pax_global_header00006660000000000000000000000064147647770030014531gustar00rootroot0000000000000052 comment=9bf846317e3c0682eb4bc268943085be8074f242 ganeti-3.1.0~rc2/000075500000000000000000000000001476477700300136045ustar00rootroot00000000000000ganeti-3.1.0~rc2/.devcontainer/000075500000000000000000000000001476477700300163435ustar00rootroot00000000000000ganeti-3.1.0~rc2/.devcontainer/devcontainer.json000064400000000000000000000010541476477700300217170ustar00rootroot00000000000000{ "name": "Ganeti Dev Container", "image": "docker.io/ganeti/ci:bookworm-py3", "customizations":{ "vscode": { "extensions": [ "ms-python.python", "ms-python.debugpy", "ms-python.vscode-pylance", "ms-python.isort", "ms-python.pylint", "haskell.haskell" ], "settings": { "python.linting.enabled": true, "python.linting.pylintEnabled": true } } } } ganeti-3.1.0~rc2/.github/000075500000000000000000000000001476477700300151445ustar00rootroot00000000000000ganeti-3.1.0~rc2/.github/workflows/000075500000000000000000000000001476477700300172015ustar00rootroot00000000000000ganeti-3.1.0~rc2/.github/workflows/ci.yml000064400000000000000000000027461476477700300203300ustar00rootroot00000000000000--- name: Build & Test on: push: branches: - master pull_request: branches: - master schedule: - cron: '42 3 * * *' permissions: read-all jobs: build: name: Build on ${{ matrix.os }}-${{ matrix.regex }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: os: [bookworm, focal, jammy, noble, debian-testing] regex: [pcre, tdfa] exclude: - os: debian-testing regex: pcre include: - os: debian-testing regex: pcre2 container: image: ganeti/ci:${{ matrix.os }}-py3 options: "--init" steps: - uses: actions/checkout@v1 - name: autogen run: ./autogen.sh - name: configure if: ${{ matrix.regex == 'pcre' }} run: ./configure --enable-haskell-tests - name: configure if: ${{ matrix.regex == 'tdfa' }} run: ./configure --enable-haskell-tests --with-haskell-pcre=tdfa - name: configure if: ${{ matrix.regex == 'pcre2' }} run: ./configure --enable-haskell-tests --with-haskell-pcre=pcre2 - name: Build run: make -j 2 - name: Check Local run: LC_ALL=C make check-local - name: Python unit tests run: make py-tests-unit - name: Python integration tests run: make py-tests-integration - name: Python legacy tests run: make py-tests-legacy - name: Haskell tests run: make -j 2 hs-tests ganeti-3.1.0~rc2/.github/workflows/codeql-analysis.yml000064400000000000000000000041161476477700300230160ustar00rootroot00000000000000name: "CodeQL" on: push: branches: [master, stable-2.16] pull_request: # The branches below must be a subset of the branches above branches: [master] schedule: - cron: '0 12 * * 3' permissions: read-all jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: # Override automatic language detection by changing the below list # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] language: ['python'] # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: - name: Checkout repository uses: actions/checkout@v2 with: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. fetch-depth: 2 # If this run was triggered by a pull request event, then checkout # the head of the pull request instead of the merge commit. - run: git checkout HEAD^2 if: ${{ github.event_name == 'pull_request' }} # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 ganeti-3.1.0~rc2/.github/workflows/qa-suite-run.yml000064400000000000000000000023221476477700300222550ustar00rootroot00000000000000--- name: Manual Ganeti QA Suite Run on: workflow_dispatch: inputs: recipe: description: "QA recipe to run" required: true default: "kvm-drbd_file_sharedfile-bridged" source: description: "Source Repository (Syntax 'user-name/repo-name')" required: true default: "ganeti/ganeti" branch: description: "Branch to build from" required: true default: "master" debian-version: description: "Which Debian version to use" required: true default: "bullseye" permissions: read-all defaults: run: working-directory: /home/rbott/ganeti-cluster-testing jobs: Run-QA-Suite: runs-on: [ qa-runner ] steps: - name: "Start the QA suite on Debian ${{ github.event.inputs.debian-version }} from ${{ github.event.inputs.source }}:${{ github.event.inputs.branch }}" run: "python3 -u run-cluster-test.py run-test --os-version '${{ github.event.inputs.debian-version }}' --recipe '${{ github.event.inputs.recipe }}' --source '${{ github.event.inputs.source }}' --branch '${{ github.event.inputs.branch }}' --remove-instances-on-success" working-directory: "/home/rbott/ganeti-cluster-testing/"ganeti-3.1.0~rc2/.gitignore000064400000000000000000000062641476477700300156040ustar00rootroot00000000000000# Lines that start with '#' are comments. # For a project mostly in C, the following would be a good set of # exclude patterns (uncomment them if you want to use them): # *.[oa] # *~ # global ignores *.py[co] *.swp *~ *.o *.hpc_o *.prof_o *.dyn_o *.hi *.hpc_hi *.prof_hi *.dyn_hi *.hp *.tix *.prof *.stat .hpc/ # files and folders belonging to common editors/IDEs /.vscode /.idea # / /.hsenv /Makefile /Makefile.ghc /Makefile.ghc.bak /Makefile.in /Makefile.local /Session.vim /TAGS* /aclocal.m4 /autom4te.cache /autotools/install-sh /autotools/missing /autotools/py-compile /autotools/replace_vars.sed /autotools/shell-env-init /cabal_macros.h /config.log /config.status /configure /devel/squeeze-amd64.tar.gz /devel/squeeze-amd64.conf /devel/wheezy-amd64.tar.gz /devel/wheezy-amd64.conf /dist/ /empty-cabal-config /epydoc.conf /exe/ /ganeti /ganeti.depsflags /stamp-srclinks /stamp-directories /vcs-version /*.patch /*.tar.bz2 /*.tar.gz /ganeti-[0-9]*.[0-9]*.[0-9]* # daemons /daemons/daemon-util /daemons/ganeti-cleaner /daemons/ganeti-masterd /daemons/ganeti-noded /daemons/ganeti-rapi /daemons/ganeti-watcher # doc /doc/api/ /doc/coverage/ /doc/html/ /doc/man-html/ /doc/news.rst /doc/upgrade.rst /doc/hs-lint.html /doc/manpages-enabled.rst /doc/man-*.rst /doc/users/groupmemberships /doc/users/groups /doc/users/users # doc/examples /doc/examples/bash_completion /doc/examples/bash_completion-debug /doc/examples/ganeti.cron /doc/examples/ganeti.initd /doc/examples/ganeti.logrotate /doc/examples/ganeti-kvm-poweroff.initd /doc/examples/ganeti-master-role.ocf /doc/examples/ganeti-node-role.ocf /doc/examples/gnt-config-backup /doc/examples/hooks/ipsec /doc/examples/systemd/ganeti-*.service # lib /lib/_constants.py /lib/_vcsversion.py /lib/_generated_rpc.py /lib/opcodes.py /lib/rpc/stub/ # man /man/*.[0-9] /man/*.html /man/*.gen # test/hs /test/hs/hail /test/hs/harep /test/hs/hbal /test/hs/hcheck /test/hs/hinfo /test/hs/hroller /test/hs/hscan /test/hs/hspace /test/hs/hsqueeze /test/hs/hpc-htools /test/hs/hpc-mon-collector /test/hs/htest # tools /tools/kvm-ifup /tools/kvm-ifup-os /tools/xen-ifup-os /tools/burnin /tools/ensure-dirs /tools/users-setup /tools/vcluster-setup /tools/vif-ganeti /tools/vif-ganeti-metad /tools/net-common /tools/node-cleanup /tools/node-daemon-setup /tools/prepare-node-join /tools/shebang/ /tools/ssh-update /tools/ssl-update # scripts /scripts/gnt-backup /scripts/gnt-cluster /scripts/gnt-debug /scripts/gnt-filter /scripts/gnt-group /scripts/gnt-instance /scripts/gnt-job /scripts/gnt-node /scripts/gnt-os /scripts/gnt-network /scripts/gnt-storage # haskell-specific rules /dist/build/hpc-mon-collector/hpc-mon-collector /dist/build/hpc-htools/hpc-htools /dist/build/hs2py/hs2py /dist/build/hs2py-constants/hs2py-constants /dist/build/ganeti-confd/ganeti-confd /dist/build/ganeti-wconfd/ganeti-wconfd /dist/build/ganeti-kvmd/ganeti-kvmd /dist/build/ganeti-luxid/ganeti-luxid /dist/build/ganeti-maintd/ganeti-maintd /dist/build/ganeti-metad/ganeti-metad /dist/build/ganeti-mond/ganeti-mond /dist/build/rpc-test/rpc-test # automatically-built Haskell files /src/AutoConf.hs /src/Ganeti/Curl/Internal.hs /src/Ganeti/Hs2Py/ListConstants.hs /src/Ganeti/Version.hs /test/hs/Test/Ganeti/TestImports.hs ganeti-3.1.0~rc2/CONTRIBUTING.md000064400000000000000000000163701476477700300160440ustar00rootroot00000000000000# Contribution Process ### tl;dr; [See the GitHub flow tutorial.](https://guides.github.com/introduction/flow/) 1. Fork the Ganeti repo on GitHub 2. Make changes in a feature branch, probably based off the master branch. 2. Test using "make pylint; make hlint; make py-tests; make hs-tests" 3. Send your changes for review by opening a Pull Request. 4. Address the reviewer comments by making new commits and ask PTAL. 5. Wait for your Pull Request to be accepted. ### 1. Forking the Repo The Ganeti project uses Pull Requests to accept contributions. In order to do so you must first [fork](https://help.github.com/articles/fork-a-repo/) the Ganeti repo: https://github.com/ganeti/ganeti. After that, clone your fork and add the original repository as the upstream: ``` git clone git@github.com:/ganeti.git git remote add upstream git@github.com:ganeti/ganeti.git ``` ### 2. Merging Upstream changes Important: Before starting a new patch it is a good idea to [merge all the upstream changes](https://help.github.com/articles/syncing-a-fork/): ``` git fetch upstream git checkout git merge upstream/ ``` ### 3. Creating your feature branch We recommend using [feature branches](https://www.atlassian.com/git/tutorials/using-branches) in your repo instead of commit directly to the master and/or stable branches. Without feature branches you could have a local clone in a separate folder named after the feature. This would work well as long as you don't want to make parallel pull requests or backup your local work on github. If your local folders map to respective feature branches on your personal fork on github, you can work in parallel on multiple features with backing up local commits on github as well as work on multiple pull requests. Let's say you want to patch an existing issue on branch . Then: ``` git checkout git checkout -b ``` Try to use a descriptive name for the feature branch. Names like "feature-", "bug-" are good when referring to an existing bug/feature request. Using a name related to the theme of the changes is also a good idea (i.e.: "predictive_locking"). ### 4. Making your changes Make the changes you need/want to do on your feature branch. Make as many commits as you want, making sure that the commit messages are small and concise within the context of your changes. The commit messages are going to be useful for the reviewer, so be reasonable. Favor small Pull Requests as they are easier to review and they provide feedback early. Avoid making huge changes just to see the reviewer asking you to restart from scratch.. Remember to use the --signoff flag when committing. You may also push these changes to your personal fork by running `git push origin ` ### 5. Testing your changes `make pylint; make hlint; make py-tests; make hs-tests` ### 6. Sending changes for review Once you and the tests are happy enough with your changes, push your changes to your personal fork. After that, [create a pull request](https://help.github.com/articles/creating-a-pull-request/) from your feature branch to the target branch in the ganeti/ganeti repo. The Pull Request message should explain WHY these changes are needed and, if they're not trivial, explain HOW they are being implemented. Also, make sure that it follows the canonical commit message criteria: * It MUST have a single line heading with no period, at most 80 characters * It MUST have a description, as described above * It MUST have a "Signed-off-by: author@org" line (Same generated when using the --signoff flag) ### 7. Addressing Reviewers comments If your pull request is more than a one-line fix, it will probably have some comments from a reviewer. The reviewer might help you to identify some typos/bugs, to improve your documentation and to improve your design/implementation. Don't feel bad about this, this is actually good both for you and for the community as a whole! Also, feel free to push back the reviewer if you think you have a point. This is a conversation, not a fight. In order to fix issues spotted by the reviewer, make the additional changes locally and commit them on the same feature branch of the pull request. Do not alter commits already submitted to review (e.g. by using rebase), but add your changes as new commits. The pull request review UI does not store history of the changes, so changing the commits would erase the context that was commented on during the review. 1. Make changes using your favorite editor. 2. Run the tests 3. git commit -m "Fixing typos spotted by the reviewer" --signoff 4. git push origin 5. Make changes to the Pull Request message as needed 1. Respond to individual comments as needed 2. Ask the reviewer to take a look (PTAL) in the main pull request thread, as individual comments on changes won't send notifications to the reviewer. Keep doing this until you receive an Approval from the reviewer. ### 8. Pushing your changes After receiving all the approvals needed, the reviewer will pull your changes to the repo. Congratulations! ## Pull Request Example Let's say that you found a bug in which a stack overflow is caused whenever your ternary_search function receive a specific set of inputs. You open an issue on GitHub detailing your findings and it creates the issue 1234. To fix this issue you first create a local branch called "bug/1234". You first either add a set of regression tests or fix the existing tests to prove that the invalid behavior is actually happening. You then decide to commit these changes: ``` git commit -m "Add regression tests for bug/1234" --sign-off (optional) git push origin ``` After reading through the code you finally find the source of the issue and fix it. You then commit these changes: ``` git commit -m "Fix algorithm for input X" --sign-off (optional) git push origin ``` Now both you and the tests are happy with the changes. So you then create a pull request from your fork to the base branch. The Pull Request message should be detailed, describing the WHY and the HOW of your changes. You then use the following message on your Pull Request: ``` Fix bug in ternary_search that would cause an stack overflow for specific inputs Whenever three consecutive numbers were used as inputs for the ternary_search function it would end up causing a stack overflow. This caused issues whenever the user tried to do action X. We fix this by handling these as a special case. This closes #1234. Signed-off-by: Your Name ``` Your Pull Request is then assigned to some reviewers. They notice that you forgot to add some comments to your code, so they ask you to do so. You then add the comments, push these changes to your fork. You then ask the reviewer to take another look on your Pull Request (PTAL = Please Take Another Look). These changes are automatically sent to the reviewer: ``` git commit -m "Adding comments for the fix" --sign-off git push origin ``` The reviewer is now happy as well, so he/she approves the pull request, adds the reviewed-by tag. When all the approvals are given, a reviewer then squashs all your commits into a single canonical commit, using the Pull Request message as the commit message. Now your changes are committed to the repo, so you can celebrate! ganeti-3.1.0~rc2/COPYING000064400000000000000000000024161476477700300146420ustar00rootroot00000000000000Copyright (C) 2006-2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ganeti-3.1.0~rc2/INSTALL000064400000000000000000000265231476477700300146450ustar00rootroot00000000000000Ganeti quick installation guide =============================== Please note that a more detailed installation procedure is described in the :doc:`install`. Refer to it if you are setting up Ganeti the first time. This quick installation guide is mainly meant as reference for experienced users. A glossary of terms can be found in the :doc:`glossary`. Distribution- and community-provided packages --------------------------------------------- - Debian provides stable packages for Ganeti - Community Ubuntu packages are provided on `Launchpad `_ - Community RHEL/CentOS RPM packages can be found `here `_ Software Requirements --------------------- .. highlight:: shell-example Before installing, please verify that you have the following programs: - `Xen Hypervisor `_, version 3.0 or above, if running on Xen - `KVM Hypervisor `_, version 2.12 or above - `DRBD `_, kernel module and userspace utils, version 8.0.7 or above, up to 8.4.x. - `RBD `_, kernel modules (``rbd.ko``/``libceph.ko``) and userspace utils (``ceph-common``) - `LVM2 `_ - `OpenSSH `_ - `iproute2 `_ - `arping `_ (part of iputils) - `ndisc6 `_ (if using IPv6) - `Python `_, version 3.6 or above - `Python OpenSSL bindings `_ - `pyparsing Python module `_, version 1.5.7 or above - `pyinotify Python module `_ - `PycURL Python module `_ - `socat `_, see :ref:`note ` below - `Paramiko `_, if you want to use ``ganeti-listrunner`` - `psutil Python module `_, optional python package for supporting CPU pinning under KVM, versions 2.x.x only; beware that versions from 2.0.0 to before 2.2.0 had a number of file handle leaks, so running at least 2.2.0 is advised - `qemu-img `_, if you want to use ``ovfconverter`` - `fping `_ - `Bitarray Python library `_ - `GNU Make `_ - `GNU M4 `_ These programs are supplied as part of most Linux distributions, so usually they can be installed via the standard package manager. Also many of them will already be installed on a standard machine. On Debian/Ubuntu, you can use this command line to install all required packages, except for RBD, DRBD and Xen:: $ apt-get install lvm2 ssh iproute iputils-arping make m4 \ ndisc6 python3 python3-openssl openssl \ python3-pyparsing python3-bitarray \ python3-pyinotify python3-pycurl socat fping Note that the previous instructions don't install optional packages. To install the optional package, run the following line.:: $ apt-get install python3-paramiko python3-psutil qemu-utils If you want to run the QA suite, you also need the follwing packages:: $ apt-get install python3-yaml python3-pytest If some of the python packages are not available in your system, you can try installing them using ``easy_install`` command. For example:: $ apt-get install python-setuptools python-dev $ cd / && easy_install \ psutil \ bitarray \ On Fedora to install all required packages except RBD, DRBD and Xen:: $ yum install openssh openssh-clients iproute ndisc6 make \ pyOpenSSL pyparsing python-inotify \ python-lxm socat fping python-bitarray python-ipaddr For optional packages use the command:: $ yum install python-paramiko python-psutil qemu-img If you want to build from source, please see doc/devnotes.rst for more dependencies. .. _socat-note: .. note:: Ganeti's import/export functionality uses ``socat`` with OpenSSL for transferring data between nodes. By default, OpenSSL 0.9.8 and above employ transparent compression of all data using zlib if supported by both sides of a connection. In cases where a lot of data is transferred, this can lead to an increased CPU usage. Additionally, Ganeti already compresses all data using ``gzip`` where it makes sense (for inter-cluster instance moves). To remedey this situation, patches implementing a new ``socat`` option for disabling OpenSSL compression have been contributed and will likely be included in the next feature release. Until then, users or distributions need to apply the patches on their own. Ganeti will use the option if it's detected by the ``configure`` script; auto-detection can be disabled by explicitly passing ``--enable-socat-compress`` (use the option to disable compression) or ``--disable-socat-compress`` (don't use the option). The patches and more information can be found on http://www.dest-unreach.org/socat/contrib/socat-opensslcompress.html. Haskell requirements ~~~~~~~~~~~~~~~~~~~~ Starting with Ganeti 2.7, the Haskell GHC compiler and a few base libraries are required in order to build Ganeti (but not to run and deploy Ganeti on production machines). More specifically: - `GHC `_ version 8.0 or higher - or even better, `The Haskell Platform `_ which gives you a simple way to bootstrap Haskell - `cabal-install `_ and `Cabal `_, the Common Architecture for Building Haskell Applications and Libraries (executable and library) - `json `_, a JSON library - `network `_, a basic network library - `parallel `_, a parallel programming library (note: tested with up to version 3.x) - `bytestring `_ and `utf8-string `_ libraries; these usually come with the GHC compiler - `text `_ - `deepseq `_, usually comes with the GHC compiler - `curl `_, tested with versions 1.3.4 and above - `hslogger `_, version 1.1 and above. - `hinotify `_, tested with version 0.3.2 - `Crypto `_, tested with version 4.2.4 - `regex-pcre `_, bindings for the ``pcre`` library - `attoparsec `_, version 0.10 and above - `vector `_ - `process `_, version 1.0.1.1 and above; usually comes with the GHC compiler - `base64-bytestring `_, version 1.0.0.0 and above - `lifted-base `_, version 0.1.1 and above. - `lens `_, version 3.10 and above. Some of these are also available as package in Debian/Ubuntu:: $ apt-get install ghc ghc-ghci cabal-install \ libghc-case-insensitive-dev libghc-curl-dev \ libghc-json-dev libghc-lens-dev \ libghc-network-dev libghc-parallel-dev \ libghc-utf8-string-dev libghc-deepseq-dev \ libghc-hslogger-dev libghc-cryptonite-dev \ libghc-text-dev libghc-hinotify-dev \ libghc-base64-bytestring-dev libghc-zlib-dev \ libghc-regex-pcre-dev libghc-attoparsec-dev libghc-vector-dev libghc-lifted-base-dev \ libghc-test-framework-quickcheck2-dev \ libghc-test-framework-hunit-dev libghc-temporary-dev \ libghc-old-time-dev libghc-old-time-dev \ libghc-lifted-base-dev libghc-temporary-dev \ libpcre3-dev In Fedora, some of them are available via packages as well:: $ yum install ghc ghc-json-devel ghc-network-devel \ ghc-parallel-devel ghc-deepseq-devel \ ghc-hslogger-devel ghc-text-devel \ ghc-regex-pcre-devel The most recent Fedora doesn't provide ``inotify``. So these need to be installed using ``cabal``. If using a distribution which does not provide these libraries, first install the Haskell platform. Then run:: $ cabal update Then install the additional native libraries:: $ apt-get install libpcre3-dev libcurl4-openssl-dev And finally the libraries required for building the packages via ``cabal`` (it will automatically pick only those that are not already installed via your distribution packages):: $ cabal install --only-dependencies cabal/ganeti.template.cabal Haskell optional features ~~~~~~~~~~~~~~~~~~~~~~~~~ Optionally, more functionality can be enabled if your build machine has a few more Haskell libraries enabled: the ``ganeti-confd`` daemon (``--enable-confd``), the monitoring daemon (``--enable-monitoring``) and the meta-data daemon (``--enable-metadata``). The extra dependencies for these are: - `snap-server` `_, version 0.8.1 and above. - `case-insensitive` `_, version 0.4.0.1 and above (it's also a dependency of ``snap-server``). - `PSQueue `_, version 1.0 and above. These libraries are available in Debian Wheezy or later, so you can use either apt:: $ apt-get install libghc-snap-server-dev libghc-psqueue-dev or ``cabal``:: $ cabal install --only-dependencies cabal/ganeti.template.cabal \ --flags="confd mond metad" to install them. .. _cabal-note: .. note:: Make sure that your ``~/.cabal/bin`` directory (or whatever else is defined as ``bindir``) is in your ``PATH``. Installation of the software ---------------------------- To install, simply run the following command:: $ ./configure --localstatedir=/var --sysconfdir=/etc && \ make && \ make install This will install the software under ``/usr/local``. Depending on your init system you then need to copy ``doc/examples/ganeti.initd`` to ``/etc/init.d/ganeti`` or install the respective systemd unit files provided in ``doc/examples/systemd/``. Also, Ganeti uses symbolic links in the sysconfdir to determine, which of potentially many installed versions currently is used. If these symbolic links should be added by the install as well, add the option ``--enable-symlinks`` to the ``configure`` call. Cluster initialisation ---------------------- Before initialising the cluster, on each node you need to create the following directories: - ``/etc/ganeti`` - ``/var/lib/ganeti`` - ``/var/log/ganeti`` - ``/srv/ganeti`` - ``/srv/ganeti/os`` - ``/srv/ganeti/export`` After this, use ``gnt-cluster init``. .. vim: set textwidth=72 syntax=rst : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/Makefile.am000064400000000000000000002736421476477700300156560ustar00rootroot00000000000000# Ganeti makefile # - Indent with tabs only. # - Keep files sorted; one line per file. # - Directories in lib/ must have their own *dir variable (see hypervisor). # - All directories must be listed DIRS. # - Use autogen.sh to generate Makefile.in and configure script. # For distcheck we need the haskell tests to be enabled. Note: # The "correct" way would be to define AM_DISTCHECK_CONFIGURE_FLAGS. # However, we still have to support older versions of autotools, # so we cannot use that yet, hence we fall back to.. DISTCHECK_CONFIGURE_FLAGS = --enable-haskell-tests # Automake doesn't export these variables before version 1.10. abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ # Helper values for calling builtin functions empty := space := $(empty) $(empty) comma := , # Helper function to strip src/ and test/hs/ from a list strip_hsroot = $(patsubst src/%,%,$(patsubst test/hs/%,%,$(1))) # Use bash in order to be able to use pipefail SHELL=/bin/bash EXTRA_DIST= # Enable colors in shelltest SHELLTESTARGS = "-c" # Use C.UTF-8 for tests/checks when available, fall back to en_US.UTF-8 else UTF8_LOCALE := $(shell locale -a | grep -m 1 -xF "C.UTF-8" || echo "en_US.UTF-8") ACLOCAL_AMFLAGS = -I autotools BUILD_BASH_COMPLETION = $(top_srcdir)/autotools/build-bash-completion RUN_IN_TEMPDIR = $(top_srcdir)/autotools/run-in-tempdir CHECK_PYTHON_CODE = $(top_srcdir)/autotools/check-python-code CHECK_HEADER = $(top_srcdir)/autotools/check-header CHECK_MAN_DASHES = $(top_srcdir)/autotools/check-man-dashes CHECK_MAN_REFERENCES = $(top_srcdir)/autotools/check-man-references CHECK_MAN_WARNINGS = $(top_srcdir)/autotools/check-man-warnings CHECK_VERSION = $(top_srcdir)/autotools/check-version CHECK_NEWS = $(top_srcdir)/autotools/check-news CHECK_IMPORTS = $(top_srcdir)/autotools/check-imports DOCPP = $(top_srcdir)/autotools/docpp REPLACE_VARS_SED = autotools/replace_vars.sed PRINT_PY_CONSTANTS = $(top_srcdir)/autotools/print-py-constants BUILD_RPC = $(top_srcdir)/autotools/build-rpc SHELL_ENV_INIT = autotools/shell-env-init # starting as of Ganeti 2.10, all files are stored in two directories, # with only symbolic links added at other places. # # $(versiondir) contains most of Ganeti and all architecture-dependent files # $(versionedsharedir) contains only architecture-independent files; all python # executables need to go directly to $(versionedsharedir), as all ganeti python # mdules are installed outside the usual python path, i.e., as private modules. # # $(defaultversiondir) and $(defaultversionedsharedir) are the corresponding # directories for "the currently running" version of Ganeti. We never install # there, but all symbolic links go there, rather than directory to $(versiondir) # or $(versionedsharedir). Note that all links to $(default*dir) need to be stable; # so, if some currently architecture-independent executable is replaced by an # architecture-dependent one (and hence has to go under $(versiondir)), add a link # under $(versionedsharedir) but do not change the external links. if USE_VERSION_FULL DIRVERSION=$(VERSION_FULL) else DIRVERSION=$(VERSION_MAJOR).$(VERSION_MINOR) endif versiondir = $(libdir)/ganeti/$(DIRVERSION) defaultversiondir = $(libdir)/ganeti/default versionedsharedir = $(prefix)/share/ganeti/$(DIRVERSION) defaultversionedsharedir = $(prefix)/share/ganeti/default # Note: these are automake-specific variables, and must be named after # the directory + 'dir' suffix pkglibdir = $(versiondir)$(libdir)/ganeti myexeclibdir = $(pkglibdir) bindir = $(versiondir)/$(BINDIR) sbindir = $(versiondir)$(SBINDIR) mandir = $(versionedsharedir)/root$(MANDIR) pkgpythondir = $(versionedsharedir)/ganeti pkgpython_rpc_stubdir = $(versionedsharedir)/ganeti/rpc/stub gntpythondir = $(versionedsharedir) pkgpython_bindir = $(versionedsharedir) gnt_python_sbindir = $(versionedsharedir) tools_pythondir = $(versionedsharedir) clientdir = $(pkgpythondir)/client cmdlibdir = $(pkgpythondir)/cmdlib cmdlib_clusterdir = $(pkgpythondir)/cmdlib/cluster configdir = $(pkgpythondir)/config hypervisordir = $(pkgpythondir)/hypervisor hypervisor_hv_kvmdir = $(pkgpythondir)/hypervisor/hv_kvm jqueuedir = $(pkgpythondir)/jqueue storagedir = $(pkgpythondir)/storage httpdir = $(pkgpythondir)/http masterddir = $(pkgpythondir)/masterd confddir = $(pkgpythondir)/confd rapidir = $(pkgpythondir)/rapi rpcdir = $(pkgpythondir)/rpc rpc_stubdir = $(pkgpythondir)/rpc/stub serverdir = $(pkgpythondir)/server watcherdir = $(pkgpythondir)/watcher impexpddir = $(pkgpythondir)/impexpd utilsdir = $(pkgpythondir)/utils toolsdir = $(pkglibdir)/tools iallocatorsdir = $(pkglibdir)/iallocators pytoolsdir = $(pkgpythondir)/tools docdir = $(versiondir)$(datadir)/doc/$(PACKAGE) ifupdir = $(sysconfdir)/ganeti if USE_BACKUP_DIR backup_dir = $(BACKUP_DIR) else backup_dir = $(localstatedir)/lib endif SYMLINK_TARGET_DIRS = \ $(sysconfdir)/ganeti \ $(libdir)/ganeti/iallocators \ $(libdir)/ganeti/tools \ $(prefix)/share/ganeti \ $(BINDIR) \ $(SBINDIR) \ $(MANDIR)/man1 \ $(MANDIR)/man7 \ $(MANDIR)/man8 # Delete output file if an error occurred while building it .DELETE_ON_ERROR: HS_DIRS = \ src \ src/Ganeti \ src/Ganeti/Confd \ src/Ganeti/Curl \ src/Ganeti/Cpu \ src/Ganeti/DataCollectors \ src/Ganeti/Daemon \ src/Ganeti/Hs2Py \ src/Ganeti/HTools \ src/Ganeti/HTools/Backend \ src/Ganeti/HTools/Cluster \ src/Ganeti/HTools/Program \ src/Ganeti/HTools/Tags \ src/Ganeti/Hypervisor \ src/Ganeti/Hypervisor/Xen \ src/Ganeti/JQScheduler \ src/Ganeti/JQueue \ src/Ganeti/Locking \ src/Ganeti/Logging \ src/Ganeti/MaintD \ src/Ganeti/Metad \ src/Ganeti/Monitoring \ src/Ganeti/Objects \ src/Ganeti/OpCodes \ src/Ganeti/Query \ src/Ganeti/Storage \ src/Ganeti/Storage/Diskstats \ src/Ganeti/Storage/Drbd \ src/Ganeti/Storage/Lvm \ src/Ganeti/THH \ src/Ganeti/Utils \ src/Ganeti/WConfd \ regex \ regex/tdfa \ regex/tdfa/Ganeti \ regex/tdfa/Ganeti/Query \ regex/pcre \ regex/pcre/Ganeti \ regex/pcre/Ganeti/Query \ regex/pcre2 \ regex/pcre2/Ganeti \ regex/pcre2/Ganeti/Query \ test/hs \ test/hs/Test \ test/hs/Test/Ganeti \ test/hs/Test/Ganeti/Storage \ test/hs/Test/Ganeti/Storage/Diskstats \ test/hs/Test/Ganeti/Storage/Drbd \ test/hs/Test/Ganeti/Storage/Lvm \ test/hs/Test/Ganeti/Confd \ test/hs/Test/Ganeti/HTools \ test/hs/Test/Ganeti/HTools/Backend \ test/hs/Test/Ganeti/Hypervisor \ test/hs/Test/Ganeti/Hypervisor/Xen \ test/hs/Test/Ganeti/JQueue \ test/hs/Test/Ganeti/Locking \ test/hs/Test/Ganeti/Objects \ test/hs/Test/Ganeti/Query \ test/hs/Test/Ganeti/THH \ test/hs/Test/Ganeti/Utils \ test/hs/Test/Ganeti/WConfd \ app # Haskell directories without the roots (src, test/hs) HS_DIRS_NOROOT = $(filter-out src,$(filter-out test/hs,$(HS_DIRS))) DIRS = \ $(HS_DIRS) \ $(PYTHON_TEST_DIRS) \ autotools \ daemons \ devel \ devel/data \ doc \ doc/css \ doc/examples \ doc/examples/gnt-debug \ doc/examples/hooks \ doc/examples/systemd \ doc/users \ test/data/htools \ test/data/htools/rapi \ test/hs/shelltests \ test/autotools \ lib \ lib/build \ lib/client \ lib/cmdlib \ lib/cmdlib/cluster \ lib/confd \ lib/config \ lib/jqueue \ lib/http \ lib/hypervisor \ lib/hypervisor/hv_kvm \ lib/impexpd \ lib/masterd \ lib/rapi \ lib/rpc \ lib/rpc/stub \ lib/server \ lib/storage \ lib/tools \ lib/utils \ lib/watcher \ man \ qa \ qa/patch \ test \ test/data \ test/data/bdev-rbd \ test/data/ovfdata \ test/data/ovfdata/other \ test/data/cgroup_root \ test/data/cgroup_root/memory \ test/data/cgroup_root/memory/lxc \ test/data/cgroup_root/memory/lxc/instance1 \ test/data/cgroup_root/cpuset \ test/data/cgroup_root/cpuset/some_group \ test/data/cgroup_root/cpuset/some_group/lxc \ test/data/cgroup_root/cpuset/some_group/lxc/instance1 \ test/data/cgroup_root/devices \ test/data/cgroup_root/devices/some_group \ test/data/cgroup_root/devices/some_group/lxc \ test/data/cgroup_root/devices/some_group/lxc/instance1 \ test/py \ test/py/legacy \ test/py/legacy/testutils \ test/py/legacy/cmdlib \ test/py/legacy/cmdlib/testsupport \ tools PYTHON_TEST_DIRS = \ test/py/unit \ test/py/unit/hypervisor \ test/py/unit/hypervisor/hv_kvm \ test/py/unit/test_data \ test/py/integration ALL_APIDOC_HS_DIRS = \ $(APIDOC_HS_DIR) \ $(patsubst %,$(APIDOC_HS_DIR)/%,$(call strip_hsroot,$(HS_DIRS_NOROOT))) BUILDTIME_DIR_AUTOCREATE = \ scripts \ $(APIDOC_DIR) \ $(ALL_APIDOC_HS_DIRS) \ $(APIDOC_PY_DIR) \ $(COVERAGE_DIR) \ $(COVERAGE_HS_DIR) \ $(COVERAGE_PY_DIR) \ .hpc # ToDo: I think all these subdirectories of dist/build # are an implementation detail of Cabal. # But Automake seems to require it this way. BUILDTIME_DIRS = \ $(BUILDTIME_DIR_AUTOCREATE) \ exe \ dist \ dist/build \ dist/build/Ganeti \ dist/build/Ganeti/Confd \ dist/build/Ganeti/Cpu \ dist/build/Ganeti/Curl \ dist/build/Ganeti/Daemon \ dist/build/Ganeti/DataCollectors \ dist/build/Ganeti/HTools \ dist/build/Ganeti/HTools/Backend \ dist/build/Ganeti/HTools/Cluster \ dist/build/Ganeti/HTools/Program \ dist/build/Ganeti/HTools/Tags \ dist/build/Ganeti/Hs2Py \ dist/build/Ganeti/Hypervisor \ dist/build/Ganeti/Hypervisor/Xen \ dist/build/Ganeti/JQScheduler \ dist/build/Ganeti/JQueue \ dist/build/Ganeti/Locking \ dist/build/Ganeti/Logging \ dist/build/Ganeti/Metad \ dist/build/Ganeti/Monitoring \ dist/build/Ganeti/Objects \ dist/build/Ganeti/OpCodes \ dist/build/Ganeti/Query \ dist/build/Ganeti/Storage \ dist/build/Ganeti/Storage/Diskstats \ dist/build/Ganeti/Storage/Drbd \ dist/build/Ganeti/Storage/Lvm \ dist/build/Ganeti/THH \ dist/build/Ganeti/Utils \ dist/build/Ganeti/WConfd \ dist/build/autogen \ dist/build/ganeti-confd \ dist/build/ganeti-confd/autogen \ dist/build/ganeti-confd/ganeti-confd-tmp \ dist/build/ganeti-kvmd \ dist/build/ganeti-kvmd/autogen \ dist/build/ganeti-kvmd/ganeti-kvmd-tmp \ dist/build/ganeti-luxid \ dist/build/ganeti-luxid/autogen \ dist/build/ganeti-luxid/ganeti-luxid-tmp \ dist/build/ganeti-metad \ dist/build/ganeti-metad/autogen \ dist/build/ganeti-metad/ganeti-metad-tmp \ dist/build/ganeti-mond \ dist/build/ganeti-mond/autogen \ dist/build/ganeti-mond/ganeti-mond-tmp \ dist/build/ganeti-wconfd \ dist/build/ganeti-wconfd/autogen \ dist/build/ganeti-wconfd/ganeti-wconfd-tmp \ dist/build/htools \ dist/build/htools/autogen \ dist/build/htools/htools-tmp \ dist/build/mon-collector \ dist/build/mon-collector/autogen \ dist/build/mon-collector/mon-collector-tmp \ dist/build/hs2py \ dist/build/hs2py/autogen \ dist/build/hs2py/hs2py-tmp \ dist/build/htest \ dist/build/htest/autogen \ dist/build/htest/htest-tmp \ dist/build/htest/htest-tmp/Test \ dist/build/htest/htest-tmp/Test/Ganeti \ dist/build/htest/htest-tmp/Test/Ganeti/Confd \ dist/build/htest/htest-tmp/Test/Ganeti/HTools \ dist/build/htest/htest-tmp/Test/Ganeti/HTools/Backend \ dist/build/htest/htest-tmp/Test/Ganeti/Hypervisor \ dist/build/htest/htest-tmp/Test/Ganeti/Hypervisor/Xen \ dist/build/htest/htest-tmp/Test/Ganeti/JQueue \ dist/build/htest/htest-tmp/Test/Ganeti/Locking \ dist/build/htest/htest-tmp/Test/Ganeti/Objects \ dist/build/htest/htest-tmp/Test/Ganeti/Query \ dist/build/htest/htest-tmp/Test/Ganeti/Storage \ dist/build/htest/htest-tmp/Test/Ganeti/Storage/Diskstats \ dist/build/htest/htest-tmp/Test/Ganeti/Storage/Drbd \ dist/build/htest/htest-tmp/Test/Ganeti/Storage/Lvm \ dist/build/htest/htest-tmp/Test/Ganeti/THH \ dist/build/htest/htest-tmp/Test/Ganeti/Utils \ dist/build/htest/htest-tmp/Test/Ganeti/WConfd \ dist/build/rpc-test \ dist/build/rpc-test/autogen \ dist/build/rpc-test/rpc-test-tmp \ dist/build/src \ dist/build/src/Ganeti \ dist/build/src/Ganeti/Confd \ dist/build/src/Ganeti/Cpu \ dist/build/src/Ganeti/DataCollectors \ dist/build/src/Ganeti/HTools \ dist/build/src/Ganeti/Hs2Py \ dist/build/src/Ganeti/JQScheduler \ dist/build/src/Ganeti/JQueue \ dist/build/src/Ganeti/Metad \ dist/build/src/Ganeti/Objects \ dist/build/src/Ganeti/OpCodes \ dist/build/src/Ganeti/Query \ dist/build/src/Ganeti/Storage \ dist/build/src/Ganeti/Storage/Diskstats \ dist/build/src/Ganeti/Storage/Lvm \ dist/build/src/Ganeti/WConfd \ dist/package.conf.inplace \ doc/html \ doc/man-html DIRCHECK_EXCLUDE = \ $(BUILDTIME_DIRS) \ ganeti-[0-9]*.[0-9]*.[0-9]* \ doc/html/_* \ doc/man-html/_* \ tools/shebang \ autom4te.cache \ .devcontainer \ .github \ .github/workflows # some helper vars COVERAGE_DIR = doc/coverage COVERAGE_PY_DIR = $(COVERAGE_DIR)/py COVERAGE_HS_DIR = $(COVERAGE_DIR)/hs APIDOC_DIR = doc/api APIDOC_PY_DIR = $(APIDOC_DIR)/py APIDOC_HS_DIR = $(APIDOC_DIR)/hs MAINTAINERCLEANFILES = \ doc/news.rst \ doc/upgrade.rst \ vcs-version maintainer-clean-local: rm -rf $(BUILDTIME_DIRS) CLEANFILES = \ $(addsuffix /*.py[co],$(DIRS)) \ $(addsuffix /*.hi,$(HS_DIRS)) \ $(addsuffix /*.dyn_hi,$(HS_DIRS)) \ $(addsuffix /*.o,$(HS_DIRS)) \ $(addsuffix /*.dyn_o,$(HS_DIRS)) \ $(addsuffix /*.$(HTEST_SUFFIX)_hi,$(HS_DIRS)) \ $(addsuffix /*.$(HTEST_SUFFIX)_o,$(HS_DIRS)) \ $(HASKELL_PACKAGE_VERSIONS_FILE) \ empty-cabal-config \ $(HASKELL_PACKAGE_IDS_FILE) \ $(HASKELL_PACKAGE_VERSIONS_FILE) \ Makefile.ghc \ Makefile.ghc.bak \ $(PYTHON_BOOTSTRAP) \ $(gnt_python_sbin_SCRIPTS) \ $(REPLACE_VARS_SED) \ $(SHELL_ENV_INIT) \ daemons/daemon-util \ daemons/ganeti-cleaner \ devel/squeeze-amd64.tar.gz \ devel/squeeze-amd64.conf \ $(mandocrst) \ doc/manpages-enabled.rst \ $(BUILT_EXAMPLES) \ doc/examples/bash_completion \ doc/examples/bash_completion-debug \ $(userspecs) \ lib/_generated_rpc.py \ $(man_MANS) \ $(manhtml) \ tools/kvm-ifup \ tools/kvm-ifup-os \ tools/xen-ifup-os \ tools/vif-ganeti \ tools/vif-ganeti-metad \ tools/net-common \ tools/users-setup \ tools/ssl-update \ tools/vcluster-setup \ tools/prepare-node-join \ tools/ssh-update \ $(python_scripts_shebang) \ stamp-directories \ stamp-srclinks \ $(nodist_pkgpython_PYTHON) \ $(nodist_pkgpython_rpc_stub_PYTHON) \ $(gnt_scripts) \ $(HS_ALL_PROGS) $(HS_BUILT_SRCS) \ $(HS_BUILT_TEST_HELPERS) \ exe/ganeti-confd \ exe/ganeti-wconfd \ exe/ganeti-luxid \ exe/ganeti-metad \ exe/ganeti-mond \ exe/htest \ .hpc/*.mix exe/*.tix test/hs/*.tix *.tix \ doc/hs-lint.html GENERATED_FILES = \ $(built_base_sources) \ $(built_python_sources) \ $(PYTHON_BOOTSTRAP) \ $(gnt_python_sbin_SCRIPTS) clean-local: rm -rf tools/shebang rm -rf dist find -D exec . -type d -name __pycache__ -exec rm -rf {} + HS_GENERATED_FILES = $(HS_PROGS) exe/ganeti-luxid exe/ganeti-confd if ENABLE_MOND HS_GENERATED_FILES += exe/ganeti-mond endif if ENABLE_METADATA HS_GENERATED_FILES += exe/ganeti-metad endif built_base_sources = \ stamp-directories \ stamp-srclinks built_python_base_sources = \ lib/_constants.py \ lib/_vcsversion.py \ lib/opcodes.py \ lib/rpc/stub/wconfd.py if ENABLE_METADATA built_python_base_sources += lib/rpc/stub/metad.py endif built_python_sources = \ $(nodist_pkgpython_PYTHON) \ $(nodist_pkgpython_rpc_stub_PYTHON) # these are all built from the underlying %.in sources BUILT_EXAMPLES = \ doc/examples/ganeti-kvm-poweroff.initd \ doc/examples/ganeti.cron \ doc/examples/ganeti.initd \ doc/examples/ganeti.logrotate \ doc/examples/ganeti-master-role.ocf \ doc/examples/ganeti-node-role.ocf \ doc/examples/gnt-config-backup \ doc/examples/hooks/ipsec \ doc/examples/systemd/ganeti-common.service \ doc/examples/systemd/ganeti-confd.service \ doc/examples/systemd/ganeti-kvmd.service \ doc/examples/systemd/ganeti-luxid.service \ doc/examples/systemd/ganeti-metad.service \ doc/examples/systemd/ganeti-mond.service \ doc/examples/systemd/ganeti-noded.service \ doc/examples/systemd/ganeti-rapi.service \ doc/examples/systemd/ganeti-wconfd.service nodist_ifup_SCRIPTS = \ tools/kvm-ifup-os \ tools/xen-ifup-os nodist_pkgpython_PYTHON = \ $(built_python_base_sources) \ lib/_generated_rpc.py nodist_pkgpython_rpc_stub_PYTHON = \ lib/rpc/stub/wconfd.py if ENABLE_METADATA nodist_pkgpython_rpc_stub_PYTHON += lib/rpc/stub/metad.py endif nodist_pkgpython_bin_SCRIPTS = \ $(nodist_pkglib_python_scripts) pkgpython_bin_SCRIPTS = \ $(pkglib_python_scripts) noinst_PYTHON = \ lib/build/__init__.py \ lib/build/shell_example_lexer.py \ lib/build/sphinx_ext.py pkgpython_PYTHON = \ lib/__init__.py \ lib/asyncnotifier.py \ lib/backend.py \ lib/bootstrap.py \ lib/cli.py \ lib/cli_opts.py \ lib/compat.py \ lib/constants.py \ lib/daemon.py \ lib/errors.py \ lib/hooksmaster.py \ lib/ht.py \ lib/jstore.py \ lib/locking.py \ lib/luxi.py \ lib/mcpu.py \ lib/metad.py \ lib/netutils.py \ lib/objects.py \ lib/opcodes_base.py \ lib/outils.py \ lib/ovf.py \ lib/pathutils.py \ lib/qlang.py \ lib/query.py \ lib/rpc_defs.py \ lib/runtime.py \ lib/serializer.py \ lib/ssconf.py \ lib/ssh.py \ lib/uidpool.py \ lib/vcluster.py \ lib/network.py \ lib/wconfd.py \ lib/workerpool.py client_PYTHON = \ lib/client/__init__.py \ lib/client/base.py \ lib/client/gnt_backup.py \ lib/client/gnt_cluster.py \ lib/client/gnt_debug.py \ lib/client/gnt_group.py \ lib/client/gnt_instance.py \ lib/client/gnt_job.py \ lib/client/gnt_node.py \ lib/client/gnt_network.py \ lib/client/gnt_os.py \ lib/client/gnt_storage.py \ lib/client/gnt_filter.py cmdlib_PYTHON = \ lib/cmdlib/__init__.py \ lib/cmdlib/backup.py \ lib/cmdlib/base.py \ lib/cmdlib/common.py \ lib/cmdlib/group.py \ lib/cmdlib/instance.py \ lib/cmdlib/instance_create.py \ lib/cmdlib/instance_helpervm.py \ lib/cmdlib/instance_migration.py \ lib/cmdlib/instance_operation.py \ lib/cmdlib/instance_query.py \ lib/cmdlib/instance_set_params.py \ lib/cmdlib/instance_storage.py \ lib/cmdlib/instance_utils.py \ lib/cmdlib/misc.py \ lib/cmdlib/network.py \ lib/cmdlib/node.py \ lib/cmdlib/operating_system.py \ lib/cmdlib/query.py \ lib/cmdlib/tags.py \ lib/cmdlib/test.py cmdlib_cluster_PYTHON = \ lib/cmdlib/cluster/__init__.py \ lib/cmdlib/cluster/verify.py config_PYTHON = \ lib/config/__init__.py \ lib/config/verify.py \ lib/config/temporary_reservations.py \ lib/config/utils.py hypervisor_PYTHON = \ lib/hypervisor/__init__.py \ lib/hypervisor/hv_base.py \ lib/hypervisor/hv_chroot.py \ lib/hypervisor/hv_fake.py \ lib/hypervisor/hv_lxc.py \ lib/hypervisor/hv_xen.py hypervisor_hv_kvm_PYTHON = \ lib/hypervisor/hv_kvm/__init__.py \ lib/hypervisor/hv_kvm/monitor.py \ lib/hypervisor/hv_kvm/netdev.py \ lib/hypervisor/hv_kvm/validation.py \ lib/hypervisor/hv_kvm/kvm_utils.py \ lib/hypervisor/hv_kvm/kvm_runtime.py jqueue_PYTHON = \ lib/jqueue/__init__.py \ lib/jqueue/exec.py storage_PYTHON = \ lib/storage/__init__.py \ lib/storage/bdev.py \ lib/storage/base.py \ lib/storage/container.py \ lib/storage/drbd.py \ lib/storage/drbd_info.py \ lib/storage/drbd_cmdgen.py \ lib/storage/extstorage.py \ lib/storage/filestorage.py \ lib/storage/gluster.py rapi_PYTHON = \ lib/rapi/__init__.py \ lib/rapi/baserlib.py \ lib/rapi/client.py \ lib/rapi/client_utils.py \ lib/rapi/connector.py \ lib/rapi/rlib2.py \ lib/rapi/testutils.py http_PYTHON = \ lib/http/__init__.py \ lib/http/auth.py \ lib/http/client.py \ lib/http/server.py confd_PYTHON = \ lib/confd/__init__.py \ lib/confd/client.py masterd_PYTHON = \ lib/masterd/__init__.py \ lib/masterd/iallocator.py \ lib/masterd/instance.py impexpd_PYTHON = \ lib/impexpd/__init__.py watcher_PYTHON = \ lib/watcher/__init__.py \ lib/watcher/nodemaint.py \ lib/watcher/state.py server_PYTHON = \ lib/server/__init__.py \ lib/server/masterd.py \ lib/server/noded.py \ lib/server/rapi.py rpc_PYTHON = \ lib/rpc/__init__.py \ lib/rpc/client.py \ lib/rpc/errors.py \ lib/rpc/node.py \ lib/rpc/transport.py rpc_stub_PYTHON = \ lib/rpc/stub/__init__.py pytools_PYTHON = \ lib/tools/__init__.py \ lib/tools/burnin.py \ lib/tools/common.py \ lib/tools/ensure_dirs.py \ lib/tools/node_cleanup.py \ lib/tools/node_daemon_setup.py \ lib/tools/prepare_node_join.py \ lib/tools/ssh_update.py \ lib/tools/ssl_update.py \ lib/tools/cfgupgrade.py utils_PYTHON = \ lib/utils/__init__.py \ lib/utils/algo.py \ lib/utils/filelock.py \ lib/utils/hash.py \ lib/utils/io.py \ lib/utils/livelock.py \ lib/utils/log.py \ lib/utils/lvm.py \ lib/utils/mlock.py \ lib/utils/nodesetup.py \ lib/utils/process.py \ lib/utils/retry.py \ lib/utils/security.py \ lib/utils/storage.py \ lib/utils/text.py \ lib/utils/tags.py \ lib/utils/version.py \ lib/utils/wrapper.py \ lib/utils/x509.py \ lib/utils/bitarrays.py docinput = \ doc/admin.rst \ doc/cluster-keys-replacement.rst \ doc/cluster-merge.rst \ doc/conf.py \ doc/design-2.0.rst \ doc/design-2.1.rst \ doc/design-2.2.rst \ doc/design-2.3.rst \ doc/design-2.4.rst \ doc/design-2.5.rst \ doc/design-2.6.rst \ doc/design-2.7.rst \ doc/design-2.8.rst \ doc/design-2.9.rst \ doc/design-2.10.rst \ doc/design-2.11.rst \ doc/design-2.12.rst \ doc/design-2.13.rst \ doc/design-2.14.rst \ doc/design-2.15.rst \ doc/design-2.16.rst \ doc/design-3.0.rst \ doc/design-3.1.rst \ doc/design-allocation-efficiency.rst \ doc/design-autorepair.rst \ doc/design-bulk-create.rst \ doc/design-ceph-ganeti-support.rst \ doc/design-configlock.rst \ doc/design-chained-jobs.rst \ doc/design-cmdlib-unittests.rst \ doc/design-cpu-pinning.rst \ doc/design-cpu-speed.rst \ doc/design-daemons.rst \ doc/design-dedicated-allocation.rst \ doc/design-device-uuid-name.rst \ doc/design-disk-conversion.rst \ doc/design-disks.rst \ doc/design-draft.rst \ doc/design-file-based-disks-ownership.rst \ doc/design-file-based-storage.rst \ doc/design-glusterfs-ganeti-support.rst \ doc/design-hotplug.rst \ doc/design-hroller.rst \ doc/design-hsqueeze.rst \ doc/design-htools-2.3.rst \ doc/design-http-server.rst \ doc/design-hugepages-support.rst \ doc/design-ifdown.rst \ doc/design-impexp2.rst \ doc/design-internal-shutdown.rst \ doc/design-kvmd.rst \ doc/design-location.rst \ doc/design-linuxha.rst \ doc/design-lu-generated-jobs.rst \ doc/design-monitoring-agent.rst \ doc/design-move-instance-improvements.rst \ doc/design-multi-reloc.rst \ doc/design-multi-storage-htools.rst \ doc/design-multi-version-tests.rst \ doc/design-network.rst \ doc/design-network2.rst \ doc/design-node-add.rst \ doc/design-node-security.rst \ doc/design-oob.rst \ doc/design-openvswitch.rst \ doc/design-opportunistic-locking.rst \ doc/design-optables.rst \ doc/design-os.rst \ doc/design-ovf-support.rst \ doc/design-partitioned.rst \ doc/design-plain-redundancy.rst \ doc/design-performance-tests.rst \ doc/design-query-splitting.rst \ doc/design-query2.rst \ doc/design-query-splitting.rst \ doc/design-qemu-blockdev.rst \ doc/design-reason-trail.rst \ doc/design-repaird.rst \ doc/design-reservations.rst \ doc/design-resource-model.rst \ doc/design-restricted-commands.rst \ doc/design-scsi-kvm.rst \ doc/design-shared-storage.rst \ doc/design-shared-storage-redundancy.rst \ doc/design-ssh-ports.rst \ doc/design-storagetypes.rst \ doc/design-sync-rate-throttling.rst \ doc/design-systemd.rst \ doc/design-upgrade.rst \ doc/design-virtual-clusters.rst \ doc/design-x509-ca.rst \ doc/dev-codestyle.rst \ doc/glossary.rst \ doc/hooks.rst \ doc/iallocator.rst \ doc/index.rst \ doc/install.rst \ doc/locking.rst \ doc/manpages-disabled.rst \ doc/monitoring-query-format.rst \ doc/move-instance.rst \ doc/news.rst \ doc/ovfconverter.rst \ doc/rapi.rst \ doc/security.rst \ doc/upgrade.rst \ doc/virtual-cluster.rst # Generates file names such as "doc/man-gnt-instance.rst" mandocrst = $(addprefix doc/man-,$(notdir $(manrst))) # Haskell programs to be installed in $PREFIX/bin HS_BIN_PROGS=exe/htools # Haskell programs to be installed in the MYEXECLIB dir if ENABLE_MOND HS_MYEXECLIB_PROGS=exe/mon-collector else HS_MYEXECLIB_PROGS= endif HS2PY_PROG = exe/hs2py # Haskell programs to be compiled by "make really-all" HS_COMPILE_PROGS = \ exe/ganeti-kvmd \ exe/ganeti-wconfd \ exe/ganeti-confd \ exe/ganeti-luxid \ $(HS2PY_PROG) \ exe/rpc-test if ENABLE_MOND HS_COMPILE_PROGS += exe/ganeti-mond endif if ENABLE_METADATA HS_COMPILE_PROGS += exe/ganeti-metad endif CABAL_SETUP = runhaskell $(top_srcdir)/Setup # All Haskell non-test programs to be compiled but not automatically installed HS_PROGS = $(HS_BIN_PROGS) $(HS_MYEXECLIB_PROGS) HS_BIN_ROLES = harep hbal hscan hspace hinfo hcheck hroller hsqueeze HS_HTOOLS_PROGS = $(HS_BIN_ROLES) hail # Haskell programs that cannot be disabled at configure (e.g., unlike # 'mon-collector') HS_DEFAULT_PROGS = \ $(HS_BIN_PROGS) \ test/hs/hpc-htools \ test/hs/hpc-mon-collector \ $(HS_COMPILE_PROGS) if HTEST HS_DEFAULT_PROGS += exe/htest endif EXTRA_DIST += test/hs/htest.hs HS_ALL_PROGS = $(HS_DEFAULT_PROGS) $(HS_MYEXECLIB_PROGS) HS_TEST_PROGS = $(filter test/%,$(HS_ALL_PROGS)) HS_SRC_PROGS = $(filter-out test/%,$(HS_ALL_PROGS)) HS_PROG_SRCS = \ app/ganeti-confd.hs \ app/ganeti-kvmd.hs \ app/ganeti-luxid.hs \ app/ganeti-metad.hs \ app/ganeti-mond.hs \ app/ganeti-wconfd.hs \ app/hs2py.hs \ app/htools.hs \ app/mon-collector.hs \ app/rpc-test.hs HS_BUILT_TEST_HELPERS = $(HS_BIN_ROLES:%=test/hs/%) test/hs/hail HFLAGS = \ -O -Wall -isrc \ -fwarn-tabs \ -optP-include -optP$(HASKELL_PACKAGE_VERSIONS_FILE) \ -hide-all-packages \ `cat $(HASKELL_PACKAGE_IDS_FILE)` \ $(GHC_BYVERSION_FLAGS) HTEST_SUFFIX = hpc HPROF_SUFFIX = prof # GHC >= 7.8 stopped putting underscores into -dep-suffix by itself # (https://ghc.haskell.org/trac/ghc/ticket/9749) so we have to put them. # It also needs -dep-suffix "" for the .o file. DEP_SUFFIXES = -dep-suffix $(HPROF_SUFFIX)_ -dep-suffix $(HTEST_SUFFIX)_ \ -dep-suffix "" # GHC > 7.6 needs -dynamic-too when using Template Haskell since its # ghci is switched to loading dynamic libraries by default. # It must only be used in non-profiling GHC invocations. # We also don't use it in compilations that use HTEST_SUFFIX (which are # compiled with -fhpc) because HPC coverage doesn't interact well with # GHCI shared lib loading (https://ghc.haskell.org/trac/ghc/ticket/9762). HFLAGS_DYNAMIC = -dynamic-too if HPROFILE HPROFFLAGS = -prof -fprof-auto-top -osuf $(HPROF_SUFFIX)_o \ -hisuf $(HPROF_SUFFIX)_hi -rtsopts endif if HCOVERAGE HFLAGS += -fhpc endif if HTEST HFLAGS += -DTEST endif HTEST_FLAGS = $(HFLAGS) -fhpc -itest/hs \ -osuf $(HTEST_SUFFIX)_o \ -hisuf $(HTEST_SUFFIX)_hi # extra flags that can be overriden on the command line (e.g. -Wwarn, etc.) HEXTRA = # combination of HEXTRA and HEXTRA_CONFIGURE HEXTRA_COMBINED = $(HEXTRA) $(HEXTRA_CONFIGURE) # extra flags used for htest # Only use the threaded runtime and run tests in parallel for GHC > 7.6. A bug # in base that wasn't fixed until 4.7.0.0 in GHC 7.8 causes random file lock # errors in the JQueue tests with the threaded runtime. # See https://ghc.haskell.org/trac/ghc/ticket/7646 HEXTRA_TEST = -threaded -with-rtsopts="-N -A32m" -rtsopts # exclude options for coverage reports HPCEXCL = --exclude Main \ --exclude Ganeti.Constants \ --exclude Ganeti.HTools.QC \ --exclude Ganeti.THH \ --exclude Ganeti.Version \ --exclude Test.Ganeti.Attoparsec \ --exclude Test.Ganeti.TestCommon \ --exclude Test.Ganeti.TestHTools \ --exclude Test.Ganeti.TestHelper \ --exclude Test.Ganeti.TestImports \ $(patsubst src.%,--exclude Test.%,$(subst /,.,$(HS_LIB_STEMS))) HS_LIB_SRCS = \ src/Ganeti/BasicTypes.hs \ src/Ganeti/Codec.hs \ src/Ganeti/Common.hs \ src/Ganeti/Compat.hs \ src/Ganeti/Confd/Client.hs \ src/Ganeti/Confd/ClientFunctions.hs \ src/Ganeti/Confd/Server.hs \ src/Ganeti/Confd/Types.hs \ src/Ganeti/Confd/Utils.hs \ src/Ganeti/Config.hs \ src/Ganeti/ConfigReader.hs \ src/Ganeti/Constants.hs \ src/Ganeti/ConstantUtils.hs \ src/Ganeti/Cpu/LoadParser.hs \ src/Ganeti/Cpu/Types.hs \ src/Ganeti/Curl/Internal.hsc \ src/Ganeti/Curl/Multi.hs \ src/Ganeti/Daemon.hs \ src/Ganeti/Daemon/Utils.hs \ src/Ganeti/DataCollectors.hs \ src/Ganeti/DataCollectors/CLI.hs \ src/Ganeti/DataCollectors/CPUload.hs \ src/Ganeti/DataCollectors/Diskstats.hs \ src/Ganeti/DataCollectors/Drbd.hs \ src/Ganeti/DataCollectors/InstStatus.hs \ src/Ganeti/DataCollectors/InstStatusTypes.hs \ src/Ganeti/DataCollectors/Lv.hs \ src/Ganeti/DataCollectors/Program.hs \ src/Ganeti/DataCollectors/Types.hs \ src/Ganeti/DataCollectors/XenCpuLoad.hs \ src/Ganeti/Errors.hs \ src/Ganeti/HTools/AlgorithmParams.hs \ src/Ganeti/HTools/Backend/IAlloc.hs \ src/Ganeti/HTools/Backend/Luxi.hs \ src/Ganeti/HTools/Backend/MonD.hs \ src/Ganeti/HTools/Backend/Rapi.hs \ src/Ganeti/HTools/Backend/Simu.hs \ src/Ganeti/HTools/Backend/Text.hs \ src/Ganeti/HTools/CLI.hs \ src/Ganeti/HTools/Cluster.hs \ src/Ganeti/HTools/Cluster/AllocatePrimitives.hs \ src/Ganeti/HTools/Cluster/AllocateSecondary.hs \ src/Ganeti/HTools/Cluster/AllocationSolution.hs \ src/Ganeti/HTools/Cluster/Evacuate.hs \ src/Ganeti/HTools/Cluster/Metrics.hs \ src/Ganeti/HTools/Cluster/Moves.hs \ src/Ganeti/HTools/Cluster/Utils.hs \ src/Ganeti/HTools/Container.hs \ src/Ganeti/HTools/Dedicated.hs \ src/Ganeti/HTools/ExtLoader.hs \ src/Ganeti/HTools/GlobalN1.hs \ src/Ganeti/HTools/Graph.hs \ src/Ganeti/HTools/Group.hs \ src/Ganeti/HTools/Instance.hs \ src/Ganeti/HTools/Loader.hs \ src/Ganeti/HTools/Nic.hs \ src/Ganeti/HTools/Node.hs \ src/Ganeti/HTools/PeerMap.hs \ src/Ganeti/HTools/Program/Hail.hs \ src/Ganeti/HTools/Program/Harep.hs \ src/Ganeti/HTools/Program/Hbal.hs \ src/Ganeti/HTools/Program/Hcheck.hs \ src/Ganeti/HTools/Program/Hinfo.hs \ src/Ganeti/HTools/Program/Hscan.hs \ src/Ganeti/HTools/Program/Hspace.hs \ src/Ganeti/HTools/Program/Hsqueeze.hs \ src/Ganeti/HTools/Program/Hroller.hs \ src/Ganeti/HTools/Program/Main.hs \ src/Ganeti/HTools/Tags.hs \ src/Ganeti/HTools/Tags/Constants.hs \ src/Ganeti/HTools/Types.hs \ src/Ganeti/Hypervisor/Xen.hs \ src/Ganeti/Hypervisor/Xen/XlParser.hs \ src/Ganeti/Hypervisor/Xen/Types.hs \ src/Ganeti/Hash.hs \ src/Ganeti/Hs2Py/GenConstants.hs \ src/Ganeti/Hs2Py/GenOpCodes.hs \ src/Ganeti/Hs2Py/OpDoc.hs \ src/Ganeti/JQScheduler.hs \ src/Ganeti/JQScheduler/Filtering.hs \ src/Ganeti/JQScheduler/ReasonRateLimiting.hs \ src/Ganeti/JQScheduler/Types.hs \ src/Ganeti/JQueue.hs \ src/Ganeti/JQueue/Lens.hs \ src/Ganeti/JQueue/Objects.hs \ src/Ganeti/JSON.hs \ src/Ganeti/Jobs.hs \ src/Ganeti/Kvmd.hs \ src/Ganeti/Lens.hs \ src/Ganeti/Locking/Allocation.hs \ src/Ganeti/Locking/Types.hs \ src/Ganeti/Locking/Locks.hs \ src/Ganeti/Locking/Waiting.hs \ src/Ganeti/Logging.hs \ src/Ganeti/Logging/Lifted.hs \ src/Ganeti/Logging/WriterLog.hs \ src/Ganeti/Luxi.hs \ src/Ganeti/Network.hs \ src/Ganeti/Objects.hs \ src/Ganeti/Objects/BitArray.hs \ src/Ganeti/Objects/Disk.hs \ src/Ganeti/Objects/Instance.hs \ src/Ganeti/Objects/Lens.hs \ src/Ganeti/Objects/Nic.hs \ src/Ganeti/OpCodes.hs \ src/Ganeti/OpCodes/Lens.hs \ src/Ganeti/OpParams.hs \ src/Ganeti/Path.hs \ src/Ganeti/Parsers.hs \ src/Ganeti/PyValue.hs \ src/Ganeti/Query/Cluster.hs \ src/Ganeti/Query/Common.hs \ src/Ganeti/Query/Exec.hs \ src/Ganeti/Query/Export.hs \ src/Ganeti/Query/Filter.hs \ src/Ganeti/Query/FilterRules.hs \ src/Ganeti/Query/Group.hs \ src/Ganeti/Query/Instance.hs \ src/Ganeti/Query/Job.hs \ src/Ganeti/Query/Language.hs \ src/Ganeti/Query/Locks.hs \ src/Ganeti/Query/Network.hs \ src/Ganeti/Query/Node.hs \ src/Ganeti/Query/Query.hs \ src/Ganeti/Query/Server.hs \ src/Ganeti/Query/Types.hs \ src/Ganeti/PartialParams.hs \ src/Ganeti/Rpc.hs \ src/Ganeti/Runtime.hs \ src/Ganeti/SlotMap.hs \ src/Ganeti/Ssconf.hs \ src/Ganeti/Storage/Diskstats/Parser.hs \ src/Ganeti/Storage/Diskstats/Types.hs \ src/Ganeti/Storage/Drbd/Parser.hs \ src/Ganeti/Storage/Drbd/Types.hs \ src/Ganeti/Storage/Lvm/LVParser.hs \ src/Ganeti/Storage/Lvm/Types.hs \ src/Ganeti/Storage/Utils.hs \ src/Ganeti/THH.hs \ src/Ganeti/THH/Compat.hs \ src/Ganeti/THH/Field.hs \ src/Ganeti/THH/HsRPC.hs \ src/Ganeti/THH/PyRPC.hs \ src/Ganeti/THH/PyType.hs \ src/Ganeti/THH/Types.hs \ src/Ganeti/THH/RPC.hs \ src/Ganeti/Types.hs \ src/Ganeti/UDSServer.hs \ src/Ganeti/Utils.hs \ src/Ganeti/Utils/Atomic.hs \ src/Ganeti/Utils/AsyncWorker.hs \ src/Ganeti/Utils/IORef.hs \ src/Ganeti/Utils/Livelock.hs \ src/Ganeti/Utils/Monad.hs \ src/Ganeti/Utils/MultiMap.hs \ src/Ganeti/Utils/MVarLock.hs \ src/Ganeti/Utils/Random.hs \ src/Ganeti/Utils/Statistics.hs \ src/Ganeti/Utils/Time.hs \ src/Ganeti/Utils/UniStd.hs \ src/Ganeti/Utils/Validate.hs \ src/Ganeti/VCluster.hs \ src/Ganeti/WConfd/ConfigState.hs \ src/Ganeti/WConfd/ConfigModifications.hs \ src/Ganeti/WConfd/ConfigVerify.hs \ src/Ganeti/WConfd/ConfigWriter.hs \ src/Ganeti/WConfd/Client.hs \ src/Ganeti/WConfd/Core.hs \ src/Ganeti/WConfd/DeathDetection.hs \ src/Ganeti/WConfd/Language.hs \ src/Ganeti/WConfd/Monad.hs \ src/Ganeti/WConfd/Persistent.hs \ src/Ganeti/WConfd/Server.hs \ src/Ganeti/WConfd/Ssconf.hs \ src/Ganeti/WConfd/TempRes.hs if ENABLE_MOND HS_LIB_SRCS += src/Ganeti/Monitoring/Server.hs else EXTRA_DIST += src/Ganeti/Monitoring/Server.hs endif if ENABLE_METADATA HS_LIB_SRCS += \ src/Ganeti/Metad/Config.hs \ src/Ganeti/Metad/ConfigCore.hs \ src/Ganeti/Metad/ConfigServer.hs \ src/Ganeti/Metad/Server.hs \ src/Ganeti/Metad/Types.hs \ src/Ganeti/Metad/WebServer.hs else EXTRA_DIST += \ src/Ganeti/Metad/Config.hs \ src/Ganeti/Metad/ConfigCore.hs \ src/Ganeti/Metad/ConfigServer.hs \ src/Ganeti/Metad/Server.hs \ src/Ganeti/Metad/Types.hs \ src/Ganeti/Metad/WebServer.hs endif HS_REGEX_SRCS = \ regex/tdfa/Ganeti/Query/RegEx.hs \ regex/pcre/Ganeti/Query/RegEx.hs \ regex/pcre2/Ganeti/Query/RegEx.hs HS_TEST_SRCS = \ test/hs/Test/AutoConf.hs \ test/hs/Test/Ganeti/Attoparsec.hs \ test/hs/Test/Ganeti/BasicTypes.hs \ test/hs/Test/Ganeti/Common.hs \ test/hs/Test/Ganeti/Confd/Types.hs \ test/hs/Test/Ganeti/Confd/Utils.hs \ test/hs/Test/Ganeti/Constants.hs \ test/hs/Test/Ganeti/Daemon.hs \ test/hs/Test/Ganeti/Errors.hs \ test/hs/Test/Ganeti/HTools/Backend/MonD.hs \ test/hs/Test/Ganeti/HTools/Backend/Simu.hs \ test/hs/Test/Ganeti/HTools/Backend/Text.hs \ test/hs/Test/Ganeti/HTools/CLI.hs \ test/hs/Test/Ganeti/HTools/Cluster.hs \ test/hs/Test/Ganeti/HTools/Container.hs \ test/hs/Test/Ganeti/HTools/Graph.hs \ test/hs/Test/Ganeti/HTools/Instance.hs \ test/hs/Test/Ganeti/HTools/Loader.hs \ test/hs/Test/Ganeti/HTools/Node.hs \ test/hs/Test/Ganeti/HTools/PeerMap.hs \ test/hs/Test/Ganeti/HTools/Types.hs \ test/hs/Test/Ganeti/Hypervisor/Xen/XlParser.hs \ test/hs/Test/Ganeti/JSON.hs \ test/hs/Test/Ganeti/Jobs.hs \ test/hs/Test/Ganeti/JQScheduler.hs \ test/hs/Test/Ganeti/JQueue.hs \ test/hs/Test/Ganeti/JQueue/Objects.hs \ test/hs/Test/Ganeti/Kvmd.hs \ test/hs/Test/Ganeti/Luxi.hs \ test/hs/Test/Ganeti/Locking/Allocation.hs \ test/hs/Test/Ganeti/Locking/Locks.hs \ test/hs/Test/Ganeti/Locking/Waiting.hs \ test/hs/Test/Ganeti/Network.hs \ test/hs/Test/Ganeti/PartialParams.hs \ test/hs/Test/Ganeti/PyValue.hs \ test/hs/Test/Ganeti/Objects.hs \ test/hs/Test/Ganeti/Objects/BitArray.hs \ test/hs/Test/Ganeti/OpCodes.hs \ test/hs/Test/Ganeti/Query/Aliases.hs \ test/hs/Test/Ganeti/Query/Filter.hs \ test/hs/Test/Ganeti/Query/Instance.hs \ test/hs/Test/Ganeti/Query/Language.hs \ test/hs/Test/Ganeti/Query/Network.hs \ test/hs/Test/Ganeti/Query/Query.hs \ test/hs/Test/Ganeti/Rpc.hs \ test/hs/Test/Ganeti/Runtime.hs \ test/hs/Test/Ganeti/SlotMap.hs \ test/hs/Test/Ganeti/Ssconf.hs \ test/hs/Test/Ganeti/Storage/Diskstats/Parser.hs \ test/hs/Test/Ganeti/Storage/Drbd/Parser.hs \ test/hs/Test/Ganeti/Storage/Drbd/Types.hs \ test/hs/Test/Ganeti/Storage/Lvm/LVParser.hs \ test/hs/Test/Ganeti/THH.hs \ test/hs/Test/Ganeti/THH/Types.hs \ test/hs/Test/Ganeti/TestCommon.hs \ test/hs/Test/Ganeti/TestHTools.hs \ test/hs/Test/Ganeti/TestHelper.hs \ test/hs/Test/Ganeti/Types.hs \ test/hs/Test/Ganeti/Utils.hs \ test/hs/Test/Ganeti/Utils/MultiMap.hs \ test/hs/Test/Ganeti/Utils/Statistics.hs \ test/hs/Test/Ganeti/Utils/Time.hs \ test/hs/Test/Ganeti/WConfd/Ssconf.hs \ test/hs/Test/Ganeti/WConfd/TempRes.hs HS_LIB_STEMS = $(patsubst %.hsc,%, $(patsubst %.hs,%, $(HS_LIB_SRCS))) HS_LIBTEST_SRCS = $(HS_LIB_SRCS) $(HS_REGEX_SRCS) $(HS_TEST_SRCS) HS_BUILT_SRCS = \ test/hs/Test/Ganeti/TestImports.hs \ src/AutoConf.hs \ src/Ganeti/Hs2Py/ListConstants.hs \ src/Ganeti/Version.hs HS_BUILT_SRCS_IN = \ $(patsubst %,%.in,$(HS_BUILT_SRCS)) \ lib/_constants.py.in \ lib/opcodes.py.in_after \ lib/opcodes.py.in_before HS_LIBTESTBUILT_SRCS = $(HS_LIBTEST_SRCS) $(HS_BUILT_SRCS) $(RUN_IN_TEMPDIR): | stamp-directories doc/html/index.html: ENABLE_MANPAGES = doc/man-html/index.html: ENABLE_MANPAGES = 1 doc/man-html/index.html: doc/manpages-enabled.rst $(mandocrst) # Note: we use here an order-only prerequisite, as the contents of # _constants.py are not actually influencing the html build output: it # has to exist in order for the sphinx module to be loaded # successfully, but we certainly don't want the docs to be rebuilt if # it changes doc/html/index.html doc/man-html/index.html: $(docinput) doc/conf.py \ configure.ac $(RUN_IN_TEMPDIR) lib/build/sphinx_ext.py \ lib/build/shell_example_lexer.py lib/ht.py \ lib/rapi/connector.py lib/rapi/rlib2.py \ $(abs_top_srcdir)/autotools/sphinx-wrapper | $(built_python_sources) @test -n "$(SPHINX)" || \ { echo 'sphinx-build' not found during configure; exit 1; } if !MANPAGES_IN_DOC if test -n '$(ENABLE_MANPAGES)'; then \ echo 'Man pages in documentation were disabled at configure time' >&2; \ exit 1; \ fi endif ## Sphinx provides little control over what content should be included. Some ## mechanisms exist, but they all have drawbacks or actual issues. Since we ## build two different versions of the documentation--once without man pages and ## once, if enabled, with them--some control is necessary. xmpp-wrapper provides ## us with this, but requires running in a temporary directory. It moves the ## correct files into place depending on environment variables. dir=$(dir $@) && \ @mkdir_p@ $$dir && \ PYTHONPATH=. ENABLE_MANPAGES=$(ENABLE_MANPAGES) COPY_DOC=1 \ $(RUN_IN_TEMPDIR) \ $(abs_top_srcdir)/autotools/sphinx-wrapper $(SPHINX) -q -W -b html \ -d . \ -D version="$(VERSION_MAJOR).$(VERSION_MINOR)" \ -D release="$(PACKAGE_VERSION)" \ -D graphviz_dot="$(DOT)" \ doc $(CURDIR)/$$dir && \ rm -f $$dir/.buildinfo $$dir/objects.inv touch $@ doc/html: doc/html/index.html doc/man-html: doc/man-html/index.html doc/news.rst: NEWS doc/upgrade.rst: UPGRADE doc/news.rst doc/upgrade.rst: set -e; \ { echo '.. This file is automatically updated at build time from $<.'; \ echo '.. Do not edit.'; \ echo; \ cat $<; \ } > $@ doc/manpages-enabled.rst: Makefile | $(built_base_sources) { echo '.. This file is automatically generated, do not edit!'; \ echo ''; \ echo 'Man pages'; \ echo '========='; \ echo; \ echo '.. toctree::'; \ echo ' :maxdepth: 1'; \ echo; \ for i in $(notdir $(mandocrst)); do \ echo " $$i"; \ done | LC_ALL=C sort; \ } > $@ doc/man-%.rst: man/%.gen Makefile $(REPLACE_VARS_SED) | $(built_base_sources) if MANPAGES_IN_DOC { echo '.. This file is automatically updated at build time from $<.'; \ echo '.. Do not edit.'; \ echo; \ echo "$*"; \ echo '=========================================='; \ tail -n +3 $< | sed -f $(REPLACE_VARS_SED); \ } > $@ else echo 'Man pages in documentation were disabled at configure time' >&2; \ exit 1; endif doc/users/%: doc/users/%.in Makefile $(REPLACE_VARS_SED) cat $< | sed -f $(REPLACE_VARS_SED) | LC_ALL=C sort | uniq | (grep -v '^root' || true) > $@ userspecs = \ doc/users/users \ doc/users/groups \ doc/users/groupmemberships # Things to build but not to install (add it to EXTRA_DIST if it should be # distributed) noinst_DATA = \ $(BUILT_EXAMPLES) \ doc/examples/bash_completion \ doc/examples/bash_completion-debug \ $(userspecs) \ $(manhtml) if HAS_SPHINX if MANPAGES_IN_DOC noinst_DATA += doc/man-html else noinst_DATA += doc/html endif endif gnt_scripts = \ scripts/gnt-backup \ scripts/gnt-cluster \ scripts/gnt-debug \ scripts/gnt-group \ scripts/gnt-instance \ scripts/gnt-job \ scripts/gnt-network \ scripts/gnt-node \ scripts/gnt-os \ scripts/gnt-storage \ scripts/gnt-filter gnt_scripts_basenames = \ $(patsubst scripts/%,%,$(patsubst daemons/%,%,$(gnt_scripts) $(gnt_python_sbin_SCRIPTS))) gnt_python_sbin_SCRIPTS = \ $(PYTHON_BOOTSTRAP_SBIN) gntpython_SCRIPTS = $(gnt_scripts) PYTHON_BOOTSTRAP_SBIN = \ daemons/ganeti-noded \ daemons/ganeti-rapi \ daemons/ganeti-watcher PYTHON_BOOTSTRAP = \ tools/burnin \ tools/ensure-dirs \ tools/node-cleanup \ tools/node-daemon-setup \ tools/prepare-node-join \ tools/ssh-update \ tools/ssl-update qa_scripts = \ qa/__init__.py \ qa/ganeti-qa.py \ qa/qa_cluster.py \ qa/qa_config.py \ qa/qa_daemon.py \ qa/qa_env.py \ qa/qa_error.py \ qa/qa_filters.py \ qa/qa_group.py \ qa/qa_instance.py \ qa/qa_instance_utils.py \ qa/qa_iptables.py \ qa/qa_job.py \ qa/qa_job_utils.py \ qa/qa_logging.py \ qa/qa_monitoring.py \ qa/qa_network.py \ qa/qa_node.py \ qa/qa_os.py \ qa/qa_performance.py \ qa/qa_rapi.py \ qa/qa_tags.py \ qa/qa_utils.py \ qa/colors.py bin_SCRIPTS = $(HS_BIN_PROGS) install-exec-hook: @mkdir_p@ $(DESTDIR)$(iallocatorsdir) # FIXME: this is a hardcoded logic, instead of auto-resolving $(LN_S) -f ../../../bin/htools \ $(DESTDIR)$(iallocatorsdir)/hail for role in $(HS_BIN_ROLES); do \ $(LN_S) -f htools $(DESTDIR)$(bindir)/$$role ; \ done HS_SRCS = $(HS_LIBTESTBUILT_SRCS) $(HS_SRC_PROGS) exe/htest: dist/app.stamp dist/app.stamp: exe/ dist/setup-config $(CABAL_SETUP) build -(cd exe; \ for name in ganeti-kvmd ganeti-wconfd ganeti-confd ganeti-luxid \ rpc-test ganeti-mond ganeti-metad \ htools mon-collector hs2py htest ; do \ $(LN_S) ../dist/build/$$name/$$name; \ done) @touch "$@" exe/: @mkdir_p@ exe dist_sbin_SCRIPTS = \ tools/ganeti-listrunner nodist_sbin_SCRIPTS = \ daemons/ganeti-cleaner \ exe/ganeti-kvmd \ exe/ganeti-luxid \ exe/ganeti-confd \ exe/ganeti-wconfd # strip path prefixes off the sbin scripts all_sbin_scripts = $(notdir $(dist_sbin_SCRIPTS) $(nodist_sbin_SCRIPTS)) if ENABLE_MOND nodist_sbin_SCRIPTS += exe/ganeti-mond endif if ENABLE_METADATA nodist_sbin_SCRIPTS += exe/ganeti-metad endif python_scripts = \ tools/cfgshell \ tools/cfgupgrade \ tools/cfgupgrade12 \ tools/cluster-merge \ tools/confd-client \ tools/fmtjson \ tools/lvmstrap \ tools/move-instance \ tools/ovfconverter \ tools/post-upgrade \ tools/sanitize-config \ tools/query-config python_scripts_shebang = \ $(patsubst tools/%,tools/shebang/%, $(python_scripts)) tools/shebang/%: tools/% mkdir -p tools/shebang head -1 $< | sed 's|#!/usr/bin/python3|#!$(PYTHON)|' > $@ echo '# Generated file; do not edit.' >> $@ tail -n +2 $< >> $@ dist_tools_SCRIPTS = \ tools/kvm-console-wrapper \ tools/master-ip-setup \ tools/xen-console-wrapper dist_tools_python_SCRIPTS = \ tools/burnin nodist_tools_python_SCRIPTS = \ tools/node-cleanup \ $(python_scripts_shebang) tools_python_basenames = \ $(patsubst shebang/%,%,\ $(patsubst tools/%,%,\ $(dist_tools_python_SCRIPTS) $(nodist_tools_python_SCRIPTS))) nodist_tools_SCRIPTS = \ tools/users-setup \ tools/vcluster-setup tools_basenames = $(patsubst tools/%,%,$(nodist_tools_SCRIPTS) $(dist_tools_SCRIPTS)) pkglib_python_scripts = \ daemons/import-export \ tools/check-cert-expired nodist_pkglib_python_scripts = \ tools/ensure-dirs \ tools/node-daemon-setup \ tools/prepare-node-join \ tools/ssh-update \ tools/ssl-update pkglib_python_basenames = \ $(patsubst daemons/%,%,$(patsubst tools/%,%,\ $(pkglib_python_scripts) $(nodist_pkglib_python_scripts))) myexeclib_SCRIPTS = \ daemons/daemon-util \ tools/kvm-ifup \ tools/kvm-ifup-os \ tools/xen-ifup-os \ tools/vif-ganeti \ tools/vif-ganeti-metad \ tools/net-common \ $(HS_MYEXECLIB_PROGS) # compute the basenames of the myexeclib_scripts myexeclib_scripts_basenames = $(notdir $(myexeclib_SCRIPTS)) EXTRA_DIST += \ NEWS \ UPGRADE \ pydoctor.ini \ pylintrc \ pylintrc-test \ autotools/build-bash-completion \ autotools/build-rpc \ autotools/check-header \ autotools/check-imports \ autotools/check-man-dashes \ autotools/check-man-references \ autotools/check-man-warnings \ autotools/check-news \ autotools/check-python-code \ autotools/check-tar \ autotools/check-version \ autotools/docpp \ autotools/gen-py-coverage \ autotools/print-py-constants \ autotools/sphinx-wrapper \ autotools/testrunner \ autotools/wrong-hardcoded-paths \ $(RUN_IN_TEMPDIR) \ daemons/daemon-util.in \ daemons/ganeti-cleaner.in \ $(pkglib_python_scripts) \ devel/build_chroot \ devel/upload \ devel/webserver \ tools/kvm-ifup.in \ tools/ifup-os.in \ tools/vif-ganeti.in \ tools/vif-ganeti-metad.in \ tools/net-common.in \ tools/vcluster-setup.in \ $(python_scripts) \ $(docinput) \ doc/html \ $(BUILT_EXAMPLES:%=%.in) \ doc/examples/ganeti.default \ doc/examples/ganeti.default-debug \ doc/examples/hooks/ethers \ doc/examples/gnt-debug/README \ doc/examples/gnt-debug/delay0.json \ doc/examples/gnt-debug/delay50.json \ doc/examples/systemd/ganeti-master.target \ doc/examples/systemd/ganeti-node.target \ doc/examples/systemd/ganeti.service \ doc/examples/systemd/ganeti.target \ doc/users/groupmemberships.in \ doc/users/groups.in \ doc/users/users.in \ ganeti.cabal \ Setup.lhs \ $(dist_TESTS) \ $(TEST_FILES) \ $(python_test_support) \ $(python_test_utils) \ man/footer.rst \ $(manrst) \ qa/qa-sample.json \ $(qa_scripts) \ $(HS_LIBTEST_SRCS) $(HS_BUILT_SRCS_IN) \ $(HS_PROG_SRCS) \ src/lint-hints.hs \ test/hs/cli-tests-defs.sh \ test/hs/offline-test.sh man_MANS = \ man/ganeti-cleaner.8 \ man/ganeti-confd.8 \ man/ganeti-luxid.8 \ man/ganeti-listrunner.8 \ man/ganeti-kvmd.8 \ man/ganeti-mond.8 \ man/ganeti-noded.8 \ man/ganeti-os-interface.7 \ man/ganeti-extstorage-interface.7 \ man/ganeti-rapi.8 \ man/ganeti-watcher.8 \ man/ganeti-wconfd.8 \ man/ganeti.7 \ man/gnt-backup.8 \ man/gnt-cluster.8 \ man/gnt-debug.8 \ man/gnt-group.8 \ man/gnt-network.8 \ man/gnt-instance.8 \ man/gnt-job.8 \ man/gnt-node.8 \ man/gnt-os.8 \ man/gnt-storage.8 \ man/gnt-filter.8 \ man/hail.1 \ man/harep.1 \ man/hbal.1 \ man/hcheck.1 \ man/hinfo.1 \ man/hscan.1 \ man/hspace.1 \ man/hsqueeze.1 \ man/hroller.1 \ man/htools.1 \ man/mon-collector.7 # Remove extensions from all filenames in man_MANS mannoext = $(patsubst %.1,%,$(patsubst %.7,%,$(patsubst %.8,%,$(man_MANS)))) manrst = $(patsubst %,%.rst,$(mannoext)) manhtml = $(patsubst %.rst,%.html,$(manrst)) mangen = $(patsubst %.rst,%.gen,$(manrst)) manfullpath = $(patsubst man/%.1,man1/%.1,\ $(patsubst man/%.7,man7/%.7,\ $(patsubst man/%.8,man8/%.8,$(man_MANS)))) TEST_FILES = \ test/autotools/autotools-check-news.test \ test/data/htools/clean-nonzero-score.data \ test/data/htools/common-suffix.data \ test/data/htools/empty-cluster.data \ test/data/htools/hail-alloc-dedicated-1.json \ test/data/htools/hail-alloc-desired-location.json \ test/data/htools/hail-alloc-drbd.json \ test/data/htools/hail-alloc-drbd-restricted.json \ test/data/htools/hail-alloc-ext.json \ test/data/htools/hail-alloc-invalid-network.json \ test/data/htools/hail-alloc-invalid-twodisks.json \ test/data/htools/hail-alloc-restricted-network.json \ test/data/htools/hail-alloc-nlocation.json \ test/data/htools/hail-alloc-plain-tags.json \ test/data/htools/hail-alloc-secondary.json \ test/data/htools/hail-alloc-spindles.json \ test/data/htools/hail-alloc-twodisks.json \ test/data/htools/hail-change-group.json \ test/data/htools/hail-invalid-reloc.json \ test/data/htools/hail-node-evac.json \ test/data/htools/hail-reloc-drbd.json \ test/data/htools/hail-reloc-drbd-crowded.json \ test/data/htools/hbal-cpu-speed.data \ test/data/htools/hbal-desiredlocation-1.data \ test/data/htools/hbal-desiredlocation-2.data \ test/data/htools/hbal-desiredlocation-3.data \ test/data/htools/hbal-desiredlocation-4.data \ test/data/htools/hbal-dyn.data \ test/data/htools/hbal-evac.data \ test/data/htools/hbal-excl-tags.data \ test/data/htools/hbal-forth.data \ test/data/htools/hbal-location-1.data \ test/data/htools/hbal-location-exclusion.data \ test/data/htools/hbal-location-2.data \ test/data/htools/hbal-migration-1.data \ test/data/htools/hbal-migration-2.data \ test/data/htools/hbal-migration-3.data \ test/data/htools/hail-multialloc-dedicated.json \ test/data/htools/hbal-soft-errors.data \ test/data/htools/hbal-soft-errors2.data \ test/data/htools/hbal-split-insts.data \ test/data/htools/hspace-bad-group.data \ test/data/htools/hspace-existing.data \ test/data/htools/hspace-groups-one.data \ test/data/htools/hspace-groups-two.data \ test/data/htools/hspace-tiered-dualspec-exclusive.data \ test/data/htools/hspace-tiered-dualspec.data \ test/data/htools/hspace-tiered-exclusive.data \ test/data/htools/hspace-tiered-ipolicy.data \ test/data/htools/hspace-tiered-mixed.data \ test/data/htools/hspace-tiered-resourcetypes.data \ test/data/htools/hspace-tiered-vcpu.data \ test/data/htools/hspace-tiered.data \ test/data/htools/invalid-node.data \ test/data/htools/missing-resources.data \ test/data/htools/multiple-master.data \ test/data/htools/multiple-tags.data \ test/data/htools/n1-failure.data \ test/data/htools/partly-used.data \ test/data/htools/rapi/groups.json \ test/data/htools/rapi/info.json \ test/data/htools/rapi/instances.json \ test/data/htools/rapi/nodes.json \ test/data/htools/hroller-full.data \ test/data/htools/hroller-nodegroups.data \ test/data/htools/hroller-nonredundant.data \ test/data/htools/hroller-online.data \ test/data/htools/hsqueeze-mixed-instances.data \ test/data/htools/hsqueeze-overutilized.data \ test/data/htools/hsqueeze-underutilized.data \ test/data/htools/plain-n1-restriction.data \ test/data/htools/shared-n1-failure.data \ test/data/htools/shared-n1-restriction.data \ test/data/htools/unique-reboot-order.data \ test/data/mond-data.txt \ test/hs/shelltests/htools-balancing.test \ test/hs/shelltests/htools-basic.test \ test/hs/shelltests/htools-dynutil.test \ test/hs/shelltests/htools-excl.test \ test/hs/shelltests/htools-hail.test \ test/hs/shelltests/htools-hbal-evac.test \ test/hs/shelltests/htools-hbal.test \ test/hs/shelltests/htools-hcheck.test \ test/hs/shelltests/htools-hroller.test \ test/hs/shelltests/htools-hspace.test \ test/hs/shelltests/htools-hsqueeze.test \ test/hs/shelltests/htools-invalid.test \ test/hs/shelltests/htools-multi-group.test \ test/hs/shelltests/htools-no-backend.test \ test/hs/shelltests/htools-rapi.test \ test/hs/shelltests/htools-single-group.test \ test/hs/shelltests/htools-text-backend.test \ test/hs/shelltests/htools-mon-collector.test \ test/data/bdev-drbd-8.0.txt \ test/data/bdev-drbd-8.3.txt \ test/data/bdev-drbd-8.4.txt \ test/data/bdev-drbd-8.4-no-disk-params.txt \ test/data/bdev-drbd-disk.txt \ test/data/bdev-drbd-net-ip4.txt \ test/data/bdev-drbd-net-ip6.txt \ test/data/bdev-rbd/json_output_empty.txt \ test/data/bdev-rbd/json_output_extra_matches.txt \ test/data/bdev-rbd/json_output_no_matches.txt \ test/data/bdev-rbd/json_output_ok.txt \ test/data/bdev-rbd/plain_output_new_extra_matches.txt \ test/data/bdev-rbd/plain_output_new_no_matches.txt \ test/data/bdev-rbd/plain_output_new_ok.txt \ test/data/bdev-rbd/plain_output_old_empty.txt \ test/data/bdev-rbd/plain_output_old_extra_matches.txt \ test/data/bdev-rbd/plain_output_old_no_matches.txt \ test/data/bdev-rbd/plain_output_old_ok.txt \ test/data/bdev-rbd/output_invalid.txt \ test/data/cert1.pem \ test/data/cert2.pem \ test/data/cgroup_root/memory/lxc/instance1/memory.limit_in_bytes \ test/data/cgroup_root/cpuset/some_group/lxc/instance1/cpuset.cpus \ test/data/cgroup_root/devices/some_group/lxc/instance1/devices.list \ test/data/cluster_config_2.7.json \ test/data/cluster_config_2.8.json \ test/data/cluster_config_2.9.json \ test/data/cluster_config_2.10.json \ test/data/cluster_config_2.11.json \ test/data/cluster_config_2.12.json \ test/data/cluster_config_2.13.json \ test/data/cluster_config_2.14.json \ test/data/cluster_config_2.15.json \ test/data/cluster_config_2.16.json \ test/data/cluster_config_3.0.json \ test/data/cluster_config_3.1.json \ test/data/instance-minor-pairing.txt \ test/data/instance-disks.txt \ test/data/ip-addr-show-dummy0.txt \ test/data/ip-addr-show-lo-ipv4.txt \ test/data/ip-addr-show-lo-ipv6.txt \ test/data/ip-addr-show-lo-oneline-ipv4.txt \ test/data/ip-addr-show-lo-oneline-ipv6.txt \ test/data/ip-addr-show-lo-oneline.txt \ test/data/ip-addr-show-lo.txt \ test/data/kvm_0.12.5_help.txt \ test/data/kvm_0.15.90_help.txt \ test/data/kvm_0.9.1_help.txt \ test/data/kvm_0.9.1_help_boot_test.txt \ test/data/kvm_1.0_help.txt \ test/data/kvm_1.1.2_help.txt \ test/data/kvm_5.2.0_help.txt \ test/data/kvm_current_help.txt \ test/data/kvm_6.0.0_machine.txt \ test/data/kvm_runtime.json \ test/data/lvs_lv.txt \ test/data/NEWS_OK.txt \ test/data/NEWS_previous_unreleased.txt \ test/data/ovfdata/compr_disk.vmdk.gz \ test/data/ovfdata/config.ini \ test/data/ovfdata/corrupted_resources.ovf \ test/data/ovfdata/empty.ini \ test/data/ovfdata/empty.ovf \ test/data/ovfdata/ganeti.mf \ test/data/ovfdata/ganeti.ovf \ test/data/ovfdata/gzip_disk.ovf \ test/data/ovfdata/new_disk.vmdk \ test/data/ovfdata/no_disk.ini \ test/data/ovfdata/no_disk_in_ref.ovf \ test/data/ovfdata/no_os.ini \ test/data/ovfdata/no_ovf.ova \ test/data/ovfdata/other/rawdisk.raw \ test/data/ovfdata/ova.ova \ test/data/ovfdata/rawdisk.raw \ test/data/ovfdata/second_disk.vmdk \ test/data/ovfdata/unsafe_path.ini \ test/data/ovfdata/virtualbox.ovf \ test/data/ovfdata/wrong_config.ini \ test/data/ovfdata/wrong_extension.ovd \ test/data/ovfdata/wrong_manifest.mf \ test/data/ovfdata/wrong_manifest.ovf \ test/data/ovfdata/wrong_ova.ova \ test/data/ovfdata/wrong_xml.ovf \ test/data/proc_cgroup.txt \ test/data/proc_diskstats.txt \ test/data/proc_drbd8.txt \ test/data/proc_drbd80-emptyline.txt \ test/data/proc_drbd80-emptyversion.txt \ test/data/proc_drbd83.txt \ test/data/proc_drbd83_sync.txt \ test/data/proc_drbd83_sync_want.txt \ test/data/proc_drbd83_sync_krnl2.6.39.txt \ test/data/proc_drbd84.txt \ test/data/proc_drbd84_emptyfirst.txt \ test/data/proc_drbd84_sync.txt \ test/data/proc_meminfo.txt \ test/data/proc_cpuinfo.txt \ test/data/qa-minimal-nodes-instances-only.json \ test/data/sys_drbd_usermode_helper.txt \ test/data/vgreduce-removemissing-2.02.02.txt \ test/data/vgreduce-removemissing-2.02.66-fail.txt \ test/data/vgreduce-removemissing-2.02.66-ok.txt \ test/data/vgs-missing-pvs-2.02.02.txt \ test/data/vgs-missing-pvs-2.02.66.txt \ test/data/xen-xl-list-4.4-crashed-instances.txt \ test/data/xen-xl-info-4.0.1.txt \ test/data/xen-xl-list-4.0.1-dom0-only.txt \ test/data/xen-xl-list-4.0.1-four-instances.txt \ test/data/xen-xl-list-long-4.0.1.txt \ test/data/xen-xl-uptime-4.0.1.txt \ test/py/legacy/ganeti-cli.test \ test/py/legacy/gnt-cli.test \ test/py/legacy/import-export_unittest-helper \ test/py/unit/test_data/serialized_disks.json \ test/py/unit/test_data/serialized_nics.json python_tests_integration = \ test/py/integration/test_dummy.py python_tests_legacy = \ doc/examples/rapi_testutils.py \ test/py/legacy/cmdlib/backup_unittest.py \ test/py/legacy/cmdlib/cluster_unittest.py \ test/py/legacy/cmdlib/cmdlib_unittest.py \ test/py/legacy/cmdlib/group_unittest.py \ test/py/legacy/cmdlib/instance_unittest.py \ test/py/legacy/cmdlib/instance_migration_unittest.py \ test/py/legacy/cmdlib/instance_storage_unittest.py \ test/py/legacy/cmdlib/node_unittest.py \ test/py/legacy/cmdlib/test_unittest.py \ test/py/legacy/cfgupgrade_unittest.py \ test/py/legacy/docs_unittest.py \ test/py/legacy/ganeti.asyncnotifier_unittest.py \ test/py/legacy/ganeti.backend_unittest-runasroot.py \ test/py/legacy/ganeti.backend_unittest.py \ test/py/legacy/ganeti.bootstrap_unittest.py \ test/py/legacy/ganeti.cli_unittest.py \ test/py/legacy/ganeti.cli_opts_unittest.py \ test/py/legacy/ganeti.client.gnt_cluster_unittest.py \ test/py/legacy/ganeti.client.gnt_instance_unittest.py \ test/py/legacy/ganeti.client.gnt_job_unittest.py \ test/py/legacy/ganeti.compat_unittest.py \ test/py/legacy/ganeti.confd.client_unittest.py \ test/py/legacy/ganeti.config_unittest.py \ test/py/legacy/ganeti.constants_unittest.py \ test/py/legacy/ganeti.daemon_unittest.py \ test/py/legacy/ganeti.errors_unittest.py \ test/py/legacy/ganeti.hooks_unittest.py \ test/py/legacy/ganeti.ht_unittest.py \ test/py/legacy/ganeti.http_unittest.py \ test/py/legacy/ganeti.hypervisor.hv_chroot_unittest.py \ test/py/legacy/ganeti.hypervisor.hv_fake_unittest.py \ test/py/legacy/ganeti.hypervisor.hv_kvm_unittest.py \ test/py/legacy/ganeti.hypervisor.hv_lxc_unittest.py \ test/py/legacy/ganeti.hypervisor.hv_xen_unittest.py \ test/py/legacy/ganeti.hypervisor_unittest.py \ test/py/legacy/ganeti.impexpd_unittest.py \ test/py/legacy/ganeti.jqueue_unittest.py \ test/py/legacy/ganeti.jstore_unittest.py \ test/py/legacy/ganeti.locking_unittest.py \ test/py/legacy/ganeti.luxi_unittest.py \ test/py/legacy/ganeti.masterd.iallocator_unittest.py \ test/py/legacy/ganeti.masterd.instance_unittest.py \ test/py/legacy/ganeti.mcpu_unittest.py \ test/py/legacy/ganeti.netutils_unittest.py \ test/py/legacy/ganeti.objects_unittest.py \ test/py/legacy/ganeti.opcodes_unittest.py \ test/py/legacy/ganeti.outils_unittest.py \ test/py/legacy/ganeti.ovf_unittest.py \ test/py/legacy/ganeti.qlang_unittest.py \ test/py/legacy/ganeti.query_unittest.py \ test/py/legacy/ganeti.rapi.baserlib_unittest.py \ test/py/legacy/ganeti.rapi.client_unittest.py \ test/py/legacy/ganeti.rapi.resources_unittest.py \ test/py/legacy/ganeti.rapi.rlib2_unittest.py \ test/py/legacy/ganeti.rapi.testutils_unittest.py \ test/py/legacy/ganeti.rpc_unittest.py \ test/py/legacy/ganeti.rpc.client_unittest.py \ test/py/legacy/ganeti.runtime_unittest.py \ test/py/legacy/ganeti.serializer_unittest.py \ test/py/legacy/ganeti.server.rapi_unittest.py \ test/py/legacy/ganeti.ssconf_unittest.py \ test/py/legacy/ganeti.ssh_unittest.py \ test/py/legacy/ganeti.storage.bdev_unittest.py \ test/py/legacy/ganeti.storage.container_unittest.py \ test/py/legacy/ganeti.storage.drbd_unittest.py \ test/py/legacy/ganeti.storage.filestorage_unittest.py \ test/py/legacy/ganeti.storage.gluster_unittest.py \ test/py/legacy/ganeti.tools.burnin_unittest.py \ test/py/legacy/ganeti.tools.ensure_dirs_unittest.py \ test/py/legacy/ganeti.tools.node_daemon_setup_unittest.py \ test/py/legacy/ganeti.tools.prepare_node_join_unittest.py \ test/py/legacy/ganeti.uidpool_unittest.py \ test/py/legacy/ganeti.utils.algo_unittest.py \ test/py/legacy/ganeti.utils.filelock_unittest.py \ test/py/legacy/ganeti.utils.hash_unittest.py \ test/py/legacy/ganeti.utils.io_unittest-runasroot.py \ test/py/legacy/ganeti.utils.io_unittest.py \ test/py/legacy/ganeti.utils.log_unittest.py \ test/py/legacy/ganeti.utils.livelock_unittest.py \ test/py/legacy/ganeti.utils.lvm_unittest.py \ test/py/legacy/ganeti.utils.mlock_unittest.py \ test/py/legacy/ganeti.utils.nodesetup_unittest.py \ test/py/legacy/ganeti.utils.process_unittest.py \ test/py/legacy/ganeti.utils.retry_unittest.py \ test/py/legacy/ganeti.utils.security_unittest.py \ test/py/legacy/ganeti.utils.storage_unittest.py \ test/py/legacy/ganeti.utils.text_unittest.py \ test/py/legacy/ganeti.utils.version_unittest.py \ test/py/legacy/ganeti.utils.wrapper_unittest.py \ test/py/legacy/ganeti.utils.x509_unittest.py \ test/py/legacy/ganeti.utils.bitarrays_unittest.py \ test/py/legacy/ganeti.utils_unittest.py \ test/py/legacy/ganeti.vcluster_unittest.py \ test/py/legacy/ganeti.workerpool_unittest.py \ test/py/legacy/qa.qa_config_unittest.py \ test/py/legacy/tempfile_fork_unittest.py python_tests_unit = \ test/py/unit/hypervisor/hv_kvm/test_monitor.py python_tests = $(python_tests_integration) $(python_tests_legacy) $(python_tests_unit) python_test_support = \ test/py/legacy/__init__.py \ test/py/legacy/lockperf.py \ test/py/legacy/testutils_ssh.py \ test/py/legacy/mocks.py \ test/py/legacy/testutils/__init__.py \ test/py/legacy/testutils/config_mock.py \ test/py/legacy/cmdlib/__init__.py \ test/py/legacy/cmdlib/testsupport/__init__.py \ test/py/legacy/cmdlib/testsupport/cmdlib_testcase.py \ test/py/legacy/cmdlib/testsupport/iallocator_mock.py \ test/py/legacy/cmdlib/testsupport/livelock_mock.py \ test/py/legacy/cmdlib/testsupport/netutils_mock.py \ test/py/legacy/cmdlib/testsupport/pathutils_mock.py \ test/py/legacy/cmdlib/testsupport/processor_mock.py \ test/py/legacy/cmdlib/testsupport/rpc_runner_mock.py \ test/py/legacy/cmdlib/testsupport/ssh_mock.py \ test/py/legacy/cmdlib/testsupport/utils_mock.py \ test/py/legacy/cmdlib/testsupport/util.py \ test/py/legacy/cmdlib/testsupport/wconfd_mock.py haskell_tests = exe/htest dist_TESTS = \ test/py/legacy/check-cert-expired_unittest.bash \ test/py/legacy/daemon-util_unittest.bash \ test/py/legacy/systemd_unittest.bash \ test/py/legacy/ganeti-cleaner_unittest.bash \ test/py/legacy/import-export_unittest.bash \ test/py/legacy/cli-test.bash \ test/py/legacy/bash_completion.bash if PY_UNIT dist_TESTS += $(python_tests) endif nodist_TESTS = check_SCRIPTS = if WANT_HSTESTS nodist_TESTS += $(haskell_tests) # test dependency test/hs/offline-test.sh: test/hs/hpc-htools test/hs/hpc-mon-collector dist_TESTS += test/hs/offline-test.sh check_SCRIPTS += \ test/hs/hpc-htools \ test/hs/hpc-mon-collector \ $(HS_BUILT_TEST_HELPERS) endif test/hs/hpc-htools: exe/htools $(LN_S) ../../$< $@ test/hs/hpc-mon-collector: exe/mon-collector $(LN_S) ../../$< $@ TESTS = $(dist_TESTS) $(nodist_TESTS) # Environment for all tests PLAIN_TESTS_ENVIRONMENT = \ PYTHONPATH=.:./test/py/legacy \ TOP_SRCDIR=$(abs_top_srcdir) TOP_BUILDDIR=$(abs_top_builddir) \ PYTHON=$(PYTHON) FAKEROOT=$(FAKEROOT_PATH) \ $(RUN_IN_TEMPDIR) # Environment for tests run by automake TESTS_ENVIRONMENT = \ $(PLAIN_TESTS_ENVIRONMENT) $(abs_top_srcdir)/autotools/testrunner all_python_code = \ $(dist_sbin_SCRIPTS) \ $(python_scripts) \ $(pkglib_python_scripts) \ $(nodist_pkglib_python_scripts) \ $(nodist_tools_python_scripts) \ $(pkgpython_PYTHON) \ $(client_PYTHON) \ $(cmdlib_PYTHON) \ $(cmdlib_cluster_PYTHON) \ $(config_PYTHON) \ $(hypervisor_PYTHON) \ $(hypervisor_hv_kvm_PYTHON) \ $(jqueue_PYTHON) \ $(storage_PYTHON) \ $(rapi_PYTHON) \ $(server_PYTHON) \ $(rpc_PYTHON) \ $(rpc_stub_PYTHON) \ $(pytools_PYTHON) \ $(http_PYTHON) \ $(confd_PYTHON) \ $(masterd_PYTHON) \ $(impexpd_PYTHON) \ $(utils_PYTHON) \ $(watcher_PYTHON) \ $(noinst_PYTHON) \ $(qa_scripts) if PY_UNIT all_python_code += $(python_tests) all_python_code += $(python_test_support) all_python_code += $(python_test_utils) endif srclink_files = \ man/footer.rst \ test/py/legacy/check-cert-expired_unittest.bash \ test/py/legacy/daemon-util_unittest.bash \ test/py/legacy/systemd_unittest.bash \ test/py/legacy/ganeti-cleaner_unittest.bash \ test/py/legacy/import-export_unittest.bash \ test/py/legacy/cli-test.bash \ test/py/legacy/bash_completion.bash \ test/hs/htest.hs \ test/hs/offline-test.sh \ test/hs/cli-tests-defs.sh \ $(all_python_code) \ $(HS_LIBTEST_SRCS) $(HS_PROG_SRCS) \ $(docinput) check_python_code = \ $(BUILD_BASH_COMPLETION) \ $(CHECK_IMPORTS) \ $(CHECK_HEADER) \ $(DOCPP) \ $(all_python_code) lint_python_code = \ ganeti \ ganeti/http/server.py \ $(dist_sbin_SCRIPTS) \ $(python_scripts) \ $(pkglib_python_scripts) \ $(BUILD_BASH_COMPLETION) \ $(CHECK_IMPORTS) \ $(CHECK_HEADER) \ $(DOCPP) \ $(gnt_python_sbin_SCRIPTS) \ $(PYTHON_BOOTSTRAP) standalone_python_modules = \ lib/rapi/client.py \ tools/ganeti-listrunner pycodestyle_python_code = \ ganeti \ ganeti/http/server.py \ $(dist_sbin_SCRIPTS) \ $(python_scripts) \ $(pkglib_python_scripts) \ $(BUILD_BASH_COMPLETION) \ $(CHECK_HEADER) \ $(DOCPP) \ $(PYTHON_BOOTSTRAP) \ $(gnt_python_sbin_SCRIPTS) \ qa \ $(python_test_support) $(python_test_utils) test/py/daemon-util_unittest.bash: daemons/daemon-util test/py/systemd_unittest.bash: daemons/daemon-util $(BUILT_EXAMPLES) test/py/ganeti-cleaner_unittest.bash: daemons/ganeti-cleaner test/py/bash_completion.bash: doc/examples/bash_completion-debug tools/kvm-ifup: tools/kvm-ifup.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) < $< > $@ chmod +x $@ tools/kvm-ifup-os: tools/ifup-os.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) -e "s/ifup-os:/kvm-ifup-os:/" < $< > $@ chmod +x $@ tools/xen-ifup-os: tools/ifup-os.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) -e "s/ifup-os:/xen-ifup-os:/" < $< > $@ chmod +x $@ tools/vif-ganeti: tools/vif-ganeti.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) < $< > $@ chmod +x $@ tools/vif-ganeti-metad: tools/vif-ganeti-metad.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) < $< > $@ chmod +x $@ tools/net-common: tools/net-common.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) < $< > $@ chmod +x $@ tools/users-setup: Makefile $(userspecs) set -e; \ { echo '#!/bin/sh'; \ echo 'if [ "x$$1" != "x--yes-do-it" ];'; \ echo 'then echo "This will do the following changes"'; \ $(AWK) -- '{print "echo + Will add group ",$$1; count++}\ END {if (count == 0) {print "echo + No groups to add"}}' doc/users/groups; \ $(AWK) -- '{if (NF > 1) {print "echo + Will add user",$$1,"with primary group",$$2} \ else {print "echo + Will add user",$$1}; count++}\ END {if (count == 0) {print "echo + No users to add"}}' doc/users/users; \ $(AWK) -- '{print "echo + Will add user",$$1,"to group",$$2}' doc/users/groupmemberships; \ echo 'echo'; \ echo 'echo "OK? (y/n)"'; \ echo 'read confirm'; \ echo 'if [ "x$$confirm" != "xy" ]; then exit 0; fi'; \ echo 'fi'; \ $(AWK) -- '{print "groupadd --system",$$1}' doc/users/groups; \ $(AWK) -- '{if (NF > 1) {print "useradd --system --gid",$$2,$$1} else {print "useradd --system",$$1}}' doc/users/users; \ $(AWK) -- '{print "usermod --append --groups",$$2,$$1}' doc/users/groupmemberships; \ } > $@ chmod +x $@ tools/vcluster-setup: tools/vcluster-setup.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) < $< > $@ chmod +x $@ daemons/%:: daemons/%.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) < $< > $@ chmod +x $@ doc/examples/%:: doc/examples/%.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) < $< > $@ doc/examples/bash_completion: BC_ARGS = --compact doc/examples/bash_completion-debug: BC_ARGS = doc/examples/bash_completion doc/examples/bash_completion-debug: \ $(BUILD_BASH_COMPLETION) $(RUN_IN_TEMPDIR) \ lib/cli.py $(gnt_scripts) $(client_PYTHON) tools/burnin \ daemons/ganeti-cleaner \ $(GENERATED_FILES) $(HS_GENERATED_FILES) PYTHONPATH=. $(RUN_IN_TEMPDIR) \ $(CURDIR)/$(BUILD_BASH_COMPLETION) $(BC_ARGS) > $@ man/%.gen: man/%.rst lib/query.py lib/build/sphinx_ext.py \ lib/build/shell_example_lexer.py \ | $(RUN_IN_TEMPDIR) $(built_python_sources) @echo "Checking $< for hardcoded paths..." @if grep -nEf autotools/wrong-hardcoded-paths $<; then \ echo "Man page $< has hardcoded paths (see above)!" 1>&2 ; \ exit 1; \ fi set -e ; \ trap 'echo auto-removing $@; rm $@' EXIT; \ PYTHONPATH=. $(RUN_IN_TEMPDIR) $(CURDIR)/$(DOCPP) < $< | \ sed -f $(REPLACE_VARS_SED) > $@ ;\ $(CHECK_MAN_REFERENCES) $@; \ trap - EXIT man/%.7 man/%.8 man/%.1: man/%.gen man/footer.rst @test -n "$(PANDOC)" || \ { echo 'pandoc' not found during configure; exit 1; } set -o pipefail -e; \ trap 'echo auto-removing $@; rm $@' EXIT; \ $(PANDOC) -s -f rst -t man $< man/footer.rst > $@; \ if test -n "$(MAN_HAS_WARNINGS)"; then LC_ALL=$(UTF8_LOCALE) $(CHECK_MAN_WARNINGS) $@; fi; \ $(CHECK_MAN_DASHES) $@; \ trap - EXIT man/%.html: man/%.gen man/footer.rst @test -n "$(PANDOC)" || \ { echo 'pandoc' not found during configure; exit 1; } set -o pipefail ; \ $(PANDOC) --toc -s -f rst -t html $< man/footer.rst > $@ vcs-version: if test -d $(top_srcdir)/.git; then \ git -C $(top_srcdir) describe | tr '"' - > $@; \ elif test ! -f $@ ; then \ echo "Cannot auto-generate $@ file"; exit 1; \ fi .PHONY: clean-vcs-version clean-vcs-version: rm -f vcs-version .PHONY: regen-vcs-version regen-vcs-version: @set -e; \ cd $(srcdir); \ if test -d .git; then \ T=`mktemp` ; trap 'rm -f $$T' EXIT; \ git describe > $$T; \ if ! cmp --quiet $$T vcs-version; then \ mv $$T vcs-version; \ fi; \ fi src/Ganeti/Version.hs: src/Ganeti/Version.hs.in \ vcs-version $(built_base_sources) set -e; \ VCSVER=`cat $(srcdir)/vcs-version`; \ sed -e 's"%ver%"'"$$VCSVER"'"' < $< > $@ src/Ganeti/Hs2Py/ListConstants.hs: src/Ganeti/Hs2Py/ListConstants.hs.in \ src/Ganeti/Constants.hs \ | stamp-directories @echo Generating $@ @set -e; \ ## Extract constant names from 'Constants.hs' by extracting the left ## side of all lines containing an equal sign (i.e., '=') and ## prepending the apostrophe sign (i.e., "'"). ## ## For example, the constant ## adminstDown = ... ## becomes ## 'adminstDown NAMES=$$(sed -e "/^--/ d" $(abs_top_srcdir)/src/Ganeti/Constants.hs |\ sed -n -e "/=/ s/\(.*\) =.*/ '\1:/g p"); \ m4 -DPY_CONSTANT_NAMES="$$NAMES" \ $(abs_top_srcdir)/src/Ganeti/Hs2Py/ListConstants.hs.in > $@ test/hs/Test/Ganeti/TestImports.hs: test/hs/Test/Ganeti/TestImports.hs.in \ $(built_base_sources) set -e; \ { cat $< ; \ echo ; \ for name in $(filter-out Ganeti.THH,$(subst /,.,$(patsubst src/%,%,$(HS_LIB_STEMS)))) ; do \ echo "import $$name ()" ; \ done ; \ echo "import Ganeti.Query.RegEx ()" ; \ } > $@ lib/_constants.py: Makefile $(HS2PY_PROG) lib/_constants.py.in | stamp-directories cat $(abs_top_srcdir)/lib/_constants.py.in > $@ $(HS2PY_PROG) --constants >> $@ lib/constants.py: lib/_constants.py src/AutoConf.hs: Makefile src/AutoConf.hs.in $(PRINT_PY_CONSTANTS) \ | $(built_base_sources) @echo "m4 ... >" $@ @m4 -DPACKAGE_VERSION="$(PACKAGE_VERSION)" \ -DVERSION_MAJOR="$(VERSION_MAJOR)" \ -DVERSION_MINOR="$(VERSION_MINOR)" \ -DVERSION_REVISION="$(VERSION_REVISION)" \ -DVERSION_SUFFIX="$(VERSION_SUFFIX)" \ -DVERSION_FULL="$(VERSION_FULL)" \ -DDIRVERSION="$(DIRVERSION)" \ -DLOCALSTATEDIR="$(localstatedir)" \ -DSYSCONFDIR="$(sysconfdir)" \ -DSSH_CONFIG_DIR="$(SSH_CONFIG_DIR)" \ -DSSH_LOGIN_USER="$(SSH_LOGIN_USER)" \ -DSSH_CONSOLE_USER="$(SSH_CONSOLE_USER)" \ -DEXPORT_DIR="$(EXPORT_DIR)" \ -DBACKUP_DIR="$(backup_dir)" \ -DOS_SEARCH_PATH="\"$(OS_SEARCH_PATH)\"" \ -DES_SEARCH_PATH="\"$(ES_SEARCH_PATH)\"" \ -DXEN_BOOTLOADER="$(XEN_BOOTLOADER)" \ -DXEN_CONFIG_DIR="$(XEN_CONFIG_DIR)" \ -DXEN_KERNEL="$(XEN_KERNEL)" \ -DXEN_INITRD="$(XEN_INITRD)" \ -DKVM_KERNEL="$(KVM_KERNEL)" \ -DSHARED_FILE_STORAGE_DIR="$(SHARED_FILE_STORAGE_DIR)" \ -DIALLOCATOR_SEARCH_PATH="\"$(IALLOCATOR_SEARCH_PATH)\"" \ -DDEFAULT_BRIDGE="$(DEFAULT_BRIDGE)" \ -DDEFAULT_VG="$(DEFAULT_VG)" \ -DKVM_PATH="$(KVM_PATH)" \ -DIP_PATH="$(IP_PATH)" \ -DSOCAT_PATH="$(SOCAT)" \ -DPYTHON_PATH="$(PYTHON)" \ -DSOCAT_USE_ESCAPE="$(SOCAT_USE_ESCAPE)" \ -DSOCAT_USE_COMPRESS="$(SOCAT_USE_COMPRESS)" \ -DLVM_STRIPECOUNT="$(LVM_STRIPECOUNT)" \ -DTOOLSDIR="$(libdir)/ganeti/tools" \ -DGNT_SCRIPTS="$(foreach i,$(notdir $(gnt_scripts)),\"$(i)\":)" \ -DHS_HTOOLS_PROGS="$(foreach i,$(HS_HTOOLS_PROGS),\"$(i)\":)" \ -DPKGLIBDIR="$(libdir)/ganeti" \ -DSHAREDIR="$(prefix)/share/ganeti" \ -DVERSIONEDSHAREDIR="$(versionedsharedir)" \ -DDRBD_BARRIERS="$(DRBD_BARRIERS)" \ -DDRBD_NO_META_FLUSH="$(DRBD_NO_META_FLUSH)" \ -DSYSLOG_USAGE="$(SYSLOG_USAGE)" \ -DDAEMONS_GROUP="$(DAEMONS_GROUP)" \ -DADMIN_GROUP="$(ADMIN_GROUP)" \ -DMASTERD_USER="$(MASTERD_USER)" \ -DMASTERD_GROUP="$(MASTERD_GROUP)" \ -DMETAD_USER="$(METAD_USER)" \ -DMETAD_GROUP="$(METAD_GROUP)" \ -DRAPI_USER="$(RAPI_USER)" \ -DRAPI_GROUP="$(RAPI_GROUP)" \ -DCONFD_USER="$(CONFD_USER)" \ -DCONFD_GROUP="$(CONFD_GROUP)" \ -DWCONFD_USER="$(WCONFD_USER)" \ -DWCONFD_GROUP="$(WCONFD_GROUP)" \ -DKVMD_USER="$(KVMD_USER)" \ -DKVMD_GROUP="$(KVMD_GROUP)" \ -DLUXID_USER="$(LUXID_USER)" \ -DLUXID_GROUP="$(LUXID_GROUP)" \ -DNODED_USER="$(NODED_USER)" \ -DNODED_GROUP="$(NODED_GROUP)" \ -DMOND_USER="$(MOND_USER)" \ -DMOND_GROUP="$(MOND_GROUP)" \ -DMETAD_USER="$(METAD_USER)" \ -DMETAD_GROUP="$(METAD_GROUP)" \ -DDISK_SEPARATOR="$(DISK_SEPARATOR)" \ -DQEMUIMG_PATH="$(QEMUIMG_PATH)" \ -DENABLE_RESTRICTED_COMMANDS="$(ENABLE_RESTRICTED_COMMANDS)" \ -DENABLE_METADATA="$(ENABLE_METADATA)" \ -DENABLE_MOND="$(ENABLE_MOND)" \ -DHAS_GNU_LN="$(HAS_GNU_LN)" \ -DMAN_PAGES="$$(for i in $(notdir $(man_MANS)); do \ echo -n "$$i" | sed -re 's/^(.*)\.([0-9]+)$$/("\1",\2):/g'; \ done)" \ -DAF_INET4="$$(PYTHONPATH=. $(PYTHON) $(PRINT_PY_CONSTANTS) AF_INET4)" \ -DAF_INET6="$$(PYTHONPATH=. $(PYTHON) $(PRINT_PY_CONSTANTS) AF_INET6)" \ $(abs_top_srcdir)/src/AutoConf.hs.in > $@ lib/_vcsversion.py: Makefile vcs-version | stamp-directories set -e; \ VCSVER=`cat $(srcdir)/vcs-version`; \ { echo '# This file is automatically generated, do not edit!'; \ echo '#'; \ echo ''; \ echo '"""Build-time VCS version number for Ganeti.'; \ echo '';\ echo 'This file is autogenerated by the build process.'; \ echo 'For any changes you need to re-run ./configure (and'; \ echo 'not edit by hand).'; \ echo ''; \ echo '"""'; \ echo ''; \ echo '# pylint: disable=C0301,C0324'; \ echo '# because this is autogenerated, we do not want'; \ echo '# style warnings' ; \ echo ''; \ echo "VCS_VERSION = '$$VCSVER'"; \ } > $@ lib/opcodes.py: Makefile $(HS2PY_PROG) lib/opcodes.py.in_before \ lib/opcodes.py.in_after | stamp-directories cat $(abs_top_srcdir)/lib/opcodes.py.in_before > $@ $(HS2PY_PROG) --opcodes >> $@ cat $(abs_top_srcdir)/lib/opcodes.py.in_after >> $@ # Generating the RPC wrappers depends on many things, so make sure # it's built at the end of the built sources lib/_generated_rpc.py: lib/rpc_defs.py $(BUILD_RPC) | $(built_base_sources) $(built_python_base_sources) PYTHONPATH=. $(RUN_IN_TEMPDIR) $(CURDIR)/$(BUILD_RPC) lib/rpc_defs.py > $@ if ENABLE_METADATA lib/rpc/stub/metad.py: Makefile $(HS2PY_PROG) | stamp-directories $(HS2PY_PROG) --metad-rpc > $@ endif lib/rpc/stub/wconfd.py: Makefile $(HS2PY_PROG) | stamp-directories $(HS2PY_PROG) --wconfd-rpc > $@ $(SHELL_ENV_INIT): Makefile stamp-directories set -e; \ { echo '# Allow overriding for tests'; \ echo 'readonly LOCALSTATEDIR=$${LOCALSTATEDIR:-$${GANETI_ROOTDIR:-}$(localstatedir)}'; \ echo 'readonly SYSCONFDIR=$${SYSCONFDIR:-$${GANETI_ROOTDIR:-}$(sysconfdir)}'; \ echo; \ echo 'readonly PKGLIBDIR=$(libdir)/ganeti'; \ echo 'readonly LOG_DIR="$$LOCALSTATEDIR/log/ganeti"'; \ echo 'readonly RUN_DIR="$$LOCALSTATEDIR/run/ganeti"'; \ echo 'readonly DATA_DIR="$$LOCALSTATEDIR/lib/ganeti"'; \ echo 'readonly CONF_DIR="$$SYSCONFDIR/ganeti"'; \ } > $@ ## Writes sed script to replace placeholders with build-time values. The ## additional quotes after the first @ sign are necessary to stop configure ## from replacing those values as well. $(REPLACE_VARS_SED): $(SHELL_ENV_INIT) Makefile stamp-directories set -e; \ { echo 's#@''PREFIX@#$(prefix)#g'; \ echo 's#@''SYSCONFDIR@#$(sysconfdir)#g'; \ echo 's#@''LOCALSTATEDIR@#$(localstatedir)#g'; \ echo 's#@''BINDIR@#$(BINDIR)#g'; \ echo 's#@''SBINDIR@#$(SBINDIR)#g'; \ echo 's#@''LIBDIR@#$(libdir)#g'; \ echo 's#@''GANETI_VERSION@#$(PACKAGE_VERSION)#g'; \ echo 's#@''CUSTOM_XEN_BOOTLOADER@#$(XEN_BOOTLOADER)#g'; \ echo 's#@''CUSTOM_XEN_KERNEL@#$(XEN_KERNEL)#g'; \ echo 's#@''CUSTOM_XEN_INITRD@#$(XEN_INITRD)#g'; \ echo 's#@''CUSTOM_IALLOCATOR_SEARCH_PATH@#$(IALLOCATOR_SEARCH_PATH)#g'; \ echo 's#@''CUSTOM_EXPORT_DIR@#$(EXPORT_DIR)#g'; \ echo 's#@''RPL_SSHD_RESTART_COMMAND@#$(SSHD_RESTART_COMMAND)#g'; \ echo 's#@''PKGLIBDIR@#$(libdir)/ganeti#g'; \ echo 's#@''GNTMASTERUSER@#$(MASTERD_USER)#g'; \ echo 's#@''GNTRAPIUSER@#$(RAPI_USER)#g'; \ echo 's#@''GNTCONFDUSER@#$(CONFD_USER)#g'; \ echo 's#@''GNTWCONFDUSER@#$(WCONFD_USER)#g'; \ echo 's#@''GNTLUXIDUSER@#$(LUXID_USER)#g'; \ echo 's#@''GNTNODEDUSER@#$(NODED_USER)#g'; \ echo 's#@''GNTMONDUSER@#$(MOND_USER)#g'; \ echo 's#@''GNTMETADUSER@#$(METAD_USER)#g'; \ echo 's#@''GNTRAPIGROUP@#$(RAPI_GROUP)#g'; \ echo 's#@''GNTADMINGROUP@#$(ADMIN_GROUP)#g'; \ echo 's#@''GNTCONFDGROUP@#$(CONFD_GROUP)#g'; \ echo 's#@''GNTNODEDGROUP@#$(NODED_GROUP)#g'; \ echo 's#@''GNTWCONFDGROUP@#$(CONFD_GROUP)#g'; \ echo 's#@''GNTLUXIDGROUP@#$(LUXID_GROUP)#g'; \ echo 's#@''GNTMASTERDGROUP@#$(MASTERD_GROUP)#g'; \ echo 's#@''GNTMONDGROUP@#$(MOND_GROUP)#g'; \ echo 's#@''GNTMETADGROUP@#$(METAD_GROUP)#g'; \ echo 's#@''GNTDAEMONSGROUP@#$(DAEMONS_GROUP)#g'; \ echo 's#@''CUSTOM_ENABLE_MOND@#$(ENABLE_MOND)#g'; \ echo 's#@''XEN_CONFIG_DIR@#$(XEN_CONFIG_DIR)#g'; \ echo; \ echo '/^@SHELL_ENV_INIT@$$/ {'; \ echo ' r $(SHELL_ENV_INIT)'; \ echo ' d'; \ echo '}'; \ } > $@ # Using deferred evaluation daemons/ganeti-%: MODULE = ganeti.server.$(patsubst ganeti-%,%,$(notdir $@)) daemons/ganeti-watcher: MODULE = ganeti.watcher scripts/%: MODULE = ganeti.client.$(subst -,_,$(notdir $@)) tools/burnin: MODULE = ganeti.tools.burnin tools/ensure-dirs: MODULE = ganeti.tools.ensure_dirs tools/node-daemon-setup: MODULE = ganeti.tools.node_daemon_setup tools/prepare-node-join: MODULE = ganeti.tools.prepare_node_join tools/ssh-update: MODULE = ganeti.tools.ssh_update tools/node-cleanup: MODULE = ganeti.tools.node_cleanup tools/ssl-update: MODULE = ganeti.tools.ssl_update $(HS_BUILT_TEST_HELPERS): TESTROLE = $(patsubst test/hs/%,%,$@) $(PYTHON_BOOTSTRAP) $(gnt_scripts) $(gnt_python_sbin_SCRIPTS): Makefile | stamp-directories test -n "$(MODULE)" || { echo Missing module; exit 1; } set -e; \ { echo '#!${PYTHON}'; \ echo '# This file is automatically generated, do not edit!'; \ echo "# Edit $(MODULE) instead."; \ echo; \ echo '"""Bootstrap script for L{$(MODULE)}"""'; \ echo; \ echo '# pylint: disable=C0103'; \ echo '# C0103: Invalid name'; \ echo; \ echo 'import sys'; \ echo 'import $(MODULE) as main'; \ echo; \ echo '# Temporarily alias commands until bash completion'; \ echo '# generator is changed'; \ echo 'if hasattr(main, "commands"):'; \ echo ' commands = main.commands # pylint: disable=E1101'; \ echo 'if hasattr(main, "aliases"):'; \ echo ' aliases = main.aliases # pylint: disable=E1101'; \ echo; \ echo 'if __name__ == "__main__":'; \ echo ' sys.exit(main.Main())'; \ } > $@ chmod u+x $@ $(HS_BUILT_TEST_HELPERS): Makefile @test -n "$(TESTROLE)" || { echo Missing TESTROLE; exit 1; } set -e; \ { echo '#!/bin/sh'; \ echo '# This file is automatically generated, do not edit!'; \ echo "# Edit Makefile.am instead."; \ echo; \ echo "HTOOLS=$(TESTROLE) exec ./test/hs/hpc-htools \"\$$@\""; \ } > $@ chmod u+x $@ stamp-directories: Makefile $(MAKE) $(AM_MAKEFLAGS) ganeti @mkdir_p@ $(DIRS) $(BUILDTIME_DIR_AUTOCREATE) touch $@ # We need to create symlinks because "make distcheck" will not install Python # files when building. stamp-srclinks: Makefile | stamp-directories set -e; \ for i in $(srclink_files); do \ if test ! -f $$i -a -f $(abs_top_srcdir)/$$i; then \ $(LN_S) $(abs_top_srcdir)/$$i $$i; \ fi; \ done touch $@ .PHONY: ganeti ganeti: cd $(top_builddir) && test -h "$@" || { rm -f $@ && $(LN_S) lib $@; } .PHONY: check-dirs check-dirs: $(GENERATED_FILES) @set -e; \ find . -type d \( -name . -o -name .git -prune -o -print \) | { \ error=; \ while read dir; do \ case "$$dir" in \ $(strip $(patsubst %,(./%) ;;,$(DIRCHECK_EXCLUDE) $(DIRS))) \ *) error=1; echo "Directory $$dir not listed in Makefile" >&2 ;; \ esac; \ done; \ for dir in $(DIRS); do \ if ! test -d "$$dir"; then \ echo "Directory $$dir listed in DIRS does not exist" >&2; \ error=1; \ fi \ done; \ test -z "$$error"; \ } .PHONY: check-news check-news: RELEASE=$(PACKAGE_VERSION) $(CHECK_NEWS) < $(top_srcdir)/NEWS .PHONY: check-local check-local: check-dirs check-news $(GENERATED_FILES) $(CHECK_PYTHON_CODE) $(check_python_code) PYTHONPATH=. $(CHECK_HEADER) \ $(filter-out $(GENERATED_FILES),$(check_python_code)) $(CHECK_VERSION) $(VERSION) $(top_srcdir)/NEWS PYTHONPATH=. $(RUN_IN_TEMPDIR) $(CURDIR)/$(CHECK_IMPORTS) . $(standalone_python_modules) error= ; \ if [ "x`echo $(VERSION_SUFFIX)|grep 'alpha'`" == "x" ]; then \ expver=$(VERSION_MAJOR).$(VERSION_MINOR); \ if test "`head -n 1 $(top_srcdir)/README.md`" != "# Ganeti $$expver"; then \ echo "Incorrect version in README.md, expected $$expver" >&2; \ error=1; \ fi; \ for file in doc/iallocator.rst doc/hooks.rst doc/virtual-cluster.rst \ doc/security.rst; do \ if test "`sed -ne '4 p' $(top_srcdir)/$$file`" != \ "Documents Ganeti version $$expver"; then \ echo "Incorrect version in $$file, expected $$expver" >&2; \ error=1; \ fi; \ done; \ if ! test -f $(top_srcdir)/doc/design-$$expver.rst; then \ echo "File $(top_srcdir)/doc/design-$$expver.rst not found" >&2; \ error=1; \ fi; \ if test "`sed -ne '5 p' $(top_srcdir)/doc/design-draft.rst`" != \ ".. Last updated for Ganeti $$expver"; then \ echo "doc/design-draft.rst was not updated for version $$expver" >&2; \ error=1; \ fi; \ fi; \ for file in configure.ac $(HS_LIBTEST_SRCS) $(HS_PROG_SRCS); do \ if test $$(wc --max-line-length < $(top_srcdir)/$$file) -gt 80; then \ echo "Longest line in $$file is longer than 80 characters" >&2; \ error=1; \ fi; \ done; \ test -z "$$error" .PHONY: hs-test-% hs-test-%: exe/htest @rm -f htest.tix $< -t $* .PHONY: hs-tests hs-tests: exe/htest @rm -f htest.tix $< .PHONY: py-tests-legacy py-tests-legacy: $(python_tests_legacy) ganeti $(built_python_sources) @if [ "$(PY_NODEV)" ]; then \ echo "Error: cannot run unittests without the development" \ " libraries (see devnotes.rst)" 1>&2; \ exit 1; \ fi error=; \ for file in $(python_tests_legacy); \ do if ! $(TESTS_ENVIRONMENT) $$file; then error=1; fi; \ done; \ test -z "$$error" .PHONY: py-tests-unit py-tests-unit: ganeti $(built_python_sources) PYTHONPATH=: $(RUN_IN_TEMPDIR) pytest-3 ./test/py/unit/ .PHONY: py-tests-integration py-tests-integration: ganeti $(built_python_sources) PYTHONPATH=: $(RUN_IN_TEMPDIR) pytest-3 ./test/py/integration/ .PHONY: hs-shell-% hs-shell-%: test/hs/hpc-htools test/hs/hpc-mon-collector \ $(HS_BUILT_TEST_HELPERS) @rm -f hpc-htools.tix hpc-mon-collector.tix HBINARY="./test/hs/hpc-htools" \ SHELLTESTARGS=$(SHELLTESTARGS) \ ./test/hs/offline-test.sh $* .PHONY: hs-shell hs-shell: test/hs/hpc-htools test/hs/hpc-mon-collector $(HS_BUILT_TEST_HELPERS) @rm -f hpc-htools.tix hpc-mon-collector.tix HBINARY="./test/hs/hpc-htools" \ SHELLTESTARGS=$(SHELLTESTARGS) \ ./test/hs/offline-test.sh .PHONY: hs-check hs-check: hs-tests hs-shell # E111: indentation is not a multiple of four # E114: indentation is not a multiple of four (comment) # E121: continuation line indentation is not a multiple of four # (since our indent level is not 4) # E125: continuation line does not distinguish itself from next logical line # (since our indent level is not 4) # E123: closing bracket does not match indentation of opening bracket's line # E127: continuation line over-indented for visual indent # (since our indent level is not 4) # note: do NOT add E128 here; it's a valid style error in most cases! # I've seen real errors, but also some cases were we indent wrongly # due to line length; try to rework the cases where it is triggered, # instead of silencing it # E261: at least two spaces before inline comment # E501: line too long (80 characters) PEP8_IGNORE = E111,E114,E121,E123,E125,E127,E261,E501 # For excluding pep8 expects filenames only, not whole paths PEP8_EXCLUDE = $(subst $(space),$(comma),$(strip $(notdir $(built_python_sources)))) # A space-separated list of pylint warnings to completely ignore: # I0013 = disable warnings for ignoring whole files # R0912 = disable too many branches warning. It's useful, but ganeti requires # a lot of refactoring to fix this. # R0204 = disable redefined-variable-type warning. There are a large number of # cases where Ganeti assigns multiple types (eg set/list, float/int) to # the same variable, and these are benign. # C0325 = disable superfluous-parens. There are a lot of cases where this is # overzealous, eg where we use parens to make it clear that we're # deliberately doing a comparison that should yield bool, or are using # parens clarify precedence or to allow multi-line expressions. # C0330 = disable wrong indentation warnings. pylint is much more strict than # pep8, and it would be too invasive to fix all these. LINT_DISABLE = I0013 R0912 R0204 C0325 C0330 # Additional pylint options LINT_OPTS = # The combined set of pylint options LINT_OPTS_ALL = $(LINT_OPTS) \ $(addprefix --disable=,$(LINT_DISABLE)) # Whitelist loading pycurl C extension for attribute checking LINT_OPTS_ALL += --extension-pkg-whitelist=pycurl LINT_TARGETS = pylint pylint-qa pylint-test if HAS_PYCODESTYLE LINT_TARGETS += pycodestyle endif if HAS_HLINT LINT_TARGETS += hlint endif .PHONY: lint lint: $(LINT_TARGETS) .PHONY: pylint pylint: $(GENERATED_FILES) @test -n "$(PYLINT)" || { echo 'pylint' not found during configure; exit 1; } $(PYLINT) $(LINT_OPTS_ALL) $(lint_python_code) .PHONY: pylint-qa pylint-qa: $(GENERATED_FILES) @test -n "$(PYLINT)" || { echo 'pylint' not found during configure; exit 1; } cd $(top_srcdir)/qa && \ PYTHONPATH=$(abs_top_srcdir) $(PYLINT) $(LINT_OPTS_ALL) \ --rcfile ../pylintrc $(patsubst qa/%.py,%,$(qa_scripts)) # FIXME: lint all test code, not just the newly added test support pylint-test: $(GENERATED_FILES) @test -n "$(PYLINT)" || { echo 'pylint' not found during configure; exit 1; } cd $(top_srcdir) && \ PYTHONPATH=.:./test/py $(PYLINT) $(LINT_OPTS_ALL) \ --rcfile=pylintrc-test $(python_test_support) $(python_test_utils) .PHONY: pycodestyle pycodestyle: $(GENERATED_FILES) @test -n "$(PYCODESTYLE)" || { echo 'pycodestyle' not found during configure; exit 1; } $(PYCODESTYLE) --ignore='$(PEP8_IGNORE)' --exclude='$(PEP8_EXCLUDE)' \ --repeat $(pycodestyle_python_code) pycodestyle-stats: $(GENERATED_FILES) @test -n "$(PYCODESTYLE)" || { echo 'pycodestyle' not found during configure; exit 1; } $(PYCODESTYLE) --ignore='$(PEP8_IGNORE)' --exclude='$(PEP8_EXCLUDE)' \ --repeat --statistics $(pycodestyle_python_code) HLINT_EXCLUDES = src/Ganeti/THH.hs test/hs/hpc-htools.hs .PHONY: hlint hlint: $(HS_BUILT_SRCS) src/lint-hints.hs @test -n "$(HLINT)" || { echo 'hlint' not found during configure; exit 1; } @rm -f doc/hs-lint.html if tty -s; then C="-c"; else C=""; fi; \ $(HLINT) --report=doc/hs-lint.html --cross $$C \ --hint src/lint-hints \ --cpp-file=$(HASKELL_PACKAGE_VERSIONS_FILE) \ $(filter-out $(HLINT_EXCLUDES),$(HS_LIBTEST_SRCS) $(HS_PROG_SRCS)) @if [ ! -f doc/hs-lint.html ]; then \ echo "All good" > doc/hs-lint.html; \ fi # a dist hook rule for updating the vcs-version file; this is # hardcoded due to where it needs to build the file... dist-hook: $(MAKE) $(AM_MAKEFLAGS) regen-vcs-version rm -f $(top_distdir)/vcs-version cp -p $(srcdir)/vcs-version $(top_distdir) # a distcheck hook rule for catching revision control directories distcheck-hook: if find $(top_distdir) -name .svn -or -name .git | grep .; then \ echo "Found revision control files in final archive." 1>&2; \ exit 1; \ fi if find $(top_distdir) -name '*.py[co]' | grep .; then \ echo "Found Python byte code in final archive." 1>&2; \ exit 1; \ fi if find $(top_distdir) -name '*~' | grep .; then \ echo "Found backup files in final archive." 1>&2; \ exit 1; \ fi # Empty files or directories should not be distributed. They can cause # unnecessary warnings for packagers. Directories used by automake during # distcheck must be excluded. if find $(top_distdir) -empty -and -not \( \ -regex '$(top_distdir)/_build\(/.*\)?' -or \ -path $(top_distdir)/_inst \) | grep .; then \ echo "Found empty files or directories in final archive." 1>&2; \ exit 1; \ fi if test -e $(top_distdir)/doc/man-html; then \ echo "Found documentation including man pages in final archive" >&2; \ exit 1; \ fi # Backwards compatible distcheck-release target distcheck-release: distcheck distrebuildcheck: dist set -e; \ builddir=$$(mktemp -d $(abs_srcdir)/distrebuildcheck.XXXXXXX); \ trap "echo Removing $$builddir; cd $(abs_srcdir); rm -rf $$builddir" EXIT; \ cd $$builddir; \ tar xzf $(abs_srcdir)/$(distdir).tar.gz; \ cd $(distdir); \ ./configure; \ $(MAKE) maintainer-clean; \ cp $(abs_srcdir)/vcs-version .; \ ./configure; \ $(MAKE) $(AM_MAKEFLAGS) dist-release: dist set -e; \ for i in $(DIST_ARCHIVES); do \ echo -n "Checking $$i ... "; \ autotools/check-tar < $$i; \ echo OK; \ done install-exec-local: @mkdir_p@ "$(DESTDIR)${localstatedir}/lib/ganeti" \ "$(DESTDIR)${localstatedir}/log/ganeti" \ "$(DESTDIR)${localstatedir}/run/ganeti" for dir in $(SYMLINK_TARGET_DIRS); do \ @mkdir_p@ $(DESTDIR)$$dir; \ done $(LN_S) -f $(sysconfdir)/ganeti/lib $(DESTDIR)$(defaultversiondir) $(LN_S) -f $(sysconfdir)/ganeti/share $(DESTDIR)$(defaultversionedsharedir) for prog in $(HS_BIN_ROLES); do \ $(LN_S) -f $(defaultversiondir)$(BINDIR)/$$prog $(DESTDIR)$(BINDIR)/$$prog; \ done $(LN_S) -f $(defaultversiondir)$(libdir)/ganeti/iallocators/hail $(DESTDIR)$(libdir)/ganeti/iallocators/hail for prog in $(all_sbin_scripts); do \ $(LN_S) -f $(defaultversiondir)$(SBINDIR)/$$prog $(DESTDIR)$(SBINDIR)/$$prog; \ done for prog in $(gnt_scripts_basenames); do \ $(LN_S) -f $(defaultversionedsharedir)/$$prog $(DESTDIR)$(SBINDIR)/$$prog; \ done for prog in $(pkglib_python_basenames); do \ $(LN_S) -f $(defaultversionedsharedir)/$$prog $(DESTDIR)$(libdir)/ganeti/$$prog; \ done for prog in $(tools_python_basenames); do \ $(LN_S) -f $(defaultversionedsharedir)/$$prog $(DESTDIR)$(libdir)/ganeti/tools/$$prog; \ done for prog in $(tools_basenames); do \ $(LN_S) -f $(defaultversiondir)/$(libdir)/ganeti/tools/$$prog $(DESTDIR)$(libdir)/ganeti/tools/$$prog; \ done if ! test -n '$(ENABLE_MANPAGES)'; then \ for man in $(manfullpath); do \ $(LN_S) -f $(defaultversionedsharedir)/root$(MANDIR)/$$man $(DESTDIR)$(MANDIR)/$$man; \ done; \ fi for prog in $(myexeclib_scripts_basenames); do \ $(LN_S) -f $(defaultversiondir)$(libdir)/ganeti/$$prog $(DESTDIR)$(libdir)/ganeti/$$prog; \ done if INSTALL_SYMLINKS $(LN_S) -f $(versionedsharedir) $(DESTDIR)$(sysconfdir)/ganeti/share $(LN_S) -f $(versiondir) $(DESTDIR)$(sysconfdir)/ganeti/lib endif .PHONY: apidoc if WANT_HSAPIDOC apidoc: py-apidoc hs-apidoc else apidoc: py-apidoc endif .PHONY: py-apidoc py-apidoc: $(RUN_IN_TEMPDIR) pydoctor -v \ --config $(CURDIR)/pydoctor.ini \ --html-output $(CURDIR)/$(APIDOC_PY_DIR) \ $(lint_python_code) .PHONY: hs-apidoc hs-apidoc: $(APIDOC_HS_DIR)/index.html $(APIDOC_HS_DIR)/index.html: $(HS_LIBTESTBUILT_SRCS) Makefile @test -n "$(HSCOLOUR)" || \ { echo 'HsColour' not found during configure; exit 1; } @test -n "$(HADDOCK)" || \ { echo 'haddock' not found during configure; exit 1; } rm -rf $(APIDOC_HS_DIR) export LC_ALL=$(UTF8_LOCALE); \ $(CABAL_SETUP) haddock mv dist/doc/html/ganeti $(APIDOC_HS_DIR) rm -r dist/doc .PHONY: TAGS TAGS: $(GENERATED_FILES) rm -f TAGS $(GHC) -e ":etags TAGS_hs" -v0 \ $(filter-out -O -Werror,$(HFLAGS)) \ -osuf tags.o \ -hisuf tags.hi \ -lcurl \ $(HS_LIBTEST_SRCS) find . -path './lib/*.py' -o -path './scripts/gnt-*' -o \ -path './daemons/ganeti-*' -o -path './tools/*' -o \ -path './qa/*.py' | \ etags --etags-include=TAGS_hs -L - .PHONY: coverage COVERAGE_TESTS= if HS_UNIT COVERAGE_TESTS += hs-coverage endif if PY_UNIT COVERAGE_TESTS += py-coverage endif coverage: $(COVERAGE_TESTS) test/py/docs_unittest.py: $(gnt_scripts) .PHONY: py-coverage py-coverage: $(GENERATED_FILES) $(python_tests) @test -n "$(PYCOVERAGE)" || \ { echo 'python-coverage' not found during configure; exit 1; } set -e; \ COVERAGE=$(PYCOVERAGE) \ COVERAGE_FILE=$(CURDIR)/$(COVERAGE_PY_DIR)/data \ TEXT_COVERAGE=$(CURDIR)/$(COVERAGE_PY_DIR)/report.txt \ HTML_COVERAGE=$(CURDIR)/$(COVERAGE_PY_DIR) \ $(PLAIN_TESTS_ENVIRONMENT) \ $(abs_top_srcdir)/autotools/gen-py-coverage \ $(python_tests) .PHONY: hs-coverage hs-coverage: $(haskell_tests) test/hs/hpc-htools test/hs/hpc-mon-collector rm -f *.tix $(MAKE) $(AM_MAKEFLAGS) hs-check @mkdir_p@ $(COVERAGE_HS_DIR) hpc sum --union $(HPCEXCL) \ htest.tix hpc-htools.tix hpc-mon-collector.tix > coverage-hs.tix hpc markup --destdir=$(COVERAGE_HS_DIR) coverage-hs.tix hpc report coverage-hs.tix | tee $(COVERAGE_HS_DIR)/report.txt $(LN_S) -f hpc_index.html $(COVERAGE_HS_DIR)/index.html # Special "kind-of-QA" target for htools, needs special setup (all # tools compiled with -fhpc) .PHONY: live-test live-test: all set -e ; \ cd src; \ rm -f .hpc; $(LN_S) ../.hpc .hpc; \ rm -f *.tix *.mix; \ ./live-test.sh; \ hpc sum --union $(HPCEXCL) $(addsuffix .tix,$(HS_PROGS:exe/%=%)) \ --output=live-test.tix ; \ @mkdir_p@ ../$(COVERAGE_HS_DIR) ; \ hpc markup --destdir=../$(COVERAGE_HS_DIR) live-test \ --srcdir=.. $(HPCEXCL) ; \ hpc report --srcdir=.. live-test $(HPCEXCL) commit-check: autotools-check distcheck lint apidoc autotools-check: TESTDATA_DIR=./test/data shelltest $(SHELLTESTARGS) \ $(abs_top_srcdir)/test/autotools/*-*.test \ -- --hide-successes .PHONY: gitignore-check gitignore-check: @if [ -n "`git status --short`" ]; then \ echo "Git status is not clean!" 1>&2 ; \ git status --short; \ exit 1; \ fi # target to rebuild all man pages (both groff and html output) .PHONY: man man: $(man_MANS) $(manhtml) dist/setup-config: ganeti.cabal $(HS_BUILT_SRCS) $(CABAL_SETUP) configure --user \ --cabal-file=$(top_srcdir)/ganeti.cabal \ -f`test $(HTEST) == yes && echo "htest" || echo "-htest"` \ -f`test $(ENABLE_MOND) == True && echo "mond" || echo "-mond"` \ -f`test $(ENABLE_METADATA) == True && echo "metad" || echo "-metad"` \ -f`test $(ENABLE_NETWORK_BSD) == True && echo "network_bsd" || echo "-network_bsd"` \ -f`test $(HS_PCRE_BACKEND) == pcre-builtin && echo "regex-pcre-builtin" || echo "-regex-pcre-builtin"` \ -f`test $(HS_PCRE_BACKEND) == tdfa && echo "regex-tdfa" || echo "-regex-tdfa"` \ -f`test $(HS_PCRE_BACKEND) == pcre2 && echo "regex-pcre2" || echo "-regex-pcre2"` # Target that builds all binaries (including those that are not # rebuilt except when running the tests) .PHONY: really-all really-all: all $(check_SCRIPTS) $(haskell_tests) $(HS_ALL_PROGS) # we don't need the ancient implicit rules: %: %,v %: RCS/%,v %: RCS/% %: s.% %: SCCS/s.% -include ./Makefile.local # support inspecting the value of a make variable print-%: @echo $($*) # vim: set noet : ganeti-3.1.0~rc2/NEWS000064400000000000000000006674521476477700300143270ustar00rootroot00000000000000News ==== Version 3.1.0 rc2 ----------------- *(Released Wed, 12 Mar 2025)* Changes since 3.1.0 rc1 ~~~~~~~~~~~~~~~~~~~~~~~ - various tests have been fixed (Python and Haskell) (#1833) - various bugs related to `make dist / distcheck` have been fixed (which is the reason there never was an official -rc1 tarball) (#1833, #1834) - builds are now reproducible (#1831) - QA suite now default to RSA instead of DSA SSH keys (#1829) - `--enable-regex-pcre-builtin` and `--enable-regex-tdfa` have been dropped from `./configure` and replaced by -`-with-haskell-pcre=auto|$library` (#1832) - support for the Haskell pcre2 library has been added (#1832) - `start-stop-daemon` is now invoked with `--remove-pidfile` to avoid stale PID files which confused tests (#1836) Version 3.1.0 rc1 ----------------- *(Released Tue, 11 Feb 2025)* Upgrade notes ~~~~~~~~~~~~~ Upgrading is supported from 3.0.x versions. If you are using the KVM hypervisor, you should consider setting the parameter `machine_version` to the latest `pc`-version supported by your version of kvm. You can query that by running `kvm -M ?`. Do not set this to `pc`, but rather to a specific version like `pc-i440fx-9.2`. Otherwise you might risk instance crashes if you live-migrate between different kvm versions (e.g. during a rolling cluster upgrade). The `q35` versions are not yet supported by Ganeti and will fail your instances to boot once kvm changes from the current default of `pc`. Ganeti now uses `-blockdev` instead of `-drive` parameters for Qemu/KVM. This has been tested with all built-in storage options. If you are using the `extstorage` backend, please make sure to test this in a non-production environment before upgrading. Important changes ~~~~~~~~~~~~~~~~~ Adapt to current Python versions ++++++++++++++++++++++++++++++++ Various changes have been incorporated to build on Python versions up to 3.13, the more relevant ones are as follows: Instead of `python-simplejson` Ganeti now uses the built-in JSON module (#1723). The deprecated `imp` module has been replaced with `importlib` (#1771). Ganeti now uses pytest along with the existing Python testing framework (#1769). Adapt to current GHC versions +++++++++++++++++++++++++++++ Various changes have been incorporated to build on GHC versions up to 9.10, the more relevant ones are as follows: Ganeti now supports three different regex implementations: regex-pcre, regex-pcre-builtin, regex-tdfa. The implementation to be used can be specified as parameters to the `configure` script. From Debian 13, the recommended option is regex-tdfa, which does not support PCRE type regexes (#1794). `Makefile.ghc` has been dropped and `cabal` is now used to build the Haskell parts (#1785). DRBD disks ++++++++++ Ganeti will now classify DRBD disks in `StandAlone` mode as an error during a `gnt-cluster verify` run (#1814). Until now, this was classified as a warning only. General hypervisor changes ++++++++++++++++++++++++++ `gnt-instance modify` now uses hotplugging by default. The parameter `--hotplug-if-possible` has been dropped and instead `--no-hotplug` has been added. The same applies to respective RAPI calls (#1730). Support for `--no-name-check` and `--no-ip-check` arguments to `gnt-instance` operations has been dropped. If you explicitly rely on these checks, please enable them by using `--name-check` and `--ip-check` instead (#1589). Hypervisor implementations are now able to add code for the assessment of cluster-wide hypervisor parameter defaults. This has been implemented for KVM with some examples (`cpu_type`, `machine_version` etc.) and executes as part of a regular `gnt-cluster verify` run. A new argument `--no-hv-param-assessment` has been added to disable the assessment (#1822). KVM hypervisor ++++++++++++++ Ganeti has dropped all usage of the human monitor protocol/socket (HMP) and entirely moved to QMP for all operations. The HMP socket is still created but its usage is deprecated and it might be removed in future versions of Ganeti (#1662, #1667). It is now impossible to set conflicting values for `disk_aio` and `disk_cache` hypervisor parameters (#1646). The `disk_aio` parameter now supports `io_uring` on QEMU 5.0 and newer (#1667). Ganeti now uses QEMUs `-blockdev` argument instead of `-drive` for all disk/storage devices (and also the corresponding QMP methods - #1667). `vhost_net` now defaults to `true` on new clusters for improved virtio-net performance (#1652). Calls to `gnt-instance info` used to execute the QMP method `query-cpus`. Since that is known to (possibly) interrupt guest execution, Ganeti now uses `query-cpus-fast` (#1756). The `kvm_extra` hypervisor parameter now correctly supports spaces (#1713). Growing a disk will now inform the hypervisor of the new disk size. This makes rebooting the instance or the use of custom hook scripts unnecessary (#1731). Xen hypervsior ++++++++++++++ Support for the `xm` toolstack has been dropped (#1634). Instance migration across clusters ++++++++++++++++++++++++++++++++++ The `move-instance` script features a new parameter `--keep-source-instance` to retain the original instance after a successful migration (#1617). Additionally it will recreate instance tags on the destination cluster (#1619). The import/export daemon (`impexpd`) has fixed support for certificate verification with more recent versions of `socat` (#1699). DRBD-based instances will now be created on the destination cluster without waiting for the DRBD device to be synced. This greatly speeds up instance moves and seems reliable enough, as the data is still available on the source cluster (#1703). Ganeti networks +++++++++++++++ `gnt-network` now supports a `rename` operation (#1591). RAPI ++++ The new parameter `--ssl-chain` adds support for certificate chain files to the RAPI daemon (#1625). Request parsing errors now produce a HTTP 400 return code (#1610). All instance operations now support the field `custom_osparams` (#1673). Ganeti watcher ++++++++++++++ The Ganeti watcher now supports a new argument `--no-strict`. If used, it will allow the watcher to skip nodes during the group-verify-disk jobs which are busy with other jobs. This will increase the chance that shutdown instances will be detected and (re)started by the watcher even if nodes in the cluster are busy with long-running jobs. An updated example cron file is provided which mixes `-no-strict` and strict watcher runs (#1704). Documentation +++++++++++++ Large parts of the documentation have been restructured/cleaned up to get rid of obsolete/redundant/outdated information (#1709). We now use pydoctor instead of epydoc to generate Python apidocs (#1676). Version 3.0.0 ------------- *(Released Wed, 23 Dec 2020)* Changes since 3.0.0 rc1 ~~~~~~~~~~~~~~~~~~~~~~~ 3.0.0 includes only a handful of improvements and fixes since 3.0.0 rc1. Automatic postcopy migration handling for KVM guests ++++++++++++++++++++++++++++++++++++++++++++++++++++ Ganeti now supports switching a KVM live migration automatically over to postcopy mode if the instance's ``migration_caps`` include the ``postcopy-ram`` capability and the live migration has already completed two full memory passes. Postcopy live migration support in Ganeti 3.0 is considered experimental; users are encouraged to test it and report bugs, but it should be used with care in production environments. We recommend using postcopy migration with at least QEMU version 3.0; QEMU versions before 3.0 do not support limiting the bandwidth of a postcopy migration, which might saturate the network and cause interference with e.g. DRBD connections. For QEMU 3.0 and on, we apply the ``migration_bandwidth`` HV parameter that limits the regular live migration bandwidth to the postcopy phase as well. Thanks to Sascha Lucas and Calum Calder for all the work related to postcopy migration. Other changes +++++++++++++ Bugfixes: - Fix multi-queue tap creation that was broken by the Python 3 migration (#1534) - Make sure we set KVM migration capabilities on both sides of the live migration and clear them once the migration is over (#1525) - Properly cleanup the dedicated spice connection used to set a KVM instance's spice password; this avoids blocking the instance on boot (#1535, #1536) - Fix non-SSL operation for Python daemons, broken by the Python 3 migration. This should be only relevant for the RAPI daemon running behind a reverse proxy; noded requires SSL to function properly (#1508, #1538) Compatibility fixes: - Correctly report the status of user-down KVM instances with QEMU >= 3.1 (#1440, #1537) Version 3.0.0 rc1 ----------------- *(Released Sat, 19 Sep 2020)* Since releasing 3.0.0 beta1 in June no critical issues have surfaced. This release includes some feature and compability improvements but no breaking changes. Upgrade notes ~~~~~~~~~~~~~ This release comes with the same restrictions as the previous one: to upgrade, you either need 2.16.2 or 3.0.0 beta1 installed. Upgrading directly from older versions or from the Ganeti-2.17 beta version is not supported. Please refer to the 3.0.0 beta1 upgrade notes for more information. Important changes ~~~~~~~~~~~~~~~~~ GHC 8.0 through 8.8 compatibility +++++++++++++++++++++++++++++++++ This release has been built/tested against GHC 8.0 through 8.8 which means it should work on most current and near-future distribution versions. Support for GHC versions < 8 has already been dropped with the previous Ganeti release. Along with this change we have also added compatibility to Cabal version 3.0. Other notable changes ~~~~~~~~~~~~~~~~~~~~~ Bugfixes: - Fix distribution of hmac.key to new nodes - this has been pulled from the 2.17 tree #(1494) Compatibility Improvements: - Open vSwitch: Do not fail to add node when the ovs_name or ovs_link already exists (#1495) - Improved support for DRBD >= 8.4 (#1496) - Relax QuickCheck version restriction (#1479) Documentation Fixes: - Various typos have been fixed (#1483, #1487, #1489, #1501) - Documentation build has been improved (#1500, #1504) - Missing information has been added (#1490, #1505, #1517) Build Environment: - We now have matrix / nightly builds using Github Actions (#1512) - We now have code analysis through Github CodeQL (#1514) Misc: - Support other values than 'none' for 'disk_cache' when using RBD (#1516) - The OS install scripts can now query the configured disk size via a new environment variable 'DISK_%N_SIZE' (#1503) Version 3.0.0 beta1 ------------------- *(Released Fri, 5 Jun 2020)* This is a major version pre-release, porting Ganeti to Python 3, fixing bugs and adding new features. This is also the first major release to be created by community contributors exclusively. As of May 2020, Google transferred the maintenance of Ganeti to the community. We would like to thank Google for the support and resources it granted to the project and for allowing the community to carry it forward! Upgrade notes ~~~~~~~~~~~~~ Ganeti versions earlier than 2.16.2 will refuse to upgrade to 3.0 using ``gnt-cluster upgrade``. If you are using your distribution packages, chances are their maintainers will provide a smooth upgrade path from older versions, so check the package release notes. If you build Ganeti from source, please upgrade to 2.16.2 as an intermediate step before upgrading to 3.0, or consult Github issue `#1423`_ for possible workarounds. .. _#1423: https://github.com/ganeti/ganeti/issues/1423 Note that at this time there is no supported upgrade path for users running Ganeti-2.17 (i.e. 2.17.0 beta1). Ganeti-2.17 was never released, so hopefully no one uses it. In case you are using it, the best option is to downgrade to 2.16 (either via a regular downgrade or manually). See Github issue `#1346`_ for a bit more discussion on this topic. .. _#1346: https://github.com/ganeti/ganeti/issues/1346#issuecomment-642763303 Important changes ~~~~~~~~~~~~~~~~~ Python >=3.6 required +++++++++++++++++++++ This release switches the whole Ganeti codebase over to Python 3. Python 2 has reached its end-of-life and is being removed from most distributions, so we opted to skip dual-version support completely and convert the code straight to Python 3-only, the only exception being the RAPI client which remains Python-2 compatible. We have tested the code as well as we can, but there is still the possibility of breakage, as the conversion touches a big part of the codebase that cannot always be tested automatically. Please test this release if possible and report any bugs on GitHub. Note that the minimum required Python version is 3.6. GHC >= 8 required +++++++++++++++++ This release removes support for ancient versions of GHC and now requires at least GHC 8.0 to build. VLAN-aware bridging +++++++++++++++++++ This version adds support for VLAN-aware bridging. Traditionally setups using multiple VLANs had to create one Linux bridge per VLAN and assign instance NICs to the correct bridge. For large setups this usually incurred a fair amount of configuration that had to be kept in sync between nodes. An alternative was to use OpenVSwitch, for which Ganeti already included VLAN support. Beginning with 3.0, Ganeti supports VLAN-aware bridging: it is now possible to have a single bridge handling traffic for multiple VLANs and have instance NICs assigned to one or more VLANs using the ``vlan`` NIC parameter with the same syntax as for OpenVSwitch (see the manpage for ``gnt-instance``). Note that Ganeti expects VLAN support for the bridge to be enabled externally, using ``ip link set dev type bridge vlan_filtering 1``. Other notable changes ~~~~~~~~~~~~~~~~~~~~~ Bugfixes: - Refactor LuxiD's job forking code to make job process creation more reliable. This fixes sporadic failures when polling jobs for status changes, as well as randomly-appearing 30-second delays when enqueueing a new job (#1411). - Wait for a Luxi job to actually finish before archiving it. This prevents job file accumulation in master candidate queues (#1266). - Avoid accidentally backing up the export directory on cluster upgrade (#1337). - This release includes all fixes from 2.16.2 as well, please refer to the 2.16.2 changelog below. Compatibility changes: - Orchestrate KVM live migrations using only QMP (and not the human monitor), ensuring compatibility with QEMU 4.2 (#1433). - Use iproute2 instead of brctl, removing the dependency on bridge-utils (#1394). - Enable ``AM_MAINTAINER_MODE``, supporting read-only VPATH builds (#1391). - Port from Haskell Crypto (unmaintained) to cryptonite (#1405) - Enable compatibility with pyopenssl >=19.1.0 (#1446) Version 2.16.2 -------------- *(Released Fri, 22 May 2020)* This is a bugfix and compatibility release. Important note ~~~~~~~~~~~~~~ Due to the way the ``gnt-cluster upgrade`` mechanism is implemented, Ganeti versions earlier than 2.16.2 will refuse to upgrade to the upcoming 3.0 release. This release changes the upgrade logic to explicitly allow upgrades from 2.16.2 and later to 3.0. See `#1423`_ for more details and the relevant discussion. .. _#1423: https://github.com/ganeti/ganeti/issues/1423 Bugfixes ~~~~~~~~ - Fix node secondary instance count. Secondary instances were counted as many times as their disk count (#1399) - RPC: remove 1-second wait introduced by ``Expect: 100-Continue``. This speeds up all RPC operations that pass through LuxiD (most notably queries like ``gnt-instance list``) by 1 second. Version 2.16.1 -------------- *(Released Mon, 1 Apr 2019)* This is a bugfix and compatibility release. Important changes ~~~~~~~~~~~~~~~~~ Updated X.509 certificate signing algorithm +++++++++++++++++++++++++++++++++++++++++++ Ganeti now uses the SHA-256 digest algorithm to sign all generated X.509 certificates used to secure the RPC communications between nodes. Previously, Ganeti was using SHA-1 which is seen as weak (but not broken) and has been deprecated by most vendors; most notably, OpenSSL — used by Ganeti on some setups — rejects SHA-1-signed certificates when configured to run on security level 2 and above. Users are advised to re-generate Ganeti's server and node certificates after installing 2.16.1 *on all nodes* using the following command: :: gnt-cluster renew-crypto --new-cluster-certificate On setups using RAPI and/or SPICE with Ganeti-generated certificates, ``--new-rapi-certificate`` and ``--new-spice-certificate`` should be appended to the command above. QEMU 3.1 compatibility ++++++++++++++++++++++ Previous versions of Ganeti used QEMU command line options that were removed in QEMU 3.1, leading to an inability to start KVM instances with QEMU 3.1. This version restores compatibility with QEMU 3.1 by adapting to these changes. This was done in a backwards-compatible way, however there is one special case: Users using VNC with X.509 support enabled, will need to be running at least QEMU 2.5. See #1342 for details. Newer GHC support +++++++++++++++++ Ganeti 2.16.0 could only be built using GHC versions prior to 7.10, as GHC 7.10 and later versions introduced breaking API changes that made the build fail. This release introduces support for building with newer GHC versions: Ganeti is now known to build with GHC 8.0, 8.2 and 8.4. Furthermore, Ganeti can now be built with snap-server 1.0 as well as hinotify 0.3.10 and later. Previously supported versions of GHC and of these libraries remain supported. Misc changes ~~~~~~~~~~~~ *(Contributor names in parentheses where available)* Compatibility fixes: - Fix initscript operation on systems with dpkg >= 1.19.4 (#1322) - Support Sphinx versions later than 1.7 (#1333) (Robin Sonnabend) - Force KVM to use ``cache=none`` when ``aio=native`` is set; this is mandatory for QEMU versions later than 2.6 (#43) (Alexandros Kosiaris) - Handle the new output format of ``rbd showmapped`` introduced in Ceph Mimic (#1339) (Ansgar Jazdzewski) - Support current versions of python-psutil (George Diamantopoulos) - Fix distcheck-hook with automake versions >= 1.15 (Apollon Oikonomopoulos) - Fix cli tests with shelltestrunner versions >= 1.9 (Apollon Oikonomopoulos) Bugfixes: - Allow IPv6 addresses in the ``vnc_bind_address`` KVM hypervisor parameter (#1257) (Brian Candler) - Fix iproute2 invocation to accept ``dev`` as a valid interface name (#26) (Arnd Hannemann) - Properly handle OpenVSwitch trunk ports without native VLANs (#1324) (George Diamantopoulos) - Fix virtio-net multiqueue support (#1268) (George Diamantopoulos) - Make the ganeti-kvm-poweroff example script work on systems with systemd/sysv integration (#1288) - Avoid triggering the CPU affinity code when the instance's CPU mask is set to ``all``, relaxing the runtime dependency on python-psutil (Calum Calder) Performance improvements: - Speed up Haskell test execution (Iustin Pop) - Speed up Python test execution (Apollon Oikonomopoulos) Documentation fixes: - Fix a couple of typos in the gnt-instance man page (#1279) (Phil Regnauld) - Fix a typo in doc/install.rst (Igor Vuk) Enhancements: - KVM process logs are now obtained and saved under /var/log/ganeti/kvm (Yiannis Tsiouris) Version 2.16.0 -------------- *(Released Tue, 18 Sep 2018)* Changes since 2.16.0 rc2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Compatibility fixes: - Support python-mock versions later than about 1.1 Bugfixes: - RAPI: Check non-SSL exceptions in _CheckIfConnectionDropped - makefile: Preserve mode on copy - Change uidpool test to skip uid 0 - Add retry behavior to detect SSConf read race in qa.qa_instance_utils - utils.livelock: use portable struct flock type Misc changes: - Set default ssh connection timeout between nodes to 10s - Add unit tests for utils.livelock Version 2.16.0 rc2 ------------------ *(Released Mon, 29 Jan 2018)* Changes since 2.16.0 rc1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ New features: - Do not prompt when force-setting a node online - Consider state-of-the-record free memory in htools memory model (#35) - Add optStaticKvmNodeMemory to HTools and IAllocator (#34) - Add discard KVM option to bdev - StartInstance restores instance state if running - Allow migrate --cleanup to adopt an instance - Add ganeti-noded and ganeti-rapi --max-clients options - Add gnt-instance rename --force option - Allow master failover to ignore offline nodes - Adding a confirmation before gnt-node --offline no Compatibility fixes: - kvm: use the current psutil CPU affinity API - Provide alternative to decompressWithErrors in zlib 6.0 - Ceph/RBD rbd showmapped -p is no longer supported - kvm: use_guest_agent: QEMU Guest Agent support - Fix LogicalVolume code to work with older /sbin/lvs Performance optimizations: - Use fork instead of spawnv in the watcher - Make executeRpcCall only compute rpcCallData once - Special case WaitForJobChange to reduce heap use - Get haskell daemons to only compress files > 4kB - Use zlib compression level 3 in Haskell RPC code - Make the TH fieldsDictsKeys more efficient - Implement localized cache for lvs commands - Reduce load in NV_NODENETTEST and NV_MASTERIP Bugfixes: - impexpd: fix certificate verification with new socat versions - impexpd: do not set socat SSL method - backend: fix key renewal on single-node clusters - hv_xen: generate correct type for paravirtualized nic (#57) - Force CleanupInstance always on InstanceShutdown (#53) - Cleanup blockdevs from target on migration failure (#50) - kvm: Add missing 'driver' in 'hvinfo' dict (#46) - Add cleanup of stale OS hvp data on cluster modify - Htools should use state-of-record instance size - Prohibit disk removal w/o hotplug on live instance - Fix tuple-unpacking from QueryInstances result - Fix index in RemoveDisks warning - Fix coexistence of location tags and non-DRBD instances - Fix backup export in case of ext disk template - Fix instance state detection in _Shutdowninstance - Fix for instance reinstall not updating config (issue #1193) - Fix optimisation: Correctly extract secondary node - Tune getNodeInstances DRBD secondary computation - Fix LogicalVolume Attach failure on missing path - Set USE_VERSION_FULL=no if --enable-versionfull=no - Don't verify disks when all disk templates are ext Misc changes: - Fixed several hlint and pylint styling errors - Reduced the verbosity of several debug messages - Fixed various typos in man pages and documentation - Improved various unit tests Fixes inherited from 2.15 branch: - Update hv_kvm to handle output from qemu >= 1.6.0 - Disable logging CallRPCMethod timings in non-debug configs - Give atomicWriteFile temp filenames a more distinct pattern - FIX: Refactor DiagnoseOS to use a loop, not an inner fn - FIX: Set INSTANCE_NICn_NETWORK_NAME only if net is defined - Fix invalid variable error for file-based disks - Fix gnt-instance console instance unpausing for xl toolstack - KVM: handle gracefully too old/too new psutil versions - Fixup compatibility with GHC 7.4/base 4.5 - mcpu: Raise caught OpPrereqErrors with too few args - Make EnsureDirs print chmod mode on errors - Use socat method string compatible with <1.73 & >=1.73 - Reduce heap when parsing & storing ConfigData 10% - Cancel RAPI job if the client drops the connection - Make JQScheduler queues more strict to avoid leaks - Fix ganeti-rapi/noded exit-under-load bug - Fix ClusterVerifyConfig() causing high mem usage - Use threaded runtime when linking Haskell unit tests - Give JQueue test dirs unique prefixes so they can't conflict - Update install-quick DRBD requirements to include DRBD 8.4 - Fix memory/perf bug in gnt-cluster verify - Improve luxid QueryInstances performance for large clusters - Optimize LXC hypervisor GetAllInstancesInfo - Bracket ConfigWriter writeConfigAndUnlock with debug logging - Bracket client LockConfig calls with debug logging - Get onInotify and onPollTimer to print filepath - Prevent InstanceShutdown from waiting on success Fixes inherited from 2.14 branch: - Support userspace disk URIs for OS import/export scripts - iallocator: only adjust memory usage for up instances Fixes inherited from 2.13 branch: - Bugfix: migrate needs HypervisorClass, not an instance Version 2.16.0 rc1 ------------------ *(Released Thu, 18 Feb 2016)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The IAllocator protocol has been extended by a new ``allocate-secondary`` request type. Currently, this new request type is only used when in disk conversion to DRBD no secondary node is specified. As long as this new feature is not used, a third-party IAllocator not aware of this extension can be continued to be used. - ``htools`` now also take into account N+1 redundancy for plain and shared storage. To obtain the old behavior, add the ``--no-capacity-checks`` option. - ``hail`` now tries to keep the overall cluster balanced; in particular it now prefers more empty groups over groups that are internally more balanced. - The option ``--no-node-setup`` of ``gnt-node add`` is disabled. Instead, the cluster configuration parameter ``modify_ssh_setup`` is used to determine whether or not to manipulate the SSH setup of a new node. - Timeouts for communication with luxid have been increased. As a consequence, Ganeti tools communicating (directly or indirectly) with luxid also time out later. Please increase all timeouts for higher level tools interacting with Ganeti accordingly. New features ~~~~~~~~~~~~ - ``hbal`` can now be made aware of common causes of failures (for nodes). Look at ``hbal`` man page's LOCATION TAGS section for more details. - ``hbal`` can now be made aware of desired location for instances. Look at ``hbal`` man page's DESIRED LOCATION TAGS section for more details. - Secret parameters are now readacted in job files New dependencies ~~~~~~~~~~~~~~~~ - Using the metadata daemon now requires the presence of the 'setcap' utility. On Debian-based systems, it is available as a part of the 'libcap2-bin' package. Changes since beta2 ~~~~~~~~~~~~~~~~~~~ - On group verify, only flush to group nodes Version 2.16.0 beta2 -------------------- *(Released Tue, 2 Feb 2016)* This was the second beta release of the 2.16 series. All important changes are listed in the latest 2.16 entry. Changes since beta1 ~~~~~~~~~~~~~~~~~~~ - Do not add a new Inotify watchers on timer - Set block buffering for UDSServer - Fix failover in case the source node is offline - Add a parameter to ignore groups in capacity checks - Make hspace correctly handle --independent-groups - Accept BoringSSL as a known good ssl library - Make CommitTemporaryIPs call out to WConfD - Fix requested instance desired location tags in IAllocator - monitor: Use hvinfo in QMP methods - KVM: Work around QEMU commit 48f364dd - KVM: Introduce scsi_controller_type and kvm_pci_reservations hvparams - Improvements in SSH key handling - Do not generate the ganeti_pub_keys file with --no-ssh-init - Support force option for deactivate disks on RAPI - Add a --dry-run option to htools - Extended logging to improve traceability - Many documentation improvements and cleanups - Performance optimizations on larger clusters - Various QA and testing improvements Fixes inherited from 2.15 branch: - Metad: ignore instances that have no communication NIC - For queries, take the correct base address of an IP block - Fix computation in network blocks - Use bulk-adding of keys in renew-crypto - Introduce bulk-adding of SSH keys - Handle SSH key distribution on auto promotion - Do not remove authorized key of node itself - Support force option for deactivate disks on RAPI - renew-crypto: use bulk-removal of SSH keys - Bulk-removal of SSH keys - Catch IOError of SSH files when removing node - Fix renew-crypto on one-node-cluster - Increase timeout of RPC adding/removing keys - After TestNodeModify, fix the pool of master candidates Fixes inherited from 2.14 branch: - bdev: Allow userspace-only disk templates - Export disk's userspace URI to OS scripts - Fix instance failover in case of DTS_EXT_MIRROR - Set node tags in iallocator htools backend - Fix faulty iallocator type check - Allow disk attachment to diskless instances - Allow disk attachment with external storage Fixes inherited from 2.13 branch: - Improve xl socat migrations - Renew-crypto: stop daemons on master node first - Extend timeout for gnt-cluster renew-crypto Fixes inherited from 2.12 branch: - Accept timeout errors when luxi down - Fix disabling of user shutdown reporting - gnt-node add: password auth is only one method - Fix inconsistency in python and haskell objects - Increase default disk size of burnin to 1G - Only search for Python-2 interpreters - Handle Xen 4.3 states better - Return the correct error code in the post-upgrade script - Make openssl refrain from DH altogether - Fix upgrades of instances with missing creation time - Check for healthy majority on master failover with voting - Pass arguments to correct daemons during master-failover Fixes inherited from 2.11 branch: - At IAlloc backend guess state from admin state - Fix default for --default-iallocator-params Fixes inherited from 2.10 branch: - Make htools tolerate missing "dtotal" and "dfree" on luxi - KVM: explicitly configure routed NICs late Fixes inherited from the 2.9 branch: - Security patch for CVE-2015-7944 RAPI Vulnerable to DoS via SSL renegotiation - Security patch for CVE-2015-7945 Leak DRBD secret via RAPI - replace-disks: fix --ignore-ipolicy Version 2.16.0 beta1 -------------------- *(Released Tue, 28 Jul 2015)* This was the first beta release of the 2.16 series. All important changes are listed in the latest 2.16 entry. Version 2.15.2 -------------- *(Released Wed, 16 Dec 2015)* Important changes and security notes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Security release. CVE-2015-7944 Ganeti provides a RESTful control interface called the RAPI. Its HTTPS implementation is vulnerable to DoS attacks via client-initiated SSL parameter renegotiation. While the interface is not meant to be exposed publicly, due to the fact that it binds to all interfaces, we believe some users might be exposing it unintentionally and are vulnerable. A DoS attack can consume resources meant for Ganeti daemons and instances running on the master node, making both perform badly. Fixes are not feasible due to the OpenSSL Python library not exposing functionality needed to disable client-side renegotiation. Instead, we offer instructions on how to control RAPI's exposure, along with info on how RAPI can be setup alongside an HTTPS proxy in case users still want or need to expose the RAPI interface. The instructions are outlined in Ganeti's security document: doc/html/security.html CVE-2015-7945 Ganeti leaks the DRBD secret through the RAPI interface. Examining job results after an instance information job reveals the secret. With the DRBD secret, access to the local cluster network, and ARP poisoning, an attacker can impersonate a Ganeti node and clone the disks of a DRBD-based instance. While an attacker with access to the cluster network is already capable of accessing any data written as DRBD traffic is unencrypted, having the secret expedites the process and allows access to the entire disk. Fixes contained in this release prevent the secret from being exposed via the RAPI. The DRBD secret can be changed by converting an instance to plain and back to DRBD, generating a new secret, but redundancy will be lost until the process completes. Since attackers with node access are capable of accessing some and potentially all data even without the secret, we do not recommend that the secret be changed for existing instances. Minor changes ~~~~~~~~~~~~~ - Allow disk aittachment to diskless instances - Reduce memory footprint: Compute lock allocation strictly - Calculate correct affected nodes set in InstanceChangeGroup (Issue 1144) - Reduce memory footprint: Don't keep input for error messages - Use bulk-adding of keys in renew-crypto - Reduce memory footprint: Send answers strictly - Reduce memory footprint: Store keys as ByteStrings - Reduce memory footprint: Encode UUIDs as ByteStrings - Do not retry all requests after connection timeouts to prevent repeated job submission - Fix reason trails of expanding opcodes - Make lockConfig call retryable - Extend timeout for gnt-cluster renew-crypto - Return the correct error code in the post-upgrade script - Make OpenSSL refrain from DH altogether - Fix faulty iallocator type check - Improve cfgupgrade output in case of errors - Fix upgrades of instances with missing creation time - Support force option for deactivate disks on RAPI - Make htools tolerate missing "dtotal" and "dfree" on luxi - Fix default for --default-iallocator-params - Renew-crypto: stop daemons on master node first - Don't warn about broken SSH setup of offline nodes (Issue 1131) - Fix computation in network blocks - At IAlloc backend guess state from admin state - Set node tags in iallocator htools backend - Only search for Python-2 interpreters - Handle Xen 4.3 states better - Improve xl socat migrations Version 2.15.1 -------------- *(Released Mon, 7 Sep 2015)* New features ~~~~~~~~~~~~ - The ext template now allows userspace-only disks to be used Bugfixes ~~~~~~~~ - Fixed the silently broken 'gnt-instance replace-disks --ignore-ipolicy' command. - User shutdown reporting can now be disabled on Xen using the '--user-shutdown' flag. - Remove falsely reported communication NIC error messages on instance start. - Fix 'gnt-node migrate' behavior when no instances are present on a node. - Fix the multi-allocation functionality for non-DRBD instances. Version 2.15.0 -------------- *(Released Wed, 29 Jul 2015)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - In order to improve allocation efficiency when using DRBD, the cluster metric now takes the total reserved memory into account. A consequence of this change is that the best possible cluster metric is no longer 0. htools(1) interprets minimal cluster scores to be offsets of the theoretical lower bound, so only users interpreting the cluster score directly should be affected. - This release contains a fix for the problem that different encodings in SSL certificates can break RPC communication (issue 1094). The fix makes it necessary to rerun 'gnt-cluster renew-crypto --new-node-certificates' after the cluster is fully upgraded to 2.14.1 New features ~~~~~~~~~~~~ - On dedicated clusters, hail will now favour allocations filling up nodes efficiently over balanced allocations. New dependencies ~~~~~~~~~~~~~~~~ - The indirect dependency on Haskell package 'case-insensitive' is now explicit. Version 2.15.0 rc1 ------------------ *(Released Wed, 17 Jun 2015)* This was the first release candidate in the 2.15 series. All important changes are listed in the latest 2.15 entry. Known issues: ~~~~~~~~~~~~~ - Issue 1094: differences in encodings in SSL certificates due to different OpenSSL versions can result in rendering a cluster uncommunicative after a master-failover. Version 2.15.0 beta1 -------------------- *(Released Thu, 30 Apr 2015)* This was the second beta release in the 2.15 series. All important changes are listed in the latest 2.15 entry. Version 2.14.2 -------------- *(Released Tue, 15 Dec 2015)* Important changes and security notes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Security release. CVE-2015-7944 Ganeti provides a RESTful control interface called the RAPI. Its HTTPS implementation is vulnerable to DoS attacks via client-initiated SSL parameter renegotiation. While the interface is not meant to be exposed publicly, due to the fact that it binds to all interfaces, we believe some users might be exposing it unintentionally and are vulnerable. A DoS attack can consume resources meant for Ganeti daemons and instances running on the master node, making both perform badly. Fixes are not feasible due to the OpenSSL Python library not exposing functionality needed to disable client-side renegotiation. Instead, we offer instructions on how to control RAPI's exposure, along with info on how RAPI can be setup alongside an HTTPS proxy in case users still want or need to expose the RAPI interface. The instructions are outlined in Ganeti's security document: doc/html/security.html CVE-2015-7945 Ganeti leaks the DRBD secret through the RAPI interface. Examining job results after an instance information job reveals the secret. With the DRBD secret, access to the local cluster network, and ARP poisoning, an attacker can impersonate a Ganeti node and clone the disks of a DRBD-based instance. While an attacker with access to the cluster network is already capable of accessing any data written as DRBD traffic is unencrypted, having the secret expedites the process and allows access to the entire disk. Fixes contained in this release prevent the secret from being exposed via the RAPI. The DRBD secret can be changed by converting an instance to plain and back to DRBD, generating a new secret, but redundancy will be lost until the process completes. Since attackers with node access are capable of accessing some and potentially all data even without the secret, we do not recommend that the secret be changed for existing instances. Minor changes ~~~~~~~~~~~~~ - Allow disk attachment to diskless instances - Calculate correct affected nodes set in InstanceChangeGroup (Issue 1144) - Do not retry all requests after connection timeouts to prevent repeated job submission - Fix reason trails of expanding opcodes - Make lockConfig call retryable - Extend timeout for gnt-cluster renew-crypto - Return the correct error code in the post-upgrade script - Make OpenSSL refrain from DH altogether - Fix faulty iallocator type check - Improve cfgupgrade output in case of errors - Fix upgrades of instances with missing creation time - Make htools tolerate missing "dtotal" and "dfree" on luxi - Fix default for --default-iallocator-params - Renew-crypto: stop daemons on master node first - Don't warn about broken SSH setup of offline nodes (Issue 1131) - At IAlloc backend guess state from admin state - Set node tags in iallocator htools backend - Only search for Python-2 interpreters - Handle Xen 4.3 states better - Improve xl socat migrations - replace-disks: fix --ignore-ipolicy - Fix disabling of user shutdown reporting - Allow userspace-only disk templates - Fix instance failover in case of DTS_EXT_MIRROR - Fix operations on empty nodes by accepting allocation of 0 jobs - Fix instance multi allocation for non-DRBD disks - Redistribute master key on downgrade - Allow more failover options when using the --no-disk-moves flag Version 2.14.1 -------------- *(Released Fri, 10 Jul 2015)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The SSH security changes reduced the number of nodes which can SSH into other nodes. Unfortunately enough, the Ganeti implementation of migration for the xl stack of Xen required SSH to be able to migrate the instance, leading to a situation where full movement of an instance around the cluster was not possible. This version fixes the issue by using socat to transfer instance data. While socat is less secure than SSH, it is about as secure as xm migrations, and occurs over the secondary network if present. As a consequence of this change, Xen instance migrations using xl cannot occur between nodes running 2.14.0 and 2.14.1. - This release contains a fix for the problem that different encodings in SSL certificates can break RPC communication (issue 1094). The fix makes it necessary to rerun 'gnt-cluster renew-crypto --new-node-certificates' after the cluster is fully upgraded to 2.14.1 Other Changes ~~~~~~~~~~~~~ - The ``htools`` now properly work also on shared-storage clusters. - Instance moves now work properly also for the plain disk template. - Filter-evaluation for run-time data filter was fixed (issue 1100). - Various improvements to the documentation have been added. Version 2.14.0 -------------- *(Released Tue, 2 Jun 2015)* New features ~~~~~~~~~~~~ - The build system now enforces external Haskell dependencies to lie in a supported range as declared by our new ganeti.cabal file. - Basic support for instance reservations has been added. Instance addition supports a --forthcoming option telling Ganeti to only reserve the resources but not create the actual instance. The instance can later be created with by passing the --commit option to the instance addition command. - Node tags starting with htools:nlocation: now have a special meaning to htools(1). They control between which nodes migration is possible, e.g., during hypervisor upgrades. See hbal(1) for details. - The node-allocation lock as been removed for good, thus speeding up parallel instance allocation and creation. - The external storage interface has been extended by optional ``open`` and ``close`` scripts. New dependencies ~~~~~~~~~~~~~~~~ - Building the Haskell part of Ganeti now requires Cabal and cabal-install. Known issues ~~~~~~~~~~~~ - Under certain conditions instance doesn't get unpaused after live migration (issue #1050) Since 2.14.0 rc1 ~~~~~~~~~~~~~~~~ - The call to the IAllocator in 'gnt-node evacuate' has been fixed. - In opportunistic locking, only ask for those node resource locks where the node lock is held. - Lock requests are repeatable now; this avoids failure of a job in a race condition with a signal sent to the job. - Various improvements to the QA. Version 2.14.0 rc2 ------------------ *(Released Tue, 19 May 2015)* This was the second release candidate in the 2.14 series. All important changes are listed in the 2.14.0 entry. Since 2.14.0 rc1 ~~~~~~~~~~~~~~~~ - private parameters are now properly exported to instance create scripts - unnecessary config unlocks and upgrades have been removed, improving performance, in particular of cluster verification - some rarely occuring file-descriptor leaks have been fixed - The checks for orphan and lost volumes have been fixed to also work correctly when multiple volume groups are used. Version 2.14.0 rc1 ------------------ *(Released Wed, 29 Apr 2015)* This was the first release candidate in the 2.14 series. All important changes are listed in the latest 2.14 entry. Since 2.14.0 beta2 ~~~~~~~~~~~~~~~~~~ The following issue has been fixed: - A race condition where a badly timed kill of WConfD could lead to an incorrect configuration. Fixes inherited from the 2.12 branch: - Upgrade from old versions (2.5 and 2.6) was failing (issues 1070, 1019). - gnt-network info outputs wrong external reservations (issue 1068) - Refuse to demote master from master capability (issue 1023) Fixes inherited from the 2.13 branch: - bugs related to ssh-key handling of master candidate (issues 1045, 1046, 1047) Version 2.14.0 beta2 -------------------- *(Released Thu, 26 Mar 2015)* This was the second beta release in the 2.14 series. All important changes are listed in the latest 2.14 entry. Since 2.14.0 beta1 ~~~~~~~~~~~~~~~~~~ The following issues have been fixed: - Issue 1018: Cluster init (and possibly other jobs) occasionally fail to start The extension of the external storage interface was not present in 2.14.0 beta1. Version 2.14.0 beta1 -------------------- *(Released Fri, 13 Feb 2015)* This was the first beta release of the 2.14 series. All important changes are listed in the latest 2.14 entry. Version 2.13.3 -------------- *(Released Mon, 14 Dec 2015)* Important changes and security notes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Security release. CVE-2015-7944 Ganeti provides a RESTful control interface called the RAPI. Its HTTPS implementation is vulnerable to DoS attacks via client-initiated SSL parameter renegotiation. While the interface is not meant to be exposed publicly, due to the fact that it binds to all interfaces, we believe some users might be exposing it unintentionally and are vulnerable. A DoS attack can consume resources meant for Ganeti daemons and instances running on the master node, making both perform badly. Fixes are not feasible due to the OpenSSL Python library not exposing functionality needed to disable client-side renegotiation. Instead, we offer instructions on how to control RAPI's exposure, along with info on how RAPI can be setup alongside an HTTPS proxy in case users still want or need to expose the RAPI interface. The instructions are outlined in Ganeti's security document: doc/html/security.html CVE-2015-7945 Ganeti leaks the DRBD secret through the RAPI interface. Examining job results after an instance information job reveals the secret. With the DRBD secret, access to the local cluster network, and ARP poisoning, an attacker can impersonate a Ganeti node and clone the disks of a DRBD-based instance. While an attacker with access to the cluster network is already capable of accessing any data written as DRBD traffic is unencrypted, having the secret expedites the process and allows access to the entire disk. Fixes contained in this release prevent the secret from being exposed via the RAPI. The DRBD secret can be changed by converting an instance to plain and back to DRBD, generating a new secret, but redundancy will be lost until the process completes. Since attackers with node access are capable of accessing some and potentially all data even without the secret, we do not recommend that the secret be changed for existing instances. Minor changes ~~~~~~~~~~~~~ - Calculate correct affected nodes set in InstanceChangeGroup (Issue 1144) - Do not retry all requests after connection timeouts to prevent repeated job submission - Fix reason trails of expanding opcodes - Make lockConfig call retryable - Extend timeout for gnt-cluster renew-crypto - Return the correct error code in the post-upgrade script - Make OpenSSL refrain from DH altogether - Fix upgrades of instances with missing creation time - Make htools tolerate missing "dtotal" and "dfree" on luxi - Fix default for --default-iallocator-params - Renew-crypto: stop daemons on master node first - Don't warn about broken SSH setup of offline nodes (Issue 1131) - At IAlloc backend guess state from admin state - Only search for Python-2 interpreters - Handle Xen 4.3 states better - Improve xl socat migrations - replace-disks: fix --ignore-ipolicy - Fix disabling of user shutdown reporting - Fix operations on empty nodes by accepting allocation of 0 jobs - Fix instance multi allocation for non-DRBD disks - Redistribute master key on downgrade - Allow more failover options when using the --no-disk-moves flag Version 2.13.2 -------------- *(Released Mon, 13 Jul 2015)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - This release contains a fix for the problem that different encodings in SSL certificates can break RPC communication (issue 1094). The fix makes it necessary to rerun 'gnt-cluster renew-crypto --new-node-certificates' after the cluster is fully upgraded to 2.13.2 Other fixes and known issues ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Inherited from 2.12: - Fixed Issue #1115: Race between starting WConfD and updating the config - Fixed Issue #1114: Binding RAPI to a specific IP makes the watcher restart the RAPI - Fixed Issue #1100: Filter-evaluation for run-time data filter - Better handling of the "crashed" Xen state - The watcher can be instructed to skip disk verification - Reduce amount of logging on successful requests - Prevent multiple communication NICs being created for instances - The ``htools`` now properly work also on shared-storage clusters - Instance moves now work properly also for the plain disk template - Various improvements to the documentation have been added Known issues: - Issue #1104: gnt-backup: dh key too small Version 2.13.1 -------------- *(Released Tue, 16 Jun 2015)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The SSH security changes reduced the number of nodes which can SSH into other nodes. Unfortunately enough, the Ganeti implementation of migration for the xl stack of Xen required SSH to be able to migrate the instance, leading to a situation where full movement of an instance around the cluster was not possible. This version fixes the issue by using socat to transfer instance data. While socat is less secure than SSH, it is about as secure as xm migrations, and occurs over the secondary network if present. As a consequence of this change, Xen instance migrations using xl cannot occur between nodes running 2.13.0 and 2.13.1. Other fixes and known issues ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Inherited from 2.12: - Fixed Issue #1082: RAPI is unresponsive after master-failover - Fixed Issue #1083: Cluster verify reports existing instance disks on non-default VGs as missing - Fixed Issue #1101: Modifying the storage directory for the shared-file disk template doesn't work - Fixed a possible file descriptor leak when forking jobs - Fixed missing private parameters in the environment for OS scripts - Fixed a performance regression when handling configuration (only upgrade it if it changes) - Adapt for compilation with GHC7.8 (compiles with warnings; cherrypicked from 2.14) Known issues: - Issue #1094: Mismatch in SSL encodings breaks RPC communication - Issue #1104: Export fails: key is too small Version 2.13.0 -------------- *(Released Tue, 28 Apr 2015)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Ganeti now internally retries the instance creation opcode if opportunistic locking did not acquire nodes with enough free resources. The internal retry will not use opportunistic locking. In particular, instance creation, even if opportunistic locking is set, will never fail with ECODE_TEMP_NORES. - The handling of SSH security had undergone a significant change. From this version on, each node has an individual SSH key pair instead of sharing one with all nodes of the cluster. From now on, we also restrict SSH access to master candidates. This means that only master candidates can ssh into other cluster nodes and all non-master-candidates cannot. Refer to the UPGRADE notes for further instructions on the creation and distribution of the keys. - Ganeti now checks hypervisor version compatibility before trying an instance migration. It errors out if the versions are not compatible. Add the option --ignore-hvversions to restore the old behavior of only warning. - Node tags starting with htools:migration: or htools:allowmigration: now have a special meaning to htools(1). See hbal(1) for details. - The LXC hypervisor code has been repaired and improved. Instances cannot be migrated and cannot have more than one disk, but should otherwise work as with other hypervisors. OS script changes should not be necessary. LXC version 1.0.0 or higher required. New features ~~~~~~~~~~~~ - A new job filter rules system allows to define iptables-like rules for the job scheduler, making it easier to (soft-)drain the job queue, perform maintenance, and rate-limit selected job types. See gnt-filter(8) for details. - Ganeti jobs can now be ad-hoc rate limited via the reason trail. For a set of jobs queued with "--reason=rate-limit:n:label", the job scheduler ensures that not more than n will be scheduled to run at the same time. See ganeti(7), section "Options", for details. - The monitoring daemon has now variable sleep times for the data collectors. This currently means that the granularity of cpu-avg-load can be configured. - The 'gnt-cluster verify' command now has the option '--verify-ssh-clutter', which verifies whether Ganeti (accidentally) cluttered up the 'authorized_keys' file. - Instance disks can now be converted from one disk template to another for many different template combinations. When available, more efficient conversions will be used, otherwise the disks are simply copied over. New dependencies ~~~~~~~~~~~~~~~~ - The monitoring daemon uses the PSQueue library. Be sure to install it if you use Mond. - The formerly optional regex-pcre is now an unconditional dependency because the new job filter rules have regular expressions as a core feature. Since 2.13.0 rc1 ~~~~~~~~~~~~~~~~~~ The following issues have been fixed: - Bugs related to ssh-key handling of master candidates (issues 1045, 1046, 1047) Fixes inherited from the 2.12 branch: - Upgrade from old versions (2.5 and 2.6) was failing (issues 1070, 1019). - gnt-network info outputs wrong external reservations (issue 1068) - Refuse to demote master from master capability (issue 1023) Version 2.13.0 rc1 ------------------ *(Released Wed, 25 Mar 2015)* This was the first release candidate of the 2.13 series. All important changes are listed in the latest 2.13 entry. Since 2.13.0 beta1 ~~~~~~~~~~~~~~~~~~ The following issues have been fixed: - Issue 1018: Cluster init (and possibly other jobs) occasionally fail to start Version 2.13.0 beta1 -------------------- *(Released Wed, 14 Jan 2015)* This was the first beta release of the 2.13 series. All important changes are listed in the latest 2.13 entry. Version 2.12.6 -------------- *(Released Mon, 14 Dec 2015)* Important changes and security notes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Security release. CVE-2015-7944 Ganeti provides a RESTful control interface called the RAPI. Its HTTPS implementation is vulnerable to DoS attacks via client-initiated SSL parameter renegotiation. While the interface is not meant to be exposed publicly, due to the fact that it binds to all interfaces, we believe some users might be exposing it unintentionally and are vulnerable. A DoS attack can consume resources meant for Ganeti daemons and instances running on the master node, making both perform badly. Fixes are not feasible due to the OpenSSL Python library not exposing functionality needed to disable client-side renegotiation. Instead, we offer instructions on how to control RAPI's exposure, along with info on how RAPI can be setup alongside an HTTPS proxy in case users still want or need to expose the RAPI interface. The instructions are outlined in Ganeti's security document: doc/html/security.html CVE-2015-7945 Ganeti leaks the DRBD secret through the RAPI interface. Examining job results after an instance information job reveals the secret. With the DRBD secret, access to the local cluster network, and ARP poisoning, an attacker can impersonate a Ganeti node and clone the disks of a DRBD-based instance. While an attacker with access to the cluster network is already capable of accessing any data written as DRBD traffic is unencrypted, having the secret expedites the process and allows access to the entire disk. Fixes contained in this release prevent the secret from being exposed via the RAPI. The DRBD secret can be changed by converting an instance to plain and back to DRBD, generating a new secret, but redundancy will be lost until the process completes. Since attackers with node access are capable of accessing some and potentially all data even without the secret, we do not recommend that the secret be changed for existing instances. Minor changes ~~~~~~~~~~~~~ - Calculate correct affected nodes set in InstanceChangeGroup (Issue 1144) - Do not retry all requests after connection timeouts to prevent repeated job submission - Fix reason trails of expanding opcodes - Make lockConfig call retryable - Return the correct error code in the post-upgrade script - Make OpenSSL refrain from DH altogether - Fix upgrades of instances with missing creation time - Make htools tolerate missing "dtotal" and "dfree" on luxi - Fix default for --default-iallocator-params - At IAlloc backend guess state from admin state - Only search for Python-2 interpreters - Handle Xen 4.3 states better - replace-disks: fix --ignore-ipolicy - Fix disabling of user shutdown reporting - Fix operations on empty nodes by accepting allocation of 0 jobs - Fix instance multi allocation for non-DRBD disks - Allow more failover options when using the --no-disk-moves flag Version 2.12.5 -------------- *(Released Mon, 13 Jul 2015)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - This release contains a fix for the problem that different encodings in SSL certificates can break RPC communication (issue 1094). The fix makes it necessary to rerun 'gnt-cluster renew-crypto --new-node-certificates' after the cluster is fully upgraded to 2.12.5. Fixed and improvements ~~~~~~~~~~~~~~~~~~~~~~ - Fixed Issue #1030: GlusterFS support breaks at upgrade to 2.12 - switches back to shared-file - Fixed Issue #1094 (see the notice in Incompatible/important changes): Differences in encodings of SSL certificates can render a cluster uncommunicative after a master-failover - Fixed Issue #1098: Support for ECDSA SSH keys - Fixed Issue #1100: Filter-evaluation for run-time data filter - Fixed Issue #1101: Modifying the storage directory for the shared-file disk template doesn't work - Fixed Issue #1108: Spurious "NIC name already used" errors during instance creation - Fixed Issue #1114: Binding RAPI to a specific IP makes the watcher restart the RAPI - Fixed Issue #1115: Race between starting WConfD and updating the config - Better handling of the "crashed" Xen state - The ``htools`` now properly work also on shared-storage clusters - Various improvements to the documentation have been added Inherited from the 2.11 branch: - Fixed Issue #1113: Reduce amount of logging on successful requests Known issues ~~~~~~~~~~~~ - Issue #1104: gnt-backup: dh key too small Version 2.12.4 -------------- *(Released Tue, 12 May 2015)* - Fixed Issue #1082: RAPI is unresponsive after master-failover - Fixed Issue #1083: Cluster verify reports existing instance disks on non-default VGs as missing - Fixed a possible file descriptor leak when forking jobs - Fixed missing private parameters in the environment for OS scripts - Fixed a performance regression when handling configuration (only upgrade it if it changes) - Adapt for compilation with GHC7.8 (compiles with warnings; cherrypicked from 2.14) Known issues ~~~~~~~~~~~~ Pending since 2.12.2: - Under certain conditions instance doesn't get unpaused after live migration (issue #1050) - GlusterFS support breaks at upgrade to 2.12 - switches back to shared-file (issue #1030) Version 2.12.3 -------------- *(Released Wed, 29 Apr 2015)* - Fixed Issue #1019: upgrade from 2.6.2 to 2.12 fails. cfgupgrade doesn't migrate the config.data file properly - Fixed Issue 1023: Master master-capable option bug - Fixed Issue 1068: gnt-network info outputs wrong external reservations - Fixed Issue 1070: Upgrade of Ganeti 2.5.2 to 2.12.0 fails due to missing UUIDs for disks - Fixed Issue 1073: ssconf_hvparams_* not distributed with ssconf Inherited from the 2.11 branch: - Fixed Issue 1032: Renew-crypto --new-node-certificates sometimes does not complete. The operation 'gnt-cluster renew-crypto --new-node-certificates' is now more robust against intermitten reachability errors. Nodes that are temporarily not reachable, are contacted with several retries. Nodes which are marked as offline are omitted right away. Inherited from the 2.10 branch: - Fixed Issue 1057: master-failover succeeds, but IP remains assigned to old master - Fixed Issue 1058: Python's os.minor() does not support devices with high minor numbers - Fixed Issue 1059: Luxid fails if DNS returns an IPv6 address that does not reverse resolve Known issues ~~~~~~~~~~~~ Pending since 2.12.2: - GHC 7.8 introduced some incompatible changes, so currently Ganeti 2.12. doesn't compile on GHC 7.8 - Under certain conditions instance doesn't get unpaused after live migration (issue #1050) - GlusterFS support breaks at upgrade to 2.12 - switches back to shared-file (issue #1030) Version 2.12.2 -------------- *(Released Wed, 25 Mar 2015)* - Support for the lens Haskell library up to version 4.7 (issue #1028) - SSH keys are now distributed only to master and master candidates (issue #377) - Improved performance for operations that frequently read the cluster configuration - Improved robustness of spawning job processes that occasionally caused newly-started jobs to timeout - Fixed race condition during cluster verify which occasionally caused it to fail Inherited from the 2.11 branch: - Fix failing automatic glusterfs mounts (issue #984) - Fix watcher failing to read its status file after an upgrade (issue #1022) - Improve Xen instance state handling, in particular of somewhat exotic transitional states Inherited from the 2.10 branch: - Fix failing to change a diskless drbd instance to plain (issue #1036) - Fixed issues with auto-upgrades from pre-2.6 (hv_state_static and disk_state_static) - Fix memory leak in the monitoring daemon Inherited from the 2.9 branch: - Fix file descriptor leak in Confd client Known issues ~~~~~~~~~~~~ - GHC 7.8 introduced some incompatible changes, so currently Ganeti 2.12. doesn't compile on GHC 7.8 - Under certain conditions instance doesn't get unpaused after live migration (issue #1050) - GlusterFS support breaks at upgrade to 2.12 - switches back to shared-file (issue #1030) Version 2.12.1 -------------- *(Released Wed, 14 Jan 2015)* - Fix users under which the wconfd and metad daemons run (issue #976) - Clean up stale livelock files (issue #865) - Fix setting up the metadata daemon's network interface for Xen - Make watcher identify itself on disk activation - Add "ignore-ipolicy" option to gnt-instance grow-disk - Check disk size ipolicy during "gnt-instance grow-disk" (issue #995) Inherited from the 2.11 branch: - Fix counting votes when doing master failover (issue #962) - Fix broken haskell dependencies (issues #758 and #912) - Check if IPv6 is used directly when running SSH (issue #892) Inherited from the 2.10 branch: - Fix typo in gnt_cluster output (issue #1015) - Use the Python path detected at configure time in the top-level Python scripts. - Fix check for sphinx-build from python2-sphinx - Properly check if an instance exists in 'gnt-instance console' Version 2.12.0 -------------- *(Released Fri, 10 Oct 2014)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Ganeti is now distributed under the 2-clause BSD license. See the COPYING file. - Do not use debug mode in production. Certain daemons will issue warnings when launched in debug mode. Some debug logging violates some of the new invariants in the system (see "New features"). The logging has been kept as it aids diagnostics and development. New features ~~~~~~~~~~~~ - OS install script parameters now come in public, private and secret varieties: - Public parameters are like all other parameters in Ganeti. - Ganeti will not log private and secret parameters, *unless* it is running in debug mode. - Ganeti will not save secret parameters to configuration. Secret parameters must be supplied every time you install, or reinstall, an instance. - Attempting to override public parameters with private or secret parameters results in an error. Similarly, you may not use secret parameters to override private parameters. - The move-instance tool can now attempt to allocate an instance by using opportunistic locking when an iallocator is used. - The build system creates sample systemd unit files, available under doc/examples/systemd. These unit files allow systemd to natively manage and supervise all Ganeti processes. - Different types of compression can be applied during instance moves, including user-specified ones. - Ganeti jobs now run as separate processes. The jobs are coordinated by a new daemon "WConfd" that manages cluster's configuration and locks for individual jobs. A consequence is that more jobs can run in parallel; the number is run-time configurable, see "New features" entry of 2.11.0. To avoid luxid being overloaded with tracking running jobs, it backs of and only occasionally, in a sequential way, checks if jobs have finished and schedules new ones. In this way, luxid keeps responsive under high cluster load. The limit as when to start backing of is also run-time configurable. - The metadata daemon is now optionally available, as part of the partial implementation of the OS-installs design. It allows pass information to OS install scripts or to instances. It is also possible to run Ganeti without the daemon, if desired. - Detection of user shutdown of instances has been implemented for Xen as well. New dependencies ~~~~~~~~~~~~~~~~ - The KVM CPU pinning no longer uses the affinity python package, but psutil instead. The package is still optional and needed only if the feature is to be used. Incomplete features ~~~~~~~~~~~~~~~~~~~ The following issues are related to features which are not completely implemented in 2.12: - Issue 885: Network hotplugging on KVM sometimes makes an instance unresponsive - Issues 708 and 602: The secret parameters are currently still written to disk in the job queue. - Setting up the metadata network interface under Xen isn't fully implemented yet. Known issues ~~~~~~~~~~~~ - *Wrong UDP checksums in DHCP network packets:* If an instance communicates with the metadata daemon and uses DHCP to obtain its IP address on the provided virtual network interface, it can happen that UDP packets have a wrong checksum, due to a bug in virtio. See for example https://bugs.launchpad.net/bugs/930962 Ganeti works around this bug by disabling the UDP checksums on the way from a host to instances (only on the special metadata communication network interface) using the ethtool command. Therefore if using the metadata daemon the host nodes should have this tool available. - The metadata daemon is run as root in the split-user mode, to be able to bind to port 80. This should be improved in future versions, see issue #949. Since 2.12.0 rc2 ~~~~~~~~~~~~~~~~ The following issues have been fixed: - Fixed passing additional parameters to RecreateInstanceDisks over RAPI. - Fixed the permissions of WConfd when running in the split-user mode. As WConfd takes over the previous master daemon to manage the configuration, it currently runs under the masterd user. - Fixed the permissions of the metadata daemon wn running in the split-user mode (see Known issues). - Watcher now properly adds a reason trail entry when initiating disk checks. - Fixed removing KVM parameters introduced in 2.12 when downgrading a cluster to 2.11: "migration_caps", "disk_aio" and "virtio_net_queues". - Improved retrying of RPC calls that fail due to network errors. Version 2.12.0 rc2 ------------------ *(Released Mon, 22 Sep 2014)* This was the second release candidate of the 2.12 series. All important changes are listed in the latest 2.12 entry. Since 2.12.0 rc1 ~~~~~~~~~~~~~~~~ The following issues have been fixed: - Watcher now checks if WConfd is running and functional. - Watcher now properly adds reason trail entries. - Fixed NIC options in Xen's config files. Inherited from the 2.10 branch: - Fixed handling of the --online option - Add warning against hvparam changes with live migrations, which might lead to dangerous situations for instances. - Only the LVs in the configured VG are checked during cluster verify. Version 2.12.0 rc1 ------------------ *(Released Wed, 20 Aug 2014)* This was the first release candidate of the 2.12 series. All important changes are listed in the latest 2.12 entry. Since 2.12.0 beta1 ~~~~~~~~~~~~~~~~~~ The following issues have been fixed: - Issue 881: Handle communication errors in mcpu - Issue 883: WConfd leaks memory for some long operations - Issue 884: Under heavy load the IAllocator fails with a "missing instance" error Inherited from the 2.10 branch: - Improve the recognition of Xen domU states - Automatic upgrades: - Create the config backup archive in a safe way - On upgrades, check for upgrades to resume first - Pause watcher during upgrade - Allow instance disks to be added with --no-wait-for-sync Version 2.12.0 beta1 -------------------- *(Released Mon, 21 Jul 2014)* This was the first beta release of the 2.12 series. All important changes are listed in the latest 2.12 entry. Version 2.11.8 -------------- *(Released Mon, 14 Dec 2015)* Important changes and security notes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Security release. CVE-2015-7944 Ganeti provides a RESTful control interface called the RAPI. Its HTTPS implementation is vulnerable to DoS attacks via client-initiated SSL parameter renegotiation. While the interface is not meant to be exposed publicly, due to the fact that it binds to all interfaces, we believe some users might be exposing it unintentionally and are vulnerable. A DoS attack can consume resources meant for Ganeti daemons and instances running on the master node, making both perform badly. Fixes are not feasible due to the OpenSSL Python library not exposing functionality needed to disable client-side renegotiation. Instead, we offer instructions on how to control RAPI's exposure, along with info on how RAPI can be setup alongside an HTTPS proxy in case users still want or need to expose the RAPI interface. The instructions are outlined in Ganeti's security document: doc/html/security.html CVE-2015-7945 Ganeti leaks the DRBD secret through the RAPI interface. Examining job results after an instance information job reveals the secret. With the DRBD secret, access to the local cluster network, and ARP poisoning, an attacker can impersonate a Ganeti node and clone the disks of a DRBD-based instance. While an attacker with access to the cluster network is already capable of accessing any data written as DRBD traffic is unencrypted, having the secret expedites the process and allows access to the entire disk. Fixes contained in this release prevent the secret from being exposed via the RAPI. The DRBD secret can be changed by converting an instance to plain and back to DRBD, generating a new secret, but redundancy will be lost until the process completes. Since attackers with node access are capable of accessing some and potentially all data even without the secret, we do not recommend that the secret be changed for existing instances. Minor changes ~~~~~~~~~~~~~ - Make htools tolerate missing "dtotal" and "dfree" on luxi - Fix default for --default-iallocator-params - At IAlloc backend guess state from admin state - replace-disks: fix --ignore-ipolicy - Fix instance multi allocation for non-DRBD disks - Trigger renew-crypto on downgrade to 2.11 - Downgrade log-message for rereading job - Downgrade log-level for successful requests - Check for gnt-cluster before running gnt-cluster upgrade Version 2.11.7 -------------- *(Released Fri, 17 Apr 2015)* - The operation 'gnt-cluster renew-crypto --new-node-certificates' is now more robust against intermitten reachability errors. Nodes that are temporarily not reachable, are contacted with several retries. Nodes which are marked as offline are omitted right away. Version 2.11.6 -------------- *(Released Mon, 22 Sep 2014)* - Ganeti is now distributed under the 2-clause BSD license. See the COPYING file. - Fix userspace access checks. - Various documentation fixes have been added. Inherited from the 2.10 branch: - The --online option now works as documented. - The watcher is paused during cluster upgrades; also, upgrade checks for upgrades to resume first. - Instance disks can be added with --no-wait-for-sync. Version 2.11.5 -------------- *(Released Thu, 7 Aug 2014)* Inherited from the 2.10 branch: Important security release. In 2.10.0, the 'gnt-cluster upgrade' command was introduced. Before performing an upgrade, the configuration directory of the cluster is backed up. Unfortunately, the archive was written with permissions that make it possible for non-privileged users to read the archive and thus have access to cluster and RAPI keys. After this release, the archive will be created with privileged access only. We strongly advise you to restrict the permissions of previously created archives. The archives are found in /var/lib/ganeti*.tar (unless otherwise configured with --localstatedir or --with-backup-dir). If you suspect that non-privileged users have accessed your archives already, we advise you to renew the cluster's crypto keys using 'gnt-cluster renew-crypto' and to reset the RAPI credentials by editing /var/lib/ganeti/rapi_users (respectively under a different path if configured differently with --localstatedir). Other changes included in this release: - Fix handling of Xen instance states. - Fix NIC configuration with absent NIC VLAN - Adapt relative path expansion in PATH to new environment - Exclude archived jobs from configuration backups - Fix RAPI for split query setup - Allow disk hot-remove even with chroot or SM Inherited from the 2.9 branch: - Make htools tolerate missing 'spfree' on luxi Version 2.11.4 -------------- *(Released Thu, 31 Jul 2014)* - Improved documentation of the instance shutdown behavior. Inherited from the 2.10 branch: - KVM: fix NIC configuration with absent NIC VLAN (Issue 893) - Adapt relative path expansion in PATH to new environment - Exclude archived jobs from configuration backup - Expose early_release for ReplaceInstanceDisks - Add backup directory for configuration backups for upgrades - Fix BlockdevSnapshot in case of non lvm-based disk - Improve RAPI error handling for queries in non-existing items - Allow disk hot-remove even with chroot or SM - Remove superflous loop in instance queries (Issue 875) Inherited from the 2.9 branch: - Make ganeti-cleaner switch to save working directory (Issue 880) Version 2.11.3 -------------- *(Released Wed, 9 Jul 2014)* - Readd nodes to their previous node group - Remove old-style gnt-network connect Inherited from the 2.10 branch: - Make network_vlan an optional OpParam - hspace: support --accept-existing-errors - Make hspace support --independent-groups - Add a modifier for a group's allocation policy - Export VLAN nicparam to NIC configuration scripts - Fix gnt-network client to accept vlan info - Support disk hotplug with userspace access Inherited from the 2.9 branch: - Make htools tolerate missing "spfree" on luxi - Move the design for query splitting to the implemented list - Add tests for DRBD setups with empty first resource Inherited from the 2.8 branch: - DRBD parser: consume initial empty resource lines Version 2.11.2 -------------- *(Released Fri, 13 Jun 2014)* - Improvements to KVM wrt to the kvmd and instance shutdown behavior. WARNING: In contrast to our standard policy, this bug fix update introduces new parameters to the configuration. This means in particular that after an upgrade from 2.11.0 or 2.11.1, 'cfgupgrade' needs to be run, either manually or explicitly by running 'gnt-cluster upgrade --to 2.11.2' (which requires that they had configured the cluster with --enable-versionfull). This also means, that it is not easily possible to downgrade from 2.11.2 to 2.11.1 or 2.11.0. The only way is to go back to 2.10 and back. Inherited from the 2.10 branch: - Check for SSL encoding inconsistencies - Check drbd helper only in VM capable nodes - Improvements in statistics utils Inherited from the 2.9 branch: - check-man-warnings: use C.UTF-8 and set LC_ALL Version 2.11.1 -------------- *(Released Wed, 14 May 2014)* - Add design-node-security.rst to docinput - kvm: use a dedicated QMP socket for kvmd Inherited from the 2.10 branch: - Set correct Ganeti version on setup commands - Add a utility to combine shell commands - Add design doc for performance tests - Fix failed DRBD disk creation cleanup - Hooking up verification for shared file storage - Fix --shared-file-storage-dir option of gnt-cluster modify - Clarify default setting of 'metavg' - Fix invocation of GetCommandOutput in QA - Clean up RunWithLocks - Add an exception-trapping thread class - Wait for delay to provide interruption information - Add an expected block option to RunWithLocks - Track if a QA test was blocked by locks - Add a RunWithLocks QA utility function - Add restricted migration - Add an example for node evacuation - Add a test for parsing version strings - Tests for parallel job execution - Fail in replace-disks if attaching disks fails - Fix passing of ispecs in cluster init during QA - Move QAThreadGroup to qa_job_utils.py - Extract GetJobStatuses and use an unified version - Run disk template specific tests only if possible Inherited from the 2.9 branch: - If Automake version > 1.11, force serial tests - KVM: set IFF_ONE_QUEUE on created tap interfaces - Add configure option to pass GHC flags Version 2.11.0 -------------- *(Released Fri, 25 Apr 2014)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - ``gnt-node list`` no longer shows disk space information for shared file disk templates because it is not a node attribute. (For example, if you have both the file and shared file disk templates enabled, ``gnt-node list`` now only shows information about the file disk template.) - The shared file disk template is now in the new 'sharedfile' storage type. As a result, ``gnt-node list-storage -t file`` now only shows information about the file disk template and you may use ``gnt-node list-storage -t sharedfile`` to query storage information for the shared file disk template. - Over luxi, syntactially incorrect queries are now rejected as a whole; before, a 'SumbmitManyJobs' request was partially executed, if the outer structure of the request was syntactically correct. As the luxi protocol is internal (external applications are expected to use RAPI), the impact of this incompatible change should be limited. - Queries for nodes, instances, groups, backups and networks are now exclusively done via the luxi daemon. Legacy python code was removed, as well as the --enable-split-queries configuration option. - Orphan volumes errors are demoted to warnings and no longer affect the exit code of ``gnt-cluster verify``. - RPC security got enhanced by using different client SSL certificates for each node. In this context 'gnt-cluster renew-crypto' got a new option '--new-node-certificates', which renews the client certificates of all nodes. After a cluster upgrade from pre-2.11, run this to create client certificates and activate this feature. New features ~~~~~~~~~~~~ - Instance moves, backups and imports can now use compression to transfer the instance data. - Node groups can be configured to use an SSH port different than the default 22. - Added experimental support for Gluster distributed file storage as the ``gluster`` disk template under the new ``sharedfile`` storage type through automatic management of per-node FUSE mount points. You can configure the mount point location at ``gnt-cluster init`` time by using the new ``--gluster-storage-dir`` switch. - Job scheduling is now handled by luxid, and the maximal number of jobs running in parallel is a run-time parameter of the cluster. - A new tool for planning dynamic power management, called ``hsqueeze``, has been added. It suggests nodes to power up or down and corresponding instance moves. New dependencies ~~~~~~~~~~~~~~~~ The following new dependencies have been added: For Haskell: - ``zlib`` library (http://hackage.haskell.org/package/base64-bytestring) - ``base64-bytestring`` library (http://hackage.haskell.org/package/zlib), at least version 1.0.0.0 - ``lifted-base`` library (http://hackage.haskell.org/package/lifted-base) - ``lens`` library (http://hackage.haskell.org/package/lens) Since 2.11.0 rc1 ~~~~~~~~~~~~~~~~ - Fix Xen instance state Inherited from the 2.10 branch: - Fix conflict between virtio + spice or soundhw - Fix bitarray ops wrt PCI slots - Allow releases scheduled 5 days in advance - Make watcher submit queries low priority - Fix specification of TIDiskParams - Add unittests for instance modify parameter renaming - Add renaming of instance custom params - Add RAPI symmetry tests for groups - Extend RAPI symmetry tests with RAPI-only aliases - Add test for group custom parameter renaming - Add renaming of group custom ndparams, ipolicy, diskparams - Add the RAPI symmetry test for nodes - Add aliases for nodes - Allow choice of HTTP method for modification - Add cluster RAPI symmetry test - Fix failing cluster query test - Add aliases for cluster parameters - Add support for value aliases to RAPI - Provide tests for GET/PUT symmetry - Sort imports - Also consider filter fields for deciding if using live data - Document the python-fdsend dependency - Verify configuration version number before parsing - KVM: use running HVPs to calc blockdev options - KVM: reserve a PCI slot for the SCSI controller - Check for LVM-based verification results only when enabled - Fix "existing" typos - Fix output of gnt-instance info after migration - Warn in UPGRADE about not tar'ing exported insts - Fix non-running test and remove custom_nicparams rename - Account for NODE_RES lock in opportunistic locking - Fix request flooding of noded during disk sync Inherited from the 2.9 branch: - Make watcher submit queries low priority - Fix failing gnt-node list-drbd command - Update installation guide wrt to DRBD version - Fix list-drbd QA test - Add messages about skipped QA disk template tests - Allow QA asserts to produce more messages - Set exclusion tags correctly in requested instance - Export extractExTags and updateExclTags - Document spindles in the hbal man page - Sample logrotate conf breaks permissions with split users - Fix 'gnt-cluster' and 'gnt-node list-storage' outputs Inherited from the 2.8 branch: - Add reason parameter to RAPI client functions - Include qa/patch in Makefile - Handle empty patches better - Move message formatting functions to separate file - Add optional ordering of QA patch files - Allow multiple QA patches - Refactor current patching code Version 2.11.0 rc1 ------------------ *(Released Thu, 20 Mar 2014)* This was the first RC release of the 2.11 series. Since 2.11.0 beta1: - Convert int to float when checking config. consistency - Rename compression option in gnt-backup export Inherited from the 2.9 branch: - Fix error introduced during merge - gnt-cluster copyfile: accept relative paths Inherited from the 2.8 branch: - Improve RAPI detection of the watcher - Add patching QA configuration files on buildbots - Enable a timeout for instance shutdown - Allow KVM commands to have a timeout - Allow xen commands to have a timeout - Fix wrong docstring Version 2.11.0 beta1 -------------------- *(Released Wed, 5 Mar 2014)* This was the first beta release of the 2.11 series. All important changes are listed in the latest 2.11 entry. Version 2.10.8 -------------- *(Released Fri, 11 Dec 2015)* Important changes and security notes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Security release. CVE-2015-7944 Ganeti provides a RESTful control interface called the RAPI. Its HTTPS implementation is vulnerable to DoS attacks via client-initiated SSL parameter renegotiation. While the interface is not meant to be exposed publicly, due to the fact that it binds to all interfaces, we believe some users might be exposing it unintentionally and are vulnerable. A DoS attack can consume resources meant for Ganeti daemons and instances running on the master node, making both perform badly. Fixes are not feasible due to the OpenSSL Python library not exposing functionality needed to disable client-side renegotiation. Instead, we offer instructions on how to control RAPI's exposure, along with info on how RAPI can be setup alongside an HTTPS proxy in case users still want or need to expose the RAPI interface. The instructions are outlined in Ganeti's security document: doc/html/security.html CVE-2015-7945 Ganeti leaks the DRBD secret through the RAPI interface. Examining job results after an instance information job reveals the secret. With the DRBD secret, access to the local cluster network, and ARP poisoning, an attacker can impersonate a Ganeti node and clone the disks of a DRBD-based instance. While an attacker with access to the cluster network is already capable of accessing any data written as DRBD traffic is unencrypted, having the secret expedites the process and allows access to the entire disk. Fixes contained in this release prevent the secret from being exposed via the RAPI. The DRBD secret can be changed by converting an instance to plain and back to DRBD, generating a new secret, but redundancy will be lost until the process completes. Since attackers with node access are capable of accessing some and potentially all data even without the secret, we do not recommend that the secret be changed for existing instances. Minor changes ~~~~~~~~~~~~~ - Make htools tolerate missing "dtotal" and "dfree" on luxi - At IAlloc backend guess state from admin state - replace-disks: fix --ignore-ipolicy - Fix instance multi allocation for non-DRBD disks - Check for gnt-cluster before running gnt-cluster upgrade - Work around a Python os.minor bug - Add IP-related checks after master-failover - Pass correct backend params in move-instance - Allow plain/DRBD conversions regardless of lack of disks - Fix MonD collector thunk leak - Stop MonD when removing a node from a cluster - Finalize backup only if successful - Fix file descriptor leak in Confd Client - Auto-upgrade hv_state_static and disk_state_static - Do not hardcode the Python path in CLI tools - Use the Python interpreter from ENV - ganeti.daemon: fix daemon mode with GnuTLS >= 3.3 (Issues 961, 964) - Ganeti.Daemon: always install SIGHUP handler (Issue 755) - Fix DRBD version check for non VM capable nodes - Fix handling of the --online option - Add warning against hvparam changes with live migrations - Only verify LVs in configured VG during cluster verify - Fix network info in case of multi NIC instances - On upgrades, check for upgrades to resume first - Pause watcher during upgrade - Allow instance disks to be added with --no-wait-for-sync Version 2.10.7 -------------- *(Released Thu, 7 Aug 2014)* Important security release. In 2.10.0, the 'gnt-cluster upgrade' command was introduced. Before performing an upgrade, the configuration directory of the cluster is backed up. Unfortunately, the archive was written with permissions that make it possible for non-privileged users to read the archive and thus have access to cluster and RAPI keys. After this release, the archive will be created with privileged access only. We strongly advise you to restrict the permissions of previously created archives. The archives are found in /var/lib/ganeti*.tar (unless otherwise configured with --localstatedir or --with-backup-dir). If you suspect that non-privileged users have accessed your archives already, we advise you to renew the cluster's crypto keys using 'gnt-cluster renew-crypto' and to reset the RAPI credentials by editing /var/lib/ganeti/rapi_users (respectively under a different path if configured differently with --localstatedir). Other changes included in this release: - Fix handling of Xen instance states. - Fix NIC configuration with absent NIC VLAN - Adapt relative path expansion in PATH to new environment - Exclude archived jobs from configuration backups - Fix RAPI for split query setup - Allow disk hot-remove even with chroot or SM Inherited from the 2.9 branch: - Make htools tolerate missing 'spfree' on luxi Version 2.10.6 -------------- *(Released Mon, 30 Jun 2014)* - Make Ganeti tolerant towards different openssl library version on different nodes (issue 853). - Allow hspace to make useful predictions in multi-group clusters with one group overfull (isse 861). - Various gnt-network related fixes. - Fix disk hotplug with userspace access. - Various documentation errors fixed. Version 2.10.5 -------------- *(Released Mon, 2 Jun 2014)* - Two new options have been added to gnt-group evacuate. The 'sequential' option forces all the evacuation steps to be carried out sequentially, thus avoiding congestion on a slow link between node groups. The 'force-failover' option disallows migrations and forces failovers to be used instead. In this way evacuation to a group with vastly differnet hypervisor is possible. - In tiered allocation, when looking for ways on how to shrink an instance, the canoncial path is tried first, i.e., in each step reduce on the resource most placements are blocked on. Only if no smaller fitting instance can be found shrinking a single resource till fit is tried. - For finding the placement of an instance, the duplicate computations in the computation of the various cluster scores are computed only once. This significantly improves the performance of hspace for DRBD on large clusters; for other clusters, a slight performance decrease might occur. Moreover, due to the changed order, floating point number inaccuracies accumulate differently, thus resulting in different cluster scores. It has been verified that the effect of these different roundings is less than 1e-12. - network queries fixed with respect to instances - relax too strict prerequisite in LUClusterSetParams for DRBD helpers - VArious improvements to QA and build-time tests Version 2.10.4 -------------- *(Released Thu, 15 May 2014)* - Support restricted migration in hbal - Fix for the --shared-file-storage-dir of gnt-cluster modify (issue 811) - Fail in replace-disks if attaching disks fails (issue 814) - Set IFF_ONE_QUEUE on created tap interfaces for KVM - Small fixes and enhancements in the build system - Various documentation fixes (e.g. issue 810) Version 2.10.3 -------------- *(Released Wed, 16 Apr 2014)* - Fix filtering of pending jobs with -o id (issue 778) - Make RAPI API calls more symmetric (issue 770) - Make parsing of old cluster configuration more robust (issue 783) - Fix wrong output of gnt-instance info after migrations - Fix reserved PCI slots for KVM hotplugging - Use runtime hypervisor parameters to calculate bockdevice options for KVM - Fix high node daemon load during disk sync if the sync is paused manually (issue 792) - Improve opportunistic locking during instance creation (issue 791) Inherited from the 2.9 branch: - Make watcher submit queries low priority (issue 772) - Add reason parameter to RAPI client functions (issue 776) - Fix failing gnt-node list-drbd command (issue 777) - Properly display fake job locks in gnt-debug. - small fixes in documentation Version 2.10.2 -------------- *(Released Mon, 24 Mar 2014)* - Fix conflict between virtio + spice or soundhw (issue 757) - accept relative paths in gnt-cluster copyfile (issue 754) - Introduce shutdown timeout for 'xm shutdown' command - Improve RAPI detection of the watcher (issue 752) Version 2.10.1 -------------- *(Released Wed, 5 Mar 2014)* - Fix incorrect invocation of hooks on offline nodes (issue 742) - Fix incorrect exit code of gnt-cluster verify in certain circumstances (issue 744) Inherited from the 2.9 branch: - Fix overflow problem in hbal that caused it to break when waiting for jobs for more than 10 minutes (issue 717) - Make hbal properly handle non-LVM storage - Properly export and import NIC parameters, and do so in a backwards compatible way (issue 716) - Fix net-common script in case of routed mode (issue 728) - Improve documentation (issues 724, 730) Version 2.10.0 -------------- *(Released Thu, 20 Feb 2014)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Adding disks with 'gnt-instance modify' now waits for the disks to sync per default. Specify --no-wait-for-sync to override this behavior. - The Ganeti python code now adheres to a private-module layout. In particular, the module 'ganeti' is no longer in the python search path. - On instance allocation, the iallocator now considers non-LVM storage properly. In particular, actual file storage space information is used when allocating space for a file/sharedfile instance. - When disabling disk templates cluster-wide, the cluster now first checks whether there are instances still using those templates. - 'gnt-node list-storage' now also reports storage information about file-based storage types. - In case of non drbd instances, export \*_SECONDARY environment variables as empty strings (and not "None") during 'instance-migrate' related hooks. New features ~~~~~~~~~~~~ - KVM hypervisors can now access RBD storage directly without having to go through a block device. - A new command 'gnt-cluster upgrade' was added that automates the upgrade procedure between two Ganeti versions that are both 2.10 or higher. - The move-instance command can now change disk templates when moving instances, and does not require any node placement options to be specified if the destination cluster has a default iallocator. - Users can now change the soundhw and cpuid settings for XEN hypervisors. - Hail and hbal now have the (optional) capability of accessing average CPU load information through the monitoring daemon, and to use it to dynamically adapt the allocation of instances. - Hotplug support. Introduce new option '--hotplug' to ``gnt-instance modify`` so that disk and NIC modifications take effect without the need of actual reboot. There are a couple of constrains currently for this feature: - only KVM hypervisor (versions >= 1.0) supports it, - one can not (yet) hotplug a disk using userspace access mode for RBD - in case of a downgrade instances should suffer a reboot in order to be migratable (due to core change of runtime files) - ``python-fdsend`` is required for NIC hotplugging. Misc changes ~~~~~~~~~~~~ - A new test framework for logical units was introduced and the test coverage for logical units was improved significantly. - Opcodes are entirely generated from Haskell using the tool 'hs2py' and the module 'src/Ganeti/OpCodes.hs'. - Constants are also generated from Haskell using the tool 'hs2py-constants' and the module 'src/Ganeti/Constants.hs', with the exception of socket related constants, which require changing the cluster configuration file, and HVS related constants, because they are part of a port of instance queries to Haskell. As a result, these changes will be part of the next release of Ganeti. New dependencies ~~~~~~~~~~~~~~~~ The following new dependencies have been added/updated. Python - The version requirements for ``python-mock`` have increased to at least version 1.0.1. It is still used for testing only. - ``python-fdsend`` (https://gitorious.org/python-fdsend) is optional but required for KVM NIC hotplugging to work. Since 2.10.0 rc3 ~~~~~~~~~~~~~~~~ - Fix integer overflow problem in hbal Version 2.10.0 rc3 ------------------ *(Released Wed, 12 Feb 2014)* This was the third RC release of the 2.10 series. Since 2.10.0 rc2: - Improved hotplug robustness - Start Ganeti daemons after ensure-dirs during upgrade - Documentation improvements Inherited from the 2.9 branch: - Fix the RAPI instances-multi-alloc call - assign unique filenames to file-based disks - gracefully handle degraded non-diskless instances with 0 disks (issue 697) - noded now runs with its specified group, which is the default group, defaulting to root (issue 707) - make using UUIDs to identify nodes in gnt-node consistently possible (issue 703) Version 2.10.0 rc2 ------------------ *(Released Fri, 31 Jan 2014)* This was the second RC release of the 2.10 series. Since 2.10.0 rc1: - Documentation improvements - Run drbdsetup syncer only on network attach - Include target node in hooks nodes for migration - Fix configure dirs - Support post-upgrade hooks during cluster upgrades Inherited from the 2.9 branch: - Ensure that all the hypervisors exist in the config file (Issue 640) - Correctly recognise the role as master node (Issue 687) - configure: allow detection of Sphinx 1.2+ (Issue 502) - gnt-instance now honors the KVM path correctly (Issue 691) Inherited from the 2.8 branch: - Change the list separator for the usb_devices parameter from comma to space. Commas could not work because they are already the hypervisor option separator (Issue 649) - Add support for blktap2 file-driver (Issue 638) - Add network tag definitions to the haskell codebase (Issue 641) - Fix RAPI network tag handling - Add the network tags to the tags searched by gnt-cluster search-tags - Fix caching bug preventing jobs from being cancelled - Start-master/stop-master was always failing if ConfD was disabled. (Issue 685) Version 2.10.0 rc1 ------------------ *(Released Tue, 17 Dec 2013)* This was the first RC release of the 2.10 series. Since 2.10.0 beta1: - All known issues in 2.10.0 beta1 have been resolved (see changes from the 2.8 branch). - Improve handling of KVM runtime files from earlier Ganeti versions - Documentation fixes Inherited from the 2.9 branch: - use custom KVM path if set for version checking - SingleNotifyPipeCondition: don't share pollers Inherited from the 2.8 branch: - Fixed Luxi daemon socket permissions after master-failover - Improve IP version detection code directly checking for colons rather than passing the family from the cluster object - Fix NODE/NODE_RES locking in LUInstanceCreate by not acquiring NODE_RES locks opportunistically anymore (Issue 622) - Allow link local IPv6 gateways (Issue 624) - Fix error printing (Issue 616) - Fix a bug in InstanceSetParams concerning names: in case no name is passed in disk modifications, keep the old one. If name=none then set disk name to None. - Update build_chroot script to work with the latest hackage packages - Add a packet number limit to "fping" in master-ip-setup (Issue 630) - Fix evacuation out of drained node (Issue 615) - Add default file_driver if missing (Issue 571) - Fix job error message after unclean master shutdown (Issue 618) - Lock group(s) when creating instances (Issue 621) - SetDiskID() before accepting an instance (Issue 633) - Allow the ext template disks to receive arbitrary parameters, both at creation time and while being modified - Xen handle domain shutdown (future proofing cherry-pick) - Refactor reading live data in htools (future proofing cherry-pick) Version 2.10.0 beta1 -------------------- *(Released Wed, 27 Nov 2013)* This was the first beta release of the 2.10 series. All important changes are listed in the latest 2.10 entry. Known issues ~~~~~~~~~~~~ The following issues are known to be present in the beta and will be fixed before rc1. - Issue 477: Wrong permissions for confd LUXI socket - Issue 621: Instance related opcodes do not aquire network/group locks - Issue 622: Assertion Error: Node locks differ from node resource locks - Issue 623: IPv6 Masterd <-> Luxid communication error Version 2.9.7 ------------- *(Released Fri, 11 Dec 2015)* Important changes and security notes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Security release. CVE-2015-7944 Ganeti provides a RESTful control interface called the RAPI. Its HTTPS implementation is vulnerable to DoS attacks via client-initiated SSL parameter renegotiation. While the interface is not meant to be exposed publicly, due to the fact that it binds to all interfaces, we believe some users might be exposing it unintentionally and are vulnerable. A DoS attack can consume resources meant for Ganeti daemons and instances running on the master node, making both perform badly. Fixes are not feasible due to the OpenSSL Python library not exposing functionality needed to disable client-side renegotiation. Instead, we offer instructions on how to control RAPI's exposure, along with info on how RAPI can be setup alongside an HTTPS proxy in case users still want or need to expose the RAPI interface. The instructions are outlined in Ganeti's security document: doc/html/security.html CVE-2015-7945 Ganeti leaks the DRBD secret through the RAPI interface. Examining job results after an instance information job reveals the secret. With the DRBD secret, access to the local cluster network, and ARP poisoning, an attacker can impersonate a Ganeti node and clone the disks of a DRBD-based instance. While an attacker with access to the cluster network is already capable of accessing any data written as DRBD traffic is unencrypted, having the secret expedites the process and allows access to the entire disk. Fixes contained in this release prevent the secret from being exposed via the RAPI. The DRBD secret can be changed by converting an instance to plain and back to DRBD, generating a new secret, but redundancy will be lost until the process completes. Since attackers with node access are capable of accessing some and potentially all data even without the secret, we do not recommend that the secret be changed for existing instances. Minor changes ~~~~~~~~~~~~~ - gnt-instance replace-disks no longer crashes when --ignore-policy is passed to it - Stop MonD when removing a node from a cluster - Fix file descriptor leak in Confd client - Always install SIGHUP handler for Haskell daemons (Issue 755) - Make ganeti-cleaner switch to a safe working directory (Issue 880) - Make htools tolerate missing "spfree" on Luxi - DRBD parser: consume initial empty resource lines (Issue 869) - KVM: set IFF_ONE_QUEUE on created tap interfaces - Set exclusion tags correctly in requested instance Version 2.9.6 ------------- *(Released Mon, 7 Apr 2014)* - Improve RAPI detection of the watcher (Issue 752) - gnt-cluster copyfile: accept relative paths (Issue 754) - Make watcher submit queries low priority (Issue 772) - Add reason parameter to RAPI client functions (Issue 776) - Fix failing gnt-node list-drbd command (Issue 777) - Properly display fake job locks in gnt-debug. - Enable timeout for instance shutdown - small fixes in documentation Version 2.9.5 ------------- *(Released Tue, 25 Feb 2014)* - Fix overflow problem in hbal that caused it to break when waiting for jobs for more than 10 minutes (issue 717) - Make hbal properly handle non-LVM storage - Properly export and import NIC parameters, and do so in a backwards compatible way (issue 716) - Fix net-common script in case of routed mode (issue 728) - Improve documentation (issues 724, 730) Version 2.9.4 ------------- *(Released Mon, 10 Feb 2014)* - Fix the RAPI instances-multi-alloc call - assign unique filenames to file-based disks - gracefully handle degraded non-diskless instances with 0 disks (issue 697) - noded now runs with its specified group, which is the default group, defaulting to root (issue 707) - make using UUIDs to identify nodes in gnt-node consistently possible (issue 703) Version 2.9.3 ------------- *(Released Mon, 27 Jan 2014)* - Ensure that all the hypervisors exist in the config file (Issue 640) - Correctly recognise the role as master node (Issue 687) - configure: allow detection of Sphinx 1.2+ (Issue 502) - gnt-instance now honors the KVM path correctly (Issue 691) Inherited from the 2.8 branch: - Change the list separator for the usb_devices parameter from comma to space. Commas could not work because they are already the hypervisor option separator (Issue 649) - Add support for blktap2 file-driver (Issue 638) - Add network tag definitions to the haskell codebase (Issue 641) - Fix RAPI network tag handling - Add the network tags to the tags searched by gnt-cluster search-tags - Fix caching bug preventing jobs from being cancelled - Start-master/stop-master was always failing if ConfD was disabled. (Issue 685) Version 2.9.2 ------------- *(Released Fri, 13 Dec 2013)* - use custom KVM path if set for version checking - SingleNotifyPipeCondition: don't share pollers Inherited from the 2.8 branch: - Fixed Luxi daemon socket permissions after master-failover - Improve IP version detection code directly checking for colons rather than passing the family from the cluster object - Fix NODE/NODE_RES locking in LUInstanceCreate by not acquiring NODE_RES locks opportunistically anymore (Issue 622) - Allow link local IPv6 gateways (Issue 624) - Fix error printing (Issue 616) - Fix a bug in InstanceSetParams concerning names: in case no name is passed in disk modifications, keep the old one. If name=none then set disk name to None. - Update build_chroot script to work with the latest hackage packages - Add a packet number limit to "fping" in master-ip-setup (Issue 630) - Fix evacuation out of drained node (Issue 615) - Add default file_driver if missing (Issue 571) - Fix job error message after unclean master shutdown (Issue 618) - Lock group(s) when creating instances (Issue 621) - SetDiskID() before accepting an instance (Issue 633) - Allow the ext template disks to receive arbitrary parameters, both at creation time and while being modified - Xen handle domain shutdown (future proofing cherry-pick) - Refactor reading live data in htools (future proofing cherry-pick) Version 2.9.1 ------------- *(Released Wed, 13 Nov 2013)* - fix bug, that kept nodes offline when readding - when verifying DRBD versions, ignore unavailable nodes - fix bug that made the console unavailable on kvm in split-user setup (issue 608) - DRBD: ensure peers are UpToDate for dual-primary (inherited 2.8.2) Version 2.9.0 ------------- *(Released Tue, 5 Nov 2013)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - hroller now also plans for capacity to move non-redundant instances off any node to be rebooted; the old behavior of completely ignoring any non-redundant instances can be restored by adding the --ignore-non-redundant option. - The cluster option '--no-lvm-storage' was removed in favor of the new option '--enabled-disk-templates'. - On instance creation, disk templates no longer need to be specified with '-t'. The default disk template will be taken from the list of enabled disk templates. - The monitoring daemon is now running as root, in order to be able to collect information only available to root (such as the state of Xen instances). - The ConfD client is now IPv6 compatible. - File and shared file storage is no longer dis/enabled at configure time, but using the option '--enabled-disk-templates' at cluster initialization and modification. - The default directories for file and shared file storage are not anymore specified at configure time, but taken from the cluster's configuration. They can be set at cluster initialization and modification with '--file-storage-dir' and '--shared-file-storage-dir'. - Cluster verification now includes stricter checks regarding the default file and shared file storage directories. It now checks that the directories are explicitly allowed in the 'file-storage-paths' file and that the directories exist on all nodes. - The list of allowed disk templates in the instance policy and the list of cluster-wide enabled disk templates is now checked for consistency on cluster or group modification. On cluster initialization, the ipolicy disk templates are ensured to be a subset of the cluster-wide enabled disk templates. New features ~~~~~~~~~~~~ - DRBD 8.4 support. Depending on the installed DRBD version, Ganeti now uses the correct command syntax. It is possible to use different DRBD versions on different nodes as long as they are compatible to each other. This enables rolling upgrades of DRBD with no downtime. As permanent operation of different DRBD versions within a node group is discouraged, ``gnt-cluster verify`` will emit a warning if it detects such a situation. - New "inst-status-xen" data collector for the monitoring daemon, providing information about the state of the xen instances on the nodes. - New "lv" data collector for the monitoring daemon, collecting data about the logical volumes on the nodes, and pairing them with the name of the instances they belong to. - New "diskstats" data collector, collecting the data from /proc/diskstats and presenting them over the monitoring daemon interface. - The ConfD client is now IPv6 compatible. New dependencies ~~~~~~~~~~~~~~~~ The following new dependencies have been added. Python - ``python-mock`` (http://www.voidspace.org.uk/python/mock/) is now a required for the unit tests (and only used for testing). Haskell - ``hslogger`` (http://software.complete.org/hslogger) is now always required, even if confd is not enabled. Since 2.9.0 rc3 ~~~~~~~~~~~~~~~ - Correctly start/stop luxid during gnt-cluster master-failover (inherited from stable-2.8) - Improved error messsages (inherited from stable-2.8) Version 2.9.0 rc3 ----------------- *(Released Tue, 15 Oct 2013)* The third release candidate in the 2.9 series. Since 2.9.0 rc2: - in implicit configuration upgrade, match ipolicy with enabled disk templates - improved harep documentation (inherited from stable-2.8) Version 2.9.0 rc2 ----------------- *(Released Wed, 9 Oct 2013)* The second release candidate in the 2.9 series. Since 2.9.0 rc1: - Fix bug in cfgupgrade that led to failure when upgrading from 2.8 with at least one DRBD instance. - Fix bug in cfgupgrade that led to an invalid 2.8 configuration after downgrading. Version 2.9.0 rc1 ----------------- *(Released Tue, 1 Oct 2013)* The first release candidate in the 2.9 series. Since 2.9.0 beta1: - various bug fixes - update of the documentation, in particular installation instructions - merging of LD_* constants into DT_* constants - python style changes to be compatible with newer versions of pylint Version 2.9.0 beta1 ------------------- *(Released Thu, 29 Aug 2013)* This was the first beta release of the 2.9 series. All important changes are listed in the latest 2.9 entry. Version 2.8.4 ------------- *(Released Thu, 23 Jan 2014)* - Change the list separator for the usb_devices parameter from comma to space. Commas could not work because they are already the hypervisor option separator (Issue 649) - Add support for blktap2 file-driver (Issue 638) - Add network tag definitions to the haskell codebase (Issue 641) - Fix RAPI network tag handling - Add the network tags to the tags searched by gnt-cluster search-tags - Fix caching bug preventing jobs from being cancelled - Start-master/stop-master was always failing if ConfD was disabled. (Issue 685) Version 2.8.3 ------------- *(Released Thu, 12 Dec 2013)* - Fixed Luxi daemon socket permissions after master-failover - Improve IP version detection code directly checking for colons rather than passing the family from the cluster object - Fix NODE/NODE_RES locking in LUInstanceCreate by not acquiring NODE_RES locks opportunistically anymore (Issue 622) - Allow link local IPv6 gateways (Issue 624) - Fix error printing (Issue 616) - Fix a bug in InstanceSetParams concerning names: in case no name is passed in disk modifications, keep the old one. If name=none then set disk name to None. - Update build_chroot script to work with the latest hackage packages - Add a packet number limit to "fping" in master-ip-setup (Issue 630) - Fix evacuation out of drained node (Issue 615) - Add default file_driver if missing (Issue 571) - Fix job error message after unclean master shutdown (Issue 618) - Lock group(s) when creating instances (Issue 621) - SetDiskID() before accepting an instance (Issue 633) - Allow the ext template disks to receive arbitrary parameters, both at creation time and while being modified - Xen handle domain shutdown (future proofing cherry-pick) - Refactor reading live data in htools (future proofing cherry-pick) Version 2.8.2 ------------- *(Released Thu, 07 Nov 2013)* - DRBD: ensure peers are UpToDate for dual-primary - Improve error message for replace-disks - More dependency checks at configure time - Placate warnings on ganeti.outils_unittest.py Version 2.8.1 ------------- *(Released Thu, 17 Oct 2013)* - Correctly start/stop luxid during gnt-cluster master-failover - Don't attempt IPv6 ssh in case of IPv4 cluster (Issue 595) - Fix path for the job queue serial file - Improved harep man page - Minor documentation improvements Version 2.8.0 ------------- *(Released Mon, 30 Sep 2013)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Instance policy can contain multiple instance specs, as described in the “Constrained instance sizes” section of :doc:`Partitioned Ganeti `. As a consequence, it's not possible to partially change or override instance specs. Bounding specs (min and max) can be specified as a whole using the new option ``--ipolicy-bounds-specs``, while standard specs use the new option ``--ipolicy-std-specs``. - The output of the info command of gnt-cluster, gnt-group, gnt-node, gnt-instance is a valid YAML object. - hail now honors network restrictions when allocating nodes. This led to an update of the IAllocator protocol. See the IAllocator documentation for details. - confd now only answers static configuration request over the network. luxid was extracted, listens on the local LUXI socket and responds to live queries. This allows finer grained permissions if using separate users. New features ~~~~~~~~~~~~ - The :doc:`Remote API ` daemon now supports a command line flag to always require authentication, ``--require-authentication``. It can be specified in ``$sysconfdir/default/ganeti``. - A new cluster attribute 'enabled_disk_templates' is introduced. It will be used to manage the disk templates to be used by instances in the cluster. Initially, it will be set to a list that includes plain, drbd, if they were enabled by specifying a volume group name, and file and sharedfile, if those were enabled at configure time. Additionally, it will include all disk templates that are currently used by instances. The order of disk templates will be based on Ganeti's history of supporting them. In the future, the first entry of the list will be used as a default disk template on instance creation. - ``cfgupgrade`` now supports a ``--downgrade`` option to bring the configuration back to the previous stable version. - Disk templates in group ipolicy can be restored to the default value. - Initial support for diskless instances and virtual clusters in QA. - More QA and unit tests for instance policies. - Every opcode now contains a reason trail (visible through ``gnt-job info``) describing why the opcode itself was executed. - The monitoring daemon is now available. It allows users to query the cluster for obtaining information about the status of the system. The daemon is only responsible for providing the information over the network: the actual data gathering is performed by data collectors (currently, only the DRBD status collector is available). - In order to help developers work on Ganeti, a new script (``devel/build_chroot``) is provided, for building a chroot that contains all the required development libraries and tools for compiling Ganeti on a Debian Squeeze system. - A new tool, ``harep``, for performing self-repair and recreation of instances in Ganeti has been added. - Split queries are enabled for tags, network, exports, cluster info, groups, jobs, nodes. - New command ``show-ispecs-cmd`` for ``gnt-cluster`` and ``gnt-group``. It prints the command line to set the current policies, to ease changing them. - Add the ``vnet_hdr`` HV parameter for KVM, to control whether the tap devices for KVM virtio-net interfaces will get created with VNET_HDR (IFF_VNET_HDR) support. If set to false, it disables offloading on the virtio-net interfaces, which prevents host kernel tainting and log flooding, when dealing with broken or malicious virtio-net drivers. It's set to true by default. - Instance failover now supports a ``--cleanup`` parameter for fixing previous failures. - Support 'viridian' parameter in Xen HVM - Support DSA SSH keys in bootstrap - To simplify the work of packaging frameworks that want to add the needed users and groups in a split-user setup themselves, at build time three files in ``doc/users`` will be generated. The ``groups`` files contains, one per line, the groups to be generated, the ``users`` file contains, one per line, the users to be generated, optionally followed by their primary group, where important. The ``groupmemberships`` file contains, one per line, additional user-group membership relations that need to be established. The syntax of these files will remain stable in all future versions. New dependencies ~~~~~~~~~~~~~~~~ The following new dependencies have been added: For Haskell: - The ``curl`` library is not optional anymore for compiling the Haskell code. - ``snap-server`` library (if monitoring is enabled). For Python: - The minimum Python version needed to run Ganeti is now 2.6. - ``yaml`` library (only for running the QA). Since 2.8.0 rc3 ~~~~~~~~~~~~~~~ - Perform proper cleanup on termination of Haskell daemons - Fix corner-case in handling of remaining retry time Version 2.8.0 rc3 ----------------- *(Released Tue, 17 Sep 2013)* - To simplify the work of packaging frameworks that want to add the needed users and groups in a split-user setup themselves, at build time three files in ``doc/users`` will be generated. The ``groups`` files contains, one per line, the groups to be generated, the ``users`` file contains, one per line, the users to be generated, optionally followed by their primary group, where important. The ``groupmemberships`` file contains, one per line, additional user-group membership relations that need to be established. The syntax of these files will remain stable in all future versions. - Add a default to file-driver when unspecified over RAPI (Issue 571) - Mark the DSA host pubkey as optional, and remove it during config downgrade (Issue 560) - Some documentation fixes Version 2.8.0 rc2 ----------------- *(Released Tue, 27 Aug 2013)* The second release candidate of the 2.8 series. Since 2.8.0. rc1: - Support 'viridian' parameter in Xen HVM (Issue 233) - Include VCS version in ``gnt-cluster version`` - Support DSA SSH keys in bootstrap (Issue 338) - Fix batch creation of instances - Use FQDN to check master node status (Issue 551) - Make the DRBD collector more failure-resilient Version 2.8.0 rc1 ----------------- *(Released Fri, 2 Aug 2013)* The first release candidate of the 2.8 series. Since 2.8.0 beta1: - Fix upgrading/downgrading from 2.7 - Increase maximum RAPI message size - Documentation updates - Split ``confd`` between ``luxid`` and ``confd`` - Merge 2.7 series up to the 2.7.1 release - Allow the ``modify_etc_hosts`` option to be changed - Add better debugging for ``luxid`` queries - Expose bulk parameter for GetJobs in RAPI client - Expose missing ``network`` fields in RAPI - Add some ``cluster verify`` tests - Some unittest fixes - Fix a malfunction in ``hspace``'s tiered allocation - Fix query compatibility between haskell and python implementations - Add the ``vnet_hdr`` HV parameter for KVM - Add ``--cleanup`` to instance failover - Change the connected groups format in ``gnt-network info`` output; it was previously displayed as a raw list by mistake. (Merged from 2.7) Version 2.8.0 beta1 ------------------- *(Released Mon, 24 Jun 2013)* This was the first beta release of the 2.8 series. All important changes are listed in the latest 2.8 entry. Version 2.7.2 ------------- *(Released Thu, 26 Sep 2013)* - Change the connected groups format in ``gnt-network info`` output; it was previously displayed as a raw list by mistake - Check disk template in right dict when copying - Support multi-instance allocs without iallocator - Fix some errors in the documentation - Fix formatting of tuple in an error message Version 2.7.1 ------------- *(Released Thu, 25 Jul 2013)* - Add logrotate functionality in daemon-util - Add logrotate example file - Add missing fields to network queries over rapi - Fix network object timestamps - Add support for querying network timestamps - Fix a typo in the example crontab - Fix a documentation typo Version 2.7.0 ------------- *(Released Thu, 04 Jul 2013)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Instance policies for disk size were documented to be on a per-disk basis, but hail applied them to the sum of all disks. This has been fixed. - ``hbal`` will now exit with status 0 if, during job execution over LUXI, early exit has been requested and all jobs are successful; before, exit status 1 was used, which cannot be differentiated from "job error" case - Compatibility with newer versions of rbd has been fixed - ``gnt-instance batch-create`` has been changed to use the bulk create opcode from Ganeti. This lead to incompatible changes in the format of the JSON file. It's now not a custom dict anymore but a dict compatible with the ``OpInstanceCreate`` opcode. - Parent directories for file storage need to be listed in ``$sysconfdir/ganeti/file-storage-paths`` now. ``cfgupgrade`` will write the file automatically based on old configuration values, but it can not distribute it across all nodes and the file contents should be verified. Use ``gnt-cluster copyfile $sysconfdir/ganeti/file-storage-paths`` once the cluster has been upgraded. The reason for requiring this list of paths now is that before it would have been possible to inject new paths via RPC, allowing files to be created in arbitrary locations. The RPC protocol is protected using SSL/X.509 certificates, but as a design principle Ganeti does not permit arbitrary paths to be passed. - The parsing of the variants file for OSes (see :manpage:`ganeti-os-interface(7)`) has been slightly changed: now empty lines and comment lines (starting with ``#``) are ignored for better readability. - The ``setup-ssh`` tool added in Ganeti 2.2 has been replaced and is no longer available. ``gnt-node add`` now invokes a new tool on the destination node, named ``prepare-node-join``, to configure the SSH daemon. Paramiko is no longer necessary to configure nodes' SSH daemons via ``gnt-node add``. - Draining (``gnt-cluster queue drain``) and un-draining the job queue (``gnt-cluster queue undrain``) now affects all nodes in a cluster and the flag is not reset after a master failover. - Python 2.4 has *not* been tested with this release. Using 2.6 or above is recommended. 2.6 will be mandatory from the 2.8 series. New features ~~~~~~~~~~~~ - New network management functionality to support automatic allocation of IP addresses and managing of network parameters. See :manpage:`gnt-network(8)` for more details. - New external storage backend, to allow managing arbitrary storage systems external to the cluster. See :manpage:`ganeti-extstorage-interface(7)`. - New ``exclusive-storage`` node parameter added, restricted to nodegroup level. When it's set to true, physical disks are assigned in an exclusive fashion to instances, as documented in :doc:`Partitioned Ganeti `. Currently, only instances using the ``plain`` disk template are supported. - The KVM hypervisor has been updated with many new hypervisor parameters, including a generic one for passing arbitrary command line values. See a complete list in :manpage:`gnt-instance(8)`. It is now compatible up to qemu 1.4. - A new tool, called ``mon-collector``, is the stand-alone executor of the data collectors for a monitoring system. As of this version, it just includes the DRBD data collector, that can be executed by calling ``mon-collector`` using the ``drbd`` parameter. See :manpage:`mon-collector(7)`. - A new user option, :pyeval:`rapi.RAPI_ACCESS_READ`, has been added for RAPI users. It allows granting permissions to query for information to a specific user without giving :pyeval:`rapi.RAPI_ACCESS_WRITE` permissions. - A new tool named ``node-cleanup`` has been added. It cleans remains of a cluster from a machine by stopping all daemons, removing certificates and ssconf files. Unless the ``--no-backup`` option is given, copies of the certificates are made. - Instance creations now support the use of opportunistic locking, potentially speeding up the (parallel) creation of multiple instances. This feature is currently only available via the :doc:`RAPI ` interface and when an instance allocator is used. If the ``opportunistic_locking`` parameter is set the opcode will try to acquire as many locks as possible, but will not wait for any locks held by other opcodes. If not enough resources can be found to allocate the instance, the temporary error code :pyeval:`errors.ECODE_TEMP_NORES` is returned. The operation can be retried thereafter, with or without opportunistic locking. - New experimental linux-ha resource scripts. - Restricted-commands support: ganeti can now be asked (via command line or rapi) to perform commands on a node. These are passed via ganeti RPC rather than ssh. This functionality is restricted to commands specified on the ``$sysconfdir/ganeti/restricted-commands`` for security reasons. The file is not copied automatically. Misc changes ~~~~~~~~~~~~ - Diskless instances are now externally mirrored (Issue 237). This for now has only been tested in conjunction with explicit target nodes for migration/failover. - Queries not needing locks or RPC access to the node can now be performed by the confd daemon, making them independent from jobs, and thus faster to execute. This is selectable at configure time. - The functionality for allocating multiple instances at once has been overhauled and is now also available through :doc:`RAPI `. There are no significant changes from version 2.7.0~rc3. Version 2.7.0 rc3 ----------------- *(Released Tue, 25 Jun 2013)* - Fix permissions on the confd query socket (Issue 477) - Fix permissions on the job archive dir (Issue 498) - Fix handling of an internal exception in replace-disks (Issue 472) - Fix gnt-node info handling of shortened names (Issue 497) - Fix gnt-instance grow-disk when wiping is enabled - Documentation improvements, and support for newer pandoc - Fix hspace honoring ipolicy for disks (Issue 484) - Improve handling of the ``kvm_extra`` HV parameter Version 2.7.0 rc2 ----------------- *(Released Fri, 24 May 2013)* - ``devel/upload`` now works when ``/var/run`` on the target nodes is a symlink. - Disks added through ``gnt-instance modify`` or created through ``gnt-instance recreate-disks`` are wiped, if the ``prealloc_wipe_disks`` flag is set. - If wiping newly created disks fails, the disks are removed. Also, partial failures in creating disks through ``gnt-instance modify`` triggers a cleanup of the partially-created disks. - Removing the master IP address doesn't fail if the address has been already removed. - Fix ownership of the OS log dir - Workaround missing SO_PEERCRED constant (Issue 191) Version 2.7.0 rc1 ----------------- *(Released Fri, 3 May 2013)* This was the first release candidate of the 2.7 series. Since beta3: - Fix kvm compatibility with qemu 1.4 (Issue 389) - Documentation updates (admin guide, upgrade notes, install instructions) (Issue 372) - Fix gnt-group list nodes and instances count (Issue 436) - Fix compilation without non-mandatory libraries (Issue 441) - Fix xen-hvm hypervisor forcing nics to type 'ioemu' (Issue 247) - Make confd logging more verbose at INFO level (Issue 435) - Improve "networks" documentation in :manpage:`gnt-instance(8)` - Fix failure path for instance storage type conversion (Issue 229) - Update htools text backend documentation - Improve the renew-crypto section of :manpage:`gnt-cluster(8)` - Disable inter-cluster instance move for file-based instances, because it is dependant on instance export, which is not supported for file-based instances. (Issue 414) - Fix gnt-job crashes on non-ascii characters (Issue 427) - Fix volume group checks on non-vm-capable nodes (Issue 432) Version 2.7.0 beta3 ------------------- *(Released Mon, 22 Apr 2013)* This was the third beta release of the 2.7 series. Since beta2: - Fix hail to verify disk instance policies on a per-disk basis (Issue 418). - Fix data loss on wrong usage of ``gnt-instance move`` - Properly export errors in confd-based job queries - Add ``users-setup`` tool - Fix iallocator protocol to report 0 as a disk size for diskless instances. This avoids hail breaking when a diskless instance is present. - Fix job queue directory permission problem that made confd job queries fail. This requires running an ``ensure-dirs --full-run`` on upgrade for access to archived jobs (Issue 406). - Limit the sizes of networks supported by ``gnt-network`` to something between a ``/16`` and a ``/30`` to prevent memory bloat and crashes. - Fix bugs in instance disk template conversion - Fix GHC 7 compatibility - Fix ``burnin`` install path (Issue 426). - Allow very small disk grows (Issue 347). - Fix a ``ganeti-noded`` memory bloat introduced in 2.5, by making sure that noded doesn't import masterd code (Issue 419). - Make sure the default metavg at cluster init is the same as the vg, if unspecified (Issue 358). - Fix cleanup of partially created disks (part of Issue 416) Version 2.7.0 beta2 ------------------- *(Released Tue, 2 Apr 2013)* This was the second beta release of the 2.7 series. Since beta1: - Networks no longer have a "type" slot, since this information was unused in Ganeti: instead of it tags should be used. - The rapi client now has a ``target_node`` option to MigrateInstance. - Fix early exit return code for hbal (Issue 386). - Fix ``gnt-instance migrate/failover -n`` (Issue 396). - Fix ``rbd showmapped`` output parsing (Issue 312). - Networks are now referenced indexed by UUID, rather than name. This will require running cfgupgrade, from 2.7.0beta1, if networks are in use. - The OS environment now includes network information. - Deleting of a network is now disallowed if any instance nic is using it, to prevent dangling references. - External storage is now documented in man pages. - The exclusive_storage flag can now only be set at nodegroup level. - Hbal can now submit an explicit priority with its jobs. - Many network related locking fixes. - Bump up the required pylint version to 0.25.1. - Fix the ``no_remember`` option in RAPI client. - Many ipolicy related tests, qa, and fixes. - Many documentation improvements and fixes. - Fix building with ``--disable-file-storage``. - Fix ``-q`` option in htools, which was broken if passed more than once. - Some haskell/python interaction improvements and fixes. - Fix iallocator in case of missing LVM storage. - Fix confd config load in case of ``--no-lvm-storage``. - The confd/query functionality is now mentioned in the security documentation. Version 2.7.0 beta1 ------------------- *(Released Wed, 6 Feb 2013)* This was the first beta release of the 2.7 series. All important changes are listed in the latest 2.7 entry. Version 2.6.2 ------------- *(Released Fri, 21 Dec 2012)* Important behaviour change: hbal won't rebalance anymore instances which have the ``auto_balance`` attribute set to false. This was the intention all along, but until now it only skipped those from the N+1 memory reservation (DRBD-specific). A significant number of bug fixes in this release: - Fixed disk adoption interaction with ipolicy checks. - Fixed networking issues when instances are started, stopped or migrated, by forcing the tap device's MAC prefix to "fe" (issue 217). - Fixed the warning in cluster verify for shared storage instances not being redundant. - Fixed removal of storage directory on shared file storage (issue 262). - Fixed validation of LVM volume group name in OpClusterSetParams (``gnt-cluster modify``) (issue 285). - Fixed runtime memory increases (``gnt-instance modify -m``). - Fixed live migration under Xen's ``xl`` mode. - Fixed ``gnt-instance console`` with ``xl``. - Fixed building with newer Haskell compiler/libraries. - Fixed PID file writing in Haskell daemons (confd); this prevents restart issues if confd was launched manually (outside of ``daemon-util``) while another copy of it was running - Fixed a type error when doing live migrations with KVM (issue 297) and the error messages for failing migrations have been improved. - Fixed opcode validation for the out-of-band commands (``gnt-node power``). - Fixed a type error when unsetting OS hypervisor parameters (issue 311); now it's possible to unset all OS-specific hypervisor parameters. - Fixed the ``dry-run`` mode for many operations: verification of results was over-zealous but didn't take into account the ``dry-run`` operation, resulting in "wrong" failures. - Fixed bash completion in ``gnt-job list`` when the job queue has hundreds of entries; especially with older ``bash`` versions, this results in significant CPU usage. And lastly, a few other improvements have been made: - Added option to force master-failover without voting (issue 282). - Clarified error message on lock conflict (issue 287). - Logging of newly submitted jobs has been improved (issue 290). - Hostname checks have been made uniform between instance rename and create (issue 291). - The ``--submit`` option is now supported by ``gnt-debug delay``. - Shutting down the master daemon by sending SIGTERM now stops it from processing jobs waiting for locks; instead, those jobs will be started once again after the master daemon is started the next time (issue 296). - Support for Xen's ``xl`` program has been improved (besides the fixes above). - Reduced logging noise in the Haskell confd daemon (only show one log entry for each config reload, instead of two). - Several man page updates and typo fixes. Version 2.6.1 ------------- *(Released Fri, 12 Oct 2012)* A small bugfix release. Among the bugs fixed: - Fixed double use of ``PRIORITY_OPT`` in ``gnt-node migrate``, that made the command unusable. - Commands that issue many jobs don't fail anymore just because some jobs take so long that other jobs are archived. - Failures during ``gnt-instance reinstall`` are reflected by the exit status. - Issue 190 fixed. Check for DRBD in cluster verify is enabled only when DRBD is enabled. - When ``always_failover`` is set, ``--allow-failover`` is not required in migrate commands anymore. - ``bash_completion`` works even if extglob is disabled. - Fixed bug with locks that made failover for RDB-based instances fail. - Fixed bug in non-mirrored instance allocation that made Ganeti choose a random node instead of one based on the allocator metric. - Support for newer versions of pylint and pep8. - Hail doesn't fail anymore when trying to add an instance of type ``file``, ``sharedfile`` or ``rbd``. - Added new Makefile target to rebuild the whole distribution, so that all files are included. Version 2.6.0 ------------- *(Released Fri, 27 Jul 2012)* .. attention:: The ``LUXI`` protocol has been made more consistent regarding its handling of command arguments. This, however, leads to incompatibility issues with previous versions. Please ensure that you restart Ganeti daemons soon after the upgrade, otherwise most ``LUXI`` calls (job submission, setting/resetting the drain flag, pausing/resuming the watcher, cancelling and archiving jobs, querying the cluster configuration) will fail. New features ~~~~~~~~~~~~ Instance run status +++++++++++++++++++ The current ``admin_up`` field, which used to denote whether an instance should be running or not, has been removed. Instead, ``admin_state`` is introduced, with 3 possible values -- ``up``, ``down`` and ``offline``. The rational behind this is that an instance being “down” can have different meanings: - it could be down during a reboot - it could be temporarily be down for a reinstall - or it could be down because it is deprecated and kept just for its disk The previous Boolean state was making it difficult to do capacity calculations: should Ganeti reserve memory for a down instance? Now, the tri-state field makes it clear: - in ``up`` and ``down`` state, all resources are reserved for the instance, and it can be at any time brought up if it is down - in ``offline`` state, only disk space is reserved for it, but not memory or CPUs The field can have an extra use: since the transition between ``up`` and ``down`` and vice-versus is done via ``gnt-instance start/stop``, but transition between ``offline`` and ``down`` is done via ``gnt-instance modify``, it is possible to given different rights to users. For example, owners of an instance could be allowed to start/stop it, but not transition it out of the offline state. Instance policies and specs +++++++++++++++++++++++++++ In previous Ganeti versions, an instance creation request was not limited on the minimum size and on the maximum size just by the cluster resources. As such, any policy could be implemented only in third-party clients (RAPI clients, or shell wrappers over ``gnt-*`` tools). Furthermore, calculating cluster capacity via ``hspace`` again required external input with regards to instance sizes. In order to improve these workflows and to allow for example better per-node group differentiation, we introduced instance specs, which allow declaring: - minimum instance disk size, disk count, memory size, cpu count - maximum values for the above metrics - and “standard” values (used in ``hspace`` to calculate the standard sized instances) The minimum/maximum values can be also customised at node-group level, for example allowing more powerful hardware to support bigger instance memory sizes. Beside the instance specs, there are a few other settings belonging to the instance policy framework. It is possible now to customise, per cluster and node-group: - the list of allowed disk templates - the maximum ratio of VCPUs per PCPUs (to control CPU oversubscription) - the maximum ratio of instance to spindles (see below for more information) for local storage All these together should allow all tools that talk to Ganeti to know what are the ranges of allowed values for instances and the over-subscription that is allowed. For the VCPU/PCPU ratio, we already have the VCPU configuration from the instance configuration, and the physical CPU configuration from the node. For the spindle ratios however, we didn't track before these values, so new parameters have been added: - a new node parameter ``spindle_count``, defaults to 1, customisable at node group or node level - at new backend parameter (for instances), ``spindle_use`` defaults to 1 Note that spindles in this context doesn't need to mean actual mechanical hard-drives; it's just a relative number for both the node I/O capacity and instance I/O consumption. Instance migration behaviour ++++++++++++++++++++++++++++ While live-migration is in general desirable over failover, it is possible that for some workloads it is actually worse, due to the variable time of the “suspend” phase during live migration. To allow the tools to work consistently over such instances (without having to hard-code instance names), a new backend parameter ``always_failover`` has been added to control the migration/failover behaviour. When set to True, all migration requests for an instance will instead fall-back to failover. Instance memory ballooning ++++++++++++++++++++++++++ Initial support for memory ballooning has been added. The memory for an instance is no longer fixed (backend parameter ``memory``), but instead can vary between minimum and maximum values (backend parameters ``minmem`` and ``maxmem``). Currently we only change an instance's memory when: - live migrating or failing over and instance and the target node doesn't have enough memory - user requests changing the memory via ``gnt-instance modify --runtime-memory`` Instance CPU pinning ++++++++++++++++++++ In order to control the use of specific CPUs by instance, support for controlling CPU pinning has been added for the Xen, HVM and LXC hypervisors. This is controlled by a new hypervisor parameter ``cpu_mask``; details about possible values for this are in the :manpage:`gnt-instance(8)`. Note that use of the most specific (precise VCPU-to-CPU mapping) form will work well only when all nodes in your cluster have the same amount of CPUs. Disk parameters +++++++++++++++ Another area in which Ganeti was not customisable were the parameters used for storage configuration, e.g. how many stripes to use for LVM, DRBD resync configuration, etc. To improve this area, we've added disks parameters, which are customisable at cluster and node group level, and which allow to specify various parameters for disks (DRBD has the most parameters currently), for example: - DRBD resync algorithm and parameters (e.g. speed) - the default VG for meta-data volumes for DRBD - number of stripes for LVM (plain disk template) - the RBD pool These parameters can be modified via ``gnt-cluster modify -D â€Ļ`` and ``gnt-group modify -D â€Ļ``, and are used at either instance creation (in case of LVM stripes, for example) or at disk “activation” time (e.g. resync speed). Rados block device support ++++++++++++++++++++++++++ A Rados (http://ceph.com/wiki/Rbd) storage backend has been added, denoted by the ``rbd`` disk template type. This is considered experimental, feedback is welcome. For details on configuring it, see the :doc:`install` document and the :manpage:`gnt-cluster(8)` man page. Master IP setup +++++++++++++++ The existing master IP functionality works well only in simple setups (a single network shared by all nodes); however, if nodes belong to different networks, then the ``/32`` setup and lack of routing information is not enough. To allow the master IP to function well in more complex cases, the system was reworked as follows: - a master IP netmask setting has been added - the master IP activation/turn-down code was moved from the node daemon to a separate script - whether to run the Ganeti-supplied master IP script or a user-supplied on is a ``gnt-cluster init`` setting Details about the location of the standard and custom setup scripts are in the man page :manpage:`gnt-cluster(8)`; for information about the setup script protocol, look at the Ganeti-supplied script. SPICE support +++++++++++++ The `SPICE `_ support has been improved. It is now possible to use TLS-protected connections, and when renewing or changing the cluster certificates (via ``gnt-cluster renew-crypto``, it is now possible to specify spice or spice CA certificates. Also, it is possible to configure a password for SPICE sessions via the hypervisor parameter ``spice_password_file``. There are also new parameters to control the compression and streaming options (e.g. ``spice_image_compression``, ``spice_streaming_video``, etc.). For details, see the man page :manpage:`gnt-instance(8)` and look for the spice parameters. Lastly, it is now possible to see the SPICE connection information via ``gnt-instance console``. OVF converter +++++++++++++ A new tool (``tools/ovfconverter``) has been added that supports conversion between Ganeti and the `Open Virtualization Format `_ (both to and from). This relies on the ``qemu-img`` tool to convert the disk formats, so the actual compatibility with other virtualization solutions depends on it. Confd daemon changes ++++++++++++++++++++ The configuration query daemon (``ganeti-confd``) is now optional, and has been rewritten in Haskell; whether to use the daemon at all, use the Python (default) or the Haskell version is selectable at configure time via the ``--enable-confd`` parameter, which can take one of the ``haskell``, ``python`` or ``no`` values. If not used, disabling the daemon will result in a smaller footprint; for larger systems, we welcome feedback on the Haskell version which might become the default in future versions. If you want to use ``gnt-node list-drbd`` you need to have the Haskell daemon running. The Python version doesn't implement the new call. User interface changes ~~~~~~~~~~~~~~~~~~~~~~ We have replaced the ``--disks`` option of ``gnt-instance replace-disks`` with a more flexible ``--disk`` option, which allows adding and removing disks at arbitrary indices (Issue 188). Furthermore, disk size and mode can be changed upon recreation (via ``gnt-instance recreate-disks``, which accepts the same ``--disk`` option). As many people are used to a ``show`` command, we have added that as an alias to ``info`` on all ``gnt-*`` commands. The ``gnt-instance grow-disk`` command has a new mode in which it can accept the target size of the disk, instead of the delta; this can be more safe since two runs in absolute mode will be idempotent, and sometimes it's also easier to specify the desired size directly. Also the handling of instances with regard to offline secondaries has been improved. Instance operations should not fail because one of it's secondary nodes is offline, even though it's safe to proceed. A new command ``list-drbd`` has been added to the ``gnt-node`` script to support debugging of DRBD issues on nodes. It provides a mapping of DRBD minors to instance name. API changes ~~~~~~~~~~~ RAPI coverage has improved, with (for example) new resources for recreate-disks, node power-cycle, etc. Compatibility ~~~~~~~~~~~~~ There is partial support for ``xl`` in the Xen hypervisor; feedback is welcome. Python 2.7 is better supported, and after Ganeti 2.6 we will investigate whether to still support Python 2.4 or move to Python 2.6 as minimum required version. Support for Fedora has been slightly improved; the provided example init.d script should work better on it and the INSTALL file should document the needed dependencies. Internal changes ~~~~~~~~~~~~~~~~ The deprecated ``QueryLocks`` LUXI request has been removed. Use ``Query(what=QR_LOCK, ...)`` instead. The LUXI requests :pyeval:`luxi.REQ_QUERY_JOBS`, :pyeval:`luxi.REQ_QUERY_INSTANCES`, :pyeval:`luxi.REQ_QUERY_NODES`, :pyeval:`luxi.REQ_QUERY_GROUPS`, :pyeval:`luxi.REQ_QUERY_EXPORTS` and :pyeval:`luxi.REQ_QUERY_TAGS` are deprecated and will be removed in a future version. :pyeval:`luxi.REQ_QUERY` should be used instead. RAPI client: ``CertificateError`` now derives from ``GanetiApiError``. This should make it more easy to handle Ganeti errors. Deprecation warnings due to PyCrypto/paramiko import in ``tools/setup-ssh`` have been silenced, as usually they are safe; please make sure to run an up-to-date paramiko version, if you use this tool. The QA scripts now depend on Python 2.5 or above (the main code base still works with Python 2.4). The configuration file (``config.data``) is now written without indentation for performance reasons; if you want to edit it, it can be re-formatted via ``tools/fmtjson``. A number of bugs has been fixed in the cluster merge tool. ``x509`` certification verification (used in import-export) has been changed to allow the same clock skew as permitted by the cluster verification. This will remove some rare but hard to diagnose errors in import-export. Version 2.6.0 rc4 ----------------- *(Released Thu, 19 Jul 2012)* Very few changes from rc4 to the final release, only bugfixes: - integrated fixes from release 2.5.2 (fix general boot flag for KVM instance, fix CDROM booting for KVM instances) - fixed node group modification of node parameters - fixed issue in LUClusterVerifyGroup with multi-group clusters - fixed generation of bash completion to ensure a stable ordering - fixed a few typos Version 2.6.0 rc3 ----------------- *(Released Fri, 13 Jul 2012)* Third release candidate for 2.6. The following changes were done from rc3 to rc4: - Fixed ``UpgradeConfig`` w.r.t. to disk parameters on disk objects. - Fixed an inconsistency in the LUXI protocol with the provided arguments (NOT backwards compatible) - Fixed a bug with node groups ipolicy where ``min`` was greater than the cluster ``std`` value - Implemented a new ``gnt-node list-drbd`` call to list DRBD minors for easier instance debugging on nodes (requires ``hconfd`` to work) Version 2.6.0 rc2 ----------------- *(Released Tue, 03 Jul 2012)* Second release candidate for 2.6. The following changes were done from rc2 to rc3: - Fixed ``gnt-cluster verify`` regarding ``master-ip-script`` on non master candidates - Fixed a RAPI regression on missing beparams/memory - Fixed redistribution of files on offline nodes - Added possibility to run activate-disks even though secondaries are offline. With this change it relaxes also the strictness on some other commands which use activate disks internally: * ``gnt-instance start|reboot|rename|backup|export`` - Made it possible to remove safely an instance if its secondaries are offline - Made it possible to reinstall even though secondaries are offline Version 2.6.0 rc1 ----------------- *(Released Mon, 25 Jun 2012)* First release candidate for 2.6. The following changes were done from rc1 to rc2: - Fixed bugs with disk parameters and ``rbd`` templates as well as ``instance_os_add`` - Made ``gnt-instance modify`` more consistent regarding new NIC/Disk behaviour. It supports now the modify operation - ``hcheck`` implemented to analyze cluster health and possibility of improving health by rebalance - ``hbal`` has been improved in dealing with split instances Version 2.6.0 beta2 ------------------- *(Released Mon, 11 Jun 2012)* Second beta release of 2.6. The following changes were done from beta2 to rc1: - Fixed ``daemon-util`` with non-root user models - Fixed creation of plain instances with ``--no-wait-for-sync`` - Fix wrong iv_names when running ``cfgupgrade`` - Export more information in RAPI group queries - Fixed bug when changing instance network interfaces - Extended burnin to do NIC changes - query: Added ``<``, ``>``, ``<=``, ``>=`` comparison operators - Changed default for DRBD barriers - Fixed DRBD error reporting for syncer rate - Verify the options on disk parameters And of course various fixes to documentation and improved unittests and QA. Version 2.6.0 beta1 ------------------- *(Released Wed, 23 May 2012)* First beta release of 2.6. The following changes were done from beta1 to beta2: - integrated patch for distributions without ``start-stop-daemon`` - adapted example init.d script to work on Fedora - fixed log handling in Haskell daemons - adapted checks in the watcher for pycurl linked against libnss - add partial support for ``xl`` instead of ``xm`` for Xen - fixed a type issue in cluster verification - fixed ssconf handling in the Haskell code (was breaking confd in IPv6 clusters) Plus integrated fixes from the 2.5 branch: - fixed ``kvm-ifup`` to use ``/bin/bash`` - fixed parallel build failures - KVM live migration when using a custom keymap Version 2.5.2 ------------- *(Released Tue, 24 Jul 2012)* A small bugfix release, with no new features: - fixed bash-isms in kvm-ifup, for compatibility with systems which use a different default shell (e.g. Debian, Ubuntu) - fixed KVM startup and live migration with a custom keymap (fixes Issue 243 and Debian bug #650664) - fixed compatibility with KVM versions that don't support multiple boot devices (fixes Issue 230 and Debian bug #624256) Additionally, a few fixes were done to the build system (fixed parallel build failures) and to the unittests (fixed race condition in test for FileID functions, and the default enable/disable mode for QA test is now customisable). Version 2.5.1 ------------- *(Released Fri, 11 May 2012)* A small bugfix release. The main issues solved are on the topic of compatibility with newer LVM releases: - fixed parsing of ``lv_attr`` field - adapted to new ``vgreduce --removemissing`` behaviour where sometimes the ``--force`` flag is needed Also on the topic of compatibility, ``tools/lvmstrap`` has been changed to accept kernel 3.x too (was hardcoded to 2.6.*). A regression present in 2.5.0 that broke handling (in the gnt-* scripts) of hook results and that also made display of other errors suboptimal was fixed; the code behaves now like 2.4 and earlier. Another change in 2.5, the cleanup of the OS scripts environment, is too aggressive: it removed even the ``PATH`` variable, which requires the OS scripts to *always* need to export it. Since this is a bit too strict, we now export a minimal PATH, the same that we export for hooks. The fix for issue 201 (Preserve bridge MTU in KVM ifup script) was integrated into this release. Finally, a few other miscellaneous changes were done (no new features, just small improvements): - Fix ``gnt-group --help`` display - Fix hardcoded Xen kernel path - Fix grow-disk handling of invalid units - Update synopsis for ``gnt-cluster repair-disk-sizes`` - Accept both PUT and POST in noded (makes future upgrade to 2.6 easier) Version 2.5.0 ------------- *(Released Thu, 12 Apr 2012)* Incompatible/important changes and bugfixes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The default of the ``/2/instances/[instance_name]/rename`` RAPI resource's ``ip_check`` parameter changed from ``True`` to ``False`` to match the underlying LUXI interface. - The ``/2/nodes/[node_name]/evacuate`` RAPI resource was changed to use body parameters, see :doc:`RAPI documentation `. The server does not maintain backwards-compatibility as the underlying operation changed in an incompatible way. The RAPI client can talk to old servers, but it needs to be told so as the return value changed. - When creating file-based instances via RAPI, the ``file_driver`` parameter no longer defaults to ``loop`` and must be specified. - The deprecated ``bridge`` NIC parameter is no longer supported. Use ``link`` instead. - Support for the undocumented and deprecated RAPI instance creation request format version 0 has been dropped. Use version 1, supported since Ganeti 2.1.3 and :doc:`documented `, instead. - Pyparsing 1.4.6 or above is required, see :doc:`installation documentation `. - The "cluster-verify" hooks are now executed per group by the ``OP_CLUSTER_VERIFY_GROUP`` opcode. This maintains the same behavior if you just run ``gnt-cluster verify``, which generates one opcode per group. - The environment as passed to the OS scripts is cleared, and thus no environment variables defined in the node daemon's environment will be inherited by the scripts. - The :doc:`iallocator ` mode ``multi-evacuate`` has been deprecated. - :doc:`New iallocator modes ` have been added to support operations involving multiple node groups. - Offline nodes are ignored when failing over an instance. - Support for KVM version 1.0, which changed the version reporting format from 3 to 2 digits. - TCP/IP ports used by DRBD disks are returned to a pool upon instance removal. - ``Makefile`` is now compatible with Automake 1.11.2 - Includes all bugfixes made in the 2.4 series New features ~~~~~~~~~~~~ - The ganeti-htools project has been merged into the ganeti-core source tree and will be built as part of Ganeti. - Implemented support for :doc:`shared storage `. - Add support for disks larger than 2 TB in ``lvmstrap`` by supporting GPT-style partition tables (requires `parted `_). - Added support for floppy drive and 2nd CD-ROM drive in KVM hypervisor. - Allowed adding tags on instance creation. - Export instance tags to hooks (``INSTANCE_TAGS``, see :doc:`hooks`) - Allow instances to be started in a paused state, enabling the user to see the complete console output on boot using the console. - Added new hypervisor flag to control default reboot behaviour (``reboot_behavior``). - Added support for KVM keymaps (hypervisor parameter ``keymap``). - Improved out-of-band management support: - Added ``gnt-node health`` command reporting the health status of nodes. - Added ``gnt-node power`` command to manage power status of nodes. - Added command for emergency power-off (EPO), ``gnt-cluster epo``. - Instance migration can fall back to failover if instance is not running. - Filters can be used when listing nodes, instances, groups and locks; see :manpage:`ganeti(7)` manpage. - Added post-execution status as variables to :doc:`hooks ` environment. - Instance tags are exported/imported together with the instance. - When given an explicit job ID, ``gnt-job info`` will work for archived jobs. - Jobs can define dependencies on other jobs (not yet supported via RAPI or command line, but used by internal commands and usable via LUXI). - Lock monitor (``gnt-debug locks``) shows jobs waiting for dependencies. - Instance failover is now available as a RAPI resource (``/2/instances/[instance_name]/failover``). - ``gnt-instance info`` defaults to static information if primary node is offline. - Opcodes have a new ``comment`` attribute. - Added basic SPICE support to KVM hypervisor. - ``tools/ganeti-listrunner`` allows passing of arguments to executable. Node group improvements ~~~~~~~~~~~~~~~~~~~~~~~ - ``gnt-cluster verify`` has been modified to check groups separately, thereby improving performance. - Node group support has been added to ``gnt-cluster verify-disks``, which now operates per node group. - Watcher has been changed to work better with node groups. - One process and state file per node group. - Slow watcher in one group doesn't block other group's watcher. - Added new command, ``gnt-group evacuate``, to move all instances in a node group to other groups. - Added ``gnt-instance change-group`` to move an instance to another node group. - ``gnt-cluster command`` and ``gnt-cluster copyfile`` now support per-group operations. - Node groups can be tagged. - Some operations switch from an exclusive to a shared lock as soon as possible. - Instance's primary and secondary nodes' groups are now available as query fields (``pnode.group``, ``pnode.group.uuid``, ``snodes.group`` and ``snodes.group.uuid``). Misc ~~~~ - Numerous updates to documentation and manpages. - :doc:`RAPI ` documentation now has detailed parameter descriptions. - Some opcode/job results are now also documented, see :doc:`RAPI `. - A lockset's internal lock is now also visible in lock monitor. - Log messages from job queue workers now contain information about the opcode they're processing. - ``gnt-instance console`` no longer requires the instance lock. - A short delay when waiting for job changes reduces the number of LUXI requests significantly. - DRBD metadata volumes are overwritten with zeros during disk creation. - Out-of-band commands no longer acquire the cluster lock in exclusive mode. - ``devel/upload`` now uses correct permissions for directories. Version 2.5.0 rc6 ----------------- *(Released Fri, 23 Mar 2012)* This was the sixth release candidate of the 2.5 series. Version 2.5.0 rc5 ----------------- *(Released Mon, 9 Jan 2012)* This was the fifth release candidate of the 2.5 series. Version 2.5.0 rc4 ----------------- *(Released Thu, 27 Oct 2011)* This was the fourth release candidate of the 2.5 series. Version 2.5.0 rc3 ----------------- *(Released Wed, 26 Oct 2011)* This was the third release candidate of the 2.5 series. Version 2.5.0 rc2 ----------------- *(Released Tue, 18 Oct 2011)* This was the second release candidate of the 2.5 series. Version 2.5.0 rc1 ----------------- *(Released Tue, 4 Oct 2011)* This was the first release candidate of the 2.5 series. Version 2.5.0 beta3 ------------------- *(Released Wed, 31 Aug 2011)* This was the third beta release of the 2.5 series. Version 2.5.0 beta2 ------------------- *(Released Mon, 22 Aug 2011)* This was the second beta release of the 2.5 series. Version 2.5.0 beta1 ------------------- *(Released Fri, 12 Aug 2011)* This was the first beta release of the 2.5 series. Version 2.4.5 ------------- *(Released Thu, 27 Oct 2011)* - Fixed bug when parsing command line parameter values ending in backslash - Fixed assertion error after unclean master shutdown - Disable HTTP client pool for RPC, significantly reducing memory usage of master daemon - Fixed queue archive creation with wrong permissions Version 2.4.4 ------------- *(Released Tue, 23 Aug 2011)* Small bug-fixes: - Fixed documentation for importing with ``--src-dir`` option - Fixed a bug in ``ensure-dirs`` with queue/archive permissions - Fixed a parsing issue with DRBD 8.3.11 in the Linux kernel Version 2.4.3 ------------- *(Released Fri, 5 Aug 2011)* Many bug-fixes and a few small features: - Fixed argument order in ``ReserveLV`` and ``ReserveMAC`` which caused issues when you tried to add an instance with two MAC addresses in one request - KVM: fixed per-instance stored UID value - KVM: configure bridged NICs at migration start - KVM: Fix a bug where instance will not start with never KVM versions (>= 0.14) - Added OS search path to ``gnt-cluster info`` - Fixed an issue with ``file_storage_dir`` where you were forced to provide an absolute path, but the documentation states it is a relative path, the documentation was right - Added a new parameter to instance stop/start called ``--no-remember`` that will make the state change to not be remembered - Implemented ``no_remember`` at RAPI level - Improved the documentation - Node evacuation: don't call IAllocator if node is already empty - Fixed bug in DRBD8 replace disks on current nodes - Fixed bug in recreate-disks for DRBD instances - Moved assertion checking locks in ``gnt-instance replace-disks`` causing it to abort with not owning the right locks for some situation - Job queue: Fixed potential race condition when cancelling queued jobs - Fixed off-by-one bug in job serial generation - ``gnt-node volumes``: Fix instance names - Fixed aliases in bash completion - Fixed a bug in reopening log files after being sent a SIGHUP - Added a flag to burnin to allow specifying VCPU count - Bugfixes to non-root Ganeti configuration Version 2.4.2 ------------- *(Released Thu, 12 May 2011)* Many bug-fixes and a few new small features: - Fixed a bug related to log opening failures - Fixed a bug in instance listing with orphan instances - Fixed a bug which prevented resetting the cluster-level node parameter ``oob_program`` to the default - Many fixes related to the ``cluster-merge`` tool - Fixed a race condition in the lock monitor, which caused failures during (at least) creation of many instances in parallel - Improved output for gnt-job info - Removed the quiet flag on some ssh calls which prevented debugging failures - Improved the N+1 failure messages in cluster verify by actually showing the memory values (needed and available) - Increased lock attempt timeouts so that when executing long operations (e.g. DRBD replace-disks) other jobs do not enter 'blocking acquire' too early and thus prevent the use of the 'fair' mechanism - Changed instance query data (``gnt-instance info``) to not acquire locks unless needed, thus allowing its use on locked instance if only static information is asked for - Improved behaviour with filesystems that do not support rename on an opened file - Fixed the behaviour of ``prealloc_wipe_disks`` cluster parameter which kept locks on all nodes during the wipe, which is unneeded - Fixed ``gnt-watcher`` handling of errors during hooks execution - Fixed bug in ``prealloc_wipe_disks`` with small disk sizes (less than 10GiB) which caused the wipe to fail right at the end in some cases - Fixed master IP activation when doing master failover with no-voting - Fixed bug in ``gnt-node add --readd`` which allowed the re-adding of the master node itself - Fixed potential data-loss in under disk full conditions, where Ganeti wouldn't check correctly the return code and would consider partially-written files 'correct' - Fixed bug related to multiple VGs and DRBD disk replacing - Added new disk parameter ``metavg`` that allows placement of the meta device for DRBD in a different volume group - Fixed error handling in the node daemon when the system libc doesn't have major number 6 (i.e. if ``libc.so.6`` is not the actual libc) - Fixed lock release during replace-disks, which kept cluster-wide locks when doing disk replaces with an iallocator script - Added check for missing bridges in cluster verify - Handle EPIPE errors while writing to the terminal better, so that piping the output to e.g. ``less`` doesn't cause a backtrace - Fixed rare case where a ^C during Luxi calls could have been interpreted as server errors, instead of simply terminating - Fixed a race condition in LUGroupAssignNodes (``gnt-group assign-nodes``) - Added a few more parameters to the KVM hypervisor, allowing a second CDROM, custom disk type for CDROMs and a floppy image - Removed redundant message in instance rename when the name is given already as a FQDN - Added option to ``gnt-instance recreate-disks`` to allow creating the disks on new nodes, allowing recreation when the original instance nodes are completely gone - Added option when converting disk templates to DRBD to skip waiting for the resync, in order to make the instance available sooner - Added two new variables to the OS scripts environment (containing the instance's nodes) - Made the root_path and optional parameter for the xen-pvm hypervisor, to allow use of ``pvgrub`` as bootloader - Changed the instance memory modifications to only check out-of-memory conditions on memory increases, and turned the secondary node warnings into errors (they can still be overridden via ``--force``) - Fixed the handling of a corner case when the Python installation gets corrupted (e.g. a bad disk) while ganeti-noded is running and we try to execute a command that doesn't exist - Fixed a bug in ``gnt-instance move`` (LUInstanceMove) when the primary node of the instance returned failures during instance shutdown; this adds the option ``--ignore-consistency`` to gnt-instance move And as usual, various improvements to the error messages, documentation and man pages. Version 2.4.1 ------------- *(Released Wed, 09 Mar 2011)* Emergency bug-fix release. ``tools/cfgupgrade`` was broken and overwrote the RAPI users file if run twice (even with ``--dry-run``). The release fixes that bug (nothing else changed). Version 2.4.0 ------------- *(Released Mon, 07 Mar 2011)* Final 2.4.0 release. Just a few small fixes: - Fixed RAPI node evacuate - Fixed the kvm-ifup script - Fixed internal error handling for special job cases - Updated man page to specify the escaping feature for options Version 2.4.0 rc3 ----------------- *(Released Mon, 28 Feb 2011)* A critical fix for the ``prealloc_wipe_disks`` feature: it is possible that this feature wiped the disks of the wrong instance, leading to loss of data. Other changes: - Fixed title of query field containing instance name - Expanded the glossary in the documentation - Fixed one unittest (internal issue) Version 2.4.0 rc2 ----------------- *(Released Mon, 21 Feb 2011)* A number of bug fixes plus just a couple functionality changes. On the user-visible side, the ``gnt-* list`` command output has changed with respect to "special" field states. The current rc1 style of display can be re-enabled by passing a new ``--verbose`` (``-v``) flag, but in the default output mode special fields states are displayed as follows: - Offline resource: ``*`` - Unavailable/not applicable: ``-`` - Data missing (RPC failure): ``?`` - Unknown field: ``??`` Another user-visible change is the addition of ``--force-join`` to ``gnt-node add``. As for bug fixes: - ``tools/cluster-merge`` has seen many fixes and is now enabled again - Fixed regression in RAPI/instance reinstall where all parameters were required (instead of optional) - Fixed ``gnt-cluster repair-disk-sizes``, was broken since Ganeti 2.2 - Fixed iallocator usage (offline nodes were not considered offline) - Fixed ``gnt-node list`` with respect to non-vm_capable nodes - Fixed hypervisor and OS parameter validation with respect to non-vm_capable nodes - Fixed ``gnt-cluster verify`` with respect to offline nodes (mostly cosmetic) - Fixed ``tools/listrunner`` with respect to agent-based usage Version 2.4.0 rc1 ----------------- *(Released Fri, 4 Feb 2011)* Many changes and fixes since the beta1 release. While there were some internal changes, the code has been mostly stabilised for the RC release. Note: the dumb allocator was removed in this release, as it was not kept up-to-date with the IAllocator protocol changes. It is recommended to use the ``hail`` command from the ganeti-htools package. Note: the 2.4 and up versions of Ganeti are not compatible with the 0.2.x branch of ganeti-htools. You need to upgrade to ganeti-htools-0.3.0 (or later). Regressions fixed from 2.3 ~~~~~~~~~~~~~~~~~~~~~~~~~~ - Fixed the ``gnt-cluster verify-disks`` command - Made ``gnt-cluster verify-disks`` work in parallel (as opposed to serially on nodes) - Fixed disk adoption breakage - Fixed wrong headers in instance listing for field aliases Other bugs fixed ~~~~~~~~~~~~~~~~ - Fixed corner case in KVM handling of NICs - Fixed many cases of wrong handling of non-vm_capable nodes - Fixed a bug where a missing instance symlink was not possible to recreate with any ``gnt-*`` command (now ``gnt-instance activate-disks`` does it) - Fixed the volume group name as reported by ``gnt-cluster verify-disks`` - Increased timeouts for the import-export code, hopefully leading to fewer aborts due network or instance timeouts - Fixed bug in ``gnt-node list-storage`` - Fixed bug where not all daemons were started on cluster initialisation, but only at the first watcher run - Fixed many bugs in the OOB implementation - Fixed watcher behaviour in presence of instances with offline secondaries - Fixed instance list output for instances running on the wrong node - a few fixes to the cluster-merge tool, but it still cannot merge multi-node groups (currently it is not recommended to use this tool) Improvements ~~~~~~~~~~~~ - Improved network configuration for the KVM hypervisor - Added e1000 as a supported NIC for Xen-HVM - Improved the lvmstrap tool to also be able to use partitions, as opposed to full disks - Improved speed of disk wiping (the cluster parameter ``prealloc_wipe_disks``, so that it has a low impact on the total time of instance creations - Added documentation for the OS parameters - Changed ``gnt-instance deactivate-disks`` so that it can work if the hypervisor is not responding - Added display of blacklisted and hidden OS information in ``gnt-cluster info`` - Extended ``gnt-cluster verify`` to also validate hypervisor, backend, NIC and node parameters, which might create problems with currently invalid (but undetected) configuration files, but prevents validation failures when unrelated parameters are modified - Changed cluster initialisation to wait for the master daemon to become available - Expanded the RAPI interface: - Added config redistribution resource - Added activation/deactivation of instance disks - Added export of console information - Implemented log file reopening on SIGHUP, which allows using logrotate(8) for the Ganeti log files - Added a basic OOB helper script as an example Version 2.4.0 beta1 ------------------- *(Released Fri, 14 Jan 2011)* User-visible ~~~~~~~~~~~~ - Fixed timezone issues when formatting timestamps - Added support for node groups, available via ``gnt-group`` and other commands - Added out-of-band framework and management, see :doc:`design document ` - Removed support for roman numbers from ``gnt-node list`` and ``gnt-instance list``. - Allowed modification of master network interface via ``gnt-cluster modify --master-netdev`` - Accept offline secondaries while shutting down instance disks - Added ``blockdev_prefix`` parameter to Xen PVM and HVM hypervisors - Added support for multiple LVM volume groups - Avoid sorting nodes for ``gnt-node list`` if specific nodes are requested - Added commands to list available fields: - ``gnt-node list-fields`` - ``gnt-group list-fields`` - ``gnt-instance list-fields`` - Updated documentation and man pages Integration ~~~~~~~~~~~ - Moved ``rapi_users`` file into separate directory, now named ``.../ganeti/rapi/users``, ``cfgupgrade`` moves the file and creates a symlink - Added new tool for running commands on many machines, ``tools/ganeti-listrunner`` - Implemented more verbose result in ``OpInstanceConsole`` opcode, also improving the ``gnt-instance console`` output - Allowed customisation of disk index separator at ``configure`` time - Export node group allocation policy to :doc:`iallocator ` - Added support for non-partitioned md disks in ``lvmstrap`` - Added script to gracefully power off KVM instances - Split ``utils`` module into smaller parts - Changed query operations to return more detailed information, e.g. whether an information is unavailable due to an offline node. To use this new functionality, the LUXI call ``Query`` must be used. Field information is now stored by the master daemon and can be retrieved using ``QueryFields``. Instances, nodes and groups can also be queried using the new opcodes ``OpQuery`` and ``OpQueryFields`` (not yet exposed via RAPI). The following commands make use of this infrastructure change: - ``gnt-group list`` - ``gnt-group list-fields`` - ``gnt-node list`` - ``gnt-node list-fields`` - ``gnt-instance list`` - ``gnt-instance list-fields`` - ``gnt-debug locks`` Remote API ~~~~~~~~~~ - New RAPI resources (see :doc:`rapi`): - ``/2/modify`` - ``/2/groups`` - ``/2/groups/[group_name]`` - ``/2/groups/[group_name]/assign-nodes`` - ``/2/groups/[group_name]/modify`` - ``/2/groups/[group_name]/rename`` - ``/2/instances/[instance_name]/disk/[disk_index]/grow`` - RAPI changes: - Implemented ``no_install`` for instance creation - Implemented OS parameters for instance reinstallation, allowing use of special settings on reinstallation (e.g. for preserving data) Misc ~~~~ - Added IPv6 support in import/export - Pause DRBD synchronization while wiping disks on instance creation - Updated unittests and QA scripts - Improved network parameters passed to KVM - Converted man pages from docbook to reStructuredText Version 2.3.1 ------------- *(Released Mon, 20 Dec 2010)* Released version 2.3.1~rc1 without any changes. Version 2.3.1 rc1 ----------------- *(Released Wed, 1 Dec 2010)* - impexpd: Disable OpenSSL compression in socat if possible (backport from master, commit e90739d625b) - Changed unittest coverage report to exclude test scripts - Added script to check version format Version 2.3.0 ------------- *(Released Wed, 1 Dec 2010)* Released version 2.3.0~rc1 without any changes. Version 2.3.0 rc1 ----------------- *(Released Fri, 19 Nov 2010)* A number of bugfixes and documentation updates: - Update ganeti-os-interface documentation - Fixed a bug related to duplicate MACs or similar items which should be unique - Fix breakage in OS state modify - Reinstall instance: disallow offline secondaries (fixes bug related to OS changing but reinstall failing) - plus all the other fixes between 2.2.1 and 2.2.2 Version 2.3.0 rc0 ----------------- *(Released Tue, 2 Nov 2010)* - Fixed clearing of the default iallocator using ``gnt-cluster modify`` - Fixed master failover race with watcher - Fixed a bug in ``gnt-node modify`` which could lead to an inconsistent configuration - Accept previously stopped instance for export with instance removal - Simplify and extend the environment variables for instance OS scripts - Added new node flags, ``master_capable`` and ``vm_capable`` - Added optional instance disk wiping prior during allocation. This is a cluster-wide option and can be set/modified using ``gnt-cluster {init,modify} --prealloc-wipe-disks``. - Added IPv6 support, see :doc:`design document ` and install-quick - Added a new watcher option (``--ignore-pause``) - Added option to ignore offline node on instance start/stop (``--ignore-offline``) - Allow overriding OS parameters with ``gnt-instance reinstall`` - Added ability to change node's secondary IP address using ``gnt-node modify`` - Implemented privilege separation for all daemons except ``ganeti-noded``, see ``configure`` options - Complain if an instance's disk is marked faulty in ``gnt-cluster verify`` - Implemented job priorities (see ``ganeti(7)`` manpage) - Ignore failures while shutting down instances during failover from offline node - Exit daemon's bootstrap process only once daemon is ready - Export more information via ``LUInstanceQuery``/remote API - Improved documentation, QA and unittests - RAPI daemon now watches ``rapi_users`` all the time and doesn't need a restart if the file was created or changed - Added LUXI protocol version sent with each request and response, allowing detection of server/client mismatches - Moved the Python scripts among gnt-* and ganeti-* into modules - Moved all code related to setting up SSH to an external script, ``setup-ssh`` - Infrastructure changes for node group support in future versions Version 2.2.2 ------------- *(Released Fri, 19 Nov 2010)* A few small bugs fixed, and some improvements to the build system: - Fix documentation regarding conversion to drbd - Fix validation of parameters in cluster modify (``gnt-cluster modify -B``) - Fix error handling in node modify with multiple changes - Allow remote imports without checked names Version 2.2.1 ------------- *(Released Tue, 19 Oct 2010)* - Disable SSL session ID cache in RPC client Version 2.2.1 rc1 ----------------- *(Released Thu, 14 Oct 2010)* - Fix interaction between Curl/GnuTLS and the Python's HTTP server (thanks Apollon Oikonomopoulos!), finally allowing the use of Curl with GnuTLS - Fix problems with interaction between Curl and Python's HTTP server, resulting in increased speed in many RPC calls - Improve our release script to prevent breakage with older aclocal and Python 2.6 Version 2.2.1 rc0 ----------------- *(Released Thu, 7 Oct 2010)* - Fixed issue 125, replace hardcoded "xenvg" in ``gnt-cluster`` with value retrieved from master - Added support for blacklisted or hidden OS definitions - Added simple lock monitor (accessible via (``gnt-debug locks``) - Added support for -mem-path in KVM hypervisor abstraction layer - Allow overriding instance parameters in tool for inter-cluster instance moves (``tools/move-instance``) - Improved opcode summaries (e.g. in ``gnt-job list``) - Improve consistency of OS listing by sorting it - Documentation updates Version 2.2.0.1 --------------- *(Released Fri, 8 Oct 2010)* - Rebuild with a newer autotools version, to fix python 2.6 compatibility Version 2.2.0 ------------- *(Released Mon, 4 Oct 2010)* - Fixed regression in ``gnt-instance rename`` Version 2.2.0 rc2 ----------------- *(Released Wed, 22 Sep 2010)* - Fixed OS_VARIANT variable for OS scripts - Fixed cluster tag operations via RAPI - Made ``setup-ssh`` exit with non-zero code if an error occurred - Disabled RAPI CA checks in watcher Version 2.2.0 rc1 ----------------- *(Released Mon, 23 Aug 2010)* - Support DRBD versions of the format "a.b.c.d" - Updated manpages - Re-introduce support for usage from multiple threads in RAPI client - Instance renames and modify via RAPI - Work around race condition between processing and archival in job queue - Mark opcodes following failed one as failed, too - Job field ``lock_status`` was removed due to difficulties making it work with the changed job queue in Ganeti 2.2; a better way to monitor locks is expected for a later 2.2.x release - Fixed dry-run behaviour with many commands - Support ``ssh-agent`` again when adding nodes - Many additional bugfixes Version 2.2.0 rc0 ----------------- *(Released Fri, 30 Jul 2010)* Important change: the internal RPC mechanism between Ganeti nodes has changed from using a home-grown http library (based on the Python base libraries) to use the PycURL library. This requires that PycURL is installed on nodes. Please note that on Debian/Ubuntu, PycURL is linked against GnuTLS by default. cURL's support for GnuTLS had known issues before cURL 7.21.0 and we recommend using the latest cURL release or linking against OpenSSL. Most other distributions already link PycURL and cURL against OpenSSL. The command:: python -c 'import pycurl; print pycurl.version' can be used to determine the libraries PycURL and cURL are linked against. Other significant changes: - Rewrote much of the internals of the job queue, in order to achieve better parallelism; this decouples job query operations from the job processing, and it should allow much nicer behaviour of the master daemon under load, and it also has uncovered some long-standing bugs related to the job serialisation (now fixed) - Added a default iallocator setting to the cluster parameters, eliminating the need to always pass nodes or an iallocator for operations that require selection of new node(s) - Added experimental support for the LXC virtualization method - Added support for OS parameters, which allows the installation of instances to pass parameter to OS scripts in order to customise the instance - Added a hypervisor parameter controlling the migration type (live or non-live), since hypervisors have various levels of reliability; this has renamed the 'live' parameter to 'mode' - Added a cluster parameter ``reserved_lvs`` that denotes reserved logical volumes, meaning that cluster verify will ignore them and not flag their presence as errors - The watcher will now reset the error count for failed instances after 8 hours, thus allowing self-healing if the problem that caused the instances to be down/fail to start has cleared in the meantime - Added a cluster parameter ``drbd_usermode_helper`` that makes Ganeti check for, and warn, if the drbd module parameter ``usermode_helper`` is not consistent with the cluster-wide setting; this is needed to make diagnose easier of failed drbd creations - Started adding base IPv6 support, but this is not yet enabled/available for use - Rename operations (cluster, instance) will now return the new name, which is especially useful if a short name was passed in - Added support for instance migration in RAPI - Added a tool to pre-configure nodes for the SSH setup, before joining them to the cluster; this will allow in the future a simplified model for node joining (but not yet fully enabled in 2.2); this needs the paramiko python library - Fixed handling of name-resolving errors - Fixed consistency of job results on the error path - Fixed master-failover race condition when executed multiple times in sequence - Fixed many bugs related to the job queue (mostly introduced during the 2.2 development cycle, so not all are impacting 2.1) - Fixed instance migration with missing disk symlinks - Fixed handling of unknown jobs in ``gnt-job archive`` - And many other small fixes/improvements Internal changes: - Enhanced both the unittest and the QA coverage - Switched the opcode validation to a generic model, and extended the validation to all opcode parameters - Changed more parts of the code that write shell scripts to use the same class for this - Switched the master daemon to use the asyncore library for the Luxi server endpoint Version 2.2.0 beta0 ------------------- *(Released Thu, 17 Jun 2010)* - Added tool (``move-instance``) and infrastructure to move instances between separate clusters (see :doc:`separate documentation ` and :doc:`design document `) - Added per-request RPC timeout - RAPI now requires a Content-Type header for requests with a body (e.g. ``PUT`` or ``POST``) which must be set to ``application/json`` (see :rfc:`2616` (HTTP/1.1), section 7.2.1) - ``ganeti-watcher`` attempts to restart ``ganeti-rapi`` if RAPI is not reachable - Implemented initial support for running Ganeti daemons as separate users, see configure-time flags ``--with-user-prefix`` and ``--with-group-prefix`` (only ``ganeti-rapi`` is supported at this time) - Instances can be removed after export (``gnt-backup export --remove-instance``) - Self-signed certificates generated by Ganeti now use a 2048 bit RSA key (instead of 1024 bit) - Added new cluster configuration file for cluster domain secret - Import/export now use SSL instead of SSH - Added support for showing estimated time when exporting an instance, see the ``ganeti-os-interface(7)`` manpage and look for ``EXP_SIZE_FD`` Version 2.1.8 ------------- *(Released Tue, 16 Nov 2010)* Some more bugfixes. Unless critical bugs occur, this will be the last 2.1 release: - Fix case of MAC special-values - Fix mac checker regex - backend: Fix typo causing "out of range" error - Add missing --units in gnt-instance list man page Version 2.1.7 ------------- *(Released Tue, 24 Aug 2010)* Bugfixes only: - Don't ignore secondary node silently on non-mirrored disk templates (issue 113) - Fix --master-netdev arg name in gnt-cluster(8) (issue 114) - Fix usb_mouse parameter breaking with vnc_console (issue 109) - Properly document the usb_mouse parameter - Fix path in ganeti-rapi(8) (issue 116) - Adjust error message when the ganeti user's .ssh directory is missing - Add same-node-check when changing the disk template to drbd Version 2.1.6 ------------- *(Released Fri, 16 Jul 2010)* Bugfixes only: - Add an option to only select some reboot types during qa/burnin. (on some hypervisors consequent reboots are not supported) - Fix infrequent race condition in master failover. Sometimes the old master ip address would be still detected as up for a short time after it was removed, causing failover to fail. - Decrease mlockall warnings when the ctypes module is missing. On Python 2.4 we support running even if no ctypes module is installed, but we were too verbose about this issue. - Fix building on old distributions, on which man doesn't have a --warnings option. - Fix RAPI not to ignore the MAC address on instance creation - Implement the old instance creation format in the RAPI client. Version 2.1.5 ------------- *(Released Thu, 01 Jul 2010)* A small bugfix release: - Fix disk adoption: broken by strict --disk option checking in 2.1.4 - Fix batch-create: broken in the whole 2.1 series due to a lookup on a non-existing option - Fix instance create: the --force-variant option was ignored - Improve pylint 0.21 compatibility and warnings with Python 2.6 - Fix modify node storage with non-FQDN arguments - Fix RAPI client to authenticate under Python 2.6 when used for more than 5 requests needing authentication - Fix gnt-instance modify -t (storage) giving a wrong error message when converting a non-shutdown drbd instance to plain Version 2.1.4 ------------- *(Released Fri, 18 Jun 2010)* A small bugfix release: - Fix live migration of KVM instances started with older Ganeti versions which had fewer hypervisor parameters - Fix gnt-instance grow-disk on down instances - Fix an error-reporting bug during instance migration - Better checking of the ``--net`` and ``--disk`` values, to avoid silently ignoring broken ones - Fix an RPC error reporting bug affecting, for example, RAPI client users - Fix bug triggered by different API version os-es on different nodes - Fix a bug in instance startup with custom hvparams: OS level parameters would fail to be applied. - Fix the RAPI client under Python 2.6 (but more work is needed to make it work completely well with OpenSSL) - Fix handling of errors when resolving names from DNS Version 2.1.3 ------------- *(Released Thu, 3 Jun 2010)* A medium sized development cycle. Some new features, and some fixes/small improvements/cleanups. Significant features ~~~~~~~~~~~~~~~~~~~~ The node daemon now tries to mlock itself into memory, unless the ``--no-mlock`` flag is passed. It also doesn't fail if it can't write its logs, and falls back to console logging. This allows emergency features such as ``gnt-node powercycle`` to work even in the event of a broken node disk (tested offlining the disk hosting the node's filesystem and dropping its memory caches; don't try this at home) KVM: add vhost-net acceleration support. It can be tested with a new enough version of the kernel and of qemu-kvm. KVM: Add instance chrooting feature. If you use privilege dropping for your VMs you can also now force them to chroot to an empty directory, before starting the emulated guest. KVM: Add maximum migration bandwith and maximum downtime tweaking support (requires a new-enough version of qemu-kvm). Cluster verify will now warn if the master node doesn't have the master ip configured on it. Add a new (incompatible) instance creation request format to RAPI which supports all parameters (previously only a subset was supported, and it wasn't possible to extend the old format to accomodate all the new features. The old format is still supported, and a client can check for this feature, before using it, by checking for its presence in the ``features`` RAPI resource. Now with ancient latin support. Try it passing the ``--roman`` option to ``gnt-instance info``, ``gnt-cluster info`` or ``gnt-node list`` (requires the python-roman module to be installed, in order to work). Other changes ~~~~~~~~~~~~~ As usual many internal code refactorings, documentation updates, and such. Among others: - Lots of improvements and cleanups to the experimental Remote API (RAPI) client library. - A new unit test suite for the core daemon libraries. - A fix to creating missing directories makes sure the umask is not applied anymore. This enforces the same directory permissions everywhere. - Better handling terminating daemons with ctrl+c (used when running them in debugging mode). - Fix a race condition in live migrating a KVM instance, when stat() on the old proc status file returned EINVAL, which is an unexpected value. - Fixed manpage checking with newer man and utf-8 charachters. But now you need the en_US.UTF-8 locale enabled to build Ganeti from git. Version 2.1.2.1 --------------- *(Released Fri, 7 May 2010)* Fix a bug which prevented untagged KVM instances from starting. Version 2.1.2 ------------- *(Released Fri, 7 May 2010)* Another release with a long development cycle, during which many different features were added. Significant features ~~~~~~~~~~~~~~~~~~~~ The KVM hypervisor now can run the individual instances as non-root, to reduce the impact of a VM being hijacked due to bugs in the hypervisor. It is possible to run all instances as a single (non-root) user, to manually specify a user for each instance, or to dynamically allocate a user out of a cluster-wide pool to each instance, with the guarantee that no two instances will run under the same user ID on any given node. An experimental RAPI client library, that can be used standalone (without the other Ganeti libraries), is provided in the source tree as ``lib/rapi/client.py``. Note this client might change its interface in the future, as we iterate on its capabilities. A new command, ``gnt-cluster renew-crypto`` has been added to easily replace the cluster's certificates and crypto keys. This might help in case they have been compromised, or have simply expired. A new disk option for instance creation has been added that allows one to "adopt" currently existing logical volumes, with data preservation. This should allow easier migration to Ganeti from unmanaged (or managed via other software) instances. Another disk improvement is the possibility to convert between redundant (DRBD) and plain (LVM) disk configuration for an instance. This should allow better scalability (starting with one node and growing the cluster, or shrinking a two-node cluster to one node). A new feature that could help with automated node failovers has been implemented: if a node sees itself as offline (by querying the master candidates), it will try to shutdown (hard) all instances and any active DRBD devices. This reduces the risk of duplicate instances if an external script automatically failovers the instances on such nodes. To enable this, the cluster parameter ``maintain_node_health`` should be enabled; in the future this option (per the name) will enable other automatic maintenance features. Instance export/import now will reuse the original instance specifications for all parameters; that means exporting an instance, deleting it and the importing it back should give an almost identical instance. Note that the default import behaviour has changed from before, where it created only one NIC; now it recreates the original number of NICs. Cluster verify has added a few new checks: SSL certificates validity, /etc/hosts consistency across the cluster, etc. Other changes ~~~~~~~~~~~~~ As usual, many internal changes were done, documentation fixes, etc. Among others: - Fixed cluster initialization with disabled cluster storage (regression introduced in 2.1.1) - File-based storage supports growing the disks - Fixed behaviour of node role changes - Fixed cluster verify for some corner cases, plus a general rewrite of cluster verify to allow future extension with more checks - Fixed log spamming by watcher and node daemon (regression introduced in 2.1.1) - Fixed possible validation issues when changing the list of enabled hypervisors - Fixed cleanup of /etc/hosts during node removal - Fixed RAPI response for invalid methods - Fixed bug with hashed passwords in ``ganeti-rapi`` daemon - Multiple small improvements to the KVM hypervisor (VNC usage, booting from ide disks, etc.) - Allow OS changes without re-installation (to record a changed OS outside of Ganeti, or to allow OS renames) - Allow instance creation without OS installation (useful for example if the OS will be installed manually, or restored from a backup not in Ganeti format) - Implemented option to make cluster ``copyfile`` use the replication network - Added list of enabled hypervisors to ssconf (possibly useful for external scripts) - Added a new tool (``tools/cfgupgrade12``) that allows upgrading from 1.2 clusters - A partial form of node re-IP is possible via node readd, which now allows changed node primary IP - Command line utilities now show an informational message if the job is waiting for a lock - The logs of the master daemon now show the PID/UID/GID of the connected client Version 2.1.1 ------------- *(Released Fri, 12 Mar 2010)* During the 2.1.0 long release candidate cycle, a lot of improvements and changes have accumulated with were released later as 2.1.1. Major changes ~~~~~~~~~~~~~ The node evacuate command (``gnt-node evacuate``) was significantly rewritten, and as such the IAllocator protocol was changed - a new request type has been added. This unfortunate change during a stable series is designed to improve performance of node evacuations; on clusters with more than about five nodes and which are well-balanced, evacuation should proceed in parallel for all instances of the node being evacuated. As such, any existing IAllocator scripts need to be updated, otherwise the above command will fail due to the unknown request. The provided "dumb" allocator has not been updated; but the ganeti-htools package supports the new protocol since version 0.2.4. Another important change is increased validation of node and instance names. This might create problems in special cases, if invalid host names are being used. Also, a new layer of hypervisor parameters has been added, that sits at OS level between the cluster defaults and the instance ones. This allows customisation of virtualization parameters depending on the installed OS. For example instances with OS 'X' may have a different KVM kernel (or any other parameter) than the cluster defaults. This is intended to help managing a multiple OSes on the same cluster, without manual modification of each instance's parameters. A tool for merging clusters, ``cluster-merge``, has been added in the tools sub-directory. Bug fixes ~~~~~~~~~ - Improved the int/float conversions that should make the code more robust in face of errors from the node daemons - Fixed the remove node code in case of internal configuration errors - Fixed the node daemon behaviour in face of inconsistent queue directory (e.g. read-only file-system where we can't open the files read-write, etc.) - Fixed the behaviour of gnt-node modify for master candidate demotion; now it either aborts cleanly or, if given the new "auto_promote" parameter, will automatically promote other nodes as needed - Fixed compatibility with (unreleased yet) Python 2.6.5 that would completely prevent Ganeti from working - Fixed bug for instance export when not all disks were successfully exported - Fixed behaviour of node add when the new node is slow in starting up the node daemon - Fixed handling of signals in the LUXI client, which should improve behaviour of command-line scripts - Added checks for invalid node/instance names in the configuration (now flagged during cluster verify) - Fixed watcher behaviour for disk activation errors - Fixed two potentially endless loops in http library, which led to the RAPI daemon hanging and consuming 100% CPU in some cases - Fixed bug in RAPI daemon related to hashed passwords - Fixed bug for unintended qemu-level bridging of multi-NIC KVM instances - Enhanced compatibility with non-Debian OSes, but not using absolute path in some commands and allowing customisation of the ssh configuration directory - Fixed possible future issue with new Python versions by abiding to the proper use of ``__slots__`` attribute on classes - Added checks that should prevent directory traversal attacks - Many documentation fixes based on feedback from users New features ~~~~~~~~~~~~ - Added an "early_release" more for instance replace disks and node evacuate, where we release locks earlier and thus allow higher parallelism within the cluster - Added watcher hooks, intended to allow the watcher to restart other daemons (e.g. from the ganeti-nbma project), but they can be used of course for any other purpose - Added a compile-time disable for DRBD barriers, to increase performance if the administrator trusts the power supply or the storage system to not lose writes - Added the option of using syslog for logging instead of, or in addition to, Ganeti's own log files - Removed boot restriction for paravirtual NICs for KVM, recent versions can indeed boot from a paravirtual NIC - Added a generic debug level for many operations; while this is not used widely yet, it allows one to pass the debug value all the way to the OS scripts - Enhanced the hooks environment for instance moves (failovers, migrations) where the primary/secondary nodes changed during the operation, by adding {NEW,OLD}_{PRIMARY,SECONDARY} vars - Enhanced data validations for many user-supplied values; one important item is the restrictions imposed on instance and node names, which might reject some (invalid) host names - Add a configure-time option to disable file-based storage, if it's not needed; this allows greater security separation between the master node and the other nodes from the point of view of the inter-node RPC protocol - Added user notification in interactive tools if job is waiting in the job queue or trying to acquire locks - Added log messages when a job is waiting for locks - Added filtering by node tags in instance operations which admit multiple instances (start, stop, reboot, reinstall) - Added a new tool for cluster mergers, ``cluster-merge`` - Parameters from command line which are of the form ``a=b,c=d`` can now use backslash escapes to pass in values which contain commas, e.g. ``a=b\\c,d=e`` where the 'a' parameter would get the value ``b,c`` - For KVM, the instance name is the first parameter passed to KVM, so that it's more visible in the process list Version 2.1.0 ------------- *(Released Tue, 2 Mar 2010)* Ganeti 2.1 brings many improvements with it. Major changes: - Added infrastructure to ease automated disk repairs - Added new daemon to export configuration data in a cheaper way than using the remote API - Instance NICs can now be routed instead of being associated with a networking bridge - Improved job locking logic to reduce impact of jobs acquiring multiple locks waiting for other long-running jobs In-depth implementation details can be found in the Ganeti 2.1 design document. Details ~~~~~~~ - Added chroot hypervisor - Added more options to xen-hvm hypervisor (``kernel_path`` and ``device_model``) - Added more options to xen-pvm hypervisor (``use_bootloader``, ``bootloader_path`` and ``bootloader_args``) - Added the ``use_localtime`` option for the xen-hvm and kvm hypervisors, and the default value for this has changed to false (in 2.0 xen-hvm always enabled it) - Added luxi call to submit multiple jobs in one go - Added cluster initialization option to not modify ``/etc/hosts`` file on nodes - Added network interface parameters - Added dry run mode to some LUs - Added RAPI resources: - ``/2/instances/[instance_name]/info`` - ``/2/instances/[instance_name]/replace-disks`` - ``/2/nodes/[node_name]/evacuate`` - ``/2/nodes/[node_name]/migrate`` - ``/2/nodes/[node_name]/role`` - ``/2/nodes/[node_name]/storage`` - ``/2/nodes/[node_name]/storage/modify`` - ``/2/nodes/[node_name]/storage/repair`` - Added OpCodes to evacuate or migrate all instances on a node - Added new command to list storage elements on nodes (``gnt-node list-storage``) and modify them (``gnt-node modify-storage``) - Added new ssconf files with master candidate IP address (``ssconf_master_candidates_ips``), node primary IP address (``ssconf_node_primary_ips``) and node secondary IP address (``ssconf_node_secondary_ips``) - Added ``ganeti-confd`` and a client library to query the Ganeti configuration via UDP - Added ability to run hooks after cluster initialization and before cluster destruction - Added automatic mode for disk replace (``gnt-instance replace-disks --auto``) - Added ``gnt-instance recreate-disks`` to re-create (empty) disks after catastrophic data-loss - Added ``gnt-node repair-storage`` command to repair damaged LVM volume groups - Added ``gnt-instance move`` command to move instances - Added ``gnt-cluster watcher`` command to control watcher - Added ``gnt-node powercycle`` command to powercycle nodes - Added new job status field ``lock_status`` - Added parseable error codes to cluster verification (``gnt-cluster verify --error-codes``) and made output less verbose (use ``--verbose`` to restore previous behaviour) - Added UUIDs to the main config entities (cluster, nodes, instances) - Added support for OS variants - Added support for hashed passwords in the Ganeti remote API users file (``rapi_users``) - Added option to specify maximum timeout on instance shutdown - Added ``--no-ssh-init`` option to ``gnt-cluster init`` - Added new helper script to start and stop Ganeti daemons (``daemon-util``), with the intent to reduce the work necessary to adjust Ganeti for non-Debian distributions and to start/stop daemons from one place - Added more unittests - Fixed critical bug in ganeti-masterd startup - Removed the configure-time ``kvm-migration-port`` parameter, this is now customisable at the cluster level for both the KVM and Xen hypervisors using the new ``migration_port`` parameter - Pass ``INSTANCE_REINSTALL`` variable to OS installation script when reinstalling an instance - Allowed ``@`` in tag names - Migrated to Sphinx (http://sphinx.pocoo.org/) for documentation - Many documentation updates - Distribute hypervisor files on ``gnt-cluster redist-conf`` - ``gnt-instance reinstall`` can now reinstall multiple instances - Updated many command line parameters - Introduced new OS API version 15 - No longer support a default hypervisor - Treat virtual LVs as inexistent - Improved job locking logic to reduce lock contention - Match instance and node names case insensitively - Reimplemented bash completion script to be more complete - Improved burnin Version 2.0.6 ------------- *(Released Thu, 4 Feb 2010)* - Fix cleaner behaviour on nodes not in a cluster (Debian bug 568105) - Fix a string formatting bug - Improve safety of the code in some error paths - Improve data validation in the master of values returned from nodes Version 2.0.5 ------------- *(Released Thu, 17 Dec 2009)* - Fix security issue due to missing validation of iallocator names; this allows local and remote execution of arbitrary executables - Fix failure of gnt-node list during instance removal - Ship the RAPI documentation in the archive Version 2.0.4 ------------- *(Released Wed, 30 Sep 2009)* - Fixed many wrong messages - Fixed a few bugs related to the locking library - Fixed MAC checking at instance creation time - Fixed a DRBD parsing bug related to gaps in /proc/drbd - Fixed a few issues related to signal handling in both daemons and scripts - Fixed the example startup script provided - Fixed insserv dependencies in the example startup script (patch from Debian) - Fixed handling of drained nodes in the iallocator framework - Fixed handling of KERNEL_PATH parameter for xen-hvm (Debian bug #528618) - Fixed error related to invalid job IDs in job polling - Fixed job/opcode persistence on unclean master shutdown - Fixed handling of partial job processing after unclean master shutdown - Fixed error reporting from LUs, previously all errors were converted into execution errors - Fixed error reporting from burnin - Decreased significantly the memory usage of the job queue - Optimised slightly multi-job submission - Optimised slightly opcode loading - Backported the multi-job submit framework from the development branch; multi-instance start and stop should be faster - Added script to clean archived jobs after 21 days; this will reduce the size of the queue directory - Added some extra checks in disk size tracking - Added an example ethers hook script - Added a cluster parameter that prevents Ganeti from modifying of /etc/hosts - Added more node information to RAPI responses - Added a ``gnt-job watch`` command that allows following the ouput of a job - Added a bind-address option to ganeti-rapi - Added more checks to the configuration verify - Enhanced the burnin script such that some operations can be retried automatically - Converted instance reinstall to multi-instance model Version 2.0.3 ------------- *(Released Fri, 7 Aug 2009)* - Added ``--ignore-size`` to the ``gnt-instance activate-disks`` command to allow using the pre-2.0.2 behaviour in activation, if any existing instances have mismatched disk sizes in the configuration - Added ``gnt-cluster repair-disk-sizes`` command to check and update any configuration mismatches for disk sizes - Added ``gnt-master cluste-failover --no-voting`` to allow master failover to work on two-node clusters - Fixed the ``--net`` option of ``gnt-backup import``, which was unusable - Fixed detection of OS script errors in ``gnt-backup export`` - Fixed exit code of ``gnt-backup export`` Version 2.0.2 ------------- *(Released Fri, 17 Jul 2009)* - Added experimental support for stripped logical volumes; this should enhance performance but comes with a higher complexity in the block device handling; stripping is only enabled when passing ``--with-lvm-stripecount=N`` to ``configure``, but codepaths are affected even in the non-stripped mode - Improved resiliency against transient failures at the end of DRBD resyncs, and in general of DRBD resync checks - Fixed a couple of issues with exports and snapshot errors - Fixed a couple of issues in instance listing - Added display of the disk size in ``gnt-instance info`` - Fixed checking for valid OSes in instance creation - Fixed handling of the "vcpus" parameter in instance listing and in general of invalid parameters - Fixed http server library, and thus RAPI, to handle invalid username/password combinations correctly; this means that now they report unauthorized for queries too, not only for modifications, allowing earlier detect of configuration problems - Added a new "role" node list field, equivalent to the master/master candidate/drained/offline flags combinations - Fixed cluster modify and changes of candidate pool size - Fixed cluster verify error messages for wrong files on regular nodes - Fixed a couple of issues with node demotion from master candidate role - Fixed node readd issues - Added non-interactive mode for ``ganeti-masterd --no-voting`` startup - Added a new ``--no-voting`` option for masterfailover to fix failover on two-nodes clusters when the former master node is unreachable - Added instance reinstall over RAPI Version 2.0.1 ------------- *(Released Tue, 16 Jun 2009)* - added ``-H``/``-B`` startup parameters to ``gnt-instance``, which will allow re-adding the start in single-user option (regression from 1.2) - the watcher writes the instance status to a file, to allow monitoring to report the instance status (from the master) based on cached results of the watcher's queries; while this can get stale if the watcher is being locked due to other work on the cluster, this is still an improvement - the watcher now also restarts the node daemon and the rapi daemon if they died - fixed the watcher to handle full and drained queue cases - hooks export more instance data in the environment, which helps if hook scripts need to take action based on the instance's properties (no longer need to query back into ganeti) - instance failovers when the instance is stopped do not check for free RAM, so that failing over a stopped instance is possible in low memory situations - rapi uses queries for tags instead of jobs (for less job traffic), and for cluster tags it won't talk to masterd at all but read them from ssconf - a couple of error handling fixes in RAPI - drbd handling: improved the error handling of inconsistent disks after resync to reduce the frequency of "there are some degraded disks for this instance" messages - fixed a bug in live migration when DRBD doesn't want to reconnect (the error handling path called a wrong function name) Version 2.0.0 ------------- *(Released Wed, 27 May 2009)* - no changes from rc5 Version 2.0 rc5 --------------- *(Released Wed, 20 May 2009)* - fix a couple of bugs (validation, argument checks) - fix ``gnt-cluster getmaster`` on non-master nodes (regression) - some small improvements to RAPI and IAllocator - make watcher automatically start the master daemon if down Version 2.0 rc4 --------------- *(Released Mon, 27 Apr 2009)* - change the OS list to not require locks; this helps with big clusters - fix ``gnt-cluster verify`` and ``gnt-cluster verify-disks`` when the volume group is broken - ``gnt-instance info``, without any arguments, doesn't run for all instances anymore; either pass ``--all`` or pass the desired instances; this helps against mistakes on big clusters where listing the information for all instances takes a long time - miscellaneous doc and man pages fixes Version 2.0 rc3 --------------- *(Released Wed, 8 Apr 2009)* - Change the internal locking model of some ``gnt-node`` commands, in order to reduce contention (and blocking of master daemon) when batching many creation/reinstall jobs - Fixes to Xen soft reboot - No longer build documentation at build time, instead distribute it in the archive, in order to reduce the need for the whole docbook/rst toolchains Version 2.0 rc2 --------------- *(Released Fri, 27 Mar 2009)* - Now the cfgupgrade scripts works and can upgrade 1.2.7 clusters to 2.0 - Fix watcher startup sequence, improves the behaviour of busy clusters - Some other fixes in ``gnt-cluster verify``, ``gnt-instance replace-disks``, ``gnt-instance add``, ``gnt-cluster queue``, KVM VNC bind address and other places - Some documentation fixes and updates Version 2.0 rc1 --------------- *(Released Mon, 2 Mar 2009)* - More documentation updates, now all docs should be more-or-less up-to-date - A couple of small fixes (mixed hypervisor clusters, offline nodes, etc.) - Added a customizable HV_KERNEL_ARGS hypervisor parameter (for Xen PVM and KVM) - Fix an issue related to $libdir/run/ganeti and cluster creation Version 2.0 beta2 ----------------- *(Released Thu, 19 Feb 2009)* - Xen PVM and KVM have switched the default value for the instance root disk to the first partition on the first drive, instead of the whole drive; this means that the OS installation scripts must be changed accordingly - Man pages have been updated - RAPI has been switched by default to HTTPS, and the exported functions should all work correctly - RAPI v1 has been removed - Many improvements to the KVM hypervisor - Block device errors are now better reported - Many other bugfixes and small improvements Version 2.0 beta1 ----------------- *(Released Mon, 26 Jan 2009)* - Version 2 is a general rewrite of the code and therefore the differences are too many to list, see the design document for 2.0 in the ``doc/`` subdirectory for more details - In this beta version there is not yet a migration path from 1.2 (there will be one in the final 2.0 release) - A few significant changes are: - all commands are executed by a daemon (``ganeti-masterd``) and the various ``gnt-*`` commands are just front-ends to it - all the commands are entered into, and executed from a job queue, see the ``gnt-job(8)`` manpage - the RAPI daemon supports read-write operations, secured by basic HTTP authentication on top of HTTPS - DRBD version 0.7 support has been removed, DRBD 8 is the only supported version (when migrating from Ganeti 1.2 to 2.0, you need to migrate to DRBD 8 first while still running Ganeti 1.2) - DRBD devices are using statically allocated minor numbers, which will be assigned to existing instances during the migration process - there is support for both Xen PVM and Xen HVM instances running on the same cluster - KVM virtualization is supported too - file-based storage has been implemented, which means that it is possible to run the cluster without LVM and DRBD storage, for example using a shared filesystem exported from shared storage (and still have live migration) Version 1.2.7 ------------- *(Released Tue, 13 Jan 2009)* - Change the default reboot type in ``gnt-instance reboot`` to "hard" - Reuse the old instance mac address by default on instance import, if the instance name is the same. - Handle situations in which the node info rpc returns incomplete results (issue 46) - Add checks for tcp/udp ports collisions in ``gnt-cluster verify`` - Improved version of batcher: - state file support - instance mac address support - support for HVM clusters/instances - Add an option to show the number of cpu sockets and nodes in ``gnt-node list`` - Support OSes that handle more than one version of the OS api (but do not change the current API in any other way) - Fix ``gnt-node migrate`` - ``gnt-debug`` man page - Fixes various more typos and small issues - Increase disk resync maximum speed to 60MB/s (from 30MB/s) Version 1.2.6 ------------- *(Released Wed, 24 Sep 2008)* - new ``--hvm-nic-type`` and ``--hvm-disk-type`` flags to control the type of disk exported to fully virtualized instances. - provide access to the serial console of HVM instances - instance auto_balance flag, set by default. If turned off it will avoid warnings on cluster verify if there is not enough memory to fail over an instance. in the future it will prevent automatically failing it over when we will support that. - batcher tool for instance creation, see ``tools/README.batcher`` - ``gnt-instance reinstall --select-os`` to interactively select a new operating system when reinstalling an instance. - when changing the memory amount on instance modify a check has been added that the instance will be able to start. also warnings are emitted if the instance will not be able to fail over, if auto_balance is true. - documentation fixes - sync fields between ``gnt-instance list/modify/add/import`` - fix a race condition in drbd when the sync speed was set after giving the device a remote peer. Version 1.2.5 ------------- *(Released Tue, 22 Jul 2008)* - note: the allowed size and number of tags per object were reduced - fix a bug in ``gnt-cluster verify`` with inconsistent volume groups - fixed twisted 8.x compatibility - fixed ``gnt-instance replace-disks`` with iallocator - add TCP keepalives on twisted connections to detect restarted nodes - disk increase support, see ``gnt-instance grow-disk`` - implement bulk node/instance query for RAPI - add tags in node/instance listing (optional) - experimental migration (and live migration) support, read the man page for ``gnt-instance migrate`` - the ``ganeti-watcher`` logs are now timestamped, and the watcher also has some small improvements in handling its state file Version 1.2.4 ------------- *(Released Fri, 13 Jun 2008)* - Experimental readonly, REST-based remote API implementation; automatically started on master node, TCP port 5080, if enabled by ``--enable-rapi`` parameter to configure script. - Instance allocator support. Add and import instance accept a ``--iallocator`` parameter, and call that instance allocator to decide which node to use for the instance. The iallocator document describes what's expected from an allocator script. - ``gnt-cluster verify`` N+1 memory redundancy checks: Unless passed the ``--no-nplus1-mem`` option ``gnt-cluster verify`` now checks that if a node is lost there is still enough memory to fail over the instances that reside on it. - ``gnt-cluster verify`` hooks: it is now possible to add post-hooks to ``gnt-cluster verify``, to check for site-specific compliance. All the hooks will run, and their output, if any, will be displayed. Any failing hook will make the verification return an error value. - ``gnt-cluster verify`` now checks that its peers are reachable on the primary and secondary interfaces - ``gnt-node add`` now supports the ``--readd`` option, to readd a node that is still declared as part of the cluster and has failed. - ``gnt-* list`` commands now accept a new ``-o +field`` way of specifying output fields, that just adds the chosen fields to the default ones. - ``gnt-backup`` now has a new ``remove`` command to delete an existing export from the filesystem. - New per-instance parameters hvm_acpi, hvm_pae and hvm_cdrom_image_path have been added. Using them you can enable/disable acpi and pae support, and specify a path for a cd image to be exported to the instance. These parameters as the name suggest only work on HVM clusters. - When upgrading an HVM cluster to Ganeti 1.2.4, the values for ACPI and PAE support will be set to the previously hardcoded values, but the (previously hardcoded) path to the CDROM ISO image will be unset and if required, needs to be set manually with ``gnt-instance modify`` after the upgrade. - The address to which an instance's VNC console is bound is now selectable per-instance, rather than being cluster wide. Of course this only applies to instances controlled via VNC, so currently just applies to HVM clusters. Version 1.2.3 ------------- *(Released Mon, 18 Feb 2008)* - more tweaks to the disk activation code (especially helpful for DRBD) - change the default ``gnt-instance list`` output format, now there is one combined status field (see the manpage for the exact values this field will have) - some more fixes for the mac export to hooks change - make Ganeti not break with DRBD 8.2.x (which changed the version format in ``/proc/drbd``) (issue 24) - add an upgrade tool from "remote_raid1" disk template to "drbd" disk template, allowing migration from DRBD0.7+MD to DRBD8 Version 1.2.2 ------------- *(Released Wed, 30 Jan 2008)* - fix ``gnt-instance modify`` breakage introduced in 1.2.1 with the HVM support (issue 23) - add command aliases infrastructure and a few aliases - allow listing of VCPUs in the ``gnt-instance list`` and improve the man pages and the ``--help`` option of ``gnt-node list``/``gnt-instance list`` - fix ``gnt-backup list`` with down nodes (issue 21) - change the tools location (move from $pkgdatadir to $pkglibdir/tools) - fix the dist archive and add a check for including svn/git files in the future - some developer-related changes: improve the burnin and the QA suite, add an upload script for testing during development Version 1.2.1 ------------- *(Released Wed, 16 Jan 2008)* - experimental HVM support, read the install document, section "Initializing the cluster" - allow for the PVM hypervisor per-instance kernel and initrd paths - add a new command ``gnt-cluster verify-disks`` which uses a new algorithm to improve the reconnection of the DRBD pairs if the device on the secondary node has gone away - make logical volume code auto-activate LVs at disk activation time - slightly improve the speed of activating disks - allow specification of the MAC address at instance creation time, and changing it later via ``gnt-instance modify`` - fix handling of external commands that generate lots of output on stderr - update documentation with regard to minimum version of DRBD8 supported Version 1.2.0 ------------- *(Released Tue, 4 Dec 2007)* - Log the ``xm create`` output to the node daemon log on failure (to help diagnosing the error) - In debug mode, log all external commands output if failed to the logs - Change parsing of lvm commands to ignore stderr Version 1.2 beta3 ----------------- *(Released Wed, 28 Nov 2007)* - Another round of updates to the DRBD 8 code to deal with more failures in the replace secondary node operation - Some more logging of failures in disk operations (lvm, drbd) - A few documentation updates - QA updates Version 1.2 beta2 ----------------- *(Released Tue, 13 Nov 2007)* - Change configuration file format from Python's Pickle to JSON. Upgrading is possible using the cfgupgrade utility. - Add support for DRBD 8.0 (new disk template ``drbd``) which allows for faster replace disks and is more stable (DRBD 8 has many improvements compared to DRBD 0.7) - Added command line tags support (see man pages for ``gnt-instance``, ``gnt-node``, ``gnt-cluster``) - Added instance rename support - Added multi-instance startup/shutdown - Added cluster rename support - Added ``gnt-node evacuate`` to simplify some node operations - Added instance reboot operation that can speedup reboot as compared to stop and start - Soften the requirement that hostnames are in FQDN format - The ``ganeti-watcher`` now activates drbd pairs after secondary node reboots - Removed dependency on debian's patched fping that uses the non-standard ``-S`` option - Now the OS definitions are searched for in multiple, configurable paths (easier for distros to package) - Some changes to the hooks infrastructure (especially the new post-configuration update hook) - Other small bugfixes .. vim: set textwidth=72 syntax=rst : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/README.md000064400000000000000000000044171476477700300150710ustar00rootroot00000000000000# Ganeti 3.1 **Welcome to the Ganeti Project** Ganeti is a virtual machine cluster management tool built on top of existing virtualisation technologies such as Xen or KVM and other open source software. ## Our Vision * a virtualisation solution that Just Works * self-contained * aim for high-availibility and reliability * mid-range virtualisation solution (3 to 100 nodes) * purely open source * low complexity * available in major Linux distributions ## Getting Started ### Debian Debian provides packages out of the box. More recent packages can usually be found through [Debian Backports](https://backports.debian.org/). ### Ubuntu Please use [this PPA](https://launchpad.net/~pkg-ganeti-devel/+archive/ubuntu/lts) if you would like to run Ganeti on Ubuntu. ### RedHat/CentOS For Redhat/CentOS based systems, please head over to [this repository](https://jfut.integ.jp/linux/ganeti/). ### Other Distributions / Source builds You can also build Ganeti directly from source - please read [these instructions](https://github.com/ganeti/ganeti/blob/master/INSTALL) carefully. ### Nightly/Developer builds We provide nightly packages for Debian and Ubuntu off the project's master branch via this repository. These packages are only for testing purposes, do not use them in production! There is no upgrade path between nightly builds! Further information about dependencies and cluster initialisation can be found in [this Guide](https://docs.ganeti.org/docs/ganeti/3.0/html/install.html). ## Community Ganeti is being used in virtualisation setups all over the world. If you want to reach or join our community, you can use one of the following options. ### Mailing Lists There are two mailing lists available: [ganeti-users](https://groups.google.com/g/ganeti) and [ganeti-devel](https://groups.google.com/g/ganeti-devel). ### IRC The `#ganeti` channel lives on the [OFTC IRC Network](https://www.oftc.net/). ### GanetiCon We have a yearly developer & users meeting organized by the community. Head over to [ganeticon.org](https://ganeticon.org/) for more information! ## Contributing You would like to contribute to the Ganeti Project? Feel free to open Github issues, submit pull requests and of course check out our [development guidelines](https://github.com/ganeti/ganeti/wiki#development). ganeti-3.1.0~rc2/Setup.lhs000075500000000000000000000001151476477700300154140ustar00rootroot00000000000000#! /usr/bin/env runhaskell > import Distribution.Simple > main = defaultMain ganeti-3.1.0~rc2/UPGRADE000064400000000000000000000351331476477700300146230ustar00rootroot00000000000000Upgrade notes ============= .. highlight:: shell-example This document details the steps needed to upgrade a cluster to newer versions of Ganeti. As a general rule the node daemons need to be restarted after each software upgrade; if using the provided example init.d script, this means running the following command on all nodes:: $ /etc/init.d/ganeti restart 2.11 and above -------------- Starting from 2.10 onwards, Ganeti has support for parallely installed versions and automated upgrades. The default configuration for 2.11 and higher already is to install as a parallel version without changing the running version. If both versions, the installed one and the one to upgrade to, are 2.10 or higher, the actual switch of the live version can be carried out by the following command on the master node.:: $ gnt-cluster upgrade --to 2.11 This will carry out the steps described below in the section on upgrades from 2.1 and above. Downgrades to the previous minor version can be done in the same way, specifiying the smaller version on the ``--to`` argument. Note that ``gnt-cluster upgrade`` only manages the actual switch between versions as described below on upgrades from 2.1 and above. It does not install or remove any binaries. Having the new binaries installed is a prerequisite of calling ``gnt-cluster upgrade`` (and the command will abort if the prerequisite is not met). The old binaries can be used to downgrade back to the previous version; once the system administrator decides that going back to the old version is not needed any more, they can be removed. Addition and removal of the Ganeti binaries should happen in the same way as for all other binaries on your system. 2.13 ---- When upgrading to 2.13, first apply the instructions of ``2.11 and above``. 2.13 comes with the new feature of enhanced SSH security through individual SSH keys. This features needs to be enabled after the upgrade by:: $ gnt-cluster renew-crypto --new-ssh-keys --no-ssh-key-check Note that new SSH keys are generated automatically without warning when upgrading with ``gnt-cluster upgrade``. If you instructed Ganeti to not touch the SSH setup (by using the ``--no-ssh-init`` option of ``gnt-cluster init``, the changes in the handling of SSH keys will not affect your cluster. If you want to be prompted for each newly created SSH key, leave out the ``--no-ssh-key-check`` option in the command listed above. Note that after a downgrade from 2.13 to 2.12, the individual SSH keys will not get removed automatically. This can lead to reachability errors under very specific circumstances (Issue 1008). In case you plan on keeping 2.12 for a while and not upgrade to 2.13 again soon, we recommend to replace all SSH key pairs of non-master nodes' with the master node's SSH key pair. 2.12 ---- Due to issue #1094 in Ganeti 2.11 and 2.12 up to version 2.12.4, we advise to rerun 'gnt-cluster renew-crypto --new-node-certificates' after an upgrade to 2.12.5 or higher. 2.11 ---- When upgrading to 2.11, first apply the instructions of ``2.11 and above``. 2.11 comes with the new feature of enhanced RPC security through client certificates. This features needs to be enabled after the upgrade by:: $ gnt-cluster renew-crypto --new-node-certificates Note that new node certificates are generated automatically without warning when upgrading with ``gnt-cluster upgrade``. 2.1 and above ------------- Starting with Ganeti 2.0, upgrades between revisions (e.g. 2.1.0 to 2.1.1) should not need manual intervention. As a safety measure, minor releases (e.g. 2.1.3 to 2.2.0) require the ``cfgupgrade`` command for changing the configuration version. Below you find the steps necessary to upgrade between minor releases. To run commands on all nodes, the `distributed shell (dsh) `_ can be used, e.g. ``dsh -M -F 8 -f /var/lib/ganeti/ssconf_online_nodes gnt-cluster --version``. #. Ensure no jobs are running (master node only):: $ gnt-job list #. Pause the watcher for an hour (master node only):: $ gnt-cluster watcher pause 1h #. Stop all daemons on all nodes:: $ /etc/init.d/ganeti stop #. Backup old configuration (master node only):: $ tar czf /var/lib/ganeti-$(date +\%FT\%T).tar.gz -C /var/lib ganeti (``/var/lib/ganeti`` can also contain exported instances, so make sure to backup only files you are interested in. Use ``--exclude export`` for example) #. Install new Ganeti version on all nodes #. Run cfgupgrade on the master node:: $ /usr/lib/ganeti/tools/cfgupgrade --verbose --dry-run $ /usr/lib/ganeti/tools/cfgupgrade --verbose (``cfgupgrade`` supports a number of parameters, run it with ``--help`` for more information) #. Upgrade the directory permissions on all nodes:: $ /usr/lib/ganeti/ensure-dirs --full-run Note that ensure-dirs does not create the directories for file and shared-file storage. This is due to security reasons. They need to be created manually. For details see ``man gnt-cluster``. #. Create the (missing) required users and make users part of the required groups on all nodes:: $ /usr/lib/ganeti/tools/users-setup This will ask for confirmation. To execute directly, add the ``--yes-do-it`` option. #. Restart daemons on all nodes:: $ /etc/init.d/ganeti restart #. Re-distribute configuration (master node only):: $ gnt-cluster redist-conf #. If you use file storage, check that the ``/etc/ganeti/file-storage-paths`` is correct on all nodes. For security reasons it's not copied automatically, but it can be copied manually via:: $ gnt-cluster copyfile /etc/ganeti/file-storage-paths #. Restart daemons again on all nodes:: $ /etc/init.d/ganeti restart #. Enable the watcher again (master node only):: $ gnt-cluster watcher continue #. Verify cluster (master node only):: $ gnt-cluster verify Reverting an upgrade ~~~~~~~~~~~~~~~~~~~~ For going back between revisions (e.g. 2.1.1 to 2.1.0) no manual intervention is required, as for upgrades. Starting from version 2.8, ``cfgupgrade`` supports ``--downgrade`` option to bring the configuration back to the previous stable version. This is useful if you upgrade Ganeti and after some time you run into problems with the new version. You can downgrade the configuration without losing the changes made since the upgrade. Any feature not supported by the old version will be removed from the configuration, of course, but you get a warning about it. If there is any new feature and you haven't changed from its default value, you don't have to worry about it, as it will get the same value whenever you'll upgrade again. Automatic downgrades .................... From version 2.11 onwards, downgrades can be done by using the ``gnt-cluster upgrade`` command.:: gnt-cluster upgrade --to 2.10 Manual downgrades ................. The procedure is similar to upgrading, but please notice that you have to revert the configuration **before** installing the old version. #. Ensure no jobs are running (master node only):: $ gnt-job list #. Pause the watcher for an hour (master node only):: $ gnt-cluster watcher pause 1h #. Stop all daemons on all nodes:: $ /etc/init.d/ganeti stop #. Backup old configuration (master node only):: $ tar czf /var/lib/ganeti-$(date +\%FT\%T).tar.gz -C /var/lib ganeti #. Run cfgupgrade on the master node:: $ /usr/lib/ganeti/tools/cfgupgrade --verbose --downgrade --dry-run $ /usr/lib/ganeti/tools/cfgupgrade --verbose --downgrade You may want to copy all the messages about features that have been removed during the downgrade, in case you want to restore them when upgrading again. #. Install the old Ganeti version on all nodes NB: in Ganeti 2.8, the ``cmdlib.py`` file was split into a series of files contained in the ``cmdlib`` directory. If Ganeti is installed from sources and not from a package, while downgrading Ganeti to a pre-2.8 version it is important to remember to remove the ``cmdlib`` directory from the directory containing the Ganeti python files (which usually is ``${PREFIX}/lib/python${VERSION}/dist-packages/ganeti``). A simpler upgrade/downgrade procedure will be made available in future versions of Ganeti. #. Restart daemons on all nodes:: $ /etc/init.d/ganeti restart #. Re-distribute configuration (master node only):: $ gnt-cluster redist-conf #. Restart daemons again on all nodes:: $ /etc/init.d/ganeti restart #. Enable the watcher again (master node only):: $ gnt-cluster watcher continue #. Verify cluster (master node only):: $ gnt-cluster verify Specific tasks for 2.11 to 2.10 downgrade ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, After running ``cfgupgrade``, the ``client.pem`` and ``ssconf_master_candidates_certs`` files need to be removed from Ganeti's data directory on all nodes. While this step is not necessary for 2.10 to run cleanly, leaving them will cause problems when upgrading again after the downgrade. 2.0 releases ------------ 2.0.3 to 2.0.4 ~~~~~~~~~~~~~~ No changes needed except restarting the daemon; but rollback to 2.0.3 might require configuration editing. If you're using Xen-HVM instances, please double-check the network configuration (``nic_type`` parameter) as the defaults might have changed: 2.0.4 adds any missing configuration items and depending on the version of the software the cluster has been installed with, some new keys might have been added. 2.0.1 to 2.0.2/2.0.3 ~~~~~~~~~~~~~~~~~~~~ Between 2.0.1 and 2.0.2 there have been some changes in the handling of block devices, which can cause some issues. 2.0.3 was then released which adds two new options/commands to fix this issue. If you use DRBD-type instances and see problems in instance start or activate-disks with messages from DRBD about "lower device too small" or similar, it is recoomended to: #. Run ``gnt-instance activate-disks --ignore-size $instance`` for each of the affected instances #. Then run ``gnt-cluster repair-disk-sizes`` which will check that instances have the correct disk sizes 1.2 to 2.0 ---------- Prerequisites: - Ganeti 1.2.7 is currently installed - All instances have been migrated from DRBD 0.7 to DRBD 8.x (i.e. no ``remote_raid1`` disk template) - Upgrade to Ganeti 2.0.0~rc2 or later (~rc1 and earlier don't have the needed upgrade tool) In the below steps, replace :file:`/var/lib` with ``$libdir`` if Ganeti was not installed with this prefix (e.g. :file:`/usr/local/var`). Same for :file:`/usr/lib`. Execution (all steps are required in the order given): #. Make a backup of the current configuration, for safety:: $ cp -a /var/lib/ganeti /var/lib/ganeti-1.2.backup #. Stop all instances:: $ gnt-instance stop --all #. Make sure no DRBD device are in use, the following command should show no active minors:: $ gnt-cluster command grep cs: /proc/drbd | grep -v cs:Unconf #. Stop the node daemons and rapi daemon on all nodes (note: should be logged in not via the cluster name, but the master node name, as the command below will remove the cluster ip from the master node):: $ gnt-cluster command /etc/init.d/ganeti stop #. Install the new software on all nodes, either from packaging (if available) or from sources; the master daemon will not start but give error messages about wrong configuration file, which is normal #. Upgrade the configuration file:: $ /usr/lib/ganeti/tools/cfgupgrade12 -v --dry-run $ /usr/lib/ganeti/tools/cfgupgrade12 -v #. Make sure ``ganeti-noded`` is running on all nodes (and start it if not) #. Start the master daemon:: $ ganeti-masterd #. Check that a simple node-list works:: $ gnt-node list #. Redistribute updated configuration to all nodes:: $ gnt-cluster redist-conf $ gnt-cluster copyfile /var/lib/ganeti/known_hosts #. Optional: if needed, install RAPI-specific certificates under :file:`/var/lib/ganeti/rapi.pem` and run:: $ gnt-cluster copyfile /var/lib/ganeti/rapi.pem #. Run a cluster verify, this should show no problems:: $ gnt-cluster verify #. Remove some obsolete files:: $ gnt-cluster command rm /var/lib/ganeti/ssconf_node_pass $ gnt-cluster command rm /var/lib/ganeti/ssconf_hypervisor #. Update the xen pvm (if this was a pvm cluster) setting for 1.2 compatibility:: $ gnt-cluster modify -H xen-pvm:root_path=/dev/sda #. Depending on your setup, you might also want to reset the initrd parameter:: $ gnt-cluster modify -H xen-pvm:initrd_path=/boot/initrd-2.6-xenU #. Reset the instance autobalance setting to default:: $ for i in $(gnt-instance list -o name --no-headers); do \ gnt-instance modify -B auto_balance=default $i; \ done #. Optional: start the RAPI demon:: $ ganeti-rapi #. Restart instances:: $ gnt-instance start --force-multiple --all At this point, ``gnt-cluster verify`` should show no errors and the migration is complete. 1.2 releases ------------ 1.2.4 to any other higher 1.2 version ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ No changes needed. Rollback will usually require manual edit of the configuration file. 1.2.3 to 1.2.4 ~~~~~~~~~~~~~~ No changes needed. Note that going back from 1.2.4 to 1.2.3 will require manual edit of the configuration file (since we added some HVM-related new attributes). 1.2.2 to 1.2.3 ~~~~~~~~~~~~~~ No changes needed. Note that the drbd7-to-8 upgrade tool does a disk format change for the DRBD metadata, so in theory this might be **risky**. It is advised to have (good) backups before doing the upgrade. 1.2.1 to 1.2.2 ~~~~~~~~~~~~~~ No changes needed. 1.2.0 to 1.2.1 ~~~~~~~~~~~~~~ No changes needed. Only some bugfixes and new additions that don't affect existing clusters. 1.2.0 beta 3 to 1.2.0 ~~~~~~~~~~~~~~~~~~~~~ No changes needed. 1.2.0 beta 2 to beta 3 ~~~~~~~~~~~~~~~~~~~~~~ No changes needed. A new version of the debian-etch-instance OS (0.3) has been released, but upgrading it is not required. 1.2.0 beta 1 to beta 2 ~~~~~~~~~~~~~~~~~~~~~~ Beta 2 switched the config file format to JSON. Steps to upgrade: #. Stop the daemons (``/etc/init.d/ganeti stop``) on all nodes #. Disable the cron job (default is :file:`/etc/cron.d/ganeti`) #. Install the new version #. Make a backup copy of the config file #. Upgrade the config file using the following command:: $ /usr/share/ganeti/cfgupgrade --verbose /var/lib/ganeti/config.data #. Start the daemons and run ``gnt-cluster info``, ``gnt-node list`` and ``gnt-instance list`` to check if the upgrade process finished successfully The OS definition also need to be upgraded. There is a new version of the debian-etch-instance OS (0.2) that goes along with beta 2. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/app/000075500000000000000000000000001476477700300143645ustar00rootroot00000000000000ganeti-3.1.0~rc2/app/ganeti-confd.hs000064400000000000000000000034701476477700300172620ustar00rootroot00000000000000{-| Ganeti configuration query daemon -} {- Copyright (C) 2009, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Main (main) where import qualified Ganeti.Confd.Server import Ganeti.Daemon import Ganeti.Runtime import qualified Ganeti.Constants as C -- | Options list and functions. options :: [OptType] options = [ oNoDaemonize , oNoUserChecks , oDebug , oPort C.defaultConfdPort , oBindAddress , oSyslogUsage ] -- | Main function. main :: IO () main = genericMain GanetiConfd options Ganeti.Confd.Server.checkMain Ganeti.Confd.Server.prepMain Ganeti.Confd.Server.main ganeti-3.1.0~rc2/app/ganeti-kvmd.hs000064400000000000000000000034161476477700300171320ustar00rootroot00000000000000{-| KVM daemon main -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} import Ganeti.Daemon (OptType) import qualified Ganeti.Daemon as Daemon import qualified Ganeti.Kvmd as Kvmd (start) import Ganeti.Runtime (GanetiDaemon(..)) -- | Options list and functions. options :: [OptType] options = [ Daemon.oNoDaemonize , Daemon.oNoUserChecks , Daemon.oDebug , Daemon.oSyslogUsage ] -- | Main function. main :: IO () main = Daemon.genericMain GanetiKvmd options (\_ -> return . Right $ ()) (\_ _ -> return ()) (\_ _ _ -> Kvmd.start) ganeti-3.1.0~rc2/app/ganeti-luxid.hs000064400000000000000000000033661476477700300173220ustar00rootroot00000000000000{-| Ganeti query daemon -} {- Copyright (C) 2009, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Main (main) where import qualified Ganeti.Query.Server import Ganeti.Daemon import Ganeti.Runtime -- | Options list and functions. options :: [OptType] options = [ oNoDaemonize , oNoUserChecks , oDebug , oSyslogUsage , oNoVoting , oYesDoIt ] -- | Main function. main :: IO () main = genericMain GanetiLuxid options Ganeti.Query.Server.checkMain Ganeti.Query.Server.prepMain Ganeti.Query.Server.main ganeti-3.1.0~rc2/app/ganeti-metad.hs000064400000000000000000000035461476477700300172670ustar00rootroot00000000000000{-| Metadata daemon. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Main (main) where import qualified Ganeti.Constants as Constants import Ganeti.Daemon (OptType) import qualified Ganeti.Daemon as Daemon import qualified Ganeti.Metad.Server as Server import qualified Ganeti.Runtime as Runtime options :: [OptType] options = [ Daemon.oBindAddress , Daemon.oDebug , Daemon.oNoDaemonize , Daemon.oNoUserChecks , Daemon.oPort Constants.defaultMetadPort ] main :: IO () main = Daemon.genericMain Runtime.GanetiMetad options (\_ -> return . Right $ ()) (\_ _ -> return ()) (\opts _ _ -> Server.start opts) ganeti-3.1.0~rc2/app/ganeti-mond.hs000064400000000000000000000044061476477700300171260ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Ganeti monitoring agent daemon -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Main (main) where import Data.List ((\\)) import Ganeti.Daemon import Ganeti.DataCollectors (collectors) import Ganeti.DataCollectors.Types (dName) import Ganeti.Runtime import qualified Ganeti.Monitoring.Server as S import qualified Ganeti.Constants as C import qualified Ganeti.ConstantUtils as CU -- Check constistency of defined data collectors and their names used for the -- Python constant generation: $(let names = map dName collectors missing = names \\ CU.toList C.dataCollectorNames in if null missing then return [] else fail $ "Please add " ++ show missing ++ " to the Ganeti.Constants.dataCollectorNames.") -- | Options list and functions. options :: [OptType] options = [ oNoDaemonize , oNoUserChecks , oDebug , oBindAddress , oPort C.defaultMondPort ] -- | Main function. main :: IO () main = genericMain GanetiMond options S.checkMain S.prepMain S.main ganeti-3.1.0~rc2/app/ganeti-wconfd.hs000064400000000000000000000031641476477700300174510ustar00rootroot00000000000000{-| Ganeti WConfD (config writer) daemon -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Main (main) where import qualified Ganeti.WConfd.Server import Ganeti.Daemon import Ganeti.Runtime -- | Main function. main :: IO () main = genericMain GanetiWConfd Ganeti.WConfd.Server.options Ganeti.WConfd.Server.checkMain Ganeti.WConfd.Server.prepMain Ganeti.WConfd.Server.main ganeti-3.1.0~rc2/app/hs2py.hs000064400000000000000000000044001476477700300157630ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Haskell to Python opcode generation program. -} {- Copyright (C) 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} import Ganeti.Hs2Py.GenOpCodes import Ganeti.Hs2Py.ListConstants import Ganeti.THH.PyRPC import qualified Ganeti.WConfd.Core as WConfd import qualified Ganeti.Metad.ConfigCore as Metad import System.Environment (getArgs) import System.Exit (exitFailure) import System.IO (hPutStrLn, stderr) main :: IO () main = do args <- getArgs case args of ["--opcodes"] -> putStrLn showPyClasses ["--constants"] -> putConstants ["--wconfd-rpc"] -> putStrLn $ $( genPyUDSRpcStubStr "ClientRpcStub" "WCONFD_SOCKET" WConfd.exportedFunctions ) ["--metad-rpc"] -> putStrLn $ $( genPyUDSRpcStubStr "ClientRpcStub" "METAD_SOCKET" Metad.exportedFunctions ) _ -> do hPutStrLn stderr "Usage: hs2py --opcodes\ \| --constants\ \| --wconfd-rpc" exitFailure ganeti-3.1.0~rc2/app/htools.hs000064400000000000000000000025701476477700300162340ustar00rootroot00000000000000{-| Main htools binary. -} {- Copyright (C) 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Main (main) where import Ganeti.HTools.Program.Main (main) ganeti-3.1.0~rc2/app/mon-collector.hs000064400000000000000000000031361476477700300175000ustar00rootroot00000000000000{-| Main binary for all stand-alone data collectors -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Main (main) where import Ganeti.Common import Ganeti.DataCollectors.CLI (genericOptions, defaultOptions) import Ganeti.DataCollectors.Program (personalities) -- | Simple main function. main :: IO () main = genericMainCmds defaultOptions personalities genericOptions ganeti-3.1.0~rc2/app/rpc-test.hs000064400000000000000000000223041476477700300164620ustar00rootroot00000000000000{-# LANGUAGE BangPatterns #-} {-| RPC test program. -} {- Copyright (C) 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} import Control.Concurrent import Control.Monad import System.Console.GetOpt import System.Environment import System.IO import Text.JSON (decode) import Text.Printf import Ganeti.BasicTypes import Ganeti.Common import Ganeti.Config import Ganeti.Errors import Ganeti.JSON import Ganeti.Objects import qualified Ganeti.Path as P import Ganeti.Rpc import Ganeti.Utils import Ganeti.Utils.Time (getCurrentTimeUSec) -- | Command line options structure. data Options = Options { optRpc :: String -- ^ RPC to execute , optDataFile :: FilePath -- ^ Path to the RPC serialised form , optVerbose :: Bool -- ^ Verbosity level , optStats :: Bool -- ^ Whether to show timing stats , optCount :: Int -- ^ Count of (multi) RPCs to do , optBatch :: Int -- ^ How many (multi) RPCs to run in parallel , optShowHelp :: Bool -- ^ Just show the help , optShowComp :: Bool -- ^ Just show the completion info , optShowVer :: Bool -- ^ Just show the program version } deriving Show -- | Default values for the command line options. defaultOptions :: Options defaultOptions = Options { optRpc = "version" , optDataFile = "rpc.json" , optVerbose = False , optStats = False , optCount = 1 , optBatch = 1 , optShowHelp = False , optShowComp = False , optShowVer = False } instance StandardOptions Options where helpRequested = optShowHelp verRequested = optShowVer compRequested = optShowComp requestHelp o = o { optShowHelp = True } requestVer o = o { optShowVer = True } requestComp o = o { optShowComp = True } -- | The rpcs we support. Sadly this duplicates the RPC list. data KnownRpc = KRInstanceInfo RpcCallInstanceInfo | KRAllInstancesInfo RpcCallAllInstancesInfo | KRInstanceConsoleInfo RpcCallInstanceConsoleInfo | KRInstanceList RpcCallInstanceList | KRNodeInfo RpcCallNodeInfo | KRVersion RpcCallVersion | KRStorageList RpcCallStorageList | KRTestDelay RpcCallTestDelay | KRExportList RpcCallExportList deriving (Show) -- | The command line options. options :: [GenericOptType Options] options = [ (Option "r" ["rpc"] (ReqArg (\ r o -> Ok o { optRpc = r }) "RPC") "the rpc to use [version]", OptComplChoices []) , (Option "f" ["data-file"] (ReqArg (\ f o -> Ok o { optDataFile = f }) "FILE") "the rpc serialised form [\"rpc.json\"]", OptComplFile) , (Option "v" ["verbose"] (NoArg (\ opts -> Ok opts { optVerbose = True})) "show more information when executing RPCs", OptComplNone) , (Option "t" ["stats"] (NoArg (\ opts -> Ok opts { optStats = True})) "show timing information summary", OptComplNone) , (Option "c" ["count"] (reqWithConversion (tryRead "reading count") (\count opts -> Ok opts { optCount = count }) "NUMBER") "Count of (multi) RPCs to execute [1]", OptComplInteger) , (Option "b" ["batch"] (reqWithConversion (tryRead "reading batch size") (\batch opts -> Ok opts { optBatch = batch }) "NUMBER") "Parallelisation factor for RPCs [1]", OptComplInteger) , oShowHelp , oShowComp , oShowVer ] -- | Arguments we expect arguments :: [ArgCompletion] arguments = [ArgCompletion OptComplOneNode 1 Nothing] -- | Log a message. logMsg :: MVar () -> String -> IO () logMsg outmvar text = withMVar outmvar $ \_ -> do let p = if null text || last text /= '\n' then putStrLn else putStr p text hFlush stdout -- | Parses a RPC. parseRpc :: String -> String -> Result KnownRpc parseRpc "instance_info" f = fromJResult "parsing rpc" (decode f) >>= Ok . KRInstanceInfo parseRpc "all_instances_info" f = fromJResult "parsing rpc" (decode f) >>= Ok . KRAllInstancesInfo parseRpc "console_instance_info" f = fromJResult "parsing rpc" (decode f) >>= Ok . KRInstanceConsoleInfo parseRpc "instance_list" f = fromJResult "parsing rpc" (decode f) >>= Ok . KRInstanceList parseRpc "node_info" f = fromJResult "parsing rpc" (decode f) >>= Ok . KRNodeInfo parseRpc "version" f = fromJResult "parsing rpc" (decode f) >>= Ok . KRVersion parseRpc "storage_list" f = fromJResult "parsing rpc" (decode f) >>= Ok . KRStorageList parseRpc "test_delay" f = fromJResult "parsing rpc" (decode f) >>= Ok . KRTestDelay parseRpc "export_list" f = fromJResult "parsing rpc" (decode f) >>= Ok . KRExportList parseRpc s _ = Bad $ "Unknown rpc '" ++ s ++ "'" -- | Executes a RPC. These duplicate definitions are needed due to the -- polymorphism of 'executeRpcCall', and the binding of the result -- based on the input rpc call. execRpc :: [Node] -> KnownRpc -> IO [[String]] execRpc n (KRInstanceInfo v) = formatRpcRes `fmap` executeRpcCall n v execRpc n (KRAllInstancesInfo v) = formatRpcRes `fmap` executeRpcCall n v execRpc n (KRInstanceConsoleInfo v) = formatRpcRes `fmap` executeRpcCall n v execRpc n (KRInstanceList v) = formatRpcRes `fmap` executeRpcCall n v execRpc n (KRNodeInfo v) = formatRpcRes `fmap` executeRpcCall n v execRpc n (KRVersion v) = formatRpcRes `fmap` executeRpcCall n v execRpc n (KRStorageList v) = formatRpcRes `fmap` executeRpcCall n v execRpc n (KRTestDelay v) = formatRpcRes `fmap` executeRpcCall n v execRpc n (KRExportList v) = formatRpcRes `fmap` executeRpcCall n v -- | Helper to format the RPC result such that it can be printed by -- 'printTable'. formatRpcRes :: (Show b) => [(Node, ERpcError b)] -> [[String]] formatRpcRes = map (\(n, r) -> [nodeName n, either explainRpcError show r]) -- | Main function. main :: IO () main = do cmd_args <- getArgs (opts, args) <- parseOpts defaultOptions cmd_args "rpc-test" options arguments rpc <- parseRpc (optRpc opts) `liftM` readFile (optDataFile opts) >>= exitIfBad "parsing RPC" cfg_file <- P.clusterConfFile cfg <- loadConfig cfg_file>>= exitIfBad "Can't load configuration" nodes <- exitIfBad "Can't find node" . errToResult $ mapM (getNode cfg) args token <- newEmptyMVar -- semaphore for batch calls outmvar <- newMVar () -- token for stdout non-interleaving let logger = if optVerbose opts then logMsg outmvar else const $ return () let batch = [1..optBatch opts] count = optCount opts rpcs = count * length nodes logger $ printf "Will execute %s multi-ops and %s RPCs" (show count) (show rpcs) tstart <- getCurrentTimeUSec _ <- forkIO $ mapM_ (\_ -> putMVar token ()) batch mapM_ (\idx -> do let str_idx = show idx logger $ "Acquiring token for run " ++ str_idx _ <- takeMVar token forkIO $ do start <- getCurrentTimeUSec logger $ "Start run " ++ str_idx !results <- execRpc nodes rpc stop <- getCurrentTimeUSec let delta = (fromIntegral (stop - start)::Double) / 1000 putMVar token () let stats = if optVerbose opts then printf "Done run %d in %7.3fmsec\n" idx delta else "" table = printTable "" ["Node", "Result"] results [False, False] logMsg outmvar $ stats ++ table ) [1..count] mapM_ (\_ -> takeMVar token) batch _ <- takeMVar outmvar when (optStats opts) $ do tstop <- getCurrentTimeUSec let delta = (fromIntegral (tstop - tstart) / 1000000)::Double printf "Total runtime: %9.3fs\n" delta :: IO () printf "Total mult-ops: %9d\n" count :: IO () printf "Total single RPCs: %9d\n" rpcs :: IO () printf "Multi-ops/sec: %9.3f\n" (fromIntegral count / delta) :: IO () printf "RPCs/sec: %9.3f\n" (fromIntegral rpcs / delta) :: IO () ganeti-3.1.0~rc2/autogen.sh000075500000000000000000000004331476477700300156050ustar00rootroot00000000000000#!/bin/sh if test ! -f configure.ac ; then echo "You must execute this script from the top level directory." exit 1 fi set -e rm -rf config.cache autom4te.cache ${ACLOCAL:-aclocal} -I autotools ${AUTOCONF:-autoconf} ${AUTOMAKE:-automake} --add-missing rm -rf autom4te.cache ganeti-3.1.0~rc2/autotools/000075500000000000000000000000001476477700300156355ustar00rootroot00000000000000ganeti-3.1.0~rc2/autotools/ac_ghc_pkg.m4000064400000000000000000000052111476477700300201430ustar00rootroot00000000000000##### # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ##### # # SYNOPSIS # # AC_GHC_PKG_CHECK(modname, action_found, action_not_found, extended) # # DESCRIPTION # # Checks for a Haskell (GHC) module. If found, execute the second # argument, if not found, the third one. # # If the fourth argument is non-empty, then the check will be some # via 'ghc-pkg list' (which supports patterns), otherwise it will # use just 'ghc-pkg latest'. # # ##### AC_DEFUN([AC_GHC_PKG_CHECK],[ if test -z $GHC_PKG; then AC_MSG_ERROR([GHC_PKG not defined]) fi AC_MSG_CHECKING([haskell library $1]) if test -n "$4"; then GHC_PKG_RESULT=$($GHC_PKG --simple-output list '$1'|tail -n1) else GHC_PKG_RESULT=$($GHC_PKG latest '$1' 2>/dev/null) fi if test -n "$GHC_PKG_RESULT"; then AC_MSG_RESULT($GHC_PKG_RESULT) $2 else AC_MSG_RESULT([no]) $3 fi ]) ##### # # SYNOPSIS # # AC_GHC_PKG_REQUIRE(modname, extended) # # DESCRIPTION # # Checks for a Haskell (GHC) module, and abort if not found. If the # second argument is non-empty, then the check will be some via # 'ghc-pkg list' (which supports patterns), otherwise it will use # just 'ghc-pkg latest'. # # ##### AC_DEFUN([AC_GHC_PKG_REQUIRE],[ AC_GHC_PKG_CHECK($1, [], [AC_MSG_FAILURE([Required Haskell module $1 not found])], $2) ]) ganeti-3.1.0~rc2/autotools/ac_python_module.m4000064400000000000000000000017511476477700300214340ustar00rootroot00000000000000##### http://autoconf-archive.cryp.to/ac_python_module.html # # SYNOPSIS # # AC_PYTHON_MODULE(modname[, fatal]) # # DESCRIPTION # # Checks for Python module. # # If fatal is non-empty then absence of a module will trigger an # error. # # LAST MODIFICATION # # 2007-01-09 # # COPYLEFT # # Copyright (c) 2007 Andrew Collier # # Copying and distribution of this file, with or without # modification, are permitted in any medium without royalty provided # the copyright notice and this notice are preserved. AC_DEFUN([AC_PYTHON_MODULE],[ if test -z $PYTHON; then PYTHON="python3" fi PYTHON_NAME=`basename $PYTHON` AC_MSG_CHECKING($PYTHON_NAME module: $1) $PYTHON -c "import $1" 2>/dev/null if test $? -eq 0; then AC_MSG_RESULT(yes) eval AS_TR_CPP(HAVE_PYMOD_$1)=yes else AC_MSG_RESULT(no) eval AS_TR_CPP(HAVE_PYMOD_$1)=no # if test -n "$2" then AC_MSG_ERROR(failed to find required module $1) exit 1 fi fi ]) ganeti-3.1.0~rc2/autotools/build-bash-completion000075500000000000000000000663751476477700300217650ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2009, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to generate bash_completion script for Ganeti. """ # pylint: disable=C0103 # [C0103] Invalid name build-bash-completion import os import os.path import re import itertools import optparse from io import StringIO # _constants shouldn't be imported from anywhere except constants.py, but we're # making an exception here because this script is only used at build time. from ganeti import _constants from ganeti import constants from ganeti import cli from ganeti import utils from ganeti import build from ganeti import pathutils from ganeti.tools import burnin #: Regular expression describing desired format of option names. Long names can #: contain lowercase characters, numbers and dashes only. _OPT_NAME_RE = re.compile(r"^-[a-zA-Z0-9]|--[a-z][-a-z0-9]+$") def _WriteGntLog(sw, support_debug): if support_debug: sw.Write("_gnt_log() {") sw.IncIndent() try: sw.Write("if [[ -n \"$GANETI_COMPL_LOG\" ]]; then") sw.IncIndent() try: sw.Write("{") sw.IncIndent() try: sw.Write("echo ---") sw.Write("echo \"$@\"") sw.Write("echo") finally: sw.DecIndent() sw.Write("} >> $GANETI_COMPL_LOG") finally: sw.DecIndent() sw.Write("fi") finally: sw.DecIndent() sw.Write("}") def _WriteNodes(sw): sw.Write("_ganeti_nodes() {") sw.IncIndent() try: node_list_path = os.path.join(pathutils.DATA_DIR, "ssconf_node_list") sw.Write("cat %s 2>/dev/null || :", utils.ShellQuote(node_list_path)) finally: sw.DecIndent() sw.Write("}") def _WriteInstances(sw): sw.Write("_ganeti_instances() {") sw.IncIndent() try: instance_list_path = os.path.join(pathutils.DATA_DIR, "ssconf_instance_list") sw.Write("cat %s 2>/dev/null || :", utils.ShellQuote(instance_list_path)) finally: sw.DecIndent() sw.Write("}") def _WriteJobs(sw): sw.Write("_ganeti_jobs() {") sw.IncIndent() try: # FIXME: this is really going into the internals of the job queue sw.Write(("local jlist=($( shopt -s nullglob &&" " cd %s 2>/dev/null && echo job-* || : ))"), utils.ShellQuote(pathutils.QUEUE_DIR)) sw.Write('echo "${jlist[@]/job-/}"') finally: sw.DecIndent() sw.Write("}") def _WriteOSAndIAllocator(sw): for (fnname, paths) in [ ("os", pathutils.OS_SEARCH_PATH), ("iallocator", constants.IALLOCATOR_SEARCH_PATH), ]: sw.Write("_ganeti_%s() {", fnname) sw.IncIndent() try: # FIXME: Make querying the master for all OSes cheap for path in paths: sw.Write("( shopt -s nullglob && cd %s 2>/dev/null && echo * || : )", utils.ShellQuote(path)) finally: sw.DecIndent() sw.Write("}") def _WriteNodegroup(sw): sw.Write("_ganeti_nodegroup() {") sw.IncIndent() try: nodegroups_path = os.path.join(pathutils.DATA_DIR, "ssconf_nodegroups") sw.Write("cat %s 2>/dev/null || :", utils.ShellQuote(nodegroups_path)) finally: sw.DecIndent() sw.Write("}") def _WriteNetwork(sw): sw.Write("_ganeti_network() {") sw.IncIndent() try: networks_path = os.path.join(pathutils.DATA_DIR, "ssconf_networks") sw.Write("cat %s 2>/dev/null || :", utils.ShellQuote(networks_path)) finally: sw.DecIndent() sw.Write("}") def _WriteFindFirstArg(sw): # Params: # Result variable: $first_arg_idx sw.Write("_ganeti_find_first_arg() {") sw.IncIndent() try: sw.Write("local w i") sw.Write("first_arg_idx=") sw.Write("for (( i=$1; i < COMP_CWORD; ++i )); do") sw.IncIndent() try: sw.Write("w=${COMP_WORDS[$i]}") # Skip option value sw.Write("""if [[ -n "$2" && "$w" == @($2) ]]; then let ++i""") # Skip sw.Write("""elif [[ -n "$3" && "$w" == @($3) ]]; then :""") # Ah, we found the first argument sw.Write("else first_arg_idx=$i; break;") sw.Write("fi") finally: sw.DecIndent() sw.Write("done") finally: sw.DecIndent() sw.Write("}") def _WriteListOptions(sw): # Params: # Input variable: $first_arg_idx # Result variables: $arg_idx, $choices sw.Write("_ganeti_list_options() {") sw.IncIndent() try: sw.Write("""if [[ -z "$first_arg_idx" ]]; then""") sw.IncIndent() try: sw.Write("arg_idx=0") # Show options only if the current word starts with a dash sw.Write("""if [[ "$cur" == -* ]]; then""") sw.IncIndent() try: sw.Write("choices=$1") finally: sw.DecIndent() sw.Write("fi") sw.Write("return") finally: sw.DecIndent() sw.Write("fi") # Calculate position of current argument sw.Write("arg_idx=$(( COMP_CWORD - first_arg_idx ))") sw.Write("choices=") finally: sw.DecIndent() sw.Write("}") def _WriteGntCheckopt(sw, support_debug): # Params: # Result variable: $optcur sw.Write("_gnt_checkopt() {") sw.IncIndent() try: sw.Write("""if [[ -n "$1" && "$cur" == @($1) ]]; then""") sw.IncIndent() try: sw.Write("optcur=\"${cur#--*=}\"") sw.Write("return 0") finally: sw.DecIndent() sw.Write("""elif [[ -n "$2" && "$prev" == @($2) ]]; then""") sw.IncIndent() try: sw.Write("optcur=\"$cur\"") sw.Write("return 0") finally: sw.DecIndent() sw.Write("fi") if support_debug: sw.Write("_gnt_log optcur=\"'$optcur'\"") sw.Write("return 1") finally: sw.DecIndent() sw.Write("}") def _WriteGntCompgen(sw, support_debug): # Params: # Result variable: $COMPREPLY sw.Write("_gnt_compgen() {") sw.IncIndent() try: sw.Write("""COMPREPLY=( $(compgen "$@") )""") if support_debug: sw.Write("_gnt_log COMPREPLY=\"${COMPREPLY[@]}\"") finally: sw.DecIndent() sw.Write("}") def WritePreamble(sw, support_debug): """Writes the script preamble. Helper functions should be written here. """ sw.Write("# This script is automatically generated at build time.") sw.Write("# Do not modify manually.") _WriteGntLog(sw, support_debug) _WriteNodes(sw) _WriteInstances(sw) _WriteJobs(sw) _WriteOSAndIAllocator(sw) _WriteNodegroup(sw) _WriteNetwork(sw) _WriteFindFirstArg(sw) _WriteListOptions(sw) _WriteGntCheckopt(sw, support_debug) _WriteGntCompgen(sw, support_debug) def WriteCompReply(sw, args, cur="\"$cur\""): sw.Write("_gnt_compgen %s -- %s", args, cur) sw.Write("return") class CompletionWriter(object): """Command completion writer class. """ def __init__(self, arg_offset, opts, args, support_debug): self.arg_offset = arg_offset self.opts = opts self.args = args self.support_debug = support_debug for opt in opts: # While documented, these variables aren't seen as public attributes by # pylint. pylint: disable=W0212 opt.all_names = sorted(opt._short_opts + opt._long_opts) invalid = list(itertools.filterfalse(_OPT_NAME_RE.match, opt.all_names)) if invalid: raise Exception("Option names don't match regular expression '%s': %s" % (_OPT_NAME_RE.pattern, utils.CommaJoin(invalid))) def _FindFirstArgument(self, sw): ignore = [] skip_one = [] for opt in self.opts: if opt.takes_value(): # Ignore value for i in opt.all_names: if i.startswith("--"): ignore.append("%s=*" % utils.ShellQuote(i)) skip_one.append(utils.ShellQuote(i)) else: ignore.extend([utils.ShellQuote(i) for i in opt.all_names]) ignore = sorted(utils.UniqueSequence(ignore)) skip_one = sorted(utils.UniqueSequence(skip_one)) if ignore or skip_one: # Try to locate first argument sw.Write("_ganeti_find_first_arg %s %s %s", self.arg_offset + 1, utils.ShellQuote("|".join(skip_one)), utils.ShellQuote("|".join(ignore))) else: # When there are no options the first argument is always at position # offset + 1 sw.Write("first_arg_idx=%s", self.arg_offset + 1) def _CompleteOptionValues(self, sw): # Group by values # "values" -> [optname1, optname2, ...] values = {} for opt in self.opts: if not opt.takes_value(): continue # Only static choices implemented so far (e.g. no node list) suggest = getattr(opt, "completion_suggest", None) # our custom option type if opt.type == "bool": suggest = ["yes", "no"] if not suggest: suggest = opt.choices if (isinstance(suggest, int) and suggest in cli.OPT_COMPL_ALL): key = suggest elif suggest: key = " ".join(sorted(suggest)) else: key = "" values.setdefault(key, []).extend(opt.all_names) # Don't write any code if there are no option values if not values: return cur = "\"$optcur\"" wrote_opt = False for (suggest, allnames) in values.items(): longnames = [i for i in allnames if i.startswith("--")] if wrote_opt: condcmd = "elif" else: condcmd = "if" sw.Write("%s _gnt_checkopt %s %s; then", condcmd, utils.ShellQuote("|".join(["%s=*" % i for i in longnames])), utils.ShellQuote("|".join(allnames))) sw.IncIndent() try: if suggest == cli.OPT_COMPL_MANY_NODES: # TODO: Implement comma-separated values WriteCompReply(sw, "-W ''", cur=cur) elif suggest == cli.OPT_COMPL_ONE_NODE: WriteCompReply(sw, "-W \"$(_ganeti_nodes)\"", cur=cur) elif suggest == cli.OPT_COMPL_ONE_INSTANCE: WriteCompReply(sw, "-W \"$(_ganeti_instances)\"", cur=cur) elif suggest == cli.OPT_COMPL_ONE_OS: WriteCompReply(sw, "-W \"$(_ganeti_os)\"", cur=cur) elif suggest == cli.OPT_COMPL_ONE_EXTSTORAGE: WriteCompReply(sw, "-W \"$(_ganeti_extstorage)\"", cur=cur) elif suggest == cli.OPT_COMPL_ONE_FILTER: WriteCompReply(sw, "-W \"$(_ganeti_filter)\"", cur=cur) elif suggest == cli.OPT_COMPL_ONE_IALLOCATOR: WriteCompReply(sw, "-W \"$(_ganeti_iallocator)\"", cur=cur) elif suggest == cli.OPT_COMPL_ONE_NODEGROUP: WriteCompReply(sw, "-W \"$(_ganeti_nodegroup)\"", cur=cur) elif suggest == cli.OPT_COMPL_ONE_NETWORK: WriteCompReply(sw, "-W \"$(_ganeti_network)\"", cur=cur) elif suggest == cli.OPT_COMPL_INST_ADD_NODES: sw.Write("local tmp= node1= pfx= curvalue=\"${optcur#*:}\"") sw.Write("if [[ \"$optcur\" == *:* ]]; then") sw.IncIndent() try: sw.Write("node1=\"${optcur%%:*}\"") sw.Write("if [[ \"$COMP_WORDBREAKS\" != *:* ]]; then") sw.IncIndent() try: sw.Write("pfx=\"$node1:\"") finally: sw.DecIndent() sw.Write("fi") finally: sw.DecIndent() sw.Write("fi") if self.support_debug: sw.Write("_gnt_log pfx=\"'$pfx'\" curvalue=\"'$curvalue'\"" " node1=\"'$node1'\"") sw.Write("for i in $(_ganeti_nodes); do") sw.IncIndent() try: sw.Write("if [[ -z \"$node1\" ]]; then") sw.IncIndent() try: sw.Write("tmp=\"$tmp $i $i:\"") finally: sw.DecIndent() sw.Write("elif [[ \"$i\" != \"$node1\" ]]; then") sw.IncIndent() try: sw.Write("tmp=\"$tmp $i\"") finally: sw.DecIndent() sw.Write("fi") finally: sw.DecIndent() sw.Write("done") WriteCompReply(sw, "-P \"$pfx\" -W \"$tmp\"", cur="\"$curvalue\"") else: WriteCompReply(sw, "-W %s" % utils.ShellQuote(suggest), cur=cur) finally: sw.DecIndent() wrote_opt = True if wrote_opt: sw.Write("fi") return def _CompleteArguments(self, sw): if not (self.opts or self.args): return all_option_names = [] for opt in self.opts: all_option_names.extend(opt.all_names) all_option_names.sort() # List options if no argument has been specified yet sw.Write("_ganeti_list_options %s", utils.ShellQuote(" ".join(all_option_names))) if self.args: last_idx = len(self.args) - 1 last_arg_end = 0 varlen_arg_idx = None wrote_arg = False sw.Write("compgenargs=") for idx, arg in enumerate(self.args): assert arg.min is not None and arg.min >= 0 assert not (idx < last_idx and arg.max is None) if arg.min != arg.max or arg.max is None: if varlen_arg_idx is not None: raise Exception("Only one argument can have a variable length") varlen_arg_idx = idx compgenargs = [] if isinstance(arg, cli.ArgUnknown): choices = "" elif isinstance(arg, cli.ArgSuggest): choices = utils.ShellQuote(" ".join(arg.choices)) elif isinstance(arg, cli.ArgInstance): choices = "$(_ganeti_instances)" elif isinstance(arg, cli.ArgNode): choices = "$(_ganeti_nodes)" elif isinstance(arg, cli.ArgGroup): choices = "$(_ganeti_nodegroup)" elif isinstance(arg, cli.ArgNetwork): choices = "$(_ganeti_network)" elif isinstance(arg, cli.ArgJobId): choices = "$(_ganeti_jobs)" elif isinstance(arg, cli.ArgOs): choices = "$(_ganeti_os)" elif isinstance(arg, cli.ArgExtStorage): choices = "$(_ganeti_extstorage)" elif isinstance(arg, cli.ArgFilter): choices = "$(_ganeti_filter)" elif isinstance(arg, cli.ArgFile): choices = "" compgenargs.append("-f") elif isinstance(arg, cli.ArgCommand): choices = "" compgenargs.append("-c") elif isinstance(arg, cli.ArgHost): choices = "" compgenargs.append("-A hostname") else: raise Exception("Unknown argument type %r" % arg) if arg.min == 1 and arg.max == 1: cmpcode = """"$arg_idx" == %d""" % (last_arg_end) elif arg.max is None: cmpcode = """"$arg_idx" -ge %d""" % (last_arg_end) elif arg.min <= arg.max: cmpcode = (""""$arg_idx" -ge %d && "$arg_idx" -lt %d""" % (last_arg_end, last_arg_end + arg.max)) else: raise Exception("Unable to generate argument position condition") last_arg_end += arg.min if choices or compgenargs: if wrote_arg: condcmd = "elif" else: condcmd = "if" sw.Write("""%s [[ %s ]]; then""", condcmd, cmpcode) sw.IncIndent() try: if choices: sw.Write("""choices="$choices "%s""", choices) if compgenargs: sw.Write("compgenargs=%s", utils.ShellQuote(" ".join(compgenargs))) finally: sw.DecIndent() wrote_arg = True if wrote_arg: sw.Write("fi") if self.args: WriteCompReply(sw, """-W "$choices" $compgenargs""") else: # $compgenargs exists only if there are arguments WriteCompReply(sw, '-W "$choices"') def WriteTo(self, sw): self._FindFirstArgument(sw) self._CompleteOptionValues(sw) self._CompleteArguments(sw) def WriteCompletion(sw, scriptname, funcname, support_debug, commands=None, opts=None, args=None): """Writes the completion code for one command. @type sw: ShellWriter @param sw: Script writer @type scriptname: string @param scriptname: Name of command line program @type funcname: string @param funcname: Shell function name @type commands: list @param commands: List of all subcommands in this program """ sw.Write("%s() {", funcname) sw.IncIndent() try: sw.Write("local " ' cur="${COMP_WORDS[COMP_CWORD]}"' ' prev="${COMP_WORDS[COMP_CWORD-1]}"' ' i first_arg_idx choices compgenargs arg_idx optcur') if support_debug: sw.Write("_gnt_log cur=\"$cur\" prev=\"$prev\"") sw.Write("[[ -n \"$GANETI_COMPL_LOG\" ]] &&" " _gnt_log \"$(set | grep ^COMP_)\"") sw.Write("COMPREPLY=()") if opts is not None and args is not None: assert not commands CompletionWriter(0, opts, args, support_debug).WriteTo(sw) else: sw.Write("""if [[ "$COMP_CWORD" == 1 ]]; then""") sw.IncIndent() try: # Complete the command name WriteCompReply(sw, ("-W %s" % utils.ShellQuote(" ".join(sorted(commands.keys()))))) finally: sw.DecIndent() sw.Write("fi") # Group commands by arguments and options grouped_cmds = {} for cmd, (_, argdef, optdef, _, _) in commands.items(): if not (argdef or optdef): continue grouped_cmds.setdefault((tuple(argdef), tuple(optdef)), set()).add(cmd) # We're doing options and arguments to commands sw.Write("""case "${COMP_WORDS[1]}" in""") sort_grouped = sorted(grouped_cmds.items(), key=lambda _y: sorted(_y[1])[0]) for ((argdef, optdef), cmds) in sort_grouped: assert argdef or optdef sw.Write("%s)", "|".join(map(utils.ShellQuote, sorted(cmds)))) sw.IncIndent() try: CompletionWriter(1, optdef, argdef, support_debug).WriteTo(sw) finally: sw.DecIndent() sw.Write(";;") sw.Write("esac") finally: sw.DecIndent() sw.Write("}") sw.Write("complete -F %s -o filenames %s", utils.ShellQuote(funcname), utils.ShellQuote(scriptname)) def GetFunctionName(name): return "_" + re.sub(r"[^a-z0-9]+", "_", name.lower()) def GetCommands(filename, module): """Returns the commands defined in a module. Aliases are also added as commands. """ try: commands = getattr(module, "commands") except AttributeError: raise Exception("Script %s doesn't have 'commands' attribute" % filename) # Add the implicit "--help" option help_option = cli.cli_option("-h", "--help", default=False, action="store_true") for name, (_, _, optdef, _, _) in commands.items(): if help_option not in optdef: optdef.append(help_option) for opt in cli.COMMON_OPTS: if opt in optdef: raise Exception("Common option '%s' listed for command '%s' in %s" % (opt, name, filename)) optdef.append(opt) # Use aliases aliases = getattr(module, "aliases", {}) if aliases: commands = commands.copy() for name, target in aliases.items(): commands[name] = commands[target] return commands def HaskellOptToOptParse(opts, kind): """Converts a Haskell options to Python cli_options. @type opts: string @param opts: comma-separated string with short and long options @type kind: string @param kind: type generated by Common.hs/complToText; needs to be kept in sync """ opts = opts.split(",") if kind == "none": return cli.cli_option(*opts, action="store_true") elif kind in ["file", "string", "host", "dir", "inetaddr"]: return cli.cli_option(*opts, type="string") elif kind == "integer": return cli.cli_option(*opts, type="int") elif kind == "float": return cli.cli_option(*opts, type="float") elif kind == "onegroup": return cli.cli_option(*opts, type="string", completion_suggest=cli.OPT_COMPL_ONE_NODEGROUP) elif kind == "onenode": return cli.cli_option(*opts, type="string", completion_suggest=cli.OPT_COMPL_ONE_NODE) elif kind == "manynodes": # FIXME: no support for many nodes return cli.cli_option(*opts, type="string") elif kind == "manyinstances": # FIXME: no support for many instances return cli.cli_option(*opts, type="string") elif kind.startswith("choices="): choices = kind[len("choices="):].split(",") return cli.cli_option(*opts, type="choice", choices=choices) else: # FIXME: there are many other currently unused completion types, # should be added on an as-needed basis raise Exception("Unhandled option kind '%s'" % kind) #: serialised kind to arg type _ARG_MAP = { "choices": cli.ArgChoice, "command": cli.ArgCommand, "file": cli.ArgFile, "host": cli.ArgHost, "jobid": cli.ArgJobId, "onegroup": cli.ArgGroup, "oneinstance": cli.ArgInstance, "onenode": cli.ArgNode, "oneos": cli.ArgOs, "string": cli.ArgUnknown, "suggests": cli.ArgSuggest, } def HaskellArgToCliArg(kind, min_cnt, max_cnt): """Converts a Haskell options to Python _Argument. @type kind: string @param kind: type generated by Common.hs/argComplToText; needs to be kept in sync """ min_cnt = int(min_cnt) if max_cnt == "none": max_cnt = None else: max_cnt = int(max_cnt) kwargs = {"min": min_cnt, "max": max_cnt} if kind.startswith("choices=") or kind.startswith("suggest="): (kind, choices) = kind.split("=", 1) kwargs["choices"] = choices.split(",") if kind not in _ARG_MAP: raise Exception("Unhandled argument kind '%s'" % kind) else: return _ARG_MAP[kind](**kwargs) def ParseHaskellOptsArgs(script, output): """Computes list of options/arguments from help-completion output. """ cli_opts = [] cli_args = [] for line in output.splitlines(): v = line.split(None) exc = lambda msg, v: Exception("Invalid %s output from %s: %s" % (msg, script, v)) if len(v) < 2: raise exc("help completion", v) if v[0].startswith("-"): if len(v) != 2: raise exc("option format", v) (opts, kind) = v cli_opts.append(HaskellOptToOptParse(opts, kind)) else: if len(v) != 3: raise exc("argument format", v) (kind, min_cnt, max_cnt) = v cli_args.append(HaskellArgToCliArg(kind, min_cnt, max_cnt)) return (cli_opts, cli_args) def WriteHaskellCompletion(sw, script, htools=True, debug=True): """Generates completion information for a Haskell program. This converts completion info from a Haskell program into 'fake' cli_opts and then builds completion for them. """ if htools: cmd = "exe/htools" env = {"HTOOLS": script} script_name = script func_name = "htools_%s" % script else: cmd = "./" + script env = {} script_name = os.path.basename(script) func_name = script_name func_name = GetFunctionName(func_name) output = utils.RunCmd([cmd, "--help-completion"], env=env, cwd=".").output (opts, args) = ParseHaskellOptsArgs(script_name, output) WriteCompletion(sw, script_name, func_name, debug, opts=opts, args=args) def WriteHaskellCmdCompletion(sw, script, debug=True): """Generates completion information for a Haskell multi-command program. This gathers the list of commands from a Haskell program and computes the list of commands available, then builds the sub-command list of options/arguments for each command, using that for building a unified help output. """ cmd = "./" + script script_name = os.path.basename(script) func_name = script_name func_name = GetFunctionName(func_name) output = utils.RunCmd([cmd, "--help-completion"], cwd=".").output commands = {} lines = output.splitlines() if len(lines) != 1: raise Exception("Invalid lines in multi-command mode: %s" % str(lines)) v = lines[0].split(None) exc = lambda msg: Exception("Invalid %s output from %s: %s" % (msg, script, v)) if len(v) != 3: raise exc("help completion in multi-command mode") if not v[0].startswith("choices="): raise exc("invalid format in multi-command mode '%s'" % v[0]) for subcmd in v[0][len("choices="):].split(","): output = utils.RunCmd([cmd, subcmd, "--help-completion"], cwd=".").output (opts, args) = ParseHaskellOptsArgs(script, output) commands[subcmd] = (None, args, opts, None, None) WriteCompletion(sw, script_name, func_name, debug, commands=commands) def main(): parser = optparse.OptionParser(usage="%prog [--compact]") parser.add_option("--compact", action="store_true", help=("Don't indent output and don't include debugging" " facilities")) options, args = parser.parse_args() if args: parser.error("Wrong number of arguments") # Whether to build debug version of completion script debug = not options.compact buf = StringIO() sw = utils.ShellWriter(buf, indent=debug) # Remember original state of extglob and enable it (required for pattern # matching; must be enabled while parsing script) sw.Write("gnt_shopt_extglob=$(shopt -p extglob || :)") sw.Write("shopt -s extglob") WritePreamble(sw, debug) # gnt-* scripts for scriptname in _constants.GNT_SCRIPTS: filename = "scripts/%s" % scriptname WriteCompletion(sw, scriptname, GetFunctionName(scriptname), debug, commands=GetCommands(filename, build.LoadModule(filename))) # Burnin script WriteCompletion(sw, "%s/burnin" % pathutils.TOOLSDIR, "_ganeti_burnin", debug, opts=burnin.OPTIONS, args=burnin.ARGUMENTS) # ganeti-cleaner WriteHaskellCompletion(sw, "daemons/ganeti-cleaner", htools=False, debug=not options.compact) # htools for script in _constants.HTOOLS_PROGS: WriteHaskellCompletion(sw, script, htools=True, debug=debug) # ganeti-confd, if enabled WriteHaskellCompletion(sw, "exe/ganeti-confd", htools=False, debug=debug) # mon-collector, if monitoring is enabled if _constants.ENABLE_MOND: WriteHaskellCmdCompletion(sw, "exe/mon-collector", debug=debug) # Reset extglob to original value sw.Write("[[ -n \"$gnt_shopt_extglob\" ]] && $gnt_shopt_extglob") sw.Write("unset gnt_shopt_extglob") print(buf.getvalue()) if __name__ == "__main__": main() ganeti-3.1.0~rc2/autotools/build-rpc000075500000000000000000000134511476477700300174500ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to generate RPC code. """ # pylint: disable=C0103 # [C0103] Invalid name import sys import re import itertools import textwrap from io import StringIO from ganeti import utils from ganeti import compat from ganeti import build _SINGLE = "single-node" _MULTI = "multi-node" #: Expected length of a rpc definition _RPC_DEF_LEN = 8 def _WritePreamble(sw): """Writes a preamble for the RPC wrapper output. """ sw.Write("# This code is automatically generated at build time.") sw.Write("# Do not modify manually.") sw.Write("") sw.Write("\"\"\"Automatically generated RPC client wrappers.") sw.Write("") sw.Write("\"\"\"") sw.Write("") sw.Write("from ganeti import rpc_defs") sw.Write("") def _WrapCode(line): """Wraps Python code. """ return textwrap.wrap(line, width=70, expand_tabs=False, fix_sentence_endings=False, break_long_words=False, replace_whitespace=True, subsequent_indent=utils.ShellWriter.INDENT_STR) def _WriteDocstring(sw, name, timeout, kind, args, desc): """Writes a docstring for an RPC wrapper. """ sw.Write("\"\"\"Wrapper for RPC call '%s'", name) sw.Write("") if desc: sw.Write(desc) sw.Write("") note = ["This is a %s call" % kind] if timeout and not callable(timeout): note.append(" with a timeout of %s" % utils.FormatSeconds(timeout)) sw.Write("@note: %s", "".join(note)) if kind == _SINGLE: sw.Write("@type node: string") sw.Write("@param node: Node name") else: sw.Write("@type node_list: list of string") sw.Write("@param node_list: List of node names") if args: for (argname, _, argtext) in args: if argtext: docline = "@param %s: %s" % (argname, argtext) for line in _WrapCode(docline): sw.Write(line) sw.Write("") sw.Write("\"\"\"") def _WriteBaseClass(sw, clsname, calls): """Write RPC wrapper class. """ sw.Write("") sw.Write("class %s(object):", clsname) sw.IncIndent() try: sw.Write("# E1101: Non-existent members") sw.Write("# R0904: Too many public methods") sw.Write("# pylint: disable=E1101,R0904") if not calls: sw.Write("pass") return sw.Write("_CALLS = rpc_defs.CALLS[%r]", clsname) sw.Write("") for v in calls: if len(v) != _RPC_DEF_LEN: raise ValueError("Procedure %s has only %d elements, expected %d" % (v[0], len(v), _RPC_DEF_LEN)) for (name, kind, _, timeout, args, _, _, desc) in sorted(calls): funcargs = ["self"] if kind == _SINGLE: funcargs.append("node") elif kind == _MULTI: funcargs.append("node_list") else: raise Exception("Unknown kind '%s'" % kind) funcargs.extend([arg[0] for arg in args]) funcargs.append("_def=_CALLS[%r]" % name) funcdef = "def call_%s(%s):" % (name, utils.CommaJoin(funcargs)) for line in _WrapCode(funcdef): sw.Write(line) sw.IncIndent() try: _WriteDocstring(sw, name, timeout, kind, args, desc) buf = StringIO() buf.write("return ") # In case line gets too long and is wrapped in a bad spot buf.write("(") buf.write("self._Call(_def, ") if kind == _SINGLE: buf.write("[node]") else: buf.write("node_list") buf.write(", [%s])" % # Function arguments utils.CommaJoin(map(compat.fst, args))) if kind == _SINGLE: buf.write("[node]") buf.write(")") for line in _WrapCode(buf.getvalue()): sw.Write(line) finally: sw.DecIndent() sw.Write("") finally: sw.DecIndent() def main(): """Main function. """ buf = StringIO() sw = utils.ShellWriter(buf) _WritePreamble(sw) for filename in sys.argv[1:]: sw.Write("# Definitions from '%s'", filename) module = build.LoadModule(filename) # Call types are re-defined in definitions file to avoid imports. Verify # here to ensure they're equal to local constants. assert module.SINGLE == _SINGLE assert module.MULTI == _MULTI dups = utils.GetRepeatedKeys(*module.CALLS.values()) if dups: raise Exception("Found duplicate RPC definitions for '%s'" % utils.CommaJoin(sorted(dups))) for (clsname, calls) in sorted(module.CALLS.items()): _WriteBaseClass(sw, clsname, calls.values()) print(buf.getvalue().rstrip()) if __name__ == "__main__": main() ganeti-3.1.0~rc2/autotools/check-header000075500000000000000000000116271476477700300200750ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to verify file header. """ # pylint: disable=C0103 # [C0103] Invalid name import sys import re import itertools from ganeti import constants from ganeti import utils from ganeti import compat #: Assume header is always in the first 8kB of a file _READ_SIZE = 8 * 1024 _BSD2 = [ "All rights reserved.", "", "Redistribution and use in source and binary forms, with or without", "modification, are permitted provided that the following conditions are", "met:", "", "1. Redistributions of source code must retain the above copyright notice,", "this list of conditions and the following disclaimer.", "", "2. Redistributions in binary form must reproduce the above copyright", "notice, this list of conditions and the following disclaimer in the", "documentation and/or other materials provided with the distribution.", "", "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS", "IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED", "TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR", "PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR", "CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,", "EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,", "PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR", "PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF", "LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING", "NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS", "SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", ] _SHEBANG = re.compile(r"^#(?:|!(?:/usr/bin/python3?(?:| -u)|/bin/(?:|ba)sh))$") _COPYRIGHT_YEAR = r"20[012][0-9]" _COPYRIGHT = re.compile(r"# Copyright \(C\) (%s(?:, %s)*) " "(Google Inc\.|the Ganeti project)$" % (_COPYRIGHT_YEAR, _COPYRIGHT_YEAR)) _COPYRIGHT_DESC = "Copyright (C) [, ...] the Ganeti project" _AUTOGEN = "# This file is automatically generated, do not edit!" class HeaderError(Exception): pass def _Fail(lineno, msg): raise HeaderError("Line %s: %s" % (lineno, msg)) def _CheckHeader(getline_fn): (lineno, line) = getline_fn() if line == _AUTOGEN: return if not _SHEBANG.match(line): _Fail(lineno, ("Must contain nothing but a hash character (#) or a" " shebang line (e.g. #!/bin/bash)")) (lineno, line) = getline_fn() if line == _AUTOGEN: return if line != "#": _Fail(lineno, "Must contain nothing but hash character (#)") (lineno, line) = getline_fn() if line: _Fail(lineno, "Must be empty") (lineno, line) = getline_fn() if not _COPYRIGHT.match(line): _Fail(lineno, "Must contain copyright information (%s)" % _COPYRIGHT_DESC) for licence_line in _BSD2: (lineno, line) = getline_fn() if line != ("# %s" % licence_line).rstrip(): _Fail(lineno, "Does not match expected licence line (%s)" % licence_line) (lineno, line) = getline_fn() if line: _Fail(lineno, "Must be empty") def Main(): """Main program. """ fail = False for filename in sys.argv[1:]: content = utils.ReadFile(filename, size=_READ_SIZE) lines = zip(itertools.count(1), content.splitlines()) try: _CheckHeader(compat.partial(next, lines)) except HeaderError as err: report = str(err) print("%s: %s" % (filename, report)) fail = True if fail: sys.exit(constants.EXIT_FAILURE) else: sys.exit(constants.EXIT_SUCCESS) if __name__ == "__main__": Main() ganeti-3.1.0~rc2/autotools/check-imports000075500000000000000000000061721476477700300203410ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to check module imports. """ # pylint: disable=C0103, C0413 # C0103: Invalid name # C0413: Wrong import position import sys # All modules imported after this line are removed from the global list before # importing a module to be checked _STANDARD_MODULES = set(sys.modules.keys()) import os.path from ganeti import build def main(): args = sys.argv[1:] # Get references to functions used later on load_module = build.LoadModule abspath = os.path.abspath commonprefix = os.path.commonprefix normpath = os.path.normpath script_path = abspath(__file__) srcdir = normpath(abspath(args.pop(0))) assert "ganeti" in sys.modules for filename in args: # Reset global state modules_to_remove = [] for name in sys.modules: if name not in _STANDARD_MODULES: modules_to_remove.append(name) for name in modules_to_remove: sys.modules.pop(name, None) assert "ganeti" not in sys.modules # Load module (this might import other modules) module = load_module(filename) result = [] for (name, checkmod) in sorted(sys.modules.items()): if checkmod is None or checkmod == module: continue try: checkmodpath = getattr(checkmod, "__file__") except AttributeError: # Built-in module continue if checkmodpath is None and hasattr(checkmod, "__path__"): # Namespace module continue abscheckmodpath = os.path.abspath(checkmodpath) if abscheckmodpath == script_path: # Ignore check script continue if commonprefix([abscheckmodpath, srcdir]) == srcdir: result.append(name) if result: raise Exception("Module '%s' has illegal imports: %s" % (filename, ", ".join(result))) if __name__ == "__main__": main() ganeti-3.1.0~rc2/autotools/check-man-dashes000075500000000000000000000027011476477700300206560ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e ! grep -F '\[em]' "$1" || \ { echo "Unescaped dashes found in $1, use \\-- instead of --" 1>&2; exit 1; } ganeti-3.1.0~rc2/autotools/check-man-references000075500000000000000000000040431476477700300215310ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e -u -o pipefail # Use array for arguments so that comments can be inline args=( # "...name*(8)" (missing backslash) -e '\w+\*+\([0-9]*\)' # "...name(8)" (no asterisk) -e '\w+\([0-9]*\)' # "...name(8)*" (asterisk after number) -e '\w+\([0-9]*\)\*' # "...name*\(8)" (only one asterisk before backslash) -e '\w+\*\\\([0-9]*\)' # ":manpage:..." (Sphinx-specific) -e ':manpage:' ) for fname; do # Ignore title and then look for faulty references if tail -n +2 $fname | grep -n -E -i "${args[@]}"; then { echo "Found faulty man page reference(s) in '$fname'."\ 'Use syntax "**name**\(number)" instead.'\ 'Example: **gnt-instance**\(8).' } >&2 exit 1 fi done ganeti-3.1.0~rc2/autotools/check-man-warnings000075500000000000000000000031741476477700300212440ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e if locale -a | grep -qF 'C.UTF-8'; then loc="C.UTF-8" else loc="en_US.UTF-8" fi ! LANG="$loc" LC_ALL="$loc" MANWIDTH=80 \ man --warnings --encoding=utf8 --local-file "$1" 2>&1 >/dev/null | \ grep -v -e "cannot adjust line" -e "can't break line" \ -e "cannot select font" | \ grep . ganeti-3.1.0~rc2/autotools/check-news000075500000000000000000000133721476477700300176200ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to check NEWS file. """ # pylint: disable=C0103 # [C0103] Invalid name from __future__ import print_function import sys import time import datetime import locale import fileinput import re import os DASHES_RE = re.compile(r"^\s*-+\s*$") RELEASED_RE = re.compile(r"^\*\(Released (?P[A-Z][a-z]{2})," r" (?P.+)\)\*$") UNRELEASED_RE = re.compile(r"^\*\(unreleased\)\*$") VERSION_RE = re.compile(r"^Version (\d+(\.\d+)+( (alpha|beta|rc)\d+)?)$") #: How many days release timestamps may be in the future TIMESTAMP_FUTURE_DAYS_MAX = 5 errors = [] def Error(msg): """Log an error for later display. """ errors.append(msg) def ReqNLines(req, count_empty, lineno, line): """Check if we have N empty lines before the current one. """ if count_empty < req: Error("Line %s: Missing empty line(s) before %s," " %d needed but got only %d" % (lineno, line, req, count_empty)) if count_empty > req: Error("Line %s: Too many empty lines before %s," " %d needed but got %d" % (lineno, line, req, count_empty)) def IsAlphaVersion(version): return "alpha" in version def UpdateAllowUnreleased(allow_unreleased, version_match, release): if not allow_unreleased: return False if IsAlphaVersion(release): return True version = version_match.group(1) if version == release: return False return True def main(): # Ensure "C" locale is used curlocale = locale.getlocale() if curlocale[0] not in (None, 'C'): Error("Invalid locale %s" % str(curlocale)) # Get the release version, but replace "~" with " " as the version # in the NEWS file uses spaces for beta and rc releases. release = os.environ.get('RELEASE', "").replace("~", " ") prevline = None expect_date = False count_empty = 0 allow_unreleased = True found_versions = set() for line in fileinput.input(): line = line.rstrip("\n") version_match = VERSION_RE.match(line) if version_match: ReqNLines(2, count_empty, fileinput.filelineno(), line) version = version_match.group(1) if version in found_versions: Error("Line %s: Duplicate release %s found" % (fileinput.filelineno(), version)) found_versions.add(version) allow_unreleased = UpdateAllowUnreleased(allow_unreleased, version_match, release) unreleased_match = UNRELEASED_RE.match(line) if unreleased_match and not allow_unreleased: Error("Line %s: Unreleased version after current release %s" % (fileinput.filelineno(), release)) if unreleased_match or RELEASED_RE.match(line): ReqNLines(1, count_empty, fileinput.filelineno(), line) if line: count_empty = 0 else: count_empty += 1 if DASHES_RE.match(line): if not VERSION_RE.match(prevline): Error("Line %s: Invalid title" % (fileinput.filelineno() - 1)) if len(line) != len(prevline): Error("Line %s: Invalid dashes length" % (fileinput.filelineno())) expect_date = True elif expect_date: if not line: # Ignore empty lines continue if UNRELEASED_RE.match(line): # Ignore unreleased versions expect_date = False continue m = RELEASED_RE.match(line) if not m: Error("Line %s: Invalid release line" % fileinput.filelineno()) expect_date = False continue # Including the weekday in the date string does not work as time.strptime # would return an inconsistent result if the weekday is incorrect. parsed_ts = time.mktime(time.strptime(m.group("date"), "%d %b %Y")) parsed = datetime.date.fromtimestamp(parsed_ts) today = datetime.date.today() if (parsed - datetime.timedelta(TIMESTAMP_FUTURE_DAYS_MAX)) > today: Error("Line %s: %s is more than %s days in the future (today is %s)" % (fileinput.filelineno(), parsed, TIMESTAMP_FUTURE_DAYS_MAX, today)) weekday = parsed.strftime("%a") # Check weekday if m.group("day") != weekday: Error("Line %s: %s was/is a %s, not %s" % (fileinput.filelineno(), parsed, weekday, m.group("day"))) expect_date = False prevline = line if errors: for msg in errors: print(msg, file=sys.stderr) sys.exit(1) else: sys.exit(0) if __name__ == "__main__": main() ganeti-3.1.0~rc2/autotools/check-python-code000075500000000000000000000053511476477700300210730ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2009, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e # Ensure the checks always use the same locale export LC_ALL=C readonly maxlinelen=$(for ((i=0; i<81; ++i)); do echo -n .; done) if [[ "${#maxlinelen}" != 81 ]]; then echo "Internal error: Check for line length is incorrect" >&2 exit 1 fi # "[...] If the last ARG evaluates to 0, let returns 1; 0 is returned # otherwise.", hence ignoring the return value. let problems=0 || : for script; do if grep -n -H -F $'\t' "$script"; then let ++problems echo "Found tabs in $script" >&2 fi if grep -n -H -E '[[:space:]]$' "$script"; then let ++problems echo "Found end-of-line-whitespace in $script" >&2 fi # FIXME: This will also match "foo.xrange(...)" if grep -n -H -E '^[^#]*\' "$script"; then let ++problems echo "Forbidden function 'xrange' used in $script" >&2 fi if grep -n -H -E -i '#[[:space:]]*(vim|Local[[:space:]]+Variables):' "$script" then let ++problems echo "Found editor-specific settings in $script" >&2 fi if grep -n -H "^$maxlinelen" "$script"; then let ++problems echo "Longest line in $script is longer than 80 characters" >&2 fi if grep -n -H -E -i \ '#.*\bpylint[[:space:]]*:[[:space:]]*disable-msg\b' "$script" then let ++problems echo "Found old-style pylint disable pragma in $script" >&2 fi done if [[ "$problems" -gt 0 ]]; then echo "Found $problems problem(s) while checking code." >&2 exit 1 fi ganeti-3.1.0~rc2/autotools/check-tar000075500000000000000000000041631476477700300174300ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to check tarball generated by Automake. """ from __future__ import print_function import sys import stat import tarfile def ReportError(member, msg): print("%s: %s" % (member.name, msg), file=sys.stderr) def main(): tf = tarfile.open(fileobj=sys.stdin.buffer) success = True for member in tf.getmembers(): if member.uid != 0: success = False ReportError(member, "Owned by UID %s, not UID 0" % member.uid) if member.gid != 0: success = False ReportError(member, "Owned by GID %s, not GID 0" % member.gid) if member.mode & (stat.S_IWGRP | stat.S_IWOTH): success = False ReportError(member, "World or group writeable (mode is %o)" % member.mode) if success: sys.exit(0) sys.exit(1) if __name__ == "__main__": main() ganeti-3.1.0~rc2/autotools/check-version000075500000000000000000000040611476477700300203240ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2010,2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e # Enable Bash-specific patterns shopt -s extglob readonly version=$1 readonly newsfile=$2 readonly numpat='+([0-9])' case "$version" in # Format "x.y.z" $numpat.$numpat.$numpat) : ;; # Format "x.y.z~rcN" or "x.y.z~betaN" or "x.y.z~alphaN" for N > 0 $numpat.$numpat.$numpat~@(rc|beta|alpha)[1-9]*([0-9])) : ;; *) echo "Invalid version format: $version" >&2 exit 1 ;; esac readonly newsver="Version ${version/\~/ }" # Only alpha versions are allowed not to have their own NEWS section yet set +e FOUND=x`echo $version | grep "alpha[1-9]*[0-9]$"` set -e if [ $FOUND == "x" ] then if ! grep -q -x "$newsver" $newsfile then echo "Unable to find heading '$newsver' in NEWS" >&2 exit 1 fi fi exit 0 ganeti-3.1.0~rc2/autotools/docpp000075500000000000000000000040331476477700300166700ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to replace special directives in documentation. """ import re import fileinput from ganeti import query from ganeti.build import sphinx_ext _DOC_RE = re.compile(r"^@(?P[A-Z_]+)_(?P[A-Z]+)@$") _DOC_CLASSES_DATA = { "CONSTANTS": (sphinx_ext.DOCUMENTED_CONSTANTS, sphinx_ext.BuildValuesDoc), "QUERY_FIELDS": (query.ALL_FIELDS, sphinx_ext.BuildQueryFields), } def main(): for line in fileinput.input(): m = _DOC_RE.match(line) if m: fields_dict, builder = _DOC_CLASSES_DATA[m.group("class")] fields = fields_dict[m.group("kind").lower()] for i in builder(fields): print(i) else: print(line, end='') if __name__ == "__main__": main() ganeti-3.1.0~rc2/autotools/gen-py-coverage000075500000000000000000000045451476477700300205630ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e set -u : ${PYTHON:=python} : ${COVERAGE:?} : ${COVERAGE_FILE:?} : ${TEXT_COVERAGE:?} : ${HTML_COVERAGE:=} : ${GANETI_TEMP_DIR:?} reportargs=( '--include=*' '--omit=test/py/*' ) $COVERAGE erase if [[ -n "$HTML_COVERAGE" ]]; then if [[ ! -d "$HTML_COVERAGE" ]]; then echo "Not a directory: $HTML_COVERAGE" >&2 exit 1 fi # At least coverage 3.4 fails to overwrite files find "$HTML_COVERAGE" \( -type f -o -type l \) -delete fi for script; do if [[ "$script" == *-runasroot.py ]]; then if [[ -z "$FAKEROOT" ]]; then echo "WARNING: FAKEROOT variable not set: skipping $script" >&2 continue fi cmdprefix="$FAKEROOT" else cmdprefix= fi $cmdprefix $COVERAGE run --branch --append "${reportargs[@]}" $script done echo "Writing text report to $TEXT_COVERAGE ..." >&2 $COVERAGE report "${reportargs[@]}" | tee "$TEXT_COVERAGE" if [[ -n "$HTML_COVERAGE" ]]; then echo "Generating HTML report in $HTML_COVERAGE ..." >&2 $COVERAGE html "${reportargs[@]}" -d "$HTML_COVERAGE" fi ganeti-3.1.0~rc2/autotools/print-py-constants000075500000000000000000000035761476477700300213720ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for printing Python constants related to sockets. These constants are the remnants of the Haskell to Python constant generation. This solution is transitional until Ganeti 2.11 because the solution for eliminating completely the Python to Haskell conversion requires updating the configuration file. """ import socket import sys def main(): if len(sys.argv) > 1: if sys.argv[1] == "AF_INET4": print("%d" % socket.AF_INET) elif sys.argv[1] == "AF_INET6": print("%d" % socket.AF_INET6) if __name__ == "__main__": main() ganeti-3.1.0~rc2/autotools/run-in-tempdir000075500000000000000000000015041476477700300204350ustar00rootroot00000000000000#!/bin/bash # Helper for running things in a temporary directory; used for docs # building, unittests, etc. set -e tmpdir=$(mktemp -d -t gntbuild.XXXXXXXX) trap "rm -rf $tmpdir" EXIT # fully copy items cp -r autotools daemons scripts lib tools qa $tmpdir if [[ -z "$COPY_DOC" ]]; then mkdir $tmpdir/doc ln -s $PWD/doc/examples $tmpdir/doc else # Building documentation requires all files cp -r doc $tmpdir fi mkdir $tmpdir/test/ cp -r test/py $tmpdir/test/py ln -s $PWD/test/data $tmpdir/test ln -s $PWD/test/hs $tmpdir/test mv $tmpdir/lib $tmpdir/ganeti ln -T -s $tmpdir/ganeti $tmpdir/lib mkdir -p $tmpdir/exe $tmpdir/test/hs for hfile in htest htools ganeti-confd mon-collector hs2py; do exe=exe/$hfile if [ -e $exe ]; then ln -s $PWD/$exe $tmpdir/exe/ fi done cd $tmpdir && GANETI_TEMP_DIR="$tmpdir" "$@" ganeti-3.1.0~rc2/autotools/sphinx-wrapper000075500000000000000000000033051476477700300205530ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e -u -o pipefail if [[ -e doc/manpages.rst ]]; then echo 'doc/manpages.rst should not exist' >&2 exit 1 fi if [[ -n "$ENABLE_MANPAGES" ]]; then mv doc/manpages-enabled.rst doc/manpages.rst rm doc/manpages-disabled.rst else mv doc/manpages-disabled.rst doc/manpages.rst if [[ -e doc/manpages-enabled.rst ]]; then rm doc/manpages-enabled.rst fi fi exec "$@" ganeti-3.1.0~rc2/autotools/testrunner000075500000000000000000000033011476477700300177710ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e filename=$1 execasroot() { local fname=$1 shift if [[ -z "$FAKEROOT" ]]; then echo "WARNING: FAKEROOT variable not set, skipping $fname" >&2 else exec "$FAKEROOT" "$@" fi } case "$filename" in *-runasroot.py) execasroot $filename $PYTHON "$@" ;; *.py) exec $PYTHON "$@" ;; *-runasroot) execasroot $filename "$@" ;; *) exec "$@" ;; esac ganeti-3.1.0~rc2/autotools/wrong-hardcoded-paths000064400000000000000000000001141476477700300217400ustar00rootroot00000000000000/etc/ganeti /usr/(local/)?lib/ganeti /(usr/local/)?var/(lib|run|log)/ganeti ganeti-3.1.0~rc2/configure.ac000064400000000000000000000721461476477700300161040ustar00rootroot00000000000000# Configure script for Ganeti m4_define([gnt_version_major], [3]) m4_define([gnt_version_minor], [1]) m4_define([gnt_version_revision], [0]) m4_define([gnt_version_suffix], [~rc2]) m4_define([gnt_version_full], m4_format([%d.%d.%d%s], gnt_version_major, gnt_version_minor, gnt_version_revision, gnt_version_suffix)) AC_PREREQ(2.59) AC_INIT(ganeti, gnt_version_full, ganeti@googlegroups.com) AC_CONFIG_AUX_DIR(autotools) AC_CONFIG_SRCDIR(configure) AM_INIT_AUTOMAKE([1.9 foreign tar-ustar -Wall -Wno-portability] m4_esyscmd([case `automake --version | head -n 1` in *1.11*);; *) echo serial-tests;; esac])) AM_MAINTAINER_MODE([enable]) AC_SUBST([VERSION_MAJOR], gnt_version_major) AC_SUBST([VERSION_MINOR], gnt_version_minor) AC_SUBST([VERSION_REVISION], gnt_version_revision) AC_SUBST([VERSION_SUFFIX], gnt_version_suffix) AC_SUBST([VERSION_FULL], gnt_version_full) AC_SUBST([BINDIR], $bindir) AC_SUBST([SBINDIR], $sbindir) AC_SUBST([MANDIR], $mandir) # --enable-versionfull AC_ARG_ENABLE([versionfull], [AS_HELP_STRING([--enable-versionfull], m4_normalize([use the full version string rather than major.minor for version directories]))], [[if test "$enableval" != no; then USE_VERSION_FULL=yes else USE_VERSION_FULL=no fi ]], [USE_VERSION_FULL=no ]) AC_SUBST(USE_VERSION_FULL, $USE_VERSION_FULL) AM_CONDITIONAL([USE_VERSION_FULL], [test "$USE_VERSION_FULL" = yes]) # --enable-symlinks AC_ARG_ENABLE([symlinks], [AS_HELP_STRING([--enable-symlinks], m4_normalize([also install version-dependent symlinks under $sysconfdir (default: disabled)]))], [[if test "$enableval" != yes; then INSTALL_SYMLINKS=no else INSTALL_SYMLINKS=yes fi ]], [INSTALL_SYMLINKS=no ]) AC_SUBST(INSTALL_SYMLINKS, $INSTALL_SYMLINKS) AM_CONDITIONAL([INSTALL_SYMLINKS], [test "$INSTALL_SYMLINKS" = yes]) # --enable-haskell-profiling AC_ARG_ENABLE([haskell-profiling], [AS_HELP_STRING([--enable-haskell-profiling], m4_normalize([enable profiling for Haskell binaries (default: disabled)]))], [[if test "$enableval" != yes; then HPROFILE=no else HPROFILE=yes fi ]], [HPROFILE=no ]) AC_SUBST(HPROFILE, $HPROFILE) AM_CONDITIONAL([HPROFILE], [test "$HPROFILE" = yes]) # --enable-haskell-coverage AC_ARG_ENABLE([haskell-coverage], [AS_HELP_STRING([--enable-haskell-coverage], m4_normalize([enable coverage for Haskell binaries (default: disabled)]))], [[if test "$enableval" != yes; then HCOVERAGE=no else HCOVERAGE=yes fi ]], [HCOVERAGE=no ]) AC_SUBST(HCOVERAGE, $HCOVERAGE) AM_CONDITIONAL([HCOVERAGE], [test "$HCOVERAGE" = yes]) # --enable-haskell-tests AC_ARG_ENABLE([haskell-tests], [AS_HELP_STRING([--enable-haskell-tests], m4_normalize([enable additional Haskell development test code (default: disabled)]))], [[if test "$enableval" != yes; then HTEST=no else HTEST=yes fi ]], [HTEST=no ]) AC_SUBST(HTEST, $HTEST) AM_CONDITIONAL([HTEST], [test "$HTEST" = yes]) # --enable-developer-mode AC_ARG_ENABLE([developer-mode], [AS_HELP_STRING([--enable-developer-mode], m4_normalize([do a developer build with additional checks and fatal warnings; this is implied by enabling the haskell tests]))], [[if test "$enableval" != no; then DEVELOPER_MODE=yes else DEVELOPER_MODE=no fi ]], [DEVELOPER_MODE=no ]) AC_SUBST(DEVELOPER_MODE, $DEVELOPER_MODE) AM_CONDITIONAL([DEVELOPER_MODE], [test "$DEVELOPER_MODE" = yes -o "$HTEST" = yes]) # --with-haskell-flags= AC_ARG_WITH([haskell-flags], [AS_HELP_STRING([--with-haskell-flags=FLAGS], [Extra flags to pass to GHC] )], [hextra_configure="$withval"], [hextra_configure=""]) AC_SUBST(HEXTRA_CONFIGURE, $hextra_configure) # --with-haskell-pcre= AC_ARG_WITH([haskell-pcre], [AS_HELP_STRING([--with-haskell-pcre=auto|pcre|pcre2|pcre-builtin|tdfa], [Haskell PCRE regex to use] )], [case "$withval" in auto|pcre|pcre2|pcre-builtin|tdfa) hs_pcre_backend="$withval" ;; *) AC_MSG_ERROR([Unsupported value ${withval} for --with-haskell-pcre]) ;; esac ], [hs_pcre_backend="auto"]) # --with-sshd-restart-command=... AC_ARG_WITH([sshd-restart-command], [AS_HELP_STRING([--with-sshd-restart-command=SCRIPT], [SSH restart command to use (default is /usr/sbin/service ssh restart)] )], [sshd_restart_command="$withval"], [sshd_restart_command="/usr/sbin/service ssh restart"]) AC_SUBST(SSHD_RESTART_COMMAND, $sshd_restart_command) # --with-export-dir=... AC_ARG_WITH([export-dir], [AS_HELP_STRING([--with-export-dir=DIR], [directory to use by default for instance image] [ exports (default is /srv/ganeti/export)] )], [export_dir="$withval"], [export_dir="/srv/ganeti/export"]) AC_SUBST(EXPORT_DIR, $export_dir) # --with-backup-dir=... AC_ARG_WITH([backup-dir], [AS_HELP_STRING([--with-backup-dir=DIR], [directory to use for configuration backups] [ on Ganeti upgrades (default is $(localstatedir)/lib)] )], [backup_dir="$withval" USE_BACKUP_DIR=yes ], [backup_dir= USE_BACKUP_DIR=no ]) AC_SUBST(BACKUP_DIR, $backup_dir) AM_CONDITIONAL([USE_BACKUP_DIR], [test "$USE_BACKUP_DIR" = yes]) # --with-ssh-config-dir=... AC_ARG_WITH([ssh-config-dir], [AS_HELP_STRING([--with-ssh-config-dir=DIR], [ directory with ssh host keys ] [ (default is /etc/ssh)] )], [ssh_config_dir="$withval"], [ssh_config_dir="/etc/ssh"]) AC_SUBST(SSH_CONFIG_DIR, $ssh_config_dir) # --with-xen-config-dir=... AC_ARG_WITH([xen-config-dir], [AS_HELP_STRING([--with-xen-config-dir=DIR], m4_normalize([Xen configuration directory (default: /etc/xen)]))], [xen_config_dir="$withval"], [xen_config_dir=/etc/xen]) AC_SUBST(XEN_CONFIG_DIR, $xen_config_dir) # --with-os-search-path=... AC_ARG_WITH([os-search-path], [AS_HELP_STRING([--with-os-search-path=LIST], [comma separated list of directories to] [ search for OS images (default is /srv/ganeti/os)] )], [os_search_path="$withval"], [os_search_path="/srv/ganeti/os"]) AC_SUBST(OS_SEARCH_PATH, $os_search_path) # --with-extstorage-search-path=... AC_ARG_WITH([extstorage-search-path], [AS_HELP_STRING([--with-extstorage-search-path=LIST], [comma separated list of directories to] [ search for External Storage Providers] [ (default is /srv/ganeti/extstorage)] )], [es_search_path="$withval"], [es_search_path="/srv/ganeti/extstorage"]) AC_SUBST(ES_SEARCH_PATH, $es_search_path) # --with-iallocator-search-path=... AC_ARG_WITH([iallocator-search-path], [AS_HELP_STRING([--with-iallocator-search-path=LIST], [comma separated list of directories to] [ search for instance allocators (default is $libdir/ganeti/iallocators)] )], [iallocator_search_path="$withval"], [iallocator_search_path="$libdir/$PACKAGE_NAME/iallocators"]) AC_SUBST(IALLOCATOR_SEARCH_PATH, $iallocator_search_path) # --with-default-vg=... AC_ARG_WITH([default-vg], [AS_HELP_STRING([--with-default-vg=VOLUMEGROUP], [default volume group (default is xenvg)] )], [default_vg="$withval"], [default_vg="xenvg"]) AC_SUBST(DEFAULT_VG, $default_vg) # --with-default-bridge=... AC_ARG_WITH([default-bridge], [AS_HELP_STRING([--with-default-bridge=BRIDGE], [default bridge (default is xen-br0)] )], [default_bridge="$withval"], [default_bridge="xen-br0"]) AC_SUBST(DEFAULT_BRIDGE, $default_bridge) # --with-xen-bootloader=... AC_ARG_WITH([xen-bootloader], [AS_HELP_STRING([--with-xen-bootloader=PATH], [bootloader for Xen hypervisor (default is empty)] )], [xen_bootloader="$withval"], [xen_bootloader=]) AC_SUBST(XEN_BOOTLOADER, $xen_bootloader) # --with-xen-kernel=... AC_ARG_WITH([xen-kernel], [AS_HELP_STRING([--with-xen-kernel=PATH], [DomU kernel image for Xen hypervisor (default is /boot/vmlinuz-3-xenU)] )], [xen_kernel="$withval"], [xen_kernel="/boot/vmlinuz-3-xenU"]) AC_SUBST(XEN_KERNEL, $xen_kernel) # --with-xen-initrd=... AC_ARG_WITH([xen-initrd], [AS_HELP_STRING([--with-xen-initrd=PATH], [DomU initrd image for Xen hypervisor (default is /boot/initrd-3-xenU)] )], [xen_initrd="$withval"], [xen_initrd="/boot/initrd-3-xenU"]) AC_SUBST(XEN_INITRD, $xen_initrd) # --with-kvm-kernel=... AC_ARG_WITH([kvm-kernel], [AS_HELP_STRING([--with-kvm-kernel=PATH], [Guest kernel image for KVM hypervisor (default is /boot/vmlinuz-3-kvmU)] )], [kvm_kernel="$withval"], [kvm_kernel="/boot/vmlinuz-3-kvmU"]) AC_SUBST(KVM_KERNEL, $kvm_kernel) # --with-kvm-path=... AC_ARG_WITH([kvm-path], [AS_HELP_STRING([--with-kvm-path=PATH], [absolute path to the kvm binary] [ (default is /usr/bin/kvm)] )], [kvm_path="$withval"], [kvm_path="/usr/bin/kvm"]) AC_SUBST(KVM_PATH, $kvm_path) # --with-lvm-stripecount=... AC_ARG_WITH([lvm-stripecount], [AS_HELP_STRING([--with-lvm-stripecount=NUM], [the default number of stripes to use for LVM volumes] [ (default is 1)] )], [lvm_stripecount="$withval"], [lvm_stripecount=1]) AC_SUBST(LVM_STRIPECOUNT, $lvm_stripecount) # --with-ssh-login-user=... AC_ARG_WITH([ssh-login-user], [AS_HELP_STRING([--with-ssh-login-user=USERNAME], [user to use for SSH logins within the cluster (default is root)] )], [ssh_login_user="$withval"], [ssh_login_user=root]) AC_SUBST(SSH_LOGIN_USER, $ssh_login_user) # --with-ssh-console-user=... AC_ARG_WITH([ssh-console-user], [AS_HELP_STRING([--with-ssh-console-user=USERNAME], [user to use for SSH logins to access instance consoles (default is root)] )], [ssh_console_user="$withval"], [ssh_console_user=root]) AC_SUBST(SSH_CONSOLE_USER, $ssh_console_user) # --with-default-user=... AC_ARG_WITH([default-user], [AS_HELP_STRING([--with-default-user=USERNAME], [default user for daemons] [ (default is to run all daemons as root)] )], [user_default="$withval"], [user_default=root]) # --with-default-group=... AC_ARG_WITH([default-group], [AS_HELP_STRING([--with-default-group=GROUPNAME], [default group for daemons] [ (default is to run all daemons under group root)] )], [group_default="$withval"], [group_default=root]) # --with-user-prefix=... AC_ARG_WITH([user-prefix], [AS_HELP_STRING([--with-user-prefix=PREFIX], [prefix for daemon users] [ (default is to run all daemons as root; use --with-default-user] [ to change the default)] )], [user_masterd="${withval}masterd"; user_metad="${withval}metad"; user_rapi="${withval}rapi"; user_confd="${withval}confd"; user_wconfd="${withval}masterd"; user_kvmd="$user_default"; user_luxid="${withval}masterd"; user_noded="$user_default"; user_mond="$user_default"], [user_masterd="$user_default"; user_metad="$user_default"; user_rapi="$user_default"; user_confd="$user_default"; user_wconfd="$user_default"; user_kvmd="$user_default"; user_luxid="$user_default"; user_noded="$user_default"; user_mond="$user_default"]) AC_SUBST(MASTERD_USER, $user_masterd) AC_SUBST(METAD_USER, $user_metad) AC_SUBST(RAPI_USER, $user_rapi) AC_SUBST(CONFD_USER, $user_confd) AC_SUBST(WCONFD_USER, $user_wconfd) AC_SUBST(KVMD_USER, $user_kvmd) AC_SUBST(LUXID_USER, $user_luxid) AC_SUBST(NODED_USER, $user_noded) AC_SUBST(MOND_USER, $user_mond) AC_SUBST(METAD_USER, $user_metad) # --with-group-prefix=... AC_ARG_WITH([group-prefix], [AS_HELP_STRING([--with-group-prefix=PREFIX], [prefix for daemon POSIX groups] [ (default is to run all daemons under group root; use] [ --with-default-group to change the default)] )], [group_rapi="${withval}rapi"; group_admin="${withval}admin"; group_confd="${withval}confd"; group_wconfd="${withval}masterd"; group_kvmd="$group_default"; group_luxid="${withval}luxid"; group_masterd="${withval}masterd"; group_metad="${withval}metad"; group_noded="$group_default"; group_daemons="${withval}daemons"; group_mond="$group_default"], [group_rapi="$group_default"; group_admin="$group_default"; group_confd="$group_default"; group_wconfd="$group_default"; group_kvmd="$group_default"; group_luxid="$group_default"; group_masterd="$group_default"; group_metad="$group_default"; group_noded="$group_default"; group_daemons="$group_default"; group_mond="$group_default"]) AC_SUBST(RAPI_GROUP, $group_rapi) AC_SUBST(ADMIN_GROUP, $group_admin) AC_SUBST(CONFD_GROUP, $group_confd) AC_SUBST(WCONFD_GROUP, $group_wconfd) AC_SUBST(KVMD_GROUP, $group_kvmd) AC_SUBST(LUXID_GROUP, $group_luxid) AC_SUBST(MASTERD_GROUP, $group_masterd) AC_SUBST(METAD_GROUP, $group_metad) AC_SUBST(NODED_GROUP, $group_noded) AC_SUBST(DAEMONS_GROUP, $group_daemons) AC_SUBST(MOND_GROUP, $group_mond) AC_SUBST(METAD_GROUP, $group_metad) # Print the config to the user AC_MSG_NOTICE([Running ganeti-masterd as $group_masterd:$group_masterd]) AC_MSG_NOTICE([Running ganeti-metad as $group_metad:$group_metad]) AC_MSG_NOTICE([Running ganeti-rapi as $user_rapi:$group_rapi]) AC_MSG_NOTICE([Running ganeti-confd as $user_confd:$group_confd]) AC_MSG_NOTICE([Running ganeti-wconfd as $user_wconfd:$group_wconfd]) AC_MSG_NOTICE([Running ganeti-luxid as $user_luxid:$group_luxid]) AC_MSG_NOTICE([Group for daemons is $group_daemons]) AC_MSG_NOTICE([Group for clients is $group_admin]) # --enable-drbd-barriers AC_ARG_ENABLE([drbd-barriers], [AS_HELP_STRING([--enable-drbd-barriers], m4_normalize([enable the DRBD barriers functionality by default (>= 8.0.12) (default: enabled)]))], [[if test "$enableval" != no; then DRBD_BARRIERS=n DRBD_NO_META_FLUSH=False else DRBD_BARRIERS=bf DRBD_NO_META_FLUSH=True fi ]], [DRBD_BARRIERS=n DRBD_NO_META_FLUSH=False ]) AC_SUBST(DRBD_BARRIERS, $DRBD_BARRIERS) AC_SUBST(DRBD_NO_META_FLUSH, $DRBD_NO_META_FLUSH) # --enable-syslog[=no/yes/only] AC_ARG_ENABLE([syslog], [AS_HELP_STRING([--enable-syslog], [enable use of syslog (default: disabled), one of no/yes/only])], [[case "$enableval" in no) SYSLOG=no ;; yes) SYSLOG=yes ;; only) SYSLOG=only ;; *) SYSLOG= ;; esac ]], [SYSLOG=no]) if test -z "$SYSLOG" then AC_MSG_ERROR([invalid value for syslog, choose one of no/yes/only]) fi AC_SUBST(SYSLOG_USAGE, $SYSLOG) # --enable-restricted-commands[=no/yes] AC_ARG_ENABLE([restricted-commands], [AS_HELP_STRING([--enable-restricted-commands], m4_normalize([enable restricted commands in the node daemon (default: disabled)]))], [[if test "$enableval" = no; then enable_restricted_commands=False else enable_restricted_commands=True fi ]], [enable_restricted_commands=False]) AC_SUBST(ENABLE_RESTRICTED_COMMANDS, $enable_restricted_commands) # --with-disk-separator=... AC_ARG_WITH([disk-separator], [AS_HELP_STRING([--with-disk-separator=STRING], [Disk index separator, useful if the default of ':' is handled] [ specially by the hypervisor] )], [disk_separator="$withval"], [disk_separator=":"]) AC_SUBST(DISK_SEPARATOR, $disk_separator) # Check common programs AC_PROG_INSTALL AC_PROG_LN_S # check if ln is the GNU version of ln (and hence supports -T) if ln --version 2> /dev/null | head -1 | grep -q GNU then AC_SUBST(HAS_GNU_LN, True) else AC_SUBST(HAS_GNU_LN, False) fi # Check for the ip command AC_ARG_VAR(IP_PATH, [ip path]) AC_PATH_PROG(IP_PATH, [ip], []) if test -z "$IP_PATH" then AC_MSG_ERROR([ip command not found]) fi # Check for pandoc AC_ARG_VAR(PANDOC, [pandoc path]) AC_PATH_PROG(PANDOC, [pandoc], []) if test -z "$PANDOC" then AC_MSG_WARN([pandoc not found, man pages rebuild will not be possible]) fi # Check for python-sphinx AC_ARG_VAR(SPHINX, [sphinx-build path]) AC_PATH_PROG(SPHINX, [sphinx-build], []) if test -z "$SPHINX" then AC_MSG_WARN(m4_normalize([sphinx-build not found, documentation rebuild will not be possible])) else # Sphinx exits with code 1 when it prints its usage sphinxver=`{ $SPHINX --version 2>&1 || :; } | head -n 3` if ! echo "$sphinxver" | grep -q -w -i -e '^Sphinx' -e '^Usage:'; then AC_MSG_ERROR([Unable to determine Sphinx version]) # Note: Character classes ([...]) need to be double quoted due to autoconf # using m4 elif ! echo "$sphinxver" | grep -q -i -E \ '^sphinx(-build\d?)?([[[:space:]]]+|\(sphinx-build\d?\)|v)*[[1-9]]\>'; then AC_MSG_ERROR([Sphinx 1.0 or higher is required]) fi fi AM_CONDITIONAL([HAS_SPHINX], [test -n "$SPHINX"]) AC_ARG_ENABLE([manpages-in-doc], [AS_HELP_STRING([--enable-manpages-in-doc], m4_normalize([include man pages in HTML documentation (requires sphinx; default disabled)]))], [case "$enableval" in yes) manpages_in_doc=yes ;; no) manpages_in_doc= ;; *) AC_MSG_ERROR([Bad value $enableval for --enable-manpages-in-doc]) ;; esac ], [manpages_in_doc=]) AM_CONDITIONAL([MANPAGES_IN_DOC], [test -n "$manpages_in_doc"]) AC_SUBST(MANPAGES_IN_DOC, $manpages_in_doc) if test -z "$SPHINX" -a -n "$manpages_in_doc"; then AC_MSG_ERROR([Including man pages in HTML documentation requires sphinx]) fi # Check for graphviz (dot) AC_ARG_VAR(DOT, [dot path]) AC_PATH_PROG(DOT, [dot], []) if test -z "$DOT" then AC_MSG_WARN(m4_normalize([dot (from the graphviz suite) not found, documentation rebuild not possible])) fi # Check for pylint AC_ARG_VAR(PYLINT, [pylint path]) AC_PATH_PROG(PYLINT, [pylint3], []) if test -z "$PYLINT" then AC_PATH_PROG(PYLINT, [pylint], []) if test -z "$PYLINT" then AC_MSG_WARN([pylint not found, checking code will not be possible]) else if $PYLINT --version 2>/dev/null | grep -q '^pylint 1\.' then # Make sure this is not pylint 1 AC_MSG_WARN([pylint 1.x found, checking code will not be possible. Please upgrade pylint to at least 2.0.]) PYLINT= fi fi fi # Check for pycodestyle, formerly pep8 AC_ARG_VAR(PYCODESTYLE, [pycodestyle path]) AC_PATH_PROG(PYCODESTYLE, [pycodestyle], []) if test -z "$PYCODESTYLE" then AC_MSG_WARN([pycodestyle not found, checking code will not be complete]) fi AM_CONDITIONAL([HAS_PYCODESTYLE], [test -n "$PYCODESTYLE"]) # Check for python3-coverage AC_ARG_VAR(PYCOVERAGE, [python3-coverage path]) AC_PATH_PROGS(PYCOVERAGE, [python3-coverage coverage], []) if test -z "$PYCOVERAGE" then AC_MSG_WARN(m4_normalize([python-coverage or coverage not found, evaluating Python test coverage will not be possible])) fi # Check for socat AC_ARG_VAR(SOCAT, [socat path]) AC_PATH_PROG(SOCAT, [socat], []) if test -z "$SOCAT" then AC_MSG_ERROR([socat not found]) fi # Check for qemu-img AC_ARG_VAR(QEMUIMG_PATH, [qemu-img path]) AC_PATH_PROG(QEMUIMG_PATH, [qemu-img], []) if test -z "$QEMUIMG_PATH" then AC_MSG_WARN([qemu-img not found, using ovfconverter will not be possible]) fi ENABLE_MOND= AC_ARG_ENABLE([monitoring], [AS_HELP_STRING([--enable-monitoring], [enable the ganeti monitoring daemon (default: check)])], [], [enable_monitoring=check]) # --enable-metadata ENABLE_METADATA= AC_ARG_ENABLE([metadata], [AS_HELP_STRING([--enable-metadata], [enable the ganeti metadata daemon (default: check)])], [], [enable_metadata=check]) # Check for ghc AC_ARG_VAR(GHC, [ghc path]) AC_PATH_PROG(GHC, [ghc], []) if test -z "$GHC"; then AC_MSG_FAILURE([ghc not found, compilation will not possible]) fi AC_MSG_CHECKING([checking for extra GHC flags]) GHC_BYVERSION_FLAGS= # check for GHC supported flags that vary across versions for flag in -fwarn-incomplete-uni-patterns; do if $GHC -e '0' $flag >/dev/null 2>/dev/null; then GHC_BYVERSION_FLAGS="$GHC_BYVERSION_FLAGS $flag" fi done AC_MSG_RESULT($GHC_BYVERSION_FLAGS) AC_SUBST(GHC_BYVERSION_FLAGS) # Check for ghc-pkg AC_ARG_VAR(GHC_PKG, [ghc-pkg path]) AC_PATH_PROG(GHC_PKG, [ghc-pkg], []) if test -z "$GHC_PKG"; then AC_MSG_FAILURE([ghc-pkg not found, compilation will not be possible]) fi # Check for cabal AC_ARG_VAR(CABAL, [cabal path]) AC_PATH_PROG(CABAL, [cabal], []) if test -z "$CABAL"; then AC_MSG_FAILURE([cabal not found, compilation will not be possible]) fi AC_MSG_CHECKING([for the appropriate cabal configure command]) if $CABAL --help 2>&1 | grep -qw v1-configure; then CABAL_CONFIGURE_CMD="v1-configure" else CABAL_CONFIGURE_CMD="configure" fi AC_MSG_RESULT($CABAL_CONFIGURE_CMD) AC_SUBST(CABAL_CONFIGURE_CMD) # check for standard modules AC_GHC_PKG_REQUIRE(Cabal) AC_GHC_PKG_REQUIRE(curl) AC_GHC_PKG_REQUIRE(json) AC_GHC_PKG_REQUIRE(network) AC_GHC_PKG_REQUIRE(mtl) AC_GHC_PKG_REQUIRE(bytestring) AC_GHC_PKG_REQUIRE(base64-bytestring-1.*, t) AC_GHC_PKG_REQUIRE(utf8-string) AC_GHC_PKG_REQUIRE(zlib) AC_GHC_PKG_REQUIRE(hslogger) AC_GHC_PKG_REQUIRE(process) AC_GHC_PKG_REQUIRE(attoparsec) AC_GHC_PKG_REQUIRE(vector) AC_GHC_PKG_REQUIRE(text) AC_GHC_PKG_REQUIRE(hinotify) AC_GHC_PKG_REQUIRE(cryptonite) AC_GHC_PKG_REQUIRE(lifted-base) AC_GHC_PKG_REQUIRE(lens) AC_GHC_PKG_REQUIRE(old-time) AC_GHC_PKG_REQUIRE(temporary) case "$hs_pcre_backend" in auto) AC_GHC_PKG_CHECK([regex-pcre], [hs_pcre_backend=pcre], [AC_GHC_PKG_CHECK([regex-pcre2], [hs_pcre_backend=pcre2], [AC_GHC_PKG_CHECK([regex-pcre-builtin], [hs_pcre_backend=pcre-builtin], [AC_GHC_PKG_CHECK([regex-tdfa], [hs_pcre_backend=tdfa], [ AC_MSG_ERROR([No supported Haskell PCRE library found]) ])])])]) ;; tdfa) AC_GHC_PKG_REQUIRE(regex-tdfa) ;; pcre) AC_GHC_PKG_REQUIRE(regex-pcre) ;; pcre2) AC_GHC_PKG_REQUIRE(regex-pcre2) ;; pcre-builtin) AC_GHC_PKG_REQUIRE(regex-pcre-builtin) ;; esac AC_SUBST(HS_PCRE_BACKEND, $hs_pcre_backend) #extra modules for monitoring daemon functionality; also needed for tests MONITORING_PKG= AC_GHC_PKG_CHECK([snap-server], [], [NS_NODEV=1; MONITORING_PKG="$MONITORING_PKG snap-server"]) AC_GHC_PKG_CHECK([PSQueue], [], [NS_NODEV=1; MONITORING_PKG="$MONITORING_PKG PSQueue"]) has_monitoring=False if test "$enable_monitoring" != no; then MONITORING_DEP= has_monitoring_pkg=False if test -z "$MONITORING_PKG"; then has_monitoring_pkg=True elif test "$enable_monitoring" = check; then AC_MSG_WARN(m4_normalize([The required extra libraries for the monitoring daemon were not found ($MONITORING_PKG), monitoring disabled])) else AC_MSG_FAILURE(m4_normalize([The monitoring functionality was requested, but required libraries were not found: $MONITORING_PKG])) fi has_monitoring_dep=False if test -z "$MONITORING_DEP"; then has_monitoring_dep=True elif test "$enable_monitoring" = check; then AC_MSG_WARN(m4_normalize([The optional Ganeti components required for the monitoring agent were not enabled ($MONITORING_DEP), monitoring disabled])) else AC_MSG_FAILURE(m4_normalize([The monitoring functionality was requested, but required optional Ganeti components were not found: $MONITORING_DEP])) fi fi if test "$has_monitoring_pkg" = True -a "$has_monitoring_dep" = True; then has_monitoring=True AC_MSG_NOTICE([Enabling the monitoring agent usage]) fi AC_SUBST(ENABLE_MOND, $has_monitoring) AM_CONDITIONAL([ENABLE_MOND], [test "$has_monitoring" = True]) # extra modules for metad functionality; also needed for tests METAD_PKG= AC_GHC_PKG_CHECK([snap-server], [], [NS_NODEV=1; METAD_PKG="$METAD_PKG snap-server"]) has_metad=False if test "$enable_metadata" != no; then if test -z "$METAD_PKG"; then has_metad=True elif test "$enable_metadata" = check; then AC_MSG_WARN(m4_normalize([The required extra libraries for metad were not found ($METAD_PKG), metad disabled])) else AC_MSG_FAILURE(m4_normalize([The metadata functionality was requested, but required libraries were not found: $METAD_PKG])) fi fi if test "$has_metad" = True; then AC_MSG_NOTICE([Enabling metadata usage]) fi AC_SUBST(ENABLE_METADATA, $has_metad) AM_CONDITIONAL([ENABLE_METADATA], [test x$has_metad = xTrue]) # network socket split package AC_GHC_PKG_CHECK([network-bsd], [HS_NETWORK_BSD=True], [HS_NETWORK_BSD=False], t) AC_SUBST(ENABLE_NETWORK_BSD, $HS_NETWORK_BSD) AM_CONDITIONAL([ENABLE_NETWORK_BSD], [test x$HS_NETWORK_BSD=xTrue]) # development modules AC_GHC_PKG_CHECK([QuickCheck-2.*], [], [HS_NODEV=1], t) AC_GHC_PKG_CHECK([test-framework-0.6*], [], [ AC_GHC_PKG_CHECK([test-framework-0.7*], [], [ AC_GHC_PKG_CHECK([test-framework-0.8*], [], [HS_NODEV=1], t) ], t) ], t) AC_GHC_PKG_CHECK([test-framework-hunit], [], [HS_NODEV=1]) AC_GHC_PKG_CHECK([test-framework-quickcheck2], [], [HS_NODEV=1]) AC_GHC_PKG_CHECK([temporary], [], [HS_NODEV=1]) if test -n "$HS_NODEV"; then AC_MSG_WARN(m4_normalize([Required development modules were not found, you won't be able to run Haskell unittests])) else AC_MSG_NOTICE([Haskell development modules found, unittests enabled]) fi AC_SUBST(HS_NODEV) AM_CONDITIONAL([HS_UNIT], [test -n $HS_NODEV]) # Check for HsColour HS_APIDOC=no AC_ARG_VAR(HSCOLOUR, [HsColour path]) AC_PATH_PROG(HSCOLOUR, [HsColour], []) if test -z "$HSCOLOUR"; then AC_MSG_WARN(m4_normalize([HsColour not found, htools API documentation will not be generated])) fi # Check for haddock AC_ARG_VAR(HADDOCK, [haddock path]) AC_PATH_PROG(HADDOCK, [haddock], []) if test -z "$HADDOCK"; then AC_MSG_WARN(m4_normalize([haddock not found, htools API documentation will not be generated])) fi if test -n "$HADDOCK" && test -n "$HSCOLOUR"; then HS_APIDOC=yes fi AC_SUBST(HS_APIDOC) # Check for hlint AC_ARG_VAR(HLINT, [hlint path]) AC_PATH_PROG(HLINT, [hlint], []) if test -z "$HLINT"; then AC_MSG_WARN([hlint not found, checking code will not be possible]) fi AM_CONDITIONAL([WANT_HSTESTS], [test "x$HS_NODEV" = x]) AM_CONDITIONAL([WANT_HSAPIDOC], [test "$HS_APIDOC" = yes]) AM_CONDITIONAL([HAS_HLINT], [test "$HLINT"]) # Check for fakeroot AC_ARG_VAR(FAKEROOT_PATH, [fakeroot path]) AC_PATH_PROG(FAKEROOT_PATH, [fakeroot], []) if test -z "$FAKEROOT_PATH"; then AC_MSG_WARN(m4_normalize([fakeroot not found, tests that must run as root will not be executed])) fi AM_CONDITIONAL([HAS_FAKEROOT], [test "x$FAKEROOT_PATH" != x]) SOCAT_USE_ESCAPE= AC_ARG_ENABLE([socat-escape], [AS_HELP_STRING([--enable-socat-escape], [use escape functionality available in socat >= 1.7 (default: detect automatically)])], [[if test "$enableval" = yes; then SOCAT_USE_ESCAPE=True else SOCAT_USE_ESCAPE=False fi ]]) if test -z "$SOCAT_USE_ESCAPE" then if $SOCAT -hh | grep -w -q escape; then SOCAT_USE_ESCAPE=True else SOCAT_USE_ESCAPE=False fi fi AC_SUBST(SOCAT_USE_ESCAPE) SOCAT_USE_COMPRESS= AC_ARG_ENABLE([socat-compress], [AS_HELP_STRING([--enable-socat-compress], [use OpenSSL compression option available in patched socat builds (see INSTALL for details; default: detect automatically)])], [[if test "$enableval" = yes; then SOCAT_USE_COMPRESS=True else SOCAT_USE_COMPRESS=False fi ]]) if test -z "$SOCAT_USE_COMPRESS" then if $SOCAT -hhh | grep -w -q openssl-compress; then SOCAT_USE_COMPRESS=True else SOCAT_USE_COMPRESS=False fi fi AC_SUBST(SOCAT_USE_COMPRESS) if man --help | grep -q -e --warnings then MAN_HAS_WARNINGS=1 else MAN_HAS_WARNINGS= AC_MSG_WARN(m4_normalize([man does not support --warnings, man page checks will not be possible])) fi AC_SUBST(MAN_HAS_WARNINGS) # Check for Python # We need a Python3 interpreter, version at least 3.6. # We tune _AM_PYTHON_INTERPRETER_LIST to first check interpreters that are # likely interpreters for Python 3. m4_define_default([_AM_PYTHON_INTERPRETER_LIST], [python3 python3.8 python3.7 python3.6 python]) AM_PATH_PYTHON([3.6]) AC_PYTHON_MODULE(OpenSSL, t) AC_PYTHON_MODULE(pyparsing, t) AC_PYTHON_MODULE(pyinotify, t) AC_PYTHON_MODULE(pycurl, t) AC_PYTHON_MODULE(bitarray, t) AC_PYTHON_MODULE(psutil) AC_PYTHON_MODULE(paramiko) # Development-only Python modules PY_NODEV= AC_PYTHON_MODULE(yaml) if test $HAVE_PYMOD_YAML == "no"; then PY_NODEV="$PY_NODEV yaml" fi AC_PYTHON_MODULE(pytest) if test $HAVE_PYMOD_PYTEST == "no"; then PY_NODEV="$PY_NODEV pytest" fi if test -n "$PY_NODEV"; then AC_MSG_WARN(m4_normalize([Required development modules ($PY_NODEV) were not found, you won't be able to run Python unittests])) else AC_MSG_NOTICE([Python development modules found, unittests enabled]) fi AC_SUBST(PY_NODEV) AM_CONDITIONAL([PY_UNIT], [test -z $PY_NODEV]) include_makefile_ghc=' ifneq ($(MAKECMDGOALS),ganeti) ifneq ($(MAKECMDGOALS),clean) ifneq ($(MAKECMDGOALS),distclean) include Makefile.ghc endif endif endif ' AC_SUBST([include_makefile_ghc]) AM_SUBST_NOTMAKE([include_makefile_ghc]) AC_CONFIG_FILES([ Makefile ]) AC_OUTPUT ganeti-3.1.0~rc2/daemons/000075500000000000000000000000001476477700300152325ustar00rootroot00000000000000ganeti-3.1.0~rc2/daemons/daemon-util.in000064400000000000000000000243311476477700300200030ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2009, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e @SHELL_ENV_INIT@ readonly defaults_file="$SYSCONFDIR/default/ganeti" # This is a list of all daemons and the order in which they're started. The # order is important as there are dependencies between them. On shutdown, # they're stopped in reverse order. DAEMONS=( ganeti-noded ganeti-confd ganeti-wconfd ganeti-rapi ganeti-luxid ganeti-kvmd ) # This is the list of daemons that are loaded on demand; they should only be # stopped, not started. ON_DEMAND_DAEMONS=( ganeti-metad ) _mond_enabled() { [[ "@CUSTOM_ENABLE_MOND@" == True ]] } if _mond_enabled; then DAEMONS+=( ganeti-mond ) fi # The full list of all daemons we know about ALL_DAEMONS=( ${DAEMONS[@]} ${ON_DEMAND_DAEMONS[@]} ) NODED_ARGS= CONFD_ARGS= WCONFD_ARGS= LUXID_ARGS= RAPI_ARGS= MOND_ARGS= METAD_ARGS= KVMD_ARGS= # Read defaults file if it exists if [[ -s $defaults_file ]]; then . $defaults_file fi # Meant to facilitate use utilities in /etc/rc.d/init.d/functions in case # start-stop-daemon is not available. _ignore_error() { eval "$@" || : } _daemon_pidfile() { echo "$RUN_DIR/$1.pid" } _daemon_executable() { echo "@PREFIX@/sbin/$1" } _daemon_usergroup() { case "$1" in confd) echo "@GNTCONFDUSER@:@GNTCONFDGROUP@" ;; wconfd) echo "@GNTWCONFDUSER@:@GNTWCONFDGROUP@" ;; luxid) echo "@GNTLUXIDUSER@:@GNTLUXIDGROUP@" ;; rapi) echo "@GNTRAPIUSER@:@GNTRAPIGROUP@" ;; noded) echo "@GNTNODEDUSER@:@GNTNODEDGROUP@" ;; mond) echo "@GNTMONDUSER@:@GNTMONDGROUP@" ;; metad) echo "@GNTMETADUSER@:@GNTMETADGROUP@" ;; *) echo "root:@GNTDAEMONSGROUP@" ;; esac } # Specifies the additional capabilities needed by individual daemons _daemon_caps() { case "$1" in metad) echo "cap_net_bind_service=+ep" ;; *) echo "" ;; esac } # Checks whether the local machine is part of a cluster check_config() { local server_pem=$DATA_DIR/server.pem local fname for fname in $server_pem; do if [[ ! -f $fname ]]; then echo "Missing configuration file $fname" >&2 return 1 fi done return 0 } # Checks the exit code of a daemon check_exitcode() { if [[ "$#" -lt 1 ]]; then echo 'Missing exit code.' >&2 return 1 fi local rc="$1"; shift case "$rc" in 0) ;; 11) echo "not master" ;; *) echo "exit code $rc" return 1 ;; esac return 0 } # Checks if we should use systemctl to start/stop daemons use_systemctl() { # Is systemd running as PID 1? [ -d /run/systemd/system ] || return 1 type -p systemctl >/dev/null || return 1 # Does systemd know about Ganeti at all? loadstate="$(systemctl show -pLoadState ganeti.target)" if [ "$loadstate" = "LoadState=loaded" ]; then return 0 fi return 1 } # Prints path to PID file for a daemon. daemon_pidfile() { if [[ "$#" -lt 1 ]]; then echo 'Missing daemon name.' >&2 return 1 fi local name="$1"; shift _daemon_pidfile $name } # Prints path to daemon executable. daemon_executable() { if [[ "$#" -lt 1 ]]; then echo 'Missing daemon name.' >&2 return 1 fi local name="$1"; shift _daemon_executable $name } # Prints a list of all daemons in the order in which they should be started list_start_daemons() { local name for name in "${DAEMONS[@]}"; do echo "$name" done } # Prints a list of all daemons in the order in which they should be stopped list_stop_daemons() { for name in "${ALL_DAEMONS[@]}"; do echo "$name" done | tac } # Checks whether a daemon name is known is_daemon_name() { if [[ "$#" -lt 1 ]]; then echo 'Missing daemon name.' >&2 return 1 fi local name="$1"; shift for i in "${ALL_DAEMONS[@]}"; do if [[ "$i" == "$name" ]]; then return 0 fi done echo "Unknown daemon name '$name'" >&2 return 1 } # Checks whether daemon is running check() { if [[ "$#" -lt 1 ]]; then echo 'Missing daemon name.' >&2 return 1 fi local name="$1"; shift local pidfile=$(_daemon_pidfile $name) local daemonexec=$(_daemon_executable $name) if use_systemctl; then activestate="$(systemctl show -pActiveState "${name}.service")" if [ "$activestate" = "ActiveState=active" ]; then return 0 else return 1 fi elif type -p start-stop-daemon >/dev/null; then start-stop-daemon --stop --signal 0 --quiet \ --pidfile $pidfile --name "$name" else _ignore_error status \ -p $pidfile \ $daemonexec fi } # Starts a daemon start() { if [[ "$#" -lt 1 ]]; then echo 'Missing daemon name.' >&2 return 1 fi local name="$1"; shift # Convert daemon name to uppercase after removing "ganeti-" prefix local plain_name=${name#ganeti-} local ucname=$(tr a-z A-Z <<<$plain_name) local pidfile=$(_daemon_pidfile $name) local usergroup=$(_daemon_usergroup $plain_name) local daemonexec=$(_daemon_executable $name) if use_systemctl; then local onetime_conf="${DATA_DIR}/${name}.onetime.conf" echo "ONETIME_ARGS=$@" > "$onetime_conf" systemctl start "${name}.service" ret=$? rm -f "$onetime_conf" return $? fi # Read $_ARGS and $EXTRA__ARGS eval local args="\"\$${ucname}_ARGS \$EXTRA_${ucname}_ARGS\"" @PKGLIBDIR@/ensure-dirs # Grant capabilities to daemons that need them local daemoncaps=$(_daemon_caps $plain_name) if [[ "$daemoncaps" != "" ]]; then if type -p setcap >/dev/null; then setcap $daemoncaps $(readlink -f $daemonexec) else echo "setcap missing, could not set capabilities for $name." >&2 return 1 fi fi if type -p start-stop-daemon >/dev/null; then start-stop-daemon --start --quiet --oknodo \ --pidfile $pidfile \ --startas $daemonexec \ --chuid $usergroup \ -- $args "$@" else # TODO: Find a way to start daemon with a group, until then the group must # be removed _ignore_error daemon \ --pidfile $pidfile \ --user ${usergroup%:*} \ $daemonexec $args "$@" fi } # Stops a daemon stop() { if [[ "$#" -lt 1 ]]; then echo 'Missing daemon name.' >&2 return 1 fi local name="$1"; shift local pidfile=$(_daemon_pidfile $name) if use_systemctl; then systemctl stop "${name}.service" elif type -p start-stop-daemon >/dev/null; then start-stop-daemon --stop --quiet --oknodo --retry 30 \ --pidfile $pidfile --remove-pidfile --name "$name" else _ignore_error killproc -p $pidfile $name fi } # Starts a daemon if it's not yet running check_and_start() { local name="$1" if ! check $name; then if use_systemctl; then echo "${name} supervised by systemd but not running, will not restart." return 1 fi start $name fi } # Starts the master role start_master() { if use_systemctl; then systemctl start ganeti-master.target else start ganeti-wconfd start ganeti-rapi start ganeti-luxid fi } # Stops the master role stop_master() { if use_systemctl; then systemctl stop ganeti-master.target else stop ganeti-luxid stop ganeti-rapi stop ganeti-wconfd fi } # Start all daemons start_all() { use_systemctl && systemctl start ganeti.target # Fall through so that we detect any errors. for i in $(list_start_daemons); do local rc=0 # Try to start daemon start $i || rc=$? if ! errmsg=$(check_exitcode $rc); then echo "$errmsg" >&2 return 1 fi done return 0 } # Stop all daemons stop_all() { if use_systemctl; then systemctl stop ganeti.target else for i in $(list_stop_daemons); do stop $i done fi } # SIGHUP a process to force re-opening its logfiles rotate_logs() { if [[ "$#" -lt 1 ]]; then echo 'Missing daemon name.' >&2 return 1 fi local name="$1"; shift local pidfile=$(_daemon_pidfile $name) local daemonexec=$(_daemon_executable $name) if type -p start-stop-daemon >/dev/null; then start-stop-daemon --stop --signal HUP --quiet \ --oknodo --pidfile $pidfile --name "$name" else _ignore_error killproc \ -p $pidfile \ $daemonexec -HUP fi } # SIGHUP all processes rotate_all_logs() { for i in $(list_stop_daemons); do rotate_logs $i done } # Reloads the SSH keys reload_ssh_keys() { @RPL_SSHD_RESTART_COMMAND@ } # Read @SYSCONFDIR@/rc.d/init.d/functions if start-stop-daemon not available if ! type -p start-stop-daemon >/dev/null && \ [[ -f @SYSCONFDIR@/rc.d/init.d/functions ]]; then _ignore_error . @SYSCONFDIR@/rc.d/init.d/functions fi if [[ "$#" -lt 1 ]]; then echo "Usage: $0 " >&2 exit 1 fi orig_action=$1; shift if [[ "$orig_action" == *_* ]]; then echo "Command must not contain underscores" >&2 exit 1 fi # Replace all dashes (-) with underlines (_) action=${orig_action//-/_} # Is it a known function? if ! declare -F "$action" >/dev/null 2>&1; then echo "Unknown command: $orig_action" >&2 exit 1 fi # Call handler function $action "$@" ganeti-3.1.0~rc2/daemons/ganeti-cleaner.in000064400000000000000000000070761476477700300204520ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2009, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e -u @SHELL_ENV_INIT@ # Overridden by unittest : ${CHECK_CERT_EXPIRED:=$PKGLIBDIR/check-cert-expired} usage() { echo "Usage: $0 node|master" 2>&1 exit $1 } if [[ "$#" -ne 1 ]]; then usage 1 fi case "$1" in node) readonly CLEANER_LOG_DIR=$LOG_DIR/cleaner ;; master) readonly CLEANER_LOG_DIR=$LOG_DIR/master-cleaner ;; --help-completion) echo "choices=node,master 1 1" exit 0 ;; --help) usage 0 ;; *) usage 1 ;; esac readonly CRYPTO_DIR=$RUN_DIR/crypto readonly QUEUE_ARCHIVE_DIR=$DATA_DIR/queue/archive in_cluster() { [[ -e $DATA_DIR/ssconf_master_node ]] } cleanup_node() { # Return if directory for crypto keys doesn't exist [[ -d $CRYPTO_DIR ]] || return 0 find $CRYPTO_DIR -mindepth 1 -maxdepth 1 -type d | \ while read dir; do if $CHECK_CERT_EXPIRED $dir/cert; then rm -vf $dir/{cert,key} rmdir -v --ignore-fail-on-non-empty $dir fi done } cleanup_watcher() { # Return if machine is not part of a cluster in_cluster || return 0 # Remove old watcher files find $DATA_DIR -maxdepth 1 -type f -mtime +$REMOVE_AFTER \ \( -name 'watcher.*-*-*-*.data' -or \ -name 'watcher.*-*-*-*.instance-status' \) -print0 | \ xargs -r0 rm -vf } cleanup_master() { # Return if machine is not part of a cluster in_cluster || return 0 # Return if queue archive directory doesn't exist [[ -d $QUEUE_ARCHIVE_DIR ]] || return 0 # Remove old jobs find $QUEUE_ARCHIVE_DIR -mindepth 2 -type f -mtime +$REMOVE_AFTER -print0 | \ xargs -r0 rm -vf } # Define how many days archived jobs should be left alone REMOVE_AFTER=21 # Define how many log files to keep around (usually one per day) KEEP_LOGS=50 # Log file for this run LOG_FILE=$CLEANER_LOG_DIR/cleaner-$(date +'%Y-%m-%dT%H_%M').$$.log # Create log directory mkdir -p $CLEANER_LOG_DIR # Redirect all output to log file exec >>$LOG_FILE 2>&1 echo "Cleaner started at $(date)" # Switch to a working directory accessible to the cleaner cd $CLEANER_LOG_DIR # Remove old cleaner log files find $CLEANER_LOG_DIR -maxdepth 1 -type f | sort | head -n -$KEEP_LOGS | \ xargs -r rm -vf case "$1" in node) cleanup_node cleanup_watcher ;; master) cleanup_master ;; esac exit 0 ganeti-3.1.0~rc2/daemons/import-export000075500000000000000000000537231476477700300200230ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Import/export daemon. """ # pylint: disable=C0103 # C0103: Invalid name import-export import errno import logging import optparse import os import select import signal import subprocess import sys import time import math from ganeti import constants from ganeti import cli from ganeti import utils from ganeti import errors from ganeti import serializer from ganeti import objects from ganeti import impexpd from ganeti import netutils #: How many lines to keep in the status file MAX_RECENT_OUTPUT_LINES = 20 #: Don't update status file more than once every 5 seconds (unless forced) MIN_UPDATE_INTERVAL = 5.0 #: How long to wait for a connection to be established DEFAULT_CONNECT_TIMEOUT = 60 #: Get dd(1) statistics every few seconds DD_STATISTICS_INTERVAL = 5.0 #: Seconds for throughput calculation DD_THROUGHPUT_INTERVAL = 60.0 #: Number of samples for throughput calculation DD_THROUGHPUT_SAMPLES = int(math.ceil(float(DD_THROUGHPUT_INTERVAL) / DD_STATISTICS_INTERVAL)) # Global variable for options options = None def SetupLogging(): """Configures the logging module. """ formatter = logging.Formatter("%(asctime)s: %(message)s") stderr_handler = logging.StreamHandler() stderr_handler.setFormatter(formatter) stderr_handler.setLevel(logging.NOTSET) root_logger = logging.getLogger("") root_logger.addHandler(stderr_handler) if options.debug: root_logger.setLevel(logging.NOTSET) elif options.verbose: root_logger.setLevel(logging.INFO) else: root_logger.setLevel(logging.ERROR) # Create special logger for child process output child_logger = logging.Logger("child output") child_logger.addHandler(stderr_handler) child_logger.setLevel(logging.NOTSET) return child_logger class StatusFile(object): """Status file manager. """ def __init__(self, path): """Initializes class. """ self._path = path self._data = objects.ImportExportStatus(ctime=time.time(), mtime=None, recent_output=[]) def AddRecentOutput(self, line): """Adds a new line of recent output. """ self._data.recent_output.append(line) # Remove old lines del self._data.recent_output[:-MAX_RECENT_OUTPUT_LINES] def SetListenPort(self, port): """Sets the port the daemon is listening on. @type port: int @param port: TCP/UDP port """ assert isinstance(port, int) and 0 < port < (2 ** 16) self._data.listen_port = port def GetListenPort(self): """Returns the port the daemon is listening on. """ return self._data.listen_port def SetConnected(self): """Sets the connected flag. """ self._data.connected = True def GetConnected(self): """Determines whether the daemon is connected. """ return self._data.connected def SetProgress(self, mbytes, throughput, percent, eta): """Sets how much data has been transferred so far. @type mbytes: number @param mbytes: Transferred amount of data in MiB. @type throughput: float @param throughput: MiB/second @type percent: number @param percent: Percent processed @type eta: number @param eta: Expected number of seconds until done """ self._data.progress_mbytes = mbytes self._data.progress_throughput = throughput self._data.progress_percent = percent self._data.progress_eta = eta def SetExitStatus(self, exit_status, error_message): """Sets the exit status and an error message. """ # Require error message when status isn't 0 assert exit_status == 0 or error_message self._data.exit_status = exit_status self._data.error_message = error_message def ExitStatusIsSuccess(self): """Returns whether the exit status means "success". """ return not bool(self._data.error_message) def Update(self, force): """Updates the status file. @type force: bool @param force: Write status file in any case, not only when minimum interval is expired """ if not (force or self._data.mtime is None or time.time() > (self._data.mtime + MIN_UPDATE_INTERVAL)): return logging.debug("Updating status file %s", self._path) self._data.mtime = time.time() utils.WriteFile(self._path, data=serializer.DumpJson(self._data.ToDict()), mode=0o400) def ProcessChildIO(child, socat_stderr_read_fd, dd_stderr_read_fd, dd_pid_read_fd, exp_size_read_fd, status_file, child_logger, signal_notify, signal_handler, mode): """Handles the child processes' output. """ assert not (signal_handler.signum - set([signal.SIGTERM, signal.SIGINT])), \ "Other signals are not handled in this function" # Buffer size 0 is important, otherwise .read() with a specified length # might buffer data while poll(2) won't mark its file descriptor as # readable again. socat_stderr_read = os.fdopen(socat_stderr_read_fd, "rb", 0) dd_stderr_read = os.fdopen(dd_stderr_read_fd, "rb", 0) dd_pid_read = os.fdopen(dd_pid_read_fd, "rb", 0) exp_size_read = os.fdopen(exp_size_read_fd, "rb", 0) tp_samples = DD_THROUGHPUT_SAMPLES if options.exp_size == constants.IE_CUSTOM_SIZE: exp_size = None else: exp_size = options.exp_size child_io_proc = impexpd.ChildIOProcessor(options.debug, status_file, child_logger, tp_samples, exp_size) try: fdmap = { child.stderr.fileno(): (child.stderr, child_io_proc.GetLineSplitter(impexpd.PROG_OTHER)), socat_stderr_read.fileno(): (socat_stderr_read, child_io_proc.GetLineSplitter(impexpd.PROG_SOCAT)), dd_pid_read.fileno(): (dd_pid_read, child_io_proc.GetLineSplitter(impexpd.PROG_DD_PID)), dd_stderr_read.fileno(): (dd_stderr_read, child_io_proc.GetLineSplitter(impexpd.PROG_DD)), exp_size_read.fileno(): (exp_size_read, child_io_proc.GetLineSplitter(impexpd.PROG_EXP_SIZE)), signal_notify.fileno(): (signal_notify, None), } poller = select.poll() for fd in fdmap: utils.SetNonblockFlag(fd, True) poller.register(fd, select.POLLIN) if options.connect_timeout and mode == constants.IEM_IMPORT: listen_timeout = utils.RunningTimeout(options.connect_timeout, True) else: listen_timeout = None exit_timeout = None dd_stats_timeout = None while True: # Break out of loop if only signal notify FD is left if len(fdmap) == 1 and signal_notify.fileno() in fdmap: break timeout = None if listen_timeout and not exit_timeout: assert mode == constants.IEM_IMPORT and options.connect_timeout if status_file.GetConnected(): listen_timeout = None elif listen_timeout.Remaining() < 0: errmsg = ("Child process didn't establish connection in time" " (%0.0fs), sending SIGTERM" % options.connect_timeout) logging.error(errmsg) status_file.AddRecentOutput(errmsg) status_file.Update(True) child.Kill(signal.SIGTERM) exit_timeout = \ utils.RunningTimeout(constants.CHILD_LINGER_TIMEOUT, True) # Next block will calculate timeout else: # Not yet connected, check again in a second timeout = 1000 if exit_timeout: timeout = exit_timeout.Remaining() * 1000 if timeout < 0: logging.info("Child process didn't exit in time") break if (not dd_stats_timeout) or dd_stats_timeout.Remaining() < 0: notify_status = child_io_proc.NotifyDd() if notify_status: # Schedule next notification dd_stats_timeout = utils.RunningTimeout(DD_STATISTICS_INTERVAL, True) else: # Try again soon (dd isn't ready yet) dd_stats_timeout = utils.RunningTimeout(1.0, True) if dd_stats_timeout: dd_timeout = max(0, dd_stats_timeout.Remaining() * 1000) if timeout is None: timeout = dd_timeout else: timeout = min(timeout, dd_timeout) for fd, event in utils.RetryOnSignal(poller.poll, timeout): if event & (select.POLLIN | event & select.POLLPRI): (from_, to) = fdmap[fd] # Read up to 1 KB of data data = from_.read(1024) # On error, remove the mapping if not data: poller.unregister(fd) del fdmap[fd] continue # If the data needs to be sent to another fd, write it if to: to.write(data) continue # Did we get a signal? if fd != signal_notify.fileno(): continue # Has it been handled? if not signal_handler.called: continue # If so, clean up after it. signal_handler.Clear() if exit_timeout: logging.info("Child process still has about %0.2f seconds" " to exit", exit_timeout.Remaining()) else: logging.info("Giving child process %0.2f seconds to exit", constants.CHILD_LINGER_TIMEOUT) exit_timeout = \ utils.RunningTimeout(constants.CHILD_LINGER_TIMEOUT, True) elif event & (select.POLLNVAL | select.POLLHUP | select.POLLERR): poller.unregister(fd) del fdmap[fd] child_io_proc.FlushAll() # If there was a timeout calculator, we were waiting for the child to # finish, e.g. due to a signal return not bool(exit_timeout) finally: child_io_proc.CloseAll() def ParseOptions(): """Parses the options passed to the program. @return: Arguments to program """ global options # pylint: disable=W0603 parser = optparse.OptionParser(usage=("%%prog {%s|%s}" % (constants.IEM_IMPORT, constants.IEM_EXPORT))) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option("--key", dest="key", action="store", type="string", help="RSA key file") parser.add_option("--cert", dest="cert", action="store", type="string", help="X509 certificate file") parser.add_option("--ca", dest="ca", action="store", type="string", help="X509 CA file") parser.add_option("--bind", dest="bind", action="store", type="string", help="Bind address") parser.add_option("--ipv4", dest="ipv4", action="store_true", help="Use IPv4 only") parser.add_option("--ipv6", dest="ipv6", action="store_true", help="Use IPv6 only") parser.add_option("--host", dest="host", action="store", type="string", help="Remote hostname") parser.add_option("--port", dest="port", action="store", type="int", help="Remote port") parser.add_option("--connect-retries", dest="connect_retries", action="store", type="int", default=0, help=("How many times the connection should be retried" " (export only)")) parser.add_option("--connect-timeout", dest="connect_timeout", action="store", type="int", default=DEFAULT_CONNECT_TIMEOUT, help="Timeout for connection to be established (seconds)") parser.add_option("--compress", dest="compress", action="store", type="string", help="Compression method", default=constants.IEC_GZIP) parser.add_option("--expected-size", dest="exp_size", action="store", type="string", default=None, help="Expected import/export size (MiB)") parser.add_option("--magic", dest="magic", action="store", type="string", default=None, help="Magic string") parser.add_option("--cmd-prefix", dest="cmd_prefix", action="store", type="string", help="Command prefix") parser.add_option("--cmd-suffix", dest="cmd_suffix", action="store", type="string", help="Command suffix") (options, args) = parser.parse_args() if len(args) != 2: # Won't return parser.error("Expected exactly two arguments") (status_file_path, mode) = args if mode not in (constants.IEM_IMPORT, constants.IEM_EXPORT): # Won't return parser.error("Invalid mode: %s" % mode) # Normalize and check parameters if options.host is not None and not netutils.IPAddress.IsValid(options.host): try: options.host = netutils.Hostname.GetNormalizedName(options.host) except errors.OpPrereqError as err: parser.error("Invalid hostname '%s': %s" % (options.host, err)) if options.port is not None: options.port = utils.ValidateServiceName(options.port) if (options.exp_size is not None and options.exp_size != constants.IE_CUSTOM_SIZE): try: options.exp_size = int(options.exp_size) except (ValueError, TypeError) as err: # Won't return parser.error("Invalid value for --expected-size: %s (%s)" % (options.exp_size, err)) if not (options.magic is None or constants.IE_MAGIC_RE.match(options.magic)): parser.error("Magic must match regular expression %s" % constants.IE_MAGIC_RE.pattern) if options.ipv4 and options.ipv6: parser.error("Can only use one of --ipv4 and --ipv6") return (status_file_path, mode) # Return code signifying that no program was found PROGRAM_NOT_FOUND_RCODE = 127 def _RunWithTimeout(cmd, timeout, silent=False): """Runs a command, killing it if a timeout was reached. Uses the alarm signal, not thread-safe. Waits regardless of whether the command exited early. @type timeout: number @param timeout: Timeout, in seconds @type silent: Boolean @param silent: Whether command output should be suppressed @rtype: tuple of (bool, int) @return: Whether the command timed out, and the return code """ try: if silent: with open(os.devnull, 'wb') as null_fd: p = subprocess.Popen(cmd, stdout=null_fd, stderr=null_fd) else: p = subprocess.Popen(cmd) except OSError: return False, PROGRAM_NOT_FOUND_RCODE time.sleep(timeout) timed_out = False status = p.poll() if status is None: timed_out = True p.kill() return timed_out, p.wait() CHECK_SWITCH = "-h" def VerifyOptions(): """Performs various runtime checks to make sure the options are valid. """ if options.compress != constants.IEC_NONE: utility_name = constants.IEC_COMPRESSION_UTILITIES.get(options.compress, options.compress) timed_out, rcode = \ _RunWithTimeout([utility_name, CHECK_SWITCH], 2, silent=True) if timed_out: raise Exception("The invoked utility has timed out - the %s switch to" " check for presence must be supported" % CHECK_SWITCH) if rcode != 0: raise Exception("Verification attempt of selected compression method %s" " failed - check that %s is present and can be invoked" " safely with the %s switch" % (options.compress, utility_name, CHECK_SWITCH)) class ChildProcess(subprocess.Popen): def __init__(self, env, cmd, pass_fds): """Initializes this class. """ # Not using close_fds because doing so would also close the socat stderr # pipe, which we still need. subprocess.Popen.__init__(self, cmd, env=env, shell=False, pass_fds=pass_fds, stderr=subprocess.PIPE, stdout=None, stdin=None, preexec_fn=self._ChildPreexec) self._SetProcessGroup() def _ChildPreexec(self): """Called before child executable is execve'd. """ # Move to separate process group. By sending a signal to its process group # we can kill the child process and all grandchildren. os.setpgid(0, 0) def _SetProcessGroup(self): """Sets the child's process group. """ assert self.pid, "Can't be called in child process" # Avoid race condition by setting child's process group (as good as # possible in Python) before sending signals to child. For an # explanation, see preexec function for child. try: os.setpgid(self.pid, self.pid) except EnvironmentError as err: # If the child process was faster we receive EPERM or EACCES if err.errno not in (errno.EPERM, errno.EACCES): raise def Kill(self, signum): """Sends signal to child process. """ logging.info("Sending signal %s to child process", signum) utils.IgnoreProcessNotFound(os.killpg, self.pid, signum) def ForceQuit(self): """Ensure child process is no longer running. """ # Final check if child process is still alive if utils.RetryOnSignal(self.poll) is None: logging.error("Child process still alive, sending SIGKILL") self.Kill(signal.SIGKILL) utils.RetryOnSignal(self.wait) def main(): """Main function. """ # Option parsing (status_file_path, mode) = ParseOptions() # Configure logging child_logger = SetupLogging() status_file = StatusFile(status_file_path) try: try: # Option verification VerifyOptions() # Pipe to receive socat's stderr output (socat_stderr_read_fd, socat_stderr_write_fd) = os.pipe() # Pipe to receive dd's stderr output (dd_stderr_read_fd, dd_stderr_write_fd) = os.pipe() # Pipe to receive dd's PID (dd_pid_read_fd, dd_pid_write_fd) = os.pipe() # Pipe to receive size predicted by export script (exp_size_read_fd, exp_size_write_fd) = os.pipe() # Get child process command cmd_builder = impexpd.CommandBuilder(mode, options, socat_stderr_write_fd, dd_stderr_write_fd, dd_pid_write_fd) cmd = cmd_builder.GetCommand() # Prepare command environment cmd_env = os.environ.copy() if options.exp_size == constants.IE_CUSTOM_SIZE: cmd_env["EXP_SIZE_FD"] = str(exp_size_write_fd) logging.debug("Starting command %r", cmd) # Start child process child = ChildProcess(cmd_env, cmd, [socat_stderr_write_fd, dd_stderr_write_fd, dd_pid_write_fd, exp_size_write_fd]) try: def _ForwardSignal(signum, _): """Forwards signals to child process. """ child.Kill(signum) signal_wakeup = utils.SignalWakeupFd() try: # TODO: There is a race condition between starting the child and # handling the signals here. While there might be a way to work around # it by registering the handlers before starting the child and # deferring sent signals until the child is available, doing so can be # complicated. signal_handler = utils.SignalHandler([signal.SIGTERM, signal.SIGINT], handler_fn=_ForwardSignal, wakeup=signal_wakeup) try: # Close child's side utils.RetryOnSignal(os.close, socat_stderr_write_fd) utils.RetryOnSignal(os.close, dd_stderr_write_fd) utils.RetryOnSignal(os.close, dd_pid_write_fd) utils.RetryOnSignal(os.close, exp_size_write_fd) if ProcessChildIO(child, socat_stderr_read_fd, dd_stderr_read_fd, dd_pid_read_fd, exp_size_read_fd, status_file, child_logger, signal_wakeup, signal_handler, mode): # The child closed all its file descriptors and there was no # signal # TODO: Implement timeout instead of waiting indefinitely utils.RetryOnSignal(child.wait) finally: signal_handler.Reset() finally: signal_wakeup.Reset() finally: child.ForceQuit() if child.returncode == 0: errmsg = None elif child.returncode < 0: errmsg = "Exited due to signal %s" % (-child.returncode, ) else: errmsg = "Exited with status %s" % (child.returncode, ) status_file.SetExitStatus(child.returncode, errmsg) except Exception as err: # pylint: disable=W0703 logging.exception("Unhandled error occurred") status_file.SetExitStatus(constants.EXIT_FAILURE, "Unhandled error occurred: %s" % (err, )) if status_file.ExitStatusIsSuccess(): sys.exit(constants.EXIT_SUCCESS) sys.exit(constants.EXIT_FAILURE) finally: status_file.Update(True) if __name__ == "__main__": main() ganeti-3.1.0~rc2/devel/000075500000000000000000000000001476477700300147035ustar00rootroot00000000000000ganeti-3.1.0~rc2/devel/build_chroot000075500000000000000000000446241476477700300173200ustar00rootroot00000000000000#!/bin/bash #Requirements for this script to work: #* Make sure that the user who uses the chroot is in group 'src', or change # the ${GROUP} variable to a group that contains the user. #* Add any path of the host system that you want to access inside the chroot # to the /etc/schroot/default/fstab file. This is important in particular if # your homedir is not in /home. #* Add this to your /etc/fstab: # tmpfs /var/lib/schroot/mount tmpfs defaults,size=3G 0 0 # tmpfs /var/lib/schroot/unpack tmpfs defaults,size=3G 0 0 #Configuration : ${ARCH:=amd64} : ${DIST_RELEASE:=wheezy} : ${VARIANT:=} : ${CONF_DIR:=/etc/schroot/chroot.d} : ${CHROOT_DIR:=/srv/chroot} : ${ALTERNATIVE_EDITOR:=/usr/bin/vim.basic} : ${CHROOT_FINAL_HOOK:=/bin/true} : ${GROUP:=src} # Additional Variables taken from the environmen # DATA_DIR # CHROOT_EXTRA_DEBIAN_PACKAGES # make the appended variant name more readable [ -n "$VARIANT" ] && VARIANT="-${VARIANT#-}" #Automatically generated variables CHROOTNAME=$DIST_RELEASE-$ARCH$VARIANT CHNAME=building_$CHROOTNAME TEMP_CHROOT_CONF=$CONF_DIR/$CHNAME.conf FINAL_CHROOT_CONF=$CHROOTNAME.conf ROOT=`pwd` CHDIR=$ROOT/$CHNAME USER=`whoami` COMP_FILENAME=$CHROOTNAME.tar.gz COMP_FILEPATH=$ROOT/$COMP_FILENAME TEMP_DATA_DIR=`mktemp -d` ACTUAL_DATA_DIR=$DATA_DIR ACTUAL_DATA_DIR=${ACTUAL_DATA_DIR:-$TEMP_DATA_DIR} SHA1_LIST=' cabal-install-1.18.0.2.tar.gz 2d1f7a48d17b1e02a1e67584a889b2ff4176a773 cabal-install-1.22.4.0.tar.gz b98eea96d321cdeed83a201c192dac116e786ec2 ghc-7.6.3-i386-unknown-linux.tar.bz2 f042b4171a2d4745137f2e425e6949c185f8ea14 ghc-7.6.3-x86_64-unknown-linux.tar.bz2 46ec3f3352ff57fba0dcbc8d9c20f7bcb6924b77 ghc-7.8.4-i386-unknown-linux-deb7.tar.bz2 4f523f854c37a43b738359506a89a37a9fa9fc5f ghc-7.8.4-x86_64-unknown-linux-deb7.tar.bz2 3f68321b064e5c1ffcb05838b85bcc00aa2315b4 ' # export all variables needed in the schroot export ARCH SHA1_LIST # Use gzip --rsyncable if available, to speed up transfers of generated files # The environment variable GZIP is read automatically by 'gzip', # see ENVIRONMENT in gzip(1). gzip --rsyncable /dev/null 2>&1 && export GZIP="--rsyncable" #Runnability checks if [ $USER != 'root' ] then echo "This script requires root permissions to run" exit fi if [ -f $TEMP_CHROOT_CONF ] then echo "The configuration file name for the temporary chroot" echo " $TEMP_CHROOT_CONF" echo "already exists." echo "Remove it or change the CHNAME value in the script." exit fi #Create configuration dir and files if they do not exist if [ ! -d $ACTUAL_DATA_DIR ] then mkdir $ACTUAL_DATA_DIR echo "The data directory" echo " $ACTUAL_DATA_DIR" echo "has been created." fi if [ ! -f $ACTUAL_DATA_DIR/final.schroot.conf.in ] then cat <$ACTUAL_DATA_DIR/final.schroot.conf.in [${CHROOTNAME}] description=Debian ${DIST_RELEASE} ${ARCH} groups=${GROUP} source-root-groups=root type=file file=${CHROOT_DIR}/${COMP_FILENAME} END echo "The file" echo " $ACTUAL_DATA_DIR/final.schroot.conf.in" echo "has been created with default configurations." fi if [ ! -f $ACTUAL_DATA_DIR/temp.schroot.conf.in ] then cat <$ACTUAL_DATA_DIR/temp.schroot.conf.in [${CHNAME}] description=Debian ${DIST_RELEASE}${VARIANT} ${ARCH} directory=${CHDIR} groups=${GROUP} users=root type=directory END echo "The file" echo " $ACTUAL_DATA_DIR/temp.schroot.conf.in" echo "has been created with default configurations." fi #Stop on errors set -e #Cleanup rm -rf $CHDIR mkdir $CHDIR #Install tools for building chroots apt-get install -y schroot debootstrap shopt -s expand_aliases alias in_chroot='schroot -c $CHNAME -d / ' function subst_variables { sed \ -e "s/\${ARCH}/$ARCH/" \ -e "s*\${CHDIR}*$CHDIR*" \ -e "s/\${CHNAME}/$CHNAME/" \ -e "s/\${CHROOTNAME}/$CHROOTNAME/" \ -e "s*\${CHROOT_DIR}*$CHROOT_DIR*" \ -e "s/\${COMP_FILENAME}/$COMP_FILENAME/" \ -e "s/\${DIST_RELEASE}/$DIST_RELEASE/" $@ } #Generate chroot configurations cat $ACTUAL_DATA_DIR/temp.schroot.conf.in | subst_variables > $TEMP_CHROOT_CONF cat $ACTUAL_DATA_DIR/final.schroot.conf.in | subst_variables > $FINAL_CHROOT_CONF #Install the base system debootstrap --arch $ARCH $DIST_RELEASE $CHDIR APT_INSTALL="apt-get install -y --no-install-recommends" if [ $DIST_RELEASE = squeeze ] then echo "deb http://backports.debian.org/debian-backports" \ "$DIST_RELEASE-backports main contrib non-free" \ > $CHDIR/etc/apt/sources.list.d/backports.list fi #Install all the packages in_chroot -- \ apt-get update # Functions for downloading and checking Haskell core components. # The functions run commands within the schroot. # arguments : file_name expected_sha1 function verify_sha1 { local SUM="$( in_chroot -- sha1sum "$1" | awk '{print $1;exit}' )" if [ "$SUM" != "$2" ] ; then echo "ERROR: The SHA1 sum $SUM of $1 doesn't match $2." >&2 return 1 else echo "SHA1 of $1 verified correct." fi } # arguments: URL function lookup_sha1 { grep -o "${1##*/}"'\s\+[0-9a-fA-F]*' <<<"$SHA1_LIST" | awk '{print $2;exit}' } # arguments : file_name URL function download { local FNAME="$1" local URL="$2" in_chroot -- wget --no-check-certificate --output-document="$FNAME" "$URL" verify_sha1 "$FNAME" "$( lookup_sha1 "$URL" )" } function install_ghc { local GHC_ARCH=$ARCH local TDIR=$( schroot -c $CHNAME -d / -- mktemp -d ) [ -n "$TDIR" ] if [ "$ARCH" == "amd64" ] ; then download "$TDIR"/ghc.tar.bz2 \ http://www.haskell.org/ghc/dist/${GHC_VERSION}/ghc-${GHC_VERSION}-x86_64-unknown-linux${GHC_VARIANT}.tar.bz2 elif [ "$ARCH" == "i386" ] ; then download "$TDIR"/ghc.tar.bz2 \ http://www.haskell.org/ghc/dist/${GHC_VERSION}/ghc-${GHC_VERSION}-i386-unknown-linux${GHC_VARIANT}.tar.bz2 else echo "Don't know what GHC to download for architecture $ARCH" >&2 return 1 fi schroot -c $CHNAME -d "$TDIR" -- \ tar xjf ghc.tar.bz2 schroot -c $CHNAME -d "$TDIR/ghc-${GHC_VERSION}" -- \ ./configure --prefix=/usr/local schroot -c $CHNAME -d "$TDIR/ghc-${GHC_VERSION}" -- \ make install schroot -c $CHNAME -d "/" -- \ rm -rf "$TDIR" } function install_cabal { local TDIR=$( schroot -c $CHNAME -d / -- mktemp -d ) [ -n "$TDIR" ] download "$TDIR"/cabal-install.tar.gz \ http://www.haskell.org/cabal/release/cabal-install-${CABAL_INSTALL_VERSION}/cabal-install-${CABAL_INSTALL_VERSION}.tar.gz schroot -c $CHNAME -d "$TDIR" -- \ tar xzf cabal-install.tar.gz schroot -c $CHNAME -d "$TDIR/cabal-install-${CABAL_INSTALL_VERSION}" -- \ bash -c 'EXTRA_CONFIGURE_OPTS="--enable-library-profiling" ./bootstrap.sh --global' schroot -c $CHNAME -d "/" -- \ rm -rf "$TDIR" } case ${DIST_RELEASE}${VARIANT} in squeeze) GHC_VERSION="7.6.3" GHC_VARIANT="" CABAL_INSTALL_VERSION="1.18.0.2" CABAL_LIB_VERSION=">=1.18.0 && <1.19" export GHC_VERSION GHC_VARIANT CABAL_INSTALL_VERSION # do not install libghc6-network-dev, since it's too old, and just # confuses the dependencies in_chroot -- \ $APT_INSTALL \ autoconf automake \ zlib1g-dev \ libgmp3-dev \ libcurl4-gnutls-dev \ libpcre3-dev \ happy \ hscolour pandoc \ graphviz qemu-utils \ python-docutils \ python-pyparsing \ python-pyinotify \ python-pycurl \ python-ipaddr \ python-yaml \ python-paramiko in_chroot -- \ $APT_INSTALL python-setuptools python-dev build-essential in_chroot -- \ easy_install \ unittest2==0.5.1 \ logilab-astng==0.24.1 \ logilab-common==0.58.3 \ pylint==0.26.0 in_chroot -- \ easy_install \ sphinx==1.1.3 \ pep8==1.3.3 \ coverage==3.4 \ bitarray==0.8.0 install_ghc install_cabal in_chroot -- \ cabal update # sinec we're using Cabal >=1.16, we can use the parallel install option in_chroot -- \ cabal install --global -j --enable-library-profiling \ attoparsec-0.11.1.0 \ base64-bytestring-1.0.0.1 \ blaze-builder-0.3.3.2 \ case-insensitive-1.1.0.3 \ Crypto-4.2.5.1 \ curl-1.3.8 \ happy \ hashable-1.2.1.0 \ hinotify-0.3.6 \ hscolour-1.20.3 \ hslogger-1.2.3 \ json-0.7 \ lifted-base-0.2.2.0 \ lens-4.0.4 \ MonadCatchIO-transformers-0.3.0.0 \ network-2.4.1.2 \ parallel-3.2.0.4 \ parsec-3.1.3 \ regex-pcre-0.94.4 \ temporary-1.2.0.1 \ vector-0.10.9.1 \ zlib-0.5.4.1 \ \ 'hlint>=1.9.12' \ HUnit-1.2.5.2 \ QuickCheck-2.6 \ test-framework-0.8.0.3 \ test-framework-hunit-0.3.0.1 \ test-framework-quickcheck2-0.3.0.2 \ \ snap-server-0.9.4.0 \ PSQueue-1.1 \ \ "Cabal $CABAL_LIB_VERSION" \ cabal-file-th-0.2.3 \ shelltestrunner #Install selected packages from backports in_chroot -- \ $APT_INSTALL -t squeeze-backports \ git \ git-email \ vim \ exuberant-ctags ;; wheezy) in_chroot -- \ $APT_INSTALL \ autoconf automake ghc ghc-haddock libghc-network-dev \ libghc-test-framework{,-hunit,-quickcheck2}-dev \ libghc-json-dev libghc-curl-dev libghc-hinotify-dev \ libghc-parallel-dev libghc-utf8-string-dev \ libghc-hslogger-dev libghc-crypto-dev \ libghc-regex-pcre-dev libghc-attoparsec-dev \ libghc-vector-dev libghc-temporary-dev \ libghc-snap-server-dev libpcre3 libpcre3-dev happy hscolour pandoc \ libghc-zlib-dev libghc-psqueue-dev \ cabal-install \ python-setuptools python-sphinx python-epydoc graphviz python-pyparsing \ python-pycurl python-paramiko \ python-bitarray python-ipaddr python-yaml qemu-utils python-coverage pep8 \ shelltestrunner python-dev openssh-client vim git git-email exuberant-ctags # We need version 0.9.4 of pyinotify because the packaged version, 0.9.3, is # incompatibile with the packaged version of python-epydoc 3.0.1. # Reason: a logger class in pyinotify calculates its superclasses at # runtime, which clashes with python-epydoc's static analysis phase. # # Problem introduced in: # https://github.com/seb-m/pyinotify/commit/2c7e8f8959d2f8528e0d90847df360 # and "fixed" in: # https://github.com/seb-m/pyinotify/commit/98c5f41a6e2e90827a63ff1b878596 in_chroot -- \ easy_install \ logilab-astng==0.24.1 \ logilab-common==0.58.3 \ pylint==0.26.0 \ pep8==1.3.3 in_chroot -- \ easy_install pyinotify==0.9.4 in_chroot -- \ cabal update in_chroot -- \ cabal install --global \ 'base64-bytestring>=1' \ lens-3.10.2 \ 'lifted-base>=0.1.2' \ 'hlint>=1.9.12' ;; jessie) in_chroot -- \ $APT_INSTALL \ autoconf automake ghc ghc-haddock libghc-network-dev \ libghc-test-framework{,-hunit,-quickcheck2}-dev \ libghc-json-dev libghc-curl-dev libghc-hinotify-dev \ libghc-parallel-dev libghc-utf8-string-dev \ libghc-hslogger-dev libghc-crypto-dev \ libghc-regex-pcre-dev libghc-attoparsec-dev \ libghc-vector-dev libghc-temporary-dev \ libghc-snap-server-dev libpcre3 libpcre3-dev happy hscolour pandoc \ libghc-zlib-dev libghc-psqueue-dev \ libghc-base64-bytestring-dev libghc-lens-dev libghc-lifted-base-dev \ libghc-cabal-dev \ cabal-install \ python-setuptools python-sphinx python-epydoc graphviz python-pyparsing \ python-pycurl python-pyinotify python-paramiko \ python-bitarray python-ipaddr python-yaml qemu-utils python-coverage pep8 \ shelltestrunner python-dev pylint openssh-client \ vim git git-email exuberant-ctags in_chroot -- \ cabal update in_chroot -- \ cabal install --global \ 'hlint>=1.9.12' ;; jessie-ghc78) GHC_VERSION="7.8.4" GHC_VARIANT="-deb7" CABAL_INSTALL_VERSION="1.22.4.0" # the version of the Cabal library below must match the version used by # CABAL_INSTALL_VERSION, see the dependencies of cabal-install CABAL_LIB_VERSION=">=1.22.2 && <1.23" export GHC_VERSION GHC_VARIANT CABAL_INSTALL_VERSION in_chroot -- \ $APT_INSTALL \ autoconf automake \ zlib1g-dev \ libgmp3-dev \ libcurl4-openssl-dev \ libpcre3-dev \ happy \ hlint hscolour pandoc \ graphviz qemu-utils \ python-docutils \ python-pyparsing \ python-pyinotify \ python-pycurl \ python-ipaddr \ python-yaml \ python-paramiko \ git \ git-email \ vim in_chroot -- \ $APT_INSTALL python-setuptools python-dev build-essential in_chroot -- \ easy_install \ logilab-astng==0.24.1 \ logilab-common==0.58.3 \ pylint==0.26.0 in_chroot -- \ easy_install \ sphinx==1.1.3 \ pep8==1.3.3 \ coverage==3.4 \ bitarray==0.8.0 install_ghc install_cabal in_chroot -- \ cabal update # since we're using Cabal >=1.16, we can use the parallel install option in_chroot -- \ cabal install --global -j --enable-library-profiling \ attoparsec==0.12.1.6 \ base64-bytestring==1.0.0.1 \ blaze-builder==0.4.0.1 \ case-insensitive==1.2.0.4 \ Crypto==4.2.5.1 \ curl==1.3.8 \ happy==1.19.5 \ hashable==1.2.3.2 \ hinotify==0.3.7 \ hscolour==1.23 \ hslogger==1.2.8 \ json==0.9.1 \ lifted-base==0.2.3.6 \ lens==4.9.1 \ MonadCatchIO-transformers==0.3.1.3 \ network==2.6.0.2 \ parallel==3.2.0.6 \ parsec==3.1.7 \ regex-pcre==0.94.4 \ temporary==1.2.0.3 \ vector==0.10.12.3 \ zlib==0.5.4.2 \ \ hlint==1.9.20 \ HUnit==1.2.5.2 \ QuickCheck==2.8.1 \ test-framework==0.8.1.1 \ test-framework-hunit==0.3.0.1 \ test-framework-quickcheck2==0.3.0.3 \ \ snap-server==0.9.5.1 \ \ "Cabal $CABAL_LIB_VERSION" \ cabal-file-th==0.2.3 \ shelltestrunner==1.3.5 ;; precise) # ghc, git-email and other dependencies are hosted in the universe # repository, which is not enabled by default. echo "Adding universe repository..." cat > $CHDIR/etc/apt/sources.list.d/universe.list <=1' \ hslogger-1.2.3 \ 'hlint>=1.9.12' \ json-0.7 \ lens-3.10.2 \ 'lifted-base>=0.1.2' \ 'network>=2.4.0.1' \ 'regex-pcre>=0.94.4' \ parsec-3.1.3 \ shelltestrunner \ 'snap-server>=0.8.1' \ test-framework-0.8.0.3 \ test-framework-hunit-0.3.0.1 \ test-framework-quickcheck2-0.3.0.2 \ 'transformers>=0.3.0.0' ;; *) in_chroot -- \ $APT_INSTALL \ autoconf automake ghc ghc-haddock libghc-network-dev \ libghc-test-framework{,-hunit,-quickcheck2}-dev \ libghc-json-dev libghc-curl-dev libghc-hinotify-dev \ libghc-parallel-dev libghc-utf8-string-dev \ libghc-hslogger-dev libghc-crypto-dev \ libghc-regex-pcre-dev libghc-attoparsec-dev \ libghc-vector-dev libghc-temporary-dev libghc-psqueue-dev \ libghc-snap-server-dev libpcre3 libpcre3-dev happy hscolour pandoc \ libghc-lens-dev libghc-lifted-base-dev \ libghc-cabal-dev \ cabal-install \ libghc-base64-bytestring-dev \ python-setuptools python-sphinx python-epydoc graphviz python-pyparsing \ python-pyinotify python-pycurl python-paramiko \ python-bitarray python-ipaddr python-yaml qemu-utils python-coverage pep8 \ shelltestrunner python-dev pylint openssh-client \ vim git git-email exuberant-ctags \ build-essential in_chroot -- \ cabal update in_chroot -- \ cabal install --global \ 'hlint>=1.9.12' ;; esac # print what packages and versions are installed: in_chroot -- \ cabal list --installed --simple-output in_chroot -- \ $APT_INSTALL sudo fakeroot rsync locales less socat # Configure the locale case $DIST_RELEASE in precise) in_chroot -- \ $APT_INSTALL language-pack-en ;; *) echo "en_US.UTF-8 UTF-8" >> $CHDIR/etc/locale.gen in_chroot -- \ locale-gen ;; esac in_chroot -- \ $APT_INSTALL lvm2 ssh bridge-utils iproute iputils-arping \ ndisc6 python-openssl openssl \ fping qemu-utils in_chroot -- \ easy_install psutil in_chroot -- \ easy_install jsonpointer \ jsonpointer \ jsonpatch in_chroot -- \ $APT_INSTALL \ python-epydoc debhelper quilt # extra debian packages for package in $CHROOT_EXTRA_DEBIAN_PACKAGES do in_chroot -- \ $APT_INSTALL $package done #Set default editor in_chroot -- \ update-alternatives --set editor $ALTERNATIVE_EDITOR # Final user hook in_chroot -- $CHROOT_FINAL_HOOK rm -f $COMP_FILEPATH echo "Creating compressed schroot image..." cd $CHDIR tar czf $COMP_FILEPATH ./* cd $ROOT rm -rf $CHDIR rm -f $TEMP_CHROOT_CONF rm -rf $TEMP_DATA_DIR echo "Chroot created. In order to run it:" echo " * sudo cp $FINAL_CHROOT_CONF $CONF_DIR/$FINAL_CHROOT_CONF" echo " * sudo mkdir -p $CHROOT_DIR" echo " * sudo cp $COMP_FILEPATH $CHROOT_DIR/$COMP_FILENAME" echo "Then run \"schroot -c $CHROOTNAME\"" ganeti-3.1.0~rc2/devel/check-split-query000075500000000000000000000054751476477700300202150ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Checks query equivalence between masterd and confd # # This is not (currently) run automatically during QA, but you can run # it manually on a test cluster. It will force all queries known to be # converted via both paths and check the difference, via both 'list' # and 'list-fields'. For best results, it should be run on a non-empty # cluster. # # Also note that this is not expected to show 100% perfect matches, # since the JSON output differs slightly for complex data types # (e.g. dictionaries with different sort order for keys, etc.). # # Current known delta: # - all dicts, sort order # - ctime is always defined in Haskell as epoch 0 if missing MA=`mktemp master.XXXXXX` CF=`mktemp confd.XXXXXX` trap 'rm -f "$MA" "$CF"' EXIT trap 'exit 1' SIGINT RET=0 SEP="--separator=," ENABLED_QUERIES="node group network backup" test_cmd() { cmd="$1" desc="$2" FORCE_LUXI_SOCKET=master $cmd > "$MA" FORCE_LUXI_SOCKET=query $cmd > "$CF" diff -u "$MA" "$CF" || { echo "Mismatch in $desc, see above." RET=1 } } for kind in $ENABLED_QUERIES; do all_fields=$(FORCE_LUXI_SOCKET=master gnt-$kind list-fields \ --no-headers --separator=,|cut -d, -f1) comma_fields=$(echo $all_fields|tr ' ' ,|sed -e 's/,$//') for op in list list-fields; do test_cmd "gnt-$kind $op $SEP" "$kind $op" done #test_cmd "gnt-$kind list $SEP -o$comma_fields" "$kind list with all fields" for field in $all_fields; do test_cmd "gnt-$kind list $SEP -o$field" "$kind list for field $field" done done exit $RET ganeti-3.1.0~rc2/devel/check_copyright000075500000000000000000000060751476477700300200060ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Script to check whether the local dirty commits are changing files # which do not have an updated copyright. # # The script will determine your current remote branch and local # branch, from which it will extract the commits to analyze. # Afterwards, for each commit, it will see which files are being # modified and, for each file, it will check the copyright. function join { local IFS="$1" shift echo "$*" } # Determine the tracking branch for the current branch readonly REMOTE=$(git branch -vv | grep -e "^\*" | sed -e "s/ \+/ /g" | awk '{ print $4 }' | grep "\[" | tr -d ":[]") if [ -z "$REMOTE" ] then echo check_copyright: failed to get remote branch exit 1 fi # Determine which commits have no been pushed (i.e, diff between the # remote branch and the current branch) COMMITS=$(git log --pretty=format:'%h' ${REMOTE}..HEAD) if [ -z "$COMMITS" ] then echo check_copyright: there are no commits to check exit 0 fi # for each commit, check its files for commit in $(echo $COMMITS | tac -s " ") do FILES=$(git diff-tree --no-commit-id --name-only -r $commit) if [ -z "$FILES" ] then echo check_copyright: commit \"$commit\" has no files to check else # for each file, check if it is in the 'lib' or 'src' dirs # and, if so, check the copyright for file in $FILES do DIR=$(echo $file | cut -d "/" -f 1) if [ "$DIR" = lib -o "$DIR" = src ] then COPYRIGHT=$(grep "Copyright (C)" $file) YEAR=$(date +%G) if [ -z "$COPYRIGHT" ] then echo check_copyright: commit \"$commit\" misses \ copyright for \"$file\" elif ! echo $COPYRIGHT | grep -o $YEAR > /dev/null then echo check_copyright: commit \"$commit\" misses \ \"$YEAR\" copyright for \"$file\" fi fi done fi done ganeti-3.1.0~rc2/devel/release000075500000000000000000000062711476477700300162570ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # This is a test script to ease development and testing on test clusters. # It should not be used to update production environments. # Usage: release v2.0.5 # Alternative: URL=file:///my/git/repo release e5823b7e2cd8a3... # It will clone the given repository from the default or passed URL, # checkout the given reference (a tag or branch) and then create a # release archive; you will need to copy the archive and delete the # temporary directory at the end set -e : ${URL:=https://github.com/ganeti/ganeti} TAG="$1" : ${PARALLEL:=$(egrep -c "^processor\s+:" /proc/cpuinfo)} if [[ -z "$TAG" ]]; then echo "Usage: $0 " >&2 exit 1 fi echo "Using Git repository $URL" TMPDIR=$(mktemp -d -t gntrelease.XXXXXXXXXX) cd $TMPDIR echo "Cloning the repository under $TMPDIR ..." git clone -q "$URL" dist cd dist git checkout $TAG # Check minimum aclocal version for releasing MIN_ACLOCAL_VERSION=( 1 11 1 ) ACLOCAL_VERSION=$(${ACLOCAL:-aclocal} --version | head -1 | \ sed -e 's/^[^0-9]*\([0-9\.]*\)$/\1/') ACLOCAL_VERSION_REST=$ACLOCAL_VERSION for v in ${MIN_ACLOCAL_VERSION[@]}; do ACLOCAL_VERSION_PART=${ACLOCAL_VERSION_REST%%.*} ACLOCAL_VERSION_REST=${ACLOCAL_VERSION_REST#$ACLOCAL_VERSION_PART.} if [[ $v -eq $ACLOCAL_VERSION_PART ]]; then continue elif [[ $v -lt $ACLOCAL_VERSION_PART ]]; then break else # gt echo "aclocal version $ACLOCAL_VERSION is too old (< 1.11.1)" exit 1 fi done ./autogen.sh ./configure VERSION=$(sed -n -e '/^PACKAGE_VERSION =/ s/^PACKAGE_VERSION = // p' Makefile) ARCHIVE="ganeti-${VERSION}.tar.gz" make -j$PARALLEL distcheck-release fakeroot make -j$PARALLEL dist-release tar tzvf "$ARCHIVE" echo echo 'MD5:' md5sum "$ARCHIVE" echo echo 'SHA1:' sha1sum "$ARCHIVE" echo echo 'SHA256:' sha256sum "$ARCHIVE" echo echo "The archive is at ${PWD}/${ARCHIVE}" echo "Please copy it and remove the temporary directory when done." ganeti-3.1.0~rc2/devel/review000075500000000000000000000122371476477700300161370ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # To set user mappings, use this command: # git config gnt-review.johndoe 'John Doe ' # To disable strict mode (enabled by default): # git config gnt-review.strict false # To enable strict mode: # git config gnt-review.strict true set -e # Get absolute path to myself me_plain="$0" me=$(readlink -f "$me_plain") add_reviewed_by() { local msgfile="$1" grep -q '^Reviewed-by: ' "$msgfile" && return perl -i -e ' my $reviewer = $ENV{"REVIEWER"}; defined($reviewer) or $reviewer = ""; my $sob = 0; while (<>) { if ($sob == 0 and m/^Signed-off-by:/) { $sob = 1; } elsif ($sob == 1 and not m/^Signed-off-by:/) { print "Reviewed-by: $reviewer\n"; $sob = -1; } print; } if ($sob == 1) { print "Reviewed-by: $reviewer\n"; } ' "$msgfile" } replace_users() { local msgfile="$1" if perl -i -e ' use strict; use warnings; my $error = 0; my $strict; sub map_username { my ($name) = @_; return $name unless $name; my @cmd = ("git", "config", "--get", "gnt-review.$name"); open(my $fh, "-|", @cmd) or die "Command \"@cmd\" failed: $!"; my $output = do { local $/ = undef; <$fh> }; close($fh); if ($? == 0) { chomp $output; $output =~ s/\s+/ /; return $output; } unless (defined $strict) { @cmd = ("git", "config", "--get", "--bool", "gnt-review.strict"); open($fh, "-|", @cmd) or die "Command \"@cmd\" failed: $!"; $output = do { local $/ = undef; <$fh> }; close($fh); $strict = ($? != 0 or not $output or $output !~ m/^false$/); } if ($strict and $name !~ m/^.+<.+\@.+>$/) { $error = 1; } return $name; } while (<>) { if (m/^Reviewed-by:(.*)$/) { my @names = grep { # Ignore empty entries !/^$/ } map { # Normalize whitespace $_ =~ s/(^\s+|\s+$)//g; $_ =~ s/\s+/ /g; # Map names $_ = map_username($_); $_; } split(m/,/, $1); # Get unique names my %saw; @names = grep(!$saw{$_}++, @names); undef %saw; foreach (sort @names) { print "Reviewed-by: $_\n"; } } else { print; } } exit($error? 33 : 0); ' "$msgfile" then : else [[ "$?" == 33 ]] && return 1 exit 1 fi if ! grep -q '^Reviewed-by: ' "$msgfile" then echo 'Missing Reviewed-by: line' >&2 sleep 1 return 1 fi return 0 } run_editor() { local filename="$1" local editor=${EDITOR:-vi} local args case "$(basename "$editor")" in vi* | *vim) # Start edit mode at Reviewed-by: line args='+/^Reviewed-by: +nohlsearch +startinsert!' ;; *) args= ;; esac $editor $args "$filename" } commit_editor() { local msgfile="$1" local tmpf=$(mktemp) trap "rm -f $tmpf" EXIT cp "$msgfile" "$tmpf" while : do add_reviewed_by "$tmpf" run_editor "$tmpf" replace_users "$tmpf" && break done cp "$tmpf" "$msgfile" } copy_commit() { local rev="$1" target_branch="$2" echo "Copying commit $rev ..." git cherry-pick -n "$rev" GIT_EDITOR="$me --commit-editor \"\$@\"" git commit -c "$rev" -s } usage() { echo "Usage: $me_plain [from..to] " >&2 echo " If not passed from..to defaults to target-branch..HEAD" >&2 exit 1 } main() { local range target_branch case "$#" in 1) target_branch="$1" range="$target_branch..$(git rev-parse HEAD)" ;; 2) range="$1" target_branch="$2" if [[ "$range" != *..* ]]; then usage fi ;; *) usage ;; esac git checkout "$target_branch" local old_head=$(git rev-parse HEAD) for rev in $(git rev-list --reverse "$range") do copy_commit "$rev" done git log "$old_head..$target_branch" } if [[ "$1" == --commit-editor ]] then shift commit_editor "$@" else main "$@" fi ganeti-3.1.0~rc2/devel/upload000075500000000000000000000106551476477700300161240ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # This is a test script to ease development and testing on test clusters. # It should not be used to update production environments. # Usage: upload node-{1,2,3} # it will upload the python libraries to # $prefix/lib/python2.X/dist-packages/ganeti and the command line utils to # $prefix/sbin. It needs passwordless root login to the nodes. set -e -u usage() { echo "Usage: $0 [--no-restart] [--no-cron] [--no-debug] hosts..." >&2 exit $1 } declare -r SED="sed -f autotools/replace_vars.sed" NO_RESTART= NO_CRON= NO_DEBUG= hosts= while [ "$#" -gt 0 ]; do opt="$1" case "$opt" in --no-restart) NO_RESTART=1 ;; --no-cron) NO_CRON=1 ;; --no-debug) NO_DEBUG=1 ;; -h|--help) usage 0 ;; -*) echo "Unknown option: $opt" >&2 usage 1 ;; *) hosts="$hosts $opt" ;; esac shift done if [ -z "$hosts" ]; then usage 1 fi set ${hosts} make regen-vcs-version TXD=`mktemp -d` trap 'rm -rf $TXD' EXIT if [[ -f /proc/cpuinfo ]]; then cpu_count=$(grep -E -c '^processor[[:space:]]*:' /proc/cpuinfo) make_args=-j$(( cpu_count + 1 )) else make_args= fi # Make sure that directories will get correct permissions umask 0022 # install ganeti as a real tree make $make_args install DESTDIR="$TXD" # at this point, make has been finished, so the configuration is # fixed; we can read the prefix vars/etc. PREFIX="$(echo @PREFIX@ | $SED)" SYSCONFDIR="$(echo @SYSCONFDIR@ | $SED)" LIBDIR="$(echo @LIBDIR@ | $SED)" PKGLIBDIR="$(echo @PKGLIBDIR@ | $SED)" # copy additional needed files [ -f doc/examples/ganeti.initd ] && \ install -D --mode=0755 doc/examples/ganeti.initd \ "$TXD/$SYSCONFDIR/init.d/ganeti" [ -f doc/examples/ganeti.logrotate ] && \ install -D --mode=0755 doc/examples/ganeti.logrotate \ "$TXD/$SYSCONFDIR/logrotate.d/ganeti" [ -f doc/examples/ganeti-master-role.ocf ] && \ install -D --mode=0755 doc/examples/ganeti-master-role.ocf \ "$TXD/$LIBDIR/ocf/resource.d/ganeti/ganeti-master-role" [ -f doc/examples/ganeti-node-role.ocf ] && \ install -D --mode=0755 doc/examples/ganeti-node-role.ocf \ "$TXD/$LIBDIR/ocf/resource.d/ganeti/ganeti-node-role" [ -f doc/examples/ganeti.default-debug -a -z "$NO_DEBUG" ] && \ install -D --mode=0644 doc/examples/ganeti.default-debug \ "$TXD/$SYSCONFDIR/default/ganeti" [ -f doc/examples/bash_completion-debug ] && \ install -D --mode=0644 doc/examples/bash_completion-debug \ "$TXD/$SYSCONFDIR/bash_completion.d/ganeti" if [ -f doc/examples/ganeti.cron -a -z "$NO_CRON" ]; then install -D --mode=0644 doc/examples/ganeti.cron \ "$TXD/$SYSCONFDIR/cron.d/ganeti" fi echo --- ( cd "$TXD" && find; ) echo --- # and now put it under $prefix on the target node(s) for host; do echo Uploading code to ${host}... rsync -v -rlKDc \ -e "ssh -oBatchMode=yes" \ --exclude="*.py[oc]" --exclude="*.pdf" --exclude="*.html" \ "$TXD/" \ root@${host}:/ & done wait if test -z "${NO_RESTART}"; then for host; do echo Restarting ganeti-noded on ${host}... ssh -oBatchMode=yes root@${host} $SYSCONFDIR/init.d/ganeti restart & done wait fi ganeti-3.1.0~rc2/devel/webserver000075500000000000000000000042351476477700300166410ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import sys import BaseHTTPServer import SimpleHTTPServer def main(): if len(sys.argv) == 2: host = "127.0.0.1" (_, port) = sys.argv elif len(sys.argv) == 3: (_, port, host) = sys.argv else: sys.stderr.write("Usage: %s []\n" % sys.argv[0]) sys.stderr.write("\n") sys.stderr.write("Provides an HTTP server on the specified TCP port") sys.stderr.write(" exporting the current working directory. Binds to") sys.stderr.write(" localhost by default.\n") sys.exit(1) try: port = int(port) except (ValueError, TypeError) as err: sys.stderr.write("Invalid port '%s': %s\n" % (port, err)) sys.exit(1) handler = SimpleHTTPServer.SimpleHTTPRequestHandler server = BaseHTTPServer.HTTPServer((host, port), handler) server.serve_forever() if __name__ == "__main__": main() ganeti-3.1.0~rc2/doc/000075500000000000000000000000001476477700300143515ustar00rootroot00000000000000ganeti-3.1.0~rc2/doc/admin.rst000064400000000000000000002354601476477700300162050ustar00rootroot00000000000000Ganeti administrator's guide ============================ Documents Ganeti version |version| .. contents:: .. highlight:: shell-example Introduction ------------ Ganeti is a virtualization cluster management software. You are expected to be a system administrator familiar with your Linux distribution and the Xen or KVM virtualization environments before using it. The various components of Ganeti all have man pages and interactive help. This manual though will help you getting familiar with the system by explaining the most common operations, grouped by related use. After a terminology glossary and a section on the prerequisites needed to use this manual, the rest of this document is divided in sections for the different targets that a command affects: instance, nodes, etc. .. _terminology-label: Ganeti terminology ++++++++++++++++++ This section provides a small introduction to Ganeti terminology, which might be useful when reading the rest of the document. Cluster ~~~~~~~ A set of machines (nodes) that cooperate to offer a coherent, highly available virtualization service under a single administration domain. Node ~~~~ A physical machine which is member of a cluster. Nodes are the basic cluster infrastructure, and they don't need to be fault tolerant in order to achieve high availability for instances. Node can be added and removed (if they host no instances) at will from the cluster. In a HA cluster and only with HA instances, the loss of any single node will not cause disk data loss for any instance; of course, a node crash will cause the crash of its primary instances. A node belonging to a cluster can be in one of the following roles at a given time: - *master* node, which is the node from which the cluster is controlled - *master candidate* node, only nodes in this role have the full cluster configuration and knowledge, and only master candidates can become the master node - *regular* node, which is the state in which most nodes will be on bigger clusters (>20 nodes) - *drained* node, nodes in this state are functioning normally but the cannot receive new instances; the intention is that nodes in this role have some issue and they are being evacuated for hardware repairs - *offline* node, in which there is a record in the cluster configuration about the node, but the daemons on the master node will not talk to this node; any instances declared as having an offline node as either primary or secondary will be flagged as an error in the cluster verify operation Depending on the role, each node will run a set of daemons: - the :command:`ganeti-noded` daemon, which controls the manipulation of this node's hardware resources; it runs on all nodes which are in a cluster - the :command:`ganeti-confd` daemon (Ganeti 2.1+) which runs on all master candidates and answers queries about the cluster configuration. - the :command:`ganeti-wconfd` daemon (Ganeti 2.12+) which runs on the master node and allows control of the cluster - the :command:`ganeti-luxid` daemon which runs on the master node and answers queries related to the live state of the cluster, and schedules cluster jobs - the :command:`ganeti-rapi` daemon which runs on the master node and offers an HTTP-based API for the cluster - the :command:`ganeti-kvmd` daemon which runs on all KVM-enabled nodes when the cluster parameter `user_shutdown` is enabled, and determines whether the instance was shutdown by a user or an administrator - the :command:`ganeti-mond` daemon which runs on all nodes and collects information about the cluster - the :command:`ganeti-metad` daemon which is started on demand and provides information about the cluster to OS install scripts or instances. Beside the node role, there are other node flags that influence its behaviour: - the *master_capable* flag denotes whether the node can ever become a master candidate; setting this to 'no' means that auto-promotion will never make this node a master candidate; this flag can be useful for a remote node that only runs local instances, and having it become a master is impractical due to networking or other constraints - the *vm_capable* flag denotes whether the node can host instances or not; for example, one might use a non-vm_capable node just as a master candidate, for configuration backups; setting this flag to no disallows placement of instances of this node, deactivates hypervisor and related checks on it (e.g. bridge checks, LVM check, etc.), and removes it from cluster capacity computations Instance ~~~~~~~~ A virtual machine which runs on a cluster. It can be a fault tolerant, highly available entity. An instance has various parameters, which are classified in three categories: hypervisor related-parameters (called ``hvparams``), general parameters (called ``beparams``) and per network-card parameters (called ``nicparams``). All these parameters can be modified either at instance level or via defaults at cluster level. Disk template ~~~~~~~~~~~~~ The are multiple options for the storage provided to an instance; while the instance sees the same virtual drive in all cases, the node-level configuration varies between them. There are several disk templates you can choose from: ``diskless`` The instance has no disks. Only used for special purpose operating systems or for testing. ``file`` ***** The instance will use plain files as backend for its disks. No redundancy is provided, and this is somewhat more difficult to configure for high performance. ``sharedfile`` ***** The instance will use plain files as backend, but Ganeti assumes that those files will be available and in sync automatically on all nodes. This allows live migration and failover of instances using this method. ``plain`` The instance will use LVM devices as backend for its disks. No redundancy is provided. ``drbd`` .. note:: This is only valid for multi-node clusters using DRBD 8.0+ A mirror is set between the local node and a remote one, which must be specified with the second value of the --node option. Use this option to obtain a highly available instance that can be failed over to a remote node should the primary one fail. .. note:: Ganeti does not support DRBD stacked devices: DRBD stacked setup is not fully symmetric and as such it is not working with live migration. ``rbd`` The instance will use Volumes inside a RADOS cluster as backend for its disks. It will access them using the RADOS block device (RBD). ``gluster`` ***** The instance will use a Gluster volume for instance storage. Disk images will be stored in the top-level ``ganeti/`` directory of the volume. This directory will be created automatically for you. ``ext`` The instance will use an external storage provider. See :manpage:`ganeti-extstorage-interface(7)` for how to implement one. .. note:: Disk templates marked with an asterisk require Ganeti to access the file system. Ganeti will refuse to do so unless you whitelist the relevant paths in the file storage paths configuration which, with default configure-time paths is located in :pyeval:`pathutils.FILE_STORAGE_PATHS_FILE`. The default paths used by Ganeti are: =============== =================================================== Disk template Default path =============== =================================================== ``file`` :pyeval:`pathutils.DEFAULT_FILE_STORAGE_DIR` ``sharedfile`` :pyeval:`pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR` ``gluster`` :pyeval:`pathutils.DEFAULT_GLUSTER_STORAGE_DIR` =============== =================================================== Those paths can be changed at ``gnt-cluster init`` time. See :manpage:`gnt-cluster(8)` for details. IAllocator ~~~~~~~~~~ A framework for using external (user-provided) scripts to compute the placement of instances on the cluster nodes. This eliminates the need to manually specify nodes in instance add, instance moves, node evacuate, etc. In order for Ganeti to be able to use these scripts, they must be place in the iallocator directory (usually ``lib/ganeti/iallocators`` under the installation prefix, e.g. ``/usr/local``). “Primary” and “secondary” concepts ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ An instance has a primary and depending on the disk configuration, might also have a secondary node. The instance always runs on the primary node and only uses its secondary node for disk replication. Similarly, the term of primary and secondary instances when talking about a node refers to the set of instances having the given node as primary, respectively secondary. Tags ~~~~ Tags are short strings that can be attached to either to cluster itself, or to nodes or instances. They are useful as a very simplistic information store for helping with cluster administration, for example by attaching owner information to each instance after it's created:: $ gnt-instance add â€Ļ %instance1% $ gnt-instance add-tags %instance1% %owner:user2% And then by listing each instance and its tags, this information could be used for contacting the users of each instance. Jobs and OpCodes ~~~~~~~~~~~~~~~~ While not directly visible by an end-user, it's useful to know that a basic cluster operation (e.g. starting an instance) is represented internally by Ganeti as an *OpCode* (abbreviation from operation code). These OpCodes are executed as part of a *Job*. The OpCodes in a single Job are processed serially by Ganeti, but different Jobs will be processed (depending on resource availability) in parallel. They will not be executed in the submission order, but depending on resource availability, locks and (starting with Ganeti 2.3) priority. An earlier job may have to wait for a lock while a newer job doesn't need any locks and can be executed right away. Operations requiring a certain order need to be submitted as a single job, or the client must submit one job at a time and wait for it to finish before continuing. For example, shutting down the entire cluster can be done by running the command ``gnt-instance shutdown --all``, which will submit for each instance a separate job containing the “shutdown instance” OpCode. Prerequisites +++++++++++++ You need to have your Ganeti cluster installed and configured before you try any of the commands in this document. Please follow the :doc:`install` for instructions on how to do that. Instance management ------------------- Adding an instance ++++++++++++++++++ The add operation might seem complex due to the many parameters it accepts, but once you have understood the (few) required parameters and the customisation capabilities you will see it is an easy operation. The add operation requires at minimum five parameters: - the OS for the instance - the disk template - the disk count and size - the node specification or alternatively the iallocator to use - and finally the instance name The OS for the instance must be visible in the output of the command ``gnt-os list`` and specifies which guest OS to install on the instance. The disk template specifies what kind of storage to use as backend for the (virtual) disks presented to the instance; note that for instances with multiple virtual disks, they all must be of the same type. The node(s) on which the instance will run can be given either manually, via the ``-n`` option, or computed automatically by Ganeti, if you have installed any iallocator script. With the above parameters in mind, the command is:: $ gnt-instance add \ -n %TARGET_NODE%:%SECONDARY_NODE% \ -o %OS_TYPE% \ -t %DISK_TEMPLATE% -s %DISK_SIZE% \ %INSTANCE_NAME% The instance name must be resolvable (e.g. exist in DNS) and usually points to an address in the same subnet as the cluster itself. The above command has the minimum required options; other options you can give include, among others: - The maximum/minimum memory size (``-B maxmem``, ``-B minmem``) (``-B memory`` can be used to specify only one size) - The number of virtual CPUs (``-B vcpus``) - Arguments for the NICs of the instance; by default, a single-NIC instance is created. The IP and/or bridge of the NIC can be changed via ``--net 0:ip=IP,link=BRIDGE`` See :manpage:`ganeti-instance(8)` for the detailed option list. For example if you want to create an highly available instance, with a single disk of 50GB and the default memory size, having primary node ``node1`` and secondary node ``node3``, use the following command:: $ gnt-instance add -n node1:node3 -o debootstrap -t drbd -s 50G \ instance1 There is a also a command for batch instance creation from a specification file, see the ``batch-create`` operation in the gnt-instance manual page. Regular instance operations +++++++++++++++++++++++++++ Removal ~~~~~~~ Removing an instance is even easier than creating one. This operation is irreversible and destroys all the contents of your instance. Use with care:: $ gnt-instance remove %INSTANCE_NAME% .. _instance-startup-label: Startup/shutdown ~~~~~~~~~~~~~~~~ Instances are automatically started at instance creation time. To manually start one which is currently stopped you can run:: $ gnt-instance startup %INSTANCE_NAME% Ganeti will start an instance with up to its maximum instance memory. If not enough memory is available Ganeti will use all the available memory down to the instance minimum memory. If not even that amount of memory is free Ganeti will refuse to start the instance. Note, that this will not work when an instance is in a permanently stopped state ``offline``. In this case, you will first have to put it back to online mode by running:: $ gnt-instance modify --online %INSTANCE_NAME% The command to stop the running instance is:: $ gnt-instance shutdown %INSTANCE_NAME% If you want to shut the instance down more permanently, so that it does not require dynamically allocated resources (memory and vcpus), after shutting down an instance, execute the following:: $ gnt-instance modify --offline %INSTANCE_NAME% .. warning:: Do not use the Xen or KVM commands directly to stop instances. If you run for example ``xl shutdown`` or ``xl destroy`` on an instance Ganeti will automatically restart it (via the :command:`ganeti-watcher(8)` command which is launched via cron). Instances can also be shutdown by the user from within the instance, in which case they will marked accordingly and the :command:`ganeti-watcher(8)` will not restart them. See :manpage:`gnt-cluster(8)` for details. Querying instances ~~~~~~~~~~~~~~~~~~ There are two ways to get information about instances: listing instances, which does a tabular output containing a given set of fields about each instance, and querying detailed information about a set of instances. The command to see all the instances configured and their status is:: $ gnt-instance list The command can return a custom set of information when using the ``-o`` option (as always, check the manpage for a detailed specification). Each instance will be represented on a line, thus making it easy to parse this output via the usual shell utilities (grep, sed, etc.). To get more detailed information about an instance, you can run:: $ gnt-instance info %INSTANCE% which will give a multi-line block of information about the instance, it's hardware resources (especially its disks and their redundancy status), etc. This is harder to parse and is more expensive than the list operation, but returns much more detailed information. Changing an instance's runtime memory +++++++++++++++++++++++++++++++++++++ Ganeti will always make sure an instance has a value between its maximum and its minimum memory available as runtime memory. As of version 2.6 Ganeti will only choose a size different than the maximum size when starting up, failing over, or migrating an instance on a node with less than the maximum memory available. It won't resize other instances in order to free up space for an instance. If you find that you need more memory on a node any instance can be manually resized without downtime, with the command:: $ gnt-instance modify -m %SIZE% %INSTANCE_NAME% The same command can also be used to increase the memory available on an instance, provided that enough free memory is available on its node, and the specified size is not larger than the maximum memory size the instance had when it was first booted (an instance will be unable to see new memory above the maximum that was specified to the hypervisor at its boot time, if it needs to grow further a reboot becomes necessary). Export/Import +++++++++++++ You can create a snapshot of an instance disk and its Ganeti configuration, which then you can backup, or import into another cluster. The way to export an instance is:: $ gnt-backup export -n %TARGET_NODE% %INSTANCE_NAME% The target node can be any node in the cluster with enough space under ``/srv/ganeti`` to hold the instance image. Use the ``--noshutdown`` option to snapshot an instance without rebooting it. Note that Ganeti only keeps one snapshot for an instance - any previous snapshot of the same instance existing cluster-wide under ``/srv/ganeti`` will be removed by this operation: if you want to keep them, you need to move them out of the Ganeti exports directory. Importing an instance is similar to creating a new one, but additionally one must specify the location of the snapshot. The command is:: $ gnt-backup import -n %TARGET_NODE% \ --src-node=%NODE% --src-dir=%DIR% %INSTANCE_NAME% By default, parameters will be read from the export information, but you can of course pass them in via the command line - most of the options available for the command :command:`gnt-instance add` are supported here too. Import of foreign instances +++++++++++++++++++++++++++ There is a possibility to import a foreign instance whose disk data is already stored as LVM volumes without going through copying it: the disk adoption mode. For this, ensure that the original, non-managed instance is stopped, then create a Ganeti instance in the usual way, except that instead of passing the disk information you specify the current volumes:: $ gnt-instance add -t plain -n %HOME_NODE% ... \ --disk 0:adopt=%lv_name%[,vg=%vg_name%] %INSTANCE_NAME% This will take over the given logical volumes, rename them to the Ganeti standard (UUID-based), and without installing the OS on them start directly the instance. If you configure the hypervisor similar to the non-managed configuration that the instance had, the transition should be seamless for the instance. For more than one disk, just pass another disk parameter (e.g. ``--disk 1:adopt=...``). Instance kernel selection +++++++++++++++++++++++++ The kernel that instances uses to bootup can come either from the node, or from instances themselves, depending on the setup. Xen-PVM ~~~~~~~ With Xen PVM, there are three options. First, you can use a kernel from the node, by setting the hypervisor parameters as such: - ``kernel_path`` to a valid file on the node (and appropriately ``initrd_path``) - ``kernel_args`` optionally set to a valid Linux setting (e.g. ``ro``) - ``root_path`` to a valid setting (e.g. ``/dev/xvda1``) - ``bootloader_path`` and ``bootloader_args`` to empty Alternatively, you can delegate the kernel management to instances, and use either ``pvgrub`` or the deprecated ``pygrub``. For this, you must install the kernels and initrds in the instance and create a valid GRUB v1 configuration file. For ``pvgrub`` (new in version 2.4.2), you need to set: - ``kernel_path`` to point to the ``pvgrub`` loader present on the node (e.g. ``/usr/lib/xen/boot/pv-grub-x86_32.gz``) - ``kernel_args`` to the path to the GRUB config file, relative to the instance (e.g. ``(hd0,0)/grub/menu.lst``) - ``root_path`` **must** be empty - ``bootloader_path`` and ``bootloader_args`` to empty While ``pygrub`` is deprecated, here is how you can configure it: - ``bootloader_path`` to the pygrub binary (e.g. ``/usr/bin/pygrub``) - the other settings are not important More information can be found in the Xen wiki pages for `pvgrub `_ and `pygrub `_. KVM ~~~ For KVM also the kernel can be loaded either way. For loading the kernels from the node, you need to set: - ``kernel_path`` to a valid value - ``initrd_path`` optionally set if you use an initrd - ``kernel_args`` optionally set to a valid value (e.g. ``ro``) If you want instead to have the instance boot from its disk (and execute its bootloader), simply set the ``kernel_path`` parameter to an empty string, and all the others will be ignored. Instance HA features -------------------- .. note:: This section only applies to multi-node clusters .. _instance-change-primary-label: Changing the primary node +++++++++++++++++++++++++ There are three ways to exchange an instance's primary and secondary nodes; the right one to choose depends on how the instance has been created and the status of its current primary node. See :ref:`rest-redundancy-label` for information on changing the secondary node. Note that it's only possible to change the primary node to the secondary and vice-versa; a direct change of the primary node with a third node, while keeping the current secondary is not possible in a single step, only via multiple operations as detailed in :ref:`instance-relocation-label`. Failing over an instance ~~~~~~~~~~~~~~~~~~~~~~~~ If an instance is built in highly available mode you can at any time fail it over to its secondary node, even if the primary has somehow failed and it's not up anymore. Doing it is really easy, on the master node you can just run:: $ gnt-instance failover %INSTANCE_NAME% That's it. After the command completes the secondary node is now the primary, and vice-versa. The instance will be started with an amount of memory between its ``maxmem`` and its ``minmem`` value, depending on the free memory on its target node, or the operation will fail if that's not possible. See :ref:`instance-startup-label` for details. If the instance's disk template is of type rbd, then you can specify the target node (which can be any node) explicitly, or specify an iallocator plugin. If you omit both, the default iallocator will be used to determine the target node:: $ gnt-instance failover -n %TARGET_NODE% %INSTANCE_NAME% Live migrating an instance ~~~~~~~~~~~~~~~~~~~~~~~~~~ If an instance is built in highly available mode, it currently runs and both its nodes are running fine, you can migrate it over to its secondary node, without downtime. On the master node you need to run:: $ gnt-instance migrate %INSTANCE_NAME% The current load on the instance and its memory size will influence how long the migration will take. In any case, for both KVM and Xen hypervisors, the migration will be transparent to the instance. If the destination node has less memory than the instance's current runtime memory, but at least the instance's minimum memory available Ganeti will automatically reduce the instance runtime memory before migrating it, unless the ``--no-runtime-changes`` option is passed, in which case the target node should have at least the instance's current runtime memory free. If the instance's disk template is of type rbd, then you can specify the target node (which can be any node) explicitly, or specify an iallocator plugin. If you omit both, the default iallocator will be used to determine the target node:: $ gnt-instance migrate -n %TARGET_NODE% %INSTANCE_NAME% Moving an instance (offline) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If an instance has not been create as mirrored, then the only way to change its primary node is to execute the move command:: $ gnt-instance move -n %NEW_NODE% %INSTANCE% This has a few prerequisites: - the instance must be stopped - its current primary node must be on-line and healthy - the disks of the instance must not have any errors Since this operation actually copies the data from the old node to the new node, expect it to take proportional to the size of the instance's disks and the speed of both the nodes' I/O system and their networking. Disk operations +++++++++++++++ Disk failures are a common cause of errors in any server deployment. Ganeti offers protection from single-node failure if your instances were created in HA mode, and it also offers ways to restore redundancy after a failure. Preparing for disk operations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is important to note that for Ganeti to be able to do any disk operation, the Linux machines on top of which Ganeti runs must be consistent; for LVM, this means that the LVM commands must not return failures; it is common that after a complete disk failure, any LVM command aborts with an error similar to:: $ vgs /dev/sdb1: read failed after 0 of 4096 at 0: Input/output error /dev/sdb1: read failed after 0 of 4096 at 750153695232: Input/output error /dev/sdb1: read failed after 0 of 4096 at 0: Input/output error Couldn't find device with uuid 't30jmN-4Rcf-Fr5e-CURS-pawt-z0jU-m1TgeJ'. Couldn't find all physical volumes for volume group xenvg. Before restoring an instance's disks to healthy status, it's needed to fix the volume group used by Ganeti so that we can actually create and manage the logical volumes. This is usually done in a multi-step process: #. first, if the disk is completely gone and LVM commands exit with “Couldn't find device with uuidâ€Ļ” then you need to run the command:: $ vgreduce --removemissing %VOLUME_GROUP% #. after the above command, the LVM commands should be executing normally (warnings are normal, but the commands will not fail completely). #. if the failed disk is still visible in the output of the ``pvs`` command, you need to deactivate it from allocations by running:: $ pvs -x n /dev/%DISK% At this point, the volume group should be consistent and any bad physical volumes should not longer be available for allocation. Note that since version 2.1 Ganeti provides some commands to automate these two operations, see :ref:`storage-units-label`. .. _rest-redundancy-label: Restoring redundancy for DRBD-based instances ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A DRBD instance has two nodes, and the storage on one of them has failed. Depending on which node (primary or secondary) has failed, you have three options at hand: - if the storage on the primary node has failed, you need to re-create the disks on it - if the storage on the secondary node has failed, you can either re-create the disks on it or change the secondary and recreate redundancy on the new secondary node Of course, at any point it's possible to force re-creation of disks even though everything is already fine. For all three cases, the ``replace-disks`` operation can be used:: # re-create disks on the primary node $ gnt-instance replace-disks -p %INSTANCE_NAME% # re-create disks on the current secondary $ gnt-instance replace-disks -s %INSTANCE_NAME% # change the secondary node, via manual specification $ gnt-instance replace-disks -n %NODE% %INSTANCE_NAME% # change the secondary node, via an iallocator script $ gnt-instance replace-disks -I %SCRIPT% %INSTANCE_NAME% # since Ganeti 2.1: automatically fix the primary or secondary node $ gnt-instance replace-disks -a %INSTANCE_NAME% Since the process involves copying all data from the working node to the target node, it will take a while, depending on the instance's disk size, node I/O system and network speed. But it is (barring any network interruption) completely transparent for the instance. Re-creating disks for non-redundant instances ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. versionadded:: 2.1 For non-redundant instances, there isn't a copy (except backups) to re-create the disks. But it's possible to at-least re-create empty disks, after which a reinstall can be run, via the ``recreate-disks`` command:: $ gnt-instance recreate-disks %INSTANCE% Note that this will fail if the disks already exists. The instance can be assigned to new nodes automatically by specifying an iallocator through the ``--iallocator`` option. Conversion of an instance's disk type ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is possible to convert between a non-redundant instance of type ``plain`` (LVM storage) and redundant ``drbd`` via the ``gnt-instance modify`` command:: # start with a non-redundant instance $ gnt-instance add -t plain ... %INSTANCE% # later convert it to redundant $ gnt-instance stop %INSTANCE% $ gnt-instance modify -t drbd -n %NEW_SECONDARY% %INSTANCE% $ gnt-instance start %INSTANCE% # and convert it back $ gnt-instance stop %INSTANCE% $ gnt-instance modify -t plain %INSTANCE% $ gnt-instance start %INSTANCE% The conversion must be done while the instance is stopped, and converting from plain to drbd template presents a small risk, especially if the instance has multiple disks and/or if one node fails during the conversion procedure). As such, it's recommended (as always) to make sure that downtime for manual recovery is acceptable and that the instance has up-to-date backups. Debugging instances +++++++++++++++++++ Accessing an instance's disks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From an instance's primary node you can have access to its disks. Never ever mount the underlying logical volume manually on a fault tolerant instance, or will break replication and your data will be inconsistent. The correct way to access an instance's disks is to run (on the master node, as usual) the command:: $ gnt-instance activate-disks %INSTANCE% And then, *on the primary node of the instance*, access the device that gets created. For example, you could mount the given disks, then edit files on the filesystem, etc. Note that with partitioned disks (as opposed to whole-disk filesystems), you will need to use a tool like :manpage:`kpartx(8)`:: # on node1 $ gnt-instance activate-disks %instance1% node3:disk/0:â€Ļ $ ssh node3 # on node 3 $ kpartx -l /dev/â€Ļ $ kpartx -a /dev/â€Ļ $ mount /dev/mapper/â€Ļ /mnt/ # edit files under mnt as desired $ umount /mnt/ $ kpartx -d /dev/â€Ļ $ exit # back to node 1 After you've finished you can deactivate them with the deactivate-disks command, which works in the same way:: $ gnt-instance deactivate-disks %INSTANCE% Note that if any process started by you is still using the disks, the above command will error out, and you **must** cleanup and ensure that the above command runs successfully before you start the instance, otherwise the instance will suffer corruption. Accessing an instance's console ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The command to access a running instance's console is:: $ gnt-instance console %INSTANCE_NAME% Use the console normally and then type ``^]`` when done, to exit. Other instance operations +++++++++++++++++++++++++ Reboot ~~~~~~ There is a wrapper command for rebooting instances:: $ gnt-instance reboot %instance2% By default, this does the equivalent of shutting down and then starting the instance, but it accepts parameters to perform a soft-reboot (via the hypervisor), a hard reboot (hypervisor shutdown and then startup) or a full one (the default, which also de-configures and then configures again the disks of the instance). Instance OS definitions debugging +++++++++++++++++++++++++++++++++ Should you have any problems with instance operating systems the command to see a complete status for all your nodes is:: $ gnt-os diagnose .. _instance-relocation-label: Instance relocation ~~~~~~~~~~~~~~~~~~~ While it is not possible to move an instance from nodes ``(A, B)`` to nodes ``(C, D)`` in a single move, it is possible to do so in a few steps:: # instance is located on A, B $ gnt-instance replace-disks -n %nodeC% %instance1% # instance has moved from (A, B) to (A, C) # we now flip the primary/secondary nodes $ gnt-instance migrate %instance1% # instance lives on (C, A) # we can then change A to D via: $ gnt-instance replace-disks -n %nodeD% %instance1% Which brings it into the final configuration of ``(C, D)``. Note that we needed to do two replace-disks operation (two copies of the instance disks), because we needed to get rid of both the original nodes (A and B). Network Management ------------------ Ganeti used to describe NICs of an Instance with an IP, a MAC, a connectivity link and mode. This had three major shortcomings: * there was no easy way to assign a unique IP to an instance * network info (subnet, gateway, domain, etc.) was not available on target node (kvm-ifup, hooks, etc) * one should explicitly pass L2 info (mode, and link) to every NIC Plus there was no easy way to get the current networking overview (which instances are on the same L2 or L3 network, which IPs are reserved, etc). All the above required an external management tool that has an overall view and provides the corresponding info to Ganeti. gnt-network aims to support a big part of this functionality inside Ganeti and abstract the network as a separate entity. Currently, a Ganeti network provides the following: * A single IPv4 pool, subnet and gateway * Connectivity info per nodegroup (mode, link) * MAC prefix for each NIC inside the network * IPv6 prefix/Gateway related to this network * Tags IP pool management ensures IP uniqueness inside this network. The user can pass `ip=pool,network=test` and will: 1. Get the first available IP in the pool 2. Inherit the connectivity mode and link of the network's netparams 3. NIC will obtain the MAC prefix of the network 4. All network related info will be available as environment variables in kvm-ifup scripts and hooks, so that they can dynamically manage all networking-related setup on the host. Hands on with gnt-network +++++++++++++++++++++++++ To create a network do:: # gnt-network add --network=192.0.2.0/24 --gateway=192.0.2.1 test Please see all other available options (--add-reserved-ips, --mac-prefix, --network6, --gateway6, --tags). Currently, IPv6 info is not used by Ganeti itself. It only gets exported to NIC configuration scripts and hooks via environment variables. To make this network available on a nodegroup you should specify the connectivity mode and link during connection:: # gnt-network connect --nic-parameters mode=bridged,link=br100 test default nodegroup1 To add a NIC inside this network:: # gnt-instance modify --net -1:add,ip=pool,network=test inst1 This will let a NIC obtain a unique IP inside this network, and inherit the nodegroup's netparams (bridged, br100). IP here is optional. If missing the NIC will just get the L2 info. To move an existing NIC from a network to another and remove its IP:: # gnt-instance modify --net -1:ip=none,network=test1 inst1 This will release the old IP from the old IP pool and the NIC will inherit the new nicparams. On the above actions there is a extra option `--no-conflicts-ckeck`. This does not check for conflicting setups. Specifically: 1. When a network is added, IPs of nodes and master are not being checked. 2. When connecting a network on a nodegroup, IPs of instances inside this nodegroup are not checked whether they reside inside the subnet or not. 3. When specifying explicitly a IP without passing a network, Ganeti will not check if this IP is included inside any available network on the nodegroup. External components +++++++++++++++++++ All the aforementioned steps assure NIC configuration from the Ganeti perspective. Of course this has nothing to do, how the instance eventually will get the desired connectivity (IPv4, IPv6, default routes, DNS info, etc) and where will the IP resolve. This functionality is managed by the external components. Let's assume that the VM will need to obtain a dynamic IP via DHCP, get a SLAAC address, and use DHCPv6 for other configuration information (in case RFC-6106 is not supported by the client, e.g. Windows). This means that the following external services are needed: 1. A DHCP server 2. An IPv6 router sending Router Advertisements 3. A DHCPv6 server exporting DNS info 4. A dynamic DNS server These components must be configured dynamically and on a per NIC basis. The way to do this is by using custom kvm-ifup scripts and hooks. Node operations --------------- There are much fewer node operations available than for instances, but they are equivalently important for maintaining a healthy cluster. Add/readd +++++++++ It is at any time possible to extend the cluster with one more node, by using the node add operation:: $ gnt-node add %NEW_NODE% If the cluster has a replication network defined, then you need to pass the ``-s REPLICATION_IP`` parameter to this option. A variation of this command can be used to re-configure a node if its Ganeti configuration is broken, for example if it has been reinstalled by mistake:: $ gnt-node add --readd %EXISTING_NODE% This will reinitialise the node as if it's been newly added, but while keeping its existing configuration in the cluster (primary/secondary IP, etc.), in other words you won't need to use ``-s`` here. Changing the node role ++++++++++++++++++++++ A node can be in different roles, as explained in the :ref:`terminology-label` section. Promoting a node to the master role is special, while the other roles are handled all via a single command. Failing over the master node ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you want to promote a different node to the master role (for whatever reason), run on any other master-candidate node the command:: $ gnt-cluster master-failover and the node you ran it on is now the new master. In case you try to run this on a non master-candidate node, you will get an error telling you which nodes are valid. Changing between the other roles ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``gnt-node modify`` command can be used to select a new role:: # change to master candidate $ gnt-node modify -C yes %NODE% # change to drained status $ gnt-node modify -D yes %NODE% # change to offline status $ gnt-node modify -O yes %NODE% # change to regular mode (reset all flags) $ gnt-node modify -O no -D no -C no %NODE% Note that the cluster requires that at any point in time, a certain number of nodes are master candidates, so changing from master candidate to other roles might fail. It is recommended to either force the operation (via the ``--force`` option) or first change the number of master candidates in the cluster - see :ref:`cluster-config-label`. Evacuating nodes ++++++++++++++++ There are two steps of moving instances off a node: - moving the primary instances (actually converting them into secondary instances) - moving the secondary instances (including any instances converted in the step above) Primary instance conversion ~~~~~~~~~~~~~~~~~~~~~~~~~~~ For this step, you can use either individual instance move commands (as seen in :ref:`instance-change-primary-label`) or the bulk per-node versions; these are:: $ gnt-node migrate %NODE% $ gnt-node evacuate -s %NODE% Note that the instance “move” command doesn't currently have a node equivalent. Both these commands, or the equivalent per-instance command, will make this node the secondary node for the respective instances, whereas their current secondary node will become primary. Note that it is not possible to change in one step the primary node to another node as primary, while keeping the same secondary node. Secondary instance evacuation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ For the evacuation of secondary instances, a command called :command:`gnt-node evacuate` is provided and its syntax is:: $ gnt-node evacuate -I %IALLOCATOR_SCRIPT% %NODE% $ gnt-node evacuate -n %DESTINATION_NODE% %NODE% The first version will compute the new secondary for each instance in turn using the given iallocator script, whereas the second one will simply move all instances to DESTINATION_NODE. Removal +++++++ Once a node no longer has any instances (neither primary nor secondary), it's easy to remove it from the cluster:: $ gnt-node remove %NODE_NAME% This will deconfigure the node, stop the ganeti daemons on it and leave it hopefully like before it joined to the cluster. Replication network changes +++++++++++++++++++++++++++ The :command:`gnt-node modify -s` command can be used to change the secondary IP of a node. This operation can only be performed if: - No instance is active on the target node - The new target IP is reachable from the master's secondary IP Also this operation will not allow to change a node from single-homed (same primary and secondary ip) to multi-homed (separate replication network) or vice versa, unless: - The target node is the master node and `--force` is passed. - The target cluster is single-homed and the new primary ip is a change to single homed for a particular node. - The target cluster is multi-homed and the new primary ip is a change to multi homed for a particular node. For example to do a single-homed to multi-homed conversion:: $ gnt-node modify --force -s %SECONDARY_IP% %MASTER_NAME% $ gnt-node modify -s %SECONDARY_IP% %NODE1_NAME% $ gnt-node modify -s %SECONDARY_IP% %NODE2_NAME% $ gnt-node modify -s %SECONDARY_IP% %NODE3_NAME% ... The same commands can be used for multi-homed to single-homed except the secondary IPs should be the same as the primaries for each node, for that case. Storage handling ++++++++++++++++ When using LVM (either standalone or with DRBD), it can become tedious to debug and fix it in case of errors. Furthermore, even file-based storage can become complicated to handle manually on many hosts. Ganeti provides a couple of commands to help with automation. Logical volumes ~~~~~~~~~~~~~~~ This is a command specific to LVM handling. It allows listing the logical volumes on a given node or on all nodes and their association to instances via the ``volumes`` command:: $ gnt-node volumes Node PhysDev VG Name Size Instance node1 /dev/sdb1 xenvg e61fbc97-â€Ļ.disk0 512M instance17 node1 /dev/sdb1 xenvg ebd1a7d1-â€Ļ.disk0 512M instance19 node2 /dev/sdb1 xenvg 0af08a3d-â€Ļ.disk0 512M instance20 node2 /dev/sdb1 xenvg cc012285-â€Ļ.disk0 512M instance16 node2 /dev/sdb1 xenvg f0fac192-â€Ļ.disk0 512M instance18 The above command maps each logical volume to a volume group and underlying physical volume and (possibly) to an instance. .. _storage-units-label: Generalized storage handling ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. versionadded:: 2.1 Starting with Ganeti 2.1, a new storage framework has been implemented that tries to abstract the handling of the storage type the cluster uses. First is listing the backend storage and their space situation:: $ gnt-node list-storage Node Type Name Size Used Free Allocatable node1 lvm-vg xenvg 3.6T 0M 3.6T Y node2 lvm-vg xenvg 3.6T 0M 3.6T Y node3 lvm-vg xenvg 3.6T 2.0G 3.6T Y The default is to list LVM physical volumes. It's also possible to list the LVM volume groups:: $ gnt-node list-storage -t lvm-vg Node Type Name Size Used Free Allocatable node1 lvm-vg xenvg 3.6T 0M 3.6T Y node2 lvm-vg xenvg 3.6T 0M 3.6T Y node3 lvm-vg xenvg 3.6T 2.0G 3.6T Y Next is repairing storage units, which is currently only implemented for volume groups and does the equivalent of ``vgreduce --removemissing``:: $ gnt-node repair-storage %node2% lvm-vg xenvg Sun Oct 25 22:21:45 2009 Repairing storage unit 'xenvg' on node2 ... Last is the modification of volume properties, which is (again) only implemented for LVM physical volumes and allows toggling the ``allocatable`` value:: $ gnt-node modify-storage --allocatable=no %node2% lvm-pv /dev/%sdb1% Use of the storage commands ~~~~~~~~~~~~~~~~~~~~~~~~~~~ All these commands are needed when recovering a node from a disk failure: - first, we need to recover from complete LVM failure (due to missing disk), by running the ``repair-storage`` command - second, we need to change allocation on any partially-broken disk (i.e. LVM still sees it, but it has bad blocks) by running ``modify-storage`` - then we can evacuate the instances as needed Cluster operations ------------------ Beside the cluster initialisation command (which is detailed in the :doc:`install` document) and the master failover command which is explained under node handling, there are a couple of other cluster operations available. .. _cluster-config-label: Standard operations +++++++++++++++++++ One of the few commands that can be run on any node (not only the master) is the ``getmaster`` command:: # on node2 $ gnt-cluster getmaster node1.example.com It is possible to query and change global cluster parameters via the ``info`` and ``modify`` commands:: $ gnt-cluster info Cluster name: cluster.example.com Cluster UUID: 07805e6f-f0af-4310-95f1-572862ee939c Creation time: 2009-09-25 05:04:15 Modification time: 2009-10-18 22:11:47 Master node: node1.example.com Architecture (this node): 64bit (x86_64) â€Ļ Tags: foo Default hypervisor: xen-pvm Enabled hypervisors: xen-pvm Hypervisor parameters: - xen-pvm: root_path: /dev/sda1 â€Ļ Cluster parameters: - candidate pool size: 10 â€Ļ Default instance parameters: - default: memory: 128 â€Ļ Default nic parameters: - default: link: xen-br0 â€Ļ There various parameters above can be changed via the ``modify`` commands as follows: - the hypervisor parameters can be changed via ``modify -H xen-pvm:root_path=â€Ļ``, and so on for other hypervisors/key/values - the "default instance parameters" are changeable via ``modify -B parameter=valueâ€Ļ`` syntax - the cluster parameters are changeable via separate options to the modify command (e.g. ``--candidate-pool-size``, etc.) For detailed option list see the :manpage:`gnt-cluster(8)` man page. The cluster version can be obtained via the ``version`` command:: $ gnt-cluster version Software version: 2.1.0 Internode protocol: 20 Configuration format: 2010000 OS api version: 15 Export interface: 0 This is not very useful except when debugging Ganeti. Global node commands ++++++++++++++++++++ There are two commands provided for replicating files to all nodes of a cluster and for running commands on all the nodes:: $ gnt-cluster copyfile %/path/to/file% $ gnt-cluster command %ls -l /path/to/file% These are simple wrappers over scp/ssh and more advanced usage can be obtained using :manpage:`dsh(1)` and similar commands. But they are useful to update an OS script from the master node, for example. Cluster verification ++++++++++++++++++++ There are three commands that relate to global cluster checks. The first one is ``verify`` which gives an overview on the cluster state, highlighting any issues. In normal operation, this command should return no ``ERROR`` messages:: $ gnt-cluster verify Sun Oct 25 23:08:58 2009 * Verifying global settings Sun Oct 25 23:08:58 2009 * Gathering data (2 nodes) Sun Oct 25 23:09:00 2009 * Verifying node status Sun Oct 25 23:09:00 2009 * Verifying instance status Sun Oct 25 23:09:00 2009 * Verifying orphan volumes Sun Oct 25 23:09:00 2009 * Verifying remaining instances Sun Oct 25 23:09:00 2009 * Verifying N+1 Memory redundancy Sun Oct 25 23:09:00 2009 * Other Notes Sun Oct 25 23:09:00 2009 - NOTICE: 5 non-redundant instance(s) found. Sun Oct 25 23:09:00 2009 * Hooks Results The second command is ``verify-disks``, which checks that the instance's disks have the correct status based on the desired instance state (up/down):: $ gnt-cluster verify-disks Note that this command will show no output when disks are healthy. The last command is used to repair any discrepancies in Ganeti's recorded disk size and the actual disk size (disk size information is needed for proper activation and growth of DRBD-based disks):: $ gnt-cluster repair-disk-sizes Sun Oct 25 23:13:16 2009 - INFO: Disk 0 of instance instance1 has mismatched size, correcting: recorded 512, actual 2048 Sun Oct 25 23:13:17 2009 - WARNING: Invalid result from node node4, ignoring node results The above shows one instance having wrong disk size, and a node which returned invalid data, and thus we ignored all primary instances of that node. Configuration redistribution ++++++++++++++++++++++++++++ If the verify command complains about file mismatches between the master and other nodes, due to some node problems or if you manually modified configuration files, you can force an push of the master configuration to all other nodes via the ``redist-conf`` command:: $ gnt-cluster redist-conf This command will be silent unless there are problems sending updates to the other nodes. Cluster renaming ++++++++++++++++ It is possible to rename a cluster, or to change its IP address, via the ``rename`` command. If only the IP has changed, you need to pass the current name and Ganeti will realise its IP has changed:: $ gnt-cluster rename %cluster.example.com% This will rename the cluster to 'cluster.example.com'. If you are connected over the network to the cluster name, the operation is very dangerous as the IP address will be removed from the node and the change may not go through. Continue? y/[n]/?: %y% Failure: prerequisites not met for this operation: Neither the name nor the IP address of the cluster has changed In the above output, neither value has changed since the cluster initialisation so the operation is not completed. Queue operations ++++++++++++++++ The job queue execution in Ganeti 2.0 and higher can be inspected, suspended and resumed via the ``queue`` command:: $ gnt-cluster queue info The drain flag is unset $ gnt-cluster queue drain $ gnt-instance stop %instance1% Failed to submit job for instance1: Job queue is drained, refusing job $ gnt-cluster queue info The drain flag is set $ gnt-cluster queue undrain This is most useful if you have an active cluster and you need to upgrade the Ganeti software, or simply restart the software on any node: #. suspend the queue via ``queue drain`` #. wait until there are no more running jobs via ``gnt-job list`` #. restart the master or another node, or upgrade the software #. resume the queue via ``queue undrain`` .. note:: this command only stores a local flag file, and if you failover the master, it will not have effect on the new master. Watcher control +++++++++++++++ The :manpage:`ganeti-watcher(8)` is a program, usually scheduled via ``cron``, that takes care of cluster maintenance operations (restarting downed instances, activating down DRBD disks, etc.). However, during maintenance and troubleshooting, this can get in your way; disabling it via commenting out the cron job is not so good as this can be forgotten. Thus there are some commands for automated control of the watcher: ``pause``, ``info`` and ``continue``:: $ gnt-cluster watcher info The watcher is not paused. $ gnt-cluster watcher pause %1h% The watcher is paused until Mon Oct 26 00:30:37 2009. $ gnt-cluster watcher info The watcher is paused until Mon Oct 26 00:30:37 2009. $ ganeti-watcher -d 2009-10-25 23:30:47,984: pid=28867 ganeti-watcher:486 DEBUG Pause has been set, exiting $ gnt-cluster watcher continue The watcher is no longer paused. $ ganeti-watcher -d 2009-10-25 23:31:04,789: pid=28976 ganeti-watcher:345 DEBUG Archived 0 jobs, left 0 2009-10-25 23:31:05,884: pid=28976 ganeti-watcher:280 DEBUG Got data from cluster, writing instance status file 2009-10-25 23:31:06,061: pid=28976 ganeti-watcher:150 DEBUG Data didn't change, just touching status file $ gnt-cluster watcher info The watcher is not paused. The exact details of the argument to the ``pause`` command are available in the manpage. .. note:: this command only stores a local flag file, and if you failover the master, it will not have effect on the new master. Node auto-maintenance +++++++++++++++++++++ If the cluster parameter ``maintain_node_health`` is enabled (see the manpage for :command:`gnt-cluster`, the init and modify subcommands), then the following will happen automatically: - the watcher will shutdown any instances running on offline nodes - the watcher will deactivate any DRBD devices on offline nodes In the future, more actions are planned, so only enable this parameter if the nodes are completely dedicated to Ganeti; otherwise it might be possible to lose data due to auto-maintenance actions. Removing a cluster entirely +++++++++++++++++++++++++++ The usual method to cleanup a cluster is to run ``gnt-cluster destroy`` however if the Ganeti installation is broken in any way then this will not run. It is possible in such a case to cleanup manually most if not all traces of a cluster installation by following these steps on all of the nodes: 1. Shutdown all instances. This depends on the virtualisation method used (Xen, KVM, etc.): - Xen: run ``xl list`` and ``xl destroy`` on all the non-Domain-0 instances - KVM: kill all the KVM processes - chroot: kill all processes under the chroot mountpoints 2. If using DRBD, shutdown all DRBD minors (which should by at this time no-longer in use by instances); on each node, run ``drbdsetup /dev/drbdN down`` for each active DRBD minor. 3. If using LVM, cleanup the Ganeti volume group; if only Ganeti created logical volumes (and you are not sharing the volume group with the OS, for example), then simply running ``lvremove -f xenvg`` (replace 'xenvg' with your volume group name) should do the required cleanup. 4. If using file-based storage, remove recursively all files and directories under your file-storage directory: ``rm -rf /srv/ganeti/file-storage/*`` replacing the path with the correct path for your cluster. 5. Stop the ganeti daemons (``/etc/init.d/ganeti stop``) and kill any that remain alive (``pgrep ganeti`` and ``pkill ganeti``). 6. Remove the ganeti state directory (``rm -rf /var/lib/ganeti/*``), replacing the path with the correct path for your installation. 7. If using RBD, run ``rbd unmap /dev/rbdN`` to unmap the RBD disks. Then remove the RBD disk images used by Ganeti, identified by their UUIDs (``rbd rm uuid.rbd.diskN``). On the master node, remove the cluster from the master-netdev (usually ``xen-br0`` for bridged mode, otherwise ``eth0`` or similar), by running ``ip a del $clusterip/32 dev xen-br0`` (use the correct cluster ip and network device name). At this point, the machines are ready for a cluster creation; in case you want to remove Ganeti completely, you need to also undo some of the SSH changes and log directories: - ``rm -rf /var/log/ganeti /srv/ganeti`` (replace with the correct paths) - remove from ``/root/.ssh`` the keys that Ganeti added (check the ``authorized_keys`` and ``id_dsa`` files) - regenerate the host's SSH keys (check the OpenSSH startup scripts) - uninstall Ganeti Otherwise, if you plan to re-create the cluster, you can just go ahead and rerun ``gnt-cluster init``. Replacing the SSH and SSL keys ++++++++++++++++++++++++++++++ Ganeti uses both SSL and SSH keys, and actively modifies the SSH keys on the nodes. As result, in order to replace these keys, a few extra steps need to be followed: :doc:`cluster-keys-replacement` Hypervisor Configuration ------------------------ Ganeti comes with a comprehensive list of hypervisor parameters (called ``hvparams`` internally) which can be set at cluster or instance level. For now, this section will only cover KVM and only the most important ones. Please refer to the :manpage:`ganeti-instance(8)` man page for a complete documentation. KVM Hypervisor ++++++++++++++ On cluster level, parameters can be changed using the ``gnt-cluster`` command: $ gnt-cluster modify -H kvm:param1=value,param2=value On instance level, please use ``gnt-instance``:abbr: $ gnt-instance modify -H param1=value,param2=value Choosing the right CPU type ~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default KVM will emulate the ``qemu64`` CPU type. This CPU lacks many features/flags of modern CPUs and recent Linux versions or Windows guests might even refuse/fail to boot. However, this will ensure maximum compatibilty for live migration between nodes with different CPU types. In general you should aim to have a homogenous cluster with matching CPU types. If that is the case, you can simply pass through the node's CPU type/model and provide your instances with all features of your hardware. $ gnt-cluster modify -H kvm:cpu_type=host If you have CPUs of different generations in your cluster, you need to find the oldest one and configure this as the common denominator. You can query your qemu version which CPU types it knows: $ qemu-system-x86_64 -cpu help You can then use ``gnt-cluster command`` to test the CPU types against all your nodes and choose one that works on all of them (replace ``$CPU_TYPE`` with e.g. ``EPYC-v3`` or ``Cooperlake``). $ gnt-cluster command 'qemu-system-x86_64 -cpu $CPU_TYPE,+pcid,+ssbd,+md-clear,enforce -machine accel=kvm -nographic -nodefaults -boot c,reboot-timeout=1 -no-reboot' You can then proceed and set ``cpu_type`` accordingly: $ gnt-cluster modify -H kvm:cpu_type=Cooperlake Serial vs VNC vs Spice console ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Ganeti offers multiple options to access your instance. If your Instance supports it, you can enable the serial console with the ``serial_console`` parameter. You can access your console from the master node using $ gnt-instance console %INSTANCE% When you use direct kernel boot (e.g. ``kernel_path`` points to a kernel on your node) Ganeti will automatically append serial options to the ``kernel_args`` parameter. If you use a fully virtualized KVM boot, make sure to configure serial parameters inside your instance (e.g. through GRUB). If you would like to use a graphical console, you can either enable VNC or Spice. While the latter has improved performance, the first has a wider range of clients available. To enable Spice, you need to configure ``spice_bind`` and set it either to a valid listening IPv4/IPv6 address (including 0.0.0.0 or 127.0.0.1) or the name of a network interface. Please check out all other ``spice_*`` parameters in the :manpage:`gnt-instance(8)` man pageto configure TLS, set compression etc. To enable VNC, you need to configure ``vnc_bind_address``. This parameter only allows IPv4 addresses to be specified. Please check out all other ``vnc_*`` parameters in the :manpage:`gnt-instance(8)` man page. Please keep in mind that you can always configure the serial console but VNC and Spice only work exclusively. Configuring both will result in an error from ``gnt-instance`` or ``gnt-cluster``. Disk I/O ~~~~~~~~ Ganeti supports both disk I/O modes ``threads`` and ``native`` and they can be controlled through the ``disk_aio`` parameter. In most cases ``native`` seems to perforem better, however ``threads`` is still Ganeti's default. If you care about reliable storage, we recommend setting ``disk_cache`` to ``none``, but depending on your usecase you should also look into the other possible values listed and explained in :manpage:`gnt-instance(8)`. Live Migration Settings ~~~~~~~~~~~~~~~~~~~~~~~ If you are using a storage backend that permits live migration (e.g. ``drbd``, ``rbd`` or ``sharedfile``) you may want to look at the relevant parameters. ``migration_bandwidth`` sets the amount of bandwidth QEMU is allowed to use for memory transfers (in megabyte/second) and ``migration_downtime`` sets the amount of time an instance is allowed to be frozen to transfer the remainding memory (in milliseconds). Keep in mind if your instance's memory changes fast and the migration downtime window is too small a live migration might run endlessly. At the same time, setting the migration downtime window to a larger value might break or at least confuse the software running in your instance. If you have configured your cluster to use a separate network for replication traffic, live migration traffic will use that network as well. You may also use ``postcopy-ram`` migration which means that an instance will migrate early over to the secondary node and fetch the remaining memory from the source afterwards. Keep in mind that regular migration will copy memory first and only migrate the instance over if that succeeds. If the destination node dies in the process, the instance will not be touched by this and keep running on the source node. With the postcopy method both nodes need to stay alive until everything has been transferred, otherwise the instance will die. Virtual Disk and Network Devices ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Ganeti defaults to using VirtIO devices for disks and networking. If an instance requires it, you may also switch the disk type to IDE or SCSI or emulate certain NIC types like ``e1000``. Please check the :manpage:`gnt-instance(8)` man page for the parameters ``disk_type`` and ``nic_type`` for all possible values. The setting always applies to *all* disks and NICs for a given instance. Configuring Storage ------------------- While setting up the details of the storage options is generally out of scope for this document, there are some tips that we can share with you Optimizing DRBD for faster networks +++++++++++++++++++++++++++++++++++ If you run your DRBD replication across 10Gbe links or faster, the default settings will usually prevent you from saturating those links. The following settings have proven to speed up things. Please keep in mind that any changes to these parameters will not affect running instances. $ gnt-cluster modify --disk-parameters drbd:disk-barriers=n,protocol=C $ gnt-cluster modify --disk-parameters drbd:dynamic-resync=true,c-plan-ahead=20,c-min-rate=104857600,c-max-rate=1073741824 $ gnt-cluster modify --disk-parameters drbd:net-custom='--max-buffers=16000 --max-epoch-size=16000' Userspace vs. Kernelspace +++++++++++++++++++++++++ With GlusterFS or RBD in combination with KVM you have the option to either use kernelspace access (through blockdevices) or through userspace implementations provided by QEMU. The latter will possibly speed things up. You can switch the access mode through ``gnt-cluster`` (default is ``kernelspace``): $ gnt-cluster modify --disk-parameters rbd:access=userspace Monitoring the cluster ---------------------- Starting with Ganeti 2.8, a monitoring daemon is available, providing information about the status and the performance of the system. The monitoring daemon runs on every node, listening on TCP port 1815. Each instance of the daemon provides information related to the node it is running on. .. include:: monitoring-query-format.rst Tags handling ------------- The tags handling (addition, removal, listing) is similar for all the objects that support it (instances, nodes, and the cluster). Limitations +++++++++++ Note that the set of characters present in a tag and the maximum tag length are restricted. Currently the maximum length is 128 characters, there can be at most 4096 tags per object, and the set of characters is comprised by alphanumeric characters and additionally ``.+*/:@-_``. Operations ++++++++++ Tags can be added via ``add-tags``:: $ gnt-instance add-tags %INSTANCE% %a% %b% %c% $ gnt-node add-tags %INSTANCE% %a% %b% %c% $ gnt-cluster add-tags %a% %b% %c% The above commands add three tags to an instance, to a node and to the cluster. Note that the cluster command only takes tags as arguments, whereas the node and instance commands first required the node and instance name. Tags can also be added from a file, via the ``--from=FILENAME`` argument. The file is expected to contain one tag per line. Tags can also be remove via a syntax very similar to the add one:: $ gnt-instance remove-tags %INSTANCE% %a% %b% %c% And listed via:: $ gnt-instance list-tags $ gnt-node list-tags $ gnt-cluster list-tags Global tag search +++++++++++++++++ It is also possible to execute a global search on the all tags defined in the cluster configuration, via a cluster command:: $ gnt-cluster search-tags %REGEXP% The parameter expected is a regular expression (see :manpage:`regex(7)`). This will return all tags that match the search, together with the object they are defined in (the names being show in a hierarchical kind of way):: $ gnt-cluster search-tags %o% /cluster foo /instances/instance1 owner:bar Autorepair ---------- The tool ``harep`` can be used to automatically fix some problems that are present in the cluster. It is mainly meant to be regularly and automatically executed as a cron job. This is quite evident by considering that, when executed, it does not immediately fix all the issues of the instances of the cluster, but it cycles the instances through a series of states, one at every ``harep`` execution. Every state performs a step towards the resolution of the problem. This process goes on until the instance is brought back to the healthy state, or the tool realizes that it is not able to fix the instance, and therefore marks it as in failure state. Allowing harep to act on the cluster ++++++++++++++++++++++++++++++++++++ By default, ``harep`` checks the status of the cluster but it is not allowed to perform any modification. Modification must be explicitly allowed by an appropriate use of tags. Tagging can be applied at various levels, and can enable different kinds of autorepair, as hereafter described. All the tags that authorize ``harep`` to perform modifications follow this syntax:: ganeti:watcher:autorepair: where ```` indicates the kind of intervention that can be performed. Every possible value of ```` includes at least all the authorization of the previous one, plus its own. The possible values, in increasing order of severity, are: - ``fix-storage`` allows a disk replacement or another operation that fixes the instance backend storage without affecting the instance itself. This can for example recover from a broken drbd secondary, but risks data loss if something is wrong on the primary but the secondary was somehow recoverable. - ``migrate`` allows an instance migration. This can recover from a drained primary, but can cause an instance crash in some cases (bugs). - ``failover`` allows instance reboot on the secondary. This can recover from an offline primary, but the instance will lose its running state. - ``reinstall`` allows disks to be recreated and an instance to be reinstalled. This can recover from primary&secondary both being offline, or from an offline primary in the case of non-redundant instances. It causes data loss. These autorepair tags can be applied to a cluster, a nodegroup or an instance, and will act where they are applied and to everything in the entities sub-tree (e.g. a tag applied to a nodegroup will apply to all the instances contained in that nodegroup, but not to the rest of the cluster). If there are multiple ``ganeti:watcher:autorepair:`` tags in an object (cluster, node group or instance), the least destructive tag takes precedence. When multiplicity happens across objects, the nearest tag wins. For example, if in a cluster with two instances, *I1* and *I2*, *I1* has ``failover``, and the cluster itself has both ``fix-storage`` and ``reinstall``, *I1* will end up with ``failover`` and *I2* with ``fix-storage``. Limiting harep ++++++++++++++ Sometimes it is useful to stop harep from performing its task temporarily, and it is useful to be able to do so without disrupting its configuration, that is, without removing the authorization tags. In order to do this, suspend tags are provided. Suspend tags can be added to cluster, nodegroup or instances, and act on the entire entities sub-tree. No operation will be performed by ``harep`` on the instances protected by a suspend tag. Their syntax is as follows:: ganeti:watcher:autorepair:suspend[:] If there are multiple suspend tags in an object, the form without timestamp takes precedence (permanent suspension); or, if all object tags have a timestamp, the one with the highest timestamp. Tags with a timestamp will be automatically removed when the time indicated by the timestamp is passed. Indefinite suspension tags have to be removed manually. Result reporting ++++++++++++++++ Harep will report about the result of its actions both through its CLI, and by adding tags to the instances it operated on. Such tags will follow the syntax hereby described:: ganeti:watcher:autorepair:result::::: If this tag is present a repair of type ``type`` has been performed on the instance and has been completed by ``timestamp``. The result is either ``success``, ``failure`` or ``enoperm``, and jobs is a *+*-separated list of jobs that were executed for this repair. An ``enoperm`` result is an error state due to permission problems. It is returned when the repair cannot proceed because it would require to perform an operation that is not allowed by the ``ganeti:watcher:autorepair:`` tag that is defining the instance autorepair permissions. NB: if an instance repair ends up in a failure state, it will not be touched again by ``harep`` until it has been manually fixed by the system administrator and the ``ganeti:watcher:autorepair:result:failure:*`` tag has been manually removed. Job operations -------------- The various jobs submitted by the instance/node/cluster commands can be examined, canceled and archived by various invocations of the ``gnt-job`` command. First is the job list command:: $ gnt-job list 17771 success INSTANCE_QUERY_DATA 17773 success CLUSTER_VERIFY_DISKS 17775 success CLUSTER_REPAIR_DISK_SIZES 17776 error CLUSTER_RENAME(cluster.example.com) 17780 success CLUSTER_REDIST_CONF 17792 success INSTANCE_REBOOT(instance1.example.com) More detailed information about a job can be found via the ``info`` command:: $ gnt-job info %17776% Job ID: 17776 Status: error Received: 2009-10-25 23:18:02.180569 Processing start: 2009-10-25 23:18:02.200335 (delta 0.019766s) Processing end: 2009-10-25 23:18:02.279743 (delta 0.079408s) Total processing time: 0.099174 seconds Opcodes: OP_CLUSTER_RENAME Status: error Processing start: 2009-10-25 23:18:02.200335 Processing end: 2009-10-25 23:18:02.252282 Input fields: name: cluster.example.com Result: OpPrereqError [Neither the name nor the IP address of the cluster has changed] Execution log: During the execution of a job, it's possible to follow the output of a job, similar to the log that one get from the ``gnt-`` commands, via the watch command:: $ gnt-instance add --submit â€Ļ %instance1% JobID: 17818 $ gnt-job watch %17818% Output from job 17818 follows ----------------------------- Mon Oct 26 00:22:48 2009 - INFO: Selected nodes for instance instance1 via iallocator dumb: node1, node2 Mon Oct 26 00:22:49 2009 * creating instance disks... Mon Oct 26 00:22:52 2009 adding instance instance1 to cluster config Mon Oct 26 00:22:52 2009 - INFO: Waiting for instance instance1 to sync disks. â€Ļ Mon Oct 26 00:23:03 2009 creating os for instance instance1 on node node1 Mon Oct 26 00:23:03 2009 * running the instance OS create scripts... Mon Oct 26 00:23:13 2009 * starting instance... $ This is useful if you need to follow a job's progress from multiple terminals. A job that has not yet started to run can be canceled:: $ gnt-job cancel %17810% But not one that has already started execution:: $ gnt-job cancel %17805% Job 17805 is no longer waiting in the queue There are two queues for jobs: the *current* and the *archive* queue. Jobs are initially submitted to the current queue, and they stay in that queue until they have finished execution (either successfully or not). At that point, they can be moved into the archive queue using e.g. ``gnt-job autoarchive all``. The ``ganeti-watcher`` script will do this automatically 6 hours after a job is finished. The ``ganeti-cleaner`` script will then remove archived the jobs from the archive directory after three weeks. Note that ``gnt-job list`` only shows jobs in the current queue. Archived jobs can be viewed using ``gnt-job info ``. Special Ganeti deployments -------------------------- Since Ganeti 2.4, it is possible to extend the Ganeti deployment with two custom scenarios: Ganeti inside Ganeti and multi-site model. Running Ganeti under Ganeti +++++++++++++++++++++++++++ It is sometimes useful to be able to use a Ganeti instance as a Ganeti node (part of another cluster, usually). One example scenario is two small clusters, where we want to have an additional master candidate that holds the cluster configuration and can be used for helping with the master voting process. However, these Ganeti instance should not host instances themselves, and should not be considered in the normal capacity planning, evacuation strategies, etc. In order to accomplish this, mark these nodes as non-``vm_capable``:: $ gnt-node modify --vm-capable=no %node3% The vm_capable status can be listed as usual via ``gnt-node list``:: $ gnt-node list -oname,vm_capable Node VMCapable node1 Y node2 Y node3 N When this flag is set, the cluster will not do any operations that relate to instances on such nodes, e.g. hypervisor operations, disk-related operations, etc. Basically they will just keep the ssconf files, and if master candidates the full configuration. Multi-site model ++++++++++++++++ If Ganeti is deployed in multi-site model, with each site being a node group (so that instances are not relocated across the WAN by mistake), it is conceivable that either the WAN latency is high or that some sites have a lower reliability than others. In this case, it doesn't make sense to replicate the job information across all sites (or even outside of a “central” node group), so it should be possible to restrict which nodes can become master candidates via the auto-promotion algorithm. Ganeti 2.4 introduces for this purpose a new ``master_capable`` flag, which (when unset) prevents nodes from being marked as master candidates, either manually or automatically. As usual, the node modify operation can change this flag:: $ gnt-node modify --auto-promote --master-capable=no %node3% Fri Jan 7 06:23:07 2011 - INFO: Demoting from master candidate Fri Jan 7 06:23:08 2011 - INFO: Promoted nodes to master candidate role: node4 Modified node node3 - master_capable -> False - master_candidate -> False And the node list operation will list this flag:: $ gnt-node list -oname,master_capable %node1% %node2% %node3% Node MasterCapable node1 Y node2 Y node3 N Note that marking a node both not ``vm_capable`` and not ``master_capable`` makes the node practically unusable from Ganeti's point of view. Hence these two flags should be used probably in contrast: some nodes will be only master candidates (master_capable but not vm_capable), and other nodes will only hold instances (vm_capable but not master_capable). Ganeti tools ------------ Beside the usual ``gnt-`` and ``ganeti-`` commands which are provided and installed in ``$prefix/sbin`` at install time, there are a couple of other tools installed which are used seldom but can be helpful in some cases. lvmstrap ++++++++ The ``lvmstrap`` tool, introduced in :ref:`configure-lvm-label` section, has two modes of operation: - ``diskinfo`` shows the discovered disks on the system and their status - ``create`` takes all not-in-use disks and creates a volume group out of them .. warning:: The ``create`` argument to this command causes data-loss! cfgupgrade ++++++++++ The ``cfgupgrade`` tools is used to upgrade between major (and minor) Ganeti versions, and to roll back. Point-releases are usually transparent for the admin. More information about the upgrade procedure is listed on the wiki at http://code.google.com/p/ganeti/wiki/UpgradeNotes. There is also a script designed to upgrade from Ganeti 1.2 to 2.0, called ``cfgupgrade12``. cfgshell ++++++++ .. note:: This command is not actively maintained; make sure you backup your configuration before using it This can be used as an alternative to direct editing of the main configuration file if Ganeti has a bug and prevents you, for example, from removing an instance or a node from the configuration file. .. _burnin-label: burnin ++++++ .. warning:: This command will erase existing instances if given as arguments! This tool is used to exercise either the hardware of machines or alternatively the Ganeti software. It is safe to run on an existing cluster **as long as you don't pass it existing instance names**. The command will, by default, execute a comprehensive set of operations against a list of instances, these being: - creation - disk replacement (for redundant instances) - failover and migration (for redundant instances) - move (for non-redundant instances) - disk growth - add disks, remove disk - add NICs, remove NICs - export and then import - rename - reboot - shutdown/startup - and finally removal of the test instances Executing all these operations will test that the hardware performs well: the creation, disk replace, disk add and disk growth will exercise the storage and network; the migrate command will test the memory of the systems. Depending on the passed options, it can also test that the instance OS definitions are executing properly the rename, import and export operations. sanitize-config +++++++++++++++ This tool takes the Ganeti configuration and outputs a "sanitized" version, by randomizing or clearing: - DRBD secrets and cluster public key (always) - host names (optional) - IPs (optional) - OS names (optional) - LV names (optional, only useful for very old clusters which still have instances whose LVs are based on the instance name) By default, all optional items are activated except the LV name randomization. When passing ``--no-randomization``, which disables the optional items (i.e. just the DRBD secrets and cluster public keys are randomized), the resulting file can be used as a safety copy of the cluster config - while not trivial, the layout of the cluster can be recreated from it and if the instance disks have not been lost it permits recovery from the loss of all master candidates. move-instance +++++++++++++ See :doc:`separate documentation for move-instance `. users-setup +++++++++++ Ganeti can either be run entirely as root, or with every daemon running as its own specific user (if the parameters ``--with-user-prefix`` and/or ``--with-group-prefix`` have been specified at ``./configure``-time). In case split users are activated, they are required to exist on the system, and they need to belong to the proper groups in order for the access permissions to files and programs to be correct. The ``users-setup`` tool, when run, takes care of setting up the proper users and groups. When invoked without parameters, the tool runs in interactive mode, showing the list of actions it will perform and asking for confirmation before proceeding. Providing the ``--yes-do-it`` parameter to the tool prevents the confirmation from being asked, and the users and groups will be created immediately. .. TODO: document cluster-merge tool Other Ganeti projects --------------------- Below is a list (which might not be up-to-date) of additional projects that can be useful in a Ganeti deployment. They can be downloaded from the project site (http://www.ganeti.org) and the repositories are also on the project git site (https://github.com/ganeti). NBMA tools ++++++++++ The ``ganeti-nbma`` software is designed to allow instances to live on a separate, virtual network from the nodes, and in an environment where nodes are not guaranteed to be able to reach each other via multicasting or broadcasting. For more information see the README in the source archive. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/cluster-keys-replacement.rst000064400000000000000000000103251476477700300220330ustar00rootroot00000000000000======================== Cluster Keys Replacement ======================== Ganeti uses both SSL and SSH keys, and actively modifies the SSH keys on the nodes. As result, in order to replace these keys, a few extra steps need to be followed. For an example when this could be needed, see the thread at `Regenerating SSL and SSH keys after the security bug in Debian's OpenSSL `_. Ganeti uses OpenSSL for encryption on the RPC layer and SSH for executing commands. The SSL certificate is automatically generated when the cluster is initialized and it's copied to added nodes automatically together with the master's SSH host key. Note that paths below may vary depending on your distribution. In general, modifications should be done on the master node and then distributed to all nodes of a cluster (possibly using a pendrive - but don't forget to use "shred" to remove files securely afterwards). Replacing SSL keys ================== The cluster-wide SSL key is stored in ``/var/lib/ganeti/server.pem``. Besides that, since Ganeti 2.11, each node has an individual node SSL key, which is stored in ``/var/lib/ganeti/client.pem``. This client certificate is signed by the cluster-wide SSL certificate. To renew the individual node certificates, run this command:: gnt-cluster renew-crypto --new-node-certificates Run the following command to generate a new cluster-wide certificate:: gnt-cluster renew-crypto --new-cluster-certificate Note that this triggers both, the renewal of the cluster certificate as well as the renewal of the individual node certificate. The reason for this is that the node certificates are signed by the cluster certificate and thus they need to be renewed and signed as soon as the changes certificate changes. Therefore, the command above is equivalent to:: gnt-cluster renew-crypto --new-cluster-certificate --new-node-certificates On older versions, which don't have this command, use this instead:: chmod 0600 /var/lib/ganeti/server.pem && openssl req -new -newkey rsa:1024 -days 1825 -nodes \ -x509 -keyout /var/lib/ganeti/server.pem \ -out /var/lib/ganeti/server.pem -batch && chmod 0400 /var/lib/ganeti/server.pem && /etc/init.d/ganeti restart gnt-cluster copyfile /var/lib/ganeti/server.pem gnt-cluster command /etc/init.d/ganeti restart Note that older versions don't have individual node certificates and thus one does not have to handle the creation and distribution of them. Replacing SSH keys ================== There are two sets of SSH keys in the cluster: the host keys (both DSA and RSA, though Ganeti only uses the RSA one) and the root's DSA key (Ganeti uses DSA for historically reasons, in the future RSA will be used). host keys +++++++++ These are the files named ``/etc/ssh/ssh_host_*``. You need to manually recreate them; it's possibly that the startup script of OpenSSH will generate them if they don't exist, or that the package system regenerates them. Also make sure to copy the master's SSH host keys to all other nodes. cluster public key file +++++++++++++++++++++++ The new public rsa host key created in the previous step must be added in two places: #. known hosts file, ``/var/lib/ganeti/known_hosts`` #. cluster configuration file, ``/var/lib/ganeti/config.data`` Edit these two files and update them with newly generated SSH host key (in the previous step, take it from the ``/etc/ssh/ssh_host_rsa_key.pub``). For the ``config.data`` file, please look for an entry named ``rsahostkeypub`` and replace the value for it with the contents of the ``.pub`` file. For the ``known_hosts`` file, you need to replace the old key with the new one on each line (for each host). root's key ++++++++++ These are the files named ``~root/.ssh/id_dsa*``. Run this command to rebuild them:: ssh-keygen -t dsa -f ~root/.ssh/id_dsa -q -N "" root's ``authorized_keys`` ++++++++++++++++++++++++++ This is the file named ``~root/.ssh/authorized_keys``. Edit file and update it with the newly generated root key, from the ``id_dsa.pub`` file generated in the previous step. Finish ====== In the end, the files mentioned above should be identical for all nodes in a cluster. Also do not forget to run ``gnt-cluster verify``. ganeti-3.1.0~rc2/doc/cluster-merge.rst000064400000000000000000000047211476477700300176650ustar00rootroot00000000000000================ Merging clusters ================ With ``cluster-merge`` from the ``tools`` directory it is possible to merge two or more clusters into one single cluster. If anything goes wrong at any point the script suggests you rollback steps you've to perform *manually* if there are any. The point of no return is when the master daemon is started the first time after merging the configuration files. A rollback at this point would involve a lot of manual work. For the internal design of this tool have a look at the `Automated Ganeti Cluster Merger ` document. Merge Clusters ============== The tool has to be invoked on the cluster you like to merge the other clusters into. The usage of ``cluster-merge`` is as follows:: cluster-merge [--debug|--verbose] [--watcher-pause-period SECONDS] \ [--groups [merge|rename]] [] You can provide multiple clusters. The tool will then go over every cluster in serial and perform the steps to merge it into the invoking cluster. These options can be used to control the behaviour of the tool: ``--debug``/``--verbose`` These options are mutually exclusive and increase the level of output to either debug output or just more verbose output like action performed right now. ``--watcher-pause-period`` Define the period of time in seconds the watcher shall be disabled, default is 1800 seconds (30 minutes). ``--groups`` This option controls how ``cluster-merge`` handles duplicate node group names on the merging clusters. If ``merge`` is specified then all node groups with the same name will be merged into one. If ``rename`` is specified, then conflicting node groups on the remove clusters will have their cluster name appended to the group name. If this option is not speicifed, then ``cluster-merge`` will refuse to continue if it finds conflicting group names, otherwise it will proceed as normal. Rollback ======== If for any reason something in the merge doesn't work the way it should ``cluster-merge`` will abort, provide an error message and optionally rollback steps. Please be aware that after a certain point there's no easy way to rollback the cluster to its previous state. If you've reached that point the tool will not provide any rollback steps. If you end up with rollback steps, please perform them before invoking the tool again. It doesn't keep state over invokations. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/conf.py000064400000000000000000000162611476477700300156560ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Ganeti documentation build configuration file, created by # sphinx-quickstart on Tue Apr 14 13:23:20 2009. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys, os enable_manpages = bool(os.getenv("ENABLE_MANPAGES")) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.append(os.path.abspath(".")) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. needs_sphinx = "1.0" # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named "sphinx.ext.*") or your custom ones. extensions = [ "sphinx.ext.todo", "sphinx.ext.graphviz", "ganeti.build.sphinx_ext", "ganeti.build.shell_example_lexer", ] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix of source filenames. source_suffix = ".rst" # The encoding of source files. source_encoding = "utf-8" # The master toctree document. master_doc = "index" # General information about the project. project = "Ganeti" copyright = "%s Google Inc." % ", ".join(map(str, range(2006, 2015 + 1))) # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # These next two will be passed via the command line, see the makefile # The short X.Y version #version = VERSION_MAJOR + "." + VERSION_MINOR # The full version, including alpha/beta/rc tags. #release = PACKAGE_VERSION # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. language = "en" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = "" # Else, today_fmt is used as the format for a strftime call. #today_fmt = "%B %d, %Y" # List of documents that shouldn't be included in the build. #unused_docs = [] if enable_manpages: exclude_patterns = [] else: exclude_patterns = [ "man-*.rst", ] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = [ "_build", "api", "coverage" "examples", ] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, "()" will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = "classic" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = [] # If not "", a "Last updated on:" timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = "%b %d, %Y" # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. html_use_modindex = False # If false, no index is generated. html_use_index = False # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = "" # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = "" # Output file base name for HTML help builder. htmlhelp_basename = "Ganetidoc" # -- Options for LaTeX output -------------------------------------------------- # The paper size ("letter" or "a4"). #latex_paper_size = "a4" # The font size ("10pt", "11pt" or "12pt"). #latex_font_size = "10pt" # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ("index", "Ganeti.tex", "Ganeti Documentation", "Google Inc.", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Additional stuff for the LaTeX preamble. #latex_preamble = "" # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. latex_domain_indices = False # We override the manpage role and sphinx issues a warning, which is treated as # error. Suppress role_add warnings to avoid FTBFS. suppress_warnings = ["app.add_role"] ganeti-3.1.0~rc2/doc/design-2.0.rst000064400000000000000000002312061476477700300166550ustar00rootroot00000000000000================= Ganeti 2.0 design ================= This document describes the major changes in Ganeti 2.0 compared to the 1.2 version. The 2.0 version will constitute a rewrite of the 'core' architecture, paving the way for additional features in future 2.x versions. .. contents:: :depth: 3 Objective ========= Ganeti 1.2 has many scalability issues and restrictions due to its roots as software for managing small and 'static' clusters. Version 2.0 will attempt to remedy first the scalability issues and then the restrictions. Background ========== While Ganeti 1.2 is usable, it severely limits the flexibility of the cluster administration and imposes a very rigid model. It has the following main scalability issues: - only one operation at a time on the cluster [#]_ - poor handling of node failures in the cluster - mixing hypervisors in a cluster not allowed It also has a number of artificial restrictions, due to historical design: - fixed number of disks (two) per instance - fixed number of NICs .. [#] Replace disks will release the lock, but this is an exception and not a recommended way to operate The 2.0 version is intended to address some of these problems, and create a more flexible code base for future developments. Among these problems, the single-operation at a time restriction is biggest issue with the current version of Ganeti. It is such a big impediment in operating bigger clusters that many times one is tempted to remove the lock just to do a simple operation like start instance while an OS installation is running. Scalability problems -------------------- Ganeti 1.2 has a single global lock, which is used for all cluster operations. This has been painful at various times, for example: - It is impossible for two people to efficiently interact with a cluster (for example for debugging) at the same time. - When batch jobs are running it's impossible to do other work (for example failovers/fixes) on a cluster. This poses scalability problems: as clusters grow in node and instance size it's a lot more likely that operations which one could conceive should run in parallel (for example because they happen on different nodes) are actually stalling each other while waiting for the global lock, without a real reason for that to happen. One of the main causes of this global lock (beside the higher difficulty of ensuring data consistency in a more granular lock model) is the fact that currently there is no long-lived process in Ganeti that can coordinate multiple operations. Each command tries to acquire the so called *cmd* lock and when it succeeds, it takes complete ownership of the cluster configuration and state. Other scalability problems are due the design of the DRBD device model, which assumed at its creation a low (one to four) number of instances per node, which is no longer true with today's hardware. Artificial restrictions ----------------------- Ganeti 1.2 (and previous versions) have a fixed two-disks, one-NIC per instance model. This is a purely artificial restrictions, but it touches multiple areas (configuration, import/export, command line) that it's more fitted to a major release than a minor one. Architecture issues ------------------- The fact that each command is a separate process that reads the cluster state, executes the command, and saves the new state is also an issue on big clusters where the configuration data for the cluster begins to be non-trivial in size. Overview ======== In order to solve the scalability problems, a rewrite of the core design of Ganeti is required. While the cluster operations themselves won't change (e.g. start instance will do the same things, the way these operations are scheduled internally will change radically. The new design will change the cluster architecture to: .. digraph:: "ganeti-2.0-architecture" compound=false concentrate=true mclimit=100.0 nslimit=100.0 edge[fontsize="8" fontname="Helvetica-Oblique"] node[width="0" height="0" fontsize="12" fontcolor="black" shape=rect] subgraph outside { rclient[label="external clients"] label="Outside the cluster" } subgraph cluster_inside { label="ganeti cluster" labeljust=l subgraph cluster_master_node { label="master node" rapi[label="RAPI daemon"] cli[label="CLI"] watcher[label="Watcher"] burnin[label="Burnin"] masterd[shape=record style=filled label="{ luxi endpoint | master I/O thread | job queue | { worker| worker | worker }}"] {rapi;cli;watcher;burnin} -> masterd:luxi [label="LUXI" labelpos=100] } subgraph cluster_nodes { label="nodes" noded1 [shape=record label="{ RPC listener | Disk management | Network management | Hypervisor } "] noded2 [shape=record label="{ RPC listener | Disk management | Network management | Hypervisor } "] noded3 [shape=record label="{ RPC listener | Disk management | Network management | Hypervisor } "] } masterd:w2 -> {noded1;noded2;noded3} [label="node RPC"] cli -> {noded1;noded2;noded3} [label="SSH"] } rclient -> rapi [label="RAPI protocol"] This differs from the 1.2 architecture by the addition of the master daemon, which will be the only entity to talk to the node daemons. Detailed design =============== The changes for 2.0 can be split into roughly three areas: - core changes that affect the design of the software - features (or restriction removals) but which do not have a wide impact on the design - user-level and API-level changes which translate into differences for the operation of the cluster Core changes ------------ The main changes will be switching from a per-process model to a daemon based model, where the individual gnt-* commands will be clients that talk to this daemon (see `Master daemon`_). This will allow us to get rid of the global cluster lock for most operations, having instead a per-object lock (see `Granular locking`_). Also, the daemon will be able to queue jobs, and this will allow the individual clients to submit jobs without waiting for them to finish, and also see the result of old requests (see `Job Queue`_). Beside these major changes, another 'core' change but that will not be as visible to the users will be changing the model of object attribute storage, and separate that into name spaces (such that an Xen PVM instance will not have the Xen HVM parameters). This will allow future flexibility in defining additional parameters. For more details see `Object parameters`_. The various changes brought in by the master daemon model and the read-write RAPI will require changes to the cluster security; we move away from Twisted and use HTTP(s) for intra- and extra-cluster communications. For more details, see the security document in the doc/ directory. Master daemon ~~~~~~~~~~~~~ In Ganeti 2.0, we will have the following *entities*: - the master daemon (on the master node) - the node daemon (on all nodes) - the command line tools (on the master node) - the RAPI daemon (on the master node) The master-daemon related interaction paths are: - (CLI tools/RAPI daemon) and the master daemon, via the so called *LUXI* API - the master daemon and the node daemons, via the node RPC There are also some additional interaction paths for exceptional cases: - CLI tools might access via SSH the nodes (for ``gnt-cluster copyfile`` and ``gnt-cluster command``) - master failover is a special case when a non-master node will SSH and do node-RPC calls to the current master The protocol between the master daemon and the node daemons will be changed from (Ganeti 1.2) Twisted PB (perspective broker) to HTTP(S), using a simple PUT/GET of JSON-encoded messages. This is done due to difficulties in working with the Twisted framework and its protocols in a multithreaded environment, which we can overcome by using a simpler stack (see the caveats section). The protocol between the CLI/RAPI and the master daemon will be a custom one (called *LUXI*): on a UNIX socket on the master node, with rights restricted by filesystem permissions, the CLI/RAPI will talk to the master daemon using JSON-encoded messages. The operations supported over this internal protocol will be encoded via a python library that will expose a simple API for its users. Internally, the protocol will simply encode all objects in JSON format and decode them on the receiver side. For more details about the RAPI daemon see `Remote API changes`_, and for the node daemon see `Node daemon changes`_. .. _luxi: The LUXI protocol +++++++++++++++++ As described above, the protocol for making requests or queries to the master daemon will be a UNIX-socket based simple RPC of JSON-encoded messages. The choice of UNIX was in order to get rid of the need of authentication and authorisation inside Ganeti; for 2.0, the permissions on the Unix socket itself will determine the access rights. We will have two main classes of operations over this API: - cluster query functions - job related functions The cluster query functions are usually short-duration, and are the equivalent of the ``OP_QUERY_*`` opcodes in Ganeti 1.2 (and they are internally implemented still with these opcodes). The clients are guaranteed to receive the response in a reasonable time via a timeout. The job-related functions will be: - submit job - query job (which could also be categorized in the query-functions) - archive job (see the job queue design doc) - wait for job change, which allows a client to wait without polling For more details of the actual operation list, see the `Job Queue`_. Both requests and responses will consist of a JSON-encoded message followed by the ``ETX`` character (ASCII decimal 3), which is not a valid character in JSON messages and thus can serve as a message delimiter. The contents of the messages will be a dictionary with two fields: :method: the name of the method called :args: the arguments to the method, as a list (no keyword arguments allowed) Responses will follow the same format, with the two fields being: :success: a boolean denoting the success of the operation :result: the actual result, or error message in case of failure There are two special value for the result field: - in the case that the operation failed, and this field is a list of length two, the client library will try to interpret is as an exception, the first element being the exception type and the second one the actual exception arguments; this will allow a simple method of passing Ganeti-related exception across the interface - for the *WaitForChange* call (that waits on the server for a job to change status), if the result is equal to ``nochange`` instead of the usual result for this call (a list of changes), then the library will internally retry the call; this is done in order to differentiate internally between master daemon hung and job simply not changed Users of the API that don't use the provided python library should take care of the above two cases. Master daemon implementation ++++++++++++++++++++++++++++ The daemon will be based around a main I/O thread that will wait for new requests from the clients, and that does the setup/shutdown of the other thread (pools). There will two other classes of threads in the daemon: - job processing threads, part of a thread pool, and which are long-lived, started at daemon startup and terminated only at shutdown time - client I/O threads, which are the ones that talk the local protocol (LUXI) to the clients, and are short-lived Master startup/failover +++++++++++++++++++++++ In Ganeti 1.x there is no protection against failing over the master to a node with stale configuration. In effect, the responsibility of correct failovers falls on the admin. This is true both for the new master and for when an old, offline master startup. Since in 2.x we are extending the cluster state to cover the job queue and have a daemon that will execute by itself the job queue, we want to have more resilience for the master role. The following algorithm will happen whenever a node is ready to transition to the master role, either at startup time or at node failover: #. read the configuration file and parse the node list contained within #. query all the nodes and make sure we obtain an agreement via a quorum of at least half plus one nodes for the following: - we have the latest configuration and job list (as determined by the serial number on the configuration and highest job ID on the job queue) - if we are not failing over (but just starting), the quorum agrees that we are the designated master - if any of the above is false, we prevent the current operation (i.e. we don't become the master) #. at this point, the node transitions to the master role #. for all the in-progress jobs, mark them as failed, with reason unknown or something similar (master failed, etc.) Since due to exceptional conditions we could have a situation in which no node can become the master due to inconsistent data, we will have an override switch for the master daemon startup that will assume the current node has the right data and will replicate all the configuration files to the other nodes. **Note**: the above algorithm is by no means an election algorithm; it is a *confirmation* of the master role currently held by a node. Logging +++++++ The logging system will be switched completely to the standard python logging module; currently it's logging-based, but exposes a different API, which is just overhead. As such, the code will be switched over to standard logging calls, and only the setup will be custom. With this change, we will remove the separate debug/info/error logs, and instead have always one logfile per daemon model: - master-daemon.log for the master daemon - node-daemon.log for the node daemon (this is the same as in 1.2) - rapi-daemon.log for the RAPI daemon logs - rapi-access.log, an additional log file for the RAPI that will be in the standard HTTP log format for possible parsing by other tools Since the :term:`watcher` will only submit jobs to the master for startup of the instances, its log file will contain less information than before, mainly that it will start the instance, but not the results. Node daemon changes +++++++++++++++++++ The only change to the node daemon is that, since we need better concurrency, we don't process the inter-node RPC calls in the node daemon itself, but we fork and process each request in a separate child. Since we don't have many calls, and we only fork (not exec), the overhead should be minimal. Caveats +++++++ A discussed alternative is to keep the current individual processes touching the cluster configuration model. The reasons we have not chosen this approach is: - the speed of reading and unserializing the cluster state today is not small enough that we can ignore it; the addition of the job queue will make the startup cost even higher. While this runtime cost is low, it can be on the order of a few seconds on bigger clusters, which for very quick commands is comparable to the actual duration of the computation itself - individual commands would make it harder to implement a fire-and-forget job request, along the lines "start this instance but do not wait for it to finish"; it would require a model of backgrounding the operation and other things that are much better served by a daemon-based model Another area of discussion is moving away from Twisted in this new implementation. While Twisted has its advantages, there are also many disadvantages to using it: - first and foremost, it's not a library, but a framework; thus, if you use twisted, all the code needs to be 'twiste-ized' and written in an asynchronous manner, using deferreds; while this method works, it's not a common way to code and it requires that the entire process workflow is based around a single *reactor* (Twisted name for a main loop) - the more advanced granular locking that we want to implement would require, if written in the async-manner, deep integration with the Twisted stack, to such an extend that business-logic is inseparable from the protocol coding; we felt that this is an unreasonable request, and that a good protocol library should allow complete separation of low-level protocol calls and business logic; by comparison, the threaded approach combined with HTTPs protocol required (for the first iteration) absolutely no changes from the 1.2 code, and later changes for optimizing the inter-node RPC calls required just syntactic changes (e.g. ``rpc.call_...`` to ``self.rpc.call_...``) Another issue is with the Twisted API stability - during the Ganeti 1.x lifetime, we had to to implement many times workarounds to changes in the Twisted version, so that for example 1.2 is able to use both Twisted 2.x and 8.x. In the end, since we already had an HTTP server library for the RAPI, we just reused that for inter-node communication. Granular locking ~~~~~~~~~~~~~~~~ We want to make sure that multiple operations can run in parallel on a Ganeti Cluster. In order for this to happen we need to make sure concurrently run operations don't step on each other toes and break the cluster. This design addresses how we are going to deal with locking so that: - we preserve data coherency - we prevent deadlocks - we prevent job starvation Reaching the maximum possible parallelism is a Non-Goal. We have identified a set of operations that are currently bottlenecks and need to be parallelised and have worked on those. In the future it will be possible to address other needs, thus making the cluster more and more parallel one step at a time. This section only talks about parallelising Ganeti level operations, aka Logical Units, and the locking needed for that. Any other synchronization lock needed internally by the code is outside its scope. Library details +++++++++++++++ The proposed library has these features: - internally managing all the locks, making the implementation transparent from their usage - automatically grabbing multiple locks in the right order (avoid deadlock) - ability to transparently handle conversion to more granularity - support asynchronous operation (future goal) Locking will be valid only on the master node and will not be a distributed operation. Therefore, in case of master failure, the operations currently running will be aborted and the locks will be lost; it remains to the administrator to cleanup (if needed) the operation result (e.g. make sure an instance is either installed correctly or removed). A corollary of this is that a master-failover operation with both masters alive needs to happen while no operations are running, and therefore no locks are held. All the locks will be represented by objects (like ``lockings.SharedLock``), and the individual locks for each object will be created at initialisation time, from the config file. The API will have a way to grab one or more than one locks at the same time. Any attempt to grab a lock while already holding one in the wrong order will be checked for, and fail. The Locks +++++++++ At the first stage we have decided to provide the following locks: - One "config file" lock - One lock per node in the cluster - One lock per instance in the cluster All the instance locks will need to be taken before the node locks, and the node locks before the config lock. Locks will need to be acquired at the same time for multiple instances and nodes, and internal ordering will be dealt within the locking library, which, for simplicity, will just use alphabetical order. Each lock has the following three possible statuses: - unlocked (anyone can grab the lock) - shared (anyone can grab/have the lock but only in shared mode) - exclusive (no one else can grab/have the lock) Handling conversion to more granularity +++++++++++++++++++++++++++++++++++++++ In order to convert to a more granular approach transparently each time we split a lock into more we'll create a "metalock", which will depend on those sub-locks and live for the time necessary for all the code to convert (or forever, in some conditions). When a metalock exists all converted code must acquire it in shared mode, so it can run concurrently, but still be exclusive with old code, which acquires it exclusively. In the beginning the only such lock will be what replaces the current "command" lock, and will acquire all the locks in the system, before proceeding. This lock will be called the "Big Ganeti Lock" because holding that one will avoid any other concurrent Ganeti operations. We might also want to devise more metalocks (eg. all nodes, all nodes+config) in order to make it easier for some parts of the code to acquire what it needs without specifying it explicitly. In the future things like the node locks could become metalocks, should we decide to split them into an even more fine grained approach, but this will probably be only after the first 2.0 version has been released. Adding/Removing locks +++++++++++++++++++++ When a new instance or a new node is created an associated lock must be added to the list. The relevant code will need to inform the locking library of such a change. This needs to be compatible with every other lock in the system, especially metalocks that guarantee to grab sets of resources without specifying them explicitly. The implementation of this will be handled in the locking library itself. When instances or nodes disappear from the cluster the relevant locks must be removed. This is easier than adding new elements, as the code which removes them must own them exclusively already, and thus deals with metalocks exactly as normal code acquiring those locks. Any operation queuing on a removed lock will fail after its removal. Asynchronous operations +++++++++++++++++++++++ For the first version the locking library will only export synchronous operations, which will block till the needed lock are held, and only fail if the request is impossible or somehow erroneous. In the future we may want to implement different types of asynchronous operations such as: - try to acquire this lock set and fail if not possible - try to acquire one of these lock sets and return the first one you were able to get (or after a timeout) (select/poll like) These operations can be used to prioritize operations based on available locks, rather than making them just blindly queue for acquiring them. The inherent risk, though, is that any code using the first operation, or setting a timeout for the second one, is susceptible to starvation and thus may never be able to get the required locks and complete certain tasks. Considering this providing/using these operations should not be among our first priorities. Locking granularity +++++++++++++++++++ For the first version of this code we'll convert each Logical Unit to acquire/release the locks it needs, so locking will be at the Logical Unit level. In the future we may want to split logical units in independent "tasklets" with their own locking requirements. A different design doc (or mini design doc) will cover the move from Logical Units to tasklets. Code examples +++++++++++++ In general when acquiring locks we should use a code path equivalent to:: lock.acquire() try: ... # other code finally: lock.release() This makes sure we release all locks, and avoid possible deadlocks. Of course extra care must be used not to leave, if possible locked structures in an unusable state. Note that with Python 2.5 a simpler syntax will be possible, but we want to keep compatibility with Python 2.4 so the new constructs should not be used. In order to avoid this extra indentation and code changes everywhere in the Logical Units code, we decided to allow LUs to declare locks, and then execute their code with their locks acquired. In the new world LUs are called like this:: # user passed names are expanded to the internal lock/resource name, # then known needed locks are declared lu.ExpandNames() ... some locking/adding of locks may happen ... # late declaration of locks for one level: this is useful because sometimes # we can't know which resource we need before locking the previous level lu.DeclareLocks() # for each level (cluster, instance, node) ... more locking/adding of locks can happen ... # these functions are called with the proper locks held lu.CheckPrereq() lu.Exec() ... locks declared for removal are removed, all acquired locks released ... The Processor and the LogicalUnit class will contain exact documentation on how locks are supposed to be declared. Caveats +++++++ This library will provide an easy upgrade path to bring all the code to granular locking without breaking everything, and it will also guarantee against a lot of common errors. Code switching from the old "lock everything" lock to the new system, though, needs to be carefully scrutinised to be sure it is really acquiring all the necessary locks, and none has been overlooked or forgotten. The code can contain other locks outside of this library, to synchronise other threaded code (eg for the job queue) but in general these should be leaf locks or carefully structured non-leaf ones, to avoid deadlock race conditions. .. _jqueue-original-design: Job Queue ~~~~~~~~~ Granular locking is not enough to speed up operations, we also need a queue to store these and to be able to process as many as possible in parallel. A Ganeti job will consist of multiple ``OpCodes`` which are the basic element of operation in Ganeti 1.2 (and will remain as such). Most command-level commands are equivalent to one OpCode, or in some cases to a sequence of opcodes, all of the same type (e.g. evacuating a node will generate N opcodes of type replace disks). Job execution—“Life of a Ganeti job” ++++++++++++++++++++++++++++++++++++ #. Job gets submitted by the client. A new job identifier is generated and assigned to the job. The job is then automatically replicated [#replic]_ to all nodes in the cluster. The identifier is returned to the client. #. A pool of worker threads waits for new jobs. If all are busy, the job has to wait and the first worker finishing its work will grab it. Otherwise any of the waiting threads will pick up the new job. #. Client waits for job status updates by calling a waiting RPC function. Log message may be shown to the user. Until the job is started, it can also be canceled. #. As soon as the job is finished, its final result and status can be retrieved from the server. #. If the client archives the job, it gets moved to a history directory. There will be a method to archive all jobs older than a a given age. .. [#replic] We need replication in order to maintain the consistency across all nodes in the system; the master node only differs in the fact that now it is running the master daemon, but it if fails and we do a master failover, the jobs are still visible on the new master (though marked as failed). Failures to replicate a job to other nodes will be only flagged as errors in the master daemon log if more than half of the nodes failed, otherwise we ignore the failure, and rely on the fact that the next update (for still running jobs) will retry the update. For finished jobs, it is less of a problem. Future improvements will look into checking the consistency of the job list and jobs themselves at master daemon startup. Job storage +++++++++++ Jobs are stored in the filesystem as individual files, serialized using JSON (standard serialization mechanism in Ganeti). The choice of storing each job in its own file was made because: - a file can be atomically replaced - a file can easily be replicated to other nodes - checking consistency across nodes can be implemented very easily, since all job files should be (at a given moment in time) identical The other possible choices that were discussed and discounted were: - single big file with all job data: not feasible due to difficult updates - in-process databases: hard to replicate the entire database to the other nodes, and replicating individual operations does not mean wee keep consistency Queue structure +++++++++++++++ All file operations have to be done atomically by writing to a temporary file and subsequent renaming. Except for log messages, every change in a job is stored and replicated to other nodes. :: /var/lib/ganeti/queue/ job-1 (JSON encoded job description and status) [â€Ļ] job-37 job-38 job-39 lock (Queue managing process opens this file in exclusive mode) serial (Last job ID used) version (Queue format version) Locking +++++++ Locking in the job queue is a complicated topic. It is called from more than one thread and must be thread-safe. For simplicity, a single lock is used for the whole job queue. A more detailed description can be found in doc/locking.rst. Internal RPC ++++++++++++ RPC calls available between Ganeti master and node daemons: jobqueue_update(file_name, content) Writes a file in the job queue directory. jobqueue_purge() Cleans the job queue directory completely, including archived job. jobqueue_rename(old, new) Renames a file in the job queue directory. Client RPC ++++++++++ RPC between Ganeti clients and the Ganeti master daemon supports the following operations: SubmitJob(ops) Submits a list of opcodes and returns the job identifier. The identifier is guaranteed to be unique during the lifetime of a cluster. WaitForJobChange(job_id, fields, [â€Ļ], timeout) This function waits until a job changes or a timeout expires. The condition for when a job changed is defined by the fields passed and the last log message received. QueryJobs(job_ids, fields) Returns field values for the job identifiers passed. CancelJob(job_id) Cancels the job specified by identifier. This operation may fail if the job is already running, canceled or finished. ArchiveJob(job_id) Moves a job into the â€Ļ/archive/ directory. This operation will fail if the job has not been canceled or finished. Job and opcode status +++++++++++++++++++++ Each job and each opcode has, at any time, one of the following states: Queued The job/opcode was submitted, but did not yet start. Waiting The job/opcode is waiting for a lock to proceed. Running The job/opcode is running. Canceled The job/opcode was canceled before it started. Success The job/opcode ran and finished successfully. Error The job/opcode was aborted with an error. If the master is aborted while a job is running, the job will be set to the Error status once the master started again. History +++++++ Archived jobs are kept in a separate directory, ``/var/lib/ganeti/queue/archive/``. This is done in order to speed up the queue handling: by default, the jobs in the archive are not touched by any functions. Only the current (unarchived) jobs are parsed, loaded, and verified (if implemented) by the master daemon. Ganeti updates ++++++++++++++ The queue has to be completely empty for Ganeti updates with changes in the job queue structure. In order to allow this, there will be a way to prevent new jobs entering the queue. Object parameters ~~~~~~~~~~~~~~~~~ Across all cluster configuration data, we have multiple classes of parameters: A. cluster-wide parameters (e.g. name of the cluster, the master); these are the ones that we have today, and are unchanged from the current model #. node parameters #. instance specific parameters, e.g. the name of disks (LV), that cannot be shared with other instances #. instance parameters, that are or can be the same for many instances, but are not hypervisor related; e.g. the number of VCPUs, or the size of memory #. instance parameters that are hypervisor specific (e.g. kernel_path or PAE mode) The following definitions for instance parameters will be used below: :hypervisor parameter: a hypervisor parameter (or hypervisor specific parameter) is defined as a parameter that is interpreted by the hypervisor support code in Ganeti and usually is specific to a particular hypervisor (like the kernel path for :term:`PVM` which makes no sense for :term:`HVM`). :backend parameter: a backend parameter is defined as an instance parameter that can be shared among a list of instances, and is either generic enough not to be tied to a given hypervisor or cannot influence at all the hypervisor behaviour. For example: memory, vcpus, auto_balance All these parameters will be encoded into constants.py with the prefix "BE\_" and the whole list of parameters will exist in the set "BES_PARAMETERS" :proper parameter: a parameter whose value is unique to the instance (e.g. the name of a LV, or the MAC of a NIC) As a general rule, for all kind of parameters, “None” (or in JSON-speak, “nil”) will no longer be a valid value for a parameter. As such, only non-default parameters will be saved as part of objects in the serialization step, reducing the size of the serialized format. Cluster parameters ++++++++++++++++++ Cluster parameters remain as today, attributes at the top level of the Cluster object. In addition, two new attributes at this level will hold defaults for the instances: - hvparams, a dictionary indexed by hypervisor type, holding default values for hypervisor parameters that are not defined/overridden by the instances of this hypervisor type - beparams, a dictionary holding (for 2.0) a single element 'default', which holds the default value for backend parameters Node parameters +++++++++++++++ Node-related parameters are very few, and we will continue using the same model for these as previously (attributes on the Node object). There are three new node flags, described in a separate section "node flags" below. Instance parameters +++++++++++++++++++ As described before, the instance parameters are split in three: instance proper parameters, unique to each instance, instance hypervisor parameters and instance backend parameters. The “hvparams” and “beparams” are kept in two dictionaries at instance level. Only non-default parameters are stored (but once customized, a parameter will be kept, even with the same value as the default one, until reset). The names for hypervisor parameters in the instance.hvparams subtree should be choosen as generic as possible, especially if specific parameters could conceivably be useful for more than one hypervisor, e.g. ``instance.hvparams.vnc_console_port`` instead of using both ``instance.hvparams.hvm_vnc_console_port`` and ``instance.hvparams.kvm_vnc_console_port``. There are some special cases related to disks and NICs (for example): a disk has both Ganeti-related parameters (e.g. the name of the LV) and hypervisor-related parameters (how the disk is presented to/named in the instance). The former parameters remain as proper-instance parameters, while the latter value are migrated to the hvparams structure. In 2.0, we will have only globally-per-instance such hypervisor parameters, and not per-disk ones (e.g. all NICs will be exported as of the same type). Starting from the 1.2 list of instance parameters, here is how they will be mapped to the three classes of parameters: - name (P) - primary_node (P) - os (P) - hypervisor (P) - status (P) - memory (BE) - vcpus (BE) - nics (P) - disks (P) - disk_template (P) - network_port (P) - kernel_path (HV) - initrd_path (HV) - hvm_boot_order (HV) - hvm_acpi (HV) - hvm_pae (HV) - hvm_cdrom_image_path (HV) - hvm_nic_type (HV) - hvm_disk_type (HV) - vnc_bind_address (HV) - serial_no (P) Parameter validation ++++++++++++++++++++ To support the new cluster parameter design, additional features will be required from the hypervisor support implementations in Ganeti. The hypervisor support implementation API will be extended with the following features: :PARAMETERS: class-level attribute holding the list of valid parameters for this hypervisor :CheckParamSyntax(hvparams): checks that the given parameters are valid (as in the names are valid) for this hypervisor; usually just comparing ``hvparams.keys()`` and ``cls.PARAMETERS``; this is a class method that can be called from within master code (i.e. cmdlib) and should be safe to do so :ValidateParameters(hvparams): verifies the values of the provided parameters against this hypervisor; this is a method that will be called on the target node, from backend.py code, and as such can make node-specific checks (e.g. kernel_path checking) Default value application +++++++++++++++++++++++++ The application of defaults to an instance is done in the Cluster object, via two new methods as follows: - ``Cluster.FillHV(instance)``, returns 'filled' hvparams dict, based on instance's hvparams and cluster's ``hvparams[instance.hypervisor]`` - ``Cluster.FillBE(instance, be_type="default")``, which returns the beparams dict, based on the instance and cluster beparams The FillHV/BE transformations will be used, for example, in the RpcRunner when sending an instance for activation/stop, and the sent instance hvparams/beparams will have the final value (noded code doesn't know about defaults). LU code will need to self-call the transformation, if needed. Opcode changes ++++++++++++++ The parameter changes will have impact on the OpCodes, especially on the following ones: - ``OpInstanceCreate``, where the new hv and be parameters will be sent as dictionaries; note that all hv and be parameters are now optional, as the values can be instead taken from the cluster - ``OpInstanceQuery``, where we have to be able to query these new parameters; the syntax for names will be ``hvparam/$NAME`` and ``beparam/$NAME`` for querying an individual parameter out of one dictionary, and ``hvparams``, respectively ``beparams``, for the whole dictionaries - ``OpModifyInstance``, where the the modified parameters are sent as dictionaries Additionally, we will need new OpCodes to modify the cluster-level defaults for the be/hv sets of parameters. Caveats +++++++ One problem that might appear is that our classification is not complete or not good enough, and we'll need to change this model. As the last resort, we will need to rollback and keep 1.2 style. Another problem is that classification of one parameter is unclear (e.g. ``network_port``, is this BE or HV?); in this case we'll take the risk of having to move parameters later between classes. Security ++++++++ The only security issue that we foresee is if some new parameters will have sensitive value. If so, we will need to have a way to export the config data while purging the sensitive value. E.g. for the drbd shared secrets, we could export these with the values replaced by an empty string. Node flags ~~~~~~~~~~ Ganeti 2.0 adds three node flags that change the way nodes are handled within Ganeti and the related infrastructure (iallocator interaction, RAPI data export). *master candidate* flag +++++++++++++++++++++++ Ganeti 2.0 allows more scalability in operation by introducing parallelization. However, a new bottleneck is reached that is the synchronization and replication of cluster configuration to all nodes in the cluster. This breaks scalability as the speed of the replication decreases roughly with the size of the nodes in the cluster. The goal of the master candidate flag is to change this O(n) into O(1) with respect to job and configuration data propagation. Only nodes having this flag set (let's call this set of nodes the *candidate pool*) will have jobs and configuration data replicated. The cluster will have a new parameter (runtime changeable) called ``candidate_pool_size`` which represents the number of candidates the cluster tries to maintain (preferably automatically). This will impact the cluster operations as follows: - jobs and config data will be replicated only to a fixed set of nodes - master fail-over will only be possible to a node in the candidate pool - cluster verify needs changing to account for these two roles - external scripts will no longer have access to the configuration file (this is not recommended anyway) The caveats of this change are: - if all candidates are lost (completely), cluster configuration is lost (but it should be backed up external to the cluster anyway) - failed nodes which are candidate must be dealt with properly, so that we don't lose too many candidates at the same time; this will be reported in cluster verify - the 'all equal' concept of ganeti is no longer true - the partial distribution of config data means that all nodes will have to revert to ssconf files for master info (as in 1.2) Advantages: - speed on a 100+ nodes simulated cluster is greatly enhanced, even for a simple operation; ``gnt-instance remove`` on a diskless instance remove goes from ~9seconds to ~2 seconds - node failure of non-candidates will be less impacting on the cluster The default value for the candidate pool size will be set to 10 but this can be changed at cluster creation and modified any time later. Testing on simulated big clusters with sequential and parallel jobs show that this value (10) is a sweet-spot from performance and load point of view. *offline* flag ++++++++++++++ In order to support better the situation in which nodes are offline (e.g. for repair) without altering the cluster configuration, Ganeti needs to be told and needs to properly handle this state for nodes. This will result in simpler procedures, and less mistakes, when the amount of node failures is high on an absolute scale (either due to high failure rate or simply big clusters). Nodes having this attribute set will not be contacted for inter-node RPC calls, will not be master candidates, and will not be able to host instances as primaries. Setting this attribute on a node: - will not be allowed if the node is the master - will not be allowed if the node has primary instances - will cause the node to be demoted from the master candidate role (if it was), possibly causing another node to be promoted to that role This attribute will impact the cluster operations as follows: - querying these nodes for anything will fail instantly in the RPC library, with a specific RPC error (RpcResult.offline == True) - they will be listed in the Other section of cluster verify The code is changed in the following ways: - RPC calls were be converted to skip such nodes: - RpcRunner-instance-based RPC calls are easy to convert - static/classmethod RPC calls are harder to convert, and were left alone - the RPC results were unified so that this new result state (offline) can be differentiated - master voting still queries in repair nodes, as we need to ensure consistency in case the (wrong) masters have old data, and nodes have come back from repairs Caveats: - some operation semantics are less clear (e.g. what to do on instance start with offline secondary?); for now, these will just fail as if the flag is not set (but faster) - 2-node cluster with one node offline needs manual startup of the master with a special flag to skip voting (as the master can't get a quorum there) One of the advantages of implementing this flag is that it will allow in the future automation tools to automatically put the node in repairs and recover from this state, and the code (should/will) handle this much better than just timing out. So, future possible improvements (for later versions): - watcher will detect nodes which fail RPC calls, will attempt to ssh to them, if failure will put them offline - watcher will try to ssh and query the offline nodes, if successful will take them off the repair list Alternatives considered: The RPC call model in 2.0 is, by default, much nicer - errors are logged in the background, and job/opcode execution is clearer, so we could simply not introduce this. However, having this state will make both the codepaths clearer (offline vs. temporary failure) and the operational model (it's not a node with errors, but an offline node). *drained* flag ++++++++++++++ Due to parallel execution of jobs in Ganeti 2.0, we could have the following situation: - gnt-node migrate + failover is run - gnt-node evacuate is run, which schedules a long-running 6-opcode job for the node - partway through, a new job comes in that runs an iallocator script, which finds the above node as empty and a very good candidate - gnt-node evacuate has finished, but now it has to be run again, to clean the above instance(s) In order to prevent this situation, and to be able to get nodes into proper offline status easily, a new *drained* flag was added to the nodes. This flag (which actually means "is being, or was drained, and is expected to go offline"), will prevent allocations on the node, but otherwise all other operations (start/stop instance, query, etc.) are working without any restrictions. Interaction between flags +++++++++++++++++++++++++ While these flags are implemented as separate flags, they are mutually-exclusive and are acting together with the master node role as a single *node status* value. In other words, a flag is only in one of these roles at a given time. The lack of any of these flags denote a regular node. The current node status is visible in the ``gnt-cluster verify`` output, and the individual flags can be examined via separate flags in the ``gnt-node list`` output. These new flags will be exported in both the iallocator input message and via RAPI, see the respective man pages for the exact names. Feature changes --------------- The main feature-level changes will be: - a number of disk related changes - removal of fixed two-disk, one-nic per instance limitation Disk handling changes ~~~~~~~~~~~~~~~~~~~~~ The storage options available in Ganeti 1.x were introduced based on then-current software (first DRBD 0.7 then later DRBD 8) and the estimated usage patters. However, experience has later shown that some assumptions made initially are not true and that more flexibility is needed. One main assumption made was that disk failures should be treated as 'rare' events, and that each of them needs to be manually handled in order to ensure data safety; however, both these assumptions are false: - disk failures can be a common occurrence, based on usage patterns or cluster size - our disk setup is robust enough (referring to DRBD8 + LVM) that we could automate more of the recovery Note that we still don't have fully-automated disk recovery as a goal, but our goal is to reduce the manual work needed. As such, we plan the following main changes: - DRBD8 is much more flexible and stable than its previous version (0.7), such that removing the support for the ``remote_raid1`` template and focusing only on DRBD8 is easier - dynamic discovery of DRBD devices is not actually needed in a cluster that where the DRBD namespace is controlled by Ganeti; switching to a static assignment (done at either instance creation time or change secondary time) will change the disk activation time from O(n) to O(1), which on big clusters is a significant gain - remove the hard dependency on LVM (currently all available storage types are ultimately backed by LVM volumes) by introducing file-based storage Additionally, a number of smaller enhancements are also planned: - support variable number of disks - support read-only disks Future enhancements in the 2.x series, which do not require base design changes, might include: - enhancement of the LVM allocation method in order to try to keep all of an instance's virtual disks on the same physical disks - add support for DRBD8 authentication at handshake time in order to ensure each device connects to the correct peer - remove the restrictions on failover only to the secondary which creates very strict rules on cluster allocation DRBD minor allocation +++++++++++++++++++++ Currently, when trying to identify or activate a new DRBD (or MD) device, the code scans all in-use devices in order to see if we find one that looks similar to our parameters and is already in the desired state or not. Since this needs external commands to be run, it is very slow when more than a few devices are already present. Therefore, we will change the discovery model from dynamic to static. When a new device is logically created (added to the configuration) a free minor number is computed from the list of devices that should exist on that node and assigned to that device. At device activation, if the minor is already in use, we check if it has our parameters; if not so, we just destroy the device (if possible, otherwise we abort) and start it with our own parameters. This means that we in effect take ownership of the minor space for that device type; if there's a user-created DRBD minor, it will be automatically removed. The change will have the effect of reducing the number of external commands run per device from a constant number times the index of the first free DRBD minor to just a constant number. Removal of obsolete device types (MD, DRBD7) ++++++++++++++++++++++++++++++++++++++++++++ We need to remove these device types because of two issues. First, DRBD7 has bad failure modes in case of dual failures (both network and disk - it cannot propagate the error up the device stack and instead just panics. Second, due to the asymmetry between primary and secondary in MD+DRBD mode, we cannot do live failover (not even if we had MD+DRBD8). File-based storage support ++++++++++++++++++++++++++ Using files instead of logical volumes for instance storage would allow us to get rid of the hard requirement for volume groups for testing clusters and it would also allow usage of SAN storage to do live failover taking advantage of this storage solution. Better LVM allocation +++++++++++++++++++++ Currently, the LV to PV allocation mechanism is a very simple one: at each new request for a logical volume, tell LVM to allocate the volume in order based on the amount of free space. This is good for simplicity and for keeping the usage equally spread over the available physical disks, however it introduces a problem that an instance could end up with its (currently) two drives on two physical disks, or (worse) that the data and metadata for a DRBD device end up on different drives. This is bad because it causes unneeded ``replace-disks`` operations in case of a physical failure. The solution is to batch allocations for an instance and make the LVM handling code try to allocate as close as possible all the storage of one instance. We will still allow the logical volumes to spill over to additional disks as needed. Note that this clustered allocation can only be attempted at initial instance creation, or at change secondary node time. At add disk time, or at replacing individual disks, it's not easy enough to compute the current disk map so we'll not attempt the clustering. DRBD8 peer authentication at handshake ++++++++++++++++++++++++++++++++++++++ DRBD8 has a new feature that allow authentication of the peer at connect time. We can use this to prevent connecting to the wrong peer more that securing the connection. Even though we never had issues with wrong connections, it would be good to implement this. LVM self-repair (optional) ++++++++++++++++++++++++++ The complete failure of a physical disk is very tedious to troubleshoot, mainly because of the many failure modes and the many steps needed. We can safely automate some of the steps, more specifically the ``vgreduce --removemissing`` using the following method: #. check if all nodes have consistent volume groups #. if yes, and previous status was yes, do nothing #. if yes, and previous status was no, save status and restart #. if no, and previous status was no, do nothing #. if no, and previous status was yes: #. if more than one node is inconsistent, do nothing #. if only one node is inconsistent: #. run ``vgreduce --removemissing`` #. log this occurrence in the Ganeti log in a form that can be used for monitoring #. [FUTURE] run ``replace-disks`` for all instances affected Failover to any node ++++++++++++++++++++ With a modified disk activation sequence, we can implement the *failover to any* functionality, removing many of the layout restrictions of a cluster: - the need to reserve memory on the current secondary: this gets reduced to a must to reserve memory anywhere on the cluster - the need to first failover and then replace secondary for an instance: with failover-to-any, we can directly failover to another node, which also does the replace disks at the same step In the following, we denote the current primary by P1, the current secondary by S1, and the new primary and secondaries by P2 and S2. P2 is fixed to the node the user chooses, but the choice of S2 can be made between P1 and S1. This choice can be constrained, depending on which of P1 and S1 has failed. - if P1 has failed, then S1 must become S2, and live migration is not possible - if S1 has failed, then P1 must become S2, and live migration could be possible (in theory, but this is not a design goal for 2.0) The algorithm for performing the failover is straightforward: - verify that S2 (the node the user has chosen to keep as secondary) has valid data (is consistent) - tear down the current DRBD association and setup a DRBD pairing between P2 (P2 is indicated by the user) and S2; since P2 has no data, it will start re-syncing from S2 - as soon as P2 is in state SyncTarget (i.e. after the resync has started but before it has finished), we can promote it to primary role (r/w) and start the instance on P2 - as soon as the P2?S2 sync has finished, we can remove the old data on the old node that has not been chosen for S2 Caveats: during the P2?S2 sync, a (non-transient) network error will cause I/O errors on the instance, so (if a longer instance downtime is acceptable) we can postpone the restart of the instance until the resync is done. However, disk I/O errors on S2 will cause data loss, since we don't have a good copy of the data anymore, so in this case waiting for the sync to complete is not an option. As such, it is recommended that this feature is used only in conjunction with proper disk monitoring. Live migration note: While failover-to-any is possible for all choices of S2, migration-to-any is possible only if we keep P1 as S2. Caveats +++++++ The dynamic device model, while more complex, has an advantage: it will not reuse by mistake the DRBD device of another instance, since it always looks for either our own or a free one. The static one, in contrast, will assume that given a minor number N, it's ours and we can take over. This needs careful implementation such that if the minor is in use, either we are able to cleanly shut it down, or we abort the startup. Otherwise, it could be that we start syncing between two instance's disks, causing data loss. Variable number of disk/NICs per instance ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Variable number of disks ++++++++++++++++++++++++ In order to support high-security scenarios (for example read-only sda and read-write sdb), we need to make a fully flexibly disk definition. This has less impact that it might look at first sight: only the instance creation has hard coded number of disks, not the disk handling code. The block device handling and most of the instance handling code is already working with "the instance's disks" as opposed to "the two disks of the instance", but some pieces are not (e.g. import/export) and the code needs a review to ensure safety. The objective is to be able to specify the number of disks at instance creation, and to be able to toggle from read-only to read-write a disk afterward. Variable number of NICs +++++++++++++++++++++++ Similar to the disk change, we need to allow multiple network interfaces per instance. This will affect the internal code (some function will have to stop assuming that ``instance.nics`` is a list of length one), the OS API which currently can export/import only one instance, and the command line interface. Interface changes ----------------- There are two areas of interface changes: API-level changes (the OS interface and the RAPI interface) and the command line interface changes. OS interface ~~~~~~~~~~~~ The current Ganeti OS interface, version 5, is tailored for Ganeti 1.2. The interface is composed by a series of scripts which get called with certain parameters to perform OS-dependent operations on the cluster. The current scripts are: create called when a new instance is added to the cluster export called to export an instance disk to a stream import called to import from a stream to a new instance rename called to perform the os-specific operations necessary for renaming an instance Currently these scripts suffer from the limitations of Ganeti 1.2: for example they accept exactly one block and one swap devices to operate on, rather than any amount of generic block devices, they blindly assume that an instance will have just one network interface to operate, they can not be configured to optimise the instance for a particular hypervisor. Since in Ganeti 2.0 we want to support multiple hypervisors, and a non-fixed number of network and disks the OS interface need to change to transmit the appropriate amount of information about an instance to its managing operating system, when operating on it. Moreover since some old assumptions usually used in OS scripts are no longer valid we need to re-establish a common knowledge on what can be assumed and what cannot be regarding Ganeti environment. When designing the new OS API our priorities are: - ease of use - future extensibility - ease of porting from the old API - modularity As such we want to limit the number of scripts that must be written to support an OS, and make it easy to share code between them by uniforming their input. We also will leave the current script structure unchanged, as far as we can, and make a few of the scripts (import, export and rename) optional. Most information will be passed to the script through environment variables, for ease of access and at the same time ease of using only the information a script needs. The Scripts +++++++++++ As in Ganeti 1.2, every OS which wants to be installed in Ganeti needs to support the following functionality, through scripts: create: used to create a new instance running that OS. This script should prepare the block devices, and install them so that the new OS can boot under the specified hypervisor. export (optional): used to export an installed instance using the given OS to a format which can be used to import it back into a new instance. import (optional): used to import an exported instance into a new one. This script is similar to create, but the new instance should have the content of the export, rather than contain a pristine installation. rename (optional): used to perform the internal OS-specific operations needed to rename an instance. If any optional script is not implemented Ganeti will refuse to perform the given operation on instances using the non-implementing OS. Of course the create script is mandatory, and it doesn't make sense to support the either the export or the import operation but not both. Incompatibilities with 1.2 __________________________ We expect the following incompatibilities between the OS scripts for 1.2 and the ones for 2.0: - Input parameters: in 1.2 those were passed on the command line, in 2.0 we'll use environment variables, as there will be a lot more information and not all OSes may care about all of it. - Number of calls: export scripts will be called once for each device the instance has, and import scripts once for every exported disk. Imported instances will be forced to have a number of disks greater or equal to the one of the export. - Some scripts are not compulsory: if such a script is missing the relevant operations will be forbidden for instances of that OS. This makes it easier to distinguish between unsupported operations and no-op ones (if any). Input _____ Rather than using command line flags, as they do now, scripts will accept inputs from environment variables. We expect the following input values: OS_API_VERSION The version of the OS API that the following parameters comply with; this is used so that in the future we could have OSes supporting multiple versions and thus Ganeti send the proper version in this parameter INSTANCE_NAME Name of the instance acted on HYPERVISOR The hypervisor the instance should run on (e.g. 'xen-pvm', 'xen-hvm', 'kvm') DISK_COUNT The number of disks this instance will have NIC_COUNT The number of NICs this instance will have DISK__PATH Path to the Nth disk. DISK__ACCESS W if read/write, R if read only. OS scripts are not supposed to touch read-only disks, but will be passed them to know. DISK__FRONTEND_TYPE Type of the disk as seen by the instance. Can be 'scsi', 'ide', 'virtio' DISK__BACKEND_TYPE Type of the disk as seen from the node. Can be 'block', 'file:loop' or 'file:blktap' NIC__MAC Mac address for the Nth network interface NIC__IP Ip address for the Nth network interface, if available NIC__BRIDGE Node bridge the Nth network interface will be connected to NIC__FRONTEND_TYPE Type of the Nth NIC as seen by the instance. For example 'virtio', 'rtl8139', etc. DEBUG_LEVEL Whether more out should be produced, for debugging purposes. Currently the only valid values are 0 and 1. These are only the basic variables we are thinking of now, but more may come during the implementation and they will be documented in the :manpage:`ganeti-os-interface(7)` man page. All these variables will be available to all scripts. Some scripts will need a few more information to work. These will have per-script variables, such as for example: OLD_INSTANCE_NAME rename: the name the instance should be renamed from. EXPORT_DEVICE export: device to be exported, a snapshot of the actual device. The data must be exported to stdout. EXPORT_INDEX export: sequential number of the instance device targeted. IMPORT_DEVICE import: device to send the data to, part of the new instance. The data must be imported from stdin. IMPORT_INDEX import: sequential number of the instance device targeted. (Rationale for INSTANCE_NAME as an environment variable: the instance name is always needed and we could pass it on the command line. On the other hand, though, this would force scripts to both access the environment and parse the command line, so we'll move it for uniformity.) Output/Behaviour ________________ As discussed scripts should only send user-targeted information to stderr. The create and import scripts are supposed to format/initialise the given block devices and install the correct instance data. The export script is supposed to export instance data to stdout in a format understandable by the the import script. The data will be compressed by Ganeti, so no compression should be done. The rename script should only modify the instance's knowledge of what its name is. Other declarative style features ++++++++++++++++++++++++++++++++ Similar to Ganeti 1.2, OS specifications will need to provide a 'ganeti_api_version' containing list of numbers matching the version(s) of the API they implement. Ganeti itself will always be compatible with one version of the API and may maintain backwards compatibility if it's feasible to do so. The numbers are one-per-line, so an OS supporting both version 5 and version 20 will have a file containing two lines. This is different from Ganeti 1.2, which only supported one version number. In addition to that an OS will be able to declare that it does support only a subset of the Ganeti hypervisors, by declaring them in the 'hypervisors' file. Caveats/Notes +++++++++++++ We might want to have a "default" import/export behaviour that just dumps all disks and restores them. This can save work as most systems will just do this, while allowing flexibility for different systems. Environment variables are limited in size, but we expect that there will be enough space to store the information we need. If we discover that this is not the case we may want to go to a more complex API such as storing those information on the filesystem and providing the OS script with the path to a file where they are encoded in some format. Remote API changes ~~~~~~~~~~~~~~~~~~ The first Ganeti remote API (RAPI) was designed and deployed with the Ganeti 1.2.5 release. That version provide read-only access to the cluster state. Fully functional read-write API demands significant internal changes which will be implemented in version 2.0. We decided to go with implementing the Ganeti RAPI in a RESTful way, which is aligned with key features we looking. It is simple, stateless, scalable and extensible paradigm of API implementation. As transport it uses HTTP over SSL, and we are implementing it with JSON encoding, but in a way it possible to extend and provide any other one. Design ++++++ The Ganeti RAPI is implemented as independent daemon, running on the same node with the same permission level as Ganeti master daemon. Communication is done through the LUXI library to the master daemon. In order to keep communication asynchronous, RAPI processes two types of client requests: - queries: server is able to answer immediately - job submission: some time is required for a useful response In the query case requested data is sent back to client in the HTTP response body. Typical examples of queries would be: list of nodes, instances, cluster info, etc. In the case of job submission, the client receive a job ID, the identifier which allows one to query the job progress in the job queue (see `Job Queue`_). Internally, each exported object has a version identifier, which is used as a state identifier in the HTTP header E-Tag field for requests/responses to avoid race conditions. Resource representation +++++++++++++++++++++++ The key difference of using REST instead of others API is that REST requires separation of services via resources with unique URIs. Each of them should have limited amount of state and support standard HTTP methods: GET, POST, DELETE, PUT. For example in Ganeti's case we can have a set of URI: - ``/{clustername}/instances`` - ``/{clustername}/instances/{instancename}`` - ``/{clustername}/instances/{instancename}/tag`` - ``/{clustername}/tag`` A GET request to ``/{clustername}/instances`` will return the list of instances, a POST to ``/{clustername}/instances`` should create a new instance, a DELETE ``/{clustername}/instances/{instancename}`` should delete the instance, a GET ``/{clustername}/tag`` should return get cluster tags. Each resource URI will have a version prefix. The resource IDs are to be determined. Internal encoding might be JSON, XML, or any other. The JSON encoding fits nicely in Ganeti RAPI needs. The client can request a specific representation via the Accept field in the HTTP header. REST uses HTTP as its transport and application protocol for resource access. The set of possible responses is a subset of standard HTTP responses. The statelessness model provides additional reliability and transparency to operations (e.g. only one request needs to be analyzed to understand the in-progress operation, not a sequence of multiple requests/responses). Security ++++++++ With the write functionality security becomes a much bigger an issue. The Ganeti RAPI uses basic HTTP authentication on top of an SSL-secured connection to grant access to an exported resource. The password is stored locally in an Apache-style ``.htpasswd`` file. Only one level of privileges is supported. Caveats +++++++ The model detailed above for job submission requires the client to poll periodically for updates to the job; an alternative would be to allow the client to request a callback, or a 'wait for updates' call. The callback model was not considered due to the following two issues: - callbacks would require a new model of allowed callback URLs, together with a method of managing these - callbacks only work when the client and the master are in the same security domain, and they fail in the other cases (e.g. when there is a firewall between the client and the RAPI daemon that only allows client-to-RAPI calls, which is usual in DMZ cases) The 'wait for updates' method is not suited to the HTTP protocol, where requests are supposed to be short-lived. Command line changes ~~~~~~~~~~~~~~~~~~~~ Ganeti 2.0 introduces several new features as well as new ways to handle instance resources like disks or network interfaces. This requires some noticeable changes in the way command line arguments are handled. - extend and modify command line syntax to support new features - ensure consistent patterns in command line arguments to reduce cognitive load The design changes that require these changes are, in no particular order: - flexible instance disk handling: support a variable number of disks with varying properties per instance, - flexible instance network interface handling: support a variable number of network interfaces with varying properties per instance - multiple hypervisors: multiple hypervisors can be active on the same cluster, each supporting different parameters, - support for device type CDROM (via ISO image) As such, there are several areas of Ganeti where the command line arguments will change: - Cluster configuration - cluster initialization - cluster default configuration - Instance configuration - handling of network cards for instances, - handling of disks for instances, - handling of CDROM devices and - handling of hypervisor specific options. There are several areas of Ganeti where the command line arguments will change: - Cluster configuration - cluster initialization - cluster default configuration - Instance configuration - handling of network cards for instances, - handling of disks for instances, - handling of CDROM devices and - handling of hypervisor specific options. Notes about device removal/addition +++++++++++++++++++++++++++++++++++ To avoid problems with device location changes (e.g. second network interface of the instance becoming the first or third and the like) the list of network/disk devices is treated as a stack, i.e. devices can only be added/removed at the end of the list of devices of each class (disk or network) for each instance. gnt-instance commands +++++++++++++++++++++ The commands for gnt-instance will be modified and extended to allow for the new functionality: - the add command will be extended to support the new device and hypervisor options, - the modify command continues to handle all modifications to instances, but will be extended with new arguments for handling devices. Network Device Options ++++++++++++++++++++++ The generic format of the network device option is: --net $DEVNUM[:$OPTION=$VALUE][,$OPTION=VALUE] :$DEVNUM: device number, unsigned integer, starting at 0, :$OPTION: device option, string, :$VALUE: device option value, string. Currently, the following device options will be defined (open to further changes): :mac: MAC address of the network interface, accepts either a valid MAC address or the string 'auto'. If 'auto' is specified, a new MAC address will be generated randomly. If the mac device option is not specified, the default value 'auto' is assumed. :bridge: network bridge the network interface is connected to. Accepts either a valid bridge name (the specified bridge must exist on the node(s)) as string or the string 'auto'. If 'auto' is specified, the default brigde is used. If the bridge option is not specified, the default value 'auto' is assumed. Disk Device Options +++++++++++++++++++ The generic format of the disk device option is: --disk $DEVNUM[:$OPTION=$VALUE][,$OPTION=VALUE] :$DEVNUM: device number, unsigned integer, starting at 0, :$OPTION: device option, string, :$VALUE: device option value, string. Currently, the following device options will be defined (open to further changes): :size: size of the disk device, either a positive number, specifying the disk size in mebibytes, or a number followed by a magnitude suffix (M for mebibytes, G for gibibytes). Also accepts the string 'auto' in which case the default disk size will be used. If the size option is not specified, 'auto' is assumed. This option is not valid for all disk layout types. :access: access mode of the disk device, a single letter, valid values are: - *w*: read/write access to the disk device or - *r*: read-only access to the disk device. If the access mode is not specified, the default mode of read/write access will be configured. :path: path to the image file for the disk device, string. No default exists. This option is not valid for all disk layout types. Adding devices ++++++++++++++ To add devices to an already existing instance, use the device type specific option to gnt-instance modify. Currently, there are two device type specific options supported: :--net: for network interface cards :--disk: for disk devices The syntax to the device specific options is similar to the generic device options, but instead of specifying a device number like for gnt-instance add, you specify the magic string add. The new device will always be appended at the end of the list of devices of this type for the specified instance, e.g. if the instance has disk devices 0,1 and 2, the newly added disk device will be disk device 3. Example: gnt-instance modify --net add:mac=auto test-instance Removing devices ++++++++++++++++ Removing devices from and instance is done via gnt-instance modify. The same device specific options as for adding instances are used. Instead of a device number and further device options, only the magic string remove is specified. It will always remove the last device in the list of devices of this type for the instance specified, e.g. if the instance has disk devices 0, 1, 2 and 3, the disk device number 3 will be removed. Example: gnt-instance modify --net remove test-instance Modifying devices +++++++++++++++++ Modifying devices is also done with device type specific options to the gnt-instance modify command. There are currently two device type options supported: :--net: for network interface cards :--disk: for disk devices The syntax to the device specific options is similar to the generic device options. The device number you specify identifies the device to be modified. Example:: gnt-instance modify --disk 2:access=r Hypervisor Options ++++++++++++++++++ Ganeti 2.0 will support more than one hypervisor. Different hypervisors have various options that only apply to a specific hypervisor. Those hypervisor specific options are treated specially via the ``--hypervisor`` option. The generic syntax of the hypervisor option is as follows:: --hypervisor $HYPERVISOR:$OPTION=$VALUE[,$OPTION=$VALUE] :$HYPERVISOR: symbolic name of the hypervisor to use, string, has to match the supported hypervisors. Example: xen-pvm :$OPTION: hypervisor option name, string :$VALUE: hypervisor option value, string The hypervisor option for an instance can be set on instance creation time via the ``gnt-instance add`` command. If the hypervisor for an instance is not specified upon instance creation, the default hypervisor will be used. Modifying hypervisor parameters +++++++++++++++++++++++++++++++ The hypervisor parameters of an existing instance can be modified using ``--hypervisor`` option of the ``gnt-instance modify`` command. However, the hypervisor type of an existing instance can not be changed, only the particular hypervisor specific option can be changed. Therefore, the format of the option parameters has been simplified to omit the hypervisor name and only contain the comma separated list of option-value pairs. Example:: gnt-instance modify --hypervisor cdrom=/srv/boot.iso,boot_order=cdrom:network test-instance gnt-cluster commands ++++++++++++++++++++ The command for gnt-cluster will be extended to allow setting and changing the default parameters of the cluster: - The init command will be extend to support the defaults option to set the cluster defaults upon cluster initialization. - The modify command will be added to modify the cluster parameters. It will support the --defaults option to change the cluster defaults. Cluster defaults The generic format of the cluster default setting option is: --defaults $OPTION=$VALUE[,$OPTION=$VALUE] :$OPTION: cluster default option, string, :$VALUE: cluster default option value, string. Currently, the following cluster default options are defined (open to further changes): :hypervisor: the default hypervisor to use for new instances, string. Must be a valid hypervisor known to and supported by the cluster. :disksize: the disksize for newly created instance disks, where applicable. Must be either a positive number, in which case the unit of megabyte is assumed, or a positive number followed by a supported magnitude symbol (M for megabyte or G for gigabyte). :bridge: the default network bridge to use for newly created instance network interfaces, string. Must be a valid bridge name of a bridge existing on the node(s). Hypervisor cluster defaults +++++++++++++++++++++++++++ The generic format of the hypervisor cluster wide default setting option is:: --hypervisor-defaults $HYPERVISOR:$OPTION=$VALUE[,$OPTION=$VALUE] :$HYPERVISOR: symbolic name of the hypervisor whose defaults you want to set, string :$OPTION: cluster default option, string, :$VALUE: cluster default option value, string. .. vim: set textwidth=72 : ganeti-3.1.0~rc2/doc/design-2.1.rst000064400000000000000000001377571476477700300166760ustar00rootroot00000000000000================= Ganeti 2.1 design ================= This document describes the major changes in Ganeti 2.1 compared to the 2.0 version. The 2.1 version will be a relatively small release. Its main aim is to avoid changing too much of the core code, while addressing issues and adding new features and improvements over 2.0, in a timely fashion. .. contents:: :depth: 4 Objective ========= Ganeti 2.1 will add features to help further automatization of cluster operations, further improve scalability to even bigger clusters, and make it easier to debug the Ganeti core. Detailed design =============== As for 2.0 we divide the 2.1 design into three areas: - core changes, which affect the master daemon/job queue/locking or all/most logical units - logical unit/feature changes - external interface changes (eg. command line, os api, hooks, ...) Core changes ------------ Storage units modelling ~~~~~~~~~~~~~~~~~~~~~~~ Currently, Ganeti has a good model of the block devices for instances (e.g. LVM logical volumes, files, DRBD devices, etc.) but none of the storage pools that are providing the space for these front-end devices. For example, there are hardcoded inter-node RPC calls for volume group listing, file storage creation/deletion, etc. The storage units framework will implement a generic handling for all kinds of storage backends: - LVM physical volumes - LVM volume groups - File-based storage directories - any other future storage method There will be a generic list of methods that each storage unit type will provide, like: - list of storage units of this type - check status of the storage unit Additionally, there will be specific methods for each method, for example: - enable/disable allocations on a specific PV - file storage directory creation/deletion - VG consistency fixing This will allow a much better modeling and unification of the various RPC calls related to backend storage pool in the future. Ganeti 2.1 is intended to add the basics of the framework, and not necessarilly move all the curent VG/FileBased operations to it. Note that while we model both LVM PVs and LVM VGs, the framework will **not** model any relationship between the different types. In other words, we model neither inheritances nor stacking, since this is too complex for our needs. While a ``vgreduce`` operation on a LVM VG could actually remove a PV from it, this will not be handled at the framework level, but at individual operation level. The goal is that this is a lightweight framework, for abstracting the different storage operation, and not for modelling the storage hierarchy. Locking improvements ~~~~~~~~~~~~~~~~~~~~ Current State and shortcomings ++++++++++++++++++++++++++++++ The class ``LockSet`` (see ``lib/locking.py``) is a container for one or many ``SharedLock`` instances. It provides an interface to add/remove locks and to acquire and subsequently release any number of those locks contained in it. Locks in a ``LockSet`` are always acquired in alphabetic order. Due to the way we're using locks for nodes and instances (the single cluster lock isn't affected by this issue) this can lead to long delays when acquiring locks if another operation tries to acquire multiple locks but has to wait for yet another operation. In the following demonstration we assume to have the instance locks ``inst1``, ``inst2``, ``inst3`` and ``inst4``. #. Operation A grabs lock for instance ``inst4``. #. Operation B wants to acquire all instance locks in alphabetic order, but it has to wait for ``inst4``. #. Operation C tries to lock ``inst1``, but it has to wait until Operation B (which is trying to acquire all locks) releases the lock again. #. Operation A finishes and releases lock on ``inst4``. Operation B can continue and eventually releases all locks. #. Operation C can get ``inst1`` lock and finishes. Technically there's no need for Operation C to wait for Operation A, and subsequently Operation B, to finish. Operation B can't continue until Operation A is done (it has to wait for ``inst4``), anyway. Proposed changes ++++++++++++++++ Non-blocking lock acquiring ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Acquiring locks for OpCode execution is always done in blocking mode. They won't return until the lock has successfully been acquired (or an error occurred, although we won't cover that case here). ``SharedLock`` and ``LockSet`` must be able to be acquired in a non-blocking way. They must support a timeout and abort trying to acquire the lock(s) after the specified amount of time. Retry acquiring locks ^^^^^^^^^^^^^^^^^^^^^ To prevent other operations from waiting for a long time, such as described in the demonstration before, ``LockSet`` must not keep locks for a prolonged period of time when trying to acquire two or more locks. Instead it should, with an increasing timeout for acquiring all locks, release all locks again and sleep some time if it fails to acquire all requested locks. A good timeout value needs to be determined. In any case should ``LockSet`` proceed to acquire locks in blocking mode after a few (unsuccessful) attempts to acquire all requested locks. One proposal for the timeout is to use ``2**tries`` seconds, where ``tries`` is the number of unsuccessful tries. In the demonstration before this would allow Operation C to continue after Operation B unsuccessfully tried to acquire all locks and released all acquired locks (``inst1``, ``inst2`` and ``inst3``) again. Other solutions discussed +++++++++++++++++++++++++ There was also some discussion on going one step further and extend the job queue (see ``lib/jqueue.py``) to select the next task for a worker depending on whether it can acquire the necessary locks. While this may reduce the number of necessary worker threads and/or increase throughput on large clusters with many jobs, it also brings many potential problems, such as contention and increased memory usage, with it. As this would be an extension of the changes proposed before it could be implemented at a later point in time, but we decided to stay with the simpler solution for now. Implementation details ++++++++++++++++++++++ ``SharedLock`` redesign ^^^^^^^^^^^^^^^^^^^^^^^ The current design of ``SharedLock`` is not good for supporting timeouts when acquiring a lock and there are also minor fairness issues in it. We plan to address both with a redesign. A proof of concept implementation was written and resulted in significantly simpler code. Currently ``SharedLock`` uses two separate queues for shared and exclusive acquires and waiters get to run in turns. This means if an exclusive acquire is released, the lock will allow shared waiters to run and vice versa. Although it's still fair in the end there is a slight bias towards shared waiters in the current implementation. The same implementation with two shared queues can not support timeouts without adding a lot of complexity. Our proposed redesign changes ``SharedLock`` to have only one single queue. There will be one condition (see Condition_ for a note about performance) in the queue per exclusive acquire and two for all shared acquires (see below for an explanation). The maximum queue length will always be ``2 + (number of exclusive acquires waiting)``. The number of queue entries for shared acquires can vary from 0 to 2. The two conditions for shared acquires are a bit special. They will be used in turn. When the lock is instantiated, no conditions are in the queue. As soon as the first shared acquire arrives (and there are holder(s) or waiting acquires; see Acquire_), the active condition is added to the queue. Until it becomes the topmost condition in the queue and has been notified, any shared acquire is added to this active condition. When the active condition is notified, the conditions are swapped and further shared acquires are added to the previously inactive condition (which has now become the active condition). After all waiters on the previously active (now inactive) and now notified condition received the notification, it is removed from the queue of pending acquires. This means shared acquires will skip any exclusive acquire in the queue. We believe it's better to improve parallelization on operations only asking for shared (or read-only) locks. Exclusive operations holding the same lock can not be parallelized. Acquire ******* For exclusive acquires a new condition is created and appended to the queue. Shared acquires are added to the active condition for shared acquires and if the condition is not yet on the queue, it's appended. The next step is to wait for our condition to be on the top of the queue (to guarantee fairness). If the timeout expired, we return to the caller without acquiring the lock. On every notification we check whether the lock has been deleted, in which case an error is returned to the caller. The lock can be acquired if we're on top of the queue (there is no one else ahead of us). For an exclusive acquire, there must not be other exclusive or shared holders. For a shared acquire, there must not be an exclusive holder. If these conditions are all true, the lock is acquired and we return to the caller. In any other case we wait again on the condition. If it was the last waiter on a condition, the condition is removed from the queue. Optimization: There's no need to touch the queue if there are no pending acquires and no current holders. The caller can have the lock immediately. .. digraph:: "design-2.1-lock-acquire" graph[fontsize=8, fontname="Helvetica"] node[fontsize=8, fontname="Helvetica", width="0", height="0"] edge[fontsize=8, fontname="Helvetica"] /* Actions */ abort[label="Abort\n(couldn't acquire)"] acquire[label="Acquire lock"] add_to_queue[label="Add condition to queue"] wait[label="Wait for notification"] remove_from_queue[label="Remove from queue"] /* Conditions */ alone[label="Empty queue\nand can acquire?", shape=diamond] have_timeout[label="Do I have\ntimeout?", shape=diamond] top_of_queue_and_can_acquire[ label="On top of queue and\ncan acquire lock?", shape=diamond, ] /* Lines */ alone->acquire[label="Yes"] alone->add_to_queue[label="No"] have_timeout->abort[label="Yes"] have_timeout->wait[label="No"] top_of_queue_and_can_acquire->acquire[label="Yes"] top_of_queue_and_can_acquire->have_timeout[label="No"] add_to_queue->wait wait->top_of_queue_and_can_acquire acquire->remove_from_queue Release ******* First the lock removes the caller from the internal owner list. If there are pending acquires in the queue, the first (the oldest) condition is notified. If the first condition was the active condition for shared acquires, the inactive condition will be made active. This ensures fairness with exclusive locks by forcing consecutive shared acquires to wait in the queue. .. digraph:: "design-2.1-lock-release" graph[fontsize=8, fontname="Helvetica"] node[fontsize=8, fontname="Helvetica", width="0", height="0"] edge[fontsize=8, fontname="Helvetica"] /* Actions */ remove_from_owners[label="Remove from owner list"] notify[label="Notify topmost"] swap_shared[label="Swap shared conditions"] success[label="Success"] /* Conditions */ have_pending[label="Any pending\nacquires?", shape=diamond] was_active_queue[ label="Was active condition\nfor shared acquires?", shape=diamond, ] /* Lines */ remove_from_owners->have_pending have_pending->notify[label="Yes"] have_pending->success[label="No"] notify->was_active_queue was_active_queue->swap_shared[label="Yes"] was_active_queue->success[label="No"] swap_shared->success Delete ****** The caller must either hold the lock in exclusive mode already or the lock must be acquired in exclusive mode. Trying to delete a lock while it's held in shared mode must fail. After ensuring the lock is held in exclusive mode, the lock will mark itself as deleted and continue to notify all pending acquires. They will wake up, notice the deleted lock and return an error to the caller. Condition ^^^^^^^^^ Note: This is not necessary for the locking changes above, but it may be a good optimization (pending performance tests). The existing locking code in Ganeti 2.0 uses Python's built-in ``threading.Condition`` class. Unfortunately ``Condition`` implements timeouts by sleeping 1ms to 20ms between tries to acquire the condition lock in non-blocking mode. This requires unnecessary context switches and contention on the CPython GIL (Global Interpreter Lock). By using POSIX pipes (see ``pipe(2)``) we can use the operating system's support for timeouts on file descriptors (see ``select(2)``). A custom condition class will have to be written for this. On instantiation the class creates a pipe. After each notification the previous pipe is abandoned and re-created (technically the old pipe needs to stay around until all notifications have been delivered). All waiting clients of the condition use ``select(2)`` or ``poll(2)`` to wait for notifications, optionally with a timeout. A notification will be signalled to the waiting clients by closing the pipe. If the pipe wasn't closed during the timeout, the waiting function returns to its caller nonetheless. Node daemon availability ~~~~~~~~~~~~~~~~~~~~~~~~ Current State and shortcomings ++++++++++++++++++++++++++++++ Currently, when a Ganeti node suffers serious system disk damage, the migration/failover of an instance may not correctly shutdown the virtual machine on the broken node causing instances duplication. The ``gnt-node powercycle`` command can be used to force a node reboot and thus to avoid duplicated instances. This command relies on node daemon availability, though, and thus can fail if the node daemon has some pages swapped out of ram, for example. Proposed changes ++++++++++++++++ The proposed solution forces node daemon to run exclusively in RAM. It uses python ctypes to to call ``mlockall(MCL_CURRENT | MCL_FUTURE)`` on the node daemon process and all its children. In addition another log handler has been implemented for node daemon to redirect to ``/dev/console`` messages that cannot be written on the logfile. With these changes node daemon can successfully run basic tasks such as a powercycle request even when the system disk is heavily damaged and reading/writing to disk fails constantly. New Features ------------ Automated Ganeti Cluster Merger ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Current situation +++++++++++++++++ Currently there's no easy way to merge two or more clusters together. But in order to optimize resources this is a needed missing piece. The goal of this design doc is to come up with a easy to use solution which allows you to merge two or more clusters together. Initial contact +++++++++++++++ As the design of Ganeti is based on an autonomous system, Ganeti by itself has no way to reach nodes outside of its cluster. To overcome this situation we're required to prepare the cluster before we can go ahead with the actual merge: We've to replace at least the ssh keys on the affected nodes before we can do any operation within ``gnt-`` commands. To make this a automated process we'll ask the user to provide us with the root password of every cluster we've to merge. We use the password to grab the current ``id_dsa`` key and then rely on that ssh key for any further communication to be made until the cluster is fully merged. Cluster merge +++++++++++++ After initial contact we do the cluster merge: 1. Grab the list of nodes 2. On all nodes add our own ``id_dsa.pub`` key to ``authorized_keys`` 3. Stop all instances running on the merging cluster 4. Disable ``ganeti-watcher`` as it tries to restart Ganeti daemons 5. Stop all Ganeti daemons on all merging nodes 6. Grab the ``config.data`` from the master of the merging cluster 7. Stop local ``ganeti-masterd`` 8. Merge the config: 1. Open our own cluster ``config.data`` 2. Open cluster ``config.data`` of the merging cluster 3. Grab all nodes of the merging cluster 4. Set ``master_candidate`` to false on all merging nodes 5. Add the nodes to our own cluster ``config.data`` 6. Grab all the instances on the merging cluster 7. Adjust the port if the instance has drbd layout: 1. In ``logical_id`` (index 2) 2. In ``physical_id`` (index 1 and 3) 8. Add the instances to our own cluster ``config.data`` 9. Start ``ganeti-masterd`` with ``--no-voting`` ``--yes-do-it`` 10. ``gnt-node add --readd`` on all merging nodes 11. ``gnt-cluster redist-conf`` 12. Restart ``ganeti-masterd`` normally 13. Enable ``ganeti-watcher`` again 14. Start all merging instances again Rollback ++++++++ Until we actually (re)add any nodes we can abort and rollback the merge at any point. After merging the config, though, we've to get the backup copy of ``config.data`` (from another master candidate node). And for security reasons it's a good idea to undo ``id_dsa.pub`` distribution by going on every affected node and remove the ``id_dsa.pub`` key again. Also we've to keep in mind, that we've to start the Ganeti daemons and starting up the instances again. Verification ++++++++++++ Last but not least we should verify that the merge was successful. Therefore we run ``gnt-cluster verify``, which ensures that the cluster overall is in a healthy state. Additional it's also possible to compare the list of instances/nodes with a list made prior to the upgrade to make sure we didn't lose any data/instance/node. Appendix ++++++++ cluster-merge.py ^^^^^^^^^^^^^^^^ Used to merge the cluster config. This is a POC and might differ from actual production code. :: #!/usr/bin/python3 import sys from ganeti import config from ganeti import constants c_mine = config.ConfigWriter(offline=True) c_other = config.ConfigWriter(sys.argv[1]) fake_id = 0 for node in c_other.GetNodeList(): node_info = c_other.GetNodeInfo(node) node_info.master_candidate = False c_mine.AddNode(node_info, str(fake_id)) fake_id += 1 for instance in c_other.GetInstanceList(): instance_info = c_other.GetInstanceInfo(instance) for dsk in instance_info.disks: if dsk.dev_type in constants.LDS_DRBD: port = c_mine.AllocatePort() logical_id = list(dsk.logical_id) logical_id[2] = port dsk.logical_id = tuple(logical_id) physical_id = list(dsk.physical_id) physical_id[1] = physical_id[3] = port dsk.physical_id = tuple(physical_id) c_mine.AddInstance(instance_info, str(fake_id)) fake_id += 1 Feature changes --------------- Ganeti Confd ~~~~~~~~~~~~ Current State and shortcomings ++++++++++++++++++++++++++++++ In Ganeti 2.0 all nodes are equal, but some are more equal than others. In particular they are divided between "master", "master candidates" and "normal". (Moreover they can be offline or drained, but this is not important for the current discussion). In general the whole configuration is only replicated to master candidates, and some partial information is spread to all nodes via ssconf. This change was done so that the most frequent Ganeti operations didn't need to contact all nodes, and so clusters could become bigger. If we want more information to be available on all nodes, we need to add more ssconf values, which is counter-balancing the change, or to talk with the master node, which is not designed to happen now, and requires its availability. Information such as the instance->primary_node mapping will be needed on all nodes, and we also want to make sure services external to the cluster can query this information as well. This information must be available at all times, so we can't query it through RAPI, which would be a single point of failure, as it's only available on the master. Proposed changes ++++++++++++++++ In order to allow fast and highly available access read-only to some configuration values, we'll create a new ganeti-confd daemon, which will run on master candidates. This daemon will talk via UDP, and authenticate messages using HMAC with a cluster-wide shared key. This key will be generated at cluster init time, and stored on the clusters alongside the ganeti SSL keys, and readable only by root. An interested client can query a value by making a request to a subset of the cluster master candidates. It will then wait to get a few responses, and use the one with the highest configuration serial number. Since the configuration serial number is increased each time the ganeti config is updated, and the serial number is included in all answers, this can be used to make sure to use the most recent answer, in case some master candidates are stale or in the middle of a configuration update. In order to prevent replay attacks queries will contain the current unix timestamp according to the client, and the server will verify that its timestamp is in the same 5 minutes range (this requires synchronized clocks, which is a good idea anyway). Queries will also contain a "salt" which they expect the answers to be sent with, and clients are supposed to accept only answers which contain salt generated by them. The configuration daemon will be able to answer simple queries such as: - master candidates list - master node - offline nodes - instance list - instance primary nodes Wire protocol ^^^^^^^^^^^^^ A confd query will look like this, on the wire:: plj0{ "msg": "{\"type\": 1, \"rsalt\": \"9aa6ce92-8336-11de-af38-001d093e835f\", \"protocol\": 1, \"query\": \"node1.example.com\"}\n", "salt": "1249637704", "hmac": "4a4139b2c3c5921f7e439469a0a45ad200aead0f" } ``plj0`` is a fourcc that details the message content. It stands for plain json 0, and can be changed as we move on to different type of protocols (for example protocol buffers, or encrypted json). What follows is a json encoded string, with the following fields: - ``msg`` contains a JSON-encoded query, its fields are: - ``protocol``, integer, is the confd protocol version (initially just ``constants.CONFD_PROTOCOL_VERSION``, with a value of 1) - ``type``, integer, is the query type. For example "node role by name" or "node primary ip by instance ip". Constants will be provided for the actual available query types - ``query`` is a multi-type field (depending on the ``type`` field): - it can be missing, when the request is fully determined by the ``type`` field - it can contain a string which denotes the search key: for example an IP, or a node name - it can contain a dictionary, in which case the actual details vary further per request type - ``rsalt``, string, is the required response salt; the client must use it to recognize which answer it's getting. - ``salt`` must be the current unix timestamp, according to the client; servers should refuse messages which have a wrong timing, according to their configuration and clock - ``hmac`` is an hmac signature of salt+msg, with the cluster hmac key If an answer comes back (which is optional, since confd works over UDP) it will be in this format:: plj0{ "msg": "{\"status\": 0, \"answer\": 0, \"serial\": 42, \"protocol\": 1}\n", "salt": "9aa6ce92-8336-11de-af38-001d093e835f", "hmac": "aaeccc0dff9328fdf7967cb600b6a80a6a9332af" } Where: - ``plj0`` the message type magic fourcc, as discussed above - ``msg`` contains a JSON-encoded answer, its fields are: - ``protocol``, integer, is the confd protocol version (initially just constants.CONFD_PROTOCOL_VERSION, with a value of 1) - ``status``, integer, is the error code; initially just ``0`` for 'ok' or ``1`` for 'error' (in which case answer contains an error detail, rather than an answer), but in the future it may be expanded to have more meanings (e.g. ``2`` if the answer is compressed) - ``answer``, is the actual answer; its type and meaning is query specific: for example for "node primary ip by instance ip" queries it will be a string containing an IP address, for "node role by name" queries it will be an integer which encodes the role (master, candidate, drained, offline) according to constants - ``salt`` is the requested salt from the query; a client can use it to recognize what query the answer is answering. - ``hmac`` is an hmac signature of salt+msg, with the cluster hmac key Redistribute Config ~~~~~~~~~~~~~~~~~~~ Current State and shortcomings ++++++++++++++++++++++++++++++ Currently LUClusterRedistConf triggers a copy of the updated configuration file to all master candidates and of the ssconf files to all nodes. There are other files which are maintained manually but which are important to keep in sync. These are: - rapi SSL key certificate file (rapi.pem) (on master candidates) - rapi user/password file rapi_users (on master candidates) Furthermore there are some files which are hypervisor specific but we may want to keep in sync: - the xen-hvm hypervisor uses one shared file for all vnc passwords, and copies the file once, during node add. This design is subject to revision to be able to have different passwords for different groups of instances via the use of hypervisor parameters, and to allow xen-hvm and kvm to use an equal system to provide password-protected vnc sessions. In general, though, it would be useful if the vnc password files were copied as well, to avoid unwanted vnc password changes on instance failover/migrate. Optionally the admin may want to also ship files such as the global xend.conf file, and the network scripts to all nodes. Proposed changes ++++++++++++++++ RedistributeConfig will be changed to copy also the rapi files, and to call every enabled hypervisor asking for a list of additional files to copy. Users will have the possibility to populate a file containing a list of files to be distributed; this file will be propagated as well. Such solution is really simple to implement and it's easily usable by scripts. This code will be also shared (via tasklets or by other means, if tasklets are not ready for 2.1) with the AddNode and SetNodeParams LUs (so that the relevant files will be automatically shipped to new master candidates as they are set). VNC Console Password ~~~~~~~~~~~~~~~~~~~~ Current State and shortcomings ++++++++++++++++++++++++++++++ Currently just the xen-hvm hypervisor supports setting a password to connect the the instances' VNC console, and has one common password stored in a file. This doesn't allow different passwords for different instances/groups of instances, and makes it necessary to remember to copy the file around the cluster when the password changes. Proposed changes ++++++++++++++++ We'll change the VNC password file to a vnc_password_file hypervisor parameter. This way it can have a cluster default, but also a different value for each instance. The VNC enabled hypervisors (xen and kvm) will publish all the password files in use through the cluster so that a redistribute-config will ship them to all nodes (see the Redistribute Config proposed changes above). The current VNC_PASSWORD_FILE constant will be removed, but its value will be used as the default HV_VNC_PASSWORD_FILE value, thus retaining backwards compatibility with 2.0. The code to export the list of VNC password files from the hypervisors to RedistributeConfig will be shared between the KVM and xen-hvm hypervisors. Disk/Net parameters ~~~~~~~~~~~~~~~~~~~ Current State and shortcomings ++++++++++++++++++++++++++++++ Currently disks and network interfaces have a few tweakable options and all the rest is left to a default we chose. We're finding that we need more and more to tweak some of these parameters, for example to disable barriers for DRBD devices, or allow striping for the LVM volumes. Moreover for many of these parameters it will be nice to have cluster-wide defaults, and then be able to change them per disk/interface. Proposed changes ++++++++++++++++ We will add new cluster level diskparams and netparams, which will contain all the tweakable parameters. All values which have a sensible cluster-wide default will go into this new structure while parameters which have unique values will not. Example of network parameters: - mode: bridge/route - link: for mode "bridge" the bridge to connect to, for mode route it can contain the routing table, or the destination interface Example of disk parameters: - stripe: lvm stripes - stripe_size: lvm stripe size - meta_flushes: drbd, enable/disable metadata "barriers" - data_flushes: drbd, enable/disable data "barriers" Some parameters are bound to be disk-type specific (drbd, vs lvm, vs files) or hypervisor specific (nic models for example), but for now they will all live in the same structure. Each component is supposed to validate only the parameters it knows about, and ganeti itself will make sure that no "globally unknown" parameters are added, and that no parameters have overridden meanings for different components. The parameters will be kept, as for the BEPARAMS into a "default" category, which will allow us to expand on by creating instance "classes" in the future. Instance classes is not a feature we plan implementing in 2.1, though. Global hypervisor parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Current State and shortcomings ++++++++++++++++++++++++++++++ Currently all hypervisor parameters are modifiable both globally (cluster level) and at instance level. However, there is no other framework to held hypervisor-specific parameters, so if we want to add a new class of hypervisor parameters that only makes sense on a global level, we have to change the hvparams framework. Proposed changes ++++++++++++++++ We add a new (global, not per-hypervisor) list of parameters which are not changeable on a per-instance level. The create, modify and query instance operations are changed to not allow/show these parameters. Furthermore, to allow transition of parameters to the global list, and to allow cleanup of inadverdently-customised parameters, the ``UpgradeConfig()`` method of instances will drop any such parameters from their list of hvparams, such that a restart of the master daemon is all that is needed for cleaning these up. Also, the framework is simple enough that if we need to replicate it at beparams level we can do so easily. Non bridged instances support ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Current State and shortcomings ++++++++++++++++++++++++++++++ Currently each instance NIC must be connected to a bridge, and if the bridge is not specified the default cluster one is used. This makes it impossible to use the vif-route xen network scripts, or other alternative mechanisms that don't need a bridge to work. Proposed changes ++++++++++++++++ The new "mode" network parameter will distinguish between bridged interfaces and routed ones. When mode is "bridge" the "link" parameter will contain the bridge the instance should be connected to, effectively making things as today. The value has been migrated from a nic field to a parameter to allow for an easier manipulation of the cluster default. When mode is "route" the ip field of the interface will become mandatory, to allow for a route to be set. In the future we may want also to accept multiple IPs or IP/mask values for this purpose. We will evaluate possible meanings of the link parameter to signify a routing table to be used, which would allow for insulation between instance groups (as today happens for different bridges). For now we won't add a parameter to specify which network script gets called for which instance, so in a mixed cluster the network script must be able to handle both cases. The default kvm vif script will be changed to do so. (Xen doesn't have a ganeti provided script, so nothing will be done for that hypervisor) Introducing persistent UUIDs ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Current state and shortcomings ++++++++++++++++++++++++++++++ Some objects in the Ganeti configurations are tracked by their name while also supporting renames. This creates an extra difficulty, because neither Ganeti nor external management tools can then track the actual entity, and due to the name change it behaves like a new one. Proposed changes part 1 +++++++++++++++++++++++ We will change Ganeti to use UUIDs for entity tracking, but in a staggered way. In 2.1, we will simply add an “uuid” attribute to each of the instances, nodes and cluster itself. This will be reported on instance creation for nodes, and on node adds for the nodes. It will be of course avaiblable for querying via the OpNodeQuery/Instance and cluster information, and via RAPI as well. Note that Ganeti will not provide any way to change this attribute. Upgrading from Ganeti 2.0 will automatically add an ‘uuid’ attribute to all entities missing it. Proposed changes part 2 +++++++++++++++++++++++ In the next release (e.g. 2.2), the tracking of objects will change from the name to the UUID internally, and externally Ganeti will accept both forms of identification; e.g. an RAPI call would be made either against ``/2/instances/foo.bar`` or against ``/2/instances/bb3b2e42â€Ļ``. Since an FQDN must have at least a dot, and dots are not valid characters in UUIDs, we will not have namespace issues. Another change here is that node identification (during cluster operations/queries like master startup, “am I the master?” and similar) could be done via UUIDs which is more stable than the current hostname-based scheme. Internal tracking refers to the way the configuration is stored; a DRBD disk of an instance refers to the node name (so that IPs can be changed easily), but this is still a problem for name changes; thus these will be changed to point to the node UUID to ease renames. The advantages of this change (after the second round of changes), is that node rename becomes trivial, whereas today node rename would require a complete lock of all instances. Automated disk repairs infrastructure ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Replacing defective disks in an automated fashion is quite difficult with the current version of Ganeti. These changes will introduce additional functionality and interfaces to simplify automating disk replacements on a Ganeti node. Fix node volume group +++++++++++++++++++++ This is the most difficult addition, as it can lead to dataloss if it's not properly safeguarded. The operation must be done only when all the other nodes that have instances in common with the target node are fine, i.e. this is the only node with problems, and also we have to double-check that all instances on this node have at least a good copy of the data. This might mean that we have to enhance the GetMirrorStatus calls, and introduce and a smarter version that can tell us more about the status of an instance. Stop allocation on a given PV +++++++++++++++++++++++++++++ This is somewhat simple. First we need a "list PVs" opcode (and its associated logical unit) and then a set PV status opcode/LU. These in combination should allow both checking and changing the disk/PV status. Instance disk status ++++++++++++++++++++ This new opcode or opcode change must list the instance-disk-index and node combinations of the instance together with their status. This will allow determining what part of the instance is broken (if any). Repair instance +++++++++++++++ This new opcode/LU/RAPI call will run ``replace-disks -p`` as needed, in order to fix the instance status. It only affects primary instances; secondaries can just be moved away. Migrate node ++++++++++++ This new opcode/LU/RAPI call will take over the current ``gnt-node migrate`` code and run migrate for all instances on the node. Evacuate node ++++++++++++++ This new opcode/LU/RAPI call will take over the current ``gnt-node evacuate`` code and run replace-secondary with an iallocator script for all instances on the node. User-id pool ~~~~~~~~~~~~ In order to allow running different processes under unique user-ids on a node, we introduce the user-id pool concept. The user-id pool is a cluster-wide configuration parameter. It is a list of user-ids and/or user-id ranges that are reserved for running Ganeti processes (including KVM instances). The code guarantees that on a given node a given user-id is only handed out if there is no other process running with that user-id. Please note, that this can only be guaranteed if all processes in the system - that run under a user-id belonging to the pool - are started by reserving a user-id first. That can be accomplished either by using the RequestUnusedUid() function to get an unused user-id or by implementing the same locking mechanism. Implementation ++++++++++++++ The functions that are specific to the user-id pool feature are located in a separate module: ``lib/uidpool.py``. Storage ^^^^^^^ The user-id pool is a single cluster parameter. It is stored in the *Cluster* object under the ``uid_pool`` name as a list of integer tuples. These tuples represent the boundaries of user-id ranges. For single user-ids, the boundaries are equal. The internal user-id pool representation is converted into a string: a newline separated list of user-ids or user-id ranges. This string representation is distributed to all the nodes via the *ssconf* mechanism. This means that the user-id pool can be accessed in a read-only way on any node without consulting the master node or master candidate nodes. Initial value ^^^^^^^^^^^^^ The value of the user-id pool cluster parameter can be initialized at cluster initialization time using the ``gnt-cluster init --uid-pool ...`` command. As there is no sensible default value for the user-id pool parameter, it is initialized to an empty list if no ``--uid-pool`` option is supplied at cluster init time. If the user-id pool is empty, the user-id pool feature is considered to be disabled. Manipulation ^^^^^^^^^^^^ The user-id pool cluster parameter can be modified from the command-line with the following commands: - ``gnt-cluster modify --uid-pool `` - ``gnt-cluster modify --add-uids `` - ``gnt-cluster modify --remove-uids `` The ``--uid-pool`` option overwrites the current setting with the supplied ````, while ``--add-uids``/``--remove-uids`` adds/removes the listed uids or uid-ranges from the pool. The ```` should be a comma-separated list of user-ids or user-id ranges. A range should be defined by a lower and a higher boundary. The boundaries should be separated with a dash. The boundaries are inclusive. The ```` is parsed into the internal representation, sanity-checked and stored in the ``uid_pool`` attribute of the *Cluster* object. It is also immediately converted into a string (formatted in the input format) and distributed to all nodes via the *ssconf* mechanism. Inspection ^^^^^^^^^^ The current value of the user-id pool cluster parameter is printed by the ``gnt-cluster info`` command. The output format is accepted by the ``gnt-cluster modify --uid-pool`` command. Locking ^^^^^^^ The ``uidpool.py`` module provides a function (``RequestUnusedUid``) for requesting an unused user-id from the pool. This will try to find a random user-id that is not currently in use. The algorithm is the following: 1) Randomize the list of user-ids in the user-id pool 2) Iterate over this randomized UID list 3) Create a lock file (it doesn't matter if it already exists) 4) Acquire an exclusive POSIX lock on the file, to provide mutual exclusion for the following non-atomic operations 5) Check if there is a process in the system with the given UID 6) If there isn't, return the UID, otherwise unlock the file and continue the iteration over the user-ids The user can than start a new process with this user-id. Once a process is successfully started, the exclusive POSIX lock can be released, but the lock file will remain in the filesystem. The presence of such a lock file means that the given user-id is most probably in use. The lack of a uid lock file does not guarantee that there are no processes with that user-id. After acquiring the exclusive POSIX lock, ``RequestUnusedUid`` always performs a check to see if there is a process running with the given uid. A user-id can be returned to the pool, by calling the ``ReleaseUid`` function. This will remove the corresponding lock file. Note, that it doesn't check if there is any process still running with that user-id. The removal of the lock file only means that there are most probably no processes with the given user-id. This helps in speeding up the process of finding a user-id that is guaranteed to be unused. There is a convenience function, called ``ExecWithUnusedUid`` that wraps the execution of a function (or any callable) that requires a unique user-id. ``ExecWithUnusedUid`` takes care of requesting an unused user-id and unlocking the lock file. It also automatically returns the user-id to the pool if the callable raises an exception. Code examples +++++++++++++ Requesting a user-id from the pool: :: from ganeti import ssconf from ganeti import uidpool # Get list of all user-ids in the uid-pool from ssconf ss = ssconf.SimpleStore() uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n") all_uids = set(uidpool.ExpandUidPool(uid_pool)) uid = uidpool.RequestUnusedUid(all_uids) try: # Once the process is started, we can release the file lock uid.Unlock() except ... as err: # Return the UID to the pool uidpool.ReleaseUid(uid) Releasing a user-id: :: from ganeti import uidpool uid = uidpool.ReleaseUid(uid) External interface changes -------------------------- OS API ~~~~~~ The OS API of Ganeti 2.0 has been built with extensibility in mind. Since we pass everything as environment variables it's a lot easier to send new information to the OSes without breaking retrocompatibility. This section of the design outlines the proposed extensions to the API and their implementation. API Version Compatibility Handling ++++++++++++++++++++++++++++++++++ In 2.1 there will be a new OS API version (eg. 15), which should be mostly compatible with api 10, except for some new added variables. Since it's easy not to pass some variables we'll be able to handle Ganeti 2.0 OSes by just filtering out the newly added piece of information. We will still encourage OSes to declare support for the new API after checking that the new variables don't provide any conflict for them, and we will drop api 10 support after ganeti 2.1 has released. New Environment variables +++++++++++++++++++++++++ Some variables have never been added to the OS api but would definitely be useful for the OSes. We plan to add an INSTANCE_HYPERVISOR variable to allow the OS to make changes relevant to the virtualization the instance is going to use. Since this field is immutable for each instance, the os can tight the install without caring of making sure the instance can run under any virtualization technology. We also want the OS to know the particular hypervisor parameters, to be able to customize the install even more. Since the parameters can change, though, we will pass them only as an "FYI": if an OS ties some instance functionality to the value of a particular hypervisor parameter manual changes or a reinstall may be needed to adapt the instance to the new environment. This is not a regression as of today, because even if the OSes are left blind about this information, sometimes they still need to make compromises and cannot satisfy all possible parameter values. OS Variants +++++++++++ Currently we are assisting to some degree of "os proliferation" just to change a simple installation behavior. This means that the same OS gets installed on the cluster multiple times, with different names, to customize just one installation behavior. Usually such OSes try to share as much as possible through symlinks, but this still causes complications on the user side, especially when multiple parameters must be cross-matched. For example today if you want to install debian etch, lenny or squeeze you probably need to install the debootstrap OS multiple times, changing its configuration file, and calling it debootstrap-etch, debootstrap-lenny or debootstrap-squeeze. Furthermore if you have for example a "server" and a "development" environment which installs different packages/configuration files and must be available for all installs you'll probably end up with deboostrap-etch-server, debootstrap-etch-dev, debootrap-lenny-server, debootstrap-lenny-dev, etc. Crossing more than two parameters quickly becomes not manageable. In order to avoid this we plan to make OSes more customizable, by allowing each OS to declare a list of variants which can be used to customize it. The variants list is mandatory and must be written, one variant per line, in the new "variants.list" file inside the main os dir. At least one supported variant must be supported. When choosing the OS exactly one variant will have to be specified, and will be encoded in the os name as +. As for today it will be possible to change an instance's OS at creation or install time. The 2.1 OS list will be the combination of each OS, plus its supported variants. This will cause the name name proliferation to remain, but at least the internal OS code will be simplified to just parsing the passed variant, without the need for symlinks or code duplication. Also we expect the OSes to declare only "interesting" variants, but to accept some non-declared ones which a user will be able to pass in by overriding the checks ganeti does. This will be useful for allowing some variations to be used without polluting the OS list (per-OS documentation should list all supported variants). If a variant which is not internally supported is forced through, the OS scripts should abort. In the future (post 2.1) we may want to move to full fledged parameters all orthogonal to each other (for example "architecture" (i386, amd64), "suite" (lenny, squeeze, ...), etc). (As opposed to the variant, which is a single parameter, and you need a different variant for all the set of combinations you want to support). In this case we envision the variants to be moved inside of Ganeti and be associated with lists parameter->values associations, which will then be passed to the OS. IAllocator changes ~~~~~~~~~~~~~~~~~~ Current State and shortcomings ++++++++++++++++++++++++++++++ The iallocator interface allows creation of instances without manually specifying nodes, but instead by specifying plugins which will do the required computations and produce a valid node list. However, the interface is quite akward to use: - one cannot set a 'default' iallocator script - one cannot use it to easily test if allocation would succeed - some new functionality, such as rebalancing clusters and calculating capacity estimates is needed Proposed changes ++++++++++++++++ There are two area of improvements proposed: - improving the use of the current interface - extending the IAllocator API to cover more automation Default iallocator names ^^^^^^^^^^^^^^^^^^^^^^^^ The cluster will hold, for each type of iallocator, a (possibly empty) list of modules that will be used automatically. If the list is empty, the behaviour will remain the same. If the list has one entry, then ganeti will behave as if '--iallocator' was specifyed on the command line. I.e. use this allocator by default. If the user however passed nodes, those will be used in preference. If the list has multiple entries, they will be tried in order until one gives a successful answer. Dry-run allocation ^^^^^^^^^^^^^^^^^^ The create instance LU will get a new 'dry-run' option that will just simulate the placement, and return the chosen node-lists after running all the usual checks. Cluster balancing ^^^^^^^^^^^^^^^^^ Instance add/removals/moves can create a situation where load on the nodes is not spread equally. For this, a new iallocator mode will be implemented called ``balance`` in which the plugin, given the current cluster state, and a maximum number of operations, will need to compute the instance relocations needed in order to achieve a "better" (for whatever the script believes it's better) cluster. Cluster capacity calculation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In this mode, called ``capacity``, given an instance specification and the current cluster state (similar to the ``allocate`` mode), the plugin needs to return: - how many instances can be allocated on the cluster with that specification - on which nodes these will be allocated (in order) .. vim: set textwidth=72 : ganeti-3.1.0~rc2/doc/design-2.10.rst000064400000000000000000000006721476477700300167370ustar00rootroot00000000000000================== Ganeti 2.10 design ================== The following design documents have been implemented in Ganeti 2.10. - :doc:`design-cmdlib-unittests` - :doc:`design-hotplug` - :doc:`design-openvswitch` - :doc:`design-performance-tests` - :doc:`design-storagetypes` - :doc:`design-upgrade` The following designs have been partially implemented in Ganeti 2.10. - :doc:`design-ceph-ganeti-support` - :doc:`design-internal-shutdown` ganeti-3.1.0~rc2/doc/design-2.11.rst000064400000000000000000000006031476477700300167320ustar00rootroot00000000000000================== Ganeti 2.11 design ================== The following design documents have been implemented in Ganeti 2.11. - :doc:`design-internal-shutdown` - :doc:`design-kvmd` - :doc:`design-glusterfs-ganeti-support` - :doc:`design-multi-version-tests` The following designs have been partially implemented in Ganeti 2.11. - :doc:`design-node-security` - :doc:`design-hsqueeze` ganeti-3.1.0~rc2/doc/design-2.12.rst000064400000000000000000000006441476477700300167400ustar00rootroot00000000000000================== Ganeti 2.12 design ================== The following design documents have been implemented in Ganeti 2.12. - :doc:`design-daemons` - :doc:`design-systemd` - :doc:`design-cpu-speed` - :doc:`design-move-instance-improvements` The following designs have been partially implemented in Ganeti 2.12. - :doc:`design-node-security` - :doc:`design-hsqueeze` - :doc:`design-os` - :doc:`design-reservations` ganeti-3.1.0~rc2/doc/design-2.13.rst000064400000000000000000000005441476477700300167400ustar00rootroot00000000000000================== Ganeti 2.13 design ================== The following design documents have been implemented in Ganeti 2.13. - :doc:`design-disk-conversion` - :doc:`design-optables` - :doc:`design-hsqueeze` The following designs have been partially implemented in Ganeti 2.13. - :doc:`design-location` - :doc:`design-node-security` - :doc:`design-os` ganeti-3.1.0~rc2/doc/design-2.14.rst000064400000000000000000000004731476477700300167420ustar00rootroot00000000000000================== Ganeti 2.14 design ================== The following designs have been implemented in Ganeti 2.14. - :doc:`design-file-based-disks-ownership` The following designs have been partially implemented in Ganeti 2.14. - :doc:`design-location` - :doc:`design-reservations` - :doc:`design-configlock` ganeti-3.1.0~rc2/doc/design-2.15.rst000064400000000000000000000005311476477700300167360ustar00rootroot00000000000000================== Ganeti 2.15 design ================== The following designs have been partially implemented in Ganeti 2.15. - :doc:`design-configlock` - :doc:`design-shared-storage-redundancy` The following designs' implementations were completed in Ganeti 2.15. - :doc:`design-allocation-efficiency` - :doc:`design-dedicated-allocation` ganeti-3.1.0~rc2/doc/design-2.16.rst000064400000000000000000000005331476477700300167410ustar00rootroot00000000000000================== Ganeti 2.16 design ================== The following designs have been partially implemented in Ganeti 2.16. - :doc:`design-configlock` - :doc:`design-os` The following designs' implementations were completed in Ganeti 2.16. - :doc:`design-location` - :doc:`design-plain-redundancy` - :doc:`design-shared-storage-redundancy` ganeti-3.1.0~rc2/doc/design-2.2.rst000064400000000000000000001110531476477700300166540ustar00rootroot00000000000000================= Ganeti 2.2 design ================= This document describes the major changes in Ganeti 2.2 compared to the 2.1 version. The 2.2 version will be a relatively small release. Its main aim is to avoid changing too much of the core code, while addressing issues and adding new features and improvements over 2.1, in a timely fashion. .. contents:: :depth: 4 As for 2.1 we divide the 2.2 design into three areas: - core changes, which affect the master daemon/job queue/locking or all/most logical units - logical unit/feature changes - external interface changes (e.g. command line, OS API, hooks, ...) Core changes ============ Master Daemon Scaling improvements ---------------------------------- Current state and shortcomings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently the Ganeti master daemon is based on four sets of threads: - The main thread (1 thread) just accepts connections on the master socket - The client worker pool (16 threads) handles those connections, one thread per connected socket, parses luxi requests, and sends data back to the clients - The job queue worker pool (25 threads) executes the actual jobs submitted by the clients - The rpc worker pool (10 threads) interacts with the nodes via http-based-rpc This means that every masterd currently runs 52 threads to do its job. Being able to reduce the number of thread sets would make the master's architecture a lot simpler. Moreover having less threads can help decrease lock contention, log pollution and memory usage. Also, with the current architecture, masterd suffers from quite a few scalability issues: Core daemon connection handling +++++++++++++++++++++++++++++++ Since the 16 client worker threads handle one connection each, it's very easy to exhaust them, by just connecting to masterd 16 times and not sending any data. While we could perhaps make those pools resizable, increasing the number of threads won't help with lock contention nor with better handling long running operations making sure the client is informed that everything is proceeding, and doesn't need to time out. Wait for job change +++++++++++++++++++ The REQ_WAIT_FOR_JOB_CHANGE luxi operation makes the relevant client thread block on its job for a relatively long time. This is another easy way to exhaust the 16 client threads, and a place where clients often time out, moreover this operation is negative for the job queue lock contention (see below). Job Queue lock ++++++++++++++ The job queue lock is quite heavily contended, and certain easily reproducible workloads show that's it's very easy to put masterd in trouble: for example running ~15 background instance reinstall jobs, results in a master daemon that, even without having finished the client worker threads, can't answer simple job list requests, or submit more jobs. Currently the job queue lock is an exclusive non-fair lock insulating the following job queue methods (called by the client workers). - AddNode - RemoveNode - SubmitJob - SubmitManyJobs - WaitForJobChanges - CancelJob - ArchiveJob - AutoArchiveJobs - QueryJobs - Shutdown Moreover the job queue lock is acquired outside of the job queue in two other classes: - jqueue._JobQueueWorker (in RunTask) before executing the opcode, after finishing its executing and when handling an exception. - jqueue._OpExecCallbacks (in NotifyStart and Feedback) when the processor (mcpu.Processor) is about to start working on the opcode (after acquiring the necessary locks) and when any data is sent back via the feedback function. Of those the major critical points are: - Submit[Many]Job, QueryJobs, WaitForJobChanges, which can easily slow down and block client threads up to making the respective clients time out. - The code paths in NotifyStart, Feedback, and RunTask, which slow down job processing between clients and otherwise non-related jobs. To increase the pain: - WaitForJobChanges is a bad offender because it's implemented with a notified condition which awakes waiting threads, who then try to acquire the global lock again - Many should-be-fast code paths are slowed down by replicating the change to remote nodes, and thus waiting, with the lock held, on remote rpcs to complete (starting, finishing, and submitting jobs) Proposed changes ~~~~~~~~~~~~~~~~ In order to be able to interact with the master daemon even when it's under heavy load, and to make it simpler to add core functionality (such as an asynchronous rpc client) we propose three subsequent levels of changes to the master core architecture. After making this change we'll be able to re-evaluate the size of our thread pool, if we see that we can make most threads in the client worker pool always idle. In the future we should also investigate making the rpc client asynchronous as well, so that we can make masterd a lot smaller in number of threads, and memory size, and thus also easier to understand, debug, and scale. Connection handling +++++++++++++++++++ We'll move the main thread of ganeti-masterd to asyncore, so that it can share the mainloop code with all other Ganeti daemons. Then all luxi clients will be asyncore clients, and I/O to/from them will be handled by the master thread asynchronously. Data will be read from the client sockets as it becomes available, and kept in a buffer, then when a complete message is found, it's passed to a client worker thread for parsing and processing. The client worker thread is responsible for serializing the reply, which can then be sent asynchronously by the main thread on the socket. Wait for job change +++++++++++++++++++ The REQ_WAIT_FOR_JOB_CHANGE luxi request is changed to be subscription-based, so that the executing thread doesn't have to be waiting for the changes to arrive. Threads producing messages (job queue executors) will make sure that when there is a change another thread is awakened and delivers it to the waiting clients. This can be either a dedicated "wait for job changes" thread or pool, or one of the client workers, depending on what's easier to implement. In either case the main asyncore thread will only be involved in pushing of the actual data, and not in fetching/serializing it. Other features to look at, when implementing this code are: - Possibility not to need the job lock to know which updates to push: if the thread producing the data pushed a copy of the update for the waiting clients, the thread sending it won't need to acquire the lock again to fetch the actual data. - Possibility to signal clients about to time out, when no update has been received, not to despair and to keep waiting (luxi level keepalive). - Possibility to defer updates if they are too frequent, providing them at a maximum rate (lower priority). Job Queue lock ++++++++++++++ In order to decrease the job queue lock contention, we will change the code paths in the following ways, initially: - A per-job lock will be introduced. All operations affecting only one job (for example feedback, starting/finishing notifications, subscribing to or watching a job) will only require the job lock. This should be a leaf lock, but if a situation arises in which it must be acquired together with the global job queue lock the global one must always be acquired last (for the global section). - The locks will be converted to a sharedlock. Any read-only operation will be able to proceed in parallel. - During remote update (which happens already per-job) we'll drop the job lock level to shared mode, so that activities reading the lock (for example job change notifications or QueryJobs calls) will be able to proceed in parallel. - The wait for job changes improvements proposed above will be implemented. In the future other improvements may include splitting off some of the work (eg replication of a job to remote nodes) to a separate thread pool or asynchronous thread, not tied with the code path for answering client requests or the one executing the "real" work. This can be discussed again after we used the more granular job queue in production and tested its benefits. Inter-cluster instance moves ---------------------------- Current state and shortcomings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ With the current design of Ganeti, moving whole instances between different clusters involves a lot of manual work. There are several ways to move instances, one of them being to export the instance, manually copying all data to the new cluster before importing it again. Manual changes to the instances configuration, such as the IP address, may be necessary in the new environment. The goal is to improve and automate this process in Ganeti 2.2. Proposed changes ~~~~~~~~~~~~~~~~ Authorization, Authentication and Security ++++++++++++++++++++++++++++++++++++++++++ Until now, each Ganeti cluster was a self-contained entity and wouldn't talk to other Ganeti clusters. Nodes within clusters only had to trust the other nodes in the same cluster and the network used for replication was trusted, too (hence the ability the use a separate, local network for replication). For inter-cluster instance transfers this model must be weakened. Nodes in one cluster will have to talk to nodes in other clusters, sometimes in other locations and, very important, via untrusted network connections. Various option have been considered for securing and authenticating the data transfer from one machine to another. To reduce the risk of accidentally overwriting data due to software bugs, authenticating the arriving data was considered critical. Eventually we decided to use socat's OpenSSL options (``OPENSSL:``, ``OPENSSL-LISTEN:`` et al), which provide us with encryption, authentication and authorization when used with separate keys and certificates. Combinations of OpenSSH, GnuPG and Netcat were deemed too complex to set up from within Ganeti. Any solution involving OpenSSH would require a dedicated user with a home directory and likely automated modifications to the user's ``$HOME/.ssh/authorized_keys`` file. When using Netcat, GnuPG or another encryption method would be necessary to transfer the data over an untrusted network. socat combines both in one program and is already a dependency. Each of the two clusters will have to generate an RSA key. The public parts are exchanged between the clusters by a third party, such as an administrator or a system interacting with Ganeti via the remote API ("third party" from here on). After receiving each other's public key, the clusters can start talking to each other. All encrypted connections must be verified on both sides. Neither side may accept unverified certificates. The generated certificate should only be valid for the time necessary to move the instance. For additional protection of the instance data, the two clusters can verify the certificates and destination information exchanged via the third party by checking an HMAC signature using a key shared among the involved clusters. By default this secret key will be a random string unique to the cluster, generated by running SHA1 over 20 bytes read from ``/dev/urandom`` and the administrator must synchronize the secrets between clusters before instances can be moved. If the third party does not know the secret, it can't forge the certificates or redirect the data. Unless disabled by a new cluster parameter, verifying the HMAC signatures must be mandatory. The HMAC signature for X509 certificates will be prepended to the certificate similar to an :rfc:`822` header and only covers the certificate (from ``-----BEGIN CERTIFICATE-----`` to ``-----END CERTIFICATE-----``). The header name will be ``X-Ganeti-Signature`` and its value will have the format ``$salt/$hash`` (salt and hash separated by slash). The salt may only contain characters in the range ``[a-zA-Z0-9]``. On the web, the destination cluster would be equivalent to an HTTPS server requiring verifiable client certificates. The browser would be equivalent to the source cluster and must verify the server's certificate while providing a client certificate to the server. Copying data ++++++++++++ To simplify the implementation, we decided to operate at a block-device level only, allowing us to easily support non-DRBD instance moves. Intra-cluster instance moves will re-use the existing export and import scripts supplied by instance OS definitions. Unlike simply copying the raw data, this allows one to use filesystem-specific utilities to dump only used parts of the disk and to exclude certain disks from the move. Compression should be used to further reduce the amount of data transferred. The export scripts writes all data to stdout and the import script reads it from stdin again. To avoid copying data and reduce disk space consumption, everything is read from the disk and sent over the network directly, where it'll be written to the new block device directly again. Workflow ++++++++ #. Third party tells source cluster to shut down instance, asks for the instance specification and for the public part of an encryption key - Instance information can already be retrieved using an existing API (``OpInstanceQueryData``). - An RSA encryption key and a corresponding self-signed X509 certificate is generated using the "openssl" command. This key will be used to encrypt the data sent to the destination cluster. - Private keys never leave the cluster. - The public part (the X509 certificate) is signed using HMAC with salting and a secret shared between Ganeti clusters. #. Third party tells destination cluster to create an instance with the same specifications as on source cluster and to prepare for an instance move with the key received from the source cluster and receives the public part of the destination's encryption key - The current API to create instances (``OpInstanceCreate``) will be extended to support an import from a remote cluster. - A valid, unexpired X509 certificate signed with the destination cluster's secret will be required. By verifying the signature, we know the third party didn't modify the certificate. - The private keys never leave their cluster, hence the third party can not decrypt or intercept the instance's data by modifying the IP address or port sent by the destination cluster. - The destination cluster generates another key and certificate, signs and sends it to the third party, who will have to pass it to the API for exporting an instance (``OpBackupExport``). This certificate is used to ensure we're sending the disk data to the correct destination cluster. - Once a disk can be imported, the API sends the destination information (IP address and TCP port) together with an HMAC signature to the third party. #. Third party hands public part of the destination's encryption key together with all necessary information to source cluster and tells it to start the move - The existing API for exporting instances (``OpBackupExport``) will be extended to export instances to remote clusters. #. Source cluster connects to destination cluster for each disk and transfers its data using the instance OS definition's export and import scripts - Before starting, the source cluster must verify the HMAC signature of the certificate and destination information (IP address and TCP port). - When connecting to the remote machine, strong certificate checks must be employed. #. Due to the asynchronous nature of the whole process, the destination cluster checks whether all disks have been transferred every time after transferring a single disk; if so, it destroys the encryption key #. After sending all disks, the source cluster destroys its key #. Destination cluster runs OS definition's rename script to adjust instance settings if needed (e.g. IP address) #. Destination cluster starts the instance if requested at the beginning by the third party #. Source cluster removes the instance if requested Instance move in pseudo code ++++++++++++++++++++++++++++ .. highlight:: python The following pseudo code describes a script moving instances between clusters and what happens on both clusters. #. Script is started, gets the instance name and destination cluster:: (instance_name, dest_cluster_name) = sys.argv[1:] # Get destination cluster object dest_cluster = db.FindCluster(dest_cluster_name) # Use database to find source cluster src_cluster = db.FindClusterByInstance(instance_name) #. Script tells source cluster to stop instance:: # Stop instance src_cluster.StopInstance(instance_name) # Get instance specification (memory, disk, etc.) inst_spec = src_cluster.GetInstanceInfo(instance_name) (src_key_name, src_cert) = src_cluster.CreateX509Certificate() #. ``CreateX509Certificate`` on source cluster:: key_file = mkstemp() cert_file = "%s.cert" % key_file RunCmd(["/usr/bin/openssl", "req", "-new", "-newkey", "rsa:1024", "-days", "1", "-nodes", "-x509", "-batch", "-keyout", key_file, "-out", cert_file]) plain_cert = utils.ReadFile(cert_file) # HMAC sign using secret key, this adds a "X-Ganeti-Signature" # header to the beginning of the certificate signed_cert = utils.SignX509Certificate(plain_cert, utils.ReadFile(constants.X509_SIGNKEY_FILE)) # The certificate now looks like the following: # # X-Ganeti-Signature: $1234$28676f0516c6ab68062b[â€Ļ] # -----BEGIN CERTIFICATE----- # MIICsDCCAhmgAwIBAgI[â€Ļ] # -----END CERTIFICATE----- # Return name of key file and signed certificate in PEM format return (os.path.basename(key_file), signed_cert) #. Script creates instance on destination cluster and waits for move to finish:: dest_cluster.CreateInstance(mode=constants.REMOTE_IMPORT, spec=inst_spec, source_cert=src_cert) # Wait until destination cluster gives us its certificate dest_cert = None disk_info = [] while not (dest_cert and len(disk_info) < len(inst_spec.disks)): tmp = dest_cluster.WaitOutput() if tmp is Certificate: dest_cert = tmp elif tmp is DiskInfo: # DiskInfo contains destination address and port disk_info[tmp.index] = tmp # Tell source cluster to export disks for disk in disk_info: src_cluster.ExportDisk(instance_name, disk=disk, key_name=src_key_name, dest_cert=dest_cert) print ("Instance %s sucessfully moved to %s" % (instance_name, dest_cluster.name)) #. ``CreateInstance`` on destination cluster:: # â€Ļ if mode == constants.REMOTE_IMPORT: # Make sure certificate was not modified since it was generated by # source cluster (which must use the same secret) if (not utils.VerifySignedX509Cert(source_cert, utils.ReadFile(constants.X509_SIGNKEY_FILE))): raise Error("Certificate not signed with this cluster's secret") if utils.CheckExpiredX509Cert(source_cert): raise Error("X509 certificate is expired") source_cert_file = utils.WriteTempFile(source_cert) # See above for X509 certificate generation and signing (key_name, signed_cert) = CreateSignedX509Certificate() SendToClient("x509-cert", signed_cert) for disk in instance.disks: # Start socat RunCmd(("socat" " OPENSSL-LISTEN:%s,â€Ļ,key=%s,cert=%s,cafile=%s,verify=1" " stdout > /dev/diskâ€Ļ") % port, GetRsaKeyPath(key_name, private=True), GetRsaKeyPath(key_name, private=False), src_cert_file) SendToClient("send-disk-to", disk, ip_address, port) DestroyX509Cert(key_name) RunRenameScript(instance_name) #. ``ExportDisk`` on source cluster:: # Make sure certificate was not modified since it was generated by # destination cluster (which must use the same secret) if (not utils.VerifySignedX509Cert(cert_pem, utils.ReadFile(constants.X509_SIGNKEY_FILE))): raise Error("Certificate not signed with this cluster's secret") if utils.CheckExpiredX509Cert(cert_pem): raise Error("X509 certificate is expired") dest_cert_file = utils.WriteTempFile(cert_pem) # Start socat RunCmd(("socat stdin" " OPENSSL:%s:%s,â€Ļ,key=%s,cert=%s,cafile=%s,verify=1" " < /dev/diskâ€Ļ") % disk.host, disk.port, GetRsaKeyPath(key_name, private=True), GetRsaKeyPath(key_name, private=False), dest_cert_file) if instance.all_disks_done: DestroyX509Cert(key_name) .. highlight:: text Miscellaneous notes +++++++++++++++++++ - A very similar system could also be used for instance exports within the same cluster. Currently OpenSSH is being used, but could be replaced by socat and SSL/TLS. - During the design of intra-cluster instance moves we also discussed encrypting instance exports using GnuPG. - While most instances should have exactly the same configuration as on the source cluster, setting them up with a different disk layout might be helpful in some use-cases. - A cleanup operation, similar to the one available for failed instance migrations, should be provided. - ``ganeti-watcher`` should remove instances pending a move from another cluster after a certain amount of time. This takes care of failures somewhere in the process. - RSA keys can be generated using the existing ``bootstrap.GenerateSelfSignedSslCert`` function, though it might be useful to not write both parts into a single file, requiring small changes to the function. The public part always starts with ``-----BEGIN CERTIFICATE-----`` and ends with ``-----END CERTIFICATE-----``. - The source and destination cluster might be different when it comes to available hypervisors, kernels, etc. The destination cluster should refuse to accept an instance move if it can't fulfill an instance's requirements. Privilege separation -------------------- Current state and shortcomings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ All Ganeti daemons are run under the user root. This is not ideal from a security perspective as for possible exploitation of any daemon the user has full access to the system. In order to overcome this situation we'll allow Ganeti to run its daemon under different users and a dedicated group. This also will allow some side effects, like letting the user run some ``gnt-*`` commands if one is in the same group. Implementation ~~~~~~~~~~~~~~ For Ganeti 2.2 the implementation will be focused on a the RAPI daemon only. This involves changes to ``daemons.py`` so it's possible to drop privileges on daemonize the process. Though, this will be a short term solution which will be replaced by a privilege drop already on daemon startup in Ganeti 2.3. It also needs changes in the master daemon to create the socket with new permissions/owners to allow RAPI access. There will be no other permission/owner changes in the file structure as the RAPI daemon is started with root permission. In that time it will read all needed files and then drop privileges before contacting the master daemon. Feature changes =============== KVM Security ------------ Current state and shortcomings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently all kvm processes run as root. Taking ownership of the hypervisor process, from inside a virtual machine, would mean a full compromise of the whole Ganeti cluster, knowledge of all Ganeti authentication secrets, full access to all running instances, and the option of subverting other basic services on the cluster (eg: ssh). Proposed changes ~~~~~~~~~~~~~~~~ We would like to decrease the surface of attack available if an hypervisor is compromised. We can do so adding different features to Ganeti, which will allow restricting the broken hypervisor possibilities, in the absence of a local privilege escalation attack, to subvert the node. Dropping privileges in kvm to a single user (easy) ++++++++++++++++++++++++++++++++++++++++++++++++++ By passing the ``-runas`` option to kvm, we can make it drop privileges. The user can be chosen by an hypervisor parameter, so that each instance can have its own user, but by default they will all run under the same one. It should be very easy to implement, and can easily be backported to 2.1.X. This mode protects the Ganeti cluster from a subverted hypervisor, but doesn't protect the instances between each other, unless care is taken to specify a different user for each. This would prevent the worst attacks, including: - logging in to other nodes - administering the Ganeti cluster - subverting other services But the following would remain an option: - terminate other VMs (but not start them again, as that requires root privileges to set up networking) (unless different users are used) - trace other VMs, and probably subvert them and access their data (unless different users are used) - send network traffic from the node - read unprotected data on the node filesystem Running kvm in a chroot (slightly harder) +++++++++++++++++++++++++++++++++++++++++ By passing the ``-chroot`` option to kvm, we can restrict the kvm process in its own (possibly empty) root directory. We need to set this area up so that the instance disks and control sockets are accessible, so it would require slightly more work at the Ganeti level. Breaking out in a chroot would mean: - a lot less options to find a local privilege escalation vector - the impossibility to write local data, if the chroot is set up correctly - the impossibility to read filesystem data on the host It would still be possible though to: - terminate other VMs - trace other VMs, and possibly subvert them (if a tracer can be installed in the chroot) - send network traffic from the node Running kvm with a pool of users (slightly harder) ++++++++++++++++++++++++++++++++++++++++++++++++++ If rather than passing a single user as an hypervisor parameter, we have a pool of useable ones, we can dynamically choose a free one to use and thus guarantee that each machine will be separate from the others, without putting the burden of this on the cluster administrator. This would mean interfering between machines would be impossible, and can still be combined with the chroot benefits. Running iptables rules to limit network interaction (easy) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ These don't need to be handled by Ganeti, but we can ship examples. If the users used to run VMs would be blocked from sending some or all network traffic, it would become impossible for a broken into hypervisor to send arbitrary data on the node network, which is especially useful when the instance and the node network are separated (using ganeti-nbma or a separate set of network interfaces), or when a separate replication network is maintained. We need to experiment to see how much restriction we can properly apply, without limiting the instance legitimate traffic. Running kvm inside a container (even harder) ++++++++++++++++++++++++++++++++++++++++++++ Recent linux kernels support different process namespaces through control groups. PIDs, users, filesystems and even network interfaces can be separated. If we can set up ganeti to run kvm in a separate container we could insulate all the host process from being even visible if the hypervisor gets broken into. Most probably separating the network namespace would require one extra hop in the host, through a veth interface, thus reducing performance, so we may want to avoid that, and just rely on iptables. Implementation plan ~~~~~~~~~~~~~~~~~~~ We will first implement dropping privileges for kvm processes as a single user, and most probably backport it to 2.1. Then we'll ship example iptables rules to show how the user can be limited in its network activities. After that we'll implement chroot restriction for kvm processes, and extend the user limitation to use a user pool. Finally we'll look into namespaces and containers, although that might slip after the 2.2 release. New OS states ------------- Separate from the OS external changes, described below, we'll add some internal changes to the OS. Current state and shortcomings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ There are two issues related to the handling of the OSes. First, it's impossible to disable an OS for new instances, since that will also break reinstallations and renames of existing instances. To phase out an OS definition, without actually having to modify the OS scripts, it would be ideal to be able to restrict new installations but keep the rest of the functionality available. Second, ``gnt-instance reinstall --select-os`` shows all the OSes available on the clusters. Some OSes might exist only for debugging and diagnose, and not for end-user availability. For this, it would be useful to "hide" a set of OSes, but keep it otherwise functional. Proposed changes ~~~~~~~~~~~~~~~~ Two new cluster-level attributes will be added, holding the list of OSes hidden from the user and respectively the list of OSes which are blacklisted from new installations. These lists will be modifiable via ``gnt-os modify`` (implemented via ``OpClusterSetParams``), such that even not-yet-existing OSes can be preseeded into a given state. For the hidden OSes, they are fully functional except that they are not returned in the default OS list (as computed via ``OpOsDiagnose``), unless the hidden state is requested. For the blacklisted OSes, they are also not shown (unless the blacklisted state is requested), and they are also prevented from installation via ``OpInstanceCreate`` (in create mode). Both these attributes are per-OS, not per-variant. Thus they apply to all of an OS' variants, and it's impossible to blacklist or hide just one variant. Further improvements might allow a given OS variant to be blacklisted, as opposed to whole OSes. External interface changes ========================== OS API ------ The OS variants implementation in Ganeti 2.1 didn't prove to be useful enough to alleviate the need to hack around the Ganeti API in order to provide flexible OS parameters. As such, for Ganeti 2.2 we will provide support for arbitrary OS parameters. However, since OSes are not registered in Ganeti, but instead discovered at runtime, the interface is not entirely straightforward. Furthermore, to support the system administrator in keeping OSes properly in sync across the nodes of a cluster, Ganeti will also verify (if existing) the consistence of a new ``os_version`` file. These changes to the OS API will bump the API version to 20. OS version ~~~~~~~~~~ A new ``os_version`` file will be supported by Ganeti. This file is not required, but if existing, its contents will be checked for consistency across nodes. The file should hold only one line of text (any extra data will be discarded), and its contents will be shown in the OS information and diagnose commands. It is recommended that OS authors increase the contents of this file for any changes; at a minimum, modifications that change the behaviour of import/export scripts must increase the version, since they break intra-cluster migration. Parameters ~~~~~~~~~~ The interface between Ganeti and the OS scripts will be based on environment variables, and as such the parameters and their values will need to be valid in this context. Names +++++ The parameter names will be declared in a new file, ``parameters.list``, together with a one-line documentation (whitespace-separated). Example:: $ cat parameters.list ns1 Specifies the first name server to add to /etc/resolv.conf extra_packages Specifies additional packages to install rootfs_size Specifies the root filesystem size (the rest will be left unallocated) track Specifies the distribution track, one of 'stable', 'testing' or 'unstable' As seen above, the documentation can be separate via multiple spaces/tabs from the names. The parameter names as read from the file will be used for the command line interface in lowercased form; as such, there shouldn't be any two parameters which differ in case only. Values ++++++ The values of the parameters are, from Ganeti's point of view, completely freeform. If a given parameter has, from the OS' point of view, a fixed set of valid values, these should be documented as such and verified by the OS, but Ganeti will not handle such parameters specially. An empty value must be handled identically as a missing parameter. In other words, the validation script should only test for non-empty values, and not for declared versus undeclared parameters. Furthermore, each parameter should have an (internal to the OS) default value, that will be used if not passed from Ganeti. More precisely, it should be possible for any parameter to specify a value that will have the same effect as not passing the parameter, and no in no case should the absence of a parameter be treated as an exceptional case (outside the value space). Environment variables ^^^^^^^^^^^^^^^^^^^^^ The parameters will be exposed in the environment upper-case and prefixed with the string ``OSP_``. For example, a parameter declared in the 'parameters' file as ``ns1`` will appear in the environment as the variable ``OSP_NS1``. Validation ++++++++++ For the purpose of parameter name/value validation, the OS scripts *must* provide an additional script, named ``verify``. This script will be called with the argument ``parameters``, and all the parameters will be passed in via environment variables, as described above. The script should signify result/failure based on its exit code, and show explanatory messages either on its standard output or standard error. These messages will be passed on to the master, and stored as in the OpCode result/error message. The parameters must be constructed to be independent of the instance specifications. In general, the validation script will only be called with the parameter variables set, but not with the normal per-instance variables, in order for Ganeti to be able to validate default parameters too, when they change. Validation will only be performed on one cluster node, and it will be up to the ganeti administrator to keep the OS scripts in sync between all nodes. Instance operations +++++++++++++++++++ The parameters will be passed, as described above, to all the other instance operations (creation, import, export). Ideally, these scripts will not abort with parameter validation errors, if the ``verify`` script has verified them correctly. Note: when changing an instance's OS type, any OS parameters defined at instance level will be kept as-is. If the parameters differ between the new and the old OS, the user should manually remove/update them as needed. Declaration and modification ++++++++++++++++++++++++++++ Since the OSes are not registered in Ganeti, we will only make a 'weak' link between the parameters as declared in Ganeti and the actual OSes existing on the cluster. It will be possible to declare parameters either globally, per cluster (where they are indexed per OS/variant), or individually, per instance. The declaration of parameters will not be tied to current existing OSes. When specifying a parameter, if the OS exists, it will be validated; if not, then it will simply be stored as-is. A special note is that it will not be possible to 'unset' at instance level a parameter that is declared globally. Instead, at instance level the parameter should be given an explicit value, or the default value as explained above. CLI interface +++++++++++++ The modification of global (default) parameters will be done via the ``gnt-os`` command, and the per-instance parameters via the ``gnt-instance`` command. Both these commands will take an addition ``--os-parameters`` or ``-O`` flag that specifies the parameters in the familiar comma-separated, key=value format. For removing a parameter, a ``-key`` syntax will be used, e.g.:: # initial modification $ gnt-instance modify -O use_dchp=true instance1 # later revert (to the cluster default, or the OS default if not # defined at cluster level) $ gnt-instance modify -O -use_dhcp instance1 Internal storage ++++++++++++++++ Internally, the OS parameters will be stored in a new ``osparams`` attribute. The global parameters will be stored on the cluster object, and the value of this attribute will be a dictionary indexed by OS name (this also accepts an OS+variant name, which will override a simple OS name, see below), and for values the key/name dictionary. For the instances, the value will be directly the key/name dictionary. Overriding rules ++++++++++++++++ Any instance-specific parameters will override any variant-specific parameters, which in turn will override any global parameters. The global parameters, in turn, override the built-in defaults (of the OS scripts). .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-2.3.rst000064400000000000000000001125011476477700300166540ustar00rootroot00000000000000================= Ganeti 2.3 design ================= This document describes the major changes in Ganeti 2.3 compared to the 2.2 version. .. contents:: :depth: 4 As for 2.1 and 2.2 we divide the 2.3 design into three areas: - core changes, which affect the master daemon/job queue/locking or all/most logical units - logical unit/feature changes - external interface changes (e.g. command line, OS API, hooks, ...) Core changes ============ Node Groups ----------- Current state and shortcomings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently all nodes of a Ganeti cluster are considered as part of the same pool, for allocation purposes: DRBD instances for example can be allocated on any two nodes. This does cause a problem in cases where nodes are not all equally connected to each other. For example if a cluster is created over two set of machines, each connected to its own switch, the internal bandwidth between machines connected to the same switch might be bigger than the bandwidth for inter-switch connections. Moreover, some operations inside a cluster require all nodes to be locked together for inter-node consistency, and won't scale if we increase the number of nodes to a few hundreds. Proposed changes ~~~~~~~~~~~~~~~~ With this change we'll divide Ganeti nodes into groups. Nothing will change for clusters with only one node group. Bigger clusters will be able to have more than one group, and each node will belong to exactly one. Node group management +++++++++++++++++++++ To manage node groups and the nodes belonging to them, the following new commands and flags will be introduced:: gnt-group add # add a new node group gnt-group remove # delete an empty node group gnt-group list # list node groups gnt-group rename # rename a node group gnt-node {list,info} -g # list only nodes belonging to a node group gnt-node modify -g # assign a node to a node group Node group attributes +++++++++++++++++++++ In clusters with more than one node group, it may be desirable to establish local policies regarding which groups should be preferred when performing allocation of new instances, or inter-group instance migrations. To help with this, we will provide an ``alloc_policy`` attribute for node groups. Such attribute will be honored by iallocator plugins when making automatic decisions regarding instance placement. The ``alloc_policy`` attribute can have the following values: - unallocable: the node group should not be a candidate for instance allocations, and the operation should fail if only groups in this state could be found that would satisfy the requirements. - last_resort: the node group should not be used for instance allocations, unless this would be the only way to have the operation succeed. Prioritization among groups in this state will be deferred to the iallocator plugin that's being used. - preferred: the node group can be used freely for allocation of instances (this is the default state for newly created node groups). Note that prioritization among groups in this state will be deferred to the iallocator plugin that's being used. Node group operations +++++++++++++++++++++ One operation at the node group level will be initially provided:: gnt-group drain The purpose of this operation is to migrate all instances in a given node group to other groups in the cluster, e.g. to reclaim capacity if there are enough free resources in other node groups that share a storage pool with the evacuated group. Instance level changes ++++++++++++++++++++++ With the introduction of node groups, instances will be required to live in only one group at a time; this is mostly important for DRBD instances, which will not be allowed to have their primary and secondary nodes in different node groups. To support this, we envision the following changes: - The iallocator interface will be augmented, and node groups exposed, so that plugins will be able to make a decision regarding the group in which to place a new instance. By default, all node groups will be considered, but it will be possible to include a list of groups in the creation job, in which case the plugin will limit itself to considering those; in both cases, the ``alloc_policy`` attribute will be honored. - If, on the other hand, a primary and secondary nodes are specified for a new instance, they will be required to be on the same node group. - Moving an instance between groups can only happen via an explicit operation, which for example in the case of DRBD will work by performing internally a replace-disks, a migration, and a second replace-disks. It will be possible to clean up an interrupted group-move operation. - Cluster verify will signal an error if an instance has nodes belonging to different groups. Additionally, changing the group of a given node will be initially only allowed if the node is empty, as a straightforward mechanism to avoid creating such situation. - Inter-group instance migration will have the same operation modes as new instance allocation, defined above: letting an iallocator plugin decide the target group, possibly restricting the set of node groups to consider, or specifying a target primary and secondary nodes. In both cases, the target group or nodes must be able to accept the instance network- and storage-wise; the operation will fail otherwise, though in the future we may be able to allow some parameter to be changed together with the move (in the meantime, an import/export will be required in this scenario). Internal changes ++++++++++++++++ We expect the following changes for cluster management: - Frequent multinode operations, such as os-diagnose or cluster-verify, will act on one group at a time, which will have to be specified in all cases, except for clusters with just one group. Command line tools will also have a way to easily target all groups, by generating one job per group. - Groups will have a human-readable name, but will internally always be referenced by a UUID, which will be immutable; for example, nodes will contain the UUID of the group they belong to. This is done to simplify referencing while keeping it easy to handle renames and movements. If we see that this works well, we'll transition other config objects (instances, nodes) to the same model. - The addition of a new per-group lock will be evaluated, if we can transition some operations now requiring the BGL to it. - Master candidate status will be allowed to be spread among groups. For the first version we won't add any restriction over how this is done, although in the future we may have a minimum number of master candidates which Ganeti will try to keep in each group, for example. Other work and future changes +++++++++++++++++++++++++++++ Commands like ``gnt-cluster command``/``gnt-cluster copyfile`` will continue to work on the whole cluster, but it will be possible to target one group only by specifying it. Commands which allow selection of sets of resources (for example ``gnt-instance start``/``gnt-instance stop``) will be able to select them by node group as well. Initially node groups won't be taggable objects, to simplify the first implementation, but we expect this to be easy to add in a future version should we see it's useful. We envision groups as a good place to enhance cluster scalability. In the future we may want to use them as units for configuration diffusion, to allow a better master scalability. For example it could be possible to change some all-nodes RPCs to contact each group once, from the master, and make one node in the group perform internal diffusion. We won't implement this in the first version, but we'll evaluate it for the future, if we see scalability problems on big multi-group clusters. When Ganeti will support more storage models (e.g. SANs, Sheepdog, Ceph) we expect groups to be the basis for this, allowing for example a different Sheepdog/Ceph cluster, or a different SAN to be connected to each group. In some cases this will mean that inter-group move operation will be necessarily performed with instance downtime, unless the hypervisor has block-migrate functionality, and we implement support for it (this would be theoretically possible, today, with KVM, for example). Scalability issues with big clusters ------------------------------------ Current and future issues ~~~~~~~~~~~~~~~~~~~~~~~~~ Assuming the node groups feature will enable bigger clusters, other parts of Ganeti will be impacted even more by the (in effect) bigger clusters. While many areas will be impacted, one is the most important: the fact that the watcher still needs to be able to repair instance data on the current 5 minutes time-frame (a shorter time-frame would be even better). This means that the watcher itself needs to have parallelism when dealing with node groups. Also, the iallocator plugins are being fed data from Ganeti but also need access to the full cluster state, and in general we still rely on being able to compute the full cluster state somewhat “cheaply” and on-demand. This conflicts with the goal of disconnecting the different node groups, and to keep the same parallelism while growing the cluster size. Another issue is that the current capacity calculations are done completely outside Ganeti (and they need access to the entire cluster state), and this prevents keeping the capacity numbers in sync with the cluster state. While this is still acceptable for smaller clusters where a small number of allocations/removal are presumed to occur between two periodic capacity calculations, on bigger clusters where we aim to parallelize heavily between node groups this is no longer true. As proposed changes, the main change is introducing a cluster state cache (not serialised to disk), and to update many of the LUs and cluster operations to account for it. Furthermore, the capacity calculations will be integrated via a new OpCode/LU, so that we have faster feedback (instead of periodic computation). Cluster state cache ~~~~~~~~~~~~~~~~~~~ A new cluster state cache will be introduced. The cache relies on two main ideas: - the total node memory, CPU count are very seldom changing; the total node disk space is also slow changing, but can change at runtime; the free memory and free disk will change significantly for some jobs, but on a short timescale; in general, these values will be mostly “constant” during the lifetime of a job - we already have a periodic set of jobs that query the node and instance state, driven the by :command:`ganeti-watcher` command, and we're just discarding the results after acting on them Given the above, it makes sense to cache the results of node and instance state (with a focus on the node state) inside the master daemon. The cache will not be serialised to disk, and will be for the most part transparent to the outside of the master daemon. Cache structure +++++++++++++++ The cache will be oriented with a focus on node groups, so that it will be easy to invalidate an entire node group, or a subset of nodes, or the entire cache. The instances will be stored in the node group of their primary node. Furthermore, since the node and instance properties determine the capacity statistics in a deterministic way, the cache will also hold, at each node group level, the total capacity as determined by the new capacity iallocator mode. Cache updates +++++++++++++ The cache will be updated whenever a query for a node state returns “full” node information (so as to keep the cache state for a given node consistent). Partial results will not update the cache (see next paragraph). Since there will be no way to feed the cache from outside, and we would like to have a consistent cache view when driven by the watcher, we'll introduce a new OpCode/LU for the watcher to run, instead of the current separate opcodes (see below in the watcher section). Updates to a node that change a node's specs “downward” (e.g. less memory) will invalidate the capacity data. Updates that increase the node will not invalidate the capacity, as we're more interested in “at least available” correctness, not “at most available”. Cache invalidation ++++++++++++++++++ If a partial node query is done (e.g. just for the node free space), and the returned values don't match with the cache, then the entire node state will be invalidated. By default, all LUs will invalidate the caches for all nodes and instances they lock. If an LU uses the BGL, then it will invalidate the entire cache. In time, it is expected that LUs will be modified to not invalidate, if they are not expected to change the node's and/or instance's state (e.g. ``LUInstanceConsole``, or ``LUInstanceActivateDisks``). Invalidation of a node's properties will also invalidate the capacity data associated with that node. Cache lifetime ++++++++++++++ The cache elements will have an upper bound on their lifetime; the proposal is to make this an hour, which should be a high enough value to cover the watcher being blocked by a medium-term job (e.g. 20-30 minutes). Cache usage +++++++++++ The cache will be used by default for most queries (e.g. a Luxi call, without locks, for the entire cluster). Since this will be a change from the current behaviour, we'll need to allow non-cached responses, e.g. via a ``--cache=off`` or similar argument (which will force the query). The cache will also be used for the iallocator runs, so that computing allocation solution can proceed independent from other jobs which lock parts of the cluster. This is important as we need to separate allocation on one group from exclusive blocking jobs on other node groups. The capacity calculations will also use the cache. This is detailed in the respective sections. Watcher operation ~~~~~~~~~~~~~~~~~ As detailed in the cluster cache section, the watcher also needs improvements in order to scale with the the cluster size. As a first improvement, the proposal is to introduce a new OpCode/LU pair that runs with locks held over the entire query sequence (the current watcher runs a job with two opcodes, which grab and release the locks individually). The new opcode will be called ``OpUpdateNodeGroupCache`` and will do the following: - try to acquire all node/instance locks (to examine in more depth, and possibly alter) in the given node group - invalidate the cache for the node group - acquire node and instance state (possibly via a new single RPC call that combines node and instance information) - update cache - return the needed data The reason for the per-node group query is that we don't want a busy node group to prevent instance maintenance in other node groups. Therefore, the watcher will introduce parallelism across node groups, and it will possible to have overlapping watcher runs. The new execution sequence will be: - the parent watcher process acquires global watcher lock - query the list of node groups (lockless or very short locks only) - fork N children, one for each node group - release the global lock - poll/wait for the children to finish Each forked children will do the following: - try to acquire the per-node group watcher lock - if fail to acquire, exit with special code telling the parent that the node group is already being managed by a watcher process - otherwise, submit a OpUpdateNodeGroupCache job - get results (possibly after a long time, due to busy group) - run the needed maintenance operations for the current group This new mode of execution means that the master watcher processes might overlap in running, but not the individual per-node group child processes. This change allows us to keep (almost) the same parallelism when using a bigger cluster with node groups versus two separate clusters. Cost of periodic cache updating +++++++++++++++++++++++++++++++ Currently the watcher only does “small” queries for the node and instance state, and at first sight changing it to use the new OpCode which populates the cache with the entire state might introduce additional costs, which must be payed every five minutes. However, the OpCodes that the watcher submits are using the so-called dynamic fields (need to contact the remote nodes), and the LUs are not selective—they always grab all the node and instance state. So in the end, we have the same cost, it just becomes explicit rather than implicit. This ‘grab all node state’ behaviour is what makes the cache worth implementing. Intra-node group scalability ++++++++++++++++++++++++++++ The design above only deals with inter-node group issues. It still makes sense to run instance maintenance for nodes A and B if only node C is locked (all being in the same node group). This problem is commonly encountered in previous Ganeti versions, and it should be handled similarly, by tweaking lock lifetime in long-duration jobs. TODO: add more ideas here. State file maintenance ++++++++++++++++++++++ The splitting of node group maintenance to different children which will run in parallel requires that the state file handling changes from monolithic updates to partial ones. There are two file that the watcher maintains: - ``$LOCALSTATEDIR/lib/ganeti/watcher.data``, its internal state file, used for deciding internal actions - ``$LOCALSTATEDIR/run/ganeti/instance-status``, a file designed for external consumption For the first file, since it's used only internally to the watchers, we can move to a per node group configuration. For the second file, even if it's used as an external interface, we will need to make some changes to it: because the different node groups can return results at different times, we need to either split the file into per-group files or keep the single file and add a per-instance timestamp (currently the file holds only the instance name and state). The proposal is that each child process maintains its own node group file, and the master process will, right after querying the node group list, delete any extra per-node group state file. This leaves the consumers to run a simple ``cat instance-status.group-*`` to obtain the entire list of instance and their states. If needed, the modify timestamp of each file can be used to determine the age of the results. Capacity calculations ~~~~~~~~~~~~~~~~~~~~~ Currently, the capacity calculations are done completely outside Ganeti. As explained in the current problems section, this needs to account better for the cluster state changes. Therefore a new OpCode will be introduced, ``OpComputeCapacity``, that will either return the current capacity numbers (if available), or trigger a new capacity calculation, via the iallocator framework, which will get a new method called ``capacity``. This method will feed the cluster state (for the complete set of node group, or alternative just a subset) to the iallocator plugin (either the specified one, or the default if none is specified), and return the new capacity in the format currently exported by the htools suite and known as the “tiered specs” (see :manpage:`hspace(1)`). tspec cluster parameters ++++++++++++++++++++++++ Currently, the “tspec” calculations done in :command:`hspace` require some additional parameters: - maximum instance size - type of instance storage - maximum ratio of virtual CPUs per physical CPUs - minimum disk free For the integration in Ganeti, there are multiple ways to pass these: - ignored by Ganeti, and being the responsibility of the iallocator plugin whether to use these at all or not - as input to the opcode - as proper cluster parameters Since the first option is not consistent with the intended changes, a combination of the last two is proposed: - at cluster level, we'll have cluster-wide defaults - at node groups, we'll allow overriding the cluster defaults - and if they are passed in via the opcode, they will override for the current computation the values Whenever the capacity is requested via different parameters, it will invalidate the cache, even if otherwise the cache is up-to-date. The new parameters are: - max_inst_spec: (int, int, int), the maximum instance specification accepted by this cluster or node group, in the order of memory, disk, vcpus; - default_template: string, the default disk template to use - max_cpu_ratio: double, the maximum ratio of VCPUs/PCPUs - max_disk_usage: double, the maximum disk usage (as a ratio) These might also be used in instance creations (to be determined later, after they are introduced). OpCode details ++++++++++++++ Input: - iallocator: string (optional, otherwise uses the cluster default) - cached: boolean, optional, defaults to true, and denotes whether we accept cached responses - the above new parameters, optional; if they are passed, they will overwrite all node group's parameters Output: - cluster: list of tuples (memory, disk, vcpu, count), in decreasing order of specifications; the first three members represent the instance specification, the last one the count of how many instances of this specification can be created on the cluster - node_groups: a dictionary keyed by node group UUID, with values a dictionary: - tspecs: a list like the cluster one - additionally, the new cluster parameters, denoting the input parameters that were used for this node group - ctime: the date the result has been computed; this represents the oldest creation time amongst all node groups (so as to accurately represent how much out-of-date the global response is) Note that due to the way the tspecs are computed, for any given specification, the total available count is the count for the given entry, plus the sum of counts for higher specifications. Node flags ---------- Current state and shortcomings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently all nodes are, from the point of view of their capabilities, homogeneous. This means the cluster considers all nodes capable of becoming master candidates, and of hosting instances. This prevents some deployment scenarios: e.g. having a Ganeti instance (in another cluster) be just a master candidate, in case all other master candidates go down (but not, of course, host instances), or having a node in a remote location just host instances but not become master, etc. Proposed changes ~~~~~~~~~~~~~~~~ Two new capability flags will be added to the node: - master_capable, denoting whether the node can become a master candidate or master - vm_capable, denoting whether the node can host instances In terms of the other flags, master_capable is a stronger version of "not master candidate", and vm_capable is a stronger version of "drained". For the master_capable flag, it will affect auto-promotion code and node modifications. The vm_capable flag will affect the iallocator protocol, capacity calculations, node checks in cluster verify, and will interact in novel ways with locking (unfortunately). It is envisaged that most nodes will be both vm_capable and master_capable, and just a few will have one of these flags removed. Ganeti itself will allow clearing of both flags, even though this doesn't make much sense currently. .. _jqueue-job-priority-design: Job priorities -------------- Current state and shortcomings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently all jobs and opcodes have the same priority. Once a job started executing, its thread won't be released until all opcodes got their locks and did their work. When a job is finished, the next job is selected strictly by its incoming order. This does not mean jobs are run in their incoming order—locks and other delays can cause them to be stalled for some time. In some situations, e.g. an emergency shutdown, one may want to run a job as soon as possible. This is not possible currently if there are pending jobs in the queue. Proposed changes ~~~~~~~~~~~~~~~~ Each opcode will be assigned a priority on submission. Opcode priorities are integers and the lower the number, the higher the opcode's priority is. Within the same priority, jobs and opcodes are initially processed in their incoming order. Submitted opcodes can have one of the priorities listed below. Other priorities are reserved for internal use. The absolute range is -20..+19. Opcodes submitted without a priority (e.g. by older clients) are assigned the default priority. - High (-10) - Normal (0, default) - Low (+10) As a change from the current model where executing a job blocks one thread for the whole duration, the new job processor must return the job to the queue after each opcode and also if it can't get all locks in a reasonable timeframe. This will allow opcodes of higher priority submitted in the meantime to be processed or opcodes of the same priority to try to get their locks. When added to the job queue's workerpool, the priority is determined by the first unprocessed opcode in the job. If an opcode is deferred, the job will go back to the "queued" status, even though it's just waiting to try to acquire its locks again later. If an opcode can not be processed after a certain number of retries or a certain amount of time, it should increase its priority. This will avoid starvation. A job's priority can never go below -20. If a job hits priority -20, it must acquire its locks in blocking mode. Opcode priorities are synchronised to disk in order to be restored after a restart or crash of the master daemon. Priorities also need to be considered inside the locking library to ensure opcodes with higher priorities get locks first. See :ref:`locking priorities ` for more details. Worker pool +++++++++++ To support job priorities in the job queue, the worker pool underlying the job queue must be enhanced to support task priorities. Currently tasks are processed in the order they are added to the queue (but, due to their nature, they don't necessarily finish in that order). All tasks are equal. To support tasks with higher or lower priority, a few changes have to be made to the queue inside a worker pool. Each task is assigned a priority when added to the queue. This priority can not be changed until the task is executed (this is fine as in all current use-cases, tasks are added to a pool and then forgotten about until they're done). A task's priority can be compared to Unix' process priorities. The lower the priority number, the closer to the queue's front it is. A task with priority 0 is going to be run before one with priority 10. Tasks with the same priority are executed in the order in which they were added. While a task is running it can query its own priority. If it's not ready yet for finishing, it can raise an exception to defer itself, optionally changing its own priority. This is useful for the following cases: - A task is trying to acquire locks, but those locks are still held by other tasks. By deferring itself, the task gives others a chance to run. This is especially useful when all workers are busy. - If a task decides it hasn't gotten its locks in a long time, it can start to increase its own priority. - Tasks waiting for long-running operations running asynchronously could defer themselves while waiting for a long-running operation. With these changes, the job queue will be able to implement per-job priorities. .. _locking-priorities: Locking +++++++ In order to support priorities in Ganeti's own lock classes, ``locking.SharedLock`` and ``locking.LockSet``, the internal structure of the former class needs to be changed. The last major change in this area was done for Ganeti 2.1 and can be found in the respective :doc:`design document `. The plain list (``[]``) used as a queue is replaced by a heap queue, similar to the `worker pool`_. The heap or priority queue does automatic sorting, thereby automatically taking care of priorities. For each priority there's a plain list with pending acquires, like the single queue of pending acquires before this change. When the lock is released, the code locates the list of pending acquires for the highest priority waiting. The first condition (index 0) is notified. Once all waiting threads received the notification, the condition is removed from the list. If the list of conditions is empty it's removed from the heap queue. Like before, shared acquires are grouped and skip ahead of exclusive acquires if there's already an existing shared acquire for a priority. To accomplish this, a separate dictionary of shared acquires per priority is maintained. To simplify the code and reduce memory consumption, the concept of the "active" and "inactive" condition for shared acquires is abolished. The lock can't predict what priorities the next acquires will use and even keeping a cache can become computationally expensive for arguable benefit (the underlying POSIX pipe, see ``pipe(2)``, needs to be re-created for each notification anyway). The following diagram shows a possible state of the internal queue from a high-level view. Conditions are shown as (waiting) threads. Assuming no modifications are made to the queue (e.g. more acquires or timeouts), the lock would be acquired by the threads in this order (concurrent acquires in parentheses): ``threadE1``, ``threadE2``, (``threadS1``, ``threadS2``, ``threadS3``), (``threadS4``, ``threadS5``), ``threadE3``, ``threadS6``, ``threadE4``, ``threadE5``. :: [ (0, [exc/threadE1, exc/threadE2, shr/threadS1/threadS2/threadS3]), (2, [shr/threadS4/threadS5]), (10, [exc/threadE3]), (33, [shr/threadS6, exc/threadE4, exc/threadE5]), ] IPv6 support ------------ Currently Ganeti does not support IPv6. This is true for nodes as well as instances. Due to the fact that IPv4 exhaustion is threateningly near the need of using IPv6 is increasing, especially given that bigger and bigger clusters are supported. Supported IPv6 setup ~~~~~~~~~~~~~~~~~~~~ In Ganeti 2.3 we introduce additionally to the ordinary pure IPv4 setup a hybrid IPv6/IPv4 mode. The latter works as follows: - all nodes in a cluster have a primary IPv6 address - the master has a IPv6 address - all nodes **must** have a secondary IPv4 address The reason for this hybrid setup is that key components that Ganeti depends on do not or only partially support IPv6. More precisely, Xen does not support instance migration via IPv6 in version 3.4 and 4.0. Similarly, KVM does not support instance migration nor VNC access for IPv6 at the time of this writing. This led to the decision of not supporting pure IPv6 Ganeti clusters, as very important cluster operations would not have been possible. Using IPv4 as secondary address does not affect any of the goals of the IPv6 support: since secondary addresses do not need to be publicly accessible, they need not be globally unique. In other words, one can practically use private IPv4 secondary addresses just for intra-cluster communication without propagating them across layer 3 boundaries. netutils: Utilities for handling common network tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently common utility functions are kept in the ``utils`` module. Since this module grows bigger and bigger network-related functions are moved to a separate module named *netutils*. Additionally all these utilities will be IPv6-enabled. Cluster initialization ~~~~~~~~~~~~~~~~~~~~~~ As mentioned above there will be two different setups in terms of IP addressing: pure IPv4 and hybrid IPv6/IPv4 address. To choose that a new cluster init parameter *--primary-ip-version* is introduced. This is needed as a given name can resolve to both an IPv4 and IPv6 address on a dual-stack host effectively making it impossible to infer that bit. Once a cluster is initialized and the primary IP version chosen all nodes that join have to conform to that setup. In the case of our IPv6/IPv4 setup all nodes *must* have a secondary IPv4 address. Furthermore we store the primary IP version in ssconf which is consulted every time a daemon starts to determine the default bind address (either *0.0.0.0* or *::*. In a IPv6/IPv4 setup we need to bind the Ganeti daemon listening on network sockets to the IPv6 address. Node addition ~~~~~~~~~~~~~ When adding a new node to a IPv6/IPv4 cluster it must have a IPv6 address to be used as primary and a IPv4 address used as secondary. As explained above, every time a daemon is started we use the cluster primary IP version to determine to which any address to bind to. The only exception to this is when a node is added to the cluster. In this case there is no ssconf available when noded is started and therefore the correct address needs to be passed to it. Name resolution ~~~~~~~~~~~~~~~ Since the gethostbyname*() functions do not support IPv6 name resolution will be done by using the recommended getaddrinfo(). IPv4-only components ~~~~~~~~~~~~~~~~~~~~ ============================ =================== ==================== Component IPv6 Status Planned Version ============================ =================== ==================== Xen instance migration Not supported Xen 4.1: libxenlight KVM instance migration Not supported Unknown KVM VNC access Not supported Unknown ============================ =================== ==================== Privilege Separation -------------------- Current state and shortcomings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In Ganeti 2.2 we introduced privilege separation for the RAPI daemon. This was done directly in the daemon's code in the process of daemonizing itself. Doing so leads to several potential issues. For example, a file could be opened while the code is still running as ``root`` and for some reason not be closed again. Even after changing the user ID, the file descriptor can be written to. Implementation ~~~~~~~~~~~~~~ To address these shortcomings, daemons will be started under the target user right away. The ``start-stop-daemon`` utility used to start daemons supports the ``--chuid`` option to change user and group ID before starting the executable. The intermediate solution for the RAPI daemon from Ganeti 2.2 will be removed again. Files written by the daemons may need to have an explicit owner and group set (easily done through ``utils.WriteFile``). All SSH-related code is removed from the ``ganeti.bootstrap`` module and core components and moved to a separate script. The core code will simply assume a working SSH setup to be in place. Security Domains ~~~~~~~~~~~~~~~~ In order to separate the permissions of file sets we separate them into the following 3 overall security domain chunks: 1. Public: ``0755`` respectively ``0644`` 2. Ganeti wide: shared between the daemons (gntdaemons) 3. Secret files: shared among a specific set of daemons/users So for point 3 this tables shows the correlation of the sets to groups and their users: === ========== ============================== ========================== Set Group Users Description === ========== ============================== ========================== A gntrapi gntrapi, gntmasterd Share data between gntrapi and gntmasterd B gntadmins gntrapi, gntmasterd, *users* Shared between users who needs to call gntmasterd C gntconfd gntconfd, gntmasterd Share data between gntconfd and gntmasterd D gntmasterd gntmasterd masterd only; Currently only to redistribute the configuration, has access to all files under ``lib/ganeti`` E gntdaemons gntmasterd, gntrapi, gntconfd Shared between the various Ganeti daemons to exchange data === ========== ============================== ========================== Restricted commands ~~~~~~~~~~~~~~~~~~~ The following commands still require root permissions to fulfill their functions: :: gnt-cluster {init|destroy|command|copyfile|rename|masterfailover|renew-crypto} gnt-node {add|remove} gnt-instance {console} Directory structure and permissions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Here's how we propose to change the filesystem hierarchy and their permissions. Assuming it follows the defaults: ``gnt${daemon}`` for user and the groups from the section `Security Domains`_:: ${localstatedir}/lib/ganeti/ (0755; gntmasterd:gntmasterd) cluster-domain-secret (0600; gntmasterd:gntmasterd) config.data (0640; gntmasterd:gntconfd) hmac.key (0440; gntmasterd:gntconfd) known_host (0644; gntmasterd:gntmasterd) queue/ (0700; gntmasterd:gntmasterd) archive/ (0700; gntmasterd:gntmasterd) * (0600; gntmasterd:gntmasterd) * (0600; gntmasterd:gntmasterd) rapi.pem (0440; gntrapi:gntrapi) rapi_users (0640; gntrapi:gntrapi) server.pem (0440; gntmasterd:gntmasterd) ssconf_* (0444; root:gntmasterd) uidpool/ (0750; root:gntmasterd) watcher.data (0600; root:gntmasterd) ${localstatedir}/run/ganeti/ (0770; gntmasterd:gntdaemons) socket/ (0750; gntmasterd:gntadmins) ganeti-master (0770; gntmasterd:gntadmins) ${localstatedir}/log/ganeti/ (0770; gntmasterd:gntdaemons) master-daemon.log (0600; gntmasterd:gntdaemons) rapi-daemon.log (0600; gntrapi:gntdaemons) conf-daemon.log (0600; gntconfd:gntdaemons) node-daemon.log (0600; gntnoded:gntdaemons) Feature changes =============== External interface changes ========================== .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-2.4.rst000064400000000000000000000003771476477700300166640ustar00rootroot00000000000000================= Ganeti 2.4 design ================= The following design documents have been implemented in Ganeti 2.4: - :doc:`design-oob` - :doc:`design-query2` .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-2.5.rst000064400000000000000000000005161476477700300166600ustar00rootroot00000000000000================= Ganeti 2.5 design ================= The following design documents have been implemented in Ganeti 2.5: - :doc:`design-lu-generated-jobs` - :doc:`design-chained-jobs` - :doc:`design-multi-reloc` - :doc:`design-shared-storage` .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-2.6.rst000064400000000000000000000004531476477700300166610ustar00rootroot00000000000000================= Ganeti 2.6 design ================= The following design documents have been implemented in Ganeti 2.6: - :doc:`design-cpu-pinning` - :doc:`design-ovf-support` - :doc:`design-resource-model` .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-2.7.rst000064400000000000000000000015251476477700300166630ustar00rootroot00000000000000================= Ganeti 2.7 design ================= The following design documents have been implemented in Ganeti 2.7: - :doc:`design-bulk-create` - :doc:`design-opportunistic-locking` - :doc:`design-restricted-commands` - :doc:`design-node-add` - :doc:`design-virtual-clusters` - :doc:`design-network` - :doc:`design-linuxha` - :doc:`design-shared-storage` (Updated to reflect the new ExtStorage Interface) The following designs have been partially implemented in Ganeti 2.7: - :doc:`design-network` - :doc:`design-query-splitting`: only queries not needing RPC are supported, through confd - :doc:`design-partitioned`: only exclusive use of disks is implemented - :doc:`design-monitoring-agent`: an example standalone DRBD data collector is included .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-2.8.rst000064400000000000000000000013631476477700300166640ustar00rootroot00000000000000================= Ganeti 2.8 design ================= The following design documents have been implemented in Ganeti 2.8: - :doc:`design-reason-trail` - :doc:`design-autorepair` - :doc:`design-device-uuid-name` The following designs have been partially implemented in Ganeti 2.8: - :doc:`design-storagetypes` - :doc:`design-hroller` - :doc:`design-query-splitting`: everything except instance queries. - :doc:`design-partitioned`: "Constrained instance sizes" implemented. - :doc:`design-monitoring-agent`: implementation of all the core functionalities of the monitoring agent. Reason trail implemented as part of the work for the instance status collector. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-2.9.rst000064400000000000000000000006231476477700300166630ustar00rootroot00000000000000================= Ganeti 2.9 design ================= The following design documents have been implemented in Ganeti 2.9. - :doc:`design-hroller` - :doc:`design-partitioned` - :doc:`design-monitoring-agent` - :doc:`design-reason-trail` - :doc:`design-query-splitting` - :doc:`design-device-uuid-name` The following designs have been partially implemented in Ganeti 2.9. - :doc:`design-storagetypes` ganeti-3.1.0~rc2/doc/design-3.0.rst000064400000000000000000000032231476477700300166520ustar00rootroot00000000000000================= Ganeti 3.0 design ================= This document describes the major changes between Ganeti 2.16 and 3.0. Objective ========= All Ganeti versions prior to 3.0 are built on Python 2 code. With Python 3 being the offical default and version 2 slowly fading out of Linux distributions, the goal of this major release is to achieve full and only Python 3 compatibility. Minimum supported Python version ================================ We require a minimum of Python 3.6 for the current code to work. This allows us to support stable distributions like Debian Buster or Ubuntu 18.04 LTS and to create working upgrade paths for our users. What happened to Ganeti 2.17? ============================= A beta version of 2.17 has been released on February 22nd, 2016. This version has not seen exhaustive testing but included a number of new features. To push forward with the migration to Python 3, we decided to ditch these changes and start over based on Ganeti 2.16. Since the upgrade to 2.17.0~beta1 introduced some breaking changes, there is currently no direct upgrade path between 2.17 and 3.0. The only known but untested way is to downgrade to Ganeti 2.16 first and then upgrade to 3.0. This will remove access to incomplete features like the Ganeti Maintenance Daemon. What other features/changes have been introduced with 3.0? ========================================================== Since 2.16, a number of bugfixes, compatibility improvements and updates to documentation have been included. Please refer to the release notes for more details. However, no changes to the on-disk data and no breaking changes to commandline tools have been included.ganeti-3.1.0~rc2/doc/design-3.1.rst000064400000000000000000000002171476477700300166530ustar00rootroot00000000000000================= Ganeti 3.1 design ================= The following designs have been implemented in Ganeti 3.1 - :doc:`design-qemu-blockdev`ganeti-3.1.0~rc2/doc/design-allocation-efficiency.rst000064400000000000000000000056601476477700300226100ustar00rootroot00000000000000========================================================================= Improving allocation efficiency by considering the total reserved memory ========================================================================= :Created: 2015-Mar-19 :Status: Implemented :Ganeti-Version: 2.15.0 This document describes a change to the cluster metric to enhance the allocation efficiency of Ganeti's ``htools``. .. contents:: :depth: 4 Current state and shortcomings ============================== Ganeti's ``htools``, which typically make all allocation and balancing decisions, greedily try to improve the cluster metric. So it is important that the cluster metric faithfully reflects the objectives of these operations. Currently the cluster metric is composed of counting violations (instances on offline nodes, nodes that are not N+1 redundant, etc) and the sum of standard deviations of relative resource usage of the individual nodes. The latter component is to ensure that all nodes equally bear the load of the instances. This is reasonable for resources where the total usage is independent of its distribution, as it is the case for CPU, disk, and total RAM. It is, however, not true for reserved memory. By distributing its secondaries more widespread over the cluster, a node can reduce its reserved memory without increasing it on other nodes. Not taking this aspect into account has lead to quite inefficient allocation of instances on the cluster (see example below). Proposed changes ================ A new additive component is added to the cluster metric. It is the sum over all nodes of the fraction of reserved memory. This way, moves and allocations that reduce the amount of memory reserved to ensure N+1 redundancy are favored. Note that this component does not have the scaling of standard deviations of fractions, but, instead counts nodes reserved for N+1 redundancy. In an ideal allocation, this will not exceed 1. But bad allocations will violate this property. As waste of reserved memory is a more future-oriented problem than, e.g., current N+1 violations, we give the new component a relatively small weight of 0.25, so that counting current violations still dominate. Another consequence of this metric change is that the value 0 is no longer obtainable: as soon as we have DRBD instance, we have to reserve memory. However, in most cases only differences of scores influence decisions made. In the few cases, were absolute values of the cluster score are specified, they are interpreted as relative to the theoretical minimum of the reserved memory score. Example ======= Consider the capacity of an empty cluster of 6 nodes, each capable of holding 10 instances; this can be measured, e.g., by ``hspace --simulate=p,6,204801,10241,21 --disk-template=drbd --standard-alloc=10240,1024,2``. Without the metric change 34 standard instances are allocated. With the metric change, 48 standard instances are allocated. This is a 41% increase in utilization. ganeti-3.1.0~rc2/doc/design-autorepair.rst000064400000000000000000000374661476477700300205450ustar00rootroot00000000000000==================== Instance auto-repair ==================== :Created: 2012-Sep-03 :Status: Implemented :Ganeti-Version: 2.8.0 .. contents:: :depth: 4 This is a design document detailing the implementation of self-repair and recreation of instances in Ganeti. It also discusses ideas that might be useful for more future self-repair situations. Current state and shortcomings ============================== Ganeti currently doesn't do any sort of self-repair or self-recreate of instances: - If a drbd instance is broken (its primary of secondary nodes go offline or need to be drained) an admin or an external tool must fail it over if necessary, and then trigger a disk replacement. - If a plain instance is broken (or both nodes of a drbd instance are) an admin or an external tool must recreate its disk and reinstall it. Moreover in an oversubscribed cluster operations mentioned above might fail for lack of capacity until a node is repaired or a new one added. In this case an external tool would also need to go through any "pending-recreate" or "pending-repair" instances and fix them. Proposed changes ================ We'd like to increase the self-repair capabilities of Ganeti, at least with regards to instances. In order to do so we plan to add mechanisms to mark an instance as "due for being repaired" and then the relevant repair to be performed as soon as it's possible, on the cluster. The self repair will be written as part of ganeti-watcher or as an extra watcher component that is called less often. As the first version we'll only handle the case in which an instance lives on an offline or drained node. In the future we may add more self-repair capabilities for errors ganeti can detect. New attributes (or tags) ------------------------ In order to know when to perform a self-repair operation we need to know whether they are allowed by the cluster administrator. This can be implemented as either new attributes or tags. Tags could be acceptable as they would only be read and interpreted by the self-repair tool (part of the watcher), and not by the ganeti core opcodes and node rpcs. The following tags would be needed: ganeti:watcher:autorepair: ++++++++++++++++++++++++++++++++ (instance/nodegroup/cluster) Allow repairs to happen on an instance that has the tag, or that lives in a cluster or nodegroup which does. Types of repair are in order of perceived risk, lower to higher, and each type includes allowing the operations in the lower ones: - ``fix-storage`` allows a disk replacement or another operation that fixes the instance backend storage without affecting the instance itself. This can for example recover from a broken drbd secondary, but risks data loss if something is wrong on the primary but the secondary was somehow recoverable. - ``migrate`` allows an instance migration. This can recover from a drained primary, but can cause an instance crash in some cases (bugs). - ``failover`` allows instance reboot on the secondary. This can recover from an offline primary, but the instance will lose its running state. - ``reinstall`` allows disks to be recreated and an instance to be reinstalled. This can recover from primary&secondary both being offline, or from an offline primary in the case of non-redundant instances. It causes data loss. Each repair type allows all the operations in the previous types, in the order above, in order to ensure a repair can be completed fully. As such a repair of a lower type might not be able to proceed if it detects an error condition that requires a more risky or drastic solution, but never vice versa (if a worse solution is allowed then so is a better one). If there are multiple ``ganeti:watcher:autorepair:`` tags in an object (cluster, node group or instance), the least destructive tag takes precedence. When multiplicity happens across objects, the nearest tag wins. For example, if in a cluster with two instances, *I1* and *I2*, *I1* has ``failover``, and the cluster itself has both ``fix-storage`` and ``reinstall``, *I1* will end up with ``failover`` and *I2* with ``fix-storage``. ganeti:watcher:autorepair:suspend[:] +++++++++++++++++++++++++++++++++++++++++++++++ (instance/nodegroup/cluster) If this tag is encountered no autorepair operations will start for the instance (or for any instance, if present at the cluster or group level). Any job which already started will be allowed to finish, but then the autorepair system will not proceed further until this tag is removed, or the timestamp passes (in which case the tag will be removed automatically by the watcher). Note that depending on how this tag is used there might still be race conditions related to it for an external tool that uses it programmatically, as no "lock tag" or tag "test-and-set" operation is present at this time. While this is known we won't solve these race conditions in the first version. It might also be useful to easily have an operation that tags all instances matching a filter on some characteristic. But again, this wouldn't be specific to this tag. If there are multiple ``ganeti:watcher:autorepair:suspend[:]`` tags in an object, the form without timestamp takes precedence (permanent suspension); or, if all object tags have a timestamp, the one with the highest timestamp. When multiplicity happens across objects, the nearest tag wins, as above. This makes it possible to suspend cluster-enabled repairs with a single tag in the cluster object; or to suspend them only for a certain node group or instance. At the same time, it is possible to re-enable cluster-suspended repairs in a particular instance or group by applying an enable tag to them. ganeti:watcher:autorepair:pending:::: ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ (instance) If this tag is present a repair of type ``type`` is pending on the target instance. This means that either jobs are being run, or it's waiting for resource availability. ``id`` is the unique id identifying this repair, ``timestamp`` is the time when this tag was first applied to this instance for this ``id`` (we will "update" the tag by adding a "new copy" of it and removing the old version as we run more jobs, but the timestamp will never change for the same repair) ``jobs`` is the list of jobs already run or being run to repair the instance (separated by a plus sign, *+*). If the instance has just been put in pending state but no job has run yet, this list is empty. This tag will be set by ganeti if an equivalent autorepair tag is present and a a repair is needed, or can be set by an external tool to request a repair as a "once off". If multiple instances of this tag are present they will be handled in order of timestamp. ganeti:watcher:autorepair:result::::: ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ (instance) If this tag is present a repair of type ``type`` has been performed on the instance and has been completed by ``timestamp``. The result is either ``success``, ``failure`` or ``enoperm``, and jobs is a *+*-separated list of jobs that were executed for this repair. An ``enoperm`` result is returned when the repair was brought on until possible, but the repair type doesn't consent to proceed further. Possible states, and transitions -------------------------------- At any point an instance can be in one of the following health states: Healthy +++++++ The instance lives on only online nodes. The autorepair system will never touch these instances. Any ``repair:pending`` tags will be removed and marked ``success`` with no jobs attached to them. This state can transition to: - Needs-repair, repair disallowed (node offlined or drained, no autorepair tag) - Needs-repair, autorepair allowed (node offlined or drained, autorepair tag present) - Suspended (a suspend tag is added) Suspended +++++++++ Whenever a ``repair:suspend`` tag is added the autorepair code won't touch the instance until the timestamp on the tag has passed, if present. The tag will be removed afterwards (and the instance will transition to its correct state, depending on its health and other tags). Note that when an instance is suspended any pending repair is interrupted, but jobs which were submitted before the suspension are allowed to finish. Needs-repair, repair disallowed +++++++++++++++++++++++++++++++ The instance lives on an offline or drained node, but no autorepair tag is set, or the autorepair tag set is of a type not powerful enough to finish the repair. The autorepair system will never touch these instances, and they can transition to: - Healthy (manual repair) - Pending repair (a ``repair:pending`` tag is added) - Needs-repair, repair allowed always (an autorepair always tag is added) - Suspended (a suspend tag is added) Needs-repair, repair allowed always +++++++++++++++++++++++++++++++++++ A ``repair:pending`` tag is added, and the instance transitions to the Pending Repair state. The autorepair tag is preserved. Of course if a ``repair:suspended`` tag is found no pending tag will be added, and the instance will instead transition to the Suspended state. Pending repair ++++++++++++++ When an instance is in this stage the following will happen: If a ``repair:suspended`` tag is found the instance won't be touched and moved to the Suspended state. Any jobs which were already running will be left untouched. If there are still jobs running related to the instance and scheduled by this repair they will be given more time to run, and the instance will be checked again later. The state transitions to itself. If no jobs are running and the instance is detected to be healthy, the ``repair:result`` tag will be added, and the current active ``repair:pending`` tag will be removed. It will then transition to the Healthy state if there are no ``repair:pending`` tags, or to the Pending state otherwise: there, the instance being healthy, those tags will be resolved without any operation as well (note that this is the same as transitioning to the Healthy state, where ``repair:pending`` tags would also be resolved). If no jobs are running and the instance still has issues: - if the last job(s) failed it can either be retried a few times, if deemed to be safe, or the repair can transition to the Failed state. The ``repair:result`` tag will be added, and the active ``repair:pending`` tag will be removed (further ``repair:pending`` tags will not be able to proceed, as explained by the Failed state, until the failure state is cleared) - if the last job(s) succeeded but there are not enough resources to proceed, the state will transition to itself and no jobs are scheduled. The tag is left untouched (and later checked again). This basically just delays any repairs, the current ``pending`` tag stays active, and any others are untouched). - if the last job(s) succeeded but the repair type cannot allow to proceed any further the ``repair:result`` tag is added with an ``enoperm`` result, and the current ``repair:pending`` tag is removed. The instance is now back to "Needs-repair, repair disallowed", "Needs-repair, autorepair allowed", or "Pending" if there is already a future tag that can repair the instance. - if the last job(s) succeeded and the repair can continue new job(s) can be submitted, and the ``repair:pending`` tag can be updated. Failed ++++++ If repairing an instance has failed a ``repair:result:failure`` is added. The presence of this tag is used to detect that an instance is in this state, and it will not be touched until the failure is investigated and the tag is removed. An external tool or person needs to investigate the state of the instance and remove this tag when he is sure the instance is repaired and safe to turn back to the normal autorepair system. (Alternatively we can use the suspended state (indefinitely or temporarily) to mark the instance as "not touch" when we think a human needs to look at it. To be decided). A graph with the possible transitions follows; note that in the graph, following the implementation, the two ``Needs repair`` states have been coalesced into one; and the ``Suspended`` state disappears, for it becomes an attribute of the instance object (its auto-repair policy). .. digraph:: "auto-repair-states" node [shape=circle, style=filled, fillcolor="#BEDEF1", width=2, fixedsize=true]; healthy [label="Healthy"]; needsrep [label="Needs repair"]; pendrep [label="Pending repair"]; failed [label="Failed repair"]; disabled [label="(no state)", width=1.25]; {rank=same; needsrep} {rank=same; healthy} {rank=same; pendrep} {rank=same; failed} {rank=same; disabled} // These nodes are needed to be the "origin" of the "initial state" arrows. node [width=.5, label="", style=invis]; inih; inin; inip; inif; inix; edge [fontsize=10, fontname="Arial Bold", fontcolor=blue] inih -> healthy [label="No tags or\nresult:success"]; inip -> pendrep [label="Tag:\nautorepair:pending"]; inif -> failed [label="Tag:\nresult:failure"]; inix -> disabled [fontcolor=black, label="ArNotEnabled"]; edge [fontcolor="orange"]; healthy -> healthy [label="No problems\ndetected"]; healthy -> needsrep [ label="Brokeness\ndetected in\nfirst half of\nthe tool run"]; pendrep -> healthy [ label="All jobs\ncompleted\nsuccessfully /\ninstance healthy"]; pendrep -> failed [label="Some job(s)\nfailed"]; edge [fontcolor="red"]; needsrep -> pendrep [ label="Repair\nallowed and\ninitial job(s)\nsubmitted"]; needsrep -> needsrep [ label="Repairs suspended\n(no-op) or enabled\nbut not powerful enough\n(result: enoperm)"]; pendrep -> pendrep [label="More jobs\nsubmitted"]; Repair operation ---------------- Possible repairs are: - Replace-disks (drbd, if the secondary is down), (or other storage specific fixes) - Migrate (shared storage, rbd, drbd, if the primary is drained) - Failover (shared storage, rbd, drbd, if the primary is down) - Recreate disks + reinstall (all nodes down, plain, files or drbd) Note that more than one of these operations may need to happen before a full repair is completed (eg. if a drbd primary goes offline first a failover will happen, then a replace-disks). The self-repair tool will first take care of all needs-repair instance that can be brought into ``pending`` state, and transition them as described above. Then it will go through any ``repair:pending`` instances and handle them as described above. Note that the repair tool MAY "group" instances by performing common repair jobs for them (eg: node evacuate). Staging of work --------------- First version: recreate-disks + reinstall (2.6.1) Second version: failover and migrate repairs (2.7) Third version: replace disks repair (2.7 or 2.8) Future work =========== One important piece of work will be reporting what the autorepair system is "thinking" and exporting this in a form that can be read by an outside user or system. In order to do this we need a better communication system than embedding this information into tags. This should be thought in an extensible way that can be used in general for Ganeti to provide "advisory" information about entities it manages, and for an external system to "advise" ganeti over what it can do, but in a less direct manner than submitting individual jobs. Note that cluster verify checks some errors that are actually instance specific, (eg. a missing backend disk on a drbd node) or node-specific (eg. an extra lvm device). If we were to split these into "instance verify", "node verify" and "cluster verify", then we could easily use this tool to perform some of those repairs as well. Finally self-repairs could also be extended to the cluster level, for example concepts like "N+1 failures", missing master candidates, etc. or node level for some specific types of errors. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-bulk-create.rst000064400000000000000000000064631476477700300205610ustar00rootroot00000000000000================== Ganeti Bulk Create ================== :Created: 2012-Sep-03 :Status: Implemented :Ganeti-Version: 2.7.0 .. contents:: :depth: 4 .. highlight:: python Current state and shortcomings ============================== Creation of instances happens a lot. A fair load is done by just creating instances and due to bad allocation shifting them around later again. Additionally, if you turn up a new cluster you already know a bunch of instances, which need to exists on the cluster. Doing this one-by-one is not only cumbersome but might also fail, due to lack of resources or lead to badly balanced clusters. Since the early Ganeti 2.0 alpha version there is a ``gnt-instance batch-create`` command to allocate a bunch of instances based on a json file. This feature, however, doesn't take any advantages of iallocator and submits jobs in a serialized manner. Proposed changes ---------------- To overcome this shortcoming we would extend the current iallocator interface to allow bulk requests. On the Ganeti side, a new opcode is introduced to handle the bulk creation and returning the resulting placement from the IAllocator_. Problems -------- Due to the design of chained jobs, we can guarantee, that with the state at which the ``multi-alloc`` opcode is run, all of the instances will fit (or all won't). But we can't guarantee that once the instance creation requests were submit, no other jobs have sneaked in between. This might still lead to failing jobs because the resources have changed in the meantime. Implementation ============== IAllocator ---------- A new additional ``type`` will be added called ``multi-allocate`` to distinguish between normal and bulk operation. For the bulk operation the ``request`` will be a finite list of request dicts. If ``multi-allocate`` is declared, ``request`` must exist and is a list of ``request`` dicts as described in :doc:`Operation specific input `. The ``result`` then is a list of instance name and node placements in the order of the ``request`` field. In addition, the old ``allocate`` request type will be deprecated and at latest in Ganeti 2.8 incorporated into this new request. Current code will need slight adaption to work with the new request. This needs careful testing. OpInstanceBulkAdd ----------------- We add a new opcode ``OpInstanceBulkAdd``. It receives a list of ``OpInstanceCreate`` on the ``instances`` field. This is done to make sure, that these two loosely coupled opcodes do not get out of sync. On the RAPI side, however, this just is a list of instance create definitions. And the client is adapted accordingly. The opcode itself does some sanity checks on the instance creation opcodes which includes: * ``mode`` is not set * ``pnode`` and ``snodes`` is not set * ``iallocator`` is not set Any of the above error will be aborted with ``OpPrereqError``. Once the list has been verified it is handed to the ``iallocator`` as described in IAllocator_. Upon success we then return the result of the IAllocator_ call. At this point the current instance allocation would work with the resources available on the cluster as perceived upon ``OpInstanceBulkAdd`` invocation. However, there might be corner cases where this is not true as described in Problems_. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-ceph-ganeti-support.rst000064400000000000000000000147271476477700300222630ustar00rootroot00000000000000============================ RADOS/Ceph support in Ganeti ============================ :Created: 2013-Jul-26 :Status: Partially Implemented :Ganeti-Version: 2.10.0 .. contents:: :depth: 4 Objective ========= The project aims to improve Ceph RBD support in Ganeti. It can be primarily divided into following tasks. - Use Qemu/KVM RBD driver to provide instances with direct RBD support. [implemented as of Ganeti 2.10] - Allow Ceph RBDs' configuration through Ganeti. [unimplemented] - Write a data collector to monitor Ceph nodes. [unimplemented] Background ========== Ceph RBD -------- Ceph is a distributed storage system which provides data access as files, objects and blocks. As part of this project, we're interested in integrating ceph's block device (RBD) directly with Qemu/KVM. Primary components/daemons of Ceph. - Monitor - Serve as authentication point for clients. - Metadata - Store all the filesystem metadata (Not configured here as they are not required for RBD) - OSD - Object storage devices. One daemon for each drive/location. RBD support in Ganeti --------------------- Currently, Ganeti supports RBD volumes on a pre-configured Ceph cluster. This is enabled through RBD disk templates. These templates allow RBD volume's access through RBD Linux driver. The volumes are mapped to host as local block devices which are then attached to the instances. This method incurs an additional overhead. We plan to resolve it by using Qemu's RBD driver to enable direct access to RBD volumes for KVM instances. Also, Ganeti currently uses RBD volumes on a pre-configured ceph cluster. Allowing configuration of ceph nodes through Ganeti will be a good addition to its prime features. Qemu/KVM Direct RBD Integration =============================== A new disk param ``access`` is introduced. It's added at cluster/node-group level to simplify prototype implementation. It will specify the access method either as ``userspace`` or ``kernelspace``. It's accessible to StartInstance() in hv_kvm.py. The device path, ``rbd:/``, is generated by RADOSBlockDevice and is added to the params dictionary as ``kvm_dev_path``. This approach ensures that no disk template specific changes are required in hv_kvm.py allowing easy integration of other distributed storage systems (like Gluster). Note that the RBD volume is mapped as a local block device as before. The local mapping won't be used during instance operation in the ``userspace`` access mode, but can be used by administrators and OS scripts. Updated commands ---------------- :: $ gnt-instance info ``access:userspace/kernelspace`` will be added to Disks category. This output applies to KVM based instances only. Ceph configuration on Ganeti nodes ================================== This document proposes configuration of distributed storage pool (Ceph or Gluster) through ganeti. Currently, this design document focuses on configuring a Ceph cluster. A prerequisite of this setup would be installation of ceph packages on all the concerned nodes. At Ganeti Cluster init, the user will set distributed-storage specific options which will be stored at cluster level. The Storage cluster will be initialized using ``gnt-storage``. For the prototype, only a single storage pool/node-group is configured. Following steps take place when a node-group is initialized as a storage cluster. - Check for an existing ceph cluster through /etc/ceph/ceph.conf file on each node. - Fetch cluster configuration parameters and create a distributed storage object accordingly. - Issue an 'init distributed storage' RPC to group nodes (if any). - On each node, ``ceph`` cli tool will run appropriate services. - Mark nodes as well as the node-group as distributed-storage-enabled. The storage cluster will operate at a node-group level. The ceph cluster will be initiated using gnt-storage. A new sub-command ``init-distributed-storage`` will be added to it. The configuration of the nodes will be handled through an init function called by the node daemons running on the respective nodes. A new RPC is introduced to handle the calls. A new object will be created to send the storage parameters to the node - storage_type, devices, node_role (mon/osd) etc. A new node can be directly assigned to the storage enabled node-group. During the 'gnt-node add' process, required ceph daemons will be started and node will be added to the ceph cluster. Only an offline node can be assigned to storage enabled node-group. ``gnt-node --readd`` needs to be performed to issue RPCs for spawning appropriate services on the newly assigned node. Updated Commands ---------------- Following are the affected commands.:: $ gnt-cluster init -S ceph:disk=/dev/sdb,option=value... During cluster initialization, ceph specific options are provided which apply at cluster-level.:: $ gnt-cluster modify -S ceph:option=value2... For now, cluster modification will be allowed when there is no initialized storage cluster.:: $ gnt-storage init-distributed-storage -s{--storage-type} ceph \ Ensure that no other node-group is configured as distributed storage cluster and configure ceph on the specified node-group. If there is no node in the node-group, it'll only be marked as distributed storage enabled and no action will be taken.:: $ gnt-group assign-nodes It ensures that the node is offline if the node-group specified is distributed storage capable. Ceph configuration on the newly assigned node is not performed at this step.:: $ gnt-node --offline If the node is part of storage node-group, an offline call will stop/remove ceph daemons.:: $ gnt-node add --readd If the node is now part of the storage node-group, issue init distributed storage RPC to the respective node. This step is required after assigning a node to the storage enabled node-group:: $ gnt-node remove A warning will be issued stating that the node is part of distributed storage, mark it offline before removal. Data collector for Ceph ----------------------- TBD Future Work ----------- Due to the loopback bug in ceph, one may run into daemon hang issues while performing writes to a RBD volumes through block device mapping. This bug is applicable only when the RBD volume is stored on the OSD running on the local node. In order to mitigate this issue, we can create storage pools on different nodegroups and access RBD volumes on different pools. http://tracker.ceph.com/issues/3076 .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-chained-jobs.rst000064400000000000000000000204261476477700300207040ustar00rootroot00000000000000============ Chained jobs ============ :Created: 2011-Jul-14 :Status: Implemented :Ganeti-Version: 2.5.0 .. contents:: :depth: 4 This is a design document about the innards of Ganeti's job processing. Readers are advised to study previous design documents on the topic: - :ref:`Original job queue ` - :ref:`Job priorities ` - :doc:`LU-generated jobs ` Current state and shortcomings ============================== Ever since the introduction of the job queue with Ganeti 2.0 there have been situations where we wanted to run several jobs in a specific order. Due to the job queue's current design, such a guarantee can not be given. Jobs are run according to their priority, their ability to acquire all necessary locks and other factors. One way to work around this limitation is to do some kind of job grouping in the client code. Once all jobs of a group have finished, the next group is submitted and waited for. There are different kinds of clients for Ganeti, some of which don't share code (e.g. Python clients vs. htools). This design proposes a solution which would be implemented as part of the job queue in the master daemon. Proposed changes ================ With the implementation of :ref:`job priorities ` the processing code was re-architected and became a lot more versatile. It now returns jobs to the queue in case the locks for an opcode can't be acquired, allowing other jobs/opcodes to be run in the meantime. The proposal is to add a new, optional property to opcodes to define dependencies on other jobs. Job X could define opcodes with a dependency on the success of job Y and would only be run once job Y is finished. If there's a dependency on success and job Y failed, job X would fail as well. Since such dependencies would use job IDs, the jobs still need to be submitted in the right order. .. pyassert:: # Update description below if finalized job status change constants.JOBS_FINALIZED == frozenset([ constants.JOB_STATUS_CANCELED, constants.JOB_STATUS_SUCCESS, constants.JOB_STATUS_ERROR, ]) The new attribute's value would be a list of two-valued tuples. Each tuple contains a job ID and a list of requested status for the job depended upon. Only final status are accepted (:pyeval:`utils.CommaJoin(sorted(constants.JOBS_FINALIZED))`). An empty list is equivalent to specifying all final status (except :pyeval:`constants.JOB_STATUS_CANCELED`, which is treated specially). An opcode runs only once all its dependency requirements have been fulfilled. Any job referring to a cancelled job is also cancelled unless it explicitly lists :pyeval:`constants.JOB_STATUS_CANCELED` as a requested status. In case a referenced job can not be found in the normal queue or the archive, referring jobs fail as the status of the referenced job can't be determined. With this change, clients can submit all wanted jobs in the right order and proceed to wait for changes on all these jobs (see ``cli.JobExecutor``). The master daemon will take care of executing them in the right order, while still presenting the client with a simple interface. Clients using the ``SubmitManyJobs`` interface can use relative job IDs (negative integers) to refer to jobs in the same submission. .. highlight:: javascript Example data structures:: // First job { "job_id": "6151", "ops": [ { "OP_ID": "OP_INSTANCE_REPLACE_DISKS", /*...*/ }, { "OP_ID": "OP_INSTANCE_FAILOVER", /*...*/ }, ], } // Second job, runs in parallel with first job { "job_id": "7687", "ops": [ { "OP_ID": "OP_INSTANCE_MIGRATE", /*...*/ } ], } // Third job, depending on success of previous jobs { "job_id": "9218", "ops": [ { "OP_ID": "OP_NODE_SET_PARAMS", "depend": [ [6151, ["success"]], [7687, ["success"]], ], "offline": True, }, ], } Implementation details ---------------------- Status while waiting for dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Jobs waiting for dependencies are certainly not in the queue anymore and therefore need to change their status from "queued". While waiting for opcode locks the job is in the "waiting" status (the constant is named ``JOB_STATUS_WAITLOCK``, but the actual value is ``waiting``). There the following possibilities: #. Introduce a new status, e.g. "waitdeps". Pro: - Clients know for sure a job is waiting for dependencies, not locks Con: - Code and tests would have to be updated/extended for the new status - List of possible state transitions certainly wouldn't get simpler - Breaks backwards compatibility, older clients might get confused #. Use existing "waiting" status. Pro: - No client changes necessary, less code churn (note that there are clients which don't live in Ganeti core) - Clients don't need to know the difference between waiting for a job and waiting for a lock; it doesn't make a difference - Fewer state transitions (see commit ``5fd6b69479c0``, which removed many state transitions and disk writes) Con: - Not immediately visible what a job is waiting for, but it's the same issue with locks; this is the reason why the lock monitor (``gnt-debug locks``) was introduced; job dependencies can be shown as "locks" in the monitor Based on these arguments, the proposal is to do the following: - Rename ``JOB_STATUS_WAITLOCK`` constant to ``JOB_STATUS_WAITING`` to reflect its actual meaning: the job is waiting for something - While waiting for dependencies and locks, jobs are in the "waiting" status - Export dependency information in lock monitor; example output:: Name Mode Owner Pending job/27491 - - success:job/34709,job/21459 job/21459 - - success,error:job/14513 Cost of deserialization ~~~~~~~~~~~~~~~~~~~~~~~ To determine the status of a dependency job the job queue must have access to its data structure. Other queue operations already do this, e.g. archiving, watching a job's progress and querying jobs. Initially (Ganeti 2.0/2.1) the job queue shared the job objects in memory and protected them using locks. Ganeti 2.2 (see :doc:`design document `) changed the queue to read and deserialize jobs from disk. This significantly reduced locking and code complexity. Nowadays inotify is used to wait for changes on job files when watching a job's progress. Reading from disk and deserializing certainly has some cost associated with it, but it's a significantly simpler architecture than synchronizing in memory with locks. At the stage where dependencies are evaluated the queue lock is held in shared mode, so different workers can read at the same time (deliberately ignoring CPython's interpreter lock). It is expected that the majority of executed jobs won't use dependencies and therefore won't be affected. Other discussed solutions ========================= Job-level attribute ------------------- At a first look it might seem to be better to put dependencies on previous jobs at a job level. However, it turns out that having the option of defining only a single opcode in a job as having such a dependency can be useful as well. The code complexity in the job queue is equivalent if not simpler. Since opcodes are guaranteed to run in order, clients can just define the dependency on the first opcode. Another reason for the choice of an opcode-level attribute is that the current LUXI interface for submitting jobs is a bit restricted and would need to be changed to allow the addition of job-level attributes, potentially requiring changes in all LUXI clients and/or breaking backwards compatibility. Client-side logic ----------------- There's at least one implementation of a batched job executor twisted into the ``burnin`` tool's code. While certainly possible, a client-side solution should be avoided due to the different clients already in use. For one, the :doc:`remote API ` client shouldn't import non-standard modules. htools are written in Haskell and can't use Python modules. A batched job executor contains quite some logic. Even if cleanly abstracted in a (Python) library, sharing code between different clients is difficult if not impossible. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-cmdlib-unittests.rst000064400000000000000000000152401476477700300216460ustar00rootroot00000000000000===================================== Unit tests for cmdlib / LogicalUnit's ===================================== :Created: 2013-Jul-22 :Status: Implemented :Ganeti-Version: 2.10.0 .. contents:: :depth: 4 This is a design document describing unit tests for the cmdlib module. Other modules are deliberately omitted, as LU's contain the most complex logic and are only sparingly tested. Current state and shortcomings ============================== The current test coverage of the cmdlib module is at only ~14%. Given the complexity of the code this is clearly too little. The reasons for this low coverage are numerous. There are organisational reasons, like no strict requirements for unit tests for each feature. But there are also design and technical reasons, which this design document wants to address. First, it's not clear which parts of LU's should be tested by unit tests, i.e. the test boundaries are not clearly defined. And secondly, it's too hard to actually write unit tests for LU's. There exists no good framework or set of tools to write easy to understand and concise tests. Proposed changes ================ This design document consists of two parts. Initially, the test boundaries for cmdlib are laid out, and considerations about writing unit tests are given. Then the test framework is described, together with a rough overview of the individual parts and how they are meant to be used. Test boundaries --------------- For the cmdlib module, every LogicalUnit is seen as a unit for testing. Unit tests for LU's may only execute the LU but make sure that no side effect (like filesystem access, network access or the like) takes place. Smaller test units (like individual methods) are sensible and will be supported by the test framework. However, they are not the main scope of this document. LU's require the following environment to be provided by the test code in order to be executed: An input opcode LU's get all the user provided input and parameters from the opcode. The command processor Used to get the execution context id and to output logging messages. It also drives the execution of LU's by calling the appropriate methods in the right order. The Ganeti context Provides node-management methods and contains * The configuration. This gives access to the cluster configuration. * The Ganeti Lock Manager. Manages locks during the execution. The RPC runner Used to communicate with node daemons on other nodes and to perform operations on them. The IAllocator runner Calls the IAllocator with a given request. All of those components have to be replaced/adapted by the test framework. The goal of unit tests at the LU level is to exercise every possible code path in the LU at least once. Shared methods which are used by multiple LU's should be made testable by themselves and explicit unit tests should be written for them. Ultimately, the code coverage for the cmdlib module should be higher than 90%. As Python is a dynamic language, a portion of those tests only exists to exercise the code without actually asserting for anything in the test. They merely make sure that no type errors exist and that potential typos etc. are caught at unit test time. Test framework -------------- The test framework will it make possible to write short and concise tests for LU's. In the simplest case, only an opcode has to be provided by the test. The framework will then use default values, like an almost empty configuration with only the master node and no instances. All aspects of the test environment will be configurable by individual tests. MCPU mocking ************ The MCPU drives the execution of LU's. It has to perform its usual sequence of actions, but additionally it has to provide easy access to the log output of LU's. It will contain utility assertion methods on the output. The mock will be a sub-class of ``mcpu.Processor`` which overrides portions of it in order to support the additional functionality. The advantage of being a sub-class of the original processor is the automatic compatibility with the code running in real clusters. Configuration mocking ********************* Per default, the mocked configuration will contain only the master node, no instances and default parameters. However, convenience methods for the following use cases will be provided: - "Shortcut" methods to add objects to the configuration. - Helper methods to quickly create standard nodes/instances/etc. - Pre-populated default configurations for standard use-cases (i.e. cluster with three nodes, five instances, etc.). - Convenience assertion methods for checking the configuration. Lock mocking ************ Initially, the mocked lock manager always grants all locks. It performs the following tasks: - It keeps track of requested/released locks. - Provides utility assertion methods for checking locks (current and already released ones). In the future, this component might be extended to prevent locks from being granted. This could eventually be used to test optimistic locking. RPC mocking *********** No actual RPC can be made during unit tests. Therefore, those calls have to be replaced and their results mocked. As this will entail a large portion of work when writing tests, mocking RPC's will be made as easy as possible. This entails: - Easy construction of RPC results. - Easy mocking of RPC calls (also multiple ones of the same type during one LU execution). - Asserting for RPC calls (including arguments, affected nodes, etc.). IAllocator mocking ****************** Calls (also multiple ones during the execution of a LU) to the IAllocator interface have to be mocked. The framework will provide, similarly to the RPC mocking, provide means to specify the mocked result and to assert on the IAllocator requests. Future work =========== With unit tests for cmdlib in place, further unit testing for other modules can and should be added. The test boundaries therefore should be aligned with the boundaries from cmdlib. The mocked locking module can be extended to allow testing of optimistic locking in LU's. In this case, on all requested locks are actually granted to the LU, so it has to adapt for this situation correctly. A higher test coverage for LU's will increase confidence in our code and tests. Refactorings will be easier to make as more problems are caught during tests. After a baseline of unit tests is established for cmdlib, efficient testing guidelines could be put in place. For example, new code could be required to not lower the test coverage in cmdlib. Additionally, every bug fix could be required to include a test which triggered the bug before the fix is created. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-configlock.rst000064400000000000000000000153261476477700300204770ustar00rootroot00000000000000=================================== Removal of the Config Lock Overhead =================================== :Created: 2013-Jul-22 :Status: Partially Implemented :Ganeti-Version: 2.14.0, 2.15.0 .. contents:: :depth: 4 This is a design document detailing how the adverse effect of the config lock can be removed in an incremental way. Current state and shortcomings ============================== As a result of the :doc:`design-daemons`, the configuration is held in a process different from the processes carrying out the Ganeti jobs. Therefore, job processes have to contact WConfD in order to change the configuration. Of course, these modifications of the configuration need to be synchronised. The current form of synchronisation is via ``ConfigLock``. Exclusive possession of this lock guarantees that no one else modifies the configuration. In other words, the current procedure for a job to update the configuration is to - acquire the ``ConfigLock`` from WConfD, - read the configuration, - write the modified configuration, and - release ``ConfigLock``. The current procedure has some drawbacks. These also affect the overall throughput of jobs in a Ganeti cluster. - At each configuration update, the whole configuration is transferred between the job and WConfD. - More importantly, however, jobs can only release the ``ConfigLock`` after the write; the write, in turn, is only confirmed once the configuration is written on disk. In particular, we can only have one update per configuration write. Also, having the ``ConfigLock`` is only confirmed to the job, once the new lock status is written to disk. Additional overhead is caused by the fact that reads are synchronised over a shared config lock. This used to make sense when the configuration was modifiable in the same process to ensure consistent read. With the new structure, all access to the configuration via WConfD are consistent anyway, and local modifications by other jobs do not happen. Proposed changes for an incremental improvement =============================================== Ideally, jobs would just send patches for the configuration to WConfD that are applied by means of atomically updating the respective ``IORef``. This, however, would require changing all of Ganeti's logical units in one big change. Therefore, we propose to keep the ``ConfigLock`` and, step by step, reduce its impact till it eventually will be just used internally in the WConfD process. Unlocked Reads -------------- In a first step, all configuration operations that are synchronised over a shared config lock, and therefore necessarily read-only, will instead use WConfD's ``readConfig`` used to obtain a snapshot of the configuration. This will be done without modifying the locks. It is sound, as reads to a Haskell ``IORef`` always yield a consistent value. From that snapshot the required view is computed locally. This saves two lock-configuration write cycles per read and, additionally, does not block any concurrent modifications. In a second step, more specialised read functions will be added to ``WConfD``. This will reduce the traffic for reads. Cached Reads ------------ As jobs synchronize with each other by means of regular locks, the parts of the configuration relevant for a job can only change while a job waits for new locks. So, if a job has a copy of the configuration and not asked for locks afterwards, all read-only access can be done from that copy. While this will not affect the ``ConfigLock``, it saves traffic. Set-and-release action ---------------------- As a typical pattern is to change the configuration and afterwards release the ``ConfigLock``. To avoid unnecessary RPC call overhead, WConfD will offer a combined call. To make that call retryable, it will do nothing if the the ``ConfigLock`` is not held by the caller; in the return value, it will indicate if the config lock was held when the call was made. Short-lived ``ConfigLock`` -------------------------- For a lot of operations, the regular locks already ensure that only one job can modify a certain part of the configuration. For example, only jobs with an exclusive lock on an instance will modify that instance. Therefore, it can update that entity atomically, without relying on the configuration lock to achieve consistency. ``WConfD`` will provide such operations. To avoid interference with non-atomic operations that still take the config lock and write the configuration as a whole, this operation will only be carried out at times the config lock is not taken. To ensure this, the thread handling the request will take the config lock itself (hence no one else has it, if that succeeds) before the change and release afterwards; both operations will be done without triggering a writeout of the lock status. Note that the thread handling the request has to take the lock in its own name and not in that of the requesting job. A writeout of the lock status can still happen, triggered by other requests. Now, if ``WConfD`` gets restarted after the lock acquisition, if that happened in the name of the job, it would own a lock without knowing about it, and hence that lock would never get released. Approaches considered, but not working ====================================== Set-and-release action with asynchronous writes ----------------------------------------------- Approach ~~~~~~~~ As a typical pattern is to change the configuration and afterwards release the ``ConfigLock``. To avoid unnecessary delay in this operation (the next modification of the configuration can already happen while the last change is written out), WConfD will offer a combined command that will - set the configuration to the specified value, - release the config lock, - and only then wait for the configuration write to finish; it will not wait for confirmation of the lock-release write. If jobs use this combined command instead of the sequential set followed by release, new configuration changes can come in during writeout of the current change; in particular, a writeout can contain more than one change. Problem ~~~~~~~ This approach works fine, as long as always either ``WConfD`` can do an ordered shutdown or the calling process dies as well. If however, we allow random kill signals to be sent to individual daemons (e.g., by an out-of-memory killer), the following race occurs. A process can ask for a combined write-and-unlock operation; while the configuration is still written out, the write out of the updated lock status already finishes. Now, if ``WConfD`` forcefully gets killed in that very moment, a restarted ``WConfD`` will read the old configuration but the new lock status. This will make the calling process believe that its call, while it didn't get an answer, succeeded nevertheless, thus resulting in a wrong configuration state. ganeti-3.1.0~rc2/doc/design-cpu-pinning.rst000064400000000000000000000177511476477700300206140ustar00rootroot00000000000000================== Ganeti CPU Pinning ================== :Created: 2011-May-28 :Status: Implemented :Ganeti-Version: 2.6.0 Objective --------- This document defines Ganeti's support for CPU pinning (aka CPU affinity). CPU pinning enables mapping and unmapping entire virtual machines or a specific virtual CPU (vCPU), to a physical CPU or a range of CPUs. At this stage Pinning will be implemented for Xen and KVM. Command Line ------------ Suggested command line parameters for controlling CPU pinning are as follows:: gnt-instance modify -H cpu_mask= cpu-pinning-info can be any of the following: * One vCPU mapping, which can be the word "all" or a combination of CPU numbers and ranges separated by comma. In this case, all vCPUs will be mapped to the indicated list. * A list of vCPU mappings, separated by a colon ':'. In this case each vCPU is mapped to an entry in the list, and the size of the list must match the number of vCPUs defined for the instance. This is enforced when setting CPU pinning or when setting the number of vCPUs using ``-B vcpus=#``. The mapping list is matched to consecutive virtual CPUs, so the first entry would be the CPU pinning information for vCPU 0, the second entry for vCPU 1, etc. The default setting for new instances is "all", which maps the entire instance to all CPUs, thus effectively turning off CPU pinning. Here are some usage examples:: # Map vCPU 0 to physical CPU 1 and vCPU 1 to CPU 3 (assuming 2 vCPUs) gnt-instance modify -H cpu_mask=1:3 my-inst # Pin vCPU 0 to CPUs 1 or 2, and vCPU 1 to any CPU gnt-instance modify -H cpu_mask=1-2:all my-inst # Pin vCPU 0 to any CPU, vCPU 1 to CPUs 1, 3, 4 or 5, and CPU 2 to # CPU 0 gnt-instance modify -H cpu_mask=all:1\\,3-5:0 my-inst # Pin entire VM to CPU 0 gnt-instance modify -H cpu_mask=0 my-inst # Turn off CPU pinning (default setting) gnt-instance modify -H cpu_mask=all my-inst Assuming an instance has 3 vCPUs, the following commands will fail:: # not enough mappings gnt-instance modify -H cpu_mask=0:1 my-inst # too many gnt-instance modify -H cpu_mask=2:1:1:all my-inst Validation ---------- CPU pinning information is validated by making sure it matches the number of vCPUs. This validation happens when changing either the cpu_mask or vcpus parameters. Changing either parameter in a way that conflicts with the other will fail with a proper error message. To make such a change, both parameters should be modified at the same time. For example: ``gnt-instance modify -B vcpus=4 -H cpu_mask=1:1:2-3:4\\,6 my-inst`` Besides validating CPU configuration, i.e. the number of vCPUs matches the requested CPU pinning, Ganeti will also verify the number of physical CPUs is enough to support the required configuration. For example, trying to run a configuration of vcpus=2,cpu_mask=0:4 on a node with 4 cores will fail (Note: CPU numbers are 0-based). This validation should repeat every time an instance is started or migrated live. See more details under Migration below. Cluster verification should also test the compatibility of other nodes in the cluster to required configuration and alert if a minimum requirement is not met. Failover -------- CPU pinning configuration can be transferred from node to node, unless the number of physical CPUs is smaller than what the configuration calls for. It is suggested that unless this is the case, all transfers and migrations will succeed. In case the number of physical CPUs is smaller than the numbers indicated by CPU pinning information, instance failover will fail. In case of emergency, to force failover to ignore mismatching CPU information, the following switch can be used: ``gnt-instance failover --fix-cpu-mismatch my-inst``. This command will try to failover the instance with the current cpu mask, but if that fails, it will change the mask to be "all". Migration --------- In case of live migration, and in addition to failover considerations, it is required to remap CPU pinning after migration. This can be done in realtime for instances for both Xen and KVM, and only depends on the number of physical CPUs being sufficient to support the migrated instance. Data ---- Pinning information will be kept as a list of integers per vCPU. To mark a mapping of any CPU, we will use (-1). A single entry, no matter what the number of vCPUs is, will always mean that all vCPUs have the same mapping. Configuration file ------------------ The pinning information is kept for each instance's hypervisor params section of the configuration file as the original string. Xen --- There are 2 ways to control pinning in Xen, either via the command line or through the configuration file. The commands to make direct pinning changes are the following:: # To pin a vCPU to a specific CPU xm vcpu-pin # To unpin a vCPU xm vcpu-pin all # To get the current pinning status xm vcpu-list Since currently controlling Xen in Ganeti is done in the configuration file, it is straight forward to use the same method for CPU pinning. There are 2 different parameters that control Xen's CPU pinning and configuration: vcpus controls the number of vCPUs cpus maps vCPUs to physical CPUs When no pinning is required (pinning information is "all"), the "cpus" entry is removed from the configuration file. For all other cases, the configuration is "translated" to Xen, which expects either ``cpus = "a"`` or ``cpus = [ "a", "b", "c", ...]``, where each a, b or c are a physical CPU number, CPU range, or a combination, and the number of entries (if a list is used) must match the number of vCPUs, and are mapped in order. For example, CPU pinning information of ``1:2,4-7:0-1`` is translated to this entry in Xen's configuration ``cpus = [ "1", "2,4-7", "0-1" ]`` KVM --- Controlling pinning in KVM is a little more complicated as there is no configuration to control pinning before instances are started. The way to change or assign CPU pinning under KVM is to use ``taskset`` or its underlying system call ``sched_setaffinity``. Setting the affinity for the VM process will change CPU pinning for the entire VM, and setting it for specific vCPU threads will control specific vCPUs. The sequence of commands to control pinning is this: start the instance with the ``-S`` switch, so it halts before starting execution, get the process ID or identify thread IDs of each vCPU by sending ``info cpus`` to the monitor, map vCPUs as required by the cpu-pinning information, and issue a ``cont`` command on the KVM monitor to allow the instance to start execution. For example, a sequence of commands to control CPU affinity under KVM may be: * Start KVM: ``/usr/bin/kvm â€Ļ â€Ļ -S`` * Use socat to connect to monitor * send ``info cpus`` to monitor to get thread/vCPU information * call ``sched_setaffinity`` for each thread with the CPU mask * send ``cont`` to KVM's monitor A CPU mask is a hexadecimal bit mask where each bit represents one physical CPU. See man page for :manpage:`sched_setaffinity(2)` for more details. For example, to run a specific thread-id on CPUs 1 or 3 the mask is 0x0000000A. As of 2.12, the psutil python package (https://github.com/giampaolo/psutil) will be used to control process and thread affinity. The affinity python package (http://pypi.python.org/pypi/affinity) was used before, but it was not invoking the two underlying system calls appropriately, using a cast instead of the CPU_SET macro, causing failures for masks referencing more than 63 CPUs. Alternative Design Options -------------------------- 1. There's an option to ignore the limitations of the underlying hypervisor and instead of requiring explicit pinning information for *all* vCPUs, assume a mapping of "all" to vCPUs not mentioned. This can lead to inadvertent missing information, but either way, since using cpu-pinning options is probably not going to be frequent, there's no real advantage. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-cpu-speed.rst000064400000000000000000000034011476477700300202350ustar00rootroot00000000000000====================================== Taking relative CPU speed into account ====================================== :Created: 2014-Apr-17 :Status: Implemented :Ganeti-Version: 2.12.0 .. contents:: :depth: 4 This document describes the suggested addition of a new node-parameter, describing the CPU speed of a node, relative to that of a normal node in the node group. Current state and shortcomings ============================== Currently, for balancing a cluster, for most resources (disk, memory), the ratio between the amount used and the amount available is taken as a measure of load for that resources. As ``hbal`` tries to even out the load in terms of these measures, larger nodes get a larger share of the instances, even for a cluster not running at full capacity. For for one resources, however, hardware differences are not taken into account: CPU speed. For CPU, the load is measured by the ratio of used virtual to physical CPUs on the node. Balancing this measure implicitly assumes equal speed of all CPUs. Proposed changes ================ It is proposed to add a new node parameter, ``cpu_speed``, that is a floating-point number, with default value ``1.0``. It can be modified in the same ways, as all other node parameters. The cluster metric used by ``htools`` will be changed to use the ratio of virtual to physical cpus weighted by speed, rather than the plain virtual-to-physical ratio. So, when balancing, nodes will be considered as if they had physical cpus equal to ``cpu_speed`` times the actual number. Finally, it should be noted that for IO load, in non-dedicated Ganeti, the ``spindle_count`` already serves the same purpose as the newly proposed ``cpu_speed``. It is a parameter to measure the amount of IO a node can handle in arbitrary units. ganeti-3.1.0~rc2/doc/design-daemons.rst000064400000000000000000000644241476477700300200120ustar00rootroot00000000000000========================== Ganeti daemons refactoring ========================== :Created: 2013-Sep-27 :Status: Implemented :Ganeti-Version: 2.12.0 .. contents:: :depth: 2 This is a design document detailing the plan for refactoring the internal structure of Ganeti, and particularly the set of daemons it is divided into. Current state and shortcomings ============================== Ganeti is comprised of a growing number of daemons, each dealing with part of the tasks the cluster has to face, and communicating with the other daemons using a variety of protocols. Specifically, as of Ganeti 2.8, the situation is as follows: ``Master daemon (MasterD)`` It is responsible for managing the entire cluster, and it's written in Python. It is executed on a single node (the master node). It receives the commands given by the cluster administrator (through the remote API daemon or the command line tools) over the LUXI protocol. The master daemon is responsible for creating and managing the jobs that will execute such commands, and for managing the locks that ensure the cluster will not incur in race conditions. Each job is managed by a separate Python thread, that interacts with the node daemons via RPC calls. The master daemon is also responsible for managing the configuration of the cluster, changing it when required by some job. It is also responsible for copying the configuration to the other master candidates after updating it. ``RAPI daemon (RapiD)`` It is written in Python and runs on the master node only. It waits for requests issued remotely through the remote API protocol. Then, it forwards them, using the LUXI protocol, to the master daemon (if they are commands) or to the query daemon if they are queries about the configuration (including live status) of the cluster. ``Node daemon (NodeD)`` It is written in Python. It runs on all the nodes. It is responsible for receiving the master requests over RPC and execute them, using the appropriate backend (hypervisors, DRBD, LVM, etc.). It also receives requests over RPC for the execution of queries gathering live data on behalf of the query daemon. ``Configuration daemon (ConfD)`` It is written in Haskell. It runs on all the master candidates. Since the configuration is replicated only on the master node, this daemon exists in order to provide information about the configuration to nodes needing them. The requests are done through ConfD's own protocol, HMAC signed, implemented over UDP, and meant to be used by parallely querying all the master candidates (or a subset thereof) and getting the most up to date answer. This is meant as a way to provide a robust service even in case master is temporarily unavailable. ``Query daemon (QueryD)`` It is written in Haskell. It runs on all the master candidates. It replies to Luxi queries about the current status of the system, including live data it obtains by querying the node daemons through RPCs. ``Monitoring daemon (MonD)`` It is written in Haskell. It runs on all nodes, including the ones that are not vm-capable. It is meant to provide information on the status of the system. Such information is related only to the specific node the daemon is running on, and it is provided as JSON encoded data over HTTP, to be easily readable by external tools. The monitoring daemon communicates with ConfD to get information about the configuration of the cluster. The choice of communicating with ConfD instead of MasterD allows it to obtain configuration information even when the cluster is heavily degraded (e.g.: when master and some, but not all, of the master candidates are unreachable). The current structure of the Ganeti daemons is inefficient because there are many different protocols involved, and each daemon needs to be able to use multiple ones, and has to deal with doing different things, thus making sometimes unclear which daemon is responsible for performing a specific task. Also, with the current configuration, jobs are managed by the master daemon using python threads. This makes terminating a job after it has started a difficult operation, and it is the main reason why this is not possible yet. The master daemon currently has too many different tasks, that could be handled better if split among different daemons. Proposed changes ================ In order to improve on the current situation, a new daemon subdivision is proposed, and presented hereafter. .. digraph:: "new-daemons-structure" {rank=same; RConfD LuxiD;} {rank=same; Jobs rconfigdata;} node [shape=box] RapiD [label="RapiD [M]"] LuxiD [label="LuxiD [M]"] WConfD [label="WConfD [M]"] Jobs [label="Jobs [M]"] RConfD [label="RConfD [MC]"] MonD [label="MonD [All]"] NodeD [label="NodeD [All]"] Clients [label="gnt-*\nclients [M]"] p1 [shape=none, label=""] p2 [shape=none, label=""] p3 [shape=none, label=""] p4 [shape=none, label=""] configdata [shape=none, label="config.data"] rconfigdata [shape=none, label="config.data\n[MC copy]"] locksdata [shape=none, label="locks.data"] RapiD -> LuxiD [label="LUXI"] LuxiD -> WConfD [label="WConfD\nproto"] LuxiD -> Jobs [label="fork/exec"] Jobs -> WConfD [label="WConfD\nproto"] Jobs -> NodeD [label="RPC"] LuxiD -> NodeD [label="RPC"] rconfigdata -> RConfD configdata -> rconfigdata [label="sync via\nNodeD RPC"] WConfD -> NodeD [label="RPC"] WConfD -> configdata WConfD -> locksdata MonD -> RConfD [label="RConfD\nproto"] Clients -> LuxiD [label="LUXI"] p1 -> MonD [label="MonD proto"] p2 -> RapiD [label="RAPI"] p3 -> RConfD [label="RConfD\nproto"] p4 -> Clients [label="CLI"] ``LUXI daemon (LuxiD)`` It will be written in Haskell. It will run on the master node and it will be the only LUXI server, replying to all the LUXI queries. These includes both the queries about the live configuration of the cluster, previously served by QueryD, and the commands actually changing the status of the cluster by submitting jobs. Therefore, this daemon will also be the one responsible with managing the job queue. When a job needs to be executed, the LuxiD will spawn a separate process tasked with the execution of that specific job, thus making it easier to terminate the job itself, if needed. When a job requires locks, LuxiD will request them from WConfD. In order to keep availability of the cluster in case of failure of the master node, LuxiD will replicate the job queue to the other master candidates, by RPCs to the NodeD running there (the choice of RPCs for this task might be reviewed at a second time, after implementing this design). ``Configuration management daemon (WConfD)`` It will run on the master node and it will be responsible for the management of the authoritative copy of the cluster configuration (that is, it will be the daemon actually modifying the ``config.data`` file). All the requests of configuration changes will have to pass through this daemon, and will be performed using a LUXI-like protocol ("WConfD proto" in the graph. The exact protocol will be defined in the separate design document that will detail the WConfD separation). Having a single point of configuration management will also allow Ganeti to get rid of possible race conditions due to concurrent modifications of the configuration. When the configuration is updated, it will have to push the received changes to the other master candidates, via RPCs, so that RConfD daemons and (in case of a failure on the master node) the WConfD daemon on the new master can access an up-to-date version of it (the choice of RPCs for this task might be reviewed at a second time). This daemon will also be the one responsible for managing the locks, granting them to the jobs requesting them, and taking care of freeing them up if the jobs holding them crash or are terminated before releasing them. In order to do this, each job, after being spawned by LuxiD, will open a local unix socket that will be used to communicate with it, and will be destroyed when the job terminates. LuxiD will be able to check, after a timeout, whether the job is still running by connecting here, and to ask WConfD to forcefully remove the locks if the socket is closed. Also, WConfD should hold a serialized list of the locks and their owners in a file (``locks.data``), so that it can keep track of their status in case it crashes and needs to be restarted (by asking LuxiD which of them are still running). Interaction with this daemon will be performed using Unix sockets. ``Configuration query daemon (RConfD)`` It is written in Haskell, and it corresponds to the old ConfD. It will run on all the master candidates and it will serve information about the static configuration of the cluster (the one contained in ``config.data``). The provided information will be highly available (as in: a response will be available as long as a stable-enough connection between the client and at least one working master candidate is available) and its freshness will be best effort (the most recent reply from any of the master candidates will be returned, but it might still be older than the one available through WConfD). The information will be served through the ConfD protocol. ``Rapi daemon (RapiD)`` It remains basically unchanged, with the only difference that all of its LUXI query are directed towards LuxiD instead of being split between MasterD and QueryD. ``Monitoring daemon (MonD)`` It remains unaffected by the changes in this design document. It will just get some of the data it needs from RConfD instead of the old ConfD, but the interfaces of the two are identical. ``Node daemon (NodeD)`` It remains unaffected by the changes proposed in the design document. The only difference being that it will receive its RPCs from LuxiD (for job queue replication), from WConfD (for configuration replication) and for the processes executing single jobs (for all the operations to be performed by nodes) instead of receiving them just from MasterD. This restructuring will allow us to reorganize and improve the codebase, introducing cleaner interfaces and giving well defined and more restricted tasks to each daemon. Furthermore, having more well-defined interfaces will allow us to have easier upgrade procedures, and to work towards the possibility of upgrading single components of a cluster one at a time, without the need for immediately upgrading the entire cluster in a single step. Implementation ============== While performing this refactoring, we aim to increase the amount of Haskell code, thus benefiting from the additional type safety provided by its wide compile-time checks. In particular, all the job queue management and the configuration management daemon will be written in Haskell, taking over the role currently fulfilled by Python code executed as part of MasterD. The changes describe by this design document are quite extensive, therefore they will not be implemented all at the same time, but through a sequence of steps, leaving the codebase in a consistent and usable state. #. Rename QueryD to LuxiD. A part of LuxiD, the one replying to configuration queries including live information about the system, already exists in the form of QueryD. This is being renamed to LuxiD, and will form the first part of the new daemon. NB: this is happening starting from Ganeti 2.8. At the beginning, only the already existing queries will be replied to by LuxiD. More queries will be implemented in the next versions. #. Let LuxiD be the interface for the queries and MasterD be their executor. Currently, MasterD is the only responsible for receiving and executing LUXI queries, and for managing the jobs they create. Receiving the queries and managing the job queue will be extracted from MasterD into LuxiD. Actually executing jobs will still be done by MasterD, that contains all the logic for doing that and for properly managing locks and the configuration. At this stage, scheduling will simply consist in starting jobs until a fixed maximum number of simultaneously running jobs is reached. #. Extract WConfD from MasterD. The logic for managing the configuration file is factored out to the dedicated WConfD daemon. All configuration changes, currently executed directly by MasterD, will be changed to be IPC requests sent to the new daemon. #. Extract locking management from MasterD. The logic for managing and granting locks is extracted to WConfD as well. Locks will not be taken directly anymore, but asked via IPC to WConfD. This step can be executed on its own or at the same time as the previous one. #. Jobs are executed as processes. The logic for running jobs is rewritten so that each job can be managed by an independent process. LuxiD will spawn a new (Python) process for every single job. The RPCs will remain unchanged, and the LU code will stay as is as much as possible. MasterD will cease to exist as a daemon on its own at this point, but not before. #. Improve job scheduling algorithm. The simple algorithm for scheduling jobs will be replaced by a more intelligent one. Also, the implementation of :doc:`design-optables` can be started. Job death detection ------------------- **Requirements:** - It must be possible to reliably detect a death of a process even under uncommon conditions such as very heavy system load. - A daemon must be able to detect a death of a process even if the daemon is restarted while the process is running. - The solution must not rely on being able to communicate with a process. - The solution must work for the current situation where multiple jobs run in a single process. - It must be POSIX compliant. These conditions rule out simple solutions like checking a process ID (because the process might be eventually replaced by another process with the same ID) or keeping an open connection to a process. **Solution:** As a job process is spawned, before attempting to communicate with any other process, it will create a designated empty lock file, open it, acquire an *exclusive* lock on it, and keep it open. When connecting to a daemon, the job process will provide it with the path of the file. If the process dies unexpectedly, the operating system kernel automatically cleans up the lock. Therefore, daemons can check if a process is dead by trying to acquire a *shared* lock on the lock file in a non-blocking mode: - If the locking operation succeeds, it means that the exclusive lock is missing, therefore the process has died, but the lock file hasn't been cleaned up yet. The daemon should release the lock immediately. Optionally, the daemon may delete the lock file. - If the file is missing, the process has died and the lock file has been cleaned up. - If the locking operation fails due to a lock conflict, it means the process is alive. Using shared locks for querying lock files ensures that the detection works correctly even if multiple daemons query a file at the same time. A job should close and remove its lock file when completely finishes. The WConfD daemon will be responsible for removing stale lock files of jobs that didn't remove its lock files themselves. **Statelessness of the protocol:** To keep our protocols stateless, the job id and the path the to lock file are sent as part of every request that deals with resources, in particular the Ganeti Locks. All resources are owned by the pair (job id, lock file). In this way, several jobs can live in the same process (as it will be in the transition period), but owner death detection still only depends on the owner of the resource. In particular, no additional lookup table is needed to obtain the lock file for a given owner. **Considered alternatives:** An alternative to creating a separate lock file would be to lock the job status file. However, file locks are kept only as long as the file is open. Therefore any operation followed by closing the file would cause the process to release the lock. In particular, with jobs as threads, the master daemon wouldn't be able to keep locks and operate on job files at the same time. Job execution ------------- As the Luxi daemon will be responsible for executing jobs, it needs to start jobs in such a way that it can properly detect if the job dies under any circumstances (such as Luxi daemon being restarted in the process). The name of the lock file will be stored in the corresponding job file so that anybody is able to check the status of the process corresponding to a job. The proposed procedure: #. The Luxi daemon saves the name of its own lock file into the job file. #. The Luxi daemon forks, creating a bi-directional pipe with the child process. #. The child process creates and locks its own, proper lock file and handles its name to the Luxi daemon through the pipe. #. The Luxi daemon saves the name of the lock file into the job file and confirms it to the child process. #. Only then the child process can replace itself by the actual job process. If the child process detects that the pipe is broken before receiving the confirmation, it must terminate, not starting the actual job. This way, the actual job is only started if it is ensured that its lock file name is written to the job file. If the Luxi daemon detects that the pipe is broken before successfully sending the confirmation in step 4., it assumes that the job has failed. If the pipe gets broken after sending the confirmation, no further action is necessary. If the child doesn't receive the confirmation, it will die and its death will be detected by Luxid eventually. If the Luxi daemon dies before completing the procedure, the job will not be started. If the job file contained the daemon's lock file name, it will be detected as dead (because the daemon process died). If the job file already contained its proper lock file, it will also be detected as dead (because the child process won't start the actual job and die). WConfD details -------------- WConfD will communicate with its clients through a Unix domain socket for both configuration management and locking. Clients can issue multiple RPC calls through one socket. For each such a call the client sends a JSON request document with a remote function name and data for its arguments. The server replies with a JSON response document containing either the result of signalling a failure. Any state associated with client processes will be mirrored on persistent storage and linked to the identity of processes so that the WConfD daemon will be able to resume its operation at any point after a restart or a crash. WConfD will track each client's process start time along with its process ID to be able detect if a process dies and it's process ID is reused. WConfD will clear all locks and other state associated with a client if it detects it's process no longer exists. Configuration management ++++++++++++++++++++++++ The new configuration management protocol will be implemented in the following steps: Step 1: #. Implement the following functions in WConfD and export them through RPC: - Obtain a single internal lock, either in shared or exclusive mode. This lock will substitute the current lock ``_config_lock`` in config.py. - Release the lock. - Return the whole configuration data to a client. - Receive the whole configuration data from a client and replace the current configuration with it. Distribute it to master candidates and distribute the corresponding *ssconf*. WConfD must detect deaths of its clients (see `Job death detection`_) and release locks automatically. #. In config.py modify public methods that access configuration: - Instead of acquiring a local lock, obtain a lock from WConfD using the above functions - Fetch the current configuration from WConfD. - Use it to perform the method's task. - If the configuration was modified, send it to WConfD at the end. - Release the lock to WConfD. This will decouple the configuration management from the master daemon, even though the specific configuration tasks will still performed by individual jobs. After this step it'll be possible access the configuration from separate processes. Step 2: #. Reimplement all current methods of ``ConfigWriter`` for reading and writing the configuration of a cluster in WConfD. #. Expose each of those functions in WConfD as a separate RPC function. This will allow easy future extensions or modifications. #. Replace ``ConfigWriter`` with a stub (preferably automatically generated from the Haskell code) that will contain the same methods as the current ``ConfigWriter`` and delegate all calls to its methods to WConfD. Step 3: In a later step, the impact of the config lock will be reduced by moving it more and more into an internal detail of WConfD. This forthcoming process of :doc:`design-configlock` is described separately. Locking +++++++ The new locking protocol will be implemented as follows: Re-implement the current locking mechanism in WConfD and expose it for RPC calls. All current locks will be mapped into a data structure that will uniquely identify them (storing lock's level together with it's name). WConfD will impose a linear order on locks. The order will be compatible with the current ordering of lock levels so that existing code will work without changes. WConfD will keep the set of currently held locks for each client. The protocol will allow the following operations on the set: *Update:* Update the current set of locks according to a given list. The list contains locks and their desired level (release / shared / exclusive). To prevent deadlocks, WConfD will check that all newly requested locks (or already held locks requested to be upgraded to *exclusive*) are greater in the sense of the linear order than all currently held locks, and fail the operation if not. Only the locks in the list will be updated, other locks already held will be left intact. If the operation fails, the client's lock set will be left intact. *Opportunistic union:* Add as much as possible locks from a given set to the current set within a given timeout. WConfD will again check the proper order of locks and acquire only the ones that are allowed wrt. the current set. Returns the set of acquired locks, possibly empty. Immediate. Never fails. (It would also be possible to extend the operation to try to wait until a given number of locks is available, or a given timeout elapses.) *List:* List the current set of held locks. Immediate, never fails. *Intersection:* Retain only a given set of locks in the current one. This function is provided for convenience, it's redundant wrt. *list* and *update*. Immediate, never fails. Additional restrictions due to lock implications: Ganeti supports locks that act as if a lock on a whole group (like all nodes) were held. To avoid dead locks caused by the additional blockage of those group locks, we impose certain restrictions. Whenever `A` is a group lock and `B` belongs to `A`, then the following holds. - `A` is in lock order before `B`. - All locks that are in the lock order between `A` and `B` also belong to `A`. - It is considered a lock-order violation to ask for an exclusive lock on `B` while holding a shared lock on `A`. After this step it'll be possible to use locks from jobs as separate processes. The above set of operations allows the clients to use various work-flows. In particular: Pessimistic strategy: Lock all potentially relevant resources (for example all nodes), determine which will be needed, and release all the others. Optimistic strategy: Determine what locks need to be acquired without holding any. Lock the required set of locks. Determine the set of required locks again and check if they are all held. If not, release everything and restart. .. COMMENTED OUT: Start with the smallest set of locks and when determining what more relevant resources will be needed, expand the set. If an *union* operation fails, release all locks, acquire the desired union and restart the operation so that all preconditions and possible concurrent changes are checked again. Future aims: - Add more fine-grained locks to prevent unnecessary blocking of jobs. This could include locks on parameters of entities or locks on their states (so that a node remains online, but otherwise can change, etc.). In particular, adding, moving and removing instances currently blocks the whole node. - Add checks that all modified configuration parameters belong to entities the client has locked and log violations. - Make the above checks mandatory. - Automate optimistic locking and checking the locks in logical units. For example, this could be accomplished by allowing some of the initial phases of `LogicalUnit` (such as `ExpandNames` and `DeclareLocks`) to be run repeatedly, checking if the set of locks requested the second time is contained in the set acquired after the first pass. - Add the possibility for a job to reserve hardware resources such as disk space or memory on nodes. Most likely as a new, special kind of instances that would only block its resources and allow to be converted to a regular instance. This would allow long-running jobs such as instance creation or move to lock the corresponding nodes, acquire the resources and turn the locks into shared ones, keeping an exclusive lock only on the instance. - Use more sophisticated algorithm for preventing deadlocks such as a `wait-for graph`_. This would allow less *union* failures and allow more optimistic, scalable acquisition of locks. .. _`wait-for graph`: http://en.wikipedia.org/wiki/Wait-for_graph Further considerations ====================== There is a possibility that a job will finish performing its task while LuxiD and/or WConfD will not be available. In order to deal with this situation, each job will update its job file in the queue. This is race free, as LuxiD will no longer touch the job file, once the job is started; a corollary of this is that the job also has to take care of replicating updates to the job file. LuxiD will watch job files for changes to determine when a job was cleanly finished. To determine jobs that died without having the chance of updating the job file, the `Job death detection`_ mechanism will be used. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-dedicated-allocation.rst000064400000000000000000000075131476477700300224110ustar00rootroot00000000000000================================= Allocation for Partitioned Ganeti ================================= :Created: 2015-Jan-22 :Status: Implemented :Ganeti-Version: 2.15.0 .. contents:: :depth: 4 Current state and shortcomings ============================== The introduction of :doc:`design-partitioned` allowed to dedicate resources, in particular storage, exclusively to an instance. The advantage is that such instances have guaranteed latency that is not affected by other instances. Typically, those instances are created once and never moved. Also, typically large chunks (full, half, or quarter) of a node are handed out to individual partitioned instances. Ganeti's allocation strategy is to keep the cluster as balanced as possible. In particular, as long as empty nodes are available, new instances, regardless of their size, will be placed there. Therefore, if a couple of small instances are placed on the cluster first, it will no longer be possible to place a big instance on the cluster despite the total usage of the cluster being low. Proposed changes ================ We propose to change the allocation strategy of hail for node groups that have the ``exclusive_storage`` flag set, as detailed below; nothing will be changed for non-exclusive node groups. The new strategy will try to keep the cluster as available for new instances as possible. Dedicated Allocation Metric --------------------------- The instance policy is a set of intervals in which the resources of the instance have to be. Typical choices for dedicated clusters have disjoint intervals with the same monotonicity in every dimension. In this case, the order is obvious. In order to make it well-defined in every case, we specify that we sort the intervals by the lower bound of the disk size. This is motivated by the fact that disk is the most critical aspect of partitioned Ganeti. For a node the *allocation vector* is the vector of, for each instance policy interval in decreasing order, the number of instances minimally compliant with that interval that still can be placed on that node. For the drbd template, it is assumed that all newly placed instances have new secondaries. The *lost-allocations vector* for an instance on a node is the difference of the allocation vectors for that node before and after placing that instance on that node. Lost-allocation vectors are ordered lexicographically, i.e., a loss of an allocation larger instance size dominates loss of allocations of smaller instance sizes. If allocating in a node group with ``exclusive_storage`` set to true, hail will try to minimise the pair of the lost-allocations vector and the remaining disk space on the node after, ordered lexicographically. Example ------- Consider the already mentioned scenario were only full, half, and quarter nodes are given to instances. Here, for the placement of a quarter-node--sized instance we would prefer a three-quarter-filled node (lost allocations: 0, 0, 1 and no left overs) over a quarter-filled node (lost allocations: 0, 0, 1 and half a node left over) over a half-filled node (lost allocations: 0, 1, 1) over an empty node (lost allocations: 1, 1, 1). A half-node sized instance, however, would prefer a half-filled node (lost allocations: 0, 1, 2 and no left-overs) over a quarter-filled node (lost allocations: 0, 1, 2 and a quarter node left over) over an empty node (lost allocations: 1, 1, 2). Note that the presence of additional policy intervals affects the preferences of instances of other sizes as well. This is by design, as additional available instance sizes make additional remaining node sizes attractive. If, in the given example, we would also allow three-quarter-node--sized instances, for a quarter-node--sized instance it would now be better to be placed on a half-full node (lost allocations: 0, 0, 1, 1) than on a quarter-filled node (lost allocations: 0, 1, 0, 1). ganeti-3.1.0~rc2/doc/design-device-uuid-name.rst000064400000000000000000000056441476477700300215040ustar00rootroot00000000000000========================================== Design for adding UUID and name to devices ========================================== :Created: 2013-Apr-17 :Status: Implemented :Ganeti-Version: 2.9.0 .. contents:: :depth: 4 This is a design document about adding UUID and name to instance devices (Disks/NICs) and the ability to reference them by those identifiers. Current state and shortcomings ============================== Currently, the only way to refer to a device (Disk/NIC) is by its index inside the VM (e.g. gnt-instance modify --disk 2:remove). Using indices as identifiers has the drawback that addition/removal of a device results in changing the identifiers(indices) of other devices and makes the net effect of commands depend on their strict ordering. A device reference is not absolute, meaning an external entity controlling Ganeti, e.g., over RAPI, cannot keep permanent identifiers for referring to devices, nor can it have more than one outstanding commands, since their order of execution is not guaranteed. Proposed Changes ================ To be able to reference a device in a unique way, we propose to extend Disks and NICs by assigning to them a UUID and a name. The UUID will be assigned by Ganeti upon creation, while the name will be an optional user parameter. Renaming a device will also be supported. Commands (e.g. `gnt-instance modify`) will be able to reference each device by its index, UUID, or name. To be able to refer to devices by name, we must guarantee that device names are unique. Unlike other objects (instances, networks, nodegroups, etc.), NIC and Disk objects will not have unique names across the cluster, since they are still not independent entities, but rather part of the instance object. This makes global uniqueness of names hard to achieve at this point. Instead their names will be unique at instance level. Apart from unique device names, we must also guarantee that a device name can not be the UUID of another device. Also, to remove ambiguity while supporting both indices and names as identifiers, we forbid purely numeric device names. Implementation Details ====================== Modify OpInstanceSetParams to accept not only indexes, but also device names and UUIDs. So, the accepted NIC and disk modifications will have the following format: identifier:action,key=value where, from now on, identifier can be an index (-1 for the last device), UUID, or name and action should be add, modify, or remove. Configuration Changes ~~~~~~~~~~~~~~~~~~~~~ Disk and NIC config objects get two extra slots: - uuid - name Instance Queries ~~~~~~~~~~~~~~~~~ We will extend the query mechanism to expose names and UUIDs of NICs and Disks. Hook Variables ~~~~~~~~~~~~~~ We will expose the name of NICs and Disks to the hook environment of instance-related operations: ``INSTANCE_NIC%d_NAME`` ``INSTANCE_DISK%d_NAME`` .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-disk-conversion.rst000064400000000000000000000314531476477700300214750ustar00rootroot00000000000000================================= Conversion between disk templates ================================= :Created: 2014-May-23 :Status: Implemented :Ganeti-Version: 2.13.0 .. contents:: :depth: 4 This design document describes the support for generic disk template conversion in Ganeti. The logic used is disk template agnostic and targets to cover the majority of conversions among the supported disk templates. Current state and shortcomings ============================== Currently, Ganeti supports choosing among different disk templates when creating an instance. However, converting the disk template of an existing instance is possible only between the ``plain`` and ``drbd`` templates. This feature was added in Ganeti since its early versions when the number of supported disk templates was limited. Now that Ganeti supports plenty of choices, this feature should be extended to provide more flexibility to the user. The procedure for converting from the plain to the drbd disk template works as follows. Firstly, a completely new disk template is generated matching the size, mode, and the count of the current instance's disks. The missing volumes are created manually both in the primary (meta disk) and the secondary node. The original LVs running on the primary node are renamed to match the new names. The last step is to manually associate the DRBD devices with their mirror block device pairs. The conversion from the drbd to the plain disk template is much simpler than the opposite. Firstly, the DRBD mirroring is manually disabled. Then the unnecessary volumes including the meta disk(s) of the primary node, and the meta and data disk(s) from the previously secondary node are removed. Proposed changes ================ This design proposes the creation of a unified interface for handling the disk template conversions in Ganeti. Currently, there is no such interface and each one of the supported conversions uses a separate code path. This proposal introduces a single, disk-agnostic interface for handling the disk template conversions in Ganeti, keeping in mind that we want it to be as generic as possible. An exception case will be the currently supported conversions between the LVM-based disk templates. Their basic functionality will not be affected and will diverge from the rest disk template conversions. The target is to provide support for conversions among the majority of the available disk templates, and also creating a mechanism that will easily support any new templates that may be probably added in Ganeti, at a future point. Design decisions ================ Currently, the supported conversions for the LVM-based templates are handled by the ``LUInstanceSetParams`` LU. Our implementation will follow the same approach. From a high-level point-of-view this design can be split in two parts: * The extension of the LU's checks to cover all the supported template conversions * The new functionality which will be introduced to provide the new feature The instance must be stopped before starting the disk template conversion, as it currently is, otherwise the operation will fail. The new mechanism will need to copy the disk's data for the conversion to be possible. We propose using the Unix ``dd`` command to copy the instance's data. It can be used to copy data from source to destination, block-by-block, regardless of their filesystem types, making it a convenient tool for the case. Since the conversion will be done via data copy it will take a long time for bigger disks to copy their data and consequently for the instance to switch to the new template. Some template conversions can be done faster without copying explicitly their disks' data. A use case is the conversions between the LVM-based templates, i.e., ``drbd`` and ``plain`` which will be done as happens now and not using the ``dd`` command. Also, this implementation will provide partial support for the ``blockdev`` disk template which will act only as a source template. Since those volumes are adopted pre-existent block devices we will not support conversions targeting this template. Another exception case will be the ``diskless`` template. Since it is a testing template that creates instances with no disks we will not provide support for conversions that include this template type. We divide the design into the following parts: * Block device changes, that include the new methods which will be introduced and will be responsible for building the commands for the data copy from/to the requested devices * Backend changes, that include a new RPC call which will concatenate the output of the above two methods and will execute the data copy command * Core changes, that include the modifications in the Logical Unit * User interface changes, i.e., command line changes Block device changes -------------------- The block device abstract class will be extended with two new methods, named ``Import`` and ``Export``. Those methods will be responsible for building the commands that will be used for the data copy between the corresponding devices. The ``Export`` method will build the command which will export the data from the source device, while the ``Import`` method will do the opposite. It will import the data to the newly created target device. Those two methods will not perform the actual data copy; they will simply return the requested commands for transferring the data from/to the individual devices. The output of the two methods will be combined using a pipe ("|") by the caller method in the backend level. By default the data import and export will be done using the ``dd`` command. All the inherited classes will use the base functionality unless there is a faster way to convert to. In that case the underlying block device will overwrite those methods with its specific functionality. A use case will be the Ceph/RADOS block devices which will make use of the ``rbd import`` and ``rbd export`` commands to copy their data instead of using the default ``dd`` command. Keeping the data copy functionality in the block device layer, provides us with a generic mechanism that works between almost all conversions and furthermore can be easily extended for new disk templates. It also covers the devices that support the ``access=userspace`` parameter and solves this problem in a generic way, by implementing the logic in the right level where we know what is the best to do for each device. Backend changes --------------- Introduce a new RPC call: * blockdev_convert(src_disk, dest_disk) where ``src_disk`` and ``dest_disk`` are the original and the new disk objects respectively. First, the actual device instances will be computed and then they will be used to build the export and import commands for the data copy. The output of those methods will be concatenated using a pipe, following a similar approach with the impexp daemon. Finally, the unified data copy command will be executed, at this level, by the ``nodeD``. Core changes ------------ The main modifications will be made in the ``LUInstanceSetParams`` LU. The implementation of the conversion mechanism will be split into the following parts: * The generation of the new disk template for the instance. The new disks will match the size, mode, and name of the original volumes. Those parameters and any other needed, .i.e., the provider's name for the ExtStorage conversions, will be computed by a new method which we will introduce, named ``ComputeDisksInfo``. The output of that function will be used as the ``disk_info`` argument of the ``GenerateDiskTemplate`` method. * The creation of the new block devices. We will make use of the ``CreateDisks`` method which creates and attaches the new block devices. * The data copy for each disk of the instance from the original to the newly created volume. The data copy will be made by the ``nodeD`` with the rpc call we have introduced earlier in this design. In case some disks fail to copy their data the operation will fail and the newly created disks will be removed. The instance will remain intact. * The detachment of the original disks of the instance when the data copy operation successfully completes by calling the ``RemoveInstanceDisk`` method for each instance's disk. * The attachment of the new disks to the instance by calling the ``AddInstanceDisk`` method for each disk we have created. * The update of the configuration file with the new values. * The removal of the original block devices from the node using the ``BlockdevRemove`` method for each one of the old disks. User interface changes ---------------------- The ``-t`` (``--disk-template``) option from the gnt-instance modify command will specify the disk template to convert *to*, as it happens now. The rest disk options such as its size, its mode, and its name will be computed from the original volumes by the conversion mechanism, and the user will not explicitly provide them. ExtStorage conversions ~~~~~~~~~~~~~~~~~~~~~~ When converting to an ExtStorage disk template the ``provider=*PROVIDER*`` option which specifies the ExtStorage provider will be mandatory. Also, arbitrary parameters can be passed to the ExtStorage provider. Those parameters will be optional and could be passed as additional comma separated options. Since it is not allowed to convert the disk template of an instance and make use of the ``--disk`` option at the same time, we propose to introduce a new option named ``--ext-params`` to handle the ``ext`` template conversions. :: gnt-instance modify -t ext --ext-params provider=pvdr1 test_vm gnt-instance modify -t ext --ext-params provider=pvdr1,param1=val1,param2=val2 test_vm File-based conversions ~~~~~~~~~~~~~~~~~~~~~~ For conversions *to* a file-based template the ``--file-storage-dir`` and the ``--file-driver`` options could be used, similarly to the **add** command, to manually configure the storage directory and the preferred driver for the file-based disks. :: gnt-instance modify -t file --file-storage-dir=mysubdir test_vm Supported template conversions ============================== This is a summary of the disk template conversions that the conversion mechanism will support: +--------------+-----------------------------------------------------------------------------------+ | Source | Target Disk Template | | Disk +---------+-------+------+------------+---------+------+------+----------+----------+ | Template | Plain | DRBD | File | Sharedfile | Gluster | RBD | Ext | BlockDev | Diskless | +==============+=========+=======+======+============+=========+======+======+==========+==========+ | Plain | - | Yes. | Yes. | Yes. | Yes. | Yes. | Yes. | No. | No. | +--------------+---------+-------+------+------------+---------+------+------+----------+----------+ | DRBD | Yes. | - | Yes. | Yes. | Yes. | Yes. | Yes. | No. | No. | +--------------+---------+-------+------+------------+---------+------+------+----------+----------+ | File | Yes. | Yes. | - | Yes. | Yes. | Yes. | Yes. | No. | No. | +--------------+---------+-------+------+------------+---------+------+------+----------+----------+ | Sharedfile | Yes. | Yes. | Yes. | - | Yes. | Yes. | Yes. | No. | No. | +--------------+---------+-------+------+------------+---------+------+------+----------+----------+ | Gluster | Yes. | Yes. | Yes. | Yes. | - | Yes. | Yes. | No. | No. | +--------------+---------+-------+------+------------+---------+------+------+----------+----------+ | RBD | Yes. | Yes. | Yes. | Yes. | Yes. | - | Yes. | No. | No. | +--------------+---------+-------+------+------------+---------+------+------+----------+----------+ | Ext | Yes. | Yes. | Yes. | Yes. | Yes. | Yes. | - | No. | No. | +--------------+---------+-------+------+------------+---------+------+------+----------+----------+ | BlockDev | Yes. | Yes. | Yes. | Yes. | Yes. | Yes. | Yes. | - | No. | +--------------+---------+-------+------+------------+---------+------+------+----------+----------+ | Diskless | No. | No. | No. | No. | No. | No. | No. | No. | - | +--------------+---------+-------+------+------------+---------+------+------+----------+----------+ Future Work =========== Expand the conversion mechanism to provide a visual indication of the data copy operation. We could monitor the progress of the data sent via a pipe, and provide to the user information such as the time elapsed, percentage completed (probably with a progress bar), total data transferred, and so on, similar to the progress tracking that is currently done by the impexp daemon. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-disks.rst000064400000000000000000000247001476477700300174720ustar00rootroot00000000000000===== Disks ===== :Created: 2014-Apr-09 :Status: Implemented :Ganeti-Version: 2.14.0 .. contents:: :depth: 4 This is a design document detailing the implementation of disks as a new top-level citizen in the config file (just like instances, nodes etc). Current state and shortcomings ============================== Currently, Disks are stored in Ganeti's config file as a list (container) of Disk objects under the Instance in which they belong. This implementation imposes a number of limitations: * A Disk object cannot live outside an Instance. This means that one cannot detach a disk from an instance (without destroying the disk) and then reattach it (to the same or even to a different instance). * Disks are not taggable objects, as only top-level citizens of the config file can be made taggable. Having taggable disks will allow for further customizations. * All disks of an instance have to be of the same template. Dropping this constraint would allow mixing different kinds of storage (e.g. an instance might have a local ``plain`` storage for the OS and a remotely replicated ``sharedstorage`` for the data). Proposed changes ================ The implementation is going to be split in four parts: * Make disks a top-level citizen in config file. The Instance object will no longer contain a list of Disk objects, but a list of disk UUIDs. * Add locks for Disk objects and make them taggable. * Allow to attach/detach an existing disk to/from an instance. * Allow creation/modification/deletion of disks that are not attached to any instance (requires new LUs for disks). * Allow disks of a single instance to be of different templates. * Remove all unnecessary distinction between disk templates and disk types. Design decisions ================ Disks as config top-level citizens ---------------------------------- The first patch-series is going to add a new top-level citizen in the config object (namely ``disks``) and separate the disk objects from the instances. In doing so there are a number of problems that we have to overcome: * How the Disk object will be represented in the config file and how it is going to be connected with the instance it belongs to. * How the existing code will get the disks belonging to an instance. * What it means for a disk to be attached/detached to/from an instance. * How disks are going to be created/deleted, attached/detached using the existing code. Disk representation ~~~~~~~~~~~~~~~~~~~ The ``Disk`` object gets two extra slots, ``_TIMESTAMPS`` and ``serial_no``. The ``Instance`` object will no longer contain the list of disk objects that are attached to it or a disk template. Instead, an Instance object will refer to its disks using their UUIDs and the disks will contain their own template. Since the order in which the disks are attached to an instance is important we are going to have a list of disk UUIDs under the Instance object which will denote the disks attached to the instance and their order at the same time. So the Instance's ``disks`` slot is going to be a list of disk UUIDs. The `Disk` object is not going to have a slot pointing to the `Instance` in which it belongs since this is redundant. Get instance's disks ~~~~~~~~~~~~~~~~~~~~ A new function ``GetInstanceDisks`` will be added to the config that given an instance will return a list of Disk objects with the disks attached to this instance. This list will be exactly the same as 'instance.disks' was before. Everywhere in the code we are going to replace the 'instance.disks' (which from now one will contain a list of disk UUIDs) with the function ``GetInstanceDisks``. Since disks will not be part of the `Instance` object any more, 'all_nodes' and 'secondary_nodes' can not be `Instance`'s properties. Instead we will use the functions ``GetInstanceNodes`` and ``GetInstanceSecondaryNodes`` from the config to compute these values. Configuration changes ~~~~~~~~~~~~~~~~~~~~~ The ``ConfigData`` object gets one extra slot: ``disks``. Also there will be two new functions, ``AddDisk`` and ``RemoveDisk`` that will create/remove a disk objects from the config. The ``VerifyConfig`` function will be changed so it can check that there are no dangling pointers from instances to disks (i.e. an instance points to a disk that doesn't exist in the config). The 'upgrade' operation for the config should check if disks are top level citizens and if not it has to extract the disk objects from the instances, replace them with their uuids, and copy the disk template. In case of the 'downgrade' operation (where disks will be made again part of the `Instance` object) all disks that are not attached to any instance at all will be ignored (removed from config). The disk template of the instance is set to the disk template of any disk attached to it. If there are multiple disk templates present, the downgrade fails and the user is requested to detach disks from the instances. Apply Disk modifications ~~~~~~~~~~~~~~~~~~~~~~~~ There are four operations that can be performed to a `Disk` object: * Create a new `Disk` object of a given template and save it to the config. * Remove an existing `Disk` object from the config. * Attach an existing `Disk` to an existing `Instance`. * Detach an existing `Disk` from an existing `Instance`. The first two operations will be performed using the config functions ``AddDisk`` and ``RemoveDisk`` respectively where the last two operations will be performed using the functions ``AttachInstanceDisk`` and ``DetachInstanceDisk``. More specifically, the `add` operation will add and attach a disk at the same time, using a wrapper that calls the ``AddDisk`` and ``AttachInstanceDisk`` functions. On the same vein, the `remove` operation will detach and remove a disk using a wrapper that calls the ``DetachInstanceDisk`` and ``RemoveInstanceDisk``. The `attach` and `detach` operations are simpler, in the sense that they only call the ``AttachInstanceDisk`` and ``DetachInstanceDisk`` functions respectively. It is important to note that the `detach` operation introduces the notion of disks that are not attached to any instance. For this reason, the configuration checks for detached disks will be removed, as the detached disks can be handled by the code. In addition since Ganeti doesn't allow for a `Disk` object to be attached to more than one `Instance` at once, when attaching a disk to an instance we have to make sure that the disk is not attached anywhere else. Backend changes ~~~~~~~~~~~~~~~ The backend needs access to the disks of an `Instance` but doesn't have access to the `GetInstanceDisks` function from the config file. Thus we will create a new `Instance` slot (namely ``disks_info``) that will get annotated (during RPC) with the instance's disk objects. So in the backend we will only have to replace the ``disks`` slot with ``disks_info``. Supporting the old interface ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The current interface is designed with a uniform disk type in mind and this interface should still be supported to not break tools and workflows downstream. The behaviour is fully compatible for instances with constantly attached, uniform disks. Whenever an operation operates on an instance, the operation will only consider the disks attached. If the operation is specific to a disk type, it will throw an error if any disks of a type not supported are attached. When setting the disk template of an instance, we convert all currently attached disks to that template. This means that all disk types currently attached must be convertible to the new template. Since the disk template as a configuration value is going away, it needs to be replaced for queries. If the instance has no disks, the disk_template will be 'diskless', if it has disks of a single type, its disk_template will be that type, and if it has disks of multiple types, the new disk template 'mixed' will be returned. Eliminating the disk template from the instance ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In order to remove the disk template from the instance model, all current uses of the disk template there need to be replaced. These uses fall into the following general categories: 1. The configuration needs to reflect the new model. `cfgupgrade` and `bootstrap` need to be fixed, creating and modifying instances and disks for instances needs to be fixed. 2. The query interface will no longer be able to return an instance disk template. 3. Several checks for the DISKLESS template will be replaced by checking if any disks are attached. 4. If an operation works disk by disk, the operation will dispatch for the functionality by disk instead of by instance. If an operation requires that all disks are of the same kind (e.g. a query if the instance is DRBD backed) then the assumption is checked beforehand. Since this is a user visible change, it will have to be announced in the NEWS file specifying the calls changed. 5. Operations that operate on the instance and extract the disk template e.g. for creation of a new disk will require an additional parameter for the disk template. Several instances already provide an optional parameter to override the instance setting, those will become required. This is incompatible as well and will need to be listed in the NEWS file. Attach/Detach disks from cli ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `attach`/`detach` options should be available through the command ``gnt-instance modify``. Like the `add`/`remove` options, the `attach`/`detach` options can be invoked using the legacy syntax or the new syntax that supports indexes. For the attach option, we can refer to the disk using either its `name` or `uuid`. The detach option on the other hand has the same syntax as the remove option, and we can refer to a disk by its `name`, `uuid` or `index` in the instance. The attach/detach syntax can be seen below: * **Legacy syntax** .. code-block:: bash gnt-instance modify --disk attach,name=*NAME* *INSTANCE* gnt-instance modify --disk attach,uuid=*UUID* *INSTANCE* gnt-instance modify --disk detach *INSTANCE* * **New syntax** .. code-block:: bash gnt-instance modify --disk *N*:attach,name=*NAME* *INSTANCE* gnt-instance modify --disk *N*:attach,uuid=*UUID* *INSTANCE* gnt-instance modify --disk *N*:detach *INSTANCE* gnt-instance modify --disk *NAME*:detach *INSTANCE* gnt-instance modify --disk *UUID*:detach *INSTANCE* .. TODO: Locks for Disk objects .. TODO: LUs for disks .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-draft.rst000064400000000000000000000010201476477700300174430ustar00rootroot00000000000000====================== Design document drafts ====================== .. Last updated for Ganeti 3.1 .. toctree:: :maxdepth: 2 design-x509-ca.rst design-http-server.rst design-impexp2.rst design-hugepages-support.rst design-ifdown.rst design-reservations.rst design-sync-rate-throttling.rst design-network2.rst design-multi-storage-htools.rst design-repaird.rst design-scsi-kvm.rst design-disks.rst .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-file-based-disks-ownership.rst000064400000000000000000000054751476477700300235070ustar00rootroot00000000000000================================= Ganeti file-based disks ownership ================================= :Created: 2015-Jan-20 :Status: Implemented :Ganeti-Version: 2.14.0 .. contents:: :depth: 2 This design document explains the issue that emerges from the usage of the `detach` operation to file-based disks and provides a simple solution to it. Note that this design document applies only to disks of template `file` and `sharedfile`, but not `gluster`. However, for brevity reasons these templates will go under the umbrella term `file-based`. Current state and shortcomings ============================== When creating a file-based disk, Ganeti stores it inside a specific directory, called `file_storage_dir`. Inside this directory, there is a folder for each file-based instance and inside each folder are the files for the instance's disks (e.g. ``//``). This way of storing disks seems simple enough, but the `detach` operation does not work well with it. The reason is that if a disk is detached from an instance and attached to another one, the file will remain to the folder of the original instance. This means that if we try to destroy an instance with detached disks, Ganeti will correctly complain that the instance folder still has disk data. In more high-level terms, we need to find a way to resolve the issue of disk ownership at the filesystem level for file-based instances. Proposed changes ================ The change we propose is simple. Once a disk is detached from an instance, it will be moved out of the instance's folder. The new location will be the `file_storage_dir`, i.e. the disk will reside on the same level as the instance folders. In order to maintain a consistent configuration, the logical_id of the disk will be updated to point to the new path. Similarly, on the `attach` operation, the file name and logical id will change and the disk will be moved under the new instance's directory. Implementation details ====================== Detach operation ~~~~~~~~~~~~~~~~ Before detaching a disk from an instance, we do the following: 1. Transform the current path to the new one. // --> / 2. Use the rpc call ``call_blockdev_rename`` to move the disk to the new path. 3. Store the new ``logical_id`` to the configuration. Attach operation ~~~~~~~~~~~~~~~~ Before attaching a disk to an instance, we do the following: 1. Create the new path for the file disk. In order to construct it properly, use the ``GenerateDiskTemplate`` function to create a dummy disk template and get its ``logical_id``. The new ``logical_id`` contains the new path for the file disk. 2. Use the rpc call ``call_blockdev_rename`` to move the disk to the new path. 3. Store the new ``logical_id`` to the configuration. ganeti-3.1.0~rc2/doc/design-file-based-storage.rst000064400000000000000000000237521476477700300220200ustar00rootroot00000000000000================== File-based Storage ================== :Created: 2014-Jan-27 :Status: Implemented :Ganeti-Version: 2.0.0 This page describes the proposed file-based storage for the 2.0 version of Ganeti. The project consists in extending Ganeti in order to support a filesystem image as Virtual Block Device (VBD) in Dom0 as the primary storage for a VM. Objective ========= Goals: * file-based storage for virtual machines running in a Xen-based Ganeti cluster * failover of file-based virtual machines between cluster-nodes * export/import file-based virtual machines * reuse existing image files * allow Ganeti to initialize the cluster without checking for a volume group (e.g. xenvg) Non Goals: * any kind of data mirroring between clusters for file-based instances (this should be achieved by using shared storage) * special support for live-migration * encryption of VBDs * compression of VBDs Background ========== Ganeti is a virtual server management software tool built on top of Xen VM monitor and other Open Source software. Since Ganeti currently supports only block devices as storage backend for virtual machines, the wish came up to provide a file-based backend. Using this file-based option provides the possibility to store the VBDs on basically every filesystem and therefore allows to deploy external data storages (e.g. SAN, NAS, etc.) in clusters. Overview ======== Introduction ++++++++++++ Xen (and other hypervisors) provide(s) the possibility to use a file as the primary storage for a VM. One file represents one VBD. Advantages/Disadvantages ++++++++++++++++++++++++ Advantages of file-backed VBD: * support of sparse allocation * easy from a management/backup point of view (e.g. you can just copy the files around) * external storage (e.g. SAN, NAS) can be used to store VMs Disadvantages of file-backed VBD: * possible performance loss for I/O-intensive workloads * using sparse files requires care to ensure the sparseness is preserved when copying, and there is no header in which metadata relating back to the VM can be stored Xen-related specifications ++++++++++++++++++++++++++ Driver ~~~~~~ There are several ways to realize the required functionality with an underlying Xen hypervisor. 1) loopback driver ^^^^^^^^^^^^^^^^^^ Advantages: * available in most precompiled kernels * stable, since it is in kernel tree for a long time * easy to set up Disadvantages: * buffer writes very aggressively, which can affect guest filesystem correctness in the event of a host crash * can even cause out-of-memory kernel crashes in Dom0 under heavy write load * substantial slowdowns under heavy I/O workloads * the default number of supported loopdevices is only 8 * doesn't support QCOW files ``blktap`` driver ^^^^^^^^^^^^^^^^^ Advantages: * higher performance than loopback driver * more scalable * better safety properties for VBD data * Xen-team strongly encourages use * already in Xen tree * supports QCOW files * asynchronous driver (i.e. high performance) Disadvantages: * not enabled in most precompiled kernels * stable, but not as much tested as loopback driver 3) ublkback driver ^^^^^^^^^^^^^^^^^^ The Xen Roadmap states "Work is well under way to implement a ``ublkback`` driver that supports all of the various qemu file format plugins". Furthermore, the Roadmap includes the following: "... A special high-performance qcow plugin is also under development, that supports better metadata caching, asynchronous IO, and allows request reordering with appropriate safety barriers to enforce correctness. It remains both forward and backward compatible with existing qcow disk images, but makes adjustments to qemu's default allocation policy when creating new disks such as to optimize performance." File types ~~~~~~~~~~ Raw disk image file ^^^^^^^^^^^^^^^^^^^ Advantages: * Resizing supported * Sparse file (filesystem dependend) * simple and easily exportable Disadvantages: * Underlying filesystem needs to support sparse files (most filesystems do, though) QCOW disk image file ^^^^^^^^^^^^^^^^^^^^ Advantages: * Smaller file size, even on filesystems which don't support holes (i.e. sparse files) * Snapshot support, where the image only represents changes made to an underlying disk image * Optional zlib based compression * Optional AES encryption Disadvantages: * Resizing not supported yet (it's on the way) VMDK disk image file ^^^^^^^^^^^^^^^^^^^^ This file format is directly based on the qemu vmdk driver, which is synchronous and thus slow. Detailed Design =============== Terminology +++++++++++ * **VBD** (Virtual Block Device): Persistent storage available to a virtual machine, providing the abstraction of an actual block storage device. VBDs may be actual block devices, filesystem images, or remote/network storage. * **Dom0** (Domain 0): The first domain to be started on a Xen machine. Domain 0 is responsible for managing the system. * **VM** (Virtual Machine): The environment in which a hosted operating system runs, providing the abstraction of a dedicated machine. A VM may be identical to the underlying hardware (as in full virtualization, or it may differ, as in paravirtualization). In the case of Xen the domU (unprivileged domain) instance is meant. * **QCOW**: QEMU (a processor emulator) image format. Implementation ++++++++++++++ Managing file-based instances ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The option for file-based storage will be added to the 'gnt-instance' utility. Add Instance ^^^^^^^^^^^^ Example: gnt-instance add -t file:[path\ =[,driver=loop[,reuse[,...]]]] \ --disk 0:size=5G --disk 1:size=10G -n node -o debian-etch instance2 This will create a file-based instance with e.g. the following files: * ``/sda`` -> 5GB * ``/sdb`` -> 10GB The default directory where files will be stored is ``/srv/ganeti/file-storage/``. This can be changed by setting the ```` option. This option denotes the full path to the directory where the files are stored. The filetype will be "raw" for the first release of Ganeti 2.0. However, the code will be extensible to more file types, since Ganeti will store information about the file type of each image file. Internally Ganeti will keep track of the used driver, the file-type and the full path to the file for every VBD. Example: "logical_id" : ``[FD_LOOP, FT_RAW, "/instance1/sda"]`` If the ``--reuse`` flag is set, Ganeti checks for existing files in the corresponding directory (e.g. ``/xen/instance2/``). If one or more files in this directory are present and correctly named (the naming conventions will be defined in Ganeti version 2.0) Ganeti will set a VM up with these. If no file can be found or the names or invalid the operation will be aborted. Remove instance ^^^^^^^^^^^^^^^ The instance removal will just differ from the actual one by deleting the VBD-files instead of the corresponding block device (e.g. a logical volume). Starting/Stopping Instance ^^^^^^^^^^^^^^^^^^^^^^^^^^ Here nothing has to be changed, as the xen tools don't differentiate between file-based or blockdevice-based instances in this case. Export/Import instance ^^^^^^^^^^^^^^^^^^^^^^ Provided "dump/restore" is used in the "export" and "import" guest-os scripts, there are no modifications needed when file-based instances are exported/imported. If any other backup-tool (which requires access to the mounted file-system) is used then the image file can be temporarily mounted. This can be done in different ways: Mount a raw image file via loopback driver:: mount -o loop /srv/ganeti/file-storage/instance1/sda1 /mnt/disk\ Mount a raw image file via blkfront driver (Dom0 kernel needs this module to do the following operation):: xm block-attach 0 tap:aio:/srv/ganeti/file-storage/instance1/sda1 /dev/xvda1 w 0\ mount /dev/xvda1 /mnt/disk Mount a qcow image file via blkfront driver (Dom0 kernel needs this module to do the following operation) xm block-attach 0 tap:qcow:/srv/ganeti/file-storage/instance1/sda1 /dev/xvda1 w 0 mount /dev/xvda1 /mnt/disk High availability features with file-based instances ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Failing over an instance ^^^^^^^^^^^^^^^^^^^^^^^^ Failover is done in the same way as with block device backends. The instance gets stopped on the primary node and started on the secondary. The roles of primary and secondary get swapped. Note: If a failover is done, Ganeti will assume that the corresponding VBD(s) location (i.e. directory) is the same on the source and destination node. In case one or more corresponding file(s) are not present on the destination node, Ganeti will abort the operation. Replacing an instance disks ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Since there is no data mirroring for file-backed VM there is no such operation. Evacuation of a node ^^^^^^^^^^^^^^^^^^^^ Since there is no data mirroring for file-backed VMs there is no such operation. Live migration ^^^^^^^^^^^^^^ Live migration is possible using file-backed VBDs. However, the administrator has to make sure that the corresponding files are exactly the same on the source and destination node. Xen Setup +++++++++ File creation ~~~~~~~~~~~~~ Creation of a raw file is simple. Example of creating a sparse file of 2 Gigabytes. The option "seek" instructs "dd" to create a sparse file:: dd if=/dev/zero of=vm1disk bs=1k seek=2048k count=1 Creation of QCOW image files can be done with the "qemu-img" utility (in debian it comes with the "qemu" package). Config file ~~~~~~~~~~~ The Xen config file will have the following modification if one chooses the file-based disk-template. 1) loopback driver and raw file ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: disk = ['file:,sda1,w'] 2) blktap driver and raw file ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: disk = ['tap:aio:,sda1,w'] 3) blktap driver and qcow file ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: disk = ['tap:qcow:,sda1,w'] Other hypervisors +++++++++++++++++ Other hypervisors have mostly different ways to make storage available to their virtual instances/machines. This is beyond the scope of this document. ganeti-3.1.0~rc2/doc/design-glusterfs-ganeti-support.rst000064400000000000000000000312471476477700300233560ustar00rootroot00000000000000======================== GlusterFS Ganeti support ======================== :Created: 2013-Jun-24 :Status: Implemented :Ganeti-Version: 2.11.0 This document describes the plan for adding GlusterFS support inside Ganeti. .. contents:: :depth: 4 .. highlight:: shell-example Gluster overview ================ Gluster is a "brick" "translation" service that can turn a number of LVM logical volume or disks (so-called "bricks") into an unified "volume" that can be mounted over the network through FUSE or NFS. This is a simplified view of what components are at play and how they interconnect as data flows from the actual disks to the instances. The parts in grey are available for Ganeti to use and included for completeness but not targeted for implementation at this stage. .. digraph:: "gluster-ganeti-overview" graph [ spline=ortho ] node [ shape=rect ] { node [ shape=none ] _volume [ label=volume ] bricks -> translators -> _volume _volume -> network [label=transport] network -> instances } { rank=same; brick1 [ shape=oval ] brick2 [ shape=oval ] brick3 [ shape=oval ] bricks } { rank=same; translators distribute } { rank=same; volume [ shape=oval ] _volume } { rank=same; instances instanceA instanceB instanceC instanceD } { rank=same; network FUSE NFS QEMUC QEMUD } { node [ shape=oval ] brick1 [ label=brick ] brick2 [ label=brick ] brick3 [ label=brick ] } { node [ shape=oval ] volume } brick1 -> distribute brick2 -> distribute brick3 -> distribute -> volume volume -> FUSE [ label=UDP> color="black:grey" ] NFS [ color=grey fontcolor=grey ] volume -> NFS [ label="TCP" color=grey fontcolor=grey ] NFS -> mountpoint [ color=grey fontcolor=grey ] mountpoint [ shape=oval ] FUSE -> mountpoint instanceA [ label=instances ] instanceB [ label=instances ] mountpoint -> instanceA mountpoint -> instanceB mountpoint [ shape=oval ] QEMUC [ label=QEMU ] QEMUD [ label=QEMU ] { instanceC [ label=instances ] instanceD [ label=instances ] } volume -> QEMUC [ label=UDP> color="black:grey" ] volume -> QEMUD [ label=UDP> color="black:grey" ] QEMUC -> instanceC QEMUD -> instanceD brick: The unit of storage in gluster. Typically a drive or LVM logical volume formatted using, for example, XFS. distribute: One of the translators in Gluster, it assigns files to bricks based on the hash of their full path inside the volume. volume: A filesystem you can mount on multiple machines; all machines see the same directory tree and files. FUSE/NFS: Gluster offers two ways to mount volumes: through FUSE or a custom NFS server that is incompatible with other NFS servers. FUSE is more compatible with other services running on the storage nodes; NFS gives better performance. For now, FUSE is a priority. QEMU: QEMU 1.3 has the ability to use Gluster volumes directly in userspace without the need for mounting anything. Ganeti still needs kernelspace access at disk creation and OS install time. transport: FUSE and QEMU allow you to connect using TCP and UDP, whereas NFS only supports TCP. Those protocols are called transports in Gluster. For now, TCP is a priority. It is the administrator's duty to set up the bricks, the translators and thus the volume as they see fit. Ganeti will take care of connecting the instances to a given volume. .. note:: The gluster mountpoint must be whitelisted by the administrator in ``/etc/ganeti/file-storage-paths`` for security reasons in order to allow Ganeti to modify the filesystem. Why not use a ``sharedfile`` disk template? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Gluster volumes `can` be used by Ganeti using the generic shared file disk template. There is a number of reasons why that is probably not a good idea, however: * Shared file, being a generic solution, cannot offer userspace access support. * Even with userspace support, Ganeti still needs kernelspace access in order to create disks and install OSes on them. Ganeti can manage the mounting for you so that the Gluster servers only have as many connections as necessary. * Experiments showed that you can't trust ``mount.glusterfs`` to give useful return codes or error messages. Ganeti can work around its oddities so administrators don't have to. * The shared file folder scheme (``../{instance.name}/disk{disk.id}``) does not work well with Gluster. The ``distribute`` translator distributes files across bricks, but directories need to be replicated on `all` bricks. As a result, if we have a dozen hundred instances, that means a dozen hundred folders being replicated on all bricks. This does not scale well. * This frees up the shared file disk template to use a different, unsupported replication scheme together with Gluster. (Storage pools are the long term solution for this, however.) So, while gluster `is` a shared file disk template, essentially, Ganeti can provide better support for it than that. Implementation strategy ======================= Working with GlusterFS in kernel space essentially boils down to: 1. Ask FUSE to mount the Gluster volume. 2. Check that the mount succeeded. 3. Use files stored in the volume as instance disks, just like sharedfile does. 4. When the instances are spun down, attempt unmounting the volume. If the gluster connection is still required, the mountpoint is allowed to remain. Since it is not strictly necessary for Gluster to mount the disk if all that's needed is userspace access, however, it is inappropriate for the Gluster storage class to inherit from FileStorage. So the implementation should resort to composition rather than inheritance: 1. Extract the ``FileStorage`` disk-facing logic into a ``FileDeviceHelper`` class. * In order not to further inflate bdev.py, Filestorage should join its helper functions in filestorage.py (thus reducing their visibility) and add Gluster to its own file, gluster.py. Moving the other classes to their own files like it's been done in ``lib/hypervisor/``) is not addressed as part of this design. 2. Use the ``FileDeviceHelper`` class to implement a ``GlusterStorage`` class in much the same way. 3. Add Gluster as a disk template that behaves like SharedFile in every way. 4. Provide Ganeti knowledge about what a ``GlusterVolume`` is and how to mount, unmount and reference them. * Before attempting a mount, we should check if the volume is not mounted already. Linux allows mounting partitions multiple times, but then you also have to unmount them as many times as you mounted them to actually free the resources; this also makes the output of commands such as ``mount`` less useful. * Every time the device could be released (after instance shutdown, OS installation scripts or file creation), a single unmount is attempted. If the device is still busy (e.g. from other instances, jobs or open administrator shells), the failure is ignored. 5. Modify ``GlusterStorage`` and customize the disk template behavior to fit Gluster's needs. Directory structure ~~~~~~~~~~~~~~~~~~~ In order to address the shortcomings of the generic shared file handling of instance disk directory structure, Gluster uses a different scheme for determining a disk's logical id and therefore path on the file system. The naming scheme is:: /ganeti/{instance.uuid}.{disk.id} ...bringing the actual path on a node's file system to:: /var/run/ganeti/gluster/ganeti/{instance.uuid}.{disk.id} This means Ganeti only uses one folder on the Gluster volume (allowing other uses of the Gluster volume in the meantime) and works better with how Gluster distributes storage over its bricks. Changes to the storage types system ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Ganeti has a number of storage types that abstract over disk templates. This matters mainly in terms of disk space reporting. Gluster support is improved by a rethinking of how disk templates are assigned to storage types in Ganeti. This is the summary of the changes: +--------------+---------+---------+-------------------------------------------+ | Disk | Current | New | Does it report storage information to... | | template | storage | storage +-------------+----------------+------------+ | | type | type | ``gnt-node | ``gnt-node | iallocator | | | | | list`` | list-storage`` | | +==============+=========+=========+=============+================+============+ | File | File | File | Yes. | Yes. | Yes. | +--------------+---------+---------+-------------+----------------+------------+ | Shared file | File | Shared | No. | Yes. | No. | +--------------+---------+ file | | | | | Gluster (new)| N/A | (new) | | | | +--------------+---------+---------+-------------+----------------+------------+ | RBD (for | RBD | No. | No. | No. | | reference) | | | | | +--------------+-------------------+-------------+----------------+------------+ Gluster or Shared File should not, like RBD, report storage information to gnt-node list or to IAllocators. Regrettably, the simplest way to do so right now is by claiming that storage reporting for the relevant storage type is not implemented. An effort was made to claim that the shared storage type did support disk reporting while refusing to provide any value, but it was not successful (``hail`` does not support this combination.) To do so without breaking the File disk template, a new storage type must be added. Like RBD, it does not claim to support disk reporting. However, we can still make an effort of reporting stats to ``gnt-node list-storage``. The rationale is simple. For shared file and gluster storage, disk space is not a function of any one node. If storage types with disk space reporting are used, Hail expects them to give useful numbers for allocation purposes, but a shared storage system means disk balancing is not affected by node-instance allocation any longer. Moreover, it would be wasteful to mount a Gluster volume on each node just for running statvfs() if no machine was actually running gluster VMs. As a result, Gluster support for gnt-node list-storage is necessarily limited and nodes on which Gluster is available but not in use will report failures. Additionally, running ``gnt-node list`` will give an output like this:: Node DTotal DFree MTotal MNode MFree Pinst Sinst node1.example.com ? ? 744M 273M 477M 0 0 node2.example.com ? ? 744M 273M 477M 0 0 This is expected and consistent with behaviour in RBD. An alternative would have been to report DTotal and DFree as 0 in order to allow ``hail`` to ignore the disk information, but this incorrectly populates the ``gnt-node list`` DTotal and DFree fields with 0s as well. New configuration switches ~~~~~~~~~~~~~~~~~~~~~~~~~~ Configurable at the cluster and node group level (``gnt-cluster modify``, ``gnt-group modify`` and other commands that support the `-D` switch to edit disk parameters): ``gluster:host`` The IP address or hostname of the Gluster server to connect to. In the default deployment of Gluster, that is any machine that is hosting a brick. Default: ``"127.0.0.1"`` ``gluster:port`` The port where the Gluster server is listening to. Default: ``24007`` ``gluster:volume`` The volume Ganeti should use. Default: ``"gv0"`` Configurable at the cluster level only (``gnt-cluster init``) and stored in ssconf for all nodes to read (just like shared file): ``--gluster-dir`` Where the Gluster volume should be mounted. Default: ``/var/run/ganeti/gluster`` The default values work if all of the Ganeti nodes also host Gluster bricks. This is possible, but `not` recommended as it can cause the host to hardlock due to deadlocks in the kernel memory (much in the same way RBD works). Future work =========== In no particular order: * Support the UDP transport. * Support mounting through NFS. * Filter ``gnt-node list`` so DTotal and DFree are not shown for RBD and shared file disk types, or otherwise report the disk storage values as "-" or some other special value to clearly distinguish it from the result of a communication failure between nodes. * Allow configuring the in-volume path Ganeti uses. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-hotplug.rst000064400000000000000000000251711476477700300200420ustar00rootroot00000000000000======= Hotplug ======= :Created: 2013-Jul-24 :Status: Implemented :Ganeti-Version: 2.10.0 .. contents:: :depth: 4 This is a design document detailing the implementation of device hotplugging in Ganeti. The logic used is hypervisor agnostic but still the initial implementation will target the KVM hypervisor. The implementation adds ``python-fdsend`` as a new dependency. In case it is not installed hotplug will not be possible and the user will be notified with a warning. Current state and shortcomings ============================== Currently, Ganeti supports addition/removal/modification of devices (NICs, Disks) but the actual modification takes place only after rebooting the instance. To this end an instance cannot change network, get a new disk etc. without a hard reboot. Until now, in case of KVM hypervisor, code does not name devices nor places them in specific PCI slots. Devices are appended in the KVM command and Ganeti lets KVM decide where to place them. This means that there is a possibility a device that resides in PCI slot 5, after a reboot (due to another device removal) to be moved to another PCI slot and probably get renamed too (due to udev rules, etc.). In order for a migration to succeed, the process on the target node should be started with exactly the same machine version, CPU architecture and PCI configuration with the running process. During instance creation/startup ganeti creates a KVM runtime file with all the necessary information to generate the KVM command. This runtime file is used during instance migration to start a new identical KVM process. The current format includes the fixed part of the final KVM command, a list of NICs', and hvparams dict. It does not favor easy manipulations concerning disks, because they are encapsulated in the fixed KVM command. Proposed changes ================ For the case of the KVM hypervisor, QEMU exposes 32 PCI slots to the instance. Disks and NICs occupy some of these slots. Recent versions of QEMU have introduced monitor commands that allow addition/removal of PCI devices. Devices are referenced based on their name or position on the virtual PCI bus. To be able to use these commands, we need to be able to assign each device a unique name. To keep track where each device is plugged into, we add the ``pci`` slot to Disk and NIC objects, but we save it only in runtime files, since it is hypervisor specific info. This is added for easy object manipulation and is ensured not to be written back to the config. We propose to make use of QEMU 1.7 QMP commands so that modifications to devices take effect instantly without the need for hard reboot. Upon hotplugging the PCI configuration of an instance is changed. Runtime files should be updated correspondingly. Currently this is impossible in case of disk hotplug because disks are included in command line entry of the runtime file, contrary to NICs that are correctly treated separately. We change the format of runtime files, we remove disks from the fixed KVM command and create new entry containing them only. KVM options concerning disk are generated during ``_ExecuteKVMCommand()``, just like NICs. Design decisions ================ Which should be each device ID? Currently KVM does not support arbitrary IDs for devices; supported are only names starting with a letter, max 32 chars length, and only including '.' '_' '-' special chars. For debugging purposes and in order to be more informative, device will be named after: --pci-. Who decides where to hotplug each device? As long as this is a hypervisor specific matter, there is no point for the master node to decide such a thing. Master node just has to request noded to hotplug a device. To this end, hypervisor specific code should parse the current PCI configuration (i.e. ``query-pci`` QMP command), find the first available slot and hotplug the device. Having noded to decide where to hotplug a device we ensure that no error will occur due to duplicate slot assignment (if masterd keeps track of PCI reservations and noded fails to return the PCI slot that the device was plugged into then next hotplug will fail). Where should we keep track of devices' PCI slots? As already mentioned, we must keep track of devices PCI slots to successfully migrate instances. First option is to save this info to config data, which would allow us to place each device at the same PCI slot after reboot. This would require to make the hypervisor return the PCI slot chosen for each device, and storing this information to config data. Additionally the whole instance configuration should be returned with PCI slots filled after instance start and each instance should keep track of current PCI reservations. We decide not to go towards this direction in order to keep it simple and do not add hypervisor specific info to configuration data (``pci_reservations`` at instance level and ``pci`` at device level). For the aforementioned reason, we decide to store this info only in KVM runtime files. Where to place the devices upon instance startup? QEMU has by default 4 pre-occupied PCI slots. So, hypervisor can use the remaining ones for disks and NICs. Currently, PCI configuration is not preserved after reboot. Each time an instance starts, KVM assigns PCI slots to devices based on their ordering in Ganeti configuration, i.e. the second disk will be placed after the first, the third NIC after the second, etc. Since we decided that there is no need to keep track of devices PCI slots, there is no need to change current functionality. How to deal with existing instances? Hotplug depends on runtime file manipulation. It stores there pci info and every device the kvm process is currently using. Existing files have no pci info in devices and have block devices encapsulated inside kvm_cmd entry. Thus hotplugging of existing devices will not be possible. Still migration and hotplugging of new devices will succeed. The workaround will happen upon loading kvm runtime: if we detect old style format we will add an empty list for block devices and upon saving kvm runtime we will include this empty list as well. Switching entirely to new format will happen upon instance reboot. Configuration changes --------------------- The ``NIC`` and ``Disk`` objects get one extra slot: ``pci``. It refers to PCI slot that the device gets plugged into. In order to be able to live migrate successfully, runtime files should be updated every time a live modification (hotplug) takes place. To this end we change the format of runtime files. The KVM options referring to instance's disks are no longer recorded as part of the KVM command line. Disks are treated separately, just as we treat NICs right now. We insert and remove entries to reflect the current PCI configuration. Backend changes --------------- Introduce one new RPC call: - hotplug_device(DEVICE_TYPE, ACTION, device, ...) where DEVICE_TYPE can be either NIC or Disk, and ACTION either REMOVE or ADD. Hypervisor changes ------------------ We implement hotplug on top of the KVM hypervisor. We take advantage of QEMU 1.7 QMP commands (``device_add``, ``device_del``, ``blockdev-add``, ``netdev_add``, ``netdev_del``). Since ``drive_del`` is not yet implemented in QMP we use the one of HMP. QEMU refers to devices based on their id. We use ``uuid`` to name them properly. If a device is about to be hotplugged we parse the output of ``query-pci`` and find the occupied PCI slots. We choose the first available and the whole device object is appended to the corresponding entry in the runtime file. Concerning NIC handling, we build on the top of the existing logic (first create a tap with _OpenTap() and then pass its file descriptor to the KVM process). To this end we need to pass access rights to the corresponding file descriptor over the QMP socket (UNIX domain socket). The open file is passed as a socket-level control message (SCM), using the ``fdsend`` python library. User interface -------------- The new ``--no-hotplug`` option to gnt-instance modify is introduced, which skips live modifications. Enabling hotplug ++++++++++++++++ Hotplug is enabled by default for gnt-instance modify if it is supported. For existing instance, after installing a version that supports hotplugging we have the restriction that hotplug will not be supported for existing devices. The reason is that old runtime files lack of: 1. Device pci configuration info. 2. Separate block device entry. Hotplug will be supported only for KVM in the first implementation. For all other hypervisors, backend will raise an Exception case hotplug is requested. NIC Hotplug +++++++++++ The user can add/modify/remove NICs either with hotplugging or not. If a NIC is to be added a tap is created first and configured properly with kvm-vif-bridge script. Then the instance gets a new network interface. Since there is no QEMU monitor command to modify a NIC, we modify a NIC by temporary removing the existing one and adding a new with the new configuration. When removing a NIC the corresponding tap gets removed as well. :: gnt-instance modify --net add test gnt-instance modify --net 1:mac=aa:00:00:55:44:33 test gnt-instance modify --net 1:remove test Disk Hotplug ++++++++++++ The user can add and remove disks with hotplugging or not. QEMU monitor supports resizing of disks, however the initial implementation will support only disk addition/deletion. :: gnt-instance modify --disk add:size=1G test gnt-instance modify --disk 1:remove test Dealing with chroot and uid pool (and disks in general) ------------------------------------------------------- The design so far covers all issues that arise without addressing the case where the kvm process will not run with root privileges. Specifically: - in case of chroot, the kvm process cannot see the newly created device - in case of uid pool security model, the kvm process is not allowed to access the device For NIC hotplug we address this problem by using the ``getfd`` QMP command and passing the file descriptor to the kvm process over the monitor socket using SCM_RIGHTS. For disk hotplug and in case of uid pool we can let the hypervisor code temporarily ``chown()`` the device before the actual hotplug. Still this is insufficient in case of chroot. In this case, we need to ``mknod()`` the device inside the chroot. Both workarounds can be avoided, if we make use of the ``add-fd`` QMP command, that was introduced in version 1.7. This command is the equivalent of NICs' `get-fd`` for disks and will allow disk hotplug in every case. So, if the QMP does not support the ``add-fd`` command, we will not allow disk hotplug and notify the user with the corresponding warning. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-hroller.rst000064400000000000000000000164271476477700300200330ustar00rootroot00000000000000============ HRoller tool ============ :Created: 2013-Feb-15 :Status: Implemented :Ganeti-Version: 2.8.0, 2.9.0 .. contents:: :depth: 4 This is a design document detailing the cluster maintenance scheduler, HRoller. Current state and shortcomings ============================== To enable automating cluster-wide reboots a new htool, called HRoller, was added to Ganeti starting from version 2.7. This tool helps parallelizing cluster offline maintenances by calculating which nodes are not both primary and secondary for a DRBD instance, and thus can be rebooted at the same time, when all instances are down. The way this is done is documented in the :manpage:`hroller(1)` manpage. We would now like to perform online maintenance on the cluster by rebooting nodes after evacuating their primary instances (rolling reboots). Proposed changes ================ New options ----------- - HRoller should be able to operate on single nodegroups (-G flag) or select its target node through some other mean (eg. via a tag, or a regexp). (Note that individual node selection is already possible via the -O flag, that makes hroller ignore a node altogether). - HRoller should handle non redundant instances: currently these are ignored but there should be a way to select its behavior between "it's ok to reboot a node when a non-redundant instance is on it" or "skip nodes with non-redundant instances". This will only be selectable globally, and not per instance. - Hroller will make sure to keep any instance which is up in its current state, via live migrations, unless explicitly overridden. The algorithm that will be used calculate the rolling reboot with live migrations is described below, and any override on considering the instance status will only be possible on the whole run, and not per-instance. Calculating rolling maintenances -------------------------------- In order to perform rolling maintenance we need to migrate instances off the nodes before a reboot. How this can be done depends on the instance's disk template and status: Down instances ++++++++++++++ If an instance was shutdown when the maintenance started it will be considered for avoiding contemporary reboot of its primary and secondary nodes, but will *not* be considered as a target for the node evacuation. This allows avoiding needlessly moving its primary around, since it won't suffer a downtime anyway. Note that a node with non-redundant instances will only ever be considered good for rolling-reboot if these are down (or the checking of status is overridden) *and* an explicit option to allow it is set. DRBD ++++ Each node must migrate all instances off to their secondaries, and then can either be rebooted, or the secondaries can be evacuated as well. Since currently doing a ``replace-disks`` on DRBD breaks redundancy, it's not any safer than temporarily rebooting a node with secondaries on them (citation needed). As such we'll implement for now just the "migrate+reboot" mode, and focus later on replace-disks as well. In order to do that we can use the following algorithm: 1) Compute node sets that don't contain both the primary and the secondary of any instance, and also don't contain the primary nodes of two instances that have the same node as secondary. These can be obtained by computing a coloring of the graph with nodes as vertexes and an edge between two nodes, if either condition prevents simultaneous maintenance. (This is the current algorithm of :manpage:`hroller(1)` with the extension that the graph to be colored has additional edges between the primary nodes of two instances sharing their secondary node.) 2) It is then possible to migrate in parallel all nodes in a set created at step 1, and then reboot/perform maintenance on them, and migrate back their original primaries, which allows the computation above to be reused for each following set without N+1 failures being triggered, if none were present before. See below about the actual execution of the maintenance. Non-DRBD ++++++++ All non-DRBD disk templates that can be migrated have no "secondary" concept. As such instances can be migrated to any node (in the same nodegroup). In order to do the job we can either: - Perform migrations on one node at a time, perform the maintenance on that node, and proceed (the node will then be targeted again to host instances automatically, as hail chooses targets for the instances between all nodes in a group. Nodes in different nodegroups can be handled in parallel. - Perform migrations on one node at a time, but without waiting for the first node to come back before proceeding. This allows us to continue, restricting the cluster, until no more capacity in the nodegroup is available, and then having to wait for some nodes to come back so that capacity is available again for the last few nodes. - Pre-Calculate sets of nodes that can be migrated together (probably with a greedy algorithm) and parallelize between them, with the migrate-back approach discussed for DRBD to perform the calculation only once. Note that for non-DRBD disks that still use local storage (eg. RBD and plain) redundancy might break anyway, and nothing except the first algorithm might be safe. This perhaps would be a good reason to consider managing better RBD pools, if those are implemented on top of nodes storage, rather than on dedicated storage machines. Full-Evacuation +++++++++++++++ If full evacuation of the nodes to be rebooted is desired, a simple migration is not enough for the DRBD instances. To keep the number of disk operations small, we restrict moves to ``migrate, replace-secondary``. That is, after migrating instances out of the nodes to be rebooted, replacement secondaries are searched for, for all instances that have their then secondary on one of the rebooted nodes. This is done by a greedy algorithm, refining the initial reboot partition, if necessary. Future work =========== Hroller should become able to execute rolling maintenances, rather than just calculate them. For this to succeed properly one of the following must happen: - HRoller handles rolling maintenances that happen at the same time as unrelated cluster jobs, and thus recalculates the maintenance at each step - HRoller can selectively drain the cluster so it's sure that only the rolling maintenance can be going on DRBD nodes' ``replace-disks``' functionality should be implemented. Note that when we will support a DRBD version that allows multi-secondary this can be done safely, without losing replication at any time, by adding a temporary secondary and only when the sync is finished dropping the previous one. Non-redundant (plain or file) instances should have a way to be moved off as well via plain storage live migration or ``gnt-instance move`` (which requires downtime). If/when RBD pools can be managed inside Ganeti, care can be taken so that the pool is evacuated as well from a node before it's put into maintenance. This is equivalent to evacuating DRBD secondaries. Master failovers during the maintenance should be performed by hroller. This requires RPC/RAPI support for master failover. Hroller should also be modified to better support running on the master itself and continuing on the new master. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-hsqueeze.rst000064400000000000000000000133321476477700300202050ustar00rootroot00000000000000============= HSqueeze tool ============= :Created: 2013-Oct-14 :Status: Implemented :Ganeti-Version: 2.11.0, 2.12.0, 2.13.0 .. contents:: :depth: 4 This is a design document detailing the node-freeing scheduler, HSqueeze. Current state and shortcomings ============================== Externally-mirrored instances can be moved between nodes at low cost. Therefore, it is attractive to free up nodes and power them down at times of low usage, even for small periods of time, like nights or weekends. Currently, the best way to find out a suitable set of nodes to shut down is to use the property of our balancedness metric to move instances away from drained nodes. So, one would manually drain more and more nodes and see, if `hbal` could find a solution freeing up all those drained nodes. Proposed changes ================ We propose the addition of a new htool command-line tool, called `hsqueeze`, that aims at keeping resource usage at a constant high level by evacuating and powering down nodes, or powering up nodes and rebalancing, as appropriate. By default, only externally-mirrored instances are moved, but options are provided to additionally take DRBD instances (which can be moved without downtimes), or even all instances into consideration. Tagging of standy nodes ----------------------- Powering down nodes that are technically healthy effectively creates a new node state: nodes on standby. To avoid further state proliferation, and as this information is only used by `hsqueeze`, this information is recorded in node tags. `hsqueeze` will assume that offline nodes having a tag with prefix `htools:standby:` can easily be powered on at any time. Minimum available resources --------------------------- To keep the squeezed cluster functional, a minimal amount of resources will be left available on every node. While the precise amount will be specifiable via command-line options, a sensible default is chosen, like enough resource to start an additional instance at standard allocation on each node. If the available resources fall below this limit, `hsqueeze` will, in fact, try to power on more nodes, till enough resources are available, or all standy nodes are online. To avoid flapping behavior, a second, higher, amount of reserve resources can be specified, and `hsqueeze` will only power down nodes, if after the power down this higher amount of reserve resources is still available. Computation of the set to free up --------------------------------- To determine which nodes can be powered down, `hsqueeze` basically follows the same algorithm as the manual process. It greedily goes through all non-master nodes and tries if the algorithm used by `hbal` would find a solution (with the appropriate move restriction) that frees up the extended set of nodes to be drained, while keeping enough resources free. Being based on the algorithm used by `hbal`, all restrictions respected by `hbal`, in particular memory reservation for N+1 redundancy, are also respected by `hsqueeze`. The order in which the nodes are tried is choosen by a suitable heuristics, like trying the nodes in order of increasing number of instances; the hope is that this reduces the number of instances that actually have to be moved. If the amount of free resources has fallen below the lower limit, `hsqueeze` will determine the set of nodes to power up in a similar way; it will hypothetically add more and more of the standby nodes (in some suitable order) till the algorithm used by `hbal` will finally balance the cluster in a way that enough resources are available, or all standy nodes are online. Instance moves and execution ---------------------------- Once the final set of nodes to power down is determined, the instance moves are determined by the algorithm used by `hbal`. If requested by the `-X` option, the nodes freed up are drained, and the instance moves are executed in the same way as `hbal` does. Finally, those of the freed-up nodes that do not already have a `htools:standby:` tag are tagged as `htools:standby:auto`, all free-up nodes are marked as offline and powered down via the :doc:`design-oob`. Similarly, if it is determined that nodes need to be added, then first the nodes are powered up via the :doc:`design-oob`, then they're marked as online and finally, the cluster is balanced in the same way, as `hbal` would do. For the newly powered up nodes, the `htools:standby:auto` tag, if present, is removed, but no other tags are removed (including other `htools:standby:` tags). Design choices ============== The proposed algorithm builds on top of the already present balancing algorithm, instead of greedily packing nodes as full as possible. The reason is, that in the end, a balanced cluster is needed anyway; therefore, basing on the balancing algorithm reduces the number of instance moves. Additionally, the final configuration will also benefit from all improvements to the balancing algorithm, like taking dynamic CPU data into account. We decided to have a separate program instead of adding an option to `hbal` to keep the interfaces, especially that of `hbal`, cleaner. It is not unlikely that, over time, additional `hsqueeze`-specific options might be added, specifying, e.g., which nodes to prefer for shutdown. With the approach of the `htools` of having a single binary showing different behaviors, having an additional program also does not introduce significant additional cost. We decided to have a whole prefix instead of a single tag reserved for marking standby nodes (we consider all tags starting with `htools:standby:` as serving only this purpose). This is not only in accordance with the tag reservations for other tools, but it also allows for further extension (like specifying priorities on which nodes to power up first) without changing name spaces. ganeti-3.1.0~rc2/doc/design-htools-2.3.rst000064400000000000000000000315401476477700300201650ustar00rootroot00000000000000==================================== Synchronising htools to Ganeti 2.3 ==================================== :Created: 2011-Mar-18 :Status: Implemented :Ganeti-Version: 2.5.0 Ganeti 2.3 introduces a number of new features that change the cluster internals significantly enough that the htools suite needs to be updated accordingly in order to function correctly. Shared storage support ====================== Currently, the htools algorithms presume a model where all of an instance's resources is served from within the cluster, more specifically from the nodes comprising the cluster. While is this usual for memory and CPU, deployments which use shared storage will invalidate this assumption for storage. To account for this, we need to move some assumptions from being implicit (and hardcoded) to being explicitly exported from Ganeti. New instance parameters ----------------------- It is presumed that Ganeti will export for all instances a new ``storage_type`` parameter, that will denote either internal storage (e.g. *plain* or *drbd*), or external storage. Furthermore, a new ``storage_pool`` parameter will classify, for both internal and external storage, the pool out of which the storage is allocated. For internal storage, this will be either ``lvm`` (the pool that provides space to both ``plain`` and ``drbd`` instances) or ``file`` (for file-storage-based instances). For external storage, this will be the respective NAS/SAN/cloud storage that backs up the instance. Note that for htools, external storage pools are opaque; we only care that they have an identifier, so that we can distinguish between two different pools. If these two parameters are not present, the instances will be presumed to be ``internal/lvm``. New node parameters ------------------- For each node, it is expected that Ganeti will export what storage types it supports and pools it has access to. So a classic 2.2 cluster will have all nodes supporting ``internal/lvm`` and/or ``internal/file``, whereas a new shared storage only 2.3 cluster could have ``external/my-nas`` storage. Whatever the mechanism that Ganeti will use internally to configure the associations between nodes and storage pools, we consider that we'll have available two node attributes inside htools: the list of internal and external storage pools. External storage and instances ------------------------------ Currently, for an instance we allow one cheap move type: failover to the current secondary, if it is a healthy node, and four other “expensive” (as in, including data copies) moves that involve changing either the secondary or the primary node or both. In presence of an external storage type, the following things will change: - the disk-based moves will be disallowed; this is already a feature in the algorithm, controlled by a boolean switch, so adapting external storage here will be trivial - instead of the current one secondary node, the secondaries will become a list of potential secondaries, based on access to the instance's storage pool Except for this, the basic move algorithm remains unchanged. External storage and nodes -------------------------- Two separate areas will have to change for nodes and external storage. First, then allocating instances (either as part of a move or a new allocation), if the instance is using external storage, then the internal disk metrics should be ignored (for both the primary and secondary cases). Second, the per-node metrics used in the cluster scoring must take into account that nodes might not have internal storage at all, and handle this as a well-balanced case (score 0). N+1 status ---------- Currently, computing the N+1 status of a node is simple: - group the current secondary instances by their primary node, and compute the sum of each instance group memory - choose the maximum sum, and check if it's smaller than the current available memory on this node In effect, computing the N+1 status is a per-node matter. However, with shared storage, we don't have secondary nodes, just potential secondaries. Thus computing the N+1 status will be a cluster-level matter, and much more expensive. A simple version of the N+1 checks would be that for each instance having said node as primary, we have enough memory in the cluster for relocation. This means we would actually need to run allocation checks, and update the cluster status from within allocation on one node, while being careful that we don't recursively check N+1 status during this relocation, which is too expensive. However, the shared storage model has some properties that changes the rules of the computation. Speaking broadly (and ignoring hard restrictions like tag based exclusion and CPU limits), the exact location of an instance in the cluster doesn't matter as long as memory is available. This results in two changes: - simply tracking the amount of free memory buckets is enough, cluster-wide - moving an instance from one node to another would not change the N+1 status of any node, and only allocation needs to deal with N+1 checks Unfortunately, this very cheap solution fails in case of any other exclusion or prevention factors. TODO: find a solution for N+1 checks. Node groups support =================== The addition of node groups has a small impact on the actual algorithms, which will simply operate at node group level instead of cluster level, but it requires the addition of new algorithms for inter-node group operations. The following two definitions will be used in the following paragraphs: local group The local group refers to a node's own node group, or when speaking about an instance, the node group of its primary node regular cluster A cluster composed of a single node group, or pre-2.3 cluster super cluster This term refers to a cluster which comprises multiple node groups, as opposed to a 2.2 and earlier cluster with a single node group In all the below operations, it's assumed that Ganeti can gather the entire super cluster state cheaply. Balancing changes ----------------- Balancing will move from cluster-level balancing to group balancing. In order to achieve a reasonable improvement in a super cluster, without needing to keep state of what groups have been already balanced previously, the balancing algorithm will run as follows: #. the cluster data is gathered #. if this is a regular cluster, as opposed to a super cluster, balancing will proceed normally as previously #. otherwise, compute the cluster scores for all groups #. choose the group with the worst score and see if we can improve it; if not choose the next-worst group, so on #. once a group has been identified, run the balancing for it Of course, explicit selection of a group will be allowed. Super cluster operations ++++++++++++++++++++++++ Beside the regular group balancing, in a super cluster we have more operations. Redistribution ^^^^^^^^^^^^^^ In a regular cluster, once we run out of resources (offline nodes which can't be fully evacuated, N+1 failures, etc.) there is nothing we can do unless nodes are added or instances are removed. In a super cluster however, there might be resources available in another group, so there is the possibility of relocating instances between groups to re-establish N+1 success within each group. One difficulty in the presence of both super clusters and shared storage is that the move paths of instances are quite complicated; basically an instance can move inside its local group, and to any other groups which have access to the same storage type and storage pool pair. In effect, the super cluster is composed of multiple ‘partitions’, each containing one or more groups, but a node is simultaneously present in multiple partitions, one for each storage type and storage pool it supports. As such, the interactions between the individual partitions are too complex for non-trivial clusters to assume we can compute a perfect solution: we might need to move some instances using shared storage pool ‘A’ in order to clear some more memory to accept an instance using local storage, which will further clear more VCPUs in a third partition, etc. As such, we'll limit ourselves at simple relocation steps within a single partition. Algorithm: #. read super cluster data, and exit if cluster doesn't allow inter-group moves #. filter out any groups that are “alone” in their partition (i.e. no other group sharing at least one storage method) #. determine list of healthy versus unhealthy groups: #. a group which contains offline nodes still hosting instances is definitely not healthy #. a group which has nodes failing N+1 is ‘weakly’ unhealthy #. if either list is empty, exit (no work to do, or no way to fix problems) #. for each unhealthy group: #. compute the instances that are causing the problems: all instances living on offline nodes, all instances living as secondary on N+1 failing nodes, all instances living as primaries on N+1 failing nodes (in this order) #. remove instances, one by one, until the source group is healthy again #. try to run a standard allocation procedure for each instance on all potential groups in its partition #. if all instances were relocated successfully, it means we have a solution for repairing the original group Compression ^^^^^^^^^^^ In a super cluster which has had many instance reclamations, it is possible that while none of the groups is empty, overall there is enough empty capacity that an entire group could be removed. The algorithm for “compressing” the super cluster is as follows: #. read super cluster data #. compute total *(memory, disk, cpu)*, and free *(memory, disk, cpu)* for the super-cluster #. computer per-group used and free *(memory, disk, cpu)* #. select candidate groups for evacuation: #. they must be connected to other groups via a common storage type and pool #. they must have fewer used resources than the global free resources (minus their own free resources) #. for each of these groups, try to relocate all its instances to connected peer groups #. report the list of groups that could be evacuated, or if instructed so, perform the evacuation of the group with the largest free resources (i.e. in order to reclaim the most capacity) Load balancing ^^^^^^^^^^^^^^ Assuming a super cluster using shared storage, where instance failover is cheap, it should be possible to do a load-based balancing across groups. As opposed to the normal balancing, where we want to balance on all node attributes, here we should look only at the load attributes; in other words, compare the available (total) node capacity with the (total) load generated by instances in a given group, and computing such scores for all groups, trying to see if we have any outliers. Once a reliable load-weighting method for groups exists, we can apply a modified version of the cluster scoring method to score not imbalances across nodes, but imbalances across groups which result in a super cluster load-related score. Allocation changes ------------------ It is important to keep the allocation method across groups internal (in the Ganeti/Iallocator combination), instead of delegating it to an external party (e.g. a RAPI client). For this, the IAllocator protocol should be extended to provide proper group support. For htools, the new algorithm will work as follows: #. read/receive cluster data from Ganeti #. filter out any groups that do not supports the requested storage method #. for remaining groups, try allocation and compute scores after allocation #. sort valid allocation solutions accordingly and return the entire list to Ganeti The rationale for returning the entire group list, and not only the best choice, is that we anyway have the list, and Ganeti might have other criteria (e.g. the best group might be busy/locked down, etc.) so even if from the point of view of resources it is the best choice, it might not be the overall best one. Node evacuation changes ----------------------- While the basic concept in the ``multi-evac`` iallocator mode remains unchanged (it's a simple local group issue), when failing to evacuate and running in a super cluster, we could have resources available elsewhere in the cluster for evacuation. The algorithm for computing this will be the same as the one for super cluster compression and redistribution, except that the list of instances is fixed to the ones living on the nodes to-be-evacuated. If the inter-group relocation is successful, the result to Ganeti will not be a local group evacuation target, but instead (for each instance) a pair *(remote group, nodes)*. Ganeti itself will have to decide (based on user input) whether to continue with inter-group evacuation or not. In case that Ganeti doesn't provide complete cluster data, just the local group, the inter-group relocation won't be attempted. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-http-server.rst000064400000000000000000000127371476477700300206470ustar00rootroot00000000000000========================================= Design for replacing Ganeti's HTTP server ========================================= :Created: 2011-Mar-23 :Status: Draft .. contents:: :depth: 4 .. _http-srv-shortcomings: Current state and shortcomings ------------------------------ The :doc:`new design for import/export ` depends on an HTTP server. Ganeti includes a home-grown HTTP server based on Python's ``BaseHTTPServer``. While it served us well so far, it only implements the very basics of the HTTP protocol. It is, for example, not structured well enough to support chunked transfers (:rfc:`2616`, section 3.6.1), which would have some advantages. In addition, it has not been designed for sending large responses. In the case of the node daemon the HTTP server can not easily be separated from the actual backend code and therefore must run as "root". The RAPI daemon does request parsing in the same process as talking to the master daemon via LUXI. Proposed changes ---------------- The proposal is to start using a full-fledged HTTP server in Ganeti and to run Ganeti's code as `FastCGI `_ applications. Reasons: - Simplify Ganeti's code by delegating the details of HTTP and SSL to another piece of software - Run HTTP frontend and handler backend as separate processes and users (esp. useful for node daemon, but also import/export and Remote API) - Allows implementation of :ref:`rpc-feedback` Software choice +++++++++++++++ Theoretically any server able of speaking FastCGI to a backend process could be used. However, to keep the number of steps required for setting up a new cluster at roughly the same level, the implementation will be geared for one specific HTTP server at the beginning. Support for other HTTP servers can still be implemented. After a rough selection of available HTTP servers `lighttpd `_ and `nginx `_ were the most likely candidates. Both are `widely used`_ and tested. .. _widely used: http://news.netcraft.com/archives/2011/01/12/ january-2011-web-server-survey-4.html Nginx' `original documentation `_ is in Russian, translations are `available in a Wiki `_. Nginx does not support old-style CGI programs. The author found `lighttpd's documentation `_ easier to understand and was able to configure a test server quickly. This, together with the support for more technologies, made deciding easier. With its use as a public-facing web server on a large number of websites (and possibly more behind proxies), lighttpd should be a safe choice. Unlike other webservers, such as the Apache HTTP Server, lighttpd's codebase is of manageable size. Initially the HTTP server would only be used for import/export transfers, but its use can be expanded to the Remote API and node daemon (see :ref:`rpc-feedback`). To reduce the attack surface, an option will be provided to configure services (e.g. import/export) to only listen on certain network interfaces. .. _rpc-feedback: RPC feedback ++++++++++++ HTTP/1.1 supports chunked transfers (:rfc:`2616`, section 3.6.1). They could be used to provide feedback from node daemons to the master, similar to the feedback from jobs. A good use would be to provide feedback to the user during long-running operations, e.g. downloading an instance's data from another cluster. .. _requirement: http://www.python.org/dev/peps/pep-0333/ #buffering-and-streaming WSGI 1.0 (:pep:`333`) includes the following `requirement`_: WSGI servers, gateways, and middleware **must not** delay the transmission of any block; they **must** either fully transmit the block to the client, or guarantee that they will continue transmission even while the application is producing its next block This behaviour was confirmed to work with lighttpd and the :ref:`flup ` library. FastCGI by itself has no such guarantee; webservers with buffering might require artificial padding to force the message to be transmitted. The node daemon can send JSON-encoded messages back to the master daemon by separating them using a predefined character (see :ref:`LUXI `). The final message contains the method's result. pycURL passes each received chunk to the callback set as ``CURLOPT_WRITEFUNCTION``. Once a message is complete, the master daemon can pass it to a callback function inside the job, which then decides on what to do (e.g. forward it as job feedback to the user). A more detailed design may have to be written before deciding whether to implement RPC feedback. .. _http-software-req: Software requirements +++++++++++++++++++++ - lighttpd 1.4.24 or above built with OpenSSL support (earlier versions `don't support SSL client certificates `_) - `flup `_ for FastCGI Lighttpd SSL configuration ++++++++++++++++++++++++++ .. highlight:: lighttpd The following sample shows how to configure SSL with client certificates in Lighttpd:: $SERVER["socket"] == ":443" { ssl.engine = "enable" ssl.pemfile = "server.pem" ssl.ca-file = "ca.pem" ssl.use-sslv2 = "disable" ssl.cipher-list = "HIGH:-DES:-3DES:-EXPORT:-ADH" ssl.verifyclient.activate = "enable" ssl.verifyclient.enforce = "enable" ssl.verifyclient.exportcert = "enable" ssl.verifyclient.username = "SSL_CLIENT_S_DN_CN" } .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-hugepages-support.rst000064400000000000000000000073151476477700300220420ustar00rootroot00000000000000=============================== Huge Pages Support for Ganeti =============================== :Created: 2013-Jul-17 :Status: Draft .. contents:: :depth: 4 This is a design document about implementing support for huge pages in Ganeti. (Please note that Ganeti works with Transparent Huge Pages i.e. THP and any reference in this document to Huge Pages refers to explicit Huge Pages). Current State and Shortcomings: ------------------------------- The Linux kernel allows using pages of larger size by setting aside a portion of the memory. Using larger page size may enhance the performance of applications that require a lot of memory by improving page hits. To use huge pages, memory has to be reserved beforehand. This portion of memory is subtracted from free memory and is considered as in use. Currently Ganeti cannot take proper advantage of huge pages. On a node, if huge pages are reserved and are available to fulfill the VM request, Ganeti fails to recognize huge pages and considers the memory reserved for huge pages as used memory. This leads to failure of launching VMs on a node where memory is available in the form of huge pages rather than normal pages. Proposed Changes: ----------------- The following components will be changed in order for Ganeti to take advantage of Huge Pages. Hypervisor Parameters: ---------------------- Currently, It is possible to set or modify huge pages mount point at cluster level via the hypervisor parameter ``mem_path`` as:: $ gnt-cluster init \ >--enabled-hypervisors=kvm -nic-parameters link=br100 \ > -H kvm:mem_path=/mount/point/for/hugepages This hypervisor parameter is inherited by all the instances as default although it can be overriden at the instance level. The following changes will be made to the inheritance behaviour. - The hypervisor parameter ``mem_path`` and all other hypervisor parameters will be made available at the node group level (in addition to the cluster level), so that users can set defaults for the node group:: $ gnt-group add/modify\ > -H hv:parameter=value This changes the hypervisor inheritance level as:: cluster -> group -> OS -> instance - Furthermore, the hypervisor parameter ``mem_path`` will be changeable only at the cluster or node group level and users must not be able to override this at OS or instance level. The following command must produce an error message that ``mem_path`` may only be set at either the cluster or the node group level:: $ gnt-instance add -H kvm:mem_path=/mount/point/for/hugepages Memory Pools: ------------- Memory management of Ganeti will be improved by creating separate pools for memory used by the node itself, memory used by the hypervisor and the memory reserved for huge pages as: - mtotal/xen (Xen memory) - mfree/xen (Xen unused memory) - mtotal/hp (Memory reserved for Huge Pages) - mfree/hp (Memory available from unused huge pages) - mpgsize/hp (Size of a huge page) mfree and mtotal will be changed to mean "the total and free memory for the default method in this cluster/nodegroup". Note that the default method depends both on the default hypervisor and its parameters. iAllocator Changes: ------------------- If huge pages are set as default for a cluster of node group, then iAllocator must consider the huge pages memory on the nodes, as a parameter when trying to find the best node for the VM. Note that the iallocator will also be changed to use the correct parameter depending on the cluster/group. hbal Changes: ------------- The cluster balancer (hbal) will be changed to use the default memory pool and recognize memory reserved for huge pages when trying to rebalance the cluster. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-ifdown.rst000064400000000000000000000155171476477700300176510ustar00rootroot00000000000000====================================== Design for adding ifdown script to KVM ====================================== :Created: 2014-Jun-23 :Status: Draft .. contents:: :depth: 4 This is a design document about adding support for an ifdown script responsible for deconfiguring network devices and cleanup changes made by the ifup script. The first implementation will target KVM but it could be ported to Xen as well especially when hotplug gets implemented. Current state and shortcomings ============================== Currently, before instance startup, instance migration and NIC hotplug, KVM creates a TAP interface and invokes explicitly the `kvm-ifup` script with the relevant environment (INTERFACE, MAC, IP, MODE, LINK, TAGS, and all the network info if any; NETWORK\_SUBNET, NETWORK\_TAGS, etc). For Xen we have the `vif-ganeti` script (associated with vif-script hypervisor parameter). The main difference is that Xen calls it by itself by passing it as an extra option in the configuration file. This `ifup` script can do several things; bridge a tap to a bridge, add IP rules, update a external DNS or DHCP server, enable proxy ARP or proxy NDP, issue openvswitch commands, etc. In general we can divide those actions in two categories: 1) Commands that change the state of the host. 2) Commands that change the state of external components. Currently those changes do not get cleaned up or modified upon instance shutdown, remove, migrate, or NIC hot-unplug. Thus we have stale entries in hosts and most important might have stale/invalid configuration on external components like routers that could affect connectivity. A workaround could be hooks, but: 1) During migrate hooks the environment is the one held in config data and not in runtime files. The NIC configuration might have changed on master but not on the running KVM process (unless hotplug is used). Plus the NIC order in config data might not be the same one on the KVM process. 2) On instance modification, changes are not available on hooks. With other words we do not know the configuration before and after modification. Since Ganeti is the orchestrator and is the one who explicitly configures host devices (tap, vif) it should be the one responsible for cleanup/ deconfiguration. Especially on a SDN approach this kind of script might be useful to cleanup flows in the cluster in order to ensure correct paths without ping pongs between hosts or connectivity loss for the instance. Proposed Changes ================ We add an new script, kvm-ifdown that is explicitly invoked after: 1) instance shutdown on primary node 2) successful instance migration on source node 3) failed instance migration on target node 4) successful NIC hot-remove on primary node If an administrator's custom ifdown script exists (e.g. `kvm-ifdown-custom`), the `kvm-ifdown` script executes that script, as happens with `kvm-ifup`. Along with that change we should rename custom ifup script from `kvm-vif-bridge` (which does not make any sense) to `kvm-ifup-custom`. In contrary to `kvm-ifup`, one cannot rely on `kvm-ifdown` script to be called. A node might die just after a successful migration or after an instance shutdown. In that case, all "undo" operations will not be invoked. Thus, this script should work "on a best effort basis" and the network should not rely on the script being called or being successful. Additionally it should modify *only* the node local dynamic configs (routes, arp entries, SDN, firewalls, etc.), whereas static ones (DNS, DHCP, etc.) should be modified via hooks. Implementation Details ====================== 1) Where to get the NIC info? We cannot account on config data since it might have changed. So the only place we keep our valid data is inside the runtime file. During instance modifications (NIC hot-remove, hot-modify) we have the NIC object from the RPC. We take its UUID and search for the corresponding entry in the runtime file to get further info. After instance shutdown and migration we just take all NICs from the runtime file and invoke the ifdown script for each one 2) Where to find the corresponding TAP? Currently TAP names are kept under /var/run/ganeti/kvm-hypervisor/nics//. This is not enough. As told above a NIC's index might change during instance's life. An example will make things clear: * The admin starts an instance with three NICs. * The admin removes the second without hotplug. * The admin removes the first with hotplug. The index that will arrive with the RPC will be 1 and if we read the relevant NIC file we will get the tap of the NIC that has been removed on second step but is still existing in the KVM process. So upon TAP creation we write another file with the same info but named after the NIC's UUID. The one named after its index can be left for compatibility (Ganeti does not use it; external tools might) Obviously this info will not be available for old instances in the cluster. The ifdown script should be aware of this corner case. 3) What should we cleanup/deconfigure? Upon NIC hot-remove we obviously want to wipe everything. But on instance migration we don't want to reset external configuration like DNS. So we choose to pass an extra positional argument to the ifdown script (it already has the TAP name) that will reflect the context it was invoked with. Please note that de-configuration of external components is not encouraged and should be done via hooks. Still we could easily support it via this extra argument. 4) What will be the script environment? In general the same environment passed to ifup script. Except instance's tags. Those are the only info not kept in runtime file and it can change between ifup and ifdown script execution. The ifdown script must be aware of it and should cleanup everything that ifup script might setup depending on instance tags (e.g. firewalls, etc) Configuration Changes ~~~~~~~~~~~~~~~~~~~~~ 1) The `kvm-ifdown` script will be an extra file installed under the same dir `kvm-ifup` resides. We could have a single script (and symbolic links to it) that shares the same code, where a second positional argument or an extra environment variable would define if we are bringing the interface up or down. Still this is not the best practice since it is not equivalent with how KVM uses `script` and `downscript` in the `netdev` option; scripts are different files that get the tap name as positional argument. Of course common code will go in `net-common` so that it can be sourced from either Xen or KVM specific scripts. 2) An extra file will be written upon TAP creation, named after the NIC's UUID and including the TAP's name. Since this should be the authoritative file, to keep backwards compatibility we create a symbolic link named after the NIC's index and pointing to this new file. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-impexp2.rst000064400000000000000000000545751476477700300177560ustar00rootroot00000000000000================================== Design for import/export version 2 ================================== :Created: 2011-Mar-23 :Status: Draft :Depends-On: - :doc:`design-x509-ca` - :doc:`design-http-server` .. contents:: :depth: 4 Current state and shortcomings ------------------------------ Ganeti 2.2 introduced :doc:`inter-cluster instance moves ` and replaced the import/export mechanism with the same technology. It's since shown that the chosen implementation was too complicated and and can be difficult to debug. The old implementation is henceforth called "version 1". It used ``socat`` in combination with a rather complex tree of ``bash`` and Python utilities to move instances between clusters and import/export them inside the cluster. Due to protocol limitations, the master daemon starts a daemon on the involved nodes and then keeps polling a status file for updates. A non-trivial number of timeouts ensures that jobs don't freeze. In version 1, the destination node would start a daemon listening on a random TCP port. Upon receiving the destination information, the source node would temporarily stop the instance, create snapshots, and start exporting the data by connecting to the destination. The random TCP port is chosen by the operating system by binding the socket to port 0. While this is a somewhat elegant solution, it causes problems in setups with restricted connectivity (e.g. iptables). Another issue encountered was with dual-stack IPv6 setups. ``socat`` can only listen on one protocol, IPv4 or IPv6, at a time. The connecting node can not simply resolve the DNS name, but it must be told the exact IP address. Instance OS definitions can provide custom import/export scripts. They were working well in the early days when a filesystem was usually created directly on the block device. Around Ganeti 2.0 there was a transition to using partitions on the block devices. Import/export scripts could no longer use simple ``dump`` and ``restore`` commands, but usually ended up doing raw data dumps. Proposed changes ---------------- Unlike in version 1, in version 2 the destination node will connect to the source. The active side is swapped. This design assumes the following design documents have been implemented: - :doc:`design-x509-ca` - :doc:`design-http-server` The following design is mostly targetted at inter-cluster instance moves. Intra-cluster import and export use the same technology, but do so in a less complicated way (e.g. reusing the node daemon certificate in version 1). Support for instance OS import/export scripts, which have been in Ganeti since the beginning, will be dropped with this design. Should the need arise, they can be re-added later. Software requirements +++++++++++++++++++++ - HTTP client: cURL/pycURL (already used for inter-node RPC and RAPI client) - Authentication: X509 certificates (server and client) Transport +++++++++ Instead of a home-grown, mostly raw protocol the widely used HTTP protocol will be used. Ganeti already uses HTTP for its :doc:`Remote API ` and inter-node communication. Encryption and authentication will be implemented using SSL and X509 certificates. SSL certificates ++++++++++++++++ The source machine will identify connecting clients by their SSL certificate. Unknown certificates will be refused. Version 1 created a new self-signed certificate per instance import/export, allowing the certificate to be used as a Certificate Authority (CA). This worked by means of starting a new ``socat`` instance per instance import/export. Under the version 2 model, a continuously running HTTP server will be used. This disallows the use of self-signed certificates for authentication as the CA needs to be the same for all issued certificates. See the :doc:`separate design document for more details on how the certificate authority will be implemented `. Local imports/exports will, like version 1, use the node daemon's certificate/key. Doing so allows the verification of local connections. The client's certificate can be exported to the CGI/FastCGI handler using lighttpd's ``ssl.verifyclient.exportcert`` setting. If a cluster-local import/export is being done, the handler verifies if the used certificate matches with the local node daemon key. Source ++++++ The source can be the same physical machine as the destination, another node in the same cluster, or a node in another cluster. A physical-to-virtual migration mechanism could be implemented as an alternative source. In the case of a traditional import, the source is usually a file on the source machine. For exports and remote imports, the source is an instance's raw disk data. In all cases the transported data is opaque to Ganeti. All nodes of a cluster will run an instance of Lighttpd. The configuration is automatically generated when starting Ganeti. The HTTP server is configured to listen on IPv4 and IPv6 simultaneously. Imports/exports will use a dedicated TCP port, similar to the Remote API. See the separate :ref:`HTTP server design document ` for why Ganeti's existing, built-in HTTP server is not a good choice. The source cluster is provided with a X509 Certificate Signing Request (CSR) for a key private to the destination cluster. After shutting down the instance, creating snapshots and restarting the instance the master will sign the destination's X509 certificate using the :doc:`X509 CA ` once per instance disk. Instead of using another identifier, the certificate's serial number (:ref:`never reused `) and fingerprint are used to identify incoming requests. Once ready, the master will call an RPC method on the source node and provide it with the input information (e.g. file paths or block devices) and the certificate identities. The RPC method will write the identities to a place accessible by the HTTP request handler, generate unique transfer IDs and return them to the master. The transfer ID could be a filename containing the certificate's serial number, fingerprint and some disk information. The file containing the per-transfer information is signed using the node daemon key and the signature written to a separate file. Once everything is in place, the master sends the certificates, the data and notification URLs (which include the transfer IDs) and the public part of the source's CA to the job submitter. Like in version 1, everything will be signed using the cluster domain secret. Upon receiving a request, the handler verifies the identity and continues to stream the instance data. The serial number and fingerprint contained in the transfer ID should be matched with the certificate used. If a cluster-local import/export was requested, the remote's certificate is verified with the local node daemon key. The signature of the information file from which the handler takes the path of the block device (and more) is verified using the local node daemon certificate. There are two options for handling requests, :ref:`CGI ` and :ref:`FastCGI `. To wait for all requests to finish, the master calls another RPC method. The destination should notify the source once it's done with downloading the data. Since this notification may never arrive (e.g. network issues), an additional timeout needs to be used. There is no good way to avoid polling as the HTTP requests will be handled asynchronously in another process. Once, and if, implemented :ref:`RPC feedback ` could be used to combine the two RPC methods. Upon completion of the transfer requests, the instance is removed if requested. .. _lighttpd-cgi-opt: Option 1: CGI ~~~~~~~~~~~~~ While easier to implement, this option requires the HTTP server to either run as "root" or a so-called SUID binary to elevate the started process to run as "root". The export data can be sent directly to the HTTP server without any further processing. .. _lighttpd-fastcgi-opt: Option 2: FastCGI ~~~~~~~~~~~~~~~~~ Unlike plain CGI, FastCGI scripts are run separately from the webserver. The webserver talks to them via a Unix socket. Webserver and scripts can run as separate users. Unlike for CGI, there are almost no bootstrap costs attached to each request. The FastCGI protocol requires data to be sent in length-prefixed packets, something which wouldn't be very efficient to do in Python for large amounts of data (instance imports/exports can be hundreds of gigabytes). For this reason the proposal is to use a wrapper program written in C (e.g. `fcgiwrap `_) and to write the handler like an old-style CGI program with standard input/output. If data should be copied from a file, ``cat``, ``dd`` or ``socat`` can be used (see note about :ref:`sendfile(2)/splice(2) with Python `). The bootstrap cost associated with starting a Python interpreter for a disk export is expected to be negligible. The `spawn-fcgi `_ program will be used to start the CGI wrapper as "root". FastCGI is, in the author's opinion, the better choice as it allows user separation. As a first implementation step the export handler can be run as a standard CGI program. User separation can be implemented as a second step. Destination +++++++++++ The destination can be the same physical machine as the source, another node in the same cluster, or a node in another cluster. While not considered in this design document, instances could be exported from the cluster by implementing an external client for exports. For traditional exports the destination is usually a file on the destination machine. For imports and remote exports, the destination is an instance's disks. All transported data is opaque to Ganeti. Before an import can be started, an RSA key and corresponding Certificate Signing Request (CSR) must be generated using the new opcode ``OpInstanceImportPrepare``. The returned information is signed using the cluster domain secret. The RSA key backing the CSR must not leave the destination cluster. After being passed through a third party, the source cluster will generate signed certificates from the CSR. Once the request for creating the instance arrives at the master daemon, it'll create the instance and call an RPC method on the instance's primary node to download all data. The RPC method does not return until the transfer is complete or failed (see :ref:`EXP_SIZE_FD ` and :ref:`RPC feedback `). The node will use pycURL to connect to the source machine and identify itself with the signed certificate received. pycURL will be configured to write directly to a file descriptor pointing to either a regular file or block device. The file descriptor needs to point to the correct offset for resuming downloads. Using cURL's multi interface, more than one transfer can be made at the same time. While parallel transfers are used by the version 1 import/export, it can be decided at a later time whether to use them in version 2 too. More investigation is necessary to determine whether ``CURLOPT_MAXCONNECTS`` is enough to limit the number of connections or whether more logic is necessary. If a transfer fails before it's finished (e.g. timeout or network issues) it should be retried using an exponential backoff delay. The opcode submitter can specify for how long the transfer should be retried. At the end of a transfer, successful or not, the source cluster must be notified. A the same time the RSA key needs to be destroyed. Support for HTTP proxies can be implemented by setting ``CURLOPT_PROXY``. Proxies could be used for moving instances in/out of restricted network environments or across protocol borders (e.g. IPv4 networks unable to talk to IPv6 networks). The big picture for instance moves ---------------------------------- #. ``OpInstanceImportPrepare`` (destination cluster) Create RSA key and CSR (certificate signing request), return signed with cluster domain secret. #. ``OpBackupPrepare`` (source cluster) Becomes a no-op in version 2, but see :ref:`backwards-compat`. #. ``OpBackupExport`` (source cluster) - Receives destination cluster's CSR, verifies signature using cluster domain secret. - Creates certificates using CSR and :doc:`cluster CA `, one for each disk - Stop instance, create snapshots, start instance - Prepare HTTP resources on node - Send certificates, URLs and CA certificate to job submitter using feedback mechanism - Wait for all transfers to finish or fail (with timeout) - Remove snapshots #. ``OpInstanceCreate`` (destination cluster) - Receives certificates signed by destination cluster, verifies certificates and URLs using cluster domain secret Note that the parameters should be implemented in a generic way allowing future extensions, e.g. to download disk images from a public, remote server. The cluster domain secret allows Ganeti to check data received from a third party, but since this won't work with such extensions, other checks will have to be designed. - Create block devices - Download every disk from source, verified using remote's CA and authenticated using signed certificates - Destroy RSA key and certificates - Start instance .. TODO: separate create from import? .. _impexp2-http-resources: HTTP resources on source ------------------------ The HTTP resources listed below will be made available by the source machine. The transfer ID is generated while preparing the export and is unique per disk and instance. No caching should be used and the ``Pragma`` (HTTP/1.0) and ``Cache-Control`` (HTTP/1.1) headers set accordingly by the server. ``GET /transfers/[transfer_id]/contents`` Dump disk contents. Important request headers: ``Accept`` (:rfc:`2616`, section 14.1) Specify preferred media types. Only one type is supported in the initial implementation: ``application/octet-stream`` Request raw disk content. If support for more media types were to be implemented in the future, the "q" parameter used for "indicating a relative quality factor" needs to be used. In the meantime parameters need to be expected, but can be ignored. If support for OS scripts were to be re-added in the future, the MIME type ``application/x-ganeti-instance-export`` is hereby reserved for disk dumps using an export script. If the source can not satisfy the request the response status code will be 406 (Not Acceptable). Successful requests will specify the used media type using the ``Content-Type`` header. Unless only exactly one media type is requested, the client must handle the different response types. ``Accept-Encoding`` (:rfc:`2616`, section 14.3) Specify desired content coding. Supported are ``identity`` for uncompressed data, ``gzip`` for compressed data and ``*`` for any. The response will include a ``Content-Encoding`` header with the actual coding used. If the client specifies an unknown coding, the response status code will be 406 (Not Acceptable). If the client specifically needs compressed data (see :ref:`impexp2-compression`) but only gets ``identity``, it can either compress locally or abort the request. ``Range`` (:rfc:`2616`, section 14.35) Raw disk dumps can be resumed using this header (e.g. after a network issue). If this header was given in the request and the source supports resuming, the status code of the response will be 206 (Partial Content) and it'll include the ``Content-Range`` header as per :rfc:`2616`. If it does not support resuming or the request was not specifying a range, the status code will be 200 (OK). Only a single byte range is supported. cURL does not support ``multipart/byteranges`` responses by itself. Even if they could be somehow implemented, doing so would be of doubtful benefit for import/export. For raw data dumps handling ranges is pretty straightforward by just dumping the requested range. cURL will fail with the error code ``CURLE_RANGE_ERROR`` if a request included a range but the server can't handle it. The request must be retried without a range. ``POST /transfers/[transfer_id]/done`` Use this resource to notify the source when transfer is finished (even if not successful). The status code will be 204 (No Content). Code samples ------------ pycURL to file ++++++++++++++ .. highlight:: python The following code sample shows how to write downloaded data directly to a file without pumping it through Python:: curl = pycurl.Curl() curl.setopt(pycurl.URL, "http://www.google.com/") curl.setopt(pycurl.WRITEDATA, open("googlecom.html", "w")) curl.perform() This works equally well if the file descriptor is a pipe to another process. .. _backwards-compat: Backwards compatibility ----------------------- .. _backwards-compat-v1: Version 1 +++++++++ The old inter-cluster import/export implementation described in the :doc:`Ganeti 2.2 design document ` will be supported for at least one minor (2.x) release. Intra-cluster imports/exports will use the new version right away. .. _exp-size-fd: ``EXP_SIZE_FD`` +++++++++++++++ Together with the improved import/export infrastructure Ganeti 2.2 allowed instance export scripts to report the expected data size. This was then used to provide the user with an estimated remaining time. Version 2 no longer supports OS import/export scripts and therefore ``EXP_SIZE_FD`` is no longer needed. .. _impexp2-compression: Compression +++++++++++ Version 1 used explicit compression using ``gzip`` for transporting data, but the dumped files didn't use any compression. Version 2 will allow the destination to specify which encoding should be used. This way the transported data is already compressed and can be directly used by the client (see :ref:`impexp2-http-resources`). The cURL option ``CURLOPT_ENCODING`` can be used to set the ``Accept-Encoding`` header. cURL will not decompress received data when ``CURLOPT_HTTP_CONTENT_DECODING`` is set to zero (if another HTTP client library were used which doesn't support disabling transparent compression, a custom content-coding type could be defined, e.g. ``x-ganeti-gzip``). Notes ----- The HTTP/1.1 protocol (:rfc:`2616`) defines trailing headers for chunked transfers in section 3.6.1. This could be used to transfer a checksum at the end of an import/export. cURL supports trailing headers since version 7.14.1. Lighttpd doesn't seem to support them for FastCGI, but they appear to be usable in combination with an NPH CGI (No Parsed Headers). .. _lighttp-sendfile: Lighttpd allows FastCGI applications to send the special headers ``X-Sendfile`` and ``X-Sendfile2`` (the latter with a range). Using these headers applications can send response headers and tell the webserver to serve regular file stored on the file system as a response body. The webserver will then take care of sending that file. Unfortunately this mechanism is restricted to regular files and can not be used for data from programs, neither direct nor via named pipes, without writing to a file first. The latter is not an option as instance data can be very large. Theoretically ``X-Sendfile`` could be used for sending the input for a file-based instance import, but that'd require the webserver to run as "root". .. _python-sendfile: Python does not include interfaces for the ``sendfile(2)`` or ``splice(2)`` system calls. The latter can be useful for faster copying of data between file descriptors. There are some 3rd-party modules (e.g. http://pypi.python.org/pypi/py-sendfile/) and discussions (http://bugs.python.org/issue10882) for including support for ``sendfile(2)``, but the later is certainly not going to happen for the Python versions supported by Ganeti. Calling the function using the ``ctypes`` module might be possible. Performance considerations -------------------------- The design described above was confirmed to be one of the better choices in terms of download performance with bigger block sizes. All numbers were gathered on the same physical machine with a single CPU and 1 GB of RAM while downloading 2 GB of zeros read from ``/dev/zero``. ``wget`` (version 1.10.2) was used as the client, ``lighttpd`` (version 1.4.28) as the server. The numbers in the first line are in megabytes per second. The second line in each row is the CPU time spent in userland respective system (measured for the CGI/FastCGI program using ``time -v``). .. highlight:: none :: ---------------------------------------------------------------------- Block size 4 KB 64 KB 128 KB 1 MB 4 MB ====================================================================== Plain CGI script reading 83 174 180 122 120 from ``/dev/zero`` 0.6/3.9 0.1/2.4 0.1/2.2 0.0/1.9 0.0/2.1 ---------------------------------------------------------------------- FastCGI with ``fcgiwrap``, 86 167 170 177 174 ``dd`` reading from ``/dev/zero`` 1.1/5 0.5/2.9 0.5/2.7 0.7/3.1 0.7/2.8 ---------------------------------------------------------------------- FastCGI with ``fcgiwrap``, 68 146 150 170 170 Python script copying from ``/dev/zero`` to stdout 1.3/5.1 0.8/3.7 0.7/3.3 0.9/2.9 0.8/3 ---------------------------------------------------------------------- FastCGI, Python script using 31 48 47 5 1 ``flup`` library (version 1.0.2) reading from ``/dev/zero`` 23.5/9.8 14.3/8.5 16.1/8 - - ---------------------------------------------------------------------- It should be mentioned that the ``flup`` library is not implemented in the most efficient way, but even with some changes it doesn't get much faster. It is fine for small amounts of data, but not for huge transfers. Other considered solutions -------------------------- Another possible solution considered was to use ``socat`` like version 1 did. Due to the changing model, a large part of the code would've required a rewrite anyway, while still not fixing all shortcomings. For example, ``socat`` could still listen on only one protocol, IPv4 or IPv6. Running two separate instances might have fixed that, but it'd get more complicated. Using an existing HTTP server will provide us with a number of other benefits as well, such as easier user separation between server and backend. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-internal-shutdown.rst000064400000000000000000000144711476477700300220460ustar00rootroot00000000000000============================================================ Detection of user-initiated shutdown from inside an instance ============================================================ :Created: 2011-Mar-23 :Status: Partially Implemented :Ganeti-Version: 2.10.0, 2.11.0 .. contents:: :depth: 2 This is a design document detailing the implementation of a way for Ganeti to detect whether an instance marked as up but not running was shutdown gracefully by the user from inside the instance itself. Current state and shortcomings ============================== Ganeti keeps track of the desired status of instances in order to be able to take proper action (e.g.: reboot) on the instances that happen to crash. Currently, the only way to properly shut down an instance is through Ganeti's own commands, which can be used to mark an instance as ``ADMIN_down``. If a user shuts down an instance from inside, through the proper command of the operating system it is running, the instance will be shutdown gracefully, but Ganeti is not aware of that: the desired status of the instance will still be marked as ``running``, so when the watcher realises that the instance is down, it will restart it. This behaviour is usually not what the user expects. Proposed changes ================ We propose to modify Ganeti in such a way that it will detect when an instance was shutdown as a result of an explicit request from the user. When such a situation is detected, instead of presenting an error as it happens now, either the state of the instance will be set to ``ADMIN_down``, or the instance will be automatically rebooted, depending on an instance-specific configuration value. The default behavior in case no such parameter is found will be to follow the apparent will of the user, and setting to ``ADMIN_down`` an instance that was shut down correctly from inside. The rest of this design document details the implementation of instance shutdown detection for Xen. The KVM implementation is detailed in :doc:`design-kvmd`. Implementation ============== Xen knows why a domain is being shut down (a crash or an explicit shutdown or poweroff request), but such information is not usually readily available externally, because all such cases lead to the virtual machine being destroyed immediately after the event is detected. Still, Xen allows the instance configuration file to define what action to be taken in all those cases through the ``on_poweroff``, ``on_shutdown`` and ``on_crash`` variables. By setting them to ``preserve``, Xen will avoid destroying the domains automatically. When the domain is not destroyed, it can be viewed by using ``xm list`` (or ``xl list`` in newer Xen versions), and the ``State`` field of the output will provide useful information. If the state is ``----c-`` it means the instance has crashed. If the state is ``---s--`` it means the instance was properly shutdown. If the instance was properly shutdown and it is still marked as ``running`` by Ganeti, it means that it was shutdown from inside by the user, and the Ganeti status of the instance needs to be changed to ``ADMIN_down``. This will be done at regular intervals by the group watcher, just before deciding which instances to reboot. On top of that, at the same time, the watcher will also need to issue ``xm destroy`` commands for all the domains that are in a crashed or shutdown state, since this will not be done automatically by Xen anymore because of the ``preserve`` setting in their config files. This behavior will be limited to the domains shut down from inside, because it will actually keep the resources of the domain busy until the watcher will do the cleaning job (that, with the default setting, is up to every 5 minutes). Still, this is considered acceptable, because it is not frequent for a domain to be shut down this way. The cleanup function will be also run automatically just before performing any job that requires resources to be available (such as when creating a new instance), in order to ensure that the new resource allocation happens starting from a clean state. Functionalities that only query the state of instances will not run the cleanup function. The cleanup operation includes both node-specific operations (the actual destruction of the stopped domains) and configuration changes, to be performed on the master node (marking as offline an instance that was shut down internally). The watcher, on the master node, will fetch the list of instances that have been shutdown from inside (recognizable by their ``oper_state`` as described below). It will then submit a series of ``InstanceShutdown`` jobs that will mark such instances as ``ADMIN_down`` and clean them up (after the functionality of ``InstanceShutdown`` will have been extended as specified in the rest of this design document). LUs performing operations other than an explicit cleanup will have to be modified to perform the cleanup as well, either by submitting a job to perform the cleanup (to be completed before actually performing the task at hand) or by explicitly performing the cleanup themselves through the RPC calls. Other required changes ++++++++++++++++++++++ The implementation of this design document will require some commands to be changed in order to cope with the new shutdown procedure. With the default shutdown action in Xen set to ``preserve``, the Ganeti command for shutting down instances would leave them in a shutdown but preserved state. Therefore, it will have to be changed in such a way to immediately perform the cleanup of the instance after verifying its correct shutdown. Also, it will correctly deal with instances that have been shutdown from inside but are still active according to Ganeti, by detecting this situation, destroying the instance and carrying out the rest of the Ganeti shutdown procedure as usual. The ``gnt-instance list`` command will need to be able to handle the situation where an instance was shutdown internally but not yet cleaned up. The ``admin_state`` field will maintain the current meaning unchanged. The ``oper_state`` field will get a new possible state, ``S``, meaning that the instance was shutdown internally. The ``gnt-instance info`` command ``State`` field, in such case, will show a message stating that the instance was supposed to be run but was shut down internally. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-kvmd.rst000064400000000000000000000255171476477700300173250ustar00rootroot00000000000000========== KVM daemon ========== :Created: 2014-Jan-09 :Status: Implemented :Ganeti-Version: 2.11.0 .. toctree:: :maxdepth: 2 This design document describes the KVM daemon, which is responsible for determining whether a given KVM instance was shutdown by an administrator or a user. Current state and shortcomings ============================== This design document describes the KVM daemon which addresses the KVM side of the user-initiated shutdown problem introduced in :doc:`design-internal-shutdown`. We are also interested in keeping this functionality optional. That is, an administrator does not necessarily have to run the KVM daemon if either he is running Xen or even, if he is running KVM, he is not interested in instance shutdown detection. This requirement is important because it means the KVM daemon should be a modular component in the overall Ganeti design, i.e., it should be easy to enable and disable it. Proposed changes ================ The instance shutdown feature for KVM requires listening on events from the Qemu Machine Protocol (QMP) Unix socket, which is created together with a KVM instance. A QMP socket typically looks like ``/var/run/ganeti/kvm-hypervisor/ctrl/.qmp`` and implements the QMP protocol. This is a bidirectional protocol that allows Ganeti to send commands, such as, system powerdown, as well as, receive events, such as, the powerdown and shutdown events. Listening in on these events allows Ganeti to determine whether a given KVM instance was shutdown by an administrator, either through ``gnt-instance stop|remove `` or ``kill -KILL ``, or by a user, through ``poweroff`` from inside the instance. Upon an administrator powerdown, the QMP protocol sends two events, namely, a powerdown event and a shutdown event, whereas upon a user shutdown only the shutdown event is sent. This is enough to distinguish between an administrator and a user shutdown. However, there is one limitation, which is, ``kill -TERM ``. Even though this is an action performed by the administrator, it will be considered a user shutdown by the approach described in this document. Several design strategies were considered. Most of these strategies consisted of spawning some process listening on the QMP socket when a KVM instance is created. However, having a listener process per KVM instance is not scalable. Therefore, a different strategy is proposed, namely, having a single process, called the KVM daemon, listening on the QMP sockets of all KVM instances within a node. That also means there is an instance of the KVM daemon on each node. In order to implement the KVM daemon, two problems need to be addressed, namely, how the KVM daemon knows when to open a connection to a given QMP socket and how the KVM daemon communicates with Ganeti whether a given instance was shutdown by an administrator or a user. QMP connections management -------------------------- As mentioned before, the QMP sockets reside in the KVM control directory, which is usually located under ``/var/run/ganeti/kvm-hypervisor/ctrl/``. When a KVM instance is created, a new QMP socket for this instance is also created in this directory. In order to simplify the design of the KVM daemon, instead of having Ganeti communicate to this daemon through a pipe or socket the creation of a new KVM instance, and thus a new QMP socket, this daemon will monitor the KVM control directory using ``inotify``. As a result, the daemon is not only able to deal with KVM instances being created and removed, but also capable of overcoming other problematic situations concerning the filesystem, such as, the case when the KVM control directory does not exist because, for example, Ganeti was not yet started, or the KVM control directory was removed, for example, as a result of a Ganeti reinstallation. Shutdown detection ------------------ As mentioned before, the KVM daemon is responsible for opening a connection to the QMP socket of a given instance and listening in on the shutdown and powerdown events, which allow the KVM daemon to determine whether the instance stopped because of an administrator or user shutdown. Once the instance is stopped, the KVM daemon needs to communicate to Ganeti whether the user was responsible for shutting down the instance. In order to achieve this, the KVM daemon writes an empty file, called the shutdown file, in the KVM control directory with a name similar to the QMP socket file but with the extension ``.qmp`` replaced with ``.shutdown``. The presence of this file indicates that the shutdown was initiated by a user, whereas the absence of this file indicates that the shutdown was caused by an administrator. This strategy also handles crashes and signals, such as, ``SIGKILL``, to be handled correctly, given that in these cases the KVM daemon never receives the powerdown and shutdown events and, therefore, never creates the shutdown file. KVM daemon launch ----------------- With the above issues addressed, a question remains as to when the KVM daemon should be started. The KVM daemon is different from other Ganeti daemons, which start together with the Ganeti service, because the KVM daemon is optional, given that it is specific to KVM and should not be run on installations containing only Xen, and, even in a KVM installation, the user might still choose not to enable it. And finally because the KVM daemon is not really necessary until the first KVM instance is started. For these reasons, the KVM daemon is started from within Ganeti when a KVM instance is started. And the job process spawned by the node daemon is responsible for starting the KVM daemon. Given the current design of Ganeti, in which the node daemon spawns a job process to handle the creation of the instance, when launching the KVM daemon it is necessary to first check whether an instance of this daemon is already running and, if this is not the case, then the KVM daemon can be safely started. Design alternatives =================== At first, it might seem natural to include the instance shutdown detection for KVM in the node daemon. After all, the node daemon is already responsible for managing instances, for example, starting and stopping an instance. Nevertheless, the node daemon is more complicated than it might seem at first. The node daemon is composed of the main loop, which runs in the main thread and is responsible for receiving requests and spawning jobs for handling these requests, and the jobs, which are independent processes spawned for executing the actual tasks, such as, creating an instance. Including instance shutdown detection in the node daemon is not viable because adding it to the main loop would cause KVM specific code to taint the generality of the node daemon. In order to add it to the job processes, it would be possible to spawn either a foreground or a background process. However, these options are also not viable because they would lead to the situation described before where there would be a monitoring process per instance, which is not scalable. Moreover, the foreground process has an additional disadvantage: it would require modifications the node daemon in order not to expect a terminating job, which is the current node daemon design. There is another design issue to have in mind. We could reconsider the place where to write the data that tell Ganeti whether an instance was shutdown by an administrator or the user. Instead of using the KVM shutdown files presented above, in which the presence of the file indicates a user shutdown and its absence an administrator shutdown, we could store a value in the KVM runtime state file, which is where the relevant KVM state information is. The advantage of this approach is that it would keep the KVM related information in one place, thus making it easier to manage. However, it would lead to a more complex implementation and, in the context of the general transition in Ganeti from Python to Haskell, a simpler implementation is preferred. Finally, it should be noted that the KVM runtime state file benefits from automatic migration. That is, when an instance is migrated so is the KVM state file. However, the instance shutdown detection for KVM does not require this feature and, in fact, migrating the instance shutdown state would be incorrect. Further considerations ====================== There are potential race conditions between Ganeti and the KVM daemon, however, in practice they seem unlikely. For example, the KVM daemon needs to add and remove watches to the parent directories of the KVM control directory until this directory is finally created. It is possible that Ganeti creates this directory and a KVM instance before the KVM daemon has a chance to add a watch to the KVM control directory, thus causing this daemon to miss the ``inotify`` creation event for the QMP socket. There are other problems which arise from the limitations of ``inotify``. For example, if the KVM daemon is started after the first Ganeti instance has been created, then the ``inotify`` will not produce any event for the creation of the QMP socket. This can happen, for example, if the KVM daemon needs to be restarted or upgraded. As a result, it might be necessary to have an additional mechanism that runs at KVM daemon startup or at regular intervals to ensure that the current KVM internal state is consistent with the actual contents of the KVM control directory. Another race condition occurs when Ganeti shuts down a KVM instance using force. Ganeti uses ``TERM`` signals to stop KVM instances when force is specified or ACPI is not enabled. However, as mentioned before, ``TERM`` signals are interpreted by the KVM daemon as a user shutdown. As a result, the KVM daemon creates a shutdown file which then must be removed by Ganeti. The race condition occurs because the KVM daemon might create the shutdown file after the hypervisor code that tries to remove this file has already run. In practice, the race condition seems unlikely because Ganeti stops the KVM instance in a retry loop, which allows Ganeti to stop the instance and cleanup its runtime information. It is possible to determine if a process, in this particular case the KVM process, was terminated by a ``TERM`` signal, using the `proc connector and socket filters `_. The proc connector is a socket connected between a userspace process and the kernel through the netlink protocol and can be used to receive notifications of process events, and the socket filters is a mechanism for subscribing only to events that are relevant. There are several `process events `_ which can be subscribed to, however, in this case, we are interested only in the exit event, which carries information about the exit signal. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-linuxha.rst000064400000000000000000000145341476477700300200310ustar00rootroot00000000000000==================== Linux HA integration ==================== :Created: 2013-Oct-24 :Status: Implemented :Ganeti-Version: 2.7.0 .. contents:: :depth: 4 This is a design document detailing the integration of Ganeti and Linux HA. Current state and shortcomings ============================== Ganeti doesn't currently support any self-healing or self-monitoring. We are now working on trying to improve the situation in this regard: - The :doc:`autorepair system ` will take care of self repairing a cluster in the presence of offline nodes. - The :doc:`monitoring agent ` will take care of exporting data to monitoring. What is still missing is a way to self-detect "obvious" failures rapidly and to: - Maintain the master role active. - Offline resource that are obviously faulty so that the autorepair system can perform its work. Proposed changes ================ Linux-HA provides software that can be used to provide high availability of services through automatic failover of resources. In particular Pacemaker can be used together with Heartbeat or Corosync to make sure a resource is kept active on a self-monitoring cluster. Ganeti OCF agents ----------------- The Ganeti agents will be slightly special in the HA world. The following will apply: - The agents will be able to be configured cluster-wise by tags (which will be read on the nodes via ssconf_cluster_tags) and locally by files on the filesystem that will allow them to "simulate" a particular condition (eg. simulate a failure even if none is detected). - The agents will be able to run in "full" or "partial" mode: in "partial" mode they will always succeed, and thus never fail a resource as long as a node is online, is running the linux HA software and is responding to the network. In "full" mode they will also check resources like the cluster master ip or master daemon, and act if they are missing Note that for what Ganeti does OCF agents are needed: simply relying on the LSB scripts will not work for the Ganeti service. Master role agent ----------------- This agent will manage the Ganeti master role. It needs to be configured as a sticky resource (you don't want to flap the master role around, do you?) that is active on only one node. You can require quorum or fencing to protect your cluster from multiple masters. The agent will implement a stateless resource that considers itself "started" only the master node, "stopped" on all master candidates and in error mode for all other nodes. Note that if not all your nodes are master candidates this resource might have problems: - if all nodes are configured to run the resource, heartbeat may decide to "fence" (aka stonith) all your non-master-candidate nodes if told to do so. This might not be what you want. - if only master candidates are configured as nodes for the resource, beware of promotions and demotions, as nothing will update automatically pacemaker should a change happen at the Ganeti level. Other solutions, such as reporting the resource just as "stopped" on non master candidates as well might mean that pacemaker would choose the "wrong" node to promote to master, which is also a bad idea. Future improvements +++++++++++++++++++ - Ability to work better with non-master-candidate nodes - Stateful resource that can "safely" transfer the master role between online nodes (with queue drain and such) - Implement "full" mode, with detection of the cluster IP and the master node daemon. Node role agent --------------- This agent will manage the Ganeti node role. It needs to be configured as a cloned resource that is active on all nodes. In partial mode it will always return success (and thus trigger a failure only upon an HA level or network failure). Full mode, which initially will not be implemented, could also check for the node daemon being unresponsive or other local conditions (TBD). When a failure happens the HA notification system will trigger on all other nodes, including the master. The master will then be able to offline the node. Any other work to restore instance availability should then be done by the autorepair system. The following cluster tags are supported: - ``ocf:node-offline:use-powercycle``: Try to powercycle a node using ``gnt-node powercycle`` when offlining. - ``ocf:node-offline:use-poweroff``: Try to power off a node using ``gnt-node power off`` when offlining (requires OOB support). Future improvements +++++++++++++++++++ - Handle draining differently than offlining - Handle different modes of "stopping" the service - Implement "full" mode Risks ----- Running Ganeti with Pacemaker increases the risk of stability for your Ganeti Cluster. Events like: - stopping heartbeat or corosync on a node - corosync or heartbeat being killed for any reason - temporary failure in a node's networking will trigger potentially dangerous operations such as node offlining or master role failover. Moreover if the autorepair system will be working they will be able to also trigger instance failovers or migrations, and disk replaces. Also note that operations like: master-failover, or manual node-modify might interact badly with this setup depending on the way your HA system is configured (see below). This of course is an inherent problem with any Linux-HA installation, but is probably more visible with Ganeti given that our resources tend to be more heavyweight than many others managed in HA clusters (eg. an IP address). Code status ----------- This code is heavily experimental, and Linux-HA is a very complex subsystem. *We might not be able to help you* if you decide to run this code: please make sure you understand fully high availability on your production machines. Ganeti only ships this code as an example but it might need customization or complex configurations on your side for it to run properly. *Ganeti does not automate HA configuration for your cluster*. You need to do this job by hand. Good luck, don't get it wrong. Future work =========== - Integrate the agents better with the ganeti monitoring - Add hooks for managing HA at node add/remove/modify/master-failover operations - Provide a stonith system through Ganeti's OOB system - Provide an OOB system that does "shunning" of offline nodes, for emulating a real OOB, at least on all nodes .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-location.rst000064400000000000000000000131571476477700300201710ustar00rootroot00000000000000====================================== Improving location awareness of Ganeti ====================================== :Created: 2014-Jul-22 :Status: Partially Implemented :Ganeti-Version: 2.13.0, 2.14.0 This document describes an enhancement of Ganeti's instance placement by taking into account that some nodes are vulnerable to common failures. .. contents:: :depth: 4 Current state and shortcomings ============================== Currently, Ganeti considers all nodes in a single node group as equal. However, this is not true in some setups. Nodes might share common causes of failure or be even located in different places with spacial redundancy being a desired feature. The similar problem for instances, i.e., instances providing the same external service should not placed on the same nodes, is solved by means of exclusion tags. However, there is no mechanism for a good choice of node pairs for a single instance. Moreover, while instances providing the same service run on different nodes, they are not spread out location wise. Proposed changes ================ We propose to the cluster metric (as used, e.g., by ``hbal`` and ``hail``) to honor additional node tags indicating nodes that might have a common cause of failure. Failure tags ------------ As for exclusion tags, cluster tags will determine which tags are considered to denote a source of common failure. More precisely, a cluster tag of the form *htools:nlocation:x* will make node tags starting with *x:* indicate a common cause of failure, that redundant instances should avoid. Metric changes -------------- The following components will be added cluster metric, weighed appropriately. - The number of pairs of an instance and a common-failure tag, where primary and secondary node both have this tag. - The number of pairs of exclusion tags and common-failure tags where there exist at least two instances with the given exclusion tag with the primary node having the given common-failure tag. The weights for these components might have to be tuned as experience with these setups grows, but as a starting point, both components will have a weight of 1.0 each. In this way, any common-failure violations are less important than any hard constraints missed (like instances on offline nodes) so that the hard constraints will be restored first when balancing a cluster. Nevertheless, with weight 1.0 the new common-failure components will still be significantly more important than all the balancedness components (cpu, disk, memory), as the latter are standard deviations of fractions. It will also dominate the disk load component which, which, when only taking static information into account, essentially amounts to counting disks. In this way, Ganeti will be willing to sacrifice equal numbers of disks on every node in order to fulfill location requirements. Apart from changing the balancedness metric, common-failure tags will not have any other effect. In particular, as opposed to exclusion tags, no hard guarantees are made: ``hail`` will try allocate an instance in a common-failure avoiding way if possible, but still allocate the instance if not. Additional migration restrictions ================================= Inequality between nodes can also restrict the set of instance migrations possible. Here, the most prominent example is updating the hypervisor where usually migrations from the new to the old hypervisor version is not possible. Migration tags -------------- As for exclusion tags, cluster tags will determine which tags are considered restricting migration. More precisely, a cluster tag of the form *htools:migration:x* will make node tags starting with *x:* a migration relevant node property. Additionally, cluster tags of the form *htools:allowmigration:y::z* where *y* and *z* are migration tags not containing *::* specify a unidirectional migration possibility from *y* to *z*. Restriction ----------- An instance migration will only be considered by ``htools``, if for all migration tags *y* present on the node migrated from, either the tag is also present on the node migrated to or there is a cluster tag *htools::allowmigration:y::z* and the target node is tagged *z* (or both). Example ------- For the simple hypervisor upgrade, where migration from old to new is possible, but not the other way round, tagging all already upgraded nodes suffices. Advise only ----------- These tags are of advisory nature only. That is, all ``htools`` will strictly obey the restrictions imposed by those tags, but Ganeti will not prevent users from manually instructing other migrations. Instance pinning ================ Sometimes, administrators want specific instances located in a particular, typically geographic, location. To support these kind of requests, instances can be assigned tags of the form *htools:desiredlocation:x* where *x* is a failure tag. Those tags indicate the the instance wants to be placed on a node tagged *x*. To make ``htools`` honor those desires, the metric is extended, appropriately weighted, by the following component. - Sum of dissatisfied desired locations number among all cluster instances. An instance desired location is dissatisfied when the instance is assigned a desired-location tag *x* where the node is not tagged with the location tag *x*. Such metric extension allows to specify multiple desired locations for each instance. These desired locations may be contradictive as well. Contradictive desired locations mean that we don't care which one of desired locations will be satisfied. Again, instance pinning is just heuristics, not a hard enforced requirement; it will only be achieved by the cluster metrics favouring such placements. ganeti-3.1.0~rc2/doc/design-lu-generated-jobs.rst000064400000000000000000000067661476477700300217000ustar00rootroot00000000000000================================== Submitting jobs from logical units ================================== :Created: 2011-Mar-23 :Status: Implemented :Ganeti-Version: 2.5.0 .. contents:: :depth: 4 This is a design document about the innards of Ganeti's job processing. Readers are advised to study previous design documents on the topic: - :ref:`Original job queue ` - :ref:`Job priorities ` Current state and shortcomings ============================== Some Ganeti operations want to execute as many operations in parallel as possible. Examples are evacuating or failing over a node (``gnt-node evacuate``/``gnt-node failover``). Without changing large parts of the code, e.g. the RPC layer, to be asynchronous, or using threads inside a logical unit, only a single operation can be executed at a time per job. Currently clients work around this limitation by retrieving the list of desired targets and then re-submitting a number of jobs. This requires logic to be kept in the client, in some cases leading to duplication (e.g. CLI and RAPI). Proposed changes ================ The job queue lock is guaranteed to be released while executing an opcode/logical unit. This means an opcode can talk to the job queue and submit more jobs. It then receives the job IDs, like any job submitter using the LUXI interface would. These job IDs are returned to the client, who then will then proceed to wait for the jobs to finish. Technically, the job queue already passes a number of callbacks to the opcode processor. These are used for giving user feedback, notifying the job queue of an opcode having gotten its locks, and checking whether the opcode has been cancelled. A new callback function is added to submit jobs. Its signature and result will be equivalent to the job queue's existing ``SubmitManyJobs`` function. Logical units can submit jobs by returning an instance of a special container class with a list of jobs, each of which is a list of opcodes (e.g. ``[[op1, op2], [op3]]``). The opcode processor will recognize instances of the special class when used a return value and will submit the contained jobs. The submission status and job IDs returned by the submission callback are used as the opcode's result. It should be encapsulated in a dictionary allowing for future extensions. .. highlight:: javascript Example:: { "jobs": [ (True, "8149"), (True, "21019"), (False, "Submission failed"), (True, "31594"), ], } Job submissions can fail for variety of reasons, e.g. a full or drained job queue. Lists of jobs can not be submitted atomically, meaning some might fail while others succeed. The client is responsible for handling such cases. Other discussed solutions ========================= Instead of requiring the client to wait for the returned jobs, another idea was to do so from within the submitting opcode in the master daemon. While technically possible, doing so would have two major drawbacks: - Opcodes waiting for other jobs to finish block one job queue worker thread - All locks must be released before starting the waiting process, failure to do so can lead to deadlocks Instead of returning the job IDs as part of the normal opcode result, introducing a new opcode field, e.g. ``op_jobids``, was discussed and dismissed. A new field would touch many areas and possibly break some assumptions. There were also questions about the semantics. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-monitoring-agent.rst000064400000000000000000000730711476477700300216430ustar00rootroot00000000000000======================= Ganeti monitoring agent ======================= :Created: 2012-Oct-18 :Status: Implemented :Ganeti-Version: 2.7.0, 2.8.0, 2.9.0 .. contents:: :depth: 4 This is a design document detailing the implementation of a Ganeti monitoring agent report system, that can be queried by a monitoring system to calculate health information for a Ganeti cluster. Current state and shortcomings ============================== There is currently no monitoring support in Ganeti. While we don't want to build something like Nagios or Pacemaker as part of Ganeti, it would be useful if such tools could easily extract information from a Ganeti machine in order to take actions (example actions include logging an outage for future reporting or alerting a person or system about it). Proposed changes ================ Each Ganeti node should export a status page that can be queried by a monitoring system. Such status page will be exported on a network port and will be encoded in JSON (simple text) over HTTP. The choice of JSON is obvious as we already depend on it in Ganeti and thus we don't need to add extra libraries to use it, as opposed to what would happen for XML or some other markup format. Location of agent report ------------------------ The report will be available from all nodes, and be concerned for all node-local resources. This allows more real-time information to be available, at the cost of querying all nodes. Information reported -------------------- The monitoring agent system will report on the following basic information: - Instance status - Instance disk status - Status of storage for instances - Ganeti daemons status, CPU usage, memory footprint - Hypervisor resources report (memory, CPU, network interfaces) - Node OS resources report (memory, CPU, network interfaces) - Node OS CPU load average report - Information from a plugin system .. _monitoring-agent-format-of-the-report: Format of the report -------------------- The report of the will be in JSON format, and it will present an array of report objects. Each report object will be produced by a specific data collector. Each report object includes some mandatory fields, to be provided by all the data collectors: ``name`` The name of the data collector that produced this part of the report. It is supposed to be unique inside a report. ``version`` The version of the data collector that produces this part of the report. Built-in data collectors (as opposed to those implemented as plugins) should have "B" as the version number. ``format_version`` The format of what is represented in the "data" field for each data collector might change over time. Every time this happens, the format_version should be changed, so that who reads the report knows what format to expect, and how to correctly interpret it. ``timestamp`` The time when the reported data were gathered. It has to be expressed in nanoseconds since the unix epoch (0:00:00 January 01, 1970). If not enough precision is available (or needed) it can be padded with zeroes. If a report object needs multiple timestamps, it can add more and/or override this one inside its own "data" section. ``category`` A collector can belong to a given category of collectors (e.g.: storage collectors, daemon collector). This means that it will have to provide a minimum set of prescribed fields, as documented for each category. This field will contain the name of the category the collector belongs to, if any, or just the ``null`` value. ``kind`` Two kinds of collectors are possible: `Performance reporting collectors`_ and `Status reporting collectors`_. The respective paragraphs will describe them and the value of this field. ``data`` This field contains all the data generated by the specific data collector, in its own independently defined format. The monitoring agent could check this syntactically (according to the JSON specifications) but not semantically. Here follows a minimal example of a report:: [ { "name" : "TheCollectorIdentifier", "version" : "1.2", "format_version" : 1, "timestamp" : 1351607182000000000, "category" : null, "kind" : 0, "data" : { "plugin_specific_data" : "go_here" } }, { "name" : "AnotherDataCollector", "version" : "B", "format_version" : 7, "timestamp" : 1351609526123854000, "category" : "storage", "kind" : 1, "data" : { "status" : { "code" : 1, "message" : "Error on disk 2" }, "plugin_specific" : "data", "some_late_data" : { "timestamp" : 1351609526123942720, ... } } } ] Performance reporting collectors ++++++++++++++++++++++++++++++++ These collectors only provide data about some component of the system, without giving any interpretation over their meaning. The value of the ``kind`` field of the report will be ``0``. Status reporting collectors +++++++++++++++++++++++++++ These collectors will provide information about the status of some component of ganeti, or managed by ganeti. The value of their ``kind`` field will be ``1``. The rationale behind this kind of collectors is that there are some situations where exporting data about the underlying subsystems would expose potential issues. But if Ganeti itself is able (and going) to fix the problem, conflicts might arise between Ganeti and something/somebody else trying to fix the same problem. Also, some external monitoring systems might not be aware of the internals of a particular subsystem (e.g.: DRBD) and might only exploit the high level response of its data collector, alerting an administrator if anything is wrong. Still, completely hiding the underlying data is not a good idea, as they might still be of use in some cases. So status reporting plugins will provide two output modes: one just exporting a high level information about the status, and one also exporting all the data they gathered. The default output mode will be the status-only one. Through a command line parameter (for stand-alone data collectors) or through the HTTP request to the monitoring agent (when collectors are executed as part of it) the verbose output mode providing all the data can be selected. When exporting just the status each status reporting collector will provide, in its ``data`` section, at least the following field: ``status`` summarizes the status of the component being monitored and consists of two subfields: ``code`` It assumes a numeric value, encoded in such a way to allow using a bitset to easily distinguish which states are currently present in the whole cluster. If the bitwise OR of all the ``status`` fields is 0, the cluster is completely healthy. The status codes are as follows: ``0`` The collector can determine that everything is working as intended. ``1`` Something is temporarily wrong but it is being automatically fixed by Ganeti. There is no need of external intervention. ``2`` The collector has failed to understand whether the status is good or bad. Further analysis is required. Interpret this status as a potentially dangerous situation. ``4`` The collector can determine that something is wrong and Ganeti has no way to fix it autonomously. External intervention is required. ``message`` A message to better explain the reason of the status. The exact format of the message string is data collector dependent. The field is mandatory, but the content can be an empty string if the ``code`` is ``0`` (working as intended) or ``1`` (being fixed automatically). If the status code is ``2``, the message should specify what has gone wrong. If the status code is ``4``, the message should explain why it was not possible to determine a proper status. The ``data`` section will also contain all the fields describing the gathered data, according to a collector-specific format. Instance status +++++++++++++++ At the moment each node knows which instances are running on it, which instances it is primary for, but not the cause why an instance might not be running. On the other hand we don't want to distribute full instance "admin" status information to all nodes, because of the performance impact this would have. As such we propose that: - Any operation that can affect instance status will have an optional "reason" attached to it (at opcode level). This can be used for example to distinguish an admin request, from a scheduled maintenance or an automated tool's work. If this reason is not passed, Ganeti will just use the information it has about the source of the request. This reason information will be structured according to the :doc:`Ganeti reason trail ` design document. - RPCs that affect the instance status will be changed so that the "reason" and the version of the config object they ran on is passed to them. They will then export the new expected instance status, together with the associated reason and object version to the status report system, which then will export those themselves. Monitoring and auditing systems can then use the reason to understand the cause of an instance status, and they can use the timestamp to understand the freshness of their data even in the absence of an atomic cross-node reporting: for example if they see an instance "up" on a node after seeing it running on a previous one, they can compare these values to understand which data is freshest, and repoll the "older" node. Of course if they keep seeing this status this represents an error (either an instance continuously "flapping" between nodes, or an instance is constantly up on more than one), which should be reported and acted upon. The instance status will be on each node, for the instances it is primary for, and its ``data`` section of the report will contain a list of instances, named ``instances``, with at least the following fields for each instance: ``name`` The name of the instance. ``uuid`` The UUID of the instance (stable on name change). ``admin_state`` The status of the instance (up/down/offline) as requested by the admin. ``actual_state`` The actual status of the instance. It can be ``up``, ``down``, or ``hung`` if the instance is up but it appears to be completely stuck. ``uptime`` The uptime of the instance (if it is up, "null" otherwise). ``mtime`` The timestamp of the last known change to the instance state. ``state_reason`` The last known reason for state change of the instance, described according to the JSON representation of a reason trail, as detailed in the :doc:`reason trail design document `. ``status`` It represents the status of the instance, and its format is the same as that of the ``status`` field of `Status reporting collectors`_. Each hypervisor should provide its own instance status data collector, possibly with the addition of more, specific, fields. The ``category`` field of all of them will be ``instance``. The ``kind`` field will be ``1``. Note that as soon as a node knows it's not the primary anymore for an instance it will stop reporting status for it: this means the instance will either disappear, if it has been deleted, or appear on another node, if it's been moved. The ``code`` of the ``status`` field of the report of the Instance status data collector will be: ``0`` if ``status`` is ``0`` for all the instances it is reporting about. ``1`` otherwise. Storage collectors ++++++++++++++++++ The storage collectors will be a series of data collectors that will gather data about storage for the current node. The collection will be performed at different granularity and abstraction levels, from the physical disks, to partitions, logical volumes and to the specific storage types used by Ganeti itself (drbd, rbd, plain, file). The ``name`` of each of these collector will reflect what storage type each of them refers to. The ``category`` field of these collector will be ``storage``. The ``kind`` field will depend on the specific collector. Each ``storage`` collector's ``data`` section will provide collector-specific fields. The various storage collectors will provide keys to join the data they provide, in order to allow the user to get a better understanding of the system. E.g.: through device names, or instance names. Diskstats collector ******************* This storage data collector will gather information about the status of the disks installed in the system, as listed in the /proc/diskstats file. This means that not only physical hard drives, but also ramdisks and loopback devices will be listed. Its ``kind`` in the report will be ``0`` (`Performance reporting collectors`_). Its ``category`` field in the report will contain the value ``storage``. When executed in verbose mode, the ``data`` section of the report of this collector will be a list of items, each representing one disk, each providing the following fields: ``major`` The major number of the device. ``minor`` The minor number of the device. ``name`` The name of the device. ``readsNum`` This is the total number of reads completed successfully. ``mergedReads`` Reads which are adjacent to each other may be merged for efficiency. Thus two 4K reads may become one 8K read before it is ultimately handed to the disk, and so it will be counted (and queued) as only one I/O. This field specifies how often this was done. ``secRead`` This is the total number of sectors read successfully. ``timeRead`` This is the total number of milliseconds spent by all reads. ``writes`` This is the total number of writes completed successfully. ``mergedWrites`` Writes which are adjacent to each other may be merged for efficiency. Thus two 4K writes may become one 8K read before it is ultimately handed to the disk, and so it will be counted (and queued) as only one I/O. This field specifies how often this was done. ``secWritten`` This is the total number of sectors written successfully. ``timeWrite`` This is the total number of milliseconds spent by all writes. ``ios`` The number of I/Os currently in progress. The only field that should go to zero, it is incremented as requests are given to appropriate struct request_queue and decremented as they finish. ``timeIO`` The number of milliseconds spent doing I/Os. This field increases so long as field ``IOs`` is nonzero. ``wIOmillis`` The weighted number of milliseconds spent doing I/Os. This field is incremented at each I/O start, I/O completion, I/O merge, or read of these stats by the number of I/Os in progress (field ``IOs``) times the number of milliseconds spent doing I/O since the last update of this field. This can provide an easy measure of both I/O completion time and the backlog that may be accumulating. Logical Volume collector ************************ This data collector will gather information about the attributes of logical volumes present in the system. Its ``kind`` in the report will be ``0`` (`Performance reporting collectors`_). Its ``category`` field in the report will contain the value ``storage``. The ``data`` section of the report of this collector will be a list of items, each representing one logical volume and providing the following fields: ``uuid`` The UUID of the logical volume. ``name`` The name of the logical volume. ``attr`` The attributes of the logical volume. ``major`` Persistent major number or -1 if not persistent. ``minor`` Persistent minor number or -1 if not persistent. ``kernel_major`` Currently assigned major number or -1 if LV is not active. ``kernel_minor`` Currently assigned minor number or -1 if LV is not active. ``size`` Size of LV in bytes. ``seg_count`` Number of segments in LV. ``tags`` Tags, if any. ``modules`` Kernel device-mapper modules required for this LV, if any. ``vg_uuid`` Unique identifier of the volume group. ``vg_name`` Name of the volume group. ``segtype`` Type of LV segment. ``seg_start`` Offset within the LV to the start of the segment in bytes. ``seg_start_pe`` Offset within the LV to the start of the segment in physical extents. ``seg_size`` Size of the segment in bytes. ``seg_tags`` Tags for the segment, if any. ``seg_pe_ranges`` Ranges of Physical Extents of underlying devices in lvs command line format. ``devices`` Underlying devices used with starting extent numbers. ``instance`` The name of the instance this LV is used by, or ``null`` if it was not possible to determine it. DRBD status *********** This data collector will run only on nodes where DRBD is actually present and it will gather information about DRBD devices. Its ``kind`` in the report will be ``1`` (`Status reporting collectors`_). Its ``category`` field in the report will contain the value ``storage``. When executed in verbose mode, the ``data`` section of the report of this collector will provide the following fields: ``versionInfo`` Information about the DRBD version number, given by a combination of any (but at least one) of the following fields: ``version`` The DRBD driver version. ``api`` The API version number. ``proto`` The protocol version. ``srcversion`` The version of the source files. ``gitHash`` Git hash of the source files. ``buildBy`` Who built the binary, and, optionally, when. ``device`` A list of structures, each describing a DRBD device (a minor) and containing the following fields: ``minor`` The device minor number. ``connectionState`` The state of the connection. If it is "Unconfigured", all the following fields are not present. ``localRole`` The role of the local resource. ``remoteRole`` The role of the remote resource. ``localState`` The status of the local disk. ``remoteState`` The status of the remote disk. ``replicationProtocol`` The replication protocol being used. ``ioFlags`` The input/output flags. ``perfIndicators`` The performance indicators. This field will contain the following sub-fields: ``networkSend`` KiB of data sent on the network. ``networkReceive`` KiB of data received from the network. ``diskWrite`` KiB of data written on local disk. ``diskRead`` KiB of date read from the local disk. ``activityLog`` Number of updates of the activity log. ``bitMap`` Number of updates to the bitmap area of the metadata. ``localCount`` Number of open requests to the local I/O subsystem. ``pending`` Number of requests sent to the partner but not yet answered. ``unacknowledged`` Number of requests received by the partner but still to be answered. ``applicationPending`` Num of block input/output requests forwarded to DRBD but that have not yet been answered. ``epochs`` (Optional) Number of epoch objects. Not provided by all DRBD versions. ``writeOrder`` (Optional) Currently used write ordering method. Not provided by all DRBD versions. ``outOfSync`` (Optional) KiB of storage currently out of sync. Not provided by all DRBD versions. ``syncStatus`` (Optional) The status of the synchronization of the disk. This is present only if the disk is being synchronized, and includes the following fields: ``percentage`` The percentage of synchronized data. ``progress`` How far the synchronization is. Written as "x/y", where x and y are integer numbers expressed in the measurement unit stated in ``progressUnit`` ``progressUnit`` The measurement unit for the progress indicator. ``timeToFinish`` The expected time before finishing the synchronization. ``speed`` The speed of the synchronization. ``want`` The desired speed of the synchronization. ``speedUnit`` The measurement unit of the ``speed`` and ``want`` values. Expressed as "size/time". ``instance`` The name of the Ganeti instance this disk is associated to. Ganeti daemons status +++++++++++++++++++++ Ganeti will report what information it has about its own daemons. This should allow identifying possible problems with the Ganeti system itself: for example memory leaks, crashes and high resource utilization should be evident by analyzing this information. The ``kind`` field will be ``1`` (`Status reporting collectors`_). Each daemon will have its own data collector, and each of them will have a ``category`` field valued ``daemon``. When executed in verbose mode, their data section will include at least: ``memory`` The amount of used memory. ``size_unit`` The measurement unit used for the memory. ``uptime`` The uptime of the daemon. ``CPU usage`` How much cpu the daemon is using (percentage). Any other daemon-specific information can be included as well in the ``data`` section. Hypervisor resources report +++++++++++++++++++++++++++ Each hypervisor has a view of system resources that sometimes is different than the one the OS sees (for example in Xen the Node OS, running as Dom0, has access to only part of those resources). In this section we'll report all information we can in a "non hypervisor specific" way. Each hypervisor can then add extra specific information that is not generic enough be abstracted. The ``kind`` field will be ``0`` (`Performance reporting collectors`_). Each of the hypervisor data collectors will be of ``category``: ``hypervisor``. Node OS resources report ++++++++++++++++++++++++ Since Ganeti assumes it's running on Linux, it's useful to export some basic information as seen by the host system. The ``category`` field of the report will be ``null``. The ``kind`` field will be ``0`` (`Performance reporting collectors`_). The ``data`` section will include: ``cpu_number`` The number of available cpus. ``cpus`` A list with one element per cpu, showing its average load. ``memory`` The current view of memory (free, used, cached, etc.) ``filesystem`` A list with one element per filesystem, showing a summary of the total/available space. ``NICs`` A list with one element per network interface, showing the amount of sent/received data, error rate, IP address of the interface, etc. ``versions`` A map using the name of a component Ganeti interacts (Linux, drbd, hypervisor, etc) as the key and its version number as the value. Note that we won't go into any hardware specific details (e.g. querying a node RAID is outside the scope of this, and can be implemented as a plugin) but we can easily just report the information above, since it's standard enough across all systems. Node OS CPU load average report +++++++++++++++++++++++++++++++ This data collector will export CPU load statistics as seen by the host system. Apart from using the data from an external monitoring system we can also use the data to improve instance allocation and/or the Ganeti cluster balance. To compute the CPU load average we will use a number of values collected inside a time window. The collection process will be done by an independent thread (see `Mode of Operation`_). This report is a subset of the previous report (`Node OS resources report`_) and they might eventually get merged, once reporting for the other fields (memory, filesystem, NICs) gets implemented too. Specifically: The ``category`` field of the report will be ``null``. The ``kind`` field will be ``0`` (`Performance reporting collectors`_). The ``data`` section will include: ``cpu_number`` The number of available cpus. ``cpus`` A list with one element per cpu, showing its average load. ``cpu_total`` The total CPU load average as a sum of the all separate cpus. The CPU load report function will get N values, collected by the CPU load collection function and calculate the above averages. Please see the section `Mode of Operation`_ for more information one how the two functions of the data collector interact. Format of the query ------------------- .. include:: monitoring-query-format.rst Instance disk status propagation -------------------------------- As for the instance status Ganeti has now only partial information about its instance disks: in particular each node is unaware of the disk to instance mapping, that exists only on the master. For this design doc we plan to fix this by changing all RPCs that create a backend storage or that put an already existing one in use and passing the relevant instance to the node. The node can then export these to the status reporting tool. While we haven't implemented these RPC changes yet, we'll use Confd to fetch this information in the data collectors. Plugin system ------------- The monitoring system will be equipped with a plugin system that can export specific local information through it. The plugin system is expected to be used by local installations to export any installation specific information that they want to be monitored, about either hardware or software on their systems. The plugin system will be in the form of either scripts or binaries whose output will be inserted in the report. Eventually support for other kinds of plugins might be added as well, such as plain text files which will be inserted into the report, or local unix or network sockets from which the information has to be read. This should allow most flexibility for implementing an efficient system, while being able to keep it as simple as possible. Data collectors --------------- In order to ease testing as well as to make it simple to reuse this subsystem it will be possible to run just the "data collectors" on each node without passing through the agent daemon. If a data collector is run independently, it should print on stdout its report, according to the format corresponding to a single data collector report object, as described in the previous paragraphs. Mode of operation ----------------- In order to be able to report information fast the monitoring agent daemon will keep an in-memory or on-disk cache of the status, which will be returned when queries are made. The status system will then periodically check resources to make sure the status is up to date. Different parts of the report will be queried at different speeds. These will depend on: - how often they vary (or we expect them to vary) - how fast they are to query - how important their freshness is Of course the last parameter is installation specific, and while we'll try to have defaults, it will be configurable. The first two instead we can use adaptively to query a certain resource faster or slower depending on those two parameters. When run as stand-alone binaries, the data collector will not using any caching system, and just fetch and return the data immediately. Since some performance collectors have to operate on a number of values collected in previous times, we need a mechanism independent of the data collector which will trigger the collection of those values and also store them, so that they are available for calculation by the data collectors. To collect data periodically, a thread will be created by the monitoring agent which will run the collection function of every data collector that provides one. The values returned by the collection function of the data collector will be saved in an appropriate map, associating each value to the corresponding collector, using the collector's name as the key of the map. This map will be stored in mond's memory. The collectors are divided in two categories: - stateless collectors, collectors who have immediate access to the reported information - stateful collectors, collectors whose report is based on data collected in a previous time window For example: the collection function of the CPU load collector will collect a CPU load value and save it in the map mentioned above. The collection function will be called by the collector thread every t milliseconds. When the report function of the collector is called, it will process the last N values of the map and calculate the corresponding average. Implementation place -------------------- The status daemon will be implemented as a standalone Haskell daemon. In the future it should be easy to merge multiple daemons into one with multiple entry points, should we find out it saves resources and doesn't impact functionality. The libekg library should be looked at for easily providing metrics in json format. Implementation order -------------------- We will implement the agent system in this order: - initial example data collectors (eg. for drbd and instance status). - initial daemon for exporting data, integrating the existing collectors - plugin system - RPC updates for instance status reasons and disk to instance mapping - cache layer for the daemon - more data collectors Future work =========== As a future step it can be useful to "centralize" all this reporting data on a single place. This for example can be just the master node, or all the master candidates. We will evaluate doing this after the first node-local version has been developed and tested. Another possible change is replacing the "read-only" RPCs with queries to the agent system, thus having only one way of collecting information from the nodes from a monitoring system and for Ganeti itself. One extra feature we may need is a way to query for only sub-parts of the report (eg. instances status only). This can be done by passing arguments to the HTTP GET, which will be defined when we get to this functionality. Finally the :doc:`autorepair system design `. system (see its design) can be expanded to use the monitoring agent system as a source of information to decide which repairs it can perform. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-move-instance-improvements.rst000064400000000000000000000475401476477700300236620ustar00rootroot00000000000000======================================== Instance move improvements ======================================== :Created: 2014-Apr-11 :Status: Partially Implemented :Ganeti-Version: 2.12.0 .. contents:: :depth: 3 Ganeti provides tools for moving instances within and between clusters. Through special export and import calls, a new instance is created with the disk data of the existing one. The tools work correctly and reliably, but depending on bandwidth and priority, an instance disk of considerable size requires a long time to transfer. The length of the transfer is inconvenient at best, but the problem becomes only worse if excessive locking causes a move operation to be delayed for a longer period of time, or to block other operations. The performance of moves is a complex topic, with available bandwidth, compression, and encryption all being candidates for choke points that bog down a transfer. Depending on the environment a move is performed in, tuning these can have significant performance benefits, but Ganeti does not expose many options needed for such tuning. The details of what to expose and what tradeoffs can be made will be presented in this document. Apart from existing functionality, some beneficial features can be introduced to help with instance moves. Zeroing empty space on instance disks can be useful for drastically improving the qualities of compression, effectively not needing to transfer unused disk space during moves. Compression itself can be improved by using different tools. The encryption used can be weakened or eliminated for certain moves. Using opportunistic locking during instance moves results in greater parallelization. As all of these approaches aim to tackle two different aspects of the problem, they do not exclude each other and will be presented independently. The performance of Ganeti moves =============================== In the current implementation, there are three possible factors limiting the speed of an instance move. The first is the network bandwidth, which Ganeti can exploit better by using compression. The second is the encryption, which is obligatory, and which can throttle an otherwise fast connection. The third is surprisingly the compression, which can cause the connection to be underutilized. Example 1: some numbers present during an intra-cluster instance move: * Network bandwidth: 105MB/s, courtesy of a gigabit switch * Encryption performance: 40MB/s, provided by OpenSSL * Compression performance: 22.3MB/s input, 7.1MB/s gzip compressed output As can be seen in this example, the obligatory encryption results in 62% of available bandwidth being wasted, while using compression further lowers the throughput to 55% of what the encryption would allow. The following sections will talk about these numbers in more detail, and suggest improvements and best practices. Encryption and Ganeti security ++++++++++++++++++++++++++++++ Turning compression and encryption off would allow for an immediate improvement, and while that is possible for compression, there are good reasons why encryption is currently not a feature a user can disable. While it is impossible to secure instance data if an attacker gains SSH access to a node, the RAPI was designed to never allow user data to be accessed through it in case of being compromised. If moves could be performed unencrypted, this property would be broken. Instance moves can take place in environments which may be hostile, and where unencrypted traffic could be intercepted. As they can be instigated through the RAPI, an attacker could access all data on all instances in a cluster by moving them unencrypted and intercepting the data in flight. This is one of the few situations where the current speed of instance moves could be considered a perk. The performance of encryption can be increased by either using a less secure form of encryption, including no encryption, or using a faster encryption algorithm. The example listed above utilizes AES-256, one of the few ciphers that Ganeti deems secure enough to use. AES-128, also allowed by Ganeti's current settings, is weaker but 46% faster. A cipher that is not allowed due to its flaws, such as RC4, could offer a 208% increase in speed. On the other hand, using an OS capable of utilizing the AES_NI chip present on modern hardware can double the performance of AES, making it the best tradeoff between security and performance. Ganeti cannot and should not detect all the factors listed above, but should rather give its users some leeway in what to choose. A precedent already exists, as intra-cluster DRBD replication is already performed unencrypted, albeit on a separate VLAN. For intra-cluster moves, Ganeti should allow its users to set OpenSSL ciphers at will, while still enforcing high-security settings for moves between clusters. Thus, two settings will be introduced: * a cluster-level setting called ``--allow-cipher-bypassing``, a boolean that cannot be set over RAPI * a gnt-instance move setting called ``--ciphers-to-use``, bypassing the default cipher list with given ciphers, filtered to ensure no other OpenSSL options are passed in within This change will serve to address the issues with moving non-redundant instances within the cluster, while keeping Ganeti security at its current level. Compression +++++++++++ Support for disk compression during instance moves was partially present before, but cleaned up and unified under the ``--compress`` option only as of Ganeti 2.11. The only option offered by Ganeti is gzip with no options passed to it, resulting in a good compression ratio, but bad compression speed. As compression can affect the speed of instance moves significantly, it is worthwhile to explore alternatives. To test compression tool performance, an 8GB drive filled with data matching the expected usage patterns (taken from a workstation) was compressed by using various tools with various settings. The two top performers were ``lzop`` and, surprisingly, ``gzip``. The improvement in the performance of ``gzip`` was obtained by explicitly optimizing for speed rather than compression. * ``gzip -6``: 22.3MB/s in, 7.1MB/s out * ``gzip -1``: 44.1MB/s in, 15.1MB/s out * ``lzop``: 71.9MB/s in, 28.1MB/s out If encryption is the limiting factor, and as in the example, limits the bandwidth to 40MB/s, ``lzop`` allows for an effective 79% increase in transfer speed. The fast ``gzip`` would also prove to be beneficial, but much less than ``lzop``. It should also be noted that as a rule of thumb, tools with a lower compression ratio had a lesser workload, with ``lzop`` straining the CPU much less than any of the competitors. With the test results present here, it is clear that ``lzop`` would be a very worthwhile addition to the compression options present in Ganeti, yet the problem is that it is not available by default on all distributions, as the option's presence might imply. In general, Ganeti may know how to use several tools, and check for their presence, but should add some way of at least hinting at which tools are available. Additionally, the user might want to use a tool that Ganeti did not account for. Allowing the tool to be named is also helpful, both for cases when multiple custom tools are to be used, and for distinguishing between various tools in case of e.g. inter-cluster moves. To this end, the ``--compression-tools`` cluster parameter will be added to Ganeti. It contains a list of names of compression tools that can be supplied as the parameter of ``--compress``, and by default it contains all the tools Ganeti knows how to use. The user can change the list as desired, removing entries that are not or should not be available on the cluster, and adding custom tools. Every custom tool is identified by its name, and Ganeti expects the name to correspond to a script invoking the compression tool. Without arguments, the script compresses input on stdin, outputting it on stdout. With the -d argument, the script does the same, only while decompressing. The -h argument is used to check for the presence of the script, and in this case, only the error code is examined. This syntax matches the ``gzip`` syntax well, which should allow most compression tools to be adapted to it easily. Ganeti will not allow arbitrary parameters to be passed to a compression tool, and will restrict the names to contain only a small but assuredly safe subset of characters - alphanumeric values and dashes and underscores. This minimizes the risk of security issues that could arise from an attacker smuggling a malicious command through RAPI. Common variations, like the speed/compression tradeoff of ``gzip``, will be handled by aliases, e.g. ``gzip-fast`` or ``gzip-slow``. It should also be noted that for some purposes - e.g. the writing of OVF files, ``gzip`` is the only allowed means of compression, and an appropriate error message should be displayed if the user attempts to use one of the other provided tools. Zeroing instance disks ====================== While compression lowers the amount of data sent, further reductions can be achieved by taking advantage of the structure of the disk - namely, sending only used disk sectors. There is no direct way to achieve this, as it would require that the move-instance tool is aware of the structure of the file system. Mounting the filesystem is not an option, primarily due to security issues. A disk primed to take advantage of a disk driver exploit could cause an attacker to breach instance isolation and gain control of a Ganeti node. An indirect way for this performance gain to be achieved is the zeroing of any hard disk space not in use. While this primarily means empty space, swap partitions can be zeroed as well. Sequences of zeroes can be compressed and thus transferred very efficiently, all without the host knowing that these are empty space. This approach can also be dangerous if a sparse disk is zeroed in this way, causing ballooning. As Ganeti does not seem to make special concessions for moving sparse disks, the only difference should be the disk space utilization on the current node. Zeroing approaches ++++++++++++++++++ Zeroing is a feasible approach, but the node cannot perform it as it cannot mount the disk. Only virtualization-based options remain, and of those, using Ganeti's own virtualization capabilities makes the most sense. There are two ways of doing this - creating a new helper instance, temporary or persistent, or reusing the target instance. Both approaches have their disadvantages. Creating a new helper instance requires managing its lifecycle, taking special care to make sure no helper instance remains left over due to a failed operation. Even if this were to be taken care of, disks are not yet separate entities in Ganeti, making the temporary transfer of disks between instances hard to implement and even harder to make robust. The reuse can be done by modifying the OS running on the instance to perform the zeroing itself when notified via the new instance communication mechanism, but this approach is neither generic, nor particularly safe. There is no guarantee that the zeroing operation will not interfere with the normal operation of the instance, nor that it will be completed if a user-initiated shutdown occurs. A better solution can be found by combining the two approaches - re-using the virtualized environment, but with a specifically crafted OS image. With the instance shut down as it should be in preparation for the move, it can be extended with an additional disk with the OS image on it. By prepending the disk and changing some instance parameters, the instance can boot from it. The OS can be configured to perform the zeroing on startup, attempting to mount any partitions with a filesystem present, and creating and deleting a zero-filled file on them. After the zeroing is complete, the OS should shut down, and the master should note the shutdown and restore the instance to its previous state. Note that the requirements above are very similar to the notion of a helper VM suggested in the OS install document. Some potentially unsafe actions are performed within a virtualized environment, acting on disks that belong or will belong to the instance. The mechanisms used will thus be developed with both approaches in mind. Implementation ++++++++++++++ There are two components to this solution - the Ganeti changes needed to boot the OS, and the OS image used for the zeroing. Due to the variety of filesystems and architectures that instances can use, no single ready-to-run disk image can satisfy the needs of all the Ganeti users. Instead, the instance-debootstrap scripts can be used to generate a zeroing-capable OS image. This might not be ideal, as there are lightweight distributions that take up less space and boot up more quickly. Generating those with the right set of drivers for the virtualization platform of choice is not easy. Thus we do not provide a script for this purpose, but the user is free to provide any OS image which performs the necessary steps: zero out all virtualization-provided devices on startup, shutdown immediately. The cluster-wide parameter controlling the image to be used would be called ``--zeroing-image``. The modifications to Ganeti code needed are minor. The zeroing functionality should be implemented as an extension of the instance export, and exposed as the ``--zero-free-space option``. Prior to beginning the export, the instance configuration is temporarily extended with a new read-only disk of sufficient size to host the zeroing image, and the changes necessary for the image to be used as the boot drive. The temporary nature of the configuration changes requires that they are not propagated to other nodes. While this would normally not be feasible with an instance using a disk template offering multi-node redundancy, experiments with the code have shown that the restriction on diverse disk templates can be bypassed to temporarily allow a plain disk-template disk to host the zeroing image. Given that one of the planned changes in Ganeti is to have instance disks as separate entities, with no restriction on templates, this assumption is useful rather than harmful by asserting the desired behavior. The image is dumped to the disk, and the instance is started up. Once the instance is started up, the zeroing will proceed until completion, when a self-initiated shutdown will occur. The instance-shutdown detection capabilities of 2.11 should prevent the watcher from restarting the instance once this happens, allowing the host to take it as a sign the zeroing was completed. Either way, the host waits until the instance is shut down, or a timeout has been reached and the instance is forcibly shut down. As the time needed to zero an instance is dependent on the size of the disk of the instance, the user can provide a fixed and a per-size timeout, recommended to be set to twice the maximum write speed of the device hosting the instance. Better progress monitoring can be implemented with the instance-host communication channel proposed by the OS install design document. The first version will most likely use only the shutdown detection, and will be improved to account for the available communication channel at a later time. After the shutdown, the temporary disk is destroyed and the instance configuration is reverted to its original state. The very same action is done if any error is encountered during the zeroing process. In the case that the zeroing is interrupted while the zero-filled file is being written, the file may remain on the disk of the instance. The script that performs the zeroing will be made to react to system signals by deleting the zero-filled file, but there is little else that can be done to recover. When to use zeroing +++++++++++++++++++ The question of when it is useful to use zeroing is hard to answer because the effectiveness of the approach depends on many factors. All compression tools compress zeroes to almost nothingness, but compressing them takes time. If the time needed to compress zeroes were equal to zero, the approach would boil down to whether it is faster to zero unused space out, performing writes to disk, or to transfer it compressed. For the example used above, the average compression ratio, and write speeds of current disk drives, the answer would almost unanimously be yes. With a more realistic setup, where zeroes take time to compress, yet less time than ordinary data, the gains depend on the previously mentioned tradeoff and the free space available. Zeroing will definitely lessen the amount of bandwidth used, but it can lead to the connection being underutilized due to the time spent compressing data. It is up to the user to make these tradeoffs, but zeroing should be seen primarily as a means of further reducing the amount of data sent while increasing disk activity, with possible speed gains that should not be relied upon. In the future, the VM created for zeroing could also undertake other tasks related to the move, such as compression and encryption, and produce a stream of data rather than just modifying the disk. This would lessen the strain on the resources of the hypervisor, both disk I/O and CPU usage, and allow moves to obey the resource constraints placed on the instance being moved. Lock reduction ============== An instance move as executed by the move-instance tool consists of several preparatory RAPI calls, leading up to two long-lasting opcodes: OpCreateInstance and OpBackupExport. While OpBackupExport locks only the instance, the locks of OpCreateInstance require more attention. When executed, this opcode attempts to lock all nodes on which the instance may be created and obtain shared locks on the groups they belong to. In the case that an IAllocator is used, this means all nodes must be locked. Any operation that requires a node lock to be present can delay the move operation, and there is no shortage of these. The concept of opportunistic locking has been introduced to remedy exactly this situation, allowing the IAllocator to lock as many nodes as possible. Depending whether the allocation can be made on these nodes, the operation either proceeds as expected, or fails noting that it is temporarily infeasible. The failure case would change the semantics of the move-instance tool, which is expected to fail only if the move is impossible. To yield the benefits of opportunistic locking yet satisfy this constraint, the move-instance tool can be extended with the --opportunistic-tries and --opportunistic-try-delay options. A number of opportunistic instance creations are attempted, with a delay between attempts. The delay is slightly altered every time to avoid timing issues. Should all attempts fail, a normal instance creation is requested, which blocks until all the locks can be acquired. While it may seem excessive to grab so many node locks, the early release mechanism is used to make the situation less dire, releasing all nodes that were not chosen as candidates for allocation. This is taken to the extreme as all the locks acquired are released prior to the start of the transfer, barring the newly-acquired lock over the new instance. This works because all operations that alter the node in a way which could affect the transfer: * are prevented by the instance lock or instance presence, e.g. gnt-node remove, gnt-node evacuate, * do not interrupt the transfer, e.g. a PV on the node can be set as unallocatable, and the transfer still proceeds as expected, * do not care, e.g. a gnt-node powercycle explicitly ignores all locks. This invariant should be kept in mind, and perhaps verified through tests. All in all, there is very little space to reduce the number of locks used, and the only improvement that can be made is introducing opportunistic locking as an option of move-instance. Introduction of changes ======================= All the changes noted will be implemented in Ganeti 2.12, in the way described in the previous chapters. They will be implemented as separate changes, first the lock reduction, then the instance zeroing, then the compression improvements, and finally the encryption changes. ganeti-3.1.0~rc2/doc/design-multi-reloc.rst000064400000000000000000000120011476477700300206000ustar00rootroot00000000000000=================================== Moving instances across node groups =================================== :Created: 2011-Mar-28 :Status: Implemented :Ganeti-Version: 2.5.0 This design document explains the changes needed in Ganeti to perform instance moves across node groups. Reader familiarity with the following existing documents is advised: - :doc:`Current IAllocator specification ` - :doc:`Shared storage model in 2.3+ ` Motivation and and design proposal ================================== At the moment, moving instances away from their primary or secondary nodes with the ``relocate`` and ``multi-evacuate`` IAllocator calls restricts target nodes to those on the same node group. This ensures a mobility domain is never crossed, and allows normal operation of each node group to be confined within itself. It is desirable, however, to have a way of moving instances across node groups so that, for example, it is possible to move a set of instances to another group for policy reasons, or completely empty a given group to perform maintenance operations. To implement this, we propose the addition of new IAllocator calls to compute inter-group instance moves and group-aware node evacuation, taking into account mobility domains as appropriate. The interface proposed below should be enough to cover the use cases mentioned above. With the implementation of this design proposal, the previous ``multi-evacuate`` mode will be deprecated. .. _multi-reloc-detailed-design: Detailed design =============== All requests honor the groups' ``alloc_policy`` attribute. Changing instance's groups -------------------------- Takes a list of instances and a list of node group UUIDs; the instances will be moved away from their current group, to any of the groups in the target list. All instances need to have their primary node in the same group, which may not be a target group. If the target group list is empty, the request is simply "change group" and the instances are placed in any group but their original one. Node evacuation --------------- Evacuates instances off their primary nodes. The evacuation mode can be given as ``primary-only``, ``secondary-only`` or ``all``. The call is given a list of instances whose primary nodes need to be in the same node group. The returned nodes need to be in the same group as the original primary node. .. _multi-reloc-result: Result ------ In all storage models, an inter-group move can be modeled as a sequence of **replace secondary**, **migration** and **failover** operations (when shared storage is used, they will all be failover or migration operations within the corresponding mobility domain). The result of the operations described above must contain two lists of instances and a list of jobs (each of which is a list of serialized opcodes) to actually execute the operation. :doc:`Job dependencies ` can be used to force jobs to run in a certain order while still making use of parallelism. The two lists of instances describe which instances could be moved/migrated and which couldn't for some reason ("unsuccessful"). The union of the instances in the two lists must be equal to the set of instances given in the original request. The successful list of instances contains elements as follows:: (instance name, target group name, [chosen node names]) The choice of names is simply for readability reasons (for example, Ganeti could log the computed solution in the job information) and for being able to check (manually) for consistency that the generated opcodes match the intended target groups/nodes. Note that for the node-evacuate operation, the group is not changed, but it should still be returned as such (as it's easier to have the same return type for both operations). The unsuccessful list of instances contains elements as follows:: (instance name, explanation) where ``explanation`` is a string describing why the plugin was not able to relocate the instance. The client is given a list of job IDs (see the :doc:`design for LU-generated jobs `) which it can watch. Failures should be reported to the user. .. highlight:: python Example job list:: [ # First job [ { "OP_ID": "OP_INSTANCE_MIGRATE", "instance_name": "inst1.example.com", }, { "OP_ID": "OP_INSTANCE_MIGRATE", "instance_name": "inst2.example.com", }, ], # Second job [ { "OP_ID": "OP_INSTANCE_REPLACE_DISKS", "depends": [ [-1, ["success"]], ], "instance_name": "inst2.example.com", "mode": "replace_new_secondary", "remote_node": "node4.example.com", }, ], # Third job [ { "OP_ID": "OP_INSTANCE_FAILOVER", "depends": [ [-2, []], ], "instance_name": "inst8.example.com", }, ], ] Accepted opcodes: - ``OP_INSTANCE_FAILOVER`` - ``OP_INSTANCE_MIGRATE`` - ``OP_INSTANCE_REPLACE_DISKS`` .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-multi-storage-htools.rst000064400000000000000000000142761476477700300224660ustar00rootroot00000000000000================================================== HTools support for multiple storage units per node ================================================== :Created: 2015-Jan-21 :Status: Draft .. contents:: :depth: 4 This design document describes changes to hbal and related components (first and foremost LUXI), that will allow it to handle nodes that can't be considered monolithic in regard to disk layout, for example because they have multiple different storage units available. Current state and shortcomings ============================== Currently the htools assume that there is one storage unit per node and that it can be arbitrarily split among instances. This leads to problems in clusters where multiple storage units are present: There might be 10GB DRBD and 10GB plain storage available on a node, for a total of 20GB. If an instance that uses 15GB of a single type of storage is requested, it can't actually fit on the node, but the current implementation of hail doesn't notice this. This behaviour is clearly wrong, but the problem doesn't arise often in current setup, due to the fact that instances currently only have a single storage type and that users typically use node groups to differentiate between different node storage layouts. For the node show action, RAPI only returns * ``dfree``: The total amount of free disk space * ``dtotal``: The total amount of disk space which is insufficient for the same reasons. Proposed changes ================ Definitions ----------- * All disks have exactly one *desired storage unit*, which determines where and how the disk can be stored. If the disk is transfered, the desired storage unit remains unchanged. The desired storage unit includes specifics like the volume group in the case of LVM based storage. * A *storage unit* is a specific storage location on a specific node. Storage units have exactly one desired storage unit they can contain. A storage unit further has an identifier (containing the storage type, a key and possibly parameters), a total capacity, and a free capacity. A node cannot contain multiple storage units of the same desired storage unit. * For the purposes of this document a *disk* has a desired storage unit and a size. * A *disk can be moved* to a node, if there is at least one storage unit on that node which can contain the desired storage unit of the disk and if the free capacity is at least the size of the disk. * An *instance can be moved* to a node, if all its disks can be moved there one-by-one. LUXI and IAllocator protocol extension -------------------------------------- The LUXI and IAllocator protocols are extended to include in the ``node``: * ``storage``: a list of objects (storage units) with #. Storage unit, containing in order: #. storage type #. storage key (e.g. volume group name) #. extra parameters (e.g. flag for exclusive storage) as a list. #. Amount free in MiB #. Amount total in MiB .. code-block:: javascript { "storage": [ { "sunit": ["drbd8", "xenvg", []] , "free": 2000, , "total": 4000 }, { "sunit": ["file", "/path/to/storage1", []] , "free": 5000, , "total": 10000 }, { "sunit": ["file", "/path/to/storage2", []] , "free": 1000, , "total": 20000 }, { "sunit": ["lvm-vg", "xenssdvg", [false]] , "free": 1024, , "total": 1024 } ] } is a node with an LVM volume group mirrored over DRBD, two file storage directories, one half full, one mostly full, and a non-mirrored volume group. The storage type ``drbd8`` needs to be added in order to differentiate between mirrored storage and non-mirrored storage. The storage key signals the volume group used and the storage unit takes no additional parameters. Text protocol extension ----------------------- The same field is optionally present in the HTools text protocol: * a new "storage" column is added to the node section, which is a semicolon separated list of comma separated fields in the order #. ``free`` #. ``total`` #. ``sunit``, which in itself contains #. the storage type #. the storage key #. extra arguments For example: 2000,4000,drbd,xenvg;5000,10000,file,/path/to/storage1;1000,20000; [...] Interpretation -------------- ``hbal`` and ``hail`` will use this information only if available, if the data file doesn't contain the ``storage`` field the old algorithm is used. If the node information contains the ``storage`` field, hbal and hail will assume that only the space compatible with the disk's requirements is available. For an instance to fit a node, all it's disks need to fit there separately. For a disk to fit a node, a storage unit of the type of the disk needs to have enough free space to contain it. The total free storage is not taken into consideration. Ignoring the old information will in theory introduce a backwards incompatibility: If the total free storage is smaller than to the sum of the free storage reported in the ``storage`` field a previously illegal move will become legal. Balancing --------- In order to determine a storage location for an instance, we collect analogous metrics to the current total node free space metric -- namely the standard deviation statistic of the free space per storage unit. The *standard deviation metric* of a desired storage unit is the sample standard deviation of the percentage of free space of storage units compatible. The *full storage metric* is a average of the standard deviation metrics of the desired storage units. This is backwards compatible in-so-far as that #. For a single storage unit per node it will have the same value. #. The weight of the storage versus the other metrics remains unchanged. Further this retains the property that scarce resources with low total will tend to have bigger impact on the metric than those with large totals, because in latter case the relative differences will not make for a large standard deviation. Ignoring nodes that do not contain the desired storage unit additionally boosts the importance of the scarce desired storage units, because having more storage units of a desired storage unit will tend to make the standard deviation metric smaller. ganeti-3.1.0~rc2/doc/design-multi-version-tests.rst000064400000000000000000000162651476477700300223410ustar00rootroot00000000000000=================== Multi-version tests =================== :Created: 2013-Oct-30 :Status: Implemented :Ganeti-Version: 2.11.0 .. contents:: :depth: 4 This is a design document describing how tests which use multiple versions of Ganeti can be introduced into the current build infrastructure. Desired improvements ==================== The testing of Ganeti is currently done by using two different approaches - unit tests and QA. While the former are useful for ensuring that the individual parts of the system work as expected, most errors are discovered only when all the components of Ganeti interact during QA. However useful otherwise, until now the QA has failed to provide support for testing upgrades and version compatibility as it was limited to using only one version of Ganeti. While these can be tested for every release manually, a systematic approach is preferred and none can exist with this restriction in place. To lift it, the buildbot scripts and QA utilities must be extended to allow a way of specifying and using diverse multi-version checks. Required use cases ================== There are two classes of multi-version tests that are interesting in Ganeti, and this chapter provides an example from each to highlight what should be accounted for in the design. Compatibility tests ------------------- One interface Ganeti exposes to clients interested in interacting with it is the RAPI. Its stability has always been a design principle followed during implementation, but whether it held true in practice was not asserted through tests. An automatic test of RAPI compatibility would have to take a diverse set of RAPI requests and perform them on two clusters of different versions, one of which would be the reference version. If the clusters had been identically configured, all of the commands successfully executed on the reference version should succeed on the newer version as well. To achieve this, two versions of Ganeti can be run separately on a cleanly setup cluster. With no guarantee that the versions can coexist, the deployment of these has to be separate. A proxy placed between the client and Ganeti records all the requests and responses. Using this data, a testing utility can decide if the newer version is compatible or not, and provide additional information to assist with debugging. Upgrade / downgrade tests ------------------------- An upgrade / downgrade test serves to examine whether the state of the cluster is unchanged after its configuration has been upgraded or downgraded to another version of Ganeti. The test works with two consecutive versions of Ganeti, both installed on the same machine. It examines whether the configuration data and instances survive the downgrade and upgrade procedures. This is done by creating a cluster with the newer version, downgrading it to the older one, and upgrading it to the newer one again. After every step, the integrity of the cluster is checked by running various operations and ensuring everything still works. Design and implementation ========================= Although the previous examples have not been selected to show use cases as diverse as possible, they still show a number of dissimilarities: - Parallel installation vs sequential deployments - Comparing with reference version vs comparing consecutive versions - Examining result dumps vs trying a sequence of operations With the first two real use cases demonstrating such diversity, it does not make sense to design multi-version test classes. Instead, the programmability of buildbot's configuration files can be leveraged to implement each test as a separate builder with a custom sequence of steps. The individual steps such as checking out a given or previous version, or installing and removing Ganeti, will be provided as utility functions for any test writer to use. Current state ------------- An upgrade / downgrade test is a part of the QA suite as of commit aa104b5e. The test and the corresponding buildbot changes are a very good first step, both by showing that multi-version tests can be done, and by providing utilities needed for builds of multiple branches. Previously, the same folder was used as the base directory of any build, and now a directory structure more accommodating to multiple builds is in place. The builder running the test has one flaw - regardless of the branch submitted, it compares versions 2.10 and 2.11 (current master). This behaviour is different from any of the other builders, which may restrict the branches a test can be performed on, but do not differentiate between them otherwise. While additional builders for different versions pairs may be added, this is not a good long-term solution. The test can be improved by making it compare the current and the previous version. As the buildbot has no notion of what a previous version is, additional utilities to handle this logic will have to be introduced. Planned changes --------------- The upgrade / downgrade test should be generalized to work for any version which can be downgraded from and upgraded to automatically, meaning versions from 2.11 onwards. This will be made challenging by the fact that the previous version has to be checked out by reading the version of the currently checked out code, identifying the previous version, and then making yet another checkout. The major and minor version can be read from a Ganeti repository in multiple ways. The two are present as constants defined in source files, but due to refactorings shifting constants from the Python to the Haskell side, their position varies across versions. A more reliable way of fetching them is by examining the news file, as it obeys strict formatting restrictions. With the version found, a script that acts as a previous version lookup table can be invoked. This script can be constructed dynamically upon buildbot startup, and specified as a build step. The checkout following it proceeds as expected. The RAPI compatibility test should be added as a separate builder afterwards. As the test requires additional comparison and proxy logic to be used, it will be enabled only on 2.11 onwards, comparing the versions to 2.6 - the reference version for the RAPI. Details on the design of this test will be added in a separate document. Potential issues ================ While there are many advantages to having a single builder representing a multi-version test, working on every branch, there is at least one disadvantage: the need to define a base or reference version, which is the only version that can be used to trigger the test, and the only one on which code changes can be tried. If an error is detected while running a test, and the issue lies with a version other than the one used to invoke the test, the fix would have to make it into the repository before the test could be tried again. For simple tests, the issue might be mitigated by running them locally. However, the multi-version tests are more likely to be complicated than not, and it could be difficult to reproduce a test by hand. The situation can be made simpler by requiring that any multi-version test can use only versions lower than the reference version. As errors are more likely to be found in new rather than old code, this would at least reduce the number of troublesome cases. ganeti-3.1.0~rc2/doc/design-network.rst000064400000000000000000000256521476477700300200550ustar00rootroot00000000000000================== Network management ================== :Created: 2011-Jun-17 :Status: Partially Implemented :Ganeti-Version: 2.7.0 .. contents:: :depth: 4 This is a design document detailing the implementation of network resource management in Ganeti. Current state and shortcomings ============================== Currently Ganeti supports two configuration modes for instance NICs: routed and bridged mode. The ``ip`` NIC parameter, which is mandatory for routed NICs and optional for bridged ones, holds the given NIC's IP address and may be filled either manually, or via a DNS lookup for the instance's hostname. This approach presents some shortcomings: a) It relies on external systems to perform network resource management. Although large organizations may already have IP pool management software in place, this is not usually the case with stand-alone deployments. For smaller installations it makes sense to allocate a pool of IP addresses to Ganeti and let it transparently assign these IPs to instances as appropriate. b) The NIC network information is incomplete, lacking netmask and gateway. Operating system providers could for example use the complete network information to fully configure an instance's network parameters upon its creation. Furthermore, having full network configuration information would enable Ganeti nodes to become more self-contained and be able to infer system configuration (e.g. /etc/network/interfaces content) from Ganeti configuration. This should make configuration of newly-added nodes a lot easier and less dependent on external tools/procedures. c) Instance placement must explicitly take network availability in different node groups into account; the same ``link`` is implicitly expected to connect to the same network across the whole cluster, which may not always be the case with large clusters with multiple node groups. Proposed changes ---------------- In order to deal with the above shortcomings, we propose to extend Ganeti with high-level network management logic, which consists of a new NIC slot called ``network``, a new ``Network`` configuration object (cluster level) and logic to perform IP address pool management, i.e. maintain a set of available and occupied IP addresses. Configuration changes +++++++++++++++++++++ We propose the introduction of a new high-level Network object, containing (at least) the following data: - Symbolic name - UUID - Network in CIDR notation (IPv4 + IPv6) - Default gateway, if one exists (IPv4 + IPv6) - IP pool management data (reservations) - Default NIC connectivity mode (bridged, routed). This is the functional equivalent of the current NIC ``mode``. - Default host interface (e.g. br0). This is the functional equivalent of the current NIC ``link``. - Tags Each network will be connected to any number of node groups. During the connection of a network to a nodegroup, we define the corresponding connectivity mode (bridged or routed) and the host interface (br100 or routing_table_200). This is achieved by adding a ``networks`` slot to the NodeGroup object and using the networks' UUIDs as keys. The value for each key is a dictionary containing the network's ``mode`` and ``link`` (netparams). Every NIC assigned to the network will eventually inherit the network's netparams, as its nicparams. IP pool management ++++++++++++++++++ A new helper library is introduced, wrapping around Network objects to give IP pool management capabilities. A network's pool is defined by two bitfields, the length of the network size each: ``reservations`` This field holds all IP addresses reserved by Ganeti instances. ``external reservations`` This field holds all IP addresses that are manually reserved by the administrator (external gateway, IPs of external servers, etc) or automatically by ganeti (the network/broadcast addresses, Cluster IPs (node addresses + cluster master)). These IPs are excluded from the IP pool and cannot be assigned automatically by ganeti to instances (via ip=pool). The bitfields are implemented using the python-bitarray package for space efficiency and their binary value stored base64-encoded for JSON compatibility. This approach gives relatively compact representations even for large IPv4 networks (e.g. /20). Cluster IP addresses (node + master IPs) are reserved automatically as external if the cluster's data network itself is placed under pool management. Helper ConfigWriter methods provide free IP address generation and reservation, using a TemporaryReservationManager. It should be noted that IP pool management is performed only for IPv4 networks, as they are expected to be densely populated. IPv6 networks can use different approaches, e.g. sequential address assignment or EUI-64 addresses. New NIC parameter: network ++++++++++++++++++++++++++ In order to be able to use the new network facility while maintaining compatibility with the current networking model, a new NIC parameter is introduced, called ``network`` to reflect the fact that the given NIC belongs to the given network and its configuration is managed by Ganeti itself. To keep backwards compatibility, existing code is executed if the ``network`` value is 'none' or omitted during NIC creation. If we want our NIC to be assigned to a network, then only the ip (optional) and the network parameters should be passed. Mode and link are inherited from the network-nodegroup mapping configuration (netparams). This provides the desired abstraction between the VM's network and the node-specific underlying infrastructure. We also introduce a new ``ip`` address value, ``constants.NIC_IP_POOL``, that specifies that a given NIC's IP address should be obtained using the first available IP address inside the pool of the specified network. (reservations OR external_reservations). This value is only valid for NICs belonging to a network. A NIC's IP address can also be specified manually, as long as it is contained in the network the NIC is connected to. In case this IP is externally reserved, Ganeti will produce an error which the user can override if explicitly requested. Of course this IP will be reserved and will not be able to be assigned to another instance. Hooks +++++ Introduce new hooks concerning network operations: ``OP_NETWORK_ADD`` Add a network to Ganeti :directory: network-add :pre-execution: master node :post-execution: master node ``OP_NETWORK_REMOVE`` Remove a network from Ganeti :directory: network-remove :pre-execution: master node :post-execution: master node ``OP_NETWORK_SET_PARAMS`` Modify a network :directory: network-modify :pre-execution: master node :post-execution: master node For connect/disconnect operations use existing: ``OP_GROUP_SET_PARAMS`` Modify a nodegroup :directory: group-modify :pre-execution: master node :post-execution: master node Hook variables ^^^^^^^^^^^^^^ During instance related operations: ``INSTANCE_NICn_NETWORK`` The friendly name of the network During network related operations: ``NETWORK_NAME`` The friendly name of the network ``NETWORK_SUBNET`` The ip range of the network ``NETWORK_GATEWAY`` The gateway of the network During nodegroup related operations: ``GROUP_NETWORK`` The friendly name of the network ``GROUP_NETWORK_MODE`` The mode (bridged or routed) of the netparams ``GROUP_NETWORK_LINK`` The link of the netparams Backend changes +++++++++++++++ To keep the hypervisor-visible changes to a minimum, and maintain compatibility with the existing network configuration scripts, the instance's hypervisor configuration will have host-level mode and link replaced by the *connectivity mode* and *host interface* (netparams) of the given network on the current node group. Network configuration scripts detect if a NIC is assigned to a Network by the presence of the new environment variable: Network configuration script variables ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``NETWORK`` The friendly name of the network Conflicting IPs +++++++++++++++ To ensure IP uniqueness inside a nodegroup, we introduce the term ``conflicting ips``. Conflicting IPs occur: (a) when creating a networkless NIC with IP contained in a network already connected to the instance's nodegroup (b) when connecting/disconnecting a network to/from a nodegroup and at the same time instances with IPs inside the network's range still exist. Conflicting IPs produce prereq errors. Handling of conflicting IP with --force option: For case (a) reserve the IP and assign the NIC to the Network. For case (b) during connect same as (a), during disconnect release IP and reset NIC's network parameter to None Userland interface ++++++++++++++++++ A new client script is introduced, ``gnt-network``, which handles network-related configuration in Ganeti. Network addition/deletion ^^^^^^^^^^^^^^^^^^^^^^^^^ :: gnt-network add --network=192.168.100.0/28 --gateway=192.168.100.1 \ --network6=2001:db8:2ffc::/64 --gateway6=2001:db8:2ffc::1 \ --add-reserved-ips=192.168.100.10,192.168.100.11 net100 (Checks for already existing name and valid IP values) gnt-network remove network_name (Checks if not connected to any nodegroup) Network modification ^^^^^^^^^^^^^^^^^^^^ :: gnt-network modify --gateway=192.168.100.5 net100 (Changes the gateway only if ip is available) gnt-network modify --add-reserved-ips=192.168.100.11 net100 (Adds externally reserved ip) gnt-network modify --remove-reserved-ips=192.168.100.11 net100 (Removes externally reserved ip) Assignment to node groups ^^^^^^^^^^^^^^^^^^^^^^^^^ :: gnt-network connect net100 nodegroup1 bridged br100 (Checks for existing bridge among nodegroup) gnt-network connect net100 nodegroup2 routed rt_table (Checks for conflicting IPs) gnt-network disconnect net101 nodegroup1 (Checks for conflicting IPs) Network listing ^^^^^^^^^^^^^^^ :: gnt-network list Network Subnet Gateway NodeGroups GroupList net100 192.168.100.0/28 192.168.100.1 1 default(bridged, br100) net101 192.168.101.0/28 192.168.101.1 1 default(routed, rt_tab) Network information ^^^^^^^^^^^^^^^^^^^ :: gnt-network info testnet1 Network name: testnet1 subnet: 192.168.100.0/28 gateway: 192.168.100.1 size: 16 free: 10 (62.50%) usage map: 0 XXXXX..........X 63 (X) used (.) free externally reserved IPs: 192.168.100.0, 192.168.100.1, 192.168.100.15 connected to node groups: default(bridged, br100) used by 3 instances: test1 : 0:192.168.100.4 test2 : 0:192.168.100.2 test3 : 0:192.168.100.3 IAllocator changes ++++++++++++++++++ The IAllocator protocol can be made network-aware, i.e. also consider network availability for node group selection. Networks, as well as future shared storage pools, can be seen as constraints used to rule out the placement on certain node groups. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-network2.rst000064400000000000000000000453061476477700300201350ustar00rootroot00000000000000============================ Network Management (revised) ============================ :Created: 2014-Sep-26 :Status: Draft .. contents:: :depth: 4 This is a design document detailing how to extend the existing network management and make it more flexible and able to deal with more generic use cases. Current state and shortcomings ------------------------------ Currently in Ganeti, networks are tightly connected with IP pools, since creation of a network implies the existence of one subnet and the corresponding IP pool. This design does not allow common scenarios like: - L2 only networks - IPv6 only networks - Networks without an IP pool - Networks with an IPv6 pool - Networks with multiple IP pools (alternative to externally reserving IPs) Additionally one cannot have multiple IP pools inside one network. Finally, from the instance perspective, a NIC cannot get more than one IPs (v4 and v6). Proposed changes ---------------- In order to deal with the above shortcomings, we propose to extend the existing networks in Ganeti and support: a) Networks with multiple subnets b) Subnets with multiple IP pools c) NICs with multiple IPs from various subnets of a single network These changes bring up some design and implementation issues that we discuss in the following sections. Semantics ++++++++++ Quoting the initial network management design doc "an IP pool consists of two bitarrays. Specifically the ``reservations`` bitarray which holds all IP addresses reserved by Ganeti instances and the ``external reservations`` bitarray with all IPs that are excluded from the IP pool and cannot be assigned automatically by Ganeti to instances (via ip=pool)". Without violating those semantics, here, we clarify the following definitions. **network**: A cluster level taggable configuration object with a user-provider name, (e.g. network1, network2), UUID and MAC prefix. **L2**: The `mode` and `link` with which we connect a network to a nodegroup. A NIC attached to a network will inherit this info, just like connecting an Ethernet cable to a physical NIC. In this sense we only have one L2 info per NIC. **L3**: A CIDR and a gateway related to the network. Since a NIC can have multiple IPs on the same cable each network can have multiple L3 info with the restriction that they do not overlap with each other. The gateway is optional (just like with current implementation). No gateway can be used for private networks that do not have a default route. **subnet**: A subnet is the above L3 info plus some additional information (see below). **ip**: A valid IP should reside in a network's subnet, and should not be used by more than one instance. An IP can be either obtained dynamically from a pool or requested explicitly from a subnet (or a pool). **range**: Sequential IPs inside one subnet calculated either from the first IP and a size (e.g. start=192.0.2.10, size=10) or the first IP and the last IP (e.g. start=192.0.2.10, end=192.0.2.19). A single IP can also be thought of as an IP range with size=1 (see configuration changes). **reservations**: All IPs that are used by instances in the cluster at any time. **external reservations**: All IPs that are supposed to be reserved by the admin for either some external component or specific instances. If one instance requests an external IP explicitly (ip=192.0.2.100), Ganeti will allow the operation only if ``--force`` is given. Still, the admin can externally reserve an IP that is already in use by an instance, as happens now. This helps to reserve an IP for future use and at the same time prevent any possible race between the instance that releases this IP and another that tries to retrieve it. **pool**: A (range, reservations, name) tuple from which instances can dynamically obtain an IP. Reservations is a bitarray with length the size of the range, and is needed so that we know which IPs are available at any time without querying all instances. The use of name is explained below. A subnet can have multiple pools. Split L2 from L3 ++++++++++++++++ Currently networks in Ganeti do not separate L2 from L3. This means that one cannot use L2 only networks. The reason is because the CIDR (passed currently with the ``--network`` option) and the derived IP pool are mandatory. This design makes L3 info optional. This way we can have an L2 only network just by connecting a Ganeti network to a nodegroup with the desired `mode` and `link`. Then one could add one or more subnets to the existing network. Multiple Subnets per Network ++++++++++++++++++++++++++++ Currently the IPv4 CIDR is mandatory for a network. Also a network can obtain at most one IPv4 CIDR and one IPv6 CIDR. These restrictions will be lifted. This design doc introduces support for multiple subnets per network. The L3 info will be moved inside the subnet. A subnet will have a `name` and a `uuid` just like NIC and Disk config objects. Additionally it will contain the `dhcp` flag which is explained below, and the `pools` and `external` fields which are mentioned in the next section. Only the `cidr` will be mandatory. Any subnet related actions will be done via the new ``--subnet`` option. Its syntax will be similar to ``--net``. The network's subnets must not overlap with each other. Logic will validate any operations related to reserving/releasing of IPs and check whether a requested IP is included inside one of the network's subnets. Just like currently, the L3 info will be exported to NIC configuration hooks and scripts as environment variables. The example below adds subnets to a network: :: gnt-network modify --subnet add:cidr=10.0.0.0/24,gateway=10.0.0.1,dhcp=true net1 gnt-network modify --subnet add:cidr=2001::/64,gateway=2001::1,dhcp=true net1 To remove a subnet from a network one should use: :: gnt-network modify --subnet some-ident:remove network1 where some-ident can be either a CIDR, a name or a UUID. Ganeti will allow this operation only if no instances use IPs from this subnet. Since DHCP is allowed only for a single CIDR on the same cable, the subnet must have a `dhcp` flag. Logic must not allow more that one subnets of the same version (4 or 6) in the same network to have DHCP enabled. To modify a subnet's name or the dhcp flag one could use: :: gnt-network modify --subnet some-ident:modify,dhcp=false,name=foo network1 This would search for a registered subnet that matches the identifier, disable DHCP on it and change its name. The ``dhcp`` parameter is used only for validation purposes and does not make Ganeti starting a DHCP service. It will just be exported to external scripts (ifup and hooks) and handled accordingly. Changing the CIDR or the gateway of a subnet should also be supported. :: gnt-network modify --subnet some-ident:modify,cidr=192.0.2.0/22 net1 gnt-network modify --subnet some-ident:modify,cidr=192.0.2.32/28 net1 gnt-network modify --subnet some-ident:modify,gateway=192.0.2.40 net1 Before expanding a subnet logic should should check for overlapping subnets. Shrinking the subnet should be allowed only if the ranges that are about to be trimmed are not included either in pool reservations or external ranges. Multiple IP pools per Subnet ++++++++++++++++++++++++++++ Currently IP pools are automatically created during network creation and include the whole subnet. Some IPs can be excluded from the pool by passing them explicitly with ``--add-reserved-ips`` option. Still for IPv6 subnets or even big IPv4 ones this might be insufficient. It is impossible to have two bitarrays for a /64 prefix. Even for IPv4 networks a /20 subnet currently requires 8K long bitarrays. And the second 4K is practically useless since the external reservations are way less than the actual reservations. This design extract IP pool management from the network logic, and pools will become optional. Currently the pool is created based on the network's CIDR. With multiple subnets per network, we should be able to create and add IP pools to a network (and eventually to the corresponding subnet). Each pool will have an optional user friendly `name` so that the end user can refer to it (see instance related operations). The user will be able to obtain dynamically an IP only if we have already defined a pool for a network's subnet. One would use ``ip=pool`` for the first available IP of the first available pool, or ``ip=some-pool-name`` for the first available IP of a specific pool. Any pool related actions will be done via the new ``--pool`` option. In order to add a pool a relevant subnet should pre-exist. Overlapping pools won't be allowed. For example: :: gnt-network modify --pool add:192.0.2.10-192.0.2.100,name=pool1 net1 gnt-network modify --pool add:10.0.0.7-10.0.0.20 net1 gnt-network modify --pool add:10.0.0.100 net1 will first parse and find the ranges. Then for each range, Ganeti will try to find a matching subnet meaning that a pool must be a subrange of the subnet. If found, the range with empty reservations will be appended to the list of the subnet's pools. Moreover, logic must be added to reserve the IPs that are currently in use by instances of this network. Adding a pool can be easier if we associate it directly with a subnet. For example on could use the following shortcuts: :: gnt-network modify --subnet add:cidr=10.0.0.0/27,pool net1 gnt-network modify --pool add:subnet=some-ident gnt-network modify --pool add:10.0.0.0/27 net1 During pool removal, logic should be added to split pools if ranges given overlap existing ones. For example: :: gnt-network modify --pool remove:192.0.2.20-192.0.2.50 net1 will split the pool previously added (10-100) into two new ones; 10-19 and 51-100. The corresponding bitarrays will be trimmed accordingly. The name will be preserved. The same things apply to external reservations. Just like now, modifications will take place via the ``--add|remove-reserved-ips`` option. Logic must be added to support IP ranges. :: gnt-network modify --add-reserved-ips 192.0.2.20-192.0.2.50 net1 Based on the aforementioned we propose the following changes: 1) Change the IP pool representation in config data. Existing `reservations` and `external_reservations` bitarrays will be removed. Instead, for each subnet we will have: * `pools`: List of (IP range, reservations bitarray) tuples. * `external`: List of IP ranges For external ranges the reservations bitarray is not needed since this will be all 1's. A configuration example could be:: net1 { subnets [ uuid1 { name: subnet1 cidr: 192.0.2.0/24 pools: [ {range:Range(192.0.2.10, 192.0.2.15), reservations: 00000, name:pool1} ] reserved: [192.0.2.15] } uuid2 { name: subnet2 cidr: 10.0.0.0/24 pools: [ {range:10.0.0.8/29, reservations: 00000000, name:pool3} {range:10.0.0.40-10.0.0.45, reservations: 000000, name:pool3} ] reserved: [Range(10.0.0.8, 10.0.0.15), 10.2.4.5] } ] } Range(start, end) will be some json representation of an IPRange(). We decide not to store external reservations as pools (and in the same list) since we get the following advantages: - Keep the existing semantics for pools and external reservations. - Each list has similar entries: one has pools the other has ranges. The pool must have a bitarray, and has an optional name. It is meaningless to add a name and a bitarray to external ranges. - Each list must not have overlapping ranges. Still external reservations can overlap with pools. - The --pool option supports add|remove|modify command just like `--net` and `--disk` and operate on single entities (a restriction that is not needed for external reservations). - Another thing, and probably the most important, is that in order to get the first available IP, only the reserved list must be checked for conflicts. The ipaddr.summarize_address_range(first, last) could be very helpful. 2) Change the network module logic. The above changes should be done in the network module and be transparent to the rest of the Ganeti code. If a random IP from the networks is requested, Ganeti searches for an available IP from the first pool of the first subnet. If it is full it gets to the next pool. Then to the next subnet and so on. Of course the `external` IP ranges will be excluded. If an IP is explicitly requested, Ganeti will try to find a matching subnet. Its pools and external will be checked for availability. All this logic will be extracted in a separate class with helper methods for easier manipulation of IP ranges and bitarrays. Bitarray processing can be optimized too. The usage of bitarrays will be reduced since (a) we no longer have `external_reservations` and (b) pools will have shorter bitarrays (i.e. will not have to cover the whole subnet). Besides that, we could keep the bitarrays in memory, so that in most cases (e.g. adding/removing reservations, querying), we don't keep converting strings to bitarrays and vice versa. Also, the Haskell code could as well keep this in memory as a bitarray, and validate it on load. 3) Changes in config module. We should not have instances with the same IP inside the same network. We introduce _AllIPs() helper config method that will hold all existing (IP, network) tuples. Config logic will check this list as well before passing it to TemporaryReservationManager. 4) Change the query mechanism. Since we have more that one subnets the new `subnets` field will include a list of: * cidr: IPv4 or IPv6 CIDR * gateway: IPv4 or IPv6 address * dhcp: True or False * name: The user friendly name for the subnet Since we want to support small pools inside big subnets, current query results are not practical as far as the `map` field is concerned. It should be replaced with the new `pools` field for each subnet, which will contain: * start: The first IP of the pool * end: The last IP of the pool * map: A string with 'X' for reserved IPs (either external or not) and with '.' for all available ones inside the pool Multiple IPs per NIC ++++++++++++++++++++ Currently IP is a simple string inside the NIC object and there is a one-to-one mapping between the `ip` and the `network` slots. The whole logic behind this is that a NIC belongs to a network (cable) and inherits its mode and link. This rational will not change. Since this design adds support for multiple subnets per network, a NIC must be able to obtain multiple IPs from various subnets of the same network. Thus we change the `ip` slot into list. We introduce a new `ipX` attribute. For backwards compatibility `ip` will denote `ip0`. During instance related operations one could use something like: :: gnt-instance add --net 0:ip0=192.0.2.4,ip1=pool,ip2=some-pool-name,network=network1 inst1 gnt-instance add --net 0:ip=pool,network1 inst1 This will be parsed, converted to a proper list (e.g. ip = [192.0.2.4, "pool", "some-pool-name"]) and finally passed to the corresponding opcode. Based on the previous example, here the first IP will match subnet1, the second IP will be retrieved from the first available pool of the first available subnet, and the third from the pool with the some-pool name. During instance modification, the `ip` option will refer to the first IP of the NIC, whereas the `ipX` will refer to the X'th IP. As with NICs we start counting from 0 so `ip1` will refer to the second IP. For example one should pass: :: --net 0:modify,ip1=1.2.3.10 to change the second IP of the first NIC to 1.2.3.10, :: --net -1:add,ip0=pool,ip1=1.2.3.4,network=test to add a new NIC with two IPs, and :: --net 1:modify,ip1=none to remove the second IP of the second NIC. Configuration changes --------------------- IPRange config object: Introduce new config object that will hold ranges needed by pools, and reservations. It will be either a tuple of (start, size, end) or a simple string. The `end` is redundant and can derive from start and size in runtime, but will appear in the representation for readability and debug reasons. Pool config object: Introduce new config object to represent a single subnet's pool. It will have the `range`, `reservations`, `name` slots. The range slot will be an IPRange config object, the reservations a bitarray and the name a simple string. Subnet config object: Introduce new config object with slots: `name`, `uuid`, `cidr`, `gateway`, `dhcp`, `pools`, `external`. Pools is a list of Pool config objects. External is a list of IPRange config objects. All ranges must reside inside the subnet's CIDR. Only `cidr` will be mandatory. The `dhcp` attribute will be False by default. Network config objects: The L3 and the IP pool representation will change. Specifically all slots besides `name`, `mac_prefix`, and `tag` will be removed. Instead the slot `subnets` with a list of Subnet config objects will be added. NIC config objects: NIC's network slot will be removed and the `ip` slot will be modified to a list of strings. KVM runtime files: Any change done in config data must be done also in KVM runtime files. For this purpose the existing _UpgradeSerializedRuntime() can be used. Exported variables ------------------ The exported variables during instance related operations will be just like Linux uses aliases for interfaces. Specifically: ``IP:i`` for the ith IP. ``NETWORK_*:i`` for the ith subnet. * is SUBNET, GATEWAY, DHCP. In case of hooks those variables will be prefixed with ``INSTANCE_NICn`` for the nth NIC. Backwards Compatibility ----------------------- The existing networks representation will be internally modified. They will obtain one subnet, and one pool with range the whole subnet. During `gnt-network add` if the deprecated ``--network`` option is passed will still create a network with one subnet, and one IP pool with the size of the subnet. Otherwise ``--subnet`` and ``--pool`` options will be needed. The query mechanism will also include the deprecated `map` field. For the newly created network this will contain only the mapping of the first pool. The deprecated `network`, `gateway`, `network6`, `gateway6` fields will point to the first IPv4 and IPv6 subnet accordingly. During instance related operation the `ip` argument of the ``--net`` option will refer to the first IP of the NIC. Hooks and scripts will still have the same environment exported in case of single IP per NIC. This design allows more fine-grained configurations which in turn yields more flexibility and a wider coverage of use cases. Still basic cases (the ones that are currently available) should be easy to set up. Documentation will be enriched with examples for both typical and advanced use cases of gnt-network. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-node-add.rst000064400000000000000000000137761476477700300200430ustar00rootroot00000000000000===================================== Design for adding a node to a cluster ===================================== :Created: 2012-Oct-16 :Status: Implemented :Ganeti-Version: 2.7.0 .. contents:: :depth: 3 Note ---- Closely related to this design is the more recent design :doc:`node security ` which extends and changes some of the aspects mentioned in this document. Make sure that you read the more recent design as well to get an up to date picture of Ganeti's procedure for adding new nodes. Current state and shortcomings ------------------------------ Before a node can be added to a cluster, its SSH daemon must be re-configured to use the cluster-wide SSH host key. Ganeti 2.3.0 changed the way this is done by moving all related code to a separate script, ``tools/setup-ssh``, using Paramiko. Before all such configuration was done from ``lib/bootstrap.py`` using the system's own SSH client and a shell script given to said client through parameters. Both solutions controlled all actions on the connecting machine; the newly added node was merely executing commands. This implies and requires a tight coupling and equality between nodes (e.g. paths to files being the same). Most of the logic and error handling is also done on the connecting machine. Once a node's SSH daemon has been configured, more than 25 files need to be copied using ``scp`` before the node daemon can be started. No verification is being done before files are copied. Once the node daemon is started, an opcode is submitted to the master daemon, which will then copy more files, such as the configuration and job queue for master candidates, using RPC. This process is somewhat fragile and requires initiating many SSH connections. Proposed changes ---------------- SSH ~~~ The main goal is to move more logic to the newly added node. Instead of having a relatively large script executed on the master node, most of it is moved over to the added node. A new script named ``prepare-node-join`` is added. It receives a JSON data structure (defined :ref:`below `) on its standard input. Once the data has been successfully decoded, it proceeds to configure the local node's SSH daemon and root's SSH settings, after which the SSH daemon is restarted. All the master node has to do to add a new node is to gather all required data, build the data structure, and invoke the script on the node to be added. This will enable us to once again use the system's own SSH client and to drop the dependency on Paramiko for Ganeti itself (``ganeti-listrunner`` is going to continue using Paramiko). Eventually ``setup-ssh`` can be removed. Node daemon ~~~~~~~~~~~ Similar to SSH setup changes, the process of copying files and starting the node daemon will be moved into a dedicated program. On its standard input it will receive a standardized JSON structure (defined :ref:`below `). Once the input data has been successfully decoded and the received values were verified for sanity, the program proceeds to write the values to files and then starts the node daemon (``ganeti-noded``). To add a new node to the cluster, the master node will have to gather all values, build the data structure, and then invoke the newly added ``node-daemon-setup`` program via SSH. In this way only a single SSH connection is needed and the values can be verified before being written to files. If the program exits successfully, the node is ready to be added to the master daemon's configuration. The node daemon will be running, but ``OpNodeAdd`` needs to be run before it becomes a full node. The opcode will copy more files, such as the :doc:`RAPI certificate `. Data structures --------------- .. _prepare-node-join-json: JSON structure for SSH setup ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The data is given in an object containing the keys described below. Unless specified otherwise, all entries are optional. ``cluster_name`` Required string with the cluster name. If a local cluster name is found, the join process is aborted unless the passed cluster name matches the local name. ``node_daemon_certificate`` Public part of cluster's node daemon certificate in PEM format. If a local node certificate and key is found, the join process is aborted unless this passed public part can be verified with the local key. ``ssh_host_key`` List containing public and private parts of SSH host key. See below for definition. ``ssh_root_key`` List containing public and private parts of root's key for SSH authorization. See below for definition. Lists of SSH keys use a tuple with three values. The first describes the key variant (``rsa`` or ``dsa``). The second and third are the private and public part of the key. Example: .. highlight:: javascript :: [ ("rsa", "-----BEGIN RSA PRIVATE KEY-----...", "ssh-rss AAAA..."), ("dsa", "-----BEGIN DSA PRIVATE KEY-----...", "ssh-dss AAAA..."), ] .. _node-daemon-setup-json: JSON structure for node daemon setup ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The data is given in an object containing the keys described below. Unless specified otherwise, all entries are optional. ``cluster_name`` Required string with the cluster name. If a local cluster name is found, the join process is aborted unless the passed cluster name matches the local name. The cluster name is also included in the dictionary given via the ``ssconf`` entry. ``node_daemon_certificate`` Public and private part of cluster's node daemon certificate in PEM format. If a local node certificate is found, the process is aborted unless it matches. ``ssconf`` Dictionary with ssconf names and their values. Both are strings. Example: .. highlight:: javascript :: { "cluster_name": "cluster.example.com", "master_ip": "192.168.2.1", "master_netdev": "br0", // ... } ``start_node_daemon`` Boolean denoting whether the node daemon should be started (or restarted if it was running for some reason). .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-node-security.rst000064400000000000000000000747731476477700300211660ustar00rootroot00000000000000============================= Improvements of Node Security ============================= :Created: 2013-Dec-05 :Status: Partially Implemented :Ganeti-Version: 2.11.0, 2.12.0, 2.13.0 This document describes an enhancement of Ganeti's security by restricting the distribution of security-sensitive data to the master and master candidates only. Note: In this document, we will use the term 'normal node' for a node that is neither master nor master-candidate. .. contents:: :depth: 4 Objective ========= Up till 2.10, Ganeti distributes security-relevant keys to all nodes, including nodes that are neither master nor master-candidates. Those keys are the private and public SSH keys for node communication and the SSL certificate and private key for RPC communication. Objective of this design is to limit the set of nodes that can establish ssh and RPC connections to the master and master candidates. As pointed out in `issue 433 `_, this is a security risk. Since all nodes have these keys, compromising any of those nodes would possibly give an attacker access to all other machines in the cluster. Reducing the set of nodes that are able to make ssh and RPC connections to the master and master candidates would significantly reduce the risk simply because fewer machines would be a valuable target for attackers. Note: For bigger installations of Ganeti, it is advisable to run master candidate nodes as non-vm-capable nodes. This would reduce the attack surface for the hypervisor exploitation. Detailed design =============== Current state and shortcomings ------------------------------ Currently (as of 2.10), all nodes hold the following information: - the ssh host keys (public and private) - the ssh root keys (public and private) - node daemon certificate (the SSL client certificate and its corresponding private key) Concerning ssh, this setup contains the following security issue. Since all nodes of a cluster can ssh as root into any other cluster node, one compromised node can harm all other nodes of a cluster. Regarding the SSL encryption of the RPC communication with the node daemon, we currently have the following setup. There is only one certificate which is used as both, client and server certificate. Besides the SSL client verification, we check if the used client certificate is the same as the certificate stored on the server. This means that any node running a node daemon can also act as an RPC client and use it to issue RPC calls to other cluster nodes. This in turn means that any compromised node could be used to make RPC calls to any node (including itself) to gain full control over VMs. This could be used by an attacker to for example bring down the VMs or exploit bugs in the virtualization stacks to gain access to the host machines as well. Proposal concerning SSH host key distribution --------------------------------------------- We propose the following design regarding the SSH host key handling. The root keys are untouched by this design. Each node gets its own ssh private/public key pair, but only the public keys of the master candidates get added to the ``authorized_keys`` file of all nodes. This has the advantages, that: - Only master candidates can ssh into other nodes, thus compromised nodes cannot compromise the cluster further. - One can remove a compromised master candidate from a cluster (including removing it's public key from all nodes' ``authorized_keys`` file) without having to regenerate and distribute new ssh keys for all master candidates. (Even though it is be good practice to do that anyway, since the compromising of the other master candidates might have taken place already.) - If a (uncompromised) master candidate is offlined to be sent for repair due to a hardware failure before Ganeti can remove any keys from it (for example when the network adapter of the machine is broken), we don't have to worry about the keys being on a machine that is physically accessible. To ensure security while transferring public key information and updating the ``authorized_keys``, there are several other changes necessary: - Any distribution of keys (in this case only public keys) is done via SSH and not via RPC. An attacker who has RPC control should not be able to get SSH access where he did not have SSH access before already. - The only RPC calls that are made in this context are from the master daemon to the node daemon on its own host and noded ensures as much as possible that the change to be made does not harm the cluster's security boundary. - The nodes that are potential master candidates keep a list of public keys of potential master candidates of the cluster in a separate file called ``ganeti_pub_keys`` to keep track of which keys could possibly be added ``authorized_keys`` files of the nodes. We come to what "potential" means in this case in the next section. The key list is only transferred via SSH or written directly by noded. It is not stored in the cluster config, because the config is distributed via RPC. The following sections describe in detail which Ganeti commands are affected by the proposed changes. RAPI ~~~~ The design goal to limit SSH powers to master candidates conflicts with the current powers a user of the RAPI interface would have. The ``master_capable`` flag of nodes can be modified via RAPI. That means, an attacker that has access to the RAPI interface, can make all non-master-capable nodes master-capable, and then increase the master candidate pool size till all machines are master candidates (or at least a particular machine that he is aiming for). This means that with RAPI access and a compromised normal node, one can make this node a master candidate and then still have the power to compromise the whole cluster. To mitigate this issue, we propose the following changes: - Add a flag ``master_capability_rapi_modifiable`` to the cluster configuration which indicates whether or not it should be possible to modify the ``master_capable`` flag of nodes via RAPI. The flag is set to ``False`` by default and can itself only be changed on the commandline. In this design doc, we refer to the flag as the "rapi flag" from here on. - Only if the ``master_capability_rapi_modifiable`` switch is set to ``True``, it is possible to modify the master-capability flag of nodes. With this setup, there are the following definitions of "potential master candidates" depending on the rapi flag: - If the rapi flag is set to ``True``, all cluster nodes are potential master candidates, because as described above, all of them can eventually be made master candidates via RAPI and thus security-wise, we haven't won anything above the current SSH handling. - If the rapi flag is set to ``False``, only the master capable nodes are considered potential master candidates, as it is not possible to make them master candidates via RAPI at all. Note that when the rapi flag is changed, the state of the ``ganeti_pub_keys`` file on all nodes has to be updated accordingly. This should be done in the client script ``gnt_cluster`` before the RPC call to update the configuration is made, because this way, if someone would try to perform that RPC call on master to trick it into thinking that the flag is enabled, this would not help as the content of the ``ganeti_pub_keys`` file is a crucial part in the design of the distribution of the SSH keys. Note: One could think of always allowing to disable the master-capability via RAPI and just restrict the enabling of it, thus making it possible to RAPI-"freeze" the nodes' master-capability state once it disabled. However, we think these are rather confusing semantics of the involved flags and thus we go with proposed design. Note that this change will break RAPI compatibility, at least if the rapi flag is not explicitly set to ``True``. We made this choice to have the more secure option as default, because otherwise it is unlikely to be widely used. Cluster initialization ~~~~~~~~~~~~~~~~~~~~~~ On cluster initialization, the following steps are taken in bootstrap.py: - A public/private key pair is generated (as before), but only used by the first (and thus master) node. In particular, the private key never leaves the node. - A mapping of node UUIDs to public SSH keys is created and stored as text file in ``/var/lib/ganeti/ganeti_pub_keys`` only accessible by root (permissions 0600). The master node's uuid and its public key is added as first entry. The format of the file is one line per node, each line composed as ``node_uuid ssh_key``. - The node's public key is added to it's own ``authorized_keys`` file. (Re-)Adding nodes to a cluster ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ According to :doc:`design-node-add`, Ganeti transfers the ssh keys to every node that gets added to the cluster. Adding a new node will require the following steps. In gnt_node.py: - On the new node, a new public/private SSH key pair is generated. - The public key of the new node is fetched (via SSH) to the master node and if it is a potential master candidate (see definition above), it is added to the ``ganeti_pub_keys`` list on the master node. - The public keys of all current master candidates are added to the new node's ``authorized_keys`` file (also via SSH). In LUNodeAdd in cmdlib/node.py: - The LUNodeAdd determines whether or not the new node is a master candidate and in any case updates the cluster's configuration with the new nodes information. (This is not changed by the proposed design.) - If the new node is a master candidate, we make an RPC call to the node daemon of the master node to add the new node's public key to all nodes' ``authorized_keys`` files. The implementation of this RPC call has to be extra careful as described in the next steps, because compromised RPC security should not compromise SSH security. RPC call execution in noded (on master node): - Check that the public key of the new node is in the ``ganeti_pub_keys`` file of the master node to make sure that no keys of nodes outside the Ganeti cluster and no keys that are not potential master candidates gain SSH access in the cluster. - Via SSH, transfer the new node's public key to all nodes (including the new node) and add it to their ``authorized_keys`` file. - The ``ganeti_pub_keys`` file is transferred via SSH to all potential master candidates nodes except the master node (including the new one). In case of readding a node that used to be in the cluster before, handling of the SSH keys would basically be the same, in particular also a new SSH key pair is generated for the node, because we cannot be sure that the old key pair has not been compromised while the node was offlined. Note that for reasons of data hygiene, a node's ``ganeti_pub_keys`` file is cleared before the node is readded. Also, Ganeti attempts to remove any Ganeti keys from the ``authorized_keys`` file before the node is readded. However, since Ganeti does not keep a list of all keys ever used in the cluster, this applies only to keys which are currently used in the cluster. Note that Ganeti won't touch any keys that were added to the ``authorized_keys`` by other systems than Ganeti. Pro- and demoting a node to/from master candidate ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If the role of a node is changed from 'normal' to 'master_candidate', the procedure is the same as for adding nodes from the step "In LUNodeAdd ..." on. If a node gets demoted to 'normal', the master daemon makes a similar RPC call to the master node's node daemon as for adding a node. In the RPC call, noded will perform the following steps: - Check that the public key of the node to be demoted is indeed in the ``ganeti_pub_keys`` file to avoid deleting ssh keys of machines that don't belong to the cluster (and thus potentially lock out the administrator). - Via SSH, remove the key from all node's ``authorized_keys`` files. This affected the behavior of the following commands: :: gnt-node modify --master-candidate=yes gnt-node modify --master-candidate=no [--auto-promote] If the node has been master candidate already before the command to promote it was issued, Ganeti does not do anything. Note that when you demote a node from master candidate to normal node, another master-capable and normal node will be promoted to master candidate. For this newly promoted node, the same changes apply as if it was explicitly promoted. The same behavior should be ensured for the corresponding rapi command. Offlining and onlining a node ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When offlining a node, it immediately loses its role as master or master candidate as well. When it is onlined again, it will become master candidate again if it was so before. The handling of the keys should be done in the same way as when the node is explicitly promoted or demoted to or from master candidate. See the previous section for details. This affects the command: :: gnt-node modify --offline=yes gnt-node modify --offline=no [--auto-promote] For offlining, the removal of the keys is particularly important, as the detection of a compromised node might be the very reason for the offlining. Of course we cannot guarantee that removal of the key is always successful, because the node might not be reachable anymore. Even though it is a best-effort operation, it is still an improvement over the status quo, because currently Ganeti does not even try to remove any keys. The same behavior should be ensured for the corresponding rapi command. Cluster verify ~~~~~~~~~~~~~~ So far, ``gnt-cluster verify`` checks the SSH connectivity of all nodes to all other nodes. We propose to replace this by the following checks: - For all master candidates, we check if they can connect any other node in the cluster (other master candidates and normal nodes). - We check if the ``ganeti_pub_keys`` file contains keys of nodes that are no longer in the cluster or that are not potential master candidates. - For all normal nodes, we check if their key does not appear in other node's ``authorized_keys``. For now, we will only emit a warning rather than an error if this check fails, because Ganeti might be run in a setup where Ganeti is not the only system manipulating the SSH keys. Upgrades ~~~~~~~~ When upgrading from a version that has the previous SSH setup to the one proposed in this design, the upgrade procedure has to involve the following steps in the post-upgrade hook: - For all nodes, new SSH key pairs are generated. - All nodes and their public keys are added to the ``ganeti_pub_keys`` file and the file is copied to all nodes. - All keys of master candidate nodes are added to the ``authorized_keys`` files of all other nodes. Since this upgrade significantly changes the configuration of the clusters' nodes, we will add a note to the UPGRADE notes to make the administrator aware of this fact (in case he intends to enable access from normal nodes to master candidates for other reasons than Ganeti uses the machines). Also, in any operation where Ganeti creates new SSH keys, the old keys will be backed up and not simply overridden. Downgrades ~~~~~~~~~~ These downgrading steps will be implemented from 2.13 to 2.12: - The master node's private/public key pair will be distributed to all nodes (via SSH) and the individual SSH keys will be backed up. - The obsolete individual ssh keys will be removed from all nodes' ``authorized_keys`` file. Renew-Crypto ~~~~~~~~~~~~ The ``gnt-cluster renew-crypto`` command will be extended by a new option ``--new-ssh-keys``, which will renew all SSH keys on all nodes and rebuild the ``authorized_keys`` files and the ``ganeti_pub_keys`` files according to the previous sections. This operation will be performed considering the already stated security considerations, for example minimizing RPC calls, distribution of keys via SSH only etc. Compliance to --no-ssh-init and --no-node-setup ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ With this design, Ganeti will do more manipulations of SSH keys and ``authorized_keys`` files than before. If this is not feasible in a Ganeti environment, the administrator has the option to prevent Ganeti from performing any manipulations on the SSH setup of the nodes. The options for doing so, are ``--no-ssh-init`` for ``gnt-cluster init``, and ``--no-node-setup`` for ``gnt-node add``. Note that these options already existed before the implementation of this design, we just confirm that they will be complied to with the new design as well. Proposal regarding node daemon certificates ------------------------------------------- Regarding the node daemon certificates, we propose the following changes in the design. - Instead of using the same certificate for all nodes as both, server and client certificate, we generate a common server certificate (and the corresponding private key) for all nodes and a different client certificate (and the corresponding private key) for each node. The server certificate will be self-signed. The client certificate will be signed by the server certificate. The client certificates will use the node UUID as serial number to ensure uniqueness within the cluster. They will use the host's hostname as the certificate common name (CN). - In addition, we store a mapping of (node UUID, client certificate digest) in the cluster's configuration and ssconf for hosts that are master or master candidate. The client certificate digest is a hash of the client certificate. We suggest a 'sha1' hash here. We will call this mapping 'candidate map' from here on. - The node daemon will be modified in a way that on an incoming RPC request, it first performs a client verification (same as before) to ensure that the requesting host is indeed the holder of the corresponding private key. Additionally, it compares the digest of the certificate of the incoming request to the respective entry of the candidate map. If the digest does not match the entry of the host in the mapping or is not included in the mapping at all, the SSL connection is refused. This design has the following advantages: - A compromised normal node cannot issue RPC calls, because it will not be in the candidate map. (See the ``Drawbacks`` section regarding an indirect way of achieving this though.) - A compromised master candidate would be able to issue RPC requests, but on detection of its compromised state, it can be removed from the cluster (and thus from the candidate map) without the need for redistribution of any certificates, because the other master candidates can continue using their own certificates. However, it is best practice to issue a complete key renewal even in this case, unless one can ensure no actions compromising other nodes have not already been carried out. - A compromised node would not be able to use the other (possibly master candidate) nodes' information from the candidate map to issue RPCs, because the config just stores the digests and not the certificate itself. - A compromised node would be able to obtain another node's certificate by waiting for incoming RPCs from this other node. However, the node cannot use the certificate to issue RPC calls, because the SSL client verification would require the node to hold the corresponding private key as well. Drawbacks of this design: - Complexity of node and certificate management will be increased (see following sections for details). - If the candidate map is not distributed fast enough to all nodes after an update of the configuration, it might be possible to issue RPC calls from a compromised master candidate node that has been removed from the Ganeti cluster already. However, this is still a better situation than before and an inherent problem when one wants to distinguish between master candidates and normal nodes. - A compromised master candidate would still be able to issue RPC calls, if it uses ssh to retrieve another master candidate's client certificate and the corresponding private SSL key. This is an issue even with the first part of the improved handling of ssh keys in this design (limiting ssh keys to master candidates), but it will be eliminated with the second part of the design (separate ssh keys for each master candidate). - Even though this proposal is an improvement towards the previous situation in Ganeti, it still does not use the full power of SSL. For further improvements, see Section "Related and future work". - Signing the client certificates with the server certificate will increase the complexity of the renew-crypto, as a renewal of the server certificates requires the renewal (and signing) of all client certificates as well. Alternative proposals: - The initial version of this document described a setup where the client certificates were also self-signed. This led to a serious problem (Issue 1094), which would only have been solvable by distributing all client certificates to all nodes and load them as trusted CAs. As this would have resulted in having to restart noded on all nodes every time a node is added, removed, demoted or promoted, this was not feasible and we switched to client certificates which are signed by the server certificate. - Instead of generating a client certificate per node, one could think of just generating two different client certificates, one for normal nodes and one for master candidates. Noded could then just check if the requesting node has the master candidate certificate. Drawback of this proposal is that once one master candidate gets compromised, all master candidates would need to get a new certificate even if the compromised master candidate had not yet fetched the certificates from the other master candidates via ssh. - In addition to our main proposal, one could think of including a piece of data (for example the node's host name or UUID) in the RPC call which is encrypted with the requesting node's private key. The node daemon could check if the datum can be decrypted using the node's certificate. However, this would ensure similar functionality as SSL's built-in client verification and add significant complexity to Ganeti's RPC protocol. In the following sections, we describe how our design affects various Ganeti operations. Cluster initialization ~~~~~~~~~~~~~~~~~~~~~~ On cluster initialization, so far only the node daemon certificate was created. With our design, two certificates (and corresponding keys) need to be created, a server certificate to be distributed to all nodes and a client certificate only to be used by this particular node. In the following, we use the term node daemon certificate for the server certificate only. In the cluster configuration, the candidate map is created. It is populated with the respective entry for the master node. It is also written to ssconf. (Re-)Adding nodes ~~~~~~~~~~~~~~~~~ When a node is added, the server certificate is copied to the node (as before). Additionally, a new client certificate (and the corresponding private key) is created on the new node to be used only by the new node as client certificate. If the new node is a master candidate, the candidate map is extended by the new node's data. As before, the updated configuration is distributed to all nodes (as complete configuration on the master candidates and ssconf on all nodes). Note that distribution of the configuration after adding a node is already implemented, since all nodes hold the list of nodes in the cluster in ssconf anyway. If the configuration for whatever reason already holds an entry for this node, it will be overriden. When readding a node, the procedure is the same as for adding a node. Promotion and demotion of master candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When a normal node gets promoted to be master candidate, an entry to the candidate map has to be added and the updated configuration has to be distributed to all nodes. If there was already an entry for the node, we override it. On demotion of a master candidate, the node's entry in the candidate map gets removed and the updated configuration gets redistributed. The same procedure applied to onlining and offlining master candidates. Cluster verify ~~~~~~~~~~~~~~ Cluster verify will be extended by the following checks: - Whether each entry in the candidate map indeed corresponds to a master candidate. - Whether the master candidate's certificate digest match their entry in the candidate map. - Whether no node tries to use the certificate of another node. In particular, it is important to check that no normal node tries to use the certificate of a master candidate. - Whether there are still self-signed client certificates in use (from a pre 2.12.4 Ganeti version). Crypto renewal ~~~~~~~~~~~~~~ Currently, when the cluster's cryptographic tokens are renewed using the ``gnt-cluster renew-crypto`` command the node daemon certificate is renewed (among others). Option ``--new-cluster-certificate`` renews the node daemon certificate only. By adding an option ``--new-node-certificates`` we offer to renew the client certificate. Whenever the client certificates are renewed, the candidate map has to be updated and redistributed. If for whatever reason, the candidate map becomes inconsistent, for example due inconsistent updating after a demotion or offlining), the user can use this option to renew the client certificates and update the candidate certificate map. Note that renewing the server certificate requires all client certificates being renewed and signed by the new server certificate, because otherwise their signature can not be verified by the server who only has the new server certificate then. As there was a different design in place in Ganeti 2.12.4 and previous versions, we have to ensure that renew-crypto works on pre 2.12 versions and 2.12.1-4. Users that got hit by Issue 1094 will be encouraged to run renew-crypto at least once after switching to 2.12.5. Those who did not encounter this bug yet, will still get nagged friendly by gnt-cluster verify. Further considerations ---------------------- Watcher ~~~~~~~ The watcher is a script that is run on all nodes in regular intervals. The changes proposed in this design will not affect the watcher's implementation, because it behaves differently on the master than on non-master nodes. Only on the master, it issues query calls which would require a client certificate of a node in the candidate mapping. This is the case for the master node. On non-master node, it's only external communication is done via the ConfD protocol, which uses the hmac key, which is present on all nodes. Besides that, the watcher does not make any ssh connections, and thus is not affected by the changes in ssh key handling either. Other Keys and Daemons ~~~~~~~~~~~~~~~~~~~~~~ Ganeti handles a couple of other keys/certificates that have not been mentioned in this design so far. Also, other daemons than the ones mentioned so far perform intra-cluster communication. Neither the keys nor the daemons will be affected by this design for several reasons: - The hmac key used by ConfD (see :doc:`design-2.1`): the hmac key is still distributed to all nodes, because it was designed to be used for communicating with ConfD, which should be possible from all nodes. For example, the monitoring daemon which runs on all nodes uses it to retrieve information from ConfD. However, since communication with ConfD is read-only, a compromised node holding the hmac key does not enable an attacker to change the cluster's state. - The WConfD daemon writes the configuration to all master candidates via RPC. Since it only runs on the master node, it's ability to run RPC requests is maintained with this design. - The rapi SSL key certificate and rapi user/password file 'rapi_users' is already only copied to the master candidates (see :doc:`design-2.1`, Section ``Redistribute Config``). - The spice certificates are still distributed to all nodes, since it should be possible to use spice to access VMs on any cluster node. - The cluster domain secret is used for inter-cluster instance moves. Since instances can be moved from any normal node of the source cluster to any normal node of the destination cluster, the presence of this secret on all nodes is necessary. Related and Future Work ~~~~~~~~~~~~~~~~~~~~~~~ There a couple of suggestions on how to improve the SSL setup even more. As a trade-off wrt to complexity and implementation effort, we did not implement them yet (as of version 2.11) but describe them here for future reference. - The server certificate is currently self-signed and the client certificates are signed by the server certificate. It would increase the security if they were signed by a common CA. There is already a design doc for a Ganeti CA which was suggested in a different context (related to import/export). This would also be a benefit for the RPC calls. See design doc :doc:`design-impexp2` for more information. Implementing a CA is rather complex, because it would mean also to support renewing the CA certificate and providing and supporting infrastructure to revoke compromised certificates. - An extension of the previous suggestion would be to even enable the system administrator to use an external CA. Especially in bigger setups, where already an SSL infrastructure exists, it would be useful if Ganeti can simply be integrated with it, rather than forcing the user to use the Ganeti CA. - Ganeti RPC calls are currently done without checking if the hostname of the node complies with the common name of the certificate. This might be a desirable feature, but would increase the effort when a node is renamed. - The typical use case for SSL is to have one certificate per node rather than one shared certificate (Ganeti's noded server certificate) and a client certificate. One could change the design in a way that only one certificate per node is used, but this would require a common CA so that the validity of the certificate can be established by every node in the cluster. - With the proposed design, the serial numbers of the client certificates are set to the node UUIDs. This is technically also not complying to how SSL is supposed to be used, as the serial numbers should reflect the enumeration of certificates created by the CA. Once a CA is implemented, it might be reasonable to change this accordingly. The implementation of the proposed design also has the drawback of the serial number not changing even if the certificate is replaced by a new one (for example when calling ``gnt-cluster renew- crypt``), which also does not comply to way SSL was designed to be used. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-oob.rst000064400000000000000000000402231476477700300171320ustar00rootroot00000000000000==================================== Ganeti Node OOB Management Framework ==================================== :Created: 2010-Nov-04 :Status: Implemented :Ganeti-Version: 2.4.0 Objective --------- Extend Ganeti with Out of Band (:term:`OOB`) Cluster Node Management Capabilities. Background ---------- Ganeti currently has no support for Out of Band management of the nodes in a cluster. It relies on the OS running on the nodes and has therefore limited possibilities when the OS is not responding. The command ``gnt-node powercycle`` can be issued to attempt a reboot of a node that crashed but there are no means to power a node off and power it back on. Supporting this is very handy in the following situations: * **Emergency Power Off**: During emergencies, time is critical and manual tasks just add latency which can be avoided through automation. If a server room overheats, halting the OS on the nodes is not enough. The nodes need to be powered off cleanly to prevent damage to equipment. * **Repairs**: In most cases, repairing a node means that the node has to be powered off. * **Crashes**: Software bugs may crash a node. Having an OS independent way to power-cycle a node helps to recover the node without human intervention. Overview -------- Ganeti will be extended with OOB capabilities through adding a new **Cluster Parameter** (``--oob-program``), a new **Node Property** (``--oob-program``), a new **Node State (powered)** and support in ``gnt-node`` for invoking an **External Helper Command** which executes the actual OOB command (``gnt-node nodename ...``). The supported commands are: ``power on``, ``power off``, ``power cycle``, ``power status`` and ``health``. .. note:: The new **Node State (powered)** is a **State of Record** (:term:`SoR`), not a **State of World** (:term:`SoW`). The maximum execution time of the **External Helper Command** will be limited to 60s to prevent the cluster from getting locked for an undefined amount of time. Detailed Design --------------- New ``gnt-cluster`` Parameter +++++++++++++++++++++++++++++ | Program: ``gnt-cluster`` | Command: ``modify|init`` | Parameters: ``--oob-program`` | Options: ``--oob-program``: executable OOB program (absolute path) New ``gnt-cluster epo`` Command +++++++++++++++++++++++++++++++ | Program: ``gnt-cluster`` | Command: ``epo`` | Parameter: ``--on`` ``--force`` ``--groups`` ``--all`` | Options: ``--on``: By default epo turns off, with ``--on`` it tries to get the | cluster back online | ``--force``: To force the operation without asking for confirmation | ``--groups``: To operate on groups instead of nodes | ``--all``: To operate on the whole cluster This is a convenience command to allow easy emergency power off of a whole cluster or part of it. It takes care of all steps needed to get the cluster into a sane state to turn off the nodes. With ``--on`` it does the reverse and tries to bring the rest of the cluster back to life. .. note:: The master node is not able to shut itself cleanly down. Therefore, this command will not do all the work on single node clusters. On multi node clusters the command tries to find another master or if that is not possible prepares everything to the point where the user has to shutdown the master node itself alone this applies also to the single node cluster configuration. New ``gnt-node`` Property +++++++++++++++++++++++++ | Program: ``gnt-node`` | Command: ``modify|add`` | Parameters: ``--oob-program`` | Options: ``--oob-program``: executable OOB program (absolute path) .. note:: If ``--oob-program`` is set to ``!`` then the node has no OOB capabilities. Otherwise, we will inherit the node group respectively the cluster wide value. I.e. the nodes have to opt out from OOB capabilities. Addition to ``gnt-cluster verify`` ++++++++++++++++++++++++++++++++++ | Program: ``gnt-cluster`` | Command: ``verify`` | Parameter: None | Option: None | Additional Checks: 1. existence and execution flag of OOB program on all Master Candidates if the cluster parameter ``--oob-program`` is set or at least one node has the property ``--oob-program`` set. The OOB helper is just invoked on the master 2. check if node state powered matches actual power state of the machine for those nodes where ``--oob-program`` is set New Node State ++++++++++++++ Ganeti supports the following two boolean states related to the nodes: **drained** The cluster still communicates with drained nodes but excludes them from allocation operations **offline** if offline, the cluster does not communicate with offline nodes; useful for nodes that are not reachable in order to avoid delays And will extend this list with the following boolean state: **powered** if not powered, the cluster does not communicate with not powered nodes if the node property ``--oob-program`` is not set, the state powered is not displayed Additionally modify the meaning of the offline state as follows: **offline** if offline, the cluster does not communicate with offline nodes (**with the exception of OOB commands for nodes where** ``--oob-program`` **is set**); useful for nodes that are not reachable in order to avoid delays The corresponding command extensions are: | Program: ``gnt-node`` | Command: ``info`` | Parameter: [ ``nodename`` ... ] | Option: None Additional Output (:term:`SoR`, ommited if node property ``--oob-program`` is not set): powered: ``[True|False]`` | Program: ``gnt-node`` | Command: ``modify`` | Parameter: nodename | Option: [ ``--powered=yes|no`` ] | Reasoning: sometimes you will need to sync the :term:`SoR` with the :term:`SoW` manually | Caveat: ``--powered`` can only be modified if ``--oob-program`` is set for | the node in question New ``gnt-node`` commands: ``power [on|off|cycle|status]`` ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ | Program: ``gnt-node`` | Command: ``power [on|off|cycle|status]`` | Parameters: [ ``nodename`` ... ] | Options: None | Caveats: * If no nodenames are passed to ``power [on|off|cycle]``, the user will be prompted with ``"Do you really want to power [on|off|cycle] the following nodes: `` with defaulting to constants.DEFAULT_OVS ``ovs-vsctl add-port `` optional: connection to the outside This will give us 2 parameters, that are needed for the OpenvSwitch Setup: switchname: Which will default to constants.DEFAULT_OVS when not given ethernet device: Which will default to None when not given, might be more than one (NIC bonding) These parameters should be set at node level for individuality, _but_ can have defined defaults on cluster and node group level, which can be inherited and thus allow a cluster or node group wide configuration. If a node is setup without parameters, it should use the settings from the parent node group or cluster. If none are given there, defaults should be used. As a first step, this will be implemented for using 1 ethernet device only. Functions for nic bonding will be added later on. Configuration changes for VLANs +++++++++++++++++++++++++++++++ nicparams shall be extended by a value "vlan" that will store the VLAN information for each NIC. This parameter will only be used if nicparams[constants.NIC_MODE] == constants.NIC_MODE_OVS, since it doesn't make sense in other modes. Each VLAN the NIC belongs to shall be stored in this single value. The format of storing this information is the same as the one which is used in Xen 4.3, since Xen 4.3 comes with functionality to support OpenvSwitch. This parameter will, at first, only be implemented for Xen and will have no effects on other hypervisors. Support for KVM will be added in the future. Example: switch1 will connect the VM to the default VLAN of the switch1. switch1.3 means that the VM is connected to an access port of VLAN 3. switch1.2:10:20 means that the VM is connected to a hybrid port on switch1, carrying VLANs 2 untagged and VLANs 10 and 20 tagged. switch1:44:55 means that the VM is connected to a trunk port on switch1, carrying VLANS 44 and 55 This configuration string is split at the dot or colon respectively and stored in nicparams[constants.NIC_LINK] and nicparams[constants.NIC_VLAN] respectively. Dot or colon are stored as well in nicparams[constants.NIC_VLAN]. For Xen hypervisors, this information can be concatenated again and stored in the vif config as the bridge parameter and will be fully compatible with vif-openvswitch as of Xen 4.3. Users of older Xen versions should be able to grab vif-openvswitch from the Xen repo and use it (tested in 4.2). gnt-instance modify shall be able to add or remove single VLANs from the vlan string without users needing to specify the complete new string. NIC bonding +++++++++++ To be done Configuration changes for QoS +++++++++++++++++++++++++++++ Instances shall be extended with configuration options for - maximum bandwidth - maximum burst rate New configuration objects need to be created for the Open vSwitch configuration. All these configuration changes need to be made available on the whole node group. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-opportunistic-locking.rst000064400000000000000000000134241476477700300227240ustar00rootroot00000000000000==================================================================== Design for parallelized instance creations and opportunistic locking ==================================================================== :Created: 2012-Dec-03 :Status: Implemented :Ganeti-Version: 2.7.0 .. contents:: :depth: 3 Current state and shortcomings ------------------------------ As of Ganeti 2.6, instance creations acquire all node locks when an :doc:`instance allocator ` (henceforth "iallocator") is used. In situations where many instance should be created in a short timeframe, there is a lot of congestion on node locks. Effectively all instance creations are serialized, even on big clusters with multiple groups. The situation gets worse when disk wiping is enabled (see :manpage:`gnt-cluster(8)`) as that can take, depending on disk size and hardware performance, from minutes to hours. Not waiting for DRBD disks to synchronize (``wait_for_sync=false``) makes instance creations slightly faster, but there's a risk of impacting I/O of other instances. Proposed changes ---------------- The target is to speed up instance creations in combination with an iallocator even when the cluster's balance is sacrificed in the process. The cluster can later be re-balanced using ``hbal``. The main objective is to reduce the number of node locks acquired for creation and to release un-used locks as fast as possible (the latter is already being done). To do this safely, several changes are necessary. Locking library ~~~~~~~~~~~~~~~ Instead of forcibly acquiring all node locks for creating an instance using an iallocator, only those currently available will be acquired. To this end, the locking library must be extended to implement opportunistic locking. Lock sets must be able to only acquire all locks available at the time, ignoring and not waiting for those held by another thread. Locks (``SharedLock``) already support a timeout of zero. The latter is different from a blocking acquisition, in which case the timeout would be ``None``. Lock sets can essentially be acquired in two different modes. One is to acquire the whole set, which in turn will also block adding new locks from other threads, and the other is to acquire specific locks by name. The function to acquire locks in a set accepts a timeout which, if not ``None`` for blocking acquisitions, counts for the whole duration of acquiring, if necessary, the lock set's internal lock, as well as the member locks. For opportunistic acquisitions the timeout is only meaningful when acquiring the whole set, in which case it is only used for acquiring the set's internal lock (used to block lock additions). For acquiring member locks the timeout is effectively zero to make them opportunistic. A new and optional boolean parameter named ``opportunistic`` is added to ``LockSet.acquire`` and re-exported through ``GanetiLockManager.acquire`` for use by ``mcpu``. Internally, lock sets do the lock acquisition using a helper function, ``__acquire_inner``. It will be extended to support opportunistic acquisitions. The algorithm is very similar to acquiring the whole set with the difference that acquisitions timing out will be ignored (the timeout in this case is zero). New lock level ~~~~~~~~~~~~~~ With opportunistic locking used for instance creations (controlled by a parameter), multiple such requests can start at (essentially) the same time and compete for node locks. Some logical units, such as ``LUClusterVerifyGroup``, need to acquire all node locks. In the latter case all instance allocations would fail to get their locks. This also applies when multiple instance creations are started at roughly the same time. To avoid situations where an opcode holding all or many node locks causes allocations to fail, a new lock level must be added to control allocations. The logical units for instance failover and migration can only safely determine whether they need all node locks after the instance lock has been acquired. Therefore the new lock level, named "node-alloc" (shorthand for "node-allocation") will be inserted after instances (``LEVEL_INSTANCE``) and before node groups (``LEVEL_NODEGROUP``). Similar to the "big cluster lock" ("BGL") there is only a single lock at this level whose name is "node allocation lock" ("NAL"). As a rule-of-thumb, the node allocation lock must be acquired in the same mode as nodes and/or node resources. If all or a large number of node locks are acquired, the node allocation lock should be acquired as well. Special attention should be given to logical units started for all node groups, such as ``LUGroupVerifyDisks``, as they also block many nodes over a short amount of time. iallocator ~~~~~~~~~~ The :doc:`iallocator interface ` does not need any modification. When an instance is created, the information for all nodes is passed to the iallocator plugin. Nodes for which the lock couldn't be acquired and therefore shouldn't be used for the instance in question, will be shown as offline. Opcodes ~~~~~~~ The opcodes ``OpInstanceCreate`` and ``OpInstanceMultiAlloc`` will gain a new parameter to enable opportunistic locking. By default this mode is disabled as to not break backwards compatibility. A new error type is added to describe a temporary lack of resources. Its name will be ``ECODE_TEMP_NORES``. With opportunistic locks the opcodes mentioned before only have a partial view of the cluster and can no longer decide if an instance could not be allocated due to the locks it has been given or whether the whole cluster is lacking resources. Therefore it is required, upon encountering the error code for a temporary lack of resources, for the job submitter to make this decision by re-submitting the job or by re-directing it to another cluster. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-optables.rst000064400000000000000000000224121476477700300201640ustar00rootroot00000000000000========================================== Filtering of jobs for the Ganeti job queue ========================================== :Created: 2013-Jul-24 :Status: Implemented :Ganeti-Version: 2.13.0 .. contents:: :depth: 4 This is a design document detailing the semantics of the fine-grained control of jobs in Ganeti. For the implementation there will be a separate design document that also describes the vision for the Ganeti daemon structure. Current state and shortcomings ============================== Control of the Ganeti job queue is quite limited. There is a single status bit, the "drained flag". If set, no new jobs are accepted to the queue. This is too coarse for some use cases. - The queue might be required to be drained for several reasons, initiated by different persons or automatic programs. Each one should be able to indicate that his reasons for draining are over without affecting the others. - There is no support for partial drains. For example, one might want to allow all jobs belonging to a manual (or externally coordinated) maintenance, while disallowing all other jobs. - There is no support for blocking jobs by their op-codes, e.g., disallowing all jobs that bring new instances to a cluster. This might be part of a maintenance preparation. - There is no support for a soft version of draining, where all jobs currently in the queue are finished, while new jobs entering the queue are delayed until the drain is over. Proposed changes ================ We propose to add filters on the job queue. These will be part of the configuration and as such are persisted with it. Conceptually, the filters are always processed when a job enters the queue and while it is still in the queue. Of course, in the implementation, reevaluation is only carried out, if something could make the result change, e.g., a new job is entered to the queue, or the filter rules are changed. There is no distinction between filter processing when a job is about to enter the queue and while it is in the queue, as this can be expressed by the filter rules themselves (see predicates below). Format of a Filter rule ----------------------- Filter rules are given by the following data. - A UUID. This ensures that there can be different filter rules that otherwise have all parameters equal. In this way, multiple drains for different reasons are possible. The UUID is used to address the filter rule, in particular for deletion. If no UUID is provided at rule addition, Ganeti will create one. - The watermark. This is the highest job id ever used, as valid in the moment when the filter was added. This data will be added automatically upon addition of the filter. - A priority. This is a non-negative integer. Filters are processed in order of increasing priority. While there is a well-defined order in which rules of the same priority are evaluated (increasing watermark, then the UUID, are taken as tie breakers), it is not recommended to have rules of the same priority that overlap and have different actions associated. - A list of predicates to be matched against the job. - An action. For the time being, one of the following, but more actions might be added in the future (in particular, future implementations might add an action making filtering continue with a different filter chain). - ACCEPT. The job will be accepted; no further filter rules are applied. - PAUSE. The job will be accepted to the queue and remain there; however, it is not executed. If an opcode is currently running, it continues, but the next opcode will not be started. For a paused job all locks it might have acquired will be released as soon as possible, at the latest when the currently running opcode has finished. The job queue will take care of this. - REJECT. The job is rejected. If it is already in the queue, it will be cancelled. - CONTINUE. The filtering continues processing with the next rule. Such a rule will never have any direct or indirect effect, but it can serve as documentation for a "normally present, but currently disabled" rule. - RATE_LIMIT ``n``, where ``n`` is a positive integer. The job will be held in the queue while ``n`` or more jobs where this rule applies are running. Jobs that are forked off from luxid are considered running. Jobs already running when this rule is added are not changed. Logically, this rule is applied job by job sequentially, so that the number of jobs where this rule applies is limited to ``n`` once the jobs running at rule addition have finished. - A reason trail, in the same format as reason trails for opcodes. This allows to find out, which maintenance (or other reason) caused the addition of this filter rule. When a filter rule applies -------------------------- A filter rule in a filter chain *applies* to a job if it is the first rule in the chain of which all predicates *match* the job, and if its action is not CONTINUE. Filter chains are processed in increasing order of priority (lowest number means highest priority), then watermark, then UUID. Predicates available for the filter rules ----------------------------------------- A predicate is a list, with the first element being the name of the predicate and the rest being parameters suitable for that predicate. In most cases, the name of the predicate will be a field of a job, and there will be a single parameter, which is a boolean expression (``filter``) in the sense of the Ganeti query language. However, no assumption should be made that all predicates are of this shape. More predicates may be added in the future. - ``jobid``. Only parameter is a boolean expression. For this expression, there is only one field available, ``id``, which represents the id the job to be filtered. In all value positions, the string ``watermark`` will be replaced by the value of the watermark. - ``opcode``. Only parameter is a boolean expression. For this expression, ``OP_ID`` and all other fields present in the opcode are available. This predicate will hold true, if the expression is true for at least one opcode in the job. - ``reason``. Only parameter is a boolean expression. For this expression, the three fields ``source``, ``reason``, ``timestamp`` of reason trail entries are available. This predicate is true, if one of the entries of one of the opcodes in this job satisfies the expression. Examples ======== Draining the queue. :: {"priority": 0, "predicates": [["jobid", [">", "id", "watermark"]]], "action": "REJECT"} Soft draining could be achieved by replacing ``REJECT`` by ``PAUSE`` in the above example. Pausing all new jobs not belonging to a specific maintenance. :: {"priority": 0, "predicates": [["reason", ["=~", "reason", "maintenance pink bunny"]]], "action": "ACCEPT"} {"priority": 1, "predicates": [["jobid", [">", "id", "watermark"]]], "action": "PAUSE"} Cancelling all queued instance creations and disallowing new such jobs. :: {"priority": 1, "predicates": [["opcode", ["=", "OP_ID", "OP_INSTANCE_CREATE"]]], "action": "REJECT"} Limiting the number of simultaneous instance disk replacements to 10 in order to throttle replication traffic. :: {"priority": 99, "predicates": [["opcode", ["=", "OP_ID", "OP_INSTANCE_REPLACE_DISKS"]]], "action": ["RATE_LIMIT", 10]} Interface ========= Since queue control is intended to be used by external maintenance-handling tools as well, the primary interface for manipulating queue filters is the :doc:`rapi`. For convenience, a command-line interface will be added as well. The following resources will be added. - /2/filters/ - GET returns the list of all currently set filters - POST adds a new filter - /2/filters/[uuid] - GET returns the description of the specified filter - DELETE removes the specified filter - PUT replaces the specified filter rule, or creates it, if it doesn't exist already. Security considerations ======================= Filtering of jobs is not a security feature. It merely serves the purpose of coordinating efforts and avoiding accidental conflicting jobs. Everybody with appropriate credentials can modify the filter rules, not just the originator of a rule. To avoid accidental lock-out, requests modifying the queue are executed directly and not going through the queue themselves. Additional Ad-Hoc Rate Limiting =============================== Besides a general policy to control the job queue, it is often very useful to have a lightweight way for one-off rate-limiting. One example would be evacuating a node but limiting the number of simultaneous instance moves to no overload the replication network. Therefore, an additional rate limiting is done over the :doc:`design-reason-trail` as follows. ``reason`` fields in a reason 3-tuple starting with ``rate-limit:n:`` where ``n`` is a positive integer are considered rate-limiting buckets. A job belongs to a rate-limiting bucket if it contains at least one op-code with at least one reason-trail 3-tuple with that particular ``reason`` field. The scheduler will ensure that, for each rate-limiting bucket, there are at most ``n`` jobs belonging to that bucket that are running in parallel. The limiting in the initial example can then be done as follows. :: # gnt-node evacuate --reason='rate-limit:7:operation pink bunny' node1 ganeti-3.1.0~rc2/doc/design-os.rst000064400000000000000000000756701476477700300170120ustar00rootroot00000000000000=============================== Ganeti OS installation redesign =============================== :Created: 2013-Dec-12 :Status: Partially Implemented :Ganeti-Version: 2.12.0, 2.13.0 .. contents:: :depth: 3 This is a design document detailing a new OS installation procedure, which is more secure, able to provide more features and easier to use for many common tasks w.r.t. the current one. Current state and shortcomings ============================== As of Ganeti 2.10, each instance is associated with an OS definition. An OS definition is a set of scripts (i.e., ``create``, ``export``, ``import``, ``rename``) that are executed with root privileges on the primary host of the instance. These scripts are responsible for performing all the OS-related tasks, namely, create an instance, setup an operating system on the instance's disks, export/import the instance, and rename the instance. These scripts receive, through environment variables, a fixed set of instance parameters (such as, the hypervisor, the name of the instance, the number of disks and their location) and a set of user defined parameters. Both the instance and user defined parameters are written in the configuration file of Ganeti, to allow future reinstalls of the instance, and in various log files, namely: * node daemon log file: contains DEBUG strings of the ``/os_validate``, ``/instance_os_add`` and ``/instance_start`` RPC calls. * master daemon log file: DEBUG strings related to the same RPC calls are stored here as well. * commands log: the CLI commands that create a new instance, including their parameters, are logged here. * RAPI log: the RAPI commands that create a new instance, including their parameters, are logged here. * job logs: the job files stored in the job queue, or in its archive, contain the parameters. The current situation presents a number of shortcomings: * Having the installation scripts run as root on the nodes does not allow user-defined OS scripts, as they would pose a huge security risk. Furthermore, even a script without malicious intentions might end up disrupting a node because of due to a bug. * Ganeti cannot be used to create instances starting from user provided disk images: even in the (hypothetical) case in which the scripts are completely secure and run not by root but by an unprivileged user with only the power to mount arbitrary files as disk images, this is still a security issue. It has been proven that a carefully crafted file system might exploit kernel vulnerabilities to gain control of the system. Therefore, directly mounting images on the Ganeti nodes is not an option. * There is no way to inject files into an existing disk image. A common use case is for the system administrator to provide a standard image of the system, to be later personalized with the network configuration, private keys identifying the machine, ssh keys of the users, and so on. A possible workaround would be for the scripts to mount the image (only if this is trusted!) and to receive the configurations and ssh keys as user defined OS parameters. Unfortunately, this is also not an option for security sensitive material (such as the ssh keys) because the OS parameters are stored in many places on the system, as already described above. * Most other virtualization software allow only instance images, but no installation scripts. This difference makes the interaction between Ganeti and other software difficult. Proposed changes ================ In order to fix the shortcomings of the current state, we plan to introduce the following changes. OS parameter categories +++++++++++++++++++++++ Change the OS parameters to have three categories: * ``public``: the current behavior. The parameter is logged and stored freely. * ``private``: the parameter is saved inside the Ganeti configuration (to allow for instance reinstall) but it is not shown in logs, job logs, or passed back via RAPI. * ``secret``: the parameter is not saved inside the Ganeti configuration. Reinstalls are impossible unless the data is passed again. The parameter will not appear in any log file. When a functionality is performed jointly by multiple daemons (such as MasterD and LuxiD), currently Ganeti sometimes serializes jobs on disk and later reloads them. Secret parameters will not be serialized to disk. They will be passed around as part of the LUXI calls exchanged by the daemons, and only kept in memory, in order to reduce their accessibility as much as possible. In case of failure of the master node, these parameters will be lost and cannot be recovered because they are not serialized. As a result, the job cannot be taken over by the new master. This is an expected and accepted side effect of jobs with secret parameters: if they fail, they'll have to be restarted manually. Metadata ++++++++ In order to allow metadata to be sent inside the instance, a communication mechanism between the instance and the host will be created. This mechanism will be bidirectional (e.g.: to allow the setup process going on inside the instance to communicate its progress to the host). Each instance will have access exclusively to its own metadata, and it will be only able to communicate with its host over this channel. This is the approach followed the ``cloud-init`` tool and more details will be provided in the `Communication mechanism`_ and `Metadata service`_ sections. Installation procedure ++++++++++++++++++++++ A new installation procedure will be introduced. There will be two sets of parameters, namely, installation parameters, which are used mainly for installs and reinstalls, and execution parameters, which are used in all the other runs that are not part of an installation procedure. Also, it will be possible to use an installation medium and/or run the OS scripts in an optional virtualized environment, and optionally use a personalization package. This section details all of these options. The set of installation parameters will allow, for example, to attach an installation floppy/cdrom/network, change the boot device order, or specify a disk image to be used. Through this set of parameters, the administrator will have to provide the hypervisor a location for an installation medium for the instance (e.g., a boot disk, a network image, etc). This medium will carry out the installation of the instance onto the instance's disks and will then be responsible for getting the parameters for configuring the instance, such as, network interfaces, IP address, and hostname. These parameters are taken from the metadata. The installation parameters will be stored in the configuration of Ganeti and used in future reinstalls, but not during normal execution. The instance is reinstalled using the same installation parameters from the first installation. However, it will be the administrator's responsibility to ensure that the installation media is still available at the proper location when a reinstall occurs. The parameter ``--os-parameters`` can still be used to specify the OS parameters. However, without OS scripts, Ganeti cannot do more than a syntactic check to validate the supplied OS parameter string. As a result, this string will be passed directly to the instance as part of the metadata. If OS scripts are used and the installation procedure is running inside a virtualized environment, Ganeti will take these parameters from the metadata and pass them to the OS scripts as environment variables. Ganeti allows the following installation options: * Use a disk image: Currently, it is already possible to specify an installation medium, such as, a cdrom, but not a disk image. Therefore, a new parameter ``--os-image`` will be used to specify the location of a disk image which will be dumped to the instance's first disk before the instance is started. The location of the image can be a URL and, if this is the case, Ganeti will download this image. * Run OS scripts: The parameter ``--os-type`` (short version: ``-o``), is currently used to specify the OS scripts. This parameter will still be used to specify the OS scripts with the difference that these scripts may optionally run inside a virtualized environment for safety reasons, depending on whether they are trusted or not. For more details on trusted and untrusted OS scripts, refer to the `Installation process in a virtualized environment`_ section. Note that this parameter will become optional thus allowing a user to create an instance specifying only, for example, a disk image or a cdrom image to boot from. * Personalization package As part of the instance creation command, it will be possible to indicate a URL for a "personalization package", which is an archive containing a set of files meant to be overlayed on top of the OS file system at the end of the setup process and before the VM is started for the first time in normal mode. Ganeti will provide a mechanism for receiving and unpacking this archive, independently of whether the installation is being performed inside the virtualized environment or not. The archive will be in TAR-GZIP format (with extension ``.tar.gz`` or ``.tgz``) and contain the files according to the directory structure that will be recreated on the installation disk. Files contained in this archive will overwrite files with the same path created during the installation procedure (if any). The URL of the "personalization package" will have to specify an extension to identify the file format (in order to allow for more formats to be supported in the future). The URL will be stored as part of the configuration of the instance (therefore, the URL should not contain confidential information, but the files there available can). It is up to the system administrator to ensure that a package is actually available at that URL at install and reinstall time. The contents of the package are allowed to change. E.g.: a system administrator might create a package containing the private keys of the instance being created. When the instance is reinstalled, a new package with new keys can be made available there, thus allowing instance reinstall without the need to store keys. A username and a password can be specified together with the URL. If the URL is a HTTP(S) URL, they will be used as basic access authentication credentials to access that URL. The username and password will not be saved in the config, and will have to be provided again in case a reinstall is requested. The downloaded personalization package will not be stored locally on the node for longer than it is needed while unpacking it and adding its files to the instance being created. The personalization package will be overlayed on top of the instance filesystem after the scripts that created it have been executed. In order for the files in the package to be automatically overlayed on top of the instance filesystem, it is required that the appliance is actually able to mount the instance's disks. As a result, this will not work for every filesystem. * Combine a disk image, OS scripts, and a personalization package It will possible to combine a disk image, OS scripts, and a personalization package, both with or without a virtualized environment (see the exception below). At least, an installation medium or OS scripts should be specified. The disk image of the actual virtual appliance, which bootstraps the virtual environment used in the installation procedure, will be read only, so that a pristine copy of the appliance can be started every time a new instance needs to be created and to further increase security. The data the instance needs to write at runtime will only be stored in RAM and disappear as soon as the instance is stopped. The parameter ``--enable-safe-install=yes|no`` will be used to give the administrator control over whether to use a virtualized environment for the installation procedure. By default, a virtualized environment will be used. Note that some feature combinations, such as, using untrusted scripts, will require the virtualized environment. In this case, Ganeti will not allow disabling the virtualized environment. Implementation ============== The implementation of this design will happen as an ordered sequence of steps, of increasing impact on the system and, in some cases, dependent on each other: #. Private and secret instance parameters #. Communication mechanism between host and instance #. Metadata service #. Personalization package (inside a virtualization environment) #. Instance creation via a disk image #. Instance creation inside a virtualized environment Some of these steps need to be more deeply specified w.r.t. what is already written in the `Proposed changes`_ Section. Extra details will be provided in the following subsections. Communication mechanism +++++++++++++++++++++++ The communication mechanism will be an exclusive, generic, bidirectional communication channel between Ganeti hosts and guests. exclusive The communication mechanism allows communication between a guest and its host, but it does not allow a guest to communicate with other guests or reach the outside world. generic The communication mechanism allows a guest to reach any service on the host, not just the metadata service. Examples of valid communication include, but are not limited to, access to the metadata service, send commands to Ganeti, request changes to parameters, such as, those related to the distribution upgrades, and let Ganeti control a helper instance, such as, the one for performing OS installs inside a safe environment. bidirectional The communication mechanism allows communication to be initiated from either party, namely, from a host to a guest or guest to host. Note that Ganeti will allow communication with any service (e.g., daemon) running on the host and, as a result, Ganeti will not be responsible for ensuring that only the metadata service is reachable. It is the responsibility of each system administrator to ensure that the extra firewalling and routing rules specified on the host provide the necessary protection on a given Ganeti installation and, at the same time, do not accidentally override the behaviour hereby described which makes the communication between the host and the guest exclusive, generic, and bidirectional, unless intended. The communication mechanism will be enabled automatically during an installation procedure that requires a virtualized environment, but, for backwards compatibility, it will be disabled when the instance is running normally, unless explicitly requested. Specifically, a new parameter ``--communication=yes|no`` (short version: ``-C``) will be added to ``gnt-instance add`` and ``gnt-instance modify``. This parameter will determine whether the communication mechanism is enabled for a particular instance. The value of this parameter will be saved as part of the instance's configuration. The communication mechanism will be implemented through network interfaces on the host and the guest, and Ganeti will be responsible for the host side, namely, creating a TAP interface for each guest and configuring these interfaces to have name ``gnt.com.%d``, where ``%d`` is a unique number within the host (e.g., ``gnt.com.0`` and ``gnt.com.1``), IP address ``169.254.169.254``, and netmask ``255.255.255.255``. The interface's name allows DHCP servers to recognize which interfaces are part of the communication mechanism. This network interface will be connected to the guest's last network interface, which is meant to be used exclusively for the communication mechanism and is defined after all the used-defined interfaces. The last interface was chosen (as opposed to the first one, for example) because the first interface is generally understood and the main gateway out, and also because it minimizes the impact on existing systems, for example, in a scenario where the system administrator has a running cluster and wants to enable the communication mechanism for already existing instances, which might have been created with older versions of Ganeti. Further, DBus should assist in keeping the guest network interfaces more stable. On the guest side, each instance will have its own MAC address and IP address. Both the guest's MAC address and IP address must be unique within a single cluster. An IP is unique within a single cluster, and not within a single host, in order to minimize disruption of connectivity, for example, during live migration, in particular since an instance is not aware when it changes host. Unfortunately, a side-effect of this decision is that a cluster can have a maximum of a ``/16`` network allowed instances (with communication enabled). If necessary to overcome this limit, it should be possible to allow different networks to be configured link-local only. The guest will use the DHCP protocol on its last network interface to contact a DHCP server running on the host and thus determine its IP address. The DHCP server is configured, started, and stopped, by Ganeti and it will be listening exclusively on the TAP network interfaces of the guests in order not to interfere with a potential DHCP server running on the same host. Furthermore, the DHCP server will only recognize MAC and IP address pairs that have been approved by Ganeti. The TAP network interfaces created for each guest share the same IP address. Therefore, it will be necessary to extend the routing table with rules specific to each guest. This can be achieved with the following command, which takes the guest's unique IP address and its TAP interface:: route add -host dev This rule has the additional advantage of preventing guests from trying to lease IP addresses from the DHCP server other than the own that has been assigned to them by Ganeti. The guest could lie about its MAC address to the DHCP server and try to steal another guest's IP address, however, this routing rule will block traffic (i.e., IP packets carrying the wrong IP) from the DHCP server to the malicious guest. Similarly, the guest could lie about its IP address (i.e., simply assign a predefined IP address, perhaps from another guest), however, replies from the host will not be routed to the malicious guest. This routing rule ensures that the communication channel is exclusive but, as mentioned before, it will not prevent guests from accessing any service on the host. It is the system administrator's responsibility to employ the necessary ``iptables`` rules. In order to achieve this, Ganeti will provide ``ifup`` hooks associated with the guest network interfaces which will give system administrator's the opportunity to customize their own ``iptables``, if necessary. Ganeti will also provide examples of such hooks. However, these are meant to personalized to each Ganeti installation and not to be taken as production ready scripts. For KVM, an instance will be started with a unique MAC address and the file descriptor for the TAP network interface meant to be used by the communication mechanism. Ganeti will be responsible for generating a unique MAC address for the guest, opening the TAP interface, and passing its file descriptor to KVM:: kvm -net nic,macaddr= -net tap,fd= ... For Xen, a network interface will be created on the host (using the ``vif`` parameter of the Xen configuration file). Each instance will have its corresponding ``vif`` network interface on the host. The ``vif-route`` script of Xen might be helpful in implementing this. dnsmasq +++++++ The previous section describes the communication mechanism and explains the role of the DHCP server. Note that any DHCP server can be used in the implementation of the communication mechanism. However, the DHCP server employed should not violate the properties described in the previous section, which state that the communication mechanism should be exclusive, generic, and bidirectional, unless this is intentional. In our experiments, we have used dnsmasq. In this section, we describe how to properly configure dnsmasq to work on a given Ganeti installation. This is particularly important if, in this Ganeti installation, dnsmasq will share the node with one or more DHCP servers running in parallel. First, it is important to become familiar with the operational modes of dnsmasq, which are well explained in the `FAQ `_ under the question ``What are these strange "bind-interface" and "bind-dynamic" options?``. The rest of this section assumes the reader is familiar with these operational modes. bind-dynamic dnsmasq SHOULD be configured in the ``bind-dynamic`` mode (if supported) in order to allow other DHCP servers to run on the same node. In this mode, dnsmasq can listen on the TAP interfaces for the communication mechanism by listening on the TAP interfaces that match the pattern ``gnt.com.*`` (e.g., ``interface=gnt.com.*``). For extra safety, interfaces matching the pattern ``eth*`` and the name ``lo`` should be configured such that dnsmasq will always ignore them (e.g., ``except-interface=eth*`` and ``except-interface=lo``). bind-interfaces dnsmasq MAY be configured in the ``bind-interfaces`` mode (if supported) in order to allow other DHCP servers to run on the same node. Unfortunately, because dnsmasq cannot dynamically adjust to TAP interfaces that are created and destroyed by the system, dnsmasq must be restarted with a new configuration file each time an instance is created or destroyed. Also, the interfaces cannot be patterns, such as, ``gnt.com.*``. Instead, the interfaces must be explictly specified, for example, ``interface=gnt.com.0,gnt.com.1``. Moreover, dnsmasq cannot bind to the TAP interfaces if they have all the same IPv4 address. As a result, it is necessary to configure these TAP interfaces to enable IPv6 and an IPv6 address must be assigned to them. wildcard dnsmasq CANNOT be configured in the ``wildcard`` mode if there is (at least) another DHCP server running on the same node. Metadata service ++++++++++++++++ An instance will be able to reach metadata service on ``169.254.169.254:80`` in order to, for example, retrieve its metadata. This IP address and port were chosen for compatibility with the OpenStack and Amazon EC2 metadata service. The metadata service will be provided by a single daemon, which will determine the source instance for a given request and reply with the metadata pertaining to that instance. Where possible, the metadata will be provided in a way compatible with Amazon EC2, at:: http://169.254.169.254//meta-data/* Ganeti-specific metadata, that does not fit this structure, will be provided at:: http://169.254.169.254/ganeti//meta_data.json where ```` is either a date in YYYY-MM-DD format, or ``latest`` to indicate the most recent available protocol version. If needed in the future, this structure also allows us to support OpenStack's metadata at:: http://169.254.169.254/openstack//meta_data.json A bi-directional, pipe-like communication channel will also be provided. The instance will be able to receive data from the host by a GET request at:: http://169.254.169.254/ganeti//read and to send data to the host by a POST request at:: http://169.254.169.254/ganeti//write As in a pipe, once the data are read, they will not be in the buffer anymore, so subsequent GET requests to ``read`` will not return the same data. However, unlike a pipe, it will not be possible to perform blocking I/O operations. The OS parameters will be accessible through a GET request at:: http://169.254.169.254/ganeti//os/parameters.json as a JSON serialized dictionary having the parameter name as the key, and the pair ``(, )`` as the value, where ```` is the user-provided value of the parameter, and ```` is either ``public``, ``private`` or ``secret``. The installation scripts to be run inside the virtualized environment will be available at:: http://169.254.169.254/ganeti//os/scripts/ where ```` is the name of the script. Rationale --------- The choice of using a network interface for instance-host communication, as opposed to VirtIO, XenBus or other methods, is due to the will of having a generic, hypervisor-independent way of creating a communication channel, that doesn't require unusual (para)virtualization drivers. At the same time, a network interface was preferred over solutions involving virtual floppy or USB devices because the latter tend to be detected and configured by the guest operating systems, sometimes even in prominent positions in the user interface, whereas it is fairly common to have an unconfigured network interface in a system, usually without any negative side effects. Installation process in a virtualized environment +++++++++++++++++++++++++++++++++++++++++++++++++ In the new OS installation scenario, we distinguish between trusted and untrusted code. The trusted installation code maintains the behavior of the current one and requires no modifications, with the scripts running on the node the instance is being created on. The untrusted code is stored in a subdirectory of the OS definition called ``untrusted``. This directory contains scripts that are equivalent to the already existing ones (``create``, ``export``, ``import``, ``rename``) but that will be run inside an virtualized environment, to protect the host from malicious tampering. The ``untrusted`` code is meant to either be untrusted itself, or to be trusted code running operations that might be dangerous (such as mounting a user-provided image). By default, all new OS definitions will have to be explicitly marked as trusted by the cluster administrator (with a new ``gnt-os modify`` command) before they can run code on the host. Otherwise, only the untrusted part of the code will be allowed to run, inside the virtual appliance. For backwards compatibility reasons, when upgrading an existing cluster, all the installed OSes will be marked as trusted, so that they can keep running with no changes. In order to allow for the highest flexibility, if both a trusted and an untrusted script are provided for the same operation (i.e. ``create``), both of them will be executed at the same time, one on the host, and one inside the installation appliance. They will be allowed to communicate with each other through the already described communication mechanism, in order to orchestrate their execution (e.g.: the untrusted code might execute the installation, while the trusted one receives status updates from it and delivers them to a user interface). The cluster administrator will have an option to completely disable scripts running on the host, leaving only the ones running in the VM. Ganeti will provide a script to be run at install time that can be used to create the virtualized environment that will perform the OS installation of new instances. This script will build a debootstrapped basic Debian system including a software that will read the metadata, setup the environment variables and launch the installation scripts inside the virtualized environment. The script will also provide hooks for personalization. It will also be possible to use other self-made virtualized environments, as long as they connect to Ganeti over the described communication mechanism and they know how to read and use the provided metadata to create a new instance. While performing an installation in the virtualized environment, a customizable timeout will be used to detect possible problems with the installation process, and to kill the virtualized environment. The timeout will be optional and set on a cluster basis by the administrator. If set, it will be the total time allowed to setup an instance inside the appliance. It is mainly meant as a safety measure to prevent an instance taken over by malicious scripts to be available for a long time. Alternatives to design and implementation ========================================= This section lists alternatives to design and implementation, which came up during the development of this design document, that will not be implemented. Please read carefully through the limitations and security concerns of each of these alternatives. Port forwarding in KVM ++++++++++++++++++++++ The communication mechanism could have been implemented in KVM using guest port forwarding, as opposed to network interfaces. There are two alternatives in KVM's guest port forwarding, namely, creating a forwarding device, such as, a TCP/IP connection, or executing a command. However, we have determined that both of these options are not viable. A TCP/IP forwarding device can be created through the following KVM invocation:: kvm -net nic -net \ user,restrict=on,net=169.254.0.0/16,host=169.254.169.253, guestfwd=tcp:169.254.169.254:80-tcp:127.0.0.1:8080 ... This invocation even has the advantage that it can block undesired traffic (i.e., traffic that is not explicitly specified in the arguments) and it can remap ports, which would have allowed the metadata service daemon to run in port 8080 instead of 80. However, in this scheme, KVM opens the TCP connection only once, when it is started, and, if the connection breaks, KVM will not reestablish the connection. Furthermore, opening the TCP connection only once interferes with the HTTP protocol, which needs to dynamically establish and close connections. The alternative to the TCP/IP forwarding device is to execute a command. The KVM invocation for this is, for example, the following:: kvm -net nic -net \ "user,restrict=on,net=169.254.0.0/16,host=169.254.169.253, guestfwd=tcp:169.254.169.254:80-netcat 127.0.0.1 8080" ... The advantage of this approach is that the command is executed each time the guest initiates a connection. This is the ideal situation, however, it is only supported in KVM 1.2 and above, and, therefore, not viable because we want to provide support for at least KVM version 1.0, which is the version provided by Ubuntu LTS. Alternatives to the DHCP server +++++++++++++++++++++++++++++++ There are alternatives to using the DHCP server, for example, by assigning a fixed IP address to guests, such as, the IP address ``169.254.169.253``. However, this introduces a routing problem, namely, how to route incoming packets from the same source IP to the host. This problem can be overcome in a number of ways. The first solution is to use NAT to translate the incoming guest IP address, for example, ``169.254.169.253``, to a unique IP address, for example, ``169.254.0.1``. Given that NAT through ``ip rule`` is deprecated, users can resort to ``iptables``. Note that this has not yet been tested. Another option, which has been tested, but only in a prototype, is to connect the TAP network interfaces of the guests to a bridge. The bridge takes the configuration from the TAP network interfaces, namely, IP address ``169.254.169.254`` and netmask ``255.255.255.255``, thus leaving those interfaces without an IP address. Note that in this setting, guests will be able to reach each other, therefore, if necessary, additional ``iptables`` rules can be put in place to prevent it. ganeti-3.1.0~rc2/doc/design-ovf-support.rst000064400000000000000000000451211476477700300206610ustar00rootroot00000000000000============================================================== Ganeti Instance Import/Export using Open Virtualization Format ============================================================== :Created: 2011-Jul-22 :Status: Implemented :Ganeti-Version: 2.6.0 Background ========== Open Virtualization Format is an open standard for packaging information regarding virtual machines. It is used, among other, by VMWare, VirtualBox and XenServer. OVF allows users to migrate between virtualization software without the need of reconfiguring hardware, network or operating system. Currently, exporting instance in Ganeti results with a configuration file that is readable only for Ganeti. It disallows the users to change the platform they use without loosing all the machine's configuration. Import function in Ganeti is also currently limited to the previously prepared instances. Implementation of OVF support allows users to migrate to Ganeti from other platforms, thus potentially increasing the usage. It also enables virtual machine end-users to create their own machines (e.g. in VirtualBox or SUSE Studio) and then add them to Ganeti cluster, thus providing better personalization. Overview ======== Open Virtualization Format description -------------------------------------- According to the DMTF document introducing the standard: "The Open Virtualization Format (OVF) Specification describes an open, secure, portable, efficient and extensible format for the packaging and distribution of software to be run in virtual machines." OVF supports both single and multiple- configurations of VMs in one package, is host- and virtualization platform-independent and optimized for distribution (e.g. by allowing usage of public key infrastructure and providing tools for management of basic software licensing). There are no limitations regarding disk images used, as long as the description is provided. Any hardware described in a proper format (i.e. CIM - Common Information Model) is accepted, although there is no guarantee that every virtualization software will support all types of hardware. OVF package should contain exactly one file with ``.ovf`` extension, which is an XML file specifying the following (per virtual machine): - virtual disks - network description - list of virtual hardware - operating system, if any Each of the elements in ``.ovf`` file may, if desired, contain a human-readable description to every piece of information given. Additionally, the package may have some disk image files and other additional resources (e.g. ISO images). In order to provide secure means of distribution for OVF packages, the manifest and certificate are provided. Manifest (``.mf`` file) contains checksums for all the files in OVF package, whereas certificate (``.cert`` file) contains X.509 certificate and a checksum of manifest file. Both files are not compulsory, but certificate requires manifest to be present. Supported disk formats ---------------------- Although OVF is claimed to support 'any disk format', what we are interested in is which formats are supported by VM managers that currently use OVF. - VMWare: ``.vmdk`` (which comes in at least 3 different flavours: ``sparse``, ``compressed`` and ``streamOptimized``) - VirtualBox: ``.vdi`` (VirtualBox's format), ``.vmdk``, ``.vhd`` (Microsoft and XenServer); export disk format is always ``.vmdk`` - XenServer: ``.vmdk``, ``.vhd``; export disk format is always ``.vhd`` - Red Hat Enterprise Virtualization: ``.raw`` (raw disk format), ``.cow`` (qemu's ``QCOW2``) - other: AbiCloud, OpenNode Cloud, SUSE Studio, Morfeo Claudia, OpenStack: mostly ``.vmdk`` In our implementation of the OVF we allow a choice between raw, cow and vmdk disk formats for both import and export. Other formats convertable using ``qemu-img`` are allowed in import mode, but not tested. The justification is the following: - Raw format is supported as it is the main format of disk images used in Ganeti, thus it is effortless to provide support for this format - Cow is used in Qemu - Vmdk is most commonly supported in virtualization software, it also has the advantage of producing relatively small disk images, which is extremely important advantage when moving instances. Import and export - the closer look =================================== This section contains an overview of how different parts of Ganeti's export info are included in ``.ovf`` configuration file. It also explains how import is designed to work with incomplete information. Ganeti's backup format vs OVF ----------------------------- .. highlight:: xml The basic structure of Ganeti ``.ovf`` file is the following:: .. note :: Tags with ``gnt:`` prefix are Ganeti-specific and are not a part of OVF standard. .. highlight:: text Whereas Ganeti's export info is of the following form, ``=>`` showing where will the data be in OVF format:: [instance] disk0_dump = filename => File in References disk0_ivname = name => generated automatically disk0_size = size_in_mb => calculated after disk conversion disk_count = number => generated automatically disk_template = disk_type => gnt:DiskTemplate hypervisor = hyp-name => gnt:Name in gnt:Hypervisor name = inst-name => Name in VirtualSystem nic0_ip = ip => gnt:IPAddress in gnt:Network nic0_link = link => gnt:Link in gnt:Network nic0_mac = mac => gnt:MACAddress in gnt:Network or Item in VirtualHardwareSection nic0_mode = mode => gnt:Mode in gnt:Network nic_count = number => generated automatically tags => gnt:Tags [backend] auto_balanced => gnt:AutoBalance memory = mem_in_mb => Item in VirtualHardwareSection vcpus = number => Item in VirtualHardwareSection [export] compression => ignored os => gnt:Name in gnt:OperatingSystem source => ignored timestamp => ignored version => gnt:VersionId or constants.EXPORT_VERSION [os] => gnt:Parameters in gnt:OperatingSystem [hypervisor] => gnt:Parameters in gnt:Hypervisor In case of multiple networks/disks used by an instance, they will all be saved in appropriate sections as specified above for the first network/disk. Import from other virtualization software ----------------------------------------- In case of importing to Ganeti OVF package generated in other software, e.g. VirtualBox, some fields required for Ganeti to properly handle import may be missing. Most often it will happen that such OVF package will lack the ``gnt:GanetiSection``. If this happens you can specify all the missing parameters in the command line. Please refer to `Command Line`_ section. In the :doc:`ovfconverter` we provide examples of options when converting from VirtualBox, VMWare and OpenSuseStudio. Export to other virtualization software --------------------------------------- When exporting to other virtualization software, you may notice that there is a section ``gnt:GanetiSection``, containing Ganeti-specific information. This may on **rare** cases cause trouble in importing your instance. If that is the case please do one of the two: 1. Export from Ganeti to OVF with ``--external`` option - this will cause to skip the non-standard information. 2. Manually remove the gnt:GanetiSection from the ``.ovf`` file. You will also have to recompute sha1 sum (``sha1sum`` command) of the .ovf file and update your ``.mf`` file with new value. .. note:: Manual change option is only recommended when you have exported your instance with ``-format`` option other that ``raw`` or selected ``--compress``. It saves you the time of converting or compressing the disk image. Planned limitations =================== The limitations regarding import of the OVF instances generated outside Ganeti will be (in general) the same, as limitations for Ganeti itself. The desired behavior in case of encountering unsupported element will be to ignore this element's tag without interruption of the import process. Package ------- There are no limitations regarding support for multiple files in package or packing the OVF package into one OVA (Open Virtual Appliance) file. As for certificates and licenses in the package, their support will be under discussion after completion of the basic features implementation. Multiple Virtual Systems ------------------------ At first only singular instances (i.e. VirtualSystem, not VirtualSystemCollection) will be supported. In the future multi-tiered appliances containing whole nodes (or even clusters) are considered an option. Disks ----- As mentioned, Ganeti will allow export in ``raw``, ``cow`` and ``vmdk`` formats. This means i.e. that the appropriate ``ovf:format`` will be provided. As for import, we will support all formats that ``qemu-img`` can convert to ``raw``. At this point this means ``raw``, ``cow``, ``qcow``, ``qcow2``, ``vmdk`` and ``cloop``. We do not plan for now to support ``vdi`` or ``vhd`` unless they become part of qemu-img supported formats. We plan to support compression both for import and export - in gzip format. There is also a possibility to provide virtual disk in chunks of equal size. The latter will not be implemented in the first version, but we do plan to support it eventually. The ``ovf:format`` tag is not used in our case when importing. Instead we use ``qemu-img info``, which provides enough information for our purposes and is better standardized. Please note, that due to security reasons we require the disk image to be in the same directory as the ``.ovf`` description file for both import and export. In order to completely ignore disk-related information in resulting config file, please use ``--disk-template=diskless`` option. Network ------- Ganeti provides support for routed and bridged mode for the networks. Since the standard OVF format does not contain any information regarding used network type, we add our own source of such information in ``gnt:GanetiSection``. In case this additional information is not present, we perform a simple check - if network name specified in ``NetworkSection`` contains words ``bridged`` or ``routed``, we consider this to be the network type. Otherwise option ``auto`` is chosen, in which case the cluster's default value for that field will be used when importing. This provides a safe fallback in case of NAT networks usage, which are commonly used e.g. in VirtualBox. Hardware -------- The supported hardware is limited to virtual CPUs, RAM memory, disks and networks. In particular, no USB support is currently provided, as Ganeti does not support them. Operating Systems ----------------- Support for different operating systems depends solely on their accessibility for Ganeti instances. List of installed OSes can be checked using ``gnt-os list`` command. References ---------- Files listed in ``ovf:References`` section cannot be hyperlinks. Other ----- The instance name (``gnt:VirtualSystem\gnt:Name`` or command line's ``--name`` option ) has to be resolvable in order for successful import using ``gnt-backup import``. _`Command Line` =============== The basic usage of the ovf tool is one of the following:: ovfconverter import filename ovfconverter export --format= filename This will result in a conversion based solely on the content of provided file. In case some information required to make the conversion is missing, an error will occur. If output directory should be different than the standard Ganeti export directory (usually ``/srv/ganeti/export``), option ``--output-dir`` can be used. If name of resulting entity should be different than the one read from the file, use ``--name`` option. Import options -------------- Import options that ``ovfconverter`` supports include options for backend, disks, hypervisor, networks and operating system. If an option is given, it overrides the values provided in the OVF file. Backend ^^^^^^^ ``--backend=option=value`` can be used to set auto balance, number of vcpus and amount of RAM memory. Please note that when you do not provide full set of options, the omitted ones will be set to cluster defaults (``auto``). Disks ^^^^^ ``--disk-template=diskless`` causes the converter to ignore all other disk option - both from .ovf file and the command line. Other disk template options include ``plain``, ``drdb``, ``file``, ``sharedfile`` and ``blockdev``. ``--disk=number:size=value`` causes to create disks instead of converting them from OVF package; numbers should start with ``0`` and be consecutive. Hypervisor ^^^^^^^^^^ ``-H hypervisor_name`` and ``-H hypervisor_name:option=value`` provide options for hypervisor. Network ^^^^^^^ ``--no-nics`` option causes converter to ignore any network information provided. ``--network=number:option=value`` sets network information according to provided data, ignoring the OVF package configuration. Operating System ^^^^^^^^^^^^^^^^ ``--os-type=type`` sets os type accordingly, this option is **required** when importing from OVF instance not created from Ganeti config file. ``--os-parameters`` provides options for chosen operating system. Tags ^^^^ ``--tags=tag1,tag2,tag3`` is a means of providing tags specific for the instance. After the conversion is completed, you may use ``gnt-backup import`` to import the instance into Ganeti. Example:: ovfconverter import file.ovf --disk-template=diskless \ --os-type=lenny-image \ --backend=vcpus=1,memory=512,auto_balance \ -H:xen-pvm \ --net=0:mode=bridged,link=xen-br0 \ --name=xen.i1 [...] gnt-backup import xen.i1 [...] gnt-instance list Export options -------------- Export options include choice of disk formats to convert the disk image (``--format``) and compression of the disk into gzip format (``--compress``). User has also the choice of allowing to skip the Ganeti-specific part of the OVF document (``--external``). By default, exported OVF package will not be contained in the OVA package, but this may be changed by adding ``--ova`` option. Please note that in order to create an OVF package, it is first required that you export your VM using ``gnt-backup export``. Example:: gnt-backup export -n node1.xen xen.i1 [...] ovfconverter export --format=vmdk --ova --external \ --output-dir=~/xen.i1 \ /srv/ganeti/export/xen.i1.node1.xen/config.ini Implementation details ====================== Disk conversion --------------- Disk conversion for both import and export is done using external tool called ``qemu-img``. The same tool is used to determine the type of disk, as well as its virtual size. Import ------ Import functionality is implemented using two classes - OVFReader and OVFImporter. OVFReader class is used to read the contents of the ``.ovf`` file. Every action that requires ``.ovf`` file access is done through that class. It also performs validation of manifest, if one is present. The result of reading some part of file is typically a dictionary or a string, containing options which correspond to the ones in ``config.ini`` file. Only in case of disks, the resulting value is different - it is then a list of disk names. The reason for that is the need for conversion. OVFImporter class performs all the command-line-like tasks, such as unpacking OVA package, removing temporary directory, converting disk file to raw format or saving the configuration file on disk. It also contains a set of functions that read the options provided in the command line. Typical workflow for the import is very simple: - read the ``.ovf`` file into memory - verify manifest - parse each element of the configuration file: name, disk template, hypervisor, operating system, backend parameters, network and disks - check if option for the element can be read from command line options - if yes: parse options from command line - otherwise: read the appropriate portion of ``.ovf`` file - save gathered information in ``config.ini`` file Export ------ Similar to import, export functionality also uses two classes - OVFWriter and OVFExporter. OVFWriter class produces XML output based on the information given. Its sole role is to separate the creation of ``.ovf`` file content. OVFExporter class gathers information from ``config.ini`` file or command line and performs necessary operations like disk conversion, disk compression, manifest creation and OVA package creation. Typical workflow for the export is even simpler, than for the import: - read the ``config.ini`` file into memory - gather information about certain parts of the instance, convert and compress disks if desired - save each of these elements as a fragment of XML tree - save the XML tree as ``.ovf`` file - create manifest file and fill it with appropriate checksums - if ``--ova`` option was chosen, pack the results into ``.ova`` tarfile Work in progress ---------------- - conversion to/from raw disk should be quicker - add graphic card memory to export information (12 MB of memory) - space requirements for conversion + compression + ova are currently enormous - add support for disks in chunks - add support for certificates - investigate why VMWare's ovftool does not work with ovfconverter's compression and ova packaging -- maybe noteworty: if OVA archive does not have a disk (i.e. in OVA package there is only .ovf ad .mf file), then the ovftool works - investigate why new versions of VirtualBox have problems with OVF created by ovfconverter (everything works fine with 3.16 version, but not with 4.0) .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-partitioned.rst000064400000000000000000000306731476477700300207050ustar00rootroot00000000000000================== Partitioned Ganeti ================== :Created: 2012-Oct-05 :Status: Implemented :Ganeti-Version: 2.7.0, 2.8.0, 2.9.0 .. contents:: :depth: 4 Current state and shortcomings ============================== Currently Ganeti can be used to easily share a node between multiple virtual instances. While it's easy to do a completely "best effort" sharing it's quite harder to completely reserve resources for the use of a particular instance. In particular this has to be done manually for CPUs and disk, is implemented for RAM under Xen, but not under KVM, and there's no provision for network level QoS. Proposed changes ================ We want to make it easy to partition a node between machines with exclusive use of hardware resources. While some sharing will anyway need to happen (e.g. for operations that use the host domain, or use resources, like buses, which are unique or very scarce on host systems) we'll strive to maintain contention at a minimum, but won't try to avoid all possible sources of it. Exclusive use of disks ---------------------- ``exclusive_storage`` is a new node parameter. When it's enabled, Ganeti will allocate entire disks to instances. Though it's possible to think of ways of doing something similar for other storage back-ends, this design targets only ``plain`` and ``drbd``. The name is generic enough in case the feature will be extended to other back-ends. The flag value should be homogeneous within a node-group; ``cluster-verify`` will report any violation of this condition. Ganeti will consider each physical volume in the destination volume group as a host disk (for proper isolation, an administrator should make sure that there aren't multiple PVs on the same physical disk). When ``exclusive_storage`` is enabled in a node group, all PVs in the node group must have the same size (within a certain margin, say 1%, defined through a new parameter). Ganeti will check this condition when the ``exclusive_storage`` flag is set, whenever a new node is added and as part of ``cluster-verify``. When creating a new disk for an instance, Ganeti will allocate the minimum number of PVs to hold the disk, and those PVs will be excluded from the pool of available PVs for further disk creations. The underlying LV will be striped, when striping is allowed by the current configuration. Ganeti will continue to track only the LVs, and query the LVM layer to figure out which PVs are available and how much space is free. Yet, creation, disk growing, and free-space reporting will ignore any partially allocated PVs, so that PVs won't be shared between instance disks. For compatibility with the DRBD template and to take into account disk variability, Ganeti will always subtract 2% (this will be a parameter) from the PV space when calculating how many PVs are needed to allocate an instance and when nodes report free space. The obvious target for this option is plain disk template, which doesn't provide redundancy. An administrator can still provide resilience against disk failures by setting up RAID under PVs, but this is transparent to Ganeti. Spindles as a resource ~~~~~~~~~~~~~~~~~~~~~~ When resources are dedicated and there are more spindles than instances on a node, it is natural to assign more spindles to instances than what is strictly needed. For this reason, we introduce a new resource: spindles. A spindle is a PV in LVM. The number of spindles required for a disk of an instance is specified together with the size. Specifying the number of spindles is possible only when ``exclusive_storage`` is enabled. It is an error to specify a number of spindles insufficient to contain the requested disk size. When ``exclusive_storage`` is not enabled, spindles are not used in free space calculation, in allocation algorithms, and policies. When it's enabled, ``hspace``, ``hbal``, and allocators will use spindles instead of disk size for their computation. For each node, the number of all the spindles in every LVM group is recorded, and different LVM groups are accounted separately in allocation and balancing. There is already a concept of spindles in Ganeti. It's not related to any actual spindle or volume count, but it's used in ``spindle_use`` to measure the pressure of an instance on the storage system and in ``spindle_ratio`` to balance the I/O load on the nodes. When ``exclusive_storage`` is enabled, these parameters as currently defined won't make any sense, so their meaning will be changed in this way: - ``spindle_use`` refers to the resource, hence to the actual spindles (PVs in LVM), used by an instance. The values specified in the instance policy specifications are compared to the run-time numbers of spindle used by an instance. The ``spindle_use`` back-end parameter will be ignored. - ``spindle_ratio`` in instance policies and ``spindle_count`` in node parameters are ignored, as the exclusive assignment of PVs already implies a value of 1.0 for the first, and the second is replaced by the actual number of spindles. When ``exclusive_storage`` is disabled, the existing spindle parameters behave as before. Dedicated CPUs -------------- ``vpcu_ratio`` can be used to tie the number of VCPUs to the number of CPUs provided by the hardware. We need to take into account the CPU usage of the hypervisor. For Xen, this means counting the number of VCPUs assigned to ``Domain-0``. For KVM, it's more difficult to limit the number of CPUs used by the node OS. ``cgroups`` could be a solution to restrict the node OS to use some of the CPUs, leaving the other ones to instances and KVM processes. For KVM, the number of CPUs for the host system should also be a hypervisor parameter (set at the node group level). Dedicated RAM ------------- Instances should not compete for RAM. This is easily done on Xen, but it is tricky on KVM. Xen ~~~ Memory is already fully segregated under Xen, if sharing mechanisms (transcendent memory, auto ballooning, etc) are not in use. KVM ~~~ Under KVM or LXC memory is fully shared between the host system and all the guests, and instances can even be swapped out by the host OS. It's not clear if the problem can be solved by limiting the size of the instances, so that there is plenty of room for the host OS. We could implement segregation using cgroups to limit the memory used by the host OS. This requires finishing the implementation of the memory hypervisor status (set at the node group level) that changes how free memory is computed under KVM systems. Then we have to add a way to enforce this limit on the host system itself, rather than leaving it as a calculation tool only. Another problem for KVM is that we need to decide about the size of the cgroup versus the size of the VM: some overhead will in particular exist, due to the fact that an instance and its encapsulating KVM process share the same space. For KVM systems the physical memory allocatable to instances should be computed by subtracting an overhead for the KVM processes, whose value can be either statically configured or set in a hypervisor status parameter. NUMA ~~~~ If instances are pinned to CPUs, and the amount of memory used for every instance is proportionate to the number of VCPUs, NUMA shouldn't be a problem, as the hypervisors allocate memory in the appropriate NUMA node. Work is in progress in Xen and the Linux kernel to always allocate memory correctly even without pinning. Therefore, we don't need to address this problem specifically; it will be solved by future versions of the hypervisors or by implementing CPU pinning. Constrained instance sizes -------------------------- In order to simplify allocation and resource provisioning we want to limit the possible sizes of instances to a finite set of specifications, defined at node-group level. Currently it's possible to define an instance policy that limits the minimum and maximum value for CPU, memory, and disk usage (and spindles and any other resource, when implemented), independently from each other. We extend the policy by allowing it to contain more occurrences of the specifications for both the limits for the instance resources. Each specification pair (minimum and maximum) has a unique priority associated to it (or in other words, specifications are ordered), which is used by ``hspace`` (see below). The standard specification doesn't change: there is one for the whole cluster. For example, a policy could be set up to allow instances with this constraints: - between 1 and 2 CPUs, 2 GB of RAM, and between 10 GB and 400 GB of disk space; - 4 CPUs, 4 GB of RAM, and between 10 GB and 800 GB of disk space. Then, an instance using 1 CPU, 2 GB of RAM and 50 GB of disk would be legal, as an instance using 4 CPUs, 4 GB of RAM, and 20 GB of disk, while an instance using 2 CPUs, 4 GB of RAM and 40 GB of disk would be illegal. Ganeti will refuse to create (or modify) instances that violate instance policy constraints, unless the flag ``--ignore-ipolicy`` is passed. While the changes needed to check constraint violations are straightforward, ``hspace`` behavior needs some adjustments for tiered allocation. ``hspace`` will start to allocate instances using the maximum specification with the highest priority, then it will try to lower the most constrained resources (without breaking the policy) before moving to the second highest priority, and so on. For consistent results in capacity calculation, the specifications inside a policy should be ordered so that the biggest specifications have the highest priorities. Also, specifications should not overlap. Ganeti won't check nor enforce such constraints, though. Implementation order ==================== We will implement this design in the following order: - Exclusive use of disks (without spindles as a resource) - Constrained instance sizes - Spindles as a resource - Dedicated CPU and memory In this way have always new features that are immediately useful. Spindles as a resource are not needed for correct capacity calculation, as long as allowed disk sizes are multiples of spindle size, so it's been moved after constrained instance sizes. If it turns out that it's easier to implement dedicated disks with spindles as a resource, then we will do that. Possible future enhancements ============================ This section briefly describes some enhancements to the current design. They may require their own design document, and must be re-evaluated when considered for implementation, as Ganeti and the hypervisors may change substantially in the meantime. Network bandwidth ----------------- A new resource is introduced: network bandwidth. An administrator must be able to assign some network bandwidth to the virtual interfaces of an instance, and set limits in instance policies. Also, a list of the physical network interfaces available for Ganeti use and their maximum bandwidth must be kept at node-group or node level. This information will be taken into account for allocation, balancing, and free-space calculation. An additional enhancement is Ganeti enforcing the values set in the bandwidth resource. This can be done by configuring limits for example via openvswitch or normal QoS for bridging or routing. The bandwidth resource represents the average bandwidth usage, so a few new back-end parameters are needed to configure how to deal with bursts (they depend on the actual way used to enforce the limit). CPU pinning ----------- In order to avoid unwarranted migrations between CPUs and to deal with NUMA effectively we may need CPU pinning. CPU scheduling is a complex topic and still under active development in Xen and the Linux kernel, so we wont' try to outsmart their developers. If we need pinning it's more to have predictable performance than to get the maximum performance (which is best done by the hypervisor), so we'll implement a very simple algorithm that allocates CPUs when an instance is assigned to a node (either when it's created or when it's moved) and takes into account NUMA and maybe CPU multithreading. A more refined version might run also when an instance is deleted, but that would involve reassigning CPUs, which could be bad with NUMA. Overcommit for RAM and disks ---------------------------- Right now it is possible to assign more VCPUs to the instances running on a node than there are CPU available. This works as normally CPU usage on average is way below 100%. There are ways to share memory pages (e.g. KSM, transcendent memory) and disk blocks, so we could add new parameters to overcommit memory and disks, similar to ``vcpu_ratio``. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-performance-tests.rst000064400000000000000000000120371476477700300220160ustar00rootroot00000000000000======================== Performance tests for QA ======================== :Created: 2014-Apr-28 :Status: Implemented :Ganeti-Version: 2.10.0 .. contents:: :depth: 4 This design document describes performance tests to be added to QA in order to measure performance changes over time. Current state and shortcomings ============================== Currently, only functional QA tests are performed. Those tests verify the correct behaviour of Ganeti in various configurations, but are not designed to continuously monitor the performance of Ganeti. The current QA tests don't execute multiple tasks/jobs in parallel. Therefore, the locking part of Ganeti does not really receive any testing, neither functional nor performance wise. On the plus side, Ganeti's QA code does already measure the runtime of individual tests, which is leveraged in this design. Proposed changes ================ The tests to be added in the context of this design document focus on two areas: * Job queue performance. How does Ganeti handle a lot of submitted jobs? * Parallel job execution performance. How well does Ganeti parallelize jobs? Jobs are submitted to the job queue in sequential order, but the execution of the jobs runs in parallel. All job submissions must complete within a reasonable timeout. In order to make it easier to recognize performance related tests, all tests added in the context of this design get a description with a "PERFORMANCE: " prefix. Job queue performance --------------------- Tests targeting the job queue should eliminate external factors (like network/disk performance or hypervisor delays) as much as possible, so they are designed to run in a vcluster QA environment. The following tests are added to the QA: * Submit the maximum amount of instance create jobs in parallel. As soon as a creation job succeeds, submit a removal job for this instance. * Submit as many instance create jobs as there are nodes in the cluster in parallel (for non-redundant instances). Removal jobs as above. * For the maximum amount of instances in the cluster, submit modify jobs (modify hypervisor and backend parameters) in parallel. * For the maximum amount of instances in the cluster, submit stop, start, reboot and reinstall jobs in parallel. * For the maximum amount of instances in the cluster, submit multiple list and info jobs in parallel. * For the maximum amount of instances in the cluster, submit move jobs in parallel. While the move operations are running, get instance information using info jobs. Those jobs are required to return within a reasonable low timeout. * For the maximum amount of instances in the cluster, submit add-, remove- and list-tags jobs. * Submit 200 `gnt-debug delay` jobs with a delay of 0.1 seconds. To speed up submission, perform multiple job submissions in parallel. Verify that submitting jobs doesn't significantly slow down during the process. Verify that querying cluster information over CLI and RAPI succeeds in a timely fashion with the delay jobs running/queued. Parallel job execution performance ---------------------------------- Tests targeting the performance of parallel execution of "real" jobs in close-to-production clusters should actually perform all operations, such as creating disks and starting instances. This way, real world locking or waiting issues can be reproduced. Performing all those operations does requires quite some time though, so only a smaller number of instances and parallel jobs can be tested realistically. The following tests are added to the QA: * Submitting twice as many instance creation request as there are nodes in the cluster, using DRBD as disk template. The job parameters are chosen according to best practice for parallel instance creation without running the risk of instance creation failing for too many parallel creation attempts. As soon as a creation job succeeds, submit a removal job for this instance. * Submitting twice as many instance creation request as there are nodes in the cluster, using Plain as disk template. As soon as a creation job succeeds, submit a removal job for this instance. This test can make better use of parallelism because only one node must be locked for an instance creation. * Create an instance using DRBD. Fail it over, migrate it, change its secondary node, reboot it and reinstall it while creating an additional instance in parallel to each of those operations. Future work =========== Based on test results of the tests listed above, additional tests can be added to cover more real-world use-cases. Also, based on user requests, specially crafted performance tests modeling those workloads can be added too. Additionally, the correlations between job submission time and job queue size could be detected. Therefore, a snapshot of the job queue before job submission could be taken to measure job submission time based on the jobs in the queue. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-plain-redundancy.rst000064400000000000000000000052211476477700300216070ustar00rootroot00000000000000====================================== Redundancy for the plain disk template ====================================== .. contents:: :depth: 4 This document describes how N+1 redundancy is achieved for instanes using the plain disk template. Current state and shortcomings ============================== Ganeti has long considered N+1 redundancy for DRBD, making sure that on the secondary nodes enough memory is reserved to host the instances, should one node fail. Recently, ``htools`` have been extended to also take :doc:`design-shared-storage-redundancy` into account. For plain instances, there is no direct notion of redundancy: if the node the instance is running on dies, the instance is lost. However, if the instance can be reinstalled (e.g, because it is providing a stateless service), it does make sense to ask if the remaining nodes have enough free capacity for the instances to be recreated. This form of capacity planning is currently not addressed by current Ganeti. Proposed changes ================ The basic considerations follow those of :doc:`design-shared-storage-redundancy`. Also, the changes to the tools follow the same pattern. Definition of N+1 redundancy in the presence of shared and plain storage ------------------------------------------------------------------------ A cluster is considered N+1 redundant, if, for every node, the following steps can be carried out. First all DRBD instances are migrated out. Then, all shared-storage instances of that node are relocated to another node in the same node group. Finally, all plain instances of that node are reinstalled on a different node in the same node group; in the search for a new nodes for the plain instances, they will be recreated in order of decreasing memory size. Note that the first two setps are those in the definition of N+1 redundancy for shared storage. In particular, this notion of redundancy strictly extends the one for shared storage. Again, checking this notion of redundancy is computationally expensive and the non-DRBD part is mainly a capacity property in the sense that we expect the majority of instance moves that are fine from a DRBD point of view will not lead from a redundant to a non-redundant situation. Modifications to existing tools ------------------------------- The changes to the exisiting tools are literally the same as for :doc:`design-shared-storage-redundancy` with the above definition of N+1 redundancy substituted in for that of redundancy for shared storage. In particular, ``gnt-cluster verify`` will not be changed and ``hbal`` will use N+1 redundancy as a final filter step to disallow moves that lead from a redundant to a non-redundant situation. ganeti-3.1.0~rc2/doc/design-qemu-blockdev.rst000064400000000000000000000201221476477700300211050ustar00rootroot00000000000000====================================== Implementing the qemu blockdev backend ====================================== :Created: 2022-Apr-20 :Status: Implemented :Ganeti-Version: 3.1 .. contents:: :depth: 2 This is a design document explaining the changes inside Ganeti while transitioning to the blockdev backend of QEMU, making the currently used `-drive` parameter obsolete. Current state and shortcomings ============================== Ganeti's KVM/QEMU code currently uses the `-drive` commandline parameter to add virtual hard-disks, floppy or CD drives to instances. This approach has been deprecated and superseded by the newer `-blockdev` parameter which has been considered stable with the 2.9 release of QEMU. Furthermore, the use of `-drive` blocks the transition from QEMU's human monitor to QMP, as the latter has never seen an implementation of the relevant methods to hot-add/hot-remove storage devices configured through `-drive`. Currently, Ganeti QEMU/KVM instances support the following storage devices: ``Virtual Hard-disks`` An instance can have none to many disks which are represented to guests as the selected `disk_type`. Ganeti supports various device types like `paravirtual` (VirtIO), `scsi-hd` (along with an emulated SCSI controller), `ide` and the like. The disk type can only be set per instance, not per disk. A disk's backing storage may be file- or block-based, depending on the available disk templates on the node. Disks can be hot-added to or hot-removed from running instances. An instance may boot off the first disk when not using direct kernel boot, specified through the `boot_order` parameter. Ganeti allows tweaking of various disk related parameters like AIO modes, caching, discard settings etc. Recent versions of QEMU (5.0+) have introduced the `io_uring` AIO mode which is currently not configurable through Ganeti. ``Virtual CD Drives`` Ganeti allows for up to two virtual CD drives to be attached to an instance. The backing storage to a CD drive must be an ISO image, which needs to be either a file accessible locally on the node or remotely through a HTTP(S) URL. Different bus types (e.g. `ide`, `scsi`, or `paravirtual`) are supported. CD drives can not be hot-added to or hot-removed from running instances. Instances not using direct kernel boot may boot from the first CD drive, when specified through the `boot_order` parameter. If the `boot_order` is set to CD, the bus type will silently be overridden to `ide`. ``Virtual Floppy Drive`` An instance can be configured to provide access to a virtual floppy drive using an image file present on the node. Floppy drives can not be hot-added to or hot-removed from running instances. Instances not using direct kernel boot may boot from the floppy drive, when specified through the `boot_order` parameter. Proposed changes ================ We have to eliminate all uses of the `-drive` parameter from the Ganeti codebase to ensure compatibility with future versions of QEMU. With this, we can also switch all hotplugging-related methods to use QMP and drop all code relating to the human monitor interface. Ganeti should support the AIO mode `io_uring` as long as the QEMU version on the node is recent enough. Implementation ============== I/O Methods +++++++++++ With QEMU 5.0, support for `io_uring` has been introduced. This should be supported by Ganeti as well, given a recent enough QEMU version is present on the node. Ganeti will still default to using the `threads` mode on new installations. Disk Cache ++++++++++ Using the following table found in `man 1 qemu-system-x86_64` we can translate the disk cache modes known to Ganeti into the settings required by `-blockdev`: .. code-block:: none ┌─────────────â”Ŧ─────────────────â”Ŧ──────────────â”Ŧ────────────────┐ │ │ cache.writeback │ cache.direct │ cache.no-flush │ ├─────────────â”ŧ─────────────────â”ŧ──────────────â”ŧ────────────────┤ │writeback │ on │ off │ off │ ├─────────────â”ŧ─────────────────â”ŧ──────────────â”ŧ────────────────┤ │none │ on │ on │ off │ ├─────────────â”ŧ─────────────────â”ŧ──────────────â”ŧ────────────────┤ │writethrough │ off │ off │ off │ ├─────────────â”ŧ─────────────────â”ŧ──────────────â”ŧ────────────────┤ │directsync │ off │ on │ off │ ├─────────────â”ŧ─────────────────â”ŧ──────────────â”ŧ────────────────┤ │unsafe │ on │ off │ on │ └─────────────┴─────────────────┴──────────────┴────────────────┘ The table also shows `directsync` and `unsafe`, which are currently not implemented by Ganeti and may be addressed in future changes. The Ganeti value of `default` should internally be mapped to `writeback`, as that reflects the values which QEMU assumes when not given explicitly according to the documentation. Construction of the command line parameters +++++++++++++++++++++++++++++++++++++++++++ The code responsible for generating the commandline parameters to add disks, cdroms and floppies needs to be changed to use the combination of `-blockdev` and `-device`. This will not change they way a disk is actually presented to the virtual guest. A small exception will be cdrom: Ganeti used to overwrite the cdrom disk type to `ide` when `boot_order` is set to `cdrom`. This is not required any more - QEMU will also boot off VirtIO or SCSI CD drives. Hot-Adding Disks ++++++++++++++++++ The code needs to be refactored to make use of the QMP method "blockdev-add", using the same parameters as for its commandline counterpart (e.g. blockdev node id, caching, aio-mode). Hot-Removing Disks ++++++++++++++++++ Hot-removing a disk consists of two steps: - removing the virtual device (or rather: ask the guest to release it) - releasing the storage backend The first step always returns immediately (QMP request `device_del`) but signals its `actual` result asynchronously through the QMP event `DEVICE_DELETED`. Ganeti currently does not support receiving QMP events - implementing this will be out of scope for this change. In Ganeti releases up to 3.0 the human monitor was used to delete the device. Executing commands through that interface was very slow (500ms to 1s) and seemingly slow enough to let the following request to remove the drive succeed as the guest had enough time to actually release the device. With the switch to QMP, both requests will fire in direct succession. Since QEMU cannot release a block device (`blockdev-del` through QMP) which is still in use by a device inside the guest, hot-removing disks will always fail. Without support for QMP events, the only feasible way will be to mimic the slow human monitor interface and block for one second after sending the `device_del` request to the guest. On upgraded clusters it will **not** be possible to hot-remove a disk before the instance has been either restarted or live-migrated (thus "upgrading" all disk related parameters to `-blockdev`). Disks added with `-drive` can not be removed using `blockdev-dev`. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-query-splitting.rst000064400000000000000000000154471476477700300215450ustar00rootroot00000000000000=========================================== Splitting the query and job execution paths =========================================== :Created: 2012-May-07 :Status: Implemented :Ganeti-Version: 2.7.0, 2.8.0, 2.9.0 Introduction ============ Currently, the master daemon does two main roles: - execute jobs that change the cluster state - respond to queries Due to the technical details of the implementation, the job execution and query paths interact with each other, and for example the "masterd hang" issue that we had late in the 2.5 release cycle was due to the interaction between job queries and job execution. Furthermore, also because technical implementations (Python lacking read-only variables being one example), we can't share internal data structures for jobs; instead, in the query path, we read them from disk in order to not block job execution due to locks. All these point to the fact that the integration of both queries and job execution in the same process (multi-threaded) creates more problems than advantages, and hence we should look into separating them. Proposed design =============== In Ganeti 2.7, we will introduce a separate, optional daemon to handle queries (note: whether this is an actual "new" daemon, or its functionality is folded into confd, remains to be seen). This daemon will expose exactly the same Luxi interface as masterd, except that job submission will be disabled. If so configured (at build time), clients will be changed to: - keep sending REQ_SUBMIT_JOB, REQ_SUBMIT_MANY_JOBS, and all requests except REQ_QUERY_* to the masterd socket (but also QR_LOCK) - redirect all REQ_QUERY_* requests to the new Luxi socket of the new daemon (except generic query with QR_LOCK) This new daemon will serve both pure configuration queries (which confd can already serve), and run-time queries (which currently only masterd can serve). Since the RPC can be done from any node to any node, the new daemon can run on all master candidates, not only on the master node. This means that all gnt-* list options can be now run on other nodes than the master node. If we implement this as a separate daemon that talks to confd, then we could actually run this on all nodes of the cluster (to be decided). During the 2.7 release, masterd will still respond to queries itself, but it will log all such queries for identification of "misbehaving" clients. Advantages ---------- As far as I can see, this will bring some significant advantages. First, we remove any interaction between the job execution and cluster query state. This means that bugs in the locking code (job execution) will not impact the query of the cluster state, nor the query of the job execution itself. Furthermore, we will be able to have different tuning parameters between job execution (e.g. 25 threads for job execution) versus query (since these are transient, we could practically have unlimited numbers of query threads). As a result of the above split, we move from the current model, where shutdown of the master daemon practically "breaks" the entire Ganeti functionality (no job execution nor queries, not even connecting to the instance console), to a split model: - if just masterd is stopped, then other cluster functionality remains available: listing instances, connecting to the console of an instance, etc. - if just "luxid" is stopped, masterd can still process jobs, and one can furthermore run queries from other nodes (MCs) - only if both are stopped, we end up with the previous state This will help, for example, in the case where the master node has crashed and we haven't failed it over yet: querying and investigating the cluster state will still be possible from other master candidates (on small clusters, this will mean from all nodes). A last advantage is that we finally will be able to reduce the footprint of masterd; instead of previous discussion of splitting individual jobs, which requires duplication of all the base functionality, this will just split the queries, a more trivial piece of code than job execution. This should be a reasonable work effort, with a much smaller impact in case of failure (we can still run masterd as before). Disadvantages ------------- We might get increased inconsistency during queries, as there will be a delay between masterd saving an updated configuration and confd/query loading and parsing it. However, this could be compensated by the fact that queries will only look at "snapshots" of the configuration, whereas before it could also look at "in-progress" modifications (due to the non-atomic updates). I think these will cancel each other out, we will have to see in practice how it works. Another disadvantage *might* be that we have a more complex setup, due to the introduction of a new daemon. However, the query path will be much simpler, and when we remove the query functionality from masterd we should have a more robust system. Finally, we have QR_LOCK, which is an internal query related to the master daemon, using the same infrastructure as the other queries (related to cluster state). This is unfortunate, and will require untangling in order to keep code duplication low. Long-term plans =============== If this works well, the plan would be (tentatively) to disable the query functionality in masterd completely in Ganeti 2.8, in order to remove the duplication. This might change based on how/if we split the configuration/locking daemon out, or not. Once we split this out, there is not technical reason why we can't execute any query from any node; except maybe practical reasons (network topology, remote nodes, etc.) or security reasons (if/whether we want to change the cluster security model). In any case, it should be possible to do this in a reliable way from all master candidates. Update: We decided to keep the restriction to run queries on the master node. The reason is that it is confusing from a usability point of view that querying will work on any node and suddenly, when the user tries to submit a job, it won't work. Some implementation details --------------------------- We will fold this in confd, at least initially, to reduce the proliferation of daemons. Haskell will limit (if used properly) any too deep integration between the old "confd" functionality and the new query one. As advantages, we'll have a single daemons that handles configuration queries. The redirection of Luxi requests can be easily done based on the request type, if we have both sockets open, or if we open on demand. We don't want the masterd to talk to the luxid itself (hidden redirection), since we want to be able to run queries while masterd is down. During the 2.7 release cycle, we can test all queries against both masterd and luxid in QA, so we know we have exactly the same interface and it is consistent. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-query2.rst000064400000000000000000000316031476477700300176040ustar00rootroot00000000000000====================== Query version 2 design ====================== :Created: 2010-Nov-12 :Status: Implemented :Ganeti-Version: 2.4.0 .. contents:: :depth: 4 .. highlight:: python Current state and shortcomings ============================== Queries are used to retrieve information about the cluster, e.g. a list of instances or nodes. For historical reasons they use a simple data structure for their result. The client submits the fields it would like to receive and the query returns a list for each item (instance, node, etc.) available. Each item consists of another list representing the fields' values. This data structure has a few drawbacks. It can't associate a status (e.g. “node offline”) with fields as using special values can lead to ambiguities. Additionally it can't mark fields as “not found” as the list of returned columns must match the fields requested. Example:: >>> cli.GetClient().QueryNodes([], ["name", "pip", "mfree"], False) [ ['node1.example.com', '192.0.2.18', 14800], ['node2.example.com', '192.0.2.19', 31280] ] There is no way for clients to determine the list of possible fields, meaning they have to be hardcoded. Selecting unknown fields raises an exception:: >>> cli.GetClient().QueryNodes([], ["name", "UnknownField"], False) ganeti.errors.OpPrereqError: (u'Unknown output fields selected: UnknownField', u'wrong_input') The client must also know each fields' kind, that is whether a field is numeric, boolean, describes a storage size, etc. Centralizing this information in one place, the master daemon, is desirable. Proposed changes ---------------- The current query result format can not be changed as it's being used in various places. Changing the format from one Ganeti version to another would cause too much disruption. For this reason the ability to explicitly request a new result format must be added while the old format stays the default. The implementation of query filters is planned for the future. To avoid having to change the calls again, a (hopefully) future-compatible interface will be implemented now. In Python code, the objects described below will be implemented using subclasses of ``objects.ConfigObject``, providing existing facilities for de-/serializing. Regular expressions +++++++++++++++++++ As it turned out, only very few fields for instances used regular expressions, all of which can easily be turned into static field names. Therefore their use in field names is dropped. Reasons: - When regexps are used and a field name is not listed as a simple string in the field dictionary, all keys in the field dictionary have to be checked whether they're a regular expression object and if so, matched (see ``utils.FindMatch``). - Code becomes simpler. There would be no need anymore to care about regular expressions as field names—they'd all be simple strings, even if there are many more. The list of field names would be static once built at module-load time. - There's the issue of formatting titles for the clients. Should it be done in the server? In the client? The field definition's title would contain backreferences to the regexp groups in the field name (``re.MatchObject.expand`` can be used). With just strings, the field definitions can be passed directly to the client. They're static. - Only a side note: In the memory consumed for 1'000 ``_sre.SRE_Pattern`` objects (as returned by ``re.compile`` for an expression with one group) one can easily store 10'000 strings of the same length (the regexp objects keep the expression string around, so compiling the expression always uses more memory). .. _item-types: Item types ++++++++++ The proposal is to implement this new interface for the following items: ``instance`` Instances ``node`` Nodes ``job`` Jobs ``lock`` Locks ``os`` Operating systems .. _data-query: Data query ++++++++++ .. _data-query-request: Request ^^^^^^^ The request is a dictionary with the following entries: ``what`` (string, required) An :ref:`item type `. ``fields`` (list of strings, required) List of names of fields to return. Example:: ["name", "mem", "nic0.ip", "disk0.size", "disk1.size"] ``filter`` (optional) This will be used to filter queries. In this implementation only names can be filtered to replace the previous ``names`` parameter to queries. An empty filter (``None``) will return all items. To retrieve specific names, the filter must be specified as follows, with the inner part repeated for each name:: ["|", ["=", "name", "node1"], ["=", "name", "node2"], ...] Filters consist of S-expressions (``["operator", ]``) and extensions will be made in the future to allow for more operators and fields. Such extensions might include a Python-style "in" operator, but for simplicity only "=" is supported in this implementation. To reiterate: Filters for this implementation must consist of exactly one OR expression (``["|", ...]``) and one or more name equality filters (``["=", "name", "..."]``). Support for synchronous queries, currently available in the interface but disabled in the master daemon, will be dropped. Direct calls to opcodes have to be used instead. .. _data-query-response: Response ^^^^^^^^ The result is a dictionary with the following entries: ``fields`` (list of :ref:`field definitions `) In-order list of a :ref:`field definition ` for each requested field, unknown fields are returned with the kind ``unknown``. Length must be equal to number of requested fields. ``data`` (list of lists of tuples) List of lists, one list for each item found. Each item's list must have one entry for each field listed in ``fields`` (meaning their length is equal). Each field entry is a tuple of ``(status, value)``. ``status`` must be one of the following values: Normal (numeric 0) Value is available and matches the kind in the :ref:`field definition `. Unknown field (numeric 1) Field for this column is not known. Value must be ``None``. No data (numeric 2) Exact meaning depends on query, e.g. node is unreachable or marked offline. Value must be ``None``. Value unavailable for item (numeric 3) Used if, for example, NIC 3 is requested for an instance with only one network interface. Value must be ``None``. Resource offline (numeric 4) Used if resource is marked offline. Value must be ``None``. Example response after requesting the fields ``name``, ``mfree``, ``xyz``, ``mtotal``, ``nic0.ip``, ``nic1.ip`` and ``nic2.ip``:: { "fields": [ { "name": "name", "title": "Name", "kind": "text", }, { "name": "mfree", "title": "MemFree", "kind": "unit", }, # Unknown field { "name": "xyz", "title": None, "kind": "unknown", }, { "name": "mtotal", "title": "MemTotal", "kind": "unit", }, { "name": "nic0.ip", "title": "Nic.IP/0", "kind": "text", }, { "name": "nic1.ip", "title": "Nic.IP/1", "kind": "text", }, { "name": "nic2.ip", "title": "Nic.IP/2", "kind": "text", }, ], "data": [ [(0, "node1"), (0, 128), (1, None), (0, 4096), (0, "192.0.2.1"), (0, "192.0.2.2"), (3, None)], [(0, "node2"), (0, 96), (1, None), (0, 5000), (0, "192.0.2.21"), (0, "192.0.2.39"), (3, "192.0.2.90")], # Node not available, can't get "mfree" or "mtotal" [(0, "node3"), (2, None), (1, None), (2, None), (0, "192.0.2.30"), (3, None), (3, None)], ], } .. _fields-query: Fields query ++++++++++++ .. _fields-query-request: Request ^^^^^^^ The request is a dictionary with the following entries: ``what`` (string, required) An :ref:`item type `. ``fields`` (list of strings, optional) List of names of fields to return. If not set, all fields are returned. Example:: ["name", "mem", "nic0.ip", "disk0.size", "disk1.size"] .. _fields-query-response: Response ^^^^^^^^ The result is a dictionary with the following entries: ``fields`` (list of :ref:`field definitions `) List of a :ref:`field definition ` for each field. If ``fields`` was set in the request and contained an unknown field, it is returned as type ``unknown``. Example:: { "fields": [ { "name": "name", "title": "Name", "kind": "text", }, { "name": "mfree", "title": "MemFree", "kind": "unit", }, { "name": "mtotal", "title": "MemTotal", "kind": "unit", }, { "name": "nic0.ip", "title": "Nic.IP/0", "kind": "text", }, { "name": "nic1.ip", "title": "Nic.IP/1", "kind": "text", }, { "name": "nic2.ip", "title": "Nic.IP/2", "kind": "text", }, { "name": "nic3.ip", "title": "Nic.IP/3", "kind": "text", }, # â€Ļ { "name": "disk0.size", "title": "Disk.Size/0", "kind": "unit", }, { "name": "disk1.size", "title": "Disk.Size/1", "kind": "unit", }, { "name": "disk2.size", "title": "Disk.Size/2", "kind": "unit", }, { "name": "disk3.size", "title": "Disk.Size/3", "kind": "unit", }, # â€Ļ ] } .. _field-def: Field definition ++++++++++++++++ A field definition is a dictionary with the following entries: ``name`` (string) Field name. Must only contain characters matching ``[a-z0-9/._]``. ``title`` (string) Human-readable title to use in output. Must not contain whitespace. ``kind`` (string) Field type, one of the following: ``unknown`` Unknown field ``text`` String ``bool`` Boolean, true/false ``number`` Numeric ``unit`` Numeric, in megabytes ``timestamp`` Unix timestamp in seconds since the epoch ``other`` Free-form type, depending on query More types can be added in the future, so clients should default to formatting any unknown types the same way as "other", which should be a string representation in most cases. ``doc`` (string) Human-readable description. Must start with uppercase character and must not end with punctuation or contain newlines. .. TODO: Investigate whether there are fields with floating point .. numbers Example 1 (item name):: { "name": "name", "title": "Name", "kind": "text", } Example 2 (free memory):: { "name": "mfree", "title": "MemFree", "kind": "unit", } Example 3 (list of primary instances):: { "name": "pinst", "title": "PrimaryInstances", "kind": "other", } .. _old-result-format: Old result format +++++++++++++++++ To limit the amount of code necessary, the :ref:`new result format ` will be converted for clients calling the old methods. Unavailable values are set to ``None``. If unknown fields were requested, the whole query fails as the client expects exactly the fields it requested. .. _query2-luxi: LUXI ++++ Currently query calls take a number of parameters, e.g. names, fields and whether to use locking. These will continue to work and return the :ref:`old result format `. Only clients using the new calls described below will be able to make use of new features such as filters. Two new calls are introduced: ``Query`` Execute a query on items, optionally filtered. Takes a single parameter, a :ref:`query object ` encoded as a dictionary and returns a :ref:`data query response `. ``QueryFields`` Return list of supported fields as :ref:`field definitions `. Takes a single parameter, a :ref:`fields query object ` encoded as a dictionary and returns a :ref:`fields query response `. Python ++++++ The LUXI API is more or less mapped directly into Python. In addition to the existing stub functions new ones will be added for the new query requests. RAPI ++++ The RAPI interface already returns dictionaries for each item, but to not break compatibility no changes should be made to the structure (e.g. to include field definitions). The proposal here is to add a new parameter to allow clients to execute the requests described in this proposal directly and to receive the unmodified result. The new formats are a lot more verbose, flexible and extensible. .. _cli-programs: CLI programs ++++++++++++ Command line programs might have difficulties to display the verbose status data to the user. There are several options: - Use colours to indicate missing values - Display status as value in parentheses, e.g. "(unavailable)" - Hide unknown columns from the result table and print a warning - Exit with non-zero code to indicate failures and/or missing data Some are better for interactive usage, some better for use by other programs. It is expected that a combination will be used. The column separator (``--separator=â€Ļ``) can be used to differentiate between interactive and programmatic usage. Other discussed solutions ------------------------- Another solution discussed was to add an additional column for each non-static field containing the status. Clients interested in the status could explicitly query for it. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-reason-trail.rst000064400000000000000000000110451476477700300207530ustar00rootroot00000000000000=================== Ganeti reason trail =================== :Created: 2013-Mar-15 :Status: Implemented :Ganeti-Version: 2.9.0 .. contents:: :depth: 2 This is a design document detailing the implementation of a way for Ganeti to track the origin and the reason of every executed command, from its starting point (command line, remote API, some htool, etc.) to its actual execution time. Current state and shortcomings ============================== There is currently no way to track why a job and all the operations part of it were executed, and who or what triggered the execution. This is an inconvenience in general, and also it makes impossible to have certain information, such as finding the reason why an instance last changed its status (i.e.: why it was started/stopped/rebooted/etc.), or distinguishing an admin request from a scheduled maintenance or an automated tool's work. Proposed changes ================ We propose to introduce a new piece of information, that will be called "reason trail", to track the path from the issuing of a command to its execution. The reason trail will be a list of 3-tuples ``(source, reason, timestamp)``, with: ``source`` The entity deciding to perform (or forward) a command. It is represented by an arbitrary string, but strings prepended by "gnt:" are reserved for Ganeti components, and they will be refused by the interfaces towards the external world. ``reason`` The reason why the entity decided to perform the operation. It is represented by an arbitrary string. The string might possibly be empty, because certain components of the system might just "pass on" the operation (therefore wanting to be recorded in the trail) but without an explicit reason. ``timestamp`` The time when the element was added to the reason trail. It has to be expressed in nanoseconds since the unix epoch (0:00:00 January 01, 1970). If not enough precision is available (or needed) it can be padded with zeroes. The reason trail will be attached at the OpCode level. When it has to be serialized externally (such as on the RAPI interface), it will be serialized in JSON format. Specifically, it will be serialized as a list of elements. Each element will be a list with two strings (for ``source`` and ``reason``) and one integer number (the ``timestamp``). Any component the operation goes through is allowed (but not required) to append it's own reason to the list. Other than this, the list shouldn't be modified. As an example here is the reason trail for a shutdown operation invoked from the command line through the gnt-instance tool:: [("user", "Cleanup of unused instances", 1363088484000000000), ("gnt:client:gnt-instance", "stop", 1363088484020000000), ("gnt:opcode:shutdown", "job=1234;index=0", 1363088484026000000), ("gnt:daemon:noded:shutdown", "", 1363088484135000000)] where the first 3-tuple is determined by a user-specified message, passed to gnt-instance through a command line parameter. The same operation, launched by an external GUI tool, and executed through the remote API, would have a reason trail like:: [("user", "Cleanup of unused instances", 1363088484000000000), ("other-app:tool-name", "gui:stop", 1363088484000300000), ("gnt:client:rapi:shutdown", "", 1363088484020000000), ("gnt:library:rlib2:shutdown", "", 1363088484023000000), ("gnt:opcode:shutdown", "job=1234;index=0", 1363088484026000000), ("gnt:daemon:noded:shutdown", "", 1363088484135000000)] Implementation ============== The OpCode base class will be modified to include a new parameter, "reason". This will receive the reason trail as built by all the previous steps. When an OpCode is added to a job (in jqueue.py) the job number and the opcode index will be recorded as the reason for the existence of that opcode. From the command line tools down to the opcodes, the implementation of this design will be shared by all the components of the system. After the opcodes have been enqueued in a job queue and are dispatched for execution, the implementation will have to be OpCode specific because of the current structure of the ganeti backend. The implementation of opcode-specific parts will start from the operations that affect the instance status (as required by the design document about the monitoring daemon, for the instance status data collector). Such opcodes will be changed so that the "reason" is passed to them and they will then export the reason trail on a file. The implementation for other opcodes will follow when required. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-repaird.rst000064400000000000000000000277711476477700300200160ustar00rootroot00000000000000========================= Ganeti Maintenance Daemon ========================= .. contents:: :depth: 4 This design document outlines the implementation of a new Ganeti daemon coordinating all maintenance operations on a cluster (rebalancing, activate disks, ERROR_down handling, node repairs actions). Current state and shortcomings ============================== With ``harep``, Ganeti has a basic mechanism for repairs of instances in a cluster. The ``harep`` tool can fix a broken DRBD status, migrate, failover, and reinstall instances. It is intended to be run regularly, e.g., via a cron job. It will submit appropriate Ganeti jobs to take action within the range allowed by instance tags and keep track of them by recoding the job ids in appropriate tags. Besides ``harep``, Ganeti offers no further support for repair automation. While useful, this setup can be insufficient in some situations. Failures in actual hardware, e.g., a physical disk, currently requires coordination around Ganeti: the hardware failure is detected on the node, Ganeti needs to be told to evacuate the node, and, once this is done, some other entity needs to coordinate the actual physical repair. Currently there is no support by Ganeti to automatically prepare everything for a hardware swap. Proposed changes ================ We propose the addition of an additional daemon, called ``maintd`` that will coordinate cluster balance actions, instance repair actions, and work for hardware repair needs of individual nodes. The information about the work to be done will be obtained from a dedicated data collector via the :doc:`design-monitoring-agent`. Self-diagnose data collector ---------------------------- The monitoring daemon will get one additional dedicated data collector for node health. The collector will call an external command supposed to do any hardware-specific diagnose for the node it is running on. That command is configurable, but needs to be white-listed ahead of time by the node. For convenience, the empty string will stand for a build-in diagnose that always reports that everything is OK; this will also be the default value for this collector. Note that the self-diagnose data collector itself can, and usually will, call separate diagnose tools for separate subsystems. However, it always has to provide a consolidated description of the overall health state of the node. Protocol ~~~~~~~~ The collector script takes no arguments and is supposed to output the string representation of a single JSON object where the individual fields have the following meaning. Note that, if several things are broken on that node, the self-diagnose collector script has to merge them into a single repair action. status ...... This is a JSON string where the value is one of ``Ok``, ``live-repair``, ``evacuate``, ``evacuate-failover``. This indicates the overall need for repair and Ganeti actions to be taken. The meaning of these states are no action needed, some action is needed that can be taken while instances continue to run on that node, it is necessary to evacuate and offline the node, and it is necessary to evacuate and offline the node without attempting live migrations, respectively. command ....... If the status is ``live-repair``, a repair command can be specified. This command will be executed as repair action following the :doc:`design-restricted-commands`, however extended to read information on ``stdin``. The whole diagnose JSON object will be provided as ``stdin`` to those commands. details ....... An opaque JSON value that the repair daemon will just pass through and export. It is intended to contain information about the type of repair that needs to be done after the respective Ganeti action is finished. E.g., it might contain information which piece of hardware is to be swapped, once the node is fully evacuated and offlined. As two failures are considered different, if the output of the script encodes a different JSON object, the collector script should ensure that as long as the hardware status does not change, the output of the script is stable; otherwise this would cause various events reported for the same failure. Security considerations ~~~~~~~~~~~~~~~~~~~~~~~ Command execution ................. Obviously, running arbitrary commands that are part of the configuration poses a security risk. Note that an underlying design goal of Ganeti is that even with RAPI credentials known to the attacker, he still cannot obtain data from within the instances. As monitoring, however, is configurable via RAPI, we require the node to white-list the command using a mechanism similar to the :doc:`design-restricted-commands`; in our case, the white-listing directory will be ``/etc/ganeti/node-diagnose-commands``. For the repair-commands, as mentioned, we extend the :doc:`design-restricted-commands` by allowing input on ``stdin``. All other restrictions, in particular the white-listing requirement, remain. The white-listing directory will be ``/etc/ganeti/node-repair-commands``. Result forging .............. As the repair daemon will take real Ganeti actions based on the diagnose reported by the self-diagnose script through the monitoring daemon, we need to verify integrity of such reports to avoid denial-of-service by fraudaulent error reports. Therefore, the monitoring daemon will sign the result by an hmac signature with the cluster hmac key, in the same way as it is done in the ``confd`` wire protocol (see :doc:`design-2.1`). Repair-event life cycle ----------------------- Once a repair event is detected, a unique identifier is assigned to it. As long as the node-health collector returns the same output (as JSON object), this is still considered the same event. This identifier can be used to cancel an observed event at any time; for this an appropriate command-line and RAPI endpoint will be provided. Cancelling an event tells the repair daemon not to take any actions (despite them being requested) for this event and forget about it, as soon as it is no longer observed. Corresponding Ganeti actions will be initiated and success or failure of these Ganeti jobs monitored. All jobs submitted by the repair daemon will have the string ``gnt:daemon:maintd`` and the event identifier in the reason trail, so that :doc:`design-optables` is possible. Once a job fails, no further jobs will be submitted for this event to avoid further damage; the repair action is considered failed in this case. Once all requested actions succeeded, or one failed, the node where the event as observed will be tagged by a tag starting with ``maintd:repairready:`` or ``maintd:repairfailed:``, respectively, where the event identifier is encoded in the rest of the tag. On the one hand, it can be used as an additional verification whether a node is ready for a specific repair. However, the main purpose is to provide a simple and uniform interface to acknowledge an event. Once a ``maintd:repairready`` tag is removed, the maintenance daemon will forget about this event, as soon as it is no longer observed by any monitoring daemon. Removing a ``maintd:repairfailed:`` tag will make the maintenance daemon to unconditionally forget the event; note that, if the underlying problem is not fixed yet, this provides an easy way of restarting a repair flow. Repair daemon ------------- The new daemon ``maintd`` will be running on the master node only. It will verify the master status of its node by popular vote in the same way as all the other master-only daemons. If started on a non-master node, it will exit immediately with exit code ``exitNotmaster``, i.e., 11. External Reporting Protocol ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Upon successful start, the daemon will bind to a port overridable at command-line, by default 1816, on the master network device. There it will serve the current repair state via HTTP. All queries will be HTTP GET requests and all answers will be encoded in JSON format. Initially, the following requests will be supported. ``/`` ..... Returns the list of supported protocol versions, initially just ``[1]``. ``/1/status`` ............. Returns a list of all non-cleared incidents. Each incident is reported as a JSON object with at least the following information. - ``id`` The unique identifier assigned to the event. - ``node`` The UUID of the node on which the even was observed. - ``original`` The very JSON object reported by self-diagnose data collector. - ``repair-status`` A string describing the progress made on this event so far. It is one of the following. + ``noted`` The event has been observed, but no action has been taken yet + ``pending`` At least one job has been submitted in reaction to the event and none of the submitted jobs has failed so far. + ``canceled`` The event has been canceled, i.e., ordered to be ignored, but is still observed. + ``failed`` At least one of the submitted jobs has failed. To avoid further damage, the repair daemon will not take any further action for this event. + ``completed`` All Ganeti actions associated with this event have been completed successfully, including tagging the node. - ``jobs`` The list of the numbers of ganeti jobs submitted in response to this event. - ``tag`` A string that is the tag that either has been added to the node, or, if the repair event is not yet finalized, will be added in case of success. State ~~~~~ As repairs, especially those involving physically swapping hardware, can take a long time, the repair daemon needs to store its state persistently. As we cannot exclude master-failovers during a repair cycle, it does so by storing it as part of the Ganeti configuration. This will be done by adding a new top-level entry to the Ganeti configuration. The SSConf will not be changed. Superseeding ``harep`` and implicit balancing ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To have a single point coordinating all repair actions, the new repair daemon will also have the ability to take over the work currently done by ``harep``. To allow a smooth transition, ``maintd`` when carrying out ``harep``'s duties will add tags in precisely the same way as ``harep`` does. As the new daemon will have to move instances, it will also have the ability to balance the cluster in a way coordinated with the necessary evacuation options; dynamic load information can be taken into account. The question on whether to do ``harep``'s work and whether to balance the cluster and if so using which strategy (e.g., taking dynamic load information into account or not, allowing disk moves or not) are configurable via the Ganeti configuration. The default will be to do neither of those tasks. ``harep`` will continue to exist unchanged as part of the ``htools``. Mode of operation ~~~~~~~~~~~~~~~~~ The repair daemon will poll the monitoring daemons for the value of the self-diagnose data collector at the same (configurable) rate the monitoring daemon collects this collector; if load-based balancing is enabled, it will also collect for the the load data needed. Repair events will be exposed on the web status page as soon as observed. The Ganeti jobs doing the actual maintenance will be submitted in rounds. A new round will be started if all jobs of the old round have finished, and there is an unhandled repair event or the cluster is unbalanced enough (provided that autobalancing is enabled). In each round, ``maintd`` will first determine the most invasive action for each node; despite the self-diagnose collector summing observations in a single action recommendation, a new, more invasive recommendation can be issued before the handling of the first recommendation is finished. For all nodes to be evacuated, the first evacuation task is scheduled, in a way that these tasks do not conflict with each other. Then, for all instances on a non-affected node, that need ``harep``-style repair (if enabled) those jobs are scheduled to the extend of not conflicting with each other. Then on the remaining nodes that are not part of a failed repair event either, the jobs of the first balancing step are scheduled. All those jobs of a round are submitted at once. As they do not conflict they will be able to run in parallel. ganeti-3.1.0~rc2/doc/design-reservations.rst000064400000000000000000000126231476477700300211020ustar00rootroot00000000000000===================== Instance Reservations ===================== :Created: 2014-Jul-22 :Status: Partially Implemented :Ganeti-Version: 2.12.0, 2.14.0 .. contents:: :depth: 4 This is a design document detailing the addition of a concept of reservations for forthcoming instances to Ganeti. Current state and shortcomings ============================== Currently, for a new instance, all information about the instance, including a resolvable full name, needs to be present before an instance creation can be attempted. Moreover, the only way to find out if a cluster can host an instance is to try creating it. This can lead to problems in situations where the host name can only be determined, and hence DNS entries created, once the cluster for the instance is chosen. If lot of instances are created in parallel, by the time the DNS entries propagated, the cluster capacity might already be exceeded. Proposed changes ================ The proposed solution to overcome this shortcoming is to support *forthcoming instances*. Those forthcoming instances only exist as entries in in the configuration, hence creation and removal is cheap. Forthcoming instances can have an arbitrary subset of the attributes of a real instance with only the UUID being mandatory. In a similar way, their disks are also considered forthcoming. If a forthcoming instance specifies resources (memory, disk sizes, number of CPUs), these resources are accounted for as if they were real. In particular, a forthcoming can always be turned into a real one without running out of resources. RAPI extension -------------- To accomdate the handling of forthcoming instances, the :doc:`rapi` will be extended as follows. The following functionality will be added to existing resources. - /2/instances/ - POST. This request will have an additional, optional, ``forthcoming`` flag with default ``False``. If the ``forthcoming`` flag is set, all parameters are optional, including the instance name. Even if ``forthcoming`` is set, the result of this request will still be the job id to be used later for polling. A job to create a forthcoming instance, however, will return the UUID of the instance instead of the hosts allocated for it. - /2/instances/[instance_name]/modify - PUT. This request will be able to handle forthcoming instances in the same way as existing ones. The following resources will be added. - /2/instances/[instance_uuid]/modify - PUT. This will behave the same as the ``modify`` indexed by instance name and is added to allow modification of an instance that does not yet have a name. - /2/instances/[instance_uuid]/rename - PUT. This will behave the same as the ``rename`` indexed by instance name. This will allow to assign a name to a forthcoming instance. - /2/instances/[instance_name]/create - POST. Create the forthcoming instance. It is a prerequisite that all mandatory parameters of the instance are specified by now. It will return the id of the creation job, for later polling. Representation in the Configuration ----------------------------------- As for most part of the system, forthcoming instances and their disks are to be treated as if they were real. Therefore, the wire representation will be by adding an additional, optional, ``forthcoming`` flag to the ``instances`` and ``disks`` objects. Additionally, the internal consistency condition will be relaxed to have all non-uuid fields optional if an instance or disk is forthcoming. Rationale ~~~~~~~~~ The alternative to the chosen approach would be to add a new top-level object ``forthcoming_instances`` to the configuration. Both approaches bear the risk of introducing subtle bugs. Adding a new top-level object bears the risk of missing in some places to take into account the resources consumed by the forthcoming instances. Adding a new attribute and relaxing the consistency conditions bears the risk that some parts of the Python code cannot handle the fact that some fields are now optional if the instance is forthcoming. The design choice is to prefer the latter kind of errors, as they will always show up immediately when a faulty part of the code is touched and will always precisely indicate the part of the code base that needs to be changed. Haskell Representation ~~~~~~~~~~~~~~~~~~~~~~ The semantical condition on the instance fields renders the type into a Pascal-style variant record (one element of an enumeration type, and the remaining fields depend on the value of that field). Of course, in the Haskell part of our code base, this will be represented in the standard way having two constructors for the type; additionally there will be accessors for all the fields of the JSON representation (yielding ``Maybe`` values, as they can be optional if we're in the ``Forthcoming`` constructor). Adaptions of htools ------------------- Forthcoming instances are handled by htools essentially the same way as real instances with more possible moves, as a forthcoming instance may change primary and secondary node simultaneously. The restriction of not changing node group without explicit user request to do so remains. Wherever possible, moves of forthcoming instances are preferred over moving real instances, as forthcoming instances can be moved for free. Implementation wise, the existing ``Instance`` data structure is used, and a new bit ``forthcoming`` is added; for forthcoming instances, the ``name`` field will carry the UUID. ganeti-3.1.0~rc2/doc/design-resource-model.rst000064400000000000000000001323301476477700300213010ustar00rootroot00000000000000======================== Resource model changes ======================== :Created: 2011-Oct-12 :Status: Implemented :Ganeti-Version: 2.6.0 Introduction ============ In order to manage virtual machines across the cluster, Ganeti needs to understand the resources present on the nodes, the hardware and software limitations of the nodes, and how much can be allocated safely on each node. Some of these decisions are delegated to IAllocator plugins, for easier site-level customisation. Similarly, the HTools suite has an internal model that simulates the hardware resource changes in response to Ganeti operations, in order to provide both an iallocator plugin and for balancing the cluster. While currently the HTools model is much more advanced than Ganeti's, neither one is flexible enough and both are heavily geared toward a specific Xen model; they fail to work well with (e.g.) KVM or LXC, or with Xen when :term:`tmem` is enabled. Furthermore, the set of metrics contained in the models is limited to historic requirements and fails to account for (e.g.) heterogeneity in the I/O performance of the nodes. Current situation ================= Ganeti ------ At this moment, Ganeti itself doesn't do any static modelling of the cluster resources. It only does some runtime checks: - when creating instances, for the (current) free disk space - when starting instances, for the (current) free memory - during cluster verify, for enough N+1 memory on the secondaries, based on the (current) free memory Basically this model is a pure :term:`SoW` one, and it works well when there are other instances/LVs on the nodes, as it allows Ganeti to deal with ‘orphan’ resource usage, but on the other hand it has many issues, described below. HTools ------ Since HTools does an pure in-memory modelling of the cluster changes as it executes the balancing or allocation steps, it had to introduce a static (:term:`SoR`) cluster model. The model is constructed based on the received node properties from Ganeti (hence it basically is constructed on what Ganeti can export). Disk ~~~~ For disk it consists of just the total (``tdsk``) and the free disk space (``fdsk``); we don't directly track the used disk space. On top of this, we compute and warn if the sum of disk sizes used by instance does not match with ``tdsk - fdsk``, but otherwise we do not track this separately. Memory ~~~~~~ For memory, the model is more complex and tracks some variables that Ganeti itself doesn't compute. We start from the total (``tmem``), free (``fmem``) and node memory (``nmem``) as supplied by Ganeti, and additionally we track: instance memory (``imem``) the total memory used by primary instances on the node, computed as the sum of instance memory reserved memory (``rmem``) the memory reserved by peer nodes for N+1 redundancy; this memory is tracked per peer-node, and the maximum value out of the peer memory lists is the node's ``rmem``; when not using DRBD, this will be equal to zero missing memory (``xmem``) memory that cannot be unaccounted for via the Ganeti model; this is computed at startup as: tmem - imem - nmem - fmem if we define state-of-record free mem as: tmem - imem - nmem then we can interpret this as the difference between the state-of-record and state-of-world free memory; it presumed to remain constant irrespective of any instance moves unallocated memory (``umem``) the memory that is guaranteed to be not allocated to existing processes; in case of a static node model this is simply: min(state-of-record_free_mem, fmem) since the state-of-record changes during instance placement simulations, we can't use that definition directly (see the above note about missing memory presumed being constant); we need to use an equivalent definiton: state-of-record_free_mem - max(0, missing_memory) available memory (``amem``) this is defined as a zero bounded difference between unallocated and reserved memory: max(0, umem - rmem) so unless we use DRBD, this will be equal to ``umem`` ``tmem``, ``nmem`` and ``xmem`` are presumed constant during the instance moves, whereas the ``fmem``, ``imem``, ``rmem``, ``umem`` and ``amem`` values are updated according to the executed moves. CPU ~~~ The CPU model is different than the disk/memory models, since it's the only one where: #. we do oversubscribe physical CPUs #. and there is no natural limit for the number of VCPUs we can allocate We therefore track the total number of VCPUs used on the node and the number of physical CPUs, and we cap the vcpu-to-cpu ratio in order to make this somewhat more similar to the other resources which are limited. Dynamic load ~~~~~~~~~~~~ There is also a model that deals with *dynamic load* values in htools. As far as we know, it is not currently used actually with load values, but it is active by default with unitary values for all instances; it currently tracks these metrics: - disk load - memory load - cpu load - network load Even though we do not assign real values to these load values, the fact that we at least sum them means that the algorithm tries to equalise these loads, and especially the network load, which is otherwise not tracked at all. The practical result (due to a combination of these four metrics) is that the number of secondaries will be balanced. Limitations ----------- There are unfortunately many limitations to the current model. Memory ~~~~~~ The memory model doesn't work well in case of KVM. For Xen, the memory for the node (i.e. ``dom0``) can be static or dynamic; we don't support the latter case, but for the former case, the static value is configured in Xen/kernel command line, and can be queried from Xen itself. Therefore, Ganeti can query the hypervisor for the memory used for the node; the same model was adopted for the chroot/KVM/LXC hypervisors, but in these cases there's no natural value for the memory used by the base OS/kernel, and we currently try to compute a value for the node memory based on current consumption. This, being variable, breaks the assumptions in both Ganeti and HTools. This problem also shows for the free memory: if the free memory on the node is not constant (Xen with :term:`tmem` auto-ballooning enabled), or if the node and instance memory are pooled together (Linux-based hypervisors like KVM and LXC), the current value of the free memory is meaningless and cannot be used for instance checks. A separate issue related to the free memory tracking is that since we don't track memory use but rather memory availability, an instance that is temporary down changes Ganeti's understanding of the memory status of the node. This can lead to problems such as: .. digraph:: "free-mem-issue" node [shape=box]; inst1 [label="instance1"]; inst2 [label="instance2"]; node [shape=note]; nodeA [label="fmem=0"]; nodeB [label="fmem=1"]; nodeC [label="fmem=0"]; node [shape=ellipse, style=filled, fillcolor=green] {rank=same; inst1 inst2} stop [label="crash!", fillcolor=orange]; migrate [label="migrate/ok"]; start [style=filled, fillcolor=red, label="start/fail"]; inst1 -> stop -> start; stop -> migrate -> start [style=invis, weight=0]; inst2 -> migrate; {rank=same; inst1 inst2 nodeA} {rank=same; stop nodeB} {rank=same; migrate nodeC} nodeA -> nodeB -> nodeC [style=invis, weight=1]; The behaviour here is wrong; the migration of *instance2* to the node in question will succeed or fail depending on whether *instance1* is running or not. And for *instance1*, it can lead to cases where it if crashes, it cannot restart anymore. Finally, not a problem but rather a missing important feature is support for memory over-subscription: both Xen and KVM support memory ballooning, even automatic memory ballooning, for a while now. The entire memory model is based on a fixed memory size for instances, and if memory ballooning is enabled, it will “break” the HTools algorithm. Even the fact that KVM instances do not use all memory from the start creates problems (although not as high, since it will grow and stabilise in the end). Disks ~~~~~ Because we only track disk space currently, this means if we have a cluster of ``N`` otherwise identical nodes but half of them have 10 drives of size ``X`` and the other half 2 drives of size ``5X``, HTools will consider them exactly the same. However, in the case of mechanical drives at least, the I/O performance will differ significantly based on spindle count, and a “fair” load distribution should take this into account (a similar comment can be made about processor/memory/network speed). Another problem related to the spindle count is the LVM allocation algorithm. Currently, the algorithm always creates (or tries to create) striped volumes, with the stripe count being hard-coded to the ``./configure`` parameter ``--with-lvm-stripecount``. This creates problems like: - when installing from a distribution package, all clusters will be either limited or overloaded due to this fixed value - it is not possible to mix heterogeneous nodes (even in different node groups) and have optimal settings for all nodes - the striping value applies both to LVM/DRBD data volumes (which are on the order of gigabytes to hundreds of gigabytes) and to DRBD metadata volumes (whose size is always fixed at 128MB); when stripping such small volumes over many PVs, their size will increase needlessly (and this can confuse HTools' disk computation algorithm) Moreover, the allocation currently allocates based on a ‘most free space’ algorithm. This balances the free space usage on disks, but on the other hand it tends to mix rather badly the data and metadata volumes of different instances. For example, it cannot do the following: - keep DRBD data and metadata volumes on the same drives, in order to reduce exposure to drive failure in a many-drives system - keep DRBD data and metadata volumes on different drives, to reduce performance impact of metadata writes Additionally, while Ganeti supports setting the volume separately for data and metadata volumes at instance creation, there are no defaults for this setting. Similar to the above stripe count problem (which is about not good enough customisation of Ganeti's behaviour), we have limited pass-through customisation of the various options of our storage backends; while LVM has a system-wide configuration file that can be used to tweak some of its behaviours, for DRBD we don't use the :command:`drbdadmin` tool, and instead we call :command:`drbdsetup` directly, with a fixed/restricted set of options; so for example one cannot tweak the buffer sizes. Another current problem is that the support for shared storage in HTools is still limited, but this problem is outside of this design document. Locking ~~~~~~~ A further problem generated by the “current free” model is that during a long operation which affects resource usage (e.g. disk replaces, instance creations) we have to keep the respective objects locked (sometimes even in exclusive mode), since we don't want any concurrent modifications to the *free* values. A classic example of the locking problem is the following: .. digraph:: "iallocator-lock-issues" rankdir=TB; start [style=invis]; node [shape=box,width=2]; job1 [label="add instance\niallocator run\nchoose A,B"]; job1e [label="finish add"]; job2 [label="add instance\niallocator run\nwait locks"]; job2s [label="acquire locks\nchoose C,D"]; job2e [label="finish add"]; job1 -> job1e; job2 -> job2s -> job2e; edge [style=invis,weight=0]; start -> {job1; job2} job1 -> job2; job2 -> job1e; job1e -> job2s [style=dotted,label="release locks"]; In the above example, the second IAllocator run will wait for locks for nodes ``A`` and ``B``, even though in the end the second instance will be placed on another set of nodes (``C`` and ``D``). This wait shouldn't be needed, since right after the first IAllocator run has finished, :command:`hail` knows the status of the cluster after the allocation, and it could answer the question for the second run too; however, Ganeti doesn't have such visibility into the cluster state and thus it is forced to wait with the second job. Similar examples can be made about replace disks (another long-running opcode). .. _label-policies: Policies ~~~~~~~~ For most of the resources, we have metrics defined by policy: e.g. the over-subscription ratio for CPUs, the amount of space to reserve, etc. Furthermore, although there are no such definitions in Ganeti such as minimum/maximum instance size, a real deployment will need to have them, especially in a fully-automated workflow where end-users can request instances via an automated interface (that talks to the cluster via RAPI, LUXI or command line). However, such an automated interface will need to also take into account cluster capacity, and if the :command:`hspace` tool is used for the capacity computation, it needs to be told the maximum instance size, however it has a built-in minimum instance size which is not customisable. It is clear that this situation leads to duplicate definition of resource policies which makes it hard to easily change per-cluster (or globally) the respective policies, and furthermore it creates inconsistencies if such policies are not enforced at the source (i.e. in Ganeti). Balancing algorithm ~~~~~~~~~~~~~~~~~~~ The balancing algorithm, as documented in the HTools ``README`` file, tries to minimise the cluster score; this score is based on a set of metrics that describe both exceptional conditions and how spread the instances are across the nodes. In order to achieve this goal, it moves the instances around, with a series of moves of various types: - disk replaces (for DRBD-based instances) - instance failover/migrations (for all types) However, the algorithm only looks at the cluster score, and not at the *“cost”* of the moves. In other words, the following can and will happen on a cluster: .. digraph:: "balancing-cost-issues" rankdir=LR; ranksep=1; start [label="score Îą", shape=hexagon]; node [shape=box, width=2]; replace1 [label="replace_disks 500G\nscore Îą-3Îĩ\ncost 3"]; replace2a [label="replace_disks 20G\nscore Îą-2Îĩ\ncost 2"]; migrate1 [label="migrate\nscore Îą-Îĩ\ncost 1"]; choose [shape=ellipse,label="choose min(score)=Îą-3Îĩ\ncost 3"]; start -> {replace1; replace2a; migrate1} -> choose; Even though a migration is much, much cheaper than a disk replace (in terms of network and disk traffic on the cluster), if the disk replace results in a score infinitesimally smaller, then it will be chosen. Similarly, between two disk replaces, one moving e.g. ``500GiB`` and one moving ``20GiB``, the first one will be chosen if it results in a score smaller than the second one. Furthermore, even if the resulting scores are equal, the first computed solution will be kept, whichever it is. Fixing this algorithmic problem is doable, but currently Ganeti doesn't export enough information about nodes to make an informed decision; in the above example, if the ``500GiB`` move is between nodes having fast I/O (both disks and network), it makes sense to execute it over a disk replace of ``100GiB`` between nodes with slow I/O, so simply relating to the properties of the move itself is not enough; we need more node information for cost computation. Allocation algorithm ~~~~~~~~~~~~~~~~~~~~ .. note:: This design document will not address this limitation, but it is worth mentioning as it directly related to the resource model. The current allocation/capacity algorithm works as follows (per node-group):: repeat: allocate instance without failing N+1 This simple algorithm, and its use of ``N+1`` criterion, has a built-in limit of 1 machine failure in case of DRBD. This means the algorithm guarantees that, if using DRBD storage, there are enough resources to (re)start all affected instances in case of one machine failure. This relates mostly to memory; there is no account for CPU over-subscription (i.e. in case of failure, make sure we can failover while still not going over CPU limits), or for any other resource. In case of shared storage, there's not even the memory guarantee, as the N+1 protection doesn't work for shared storage. If a given cluster administrator wants to survive up to two machine failures, or wants to ensure CPU limits too for DRBD, there is no possibility to configure this in HTools (neither in :command:`hail` nor in :command:`hspace`). Current workaround employ for example deducting a certain number of instances from the size computed by :command:`hspace`, but this is a very crude method, and requires that instance creations are limited before Ganeti (otherwise :command:`hail` would allocate until the cluster is full). Proposed architecture ===================== There are two main changes proposed: - changing the resource model from a pure :term:`SoW` to a hybrid :term:`SoR`/:term:`SoW` one, where the :term:`SoR` component is heavily emphasised - extending the resource model to cover additional properties, completing the “holes” in the current coverage The second change is rather straightforward, but will add more complexity in the modelling of the cluster. The first change, however, represents a significant shift from the current model, which Ganeti had from its beginnings. Lock-improved resource model ---------------------------- Hybrid SoR/SoW model ~~~~~~~~~~~~~~~~~~~~ The resources of a node can be characterised in two broad classes: - mostly static resources - dynamically changing resources In the first category, we have things such as total core count, total memory size, total disk size, number of network interfaces etc. In the second category we have things such as free disk space, free memory, CPU load, etc. Note that nowadays we don't have (anymore) fully-static resources: features like CPU and memory hot-plug, online disk replace, etc. mean that theoretically all resources can change (there are some practical limitations, of course). Even though the rate of change of the two resource types is wildly different, right now Ganeti handles both the same. Given that the interval of change of the semi-static ones is much bigger than most Ganeti operations, even more than lengthy sequences of Ganeti jobs, it makes sense to treat them separately. The proposal is then to move the following resources into the configuration and treat the configuration as the authoritative source for them (a :term:`SoR` model): - CPU resources: - total core count - node core usage (*new*) - memory resources: - total memory size - node memory size - hypervisor overhead (*new*) - disk resources: - total disk size - disk overhead (*new*) Since these resources can though change at run-time, we will need functionality to update the recorded values. Pre-computing dynamic resource values ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Remember that the resource model used by HTools models the clusters as obeying the following equations: disk\ :sub:`free` = disk\ :sub:`total` - ∑ disk\ :sub:`instances` mem\ :sub:`free` = mem\ :sub:`total` - ∑ mem\ :sub:`instances` - mem\ :sub:`node` - mem\ :sub:`overhead` As this model worked fine for HTools, we can consider it valid and adopt it in Ganeti. Furthermore, note that all values in the right-hand side come now from the configuration: - the per-instance usage values were already stored in the configuration - the other values will are moved to the configuration per the previous section This means that we can now compute the free values without having to actually live-query the nodes, which brings a significant advantage. There are a couple of caveats to this model though. First, as the run-time state of the instance is no longer taken into consideration, it means that we have to introduce a new *offline* state for an instance (similar to the node one). In this state, the instance's runtime resources (memory and VCPUs) are no longer reserved for it, and can be reused by other instances. Static resources like disk and MAC addresses are still reserved though. Transitioning into and out of this reserved state will be more involved than simply stopping/starting the instance (e.g. de-offlining can fail due to missing resources). This complexity is compensated by the increased consistency of what guarantees we have in the stopped state (we always guarantee resource reservation), and the potential for management tools to restrict which users can transition into/out of this state separate from which users can stop/start the instance. Separating per-node resource locks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Many of the current node locks in Ganeti exist in order to guarantee correct resource state computation, whereas others are designed to guarantee reasonable run-time performance of nodes (e.g. by not overloading the I/O subsystem). This is an unfortunate coupling, since it means for example that the following two operations conflict in practice even though they are orthogonal: - replacing a instance's disk on a node - computing node disk/memory free for an IAllocator run This conflict increases significantly the lock contention on a big/busy cluster and at odds with the goal of increasing the cluster size. The proposal is therefore to add a new level of locking that is only used to prevent concurrent modification to the resource states (either node properties or instance properties) and not for long-term operations: - instance creation needs to acquire and keep this lock until adding the instance to the configuration - instance modification needs to acquire and keep this lock until updating the instance - node property changes will need to acquire this lock for the modification The new lock level will sit before the instance level (right after BGL) and could either be single-valued (like the “Big Ganeti Lock”), in which case we won't be able to modify two nodes at the same time, or per-node, in which case the list of locks at this level needs to be synchronised with the node lock level. To be determined. Lock contention reduction ~~~~~~~~~~~~~~~~~~~~~~~~~ Based on the above, the locking contention will be reduced as follows: IAllocator calls will no longer need the ``LEVEL_NODE: ALL_SET`` lock, only the resource lock (in exclusive mode). Hence allocating/computing evacuation targets will no longer conflict for longer than the time to compute the allocation solution. The remaining long-running locks will be the DRBD replace-disks ones (exclusive mode). These can also be removed, or changed into shared locks, but that is a separate design change. .. admonition:: FIXME Need to rework instance replace disks. I don't think we need exclusive locks for replacing disks: it is safe to stop/start the instance while it's doing a replace disks. Only modify would need exclusive, and only for transitioning into/out of offline state. Instance memory model --------------------- In order to support ballooning, the instance memory model needs to be changed from a “memory size” one to a “min/max memory size”. This interacts with the new static resource model, however, and thus we need to declare a-priori the expected oversubscription ratio on the cluster. The new minimum memory size parameter will be similar to the current memory size; the cluster will guarantee that in all circumstances, all instances will have available their minimum memory size. The maximum memory size will permit burst usage of more memory by instances, with the restriction that the sum of maximum memory usage will not be more than the free memory times the oversubscription factor: ∑ memory\ :sub:`min` ≤ memory\ :sub:`available` ∑ memory\ :sub:`max` ≤ memory\ :sub:`free` * oversubscription_ratio The hypervisor will have the possibility of adjusting the instance's memory size dynamically between these two boundaries. Note that the minimum memory is related to the available memory on the node, whereas the maximum memory is related to the free memory. On DRBD-enabled clusters, this will have the advantage of using the reserved memory for N+1 failover for burst usage, instead of having it completely idle. .. admonition:: FIXME Need to document how Ganeti forces minimum size at runtime, overriding the hypervisor, in cases of failover/lack of resources. New parameters -------------- Unfortunately the design will add a significant number of new parameters, and change the meaning of some of the current ones. Instance size limits ~~~~~~~~~~~~~~~~~~~~ As described in :ref:`label-policies`, we currently lack a clear definition of the support instance sizes (minimum, maximum and standard). As such, we will add the following structure to the cluster parameters: - ``min_ispec``, ``max_ispec``: minimum and maximum acceptable instance specs - ``std_ispec``: standard instance size, which will be used for capacity computations and for default parameters on the instance creation request Ganeti will by default reject non-standard instance sizes (lower than ``min_ispec`` or greater than ``max_ispec``), but as usual a ``--ignore-ipolicy`` option on the command line or in the RAPI request will override these constraints. The ``std_spec`` structure will be used to fill in missing instance specifications on create. Each of the ispec structures will be a dictionary, since the contents can change over time. Initially, we will define the following variables in these structures: +---------------+----------------------------------+--------------+ |Name |Description |Type | +===============+==================================+==============+ |mem_size |Allowed memory size |int | +---------------+----------------------------------+--------------+ |cpu_count |Allowed vCPU count |int | +---------------+----------------------------------+--------------+ |disk_count |Allowed disk count |int | +---------------+----------------------------------+--------------+ |disk_size |Allowed disk size |int | +---------------+----------------------------------+--------------+ |nic_count |Allowed NIC count |int | +---------------+----------------------------------+--------------+ Inheritance +++++++++++ In a single-group cluster, the above structure is sufficient. However, on a multi-group cluster, it could be that the hardware specifications differ across node groups, and thus the following problem appears: how can Ganeti present unified specifications over RAPI? Since the set of instance specs is only partially ordered (as opposed to the sets of values of individual variable in the spec, which are totally ordered), it follows that we can't present unified specs. As such, the proposed approach is to allow the ``min_ispec`` and ``max_ispec`` to be customised per node-group (and export them as a list of specifications), and a single ``std_spec`` at cluster level (exported as a single value). Allocation parameters ~~~~~~~~~~~~~~~~~~~~~ Beside the limits of min/max instance sizes, there are other parameters related to capacity and allocation limits. These are mostly related to the problems related to over allocation. +-----------------+----------+---------------------------+----------+------+ | Name |Level(s) |Description |Current |Type | | | | |value | | +=================+==========+===========================+==========+======+ |vcpu_ratio |cluster, |Maximum ratio of virtual to|64 (only |float | | |node group|physical CPUs |in htools)| | +-----------------+----------+---------------------------+----------+------+ |spindle_ratio |cluster, |Maximum ratio of instances |none |float | | |node group|to spindles; when the I/O | | | | | |model doesn't map directly | | | | | |to spindles, another | | | | | |measure of I/O should be | | | | | |used instead | | | +-----------------+----------+---------------------------+----------+------+ |max_node_failures|cluster, |Cap allocation/capacity so |1 |int | | |node group|that the cluster can |(hardcoded| | | | |survive this many node |in htools)| | | | |failures | | | +-----------------+----------+---------------------------+----------+------+ Since these are used mostly internally (in htools), they will be exported as-is from Ganeti, without explicit handling of node-groups grouping. Regarding ``spindle_ratio``, in this context spindles do not necessarily have to mean actual mechanical hard-drivers; it's rather a measure of I/O performance for internal storage. Disk parameters ~~~~~~~~~~~~~~~ The proposed model for the new disk parameters is a simple free-form one based on dictionaries, indexed per disk template and parameter name. Only the disk template parameters are visible to the user, and those are internally translated to logical disk level parameters. This is a simplification, because each parameter is applied to a whole nested structure and there is no way of fine-tuning each level's parameters, but it is good enough for the current parameter set. This model could need to be expanded, e.g., if support for three-nodes stacked DRBD setups is added to Ganeti. At JSON level, since the object key has to be a string, the keys can be encoded via a separator (e.g. slash), or by having two dict levels. When needed, the unit of measurement is expressed inside square brackets. +--------+--------------+-------------------------+---------------------+------+ |Disk |Name |Description |Current status |Type | |template| | | | | +========+==============+=========================+=====================+======+ |plain |stripes |How many stripes to use |Configured at |int | | | |for newly created (plain)|./configure time, not| | | | |logical volumes |overridable at | | | | | |runtime | | +--------+--------------+-------------------------+---------------------+------+ |drbd |data-stripes |How many stripes to use |Same as for |int | | | |for data volumes |plain/stripes | | +--------+--------------+-------------------------+---------------------+------+ |drbd |metavg |Default volume group for |Same as the main |string| | | |the metadata LVs |volume group, | | | | | |overridable via | | | | | |'metavg' key | | +--------+--------------+-------------------------+---------------------+------+ |drbd |meta-stripes |How many stripes to use |Same as for lvm |int | | | |for meta volumes |'stripes', suboptimal| | | | | |as the meta LVs are | | | | | |small | | +--------+--------------+-------------------------+---------------------+------+ |drbd |disk-barriers |What kind of barriers to |Either all enabled or|string| | | |*disable* for disks; |all disabled, per | | | | |either "n" or a string |./configure time | | | | |containing a subset of |option | | | | |"bfd" | | | +--------+--------------+-------------------------+---------------------+------+ |drbd |meta-barriers |Whether to disable or not|Handled together with|bool | | | |the barriers for the meta|disk-barriers | | | | |volume | | | +--------+--------------+-------------------------+---------------------+------+ |drbd |resync-rate |The (static) resync rate |Hardcoded in |int | | | |for drbd, when using the |constants.py, not | | | | |static syncer, in KiB/s |changeable via Ganeti| | +--------+--------------+-------------------------+---------------------+------+ |drbd |dynamic-resync|Whether to use the |Not supported. |bool | | | |dynamic resync speed | | | | | |controller or not. If | | | | | |enabled, c-plan-ahead | | | | | |must be non-zero and all | | | | | |the c-* parameters will | | | | | |be used by DRBD. | | | | | |Otherwise, the value of | | | | | |resync-rate will be used | | | | | |as a static resync speed.| | | +--------+--------------+-------------------------+---------------------+------+ |drbd |c-plan-ahead |Agility factor of the |Not supported. |int | | | |dynamic resync speed | | | | | |controller. (the higher, | | | | | |the slower the algorithm | | | | | |will adapt the resync | | | | | |speed). A value of 0 | | | | | |(that is the default) | | | | | |disables the controller | | | | | |[ds] | | | +--------+--------------+-------------------------+---------------------+------+ |drbd |c-fill-target |Maximum amount of |Not supported. |int | | | |in-flight resync data | | | | | |for the dynamic resync | | | | | |speed controller | | | | | |[sectors] | | | +--------+--------------+-------------------------+---------------------+------+ |drbd |c-delay-target|Maximum estimated peer |Not supported. |int | | | |response latency for the | | | | | |dynamic resync speed | | | | | |controller [ds] | | | +--------+--------------+-------------------------+---------------------+------+ |drbd |c-max-rate |Upper bound on resync |Not supported. |int | | | |speed for the dynamic | | | | | |resync speed controller | | | | | |[KiB/s] | | | +--------+--------------+-------------------------+---------------------+------+ |drbd |c-min-rate |Minimum resync speed for |Not supported. |int | | | |the dynamic resync speed | | | | | |controller [KiB/s] | | | +--------+--------------+-------------------------+---------------------+------+ |drbd |disk-custom |Free-form string that |Not supported |string| | | |will be appended to the | | | | | |drbdsetup disk command | | | | | |line, for custom options | | | | | |not supported by Ganeti | | | | | |itself | | | +--------+--------------+-------------------------+---------------------+------+ |drbd |net-custom |Free-form string for |Not supported |string| | | |custom net setup options | | | +--------+--------------+-------------------------+---------------------+------+ Currently Ganeti supports only DRBD 8.0.x, 8.2.x, 8.3.x. It will refuse to work with DRBD 8.4 since the :command:`drbdsetup` syntax has changed significantly. The barriers-related parameters have been introduced in different DRBD versions; please make sure that your version supports all the barrier parameters that you pass to Ganeti. Any version later than 8.3.0 implements all of them. The minimum DRBD version for using the dynamic resync speed controller is 8.3.9, since previous versions implement different parameters. A more detailed discussion of the dynamic resync speed controller parameters is outside the scope of the present document. Please refer to the ``drbdsetup`` man page (`8.3 `_ and `8.4 `_). An interesting discussion about them can also be found in a `drbd-user mailing list post `_. All the above parameters are at cluster and node group level; as in other parts of the code, the intention is that all nodes in a node group should be equal. It will later be decided to which node group give precedence in case of instances split over node groups. .. admonition:: FIXME Add details about when each parameter change takes effect (device creation vs. activation) Node parameters ~~~~~~~~~~~~~~~ For the new memory model, we'll add the following parameters, in a dictionary indexed by the hypervisor name (node attribute ``hv_state``). The rationale is that, even though multi-hypervisor clusters are rare, they make sense sometimes, and thus we need to support multiple node states (one per hypervisor). Since usually only one of the multiple hypervisors is the 'main' one (and the others used sparringly), capacity computation will still only use the first hypervisor, and not all of them. Thus we avoid possible inconsistencies. +----------+-----------------------------------+---------------+-------+ |Name |Description |Current state |Type | | | | | | +==========+===================================+===============+=======+ |mem_total |Total node memory, as discovered by|Queried at |int | | |this hypervisor |runtime | | +----------+-----------------------------------+---------------+-------+ |mem_node |Memory used by, or reserved for, |Queried at |int | | |the node itself; not that some |runtime | | | |hypervisors can report this in an | | | | |authoritative way, other not | | | +----------+-----------------------------------+---------------+-------+ |mem_hv |Memory used either by the |Not used, |int | | |hypervisor itself or lost due to |htools computes| | | |instance allocation rounding; |it internally | | | |usually this cannot be precisely | | | | |computed, but only roughly | | | | |estimated | | | +----------+-----------------------------------+---------------+-------+ |cpu_total |Total node cpu (core) count; |Queried at |int | | |usually this can be discovered |runtime | | | |automatically | | | | | | | | | | | | | | | | | | +----------+-----------------------------------+---------------+-------+ |cpu_node |Number of cores reserved for the |Not used at all|int | | |node itself; this can either be | | | | |discovered or set manually. Only | | | | |used for estimating how many VCPUs | | | | |are left for instances | | | | | | | | +----------+-----------------------------------+---------------+-------+ Of the above parameters, only ``_total`` ones are straight-forward. The others have sometimes strange semantics: - Xen can report ``mem_node``, if configured statically (as we recommend); but Linux-based hypervisors (KVM, chroot, LXC) do not, and this needs to be configured statically for these values - ``mem_hv``, representing unaccounted for memory, is not directly computable; on Xen, it can be seen that on a N GB machine, with 1 GB for dom0 and N-2 GB for instances, there's just a few MB left, instead fo a full 1 GB of RAM; however, the exact value varies with the total memory size (at least) - ``cpu_node`` only makes sense on Xen (currently), in the case when we restrict dom0; for Linux-based hypervisors, the node itself cannot be easily restricted, so it should be set as an estimate of how "heavy" the node loads will be Since these two values cannot be auto-computed from the node, we need to be able to declare a default at cluster level (debatable how useful they are at node group level); the proposal is to do this via a cluster-level ``hv_state`` dict (per hypervisor). Beside the per-hypervisor attributes, we also have disk attributes, which are queried directly on the node (without hypervisor involvement). The are stored in a separate attribute (``disk_state``), which is indexed per storage type and name; currently this will be just ``DT_PLAIN`` and the volume name as key. +-------------+-------------------------+--------------------+--------+ |Name |Description |Current state |Type | | | | | | +=============+=========================+====================+========+ |disk_total |Total disk size |Queried at runtime |int | | | | | | +-------------+-------------------------+--------------------+--------+ |disk_reserved|Reserved disk size; this |None used in Ganeti;|int | | |is a lower limit on the |htools has a | | | |free space, if such a |parameter for this | | | |limit is desired | | | +-------------+-------------------------+--------------------+--------+ |disk_overhead|Disk that is expected to |None used in Ganeti;|int | | |be used by other volumes |htools detects this | | | |(set via |at runtime | | | |``reserved_lvs``); | | | | |usually should be zero | | | +-------------+-------------------------+--------------------+--------+ Instance parameters ~~~~~~~~~~~~~~~~~~~ New instance parameters, needed especially for supporting the new memory model: +--------------+----------------------------------+-----------------+------+ |Name |Description |Current status |Type | | | | | | +==============+==================================+=================+======+ |offline |Whether the instance is in |Not supported |bool | | |“permanent” offline mode; this is | | | | |stronger than the "admin_down” | | | | |state, and is similar to the node | | | | |offline attribute | | | +--------------+----------------------------------+-----------------+------+ |be/max_memory |The maximum memory the instance is|Not existent, but|int | | |allowed |virtually | | | | |identical to | | | | |memory | | +--------------+----------------------------------+-----------------+------+ HTools changes -------------- All the new parameters (node, instance, cluster, not so much disk) will need to be taken into account by HTools, both in balancing and in capacity computation. Since the Ganeti's cluster model is much enhanced, Ganeti can also export its own reserved/overhead variables, and as such HTools can make less “guesses” as to the difference in values. .. admonition:: FIXME Need to detail more the htools changes; the model is clear to me, but need to write it down. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-restricted-commands.rst000064400000000000000000000045551476477700300223320ustar00rootroot00000000000000===================================== Design for executing commands via RPC ===================================== :Created: 2012-Oct-16 :Status: Implemented :Ganeti-Version: 2.7.0 .. contents:: :depth: 3 Current state and shortcomings ------------------------------ We have encountered situations where a node was no longer responding to attempts at connecting via SSH or SSH became unavailable through other means. Quite often the node daemon is still available, even in situations where there's little free memory. The latter is due to the node daemon being locked into main memory using ``mlock(2)``. Since the node daemon does not allow the execution of arbitrary commands, quite often the only solution left was either to attempt a powercycle request via said node daemon or to physically reset the node. Proposed changes ---------------- The goal of this design is to allow the execution of non-arbitrary commands via RPC requests. Since this can be dangerous in case the cluster certificate (``server.pem``) is leaked, some precautions need to be taken: - No parameters may be passed - No absolute or relative path may be passed, only a filename - Executable must reside in ``/etc/ganeti/restricted-commands``, which must be owned by root:root and have mode 0755 or stricter - Must be regular files or symlinks - Must be executable by root:root There shall be no way to list available commands or to retrieve an executable's contents. The result from a request to execute a specific command will either be its output and exit code, or a generic error message. Only the receiving node's log files shall contain information as to why executing the command failed. To slow down dictionary attacks on command names in case an attacker manages to obtain a copy of ``server.pem``, a system-wide, file-based lock is acquired before verifying the command name and its executable. If a command can not be executed for some reason, the lock is only released with a delay of several seconds, after which the generic error message will be returned to the caller. At first, restricted commands will not be made available through the :doc:`remote API `, though that could be done at a later point (with a separate password). On the command line, a new sub-command will be added to the ``gnt-node`` script. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-scsi-kvm.rst000064400000000000000000000256351476477700300201210ustar00rootroot00000000000000========== KVM + SCSI ========== .. contents:: :depth: 4 This is a design document detailing the refactoring of device handling in the KVM Hypervisor. More specifically, it will use the latest QEMU device model and modify the hotplug implementation so that both PCI and SCSI devices can be managed. Current state and shortcomings ============================== Ganeti currently supports SCSI virtual devices in the KVM hypervisor by setting the `disk_type` hvparam to `scsi`. Ganeti will eventually instruct QEMU to use the deprecated device model (i.e. -drive if=scsi), which will expose the backing store as an emulated SCSI device. This means that currently SCSI pass-through is not supported. On the other hand, the current hotplug implementation :doc:`design-hotplug` uses the latest QEMU device model (via the -device option) and is tailored to paravirtual devices, which leads to buggy behavior: if we hotplug a disk to an instance that is configured with disk_type=scsi hvparam, the disk which will get hot-plugged eventually will be a VirtIO device (i.e., virtio-blk-pci) on the PCI bus. The current implementation of creating the QEMU command line is error-prone, since an instance might not be able to boot due to PCI slot congestion. Proposed changes ================ We change the way that the KVM hypervisor handles block devices by introducing latest QEMU device model for SCSI devices as well, so that scsi-cd, scsi-hd, scsi-block, and scsi-generic device drivers are supported too. Additionally we refactor the hotplug implementation in order to support hotplugging of SCSI devices too. Finally, we change the way we keep track of device info inside runtime files, and the way we place each device upon instance startup. Design decisions ================ How to identify each device? Currently KVM does not support arbitrary IDs for devices; supported are only names starting with a letter, with max 32 chars length, and only including the '.', '_', '-' special chars. Currently we generate an ID with the following format: --pci-. This assumes that the device will be plugged in a certain slot on the PCI bus. Since we want to support devices on a SCSI bus too and adding the PCI slot to the ID is redundant, we dump the last two parts of the existing ID. Additionally we get rid of the 'hot' prefix of device type, and we add the next two parts of the UUID so the chance of collitions is reduced significantly. So, as an example, the device ID of a disk with UUID '9e7c85f6-b6e5-4243-b27d-680b78c6d203' would be now 'disk-9e7c85f6-b6e5-4243'. Which buses does the guest eventually see? By default QEMU starts with a single PCI bus named "pci.0". In case a SCSI controller is added on this bus, a SCSI bus is created with the corresponding name: "scsi.0". Any SCSI disks will be attached on this SCSI bus. Currently Ganeti does not explicitly use a SCSI controller via a command line option, but lets QEMU add one automatically if needed. Here, in case we have a SCSI disk, a SCSI controller is explicitly added via the -device option. For the SCSI controller, we do not specify the PCI slot to use, but let QEMU find the first available (see below). What type of SCSI controller to use? QEMU uses the `lsi` controller by default. To make this configurable we add a new hvparam, `scsi_controller_type`. The available types will be `lsi`, `megasas`, and `virtio-scsi-pci`. Where to place the devices upon instance startup? The default QEMU machine type, `pc`, adds a `i440FX-pcihost` controller on the root bus that creates a PCI bus with `pci.0` alias. By default the first three slots of this bus are occupied: slot 0 for Host bridge, slot 1 for ISA bridge, and slot 2 for VGA controller. Thereafter, the slots depend on the QEMU options passed in the command line. The main reason that we want to be fully aware of the configuration of a running instance (machine type, PCI and SCSI bus state, devices, etc.) is that in case of migration a QEMU process with the exact same configuration should be created on the target node. The configuration is kept in the runtime file created just before starting the instance. Since hotplug has been introduced, the only thing that can change after starting an instance is the configuration related to NICs and Disks. Before implementing hotplug, Ganeti did not specify PCI slots explicitly, but let QEMU decide how to place the devices on the corresponding bus. This does not work if we want to have hotplug-able devices and migrate-able VMs. Currently, upon runtime file creation, we try to reserve PCI slots based on the hvparams, the disks, and the NICs of the instance. This has three major shortcomings: first, we have to be aware which options modify the PCI bus which is practically impossible due to the huge amount of QEMU options, second, QEMU may change the default PCI configuration from version to version, and third, we cannot know if the extra options passed by the user via the `kvm_extra` hvparam modify the PCI bus. All the above makes the current implementation error prone: an instance might not be able to boot if we explicitly add a NIC/Disk on a specific PCI slot that QEMU has already used for another device while parsing its command line options. Besides that, now, we want to use the SCSI bus as well so the above mechanism is insufficient. Here, we decide to put only disks and NICs on specific slots on the corresponding bus, and let QEMU put everything else automatically. To this end, we decide to let the first 12 PCI slots be managed by QEMU, and we start adding PCI devices (VirtIO block and network devices) from the 13th slot onwards. As far as the SCSI bus is concerned, we decide to put each SCSI disk on a different scsi-id (which corresponds to a different target number in SCSI terminology). The SCSI bus will not have any default reservations. How to support the theoretical maximum of devices, 16 disks and 8 NICs? By default, one could add up to 20 devices on the PCI bus; that is the 32 slots of the PCI bus, minus the starting 12 slots that Ganeti allows QEMU to manage on its own. In order to by able to add more PCI devices, we add the new `kvm_pci_reservations` hvparam to denote how many PCI slots QEMU will handle implicitly. The rest will be available for disk and NICs inserted explicitly by Ganeti. By default the default PCI reservations will be 12 as explained above. How to keep track of the bus state of a running instance? To be able to hotplug a device, we need to know which slot is available on the desired bus. Until now, we were using the ``query-pci`` QMP command that returns the state of the PCI buses (i.e., which devices occupy which slots). Unfortunately, there is no equivalent for the SCSI buses. We could use the ``info qtree`` HMP command that practically dumps in plain text the whole device tree. This makes it really hard to parse. So we decide to generate the bus state of a running instance through our local runtime files. What info should be kept in runtime files? Runtime files are used for instance migration (to run a QEMU process on the target node with the same configuration) and for hotplug actions (to update the configuration of a running instance so that it can be migrated). Until now we were using devices only on the PCI bus, so only each device's PCI slot should be kept in the runtime file. This is obviously not enough. We decide to replace the `pci` slot of Disk and NIC configuration objects, with an `hvinfo` dict. It will contain all necessary info for constructing the appropriate -device QEMU option. Specifically the `driver`, `id`, and `bus` parameters will be present to all kind of devices. PCI devices will have the `addr` parameter, SCSI devices will have `channel`, `scsi-id`, and `lun`. NICs and Disks will have the extra `netdev` and `drive` parameters correspondingly. How to deal with existing instances? Only existing instances with paravirtual devices (configured via the disk_type and nic_type hvparam) use the latest QEMU device model. Only these have the `pci` slot filled. We will use the existing _UpgradeSerializedRuntime() method to migrate the old runtime format with `pci` slot in Disk and NIC configuration objects to the new one with `hvinfo` instead. The new hvinfo will contain the old driver (either virtio-blk-pci or virtio-net-pci), the old id (hotdisk-123456-pci-4), the default PCI bus (pci.0), and the old PCI slot (addr=4). This way those devices will still be hotplug-able, and the instance will still be migrate-able. When those instances are rebooted, the hvinfo will be re-generated. How to support downgrades? There are two possible ways, both not very pretty. The first one is to use _UpgradeSerializedRuntime() to remove the hvinfo slot. This would require the patching of all Ganeti versions down to 2.10 which is practically imposible. Another way is to ssh to all nodes and remove this slot upon a cluster downgrade. This ugly hack would go away on 2.17 since we support downgrades only to the previous minor version. Configuration changes --------------------- The ``NIC`` and ``Disk`` objects get one extra slot: ``hvinfo``. It is hypervisor-specific and will never reach config.data. In case of the KVM Hypervisor it will contain all necessary info for constructing the -device QEMU option. Existing entries in runtime files that had a `pci` slot will be upgraded to have the corresponding `hvinfo` (see above). The new `scsi_controller_type` hvparam is added to denote what type of SCSI controller should be added to PCI bus if we have a SCSI disk. Allowed values will be `lsi`, `virtio-scsi-pci`, and `megasas`. We decide to use `lsi` by default since this is the one that QEMU adds automatically if not specified explicitly by an option. Hypervisor changes ------------------ The current implementation verifies if a hotplug action has succeeded by scanning the PCI bus and searching for a specific device ID. This will change, and we will use the ``query-block`` along with the ``query-pci`` QMP command to find block devices that are attached to the SCSI bus as well. Up until now, if `disk_type` hvparam was set to `scsi`, QEMU would use the deprecated device model and end up using SCSI emulation, e.g.: :: -drive file=/var/run/ganeti/instance-disks/test:0,if=scsi,format=raw Now the equivalent, which will also enable hotplugging, will be to set disk_type to `scsi-hd`. The QEMU command line will include: :: -drive file=/var/run/ganeti/instance-disks/test:0,if=none,format=raw,id=disk-9e7c85f6-b6e5-4243 -device scsi-hd,id=disk-9e7c85f6-b6e5-4243,drive=disk-9e7c85f6-b6e5-4243,bus=scsi.0,channel=0,scsi-id=0,lun=0 User interface -------------- The `disk_type` hvparam will additionally support the `scsi-hd`, `scsi-block`, and `scsi-generic` values. The first one is equivalent to the existing `scsi` value and will make QEMU emulate a SCSI device, while the last two will add support for SCSI pass-through and will require a real SCSI device on the host. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-shared-storage-redundancy.rst000064400000000000000000000110611476477700300234130ustar00rootroot00000000000000================================= N+1 redundancy for shared storage ================================= :Created: 2015-Apr-13 :Status: Partially Implemented :Ganeti-Version: 2.15 .. contents:: :depth: 4 This document describes how N+1 redundancy is achieved for instances using shared storage. Current state and shortcomings ============================== For instances with DRBD as disk template, in case of failures of their primary node, there is only one node where the instance can be restarted immediately. Therefore, ``htools`` reserve enough memory on that node to cope with failure of a single node. For instances using shared storage, however, they can be restarted on any node---implying that on no particular node memory has to be reserved. This, however, motivated the current state where no memory is reserved at all. And even a large cluster can run out of capacity. Proposed changes ================ Definition on N+1 redundancy in the presence of shared storage -------------------------------------------------------------- A cluster is considered N+1 redundant, if, for every node, all DRBD instances can be migrated out and then all shared-storage instances can be relocated to a different node without moving instances on other nodes. This is precisely the operation done after a node breaking. Obviously, simulating failure and evacuation for every single node is an expensive operation. Basic Considerations -------------------- For DRBD, keeping N+1 redundancy is affected by moving instances and balancing the cluster. Moreover, taking is into account for balancing can help :doc:`design-allocation-efficiency`. Hence, N+1 redundancy for DRBD is to be taken into account for all choices affecting instance location, including instance allocation and balancing. For shared-storage instances, they can move everywhere within the node group. So, in practice, this is mainly a question of capacity planing, especially is most instances have the same size. Nevertheless, offcuts if instances don't fill a node entirely may not be ignored. Modifications to existing tools ------------------------------- - ``hail`` will compute and rank possible allocations as usual. However, before returning a choice it will filter out allocations that are not N+1 redundant. - Normal ``gnt-cluster verify`` will not be changed; in particular, it will still check for DRBD N+1 redundancy, but not for shared storage N+1 redundancy. However, ``hcheck`` will verify shared storage N+1 redundancy and report it that fails. - ``hbal`` will consider and rank moves as usual. However, before deciding on the next move, it will filter out those moves that lead from a shared storage N+1 redundant configuration into one that isn't. - ``hspace`` computing the capacity for DRBD instances will be unchanged; In particular, the options ``--accept-exisiting`` and ``--independent-groups`` will continue to work. For shared storage instances, however, will strictly iterate over the same allocation step as hail does. Other modifications related to opportunistic locking ---------------------------------------------------- To allow parallel instance creation, instance creation jobs can be instructed to run with just whatever node locks currently available. In this case, an allocation has to be chosen from that restricted set of nodes. Currently, this is achieved by sending ``hail`` a cluster description where all other nodes are marked offline; that works as long as only local properties are considered. With global properties, however, the capacity of the cluster is materially underestimated, causing spurious global N+1 failures. Therefore, we conservatively extend the request format of ``hail`` by an optional parameter ``restrict-to-nodes``. If that parameter is given, only allocations on those nodes will be considered. This will be an additional restriction to ones currently considered (e.g., node must be online, a particular group might have been requested). If opportunistic locking is enabled, calls to the IAllocator will use this extension to signal which nodes to restrict to, instead of marking other nodes offline. It should be noted that this change brings a race. Two concurrent allocations might bring the cluster over the global N+1 capacity limit. As, however, the reason for opportunistic locking is an urgent need for instances, this seems acceptable; Ganeti generally follows the guideline that current problems are more important than future ones. Also, even with that change allocation is more careful than the current approach of completely ignoring N+1 redundancy for shared storage. ganeti-3.1.0~rc2/doc/design-shared-storage.rst000064400000000000000000000364561476477700300213000ustar00rootroot00000000000000============================= Ganeti shared storage support ============================= :Created: 2011-Mar-01 :Status: Implemented :Ganeti-Version: 2.7.0 This document describes the changes in Ganeti 2.3+ compared to Ganeti 2.3 storage model. It also documents the ExtStorage Interface. .. contents:: :depth: 4 .. highlight:: shell-example Objective ========= The aim is to introduce support for externally mirrored, shared storage. This includes two distinct disk templates: - A shared filesystem containing instance disks as regular files typically residing on a networked or cluster filesystem (e.g. NFS, AFS, Ceph, OCFS2, etc.). - Instance images being shared block devices, typically LUNs residing on a SAN appliance. Background ========== DRBD is currently the only shared storage backend supported by Ganeti. DRBD offers the advantages of high availability while running on commodity hardware at the cost of high network I/O for block-level synchronization between hosts. DRBD's master-slave model has greatly influenced Ganeti's design, primarily by introducing the concept of primary and secondary nodes and thus defining an instance's “mobility domain”. Although DRBD has many advantages, many sites choose to use networked storage appliances for Virtual Machine hosting, such as SAN and/or NAS, which provide shared storage without the administrative overhead of DRBD nor the limitation of a 1:1 master-slave setup. Furthermore, new distributed filesystems such as Ceph are becoming viable alternatives to expensive storage appliances. Support for both modes of operation, i.e. shared block storage and shared file storage backend would make Ganeti a robust choice for high-availability virtualization clusters. Throughout this document, the term “externally mirrored storage” will refer to both modes of shared storage, suggesting that Ganeti does not need to take care about the mirroring process from one host to another. Use cases ========= We consider the following use cases: - A virtualization cluster with FibreChannel shared storage, mapping at least one LUN per instance, accessible by the whole cluster. - A virtualization cluster with instance images stored as files on an NFS server. - A virtualization cluster storing instance images on a Ceph volume. Design Overview =============== The design addresses the following procedures: - Refactoring of all code referring to constants.DTS_NET_MIRROR. - Obsolescence of the primary-secondary concept for externally mirrored storage. - Introduction of a shared file storage disk template for use with networked filesystems. - Introduction of a shared block device disk template with device adoption. - Introduction of the External Storage Interface. Additionally, mid- to long-term goals include: - Support for external “storage pools”. Refactoring of all code referring to constants.DTS_NET_MIRROR ============================================================= Currently, all storage-related decision-making depends on a number of frozensets in lib/constants.py, typically constants.DTS_NET_MIRROR. However, constants.DTS_NET_MIRROR is used to signify two different attributes: - A storage device that is shared - A storage device whose mirroring is supervised by Ganeti We propose the introduction of two new frozensets to ease decision-making: - constants.DTS_EXT_MIRROR, holding externally mirrored disk templates - constants.DTS_MIRRORED, being a union of constants.DTS_EXT_MIRROR and DTS_NET_MIRROR. Additionally, DTS_NET_MIRROR will be renamed to DTS_INT_MIRROR to reflect the status of the storage as internally mirrored by Ganeti. Thus, checks could be grouped into the following categories: - Mobility checks, like whether an instance failover or migration is possible should check against constants.DTS_MIRRORED - Syncing actions should be performed only for templates in constants.DTS_NET_MIRROR Obsolescence of the primary-secondary node model ================================================ The primary-secondary node concept has primarily evolved through the use of DRBD. In a globally shared storage framework without need for external sync (e.g. SAN, NAS, etc.), such a notion does not apply for the following reasons: 1. Access to the storage does not necessarily imply different roles for the nodes (e.g. primary vs secondary). 2. The same storage is available to potentially more than 2 nodes. Thus, an instance backed by a SAN LUN for example may actually migrate to any of the other nodes and not just a pre-designated failover node. The proposed solution is using the iallocator framework for run-time decision making during migration and failover, for nodes with disk templates in constants.DTS_EXT_MIRROR. Modifications to gnt-instance and gnt-node will be required to accept target node and/or iallocator specification for these operations. Modifications of the iallocator protocol will be required to address at least the following needs: - Allocation tools must be able to distinguish between internal and external storage - Migration/failover decisions must take into account shared storage availability Introduction of a shared file disk template =========================================== Basic shared file storage support can be implemented by creating a new disk template based on the existing FileStorage class, with only minor modifications in lib/bdev.py. The shared file disk template relies on a shared filesystem (e.g. NFS, AFS, Ceph, OCFS2 over SAN or DRBD) being mounted on all nodes under the same path, where instance images will be saved. A new cluster initialization option is added to specify the mountpoint of the shared filesystem. The remainder of this document deals with shared block storage. Introduction of a shared block device template ============================================== Basic shared block device support will be implemented with an additional disk template. This disk template will not feature any kind of storage control (provisioning, removal, resizing, etc.), but will instead rely on the adoption of already-existing block devices (e.g. SAN LUNs, NBD devices, remote iSCSI targets, etc.). The shared block device template will make the following assumptions: - The adopted block device has a consistent name across all nodes, enforced e.g. via udev rules. - The device will be available with the same path under all nodes in the node group. Introduction of the External Storage Interface ============================================== Overview -------- To extend the shared block storage template and give Ganeti the ability to control and manipulate external storage (provisioning, removal, growing, etc.) we need a more generic approach. The generic method for supporting external shared storage in Ganeti will be to have an ExtStorage provider for each external shared storage hardware type. The ExtStorage provider will be a set of files (executable scripts and text files), contained inside a directory which will be named after the provider. This directory must be present across all nodes of a nodegroup (Ganeti doesn't replicate it), in order for the provider to be usable by Ganeti for this nodegroup (valid). The external shared storage hardware should also be accessible by all nodes of this nodegroup too. An “ExtStorage provider” will have to provide the following methods: - Create a disk - Remove a disk - Grow a disk - Attach a disk to a given node - Detach a disk from a given node - SetInfo to a disk (add metadata) - Verify its supported parameters - Snapshot a disk (optional) - Open a disk (optional) - Close a disk (optional) The proposed ExtStorage interface borrows heavily from the OS interface and follows a one-script-per-function approach. An ExtStorage provider is expected to provide the following scripts: - ``create`` - ``remove`` - ``grow`` - ``attach`` - ``detach`` - ``setinfo`` - ``verify`` - ``snapshot`` (optional) - ``open`` (optional) - ``close`` (optional) All scripts will be called with no arguments and get their input via environment variables. A common set of variables will be exported for all commands, and some commands might have extra variables. ``VOL_NAME`` The name of the volume. This is unique for Ganeti and it uses it to refer to a specific volume inside the external storage. ``VOL_SIZE`` The volume's size in mebibytes. Available only to the `create` and `grow` scripts. ``VOL_NEW_SIZE`` Available only to the `grow` script. It declares the new size of the volume after grow (in mebibytes). ``EXTP_name`` ExtStorage parameter, where `name` is the parameter in upper-case (same as OS interface's ``OSP_*`` parameters). ``VOL_METADATA`` A string containing metadata to be set for the volume. This is exported only to the ``setinfo`` script. ``VOL_CNAME`` The human readable name of the disk (if any). ``VOL_SNAPSHOT_NAME`` The name of the volume's snapshot. Available only to the `snapshot` script. ``VOL_SNAPSHOT_SIZE`` The size of the volume's snapshot. Available only to the `snapshot` script. ``VOL_OPEN_EXCLUSIVE`` Whether the volume will be accessed exclusively or not. Available only to the `open` script. All scripts except `attach` should return 0 on success and non-zero on error, accompanied by an appropriate error message on stderr. The `attach` script should return a string on stdout on success, which is the block device's full path, after it has been successfully attached to the host node. On error it should return non-zero. The ``snapshot``, ``open`` and ``close`` scripts are introduced after the first implementation of the ExtStorage Interface. To keep backwards compatibility with the first implementation, we make these scripts optional. The ``snapshot`` script, if present, will be used for instance backup export. The ``open`` script makes the device ready for I/O. The ``close`` script disables the I/O on the device. Implementation -------------- To support the ExtStorage interface, we will introduce a new disk template called `ext`. This template will implement the existing Ganeti disk interface in `lib/bdev.py` (create, remove, attach, assemble, shutdown, grow, setinfo, open, close), and will simultaneously pass control to the external scripts to actually handle the above actions. The `ext` disk template will act as a translation layer between the current Ganeti disk interface and the ExtStorage providers. We will also introduce a new IDISK_PARAM called `IDISK_PROVIDER = provider`, which will be used at the command line to select the desired ExtStorage provider. This parameter will be valid only for template `ext` e.g.:: $ gnt-instance add -t ext --disk=0:size=2G,provider=sample_provider1 The Extstorage interface will support different disks to be created by different providers. e.g.:: $ gnt-instance add -t ext --disk=0:size=2G,provider=sample_provider1 \ --disk=1:size=1G,provider=sample_provider2 \ --disk=2:size=3G,provider=sample_provider1 Finally, the ExtStorage interface will support passing of parameters to the ExtStorage provider. This will also be done per disk, from the command line:: $ gnt-instance add -t ext --disk=0:size=1G,provider=sample_provider1,\ param1=value1,param2=value2 The above parameters will be exported to the ExtStorage provider's scripts as the environment variables: - `EXTP_PARAM1 = str(value1)` - `EXTP_PARAM2 = str(value2)` We will also introduce a new Ganeti client called `gnt-storage` which will be used to diagnose ExtStorage providers and show information about them, similarly to the way `gnt-os diagnose` and `gnt-os info` handle OS definitions. ExtStorage Interface support for userspace access ================================================= Overview -------- The ExtStorage Interface gets extended to cater for ExtStorage providers that support userspace access. This will allow the instances to access their external storage devices directly without going through a block device, avoiding expensive context switches with kernel space and the potential for deadlocks in low memory scenarios. The implementation should be backwards compatible and allow existing ExtStorage providers to work as is. Implementation -------------- Since the implementation should be backwards compatible we are not going to add a new script in the set of scripts an ExtStorage provider should ship with. Instead, the 'attach' script, which is currently responsible to map the block device and return a valid device path, should also be responsible for providing the URIs that will be used by each hypervisor. Even though Ganeti currently allows userspace access only for the KVM hypervisor, we want the implementation to enable the extstorage providers to support more than one hypervisors for future compliance. More specifically, the 'attach' script will be allowed to return more than one line. The first line will contain as always the block device path. Each one of the extra lines will contain a URI to be used for the userspace access by a specific hypervisor. Each URI should be prefixed with the hypervisor it corresponds to (e.g. kvm:). The prefix will be case insensitive. If the 'attach' script doesn't return any extra lines, we assume that the ExtStorage provider doesn't support userspace access (this way we maintain backward compatibility with the existing 'attach' scripts). In case the provider supports *only* userspace access and thus a local block device is not available, then the first line should be an empty line. The 'GetUserspaceAccessUri' method of the 'ExtStorageDevice' class will parse the output of the 'attach' script and if the provider supports userspace access for the requested hypervisor, it will use the corresponding URI instead of the block device itself. Long-term shared storage goals ============================== Storage pool handling --------------------- A new cluster configuration attribute will be introduced, named “storage_pools”, modeled as a dictionary mapping storage pools to external storage providers (see below), e.g.:: { "nas1": "foostore", "nas2": "foostore", "cloud1": "barcloud", } Ganeti will not interpret the contents of this dictionary, although it will provide methods for manipulating them under some basic constraints (pool identifier uniqueness, driver existence). The manipulation of storage pools will be performed by implementing new options to the `gnt-cluster` command:: $ gnt-cluster modify --add-pool nas1 foostore $ gnt-cluster modify --remove-pool nas1 # There must be no instances using # the pool to remove it Furthermore, the storage pools will be used to indicate the availability of storage pools to different node groups, thus specifying the instances' “mobility domain”. The pool, in which to put the new instance's disk, will be defined at the command line during `instance add`. This will become possible by replacing the IDISK_PROVIDER parameter with a new one, called `IDISK_POOL = pool`. The cmdlib logic will then look at the cluster-level mapping dictionary to determine the ExtStorage provider for the given pool. gnt-storage ----------- The ``gnt-storage`` client can be extended to support pool management (creation/modification/deletion of pools, connection/disconnection of pools to nodegroups, etc.). It can also be extended to diagnose and provide information for internal disk templates too, such as lvm and drbd. .. vim: set textwidth=72 : ganeti-3.1.0~rc2/doc/design-ssh-ports.rst000064400000000000000000000042411476477700300203150ustar00rootroot00000000000000================================================ Design for supporting custom SSH ports for nodes ================================================ :Created: 2013-Nov-08 :Status: Implemented :Ganeti-Version: 2.11.0 .. contents:: :depth: 4 This design document describes the intention of supporting running SSH servers on nodes with non-standard port numbers. Current state and shortcomings ============================== All SSH daemons are expected to be running on the default port 22. It has been requested by Ganeti users (`Issue 291`_) to allow SSH daemons run on non-standard ports as well. .. _`Issue 291`: https://github.com/ganeti/ganeti/issues/291 Proposed Changes ================ Allow users to configure groups with custom SSH ports. All nodes in such a group will then be using its configured SSH port. The configuration will be on the group level only as we expect all nodes in a group to have identical configurations. Users will be responsible for configuring the SSH daemons on machines before adding them as nodes to a group with a non-standard port number, or when modifying the port number of an existing group. Ganeti will not update SSH configuration by itself. Implementation Details ====================== We must ensure that all operations that use SSH will use custom ports as configured. This includes: - gnt-cluster verify - gnt-cluster renew-crypto - gnt-cluster upgrade - gnt-node add - gnt-instance console Configuration Changes ~~~~~~~~~~~~~~~~~~~~~ The node group *ndparams* will get an additional integer valued parameter *ssh_port*. Upgrades/downgrades ~~~~~~~~~~~~~~~~~~~ To/from version 2.10 -------------------- During upgrade from 2.10, the default value 22 will be supplemented. During downgrade to 2.10 the downgrading script will check that there are no configured ports other than 22 (because this would result in a broken cluster) and then will remove the corresponding key/value pairs from the configuration. Future versions --------------- For future versions the up/downgrade operation will need to know the configured SSH ports. Because all daemons are stopped during the process, it will be necessary to include SSH ports in *ssconf*. ganeti-3.1.0~rc2/doc/design-storagetypes.rst000064400000000000000000000367061476477700300211170ustar00rootroot00000000000000============================================================================= Management of storage types and disk templates, incl. storage space reporting ============================================================================= :Created: 2013-Feb-15 :Status: Implemented :Ganeti-Version: 2.8.0, 2.9.0, 2.10.0 .. contents:: :depth: 4 Background ========== Currently, there is no consistent management of different variants of storage in Ganeti. One direct consequence is that storage space reporting is currently broken for all storage that is not based on lvm technology. This design looks at the root causes and proposes a way to fix it. Proposed changes ================ We propose to streamline handling of different storage types and disk templates. Currently, there is no consistent implementation for dis/enabling of disk templates and/or storage types. Our idea is to introduce a list of enabled disk templates, which can be used by instances in the cluster. Based on this list, we want to provide storage reporting mechanisms for the available disk templates. Since some disk templates share the same underlying storage technology (for example ``drbd`` and ``plain`` are based on ``lvm``), we map disk templates to storage types and implement storage space reporting for each storage type. Configuration changes --------------------- Add a new attribute "enabled_disk_templates" (type: list of strings) to the cluster config which holds disk templates, for example, "drbd", "file", or "ext". This attribute represents the list of disk templates that are enabled cluster-wide for usage by the instances. It will not be possible to create instances with a disk template that is not enabled, as well as it will not be possible to remove a disk template from the list if there are still instances using it. The list of enabled disk templates can contain any non-empty subset of the currently implemented disk templates: ``blockdev``, ``diskless``, ``drbd``, ``ext``, ``file``, ``plain``, ``rbd``, and ``sharedfile``. See ``DISK_TEMPLATES`` in ``constants.py``. Note that the above-mentioned list of enabled disk types is just a "mechanism" parameter that defines which disk templates the cluster can use. Further filtering about what's allowed can go in the ipolicy, which is not covered in this design doc. Note that it is possible to force an instance to use a disk template that is not allowed by the ipolicy. This is not possible if the template is not enabled by the cluster. The ipolicy also contains a list of enabled disk templates. Since the cluster- wide enabled disk templates should be a stronger constraint, the list of enabled disk templates in the ipolicy should be a subset of those. In case the user tries to create an inconsistent situation here, gnt-cluster should display this as an error. We consider the first disk template in the list to be the default template for instance creation and storage reporting. This will remove the need to specify the disk template with ``-t`` on instance creation. Note: It would be better to take the default disk template from the node-group-specific ipolicy. However, when using the iallocator, the nodegroup can only be determined from the node which is determined by the iallocator, which in turn needs the disk-template first. To solve this chicken-and-egg-problem we first need to extend 'gnt-instance add' to accept a nodegroup in the first place. Currently, cluster-wide dis/enabling of disk templates is not implemented consistently. ``lvm`` based disk templates are enabled by specifying a volume group name on cluster initialization and can only be disabled by explicitly using the option ``--no-lvm-storage``. This will be replaced by adding/removing ``drbd`` and ``plain`` from the set of enabled disk templates. The option ``--no-drbd-storage`` is also subsumed by dis/enabling the disk template ``drbd`` on the cluster. Up till now, file storage and shared file storage could be dis/enabled at ``./configure`` time. This will also be replaced by adding/removing the respective disk templates from the set of enabled disk templates. There is currently no possibility to dis/enable the disk templates ``diskless``, ``blockdev``, ``ext``, and ``rdb``. By introducing the set of enabled disk templates, we will require these disk templates to be explicitly enabled in order to be used. The idea is that the administrator of the cluster can tailor the cluster configuration to what is actually needed in the cluster. There is hope that this will lead to cleaner code, better performance and fewer bugs. When upgrading the configuration from a version that did not have the list of enabled disk templates, we have to decide which disk templates are enabled based on the current configuration of the cluster. We propose the following update logic to be implemented in the online update of the config in the ``Cluster`` class in ``objects.py``: - If a ``volume_group_name`` is existing, then enable ``drbd`` and ``plain``. - If ``file`` or ``sharedfile`` was enabled at configure time, add the respective disk template to the list of enabled disk templates. - For disk templates ``diskless``, ``blockdev``, ``ext``, and ``rbd``, we inspect the current cluster configuration regarding whether or not there are instances that use one of those disk templates. We will add only those that are currently in use. The order in which the list of enabled disk templates is built up will be determined by a preference order based on when in the history of Ganeti the disk templates were introduced (thus being a heuristic for which are used more than others). The list of enabled disk templates can be specified on cluster initialization with ``gnt-cluster init`` using the optional parameter ``--enabled-disk-templates``. If it is not set, it will be set to a default set of enabled disk templates, which includes the following disk templates: ``drbd`` and ``plain``. The list can be shrunk or extended by ``gnt-cluster modify`` using the same parameter. Storage reporting ----------------- The storage reporting in ``gnt-node list`` will be the first user of the newly introduced list of enabled disk templates. Currently, storage reporting works only for lvm-based storage. We want to extend that and report storage for the enabled disk templates. The default of ``gnt-node list`` will only report on storage of the default disk template (the first in the list of enabled disk templates). One can explicitly ask for storage reporting on the other enabled disk templates with the ``-o`` option. Some of the currently implemented disk templates share the same base storage technology. Since the storage reporting is based on the underlying technology rather than on the user-facing disk templates, we introduce storage types to represent the underlying technology. There will be a mapping from disk templates to storage types, which will be used by the storage reporting backend to pick the right method for estimating the storage for the different disk templates. The proposed storage types are ``blockdev``, ``diskless``, ``ext``, ``file``, ``lvm-pv``, ``lvm-vg``, ``rados``. The mapping from disk templates to storage types will be: ``drbd`` and ``plain`` to ``lvm-vg``, ``file`` and ``sharedfile`` to ``file``, and all others to their obvious counterparts. Note that there is no disk template mapping to ``lvm-pv``, because this storage type is currently only used to enable the user to mark it as (un)allocatable. (See ``man gnt-node``.) It is not possible to create an instance on a storage unit that is of type ``lvm-pv`` directly, therefore it is not included in the mapping. The storage reporting for file and sharedfile storage will report space on the file storage dir, which is currently limited to one directory. In the future, if we'll have support for more directories, or for per-nodegroup directories this can be changed. For now, we will implement only the storage reporting for lvm-based and file-based storage, that is disk templates ``file``, ``sharedfile``, ``lvm``, and ``drbd``. For disk template ``diskless``, there is obviously nothing to report about. When implementing storage reporting for file, we can also use it for ``sharedfile``, since it uses the same file system mechanisms to determine the free space. In the future, we can optimize storage reporting for shared storage by not querying all nodes that use a common shared file for the same space information. In the future, we extend storage reporting for shared storage types like ``rados`` and ``ext``. Note that it will not make sense to query each node for storage reporting on a storage unit that is used by several nodes. We will not implement storage reporting for the ``blockdev`` disk template, because block devices are always adopted after being provided by the system administrator, thus coming from outside Ganeti. There is no point in storage reporting for block devices, because Ganeti will never try to allocate storage inside a block device. RPC changes ----------- The noded RPC call that reports node storage space will be changed to accept a list of , string tuples. For each of them, it will report the free amount of storage space found on storage as known by the requested storage_type. Depending on the storage_type, the key would be a volume group name in case of lvm, a directory name for the file-based storage, and a rados pool name for rados storage. Masterd will know through the mapping of storage types to storage calculation functions which storage type uses which mechanism for storage calculation and invoke only the needed ones. Note that for file and sharedfile the node knows which directories are allowed and won't allow any other directory to be queried for security reasons. The actual path still needs to be passed to distinguish the two, as the type will be the same for both. These calculations will be implemented in the node storage system (currently lib/storage.py) but querying will still happen through the ``node info`` call, to avoid requiring an extra RPC each time. Ganeti reporting ---------------- `gnt-node list`` can be queried for the different disk templates, if they are enabled. By default, it will just report information about the default disk template. Examples:: > gnt-node list Node DTotal DFree MTotal MNode MFree Pinst Sinst mynode1 3.6T 3.6T 64.0G 1023M 62.2G 1 0 mynode2 3.6T 3.6T 64.0G 1023M 62.0G 2 1 mynode3 3.6T 3.6T 64.0G 1023M 62.3G 0 2 > gnt-node list -o dtotal/drbd,dfree/file Node DTotal (drbd, myvg) DFree (file, mydir) mynode1 3.6T - mynode2 3.6T - Note that for drbd, we only report the space of the vg and only if it was not renamed to something different than the default volume group name. With this design, there is also no possibility to ask about the meta volume group. We restrict the design here to make the transition to storage pools easier (as it is an interim state only). It is the administrator's responsibility to ensure that there is enough space for the meta volume group. When storage pools are implemented, we switch from referencing the disk template to referencing the storage pool name. For that, of course, the pool names need to be unique over all storage types. For drbd, we will use the default 'drbd' storage pool and possibly a second lvm-based storage pool for the metavg. It will be possible to rename storage pools (thus also the default lvm storage pool). There will be new functionality to ask about what storage pools are available and of what type. Storage pools will have a storage pool type which is one of the disk templates. There can be more than one storage pool based on the same disk template, therefore we will then start referencing the storage pool name instead of the disk template. Note: As of version 2.10, ``gnt-node list`` only reports storage space information for the default disk template, as supporting more options turned out to be not feasible without storage pools. Besides in ``gnt-node list``, storage space information is also displayed in ``gnt-node list-storage``. This will also adapt to the extended storage reporting capabilities. The user can specify a storage type using ``--storage-type``. If he requests storage information about a storage type which does not support space reporting, a warning is emitted. If no storage type is specified explicitly, ``gnt-node list-storage`` will try to report storage on the storage type of the default disk template. If the default disk template's storage type does not support space reporting, an error message is emitted. ``gnt-cluster info`` will report which disk templates are enabled, i.e. which ones are supported according to the cluster configuration. Example output:: > gnt-cluster info [...] Cluster parameters: - [...] - enabled disk templates: plain, drbd, sharedfile, rados - [...] ``gnt-node list-storage`` will not be affected by any changes, since this design is restricted only to free storage reporting for non-shared storage types. Allocator changes ----------------- The iallocator protocol doesn't need to change: since we know which disk template an instance has, we'll pass only the "free" value for that disk template to the iallocator, when asking for an allocation to be made. Note that for DRBD nowadays we ignore the case when vg and metavg are different, and we only consider the main volume group. Fixing this is outside the scope of this design. Although the iallocator protocol itself does not need change, the invocation of the iallocator needs quite some adaption. So far, it always requested LVM storage information no matter if that was the disk template to be considered for the allocation. For instance allocation, this is the disk template of the instance. TODO: consider other allocator requests. With this design, we ensure forward-compatibility with respect to storage pools. For now, we'll report space for all available disk templates that are based on non-shared storage types, in the future, for all available storage pools. Rebalancing changes ------------------- Hbal will not need changes, as it handles it already. We don't forecast any changes needed to it. Space reporting changes ----------------------- Hspace will by default report by assuming the allocation will happen on the default disk template for the cluster/nodegroup. An option will be added to manually specify a different storage. Interactions with Partitioned Ganeti ------------------------------------ Also the design for :doc:`Partitioned Ganeti ` deals with reporting free space. Partitioned Ganeti has a different way to report free space for LVM on nodes where the ``exclusive_storage`` flag is set. That doesn't interact directly with this design, as the specifics of how the free space is computed is not in the scope of this design. But the ``node info`` call contains the value of the ``exclusive_storage`` flag, which is currently only meaningful for the LVM storage type. Additional flags like the ``exclusive_storage`` flag for lvm might be useful for other disk templates / storage types as well. We therefore extend the RPC call with , to ,,[] to include any disk-template-specific (or storage-type specific) parameters in the RPC call. The reporting of free spindles, also part of Partitioned Ganeti, is not concerned with this design doc, as those are seen as a separate resource. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-sync-rate-throttling.rst000064400000000000000000000036741476477700300224650ustar00rootroot00000000000000========================= DRBD Sync Rate Throttling ========================= :Created: 2014-Sep-16 :Status: Draft Objective --------- This document outlines the functionality to conveniently set rate limits for synchronization tasks. A use-case of this is that moving instances might otherwise clog the network for the nodes. If the replication network differs from the network used by the instances, there would be no benefits. Namely there should be two limits that can be set: * `resync-rate`: which should not be exceeded for each device. This exists already. * `total-resync-rate`: which should not be exceeded collectively for each node. Configuration ------------- Suggested command line parameters for controlling throttling are as follows:: gnt-cluster modify -D resync-rate= gnt-cluster modify -D total-resync-rate= Where ``bytes-per-second`` can be in the format ``{b,k,M,G}`` to set the limit to N bytes, kilobytes, megabytes, and gigabytes respectively. Semantics --------- The rate limit that is set for the drbdsetup is at least rate = min(resync-rate, total-resync-rate/number-of-syncing-devices) where number-of-syncing-devices is checked on beginning and end of syncs. This is set on each node with drbdsetup disk-options --resync-rate --c-max-rate Later versions might free additional bandwidth on the source/target if the target/source is more throttled. Architecture ------------ The code to adjust the sync rates is collected in a separate tool ``hrates`` that #. is run when a new sync is started or finished. #. can be run manually if necessary. Since the rates don't depend on the job, an unparameterized RPC ``perspective_node_run_hrates`` to NodeD will trigger the execution of the tool. A first version will query ConfD for the other nodes of the group and request the sync state for all of them. .. TODO: second version that avoids overhead. ganeti-3.1.0~rc2/doc/design-systemd.rst000064400000000000000000000236161476477700300200520ustar00rootroot00000000000000=================== Systemd integration =================== :Created: 2014-Mar-26 :Status: Implemented :Ganeti-Version: 2.12.0 .. contents:: :depth: 4 This design document outlines the implementation of native systemd support in Ganeti by providing systemd unit files. It also briefly discusses the possibility of supporting socket activation. Current state and shortcomings ============================== Ganeti currently ships an example init script, compatible with Debian (and derivatives) and RedHat/Fedora (and derivatives). The initscript treats the whole Ganeti system as a single service wrt. starting and stopping (but allows starting/stopping/restarting individual daemons). The initscript is aided by ``daemon-util``, which takes care of correctly ordering the startup/shutdown of daemons using an explicit order. Finally, process supervision is achieved by (optionally) running ``ganeti-watcher`` via cron every 5 minutes. ``ganeti-watcher`` will - among other things - try to start services that should be running but are not. The example initscript currently shipped with Ganeti will work with systemd's LSB compatibility wrappers out of the box, however there are a number of areas where we can benefit from providing native systemd unit files: - systemd is the `de-facto choice`_ of almost all major Linux distributions. Since it offers a stable API for service control, providing our own systemd unit files means that Ganeti will run out-of-the-box and in a predictable way in all distributions using systemd. - systemd performs constant process supervision with immediate service restarts and configurable back-off. Ganeti currently offers supervision only via ganeti-watcher, running via cron in 5-minute intervals and unconditionally starting missing daemons even if they have been manually stopped. - systemd offers `socket activation`_ support, which may be of interest for use at least with masterd, luxid and noded. Socket activation offers two main advantages: no explicit service dependencies or ordering needs to be defined as services will be activated when needed; and seamless restarts / upgrades are possible without rejecting new client connections. - systemd offers a number of `security features`_, primarily using the Linux kernel's namespace support, which may be of interest to better restrict daemons running as root (noded and mond). .. _de-facto choice: https://en.wikipedia.org/wiki/Systemd#Adoption .. _socket activation: http://0pointer.de/blog/projects/socket-activation.html .. _security features: http://0pointer.de/blog/projects/security.html Proposed changes ================ We propose to extend Ganeti to natively support systemd, in addition to shipping the init-script as is. This requires the addition of systemd unit files, as well as some changes in daemon-util and ganeti-watcher to use ``systemctl`` on systems where Ganeti is managed by systemd. systemd unit files ------------------ Systemd uses unit files to store information about a service, device, mount point, or other resource it controls. Each unit file contains exactly one unit definition, consisting of a ``Unit`` an (optional) ``Install`` section and an (optional) type-specific section (e.g. ``Service``). Unit files are dropped in pre-determined locations in the system, where systemd is configured to read them from. Systemd allows complete or partial overrides of the unit files, using overlay directories. For more information, see `systemd.unit(5)`_. .. _systemd.unit(5): http://www.freedesktop.org/software/systemd/man/systemd.unit.html We will create one systemd `service unit`_ per daemon (masterd, noded, mond, luxid, confd, rapi) and an additional oneshot service for ensure-dirs (``ganeti-common.service``). All services will ``Require`` ``ganeti-common.service``, which will thus run exactly once per transaction (regardless of starting one or all daemons). .. _service unit: http://www.freedesktop.org/software/systemd/man/systemd.service.html All daemons will run in the foreground (already implemented by the ``-f`` flag), directly supervised by systemd, using ``Restart=on-failure`` in the respective units. Master role units will also treat ``EXIT_NOTMASTER`` as a successful exit and not trigger restarts. Additionally, systemd's conditional directives will be used to avoid starting daemons when they will certainly fail (e.g. because of missing configuration). Apart from the individual daemon units, we will also provide three `target units`_ as synchronization points: - ``ganeti-node.target``: Regular node/master candidate functionality, including ``ganeti-noded.service``, ``ganeti-mond.service`` and ``ganeti-confd.service``. - ``ganeti-master.target``: Master node functionality, including ``ganeti-masterd.service``, ``ganeti-luxid.service`` and ``ganeti-rapi.service``. - ``ganeti.target``: A "meta-target" depending on ``ganeti-node.target`` and ``ganti-master.target``. ``ganeti.target`` itself will be ``WantedBy`` ``multi-user.target``, so that Ganeti starts automatically on boot. .. _target units: http://www.freedesktop.org/software/systemd/man/systemd.target.html To allow starting/stopping/restarting the different roles, all units will include a ``PartOf`` directive referencing their direct ancestor target. In this way ``systemctl restart ganeti-node.target`` or ``systemctl restart ganeti.target`` will work as expected, i.e. restart only the node daemons or all daemons respectively. The full dependency tree is as follows: :: ganeti.target ├─ganeti-master.target │ ├─ganeti-luxid.service │ │ └─ganeti-common.service │ ├─ganeti-masterd.service │ │ └─ganeti-common.service │ └─ganeti-rapi.service │ └─ganeti-common.service └─ganeti-node.target ├─ganeti-confd.service │ └─ganeti-common.service ├─ganeti-mond.service │ └─ganeti-common.service └─ganeti-noded.service └─ganeti-common.service Installation ~~~~~~~~~~~~ The systemd unit files will be built from templates under doc/examples/systemd, much like what is currently done for the initscript. They will not be installed with ``make install``, but left up to the distribution packagers to ship them at the appropriate locations. SysV compatibility ~~~~~~~~~~~~~~~~~~ Systemd automatically creates a service for each SysV initscript on the system, appending ``.service`` to the initscript name, except if a service with the given name already exists. In our case however, the initscript's functionality is implemented by ``ganeti.target``. Systemd provides the ability to *mask* a given service, rendering it unusable, but in the case of SysV services this also results in failure to use tools like ``invoke-rc.d`` or ``service``. Thus we have to ship a ``ganeti.service`` (calling ``/bin/true``) of type ``oneshot``, that depends on ``ganeti.target`` for these tools to continue working as expected. ``ganeti.target`` on the other hand will be marked as ``PartOf = ganeti.service`` for stop and restart to be propagated to the whole service. The ``ganeti.service`` unit will not be marked to be enabled by systemd (i.e. will not be started at boot), but will be available for manual invocation and only be used for compatibility purposes. Changes to daemon-util ---------------------- ``daemon-util`` is used wherever daemon control is required: - In the sample initscript, to start and stop all daemons. - In ``ganeti.backend`` to start the master daemons on master failover and to stop confd when leaving the cluster. - In ``ganeti.bootstrap``, to start the daemons on cluster initialization. - In ``ganeti.cli``, to control the daemon run state during certain operations (e.g. renew-crypto). Currently, ``daemon-util`` uses two auxiliary tools for managing daemons ``start-stop-daemon`` and ``daemon``, in this order of preference. In order not to confuse systemd in its process supervision, ``daemon-util`` will have to be modified to start and stop the daemons via ``systemctl`` in preference to ``start-stop-daemon`` and ``daemon``. This will require a basic check against run-time environment integrity: - Make sure that ``systemd`` runs as PID 1, which is a `simple check`_ against the existence of ``/run/systemd/system``. - Make sure ``systemd`` knows how to handle Ganeti natively. This can be a check against the ``LoadState`` of the ``ganeti.target`` unit. Unless both of these checks pass, ``daemon-util`` will fall back to its current behavior. .. _simple check: http://www.freedesktop.org/software/systemd/man/sd_booted.html Changes to ganeti-watcher ------------------------- Since the daemon process supervision will be systemd's responsibility, the watcher must detect systemd's presence and not attempt to start any missing services. Again, systemd can be detected by the existence of ``/run/systemd/system``. Future work =========== Socket activation ----------------- Systemd offers support for `socket activation`_. A daemon supporting socket-based activation, can inherit its listening socket(s) by systemd. This in turn means that the socket can be created and bound by systemd during early boot and it can be used to provide implicit startup ordering; as soon as a client connects to the listening socket, the respective service (and all its dependencies) will be started and the client will wait until its connection is accepted. Also, because the socket remains bound even if the service is restarting, new client connections will never be rejected, making service restarts and upgrades seamless. Socket activation support is trivial to implement (see `sd_listen_fds(3)`_) and relies on information passed by systemd via environment variables to the started processes. .. _sd_listen_fds(3): http://www.freedesktop.org/software/systemd/man/sd_listen_fds.html .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-upgrade.rst000064400000000000000000000301561476477700300200060ustar00rootroot00000000000000======================================== Automatized Upgrade Procedure for Ganeti ======================================== :Created: 2013-Aug-20 :Status: Implemented :Ganeti-Version: 2.10.0 .. contents:: :depth: 4 This is a design document detailing the proposed changes to the upgrade process, in order to allow it to be more automatic. Current state and shortcomings ============================== Ganeti requires to run the same version of Ganeti to be run on all nodes of a cluster and this requirement is unlikely to go away in the foreseeable future. Also, the configuration may change between minor versions (and in the past has proven to do so). This requires a quite involved manual upgrade process of draining the queue, stopping ganeti, changing the binaries, upgrading the configuration, starting ganeti, distributing the configuration, and undraining the queue. Proposed changes ================ While we will not remove the requirement of the same Ganeti version running on all nodes, the transition from one version to the other will be made more automatic. It will be possible to install new binaries ahead of time, and the actual switch between versions will be a single command. While changing the file layout anyway, we install the python code, which is architecture independent, under ``${prefix}/share``, in a way that properly separates the Ganeti libraries of the various versions. Path changes to allow multiple versions installed ------------------------------------------------- Currently, Ganeti installs to ``${PREFIX}/bin``, ``${PREFIX}/sbin``, and so on, as well as to ``${pythondir}/ganeti``. These paths will be changed in the following way. - The python package will be installed to ``${PREFIX}/share/ganeti/${VERSION}/ganeti``. Here ${VERSION} is, depending on configure options, either the full qualified version number, consisting of major, minor, revision, and suffix, or it is just a major.minor pair. All python executables will be installed under ``${PREFIX}/share/ganeti/${VERSION}`` so that they see their respective Ganeti library. ``${PREFIX}/share/ganeti/default`` is a symbolic link to ``${sysconfdir}/ganeti/share`` which, in turn, is a symbolic link to ``${PREFIX}/share/ganeti/${VERSION}``. For all python executables (like ``gnt-cluster``, ``gnt-node``, etc) symbolic links going through ``${PREFIX}/share/ganeti/default`` are added under ``${PREFIX}/sbin``. - All other files will be installed to the corresponding path under ``${libdir}/ganeti/${VERSION}`` instead of under ``${PREFIX}`` directly, where ``${libdir}`` defaults to ``${PREFIX}/lib``. ``${libdir}/ganeti/default`` will be a symlink to ``${sysconfdir}/ganeti/lib`` which, in turn, is a symlink to ``${libdir}/ganeti/${VERSION}``. Symbolic links to the files installed under ``${libdir}/ganeti/${VERSION}`` will be added under ``${PREFIX}/bin``, ``${PREFIX}/sbin``, and so on. These symbolic links will go through ``${libdir}/ganeti/default`` so that the version can easily be changed by updating the symbolic link in ``${sysconfdir}``. The set of links for ganeti binaries might change between the versions. However, as the file structure under ``${libdir}/ganeti/${VERSION}`` reflects that of ``/``, two links of different versions will never conflict. Similarly, the symbolic links for the python executables will never conflict, as they always point to a file with the same basename directly under ``${PREFIX}/share/ganeti/default``. Therefore, each version will make sure that enough symbolic links are present in ``${PREFIX}/bin``, ``${PREFIX}/sbin`` and so on, even though some might be dangling, if a different version of ganeti is currently active. The extra indirection through ``${sysconfdir}`` allows installations that choose to have ``${sysconfdir}`` and ``${localstatedir}`` outside ``${PREFIX}`` to mount ``${PREFIX}`` read-only. The latter is important for systems that choose ``/usr`` as ``${PREFIX}`` and are following the Filesystem Hierarchy Standard. For example, choosing ``/usr`` as ``${PREFIX}`` and ``/etc`` as ``${sysconfdir}``, the layout for version 2.10 will look as follows. :: / | +-- etc | | | +-- ganeti | | | +-- lib -> /usr/lib/ganeti/2.10 | | | +-- share -> /usr/share/ganeti/2.10 +-- usr | +-- bin | | | +-- harep -> /usr/lib/ganeti/default/usr/bin/harep | | | ... | +-- sbin | | | +-- gnt-cluster -> /usr/share/ganeti/default/gnt-cluster | | | ... | +-- ... | +-- lib | | | +-- ganeti | | | +-- default -> /etc/ganeti/lib | | | +-- 2.10 | | | +-- usr | | | +-- bin | | | | | +-- htools | | | | | +-- harep -> htools | | | | | ... | ... | +-- share | +-- ganeti | +-- default -> /etc/ganeti/share | +-- 2.10 | + -- gnt-cluster | + -- gnt-node | + -- ... | + -- ganeti | +-- backend.py | +-- ... | +-- cmdlib | | | ... ... gnt-cluster upgrade ------------------- The actual upgrade process will be done by a new command ``upgrade`` to ``gnt-cluster``. If called with the option ``--to`` which take precisely one argument, the version to upgrade (or downgrade) to, given as full string with major, minor, revision, and suffix. To be compatible with current configuration upgrade and downgrade procedures, the new version must be of the same major version and either an equal or higher minor version, or precisely the previous minor version. When executed, ``gnt-cluster upgrade --to=`` will perform the following actions. - It verifies that the version to change to is installed on all nodes of the cluster that are not marked as offline. If this is not the case it aborts with an error. This initial testing is an optimization to allow for early feedback. - An intent-to-upgrade file is created that contains the current version of ganeti, the version to change to, and the process ID of the ``gnt-cluster upgrade`` process. The latter is not used automatically, but allows manual detection if the upgrade process died unintentionally. The intend-to-upgrade file is persisted to disk before continuing. - The Ganeti job queue is drained, and the executable waits till there are no more jobs in the queue. Once :doc:`design-optables` is implemented, for upgrades, and only for upgrades, all jobs are paused instead (in the sense that the currently running opcode continues, but the next opcode is not started) and it is continued once all jobs are fully paused. - All ganeti daemons on the master node are stopped. - It is verified again that all nodes at this moment not marked as offline have the new version installed. If this is not the case, then all changes so far (stopping ganeti daemons and draining the queue) are undone and failure is reported. This second verification is necessary, as the set of online nodes might have changed during the draining period. - All ganeti daemons on all remaining (non-offline) nodes are stopped. - A backup of all Ganeti-related status information is created for manual rollbacks. While the normal way of rolling back after an upgrade should be calling ``gnt-cluster upgrade`` from the newer version with the older version as argument, a full backup provides an additional safety net, especially for jump-upgrades (skipping intermediate minor versions). - If the action is a downgrade to the previous minor version, the configuration is downgraded now, using ``cfgupgrade --downgrade``. - If the action is downgrade, any version-specific additional downgrade actions are carried out. - The ``${sysconfdir}/ganeti/lib`` and ``${sysconfdir}/ganeti/share`` symbolic links are updated. - If the action is an upgrade to a higher minor version, the configuration is upgraded now, using ``cfgupgrade``. - ``ensure-dirs --full-run`` is run on all nodes. - All daemons are started on all nodes. - ``gnt-cluster redist-conf`` is run on the master node. - All daemons are restarted on all nodes. - The Ganeti job queue is undrained. - The intent-to-upgrade file is removed. - ``post-upgrade`` is run with the original version as argument. - ``gnt-cluster verify`` is run and the result reported. Considerations on unintended reboots of the master node ======================================================= During the upgrade procedure, the only ganeti process still running is the one instance of ``gnt-cluster upgrade``. This process is also responsible for eventually removing the queue drain. Therefore, we have to provide means to resume this process, if it dies unintentionally. The process itself will handle SIGTERM gracefully by either undoing all changes done so far, or by ignoring the signal all together and continuing to the end; the choice between these behaviors depends on whether change of the configuration has already started (in which case it goes through to the end), or not (in which case the actions done so far are rolled back). To achieve this, ``gnt-cluster upgrade`` will support a ``--resume`` option. It is recommended to have ``gnt-cluster upgrade --resume`` as an at-reboot task in the crontab. The ``gnt-cluster upgrade --resume`` command first verifies that it is running on the master node, using the same requirement as for starting the master daemon, i.e., confirmed by a majority of all nodes. If it is not the master node, it will remove any possibly existing intend-to-upgrade file and exit. If it is running on the master node, it will check for the existence of an intend-to-upgrade file. If no such file is found, it will simply exit. If found, it will resume at the appropriate stage. - If the configuration file still is at the initial version, ``gnt-cluster upgrade`` is resumed at the step immediately following the writing of the intend-to-upgrade file. It should be noted that all steps before changing the configuration are idempotent, so redoing them does not do any harm. - If the configuration is already at the new version, all daemons on all nodes are stopped (as they might have been started again due to a reboot) and then it is resumed at the step immediately following the configuration change. All actions following the configuration change can be repeated without bringing the cluster into a worse state. Caveats ======= Since ``gnt-cluster upgrade`` drains the queue and undrains it later, so any information about a previous drain gets lost. This problem will disappear, once :doc:`design-optables` is implemented, as then the undrain will then be restricted to filters by gnt-upgrade. Requirement of job queue update =============================== Since for upgrades we only pause jobs and do not fully drain the queue, we need to be able to transform the job queue into a queue for the new version. The preferred way to obtain this is to keep the serialization format backwards compatible, i.e., only adding new opcodes and new optional fields. However, even with soft drain, no job is running at the moment `cfgupgrade` is running. So, if we change the queue representation, including the representation of individual opcodes in any way, `cfgupgrade` will also modify the queue accordingly. In a jobs-as-processes world, pausing a job will be implemented in such a way that the corresponding process stops after finishing the current opcode, and a new process is created if and when the job is unpaused again. ganeti-3.1.0~rc2/doc/design-virtual-clusters.rst000064400000000000000000000303421476477700300217040ustar00rootroot00000000000000=================================== Design for virtual clusters support =================================== :Created: 2011-Oct-14 :Status: Partial Implementation :Ganeti-Version: 2.7.0 Introduction ============ Currently there are two ways to test the Ganeti (including HTools) code base: - unittests, which run using mocks as normal user and test small bits of the code - QA/burnin/live-test, which require actual hardware (either physical or virtual) and will build an actual cluster, with one machine to one node correspondence The difference in time between these two is significant: - the unittests run in about 1-2 minutes - a so-called ‘quick’ QA (without burnin) runs in about an hour, and a full QA could be double that time On one hand, the unittests have a clear advantage: quick to run, not requiring many machines, but on the other hand QA is actually able to run end-to-end tests (including HTools, for example). Ideally, we would have an intermediate step between these two extremes: be able to test most, if not all, of Ganeti's functionality but without requiring actual hardware, full machine ownership or root access. Current situation ================= Ganeti ------ It is possible, given a manually built ``config.data`` and ``_autoconf.py``, to run the masterd under the current user as a single-node cluster master. However, the node daemon and related functionality (cluster initialisation, master failover, etc.) are not directly runnable in this model. Also, masterd only works as a master of a single node cluster, due to our current “hostname” method of identifying nodes, which results in a limit of maximum one node daemon per machine, unless we use multiple name and IP aliases. HTools ------ In HTools the situation is better, since it doesn't have to deal with actual machine management: all tools can use a custom LUXI path, and can even load RAPI data from the filesystem (so the RAPI backend can be tested), and both the ‘text’ backend for hbal/hspace and the input files for hail are text-based, loaded from the file-system. Proposed changes ================ The end-goal is to have full support for “virtual clusters”, i.e. be able to run a “big” (hundreds of virtual nodes and towards thousands of virtual instances) on a reasonably powerful, but single machine, under a single user account and without any special privileges. This would have significant advantages: - being able to test end-to-end certain changes, without requiring a complicated setup - better able to estimate Ganeti's behaviour and performance as the cluster size grows; this is something that we haven't been able to test reliably yet, and as such we still have not yet diagnosed scaling problems - easier integration with external tools (and even with HTools) ``masterd`` ----------- As described above, ``masterd`` already works reasonably well in a virtual setup, as it won't execute external programs and it shouldn't directly read files from the local filesystem (or at least not virtualisation-related, as the master node can be a non-vm_capable node). ``noded`` --------- The node daemon executes many privileged operations, but they can be split in a few general categories: +---------------+-----------------------+------------------------------------+ |Category |Description |Solution | +===============+=======================+====================================+ |disk operations|Disk creation and |Use only diskless or file-based | | |removal |instances | +---------------+-----------------------+------------------------------------+ |disk query |Node disk total/free, |Not supported currently, could use | | |used in node listing |file-based | | |and htools | | +---------------+-----------------------+------------------------------------+ |hypervisor |Instance start, stop |Use the *fake* hypervisor | |operations |and query | | +---------------+-----------------------+------------------------------------+ |instance |Bridge existence query |Unprivileged operation, can be used | |networking | |with an existing bridge at system | | | |level or use NIC-less instances | +---------------+-----------------------+------------------------------------+ |instance OS |OS add, OS rename, |Only used with non-diskless | |operations |export and import |instances; could work with custom OS| | | |scripts that just ``dd`` without | | | |mounting filesystems | +---------------+-----------------------+------------------------------------+ |node networking|IP address management |Not supported; Ganeti will need to | | |(master ip), IP query, |work without a master IP; for the IP| | |etc. |query operations the test machine | | | |would need externally-configured IPs| +---------------+-----------------------+------------------------------------+ |node add |- |SSH command must be adjusted | +---------------+-----------------------+------------------------------------+ |node setup |ssh, /etc/hosts, so on |Can already be disabled from the | | | |cluster config | +---------------+-----------------------+------------------------------------+ |master failover|start/stop the master |Doable (as long as we use a single | | |daemon |user), might get tricky w.r.t. paths| | | |to executables | +---------------+-----------------------+------------------------------------+ |file upload |Uploading of system |The only issue could be with system | | |files, job queue files |files, which are not owned by the | | |and ganeti config |current user; internal ganeti files | | | |should be working fine | +---------------+-----------------------+------------------------------------+ |node oob |Out-of-band commands |Since these are user-defined, we can| | | |mock them easily | +---------------+-----------------------+------------------------------------+ |node OS |List the existing OSes |No special privileges needed, so | |discovery |and their properties |works fine as-is | +---------------+-----------------------+------------------------------------+ |hooks |Running hooks for given|No special privileges needed | | |operations | | +---------------+-----------------------+------------------------------------+ |iallocator |Calling an iallocator |No special privileges needed | | |script | | +---------------+-----------------------+------------------------------------+ |export/import |Exporting and importing|When exporting/importing file-based | | |instances |instances, this should work, as the | | | |listening ports are dynamically | | | |chosen | +---------------+-----------------------+------------------------------------+ |hypervisor |The validation of |As long as the hypervisors don't | |validation |hypervisor parameters |call to privileged commands, it | | | |should work | +---------------+-----------------------+------------------------------------+ |node powercycle|The ability to power |Privileged, so not supported, but | | |cycle a node remotely |anyway not very interesting for | | | |testing | +---------------+-----------------------+------------------------------------+ It seems that much of the functionality works as is, or could work with small adjustments, even in a non-privileged setup. The bigger problem is the actual use of multiple node daemons per machine. Multiple ``noded`` per machine ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently Ganeti identifies node simply by their hostname. Since changing this method would imply significant changes to tracking the nodes, the proposal is to simply have as many IPs per the (single) machine that is used for tests as nodes, and have each IP correspond to a different name, and thus no changes are needed to the core RPC library. Unfortunately this has the downside of requiring root rights for setting up the extra IPs and hostnames. An alternative option is to implement per-node IP/port support in Ganeti (especially in the RPC layer), which would eliminate the root rights. We expect that this will get implemented as a second step of this design, but as the port is currently static will require changes in many places. The only remaining problem is with sharing the ``localstatedir`` structure (lib, run, log) amongst the daemons, for which we propose to introduce an environment variable (``GANETI_ROOTDIR``) acting as a prefix for essentially all paths. An environment variable is easier to transport through several levels of programs (shell scripts, Python, etc.) than a command line parameter. In Python code this prefix will be applied to all paths in ``constants.py``. Every virtual node will get its own root directory. The rationale for this is two-fold: - having two or more node daemons writing to the same directory might introduce artificial scenarios not existent in real life; currently noded either owns the entire ``/var/lib/ganeti`` directory or shares it with masterd, but never with another noded - having separate directories allows cluster verify to check correctly consistency of file upload operations; otherwise, as long as one node daemon wrote a file successfully, the results from all others are “lost” In case the use of an environment variable turns out to be too difficult a compile-time prefix path could be used. This would then require one Ganeti installation per virtual node, but it might be good enough. ``rapi`` -------- The RAPI daemon is not privileged and furthermore we only need one per cluster, so it presents no issues. ``confd`` --------- ``confd`` has somewhat the same issues as the node daemon regarding multiple daemons per machine, but the per-address binding still works. ``ganeti-watcher`` ------------------ Since the startup of daemons will be customised with per-IP binds, the watcher either has to be modified to not activate the daemons, or the start-stop tool has to take this into account. Due to watcher's use of the hostname, it's recommended that the master node is set to the machine hostname (also a requirement for the master daemon). CLI scripts ----------- As long as the master node is set to the machine hostname, these should work fine. Cluster initialisation ---------------------- It could be possible that the cluster initialisation procedure is a bit more involved (this was not tried yet). A script will be used to set up all necessary IP addresses and hostnames, as well as creating the initial directory structure. Building ``config.data`` manually should not be necessary. Needed tools ============ With the above investigation results in mind, the only thing we need are: - a tool to setup per-virtual node tree structure of ``localstatedir`` (with the help of ``ensure-dirs``) and setup correctly the extra IP/hostnames - changes to the startup daemon tools to launch correctly the daemons per virtual node - changes to ``constants.py`` to override the ``localstatedir`` path - documentation for running such a virtual cluster - and eventual small fixes to the node daemon backend functionality, to better separate privileged and non-privileged code .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/design-x509-ca.rst000064400000000000000000000146741476477700300174540ustar00rootroot00000000000000======================================= Design for a X509 Certificate Authority ======================================= :Created: 2011-Mar-23 :Status: Draft .. contents:: :depth: 4 Current state and shortcomings ------------------------------ Import/export in Ganeti have a need for many unique X509 certificates. So far these were all self-signed, but with the :doc:`new design for import/export ` they need to be signed by a Certificate Authority (CA). Proposed changes ---------------- The plan is to implement a simple CA in Ganeti. Interacting with an external CA is too difficult or impossible for automated processes like exporting instances, so each Ganeti cluster will have its own CA. The public key will be stored in ``â€Ļ/lib/ganeti/ca/cert.pem``, the private key (only readable by the master daemon) in ``â€Ļ/lib/ganeti/ca/key.pem``. Similar to the RAPI certificate, a new CA certificate can be installed using the ``gnt-cluster renew-crypto`` command. Such a CA could be an intermediate of a third-party CA. By default a self-signed CA is generated and used. .. _x509-ca-serial: Each certificate signed by the CA is required to have a unique serial number. The serial number is stored in the file ``â€Ļ/lib/ganeti/ca/serial``, replicated to all master candidates and never reset, even when a new CA is installed. The threat model is expected to be the same as with self-signed certificates. To reinforce this, all certificates signed by the CA must be valid for less than one week (168 hours). Implementing support for Certificate Revocation Lists (CRL) using OpenSSL is non-trivial. Lighttpd doesn't support them at all and `apparently never will in version 1.4.x `_. Some CRL-related parts have only been added in the most recent version of pyOpenSSL (0.11). Instead of a CRL, Ganeti will gain a new cluster configuration property defining the minimum accepted serial number. In case of a lost or compromised private key this property can be set to the most recently generated serial number. While possible to implement in the future, other X509 certificates used by the cluster (e.g. RAPI or inter-node communication) will not be automatically signed by the per-cluster CA. The ``commonName`` attribute of signed certificates must be set to the the cluster name or the name of a node in the cluster. Software requirements --------------------- - pyOpenSSL 0.10 or above (lower versions can't set the X509v3 extension ``subjectKeyIdentifier`` recommended for certificate authority certificates by :rfc:`3280`, section 4.2.1.2) Code samples ------------ Generating X509 CA using pyOpenSSL ++++++++++++++++++++++++++++++++++ .. highlight:: python The following code sample shows how to generate a CA certificate using pyOpenSSL:: key = OpenSSL.crypto.PKey() key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) ca = OpenSSL.crypto.X509() ca.set_version(3) ca.set_serial_number(1) ca.get_subject().CN = "ca.example.com" ca.gmtime_adj_notBefore(0) ca.gmtime_adj_notAfter(24 * 60 * 60) ca.set_issuer(ca.get_subject()) ca.set_pubkey(key) ca.add_extensions([ OpenSSL.crypto.X509Extension("basicConstraints", True, "CA:TRUE, pathlen:0"), OpenSSL.crypto.X509Extension("keyUsage", True, "keyCertSign, cRLSign"), OpenSSL.crypto.X509Extension("subjectKeyIdentifier", False, "hash", subject=ca), ]) ca.sign(key, "sha1") Signing X509 certificate using CA +++++++++++++++++++++++++++++++++ .. highlight:: python The following code sample shows how to sign an X509 certificate using a CA:: ca_cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, "ca.pem") ca_key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, "ca.pem") key = OpenSSL.crypto.PKey() key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) cert = OpenSSL.crypto.X509() cert.get_subject().CN = "node1.example.com" cert.set_serial_number(1) cert.gmtime_adj_notBefore(0) cert.gmtime_adj_notAfter(24 * 60 * 60) cert.set_issuer(ca_cert.get_subject()) cert.set_pubkey(key) cert.sign(ca_key, "sha1") How to generate Certificate Signing Request +++++++++++++++++++++++++++++++++++++++++++ .. highlight:: python The following code sample shows how to generate an X509 Certificate Request (CSR):: key = OpenSSL.crypto.PKey() key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) req = OpenSSL.crypto.X509Req() req.get_subject().CN = "node1.example.com" req.set_pubkey(key) req.sign(key, "sha1") # Write private key print(OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)) # Write request print(OpenSSL.crypto.dump_certificate_request(OpenSSL.crypto.FILETYPE_PEM, req)) X509 certificate from Certificate Signing Request +++++++++++++++++++++++++++++++++++++++++++++++++ .. highlight:: python The following code sample shows how to create an X509 certificate from a Certificate Signing Request and sign it with a CA:: ca_cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, "ca.pem") ca_key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, "ca.pem") req = OpenSSL.crypto.load_certificate_request(OpenSSL.crypto.FILETYPE_PEM, open("req.csr").read()) cert = OpenSSL.crypto.X509() cert.set_subject(req.get_subject()) cert.set_serial_number(1) cert.gmtime_adj_notBefore(0) cert.gmtime_adj_notAfter(24 * 60 * 60) cert.set_issuer(ca_cert.get_subject()) cert.set_pubkey(req.get_pubkey()) cert.sign(ca_key, "sha1") print(OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) Verify whether X509 certificate matches private key +++++++++++++++++++++++++++++++++++++++++++++++++++ .. highlight:: python The code sample below shows how to check whether a certificate matches with a certain private key. OpenSSL has a function for this, ``X509_check_private_key``, but pyOpenSSL provides no access to it. :: ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD) ctx.use_privatekey(key) ctx.use_certificate(cert) try: ctx.check_privatekey() except OpenSSL.SSL.Error: print("Incorrect key") else: print("Key matches certificate") .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/dev-codestyle.rst000064400000000000000000000437361476477700300176670ustar00rootroot00000000000000Code style guide ================ Python ------ .. highlight:: python These are a few guidelines for Ganeti code and documentation. In simple terms: try to stay consistent with the existing code. `PEP 8`_ says: .. _PEP 8: http://www.python.org/dev/peps/pep-0008/ A style guide is about consistency. Consistency with this style guide is important. Consistency within a project is more important. Consistency within one module or function is most important. .. note:: You might also want to take a look at the `Google style guide`_, since we have some things in common with it. .. _Google style guide: http://google-styleguide.googlecode.com/svn/trunk/pyguide.html Indentation ~~~~~~~~~~~ In general, always indent using two (2) spaces and don't use tabs. The two spaces should always be relative to the previous level of indentation, even if this means that the final number of spaces is not a multiple of 2. When going on a new line inside an open parenthesis, align with the content of the parenthesis on the previous line. Valid example:: v = (somevalue, a_function([ list_elem, # 7 spaces, but 2 from the previous indentation level another_elem, ])) Formatting strings ~~~~~~~~~~~~~~~~~~ Always use double quotes (``""``), never single quotes (``''``), except for existing code. Examples for formatting strings:: var = "value" # Note: The space character is always on the second line var = ("The quick brown fox jumps over the lazy dog. The quick brown fox" " jumps over the lazy dog. The quick brown fox jumps over the lazy" " dog.") fn("The quick brown fox jumps over the lazy dog. The quick brown fox jumps" " over the lazy dog.") fn(constants.CONFIG_VERSION, ("The quick brown fox jumps over the lazy dog. The quick brown fox" " jumps over the lazy dog. The quick brown fox jumps over the lazy" " dog.")) Don't format strings like this:: # Don't use single quotes var = 'value' # Don't use backslash for line continuation var = "The quick brown fox jumps over the lazy dog. The quick brown fox"\ " jumps over the lazy dog." # Space character goes to the beginning of a new line var = ("The quick brown fox jumps over the lazy dog. The quick brown fox " "jumps over the lazy dog. The quick brown fox jumps over the lazy " "dog.") Formatting sequences ~~~~~~~~~~~~~~~~~~~~ Built-in sequence types are list (``[]``), tuple (``()``) and dict (``{}``). When splitting to multiple lines, each item should be on its own line and a comma must be added on the last line. Don't write multiline dictionaries in function calls, except when it's the only parameter. Always indent items by two spaces. :: # Short lists var = ["foo", "bar"] var = ("foo", "bar") # Longer sequences and dictionary var = [ constants.XYZ_FILENAME_EXTENSION, constants.FOO_BAR_BAZ, ] var = { "key": func(), "otherkey": None, } # Multiline tuples as dictionary values var = { "key": ("long value taking the whole line, requiring you to go to a new one", other_value), } # Function calls var = frozenset([1, 2, 3]) var = F({ "xyz": constants.XYZ, "abc": constants.ABC, }) # Wrong F(123, "Hello World", { "xyz": constants.XYZ }) We consider tuples as data structures, not containers. So in general please use lists when dealing with a sequence of homogeneous items, and tuples when dealing with heterogeneous items. Passing arguments ~~~~~~~~~~~~~~~~~ Positional arguments must be passed as positional arguments, keyword arguments must be passed as keyword arguments. Everything else will be difficult to maintain. :: # Function signature def F(data, key, salt=None, key_selector=None): pass # Yes F("The quick brown fox", "123456") F("The quick brown fox", "123456", salt="abc") F("The quick brown fox", "123456", key_selector="xyz") F("The quick brown fox", "123456", salt="foo", key_selector="xyz") # No: Passing keyword arguments as positional argument F("The quick brown fox", "123456", "xyz", "bar") # No: Passing positional arguments as keyword argument F(salt="xyz", data="The quick brown fox", key="123456", key_selector="xyz") Docstrings ~~~~~~~~~~ .. note:: `PEP 257`_ is the canonical document, unless epydoc overrules it (e.g. in how to document the type of an argument). For docstrings, the recommended format is epytext_, to be processed via epydoc_. There is an ``apidoc`` target that builds the documentation and puts it into the doc/api subdir. Note that we currently use epydoc version 3.0. .. _PEP 257: http://www.python.org/dev/peps/pep-0257/ .. _epytext: http://epydoc.sourceforge.net/manual-epytext.html .. _epydoc: http://epydoc.sourceforge.net/ Note that one-line docstrings are only accepted in the unittests. Rules for writing the docstrings (mostly standard Python rules): * the docstring should start with a sentence, with punctuation at the end, summarizing the the aim of what is being described. This sentence cannot be longer than one line * the second line should be blank * afterwards the rest of the docstring * special epytext tags should come at the end * multi-line docstrings must finish with an empty line * do not try to make a table using lots of whitespace * use ``L{}`` and ``C{}`` where appropriate Here's an example:: def fn(foo, bar): """Compute the sum of foo and bar. This functions builds the sum of foo and bar. It's a simple function. @type foo: int @param foo: First parameter. @type bar: float @param bar: The second parameter. This line is longer to show wrapping. @rtype: float @return: the sum of the two numbers """ return foo + bar Some rules of thumb which should be applied with good judgement on a case-to- case basis: * If the meaning of parameters is already obvious given its name and the methods description, don't document it again. Just add a ``@type`` tag. * Refer to the base methods documentation when overwriting methods. Only document more if it applies to the current subclass only, or if you want to clarify on the meaning of parameters for the special subclass. Rules for classes and modules ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ As `PEP 257`_ says, the docstrings of classes should document their attributes and the docstrings of modules should shortly document the exported functions/variables/etc. See for example the pydoc output for the ``os`` or ``ConfigParser`` standard modules. Haskell ------- .. highlight:: haskell The most important consideration is, as usual, to stay consistent with the existing code. As there's no "canonical" style guide for Haskell, this code style has been inspired from a few online resources, including the style guide for the `Snap framework`_, `this style guide`_ and `this other style guide`_. .. _Snap framework: http://snapframework.com/docs/style-guide .. _this style guide: https://github.com/tibbe/haskell-style-guide/blob/master/haskell-style.md .. _this other style guide: http://www.cs.caltech.edu/courses/cs11/material/haskell/misc/haskell_style_guide.html Files ~~~~~ Use ordinary, non-`literate`_ Haskell ``.hs`` files. .. _literate: http://www.haskell.org/haskellwiki/Literate_programming Use proper copyright headers, and proper Haddock style documentation headers:: {-| Short module summary. Longer module description. -} {- Copyright (C) ... This program is free software ... -} If there are module-level pragmas add them right at the top, before the short summary. Imports ~~~~~~~ Imports should be grouped into the following groups and inside each group they should be sorted alphabetically: 1. import of non-Ganeti libaries 2. import of Ganeti libraries It is allowed to use qualified imports with short names for: * standard library (e.g. ``import qualified Data.Map as M``) * local imports (e.g. ``import qualified Ganeti.Constants as C``) Whenever possible, prefer explicit imports, either in form of qualified imports, or by naming the imported functions (e.g., ``import Control.Arrow ((&&&))``, ``import Data.Foldable(fold, toList)``) Indentation ~~~~~~~~~~~ Use only spaces, never tabs. Indentation level is 2 characters. For Emacs, this means setting the variable ``haskell-indent-offset`` to 2. Line length should be at most 78 chars, and 72 chars inside comments. Use indentation-based structure, and not braces/semicolons. .. note:: Special indendation of if/then/else construct For the ``do`` notation, the ``if-then-else`` construct has a non-intuitive behaviour. As such, the indentation of ``if-then-else`` (both in ``do`` blocks and in normal blocks) should be as follows:: if condition then expr1 else expr2 i.e. indent the then/else lines with another level. This can be accomplished in Emacs by setting the variable ``haskell-indent-thenelse`` to 2 (from the default of zero). If you have more than one line of code please newline/indent after the "=". Do `not` do:: f x = let y = x + 1 in y Instead do:: f x = let y = x + 1 in y or if it is just one line:: f x = x + 1 Multiline strings ~~~~~~~~~~~~~~~~~ Multiline strings are created by closing a line with a backslash and starting the following line with a backslash, keeping the indentation level constant. Whitespaces go on the new line, right after the backslash. :: longString :: String longString = "This is a very very very long string that\ \ needs to be split in two lines" Data declarations ~~~~~~~~~~~~~~~~~ .. warning:: Note that this is different from the Python style! When declaring either data types, or using list literals, etc., the columns should be aligned, and for lists use a comma at the start of the line, not at the end. Examples:: data OpCode = OpStartupInstance ... | OpShutdownInstance ... | ... data Node = Node { name :: String , ip :: String , ... } myList = [ value1 , value2 , value3 ] The choice of whether to wrap the first element or not is up to you; the following is also allowed:: myList = [ value1 , value2 ] For records, always add spaces around the braces and the equality sign. :: foo = Foo { fBar = "bar", fBaz = 4711 } foo' = Foo { fBar = "bar 2" , fBaz = 4712 } node' = node { ip = "127.0.0.1" } White space ~~~~~~~~~~~ Like in Python, surround binary operators with one space on either side. Do no insert a space after a lamda:: -- bad map (\ n -> ...) lst -- good foldl (\x y -> ...) ... Use a blank line between top-level definitions, but no blank lines between either the comment and the type signature or between the type signature and the actual function definition. .. note:: Ideally it would be two blank lines between top-level definitions, but the code only has one now. As always, no trailing spaces. Ever. Spaces after comma ****************** Instead of:: ("a","b") write:: ("a", "b") Naming ~~~~~~ Functions should be named in mixedCase style, and types in CamelCase. Function arguments and local variables should be mixedCase. When using acronyms, ones longer than 2 characters should be typed capitalised, not fully upper-cased (e.g. ``Http``, not ``HTTP``). For variable names, use descriptive names; it is only allowed to use very short names (e.g. ``a``, ``b``, ``i``, ``j``, etc.) when: * the function is trivial, e.g.:: sum x y = x + y * we talk about some very specific cases, e.g. iterators or accumulators in folds:: map (\v -> v + 1) lst * using ``x:xs`` for list elements and lists, etc. In general, short/one-letter names are allowed when we deal with polymorphic values; for example the standard map definition from Prelude:: map :: (a -> b) -> [a] -> [b] map _ [] = [] map f (x:xs) = f x : map f xs In this example, neither the ``a`` nor ``b`` types are known to the map function, so we cannot give them more explicit names. Since the body of the function is trivial, the variables used are longer. However, if we deal with explicit types or values, their names should be descriptive. .. todo: add a nice example here. Finally, the naming should look familiar to people who just read the Prelude/standard libraries. Naming for updated values ************************* .. highlight:: python Since one cannot update a value in Haskell, this presents a particular problem on the naming of new versions of the same value. For example, the following code in Python:: def failover(pri, sec, inst): pri.removePrimary(inst) pri.addSecondary(inst) sec.removeSecondary(inst) sec.addPrimary(inst) .. highlight:: haskell becomes in Haskell something like the following:: failover pri sec inst = let pri' = removePrimary pri inst pri'' = addSecondary pri' inst sec' = removeSecondary sec inst sec'' = addPrimary sec' inst in (pri'', sec'') When updating values, one should add single quotes to the name for up to three new names (e.g. ``inst``, ``inst'``, ``inst''``, ``inst'''``) and otherwise use numeric suffixes (``inst1``, ``inst2``, ``inst3``, ..., ``inst8``), but that many updates is already bad style and thus should be avoided. Type signatures ~~~~~~~~~~~~~~~ Always declare types for functions (and any other top-level bindings). If in doubt, feel free to declare the type of the variables/bindings in a complex expression; this usually means the expression is too complex, however. Similarly, provide Haddock-style comments for top-level definitions. Use sum types instead of exceptions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Exceptions make it hard to write functional code, as alternative control flows need to be considered and compiler support is limited. Therefore, Ganeti functions should never allow exceptions to escape. Function that can fail should report failure by returning an appropriate sum type (``Either`` or one of its glorified variants like ``Maybe`` or ``Result``); the preferred sum type for reporting errors is ``Result``. As other Ganeti functions also follow these guide lines, they can safely be composed. However, be careful when using functions from other libraries; if they can raise exceptions, catch them, preferably as close to their origin as reasonably possible. Parentheses, point free style ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Prefer the so-called `point-free`_ style style when declaring functions, if applicable:: -- bad let a x = f (g (h x)) -- good let a = f . g . h Also use function composition in a similar manner in expressions to avoid extra parentheses:: -- bad f (g (h x)) -- better f $ g $ h x -- best f . g . h $ x .. _`point-free`: http://www.haskell.org/haskellwiki/Pointfree Language features ~~~~~~~~~~~~~~~~~ Extensions ********** It is recommended to keep the use of extensions to a minimum, so that the code can be understood even if one is familiar with just Haskel98/Haskell2010. That said, some extensions are very common and useful, so they are recommended: * `Bang patterns`_: useful when you want to enforce strict evaluation (and better than repeated use of ``seq``) * CPP: a few modules need this in order to account for configure-time options; don't overuse it, since it breaks multi-line strings * `Template Haskell`_: we use this for automatically deriving JSON instances and other similar boiler-plate .. _Bang patterns: http://www.haskell.org/ghc/docs/latest/html/users_guide/bang-patterns.html .. _Template Haskell: http://www.haskell.org/ghc/docs/latest/html/users_guide/template-haskell.html Such extensions should be declared using the ``Language`` pragma:: {-# Language BangPatterns #-} {-| This is a small module... -} Comments ******** Always use proper sentences; start with a capital letter and use punctuation in top level comments:: -- | A function that does something. f :: ... For inline comments, start with a capital letter but no ending punctuation. Furthermore, align the comments together with a 2-space width from the end of the item being commented:: data Maybe a = Nothing -- ^ Represents empty container | Just a -- ^ Represents a single value The comments should be clear enough so that one doesn't need to look at the code to understand what the item does/is. Use ``-- |`` to write doc strings rather than bare comment with ``--``. Tools ***** We generate the API documentation via Haddock, and as such the comments should be correct (syntax-wise) for it. Use markup, but sparingly. We use hlint_ as a lint checker; the code is currently lint-clean, so you must not add any warnings/errors. .. _hlint: http://community.haskell.org/~ndm/darcs/hlint/hlint.htm Use these two commands during development:: make hs-apidoc make hlint QuickCheck best practices ************************* If you have big type that takes time to generate and several properties to test on that, by default 500 of those big instances are generated for each property. In many cases, it would be sufficient to only generate those 500 instances once and test all properties on those. To do this, create a property that uses ``conjoin`` to combine several properties into one. Use ``counterexample`` to add expressive error messages. For example:: prop_myMegaProp :: myBigType -> Property prop_myMegaProp b = conjoin [ counterexample ("Something failed horribly here: " ++ show b) (subProperty1 b) , counterexample ("Something else failed horribly here: " ++ show b) (subProperty2 b) , -- more properties here ... ] subProperty1 :: myBigType -> Bool subProperty1 b = ... subProperty2 :: myBigType -> Property subProperty2 b = ... ... Maybe Generation '''''''''''''''' Use ``genMaybe genSomething`` to create ``Maybe`` instances of something including some ``Nothing`` instances. Use ``Just <$> genSomething`` to generate only ``Just`` instances of something. String Generation ''''''''''''''''' To generate strings, consider using ``genName`` instead of ``arbitrary``. ``arbitrary`` has the tendency to generate strings that are too long. ganeti-3.1.0~rc2/doc/examples/000075500000000000000000000000001476477700300161675ustar00rootroot00000000000000ganeti-3.1.0~rc2/doc/examples/basic-oob000075500000000000000000000053731476477700300177630ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SSH_USER='root' SSH_FLAGS='-q -oStrictHostKeyChecking=no' EXIT_SUCCESS=0 EXIT_FAILURE=1 EXIT_UNKNOWN=2 _run_ssh() { local host="$1" local command="$2" ssh $SSH_FLAGS "$SSH_USER@$host" "$command" 1>&2 return $? } _power_on() { echo 'power-on not supported in this script' >&2 exit $EXIT_FAILURE } _power_off() { local host="$1" if ! _run_ssh "$host" 'shutdown -h now'; then echo "Failure during ssh to $host" >&2 exit $EXIT_FAILURE fi } _power_cycle() { local host="$1" if ! _run_ssh "$host" 'shutdown -r now'; then echo "Failure during ssh to $host" >&2 exit $EXIT_FAILURE fi } _power_status() { local host="$1" if fping -q "$host" > /dev/null 2>&1; then echo '{ "powered": true }' else echo '{ "powered": false }' fi } _health() { echo 'health not supported in this script' >&2 exit $EXIT_FAILURE } _action() { local command="$1" local host="$2" case "$command" in power-on) _power_on "$host" ;; power-off) _power_off "$host" ;; power-cycle) _power_cycle "$host" ;; power-status) _power_status "$host" ;; health) _health "$host" ;; *) echo "Unsupported command '$command'" >&2 exit $EXIT_FAILURE ;; esac } main() { if [[ $# != 2 ]]; then echo "Wrong argument count, got $#, expected 2" >&2 exit $EXIT_FAILURE fi _action "$@" exit $EXIT_SUCCESS } main "$@" ganeti-3.1.0~rc2/doc/examples/batcher-instances.json000064400000000000000000000004071476477700300224600ustar00rootroot00000000000000{ "instance1.example.com": { "template": "drbd", "os": "debootstrap", "disk_size": ["25G"], "ram_size": 512 }, "instance2.example.com": { "template": "plain", "os": "debootstrap", "disk_size": ["100G"], "ram_size": 512 } } ganeti-3.1.0~rc2/doc/examples/ganeti-kvm-poweroff.initd.in000064400000000000000000000036331476477700300235210ustar00rootroot00000000000000#!/bin/bash # ganeti kvm instance poweroff # based on skeleton from Debian GNU/Linux ### BEGIN INIT INFO # Provides: ganeti-kvm-poweroff # Required-Start: # Required-Stop: drbd qemu-kvm $local_fs # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Poweroff Ganeti KVM instances # Description: Sends system_powerdown command to Ganeti instances, otherwise # they will be killed. ### END INIT INFO shopt -s nullglob PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin DESC="Ganeti KVM instance poweroff " . /lib/lsb/init-functions CONTROL_PATH="@LOCALSTATEDIR@/run/ganeti/kvm-hypervisor/ctrl" SCRIPTNAME="@SYSCONFDIR@/init.d/ganeti-kvm-poweroff" TIMEOUT=60 do_kvm_poweroff () { # shutdown VMs and remove sockets of those not running for vm_monitor in $CONTROL_PATH/*.monitor; do if ! echo system_powerdown | \ socat - UNIX-CONNECT:$vm_monitor > /dev/null 2>&1; then # remove disconnected socket rm -f $vm_monitor fi done log_action_begin_msg "Waiting VMs to poweroff" waiting=true remaining=$TIMEOUT while $waiting && [ $remaining -ne 0 ]; do if [[ -z "$(find $CONTROL_PATH -name '*.monitor')" ]]; then break fi echo -n "." for vm_monitor in $CONTROL_PATH/*.monitor; do if ! echo | socat - UNIX-CONNECT:$vm_monitor > /dev/null 2>&1; then rm -rf $vm_monitor fi done sleep 5 let remaining-=5 1 done if [[ -z "$(find $CONTROL_PATH -name '*.monitor')" ]]; then log_action_end_msg 0 else log_action_end_msg 1 "some VMs did not shutdown" fi } case "$1" in start) # No-op ;; restart|reload|force-reload) echo "Error: argument '$1' not supported" >&2 exit 3 ;; stop) do_kvm_poweroff ;; *) echo "Usage: $0 start|stop" >&2 exit 3 ;; esac ganeti-3.1.0~rc2/doc/examples/ganeti-master-role.ocf.in000064400000000000000000000056301476477700300227700ustar00rootroot00000000000000#!/bin/bash # ganeti master role OCF resource # See http://linux-ha.org/wiki/OCF_Resource_Agents set -e -u @SHELL_ENV_INIT@ PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin SCRIPTNAME="@LIBDIR@/ocf/resource.d/ganeti/ganeti-master-role" # Master candidates list file MCFILE="$DATA_DIR/ssconf_master_candidates" # We'll need the hostname in a few places, so we'll get it once, now. MYHOSTNAME=$(hostname --fqdn) is_master() { local -r master=$(gnt-cluster getmaster) [[ "$MYHOSTNAME" == "$master" ]] } is_candidate() { grep -Fx $MYHOSTNAME $MCFILE } start_action() { if is_master; then exit 0 elif is_candidate; then gnt-cluster master-failover || exit 1 # OCF_ERR_GENERIC else exit 5 # OCF_ERR_INSTALLED (vital component missing) fi } stop_action() { # We can't really "stop" being a master. # TODO: investigate whether a fake approach will do. exit 1 # OCF_ERR_GENERIC } recover_action() { if is_master; then gnt-cluster redist-conf || exit 1 # OCF_ERR_GENERIC elif is_candidate; then gnt-cluster master-failover || exit 1 # OCF_ERR_GENERIC else exit 5 # OCF_ERR_INSTALLED (vital component missing) fi } monitor_action() { # monitor should exit: # 7 if the resource is not running # 1 if it failed # 0 if it's running if is_master; then exit 0 elif is_candidate; then exit 7 # OCF_NOT_RUNNING else exit 5 # OCF_ERR_INSTALLED (vital component missing) fi } return_meta() { cat < 0.1 OCF script to manage the ganeti master role in a cluster. Can be used to failover the ganeti master between master candidate nodes. Manages the ganeti cluster master END exit 0 } case "$1" in # Mandatory OCF commands start) start_action ;; stop) stop_action ;; monitor) monitor_action ;; meta-data) return_meta ;; # Optional OCF commands recover) recover_action ;; reload) # The ganeti master role has no "configuration" that is reloadable on # the pacemaker side. We declare the operation anyway to make sure # pacemaker doesn't decide to stop and start the service needlessly. exit 0 ;; promote|demote|migrate_to|migrate_from|validate-all) # Not implemented (nor declared by meta-data) exit 3 # OCF_ERR_UNIMPLEMENTED ;; *) log_success_msg "Usage: $SCRIPTNAME {start|stop|monitor|meta-data|recover|reload}" exit 1 ;; esac exit 0 ganeti-3.1.0~rc2/doc/examples/ganeti-node-role.ocf.in000064400000000000000000000075751476477700300224340ustar00rootroot00000000000000#!/bin/bash # ganeti node role OCF resource # See http://linux-ha.org/wiki/OCF_Resource_Agents set -e -u @SHELL_ENV_INIT@ PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin SCRIPTNAME="@LIBDIR@/ocf/resource.d/ganeti/ganeti-node-role" # If this file exists don't act on notifications, thus allowing them to happen # during the service configuration. NORUNFILE="$DATA_DIR/ha_node_role_config" # Where to grep for tags TAGSFILE="$DATA_DIR/ssconf_cluster_tags" # If this tag is set we won't try to powercycle nodes POWERCYCLETAG="ocf:node-offline:use-powercycle" # If this tag is set will use IPMI to power off an offline node POWEROFFTAG="ocf:node-offline:use-poweroff" # We'll need the hostname in a few places, so we'll get it once, now. MYHOSTNAME=$(hostname --fqdn) is_master() { local -r master=$(gnt-cluster getmaster) [[ "$MYHOSTNAME" == "$master" ]] } start_action() { # If we're alive we consider ourselves a node, without starting anything. # TODO: improve on this exit 0 } stop_action() { # We can't "really" stop the service locally. # TODO: investigate whether a "fake" stop will work. exit 1 } recover_action() { # Nothing to recover, as long as we're alive. exit 0 } monitor_action() { # If we're alive we consider ourselves a working node. # TODO: improve on this exit 0 } offline_node() { local -r node=$1 grep -Fx $POWERCYCLETAG $TAGSFILE && gnt-node powercycle $node grep -Fx $POWEROFFTAG $TAGSFILE && gnt-node power off $node # TODO: do better than just --auto-promote # (or make sure auto-promote gets better in Ganeti) gnt-node modify -O yes --auto-promote $node } drain_node() { node=$1 # TODO: do better than just --auto-promote # (or make sure auto-promote gets better in Ganeti) gnt-node modify -D yes --auto-promote $node || return 1 } notify_action() { is_master || exit 0 [[ -f $NORUNFILE ]] && exit 0 # TODO: also implement the "start" operation for readding a node [[ $OCF_RESKEY_CRM_meta_notify_operation == "stop" ]] || exit 0 [[ $OCF_RESKEY_CRM_meta_notify_type == "post" ]] || exit 0 local -r target=$OCF_RESKEY_CRM_meta_notify_stop_uname local -r node=$(gnt-node list --no-headers -o name $target) # TODO: use drain_node when we can offline_node $node exit 0 } return_meta() { cat < 0.1 OCF script to manage the ganeti node role in a cluster. Can be used to online and offline nodes. Should be cloned on all nodes of the cluster, with notification enabled. Manages the ganeti cluster nodes END exit 0 } case "$1" in # Mandatory OCF commands start) start_action ;; stop) stop_action ;; monitor) monitor_action ;; meta-data) return_meta ;; # Optional OCF commands recover) recover_action ;; reload) # The ganeti node role has no "configuration" that is reloadable on # the pacemaker side. We declare the operation anyway to make sure # pacemaker doesn't decide to stop and start the service needlessly. exit 0 ;; notify) # Notification of a change to the ganeti node role notify_action exit 0 ;; promote|demote|migrate_to|migrate_from|validate-all) # Not implemented (nor declared by meta-data) exit 3 # OCF_ERR_UNIMPLEMENTED ;; *) log_success_msg "Usage: $SCRIPTNAME {start|stop|monitor|meta-data|recover|reload}" exit 1 ;; esac exit 0 ganeti-3.1.0~rc2/doc/examples/ganeti.cron.in000064400000000000000000000014061476477700300207270ustar00rootroot00000000000000PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin # On reboot, continue a Ganeti upgrade, if one was in progress @reboot root [ -x @SBINDIR@/gnt-cluster ] && @SBINDIR@/gnt-cluster upgrade --resume # Restart failed instances (in non-strict mode every 5 minutes) 5-25/5,35-55/5 * * * * root [ -x @SBINDIR@/ganeti-watcher ] && @SBINDIR@/ganeti-watcher --no-strict # Restart failed instances (in strict mode every 30 minutes) */30 * * * * root [ -x @SBINDIR@/ganeti-watcher ] && @SBINDIR@/ganeti-watcher # Clean job archive (at 01:45 AM) 45 1 * * * @GNTMASTERUSER@ [ -x @SBINDIR@/ganeti-cleaner ] && @SBINDIR@/ganeti-cleaner master # Clean job archive (at 02:45 AM) 45 2 * * * @GNTNODEDUSER@ [ -x @SBINDIR@/ganeti-cleaner ] && @SBINDIR@/ganeti-cleaner node ganeti-3.1.0~rc2/doc/examples/ganeti.default000064400000000000000000000002721476477700300210050ustar00rootroot00000000000000# Default arguments for Ganeti daemons NODED_ARGS="" RAPI_ARGS="-b 127.0.0.1 --require-authentication" CONFD_ARGS="" MOND_ARGS="" WCONFD_ARGS="" LUXID_ARGS="" METAD_ARGS="" KVMD_ARGS="" ganeti-3.1.0~rc2/doc/examples/ganeti.default-debug000064400000000000000000000002621476477700300220700ustar00rootroot00000000000000# Default arguments for Ganeti daemons (debug mode) NODED_ARGS="-d" RAPI_ARGS="-d" CONFD_ARGS="-d" MOND_ARGS="-d" WCONFD_ARGS="-d" LUXID_ARGS="-d" METAD_ARGS="-d" KVMD_ARGS="-d" ganeti-3.1.0~rc2/doc/examples/ganeti.initd.in000064400000000000000000000055401476477700300211000ustar00rootroot00000000000000#!/bin/sh # ganeti daemons init script # # chkconfig: 2345 99 01 # description: Ganeti Cluster Manager ### BEGIN INIT INFO # Provides: ganeti # Required-Start: $syslog $remote_fs # Required-Stop: $syslog $remote_fs # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Ganeti Cluster Manager # Description: Ganeti Cluster Manager ### END INIT INFO PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin DESC="Ganeti cluster" DAEMON_UTIL=@PKGLIBDIR@/daemon-util SCRIPTNAME="@SYSCONFDIR@/init.d/ganeti" test -f "$DAEMON_UTIL" || exit 0 if [ -r /lib/lsb/init-functions ]; then . /lib/lsb/init-functions elif [ -r /etc/rc.d/init.d/functions ]; then . /etc/rc.d/init.d/functions else echo "Unable to find init functions" exit 1 fi check_exitcode() { RC=$1 if errmsg=$($DAEMON_UTIL check-exitcode $RC) then log_action_end_msg 0 "$errmsg" else log_action_end_msg 1 "$errmsg" fi } start_action() { # called as start_action daemon-name local daemon="$1" log_action_begin_msg "$daemon" $DAEMON_UTIL start "$@" check_exitcode $? } stop_action() { # called as stop_action daemon-name local daemon="$1" log_action_begin_msg "$daemon" $DAEMON_UTIL stop "$@" check_exitcode $? } maybe_do() { requested="$1"; shift action="$1"; shift target="$1" if [ -z "$requested" -o "$requested" = "$target" ]; then $action "$@" fi } start_all() { if ! $DAEMON_UTIL check-config; then log_warning_msg "Incomplete configuration, will not run." exit 0 fi for i in $($DAEMON_UTIL list-start-daemons); do maybe_do "$1" start_action $i done } stop_all() { for i in $($DAEMON_UTIL list-stop-daemons); do maybe_do "$1" stop_action $i done } status_all() { local daemons="$1" status ret if [ -z "$daemons" ]; then daemons=$($DAEMON_UTIL list-start-daemons) fi status=0 for i in $daemons; do if status_of_proc $($DAEMON_UTIL daemon-executable $i) $i; then ret=0 else ret=$? # Use exit code from first failed call if [ "$status" -eq 0 ]; then status=$ret fi fi done exit $status } if [ -n "$2" ] && ! errmsg=$($DAEMON_UTIL is-daemon-name "$2" 2>&1); then log_failure_msg "$errmsg" exit 1 fi case "$1" in start) log_daemon_msg "Starting $DESC" "$2" start_all "$2" ;; stop) log_daemon_msg "Stopping $DESC" "$2" stop_all "$2" ;; restart|force-reload) log_daemon_msg "Restarting $DESC" "$2" stop_all "$2" start_all "$2" ;; status) status_all "$2" ;; *) log_success_msg "Usage: $SCRIPTNAME {start|stop|force-reload|restart}" exit 1 ;; esac exit 0 ganeti-3.1.0~rc2/doc/examples/ganeti.logrotate.in000064400000000000000000000003541476477700300217670ustar00rootroot00000000000000/var/log/ganeti/*.log { weekly missingok rotate 52 notifempty compress delaycompress sharedscripts postrotate @PKGLIBDIR@/daemon-util rotate-all-logs endscript } ganeti-3.1.0~rc2/doc/examples/gnt-config-backup.in000064400000000000000000000053461476477700300220250ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # This is an example ganeti script that should be run from cron on all # nodes; it will archive the ganeti configuration into a separate # directory tree via GIT, so that it is possible to restore the # history of cluster configuration changes if needed # The script requires the lockfile-progs package and the git software # Note that since Ganeti 2.0, config.data is the authoritative source # of configuration; as such, we don't need to backup the ssconf files, # and the other files (server.pem, rapi.pem, hmac.key, known_hosts, # etc.) do no hold critical data (they can be regenerated at will, as # long as they are synchronised). set -e LOCALSTATEDIR=@LOCALSTATEDIR@ SYSCONFDIR=@SYSCONFDIR@ GANETIDIR=${LOCALSTATEDIR}/lib/ganeti CONFIGDATA=${GANETIDIR}/config.data GNTBKDIR=${LOCALSTATEDIR}/lib/gnt-config-backup LOCKFILE=${LOCALSTATEDIR}/lock/gnt-config-backup # exit if no ganeti config file (no cluster configured, or not M/MC) test -f $CONFIGDATA || exit 0 # We use a simple lock method, since our script should be fast enough # (no network, not talking to ganeti-masterd) that we don't expect to # run over 5 minutes if the system is healthy lockfile-create "$LOCKFILE" || exit 1 trap 'lockfile-remove $LOCKFILE' EXIT test -d $GNTBKDIR || mkdir $GNTBKDIR cd $GNTBKDIR test -d .git || git init cp -f $CONFIGDATA config.data git add config.data git commit -q -m "Automatic commit by gnt-config-backup" touch last_run ganeti-3.1.0~rc2/doc/examples/gnt-debug/000075500000000000000000000000001476477700300200435ustar00rootroot00000000000000ganeti-3.1.0~rc2/doc/examples/gnt-debug/README000064400000000000000000000015771476477700300207350ustar00rootroot00000000000000In order to submit arbitrary jobs to ganeti one can call gnt-debug submit-job passing a suitably formatted json file. A few examples of those files are included here. Using delay0.json and delay50.json in conjunction with submit-job for example allows one to submit rapidly many short delay job (using --job-repeat), repeating the sleep opcode any number of times (using --op-repeat), either all at the same time or one at a time (with --each). This can be used to check the performance of the job queue. Examples: # Run 40 jobs with 10 opcodes each: gnt-debug submit-job --op-repeat 10 --job-repeat 40 --timing-stats delay0.json # Run 40 jobs with 1 opcode each: gnt-debug submit-job --op-repeat 1 --job-repeat 40 --timing-stats delay0.json # Run 40 jobs with 10 opcodes each and submit one at a time: gnt-debug submit-job --op-repeat 10 --job-repeat 40 --timing-stats --each delay0.json ganeti-3.1.0~rc2/doc/examples/gnt-debug/delay0.json000064400000000000000000000001671476477700300221200ustar00rootroot00000000000000[ {"OP_ID": "OP_TEST_DELAY", "debug_level": 0, "dry_run": false, "duration": 0.0, "on_master": true, "on_nodes": []} ] ganeti-3.1.0~rc2/doc/examples/gnt-debug/delay50.json000064400000000000000000000001701476477700300221770ustar00rootroot00000000000000[ {"OP_ID": "OP_TEST_DELAY", "debug_level": 0, "dry_run": false, "duration": 0.05, "on_master": true, "on_nodes": []} ] ganeti-3.1.0~rc2/doc/examples/gnt-debug/delayR.json000064400000000000000000000002041476477700300221520ustar00rootroot00000000000000[ {"OP_ID": "OP_TEST_DELAY", "debug_level": 0, "dry_run": false, "duration": 0.0, "on_master": true, "on_nodes": [], "repeat": 5} ] ganeti-3.1.0~rc2/doc/examples/hooks/000075500000000000000000000000001476477700300173125ustar00rootroot00000000000000ganeti-3.1.0~rc2/doc/examples/hooks/ethers000075500000000000000000000076041476477700300205410ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # This is an example ganeti hook that writes the instance mac addresses in the # node's /etc/ether file. It will pic up the first nic connected to the # TARGET_BRIDGE bridge, and write it down with the syntax "MAC INSTANCE_NAME". # The hook will also send a HUP signal the daemon whose PID is in # DAEMON_PID_FILE, so that it can load the new /etc/ethers file and use it. # This has been tested in conjunction with dnsmasq's dhcp implementation. # It will also remove any other occurrences for the same instance in the # aformentioned file. This hook supports the "instance-add", "instance-modify" # "instance-remove", and "instance-mirror-replace" ganeti post hook paths. To # install it add a symlink from those hooks' directories to where this file is # installed (with a mode which permits execution). # TARGET_BRIDGE: We'll only add the first nic which gets connected to this # bridge to /etc/ethers. TARGET_BRIDGE="br0" DAEMON_PID_FILES="/var/run/dnsmasq.pid /var/run/dnsmasq/dnsmasq.pid" # In order to handle concurrent execution of this lock, we use the $LOCKFILE. # LOCKFILE_CREATE and LOCKFILE_REMOVE are the path names for the lockfile-progs # programs which we use as helpers. LOCKFILE="/var/lock/ganeti_ethers" LOCKFILE_CREATE="/usr/bin/lockfile-create" LOCKFILE_REMOVE="/usr/bin/lockfile-remove" hooks_path=$GANETI_HOOKS_PATH [ -n "$hooks_path" ] || exit 1 instance=$GANETI_INSTANCE_NAME [ -n "$instance" ] || exit 1 nic_count=$GANETI_INSTANCE_NIC_COUNT acquire_lockfile() { $LOCKFILE_CREATE $LOCKFILE || exit 1 trap "$LOCKFILE_REMOVE $LOCKFILE" EXIT } update_ethers_from_new() { chmod 644 /etc/ethers.new mv /etc/ethers.new /etc/ethers for file in $DAEMON_PID_FILES; do [ -f "$file" ] && kill -HUP $(< $file) done } if [ "$hooks_path" = "instance-add" -o \ "$hooks_path" = "instance-modify" -o \ "$hooks_path" = "instance-mirror-replace" ] then for i in $(seq 0 $((nic_count - 1)) ); do bridge_var="GANETI_INSTANCE_NIC${i}_BRIDGE" bridge=${!bridge_var} if [ -n "$bridge" -a "$bridge" = "$TARGET_BRIDGE" ]; then mac_var="GANETI_INSTANCE_NIC${i}_MAC" mac=${!mac_var} acquire_lockfile cat /etc/ethers | awk -- "! /^([[:xdigit:]:]*)[[:blank:]]+$instance\>/; END {print \"$mac\t$instance\"}" > /etc/ethers.new update_ethers_from_new break fi done fi if [ "$hooks_path" = "instance-remove" -o \ \( "$hooks_path" = "instance-modify" -a "$nic_count" -eq 0 \) ]; then acquire_lockfile cat /etc/ethers | awk -- "! /^([[:xdigit:]:]*)[[:blank:]]+$instance\>/" \ > /etc/ethers.new update_ethers_from_new fi ganeti-3.1.0~rc2/doc/examples/hooks/ipsec.in000064400000000000000000000177421476477700300207600ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # This is an example ganeti hook that sets up an IPsec ESP link between all the # nodes of a cluster for a given list of protocols. # When run on cluster initialization it will create the shared key to be used # for all the links. When run on node add/removal it will reconfigure IPsec # on each node of the cluster. set -e LOCALSTATEDIR=@LOCALSTATEDIR@ SYSCONFDIR=@SYSCONFDIR@ GNTDATA=${LOCALSTATEDIR}/lib/ganeti LOCKFILE=${LOCALSTATEDIR}/lock/ganeti_ipsec CRYPTALGO=rijndael-cbc KEYPATH=${GNTDATA}/ipsec.key KEYSIZE=24 PROTOSTOSEC="icmp tcp" TCPTOIGNORE="22 1811" # On debian/ubuntu this file is automatically reloaded on boot SETKEYCONF=${SYSCONFDIR}/ipsec-tools.conf SETKEYCUSTOMCONF=${SYSCONFDIR}/ipsec-tools-custom.conf AUTOMATIC_MARKER="# Automatically generated rules" REGEN_KEY_WAIT=2 NODES=${GNTDATA}/ssconf_node_secondary_ips MASTERNAME_FILE=${GNTDATA}/ssconf_master_node MASTERIP_FILE=${GNTDATA}/ssconf_master_ip SSHOPTS="-q -oUserKnownHostsFile=/dev/null -oStrictHostKeyChecking=no \ -oGlobalKnownHostsFile=${GNTDATA}/known_hosts" SCPOPTS="-p $SSHOPTS" CLEANUP=( ) cleanup() { # Perform all registered cleanup operation local i for (( i=${#CLEANUP[@]}; i >= 0 ; --i )); do ${CLEANUP[$i]} done } acquire_lockfile() { # Acquire the lockfile associated with system ipsec configuration. lockfile-create "$LOCKFILE" || exit 1 CLEANUP+=("lockfile-remove $LOCKFILE") } update_system_ipsec() { # Update system ipsec configuration. # $1 : temporary location of a working configuration local TMPCONF="$1" acquire_lockfile mv "$TMPCONF" "$SETKEYCONF" setkey -f "$SETKEYCONF" } update_keyfile() { # Obtain the IPsec keyfile from the master. local MASTERIP=$(< "$MASTERIP_FILE") scp $SCPOPTS "$MASTERIP":"$KEYPATH" "$KEYPATH" } gather_key() { # Output IPsec key, if no key is present on the node # obtain it from master. if [[ ! -f "$KEYPATH" ]]; then update_keyfile fi cut -d ' ' -f2 "$KEYPATH" } gather_key_seqno() { # Output IPsec key sequence number, if no key is present # on the node exit with error. if [[ ! -f "$KEYPATH" ]]; then echo 'Cannot obtain key timestamp, no key file.' >&2 exit 1 fi cut -d ' ' -f1 "$KEYPATH" } update_ipsec_conf() { # Generate a new IPsec configuration and update the system. local TMPCONF=$(mktemp) CLEANUP+=("rm -f $TMPCONF") ESCAPED_HOSTNAME=$(sed 's/\./\\./g' <<< "$HOSTNAME") local MYADDR=$(grep -E "^$ESCAPED_HOSTNAME\\>" "$NODES" | cut -d ' ' -f2) local KEY=$(gather_key) local SETKEYPATH=$(which setkey) { echo "#!$SETKEYPATH -f" echo echo "# Configuration for $MYADDR" echo echo '# This file has been automatically generated. Do not modify by hand,' echo "# add your own rules to $SETKEYCUSTOMCONF instead." echo echo '# Flush SAD and SPD' echo 'flush;' echo 'spdflush;' echo if [[ -f "$SETKEYCUSTOMCONF" ]]; then echo "# Begin custom rules from $SETKEYCUSTOMCONF" cat "$SETKEYCUSTOMCONF" echo "# End custom rules from $SETKEYCUSTOMCONF" echo fi echo "$AUTOMATIC_MARKER" for node in $(cut -d ' ' -f2 "$NODES") ; do if [[ "$node" != "$MYADDR" ]]; then # Traffic to ignore for port in $TCPTOIGNORE ; do echo "spdadd $MYADDR[$port] $node tcp -P out none;" echo "spdadd $node $MYADDR[$port] tcp -P in none;" echo "spdadd $MYADDR $node[$port] tcp -P out none;" echo "spdadd $node[$port] $MYADDR tcp -P in none;" done # IPsec ESP rules echo "add $MYADDR $node esp 0x201 -E $CRYPTALGO $KEY;" echo "add $node $MYADDR esp 0x201 -E $CRYPTALGO $KEY;" for proto in $PROTOSTOSEC ; do echo "spdadd $MYADDR $node $proto -P out ipsec esp/transport//require;" echo "spdadd $node $MYADDR $proto -P in ipsec esp/transport//require;" done echo fi done } > "$TMPCONF" chmod 400 "$TMPCONF" update_system_ipsec "$TMPCONF" } regen_ipsec_conf() { # Reconfigure IPsec on the system when a new key is generated # on the master (assuming the current configuration is working # and a new key is about to be generated on the master). if [[ ! -f "$KEYPATH" ]]; then echo 'Asking to regenerate with new key, but no old key.' >&2 exit 1 fi local CURSEQNO=$(gather_key_seqno) update_keyfile local NEWSEQNO=$(gather_key_seqno) while [[ $NEWSEQNO -le $CURSEQNO ]]; do # Master did not update yet, wait.. sleep $REGEN_KEY_WAIT update_keyfile NEWSEQNO=$(gather_key_seqno) done update_ipsec_conf } clean_ipsec_conf() { # Unconfigure IPsec on the system, removing the key and # the rules previously generated. rm -f "$KEYPATH" local TMPCONF=$(mktemp) CLEANUP+=("rm -f $TMPCONF") # Remove all auto-generated rules sed "/$AUTOMATIC_MARKER/q" "$SETKEYCONF" > "$TMPCONF" chmod 400 "$TMPCONF" update_system_ipsec "$TMPCONF" } generate_secret() { # Generate a random HEX string (length specified by global variable KEYSIZE) python -c "from ganeti import utils; print(utils.GenerateSecret($KEYSIZE))" } gen_key() { # Generate a new random key to be used for IPsec, the key is associated with # a sequence number. local KEY=$(generate_secret) if [[ ! -f "$KEYPATH" ]]; then # New environment/cluster, let's start from scratch local SEQNO="0" else local SEQNO=$(( $(gather_key_seqno) + 1 )) fi local TMPKEYPATH=$(mktemp) CLEANUP+=("rm -f $TMPKEYPATH") echo -n "$SEQNO 0x$KEY" > "$TMPKEYPATH" chmod 400 "$TMPKEYPATH" mv "$TMPKEYPATH" "$KEYPATH" } trap cleanup EXIT hooks_path="$GANETI_HOOKS_PATH" if [[ ! -n "$hooks_path" ]]; then echo '\$GANETI_HOOKS_PATH not specified.' >&2 exit 1 fi hooks_phase="$GANETI_HOOKS_PHASE" if [[ ! -n "$hooks_phase" ]]; then echo '\$GANETI_HOOKS_PHASE not specified.' >&2 exit 1 fi if [[ "$hooks_phase" = post ]]; then case "$hooks_path" in cluster-init) gen_key ;; cluster-destroy) clean_ipsec_conf ;; cluster-regenkey) # This hook path is not yet implemented in Ganeti, here we suppose it # runs on all the nodes. MASTERNAME=$(< "$MASTERNAME_FILE") if [[ "$MASTERNAME" = "$HOSTNAME" ]]; then gen_key update_ipsec_conf else regen_ipsec_conf fi ;; node-add) update_ipsec_conf ;; node-remove) node_name="$GANETI_NODE_NAME" if [[ ! -n "$node_name" ]]; then echo '\$GANETI_NODE_NAME not specified.' >&2 exit 1 fi if [[ "$node_name" = "$HOSTNAME" ]]; then clean_ipsec_conf else update_ipsec_conf fi ;; *) echo "Hooks path $hooks_path is not for us." >&2 ;; esac else echo "Hooks phase $hooks_phase is not for us." >&2 fi ganeti-3.1.0~rc2/doc/examples/rapi_testutils.py000075500000000000000000000051461476477700300216250ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Example for using L{ganeti.rapi.testutils}""" import logging from ganeti import rapi import ganeti.rapi.testutils def main(): # Disable log output logging.getLogger("").setLevel(logging.CRITICAL) cl = rapi.testutils.InputTestClient() print("Testing features ...") assert isinstance(cl.GetFeatures(), list) print("Testing node evacuation ...") result = cl.EvacuateNode("inst1.example.com", mode=rapi.client.NODE_EVAC_PRI) assert result is NotImplemented print("Testing listing instances ...") for bulk in [False, True]: result = cl.GetInstances(bulk=bulk) assert result is NotImplemented print("Testing renaming instance ...") result = cl.RenameInstance("inst1.example.com", "inst2.example.com") assert result is NotImplemented print("Testing renaming instance with error ...") try: # This test deliberately uses an invalid value for the boolean parameter # "ip_check" result = cl.RenameInstance("inst1.example.com", "inst2.example.com", ip_check=["non-boolean", "value"]) except rapi.testutils.VerificationError: # Verification failed as expected pass else: raise Exception("This test should have failed") print("Success!") if __name__ == "__main__": main() ganeti-3.1.0~rc2/doc/examples/systemd/000075500000000000000000000000001476477700300176575ustar00rootroot00000000000000ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-common.service.in000064400000000000000000000002361476477700300244040ustar00rootroot00000000000000[Unit] Description = Ganeti one-off setup [Service] Type = oneshot ExecStart = @PKGLIBDIR@/ensure-dirs # Disable the start rate limiting StartLimitBurst = 0 ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-confd.service.in000064400000000000000000000010771476477700300242110ustar00rootroot00000000000000[Unit] Description = Ganeti configuration daemon (confd) Documentation = man:ganeti-confd(8) Requires = ganeti-common.service After = ganeti-common.service PartOf = ganeti-node.target ConditionPathExists = @LOCALSTATEDIR@/lib/ganeti/config.data [Service] Type = simple User = @GNTCONFDUSER@ Group = @GNTCONFDGROUP@ EnvironmentFile = -@SYSCONFDIR@/default/ganeti EnvironmentFile = -@LOCALSTATEDIR@/lib/ganeti/ganeti-confd.onetime.conf ExecStart = @SBINDIR@/ganeti-confd -f $CONFD_ARGS $ONETIME_ARGS Restart = on-failure [Install] WantedBy = ganeti-node.target ganeti.target ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-kvmd.service.in000064400000000000000000000007401476477700300240550ustar00rootroot00000000000000[Unit] Description = Ganeti KVM daemon (kvmd) Documentation = man:ganeti-kvmd(8) Requires = ganeti-common.service After = ganeti-common.service PartOf = ganeti-noded.target [Service] Type = simple Group = @GNTDAEMONSGROUP@ EnvironmentFile = -@SYSCONFDIR@/default/ganeti EnvironmentFile = -@LOCALSTATEDIR@/lib/ganeti/ganeti-kvmd.onetime.conf ExecStart = @SBINDIR@/ganeti-kvmd -f $KVMD_ARGS $ONETIME_ARGS Restart = on-failure [Install] WantedBy = ganeti-node.target ganeti.target ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-luxid.service.in000064400000000000000000000011241476477700300242360ustar00rootroot00000000000000[Unit] Description = Ganeti query daemon (luxid) Documentation = man:ganeti-luxid(8) Requires = ganeti-common.service After = ganeti-common.service PartOf = ganeti-master.target ConditionPathExists = @LOCALSTATEDIR@/lib/ganeti/config.data [Service] Type = simple User = @GNTLUXIDUSER@ Group = @GNTLUXIDGROUP@ EnvironmentFile = -@SYSCONFDIR@/default/ganeti EnvironmentFile = -@LOCALSTATEDIR@/lib/ganeti/ganeti-luxid.onetime.conf ExecStart = @SBINDIR@/ganeti-luxid -f $LUXID_ARGS $ONETIME_ARGS Restart = on-failure SuccessExitStatus = 0 11 [Install] WantedBy = ganeti-master.target ganeti.target ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-master.target000064400000000000000000000002401476477700300236230ustar00rootroot00000000000000[Unit] Description = Ganeti master functionality Documentation = man:ganeti(7) After = syslog.target PartOf = ganeti.target [Install] WantedBy = ganeti.target ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-metad.service.in000064400000000000000000000011261476477700300242050ustar00rootroot00000000000000[Unit] Description = Ganeti instance metadata daemon (metad) Requires = ganeti-common.service After = ganeti-common.service PartOf = ganeti-noded.target [Service] Type = simple User = @GNTMETADUSER@ Group = @GNTMETADGROUP@ EnvironmentFile = -@SYSCONFDIR@/default/ganeti EnvironmentFile = -@LOCALSTATEDIR@/lib/ganeti/ganeti-metad.onetime.conf ExecStart = @SBINDIR@/ganeti-metad -f $METAD_ARGS $ONETIME_ARGS Restart = on-failure CapabilityBoundingSet=CAP_NET_BIND_SERVICE AmbientCapabilities=CAP_NET_BIND_SERVICE # ganeti-metad is started on-demand by noded, so there must be no Install # section. ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-mond.service.in000064400000000000000000000007701476477700300240540ustar00rootroot00000000000000[Unit] Description = Ganeti monitoring daemon (mond) Documentation = man:ganeti-mond(8) Requires = ganeti-common.service After = ganeti-common.service PartOf = ganeti-node.target [Service] Type = simple User = @GNTMONDUSER@ Group = @GNTMONDGROUP@ EnvironmentFile = -@SYSCONFDIR@/default/ganeti EnvironmentFile = -@LOCALSTATEDIR@/lib/ganeti/ganeti-mond.onetime.conf ExecStart = @SBINDIR@/ganeti-mond -f $MOND_ARGS $ONETIME_ARGS Restart = on-failure [Install] WantedBy = ganeti-node.target ganeti.target ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-node.target000064400000000000000000000002371476477700300232630ustar00rootroot00000000000000[Unit] Description = Ganeti node functionality Documentation = man:ganeti(7) After = syslog.service PartOf = ganeti.target [Install] WantedBy = ganeti.target ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-noded.service.in000064400000000000000000000011631476477700300242050ustar00rootroot00000000000000[Unit] Description = Ganeti node daemon (noded) Documentation = man:ganeti-noded(8) After = ganeti-common.service Requires = ganeti-common.service PartOf = ganeti-node.target ConditionPathExists = @LOCALSTATEDIR@/lib/ganeti/server.pem [Service] Type = simple User = @GNTNODEDUSER@ Group = @GNTNODEDGROUP@ EnvironmentFile = -@SYSCONFDIR@/default/ganeti EnvironmentFile = -@LOCALSTATEDIR@/lib/ganeti/ganeti-noded.onetime.conf ExecStart = @SBINDIR@/ganeti-noded -f $NODED_ARGS $ONETIME_ARGS Restart = on-failure # Important: do not kill any KVM processes KillMode = process [Install] WantedBy = ganeti-node.target ganeti.target ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-rapi.service.in000064400000000000000000000011521476477700300240450ustar00rootroot00000000000000[Unit] Description = Ganeti RAPI daemon (rapi) Documentation = man:ganeti-rapi(8) Requires = ganeti-common.service Requisite = ganeti-luxid.service After = ganeti-common.service PartOf = ganeti-master.target ConditionPathExists = @LOCALSTATEDIR@/lib/ganeti/rapi.pem [Service] Type = simple User = @GNTRAPIUSER@ Group = @GNTRAPIGROUP@ EnvironmentFile = -@SYSCONFDIR@/default/ganeti EnvironmentFile = -@LOCALSTATEDIR@/lib/ganeti/ganeti-rapi.onetime.conf ExecStart = @SBINDIR@/ganeti-rapi -f $RAPI_ARGS $ONETIME_ARGS SuccessExitStatus = 0 11 Restart = on-failure [Install] WantedBy = ganeti-master.target ganeti.target ganeti-3.1.0~rc2/doc/examples/systemd/ganeti-wconfd.service.in000064400000000000000000000011431476477700300243720ustar00rootroot00000000000000[Unit] Description = Ganeti config writer daemon (wconfd) Documentation = man:ganeti-wconfd(8) Requires = ganeti-common.service After = ganeti-common.service PartOf = ganeti-master.target ConditionPathExists = @LOCALSTATEDIR@/lib/ganeti/config.data [Service] Type = simple User = @GNTWCONFDUSER@ Group = @GNTWCONFDGROUP@ EnvironmentFile = -@SYSCONFDIR@/default/ganeti EnvironmentFile = -@LOCALSTATEDIR@/lib/ganeti/ganeti-wconfd.onetime.conf ExecStart = @SBINDIR@/ganeti-wconfd -f $WCONFD_ARGS $ONETIME_ARGS Restart = on-failure SuccessExitStatus = 0 11 [Install] WantedBy = ganeti-master.target ganeti.target ganeti-3.1.0~rc2/doc/examples/systemd/ganeti.service000064400000000000000000000011521476477700300225070ustar00rootroot00000000000000# This is a dummy service, provided only for compatibility with SysV. # Systemd will automatically create a SysV service called # ganeti.service, attempting to start the initscript. Since there is no # way to tell systemd that the initscript acts as ganeti.target (and not # ganeti.service), we create a stub service requiring ganeti.target. # # This service is for compatibility only and so will not be marked for # installation. [Unit] Description = Dummy Ganeti SysV compatibility service Documentation = man:ganeti(7) After = ganeti.target Requires = ganeti.target [Service] Type = oneshot ExecStart = /bin/true ganeti-3.1.0~rc2/doc/examples/systemd/ganeti.target000064400000000000000000000003101476477700300223300ustar00rootroot00000000000000[Unit] Description = Ganeti virtualization cluster manager Documentation = man:ganeti(7) PartOf = ganeti.service [Install] WantedBy = multi-user.target Also = ganeti-node.target ganeti-master.target ganeti-3.1.0~rc2/doc/glossary.rst000064400000000000000000000061321476477700300167500ustar00rootroot00000000000000======== Glossary ======== .. if you add new entries, keep the alphabetical sorting! .. glossary:: :sorted: ballooning A term describing dynamic changes to an instance's memory while the instance is running that don't require an instance reboot. Depending on the hypervisor and configuration, changes may be automatically initiated by the hypervisor (based on the memory usage of the node and instance), or may need to be initiated manually. BE parameter BE stands for *backend*. BE parameters are hypervisor-independent instance parameters, such as the amount of RAM/virtual CPUs allocated to an instance. DRBD A block device driver that can be used to build RAID1 across the network or across shared storage, while using only locally-attached storage. HV parameter HV stands for *hypervisor*. HV parameters describe the virtualization- specific aspects of the instance. For example, a HV parameter might describe what kernel (if any) to use to boot the instance or what emulation model to use for the emulated hard drives. HVM *Hardware Virtualization Mode*. In this mode, the virtual machine is oblivious to the fact that it is virtualized and all its hardware is emulated. LogicalUnit The code associated with an :term:`OpCode`; for example, the code that implements the startup of an instance. LUXI Local UniX Interface. The IPC method over :manpage:`unix(7)` sockets used between the CLI tools/RAPI daemon and the master daemon. OOB *Out of Band*. This term describes methods of accessing a machine (or parts of a machine) by means other than the usual network connection. Examples include accessing a remote server via a physical serial console or via a virtual console. IPMI is also considered OOB access. OpCode A data structure encapsulating a basic cluster operation; for example: start instance, add instance, etc. PVM (Xen) *Para-virtualization mode*. In this mode, the virtual machine is aware that it is virtualized; therefore, there is no need for hardware emulation or virtualization. SoR *State of Record*. Refers to values/properties that come from an authoritative configuration source. For example, the maximum VCPU over- subscription ratio is a SoR value, but the current over-subscription ratio (based upon how many instances live on the node) is a :term:`SoW` value. SoW *State of the World*. Refers to values that directly describe the world, as opposed to values that come from the configuration (which are considered :term:`SoR`). tmem Xen Transcendent Memory (http://en.wikipedia.org/wiki/Transcendent_memory). tmem is a mechanism used by Xen to provide memory over-subscription. watcher :command:`ganeti-watcher` is a tool that should be run regularly from cron. The tool executes tasks such as restarting failed instances and restarting secondary DRBD devices. For more details, see the man page :manpage:`ganeti-watcher(8)`. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/hooks.rst000064400000000000000000000544341476477700300162400ustar00rootroot00000000000000Ganeti customisation using hooks ================================ Documents Ganeti version 3.1 .. contents:: Introduction ------------ In order to allow customisation of operations, Ganeti runs scripts in sub-directories of ``@SYSCONFDIR@/ganeti/hooks`` (that is usually ``/etc/ganeti/hooks``). These sub-directories are named ``$hook-$phase.d``, where ``$phase`` is either ``pre`` or ``post`` and ``$hook`` matches the directory name given for a hook (e.g. ``cluster-verify-post.d`` or ``node-add-pre.d``). This is similar to the ``/etc/network/`` structure present in Debian for network interface handling. Note that Ganeti does not create its ``hooks`` directory by default. If you want to use hooks scripts, create it on all nodes. This applies also to all sub directories such as ``node-add-pre.d``. Organisation ------------ For every operation, two sets of scripts are run: - pre phase (for authorization/checking) - post phase (for logging) Also, for each operation, the scripts are run on one or more nodes, depending on the operation type. Note that, even though we call them scripts, we are actually talking about any executable. The filenames of the scripts need to match the regular expression ``^[a-zA-Z0-9_-]+$``. This means in particular, that scripts having a filename extension (such as ``myhook.sh``) are silently ignored by Ganeti. *pre* scripts ~~~~~~~~~~~~~ The *pre* scripts have a definite target: to check that the operation is allowed given the site-specific constraints. You could have, for example, a rule that says every new instance is required to exists in a database; to implement this, you could write a script that checks the new instance parameters against your database. The objective of these scripts should be their return code (zero or non-zero for success and failure). However, if they modify the environment in any way, they should be idempotent, as failed executions could be restarted and thus the script(s) run again with exactly the same parameters. Note that if a node is unreachable at the time a hooks is run, this will not be interpreted as a deny for the execution. In other words, only an actual error returned from a script will cause abort, and not an unreachable node. Therefore, if you want to guarantee that a hook script is run and denies an action, it's best to put it on the master node. *post* scripts ~~~~~~~~~~~~~~ These scripts should do whatever you need as a reaction to the completion of an operation. Their return code is not checked (but logged), and they should not depend on the fact that the *pre* scripts have been run. Naming ~~~~~~ The allowed names for the scripts consist of (similar to *run-parts*) upper and lower case, digits, underscores and hyphens. In other words, the regexp ``^[a-zA-Z0-9_-]+$``. Also, non-executable scripts will be ignored. Order of execution ~~~~~~~~~~~~~~~~~~ On a single node, the scripts in a directory are run in lexicographic order (more exactly, the python string comparison order). It is advisable to implement the usual *NN-name* convention where *NN* is a two digit number. For an operation whose hooks are run on multiple nodes, there is no specific ordering of nodes with regard to hooks execution; you should assume that the scripts are run in parallel on the target nodes (keeping on each node the above specified ordering). If you need any kind of inter-node synchronisation, you have to implement it yourself in the scripts. Execution environment ~~~~~~~~~~~~~~~~~~~~~ The scripts will be run as follows: - no command line arguments - no controlling *tty* - stdin is actually */dev/null* - stdout and stderr are directed to files - PATH is reset to :pyeval:`constants.HOOKS_PATH` - the environment is cleared, and only ganeti-specific variables will be left All information about the cluster is passed using environment variables. Different operations will have sligthly different environments, but most of the variables are common. Operation list -------------- Node operations ~~~~~~~~~~~~~~~ OP_NODE_ADD +++++++++++ Adds a node to the cluster. :directory: node-add :env. vars: NODE_NAME, NODE_PIP, NODE_SIP, MASTER_CAPABLE, VM_CAPABLE :pre-execution: all existing nodes :post-execution: all nodes plus the new node OP_NODE_REMOVE ++++++++++++++ Removes a node from the cluster. On the removed node the hooks are called during the execution of the operation and not after its completion. :directory: node-remove :env. vars: NODE_NAME :pre-execution: all existing nodes except the removed node :post-execution: all existing nodes OP_NODE_SET_PARAMS ++++++++++++++++++ Changes a node's parameters. :directory: node-modify :env. vars: MASTER_CANDIDATE, OFFLINE, DRAINED, MASTER_CAPABLE, VM_CAPABLE :pre-execution: master node, the target node :post-execution: master node, the target node OP_NODE_MIGRATE ++++++++++++++++ Relocate secondary instances from a node. :directory: node-migrate :env. vars: NODE_NAME :pre-execution: master node :post-execution: master node Node group operations ~~~~~~~~~~~~~~~~~~~~~ OP_GROUP_ADD ++++++++++++ Adds a node group to the cluster. :directory: group-add :env. vars: GROUP_NAME :pre-execution: master node :post-execution: master node OP_GROUP_SET_PARAMS +++++++++++++++++++ Changes a node group's parameters. :directory: group-modify :env. vars: GROUP_NAME, NEW_ALLOC_POLICY :pre-execution: master node :post-execution: master node OP_GROUP_REMOVE +++++++++++++++ Removes a node group from the cluster. Since the node group must be empty for removal to succeed, the concept of "nodes in the group" does not exist, and the hook is only executed in the master node. :directory: group-remove :env. vars: GROUP_NAME :pre-execution: master node :post-execution: master node OP_GROUP_RENAME +++++++++++++++ Renames a node group. :directory: group-rename :env. vars: OLD_NAME, NEW_NAME :pre-execution: master node and all nodes in the group :post-execution: master node and all nodes in the group OP_GROUP_EVACUATE +++++++++++++++++ Evacuates a node group. :directory: group-evacuate :env. vars: GROUP_NAME, TARGET_GROUPS :pre-execution: master node and all nodes in the group :post-execution: master node and all nodes in the group Network operations ~~~~~~~~~~~~~~~~~~ OP_NETWORK_ADD ++++++++++++++ Adds a network to the cluster. :directory: network-add :env. vars: NETWORK_NAME, NETWORK_SUBNET, NETWORK_GATEWAY, NETWORK_SUBNET6, NETWORK_GATEWAY6, NETWORK_MAC_PREFIX, NETWORK_TAGS :pre-execution: master node :post-execution: master node OP_NETWORK_REMOVE +++++++++++++++++ Removes a network from the cluster. :directory: network-remove :env. vars: NETWORK_NAME :pre-execution: master node :post-execution: master node OP_NETWORK_RENAME +++++++++++++++++ Renames a network. :directory: network-rename :env. vars: OLD_NAME, NEW_NAME :pre-execution: master node and all nodes belonging to connected groups :post-execution: master node and all nodes belonging to connected groups OP_NETWORK_CONNECT ++++++++++++++++++ Connects a network to a nodegroup. :directory: network-connect :env. vars: GROUP_NAME, NETWORK_NAME, GROUP_NETWORK_MODE, GROUP_NETWORK_LINK, GROUP_NETWORK_VLAN, NETWORK_SUBNET, NETWORK_GATEWAY, NETWORK_SUBNET6, NETWORK_GATEWAY6, NETWORK_MAC_PREFIX, NETWORK_TAGS :pre-execution: nodegroup nodes :post-execution: nodegroup nodes OP_NETWORK_DISCONNECT +++++++++++++++++++++ Disconnects a network from a nodegroup. :directory: network-disconnect :env. vars: GROUP_NAME, NETWORK_NAME, GROUP_NETWORK_MODE, GROUP_NETWORK_LINK, GROUP_NETWORK_VLAN, NETWORK_SUBNET, NETWORK_GATEWAY, NETWORK_SUBNET6, NETWORK_GATEWAY6, NETWORK_MAC_PREFIX, NETWORK_TAGS :pre-execution: nodegroup nodes :post-execution: nodegroup nodes OP_NETWORK_SET_PARAMS +++++++++++++++++++++ Modifies a network. :directory: network-modify :env. vars: NETWORK_NAME, NETWORK_SUBNET, NETWORK_GATEWAY, NETWORK_SUBNET6, NETWORK_GATEWAY6, NETWORK_MAC_PREFIX, NETWORK_TAGS :pre-execution: master node :post-execution: master node Instance operations ~~~~~~~~~~~~~~~~~~~ All instance operations take at least the following variables: INSTANCE_NAME, INSTANCE_PRIMARY, INSTANCE_SECONDARY, INSTANCE_OS_TYPE, INSTANCE_DISK_TEMPLATE, INSTANCE_MEMORY, INSTANCE_DISK_SIZES, INSTANCE_VCPUS, INSTANCE_NIC_COUNT, INSTANCE_NICn_IP, INSTANCE_NICn_BRIDGE, INSTANCE_NICn_MAC, INSTANCE_NICn_NETWORK, INSTANCE_NICn_NETWORK_UUID, INSTANCE_NICn_NETWORK_SUBNET, INSTANCE_NICn_NETWORK_GATEWAY, INSTANCE_NICn_NETWORK_SUBNET6, INSTANCE_NICn_NETWORK_GATEWAY6, INSTANCE_NICn_NETWORK_MAC_PREFIX, INSTANCE_DISK_COUNT, INSTANCE_DISKn_SIZE, INSTANCE_DISKn_MODE, INSTANCE_DISKn_NAME, INSTANCE_DISKn_UUID, INSTANCE_DISKn_DEV_TYPE. The INSTANCE_NICn_* and INSTANCE_DISKn_* variables represent the properties of the *n* -th NIC and disk, and are zero-indexed. Depending on the disk template, Ganeti exports some info related to the logical id of the disk, that is basically its driver and id. The INSTANCE_NICn_NETWORK_* variables are only passed if a NIC's network parameter is set (that is if the NIC is associated to a network defined via ``gnt-network``) OP_INSTANCE_CREATE ++++++++++++++++++ Creates a new instance. :directory: instance-add :env. vars: ADD_MODE, SRC_NODE, SRC_PATH, SRC_IMAGES :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_INSTANCE_REINSTALL +++++++++++++++++++++ Reinstalls an instance. :directory: instance-reinstall :env. vars: only the standard instance vars :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_BACKUP_EXPORT ++++++++++++++++ Exports the instance. :directory: instance-export :env. vars: EXPORT_MODE, EXPORT_NODE, EXPORT_DO_SHUTDOWN, REMOVE_INSTANCE :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_INSTANCE_STARTUP +++++++++++++++++++ Starts an instance. :directory: instance-start :env. vars: FORCE :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_INSTANCE_SHUTDOWN ++++++++++++++++++++ Stops an instance. :directory: instance-stop :env. vars: TIMEOUT :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_INSTANCE_REBOOT ++++++++++++++++++ Reboots an instance. :directory: instance-reboot :env. vars: IGNORE_SECONDARIES, REBOOT_TYPE, SHUTDOWN_TIMEOUT :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_INSTANCE_SET_PARAMS ++++++++++++++++++++++ Modifies the instance parameters. :directory: instance-modify :env. vars: NEW_DISK_TEMPLATE, RUNTIME_MEMORY :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_INSTANCE_FAILOVER ++++++++++++++++++++ Failovers an instance. In the post phase INSTANCE_PRIMARY and INSTANCE_SECONDARY refer to the nodes that were repectively primary and secondary before failover. :directory: instance-failover :env. vars: IGNORE_CONSISTENCY, SHUTDOWN_TIMEOUT, OLD_PRIMARY, OLD_SECONDARY, NEW_PRIMARY, NEW_SECONDARY :pre-execution: master node, secondary (target) node :post-execution: master node, primary (source) and secondary (target) nodes OP_INSTANCE_MIGRATE ++++++++++++++++++++ Migrates an instance. In the post phase INSTANCE_PRIMARY and INSTANCE_SECONDARY refer to the nodes that were repectively primary and secondary before migration. :directory: instance-migrate :env. vars: MIGRATE_LIVE, MIGRATE_CLEANUP, OLD_PRIMARY, OLD_SECONDARY, NEW_PRIMARY, NEW_SECONDARY :pre-execution: master node, primary (source) and secondary (target) nodes :post-execution: master node, primary (source) and secondary (target) nodes OP_INSTANCE_REMOVE ++++++++++++++++++ Remove an instance. :directory: instance-remove :env. vars: SHUTDOWN_TIMEOUT :pre-execution: master node :post-execution: master node, primary and secondary nodes OP_INSTANCE_GROW_DISK +++++++++++++++++++++ Grows the disk of an instance. :directory: disk-grow :env. vars: DISK, AMOUNT :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_INSTANCE_RENAME ++++++++++++++++++ Renames an instance. :directory: instance-rename :env. vars: INSTANCE_NEW_NAME :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_INSTANCE_MOVE ++++++++++++++++ Move an instance by data-copying. :directory: instance-move :env. vars: TARGET_NODE, SHUTDOWN_TIMEOUT :pre-execution: master node, primary and target nodes :post-execution: master node, primary and target nodes OP_INSTANCE_RECREATE_DISKS ++++++++++++++++++++++++++ Recreate an instance's missing disks. :directory: instance-recreate-disks :env. vars: only the standard instance vars :pre-execution: master node, primary and secondary nodes :post-execution: master node, primary and secondary nodes OP_INSTANCE_REPLACE_DISKS +++++++++++++++++++++++++ Replace the disks of an instance. :directory: mirrors-replace :env. vars: MODE, NEW_SECONDARY, OLD_SECONDARY :pre-execution: master node, primary and new secondary nodes :post-execution: master node, primary and new secondary nodes OP_INSTANCE_CHANGE_GROUP ++++++++++++++++++++++++ Moves an instance to another group. :directory: instance-change-group :env. vars: TARGET_GROUPS :pre-execution: master node :post-execution: master node Cluster operations ~~~~~~~~~~~~~~~~~~ OP_CLUSTER_POST_INIT ++++++++++++++++++++ This hook is called via a special "empty" LU right after cluster initialization. :directory: cluster-init :env. vars: none :pre-execution: none :post-execution: master node OP_CLUSTER_DESTROY ++++++++++++++++++ The post phase of this hook is called during the execution of destroy operation and not after its completion. :directory: cluster-destroy :env. vars: none :pre-execution: none :post-execution: master node OP_CLUSTER_VERIFY_GROUP +++++++++++++++++++++++ Verifies all nodes in a group. This is a special LU with regard to hooks, as the result of the opcode will be combined with the result of post-execution hooks, in order to allow administrators to enhance the cluster verification procedure. :directory: cluster-verify :env. vars: CLUSTER, MASTER, CLUSTER_TAGS, NODE_TAGS_ :pre-execution: none :post-execution: all nodes in a group OP_CLUSTER_RENAME +++++++++++++++++ Renames the cluster. :directory: cluster-rename :env. vars: NEW_NAME :pre-execution: master-node :post-execution: master-node OP_CLUSTER_SET_PARAMS +++++++++++++++++++++ Modifies the cluster parameters. :directory: cluster-modify :env. vars: NEW_VG_NAME :pre-execution: master node :post-execution: master node Virtual operation :pyeval:`constants.FAKE_OP_MASTER_TURNUP` +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ This doesn't correspond to an actual op-code, but it is called when the master IP is activated. :directory: master-ip-turnup :env. vars: MASTER_NETDEV, MASTER_IP, MASTER_NETMASK, CLUSTER_IP_VERSION :pre-execution: master node :post-execution: master node Virtual operation :pyeval:`constants.FAKE_OP_MASTER_TURNDOWN` +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ This doesn't correspond to an actual op-code, but it is called when the master IP is deactivated. :directory: master-ip-turndown :env. vars: MASTER_NETDEV, MASTER_IP, MASTER_NETMASK, CLUSTER_IP_VERSION :pre-execution: master node :post-execution: master node Obsolete operations ~~~~~~~~~~~~~~~~~~~ The following operations are no longer present or don't execute hooks anymore in Ganeti 2.0: - OP_INIT_CLUSTER - OP_MASTER_FAILOVER - OP_INSTANCE_ADD_MDDRBD - OP_INSTANCE_REMOVE_MDDRBD Environment variables --------------------- Note that all variables listed here are actually prefixed with *GANETI_* in order to provide a clear namespace. In addition, post-execution scripts receive another set of variables, prefixed with *GANETI_POST_*, representing the status after the opcode executed. Common variables ~~~~~~~~~~~~~~~~ This is the list of environment variables supported by all operations: HOOKS_VERSION Documents the hooks interface version. In case this doesnt match what the script expects, it should not run. The documents conforms to the version 2. HOOKS_PHASE One of *PRE* or *POST* denoting which phase are we in. CLUSTER The cluster name. MASTER The master node. OP_CODE One of the *OP_* values from the list of operations. OBJECT_TYPE One of ``INSTANCE``, ``NODE``, ``CLUSTER``. DATA_DIR The path to the Ganeti configuration directory (to read, for example, the *ssconf* files). Specialised variables ~~~~~~~~~~~~~~~~~~~~~ This is the list of variables which are specific to one or more operations. CLUSTER_IP_VERSION IP version of the master IP (4 or 6) INSTANCE_NAME The name of the instance which is the target of the operation. INSTANCE_BE_x,y,z,... Instance BE params. There is one variable per BE param. For instance, GANETI_INSTANCE_BE_auto_balance INSTANCE_DISK_TEMPLATE The disk type for the instance. NEW_DISK_TEMPLATE The new disk type for the instance. INSTANCE_DISK_COUNT The number of disks for the instance. INSTANCE_DISKn_SIZE The size of disk *n* for the instance. INSTANCE_DISKn_MODE Either *rw* for a read-write disk or *ro* for a read-only one. INSTANCE_HV_x,y,z,... Instance hypervisor options. There is one variable per option. For instance, GANETI_INSTANCE_HV_use_bootloader INSTANCE_HYPERVISOR The instance hypervisor. INSTANCE_NIC_COUNT The number of NICs for the instance. INSTANCE_NICn_BRIDGE The bridge to which the *n* -th NIC of the instance is attached. INSTANCE_NICn_IP The IP (if any) of the *n* -th NIC of the instance. INSTANCE_NICn_MAC The MAC address of the *n* -th NIC of the instance. INSTANCE_NICn_MODE The mode of the *n* -th NIC of the instance. INSTANCE_OS_TYPE The name of the instance OS. INSTANCE_PRIMARY The name of the node which is the primary for the instance. Note that for migrations/failovers, you shouldn't rely on this variable since the nodes change during the exectution, but on the OLD_PRIMARY/NEW_PRIMARY values. INSTANCE_SECONDARY Space-separated list of secondary nodes for the instance. Note that for migrations/failovers, you shouldn't rely on this variable since the nodes change during the exectution, but on the OLD_SECONDARY/NEW_SECONDARY values. INSTANCE_MEMORY The memory size (in MiBs) of the instance. INSTANCE_VCPUS The number of virtual CPUs for the instance. INSTANCE_STATUS The run status of the instance. MASTER_CAPABLE Whether a node is capable of being promoted to master. VM_CAPABLE Whether the node can host instances. MASTER_NETDEV Network device of the master IP MASTER_IP The master IP MASTER_NETMASK Netmask of the master IP INSTANCE_TAGS A space-delimited list of the instance's tags. NODE_NAME The target node of this operation (not the node on which the hook runs). NODE_PIP The primary IP of the target node (the one over which inter-node communication is done). NODE_SIP The secondary IP of the target node (the one over which drbd replication is done). This can be equal to the primary ip, in case the cluster is not dual-homed. FORCE This is provided by some operations when the user gave this flag. IGNORE_CONSISTENCY The user has specified this flag. It is used when failing over instances in case the primary node is down. ADD_MODE The mode of the instance create: either *create* for create from scratch or *import* for restoring from an exported image. SRC_NODE, SRC_PATH, SRC_IMAGE In case the instance has been added by import, these variables are defined and point to the source node, source path (the directory containing the image and the config file) and the source disk image file. NEW_SECONDARY The name of the node on which the new mirror component is being added (for replace disk). This can be the name of the current secondary, if the new mirror is on the same secondary. For migrations/failovers, this is the old primary node. OLD_SECONDARY The name of the old secondary in the replace-disks command. Note that this can be equal to the new secondary if the secondary node hasn't actually changed. For migrations/failovers, this is the new primary node. OLD_PRIMARY, NEW_PRIMARY For migrations/failovers, the old and respectively new primary nodes. These two mirror the NEW_SECONDARY/OLD_SECONDARY variables EXPORT_MODE The instance export mode. Either "remote" or "local". EXPORT_NODE The node on which the exported image of the instance was done. EXPORT_DO_SHUTDOWN This variable tells if the instance has been shutdown or not while doing the export. In the "was shutdown" case, it's likely that the filesystem is consistent, whereas in the "did not shutdown" case, the filesystem would need a check (journal replay or full fsck) in order to guarantee consistency. REMOVE_INSTANCE Whether the instance was removed from the node. SHUTDOWN_TIMEOUT Amount of time to wait for the instance to shutdown. TIMEOUT Amount of time to wait before aborting the op. OLD_NAME, NEW_NAME Old/new name of the node group. GROUP_NAME The name of the node group. NEW_ALLOC_POLICY The new allocation policy for the node group. CLUSTER_TAGS The list of cluster tags, space separated. NODE_TAGS_ The list of tags for node **, space separated. Examples -------- The startup of an instance will pass this environment to the hook script:: GANETI_CLUSTER=cluster1.example.com GANETI_DATA_DIR=/var/lib/ganeti GANETI_FORCE=False GANETI_HOOKS_PATH=instance-start GANETI_HOOKS_PHASE=post GANETI_HOOKS_VERSION=2 GANETI_INSTANCE_DISK0_MODE=rw GANETI_INSTANCE_DISK0_SIZE=128 GANETI_INSTANCE_DISK_COUNT=1 GANETI_INSTANCE_DISK_TEMPLATE=drbd GANETI_INSTANCE_MEMORY=128 GANETI_INSTANCE_NAME=instance2.example.com GANETI_INSTANCE_NIC0_BRIDGE=xen-br0 GANETI_INSTANCE_NIC0_IP= GANETI_INSTANCE_NIC0_MAC=aa:00:00:a5:91:58 GANETI_INSTANCE_NIC_COUNT=1 GANETI_INSTANCE_OS_TYPE=debootstrap GANETI_INSTANCE_PRIMARY=node3.example.com GANETI_INSTANCE_SECONDARY=node5.example.com GANETI_INSTANCE_STATUS=down GANETI_INSTANCE_VCPUS=1 GANETI_MASTER=node1.example.com GANETI_OBJECT_TYPE=INSTANCE GANETI_OP_CODE=OP_INSTANCE_STARTUP GANETI_OP_TARGET=instance2.example.com .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/iallocator.rst000064400000000000000000000436461476477700300172510ustar00rootroot00000000000000Ganeti automatic instance allocation ==================================== Documents Ganeti version 3.1 .. contents:: Introduction ------------ Currently in Ganeti the admin has to specify the exact locations for an instance's node(s). This prevents a completely automatic node evacuation, and is in general a nuisance. The *iallocator* framework will enable automatic placement via external scripts, which allows customization of the cluster layout per the site's requirements. User-visible changes ~~~~~~~~~~~~~~~~~~~~ There are two parts of the ganeti operation that are impacted by the auto-allocation: how the cluster knows what the allocator algorithms are and how the admin uses these in creating instances. An allocation algorithm is just the filename of a program installed in a defined list of directories. Cluster configuration ~~~~~~~~~~~~~~~~~~~~~ At configure time, the list of the directories can be selected via the ``--with-iallocator-search-path=LIST`` option, where *LIST* is a comma-separated list of directories. If not given, this defaults to ``$libdir/ganeti/iallocators``, i.e. for an installation under ``/usr``, this will be ``/usr/lib/ganeti/iallocators``. Ganeti will then search for allocator script in the configured list, using the first one whose filename matches the one given by the user. Command line interface changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The node selection options in instance add and instance replace disks can be replace by the new ``--iallocator=NAME`` option (shortened to ``-I``), which will cause the auto-assignement of nodes with the passed iallocator. The selected node(s) will be shown as part of the command output. IAllocator API -------------- The protocol for communication between Ganeti and an allocator script will be the following: #. ganeti launches the program with a single argument, a filename that contains a JSON-encoded structure (the input message) #. if the script finishes with exit code different from zero, it is considered a general failure and the full output will be reported to the users; this can be the case when the allocator can't parse the input message #. if the allocator finishes with exit code zero, it is expected to output (on its stdout) a JSON-encoded structure (the response) Input message ~~~~~~~~~~~~~ The input message will be the JSON encoding of a dictionary containing all the required information to perform the operation. We explain the contents of this dictionary in two parts: common information that every type of operation requires, and operation-specific information. Common information ++++++++++++++++++ All input dictionaries to the IAllocator must carry the following keys: version the version of the protocol; this document specifies version 2 cluster_name the cluster name cluster_tags the list of cluster tags enabled_hypervisors the list of enabled hypervisors ipolicy the cluster-wide instance policy (for information; the per-node group values take precedence and should be used instead) request a dictionary containing the details of the request; the keys vary depending on the type of operation that's being requested, as explained in `Operation-specific input`_ below. nodegroups a dictionary with the data for the cluster's node groups; it is keyed on the group UUID, and the values are a dictionary with the following keys: name the node group name alloc_policy the allocation policy of the node group (consult the semantics of this attribute in the :manpage:`gnt-group(8)` manpage) networks the list of network UUID's this node group is connected to ipolicy the instance policy of the node group tags the list of node group tags instances a dictionary with the data for the current existing instance on the cluster, indexed by instance name; the contents are similar to the instance definitions for the allocate mode, with the addition of: admin_state if this instance is set to run (but not the actual status of the instance) nodes list of nodes on which this instance is placed; the primary node of the instance is always the first one nodes dictionary with the data for the nodes in the cluster, indexed by the node name; the dict contains [*]_ : total_disk the total disk size of this node (mebibytes) free_disk the free disk space on the node total_memory the total memory size free_memory free memory on the node; note that currently this does not take into account the instances which are down on the node total_cpus the physical number of CPUs present on the machine; depending on the hypervisor, this might or might not be equal to how many CPUs the node operating system sees; primary_ip the primary IP address of the node secondary_ip the secondary IP address of the node (the one used for the DRBD replication); note that this can be the same as the primary one tags list with the tags of the node master_candidate: a boolean flag denoting whether this node is a master candidate drained: a boolean flag denoting whether this node is being drained offline: a boolean flag denoting whether this node is offline i_pri_memory: total memory required by primary instances i_pri_up_memory: total memory required by running primary instances group: the node group that this node belongs to No allocations should be made on nodes having either the ``drained`` or ``offline`` flags set. More details about these of node status flags is available in the manpage :manpage:`ganeti(7)`. .. [*] Note that no run-time data is present for offline, drained or non-vm_capable nodes; this means the tags total_memory, reserved_memory, free_memory, total_disk, free_disk, total_cpus, i_pri_memory and i_pri_up memory will be absent Operation-specific input ++++++++++++++++++++++++ All input dictionaries to the IAllocator carry, in the ``request`` dictionary, detailed information about the operation that's being requested. The required keys vary depending on the type of operation, as follows. In all cases, it includes: type the request type; this can be either ``allocate``, ``relocate``, ``change-group`` or ``node-evacuate``. The ``allocate`` request is used when a new instance needs to be placed on the cluster. The ``relocate`` request is used when an existing instance needs to be moved within its node group. The ``multi-evacuate`` protocol used to request that the script computes the optimal relocate solution for all secondary instances of the given nodes. It is now deprecated and needs only be implemented if backwards compatibility with Ganeti 2.4 and lower is needed. The ``change-group`` request is used to relocate multiple instances across multiple node groups. ``node-evacuate`` evacuates instances off their node(s). These are described in a separate :ref:`design document `. The ``multi-allocate`` request is used to allocate multiple instances on the cluster. The request is beside of that very similiar to the ``allocate`` one. For more details look at :doc:`Ganeti bulk create `. For both allocate and relocate mode, the following extra keys are needed in the ``request`` dictionary: name the name of the instance; if the request is a realocation, then this name will be found in the list of instances (see below), otherwise is the FQDN of the new instance; type *string* required_nodes how many nodes should the algorithm return; while this information can be deduced from the instace's disk template, it's better if this computation is left to Ganeti as then allocator scripts are less sensitive to changes to the disk templates; type *integer* disk_space_total the total disk space that will be used by this instance on the (new) nodes; again, this information can be computed from the list of instance disks and its template type, but Ganeti is better suited to compute it; type *integer* .. pyassert:: constants.DISK_ACCESS_SET == set([constants.DISK_RDONLY, constants.DISK_RDWR]) Allocation needs, in addition: disks list of dictionaries holding the disk definitions for this instance (in the order they are exported to the hypervisor): mode either :pyeval:`constants.DISK_RDONLY` or :pyeval:`constants.DISK_RDWR` denoting if the disk is read-only or writable size the size of this disk in mebibytes nics a list of dictionaries holding the network interfaces for this instance, containing: ip the IP address that Ganeti know for this instance, or null mac the MAC address for this interface bridge the bridge to which this interface will be connected vcpus the number of VCPUs for the instance disk_template the disk template for the instance memory the memory size for the instance os the OS type for the instance tags the list of the instance's tags hypervisor the hypervisor of this instance Relocation: relocate_from a list of nodes to move the instance away from; for DRBD-based instances, this will contain a single node, the current secondary of the instance, whereas for shared-storage instance, this will contain also a single node, the current primary of the instance; type *list of strings* As for ``node-evacuate``, it needs the following request arguments: instances a list of instance names to evacuate; type *list of strings* evac_mode specify which instances to evacuate; one of ``primary-only``, ``secondary-only``, ``all``, type *string* ``change-group`` needs the following request arguments: instances a list of instance names whose group to change; type *list of strings* target_groups must either be the empty list, or contain a list of group UUIDs that should be considered for relocating instances to; type *list of strings* ``multi-allocate`` needs the following request arguments: instances a list of request dicts MonD data +++++++++ Additional information is available from mond. Mond's data collectors provide information that can help an allocator script make better decisions when allocating a new instance. Mond's information may also be accessible from a mock file mainly for testing purposes. The file will be in JSON format and will present an array of :ref:`report objects `. Response message ~~~~~~~~~~~~~~~~ The response message is much more simple than the input one. It is also a dict having three keys: success a boolean value denoting if the allocation was successful or not info a string with information from the scripts; if the allocation fails, this will be shown to the user result the output of the algorithm; even if the algorithm failed (i.e. success is false), this must be returned as an empty list for allocate/relocate, this is the list of node(s) for the instance; note that the length of this list must equal the ``requested_nodes`` entry in the input message, otherwise Ganeti will consider the result as failed for the ``node-evacuate`` and ``change-group`` modes, this is a dictionary containing, among other information, a list of lists of serialized opcodes; see the :ref:`design document ` for a detailed description for the ``multi-allocate`` mode this is a tuple of 2 lists, the first being element of the tuple is a list of succeeded allocation, with the instance name as first element of each entry and the node placement in the second. The second element of the tuple is the instance list of failed allocations. .. note:: Current Ganeti version accepts either ``result`` or ``nodes`` as a backwards-compatibility measure (older versions only supported ``nodes``) Examples -------- Input messages to scripts ~~~~~~~~~~~~~~~~~~~~~~~~~ Input message, new instance allocation (common elements are listed this time, but not included in further examples below):: { "version": 2, "cluster_name": "cluster1.example.com", "cluster_tags": [], "enabled_hypervisors": [ "xen-pvm" ], "nodegroups": { "f4e06e0d-528a-4963-a5ad-10f3e114232d": { "name": "default", "alloc_policy": "preferred", "networks": ["net-uuid-1", "net-uuid-2"], "ipolicy": { "disk-templates": ["drbd", "plain"], "minmax": [ { "max": { "cpu-count": 2, "disk-count": 8, "disk-size": 2048, "memory-size": 12800, "nic-count": 8, "spindle-use": 8 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 4.0 }, "tags": ["ng-tag-1", "ng-tag-2"] } }, "instances": { "instance1.example.com": { "tags": [], "should_run": false, "disks": [ { "mode": "w", "size": 64 }, { "mode": "w", "size": 512 } ], "nics": [ { "ip": null, "mac": "aa:00:00:00:60:bf", "bridge": "xen-br0" } ], "vcpus": 1, "disk_template": "plain", "memory": 128, "nodes": [ "nodee1.com" ], "os": "debootstrap+default" }, "instance2.example.com": { "tags": [], "should_run": false, "disks": [ { "mode": "w", "size": 512 }, { "mode": "w", "size": 256 } ], "nics": [ { "ip": null, "mac": "aa:00:00:55:f8:38", "bridge": "xen-br0" } ], "vcpus": 1, "disk_template": "drbd", "memory": 512, "nodes": [ "node2.example.com", "node3.example.com" ], "os": "debootstrap+default" } }, "nodes": { "node1.example.com": { "total_disk": 858276, "primary_ip": "198.51.100.1", "secondary_ip": "192.0.2.1", "tags": [], "group": "f4e06e0d-528a-4963-a5ad-10f3e114232d", "free_memory": 3505, "free_disk": 856740, "total_memory": 4095 }, "node2.example.com": { "total_disk": 858240, "primary_ip": "198.51.100.2", "secondary_ip": "192.0.2.2", "tags": ["test"], "group": "f4e06e0d-528a-4963-a5ad-10f3e114232d", "free_memory": 3505, "free_disk": 848320, "total_memory": 4095 }, "node3.example.com.com": { "total_disk": 572184, "primary_ip": "198.51.100.3", "secondary_ip": "192.0.2.3", "tags": [], "group": "f4e06e0d-528a-4963-a5ad-10f3e114232d", "free_memory": 3505, "free_disk": 570648, "total_memory": 4095 } }, "request": { "type": "allocate", "name": "instance3.example.com", "required_nodes": 2, "disk_space_total": 3328, "disks": [ { "mode": "w", "size": 1024 }, { "mode": "w", "size": 2048 } ], "nics": [ { "ip": null, "mac": "00:11:22:33:44:55", "bridge": null } ], "vcpus": 1, "disk_template": "drbd", "memory": 2048, "os": "debootstrap+default", "tags": [ "type:test", "owner:foo" ], hypervisor: "xen-pvm" } } Input message, reallocation:: { "version": 2, ... "request": { "type": "relocate", "name": "instance2.example.com", "required_nodes": 1, "disk_space_total": 832, "relocate_from": [ "node3.example.com" ] } } Response messages ~~~~~~~~~~~~~~~~~ Successful response message:: { "success": true, "info": "Allocation successful", "result": [ "node2.example.com", "node1.example.com" ] } Failed response message:: { "success": false, "info": "Can't find a suitable node for position 2 (already selected: node2.example.com)", "result": [] } Successful node evacuation message:: { "success": true, "info": "Request successful", "result": [ [ "instance1", "node3" ], [ "instance2", "node1" ] ] } Command line messages ~~~~~~~~~~~~~~~~~~~~~ :: # gnt-instance add -t plain -m 2g --os-size 1g --swap-size 512m --iallocator hail -o debootstrap+default instance3 Selected nodes for the instance: node1.example.com * creating instance disks... [...] # gnt-instance add -t plain -m 3400m --os-size 1g --swap-size 512m --iallocator hail -o debootstrap+default instance4 Failure: prerequisites not met for this operation: Can't compute nodes using iallocator 'hail': Can't find a suitable node for position 1 (already selected: ) # gnt-instance add -t drbd -m 1400m --os-size 1g --swap-size 512m --iallocator hail -o debootstrap+default instance5 Failure: prerequisites not met for this operation: Can't compute nodes using iallocator 'hail': Can't find a suitable node for position 2 (already selected: node1.example.com) Reference implementation ~~~~~~~~~~~~~~~~~~~~~~~~ Ganeti's default iallocator is "hail" which is available when "htools" components have been enabled at build time. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/index.rst000064400000000000000000000116121476477700300162130ustar00rootroot00000000000000.. Ganeti documentation master file, created by sphinx-quickstart Welcome to Ganeti's documentation! ================================== This page is the starting point for browsing the Ganeti documentation. Below, the corpus of Ganeti documentation is grouped by topic. A few quick references: - :doc:`glossary`: Provides explanations of basic Ganeti terminology. - :doc:`news` file: Lists changes between Ganeti versions. - :ref:`search`: Allows you to search for key terms across Ganeti documentation. Installing Ganeti +++++++++++++++++ Use the following resources to install and/or upgrade Ganeti: - :doc:`install`: Comprehensive instructions for installing Ganeti. - :doc:`upgrade`: Instructions for upgrading an existing Ganeti installation to the latest version. Using Ganeti ++++++++++++ The following resources provide guidance on how to use Ganeti: - :doc:`admin`: Information about how to manage a Ganeti cluster after it is installed (including management of nodes and instances, and information about Ganeti's tools and monitoring agent). - :doc:`manpages`: Descriptions of the various tools that are part of Ganeti. - :doc:`security`: A description of the security model underlying a Ganeti cluster. - :doc:`hooks`: Information on hooking scripts, which extend Ganeti functionalities by automatically activating when certain events occur. - :doc:`rapi`: Description of the Ganeti remote API, which allows programmatic access to most of the functionalities of Ganeti. - :doc:`ovfconverter`: Description of a tool that provides compatibility with the standard OVF virtual machine interchange format. Some features are explicitly targeted for large Ganeti installations, in which multiple clusters are present: - :doc:`cluster-merge`: Describes a tool for merging two existing clusters. - :doc:`move-instance`: Describes how to move instances between clusters. Developing Ganeti +++++++++++++++++ There are a few documents particularly useful for developers who want to modify or extend Ganeti: - :doc:`locking`: Describes Ganeti's locking strategy and lock order dependencies. - :doc:`iallocator`: Description of the API for external tools, which can allocate instances either manually or automatically. - :doc:`virtual-cluster`: Explanation of how to use virtual cluster support, which is utilized mainly for testing reasons. Implemented designs ------------------- Before actual implementation, all Ganeti features are described in a design document. Designs fall into two categories: released versions and draft versions (which are either incomplete or not implemented). .. toctree:: :maxdepth: 1 design-2.0.rst design-2.1.rst design-2.2.rst design-2.3.rst design-htools-2.3.rst design-2.4.rst design-2.5.rst design-2.6.rst design-2.7.rst design-2.8.rst design-2.9.rst design-2.10.rst design-2.11.rst design-2.12.rst design-2.13.rst design-2.14.rst design-2.15.rst design-2.16.rst design-3.0.rst design-3.1.rst Draft designs ------------- .. toctree:: :maxdepth: 2 design-draft.rst .. toctree:: :hidden: admin.rst cluster-merge.rst cluster-keys-replacement.rst design-allocation-efficiency.rst design-autorepair.rst design-bulk-create.rst design-ceph-ganeti-support.rst design-chained-jobs.rst design-cmdlib-unittests.rst design-configlock.rst design-cpu-speed.rst design-cpu-pinning.rst design-dedicated-allocation.rst design-device-uuid-name.rst design-daemons.rst design-disk-conversion.rst design-disks.rst design-file-based-storage.rst design-file-based-disks-ownership.rst design-glusterfs-ganeti-support.rst design-hroller.rst design-hsqueeze.rst design-hotplug.rst design-internal-shutdown.rst design-kvmd.rst design-linuxha.rst design-location.rst design-lu-generated-jobs.rst design-monitoring-agent.rst design-move-instance-improvements.rst design-multi-reloc.rst design-multi-version-tests.rst design-network.rst design-node-add.rst design-node-security.rst design-oob.rst design-openvswitch.rst design-opportunistic-locking.rst design-optables.rst design-os.rst design-ovf-support.rst design-partitioned design-performance-tests.rst design-plain-redundancy.rst design-query2.rst design-query-splitting.rst design-qemu-blockdev.rst design-reason-trail.rst design-reservations.rst design-resource-model.rst design-restricted-commands.rst design-shared-storage.rst design-shared-storage-redundancy.rst design-ssh-ports.rst design-storagetypes.rst design-systemd.rst design-upgrade.rst design-virtual-clusters.rst dev-codestyle.rst glossary.rst hooks.rst iallocator.rst install.rst locking.rst manpages.rst monitoring-query-format.rst move-instance.rst news.rst ovfconverter.rst rapi.rst security.rst upgrade.rst virtual-cluster.rst .. vim: set textwidth=72 : ganeti-3.1.0~rc2/doc/install.rst000064400000000000000000000623001476477700300165520ustar00rootroot00000000000000Ganeti installation tutorial ============================ Documents Ganeti version |version| .. contents:: .. highlight:: shell-example Introduction ------------ Ganeti is a cluster virtualization management system based on Xen or KVM. This document explains how to bootstrap a Ganeti node (Xen *dom0*, the host Linux system for KVM), create a running cluster and install virtual instances (Xen *domUs*, KVM guests). You need to repeat most of the steps in this document for every node you want to install, but of course we recommend creating some semi-automatic procedure if you plan to deploy Ganeti on a medium/large scale. A basic Ganeti terminology glossary is provided in the introductory section of the :doc:`admin`. Please refer to that document if you are uncertain about the terms we are using. Ganeti has been developed for Linux and should be distribution-agnostic. This documentation will use Debian Bookworm as an example system but the examples can be translated to any other distribution. You are expected to be familiar with your distribution, its package management system, and Xen or KVM before trying to use Ganeti. This document is divided into two main sections: - Installation of the base system and base components - Configuration of the environment for Ganeti Each of these is divided into sub-sections. While a full Ganeti system will need all of the steps specified, some are not strictly required for every environment. Which ones they are, and why, is specified in the corresponding sections. Installing the base system and base components ---------------------------------------------- Hardware requirements +++++++++++++++++++++ Any system supported by your Linux distribution is fine. Please note that official testing is only performed in amd64 environments. Ganeti offers multiple storage options which may or may not be shared between two or more nodes. Please note that also with non-shared storage backends Ganeti is able to move instances between cluster nodes for you. However, there will be no high-availability features and no live-migration. Please not that your nodes should all share the same hardware configuration with regards to CPU, storage throughput and network. Different CPU models will require you to emulate a common subset of e.g. CPU flags (hence wasting performance). Different storage systems or NIC speeds will slow down faster nodes in replicated environments (e.g. with the DRBD, Ceph or GlusterFS backends). Installing the base system ++++++++++++++++++++++++++ **Mandatory** on all nodes. It is advised to start with a clean, minimal install of the operating system. If you plan on using the ``plain`` or ``drbd`` storage backends please make sure to configure LVM and create a volume group with at least 20GiB of storage assigned. If you plan on using ``file`` please make sure to have enough available disk space on your root partition or (**recommended**) a dedicated filesystem mounted that will hold your instances. We do not recommend any specific type of filesystem at this time. If you plan on using ``sharedfile`` please make sure all your nodes have access to the shared storage (e.g. NFS). If you plan on using ``rbd`` (Ceph) or ``gluster`` you need to setup that first. This will not be covered by this document. Hostname issues ~~~~~~~~~~~~~~~ Note that Ganeti requires the hostnames of the systems (i.e. what the ``hostname`` command outputs to be a fully-qualified name, not a short name. In other words, you should use *node1.example.com* as a hostname and not just *node1*. .. admonition:: Debian Debian usually configures the hostname differently than you need it for Ganeti. For example, this is what it puts in ``/etc/hosts`` in certain situations:: 127.0.0.1 localhost 127.0.1.1 node1.example.com node1 but for Ganeti you need to have:: 127.0.0.1 localhost 192.0.2.1 node1.example.com node1 replacing ``192.0.2.1`` with your node's address. Also, the file ``/etc/hostname`` which configures the hostname of the system should contain ``node1.example.com`` and not just ``node1`` (you need to run the command ``/etc/init.d/hostname.sh start`` after changing the file). .. admonition:: Why a fully qualified host name Although most distributions use only the short name in the /etc/hostname file, we still think Ganeti nodes should use the full name. The reason for this is that calling 'hostname --fqdn' requires the resolver library to work and is a 'guess' via heuristics at what is your domain name. Since Ganeti can be used among other things to host DNS servers, we don't want to depend on them as much as possible, and we'd rather have the uname() syscall return the full node name. We haven't ever found any breakage in using a full hostname on a Linux system, and anyway we recommend to have only a minimal installation on Ganeti nodes, and to use instances (or other dedicated machines) to run the rest of your network services. By doing this you can change the /etc/hostname file to contain an FQDN without the fear of breaking anything unrelated. Installing The Hypervisor +++++++++++++++++++++++++ **Mandatory** on all nodes. While Ganeti is developed with the ability to modularly run on different virtualization environments in mind the only two currently useable on a live system are Xen (both in PVM and HVM mode) and KVM. Supported Xen versions are: 3.0.3 and later 3.x versions, and 4.x (tested up to 4.1). Supported KVM versions are 72 and above. Please follow your distribution's recommended way to install and set up Xen, or install Xen from the upstream source, if you wish, following their manual. For KVM, make sure you have a KVM-enabled kernel and the KVM tools. After installing Xen, you need to reboot into your new system. On some distributions this might involve configuring GRUB appropriately, whereas others will configure it automatically when you install the respective kernels. For KVM no reboot should be necessary. .. admonition:: Xen on Debian Under Debian you can install the relevant ``xen-system-amd64`` package, which will pull in both the hypervisor and the relevant kernel. .. admonition:: KVM on Debian It should be sufficient to install the packages ``qemu-kvm`` and ``qemu-utils``. Xen settings ~~~~~~~~~~~~ Some useful best practices for Xen are to restrict the amount of memory dom0 has available, and pin the dom0 to a limited number of CPUs. Instructions for how to achieve this for various toolstacks can be found on the Xen wiki_. .. _wiki: http://wiki.xenproject.org/wiki/Xen_Project_Best_Practices It is recommended that you disable Xen's automatic save of virtual machines at system shutdown and subsequent restore of them at reboot. To obtain this make sure the variable ``XENDOMAINS_SAVE`` in the file ``/etc/default/xendomains`` is set to an empty value. You may need to restart the Xen daemon for some of these settings to take effect. The best way to do this depends on your distribution. Selecting the instance kernel ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ After you have installed Xen, you need to tell Ganeti exactly what kernel to use for the instances it will create. This is done by creating a symlink from your actual kernel to ``/boot/vmlinuz-3-xenU``, and one from your initrd to ``/boot/initrd-3-xenU`` [#defkernel]_. Note that if you don't use an initrd for the domU kernel, you don't need to create the initrd symlink. .. _configure-lvm-label: Configuring LVM +++++++++++++++ **Mandatory** on all nodes if you want to use ``plain`` or ``DRBD`` storage backends. The volume group is required to be at least 20GiB. If you haven't configured your LVM volume group at install time you need to do it before trying to initialize the Ganeti cluster. This is done by formatting the devices/partitions you want to use for it and then adding them to the relevant volume group:: $ pvcreate /dev/%sda3% $ vgcreate xenvg /dev/%sda3% or:: $ pvcreate /dev/%sdb1% $ pvcreate /dev/%sdc1% $ vgcreate xenvg /dev/%sdb1% /dev/%sdc1% If you want to add a device later you can do so with the *vgextend* command:: $ pvcreate /dev/%sdd1% $ vgextend xenvg /dev/%sdd1% Optional: it is recommended to only scan relevant devices for LVM signatures. Otherwise the LVM on your node might find LVM signatures *inside* your instance's disks and activate them on the node! This can be accomplished by editing ``/etc/lvm/lvm.conf`` and adding your devices as regular expression to the ``global_filter`` variable, like this: .. code-block:: text global_filter = [ "a|^/dev/(sd|nvme).+$|", "r/.*/" ] Note that with Ganeti a helper script is provided - ``lvmstrap`` which will erase and configure as LVM any not in-use disk on your system. This is dangerous and it's recommended to read its ``--help`` output if you want to use it. Installing DRBD +++++++++++++++ DRBD_ is one option if you want to use the high-availability (HA) features of Ganeti, but optional if you don't require them or only run Ganeti on single-node clusters. You can upgrade a non-HA cluster to an HA one later, but you might need to convert all your instances to DRBD to take advantage of the new features. .. _DRBD: http://www.drbd.org/ Supported DRBD versions: 8.0-8.4. It's recommended to have at least version 8.0.12. Note that for version 8.2 and newer it is needed to pass the ``usermode_helper=/bin/true`` parameter to the module, either by configuring ``/etc/modules`` or when inserting it manually. When using Xen and DRBD 8.3.2 or higher with Xen, it is recommended_ to use the ``disable_sendpage=1`` setting as well. .. _recommended: https://docs.linbit.com/docs/users-guide-8.4/#s-xen-drbd-mod-params .. admonition:: Debian On Debian, you only need to install the drbd utils with the following command, making sure you are running the target (Xen or KVM) kernel:: $ apt-get install drbd8-utils Then to configure it for Ganeti:: $ echo "options drbd minor_count=128 usermode_helper=/bin/true" \ > /etc/modprobe.d/drbd.conf $ echo "drbd" >> /etc/modules $ depmod -a $ modprobe drbd Installing RBD ++++++++++++++ Another way of making use of Ganeti's high-availability features is to configure and install RBD_ (Ceph) on all of your nodes. .. _RBD: https://ceph.com/en/users/getting-started/ Documenting the steps required to use RBD is out of scope for this document. Please refer to your distribution's documentation or to the official Ceph documentation to find the optimal way to install RBD in your environment. Installing Gluster ++++++++++++++++++ For GlusterFS_ integration, Ganeti requires that ``mount.glusterfs`` is installed on each and every node. On Debian systems, you can satisfy this requirement with the ``glusterfs-client`` package. Further steps for optimal GlusterFS configuration are out of scope for this document. Please refer to your distribution's documentation or to the official GlusterFS documentation to find the optimal way to install GlusterFS in your environment. .. _GlusterFS: https://docs.gluster.org/en/latest/Quick-Start-Guide/Quickstart/ Other required software +++++++++++++++++++++++ If you plan on building Ganeti yourself, please install all (build) dependencies as noted in the `INSTALL` file. If you use your distribution's packages there is nothing more to do here. Setting up the environment for Ganeti ------------------------------------- Configuring the network +++++++++++++++++++++++ **Mandatory** on all nodes. Ganeti can operate on a single network interface but you can also split this into up to three separate interfaces: .. admonition:: The main interface This interface will hold your Ganeti node's main/public IP address and this is where you will most likely SSH in for management. A cluster also has a dedicated cluster IP address which will be configured by Ganeti on the master node on this interface. .. admonition:: The replication interface This optional interface will only be used for replication (e.g. DRBD) and live migration traffic. If not configured, said traffic will use the main interface. You need to make sure all nodes are connected to this network and can reach each other. .. admonition:: The instance network You will most likely use a bridge to connect your instances to the outside world. While you *could* make this bridge your main interface you can also configure the bridge to use a separate interface and hence separate instance traffic from replication and management/cluster traffic. With vlan-aware bridges (only supported with KVM) you can provision instances easily on different vlans without altering your node's configuration (e.g. create one bridge per vlan). You can use plain network interfaces or make use of the linux bonding driver to achieve redundant connectivity for each of the above. In additional to "bridged mode" Ganeti also supports "routed mode" or "openvswitch mode" for your instance network. In order to use "routed mode" under Xen, you'll need to change the relevant parameters in the Xen config file. Under KVM instead, no config change is necessary, but you still need to set up your network interfaces correctly. By default, under KVM, the "link" parameter you specify per-nic will represent, if non-empty, a different routing table name or number to use for your instances. This allows isolation between different instance groups, and different routing policies between node traffic and instance traffic. You will need to configure your routing table basic routes and rules outside of ganeti. The vif scripts will only add /32 routes to your instances, through their interface, in the table you specified (under KVM, and in the main table under Xen). Also for "openvswitch mode" under Xen a custom network script is needed. Under KVM everything should work, but you'll need to configure your switches outside of Ganeti (as for bridges). It is recommended to use a dedicated network interface for your instances .. admonition:: Bridging under Debian The recommended way to configure the bridge is to edit your ``/etc/network/interfaces`` file and substitute your normal Ethernet stanza with the following snippet if you want to have instance traffic on your main network interface:: auto gnt-bridge iface gnt-bridge inet static address %YOUR_IP_ADDRESS%/%YOUR_PREFIX% gateway %YOUR_GATEWAY% bridge_ports eth0 bridge_stp off bridge_waitport 0 bridge_fd 0 The following configures a bridge to a dedicated interface (``eth1``) and also enables vlan-aware bridging:: auto gnt-bridge iface gnt-bridge inet manual bridge_ports eth1 bridge_vlan_aware yes bridge_stp off bridge_waitport 0 bridge_fd 0 In order to have a custom and more advanced networking configuration in Xen which can vary among instances, after having successfully installed Ganeti you have to create a symbolic link to the vif-script provided by Ganeti inside /etc/xen/scripts (assuming you installed Ganeti under /usr/lib):: $ ln -s /usr/lib/ganeti/vif-ganeti /etc/xen/scripts/vif-ganeti This has to be done on all nodes. Afterwards you can set the ``vif_script`` hypervisor parameter to point to that script by:: $ gnt-cluster modify -H xen-pvm:vif_script=/etc/xen/scripts/vif-ganeti Having this hypervisor parameter you are able to create your own scripts and create instances with different networking configurations. Installing Ganeti +++++++++++++++++ **Mandatory** on all nodes. .. admonition:: Use distribution packages If possible use your distribution's packages. For Debian you only need to install ``ganeti`` and ``ganeti-3.0``. It's now time to install the Ganeti software itself. Download the source from the project page at ``_, and install it (replace 3.0.2 with the latest version):: $ tar xvzf ganeti-%3.0.2%.tar.gz $ cd ganeti-%3.0.2% $ ./configure --localstatedir=/var --sysconfdir=/etc $ make $ make install $ mkdir /srv/ganeti/ /srv/ganeti/os /srv/ganeti/export You also need to copy the file ``doc/examples/ganeti.initd`` from the source archive to ``/etc/init.d/ganeti`` and register it with your distribution's startup scripts, for example in Debian:: $ chmod +x /etc/init.d/ganeti $ update-rc.d ganeti defaults 20 80 There are also unit files provided for use with systemd: ``doc/examples/systemd`` In order to automatically restart failed instances, you need to setup a cron job run the *ganeti-watcher* command. A sample cron file is provided in the source at ``doc/examples/ganeti.cron`` and you can copy that (eventually altering the path) to ``/etc/cron.d/ganeti``. Finally, a sample logrotate snippet is provided in the source at ``doc/examples/ganeti.logrotate`` and you can copy it to ``/etc/logrotate.d/ganeti`` to have Ganeti's logs rotated automatically. What gets installed ~~~~~~~~~~~~~~~~~~~ The above ``make install`` invocation, or installing via your distribution mechanisms, will install on the system: - a set of python libraries under the *ganeti* namespace (depending on the python version this can be located in either ``lib/python-$ver/site-packages`` or various other locations) - a set of programs under ``/usr/local/sbin`` or ``/usr/sbin`` - if the htools component was enabled, a set of programs under ``/usr/local/bin`` or ``/usr/bin/`` - man pages for the above programs - a set of tools under the ``lib/ganeti/tools`` directory - an example iallocator script (see the admin guide for details) under ``lib/ganeti/iallocators`` - a cron job that is needed for cluster maintenance - an init script or systemd unit files for automatic startup of Ganeti daemons - provided but not installed automatically by ``make install`` is a bash completion script that hopefully will ease working with the many cluster commands Installing the Operating System support packages ++++++++++++++++++++++++++++++++++++++++++++++++ **Mandatory** on all nodes. To be able to install instances you need to have an Operating System installation script. An example OS that works under Debian and can install Debian and Ubuntu instace OSes is provided on the project web site. Download it from the project page and follow the instructions in the ``README`` file. Here is the installation procedure (replace 0.14 with the latest version that is compatible with your ganeti version):: $ cd /usr/local/src/ $ wget https://github.com/ganeti/instance-debootstrap/archive/v%0.16%.tar.gz $ tar xzf v%0.16%.tar.gz $ cd instance-debootstrap-%0.16% $ ./configure --with-os-dir=/srv/ganeti/os $ make $ make install In order to use this OS definition, you need to have internet access from your nodes and have the *debootstrap*, *dump* and *restore* commands installed on all nodes. Also, if the OS is configured to partition the instance's disk in ``/etc/default/ganeti-instance-debootstrap``, you will need *kpartx* installed. .. admonition:: Debian Use this command on all nodes to install the required packages:: $ apt-get install debootstrap dump kpartx Or alternatively install the OS definition from the Debian package:: $ apt-get install ganeti-instance-debootstrap Please refer to the ``README`` file of ``ganeti-instance-debootstrap`` for further documentation. .. admonition:: no-op OS Provider On Debian you can also install ``ganeti-os-noop``. This dummy OS provider will not do anything and can be used to e.g. bootstrap KVM instances using PXE boot. Alternatively, you can create your own OS definitions. See the manpage :manpage:`ganeti-os-interface(7)`. Initializing the cluster ++++++++++++++++++++++++ **Mandatory** once per cluster, on the first node. The last step is to initialize the cluster. After you have repeated the above process on all of your nodes and choose one as the master. Make sure there is a SSH key pair on the master node (optionally generating one using ``ssh-keygen``). Before we can run the command ``gnt-cluster init``, we need to decide how this cluster is supposed to operate. .. admonition:: Hypervisor selection Choose which hypervisor to enable:: --enabled-hypervisor kvm [or xen-pvm, xen-hvm] .. admonition:: LVM If you have LVM included in your setup, you may need to specify your volume group's name:: --vg-name vg-ganeti .. admonition:: Network You need to specify your main network interface (e.g. where your node's main IP address resides). Ganeti will use this interface to configure the cluster IP address on the master node:: --master-netdev eth0 Each Ganeti cluster has a name which needs to resolve to an available IP on your node's IP network. Ganeti will resolve the IP address by itself but you need to specify the netmask that goes along with it:: --master-netmask 24 You should also configure the default network, the following configures your instances to use 'bridged mode' with ``gnt-bridge`` as default bridge:: --nic-parameters mode=bridged,link=gnt-bridge .. admonition:: Storage You can enable multiple storage backends (comma separated), but choose at least one:: --enabled-disk-templates drbd [,plain,file,...] Please note that most storage backends require additional parameters - refer to :manpage:`gnt-cluster(8)` for additional details. .. admonition:: iAllocator While you *can* place your instances manually on your cluster it is recommended to use an iallocator script for this. Ganeti ships ``hail`` as a built-in solution and it should be enabled by default:: --default-iallocator hail .. admonition:: Hypervisor parameters It is a good practice to set sane default hypervisor parameters for all of your instances (they can still be overriden at instance level later). The following configures KVM for full boot emulation and makes all of the node's CPU features available to the guest (if you plan on using live migration all CPUs on your cluster need to be the same for this to work!):: --hypervisor-parameters kvm:kernel_path=,initrd_path=,cpu_type=host Please refer to :manpage:`gnt-instance(8)` for a full list of hypervisor parameters and their values/defaults. Finally execute:: $ gnt-cluster init [your parameters here] %CLUSTERNAME% The *CLUSTERNAME* is a hostname, which must be resolvable (e.g. it must exist in DNS or in ``/etc/hosts``) by all the nodes in the cluster. You must choose a name different from any of the nodes names for a multi-node cluster. In general the best choice is to have a unique name for a cluster, even if it consists of only one machine, as you will be able to expand it later without any problems. Please note that the hostname used for this must resolve to an IP address reserved **exclusively** for this purpose, and cannot be the name of the first (master) node. You can also invoke the command with the ``--help`` option in order to see all the possibilities. Hypervisor/Network/Cluster parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Please note that the default hypervisor/network/cluster parameters may not be the correct one for your environment. Carefully check them, and change them either at cluster init time, or later with ``gnt-cluster modify``. Your instance types, networking environment, hypervisor type and version may all affect what kind of parameters should be used on your cluster. Joining the nodes to the cluster ++++++++++++++++++++++++++++++++ **Mandatory** for all the other nodes. After you have initialized your cluster you need to join the other nodes to it. You can do so by executing the following command on the master node:: $ gnt-node add %NODENAME% Separate replication network ++++++++++++++++++++++++++++ **Optional** Ganeti uses DRBD to mirror the disk of the virtual instances between nodes. To use a dedicated network interface for this (in order to improve performance or to enhance security) you need to configure an additional interface for each node. Use the *-s* option with ``gnt-cluster init`` and ``gnt-node add`` to specify the IP address of this secondary interface to use for each node. Note that if you specified this option at cluster setup time, you must afterwards use it for every node add operation. Testing the setup +++++++++++++++++ Execute the ``gnt-node list`` command to see all nodes in the cluster:: $ gnt-node list Node DTotal DFree MTotal MNode MFree Pinst Sinst node1.example.com 197404 197404 2047 1896 125 0 0 The above shows a couple of things: - The various Ganeti daemons can talk to each other - Ganeti can examine the storage of the node (DTotal/DFree) - Ganeti can talk to the selected hypervisor (MTotal/MNode/MFree) Cluster burnin ~~~~~~~~~~~~~~ With Ganeti a tool called :command:`burnin` is provided that can test most of the Ganeti functionality. The tool is installed under the ``lib/ganeti/tools`` directory (either under ``/usr`` or ``/usr/local`` based on the installation method). See more details under :ref:`burnin-label`. Further steps ------------- You can now proceed either to the :doc:`admin`, or read the manpages of the various commands (:manpage:`ganeti(7)`, :manpage:`gnt-cluster(8)`, :manpage:`gnt-node(8)`, :manpage:`gnt-instance(8)`, :manpage:`gnt-job(8)`). .. rubric:: Footnotes .. [#defkernel] The kernel and initrd paths can be changed at either cluster level (which changes the default for all instances) or at instance level. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/locking.rst000064400000000000000000000063031476477700300165330ustar00rootroot00000000000000Ganeti locking ============== Introduction ------------ This document describes lock order dependencies in Ganeti. It is divided by functional sections Opcode Execution Locking ------------------------ These locks are declared by Logical Units (LUs) (in cmdlib.py) and acquired by the Processor (in mcpu.py) with the aid of the Ganeti Locking Library (locking.py). They are acquired in the following order: * BGL: this is the Big Ganeti Lock, it exists for retrocompatibility. New LUs acquire it in a shared fashion, and are able to execute all toghether (baring other lock waits) while old LUs acquire it exclusively and can only execute one at a time, and not at the same time with new LUs. * Instance locks: can be declared in ExpandNames() or DeclareLocks() by an LU, and have the same name as the instance itself. They are acquired as a set. Internally the locking library acquired them in alphabetical order. * Node locks: can be declared in ExpandNames() or DeclareLocks() by an LU, and have the same name as the node itself. They are acquired as a set. Internally the locking library acquired them in alphabetical order. Given this order it's possible to safely acquire a set of instances, and then the nodes they reside on. The ConfigWriter (in config.py) is also protected by a SharedLock, which is shared by functions that read the config and acquired exclusively by functions that modify it. Since the ConfigWriter calls rpc.call_upload_file to all nodes to distribute the config without holding the node locks, this call must be able to execute on the nodes in parallel with other operations (but not necessarily concurrently with itself on the same file, as inside the ConfigWriter this is called with the internal config lock held. Job Queue Locking ----------------- The job queue is designed to be thread-safe. This means that its public functions can be called from any thread. The job queue can be called from functions called by the queue itself (e.g. logical units), but special attention must be paid not to create deadlocks or an invalid state. The single queue lock is used from all classes involved in the queue handling. During development we tried to split locks, but deemed it to be too dangerous and difficult at the time. Job queue functions acquiring the lock can be safely called from all the rest of the code, as the lock is released before leaving the job queue again. Unlocked functions should only be called from job queue related classes (e.g. in jqueue.py) and the lock must be acquired beforehand. In the job queue worker (``_JobQueueWorker``), the lock must be released before calling the LU processor. Otherwise a deadlock can occur when log messages are added to opcode results. Node Daemon Locking ------------------- The node daemon contains a lock for the job queue. In order to avoid conflicts and/or corruption when an eventual master daemon or another node daemon is running, it must be held for all job queue operations There's one special case for the node daemon running on the master node. If grabbing the lock in exclusive fails on startup, the code assumes all checks have been done by the process keeping the lock. .. vim: set textwidth=72 : ganeti-3.1.0~rc2/doc/manpages-disabled.rst000064400000000000000000000003521476477700300204430ustar00rootroot00000000000000Man pages ========= Inclusion of man pages into documentation was not enabled at build time; use ``./configure [...] --enable-manpages-in-doc``. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/monitoring-query-format.rst000064400000000000000000000031561476477700300217260ustar00rootroot00000000000000The queries to the monitoring agent will be HTTP GET requests on port 1815. The answer will be encoded in JSON format and will depend on the specific accessed resource. If a request is sent to a non-existing resource, a 404 error will be returned by the HTTP server. The following paragraphs will present the existing resources supported by the current protocol version, that is version 1. ``/`` +++++ The root resource. It will return the list of the supported protocol version numbers. Currently, this will include only version 1. ``/1`` ++++++ Not an actual resource per-se, it is the root of all the resources of protocol version 1. If requested through GET, the null JSON value will be returned. ``/1/list/collectors`` ++++++++++++++++++++++ Returns a list of tuples (kind, category, name) showing all the collectors available in the system. ``/1/report/all`` +++++++++++++++++ A list of the reports of all the data collectors, as a JSON list. Status reporting collectors will provide their output in non-verbose format. The verbose format can be requested by adding the parameter ``verbose=1`` to the request. ``/1/report/[category]/[collector_name]`` +++++++++++++++++++++++++++++++++++++++++ Returns the report of the collector ``[collector_name]`` that belongs to the specified ``[category]``. The ``category`` has to be written in lowercase. If a collector does not belong to any category, ``default`` will have to be used as the value for ``[category]``. Status reporting collectors will provide their output in non-verbose format. The verbose format can be requested by adding the parameter ``verbose=1`` to the request. ganeti-3.1.0~rc2/doc/move-instance.rst000064400000000000000000000124661476477700300176640ustar00rootroot00000000000000================================= Moving instances between clusters ================================= Starting with Ganeti 2.2, instances can be moved between separate Ganeti clusters using a new tool, ``move-instance``. The tool has a number of features: - Moving a single or multiple instances - Moving instances in parallel (``--parallel`` option) - Renaming instance (only when moving a single instance) - SSL certificate verification for RAPI connections The design of the inter-cluster instances moves is described in detail in the :doc:`Ganeti 2.2 design document `. The instance move tool talks to the Ganeti clusters via RAPI and can run on any machine which can connect to the cluster's RAPI. Despite their similar name, the instance move tool should not be confused with the ``gnt-instance move`` command, which is used to move without changes (instead of export/import plus rename) an instance within the cluster. Configuring clusters for instance moves --------------------------------------- To prevent third parties from accessing the instance data, all data exchanged between the clusters is signed using a secret key, the "cluster domain secret". It is recommended to assign the same domain secret to all clusters of the same security domain, so that instances can be easily moved between them. By checking the signatures, the destination cluster can be sure the third party (e.g. this tool) didn't modify the received crypto keys and connection information. .. highlight:: shell-example To create a new, random cluster domain secret, run the following command on the master node:: $ gnt-cluster renew-crypto --new-cluster-domain-secret To read and set the cluster domain secret from the contents of a file, run the following command on the master node:: $ gnt-cluster renew-crypto --cluster-domain-secret=%/.../ganeti.cds% More information about the ``renew-crypto`` command can be found in :manpage:`gnt-cluster(8)`. Moving instances ---------------- As soon as the clusters share a cluster domain secret, instances can be moved. The tool usage is as follows:: $ move-instance %[options]% %source-cluster% %destination-cluster% %instance-name...% Multiple instances can be moved with one invocation of the instance move tool, though a few options are only available when moving a single instance. The most important options are listed below. Unless specified otherwise, destination-related options default to the source value (e.g. setting ``--src-rapi-port=1234`` will make ``--dest-rapi-port``'s default 1234). ``--src-rapi-port``/``--dest-rapi-port`` RAPI server TCP port, defaults to 5080. ``--src-ca-file``/``--dest-ca-file`` Path to file containing source cluster Certificate Authority (CA) in PEM format. For self-signed certificates, this is the certificate itself (see more details below in :ref:`instance-move-certificates`). For certificates signed by a third party CA, the complete chain must be in the file (see documentation for :manpage:`SSL_CTX_load_verify_locations(3)`). ``--src-username``/``--dest-username`` RAPI username, must have write access to cluster. ``--src-password-file``/``--dest-password-file`` Path to file containing RAPI password (make sure to restrict access to this file). ``--dest-instance-name`` When moving a single instance: Change name of instance on destination cluster. ``--dest-primary-node`` When moving a single instance: Primary node on destination cluster. ``--dest-secondary-node`` When moving a single instance: Secondary node on destination cluster. ``--dest-disk-template`` Disk template to use after the move. Can be used to change disk templates. ``--compress`` Compression mode to use during the instance move. This mode has to be supported by both the source and the destination cluster. ``--iallocator`` Iallocator for creating instance on destination cluster. ``--hypervisor-parameters``/``--backend-parameters``/``--os-parameters``/``--net`` When moving a single instance: Override instances' parameters. ``--parallel`` Number of instance moves to run in parallel. ``--verbose``/``--debug`` Increase output verbosity. ``--keep-source-instance`` Do not delete the instance on the source cluster after successful migration. Please be aware, that the instance will end up running on both clusters, if it was running before the migration was started. The exit value of the tool is zero if and only if all instance moves were successful. .. _instance-move-certificates: Certificates ------------ If using certificates signed by a CA, then you need to pass the same CA certificate via both ``--src-ca-file`` and ``dest-ca-file``. However, if you're using self-signed certificates, this has a few (security) implications: - the certificates of both the source and destinations clusters (``rapi.pem`` from the Ganeti configuration directory, usually ``/var/lib/ganeti/rapi.pem``) must be available to the tool - by default, the certificates include the private key as well, so simply copying them to a third machine means that machine can now impersonate both the source and destination clusters RAPI endpoint It is therefore recommended to copy only the certificate from the ``rapi.pem`` files, and pass these to ``--src-ca-file`` and ``--dest-ca-file`` appropriately. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/ovfconverter.rst000064400000000000000000000157301476477700300176330ustar00rootroot00000000000000============= OVF converter ============= Using ``ovfconverter`` from the ``tools`` directory, one can easily convert previously exported Ganeti instance into OVF package, supported by VMWare, VirtualBox and some other virtualization software. It is also possible to use instance exported from such a tool and convert it to Ganeti config file, used by ``gnt-backup import`` command. For the internal design of the converter and more detailed description, including listing of available command line options, please refer to :doc:`design-ovf-support` As the amount of Ganeti-specific details, that need to be provided in order to import an external instance, is rather large, we will present here some examples of importing instances from different sources. It is also worth noting that there are some limitations regarding support for different hardware. Limitations on import ===================== Network ------- Available modes for the network include ``bridged`` and ``routed``. There is no ``NIC`` mode, which is typically used e.g. by VirtualBox. For most usecases this should not be of any effect, since if ``NetworkSection`` contains any networks which are not discovered as ``bridged`` or ``routed``, the network mode is assigned automatically, using Ganeti's cluster defaults. Backend ------- The only values that are taken into account regarding Virtual Hardware (described in ``VirtualHardwareSection`` of the ``.ovf`` file) are: - number of virtual CPUs - RAM memory - hard disks - networks Neither USB nor CD-ROM drive are used in Ganeti. We decided to simply ignore unused elements of this section, so their presence won't raise any warnings. Operating System ---------------- List of operating systems available on a cluster is viewable using ``gnt-os list`` command. When importing from external source, providing OS type in a command line (``--os-type=...``) is **required**. This is because even if the type is given in OVF description, it is not detailed enough for Ganeti to know which os-specific scripts to use. Please note, that instance containing disks may only be imported using OS script that supports raw disk images. References ---------- Files listed in ``ovf:References`` section cannot be hyperlinks. Limitations on export ===================== Disk content ------------ Most Ganeti instances do not contain grub. This results in some problems when importing to virtualization software that does expect it. Examples of such software include VirtualBox and VMWare. To avoid trouble, please install grub inside the instance before exporting it. Import to VirtualBox -------------------- ``format`` option should be set to ``vmdk`` in order for instance to be importable by VirtualBox. Tests using existing versions of VirtualBox (3.16) suggest, that VirtualBox does not support disk compression or OVA packaging. In future versions this might change. Import to VMWare ---------------- Importing Ganeti instance to VMWare was tested using ``ovftool``. ``format`` option should be set to ``vmdk`` in order for instance to be importable by VMWare. Presence of Ganeti section does seem to cause some problems and therefore it is recommended to use ``--external`` option on export. Import of compressed disks generated by ovfconverter was impossible in current version of ``ovftool`` (2.1.0). This seems to be related to old ``vmdk`` version. Since the conversion to ``vmdk`` format is done using ``qemu-img``, it is possible and in fact expected, that future versions of the latter tool will resolve this problem. Import examples =============== Ganeti's OVF ------------ If you are importing instance created using ``ovfconverter export`` -- you most probably will not have to provide any additional information. In that case, the following is all you need (unless you wish to change some configuration options):: ovfconverter import ganeti.ovf [...] gnt-instance import -n Virtualbox, VMWare and other external sources --------------------------------------------- In case of importing from external source, you will most likely have to provide the following details: - ``os-type`` can be any operating system listed on ``gnt-os list`` - ``name`` that has to be resolvable, as it will be used as instance name (even if your external instance has a name, it most probably is not resolvable to an IP address) These are not the only options, but the recommended ones. For the complete list of available options please refer to `Command Line description ` Minimalistic but complete example of importing Virtualbox's OVF instance may look like:: ovfconverter virtualbox.ovf --os-type=lenny-image \ --name=xen.test.i1 --disk-template=diskless [...] gnt-instance import -n node1.xen xen.test.i1 Export example ============== Exporting instance into ``.ovf`` format is pretty streightforward and requires little - if any - explanation. The only compulsory detail is the required disk format, provided using the ``--format`` option. Export to another Ganeti instance --------------------------------- If for some reason it is convenient for you to use ``ovfconverter`` to move instance between clusters (e.g. because of the disk compression), the complete example of export may look like this:: gnt-backup export -n node1.xen xen.test.i1 [...] ovfconverter export --format=vmdk --ova \ /srv/ganeti/export/xen.i1.node1.xen/config.ini [...] The result is then in ``/srv/ganeti/export/xen.i1.node1.xen/xen.test.i1.ova`` Export to Virtualbox/VMWare/other external tool ----------------------------------------------- Typically, when exporting to external tool we do not want Ganeti-specific configuration to be saved. In that case, simply use the ``--external`` option:: gnt-backup export -n node1.xen xen.test.i1 [...] ovfconverter export --external --output-dir ~/ganeti-instance/ \ /srv/ganeti/export/xen.i1.node1.xen/config.ini Known issues ============ Conversion errors ----------------- If you are encountering trouble when converting the disk, please ensure that you have newest ``qemu-img`` version. OVA and compression ------------------- The compressed disks and OVA packaging do not work correctly in either VirtualBox (old version) or VMWare. VirtualBox (3.16 OSE) does not seem to support those two, so there is very little we can do about this. As for VMWare, the reason behind it not accepting compressed or packed instances created by ovfconverter seems to be related to the old vmdk version. Problems on newest VirtualBox ----------------------------- In Oracle VM Virtualbox 4.0+ there seems to be a problem when importing any OVF instance created by ovfconverter. Reasons are again unknown, this will be investigated. Disk space ---------- The disk space requirements for both import and export are at the moment very large - we require free space up to about 3-4 times the size of disks. This will most likely be changed in future versions. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/rapi.rst000064400000000000000000001535461476477700300160540ustar00rootroot00000000000000Ganeti remote API ================= Documents Ganeti version |version| .. contents:: Introduction ------------ Ganeti supports a remote API for enable external tools to easily retrieve information about a cluster's state. The remote API daemon, *ganeti-rapi*, is automatically started on the master node. By default it runs on TCP port 5080, but this can be changed either in ``.../constants.py`` or via the command line parameter *-p*. SSL mode, which is used by default, can also be disabled by passing command line parameters. .. _rapi-users: Users and passwords ------------------- ``ganeti-rapi`` reads users and passwords from a file (usually ``/var/lib/ganeti/rapi/users``) on startup. Changes to the file will be read automatically. Lines starting with the hash sign (``#``) are treated as comments. Each line consists of two or three fields separated by whitespace. The first two fields are for username and password. The third field is optional and can be used to specify per-user options (separated by comma without spaces). Passwords can either be written in clear text or as a hash. Clear text passwords may not start with an opening brace (``{``) or they must be prefixed with ``{cleartext}``. To use the hashed form, get the MD5 hash of the string ``$username:Ganeti Remote API:$password`` (e.g. ``echo -n 'jack:Ganeti Remote API:abc123' | openssl md5``) [#pwhash]_ and prefix it with ``{ha1}``. Using the scheme prefix for all passwords is recommended. Scheme prefixes are case insensitive. Options control a user's access permissions. The section :ref:`rapi-access-permissions` lists the permissions required for each resource. If the ``--require-authentication`` command line option is given to the ``ganeti-rapi`` daemon, all requests require authentication. Available options: .. pyassert:: rapi.RAPI_ACCESS_ALL == set([ rapi.RAPI_ACCESS_WRITE, rapi.RAPI_ACCESS_READ, ]) .. pyassert:: rlib2.R_2_nodes_name_storage.GET_ACCESS == [rapi.RAPI_ACCESS_WRITE] .. pyassert:: rlib2.R_2_jobs_id_wait.GET_ACCESS == [rapi.RAPI_ACCESS_WRITE] :pyeval:`rapi.RAPI_ACCESS_WRITE` Enables the user to execute operations modifying the cluster. Implies :pyeval:`rapi.RAPI_ACCESS_READ` access. Resources blocking other operations for read-only access, such as :ref:`/2/nodes/[node_name]/storage ` or blocking server-side processes, such as :ref:`/2/jobs/[job_id]/wait `, use :pyeval:`rapi.RAPI_ACCESS_WRITE` to control access to their :pyeval:`http.HTTP_GET` method. :pyeval:`rapi.RAPI_ACCESS_READ` Allow access to operations querying for information. Example:: # Give Jack and Fred read-only access jack abc123 fred {cleartext}foo555 # Give write access to an imaginary instance creation script autocreator xyz789 write # Hashed password for Jessica jessica {HA1}7046452df2cbb530877058712cf17bd4 write # Monitoring can query for values monitoring {HA1}ec018ffe72b8e75bb4d508ed5b6d079c read # A user who can read and write (the former is implied by granting # write access) superuser {HA1}ec018ffe72b8e75bb4d508ed5b6d079c read,write When using the RAPI, username and password can be sent to the server by using the standard HTTP basic access authentication. This means that for accessing the protected URL ``https://cluster.example.com/resource``, the address ``https://username:password@cluster.example.com/resource`` should be used instead. Alternatively, the appropriate parameter of your HTTP client (such as ``-u`` for ``curl``) can be used. .. [#pwhash] Using the MD5 hash of username, realm and password is described in :rfc:`2617` ("HTTP Authentication"), sections 3.2.2.2 and 3.3. The reason for using it over another algorithm is forward compatibility. If ``ganeti-rapi`` were to implement HTTP Digest authentication in the future, the same hash could be used. In the current version ``ganeti-rapi``'s realm, ``Ganeti Remote API``, can only be changed by modifying the source code. Protocol -------- The protocol used is JSON_ over HTTP designed after the REST_ principle. HTTP Basic authentication as per :rfc:`2617` is supported. .. _JSON: http://www.json.org/ .. _REST: http://en.wikipedia.org/wiki/Representational_State_Transfer HTTP requests with a body (e.g. ``PUT`` or ``POST``) require the request header ``Content-type`` be set to ``application/json`` (see :rfc:`2616` (HTTP/1.1), section 7.2.1). A note on JSON as used by RAPI ++++++++++++++++++++++++++++++ JSON_ as used by Ganeti RAPI does not conform to the specification in :rfc:`4627`. Section 2 defines a JSON text to be either an object (``{"key": "value", â€Ļ}``) or an array (``[1, 2, 3, â€Ļ]``). In violation of this RAPI uses plain strings (``"master-candidate"``, ``"1234"``) for some requests or responses. Changing this now would likely break existing clients and cause a lot of trouble. .. highlight:: ruby Unlike Python's `JSON encoder and decoder `_, other programming languages or libraries may only provide a strict implementation, not allowing plain values. For those, responses can usually be wrapped in an array whose first element is then used, e.g. the response ``"1234"`` becomes ``["1234"]``. This works equally well for more complex values. Example in Ruby:: require "json" # Insert code to get response here response = "\"1234\"" decoded = JSON.parse("[#{response}]").first Short of modifying the encoder to allow encoding to a less strict format, requests will have to be formatted by hand. Newer RAPI requests already use a dictionary as their input data and shouldn't cause any problems. PUT or POST? ------------ According to :rfc:`2616` the main difference between PUT and POST is that POST can create new resources but PUT can only create the resource the URI was pointing to on the PUT request. Unfortunately, due to historic reasons, the Ganeti RAPI library is not consistent with this usage, so just use the methods as documented below for each resource. For more details have a look in the source code at ``lib/rapi/rlib2.py``. Generic parameter types ----------------------- A few generic refered parameter types and the values they allow. ``bool`` ++++++++ A boolean option will accept ``1`` or ``0`` as numbers but not i.e. ``True`` or ``False``. Generic parameters ------------------ A few parameter mean the same thing across all resources which implement it. ``bulk`` ++++++++ Bulk-mode means that for the resources which usually return just a list of child resources (e.g. ``/2/instances`` which returns just instance names), the output will instead contain detailed data for all these subresources. This is more efficient than query-ing the sub-resources themselves. ``dry-run`` +++++++++++ The boolean *dry-run* argument, if provided and set, signals to Ganeti that the job should not be executed, only the pre-execution checks will be done. This is useful in trying to determine (without guarantees though, as in the meantime the cluster state could have changed) if the operation is likely to succeed or at least start executing. ``force`` +++++++++++ Force operation to continue even if it will cause the cluster to become inconsistent (e.g. because there are not enough master candidates). Parameter details ----------------- Some parameters are not straight forward, so we describe them in details here. .. _rapi-ipolicy: ``ipolicy`` +++++++++++ The instance policy specification is a dict with the following fields: .. pyassert:: constants.IPOLICY_ALL_KEYS == set([constants.ISPECS_MINMAX, constants.ISPECS_STD, constants.IPOLICY_DTS, constants.IPOLICY_VCPU_RATIO, constants.IPOLICY_SPINDLE_RATIO]) .. pyassert:: (set(constants.ISPECS_PARAMETER_TYPES.keys()) == set([constants.ISPEC_MEM_SIZE, constants.ISPEC_DISK_SIZE, constants.ISPEC_DISK_COUNT, constants.ISPEC_CPU_COUNT, constants.ISPEC_NIC_COUNT, constants.ISPEC_SPINDLE_USE])) .. |ispec-min| replace:: :pyeval:`constants.ISPECS_MIN` .. |ispec-max| replace:: :pyeval:`constants.ISPECS_MAX` .. |ispec-std| replace:: :pyeval:`constants.ISPECS_STD` :pyeval:`constants.ISPECS_MINMAX` A list of dictionaries, each with the following two fields: |ispec-min|, |ispec-max| A sub- `dict` with the following fields, which sets the limit of the instances: :pyeval:`constants.ISPEC_MEM_SIZE` The size in MiB of the memory used :pyeval:`constants.ISPEC_DISK_SIZE` The size in MiB of the disk used :pyeval:`constants.ISPEC_DISK_COUNT` The numbers of disks used :pyeval:`constants.ISPEC_CPU_COUNT` The numbers of cpus used :pyeval:`constants.ISPEC_NIC_COUNT` The numbers of nics used :pyeval:`constants.ISPEC_SPINDLE_USE` The numbers of virtual disk spindles used by this instance. They are not real in the sense of actual HDD spindles, but useful for accounting the spindle usage on the residing node |ispec-std| A sub- `dict` with the same fields as |ispec-min| and |ispec-max| above, which sets the standard values of the instances. :pyeval:`constants.IPOLICY_DTS` A `list` of disk templates allowed for instances using this policy :pyeval:`constants.IPOLICY_VCPU_RATIO` Maximum ratio of virtual to physical CPUs (`float`) :pyeval:`constants.IPOLICY_SPINDLE_RATIO` Maximum ratio of instances to their node's ``spindle_count`` (`float`) Usage examples -------------- You can access the API using your favorite programming language as long as it supports network connections. Ganeti RAPI client ++++++++++++++++++ Ganeti includes a standalone RAPI client, ``lib/rapi/client.py``. Shell +++++ .. highlight:: shell-example Using ``wget``:: $ wget -q -O - https://%CLUSTERNAME%:5080/2/info or ``curl``:: $ curl https://%CLUSTERNAME%:5080/2/info Note: with ``curl``, the request method (GET, POST, PUT) can be specified using the ``-X`` command line option, and the username/password can be specified with the ``-u`` option. In case of POST requests with a body, the Content-Type can be set to JSON (as per the Protocol_ section) using the parameter ``-H "Content-Type: application/json"``. Python ++++++ .. highlight:: python :: import urllib2 f = urllib2.urlopen('https://CLUSTERNAME:5080/2/info') print(f.read()) JavaScript ++++++++++ .. warning:: While it's possible to use JavaScript, it poses several potential problems, including browser blocking request due to non-standard ports or different domain names. Fetching the data on the webserver is easier. .. highlight:: javascript :: var url = 'https://CLUSTERNAME:5080/2/info'; var info; var xmlreq = new XMLHttpRequest(); xmlreq.onreadystatechange = function () { if (xmlreq.readyState != 4) return; if (xmlreq.status == 200) { info = eval("(" + xmlreq.responseText + ")"); alert(info); } else { alert('Error fetching cluster info'); } xmlreq = null; }; xmlreq.open('GET', url, true); xmlreq.send(null); Resources --------- .. highlight:: javascript ``/`` +++++ The root resource. Has no function, but for legacy reasons the ``GET`` method is supported. ``/2`` ++++++ Has no function, but for legacy reasons the ``GET`` method is supported. .. _rapi-res-info: ``/2/info`` +++++++++++ Cluster information resource. .. rapi_resource_details:: /2/info .. _rapi-res-info+get: ``GET`` ~~~~~~~ Returns cluster information. Example:: { "config_version": 2000000, "name": "cluster", "software_version": "2.0.0~beta2", "os_api_version": 10, "export_version": 0, "candidate_pool_size": 10, "enabled_hypervisors": [ "fake" ], "hvparams": { "fake": {} }, "default_hypervisor": "fake", "master": "node1.example.com", "architecture": [ "64bit", "x86_64" ], "protocol_version": 20, "beparams": { "default": { "auto_balance": true, "vcpus": 1, "memory": 128 } }, // ... } .. _rapi-res-redistribute-config: ``/2/redistribute-config`` ++++++++++++++++++++++++++ Redistribute configuration to all nodes. .. rapi_resource_details:: /2/redistribute-config .. _rapi-res-redistribute-config+put: ``PUT`` ~~~~~~~ Redistribute configuration to all nodes. The result will be a job id. Job result: .. opcode_result:: OP_CLUSTER_REDIST_CONF .. _rapi-res-features: ``/2/features`` +++++++++++++++ .. rapi_resource_details:: /2/features .. _rapi-res-features+get: ``GET`` ~~~~~~~ Returns a list of features supported by the RAPI server. Available features: .. pyassert:: rlib2.ALL_FEATURES == set([rlib2._INST_CREATE_REQV1, rlib2._INST_REINSTALL_REQV1, rlib2._NODE_MIGRATE_REQV1, rlib2._NODE_EVAC_RES1]) :pyeval:`rlib2._INST_CREATE_REQV1` Instance creation request data version 1 supported :pyeval:`rlib2._INST_REINSTALL_REQV1` Instance reinstall supports body parameters :pyeval:`rlib2._NODE_MIGRATE_REQV1` Whether migrating a node (``/2/nodes/[node_name]/migrate``) supports request body parameters :pyeval:`rlib2._NODE_EVAC_RES1` Whether evacuating a node (``/2/nodes/[node_name]/evacuate``) returns a new-style result (see resource description) .. _rapi-res-filters: ``/2/filters`` +++++++++++++++ The filters resource. .. rapi_resource_details:: /2/filters .. _rapi-res-filters+get: ``GET`` ~~~~~~~ Returns a list of all existing filters. Example:: [ { "id": "8b53f7de-f8e2-4470-99bd-1efe746e434f", "uri": "/2/filters/8b53f7de-f8e2-4470-99bd-1efe746e434f" }, { "id": "b296f0c9-4809-46a8-b928-5ccf7720fa8c", "uri": "/2/filters/b296f0c9-4809-46a8-b928-5ccf7720fa8c" } ] If the optional bool *bulk* argument is provided and set to a true value (i.e ``?bulk=1``), the output contains detailed information about filters as a list. Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.FILTER_RULE_FIELDS))`. Example:: [ { "uuid": "8b53f7de-f8e2-4470-99bd-1efe746e434f", "watermark": 12534, "reason_trail": [ ["luxid", "someFilterReason", 1409249801259897000] ], "priority": 0, "action": "REJECT", "predicates": [ ["jobid", [">", "id", "watermark"]] ] }, { "uuid": "b296f0c9-4809-46a8-b928-5ccf7720fa8c", "watermark": 12534, "reason_trail": [ ["luxid", "someFilterReason", 1409249917268978000] ], "priority": 1, "action": "REJECT", "predicates": [ ["opcode", ["=", "OP_ID", "OP_INSTANCE_CREATE"]] ] } ] .. _rapi-res-filters+post: ``POST`` ~~~~~~~~ Creates a filter. Body parameters: ``priority`` (int, defaults to ``0``) Must be non-negative. Lower numbers mean higher filter priority. ``predicates`` (list, defaults to ``[]``) The first element is the name (``str``) of the predicate and the rest are parameters suitable for that predicate. Most predicates take a single parameter: A boolean expression in the Ganeti query language. ``action`` (defaults to ``"CONTINUE"``) The effect of the filter. Can be one of ``"ACCEPT"``, ``"PAUSE"``, ``"REJECT"``, ``"CONTINUE"`` and ``["RATE_LIMIT", n]``, where ``n`` is a positive integer. ``reason`` (list, defaults to ``[]``) An initial reason trail for this filter. Each element in this list is a list with 3 elements: ``[source, reason, timestamp]``, where ``source`` and ``reason`` are strings and ``timestamp`` is a time since the UNIX epoch in nanoseconds as an integer. Returns: A filter UUID (``str``) that can be used for accessing the filter later. .. _rapi-res-filters-filter_uuid: ``/2/filters/[filter_uuid]`` ++++++++++++++++++++++++++++++ Returns information about a filter. .. rapi_resource_details:: /2/filters/[filter_uuid] .. _rapi-res-filters-filter_uuid+get: ``GET`` ~~~~~~~ Returns information about a filter, similar to the bulk output from the filter list. Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.FILTER_RULE_FIELDS))`. .. _rapi-res-filters-filter_uuid+put: ``PUT`` ~~~~~~~ Replaces a filter with given UUID, or creates it with the given UUID if it doesn't already exist. Body parameters: All parameters for adding a new filter via ``POST``, plus the following: ``uuid``: (string) The UUID of the filter to replace or create. Returns: The filter UUID (``str``) of the replaced or created filter. This will be the ``uuid`` body parameter if given, and a freshly generated UUID otherwise. .. _rapi-res-filters-filter_uuid+delete: ``DELETE`` ~~~~~~~~~~ Deletes a filter. Returns: ``None`` .. _rapi-res-modify: ``/2/modify`` ++++++++++++++++++++++++++++++++++++++++ Modifies cluster parameters. .. rapi_resource_details:: /2/modify .. _rapi-res-modify+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_CLUSTER_SET_PARAMS Job result: .. opcode_result:: OP_CLUSTER_SET_PARAMS .. _rapi-res-groups: ``/2/groups`` +++++++++++++ The groups resource. .. rapi_resource_details:: /2/groups .. _rapi-res-groups+get: ``GET`` ~~~~~~~ Returns a list of all existing node groups. Example:: [ { "name": "group1", "uri": "/2/groups/group1" }, { "name": "group2", "uri": "/2/groups/group2" } ] If the optional bool *bulk* argument is provided and set to a true value (i.e ``?bulk=1``), the output contains detailed information about node groups as a list. Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.G_FIELDS))`. Example:: [ { "name": "group1", "node_cnt": 2, "node_list": [ "node1.example.com", "node2.example.com" ], "uuid": "0d7d407c-262e-49af-881a-6a430034bf43", // ... }, { "name": "group2", "node_cnt": 1, "node_list": [ "node3.example.com" ], "uuid": "f5a277e7-68f9-44d3-a378-4b25ecb5df5c", // ... }, // ... ] .. _rapi-res-groups+post: ``POST`` ~~~~~~~~ Creates a node group. If the optional bool *dry-run* argument is provided, the job will not be actually executed, only the pre-execution checks will be done. Returns: a job ID that can be used later for polling. Body parameters: .. opcode_params:: OP_GROUP_ADD Earlier versions used a parameter named ``name`` which, while still supported, has been renamed to ``group_name``. Job result: .. opcode_result:: OP_GROUP_ADD .. _rapi-res-groups-group_name: ``/2/groups/[group_name]`` ++++++++++++++++++++++++++ Returns information about a node group. .. rapi_resource_details:: /2/groups/[group_name] .. _rapi-res-groups-group_name+get: ``GET`` ~~~~~~~ Returns information about a node group, similar to the bulk output from the node group list. Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.G_FIELDS))`. .. _rapi-res-groups-group_name+delete: ``DELETE`` ~~~~~~~~~~ Deletes a node group. It supports the ``dry-run`` argument. Job result: .. opcode_result:: OP_GROUP_REMOVE .. _rapi-res-groups-group_name-modify: ``/2/groups/[group_name]/modify`` +++++++++++++++++++++++++++++++++ Modifies the parameters of a node group. .. rapi_resource_details:: /2/groups/[group_name]/modify .. _rapi-res-groups-group_name-modify+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_GROUP_SET_PARAMS :exclude: group_name Job result: .. opcode_result:: OP_GROUP_SET_PARAMS .. _rapi-res-groups-group_name-rename: ``/2/groups/[group_name]/rename`` +++++++++++++++++++++++++++++++++ Renames a node group. .. rapi_resource_details:: /2/groups/[group_name]/rename .. _rapi-res-groups-group_name-rename+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_GROUP_RENAME :exclude: group_name Job result: .. opcode_result:: OP_GROUP_RENAME .. _rapi-res-groups-group_name-assign-nodes: ``/2/groups/[group_name]/assign-nodes`` +++++++++++++++++++++++++++++++++++++++ Assigns nodes to a group. .. rapi_resource_details:: /2/groups/[group_name]/assign-nodes .. _rapi-res-groups-group_name-assign-nodes+put: ``PUT`` ~~~~~~~ Returns a job ID. It supports the ``dry-run`` and ``force`` arguments. Body parameters: .. opcode_params:: OP_GROUP_ASSIGN_NODES :exclude: group_name, force, dry_run Job result: .. opcode_result:: OP_GROUP_ASSIGN_NODES .. _rapi-res-groups-group_name-tags: ``/2/groups/[group_name]/tags`` +++++++++++++++++++++++++++++++ Manages per-nodegroup tags. .. rapi_resource_details:: /2/groups/[group_name]/tags .. _rapi-res-groups-group_name-tags+get: ``GET`` ~~~~~~~ Returns a list of tags. Example:: ["tag1", "tag2", "tag3"] .. _rapi-res-groups-group_name-tags+put: ``PUT`` ~~~~~~~ Add a set of tags. The request as a list of strings should be ``PUT`` to this URI. The result will be a job id. It supports the ``dry-run`` argument. .. _rapi-res-groups-group_name-tags+delete: ``DELETE`` ~~~~~~~~~~ .. highlight:: none Delete a tag. In order to delete a set of tags, the DELETE request should be addressed to URI like:: /tags?tag=[tag]&tag=[tag] It supports the ``dry-run`` argument. .. highlight:: javascript .. _rapi-res-networks: ``/2/networks`` +++++++++++++++ The networks resource. .. rapi_resource_details:: /2/networks .. _rapi-res-networks+get: ``GET`` ~~~~~~~ Returns a list of all existing networks. Example:: [ { "name": "network1", "uri": "/2/networks/network1" }, { "name": "network2", "uri": "/2/networks/network2" } ] If the optional bool *bulk* argument is provided and set to a true value (i.e ``?bulk=1``), the output contains detailed information about networks as a list. Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.NET_FIELDS))`. Example:: [ { 'external_reservations': '10.0.0.0, 10.0.0.1, 10.0.0.15', 'free_count': 13, 'gateway': '10.0.0.1', 'gateway6': null, 'group_list': ['default(bridged, prv0)'], 'inst_list': [], 'mac_prefix': null, 'map': 'XX.............X', 'name': 'nat', 'network': '10.0.0.0/28', 'network6': null, 'reserved_count': 3, 'tags': ['nfdhcpd'], // ... }, // ... ] .. _rapi-res-networks+post: ``POST`` ~~~~~~~~ Creates a network. If the optional bool *dry-run* argument is provided, the job will not be actually executed, only the pre-execution checks will be done. Returns: a job ID that can be used later for polling. Body parameters: .. opcode_params:: OP_NETWORK_ADD Job result: .. opcode_result:: OP_NETWORK_ADD .. _rapi-res-networks-network_name: ``/2/networks/[network_name]`` ++++++++++++++++++++++++++++++ Returns information about a network. .. rapi_resource_details:: /2/networks/[network_name] .. _rapi-res-networks-network_name+get: ``GET`` ~~~~~~~ Returns information about a network, similar to the bulk output from the network list. Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.NET_FIELDS))`. .. _rapi-res-networks-network_name+delete: ``DELETE`` ~~~~~~~~~~ Deletes a network. It supports the ``dry-run`` argument. Job result: .. opcode_result:: OP_NETWORK_REMOVE .. _rapi-res-networks-network_name-modify: ``/2/networks/[network_name]/modify`` +++++++++++++++++++++++++++++++++++++ Modifies the parameters of a network. .. rapi_resource_details:: /2/networks/[network_name]/modify .. _rapi-res-networks-network_name-modify+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_NETWORK_SET_PARAMS Job result: .. opcode_result:: OP_NETWORK_SET_PARAMS .. _rapi-res-networks-network_name-connect: ``/2/networks/[network_name]/connect`` ++++++++++++++++++++++++++++++++++++++ Connects a network to a nodegroup. .. rapi_resource_details:: /2/networks/[network_name]/connect .. _rapi-res-networks-network_name-connect+put: ``PUT`` ~~~~~~~ Returns a job ID. It supports the ``dry-run`` arguments. Body parameters: .. opcode_params:: OP_NETWORK_CONNECT Job result: .. opcode_result:: OP_NETWORK_CONNECT .. _rapi-res-networks-network_name-disconnect: ``/2/networks/[network_name]/disconnect`` +++++++++++++++++++++++++++++++++++++++++ Disonnects a network from a nodegroup. .. rapi_resource_details:: /2/networks/[network_name]/disconnect .. _rapi-res-networks-network_name-disconnect+put: ``PUT`` ~~~~~~~ Returns a job ID. It supports the ``dry-run`` arguments. Body parameters: .. opcode_params:: OP_NETWORK_DISCONNECT Job result: .. opcode_result:: OP_NETWORK_DISCONNECT .. _rapi-res-networks-network_name-tags: ``/2/networks/[network_name]/tags`` +++++++++++++++++++++++++++++++++++ Manages per-network tags. .. rapi_resource_details:: /2/networks/[network_name]/tags .. _rapi-res-networks-network_name-tags+get: ``GET`` ~~~~~~~ Returns a list of tags. Example:: ["tag1", "tag2", "tag3"] .. _rapi-res-networks-network_name-tags+put: ``PUT`` ~~~~~~~ Add a set of tags. The request as a list of strings should be ``PUT`` to this URI. The result will be a job id. It supports the ``dry-run`` argument. .. _rapi-res-networks-network_name-tags+delete: ``DELETE`` ~~~~~~~~~~ .. highlight:: none Delete a tag. In order to delete a set of tags, the DELETE request should be addressed to URI like:: /tags?tag=[tag]&tag=[tag] It supports the ``dry-run`` argument. ..highlight:: javascript .. _rapi-res-networks-network_name-rename: ``/2/networks/[network_name]/rename`` +++++++++++++++++++++++++++++++++++++ Renames a network. .. rapi_resource_details:: /2/networks/[network_name]/rename .. _rapi-res-networks-network_name-rename+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_NETWORK_RENAME :exclude: network_name Job result: .. opcode_result:: OP_NETWORK_RENAME .. _rapi-res-instances-multi-alloc: ``/2/instances-multi-alloc`` ++++++++++++++++++++++++++++ Tries to allocate multiple instances. .. rapi_resource_details:: /2/instances-multi-alloc .. _rapi-res-instances-multi-alloc+post: ``POST`` ~~~~~~~~ The parameters: .. opcode_params:: OP_INSTANCE_MULTI_ALLOC Job result: .. opcode_result:: OP_INSTANCE_MULTI_ALLOC .. _rapi-res-instances: ``/2/instances`` ++++++++++++++++ The instances resource. .. rapi_resource_details:: /2/instances .. _rapi-res-instances+get: ``GET`` ~~~~~~~ Returns a list of all available instances. Example:: [ { "name": "web.example.com", "uri": "/instances/web.example.com" }, { "name": "mail.example.com", "uri": "/instances/mail.example.com" } ] If the optional bool *bulk* argument is provided and set to a true value (i.e ``?bulk=1``), the output contains detailed information about instances as a list. Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.I_FIELDS))`. Example:: [ { "status": "running", "disk_usage": 20480, "nic.bridges": [ "xen-br0" ], "name": "web.example.com", "tags": ["tag1", "tag2"], "beparams": { "vcpus": 2, "memory": 512 }, "disk.sizes": [ 20480 ], "pnode": "node1.example.com", "nic.macs": ["01:23:45:67:89:01"], "snodes": ["node2.example.com"], "disk_template": "drbd", "admin_state": true, "os": "debian-etch", "oper_state": true, // ... }, // ... ] .. _rapi-res-instances+post: ``POST`` ~~~~~~~~ Creates an instance. If the optional bool *dry-run* argument is provided, the job will not be actually executed, only the pre-execution checks will be done. Query-ing the job result will return, in both dry-run and normal case, the list of nodes selected for the instance. Returns: a job ID that can be used later for polling. Body parameters: ``__version__`` (int, required) Must be ``1`` (older Ganeti versions used a different format for instance creation requests, version ``0``, but that format is no longer supported) .. opcode_params:: OP_INSTANCE_CREATE Earlier versions used parameters named ``name`` and ``os``. These have been replaced by ``instance_name`` and ``os_type`` to match the underlying opcode. The old names can still be used. Job result: .. opcode_result:: OP_INSTANCE_CREATE .. _rapi-res-instances-instance_name: ``/2/instances/[instance_name]`` ++++++++++++++++++++++++++++++++ Instance-specific resource. .. rapi_resource_details:: /2/instances/[instance_name] .. _rapi-res-instances-instance_name+get: ``GET`` ~~~~~~~ Returns information about an instance, similar to the bulk output from the instance list. Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.I_FIELDS))`. .. _rapi-res-instances-instance_name+delete: ``DELETE`` ~~~~~~~~~~ Deletes an instance. It supports the ``dry-run`` argument. Job result: .. opcode_result:: OP_INSTANCE_REMOVE .. _rapi-res-instances-instance_name-info: ``/2/instances/[instance_name]/info`` +++++++++++++++++++++++++++++++++++++++ .. rapi_resource_details:: /2/instances/[instance_name]/info .. _rapi-res-instances-instance_name-info+get: ``GET`` ~~~~~~~ Requests detailed information about the instance. An optional parameter, ``static`` (bool), can be set to return only static information from the configuration without querying the instance's nodes. The result will be a job id. Job result: .. opcode_result:: OP_INSTANCE_QUERY_DATA .. _rapi-res-instances-instance_name-reboot: ``/2/instances/[instance_name]/reboot`` +++++++++++++++++++++++++++++++++++++++ Reboots URI for an instance. .. rapi_resource_details:: /2/instances/[instance_name]/reboot .. _rapi-res-instances-instance_name-reboot+post: ``POST`` ~~~~~~~~ Reboots the instance. The URI takes optional ``type=soft|hard|full`` and ``ignore_secondaries=0|1`` parameters. ``type`` defines the reboot type. ``soft`` is just a normal reboot, without terminating the hypervisor. ``hard`` means full shutdown (including terminating the hypervisor process) and startup again. ``full`` is like ``hard`` but also recreates the configuration from ground up as if you would have done a ``gnt-instance shutdown`` and ``gnt-instance start`` on it. ``ignore_secondaries`` is a bool argument indicating if we start the instance even if secondary disks are failing. It supports the ``dry-run`` argument. Job result: .. opcode_result:: OP_INSTANCE_REBOOT .. _rapi-res-instances-instance_name-shutdown: ``/2/instances/[instance_name]/shutdown`` +++++++++++++++++++++++++++++++++++++++++ Instance shutdown URI. .. rapi_resource_details:: /2/instances/[instance_name]/shutdown .. _rapi-res-instances-instance_name-shutdown+put: ``PUT`` ~~~~~~~ Shutdowns an instance. It supports the ``dry-run`` argument. .. opcode_params:: OP_INSTANCE_SHUTDOWN :exclude: instance_name, dry_run Job result: .. opcode_result:: OP_INSTANCE_SHUTDOWN .. _rapi-res-instances-instance_name-startup: ``/2/instances/[instance_name]/startup`` ++++++++++++++++++++++++++++++++++++++++ Instance startup URI. .. rapi_resource_details:: /2/instances/[instance_name]/startup .. _rapi-res-instances-instance_name-startup+put: ``PUT`` ~~~~~~~ Startup an instance. The URI takes an optional ``force=1|0`` parameter to start the instance even if secondary disks are failing. It supports the ``dry-run`` argument. Job result: .. opcode_result:: OP_INSTANCE_STARTUP .. _rapi-res-instances-instance_name-reinstall: ``/2/instances/[instance_name]/reinstall`` ++++++++++++++++++++++++++++++++++++++++++++++ Installs the operating system again. .. rapi_resource_details:: /2/instances/[instance_name]/reinstall .. _rapi-res-instances-instance_name-reinstall+post: ``POST`` ~~~~~~~~ Returns a job ID. Body parameters: ``os`` (string, required) Instance operating system. ``start`` (bool, defaults to true) Whether to start instance after reinstallation. ``osparams`` (dict) Dictionary with (temporary) OS parameters. For backwards compatbility, this resource also takes the query parameters ``os`` (OS template name) and ``nostartup`` (bool). New clients should use the body parameters. .. _rapi-res-instances-instance_name-replace-disks: ``/2/instances/[instance_name]/replace-disks`` ++++++++++++++++++++++++++++++++++++++++++++++ Replaces disks on an instance. .. rapi_resource_details:: /2/instances/[instance_name]/replace-disks .. _rapi-res-instances-instance_name-replace-disks+post: ``POST`` ~~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_INSTANCE_REPLACE_DISKS :exclude: instance_name Ganeti 2.4 and below used query parameters. Those are deprecated and should no longer be used. Job result: .. opcode_result:: OP_INSTANCE_REPLACE_DISKS .. _rapi-res-instances-instance_name-activate-disks: ``/2/instances/[instance_name]/activate-disks`` +++++++++++++++++++++++++++++++++++++++++++++++ Activate disks on an instance. .. rapi_resource_details:: /2/instances/[instance_name]/activate-disks .. _rapi-res-instances-instance_name-activate-disks+put: ``PUT`` ~~~~~~~ Takes the bool parameter ``ignore_size``. When set ignore the recorded size (useful for forcing activation when recorded size is wrong). Job result: .. opcode_result:: OP_INSTANCE_ACTIVATE_DISKS .. _rapi-res-instances-instance_name-deactivate-disks: ``/2/instances/[instance_name]/deactivate-disks`` +++++++++++++++++++++++++++++++++++++++++++++++++ Deactivate disks on an instance. .. rapi_resource_details:: /2/instances/[instance_name]/deactivate-disks .. _rapi-res-instances-instance_name-deactivate-disks+put: ``PUT`` ~~~~~~~ Takes no parameters. Job result: .. opcode_result:: OP_INSTANCE_DEACTIVATE_DISKS .. _rapi-res-instances-instance_name-recreate-disks: ``/2/instances/[instance_name]/recreate-disks`` +++++++++++++++++++++++++++++++++++++++++++++++++ Recreate disks of an instance. .. rapi_resource_details:: /2/instances/[instance_name]/recreate-disks .. _rapi-res-instances-instance_name-recreate-disks+post: ``POST`` ~~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_INSTANCE_RECREATE_DISKS :exclude: instance_name Job result: .. opcode_result:: OP_INSTANCE_RECREATE_DISKS .. _rapi-res-instances-instance_name-disk-disk_index-grow: ``/2/instances/[instance_name]/disk/[disk_index]/grow`` +++++++++++++++++++++++++++++++++++++++++++++++++++++++ Grows one disk of an instance. .. rapi_resource_details:: /2/instances/[instance_name]/disk/[disk_index]/grow .. _rapi-res-instances-instance_name-disk-disk_index-grow+post: ``POST`` ~~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_INSTANCE_GROW_DISK :exclude: instance_name, disk Job result: .. opcode_result:: OP_INSTANCE_GROW_DISK .. _rapi-res-instances-instance_name-prepare-export: ``/2/instances/[instance_name]/prepare-export`` +++++++++++++++++++++++++++++++++++++++++++++++++ Prepares an export of an instance. .. rapi_resource_details:: /2/instances/[instance_name]/prepare-export .. _rapi-res-instances-instance_name-prepare-export+put: ``PUT`` ~~~~~~~ Takes one parameter, ``mode``, for the export mode. Returns a job ID. Job result: .. opcode_result:: OP_BACKUP_PREPARE .. _rapi-res-instances-instance_name-export: ``/2/instances/[instance_name]/export`` +++++++++++++++++++++++++++++++++++++++++++++++++ Exports an instance. .. rapi_resource_details:: /2/instances/[instance_name]/export .. _rapi-res-instances-instance_name-export+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_BACKUP_EXPORT :exclude: instance_name :alias: target_node=destination Job result: .. opcode_result:: OP_BACKUP_EXPORT .. _rapi-res-instances-instance_name-migrate: ``/2/instances/[instance_name]/migrate`` ++++++++++++++++++++++++++++++++++++++++ Migrates an instance. .. rapi_resource_details:: /2/instances/[instance_name]/migrate .. _rapi-res-instances-instance_name-migrate+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_INSTANCE_MIGRATE :exclude: instance_name, live Job result: .. opcode_result:: OP_INSTANCE_MIGRATE .. _rapi-res-instances-instance_name-failover: ``/2/instances/[instance_name]/failover`` +++++++++++++++++++++++++++++++++++++++++ Does a failover of an instance. .. rapi_resource_details:: /2/instances/[instance_name]/failover .. _rapi-res-instances-instance_name-failover+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_INSTANCE_FAILOVER :exclude: instance_name Job result: .. opcode_result:: OP_INSTANCE_FAILOVER .. _rapi-res-instances-instance_name-rename: ``/2/instances/[instance_name]/rename`` ++++++++++++++++++++++++++++++++++++++++ Renames an instance. .. rapi_resource_details:: /2/instances/[instance_name]/rename .. _rapi-res-instances-instance_name-rename+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_INSTANCE_RENAME :exclude: instance_name Job result: .. opcode_result:: OP_INSTANCE_RENAME .. _rapi-res-instances-instance_name-modify: ``/2/instances/[instance_name]/modify`` ++++++++++++++++++++++++++++++++++++++++ Modifies an instance. .. rapi_resource_details:: /2/instances/[instance_name]/modify .. _rapi-res-instances-instance_name-modify+put: ``PUT`` ~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_INSTANCE_SET_PARAMS :exclude: instance_name Job result: .. opcode_result:: OP_INSTANCE_SET_PARAMS .. _rapi-res-instances-instance_name-console: ``/2/instances/[instance_name]/console`` ++++++++++++++++++++++++++++++++++++++++ Request information for connecting to instance's console. .. rapi_resource_details:: /2/instances/[instance_name]/console .. _rapi-res-instances-instance_name-console+get: ``GET`` ~~~~~~~ Returns a dictionary containing information about the instance's console. Contained keys: .. pyassert:: constants.CONS_ALL == frozenset([ constants.CONS_MESSAGE, constants.CONS_SSH, constants.CONS_VNC, constants.CONS_SPICE, ]) .. pyassert:: frozenset(objects.InstanceConsole.GetAllSlots()) == frozenset([ "command", "display", "host", "instance", "kind", "message", "port", "user", ]) ``instance`` Instance name ``kind`` Console type, one of :pyeval:`constants.CONS_SSH`, :pyeval:`constants.CONS_VNC`, :pyeval:`constants.CONS_SPICE` or :pyeval:`constants.CONS_MESSAGE` ``message`` Message to display (:pyeval:`constants.CONS_MESSAGE` type only) ``host`` Host to connect to (:pyeval:`constants.CONS_SSH`, :pyeval:`constants.CONS_VNC` or :pyeval:`constants.CONS_SPICE` only) ``port`` TCP port to connect to (:pyeval:`constants.CONS_VNC` or :pyeval:`constants.CONS_SPICE` only) ``user`` Username to use (:pyeval:`constants.CONS_SSH` only) ``command`` Command to execute on machine (:pyeval:`constants.CONS_SSH` only) ``display`` VNC display number (:pyeval:`constants.CONS_VNC` only) .. _rapi-res-instances-instance_name-tags: ``/2/instances/[instance_name]/tags`` +++++++++++++++++++++++++++++++++++++ Manages per-instance tags. .. rapi_resource_details:: /2/instances/[instance_name]/tags .. _rapi-res-instances-instance_name-tags+get: ``GET`` ~~~~~~~ Returns a list of tags. Example:: ["tag1", "tag2", "tag3"] .. _rapi-res-instances-instance_name-tags+put: ``PUT`` ~~~~~~~ Add a set of tags. The request as a list of strings should be ``PUT`` to this URI. The result will be a job id. It supports the ``dry-run`` argument. .. _rapi-res-instances-instance_name-tags+delete: ``DELETE`` ~~~~~~~~~~ Delete a tag. In order to delete a set of tags, the DELETE request should be addressed to URI like:: /tags?tag=[tag]&tag=[tag] It supports the ``dry-run`` argument. .. _rapi-res-jobs: ``/2/jobs`` +++++++++++ The ``/2/jobs`` resource. .. rapi_resource_details:: /2/jobs .. _rapi-res-jobs+get: ``GET`` ~~~~~~~ Returns a dictionary of jobs. Returns: a dictionary with jobs id and uri. If the optional bool *bulk* argument is provided and set to a true value (i.e. ``?bulk=1``), the output contains detailed information about jobs as a list. Returned fields for bulk requests (unlike other bulk requests, these fields are not the same as for per-job requests): :pyeval:`utils.CommaJoin(sorted(rlib2.J_FIELDS_BULK))`. .. _rapi-res-jobs-job_id: ``/2/jobs/[job_id]`` ++++++++++++++++++++ Individual job URI. .. rapi_resource_details:: /2/jobs/[job_id] .. _rapi-res-jobs-job_id+get: ``GET`` ~~~~~~~ Returns a dictionary with job parameters, containing the fields :pyeval:`utils.CommaJoin(sorted(rlib2.J_FIELDS))`. The result includes: - id: job ID as a number - status: current job status as a string - ops: involved OpCodes as a list of dictionaries for each opcodes in the job - opstatus: OpCodes status as a list - opresult: OpCodes results as a list For a successful opcode, the ``opresult`` field corresponding to it will contain the raw result from its :term:`LogicalUnit`. In case an opcode has failed, its element in the opresult list will be a list of two elements: - first element the error type (the Ganeti internal error name) - second element a list of either one or two elements: - the first element is the textual error description - the second element, if any, will hold an error classification The error classification is most useful for the ``OpPrereqError`` error type - these errors happen before the OpCode has started executing, so it's possible to retry the OpCode without side effects. But whether it make sense to retry depends on the error classification: .. pyassert:: errors.ECODE_ALL == set([errors.ECODE_RESOLVER, errors.ECODE_NORES, errors.ECODE_INVAL, errors.ECODE_STATE, errors.ECODE_NOENT, errors.ECODE_EXISTS, errors.ECODE_NOTUNIQUE, errors.ECODE_FAULT, errors.ECODE_ENVIRON, errors.ECODE_TEMP_NORES]) :pyeval:`errors.ECODE_RESOLVER` Resolver errors. This usually means that a name doesn't exist in DNS, so if it's a case of slow DNS propagation the operation can be retried later. :pyeval:`errors.ECODE_NORES` Not enough resources (iallocator failure, disk space, memory, etc.). If the resources on the cluster increase, the operation might succeed. :pyeval:`errors.ECODE_TEMP_NORES` Simliar to :pyeval:`errors.ECODE_NORES`, but indicating the operation should be attempted again after some time. :pyeval:`errors.ECODE_INVAL` Wrong arguments (at syntax level). The operation will not ever be accepted unless the arguments change. :pyeval:`errors.ECODE_STATE` Wrong entity state. For example, live migration has been requested for a down instance, or instance creation on an offline node. The operation can be retried once the resource has changed state. :pyeval:`errors.ECODE_NOENT` Entity not found. For example, information has been requested for an unknown instance. :pyeval:`errors.ECODE_EXISTS` Entity already exists. For example, instance creation has been requested for an already-existing instance. :pyeval:`errors.ECODE_NOTUNIQUE` Resource not unique (e.g. MAC or IP duplication). :pyeval:`errors.ECODE_FAULT` Internal cluster error. For example, a node is unreachable but not set offline, or the ganeti node daemons are not working, etc. A ``gnt-cluster verify`` should be run. :pyeval:`errors.ECODE_ENVIRON` Environment error (e.g. node disk error). A ``gnt-cluster verify`` should be run. Note that in the above list, by entity we refer to a node or instance, while by a resource we refer to an instance's disk, or NIC, etc. .. _rapi-res-jobs-job_id+delete: ``DELETE`` ~~~~~~~~~~ Cancel a not-yet-started job. .. _rapi-res-jobs-job_id-wait: ``/2/jobs/[job_id]/wait`` +++++++++++++++++++++++++ .. rapi_resource_details:: /2/jobs/[job_id]/wait .. _rapi-res-jobs-job_id-wait+get: ``GET`` ~~~~~~~ Waits for changes on a job. Takes the following body parameters in a dict: ``fields`` The job fields on which to watch for changes ``previous_job_info`` Previously received field values or None if not yet available ``previous_log_serial`` Highest log serial number received so far or None if not yet available Returns None if no changes have been detected and a dict with two keys, ``job_info`` and ``log_entries`` otherwise. .. _rapi-res-nodes: ``/2/nodes`` ++++++++++++ Nodes resource. .. rapi_resource_details:: /2/nodes .. _rapi-res-nodes+get: ``GET`` ~~~~~~~ Returns a list of all nodes. Example:: [ { "id": "node1.example.com", "uri": "/nodes/node1.example.com" }, { "id": "node2.example.com", "uri": "/nodes/node2.example.com" } ] If the optional bool *bulk* argument is provided and set to a true value (i.e ``?bulk=1``), the output contains detailed information about nodes as a list. Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.N_FIELDS))`. Example:: [ { "pinst_cnt": 1, "mfree": 31280, "mtotal": 32763, "name": "www.example.com", "tags": [], "mnode": 512, "dtotal": 5246208, "sinst_cnt": 2, "dfree": 5171712, "offline": false, // ... }, // ... ] .. _rapi-res-nodes-node_name: ``/2/nodes/[node_name]`` +++++++++++++++++++++++++++++++++ Returns information about a node. .. rapi_resource_details:: /2/nodes/[node_name] .. _rapi-res-nodes-node_name+get: ``GET`` ~~~~~~~ Returned fields: :pyeval:`utils.CommaJoin(sorted(rlib2.N_FIELDS))`. .. _rapi-res-nodes-node_name-powercycle: ``/2/nodes/[node_name]/powercycle`` +++++++++++++++++++++++++++++++++++ Powercycles a node. .. rapi_resource_details:: /2/nodes/[node_name]/powercycle .. _rapi-res-nodes-node_name-powercycle+post: ``POST`` ~~~~~~~~ Returns a job ID. Job result: .. opcode_result:: OP_NODE_POWERCYCLE .. _rapi-res-nodes-node_name-evacuate: ``/2/nodes/[node_name]/evacuate`` +++++++++++++++++++++++++++++++++ Evacuates instances off a node. .. rapi_resource_details:: /2/nodes/[node_name]/evacuate .. _rapi-res-nodes-node_name-evacuate+post: ``POST`` ~~~~~~~~ Returns a job ID. The result of the job will contain the IDs of the individual jobs submitted to evacuate the node. Body parameters: .. opcode_params:: OP_NODE_EVACUATE :exclude: nodes Up to and including Ganeti 2.4 query arguments were used. Those are no longer supported. The new request can be detected by the presence of the :pyeval:`rlib2._NODE_EVAC_RES1` feature string. Job result: .. opcode_result:: OP_NODE_EVACUATE .. _rapi-res-nodes-node_name-migrate: ``/2/nodes/[node_name]/migrate`` +++++++++++++++++++++++++++++++++ Migrates all primary instances from a node. .. rapi_resource_details:: /2/nodes/[node_name]/migrate .. _rapi-res-nodes-node_name-migrate+post: ``POST`` ~~~~~~~~ If no mode is explicitly specified, each instances' hypervisor default migration mode will be used. Body parameters: .. opcode_params:: OP_NODE_MIGRATE :exclude: node_name The query arguments used up to and including Ganeti 2.4 are deprecated and should no longer be used. The new request format can be detected by the presence of the :pyeval:`rlib2._NODE_MIGRATE_REQV1` feature string. Job result: .. opcode_result:: OP_NODE_MIGRATE .. _rapi-res-nodes-node_name-role: ``/2/nodes/[node_name]/role`` +++++++++++++++++++++++++++++ Manages node role. .. rapi_resource_details:: /2/nodes/[node_name]/role The role is always one of the following: - drained - master-candidate - offline - regular Note that the 'master' role is a special, and currently it can't be modified via RAPI, only via the command line (``gnt-cluster master-failover``). .. _rapi-res-nodes-node_name-role+get: ``GET`` ~~~~~~~ Returns the current node role. Example:: "master-candidate" .. _rapi-res-nodes-node_name-role+put: ``PUT`` ~~~~~~~ Change the node role. The request is a string which should be PUT to this URI. The result will be a job id. It supports the bool ``force`` argument. Job result: .. opcode_result:: OP_NODE_SET_PARAMS .. _rapi-res-nodes-node_name-modify: ``/2/nodes/[node_name]/modify`` +++++++++++++++++++++++++++++++ Modifies the parameters of a node. .. rapi_resource_details:: /2/nodes/[node_name]/modify .. _rapi-res-nodes-node_name-modify+post: ``POST`` ~~~~~~~~ Returns a job ID. Body parameters: .. opcode_params:: OP_NODE_SET_PARAMS :exclude: node_name Job result: .. opcode_result:: OP_NODE_SET_PARAMS .. _rapi-res-nodes-node_name-storage: ``/2/nodes/[node_name]/storage`` ++++++++++++++++++++++++++++++++ Manages storage units on the node. .. rapi_resource_details:: /2/nodes/[node_name]/storage .. _rapi-res-nodes-node_name-storage+get: ``GET`` ~~~~~~~ .. pyassert:: constants.STS_REPORT == set([constants.ST_FILE, constants.ST_LVM_PV, constants.ST_LVM_VG]) Requests a list of storage units on a node. Requires the parameters ``storage_type`` for storage types that support space reporting (one of :pyeval:`constants.ST_FILE`, :pyeval:`constants.ST_LVM_PV` or :pyeval:`constants.ST_LVM_VG`) and ``output_fields``. The result will be a job id, using which the result can be retrieved. .. _rapi-res-nodes-node_name-storage-modify: ``/2/nodes/[node_name]/storage/modify`` +++++++++++++++++++++++++++++++++++++++ Modifies storage units on the node. .. rapi_resource_details:: /2/nodes/[node_name]/storage/modify .. _rapi-res-nodes-node_name-storage-modify+put: ``PUT`` ~~~~~~~ Modifies parameters of storage units on the node. Requires the parameters ``storage_type`` (one of :pyeval:`constants.ST_FILE`, :pyeval:`constants.ST_LVM_PV` or :pyeval:`constants.ST_LVM_VG`) and ``name`` (name of the storage unit). Parameters can be passed additionally. Currently only :pyeval:`constants.SF_ALLOCATABLE` (bool) is supported. The result will be a job id. Job result: .. opcode_result:: OP_NODE_MODIFY_STORAGE .. _rapi-res-nodes-node_name-storage-repair: ``/2/nodes/[node_name]/storage/repair`` +++++++++++++++++++++++++++++++++++++++ Repairs a storage unit on the node. .. rapi_resource_details:: /2/nodes/[node_name]/storage/repair .. _rapi-res-nodes-node_name-storage-repair+put: ``PUT`` ~~~~~~~ .. pyassert:: constants.VALID_STORAGE_OPERATIONS == { constants.ST_LVM_VG: set([constants.SO_FIX_CONSISTENCY]), } Repairs a storage unit on the node. Requires the parameters ``storage_type`` (currently only :pyeval:`constants.ST_LVM_VG` can be repaired) and ``name`` (name of the storage unit). The result will be a job id. Job result: .. opcode_result:: OP_REPAIR_NODE_STORAGE .. _rapi-res-nodes-node_name-tags: ``/2/nodes/[node_name]/tags`` +++++++++++++++++++++++++++++ Manages per-node tags. .. rapi_resource_details:: /2/nodes/[node_name]/tags .. _rapi-res-nodes-node_name-tags+get: ``GET`` ~~~~~~~ Returns a list of tags. Example:: ["tag1", "tag2", "tag3"] .. _rapi-res-nodes-node_name-tags+put: ``PUT`` ~~~~~~~ Add a set of tags. The request as a list of strings should be PUT to this URI. The result will be a job id. It supports the ``dry-run`` argument. .. _rapi-res-nodes-node_name-tags+delete: ``DELETE`` ~~~~~~~~~~ Deletes tags. In order to delete a set of tags, the DELETE request should be addressed to URI like:: /tags?tag=[tag]&tag=[tag] It supports the ``dry-run`` argument. .. _rapi-res-query-resource: ``/2/query/[resource]`` +++++++++++++++++++++++ Requests resource information. Available fields can be found in man pages and using ``/2/query/[resource]/fields``. The resource is one of :pyeval:`utils.CommaJoin(constants.QR_VIA_RAPI)`. See the :doc:`query2 design document ` for more details. .. rapi_resource_details:: /2/query/[resource] .. _rapi-res-query-resource+get: ``GET`` ~~~~~~~ Returns list of included fields and actual data. Takes a query parameter named "fields", containing a comma-separated list of field names. Does not support filtering. .. _rapi-res-query-resource+put: ``PUT`` ~~~~~~~ Returns list of included fields and actual data. The list of requested fields can either be given as the query parameter "fields" or as a body parameter with the same name. The optional body parameter "filter" can be given and must be either ``null`` or a list containing filter operators. .. _rapi-res-query-resource-fields: ``/2/query/[resource]/fields`` ++++++++++++++++++++++++++++++ Request list of available fields for a resource. The resource is one of :pyeval:`utils.CommaJoin(constants.QR_VIA_RAPI)`. See the :doc:`query2 design document ` for more details. .. rapi_resource_details:: /2/query/[resource]/fields .. _rapi-res-query-resource-fields+get: ``GET`` ~~~~~~~ Returns a list of field descriptions for available fields. Takes an optional query parameter named "fields", containing a comma-separated list of field names. .. _rapi-res-os: ``/2/os`` +++++++++ OS resource. .. rapi_resource_details:: /2/os .. _rapi-res-os+get: ``GET`` ~~~~~~~ Return a list of all OSes. Can return error 500 in case of a problem. Since this is a costly operation for Ganeti 2.0, it is not recommended to execute it too often. Example:: ["debian-etch"] .. _rapi-res-tags: ``/2/tags`` +++++++++++ Manages cluster tags. .. rapi_resource_details:: /2/tags .. _rapi-res-tags+get: ``GET`` ~~~~~~~ Returns the cluster tags. Example:: ["tag1", "tag2", "tag3"] .. _rapi-res-tags+put: ``PUT`` ~~~~~~~ Adds a set of tags. The request as a list of strings should be PUT to this URI. The result will be a job id. It supports the ``dry-run`` argument. .. _rapi-res-tags+delete: ``DELETE`` ~~~~~~~~~~ Deletes tags. In order to delete a set of tags, the DELETE request should be addressed to URI like:: /tags?tag=[tag]&tag=[tag] It supports the ``dry-run`` argument. .. _rapi-res-version: ``/version`` ++++++++++++ The version resource. This resource should be used to determine the remote API version and to adapt clients accordingly. .. rapi_resource_details:: /version .. _rapi-res-version+get: ``GET`` ~~~~~~~ Returns the remote API version. Ganeti 1.2 returned ``1`` and Ganeti 2.0 returns ``2``. .. _rapi-access-permissions: Access permissions ------------------ The following list describes the access permissions required for each resource. See :ref:`rapi-users` for more details. .. rapi_access_table:: .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/security.rst000064400000000000000000000305031476477700300167530ustar00rootroot00000000000000Security in Ganeti ================== Documents Ganeti version 3.1 Ganeti was developed to run on internal, trusted systems. As such, the security model is all-or-nothing. Up to version 2.3 all Ganeti code ran as root. Since version 2.4 it is possible to run all daemons except the node daemon and the monitoring daemon as non-root users by specifying user names and groups at build time. The node daemon continues to require root privileges to create logical volumes, DRBD devices, start instances, etc. Cluster commands can be run as root or by users in a group specified at build time. The monitoring daemon requires root privileges in order to be able to access and present information that are only avilable to root (such as the output of the ``xl`` command of Xen). Host issues ----------- For a host on which the Ganeti software has been installed, but not joined to a cluster, there are no changes to the system. For a host that has been joined to the cluster, there are very important changes: - The host will have its SSH host key replaced with the one of the cluster (which is the one the initial node had at the cluster creation) - A new public key will be added to root's ``authorized_keys`` file, granting root access to all nodes of the cluster. The private part of the key is also distributed to all nodes. Old files are renamed. - Communication between nodes is encrypted using SSL/TLS. A common key and certificate combo is shared between all nodes of the cluster. At this time, no CA is used. - The Ganeti node daemon will accept RPC requests from any host that is master candidate within the cluster, and the operations it will do as a result of these requests are: - running commands under the ``/etc/ganeti/hooks`` directory - creating DRBD disks between it and the IP it has been told - overwrite a defined list of files on the host As you can see, as soon as a node is joined, it becomes equal to all other nodes in the cluster wrt to SSH and equal to all non-master candidate nodes wrt to RPC, and the security of the cluster is determined by the weakest node. Note that only the SSH key will allow other machines to run any command on this node; the RPC method will run only: - well defined commands to create, remove, activate logical volumes, drbd devices, start/stop instances, etc; - run well-defined SSH commands on other nodes in the cluster - scripts under the ``/etc/ganeti/hooks`` directory - scripts under the ``/etc/ganeti/restricted-commands`` directory, if this feature has been enabled at build time (see below) It is therefore important to make sure that the contents of the ``/etc/ganeti/hooks`` and ``/etc/ganeti/restricted-commands`` directories are supervised and only trusted sources can populate them. Restricted commands ~~~~~~~~~~~~~~~~~~~ The restricted commands feature is new in Ganeti 2.7. It enables the administrator to run any commands in the ``/etc/ganeti/restricted-commands`` directory, if the feature has been enabled at build time, subject to the following restrictions: - No parameters may be passed - No absolute or relative path may be passed, only a filename - The ``/etc/ganeti/restricted-commands`` directory must be owned by root:root and have mode 0755 or stricter - Executables must be regular files or symlinks, and must be executable by root:root Note that it's not possible to list the contents of the directory, and there is an intentional delay when trying to execute a non-existing command (to slow-down dictionary attacks). Since for Ganeti itself this functionality is not needed, and is only provided as a way to help administrate or recover nodes, it is a local site decision whether to enable or not the restricted commands feature. By default, this feature is disabled. Cluster issues -------------- As mentioned above, there are multiple ways of communication between cluster nodes: - SSH-based, for high-volume traffic like image dumps or for low-level command, e.g. restarting the Ganeti node daemon - RPC communication between master and nodes - DRBD real-time disk replication traffic The SSH traffic is protected (after the initial login to a new node) by the cluster-wide shared SSH key. RPC communication between the master and nodes is protected using SSL/TLS encryption. The server must have must have the cluster-wide shared SSL/TLS certificate. When acting as a client, the nodes use an individual SSL/TLS certificate. On incoming requests, the server checks whether the client's certificate is that of a master candidate by verifying its finterprint to a list of known master candidate certificates. We decided not to use a CA (yet) to simplify the key handling. The DRBD traffic is not protected by encryption, as DRBD does not support this. It's therefore recommended to implement host-level firewalling or to use a separate range of IP addresses for the DRBD traffic (this is supported in Ganeti through the use of a secondary interface) which is not routed outside the cluster. DRBD connections are protected from erroneous connections to other machines (as may happen due to software issues), and from accepting connections from other machines, by using a shared secret, exchanged via RPC requests from the master to the nodes when configuring the device. Master daemon ------------- The command-line tools to master daemon communication is done via a UNIX socket, whose permissions are reset to ``0660`` after listening but before serving requests. This permission-based protection is documented and works on Linux, but is not-portable; however, Ganeti doesn't work on non-Linux system at the moment. Luxi daemon ----------- The ``luxid`` daemon (automatically enabled if ``confd`` is enabled at build time) serves local (UNIX socket) queries about the run-time configuration. Answering these means talking to other cluster nodes, exactly as ``masterd`` does. See the notes for ``masterd`` regarding permission-based protection. Conf daemon ----------- In Ganeti 2.8, the ``confd`` daemon (if enabled at build time), serves network-originated queries about parts of the static cluster configuration. If Ganeti is not configured (at build time) to use separate users, ``confd`` has access to all Ganeti related files (including internal RPC SSL certificates). This makes it a bit more sensitive to bugs (a remote attacker could get direct access to the intra-cluster RPC), so to harden security it's recommended to: - disable confd at build time if it (and ``luxid``) is not needed in your setup. - configure Ganeti (at build time) to use separate users, so that the confd daemon doesn't also have access to the server SSL/TLS certificates. - add firewall rules to protect the ``confd`` port or bind it to a trusted address. Make sure that all nodes can access the daemon, as the monitoring daemon requires it. Monitoring daemon ----------------- The monitoring daemon provides information about the status and the performance of the cluster over HTTP. It is currently unencrypted and non-authenticated, therefore it is strongly advised to set proper firewalling rules to prevent unwanted access. The monitoring daemon runs as root, because it needs to be able to access privileged information (such as the state of the instances as provided by the Xen hypervisor). Nevertheless, the security implications are mitigated by the fact that the agent only provides reporting functionalities, without the ability to actually modify the state of the cluster. Remote API ---------- Starting with Ganeti 2.0, Remote API traffic is encrypted using SSL/TLS by default. It supports Basic authentication as per :rfc:`2617`. Users can be granted different capabilities. Details can be found in the :ref:`RAPI documentation `. Paths for certificate, private key and CA files required for SSL/TLS will be set at source configure time. Symlinks or command line parameters may be used to use different files. The RAPI binds to all interfaces by default, and allows read-only requests without the need for authentication. In the case that one of the interfaces RAPI binds to is publicly exposed, this will allow anyone in the world to read the state of the cluster, divulging potentially useful data such as the names of instances, their IP addresses, etc. Since the RAPI daemon resides on the master node as well, DoS attacks can result in Ganeti outages or issues with instances located on the master node. We recommend that you reduce the attack surface by either placing RAPI in an environment where you can control access to it, or should you need to expose it publicly, use various RAPI daemon options to lock functionality down to only what you need. RAPI daemon options are best added to ``/etc/default/ganeti``, the ``RAPI_ARGS`` variable. Some examples of situations where you might want to expose the RAPI are cross-cluster instance moves, which can be done only via the RAPI. If you do not use RAPI at all, we recommend that you lock it down by binding it to the loopback interface. This can be done by passing the ``-b 127.0.0.1`` parameter to the RAPI daemon. Preventing the RAPI from starting or making it unreachable on the master node is not recommended, as the watcher performs health checks and will attempt to restart the daemon repeatedly. If you intend to use the RAPI and to expose it to the public, make sure to use the ``--require-authentication`` flag, disabling anonymous HTTP requests. Ganeti currently cannot protect users adequately from DoS attacks based on client-side HTTPS parameter renegotiation due to the Python OpenSSL library lacking necessary features. To protect yourself from these, the use of a HTTPS proxy handling this correctly is needed (e.g. nginx). Useful options for setting RAPI up for cooperation with the proxy are: - ``-p PORT`` for allowing the default RAPI port to be used by the proxy - ``--no-ssl`` to disable SSL as it will be handled by the proxy anyway Inter-cluster instance moves ---------------------------- To move instances between clusters, different clusters must be able to communicate with each other over a secure channel. Up to and including Ganeti 2.1, clusters were self-contained entities and had no knowledge of other clusters. With Ganeti 2.2, clusters can exchange data if tokens (an encryption certificate) was exchanged by a trusted third party before. KVM Security ------------ When running KVM instances under Ganeti three security models ara available: "none", "user" and "pool". Under security model "none" instances run by default as root. This means that, if an instance gets jail broken, it will be able to own the host node, and thus the ganeti cluster. This is the default model, and the only one available before Ganeti 2.1.2. Under security model "user" an instance is run as the user specified by the hypervisor parameter "security_domain". This makes it easy to run all instances as non privileged users, and allows one to manually allocate specific users to specific instances or sets of instances. If the specified user doesn't have permissions a jail broken instance will need some local privilege escalation before being able to take over the node and the cluster. It's possible though for a jail broken instance to affect other ones running under the same user. Under security model "pool" a global cluster-level uid pool is used to start each instance on the same node under a different user. The uids in the cluster pool can be set with ``gnt-cluster init`` and ``gnt-cluster modify``, and must correspond to existing users on all nodes. Ganeti will then allocate one to each instance, as needed. This way a jail broken instance won't be able to affect any other. Since the users are handed out by ganeti in a per-node randomized way, in this mode there is no way to make sure a particular instance is always run as a certain user. Use mode "user" for that. In addition to these precautions, if you want to avoid instances sending traffic on your node network, you can use an iptables rule such as:: iptables -A OUTPUT -m owner --uid-owner [-] -j LOG \ --log-prefix "ganeti uid pool user network traffic" iptables -A OUTPUT -m owner --uid-owner [-] -j DROP This won't affect regular instance traffic (that comes out of the tapX allocated to the instance, and can be filtered or subject to appropriate policy routes) but will stop any user generated traffic that might come from a jailbroken instance. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/doc/users/000075500000000000000000000000001476477700300155125ustar00rootroot00000000000000ganeti-3.1.0~rc2/doc/users/groupmemberships.in000064400000000000000000000006451476477700300214420ustar00rootroot00000000000000@GNTMASTERUSER@ @GNTDAEMONSGROUP@ @GNTCONFDUSER@ @GNTDAEMONSGROUP@ @GNTWCONFDUSER@ @GNTDAEMONSGROUP@ @GNTLUXIDUSER@ @GNTDAEMONSGROUP@ @GNTRAPIUSER@ @GNTDAEMONSGROUP@ @GNTMONDUSER@ @GNTDAEMONSGROUP@ @GNTMETADUSER@ @GNTDAEMONSGROUP@ @GNTMASTERUSER@ @GNTADMINGROUP@ @GNTRAPIUSER@ @GNTADMINGROUP@ @GNTMASTERUSER@ @GNTCONFDGROUP@ @GNTMONDUSER@ @GNTMASTERDGROUP@ @GNTLUXIDUSER@ @GNTMASTERDGROUP@ @GNTLUXIDUSER@ @GNTCONFDGROUP@ ganeti-3.1.0~rc2/doc/users/groups.in000064400000000000000000000002231476477700300173560ustar00rootroot00000000000000@GNTDAEMONSGROUP@ @GNTADMINGROUP@ @GNTMASTERDGROUP@ @GNTRAPIGROUP@ @GNTCONFDGROUP@ @GNTWCONFDGROUP@ @GNTLUXIDGROUP@ @GNTMONDGROUP@ @GNTMETADGROUP@ ganeti-3.1.0~rc2/doc/users/users.in000064400000000000000000000003511476477700300172020ustar00rootroot00000000000000@GNTMASTERUSER@ @GNTMASTERDGROUP@ @GNTRAPIUSER@ @GNTRAPIGROUP@ @GNTCONFDUSER@ @GNTCONFDGROUP@ @GNTWCONFDUSER@ @GNTWCONFDGROUP@ @GNTLUXIDUSER@ @GNTLUXIDGROUP@ @GNTMONDUSER@ @GNTMONDGROUP@ @GNTMETADUSER@ @GNTMETADGROUP@ @GNTNODEDUSER@ ganeti-3.1.0~rc2/doc/virtual-cluster.rst000064400000000000000000000102701476477700300202500ustar00rootroot00000000000000Virtual cluster support ======================= Documents Ganeti version 3.1 .. contents:: Introduction ------------ This is a description of Ganeti's support for virtual clusters introduced in version 2.7. The original design is described in a separate :doc:`design document `. A virtual cluster consists of multiple virtual nodes (instances of Ganeti daemons) running on the same physical machine within one operating system. This way multiple (virtual) nodes can be simulated using a single machine. Virtual clusters can be run as a user without root privileges (see :ref:`limitations `). While not implemented in the helper setup script at the time of this writing, virtual clusters can also be split over multiple physical machines, allowing for even more virtual nodes. .. _limitations: Limitations ----------- Due to historical and practical design decisions virtual clusters have several limitations. - "fake" hypervisor only - Instances must be diskless or file-backed - Node information is the same over multiple virtual nodes (e.g. free memory) - If running as a user without root privileges, certain operations are not available; some operations are not useful even when running as root (e.g. powercycle) - OS definitions must be prepared for this setup - Setup is partially manual, especially when not running as root Basics ------ Ganeti programs act as running on a virtual node if the environment variables ``GANETI_ROOTDIR`` and ``GANETI_HOSTNAME`` are set. The former must be an absolute path to a directory with the last component being equal to the value of ``GANETI_HOSTNAME``, which contains the name of the virtual node. The reason for this requirement is that one virtual node must be able to compute an absolute path on another node for copying files via SSH. The whole content of ``GANETI_ROOTDIR`` is the node directory, its parent directory (without hostname) is the cluster directory. Example for environment variables:: GANETI_ROOTDIR=/tmp/vcluster/node1.example.com GANETI_HOSTNAME=node1.example.com With this example the node directory is ``/tmp/vcluster/node1.example.com`` and the cluster directory ``/tmp/vcluster``. .. _vcluster-setup: Setup ----- A script to configure virtual clusters is included with Ganeti as ``tools/vcluster-setup`` (usually installed as ``/usr/lib/ganeti/tools/vcluster-setup``). Running it with the ``-h`` option prints a usage description. The script creates all necessary directories, configures network interfaces, adds or updates entries in ``/etc/hosts`` and generates a small number of helper scripts. .. TODO: Describe setup of non-root virtual cluster Use --- Once the virtual cluster has been :ref:`set up `, the cluster can be initialized. The instructions for doing so have been printed by the ``vcluster-setup`` script together with other useful information, such as the list of virtual nodes. The commands printed should be used to configure the list of enabled hypervisors and other settings. To run commands for a specific virtual node, the script named ``cmd`` located in the node directory can be used. It takes a command as its argument(s), sets the environment variables ``GANETI_ROOTDIR`` and ``GANETI_HOSTNAME`` and then runs the command. Example: .. highlight:: shell-example :: # Let's create a cluster with node1 as its master node $ cd /tmp/vcluster $ node1.example.com/cmd gnt-cluster info Cluster name: cluster.example.com â€Ļ Master node: node1.example.com â€Ļ # Configure cluster as per "vcluster-setup" script $ node1.example.com/cmd gnt-cluster modify â€Ļ Scripts are provided in the cluster root directory to start, stop or restart all daemons for all virtual nodes. These are named ``start-all``, ``stop-all`` and ``restart-all``. ``ganeti-watcher`` can be run for all virtual nodes using ``watcher-all``. Adding an instance (assuming node1.example.com is the master node as per the example above): .. highlight:: shell-example :: $ node1.example.com/cmd gnt-instance add --os-size 1G \ --disk-template=file --os-type dummy -B memory=192 -I hail \ instance1.example.com .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/ganeti.cabal000064400000000000000000000305151476477700300160430ustar00rootroot00000000000000Cabal-Version: 2.4 Name: ganeti Version: 3.1 Homepage: http://www.ganeti.org License: BSD-2-Clause License-File: COPYING Author: Google Inc. Maintainer: ganeti-devel@googlegroups.com Copyright: 2006-2015 Google Inc. Category: System Build-Type: Simple Extra-Source-Files: README Synopsis: Cluster-based virtualization management software Description: Cluster-based virtualization management software . See Flag mond Description: enable the ganeti monitoring daemon Default: True Flag metad Description: enable the ganeti metadata daemon Default: True Flag htest Description: enable tests Default: True Flag mcio_xformers Description: use MonadCatchIO-transformers (deprecated) Default: True Flag network_bsd Description: use the split network-bsd package Default: False Flag regex-tdfa Description: use regex-tdfa instead of regex-pcre Default: False Flag regex-pcre-builtin Description: use regex-pcre-builtin instead of regex-pcre Default: False Flag regex-pcre2 Description: use regex-pcre2 instead of regex-pcre Default: False Library Exposed-Modules: AutoConf Ganeti.BasicTypes Ganeti.Codec Ganeti.Common Ganeti.Compat Ganeti.Confd.Client Ganeti.Confd.ClientFunctions Ganeti.Confd.Server Ganeti.Confd.Types Ganeti.Confd.Utils Ganeti.Config Ganeti.ConfigReader Ganeti.ConstantUtils Ganeti.Constants Ganeti.Cpu.LoadParser Ganeti.Cpu.Types Ganeti.Curl.Internal Ganeti.Curl.Multi Ganeti.Daemon Ganeti.Daemon.Utils Ganeti.DataCollectors Ganeti.DataCollectors.CLI Ganeti.DataCollectors.CPUload Ganeti.DataCollectors.Diskstats Ganeti.DataCollectors.Drbd Ganeti.DataCollectors.InstStatus Ganeti.DataCollectors.InstStatusTypes Ganeti.DataCollectors.Lv Ganeti.DataCollectors.Program Ganeti.DataCollectors.Types Ganeti.DataCollectors.XenCpuLoad Ganeti.Errors Ganeti.HTools.AlgorithmParams Ganeti.HTools.Backend.IAlloc Ganeti.HTools.Backend.Luxi Ganeti.HTools.Backend.MonD Ganeti.HTools.Backend.Rapi Ganeti.HTools.Backend.Simu Ganeti.HTools.Backend.Text Ganeti.HTools.CLI Ganeti.HTools.Cluster Ganeti.HTools.Cluster.AllocatePrimitives Ganeti.HTools.Cluster.AllocateSecondary Ganeti.HTools.Cluster.AllocationSolution Ganeti.HTools.Cluster.Evacuate Ganeti.HTools.Cluster.Metrics Ganeti.HTools.Cluster.Moves Ganeti.HTools.Cluster.Utils Ganeti.HTools.Container Ganeti.HTools.Dedicated Ganeti.HTools.ExtLoader Ganeti.HTools.GlobalN1 Ganeti.HTools.Graph Ganeti.HTools.Group Ganeti.HTools.Instance Ganeti.HTools.Loader Ganeti.HTools.Nic Ganeti.HTools.Node Ganeti.HTools.PeerMap Ganeti.HTools.Program.Hail Ganeti.HTools.Program.Harep Ganeti.HTools.Program.Hbal Ganeti.HTools.Program.Hcheck Ganeti.HTools.Program.Hinfo Ganeti.HTools.Program.Hroller Ganeti.HTools.Program.Hscan Ganeti.HTools.Program.Hspace Ganeti.HTools.Program.Hsqueeze Ganeti.HTools.Program.Main Ganeti.HTools.Tags Ganeti.HTools.Tags.Constants Ganeti.HTools.Types Ganeti.Hash Ganeti.Hs2Py.GenConstants Ganeti.Hs2Py.GenOpCodes Ganeti.Hs2Py.ListConstants Ganeti.Hs2Py.OpDoc Ganeti.Hypervisor.Xen Ganeti.Hypervisor.Xen.Types Ganeti.Hypervisor.Xen.XlParser Ganeti.JQScheduler Ganeti.JQScheduler.Filtering Ganeti.JQScheduler.ReasonRateLimiting Ganeti.JQScheduler.Types Ganeti.JQueue Ganeti.JQueue.Lens Ganeti.JQueue.Objects Ganeti.JSON Ganeti.Jobs Ganeti.Kvmd Ganeti.Lens Ganeti.Locking.Allocation Ganeti.Locking.Locks Ganeti.Locking.Types Ganeti.Locking.Waiting Ganeti.Logging Ganeti.Logging.Lifted Ganeti.Logging.WriterLog Ganeti.Luxi -- some Metad modules are also used by hs2py Ganeti.Metad.Config Ganeti.Metad.ConfigCore Ganeti.Metad.Types Ganeti.Network Ganeti.Objects Ganeti.Objects.BitArray Ganeti.Objects.Disk Ganeti.Objects.Instance Ganeti.Objects.Lens Ganeti.Objects.Nic Ganeti.OpCodes Ganeti.OpCodes.Lens Ganeti.OpParams Ganeti.Parsers Ganeti.PartialParams Ganeti.Path Ganeti.PyValue Ganeti.Query.Cluster Ganeti.Query.Common Ganeti.Query.Exec Ganeti.Query.Export Ganeti.Query.Filter Ganeti.Query.FilterRules Ganeti.Query.Group Ganeti.Query.Instance Ganeti.Query.Job Ganeti.Query.Language Ganeti.Query.Locks Ganeti.Query.Network Ganeti.Query.Node Ganeti.Query.Query Ganeti.Query.Server Ganeti.Query.Types Ganeti.Rpc Ganeti.Runtime Ganeti.SlotMap Ganeti.Ssconf Ganeti.Storage.Diskstats.Parser Ganeti.Storage.Diskstats.Types Ganeti.Storage.Drbd.Parser Ganeti.Storage.Drbd.Types Ganeti.Storage.Lvm.LVParser Ganeti.Storage.Lvm.Types Ganeti.Storage.Utils Ganeti.THH Ganeti.THH.Compat Ganeti.THH.Field Ganeti.THH.HsRPC Ganeti.THH.PyRPC Ganeti.THH.PyType Ganeti.THH.RPC Ganeti.THH.Types Ganeti.Types Ganeti.UDSServer Ganeti.Utils Ganeti.Utils.AsyncWorker Ganeti.Utils.Atomic Ganeti.Utils.IORef Ganeti.Utils.Livelock Ganeti.Utils.MVarLock Ganeti.Utils.Monad Ganeti.Utils.MultiMap Ganeti.Utils.Random Ganeti.Utils.Statistics Ganeti.Utils.Time Ganeti.Utils.UniStd Ganeti.Utils.Validate Ganeti.VCluster Ganeti.Version Ganeti.WConfd.Client Ganeti.WConfd.ConfigModifications Ganeti.WConfd.ConfigState Ganeti.WConfd.ConfigVerify Ganeti.WConfd.ConfigWriter Ganeti.WConfd.Core Ganeti.WConfd.DeathDetection Ganeti.WConfd.Language Ganeti.WConfd.Monad Ganeti.WConfd.Persistent Ganeti.WConfd.Server Ganeti.WConfd.Ssconf Ganeti.WConfd.TempRes Other-Extensions: TemplateHaskell Build-Depends: base >= 4.9.0.0 , array >= 0.4.0.0 , bytestring >= 0.9.2.1 , containers >= 0.4.2.1 , cryptonite >= 0.23 , deepseq >= 1.3.0.0 , directory >= 1.1.0.2 , filepath >= 1.3.0.0 , mtl >= 2.2.1 , old-time >= 1.1.0.0 , pretty >= 1.1.1.0 , process >= 1.1.0.1 , random >= 1.0.1.1 , template-haskell >= 2.11.0.0 , text >= 0.11.1.13 , transformers >= 0.3.0.0 && < 0.7 , utf8-string >= 0.3.7 , attoparsec >= 0.10.1.1 && < 0.15 , base64-bytestring >= 1.0.0.1 && < 1.3 , case-insensitive >= 0.4.0.1 && < 1.3 , curl >= 1.3.7 && < 1.4 , hinotify >= 0.3.2 && < 0.5 , hslogger >= 1.1.4 && < 1.4 , json >= 0.5 && < 1.0 , lens >= 3.10 && < 6.0 , lifted-base >= 0.2.0.3 && < 0.3 , monad-control >= 0.3.1.3 && < 1.1 , parallel >= 3.2.0.2 && < 3.3 , transformers-base >= 0.4.1 && < 0.5 , unix >= 2.5.1.0 && < 2.9 , zlib >= 0.5.3.3 && < 0.8 If flag(network_bsd) Build-Depends: network >= 2.9 && < 3.3 , network-bsd >= 2.8 && < 2.9 Else Build-Depends: network >= 2.3.0.13 && < 2.9 If flag(mond) Build-Depends: PSQueue >= 1.1 && < 1.3 , snap-core >= 1.0.0 , snap-server >= 1.0.0 Exposed-Modules: Ganeti.Monitoring.Server If flag(metad) Build-Depends: snap-core >= 1.0.0 , snap-server >= 1.0.0 Exposed-Modules: Ganeti.Metad.ConfigServer Ganeti.Metad.Server Ganeti.Metad.WebServer If flag(mcio_xformers) Build-Depends: MonadCatchIO-transformers Exposed-Modules: Ganeti.Query.RegEx If flag(regex-tdfa) Hs-Source-Dirs: regex/tdfa Build-Depends: regex-tdfa >= 1.2 && < 1.4 Elif flag(regex-pcre2) Hs-Source-Dirs: regex/pcre2 Build-Depends: regex-pcre2 >= 1.0.0.0 && < 1.1 Else Hs-Source-Dirs: regex/pcre If flag(regex-pcre-builtin) Build-Depends: regex-pcre-builtin >= 0.94.2 && < 0.96 Else Build-Depends: regex-pcre >= 0.94.2 && < 0.96 Hs-Source-Dirs: src Build-Tool-Depends: hsc2hs:hsc2hs Default-Language: Haskell2010 GHC-Options: -Wall -ddump-splices -ddump-to-file Common app Hs-Source-Dirs: app Default-Language: Haskell2010 GHC-Options: -Wall Build-Depends: ganeti, base Executable htools Import: app Main-Is: htools.hs Executable mon-collector Import: app Main-Is: mon-collector.hs Executable ganeti-kvmd Import: app Main-Is: ganeti-kvmd.hs Executable ganeti-wconfd Import: app Main-Is: ganeti-wconfd.hs Executable ganeti-confd Import: app Main-Is: ganeti-confd.hs Executable ganeti-luxid Import: app Main-Is: ganeti-luxid.hs Executable hs2py Import: app Main-Is: hs2py.hs Executable rpc-test Import: app Main-Is: rpc-test.hs Build-Depends: json Executable ganeti-mond Import: app Main-Is: ganeti-mond.hs If !flag(mond) Buildable: False Executable ganeti-metad Import: app Main-Is: ganeti-metad.hs If !flag(metad) Buildable: False Executable htest Import: app Hs-Source-Dirs: test/hs Main-Is: htest.hs Build-Depends: HUnit >= 1.2.4.2 && < 1.7, QuickCheck >= 2.8 && < 2.16, test-framework >= 0.6 && < 0.9, test-framework-hunit >= 0.2.7 && < 0.4, test-framework-quickcheck2 >= 0.2.12.1 && < 0.4, temporary >= 1.1.2.3 && < 1.4, attoparsec, containers, text, bytestring, utf8-string, directory, process, json, template-haskell, hslogger, base64-bytestring, filepath, lens, network, unix, old-time Other-Modules: Test.AutoConf Test.Ganeti.Attoparsec Test.Ganeti.BasicTypes Test.Ganeti.Common Test.Ganeti.Confd.Types Test.Ganeti.Confd.Utils Test.Ganeti.Constants Test.Ganeti.Daemon Test.Ganeti.Errors Test.Ganeti.HTools.Backend.MonD Test.Ganeti.HTools.Backend.Simu Test.Ganeti.HTools.Backend.Text Test.Ganeti.HTools.CLI Test.Ganeti.HTools.Cluster Test.Ganeti.HTools.Container Test.Ganeti.HTools.Graph Test.Ganeti.HTools.Instance Test.Ganeti.HTools.Loader Test.Ganeti.HTools.Node Test.Ganeti.HTools.PeerMap Test.Ganeti.HTools.Types Test.Ganeti.Hypervisor.Xen.XlParser Test.Ganeti.JQScheduler Test.Ganeti.JQueue Test.Ganeti.JQueue.Objects Test.Ganeti.JSON Test.Ganeti.Jobs Test.Ganeti.Kvmd Test.Ganeti.Locking.Allocation Test.Ganeti.Locking.Locks Test.Ganeti.Locking.Waiting Test.Ganeti.Luxi Test.Ganeti.Network Test.Ganeti.Objects Test.Ganeti.Objects.BitArray Test.Ganeti.OpCodes Test.Ganeti.PartialParams Test.Ganeti.PyValue Test.Ganeti.Query.Aliases Test.Ganeti.Query.Filter Test.Ganeti.Query.Instance Test.Ganeti.Query.Language Test.Ganeti.Query.Network Test.Ganeti.Query.Query Test.Ganeti.Rpc Test.Ganeti.Runtime Test.Ganeti.SlotMap Test.Ganeti.Ssconf Test.Ganeti.Storage.Diskstats.Parser Test.Ganeti.Storage.Drbd.Parser Test.Ganeti.Storage.Drbd.Types Test.Ganeti.Storage.Lvm.LVParser Test.Ganeti.THH Test.Ganeti.THH.Types Test.Ganeti.TestCommon Test.Ganeti.TestHTools Test.Ganeti.TestHelper Test.Ganeti.TestImports Test.Ganeti.Types Test.Ganeti.Utils Test.Ganeti.Utils.MultiMap Test.Ganeti.Utils.Statistics Test.Ganeti.Utils.Time Test.Ganeti.WConfd.Ssconf Test.Ganeti.WConfd.TempRes ganeti-3.1.0~rc2/lib/000075500000000000000000000000001476477700300143525ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/__init__.py000064400000000000000000000034251476477700300164670ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # empty file for package definition """Ganeti python modules""" try: from ganeti import ganeti # pylint: disable=W0406 except ImportError: pass else: raise Exception("A module named \"ganeti.ganeti\" was successfully imported" " and should be removed as it can lead to importing the" " wrong module(s) in other parts of the code, consequently" " leading to failures which are difficult to debug") ganeti-3.1.0~rc2/lib/_constants.py.in000064400000000000000000000004351476477700300175060ustar00rootroot00000000000000# This file is automatically generated, do not edit! # """Automatically generated constants for Python Note that this file is autogenerated with @lib/_constants.py.in@ as a header. """ # pylint: disable=C0301,C0324 # because this is autogenerated, we do not want # style warnings ganeti-3.1.0~rc2/lib/asyncnotifier.py000064400000000000000000000153361476477700300176110ustar00rootroot00000000000000# # # Copyright (C) 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Asynchronous pyinotify implementation""" import asyncore import logging try: # pylint: disable=E0611 from pyinotify import pyinotify except ImportError: import pyinotify from ganeti import daemon from ganeti import errors # We contributed the AsyncNotifier class back to python-pyinotify, and it's # part of their codebase since version 0.8.7. This code can be removed once # we'll be ready to depend on python-pyinotify >= 0.8.7 class AsyncNotifier(asyncore.file_dispatcher): """An asyncore dispatcher for inotify events. """ # pylint: disable=W0622,W0212 def __init__(self, watch_manager, default_proc_fun=None, map=None): """Initializes this class. This is a a special asyncore file_dispatcher that actually wraps a pyinotify Notifier, making it asyncronous. """ if default_proc_fun is None: default_proc_fun = pyinotify.ProcessEvent() self.notifier = pyinotify.Notifier(watch_manager, default_proc_fun) # here we need to steal the file descriptor from the notifier, so we can # use it in the global asyncore select, and avoid calling the # check_events() function of the notifier (which doesn't allow us to select # together with other file descriptors) self.fd = self.notifier._fd asyncore.file_dispatcher.__init__(self, self.fd, map) def handle_read(self): self.notifier.read_events() self.notifier.process_events() class ErrorLoggingAsyncNotifier(AsyncNotifier, daemon.GanetiBaseAsyncoreDispatcher): """An asyncnotifier that can survive errors in the callbacks. We define this as a separate class, since we don't want to make AsyncNotifier diverge from what we contributed upstream. """ class FileEventHandlerBase(pyinotify.ProcessEvent): """Base class for file event handlers. @ivar watch_manager: Inotify watch manager """ def __init__(self, watch_manager): """Initializes this class. @type watch_manager: pyinotify.WatchManager @param watch_manager: inotify watch manager """ # pylint: disable=W0231 # no need to call the parent's constructor self.watch_manager = watch_manager def process_default(self, event): logging.error("Received unhandled inotify event: %s", event) def AddWatch(self, filename, mask): """Adds a file watch. @param filename: Path to file @param mask: Inotify event mask @return: Result """ result = self.watch_manager.add_watch(filename, mask) ret = result.get(filename, -1) if ret <= 0: raise errors.InotifyError("Could not add inotify watcher (error code %s);" " increasing fs.inotify.max_user_watches sysctl" " might be necessary" % ret) return result[filename] def RemoveWatch(self, handle): """Removes a handle from the watcher. @param handle: Inotify handle @return: Whether removal was successful """ result = self.watch_manager.rm_watch(handle) return result[handle] class SingleFileEventHandler(FileEventHandlerBase): """Handle modify events for a single file. """ def __init__(self, watch_manager, callback, filename): """Constructor for SingleFileEventHandler @type watch_manager: pyinotify.WatchManager @param watch_manager: inotify watch manager @type callback: function accepting a boolean @param callback: function to call when an inotify event happens @type filename: string @param filename: config file to watch """ FileEventHandlerBase.__init__(self, watch_manager) self._callback = callback self._filename = filename self._watch_handle = None def enable(self): """Watch the given file. """ if self._watch_handle is not None: return # Different Pyinotify versions have the flag constants at different places, # hence not accessing them directly mask = (pyinotify.EventsCodes.ALL_FLAGS["IN_MODIFY"] | pyinotify.EventsCodes.ALL_FLAGS["IN_IGNORED"]) self._watch_handle = self.AddWatch(self._filename, mask) def disable(self): """Stop watching the given file. """ if self._watch_handle is not None and self.RemoveWatch(self._watch_handle): self._watch_handle = None # pylint: disable=C0103 # this overrides a method in pyinotify.ProcessEvent def process_IN_IGNORED(self, event): # Since we monitor a single file rather than the directory it resides in, # when that file is replaced with another one (which is what happens when # utils.WriteFile, the most normal way of updating files in ganeti, is # called) we're going to receive an IN_IGNORED event from inotify, because # of the file removal (which is contextual with the replacement). In such a # case we'll need to create a watcher for the "new" file. This can be done # by the callback by calling "enable" again on us. logging.debug("Received 'ignored' inotify event for %s", event.path) self._watch_handle = None self._callback(False) # pylint: disable=C0103 # this overrides a method in pyinotify.ProcessEvent def process_IN_MODIFY(self, event): # This gets called when the monitored file is modified. Note that this # doesn't usually happen in Ganeti, as most of the time we're just # replacing any file with a new one, at filesystem level, rather than # actually changing it. (see utils.WriteFile) logging.debug("Received 'modify' inotify event for %s", event.path) self._callback(True) ganeti-3.1.0~rc2/lib/backend.py000064400000000000000000006231541476477700300163260ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Functions used by the node daemon @var _ALLOWED_UPLOAD_FILES: denotes which files are accepted in the L{UploadFile} function @var _ALLOWED_CLEAN_DIRS: denotes which directories are accepted in the L{_CleanDirectory} function """ # pylint: disable=E1103,C0302 # E1103: %s %r has no %r member (but some types could not be # inferred), because the _TryOSFromDisk returns either (True, os_obj) # or (False, "string") which confuses pylint # C0302: This module has become too big and should be split up import base64 import contextlib import collections import errno import logging import os import os.path import random import re import shutil import signal import stat import tempfile import time import zlib import pycurl from ganeti import errors from ganeti import http from ganeti import utils from ganeti import ssh from ganeti import hypervisor from ganeti.hypervisor import hv_base from ganeti import constants from ganeti.storage import bdev from ganeti.storage import drbd from ganeti.storage import extstorage from ganeti.storage import filestorage from ganeti import objects from ganeti import ssconf from ganeti import serializer from ganeti import netutils from ganeti import runtime from ganeti import compat from ganeti import pathutils from ganeti import vcluster from ganeti import ht from ganeti.storage.base import BlockDev from ganeti.storage.drbd import DRBD8 from ganeti import hooksmaster import ganeti.metad as metad _BOOT_ID_PATH = "/proc/sys/kernel/random/boot_id" _ALLOWED_CLEAN_DIRS = compat.UniqueFrozenset([ pathutils.DATA_DIR, pathutils.JOB_QUEUE_ARCHIVE_DIR, pathutils.QUEUE_DIR, pathutils.CRYPTO_KEYS_DIR, ]) _MAX_SSL_CERT_VALIDITY = 7 * 24 * 60 * 60 _X509_KEY_FILE = "key" _X509_CERT_FILE = "cert" _IES_STATUS_FILE = "status" _IES_PID_FILE = "pid" _IES_CA_FILE = "ca" #: Valid LVS output line regex _LVSLINE_REGEX = re.compile(r"^ *([^|]+)\|([^|]+)\|([0-9.]+)\|([^|]{6,})\|?$") # Actions for the master setup script _MASTER_START = "start" _MASTER_STOP = "stop" #: Maximum file permissions for restricted command directory and executables _RCMD_MAX_MODE = (stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) #: Delay before returning an error for restricted commands _RCMD_INVALID_DELAY = 10 #: How long to wait to acquire lock for restricted commands (shorter than #: L{_RCMD_INVALID_DELAY}) to reduce blockage of noded forks when many #: command requests arrive _RCMD_LOCK_TIMEOUT = _RCMD_INVALID_DELAY * 0.8 class RPCFail(Exception): """Class denoting RPC failure. Its argument is the error message. """ def _GetInstReasonFilename(instance_name): """Path of the file containing the reason of the instance status change. @type instance_name: string @param instance_name: The name of the instance @rtype: string @return: The path of the file """ return utils.PathJoin(pathutils.INSTANCE_REASON_DIR, instance_name) def _StoreInstReasonTrail(instance_name, trail): """Serialize a reason trail related to an instance change of state to file. The exact location of the file depends on the name of the instance and on the configuration of the Ganeti cluster defined at deploy time. @type instance_name: string @param instance_name: The name of the instance @type trail: list of reasons @param trail: reason trail @rtype: None """ json = serializer.DumpJson(trail) filename = _GetInstReasonFilename(instance_name) utils.WriteFile(filename, data=json) def _Fail(msg, *args, **kwargs): """Log an error and the raise an RPCFail exception. This exception is then handled specially in the ganeti daemon and turned into a 'failed' return type. As such, this function is a useful shortcut for logging the error and returning it to the master daemon. @type msg: string @param msg: the text of the exception @raise RPCFail """ if args: msg = msg % args if "log" not in kwargs or kwargs["log"]: # if we should log this error if "exc" in kwargs and kwargs["exc"]: logging.exception(msg) else: logging.error(msg) raise RPCFail(msg) def _GetConfig(): """Simple wrapper to return a SimpleStore. @rtype: L{ssconf.SimpleStore} @return: a SimpleStore instance """ return ssconf.SimpleStore() def _GetSshRunner(cluster_name): """Simple wrapper to return an SshRunner. @type cluster_name: str @param cluster_name: the cluster name, which is needed by the SshRunner constructor @rtype: L{ssh.SshRunner} @return: an SshRunner instance """ return ssh.SshRunner(cluster_name) def _Decompress(data): """Unpacks data compressed by the RPC client. @type data: list or tuple @param data: Data sent by RPC client @rtype: str @return: Decompressed data """ assert isinstance(data, (list, tuple)) assert len(data) == 2 (encoding, content) = data if encoding == constants.RPC_ENCODING_NONE: return content elif encoding == constants.RPC_ENCODING_ZLIB_BASE64: return zlib.decompress(base64.b64decode(content)) else: raise AssertionError("Unknown data encoding") def _CleanDirectory(path, exclude=None): """Removes all regular files in a directory. @type path: str @param path: the directory to clean @type exclude: list @param exclude: list of files to be excluded, defaults to the empty list """ if path not in _ALLOWED_CLEAN_DIRS: _Fail("Path passed to _CleanDirectory not in allowed clean targets: '%s'", path) if not os.path.isdir(path): return if exclude is None: exclude = [] else: # Normalize excluded paths exclude = [os.path.normpath(i) for i in exclude] for rel_name in utils.ListVisibleFiles(path): full_name = utils.PathJoin(path, rel_name) if full_name in exclude: continue if os.path.isfile(full_name) and not os.path.islink(full_name): utils.RemoveFile(full_name) def _BuildUploadFileList(): """Build the list of allowed upload files. This is abstracted so that it's built only once at module import time. """ allowed_files = set([ pathutils.CLUSTER_CONF_FILE, pathutils.ETC_HOSTS, pathutils.SSH_KNOWN_HOSTS_FILE, pathutils.VNC_PASSWORD_FILE, pathutils.RAPI_CERT_FILE, pathutils.SPICE_CERT_FILE, pathutils.SPICE_CACERT_FILE, pathutils.RAPI_USERS_FILE, pathutils.CONFD_HMAC_KEY, pathutils.CLUSTER_DOMAIN_SECRET_FILE, ]) for hv_name in constants.HYPER_TYPES: hv_class = hypervisor.GetHypervisorClass(hv_name) allowed_files.update(hv_class.GetAncillaryFiles()[0]) assert pathutils.FILE_STORAGE_PATHS_FILE not in allowed_files, \ "Allowed file storage paths should never be uploaded via RPC" return frozenset(allowed_files) _ALLOWED_UPLOAD_FILES = _BuildUploadFileList() def JobQueuePurge(): """Removes job queue files and archived jobs. @rtype: tuple @return: True, None """ _CleanDirectory(pathutils.QUEUE_DIR, exclude=[pathutils.JOB_QUEUE_LOCK_FILE]) _CleanDirectory(pathutils.JOB_QUEUE_ARCHIVE_DIR) def GetMasterNodeName(): """Returns the master node name. @rtype: string @return: name of the master node @raise RPCFail: in case of errors """ try: return _GetConfig().GetMasterNode() except errors.ConfigurationError as err: _Fail("Cluster configuration incomplete: %s", err, exc=True) def RunLocalHooks(hook_opcode, hooks_path, env_builder_fn): """Decorator that runs hooks before and after the decorated function. @type hook_opcode: string @param hook_opcode: opcode of the hook @type hooks_path: string @param hooks_path: path of the hooks @type env_builder_fn: function @param env_builder_fn: function that returns a dictionary containing the environment variables for the hooks. Will get all the parameters of the decorated function. @raise RPCFail: in case of pre-hook failure """ def decorator(fn): def wrapper(*args, **kwargs): _, myself = ssconf.GetMasterAndMyself() nodes = ([myself], [myself]) # these hooks run locally env_fn = compat.partial(env_builder_fn, *args, **kwargs) cfg = _GetConfig() hr = HooksRunner() hm = hooksmaster.HooksMaster(hook_opcode, hooks_path, nodes, hr.RunLocalHooks, None, env_fn, None, logging.warning, cfg.GetClusterName(), cfg.GetMasterNode()) hm.RunPhase(constants.HOOKS_PHASE_PRE) result = fn(*args, **kwargs) hm.RunPhase(constants.HOOKS_PHASE_POST) return result return wrapper return decorator def _BuildMasterIpEnv(master_params, use_external_mip_script=None): """Builds environment variables for master IP hooks. @type master_params: L{objects.MasterNetworkParameters} @param master_params: network parameters of the master @type use_external_mip_script: boolean @param use_external_mip_script: whether to use an external master IP address setup script (unused, but necessary per the implementation of the _RunLocalHooks decorator) """ # pylint: disable=W0613 ver = netutils.IPAddress.GetVersionFromAddressFamily(master_params.ip_family) env = { "MASTER_NETDEV": master_params.netdev, "MASTER_IP": master_params.ip, "MASTER_NETMASK": str(master_params.netmask), "CLUSTER_IP_VERSION": str(ver), } return env def _RunMasterSetupScript(master_params, action, use_external_mip_script): """Execute the master IP address setup script. @type master_params: L{objects.MasterNetworkParameters} @param master_params: network parameters of the master @type action: string @param action: action to pass to the script. Must be one of L{backend._MASTER_START} or L{backend._MASTER_STOP} @type use_external_mip_script: boolean @param use_external_mip_script: whether to use an external master IP address setup script @raise backend.RPCFail: if there are errors during the execution of the script """ env = _BuildMasterIpEnv(master_params) if use_external_mip_script: setup_script = pathutils.EXTERNAL_MASTER_SETUP_SCRIPT else: setup_script = pathutils.DEFAULT_MASTER_SETUP_SCRIPT result = utils.RunCmd([setup_script, action], env=env, reset_env=True) if result.failed: _Fail("Failed to %s the master IP. Script return value: %s, output: '%s'" % (action, result.exit_code, result.output), log=True) @RunLocalHooks(constants.FAKE_OP_MASTER_TURNUP, "master-ip-turnup", _BuildMasterIpEnv) def ActivateMasterIp(master_params, use_external_mip_script): """Activate the IP address of the master daemon. @type master_params: L{objects.MasterNetworkParameters} @param master_params: network parameters of the master @type use_external_mip_script: boolean @param use_external_mip_script: whether to use an external master IP address setup script @raise RPCFail: in case of errors during the IP startup """ _RunMasterSetupScript(master_params, _MASTER_START, use_external_mip_script) def StartMasterDaemons(no_voting): """Activate local node as master node. The function will start the master daemons (ganeti-masterd and ganeti-rapi). @type no_voting: boolean @param no_voting: whether to start ganeti-masterd without a node vote but still non-interactively @rtype: None """ if no_voting: daemon_args = "--no-voting --yes-do-it" else: daemon_args = "" env = { "EXTRA_LUXID_ARGS": daemon_args, "EXTRA_WCONFD_ARGS": daemon_args, } result = utils.RunCmd([pathutils.DAEMON_UTIL, "start-master"], env=env) if result.failed: msg = "Can't start Ganeti master: %s" % result.output logging.error(msg) _Fail(msg) @RunLocalHooks(constants.FAKE_OP_MASTER_TURNDOWN, "master-ip-turndown", _BuildMasterIpEnv) def DeactivateMasterIp(master_params, use_external_mip_script): """Deactivate the master IP on this node. @type master_params: L{objects.MasterNetworkParameters} @param master_params: network parameters of the master @type use_external_mip_script: boolean @param use_external_mip_script: whether to use an external master IP address setup script @raise RPCFail: in case of errors during the IP turndown """ _RunMasterSetupScript(master_params, _MASTER_STOP, use_external_mip_script) def StopMasterDaemons(): """Stop the master daemons on this node. Stop the master daemons (ganeti-masterd and ganeti-rapi) on this node. @rtype: None """ # TODO: log and report back to the caller the error failures; we # need to decide in which case we fail the RPC for this result = utils.RunCmd([pathutils.DAEMON_UTIL, "stop-master"]) if result.failed: logging.error("Could not stop Ganeti master, command %s had exitcode %s" " and error %s", result.cmd, result.exit_code, result.output) def ChangeMasterNetmask(old_netmask, netmask, master_ip, master_netdev): """Change the netmask of the master IP. @param old_netmask: the old value of the netmask @param netmask: the new value of the netmask @param master_ip: the master IP @param master_netdev: the master network device """ if old_netmask == netmask: return if not netutils.IPAddress.Own(master_ip): _Fail("The master IP address is not up, not attempting to change its" " netmask") result = utils.RunCmd([constants.IP_COMMAND_PATH, "address", "add", "%s/%s" % (master_ip, netmask), "dev", master_netdev, "label", "%s:0" % master_netdev]) if result.failed: _Fail("Could not set the new netmask on the master IP address") result = utils.RunCmd([constants.IP_COMMAND_PATH, "address", "del", "%s/%s" % (master_ip, old_netmask), "dev", master_netdev, "label", "%s:0" % master_netdev]) if result.failed: _Fail("Could not bring down the master IP address with the old netmask") def EtcHostsModify(mode, host, ip): """Modify a host entry in /etc/hosts. @param mode: The mode to operate. Either add or remove entry @param host: The host to operate on @param ip: The ip associated with the entry """ if mode == constants.ETC_HOSTS_ADD: if not ip: RPCFail("Mode 'add' needs 'ip' parameter, but parameter not" " present") utils.AddHostToEtcHosts(host, ip) elif mode == constants.ETC_HOSTS_REMOVE: if ip: RPCFail("Mode 'remove' does not allow 'ip' parameter, but" " parameter is present") utils.RemoveHostFromEtcHosts(host) else: RPCFail("Mode not supported") def LeaveCluster(modify_ssh_setup): """Cleans up and remove the current node. This function cleans up and prepares the current node to be removed from the cluster. If processing is successful, then it raises an L{errors.QuitGanetiException} which is used as a special case to shutdown the node daemon. @param modify_ssh_setup: boolean """ _CleanDirectory(pathutils.DATA_DIR) _CleanDirectory(pathutils.CRYPTO_KEYS_DIR) JobQueuePurge() if modify_ssh_setup: try: priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.SSH_LOGIN_USER) ssh.RemoveAuthorizedKey(auth_keys, utils.ReadFile(pub_key)) utils.RemoveFile(priv_key) utils.RemoveFile(pub_key) except errors.OpExecError: logging.exception("Error while processing ssh files") except IOError: logging.exception("At least one SSH file was not accessible.") try: utils.RemoveFile(pathutils.CONFD_HMAC_KEY) utils.RemoveFile(pathutils.RAPI_CERT_FILE) utils.RemoveFile(pathutils.SPICE_CERT_FILE) utils.RemoveFile(pathutils.SPICE_CACERT_FILE) utils.RemoveFile(pathutils.NODED_CERT_FILE) except: # pylint: disable=W0702 logging.exception("Error while removing cluster secrets") utils.StopDaemon(constants.CONFD) utils.StopDaemon(constants.MOND) utils.StopDaemon(constants.KVMD) # Raise a custom exception (handled in ganeti-noded) raise errors.QuitGanetiException(True, "Shutdown scheduled") def _CheckStorageParams(params, num_params): """Performs sanity checks for storage parameters. @type params: list @param params: list of storage parameters @type num_params: int @param num_params: expected number of parameters """ if params is None: raise errors.ProgrammerError("No storage parameters for storage" " reporting is provided.") if not isinstance(params, list): raise errors.ProgrammerError("The storage parameters are not of type" " list: '%s'" % params) if not len(params) == num_params: raise errors.ProgrammerError("Did not receive the expected number of" "storage parameters: expected %s," " received '%s'" % (num_params, len(params))) def _CheckLvmStorageParams(params): """Performs sanity check for the 'exclusive storage' flag. @see: C{_CheckStorageParams} """ _CheckStorageParams(params, 1) excl_stor = params[0] if not isinstance(params[0], bool): raise errors.ProgrammerError("Exclusive storage parameter is not" " boolean: '%s'." % excl_stor) return excl_stor def _GetLvmVgSpaceInfo(name, params): """Wrapper around C{_GetVgInfo} which checks the storage parameters. @type name: string @param name: name of the volume group @type params: list @param params: list of storage parameters, which in this case should be containing only one for exclusive storage """ excl_stor = _CheckLvmStorageParams(params) return _GetVgInfo(name, excl_stor) def _GetVgInfo( name, excl_stor, info_fn=bdev.LogicalVolume.GetVGInfo): """Retrieves information about a LVM volume group. """ # TODO: GetVGInfo supports returning information for multiple VGs at once vginfo = info_fn([name], excl_stor) if vginfo: vg_free = int(round(vginfo[0][0], 0)) vg_size = int(round(vginfo[0][1], 0)) else: vg_free = None vg_size = None return { "type": constants.ST_LVM_VG, "name": name, "storage_free": vg_free, "storage_size": vg_size, } def _GetLvmPvSpaceInfo(name, params): """Wrapper around C{_GetVgSpindlesInfo} with sanity checks. @see: C{_GetLvmVgSpaceInfo} """ excl_stor = _CheckLvmStorageParams(params) return _GetVgSpindlesInfo(name, excl_stor) def _GetVgSpindlesInfo( name, excl_stor, info_fn=bdev.LogicalVolume.GetVgSpindlesInfo): """Retrieves information about spindles in an LVM volume group. @type name: string @param name: VG name @type excl_stor: bool @param excl_stor: exclusive storage @rtype: dict @return: dictionary whose keys are "name", "vg_free", "vg_size" for VG name, free spindles, total spindles respectively """ if excl_stor: (vg_free, vg_size) = info_fn(name) else: vg_free = 0 vg_size = 0 return { "type": constants.ST_LVM_PV, "name": name, "storage_free": vg_free, "storage_size": vg_size, } def _GetHvInfo(name, hvparams, get_hv_fn=hypervisor.GetHypervisor): """Retrieves node information from a hypervisor. The information returned depends on the hypervisor. Common items: - vg_size is the size of the configured volume group in MiB - vg_free is the free size of the volume group in MiB - memory_dom0 is the memory allocated for domain0 in MiB - memory_free is the currently available (free) ram in MiB - memory_total is the total number of ram in MiB - hv_version: the hypervisor version, if available @type hvparams: dict of string @param hvparams: the hypervisor's hvparams """ return get_hv_fn(name).GetNodeInfo(hvparams=hvparams) def _GetHvInfoAll(hv_specs, get_hv_fn=hypervisor.GetHypervisor): """Retrieves node information for all hypervisors. See C{_GetHvInfo} for information on the output. @type hv_specs: list of pairs (string, dict of strings) @param hv_specs: list of pairs of a hypervisor's name and its hvparams """ if hv_specs is None: return None result = [] for hvname, hvparams in hv_specs: result.append(_GetHvInfo(hvname, hvparams, get_hv_fn)) return result def _GetNamedNodeInfo(names, fn): """Calls C{fn} for all names in C{names} and returns a list of dictionaries. @rtype: None or list of dict """ if names is None: return None else: return [fn(n) for n in names] def GetNodeInfo(storage_units, hv_specs): """Gives back a hash with different information about the node. @type storage_units: list of tuples (string, string, list) @param storage_units: List of tuples (storage unit, identifier, parameters) to ask for disk space information. In case of lvm-vg, the identifier is the VG name. The parameters can contain additional, storage-type-specific parameters, for example exclusive storage for lvm storage. @type hv_specs: list of pairs (string, dict of strings) @param hv_specs: list of pairs of a hypervisor's name and its hvparams @rtype: tuple; (string, None/list of dict, None/dict) @return: Tuple containing boot ID, volume group information and hypervisor information """ bootid = utils.ReadFile(_BOOT_ID_PATH, size=128).rstrip("\n") storage_info = _GetNamedNodeInfo( storage_units, (lambda type_key_params: _ApplyStorageInfoFunction(type_key_params[0], type_key_params[1], type_key_params[2]))) hv_info = _GetHvInfoAll(hv_specs) return (bootid, storage_info, hv_info) def _GetFileStorageSpaceInfo(path, params): """Wrapper around filestorage.GetSpaceInfo. The purpose of this wrapper is to call filestorage.GetFileStorageSpaceInfo and ignore the *args parameter to not leak it into the filestorage module's code. @see: C{filestorage.GetFileStorageSpaceInfo} for description of the parameters. """ _CheckStorageParams(params, 0) return filestorage.GetFileStorageSpaceInfo(path) # FIXME: implement storage reporting for all missing storage types. _STORAGE_TYPE_INFO_FN = { constants.ST_BLOCK: None, constants.ST_DISKLESS: None, constants.ST_EXT: None, constants.ST_FILE: _GetFileStorageSpaceInfo, constants.ST_LVM_PV: _GetLvmPvSpaceInfo, constants.ST_LVM_VG: _GetLvmVgSpaceInfo, constants.ST_SHARED_FILE: None, constants.ST_GLUSTER: None, constants.ST_RADOS: None, } def _ApplyStorageInfoFunction(storage_type, storage_key, *args): """Looks up and applies the correct function to calculate free and total storage for the given storage type. @type storage_type: string @param storage_type: the storage type for which the storage shall be reported. @type storage_key: string @param storage_key: identifier of a storage unit, e.g. the volume group name of an LVM storage unit @type args: any @param args: various parameters that can be used for storage reporting. These parameters and their semantics vary from storage type to storage type and are just propagated in this function. @return: the results of the application of the storage space function (see _STORAGE_TYPE_INFO_FN) if storage space reporting is implemented for that storage type @raises NotImplementedError: for storage types who don't support space reporting yet """ fn = _STORAGE_TYPE_INFO_FN[storage_type] if fn is not None: return fn(storage_key, *args) else: raise NotImplementedError def _CheckExclusivePvs(pvi_list): """Check that PVs are not shared among LVs @type pvi_list: list of L{objects.LvmPvInfo} objects @param pvi_list: information about the PVs @rtype: list of tuples (string, list of strings) @return: offending volumes, as tuples: (pv_name, [lv1_name, lv2_name...]) """ res = [] for pvi in pvi_list: if len(pvi.lv_list) > 1: res.append((pvi.name, pvi.lv_list)) return res def _VerifyHypervisors(what, vm_capable, result, all_hvparams, get_hv_fn=hypervisor.GetHypervisor): """Verifies the hypervisor. Appends the results to the 'results' list. @type what: C{dict} @param what: a dictionary of things to check @type vm_capable: boolean @param vm_capable: whether or not this node is vm capable @type result: dict @param result: dictionary of verification results; results of the verifications in this function will be added here @type all_hvparams: dict of dict of string @param all_hvparams: dictionary mapping hypervisor names to hvparams @type get_hv_fn: function @param get_hv_fn: function to retrieve the hypervisor, to improve testability """ if not vm_capable: return if constants.NV_HYPERVISOR in what: result[constants.NV_HYPERVISOR] = {} for hv_name in what[constants.NV_HYPERVISOR]: hvparams = all_hvparams[hv_name] try: val = get_hv_fn(hv_name).Verify(hvparams=hvparams) except errors.HypervisorError as err: val = "Error while checking hypervisor: %s" % str(err) result[constants.NV_HYPERVISOR][hv_name] = val def _VerifyHvparams(what, vm_capable, result, get_hv_fn=hypervisor.GetHypervisor): """Verifies the hvparams. Appends the results to the 'results' list. @type what: C{dict} @param what: a dictionary of things to check @type vm_capable: boolean @param vm_capable: whether or not this node is vm capable @type result: dict @param result: dictionary of verification results; results of the verifications in this function will be added here @type get_hv_fn: function @param get_hv_fn: function to retrieve the hypervisor, to improve testability """ if not vm_capable: return if constants.NV_HVPARAMS in what: result[constants.NV_HVPARAMS] = [] for source, hv_name, hvparms in what[constants.NV_HVPARAMS]: try: logging.info("Validating hv %s, %s", hv_name, hvparms) get_hv_fn(hv_name).ValidateParameters(hvparms) except errors.HypervisorError as err: result[constants.NV_HVPARAMS].append((source, hv_name, str(err))) def _VerifyInstanceList(what, vm_capable, result, all_hvparams): """Verifies the instance list. @type what: C{dict} @param what: a dictionary of things to check @type vm_capable: boolean @param vm_capable: whether or not this node is vm capable @type result: dict @param result: dictionary of verification results; results of the verifications in this function will be added here @type all_hvparams: dict of dict of string @param all_hvparams: dictionary mapping hypervisor names to hvparams """ if constants.NV_INSTANCELIST in what and vm_capable: # GetInstanceList can fail try: val = GetInstanceList(what[constants.NV_INSTANCELIST], all_hvparams=all_hvparams) except RPCFail as err: val = str(err) result[constants.NV_INSTANCELIST] = val def _VerifyNodeInfo(what, vm_capable, result, all_hvparams): """Verifies the node info. @type what: C{dict} @param what: a dictionary of things to check @type vm_capable: boolean @param vm_capable: whether or not this node is vm capable @type result: dict @param result: dictionary of verification results; results of the verifications in this function will be added here @type all_hvparams: dict of dict of string @param all_hvparams: dictionary mapping hypervisor names to hvparams """ if constants.NV_HVINFO in what and vm_capable: hvname = what[constants.NV_HVINFO] hyper = hypervisor.GetHypervisor(hvname) hvparams = all_hvparams[hvname] result[constants.NV_HVINFO] = hyper.GetNodeInfo(hvparams=hvparams) def _VerifyClientCertificate(cert_file=pathutils.NODED_CLIENT_CERT_FILE): """Verify the existance and validity of the client SSL certificate. Also, verify that the client certificate is not self-signed. Self- signed client certificates stem from Ganeti versions 2.12.0 - 2.12.4 and should be replaced by client certificates signed by the server certificate. Hence we output a warning when we encounter a self-signed one. """ create_cert_cmd = "gnt-cluster renew-crypto --new-node-certificates" if not os.path.exists(cert_file): return (constants.CV_ERROR, "The client certificate does not exist. Run '%s' to create" " client certificates for all nodes." % create_cert_cmd) (errcode, msg) = utils.VerifyCertificate(cert_file) if errcode is not None: return (errcode, msg) (errcode, msg) = utils.IsCertificateSelfSigned(cert_file) if errcode is not None: return (errcode, msg) # if everything is fine, we return the digest to be compared to the config return (None, utils.GetCertificateDigest(cert_filename=cert_file)) def _VerifySshSetup(node_status_list, my_name, ssh_key_type, ganeti_pub_keys_file=pathutils.SSH_PUB_KEYS): """Verifies the state of the SSH key files. @type node_status_list: list of tuples @param node_status_list: list of nodes of the cluster associated with a couple of flags: (uuid, name, is_master_candidate, is_potential_master_candidate, online) @type my_name: str @param my_name: name of this node @type ssh_key_type: one of L{constants.SSHK_ALL} @param ssh_key_type: type of key used on nodes @type ganeti_pub_keys_file: str @param ganeti_pub_keys_file: filename of the public keys file """ if node_status_list is None: return ["No node list to check against the pub_key_file received."] my_status_list = [(my_uuid, name, mc, pot_mc, online) for (my_uuid, name, mc, pot_mc, online) in node_status_list if name == my_name] if len(my_status_list) == 0: return ["Cannot find node information for node '%s'." % my_name] (my_uuid, _, _, potential_master_candidate, online) = \ my_status_list[0] result = [] if not os.path.exists(ganeti_pub_keys_file): result.append("The public key file '%s' does not exist. Consider running" " 'gnt-cluster renew-crypto --new-ssh-keys" " [--no-ssh-key-check]' to fix this." % ganeti_pub_keys_file) return result pot_mc_uuids = [uuid for (uuid, _, _, _, _) in node_status_list] offline_nodes = [uuid for (uuid, _, _, _, online) in node_status_list if not online] pub_keys = ssh.QueryPubKeyFile(None, key_file=ganeti_pub_keys_file) if potential_master_candidate: # Check that the set of potential master candidates matches the # public key file pub_uuids_set = set(pub_keys) - set(offline_nodes) pot_mc_uuids_set = set(pot_mc_uuids) - set(offline_nodes) missing_uuids = set([]) if pub_uuids_set != pot_mc_uuids_set: unknown_uuids = pub_uuids_set - pot_mc_uuids_set pub_key_path = "%s:%s" % (my_name, ganeti_pub_keys_file) if unknown_uuids: result.append("The following node UUIDs are listed in the shared public" " keys file %s, but are not potential master" " candidates: %s." % (pub_key_path, ", ".join(list(unknown_uuids)))) missing_uuids = pot_mc_uuids_set - pub_uuids_set if missing_uuids: result.append("The following node UUIDs of potential master candidates" " are missing in the shared public keys file %s: %s." % (pub_key_path, ", ".join(list(missing_uuids)))) (_, key_files) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=False, dircheck=False) (_, node_pub_key_file) = key_files[ssh_key_type] my_keys = pub_keys[my_uuid] node_pub_key = utils.ReadFile(node_pub_key_file) node_pub_key_path = "%s:%s" % (my_name, node_pub_key_file) if node_pub_key.strip() not in my_keys: result.append("The key for node %s in the cluster config does not match" " this node's key in the node public key file %s." % (my_name, node_pub_key_path)) if len(my_keys) != 1: result.append("There is more than one key for node %s in the node public" " key file %s." % (my_name, node_pub_key_path)) else: if len(pub_keys) > 0: result.append("The public key file %s is not empty, although" " the node is not a potential master candidate." % node_pub_key_path) # Check that all master candidate keys are in the authorized_keys file (auth_key_file, _) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=False, dircheck=False) for (uuid, name, mc, _, online) in node_status_list: if not online: continue if uuid in missing_uuids: continue if mc: for key in pub_keys[uuid]: if not ssh.HasAuthorizedKey(auth_key_file, key): result.append("A SSH key of master candidate '%s' (UUID: '%s') is" " not in the 'authorized_keys' file of node '%s'." % (name, uuid, my_name)) else: for key in pub_keys[uuid]: if name != my_name and ssh.HasAuthorizedKey(auth_key_file, key): result.append("A SSH key of normal node '%s' (UUID: '%s') is in the" " 'authorized_keys' file of node '%s'." % (name, uuid, my_name)) if name == my_name and not ssh.HasAuthorizedKey(auth_key_file, key): result.append("A SSH key of normal node '%s' (UUID: '%s') is not" " in the 'authorized_keys' file of itself." % (my_name, uuid)) return result def _VerifySshClutter(node_status_list, my_name): """Verifies that the 'authorized_keys' files are not cluttered up. @type node_status_list: list of tuples @param node_status_list: list of nodes of the cluster associated with a couple of flags: (uuid, name, is_master_candidate, is_potential_master_candidate, online) @type my_name: str @param my_name: name of this node """ result = [] (auth_key_file, _) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=False, dircheck=False) node_names = [name for (_, name, _, _) in node_status_list] multiple_occurrences = ssh.CheckForMultipleKeys(auth_key_file, node_names) if multiple_occurrences: msg = "There are hosts which have more than one SSH key stored for the" \ " same user in the 'authorized_keys' file of node %s. This can be" \ " due to an unsuccessful operation which cluttered up the" \ " 'authorized_keys' file. We recommend to clean this up manually. " \ % my_name for host, occ in multiple_occurrences.items(): msg += "Entry for '%s' in lines %s. " % (host, utils.CommaJoin(occ)) result.append(msg) return result def VerifyNodeNetTest(my_name, test_config): """Verify nodes are reachable. @type my_name: string @param my_name: name of the node this test is running on @type test_config: tuple (node_list, master_candidate_list) @param test_config: configuration for test as passed from LUClusterVerify() in what[constants.NV_NODENETTEST] @rtype: dict @return: a dictionary with node names as keys and error messages as values """ result = {} nodes, master_candidates = test_config port = netutils.GetDaemonPort(constants.NODED) if my_name not in master_candidates: return result my_pip, my_sip = next( ((pip, sip) for name, pip, sip in nodes if name == my_name), (None, None) ) if not my_pip: result[my_name] = ("Can't find my own primary/secondary IP" " in the node list") return result for name, pip, sip in nodes: fail = [] if not netutils.TcpPing(pip, port, source=my_pip): fail.append("primary") if sip != pip: if not netutils.TcpPing(sip, port, source=my_sip): fail.append("secondary") if fail: result[name] = ("failure using the %s interface(s)" % " and ".join(fail)) return result def VerifyMasterIP(my_name, test_config): """Verify master IP is reachable. @type my_name: string @param my_name: name of the node this test is running on @type test_config: tuple (master_name, master_up, master_candidates) @param test_config: configuration for test as passed from LUClusterVerify() in what[constants.NV_MASTERIP] @rtype: bool or None @return: Boolean test result, None if skipped """ master_name, master_ip, master_candidates = test_config port = netutils.GetDaemonPort(constants.NODED) if my_name not in master_candidates: return None if master_name == my_name: source = constants.IP4_ADDRESS_LOCALHOST else: source = None return netutils.TcpPing(master_ip, port, source=source) def VerifyNode(what, cluster_name, all_hvparams): """Verify the status of the local node. Based on the input L{what} parameter, various checks are done on the local node. If the I{filelist} key is present, this list of files is checksummed and the file/checksum pairs are returned. If the I{nodelist} key is present, we check that we have connectivity via ssh with the target nodes (and check the hostname report). If the I{node-net-test} key is present, we check that we have connectivity to the given nodes via both primary IP and, if applicable, secondary IPs. @type what: C{dict} @param what: a dictionary of things to check: - filelist: list of files for which to compute checksums - nodelist: list of nodes we should check ssh communication with - node-net-test: list of nodes we should check node daemon port connectivity with - hypervisor: list with hypervisors to run the verify for @type cluster_name: string @param cluster_name: the cluster's name @type all_hvparams: dict of dict of strings @param all_hvparams: a dictionary mapping hypervisor names to hvparams @rtype: dict @return: a dictionary with the same keys as the input dict, and values representing the result of the checks """ result = {} my_name = netutils.Hostname.GetSysName() vm_capable = my_name not in what.get(constants.NV_NONVMNODES, []) _VerifyHypervisors(what, vm_capable, result, all_hvparams) _VerifyHvparams(what, vm_capable, result) if constants.NV_FILELIST in what: fingerprints = utils.FingerprintFiles(map(vcluster.LocalizeVirtualPath, what[constants.NV_FILELIST])) result[constants.NV_FILELIST] = \ dict((vcluster.MakeVirtualPath(key), value) for (key, value) in fingerprints.items()) if constants.NV_CLIENT_CERT in what: result[constants.NV_CLIENT_CERT] = _VerifyClientCertificate() if constants.NV_SSH_SETUP in what: node_status_list, key_type = what[constants.NV_SSH_SETUP] result[constants.NV_SSH_SETUP] = \ _VerifySshSetup(node_status_list, my_name, key_type) if constants.NV_SSH_CLUTTER in what: result[constants.NV_SSH_CLUTTER] = \ _VerifySshClutter(what[constants.NV_SSH_SETUP], my_name) if constants.NV_NODELIST in what: (nodes, bynode, mcs) = what[constants.NV_NODELIST] # Add nodes from other groups (different for each node) try: nodes.extend(bynode[my_name]) except KeyError: pass # Use a random order random.shuffle(nodes) # Try to contact all nodes val = {} ssh_port_map = ssconf.SimpleStore().GetSshPortMap() for node in nodes: # We only test if master candidates can communicate to other nodes. # We cannot test if normal nodes cannot communicate with other nodes, # because the administrator might have installed additional SSH keys, # over which Ganeti has no power. if my_name in mcs: success, message = _GetSshRunner(cluster_name). \ VerifyNodeHostname(node, ssh_port_map[node]) if not success: val[node] = message result[constants.NV_NODELIST] = val if constants.NV_NODENETTEST in what: result[constants.NV_NODENETTEST] = VerifyNodeNetTest( my_name, what[constants.NV_NODENETTEST]) if constants.NV_MASTERIP in what: result[constants.NV_MASTERIP] = VerifyMasterIP( my_name, what[constants.NV_MASTERIP]) if constants.NV_USERSCRIPTS in what: result[constants.NV_USERSCRIPTS] = \ [script for script in what[constants.NV_USERSCRIPTS] if not utils.IsExecutable(script)] if constants.NV_OOB_PATHS in what: result[constants.NV_OOB_PATHS] = tmp = [] for path in what[constants.NV_OOB_PATHS]: try: st = os.stat(path) except OSError as err: tmp.append("error stating out of band helper: %s" % err) else: if stat.S_ISREG(st.st_mode): if stat.S_IMODE(st.st_mode) & stat.S_IXUSR: tmp.append(None) else: tmp.append("out of band helper %s is not executable" % path) else: tmp.append("out of band helper %s is not a file" % path) if constants.NV_LVLIST in what and vm_capable: try: val = GetVolumeList(list(utils.ListVolumeGroups())) except RPCFail as err: val = str(err) result[constants.NV_LVLIST] = val _VerifyInstanceList(what, vm_capable, result, all_hvparams) if constants.NV_VGLIST in what and vm_capable: result[constants.NV_VGLIST] = utils.ListVolumeGroups() if constants.NV_PVLIST in what and vm_capable: check_exclusive_pvs = constants.NV_EXCLUSIVEPVS in what val = bdev.LogicalVolume.GetPVInfo(what[constants.NV_PVLIST], filter_allocatable=False, include_lvs=check_exclusive_pvs) if check_exclusive_pvs: result[constants.NV_EXCLUSIVEPVS] = _CheckExclusivePvs(val) for pvi in val: # Avoid sending useless data on the wire pvi.lv_list = [] result[constants.NV_PVLIST] = [objects.LvmPvInfo.ToDict(v) for v in val] if constants.NV_VERSION in what: result[constants.NV_VERSION] = (constants.PROTOCOL_VERSION, constants.RELEASE_VERSION) _VerifyNodeInfo(what, vm_capable, result, all_hvparams) if constants.NV_DRBDVERSION in what and vm_capable: try: drbd_version = DRBD8.GetProcInfo().GetVersionString() except errors.BlockDeviceError as err: logging.warning("Can't get DRBD version", exc_info=True) drbd_version = str(err) result[constants.NV_DRBDVERSION] = drbd_version if constants.NV_DRBDLIST in what and vm_capable: try: used_minors = drbd.DRBD8.GetUsedDevs() except errors.BlockDeviceError as err: logging.warning("Can't get used minors list", exc_info=True) used_minors = str(err) result[constants.NV_DRBDLIST] = used_minors if constants.NV_DRBDHELPER in what and vm_capable: status = True try: payload = drbd.DRBD8.GetUsermodeHelper() except errors.BlockDeviceError as err: logging.error("Can't get DRBD usermode helper: %s", str(err)) status = False payload = str(err) result[constants.NV_DRBDHELPER] = (status, payload) if constants.NV_NODESETUP in what: result[constants.NV_NODESETUP] = tmpr = [] if not os.path.isdir("/sys/block") or not os.path.isdir("/sys/class/net"): tmpr.append("The sysfs filesytem doesn't seem to be mounted" " under /sys, missing required directories /sys/block" " and /sys/class/net") if (not os.path.isdir("/proc/sys") or not os.path.isfile("/proc/sysrq-trigger")): tmpr.append("The procfs filesystem doesn't seem to be mounted" " under /proc, missing required directory /proc/sys and" " the file /proc/sysrq-trigger") if constants.NV_TIME in what: result[constants.NV_TIME] = utils.SplitTime(time.time()) if constants.NV_OSLIST in what and vm_capable: result[constants.NV_OSLIST] = DiagnoseOS() if constants.NV_BRIDGES in what and vm_capable: result[constants.NV_BRIDGES] = [bridge for bridge in what[constants.NV_BRIDGES] if not utils.BridgeExists(bridge)] if what.get(constants.NV_ACCEPTED_STORAGE_PATHS) == my_name: result[constants.NV_ACCEPTED_STORAGE_PATHS] = \ filestorage.ComputeWrongFileStoragePaths() if what.get(constants.NV_FILE_STORAGE_PATH): pathresult = filestorage.CheckFileStoragePath( what[constants.NV_FILE_STORAGE_PATH]) if pathresult: result[constants.NV_FILE_STORAGE_PATH] = pathresult if what.get(constants.NV_SHARED_FILE_STORAGE_PATH): pathresult = filestorage.CheckFileStoragePath( what[constants.NV_SHARED_FILE_STORAGE_PATH]) if pathresult: result[constants.NV_SHARED_FILE_STORAGE_PATH] = pathresult return result def GetCryptoTokens(token_requests): """Perform actions on the node's cryptographic tokens. Token types can be 'ssl' or 'ssh'. So far only some actions are implemented for 'ssl'. Action 'get' returns the digest of the public client ssl certificate. Action 'create' creates a new client certificate and private key and also returns the digest of the certificate. The third parameter of a token request are optional parameters for the actions, so far only the filename is supported. @type token_requests: list of tuples of (string, string, dict), where the first string is in constants.CRYPTO_TYPES, the second in constants.CRYPTO_ACTIONS. The third parameter is a dictionary of string to string. @param token_requests: list of requests of cryptographic tokens and actions to perform on them. The actions come with a dictionary of options. @rtype: list of tuples (string, string) @return: list of tuples of the token type and the public crypto token """ tokens = [] for (token_type, action, _) in token_requests: if token_type not in constants.CRYPTO_TYPES: raise errors.ProgrammerError("Token type '%s' not supported." % token_type) if action not in constants.CRYPTO_ACTIONS: raise errors.ProgrammerError("Action '%s' is not supported." % action) if token_type == constants.CRYPTO_TYPE_SSL_DIGEST: tokens.append((token_type, utils.GetCertificateDigest())) return tokens def EnsureDaemon(daemon_name, run): """Ensures the given daemon is running or stopped. @type daemon_name: string @param daemon_name: name of the daemon (e.g., constants.KVMD) @type run: bool @param run: whether to start or stop the daemon @rtype: bool @return: 'True' if daemon successfully started/stopped, 'False' otherwise """ allowed_daemons = [constants.KVMD] if daemon_name not in allowed_daemons: fn = lambda _: False elif run: fn = utils.EnsureDaemon else: fn = utils.StopDaemon return fn(daemon_name) def _InitSshUpdateData(data, noded_cert_file, ssconf_store): (_, noded_cert) = \ utils.ExtractX509Certificate(utils.ReadFile(noded_cert_file)) data[constants.SSHS_NODE_DAEMON_CERTIFICATE] = noded_cert cluster_name = ssconf_store.GetClusterName() data[constants.SSHS_CLUSTER_NAME] = cluster_name def AddNodeSshKey(node_uuid, node_name, potential_master_candidates, to_authorized_keys=False, to_public_keys=False, get_public_keys=False, pub_key_file=pathutils.SSH_PUB_KEYS, ssconf_store=None, noded_cert_file=pathutils.NODED_CERT_FILE, run_cmd_fn=ssh.RunSshCmdWithStdin): """Distributes a node's public SSH key across the cluster. Note that this function should only be executed on the master node, which then will copy the new node's key to all nodes in the cluster via SSH. Also note: at least one of the flags C{to_authorized_keys}, C{to_public_keys}, and C{get_public_keys} has to be set to C{True} for the function to actually perform any actions. @type node_uuid: str @param node_uuid: the UUID of the node whose key is added @type node_name: str @param node_name: the name of the node whose key is added @type potential_master_candidates: list of str @param potential_master_candidates: list of node names of potential master candidates; this should match the list of uuids in the public key file @type to_authorized_keys: boolean @param to_authorized_keys: whether the key should be added to the C{authorized_keys} file of all nodes @type to_public_keys: boolean @param to_public_keys: whether the keys should be added to the public key file @type get_public_keys: boolean @param get_public_keys: whether the node should add the clusters' public keys to its {ganeti_pub_keys} file """ node_list = [SshAddNodeInfo(name=node_name, uuid=node_uuid, to_authorized_keys=to_authorized_keys, to_public_keys=to_public_keys, get_public_keys=get_public_keys)] return AddNodeSshKeyBulk(node_list, potential_master_candidates, pub_key_file=pub_key_file, ssconf_store=ssconf_store, noded_cert_file=noded_cert_file, run_cmd_fn=run_cmd_fn) # Node info named tuple specifically for the use with AddNodeSshKeyBulk SshAddNodeInfo = collections.namedtuple( "SshAddNodeInfo", ["uuid", "name", "to_authorized_keys", "to_public_keys", "get_public_keys"]) def AddNodeSshKeyBulk(node_list, potential_master_candidates, pub_key_file=pathutils.SSH_PUB_KEYS, ssconf_store=None, noded_cert_file=pathutils.NODED_CERT_FILE, run_cmd_fn=ssh.RunSshCmdWithStdin): """Distributes a node's public SSH key across the cluster. Note that this function should only be executed on the master node, which then will copy the new node's key to all nodes in the cluster via SSH. Also note: at least one of the flags C{to_authorized_keys}, C{to_public_keys}, and C{get_public_keys} has to be set to C{True} for the function to actually perform any actions. @type node_list: list of SshAddNodeInfo tuples @param node_list: list of tuples containing the necessary node information for adding their keys @type potential_master_candidates: list of str @param potential_master_candidates: list of node names of potential master candidates; this should match the list of uuids in the public key file """ # whether there are any keys to be added or retrieved at all to_authorized_keys = any([node_info.to_authorized_keys for node_info in node_list]) to_public_keys = any([node_info.to_public_keys for node_info in node_list]) if not ssconf_store: ssconf_store = ssconf.SimpleStore() for node_info in node_list: # replacement not necessary for keys that are not supposed to be in the # list of public keys if not node_info.to_public_keys: continue # Check and fix sanity of key file keys_by_name = ssh.QueryPubKeyFile([node_info.name], key_file=pub_key_file) keys_by_uuid = ssh.QueryPubKeyFile([node_info.uuid], key_file=pub_key_file) if (not keys_by_name or node_info.name not in keys_by_name) \ and (not keys_by_uuid or node_info.uuid not in keys_by_uuid): raise errors.SshUpdateError( "No keys found for the new node '%s' (UUID %s) in the list of public" " SSH keys, neither for the name or the UUID" % (node_info.name, node_info.uuid)) else: if node_info.name in keys_by_name: # Replace the name by UUID in the file as the name should only be used # temporarily ssh.ReplaceNameByUuid(node_info.uuid, node_info.name, error_fn=errors.SshUpdateError, key_file=pub_key_file) # Retrieve updated map of UUIDs to keys keys_by_uuid = ssh.QueryPubKeyFile( [node_info.uuid for node_info in node_list], key_file=pub_key_file) # Update the master node's key files (auth_key_file, _) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=False, dircheck=False) for node_info in node_list: if node_info.to_authorized_keys: ssh.AddAuthorizedKeys(auth_key_file, keys_by_uuid[node_info.uuid]) base_data = {} _InitSshUpdateData(base_data, noded_cert_file, ssconf_store) cluster_name = base_data[constants.SSHS_CLUSTER_NAME] ssh_port_map = ssconf_store.GetSshPortMap() # Update the target nodes themselves for node_info in node_list: logging.debug("Updating SSH key files of target node '%s'.", node_info.name) if node_info.get_public_keys: node_data = {} _InitSshUpdateData(node_data, noded_cert_file, ssconf_store) all_keys = ssh.QueryPubKeyFile(None, key_file=pub_key_file) node_data[constants.SSHS_SSH_PUBLIC_KEYS] = \ (constants.SSHS_OVERRIDE, all_keys) try: utils.RetryByNumberOfTimes( constants.SSHS_MAX_RETRIES, errors.SshUpdateError, run_cmd_fn, cluster_name, node_info.name, pathutils.SSH_UPDATE, ssh_port_map.get(node_info.name), node_data, debug=False, verbose=False, use_cluster_key=False, ask_key=False, strict_host_check=False) except errors.SshUpdateError as e: # Clean up the master's public key file if adding key fails if node_info.to_public_keys: ssh.RemovePublicKey(node_info.uuid) raise e # Update all nodes except master and the target nodes keys_by_uuid_auth = ssh.QueryPubKeyFile( [node_info.uuid for node_info in node_list if node_info.to_authorized_keys], key_file=pub_key_file) if to_authorized_keys: base_data[constants.SSHS_SSH_AUTHORIZED_KEYS] = \ (constants.SSHS_ADD, keys_by_uuid_auth) pot_mc_data = base_data.copy() keys_by_uuid_pub = ssh.QueryPubKeyFile( [node_info.uuid for node_info in node_list if node_info.to_public_keys], key_file=pub_key_file) if to_public_keys: pot_mc_data[constants.SSHS_SSH_PUBLIC_KEYS] = \ (constants.SSHS_REPLACE_OR_ADD, keys_by_uuid_pub) all_nodes = ssconf_store.GetNodeList() master_node = ssconf_store.GetMasterNode() online_nodes = ssconf_store.GetOnlineNodeList() node_errors = [] for node in all_nodes: if node == master_node: logging.debug("Skipping master node '%s'.", master_node) continue if node not in online_nodes: logging.debug("Skipping offline node '%s'.", node) continue if node in potential_master_candidates: logging.debug("Updating SSH key files of node '%s'.", node) try: utils.RetryByNumberOfTimes( constants.SSHS_MAX_RETRIES, errors.SshUpdateError, run_cmd_fn, cluster_name, node, pathutils.SSH_UPDATE, ssh_port_map.get(node), pot_mc_data, debug=False, verbose=False, use_cluster_key=False, ask_key=False, strict_host_check=False) except errors.SshUpdateError as last_exception: error_msg = ("When adding the key of node '%s', updating SSH key" " files of node '%s' failed after %s retries." " Not trying again. Last error was: %s." % (node, node_info.name, constants.SSHS_MAX_RETRIES, last_exception)) node_errors.append((node, error_msg)) # We only log the error and don't throw an exception, because # one unreachable node shall not abort the entire procedure. logging.error(error_msg) else: if to_authorized_keys: run_cmd_fn(cluster_name, node, pathutils.SSH_UPDATE, ssh_port_map.get(node), base_data, debug=False, verbose=False, use_cluster_key=False, ask_key=False, strict_host_check=False) return node_errors def RemoveNodeSshKey(node_uuid, node_name, master_candidate_uuids, potential_master_candidates, master_uuid=None, keys_to_remove=None, from_authorized_keys=False, from_public_keys=False, clear_authorized_keys=False, clear_public_keys=False, pub_key_file=pathutils.SSH_PUB_KEYS, ssconf_store=None, noded_cert_file=pathutils.NODED_CERT_FILE, readd=False, run_cmd_fn=ssh.RunSshCmdWithStdin): """Removes the node's SSH keys from the key files and distributes those. Note that at least one of the flags C{from_authorized_keys}, C{from_public_keys}, C{clear_authorized_keys}, and C{clear_public_keys} has to be set to C{True} for the function to perform any action at all. Not doing so will trigger an assertion in the function. @type node_uuid: str @param node_uuid: UUID of the node whose key is removed @type node_name: str @param node_name: name of the node whose key is remove @type master_candidate_uuids: list of str @param master_candidate_uuids: list of UUIDs of the current master candidates @type potential_master_candidates: list of str @param potential_master_candidates: list of names of potential master candidates @type keys_to_remove: dict of str to list of str @param keys_to_remove: a dictionary mapping node UUIDS to lists of SSH keys to be removed. This list is supposed to be used only if the keys are not in the public keys file. This is for example the case when removing a master node's key. @type from_authorized_keys: boolean @param from_authorized_keys: whether or not the key should be removed from the C{authorized_keys} file @type from_public_keys: boolean @param from_public_keys: whether or not the key should be remove from the C{ganeti_pub_keys} file @type clear_authorized_keys: boolean @param clear_authorized_keys: whether or not the C{authorized_keys} file should be cleared on the node whose keys are removed @type clear_public_keys: boolean @param clear_public_keys: whether to clear the node's C{ganeti_pub_key} file @type readd: boolean @param readd: whether this is called during a readd operation. @rtype: list of string @returns: list of feedback messages """ node_list = [SshRemoveNodeInfo(uuid=node_uuid, name=node_name, from_authorized_keys=from_authorized_keys, from_public_keys=from_public_keys, clear_authorized_keys=clear_authorized_keys, clear_public_keys=clear_public_keys)] return RemoveNodeSshKeyBulk(node_list, master_candidate_uuids, potential_master_candidates, master_uuid=master_uuid, keys_to_remove=keys_to_remove, pub_key_file=pub_key_file, ssconf_store=ssconf_store, noded_cert_file=noded_cert_file, readd=readd, run_cmd_fn=run_cmd_fn) # Node info named tuple specifically for the use with RemoveNodeSshKeyBulk SshRemoveNodeInfo = collections.namedtuple( "SshRemoveNodeInfo", ["uuid", "name", "from_authorized_keys", "from_public_keys", "clear_authorized_keys", "clear_public_keys"]) def RemoveNodeSshKeyBulk(node_list, master_candidate_uuids, potential_master_candidates, master_uuid=None, keys_to_remove=None, pub_key_file=pathutils.SSH_PUB_KEYS, ssconf_store=None, noded_cert_file=pathutils.NODED_CERT_FILE, readd=False, run_cmd_fn=ssh.RunSshCmdWithStdin): """Removes the node's SSH keys from the key files and distributes those. Note that at least one of the flags C{from_authorized_keys}, C{from_public_keys}, C{clear_authorized_keys}, and C{clear_public_keys} of at least one node has to be set to C{True} for the function to perform any action at all. Not doing so will trigger an assertion in the function. @type node_list: list of C{SshRemoveNodeInfo}. @param node_list: list of information about nodes whose keys are being removed @type master_candidate_uuids: list of str @param master_candidate_uuids: list of UUIDs of the current master candidates @type potential_master_candidates: list of str @param potential_master_candidates: list of names of potential master candidates @type keys_to_remove: dict of str to list of str @param keys_to_remove: a dictionary mapping node UUIDS to lists of SSH keys to be removed. This list is supposed to be used only if the keys are not in the public keys file. This is for example the case when removing a master node's key. @type readd: boolean @param readd: whether this is called during a readd operation. @rtype: list of string @returns: list of feedback messages """ # Non-disruptive error messages, list of (node, msg) pairs result_msgs = [] # whether there are any keys to be added or retrieved at all from_authorized_keys = any([node_info.from_authorized_keys for node_info in node_list]) from_public_keys = any([node_info.from_public_keys for node_info in node_list]) clear_authorized_keys = any([node_info.clear_authorized_keys for node_info in node_list]) clear_public_keys = any([node_info.clear_public_keys for node_info in node_list]) # Make sure at least one of these flags is true. if not (from_authorized_keys or from_public_keys or clear_authorized_keys or clear_public_keys): raise errors.SshUpdateError("No removal from any key file was requested.") if not ssconf_store: ssconf_store = ssconf.SimpleStore() master_node = ssconf_store.GetMasterNode() ssh_port_map = ssconf_store.GetSshPortMap() all_keys_to_remove = {} if from_authorized_keys or from_public_keys: for node_info in node_list: # Skip nodes that don't actually need any keys to be removed. if not (node_info.from_authorized_keys or node_info.from_public_keys): continue if node_info.name == master_node and not keys_to_remove: raise errors.SshUpdateError("Cannot remove the master node's keys.") if keys_to_remove: keys = keys_to_remove else: keys = ssh.QueryPubKeyFile([node_info.uuid], key_file=pub_key_file) if (not keys or node_info.uuid not in keys) and not readd: raise errors.SshUpdateError("Node '%s' not found in the list of" " public SSH keys. It seems someone" " tries to remove a key from outside" " the cluster!" % node_info.uuid) # During an upgrade all nodes have the master key. In this case we # should not remove it to avoid accidentally shutting down cluster # SSH communication master_keys = None if master_uuid: master_keys = ssh.QueryPubKeyFile([master_uuid], key_file=pub_key_file) # Remove any master keys from the list of keys to remove from the node keys[node_info.uuid] = list( set(keys[node_info.uuid]) - set(master_keys)) all_keys_to_remove.update(keys) if all_keys_to_remove: base_data = {} _InitSshUpdateData(base_data, noded_cert_file, ssconf_store) cluster_name = base_data[constants.SSHS_CLUSTER_NAME] if from_authorized_keys: # UUIDs of nodes that are supposed to be removed from the # authorized_keys files. nodes_remove_from_authorized_keys = [ node_info.uuid for node_info in node_list if node_info.from_authorized_keys] keys_to_remove_from_authorized_keys = dict([ (uuid, keys) for (uuid, keys) in all_keys_to_remove.items() if uuid in nodes_remove_from_authorized_keys]) base_data[constants.SSHS_SSH_AUTHORIZED_KEYS] = \ (constants.SSHS_REMOVE, keys_to_remove_from_authorized_keys) (auth_key_file, _) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=False, dircheck=False) for uuid in nodes_remove_from_authorized_keys: ssh.RemoveAuthorizedKeys(auth_key_file, keys_to_remove_from_authorized_keys[uuid]) pot_mc_data = base_data.copy() if from_public_keys: nodes_remove_from_public_keys = [ node_info.uuid for node_info in node_list if node_info.from_public_keys] keys_to_remove_from_public_keys = dict([ (uuid, keys) for (uuid, keys) in all_keys_to_remove.items() if uuid in nodes_remove_from_public_keys]) pot_mc_data[constants.SSHS_SSH_PUBLIC_KEYS] = \ (constants.SSHS_REMOVE, keys_to_remove_from_public_keys) all_nodes = ssconf_store.GetNodeList() online_nodes = ssconf_store.GetOnlineNodeList() all_nodes_to_remove = [node_info.name for node_info in node_list] logging.debug("Removing keys of nodes '%s' from all nodes but itself and" " master.", ", ".join(all_nodes_to_remove)) for node in all_nodes: if node == master_node: logging.debug("Skipping master node '%s'.", master_node) continue if node not in online_nodes: logging.debug("Skipping offline node '%s'.", node) continue if node in all_nodes_to_remove: logging.debug("Skipping node whose key is removed itself '%s'.", node) continue ssh_port = ssh_port_map.get(node) if not ssh_port: raise errors.OpExecError("No SSH port information available for" " node '%s', map: %s." % (node, ssh_port_map)) error_msg_final = ("When removing the key of node '%s', updating the" " SSH key files of node '%s' failed. Last error" " was: %s.") if node in potential_master_candidates or from_authorized_keys: if node in potential_master_candidates: node_desc = "potential master candidate" else: node_desc = "normal" logging.debug("Updating key setup of %s node %s.", node_desc, node) try: utils.RetryByNumberOfTimes( constants.SSHS_MAX_RETRIES, errors.SshUpdateError, run_cmd_fn, cluster_name, node, pathutils.SSH_UPDATE, ssh_port, pot_mc_data, debug=False, verbose=False, use_cluster_key=False, ask_key=False, strict_host_check=False) except errors.SshUpdateError as last_exception: error_msg = error_msg_final % ( node_info.name, node, last_exception) result_msgs.append((node, error_msg)) logging.error(error_msg) for node_info in node_list: if node_info.clear_authorized_keys or node_info.from_public_keys or \ node_info.clear_public_keys: data = {} _InitSshUpdateData(data, noded_cert_file, ssconf_store) cluster_name = data[constants.SSHS_CLUSTER_NAME] ssh_port = ssh_port_map.get(node_info.name) if not ssh_port: raise errors.OpExecError("No SSH port information available for" " node '%s', which is leaving the cluster.") if node_info.clear_authorized_keys: # The 'authorized_keys' file is not solely managed by Ganeti. Therefore, # we have to specify exactly which keys to clear to leave keys untouched # that were not added by Ganeti. other_master_candidate_uuids = [uuid for uuid in master_candidate_uuids if uuid != node_info.uuid] candidate_keys = ssh.QueryPubKeyFile(other_master_candidate_uuids, key_file=pub_key_file) data[constants.SSHS_SSH_AUTHORIZED_KEYS] = \ (constants.SSHS_REMOVE, candidate_keys) if node_info.clear_public_keys: data[constants.SSHS_SSH_PUBLIC_KEYS] = \ (constants.SSHS_CLEAR, {}) elif node_info.from_public_keys: # Since clearing the public keys subsumes removing just a single key, # we only do it if clear_public_keys is 'False'. if all_keys_to_remove: data[constants.SSHS_SSH_PUBLIC_KEYS] = \ (constants.SSHS_REMOVE, all_keys_to_remove) # If we have no changes to any keyfile, just return if not (constants.SSHS_SSH_PUBLIC_KEYS in data or constants.SSHS_SSH_AUTHORIZED_KEYS in data): return logging.debug("Updating SSH key setup of target node '%s'.", node_info.name) try: utils.RetryByNumberOfTimes( constants.SSHS_MAX_RETRIES, errors.SshUpdateError, run_cmd_fn, cluster_name, node_info.name, pathutils.SSH_UPDATE, ssh_port, data, debug=False, verbose=False, use_cluster_key=False, ask_key=False, strict_host_check=False) except errors.SshUpdateError as last_exception: result_msgs.append( (node_info.name, ("Removing SSH keys from node '%s' failed." " This can happen when the node is already unreachable." " Error: %s" % (node_info.name, last_exception)))) if all_keys_to_remove and from_public_keys: for node_uuid in nodes_remove_from_public_keys: ssh.RemovePublicKey(node_uuid, key_file=pub_key_file) return result_msgs def _GenerateNodeSshKey(node_uuid, node_name, ssh_port_map, ssh_key_type, ssh_key_bits, pub_key_file=pathutils.SSH_PUB_KEYS, ssconf_store=None, noded_cert_file=pathutils.NODED_CERT_FILE, run_cmd_fn=ssh.RunSshCmdWithStdin, suffix=""): """Generates the root SSH key pair on the node. @type node_uuid: str @param node_uuid: UUID of the node whose key is removed @type node_name: str @param node_name: name of the node whose key is remove @type ssh_port_map: dict of str to int @param ssh_port_map: mapping of node names to their SSH port @type ssh_key_type: One of L{constants.SSHK_ALL} @param ssh_key_type: the type of SSH key to be generated @type ssh_key_bits: int @param ssh_key_bits: the length of the key to be generated """ if not ssconf_store: ssconf_store = ssconf.SimpleStore() keys_by_uuid = ssh.QueryPubKeyFile([node_uuid], key_file=pub_key_file) if not keys_by_uuid or node_uuid not in keys_by_uuid: raise errors.SshUpdateError("Node %s (UUID: %s) whose key is requested to" " be regenerated is not registered in the" " public keys file." % (node_name, node_uuid)) data = {} _InitSshUpdateData(data, noded_cert_file, ssconf_store) cluster_name = data[constants.SSHS_CLUSTER_NAME] data[constants.SSHS_GENERATE] = (ssh_key_type, ssh_key_bits, suffix) run_cmd_fn(cluster_name, node_name, pathutils.SSH_UPDATE, ssh_port_map.get(node_name), data, debug=False, verbose=False, use_cluster_key=False, ask_key=False, strict_host_check=False) def _GetMasterNodeUUID(node_uuid_name_map, master_node_name): master_node_uuids = [node_uuid for (node_uuid, node_name) in node_uuid_name_map if node_name == master_node_name] if len(master_node_uuids) != 1: raise errors.SshUpdateError("No (unique) master UUID found. Master node" " name: '%s', Master UUID: '%s'" % (master_node_name, master_node_uuids)) return master_node_uuids[0] def _GetOldMasterKeys(master_node_uuid, pub_key_file): old_master_keys_by_uuid = ssh.QueryPubKeyFile([master_node_uuid], key_file=pub_key_file) if not old_master_keys_by_uuid: raise errors.SshUpdateError("No public key of the master node (UUID '%s')" " found, not generating a new key." % master_node_uuid) return old_master_keys_by_uuid def _GetNewMasterKey(root_keyfiles, master_node_uuid): new_master_keys = [] for (_, (_, public_key_file)) in root_keyfiles.items(): public_key_dir = os.path.dirname(public_key_file) public_key_file_tmp_filename = \ os.path.splitext(os.path.basename(public_key_file))[0] \ + constants.SSHS_MASTER_SUFFIX + ".pub" public_key_path_tmp = os.path.join(public_key_dir, public_key_file_tmp_filename) if os.path.exists(public_key_path_tmp): # for some key types, there might not be any keys key = utils.ReadFile(public_key_path_tmp) new_master_keys.append(key) if not new_master_keys: raise errors.SshUpdateError("Cannot find any type of temporary SSH key.") return {master_node_uuid: new_master_keys} def _ReplaceMasterKeyOnMaster(root_keyfiles): number_of_moves = 0 for (_, (private_key_file, public_key_file)) in root_keyfiles.items(): key_dir = os.path.dirname(public_key_file) private_key_file_tmp = \ os.path.basename(private_key_file) + constants.SSHS_MASTER_SUFFIX public_key_file_tmp = private_key_file_tmp + ".pub" private_key_path_tmp = os.path.join(key_dir, private_key_file_tmp) public_key_path_tmp = os.path.join(key_dir, public_key_file_tmp) if os.path.exists(public_key_file): utils.CreateBackup(public_key_file) utils.RemoveFile(public_key_file) if os.path.exists(private_key_file): utils.CreateBackup(private_key_file) utils.RemoveFile(private_key_file) if os.path.exists(public_key_path_tmp) and \ os.path.exists(private_key_path_tmp): # for some key types, there might not be any keys shutil.move(public_key_path_tmp, public_key_file) shutil.move(private_key_path_tmp, private_key_file) number_of_moves += 1 if not number_of_moves: raise errors.SshUpdateError("Could not move at least one master SSH key.") def RenewSshKeys(node_uuids, node_names, master_candidate_uuids, potential_master_candidates, old_key_type, new_key_type, new_key_bits, ganeti_pub_keys_file=pathutils.SSH_PUB_KEYS, ssconf_store=None, noded_cert_file=pathutils.NODED_CERT_FILE, run_cmd_fn=ssh.RunSshCmdWithStdin): """Renews all SSH keys and updates authorized_keys and ganeti_pub_keys. @type node_uuids: list of str @param node_uuids: list of node UUIDs whose keys should be renewed @type node_names: list of str @param node_names: list of node names whose keys should be removed. This list should match the C{node_uuids} parameter @type master_candidate_uuids: list of str @param master_candidate_uuids: list of UUIDs of master candidates or master node @type old_key_type: One of L{constants.SSHK_ALL} @param old_key_type: the type of SSH key already present on nodes @type new_key_type: One of L{constants.SSHK_ALL} @param new_key_type: the type of SSH key to be generated @type new_key_bits: int @param new_key_bits: the length of the key to be generated @type ganeti_pub_keys_file: str @param ganeti_pub_keys_file: file path of the the public key file @type noded_cert_file: str @param noded_cert_file: path of the noded SSL certificate file @type run_cmd_fn: function @param run_cmd_fn: function to run commands on remote nodes via SSH @raises ProgrammerError: if node_uuids and node_names don't match; SshUpdateError if a node's key is missing from the public key file, if a node's new SSH key could not be fetched from it, if there is none or more than one entry in the public key list for the master node. """ if not ssconf_store: ssconf_store = ssconf.SimpleStore() cluster_name = ssconf_store.GetClusterName() if not len(node_uuids) == len(node_names): raise errors.ProgrammerError("List of nodes UUIDs and node names" " does not match in length.") (_, root_keyfiles) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=False, dircheck=False) (_, old_pub_keyfile) = root_keyfiles[old_key_type] (_, new_pub_keyfile) = root_keyfiles[new_key_type] old_master_key = utils.ReadFile(old_pub_keyfile) node_uuid_name_map = list(zip(node_uuids, node_names)) master_node_name = ssconf_store.GetMasterNode() master_node_uuid = _GetMasterNodeUUID(node_uuid_name_map, master_node_name) ssh_port_map = ssconf_store.GetSshPortMap() # List of all node errors that happened, but which did not abort the # procedure as a whole. It is important that this is a list to have a # somewhat chronological history of events. all_node_errors = [] # process non-master nodes # keys to add in bulk at the end node_keys_to_add = [] # list of all nodes node_list = [] # list of keys to be removed before generating new keys node_info_to_remove = [] for node_uuid, node_name in node_uuid_name_map: if node_name == master_node_name: continue master_candidate = node_uuid in master_candidate_uuids potential_master_candidate = node_name in potential_master_candidates node_list.append((node_uuid, node_name, master_candidate, potential_master_candidate)) keys_by_uuid = ssh.QueryPubKeyFile([node_uuid], key_file=ganeti_pub_keys_file) if not keys_by_uuid: raise errors.SshUpdateError("No public key of node %s (UUID %s) found," " not generating a new key." % (node_name, node_uuid)) if master_candidate: logging.debug("Fetching old SSH key from node '%s'.", node_name) old_pub_key = ssh.ReadRemoteSshPubKeys(old_pub_keyfile, node_name, cluster_name, ssh_port_map[node_name], False, # ask_key False) # key_check if old_pub_key != old_master_key: # If we are already in a multi-key setup (that is past Ganeti 2.12), # we can safely remove the old key of the node. Otherwise, we cannot # remove that node's key, because it is also the master node's key # and that would terminate all communication from the master to the # node. node_info_to_remove.append(SshRemoveNodeInfo( uuid=node_uuid, name=node_name, from_authorized_keys=master_candidate, from_public_keys=False, clear_authorized_keys=False, clear_public_keys=False)) else: logging.debug("Old key of node '%s' is the same as the current master" " key. Not deleting that key on the node.", node_name) logging.debug("Removing old SSH keys of all master candidates.") if node_info_to_remove: node_errors = RemoveNodeSshKeyBulk( node_info_to_remove, master_candidate_uuids, potential_master_candidates, master_uuid=master_node_uuid) if node_errors: all_node_errors = all_node_errors + node_errors for (node_uuid, node_name, master_candidate, potential_master_candidate) \ in node_list: logging.debug("Generating new SSH key for node '%s'.", node_name) _GenerateNodeSshKey(node_uuid, node_name, ssh_port_map, new_key_type, new_key_bits, pub_key_file=ganeti_pub_keys_file, ssconf_store=ssconf_store, noded_cert_file=noded_cert_file, run_cmd_fn=run_cmd_fn) try: logging.debug("Fetching newly created SSH key from node '%s'.", node_name) pub_key = ssh.ReadRemoteSshPubKeys(new_pub_keyfile, node_name, cluster_name, ssh_port_map[node_name], False, # ask_key False) # key_check except: raise errors.SshUpdateError("Could not fetch key of node %s" " (UUID %s)" % (node_name, node_uuid)) if potential_master_candidate: ssh.RemovePublicKey(node_uuid, key_file=ganeti_pub_keys_file) ssh.AddPublicKey(node_uuid, pub_key, key_file=ganeti_pub_keys_file) node_info = SshAddNodeInfo(name=node_name, uuid=node_uuid, to_authorized_keys=master_candidate, to_public_keys=potential_master_candidate, get_public_keys=True) node_keys_to_add.append(node_info) if node_keys_to_add: node_errors = AddNodeSshKeyBulk( node_keys_to_add, potential_master_candidates, pub_key_file=ganeti_pub_keys_file, ssconf_store=ssconf_store, noded_cert_file=noded_cert_file, run_cmd_fn=run_cmd_fn) if node_errors: all_node_errors = all_node_errors + node_errors # Renewing the master node's key # Preserve the old keys for now old_master_keys_by_uuid = _GetOldMasterKeys(master_node_uuid, ganeti_pub_keys_file) # Generate a new master key with a suffix, don't touch the old one for now logging.debug("Generate new ssh key of master.") _GenerateNodeSshKey(master_node_uuid, master_node_name, ssh_port_map, new_key_type, new_key_bits, pub_key_file=ganeti_pub_keys_file, ssconf_store=ssconf_store, noded_cert_file=noded_cert_file, run_cmd_fn=run_cmd_fn, suffix=constants.SSHS_MASTER_SUFFIX) # Read newly created master key new_master_key_dict = _GetNewMasterKey(root_keyfiles, master_node_uuid) # Replace master key in the master nodes' public key file ssh.RemovePublicKey(master_node_uuid, key_file=ganeti_pub_keys_file) for pub_key in new_master_key_dict[master_node_uuid]: ssh.AddPublicKey(master_node_uuid, pub_key, key_file=ganeti_pub_keys_file) # Add new master key to all node's public and authorized keys logging.debug("Add new master key to all nodes.") node_errors = AddNodeSshKey( master_node_uuid, master_node_name, potential_master_candidates, to_authorized_keys=True, to_public_keys=True, get_public_keys=False, pub_key_file=ganeti_pub_keys_file, ssconf_store=ssconf_store, noded_cert_file=noded_cert_file, run_cmd_fn=run_cmd_fn) if node_errors: all_node_errors = all_node_errors + node_errors # Remove the old key file and rename the new key to the non-temporary filename _ReplaceMasterKeyOnMaster(root_keyfiles) # Remove old key from authorized keys (auth_key_file, _) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=False, dircheck=False) ssh.RemoveAuthorizedKeys(auth_key_file, old_master_keys_by_uuid[master_node_uuid]) # Remove the old key from all node's authorized keys file logging.debug("Remove the old master key from all nodes.") node_errors = RemoveNodeSshKey( master_node_uuid, master_node_name, master_candidate_uuids, potential_master_candidates, keys_to_remove=old_master_keys_by_uuid, from_authorized_keys=True, from_public_keys=False, clear_authorized_keys=False, clear_public_keys=False) if node_errors: all_node_errors = all_node_errors + node_errors return all_node_errors def GetBlockDevSizes(devices): """Return the size of the given block devices @type devices: list @param devices: list of block device nodes to query @rtype: dict @return: dictionary of all block devices under /dev (key). The value is their size in MiB. {'/dev/disk/by-uuid/123456-12321231-312312-312': 124} """ DEV_PREFIX = "/dev/" blockdevs = {} for devpath in devices: if not utils.IsBelowDir(DEV_PREFIX, devpath): continue try: st = os.stat(devpath) except EnvironmentError as err: logging.warning("Error stat()'ing device %s: %s", devpath, str(err)) continue if stat.S_ISBLK(st.st_mode): result = utils.RunCmd(["blockdev", "--getsize64", devpath]) if result.failed: # We don't want to fail, just do not list this device as available logging.warning("Cannot get size for block device %s", devpath) continue size = int(result.stdout) // (1024 * 1024) blockdevs[devpath] = size return blockdevs def GetVolumeList(vg_names): """Compute list of logical volumes and their size. @type vg_names: list @param vg_names: the volume groups whose LVs we should list, or empty for all volume groups @rtype: dict @return: dictionary of all partions (key) with value being a tuple of their size (in MiB), inactive and online status:: {'xenvg/test1': ('20.06', True, True)} in case of errors, a string is returned with the error details. """ lvs = {} sep = "|" if not vg_names: vg_names = [] result = utils.RunCmd(["lvs", "--noheadings", "--units=m", "--nosuffix", "--separator=%s" % sep, "-ovg_name,lv_name,lv_size,lv_attr"] + vg_names) if result.failed: _Fail("Failed to list logical volumes, lvs output: %s", result.output) for line in result.stdout.splitlines(): line = line.strip() match = _LVSLINE_REGEX.match(line) if not match: logging.error("Invalid line returned from lvs output: '%s'", line) continue vg_name, name, size, attr = match.groups() inactive = attr[4] == "-" online = attr[5] == "o" virtual = attr[0] == "v" if virtual: # we don't want to report such volumes as existing, since they # don't really hold data continue lvs[vg_name + "/" + name] = (size, inactive, online) return lvs def ListVolumeGroups(): """List the volume groups and their size. @rtype: dict @return: dictionary with keys volume name and values the size of the volume """ return utils.ListVolumeGroups() def NodeVolumes(): """List all volumes on this node. @rtype: list @return: A list of dictionaries, each having four keys: - name: the logical volume name, - size: the size of the logical volume - dev: the physical device on which the LV lives - vg: the volume group to which it belongs In case of errors, we return an empty list and log the error. Note that since a logical volume can live on multiple physical volumes, the resulting list might include a logical volume multiple times. """ result = utils.RunCmd(["lvs", "--noheadings", "--units=m", "--nosuffix", "--separator=|", "--options=lv_name,lv_size,devices,vg_name"]) if result.failed: _Fail("Failed to list logical volumes, lvs output: %s", result.output) def parse_dev(dev): return dev.split("(")[0] def handle_dev(dev): return [parse_dev(x) for x in dev.split(",")] def map_line(line): line = [v.strip() for v in line] return [{"name": line[0], "size": line[1], "dev": dev, "vg": line[3]} for dev in handle_dev(line[2])] all_devs = [] for line in result.stdout.splitlines(): if line.count("|") >= 3: all_devs.extend(map_line(line.split("|"))) else: logging.warning("Strange line in the output from lvs: '%s'", line) return all_devs def BridgesExist(bridges_list): """Check if a list of bridges exist on the current node. @rtype: boolean @return: C{True} if all of them exist, C{False} otherwise """ missing = [] for bridge in bridges_list: if not utils.BridgeExists(bridge): missing.append(bridge) if missing: _Fail("Missing bridges %s", utils.CommaJoin(missing)) def GetInstanceListForHypervisor(hname, hvparams=None, get_hv_fn=hypervisor.GetHypervisor): """Provides a list of instances of the given hypervisor. @type hname: string @param hname: name of the hypervisor @type hvparams: dict of strings @param hvparams: hypervisor parameters for the given hypervisor @type get_hv_fn: function @param get_hv_fn: function that returns a hypervisor for the given hypervisor name; optional parameter to increase testability @rtype: list @return: a list of all running instances on the current node - instance1.example.com - instance2.example.com """ try: return get_hv_fn(hname).ListInstances(hvparams=hvparams) except errors.HypervisorError as err: _Fail("Error enumerating instances (hypervisor %s): %s", hname, err, exc=True) def GetInstanceList(hypervisor_list, all_hvparams=None, get_hv_fn=hypervisor.GetHypervisor): """Provides a list of instances. @type hypervisor_list: list @param hypervisor_list: the list of hypervisors to query information @type all_hvparams: dict of dict of strings @param all_hvparams: a dictionary mapping hypervisor types to respective cluster-wide hypervisor parameters @type get_hv_fn: function @param get_hv_fn: function that returns a hypervisor for the given hypervisor name; optional parameter to increase testability @rtype: list @return: a list of all running instances on the current node - instance1.example.com - instance2.example.com """ results = [] for hname in hypervisor_list: hvparams = all_hvparams[hname] results.extend(GetInstanceListForHypervisor(hname, hvparams=hvparams, get_hv_fn=get_hv_fn)) return results def GetInstanceInfo(instance, hname, hvparams=None): """Gives back the information about an instance as a dictionary. @type instance: string @param instance: the instance name @type hname: string @param hname: the hypervisor type of the instance @type hvparams: dict of strings @param hvparams: the instance's hvparams @rtype: dict @return: dictionary with the following keys: - memory: memory size of instance (int) - state: state of instance (HvInstanceState) - time: cpu time of instance (float) - vcpus: the number of vcpus (int) """ output = {} iinfo = hypervisor.GetHypervisor(hname).GetInstanceInfo(instance, hvparams=hvparams) if iinfo is not None: output["memory"] = iinfo[2] output["vcpus"] = iinfo[3] output["state"] = iinfo[4] output["time"] = iinfo[5] return output def GetInstanceMigratable(instance): """Computes whether an instance can be migrated. @type instance: L{objects.Instance} @param instance: object representing the instance to be checked. @rtype: tuple @return: tuple of (result, description) where: - result: whether the instance can be migrated or not - description: a description of the issue, if relevant """ hyper = hypervisor.GetHypervisor(instance.hypervisor) iname = instance.name if iname not in hyper.ListInstances(hvparams=instance.hvparams): _Fail("Instance %s is not running", iname) for idx in range(len(instance.disks_info)): link_name = _GetBlockDevSymlinkPath(iname, idx) if not os.path.islink(link_name): logging.warning("Instance %s is missing symlink %s for disk %d", iname, link_name, idx) def GetAllInstancesInfo(hypervisor_list, all_hvparams): """Gather data about all instances. This is the equivalent of L{GetInstanceInfo}, except that it computes data for all instances at once, thus being faster if one needs data about more than one instance. @type hypervisor_list: list @param hypervisor_list: list of hypervisors to query for instance data @type all_hvparams: dict of dict of strings @param all_hvparams: mapping of hypervisor names to hvparams @rtype: dict @return: dictionary of instance: data, with data having the following keys: - memory: memory size of instance (int) - state: xen state of instance (string) - time: cpu time of instance (float) - vcpus: the number of vcpus """ output = {} for hname in hypervisor_list: hvparams = all_hvparams[hname] iinfo = hypervisor.GetHypervisor(hname).GetAllInstancesInfo(hvparams) if iinfo: for name, _, memory, vcpus, state, times in iinfo: value = { "memory": memory, "vcpus": vcpus, "state": state, "time": times, } if name in output: # we only check static parameters, like memory and vcpus, # and not state and time which can change between the # invocations of the different hypervisors for key in "memory", "vcpus": if value[key] != output[name][key]: _Fail("Instance %s is running twice" " with different parameters", name) output[name] = value return output def GetInstanceConsoleInfo(instance_param_dict, get_hv_fn=hypervisor.GetHypervisor): """Gather data about the console access of a set of instances of this node. This function assumes that the caller already knows which instances are on this node, by calling a function such as L{GetAllInstancesInfo} or L{GetInstanceList}. For every instance, a large amount of configuration data needs to be provided to the hypervisor interface in order to receive the console information. Whether this could or should be cut down can be discussed. The information is provided in a dictionary indexed by instance name, allowing any number of instance queries to be done. @type instance_param_dict: dict of string to tuple of dictionaries, where the dictionaries represent: L{objects.Instance}, L{objects.Node}, L{objects.NodeGroup}, HvParams, BeParams @param instance_param_dict: mapping of instance name to parameters necessary for console information retrieval @rtype: dict @return: dictionary of instance: data, with data having the following keys: - instance: instance name - kind: console kind - message: used with kind == CONS_MESSAGE, indicates console to be unavailable, supplies error message - host: host to connect to - port: port to use - user: user for login - command: the command, broken into parts as an array - display: unknown, potentially unused? """ output = {} for inst_name in instance_param_dict: instance = instance_param_dict[inst_name]["instance"] pnode = instance_param_dict[inst_name]["node"] group = instance_param_dict[inst_name]["group"] hvparams = instance_param_dict[inst_name]["hvParams"] beparams = instance_param_dict[inst_name]["beParams"] instance = objects.Instance.FromDict(instance) pnode = objects.Node.FromDict(pnode) group = objects.NodeGroup.FromDict(group) h = get_hv_fn(instance.hypervisor) output[inst_name] = h.GetInstanceConsole(instance, pnode, group, hvparams, beparams).ToDict() return output def _InstanceLogName(kind, os_name, instance, component): """Compute the OS log filename for a given instance and operation. The instance name and os name are passed in as strings since not all operations have these as part of an instance object. @type kind: string @param kind: the operation type (e.g. add, import, etc.) @type os_name: string @param os_name: the os name @type instance: string @param instance: the name of the instance being imported/added/etc. @type component: string or None @param component: the name of the component of the instance being transferred """ # TODO: Use tempfile.mkstemp to create unique filename if component: assert "/" not in component c_msg = "-%s" % component else: c_msg = "" base = ("%s-%s-%s%s-%s.log" % (kind, os_name, instance, c_msg, utils.TimestampForFilename())) return utils.PathJoin(pathutils.LOG_OS_DIR, base) def InstanceOsAdd(instance, reinstall, debug): """Add an OS to an instance. @type instance: L{objects.Instance} @param instance: Instance whose OS is to be installed @type reinstall: boolean @param reinstall: whether this is an instance reinstall @type debug: integer @param debug: debug level, passed to the OS scripts @rtype: None """ inst_os = OSFromDisk(instance.os) create_env = OSEnvironment(instance, inst_os, debug) if reinstall: create_env["INSTANCE_REINSTALL"] = "1" logfile = _InstanceLogName("add", instance.os, instance.name, None) result = utils.RunCmd([inst_os.create_script], env=create_env, cwd=inst_os.path, output=logfile, reset_env=True) if result.failed: logging.error("os create command '%s' returned error: %s, logfile: %s," " output: %s", result.cmd, result.fail_reason, logfile, result.output) lines = [utils.SafeEncode(val) for val in utils.TailFile(logfile, lines=20)] _Fail("OS create script failed (%s), last lines in the" " log file:\n%s", result.fail_reason, "\n".join(lines), log=False) def RunRenameInstance(instance, old_name, debug): """Run the OS rename script for an instance. @type instance: L{objects.Instance} @param instance: Instance whose OS is to be installed @type old_name: string @param old_name: previous instance name @type debug: integer @param debug: debug level, passed to the OS scripts @rtype: boolean @return: the success of the operation """ inst_os = OSFromDisk(instance.os) rename_env = OSEnvironment(instance, inst_os, debug) rename_env["OLD_INSTANCE_NAME"] = old_name logfile = _InstanceLogName("rename", instance.os, "%s-%s" % (old_name, instance.name), None) result = utils.RunCmd([inst_os.rename_script], env=rename_env, cwd=inst_os.path, output=logfile, reset_env=True) if result.failed: logging.error("os create command '%s' returned error: %s output: %s", result.cmd, result.fail_reason, result.output) lines = [utils.SafeEncode(val) for val in utils.TailFile(logfile, lines=20)] _Fail("OS rename script failed (%s), last lines in the" " log file:\n%s", result.fail_reason, "\n".join(lines), log=False) def _GetBlockDevSymlinkPath(instance_name, idx, _dir=None): """Returns symlink path for block device. """ if _dir is None: _dir = pathutils.DISK_LINKS_DIR return utils.PathJoin(_dir, ("%s%s%s" % (instance_name, constants.DISK_SEPARATOR, idx))) def _SymlinkBlockDev(instance_name, device_path, idx): """Set up symlinks to a instance's block device. This is an auxiliary function run when an instance is start (on the primary node) or when an instance is migrated (on the target node). @param instance_name: the name of the target instance @param device_path: path of the physical block device, on the node @param idx: the disk index @return: absolute path to the disk's symlink """ # In case we have only a userspace access URI, device_path is None if not device_path: return None link_name = _GetBlockDevSymlinkPath(instance_name, idx) try: os.symlink(device_path, link_name) except OSError as err: if err.errno == errno.EEXIST: if (not os.path.islink(link_name) or os.readlink(link_name) != device_path): os.remove(link_name) os.symlink(device_path, link_name) else: raise return link_name def _RemoveBlockDevLinks(instance_name, disks): """Remove the block device symlinks belonging to the given instance. """ for idx, _ in enumerate(disks): link_name = _GetBlockDevSymlinkPath(instance_name, idx) if os.path.islink(link_name): try: os.remove(link_name) except OSError: logging.exception("Can't remove symlink '%s'", link_name) def _CalculateDeviceURI(instance, disk, device): """Get the URI for the device. @type instance: L{objects.Instance} @param instance: the instance which disk belongs to @type disk: L{objects.Disk} @param disk: the target disk object @type device: L{bdev.BlockDev} @param device: the corresponding BlockDevice @rtype: string @return: the device uri if any else None """ access_mode = disk.params.get(constants.LDP_ACCESS, constants.DISK_KERNELSPACE) if access_mode == constants.DISK_USERSPACE: # This can raise errors.BlockDeviceError return device.GetUserspaceAccessUri(instance.hypervisor) else: return None def _GatherAndLinkBlockDevs(instance): """Set up an instance's block device(s). This is run on the primary node at instance startup. The block devices must be already assembled. @type instance: L{objects.Instance} @param instance: the instance whose disks we should assemble @rtype: list @return: list of (disk_object, link_name, drive_uri) """ block_devices = [] for idx, disk in enumerate(instance.disks_info): device = _RecursiveFindBD(disk) if device is None: raise errors.BlockDeviceError("Block device '%s' is not set up." % str(disk)) device.Open() try: link_name = _SymlinkBlockDev(instance.name, device.dev_path, idx) except OSError as e: raise errors.BlockDeviceError("Cannot create block device symlink: %s" % e.strerror) uri = _CalculateDeviceURI(instance, disk, device) block_devices.append((disk, link_name, uri)) return block_devices def _IsInstanceUserDown(instance_info): return instance_info and \ "state" in instance_info and \ hv_base.HvInstanceState.IsShutdown(instance_info["state"]) def _GetInstanceInfo(instance): """Helper function L{GetInstanceInfo}""" return GetInstanceInfo(instance.name, instance.hypervisor, hvparams=instance.hvparams) def StartInstance(instance, startup_paused, reason, store_reason=True): """Start an instance. @type instance: L{objects.Instance} @param instance: the instance object @type startup_paused: bool @param instance: pause instance at startup? @type reason: list of reasons @param reason: the reason trail for this startup @type store_reason: boolean @param store_reason: whether to store the shutdown reason trail on file @rtype: None """ try: instance_info = _GetInstanceInfo(instance) hyper = hypervisor.GetHypervisor(instance.hypervisor) if instance_info and not _IsInstanceUserDown(instance_info): logging.info("Instance '%s' already running, not starting", instance.name) if hyper.VerifyInstance(instance): return logging.info("Instance '%s' hypervisor config out of date. Restoring.", instance.name) block_devices = _GatherAndLinkBlockDevs(instance) hyper.RestoreInstance(instance, block_devices) return block_devices = _GatherAndLinkBlockDevs(instance) hyper.StartInstance(instance, block_devices, startup_paused) if store_reason: _StoreInstReasonTrail(instance.name, reason) except errors.BlockDeviceError as err: _Fail("Block device error: %s", err, exc=True) except errors.HypervisorError as err: _RemoveBlockDevLinks(instance.name, instance.disks_info) _Fail("Hypervisor error: %s", err, exc=True) def InstanceShutdown(instance, timeout, reason, store_reason=True): """Shut an instance down. @note: this functions uses polling with a hardcoded timeout. @type instance: L{objects.Instance} @param instance: the instance object @type timeout: integer @param timeout: maximum timeout for soft shutdown @type reason: list of reasons @param reason: the reason trail for this shutdown @type store_reason: boolean @param store_reason: whether to store the shutdown reason trail on file @rtype: None """ hyper = hypervisor.GetHypervisor(instance.hypervisor) if not _GetInstanceInfo(instance): logging.info("Instance '%s' not running, doing nothing", instance.name) else: class _TryShutdown(object): def __init__(self): self.tried_once = False def __call__(self): try: hyper.StopInstance(instance, retry=self.tried_once, timeout=timeout) if store_reason: _StoreInstReasonTrail(instance.name, reason) except errors.HypervisorError as err: # if the instance does no longer exist, consider this success and go # to cleanup, otherwise fail without retrying if _GetInstanceInfo(instance): _Fail("Failed to stop instance '%s': %s", instance.name, err) return # TODO: Cleanup hypervisor implementations to prevent them from failing # silently. We could easily decide if we want to retry or not by using # HypervisorSoftError()/HypervisorHardError() self.tried_once = True if _GetInstanceInfo(instance): raise utils.RetryAgain() try: utils.Retry(_TryShutdown(), 5, timeout) except utils.RetryTimeout: # the shutdown did not succeed logging.error("Shutdown of '%s' unsuccessful, forcing", instance.name) try: hyper.StopInstance(instance, force=True) except errors.HypervisorError as err: # only raise an error if the instance still exists, otherwise # the error could simply be "instance ... unknown"! if _GetInstanceInfo(instance): _Fail("Failed to force stop instance '%s': %s", instance.name, err) time.sleep(1) if _GetInstanceInfo(instance): _Fail("Could not shutdown instance '%s' even by destroy", instance.name) try: hyper.CleanupInstance(instance.name) except errors.HypervisorError as err: logging.warning("Failed to execute post-shutdown cleanup step: %s", err) _RemoveBlockDevLinks(instance.name, instance.disks_info) def InstanceReboot(instance, reboot_type, shutdown_timeout, reason): """Reboot an instance. @type instance: L{objects.Instance} @param instance: the instance object to reboot @type reboot_type: str @param reboot_type: the type of reboot, one the following constants: - L{constants.INSTANCE_REBOOT_SOFT}: only reboot the instance OS, do not recreate the VM - L{constants.INSTANCE_REBOOT_HARD}: tear down and restart the VM (at the hypervisor level) - the other reboot type (L{constants.INSTANCE_REBOOT_FULL}) is not accepted here, since that mode is handled differently, in cmdlib, and translates into full stop and start of the instance (instead of a call_instance_reboot RPC) @type shutdown_timeout: integer @param shutdown_timeout: maximum timeout for soft shutdown @type reason: list of reasons @param reason: the reason trail for this reboot @rtype: None """ # TODO: this is inconsistent with 'StartInstance' and 'InstanceShutdown' # because those functions simply 'return' on error whereas this one # raises an exception with '_Fail' if not _GetInstanceInfo(instance): _Fail("Cannot reboot instance '%s' that is not running", instance.name) hyper = hypervisor.GetHypervisor(instance.hypervisor) if reboot_type == constants.INSTANCE_REBOOT_SOFT: try: hyper.RebootInstance(instance) except errors.HypervisorError as err: _Fail("Failed to soft reboot instance '%s': %s", instance.name, err) elif reboot_type == constants.INSTANCE_REBOOT_HARD: try: InstanceShutdown(instance, shutdown_timeout, reason, store_reason=False) StartInstance(instance, False, reason, store_reason=False) _StoreInstReasonTrail(instance.name, reason) except errors.HypervisorError as err: _Fail("Failed to hard reboot instance '%s': %s", instance.name, err) else: _Fail("Invalid reboot_type received: '%s'", reboot_type) def InstanceBalloonMemory(instance, memory): """Resize an instance's memory. @type instance: L{objects.Instance} @param instance: the instance object @type memory: int @param memory: new memory amount in MB @rtype: None """ hyper = hypervisor.GetHypervisor(instance.hypervisor) running = hyper.ListInstances(hvparams=instance.hvparams) if instance.name not in running: logging.info("Instance %s is not running, cannot balloon", instance.name) return try: hyper.BalloonInstanceMemory(instance, memory) except errors.HypervisorError as err: _Fail("Failed to balloon instance memory: %s", err, exc=True) def MigrationInfo(instance): """Gather information about an instance to be migrated. @type instance: L{objects.Instance} @param instance: the instance definition """ hyper = hypervisor.GetHypervisor(instance.hypervisor) try: info = hyper.MigrationInfo(instance) except errors.HypervisorError as err: _Fail("Failed to fetch migration information: %s", err, exc=True) return info def AcceptInstance(instance, info, target): """Prepare the node to accept an instance. @type instance: L{objects.Instance} @param instance: the instance definition @type info: string/data (opaque) @param info: migration information, from the source node @type target: string @param target: target host (usually ip), on this node """ hyper = hypervisor.GetHypervisor(instance.hypervisor) try: hyper.AcceptInstance(instance, info, target) except errors.HypervisorError as err: _Fail("Failed to accept instance: %s", err, exc=True) def FinalizeMigrationDst(instance, info, success): """Finalize any preparation to accept an instance. @type instance: L{objects.Instance} @param instance: the instance definition @type info: string/data (opaque) @param info: migration information, from the source node @type success: boolean @param success: whether the migration was a success or a failure """ hyper = hypervisor.GetHypervisor(instance.hypervisor) try: hyper.FinalizeMigrationDst(instance, info, success) except errors.HypervisorError as err: _Fail("Failed to finalize migration on the target node: %s", err, exc=True) def MigrateInstance(cluster_name, instance, target, live): """Migrates an instance to another node. @type cluster_name: string @param cluster_name: name of the cluster @type instance: L{objects.Instance} @param instance: the instance definition @type target: string @param target: the target node name @type live: boolean @param live: whether the migration should be done live or not (the interpretation of this parameter is left to the hypervisor) @raise RPCFail: if migration fails for some reason """ hyper = hypervisor.GetHypervisor(instance.hypervisor) try: hyper.MigrateInstance(cluster_name, instance, target, live) except errors.HypervisorError as err: _Fail("Failed to migrate instance: %s", err, exc=True) def FinalizeMigrationSource(instance, success, live): """Finalize the instance migration on the source node. @type instance: L{objects.Instance} @param instance: the instance definition of the migrated instance @type success: bool @param success: whether the migration succeeded or not @type live: bool @param live: whether the user requested a live migration or not @raise RPCFail: If the execution fails for some reason """ hyper = hypervisor.GetHypervisor(instance.hypervisor) try: hyper.FinalizeMigrationSource(instance, success, live) except Exception as err: # pylint: disable=W0703 _Fail("Failed to finalize the migration on the source node: %s", err, exc=True) def GetMigrationStatus(instance): """Get the migration status @type instance: L{objects.Instance} @param instance: the instance that is being migrated @rtype: L{objects.MigrationStatus} @return: the status of the current migration (one of L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional progress info that can be retrieved from the hypervisor @raise RPCFail: If the migration status cannot be retrieved """ hyper = hypervisor.GetHypervisor(instance.hypervisor) try: return hyper.GetMigrationStatus(instance) except Exception as err: # pylint: disable=W0703 _Fail("Failed to get migration status: %s", err, exc=True) def ResizeDisk(instance, disk, new_size): """Notify the hypervisor about a disk change. @type disk: L{objects.Disk} @param disk: the disk to be changed @type new_size: int @param new_size: the new size in bytes @raise errors.HotplugError: if disk resize is not supported """ hyper = hypervisor.GetHypervisor(instance.hypervisor) return hyper.ResizeDisk(instance, disk, new_size) def HotplugDevice(instance, action, dev_type, device, extra, seq): """Hotplug a device Hotplug is currently supported only for KVM Hypervisor. @type instance: L{objects.Instance} @param instance: the instance to which we hotplug a device @type action: string @param action: the hotplug action to perform @type dev_type: string @param dev_type: the device type to hotplug @type device: either L{objects.NIC} or L{objects.Disk} @param device: the device object to hotplug @type extra: tuple @param extra: extra info used for disk hotplug (disk link, drive uri) @type seq: int @param seq: the index of the device from master perspective @raise RPCFail: in case instance does not have KVM hypervisor """ hyper = hypervisor.GetHypervisor(instance.hypervisor) try: hyper.VerifyHotplugSupport(instance, action, dev_type) except errors.HotplugError as err: _Fail("Hotplug is not supported: %s", err) if action == constants.HOTPLUG_ACTION_ADD: fn = hyper.HotAddDevice elif action == constants.HOTPLUG_ACTION_REMOVE: fn = hyper.HotDelDevice elif action == constants.HOTPLUG_ACTION_MODIFY: fn = hyper.HotModDevice else: assert action in constants.HOTPLUG_ALL_ACTIONS return fn(instance, dev_type, device, extra, seq) def HotplugSupported(instance): """Checks if hotplug is generally supported. """ hyper = hypervisor.GetHypervisor(instance.hypervisor) try: hyper.HotplugSupported(instance) except errors.HotplugError as err: _Fail("Hotplug is not supported: %s", err) def ModifyInstanceMetadata(metadata): """Sends instance data to the metadata daemon. Uses the Luxi transport layer to communicate with the metadata daemon configuration server. It starts the metadata daemon if it is not running. The daemon must be enabled during at configuration time. @type metadata: dict @param metadata: instance metadata obtained by calling L{objects.Instance.ToDict} on an instance object """ if not constants.ENABLE_METAD: raise errors.ProgrammerError("The metadata deamon is disabled, yet" " ModifyInstanceMetadata has been called") if not utils.IsDaemonAlive(constants.METAD): result = utils.RunCmd([pathutils.DAEMON_UTIL, "start", constants.METAD]) if result.failed: raise errors.HypervisorError("Failed to start metadata daemon") with contextlib.closing(metad.Client()) as client: client.UpdateConfig(metadata) def BlockdevCreate(disk, size, owner, on_primary, info, excl_stor): """Creates a block device for an instance. @type disk: L{objects.Disk} @param disk: the object describing the disk we should create @type size: int @param size: the size of the physical underlying device, in MiB @type owner: str @param owner: the name of the instance for which disk is created, used for device cache data @type on_primary: boolean @param on_primary: indicates if it is the primary node or not @type info: string @param info: string that will be sent to the physical device creation, used for example to set (LVM) tags on LVs @type excl_stor: boolean @param excl_stor: Whether exclusive_storage is active @return: the new unique_id of the device (this can sometime be computed only after creation), or None. On secondary nodes, it's not required to return anything. """ # TODO: remove the obsolete "size" argument # pylint: disable=W0613 clist = [] if disk.children: for child in disk.children: try: crdev = _RecursiveAssembleBD(child, owner, on_primary) except errors.BlockDeviceError as err: _Fail("Can't assemble device %s: %s", child, err) if on_primary or disk.AssembleOnSecondary(): # we need the children open in case the device itself has to # be assembled try: # pylint: disable=E1103 crdev.Open() except errors.BlockDeviceError as err: _Fail("Can't make child '%s' read-write: %s", child, err) clist.append(crdev) try: device = bdev.Create(disk, clist, excl_stor) except errors.BlockDeviceError as err: _Fail("Can't create block device: %s", err) if on_primary or disk.AssembleOnSecondary(): try: device.Assemble() except errors.BlockDeviceError as err: _Fail("Can't assemble device after creation, unusual event: %s", err) if on_primary or disk.OpenOnSecondary(): try: device.Open(force=True) except errors.BlockDeviceError as err: _Fail("Can't make device r/w after creation, unusual event: %s", err) DevCacheManager.UpdateCache(device.dev_path, owner, on_primary, disk.iv_name) device.SetInfo(info) return device.unique_id def _DumpDevice(source_path, target_path, offset, size, truncate): """This function images/wipes the device using a local file. @type source_path: string @param source_path: path of the image or data source (e.g., "/dev/zero") @type target_path: string @param target_path: path of the device to image/wipe @type offset: int @param offset: offset in MiB in the output file @type size: int @param size: maximum size in MiB to write (data source might be smaller) @type truncate: bool @param truncate: whether the file should be truncated @return: None @raise RPCFail: in case of failure """ # Internal sizes are always in Mebibytes; if the following "dd" command # should use a different block size the offset and size given to this # function must be adjusted accordingly before being passed to "dd". block_size = constants.DD_BLOCK_SIZE cmd = [constants.DD_CMD, "if=%s" % source_path, "seek=%d" % offset, "bs=%s" % block_size, "oflag=direct", "of=%s" % target_path, "count=%d" % size] if not truncate: cmd.append("conv=notrunc") result = utils.RunCmd(cmd) if result.failed: _Fail("Dump command '%s' exited with error: %s; output: %s", result.cmd, result.fail_reason, result.output) def _DownloadAndDumpDevice(source_url, target_path, size): """This function images a device using a downloaded image file. @type source_url: string @param source_url: URL of image to dump to disk @type target_path: string @param target_path: path of the device to image @type size: int @param size: maximum size in MiB to write (data source might be smaller) @rtype: NoneType @return: None @raise RPCFail: in case of download or write failures """ class DDParams(object): def __init__(self, current_size, total_size): self.current_size = current_size self.total_size = total_size self.image_size_error = False def dd_write(ddparams, out): if ddparams.current_size < ddparams.total_size: ddparams.current_size += len(out) target_file.write(out) else: ddparams.image_size_error = True return -1 target_file = open(target_path, "r+") ddparams = DDParams(0, 1024 * 1024 * size) curl = pycurl.Curl() curl.setopt(pycurl.VERBOSE, True) curl.setopt(pycurl.NOSIGNAL, True) curl.setopt(pycurl.USERAGENT, http.HTTP_GANETI_VERSION) curl.setopt(pycurl.URL, source_url) curl.setopt(pycurl.WRITEFUNCTION, lambda out: dd_write(ddparams, out)) try: curl.perform() except pycurl.error: if ddparams.image_size_error: _Fail("Disk image larger than the disk") else: raise target_file.close() def BlockdevConvert(src_disk, target_disk): """Copies data from source block device to target. This function gets the export and import commands from the source and target devices respectively, and then concatenates them to a single command using a pipe ("|"). Finally, executes the unified command that will transfer the data between the devices during the disk template conversion operation. @type src_disk: L{objects.Disk} @param src_disk: the disk object we want to copy from @type target_disk: L{objects.Disk} @param target_disk: the disk object we want to copy to @rtype: NoneType @return: None @raise RPCFail: in case of failure """ src_dev = _RecursiveFindBD(src_disk) if src_dev is None: _Fail("Cannot copy from device '%s': device not found", src_disk.uuid) dest_dev = _RecursiveFindBD(target_disk) if dest_dev is None: _Fail("Cannot copy to device '%s': device not found", target_disk.uuid) src_cmd = src_dev.Export() dest_cmd = dest_dev.Import() command = "%s | %s" % (utils.ShellQuoteArgs(src_cmd), utils.ShellQuoteArgs(dest_cmd)) result = utils.RunCmd(command) if result.failed: _Fail("Disk conversion command '%s' exited with error: %s; output: %s", result.cmd, result.fail_reason, result.output) def BlockdevWipe(disk, offset, size): """Wipes a block device. @type disk: L{objects.Disk} @param disk: the disk object we want to wipe @type offset: int @param offset: The offset in MiB in the file @type size: int @param size: The size in MiB to write """ try: rdev = _RecursiveFindBD(disk) except errors.BlockDeviceError: rdev = None if not rdev: _Fail("Cannot wipe device %s: device not found", disk.iv_name) if offset < 0: _Fail("Negative offset") if size < 0: _Fail("Negative size") if offset > rdev.size: _Fail("Wipe offset is bigger than device size") if (offset + size) > rdev.size: _Fail("Wipe offset and size are bigger than device size") _DumpDevice("/dev/zero", rdev.dev_path, offset, size, True) def BlockdevImage(disk, image, size): """Images a block device either by dumping a local file or downloading a URL. @type disk: L{objects.Disk} @param disk: the disk object we want to image @type image: string @param image: file path to the disk image be dumped @type size: int @param size: The size in MiB to write @rtype: NoneType @return: None @raise RPCFail: in case of failure """ if not (utils.IsUrl(image) or os.path.exists(image)): _Fail("Image '%s' not found", image) try: rdev = _RecursiveFindBD(disk) except errors.BlockDeviceError: rdev = None if not rdev: _Fail("Cannot image device %s: device not found", disk.iv_name) if size < 0: _Fail("Negative size") if size > rdev.size: _Fail("Image size is bigger than device size") if utils.IsUrl(image): _DownloadAndDumpDevice(image, rdev.dev_path, size) else: _DumpDevice(image, rdev.dev_path, 0, size, False) def BlockdevPauseResumeSync(disks, pause): """Pause or resume the sync of the block device. @type disks: list of L{objects.Disk} @param disks: the disks object we want to pause/resume @type pause: bool @param pause: Wheater to pause or resume """ success = [] for disk in disks: try: rdev = _RecursiveFindBD(disk) except errors.BlockDeviceError: rdev = None if not rdev: success.append((False, ("Cannot change sync for device %s:" " device not found" % disk.iv_name))) continue result = rdev.PauseResumeSync(pause) if result: success.append((result, None)) else: if pause: msg = "Pause" else: msg = "Resume" success.append((result, "%s for device %s failed" % (msg, disk.iv_name))) return success def BlockdevRemove(disk): """Remove a block device. @note: This is intended to be called recursively. @type disk: L{objects.Disk} @param disk: the disk object we should remove @rtype: boolean @return: the success of the operation """ msgs = [] try: rdev = _RecursiveFindBD(disk) except errors.BlockDeviceError as err: # probably can't attach logging.info("Can't attach to device %s in remove", disk) rdev = None if rdev is not None: r_path = rdev.dev_path def _TryRemove(): try: rdev.Remove() return [] except errors.BlockDeviceError as err: return [str(err)] msgs.extend(utils.SimpleRetry([], _TryRemove, constants.DISK_REMOVE_RETRY_INTERVAL, constants.DISK_REMOVE_RETRY_TIMEOUT)) if not msgs: DevCacheManager.RemoveCache(r_path) if disk.children: for child in disk.children: try: BlockdevRemove(child) except RPCFail as err: msgs.append(str(err)) if msgs: _Fail("; ".join(msgs)) def _RecursiveAssembleBD(disk, owner, as_primary): """Activate a block device for an instance. This is run on the primary and secondary nodes for an instance. @note: this function is called recursively. @type disk: L{objects.Disk} @param disk: the disk we try to assemble @type owner: str @param owner: the name of the instance which owns the disk @type as_primary: boolean @param as_primary: if we should make the block device read/write @return: the assembled device or None (in case no device was assembled) @raise errors.BlockDeviceError: in case there is an error during the activation of the children or the device itself """ children = [] if disk.children: mcn = disk.ChildrenNeeded() if mcn == -1: mcn = 0 # max number of Nones allowed else: mcn = len(disk.children) - mcn # max number of Nones for chld_disk in disk.children: try: cdev = _RecursiveAssembleBD(chld_disk, owner, as_primary) except errors.BlockDeviceError as err: if children.count(None) >= mcn: raise cdev = None logging.error("Error in child activation (but continuing): %s", str(err)) children.append(cdev) if as_primary or disk.AssembleOnSecondary(): r_dev = bdev.Assemble(disk, children) result = r_dev if as_primary or disk.OpenOnSecondary(): r_dev.Open() DevCacheManager.UpdateCache(r_dev.dev_path, owner, as_primary, disk.iv_name) else: result = True return result def BlockdevAssemble(disk, instance, as_primary, idx): """Activate a block device for an instance. This is a wrapper over _RecursiveAssembleBD. @rtype: str or boolean @return: a tuple with the C{/dev/...} path and the created symlink for primary nodes, and (C{True}, C{True}) for secondary nodes """ try: result = _RecursiveAssembleBD(disk, instance.name, as_primary) if isinstance(result, BlockDev): # pylint: disable=E1103 dev_path = result.dev_path link_name = None uri = None if as_primary: link_name = _SymlinkBlockDev(instance.name, dev_path, idx) uri = _CalculateDeviceURI(instance, disk, result) elif result: return result, result else: _Fail("Unexpected result from _RecursiveAssembleBD") except errors.BlockDeviceError as err: _Fail("Error while assembling disk: %s", err, exc=True) except OSError as err: _Fail("Error while symlinking disk: %s", err, exc=True) return dev_path, link_name, uri def BlockdevShutdown(disk): """Shut down a block device. First, if the device is assembled (Attach() is successful), then the device is shutdown. Then the children of the device are shutdown. This function is called recursively. Note that we don't cache the children or such, as oppossed to assemble, shutdown of different devices doesn't require that the upper device was active. @type disk: L{objects.Disk} @param disk: the description of the disk we should shutdown @rtype: None """ msgs = [] r_dev = _RecursiveFindBD(disk) if r_dev is not None: r_path = r_dev.dev_path try: r_dev.Shutdown() DevCacheManager.RemoveCache(r_path) except errors.BlockDeviceError as err: msgs.append(str(err)) if disk.children: for child in disk.children: try: BlockdevShutdown(child) except RPCFail as err: msgs.append(str(err)) if msgs: _Fail("; ".join(msgs)) def BlockdevAddchildren(parent_cdev, new_cdevs): """Extend a mirrored block device. @type parent_cdev: L{objects.Disk} @param parent_cdev: the disk to which we should add children @type new_cdevs: list of L{objects.Disk} @param new_cdevs: the list of children which we should add @rtype: None """ parent_bdev = _RecursiveFindBD(parent_cdev) if parent_bdev is None: _Fail("Can't find parent device '%s' in add children", parent_cdev) new_bdevs = [_RecursiveFindBD(disk) for disk in new_cdevs] if new_bdevs.count(None) > 0: _Fail("Can't find new device(s) to add: %s:%s", new_bdevs, new_cdevs) parent_bdev.AddChildren(new_bdevs) def BlockdevRemovechildren(parent_cdev, new_cdevs): """Shrink a mirrored block device. @type parent_cdev: L{objects.Disk} @param parent_cdev: the disk from which we should remove children @type new_cdevs: list of L{objects.Disk} @param new_cdevs: the list of children which we should remove @rtype: None """ parent_bdev = _RecursiveFindBD(parent_cdev) if parent_bdev is None: _Fail("Can't find parent device '%s' in remove children", parent_cdev) devs = [] for disk in new_cdevs: rpath = disk.StaticDevPath() if rpath is None: bd = _RecursiveFindBD(disk) if bd is None: _Fail("Can't find device %s while removing children", disk) else: devs.append(bd.dev_path) else: if not utils.IsNormAbsPath(rpath): _Fail("Strange path returned from StaticDevPath: '%s'", rpath) devs.append(rpath) parent_bdev.RemoveChildren(devs) def BlockdevGetmirrorstatus(disks): """Get the mirroring status of a list of devices. @type disks: list of L{objects.Disk} @param disks: the list of disks which we should query @rtype: disk @return: List of L{objects.BlockDevStatus}, one for each disk @raise errors.BlockDeviceError: if any of the disks cannot be found """ stats = [] for dsk in disks: rbd = _RecursiveFindBD(dsk) if rbd is None: _Fail("Can't find device %s", dsk) stats.append(rbd.CombinedSyncStatus()) return stats def BlockdevGetmirrorstatusMulti(disks): """Get the mirroring status of a list of devices. @type disks: list of L{objects.Disk} @param disks: the list of disks which we should query @rtype: disk @return: List of tuples, (bool, status), one for each disk; bool denotes success/failure, status is L{objects.BlockDevStatus} on success, string otherwise """ result = [] lvs_cache = None is_plain_disk = compat.any([_CheckForPlainDisk(d) for d in disks]) if is_plain_disk: lvs_cache = bdev.LogicalVolume.GetLvGlobalInfo() for disk in disks: try: rbd = _RecursiveFindBD(disk, lvs_cache=lvs_cache) if rbd is None: result.append((False, "Can't find device %s" % disk)) continue status = rbd.CombinedSyncStatus() except errors.BlockDeviceError as err: logging.exception("Error while getting disk status") result.append((False, str(err))) else: result.append((True, status)) assert len(disks) == len(result) return result def _CheckForPlainDisk(disk): """Check within a disk and its children if there is a plain disk type. @type disk: L{objects.Disk} @param disk: the disk we are checking @rtype: bool @return: whether or not there is a plain disk type """ if disk.dev_type == constants.DT_PLAIN: return True if disk.children: return compat.any([_CheckForPlainDisk(d) for d in disk.children]) return False def _RecursiveFindBD(disk, lvs_cache=None): """Check if a device is activated. If so, return information about the real device. @type disk: L{objects.Disk} @param disk: the disk object we need to find @return: None if the device can't be found, otherwise the device instance """ children = [] if disk.children: for chdisk in disk.children: children.append(_RecursiveFindBD(chdisk, lvs_cache=lvs_cache)) return bdev.FindDevice(disk, children, lvs_cache=lvs_cache) def _OpenRealBD(disk): """Opens the underlying block device of a disk. @type disk: L{objects.Disk} @param disk: the disk object we want to open """ real_disk = _RecursiveFindBD(disk) if real_disk is None: _Fail("Block device '%s' is not set up", disk) real_disk.Open() return real_disk def BlockdevFind(disk): """Check if a device is activated. If it is, return information about the real device. @type disk: L{objects.Disk} @param disk: the disk to find @rtype: None or objects.BlockDevStatus @return: None if the disk cannot be found, otherwise a the current information """ try: rbd = _RecursiveFindBD(disk) except errors.BlockDeviceError as err: _Fail("Failed to find device: %s", err, exc=True) if rbd is None: return None return rbd.GetSyncStatus() def BlockdevGetdimensions(disks): """Computes the size of the given disks. If a disk is not found, returns None instead. @type disks: list of L{objects.Disk} @param disks: the list of disk to compute the size for @rtype: list @return: list with elements None if the disk cannot be found, otherwise the pair (size, spindles), where spindles is None if the device doesn't support that """ result = [] for cf in disks: try: rbd = _RecursiveFindBD(cf) except errors.BlockDeviceError: result.append(None) continue if rbd is None: result.append(None) else: result.append(rbd.GetActualDimensions()) return result def UploadFile(file_name, data, mode, uid, gid, atime, mtime): """Write a file to the filesystem. This allows the master to overwrite(!) a file. It will only perform the operation if the file belongs to a list of configuration files. @type file_name: str @param file_name: the target file name @type data: str @param data: the new contents of the file @type mode: int @param mode: the mode to give the file (can be None) @type uid: string @param uid: the owner of the file @type gid: string @param gid: the group of the file @type atime: float @param atime: the atime to set on the file (can be None) @type mtime: float @param mtime: the mtime to set on the file (can be None) @rtype: None """ file_name = vcluster.LocalizeVirtualPath(file_name) if not os.path.isabs(file_name): _Fail("Filename passed to UploadFile is not absolute: '%s'", file_name) if file_name not in _ALLOWED_UPLOAD_FILES: _Fail("Filename passed to UploadFile not in allowed upload targets: '%s'", file_name) raw_data = _Decompress(data) if not (isinstance(uid, str) and isinstance(gid, str)): _Fail("Invalid username/groupname type") getents = runtime.GetEnts() uid = getents.LookupUser(uid) gid = getents.LookupGroup(gid) utils.SafeWriteFile(file_name, None, data=raw_data, mode=mode, uid=uid, gid=gid, atime=atime, mtime=mtime) def RunOob(oob_program, command, node, timeout): """Executes oob_program with given command on given node. @param oob_program: The path to the executable oob_program @param command: The command to invoke on oob_program @param node: The node given as an argument to the program @param timeout: Timeout after which we kill the oob program @return: stdout @raise RPCFail: If execution fails for some reason """ result = utils.RunCmd([oob_program, command, node], timeout=timeout) if result.failed: _Fail("'%s' failed with reason '%s'; output: %s", result.cmd, result.fail_reason, result.output) return result.stdout def _OSOndiskAPIVersion(os_dir): """Compute and return the API version of a given OS. This function will try to read the API version of the OS residing in the 'os_dir' directory. @type os_dir: str @param os_dir: the directory in which we should look for the OS @rtype: tuple @return: tuple (status, data) with status denoting the validity and data holding either the valid versions or an error message """ api_file = utils.PathJoin(os_dir, constants.OS_API_FILE) try: st = os.stat(api_file) except EnvironmentError as err: return False, ("Required file '%s' not found under path %s: %s" % (constants.OS_API_FILE, os_dir, utils.ErrnoOrStr(err))) if not stat.S_ISREG(stat.S_IFMT(st.st_mode)): return False, ("File '%s' in %s is not a regular file" % (constants.OS_API_FILE, os_dir)) try: api_versions = utils.ReadFile(api_file).splitlines() except EnvironmentError as err: return False, ("Error while reading the API version file at %s: %s" % (api_file, utils.ErrnoOrStr(err))) try: api_versions = [int(version.strip()) for version in api_versions] except (TypeError, ValueError) as err: return False, ("API version(s) can't be converted to integer: %s" % str(err)) return True, api_versions def DiagnoseOS(top_dirs=None): """Compute the validity for all OSes. @type top_dirs: list @param top_dirs: the list of directories in which to search (if not given defaults to L{pathutils.OS_SEARCH_PATH}) @rtype: list of L{objects.OS} @return: a list of tuples (name, path, status, diagnose, variants, parameters, api_version) for all (potential) OSes under all search paths, where: - name is the (potential) OS name - path is the full path to the OS - status True/False is the validity of the OS - diagnose is the error message for an invalid OS, otherwise empty - variants is a list of supported OS variants, if any - parameters is a list of (name, help) parameters, if any - api_version is a list of support OS API versions """ if top_dirs is None: top_dirs = pathutils.OS_SEARCH_PATH result = [] for dir_name in top_dirs: if os.path.isdir(dir_name): try: f_names = utils.ListVisibleFiles(dir_name) except EnvironmentError as err: logging.exception("Can't list the OS directory %s: %s", dir_name, err) break for name in f_names: os_path = utils.PathJoin(dir_name, name) status, os_inst = _TryOSFromDisk(name, base_dir=dir_name) if status: diagnose = "" variants = os_inst.supported_variants parameters = os_inst.supported_parameters api_versions = os_inst.api_versions trusted = False if os_inst.create_script_untrusted else True else: diagnose = os_inst variants = parameters = api_versions = [] trusted = True result.append((name, os_path, status, diagnose, variants, parameters, api_versions, trusted)) return result def _TryOSFromDisk(name, base_dir=None): """Create an OS instance from disk. This function will return an OS instance if the given name is a valid OS name. @type base_dir: string @keyword base_dir: Base directory containing OS installations. Defaults to a search in all the OS_SEARCH_PATH dirs. @rtype: tuple @return: success and either the OS instance if we find a valid one, or error message """ if base_dir is None: os_dir = utils.FindFile(name, pathutils.OS_SEARCH_PATH, os.path.isdir) else: os_dir = utils.FindFile(name, [base_dir], os.path.isdir) if os_dir is None: return False, "Directory for OS %s not found in search path" % name status, api_versions = _OSOndiskAPIVersion(os_dir) if not status: # push the error up return status, api_versions if not constants.OS_API_VERSIONS.intersection(api_versions): return False, ("API version mismatch for path '%s': found %s, want %s." % (os_dir, api_versions, constants.OS_API_VERSIONS)) # OS Files dictionary, we will populate it with the absolute path # names; if the value is True, then it is a required file, otherwise # an optional one os_files = dict.fromkeys(constants.OS_SCRIPTS, True) os_files[constants.OS_SCRIPT_CREATE] = False os_files[constants.OS_SCRIPT_CREATE_UNTRUSTED] = False if max(api_versions) >= constants.OS_API_V15: os_files[constants.OS_VARIANTS_FILE] = False if max(api_versions) >= constants.OS_API_V20: os_files[constants.OS_PARAMETERS_FILE] = True else: del os_files[constants.OS_SCRIPT_VERIFY] for (filename, required) in list(os_files.items()): os_files[filename] = utils.PathJoin(os_dir, filename) try: st = os.stat(os_files[filename]) except EnvironmentError as err: if err.errno == errno.ENOENT and not required: del os_files[filename] continue return False, ("File '%s' under path '%s' is missing (%s)" % (filename, os_dir, utils.ErrnoOrStr(err))) if not stat.S_ISREG(stat.S_IFMT(st.st_mode)): return False, ("File '%s' under path '%s' is not a regular file" % (filename, os_dir)) if filename in constants.OS_SCRIPTS: if stat.S_IMODE(st.st_mode) & stat.S_IXUSR != stat.S_IXUSR: return False, ("File '%s' under path '%s' is not executable" % (filename, os_dir)) if not constants.OS_SCRIPT_CREATE in os_files and \ not constants.OS_SCRIPT_CREATE_UNTRUSTED in os_files: return False, ("A create script (trusted or untrusted) under path '%s'" " must exist" % os_dir) create_script = os_files.get(constants.OS_SCRIPT_CREATE, None) create_script_untrusted = os_files.get(constants.OS_SCRIPT_CREATE_UNTRUSTED, None) variants = [] if constants.OS_VARIANTS_FILE in os_files: variants_file = os_files[constants.OS_VARIANTS_FILE] try: variants = \ utils.FilterEmptyLinesAndComments(utils.ReadFile(variants_file)) except EnvironmentError as err: # we accept missing files, but not other errors if err.errno != errno.ENOENT: return False, ("Error while reading the OS variants file at %s: %s" % (variants_file, utils.ErrnoOrStr(err))) parameters = [] if constants.OS_PARAMETERS_FILE in os_files: parameters_file = os_files[constants.OS_PARAMETERS_FILE] try: parameters = utils.ReadFile(parameters_file).splitlines() except EnvironmentError as err: return False, ("Error while reading the OS parameters file at %s: %s" % (parameters_file, utils.ErrnoOrStr(err))) parameters = [v.split(None, 1) for v in parameters] os_obj = objects.OS(name=name, path=os_dir, create_script=create_script, create_script_untrusted=create_script_untrusted, export_script=os_files[constants.OS_SCRIPT_EXPORT], import_script=os_files[constants.OS_SCRIPT_IMPORT], rename_script=os_files[constants.OS_SCRIPT_RENAME], verify_script=os_files.get(constants.OS_SCRIPT_VERIFY, None), supported_variants=variants, supported_parameters=parameters, api_versions=api_versions) return True, os_obj def OSFromDisk(name, base_dir=None): """Create an OS instance from disk. This function will return an OS instance if the given name is a valid OS name. Otherwise, it will raise an appropriate L{RPCFail} exception, detailing why this is not a valid OS. This is just a wrapper over L{_TryOSFromDisk}, which doesn't raise an exception but returns true/false status data. @type base_dir: string @keyword base_dir: Base directory containing OS installations. Defaults to a search in all the OS_SEARCH_PATH dirs. @rtype: L{objects.OS} @return: the OS instance if we find a valid one @raise RPCFail: if we don't find a valid OS """ name_only = objects.OS.GetName(name) status, payload = _TryOSFromDisk(name_only, base_dir) if not status: _Fail(payload) return payload def OSCoreEnv(os_name, inst_os, os_params, debug=0): """Calculate the basic environment for an os script. @type os_name: str @param os_name: full operating system name (including variant) @type inst_os: L{objects.OS} @param inst_os: operating system for which the environment is being built @type os_params: dict @param os_params: the OS parameters @type debug: integer @param debug: debug level (0 or 1, for OS Api 10) @rtype: dict @return: dict of environment variables @raise errors.BlockDeviceError: if the block device cannot be found """ result = {} api_version = \ max(constants.OS_API_VERSIONS.intersection(inst_os.api_versions)) result["OS_API_VERSION"] = "%d" % api_version result["OS_NAME"] = inst_os.name result["DEBUG_LEVEL"] = "%d" % debug # OS variants if api_version >= constants.OS_API_V15 and inst_os.supported_variants: variant = objects.OS.GetVariant(os_name) if not variant: variant = inst_os.supported_variants[0] else: variant = "" result["OS_VARIANT"] = variant # OS params for pname, pvalue in os_params.items(): result["OSP_%s" % pname.upper().replace("-", "_")] = pvalue # Set a default path otherwise programs called by OS scripts (or # even hooks called from OS scripts) might break, and we don't want # to have each script require setting a PATH variable result["PATH"] = constants.HOOKS_PATH return result def OSEnvironment(instance, inst_os, debug=0): """Calculate the environment for an os script. @type instance: L{objects.Instance} @param instance: target instance for the os script run @type inst_os: L{objects.OS} @param inst_os: operating system for which the environment is being built @type debug: integer @param debug: debug level (0 or 1, for OS Api 10) @rtype: dict @return: dict of environment variables @raise errors.BlockDeviceError: if the block device cannot be found """ result = OSCoreEnv(instance.os, inst_os, objects.FillDict(instance.osparams, instance.osparams_private.Unprivate()), debug=debug) for attr in ["name", "os", "uuid", "ctime", "mtime", "primary_node"]: result["INSTANCE_%s" % attr.upper()] = str(getattr(instance, attr)) result["HYPERVISOR"] = instance.hypervisor result["DISK_COUNT"] = "%d" % len(instance.disks_info) result["NIC_COUNT"] = "%d" % len(instance.nics) result["INSTANCE_SECONDARY_NODES"] = \ ("%s" % " ".join(instance.secondary_nodes)) # Disks for idx, disk in enumerate(instance.disks_info): real_disk = _OpenRealBD(disk) uri = _CalculateDeviceURI(instance, disk, real_disk) result["DISK_%d_ACCESS" % idx] = disk.mode result["DISK_%d_UUID" % idx] = disk.uuid result["DISK_%d_SIZE" % idx] = str(disk.size) if real_disk.dev_path: result["DISK_%d_PATH" % idx] = real_disk.dev_path if uri: result["DISK_%d_URI" % idx] = uri if disk.name: result["DISK_%d_NAME" % idx] = disk.name if constants.HV_DISK_TYPE in instance.hvparams: result["DISK_%d_FRONTEND_TYPE" % idx] = \ instance.hvparams[constants.HV_DISK_TYPE] if disk.dev_type in constants.DTS_BLOCK: result["DISK_%d_BACKEND_TYPE" % idx] = "block" elif disk.dev_type in constants.DTS_FILEBASED: result["DISK_%d_BACKEND_TYPE" % idx] = \ "file:%s" % disk.logical_id[0] # NICs for idx, nic in enumerate(instance.nics): result["NIC_%d_MAC" % idx] = nic.mac result["NIC_%d_UUID" % idx] = nic.uuid if nic.name: result["NIC_%d_NAME" % idx] = nic.name if nic.ip: result["NIC_%d_IP" % idx] = nic.ip result["NIC_%d_MODE" % idx] = nic.nicparams[constants.NIC_MODE] if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: result["NIC_%d_BRIDGE" % idx] = nic.nicparams[constants.NIC_LINK] if nic.nicparams[constants.NIC_LINK]: result["NIC_%d_LINK" % idx] = nic.nicparams[constants.NIC_LINK] if nic.netinfo: nobj = objects.Network.FromDict(nic.netinfo) result.update(nobj.HooksDict("NIC_%d_" % idx)) if constants.HV_NIC_TYPE in instance.hvparams: result["NIC_%d_FRONTEND_TYPE" % idx] = \ instance.hvparams[constants.HV_NIC_TYPE] # HV/BE params for source, kind in [(instance.beparams, "BE"), (instance.hvparams, "HV")]: for key, value in source.items(): result["INSTANCE_%s_%s" % (kind, key)] = str(value) return result def DiagnoseExtStorage(top_dirs=None): """Compute the validity for all ExtStorage Providers. @type top_dirs: list @param top_dirs: the list of directories in which to search (if not given defaults to L{pathutils.ES_SEARCH_PATH}) @rtype: list of L{objects.ExtStorage} @return: a list of tuples (name, path, status, diagnose, parameters) for all (potential) ExtStorage Providers under all search paths, where: - name is the (potential) ExtStorage Provider - path is the full path to the ExtStorage Provider - status True/False is the validity of the ExtStorage Provider - diagnose is the error message for an invalid ExtStorage Provider, otherwise empty - parameters is a list of (name, help) parameters, if any """ if top_dirs is None: top_dirs = pathutils.ES_SEARCH_PATH result = [] for dir_name in top_dirs: if os.path.isdir(dir_name): try: f_names = utils.ListVisibleFiles(dir_name) except EnvironmentError as err: logging.exception("Can't list the ExtStorage directory %s: %s", dir_name, err) break for name in f_names: es_path = utils.PathJoin(dir_name, name) status, es_inst = extstorage.ExtStorageFromDisk(name, base_dir=dir_name) if status: diagnose = "" parameters = es_inst.supported_parameters else: diagnose = es_inst parameters = [] result.append((name, es_path, status, diagnose, parameters)) return result def BlockdevGrow(disk, amount, dryrun, backingstore, excl_stor): """Grow a stack of block devices. This function is called recursively, with the childrens being the first ones to resize. @type disk: L{objects.Disk} @param disk: the disk to be grown @type amount: integer @param amount: the amount (in mebibytes) to grow with @type dryrun: boolean @param dryrun: whether to execute the operation in simulation mode only, without actually increasing the size @param backingstore: whether to execute the operation on backing storage only, or on "logical" storage only; e.g. DRBD is logical storage, whereas LVM, file, RBD are backing storage @rtype: (status, result) @type excl_stor: boolean @param excl_stor: Whether exclusive_storage is active @return: a tuple with the status of the operation (True/False), and the errors message if status is False """ r_dev = _RecursiveFindBD(disk) if r_dev is None: _Fail("Cannot find block device %s", disk) try: r_dev.Grow(amount, dryrun, backingstore, excl_stor) except errors.BlockDeviceError as err: _Fail("Failed to grow block device: %s", err, exc=True) def BlockdevSnapshot(disk, snap_name, snap_size): """Create a snapshot copy of a block device. This function is called recursively, and the snapshot is actually created just for the leaf lvm backend device. @type disk: L{objects.Disk} @param disk: the disk to be snapshotted @type snap_name: string @param snap_name: the name of the snapshot @type snap_size: int @param snap_size: the size of the snapshot @rtype: string @return: snapshot disk ID as (vg, lv) """ def _DiskSnapshot(disk, snap_name=None, snap_size=None): r_dev = _RecursiveFindBD(disk) if r_dev is not None: return r_dev.Snapshot(snap_name=snap_name, snap_size=snap_size) else: _Fail("Cannot find block device %s", disk) if disk.SupportsSnapshots(): if disk.dev_type == constants.DT_DRBD8: if not disk.children: _Fail("DRBD device '%s' without backing storage cannot be snapshotted", disk.unique_id) return BlockdevSnapshot(disk.children[0], snap_name, snap_size) else: return _DiskSnapshot(disk, snap_name, snap_size) else: _Fail("Cannot snapshot block device '%s' of type '%s'", disk.logical_id, disk.dev_type) def BlockdevSetInfo(disk, info): """Sets 'metadata' information on block devices. This function sets 'info' metadata on block devices. Initial information is set at device creation; this function should be used for example after renames. @type disk: L{objects.Disk} @param disk: the disk to be grown @type info: string @param info: new 'info' metadata @rtype: (status, result) @return: a tuple with the status of the operation (True/False), and the errors message if status is False """ r_dev = _RecursiveFindBD(disk) if r_dev is None: _Fail("Cannot find block device %s", disk) try: r_dev.SetInfo(info) except errors.BlockDeviceError as err: _Fail("Failed to set information on block device: %s", err, exc=True) def FinalizeExport(instance, snap_disks): """Write out the export configuration information. @type instance: L{objects.Instance} @param instance: the instance which we export, used for saving configuration @type snap_disks: list of L{objects.Disk} @param snap_disks: list of snapshot block devices, which will be used to get the actual name of the dump file @rtype: None """ destdir = utils.PathJoin(pathutils.EXPORT_DIR, instance.name + ".new") finaldestdir = utils.PathJoin(pathutils.EXPORT_DIR, instance.name) disk_template = utils.GetDiskTemplate(snap_disks) config = objects.SerializableConfigParser() config.add_section(constants.INISECT_EXP) config.set(constants.INISECT_EXP, "version", str(constants.EXPORT_VERSION)) config.set(constants.INISECT_EXP, "timestamp", "%d" % int(time.time())) config.set(constants.INISECT_EXP, "source", instance.primary_node) config.set(constants.INISECT_EXP, "os", instance.os) config.set(constants.INISECT_EXP, "compression", "none") config.add_section(constants.INISECT_INS) config.set(constants.INISECT_INS, "name", instance.name) config.set(constants.INISECT_INS, "maxmem", "%d" % instance.beparams[constants.BE_MAXMEM]) config.set(constants.INISECT_INS, "minmem", "%d" % instance.beparams[constants.BE_MINMEM]) # "memory" is deprecated, but useful for exporting to old ganeti versions config.set(constants.INISECT_INS, "memory", "%d" % instance.beparams[constants.BE_MAXMEM]) config.set(constants.INISECT_INS, "vcpus", "%d" % instance.beparams[constants.BE_VCPUS]) config.set(constants.INISECT_INS, "disk_template", disk_template) config.set(constants.INISECT_INS, "hypervisor", instance.hypervisor) config.set(constants.INISECT_INS, "tags", " ".join(instance.GetTags())) nic_total = 0 for nic_count, nic in enumerate(instance.nics): nic_total += 1 config.set(constants.INISECT_INS, "nic%d_mac" % nic_count, "%s" % nic.mac) config.set(constants.INISECT_INS, "nic%d_ip" % nic_count, "%s" % nic.ip) config.set(constants.INISECT_INS, "nic%d_network" % nic_count, "%s" % nic.network) config.set(constants.INISECT_INS, "nic%d_name" % nic_count, "%s" % nic.name) for param in constants.NICS_PARAMETER_TYPES: config.set(constants.INISECT_INS, "nic%d_%s" % (nic_count, param), "%s" % nic.nicparams.get(param, None)) # TODO: redundant: on load can read nics until it doesn't exist config.set(constants.INISECT_INS, "nic_count", "%d" % nic_total) disk_total = 0 for disk_count, disk in enumerate(snap_disks): if disk: disk_total += 1 config.set(constants.INISECT_INS, "disk%d_ivname" % disk_count, ("%s" % disk.iv_name)) config.set(constants.INISECT_INS, "disk%d_dump" % disk_count, ("%s" % disk.uuid)) config.set(constants.INISECT_INS, "disk%d_size" % disk_count, ("%d" % disk.size)) config.set(constants.INISECT_INS, "disk%d_name" % disk_count, "%s" % disk.name) config.set(constants.INISECT_INS, "disk_count", "%d" % disk_total) # New-style hypervisor/backend parameters config.add_section(constants.INISECT_HYP) for name, value in instance.hvparams.items(): if name not in constants.HVC_GLOBALS: config.set(constants.INISECT_HYP, name, str(value)) config.add_section(constants.INISECT_BEP) for name, value in instance.beparams.items(): config.set(constants.INISECT_BEP, name, str(value)) config.add_section(constants.INISECT_OSP) for name, value in instance.osparams.items(): config.set(constants.INISECT_OSP, name, str(value)) config.add_section(constants.INISECT_OSP_PRIVATE) for name, value in instance.osparams_private.items(): config.set(constants.INISECT_OSP_PRIVATE, name, str(value.Get())) utils.WriteFile(utils.PathJoin(destdir, constants.EXPORT_CONF_FILE), data=config.Dumps()) shutil.rmtree(finaldestdir, ignore_errors=True) shutil.move(destdir, finaldestdir) def ExportInfo(dest): """Get export configuration information. @type dest: str @param dest: directory containing the export @rtype: L{objects.SerializableConfigParser} @return: a serializable config file containing the export info """ cff = utils.PathJoin(dest, constants.EXPORT_CONF_FILE) config = objects.SerializableConfigParser() config.read(cff) if (not config.has_section(constants.INISECT_EXP) or not config.has_section(constants.INISECT_INS)): _Fail("Export info file doesn't have the required fields") return config.Dumps() def ListExports(): """Return a list of exports currently available on this machine. @rtype: list @return: list of the exports """ if os.path.isdir(pathutils.EXPORT_DIR): return sorted(utils.ListVisibleFiles(pathutils.EXPORT_DIR)) else: _Fail("No exports directory") def RemoveExport(export): """Remove an existing export from the node. @type export: str @param export: the name of the export to remove @rtype: None """ target = utils.PathJoin(pathutils.EXPORT_DIR, export) try: shutil.rmtree(target) except EnvironmentError as err: _Fail("Error while removing the export: %s", err, exc=True) def BlockdevRename(devlist): """Rename a list of block devices. @type devlist: list of tuples @param devlist: list of tuples of the form (disk, new_unique_id); disk is an L{objects.Disk} object describing the current disk, and new unique_id is the name we rename it to @rtype: boolean @return: True if all renames succeeded, False otherwise """ msgs = [] result = True for disk, unique_id in devlist: dev = _RecursiveFindBD(disk) if dev is None: msgs.append("Can't find device %s in rename" % str(disk)) result = False continue try: old_rpath = dev.dev_path dev.Rename(unique_id) new_rpath = dev.dev_path if old_rpath != new_rpath: DevCacheManager.RemoveCache(old_rpath) # FIXME: we should add the new cache information here, like: # DevCacheManager.UpdateCache(new_rpath, owner, ...) # but we don't have the owner here - maybe parse from existing # cache? for now, we only lose lvm data when we rename, which # is less critical than DRBD or MD except errors.BlockDeviceError as err: msgs.append("Can't rename device '%s' to '%s': %s" % (dev, unique_id, err)) logging.exception("Can't rename device '%s' to '%s'", dev, unique_id) result = False if not result: _Fail("; ".join(msgs)) def _TransformFileStorageDir(fs_dir): """Checks whether given file_storage_dir is valid. Checks wheter the given fs_dir is within the cluster-wide default file_storage_dir or the shared_file_storage_dir, which are stored in SimpleStore. Only paths under those directories are allowed. @type fs_dir: str @param fs_dir: the path to check @return: the normalized path if valid, None otherwise """ filestorage.CheckFileStoragePath(fs_dir) return os.path.normpath(fs_dir) def CreateFileStorageDir(file_storage_dir): """Create file storage directory. @type file_storage_dir: str @param file_storage_dir: directory to create @rtype: tuple @return: tuple with first element a boolean indicating wheter dir creation was successful or not """ file_storage_dir = _TransformFileStorageDir(file_storage_dir) if os.path.exists(file_storage_dir): if not os.path.isdir(file_storage_dir): _Fail("Specified storage dir '%s' is not a directory", file_storage_dir) else: try: os.makedirs(file_storage_dir, 0o750) except OSError as err: _Fail("Cannot create file storage directory '%s': %s", file_storage_dir, err, exc=True) def RemoveFileStorageDir(file_storage_dir): """Remove file storage directory. Remove it only if it's empty. If not log an error and return. @type file_storage_dir: str @param file_storage_dir: the directory we should cleanup @rtype: tuple (success,) @return: tuple of one element, C{success}, denoting whether the operation was successful """ file_storage_dir = _TransformFileStorageDir(file_storage_dir) if os.path.exists(file_storage_dir): if not os.path.isdir(file_storage_dir): _Fail("Specified Storage directory '%s' is not a directory", file_storage_dir) # deletes dir only if empty, otherwise we want to fail the rpc call try: os.rmdir(file_storage_dir) except OSError as err: _Fail("Cannot remove file storage directory '%s': %s", file_storage_dir, err) def RenameFileStorageDir(old_file_storage_dir, new_file_storage_dir): """Rename the file storage directory. @type old_file_storage_dir: str @param old_file_storage_dir: the current path @type new_file_storage_dir: str @param new_file_storage_dir: the name we should rename to @rtype: tuple (success,) @return: tuple of one element, C{success}, denoting whether the operation was successful """ old_file_storage_dir = _TransformFileStorageDir(old_file_storage_dir) new_file_storage_dir = _TransformFileStorageDir(new_file_storage_dir) if not os.path.exists(new_file_storage_dir): if os.path.isdir(old_file_storage_dir): try: os.rename(old_file_storage_dir, new_file_storage_dir) except OSError as err: _Fail("Cannot rename '%s' to '%s': %s", old_file_storage_dir, new_file_storage_dir, err) else: _Fail("Specified storage dir '%s' is not a directory", old_file_storage_dir) else: if os.path.exists(old_file_storage_dir): _Fail("Cannot rename '%s' to '%s': both locations exist", old_file_storage_dir, new_file_storage_dir) def _EnsureJobQueueFile(file_name): """Checks whether the given filename is in the queue directory. @type file_name: str @param file_name: the file name we should check @rtype: None @raises RPCFail: if the file is not valid """ if not utils.IsBelowDir(pathutils.QUEUE_DIR, file_name): _Fail("Passed job queue file '%s' does not belong to" " the queue directory '%s'", file_name, pathutils.QUEUE_DIR) def JobQueueUpdate(file_name, content): """Updates a file in the queue directory. This is just a wrapper over L{utils.io.WriteFile}, with proper checking. @type file_name: str @param file_name: the job file name @type content: str @param content: the new job contents @rtype: boolean @return: the success of the operation """ file_name = vcluster.LocalizeVirtualPath(file_name) _EnsureJobQueueFile(file_name) getents = runtime.GetEnts() # Write and replace the file atomically utils.WriteFile(file_name, data=_Decompress(content), uid=getents.masterd_uid, gid=getents.daemons_gid, mode=constants.JOB_QUEUE_FILES_PERMS) def JobQueueRename(old, new): """Renames a job queue file. This is just a wrapper over os.rename with proper checking. @type old: str @param old: the old (actual) file name @type new: str @param new: the desired file name @rtype: tuple @return: the success of the operation and payload """ old = vcluster.LocalizeVirtualPath(old) new = vcluster.LocalizeVirtualPath(new) _EnsureJobQueueFile(old) _EnsureJobQueueFile(new) getents = runtime.GetEnts() utils.RenameFile(old, new, mkdir=True, mkdir_mode=0o750, dir_uid=getents.masterd_uid, dir_gid=getents.daemons_gid) def BlockdevClose(instance_name, disks): """Closes the given block devices. This means they will be switched to secondary mode (in case of DRBD). @param instance_name: if the argument is not empty, the symlinks of this instance will be removed @type disks: list of L{objects.Disk} @param disks: the list of disks to be closed @rtype: tuple (success, message) @return: a tuple of success and message, where success indicates the succes of the operation, and message which will contain the error details in case we failed """ bdevs = [] for cf in disks: rd = _RecursiveFindBD(cf) if rd is None: _Fail("Can't find device %s", cf) bdevs.append(rd) msg = [] for rd in bdevs: try: rd.Close() except errors.BlockDeviceError as err: msg.append(str(err)) if msg: _Fail("Can't close devices: %s", ",".join(msg)) else: if instance_name: _RemoveBlockDevLinks(instance_name, disks) def BlockdevOpen(instance_name, disks, exclusive): """Opens the given block devices. """ bdevs = [] for cf in disks: rd = _RecursiveFindBD(cf) if rd is None: _Fail("Can't find device %s", cf) bdevs.append(rd) msg = [] for idx, rd in enumerate(bdevs): try: rd.Open(exclusive=exclusive) _SymlinkBlockDev(instance_name, rd.dev_path, idx) except errors.BlockDeviceError as err: msg.append(str(err)) if msg: _Fail("Can't open devices: %s", ",".join(msg)) def ValidateHVParams(hvname, hvparams): """Validates the given hypervisor parameters. @type hvname: string @param hvname: the hypervisor name @type hvparams: dict @param hvparams: the hypervisor parameters to be validated @rtype: None """ try: hv_type = hypervisor.GetHypervisor(hvname) hv_type.ValidateParameters(hvparams) except errors.HypervisorError as err: _Fail(str(err), log=False) def _CheckOSPList(os_obj, parameters): """Check whether a list of parameters is supported by the OS. @type os_obj: L{objects.OS} @param os_obj: OS object to check @type parameters: list @param parameters: the list of parameters to check """ supported = [v[0] for v in os_obj.supported_parameters] delta = frozenset(parameters).difference(supported) if delta: _Fail("The following parameters are not supported" " by the OS %s: %s" % (os_obj.name, utils.CommaJoin(delta))) def _CheckOSVariant(os_obj, name): """Check whether an OS name conforms to the os variants specification. @type os_obj: L{objects.OS} @param os_obj: OS object to check @type name: string @param name: OS name passed by the user, to check for validity @rtype: NoneType @return: None @raise RPCFail: if OS variant is not valid """ variant = objects.OS.GetVariant(name) if not os_obj.supported_variants: if variant: _Fail("OS '%s' does not support variants ('%s' passed)" % (os_obj.name, variant)) else: return if not variant: _Fail("OS name '%s' must include a variant" % name) if variant not in os_obj.supported_variants: _Fail("OS '%s' does not support variant '%s'" % (os_obj.name, variant)) def ValidateOS(required, osname, checks, osparams, force_variant): """Validate the given OS parameters. @type required: boolean @param required: whether absence of the OS should translate into failure or not @type osname: string @param osname: the OS to be validated @type checks: list @param checks: list of the checks to run (currently only 'parameters') @type osparams: dict @param osparams: dictionary with OS parameters, some of which may be private. @rtype: boolean @return: True if the validation passed, or False if the OS was not found and L{required} was false """ if not constants.OS_VALIDATE_CALLS.issuperset(checks): _Fail("Unknown checks required for OS %s: %s", osname, set(checks).difference(constants.OS_VALIDATE_CALLS)) name_only = objects.OS.GetName(osname) status, tbv = _TryOSFromDisk(name_only, None) if not status: if required: _Fail(tbv) else: return False if not force_variant: _CheckOSVariant(tbv, osname) if max(tbv.api_versions) < constants.OS_API_V20: return True if constants.OS_VALIDATE_PARAMETERS in checks: _CheckOSPList(tbv, list(osparams)) validate_env = OSCoreEnv(osname, tbv, osparams) result = utils.RunCmd([tbv.verify_script] + checks, env=validate_env, cwd=tbv.path, reset_env=True) if result.failed: logging.error("os validate command '%s' returned error: %s output: %s", result.cmd, result.fail_reason, result.output) _Fail("OS validation script failed (%s), output: %s", result.fail_reason, result.output, log=False) return True def ExportOS(instance, override_env): """Creates a GZIPed tarball with an OS definition and environment. The archive contains a file with the environment variables needed by the OS scripts. @type instance: L{objects.Instance} @param instance: instance for which the OS definition is exported @type override_env: dict of string to string @param override_env: if supplied, it overrides the environment on a key-by-key basis that is part of the archive @rtype: string @return: filepath of the archive """ assert instance assert instance.os temp_dir = tempfile.mkdtemp() inst_os = OSFromDisk(instance.os) result = utils.RunCmd(["ln", "-s", inst_os.path, utils.PathJoin(temp_dir, "os")]) if result.failed: _Fail("Failed to copy OS package '%s' to '%s': %s, output '%s'", inst_os, temp_dir, result.fail_reason, result.output) env = OSEnvironment(instance, inst_os) env.update(override_env) with open(utils.PathJoin(temp_dir, "environment"), "w") as f: for var in env: f.write(var + "=" + env[var] + "\n") (fd, os_package) = tempfile.mkstemp(suffix=".tgz") os.close(fd) result = utils.RunCmd(["tar", "--dereference", "-czv", "-f", os_package, "-C", temp_dir, "."]) if result.failed: _Fail("Failed to create OS archive '%s': %s, output '%s'", os_package, result.fail_reason, result.output) result = utils.RunCmd(["rm", "-rf", temp_dir]) if result.failed: _Fail("Failed to remove copy of OS package '%s' in '%s': %s, output '%s'", inst_os, temp_dir, result.fail_reason, result.output) return os_package def DemoteFromMC(): """Demotes the current node from master candidate role. """ # try to ensure we're not the master by mistake master, myself = ssconf.GetMasterAndMyself() if master == myself: _Fail("ssconf status shows I'm the master node, will not demote") result = utils.RunCmd([pathutils.DAEMON_UTIL, "check", constants.MASTERD]) if not result.failed: _Fail("The master daemon is running, will not demote") try: if os.path.isfile(pathutils.CLUSTER_CONF_FILE): utils.CreateBackup(pathutils.CLUSTER_CONF_FILE) except EnvironmentError as err: if err.errno != errno.ENOENT: _Fail("Error while backing up cluster file: %s", err, exc=True) utils.RemoveFile(pathutils.CLUSTER_CONF_FILE) def _GetX509Filenames(cryptodir, name): """Returns the full paths for the private key and certificate. """ return (utils.PathJoin(cryptodir, name), utils.PathJoin(cryptodir, name, _X509_KEY_FILE), utils.PathJoin(cryptodir, name, _X509_CERT_FILE)) def CreateX509Certificate(validity, cryptodir=pathutils.CRYPTO_KEYS_DIR): """Creates a new X509 certificate for SSL/TLS. @type validity: int @param validity: Validity in seconds @rtype: tuple; (string, string) @return: Certificate name and public part """ serial_no = int(time.time()) (key_pem, cert_pem) = \ utils.GenerateSelfSignedX509Cert(netutils.Hostname.GetSysName(), min(validity, _MAX_SSL_CERT_VALIDITY), serial_no) cert_dir = tempfile.mkdtemp(dir=cryptodir, prefix="x509-%s-" % utils.TimestampForFilename()) try: name = os.path.basename(cert_dir) assert len(name) > 5 (_, key_file, cert_file) = _GetX509Filenames(cryptodir, name) utils.WriteFile(key_file, mode=0o400, data=key_pem) utils.WriteFile(cert_file, mode=0o400, data=cert_pem) # Never return private key as it shouldn't leave the node return (name, cert_pem) except Exception: shutil.rmtree(cert_dir, ignore_errors=True) raise def RemoveX509Certificate(name, cryptodir=pathutils.CRYPTO_KEYS_DIR): """Removes a X509 certificate. @type name: string @param name: Certificate name """ (cert_dir, key_file, cert_file) = _GetX509Filenames(cryptodir, name) utils.RemoveFile(key_file) utils.RemoveFile(cert_file) try: os.rmdir(cert_dir) except EnvironmentError as err: _Fail("Cannot remove certificate directory '%s': %s", cert_dir, err) def _GetImportExportIoCommand(instance, mode, ieio, ieargs): """Returns the command for the requested input/output. @type instance: L{objects.Instance} @param instance: The instance object @param mode: Import/export mode @param ieio: Input/output type @param ieargs: Input/output arguments """ assert mode in (constants.IEM_IMPORT, constants.IEM_EXPORT) env = None prefix = None suffix = None exp_size = None if ieio == constants.IEIO_FILE: (filename, ) = ieargs if not utils.IsNormAbsPath(filename): _Fail("Path '%s' is not normalized or absolute", filename) real_filename = os.path.realpath(filename) directory = os.path.dirname(real_filename) if not utils.IsBelowDir(pathutils.EXPORT_DIR, real_filename): _Fail("File '%s' is not under exports directory '%s': %s", filename, pathutils.EXPORT_DIR, real_filename) # Create directory utils.Makedirs(directory, mode=0o750) quoted_filename = utils.ShellQuote(filename) if mode == constants.IEM_IMPORT: suffix = "> %s" % quoted_filename elif mode == constants.IEM_EXPORT: suffix = "< %s" % quoted_filename # Retrieve file size try: st = os.stat(filename) except EnvironmentError as err: logging.error("Can't stat(2) %s: %s", filename, err) else: exp_size = utils.BytesToMebibyte(st.st_size) elif ieio == constants.IEIO_RAW_DISK: (disk, ) = ieargs real_disk = _OpenRealBD(disk) if mode == constants.IEM_IMPORT: suffix = "| %s" % utils.ShellQuoteArgs(real_disk.Import()) elif mode == constants.IEM_EXPORT: prefix = "%s |" % utils.ShellQuoteArgs(real_disk.Export()) exp_size = disk.size elif ieio == constants.IEIO_SCRIPT: (disk, disk_index, ) = ieargs assert isinstance(disk_index, int) inst_os = OSFromDisk(instance.os) env = OSEnvironment(instance, inst_os) if mode == constants.IEM_IMPORT: disk_path_var = "DISK_%d_PATH" % disk_index if disk_path_var in env: env["IMPORT_DEVICE"] = env[disk_path_var] env["IMPORT_DISK_PATH"] = env[disk_path_var] disk_uri_var = "DISK_%d_URI" % disk_index if disk_uri_var in env: env["IMPORT_DISK_URI"] = env[disk_uri_var] env["IMPORT_INDEX"] = str(disk_index) script = inst_os.import_script elif mode == constants.IEM_EXPORT: real_disk = _OpenRealBD(disk) if real_disk.dev_path: env["EXPORT_DEVICE"] = real_disk.dev_path env["EXPORT_DISK_PATH"] = real_disk.dev_path uri = _CalculateDeviceURI(instance, disk, real_disk) if uri: env["EXPORT_DISK_URI"] = uri env["EXPORT_INDEX"] = str(disk_index) script = inst_os.export_script # TODO: Pass special environment only to script script_cmd = utils.BuildShellCmd("( cd %s && %s; )", inst_os.path, script) if mode == constants.IEM_IMPORT: suffix = "| %s" % script_cmd elif mode == constants.IEM_EXPORT: prefix = "%s |" % script_cmd # Let script predict size exp_size = constants.IE_CUSTOM_SIZE else: _Fail("Invalid %s I/O mode %r", mode, ieio) return (env, prefix, suffix, exp_size) def _CreateImportExportStatusDir(prefix): """Creates status directory for import/export. """ return tempfile.mkdtemp(dir=pathutils.IMPORT_EXPORT_DIR, prefix=("%s-%s-" % (prefix, utils.TimestampForFilename()))) def StartImportExportDaemon(mode, opts, host, port, instance, component, ieio, ieioargs): """Starts an import or export daemon. @param mode: Import/output mode @type opts: L{objects.ImportExportOptions} @param opts: Daemon options @type host: string @param host: Remote host for export (None for import) @type port: int @param port: Remote port for export (None for import) @type instance: L{objects.Instance} @param instance: Instance object @type component: string @param component: which part of the instance is transferred now, e.g. 'disk/0' @param ieio: Input/output type @param ieioargs: Input/output arguments """ # Use Import/Export over socat. # # Export() gives a command that produces a flat stream. # Import() gives a command that reads a flat stream to a disk template. if mode == constants.IEM_IMPORT: prefix = "import" if not (host is None and port is None): _Fail("Can not specify host or port on import") elif mode == constants.IEM_EXPORT: prefix = "export" if host is None or port is None: _Fail("Host and port must be specified for an export") else: _Fail("Invalid mode %r", mode) if (opts.key_name is None) ^ (opts.ca_pem is None): _Fail("Cluster certificate can only be used for both key and CA") (cmd_env, cmd_prefix, cmd_suffix, exp_size) = \ _GetImportExportIoCommand(instance, mode, ieio, ieioargs) if opts.key_name is None: # Use server.pem key_path = pathutils.NODED_CERT_FILE cert_path = pathutils.NODED_CERT_FILE assert opts.ca_pem is None else: (_, key_path, cert_path) = _GetX509Filenames(pathutils.CRYPTO_KEYS_DIR, opts.key_name) assert opts.ca_pem is not None for i in [key_path, cert_path]: if not os.path.exists(i): _Fail("File '%s' does not exist" % i) status_dir = _CreateImportExportStatusDir("%s-%s" % (prefix, component)) try: status_file = utils.PathJoin(status_dir, _IES_STATUS_FILE) pid_file = utils.PathJoin(status_dir, _IES_PID_FILE) ca_file = utils.PathJoin(status_dir, _IES_CA_FILE) if opts.ca_pem is None: # Use server.pem ca = utils.ReadFile(pathutils.NODED_CERT_FILE) else: ca = opts.ca_pem # Write CA file utils.WriteFile(ca_file, data=ca, mode=0o400) cmd = [ pathutils.IMPORT_EXPORT_DAEMON, status_file, mode, "--key=%s" % key_path, "--cert=%s" % cert_path, "--ca=%s" % ca_file, ] if host: cmd.append("--host=%s" % host) if port: cmd.append("--port=%s" % port) if opts.ipv6: cmd.append("--ipv6") else: cmd.append("--ipv4") if opts.compress: cmd.append("--compress=%s" % opts.compress) if opts.magic: cmd.append("--magic=%s" % opts.magic) if exp_size is not None: cmd.append("--expected-size=%s" % exp_size) if cmd_prefix: cmd.append("--cmd-prefix=%s" % cmd_prefix) if cmd_suffix: cmd.append("--cmd-suffix=%s" % cmd_suffix) if mode == constants.IEM_EXPORT: # Retry connection a few times when connecting to remote peer cmd.append("--connect-retries=%s" % constants.RIE_CONNECT_RETRIES) cmd.append("--connect-timeout=%s" % constants.RIE_CONNECT_ATTEMPT_TIMEOUT) elif opts.connect_timeout is not None: assert mode == constants.IEM_IMPORT # Overall timeout for establishing connection while listening cmd.append("--connect-timeout=%s" % opts.connect_timeout) logfile = _InstanceLogName(prefix, instance.os, instance.name, component) # TODO: Once _InstanceLogName uses tempfile.mkstemp, StartDaemon has # support for receiving a file descriptor for output utils.StartDaemon(cmd, env=cmd_env, pidfile=pid_file, output=logfile) # The import/export name is simply the status directory name return os.path.basename(status_dir) except Exception: shutil.rmtree(status_dir, ignore_errors=True) raise def GetImportExportStatus(names): """Returns import/export daemon status. @type names: sequence @param names: List of names @rtype: List of dicts @return: Returns a list of the state of each named import/export or None if a status couldn't be read """ result = [] for name in names: status_file = utils.PathJoin(pathutils.IMPORT_EXPORT_DIR, name, _IES_STATUS_FILE) try: data = utils.ReadFile(status_file) except EnvironmentError as err: if err.errno != errno.ENOENT: raise data = None if not data: result.append(None) continue result.append(serializer.LoadJson(data)) return result def AbortImportExport(name): """Sends SIGTERM to a running import/export daemon. """ logging.info("Abort import/export %s", name) status_dir = utils.PathJoin(pathutils.IMPORT_EXPORT_DIR, name) pid = utils.ReadLockedPidFile(utils.PathJoin(status_dir, _IES_PID_FILE)) if pid: logging.info("Import/export %s is running with PID %s, sending SIGTERM", name, pid) utils.IgnoreProcessNotFound(os.kill, pid, signal.SIGTERM) def CleanupImportExport(name): """Cleanup after an import or export. If the import/export daemon is still running it's killed. Afterwards the whole status directory is removed. """ logging.info("Finalizing import/export %s", name) status_dir = utils.PathJoin(pathutils.IMPORT_EXPORT_DIR, name) pid = utils.ReadLockedPidFile(utils.PathJoin(status_dir, _IES_PID_FILE)) if pid: logging.info("Import/export %s is still running with PID %s", name, pid) utils.KillProcess(pid, waitpid=False) shutil.rmtree(status_dir, ignore_errors=True) def _FindDisks(disks): """Finds attached L{BlockDev}s for the given disks. @type disks: list of L{objects.Disk} @param disks: the disk objects we need to find @return: list of L{BlockDev} objects or C{None} if a given disk was not found or was no attached. """ bdevs = [] for disk in disks: rd = _RecursiveFindBD(disk) if rd is None: _Fail("Can't find device %s", disk) bdevs.append(rd) return bdevs def DrbdDisconnectNet(disks): """Disconnects the network on a list of drbd devices. """ bdevs = _FindDisks(disks) # disconnect disks for rd in bdevs: try: rd.DisconnectNet() except errors.BlockDeviceError as err: _Fail("Can't change network configuration to standalone mode: %s", err, exc=True) def DrbdAttachNet(disks, multimaster): """Attaches the network on a list of drbd devices. """ bdevs = _FindDisks(disks) # reconnect disks, switch to new master configuration and if # needed primary mode for rd in bdevs: try: rd.AttachNet(multimaster) except errors.BlockDeviceError as err: _Fail("Can't change network configuration: %s", err) # wait until the disks are connected; we need to retry the re-attach # if the device becomes standalone, as this might happen if the one # node disconnects and reconnects in a different mode before the # other node reconnects; in this case, one or both of the nodes will # decide it has wrong configuration and switch to standalone def _Attach(): all_connected = True for rd in bdevs: stats = rd.GetProcStatus() if multimaster: # In the multimaster case we have to wait explicitly until # the resource is Connected and UpToDate/UpToDate, because # we promote *both nodes* to primary directly afterwards. # Being in resync is not enough, since there is a race during which we # may promote a node with an Outdated disk to primary, effectively # tearing down the connection. all_connected = (all_connected and stats.is_connected and stats.is_disk_uptodate and stats.peer_disk_uptodate) else: all_connected = (all_connected and (stats.is_connected or stats.is_in_resync)) if stats.is_standalone: # peer had different config info and this node became # standalone, even though this should not happen with the # new staged way of changing disk configs try: rd.AttachNet(multimaster) except errors.BlockDeviceError as err: _Fail("Can't change network configuration: %s", err) if not all_connected: raise utils.RetryAgain() try: # Start with a delay of 100 miliseconds and go up to 5 seconds utils.Retry(_Attach, (0.1, 1.5, 5.0), 2 * 60) except utils.RetryTimeout: _Fail("Timeout in disk reconnecting") def DrbdWaitSync(disks): """Wait until DRBDs have synchronized. """ def _helper(rd): stats = rd.GetProcStatus() if not (stats.is_connected or stats.is_in_resync): raise utils.RetryAgain() return stats bdevs = _FindDisks(disks) min_resync = 100 alldone = True for rd in bdevs: try: # poll each second for 15 seconds stats = utils.Retry(_helper, 1, 15, args=[rd]) except utils.RetryTimeout: stats = rd.GetProcStatus() # last check if not (stats.is_connected or stats.is_in_resync): _Fail("DRBD device %s is not in sync: stats=%s", rd, stats) alldone = alldone and (not stats.is_in_resync) if stats.sync_percent is not None: min_resync = min(min_resync, stats.sync_percent) return (alldone, min_resync) def DrbdNeedsActivation(disks): """Checks which of the passed disks needs activation and returns their UUIDs. """ faulty_disks = [] is_plain_disk = compat.any([_CheckForPlainDisk(d) for d in disks]) lvs_cache = bdev.LogicalVolume.GetLvGlobalInfo() if is_plain_disk else None for disk in disks: rd = _RecursiveFindBD(disk, lvs_cache=lvs_cache) if rd is None: faulty_disks.append(disk) continue stats = rd.GetProcStatus() if stats.is_standalone or stats.is_diskless: faulty_disks.append(disk) return [disk.uuid for disk in faulty_disks] def GetDrbdUsermodeHelper(): """Returns DRBD usermode helper currently configured. """ try: return drbd.DRBD8.GetUsermodeHelper() except errors.BlockDeviceError as err: _Fail(str(err)) def PowercycleNode(hypervisor_type, hvparams=None): """Hard-powercycle the node. Because we need to return first, and schedule the powercycle in the background, we won't be able to report failures nicely. """ hyper = hypervisor.GetHypervisor(hypervisor_type) try: pid = os.fork() except OSError: # if we can't fork, we'll pretend that we're in the child process pid = 0 if pid > 0: return "Reboot scheduled in 5 seconds" # ensure the child is running on ram try: utils.Mlockall() except Exception: # pylint: disable=W0703 pass time.sleep(5) hyper.PowercycleNode(hvparams=hvparams) def _VerifyRestrictedCmdName(cmd): """Verifies a restricted command name. @type cmd: string @param cmd: Command name @rtype: tuple; (boolean, string or None) @return: The tuple's first element is the status; if C{False}, the second element is an error message string, otherwise it's C{None} """ if not cmd.strip(): return (False, "Missing command name") if os.path.basename(cmd) != cmd: return (False, "Invalid command name") if not constants.EXT_PLUGIN_MASK.match(cmd): return (False, "Command name contains forbidden characters") return (True, None) def _CommonRestrictedCmdCheck(path, owner): """Common checks for restricted command file system directories and files. @type path: string @param path: Path to check @param owner: C{None} or tuple containing UID and GID @rtype: tuple; (boolean, string or C{os.stat} result) @return: The tuple's first element is the status; if C{False}, the second element is an error message string, otherwise it's the result of C{os.stat} """ if owner is None: # Default to root as owner owner = (0, 0) try: st = os.stat(path) except EnvironmentError as err: return (False, "Can't stat(2) '%s': %s" % (path, err)) if stat.S_IMODE(st.st_mode) & (~_RCMD_MAX_MODE): return (False, "Permissions on '%s' are too permissive" % path) if (st.st_uid, st.st_gid) != owner: (owner_uid, owner_gid) = owner return (False, "'%s' is not owned by %s:%s" % (path, owner_uid, owner_gid)) return (True, st) def _VerifyRestrictedCmdDirectory(path, _owner=None): """Verifies restricted command directory. @type path: string @param path: Path to check @rtype: tuple; (boolean, string or None) @return: The tuple's first element is the status; if C{False}, the second element is an error message string, otherwise it's C{None} """ (status, value) = _CommonRestrictedCmdCheck(path, _owner) if not status: return (False, value) if not stat.S_ISDIR(value.st_mode): return (False, "Path '%s' is not a directory" % path) return (True, None) def _VerifyRestrictedCmd(path, cmd, _owner=None): """Verifies a whole restricted command and returns its executable filename. @type path: string @param path: Directory containing restricted commands @type cmd: string @param cmd: Command name @rtype: tuple; (boolean, string) @return: The tuple's first element is the status; if C{False}, the second element is an error message string, otherwise the second element is the absolute path to the executable """ executable = utils.PathJoin(path, cmd) (status, msg) = _CommonRestrictedCmdCheck(executable, _owner) if not status: return (False, msg) if not utils.IsExecutable(executable): return (False, "access(2) thinks '%s' can't be executed" % executable) return (True, executable) def _PrepareRestrictedCmd(path, cmd, _verify_dir=_VerifyRestrictedCmdDirectory, _verify_name=_VerifyRestrictedCmdName, _verify_cmd=_VerifyRestrictedCmd): """Performs a number of tests on a restricted command. @type path: string @param path: Directory containing restricted commands @type cmd: string @param cmd: Command name @return: Same as L{_VerifyRestrictedCmd} """ # Verify the directory first (status, msg) = _verify_dir(path) if status: # Check command if everything was alright (status, msg) = _verify_name(cmd) if not status: return (False, msg) # Check actual executable return _verify_cmd(path, cmd) def RunRestrictedCmd(cmd, _lock_timeout=_RCMD_LOCK_TIMEOUT, _lock_file=pathutils.RESTRICTED_COMMANDS_LOCK_FILE, _path=pathutils.RESTRICTED_COMMANDS_DIR, _sleep_fn=time.sleep, _prepare_fn=_PrepareRestrictedCmd, _runcmd_fn=utils.RunCmd, _enabled=constants.ENABLE_RESTRICTED_COMMANDS): """Executes a restricted command after performing strict tests. @type cmd: string @param cmd: Command name @rtype: string @return: Command output @raise RPCFail: In case of an error """ logging.info("Preparing to run restricted command '%s'", cmd) if not _enabled: _Fail("Restricted commands disabled at configure time") lock = None try: cmdresult = None try: lock = utils.FileLock.Open(_lock_file) lock.Exclusive(blocking=True, timeout=_lock_timeout) (status, value) = _prepare_fn(_path, cmd) if status: cmdresult = _runcmd_fn([value], env={}, reset_env=True, postfork_fn=lambda _: lock.Unlock()) else: logging.error(value) except Exception: # pylint: disable=W0703 # Keep original error in log logging.exception("Caught exception") if cmdresult is None: logging.info("Sleeping for %0.1f seconds before returning", _RCMD_INVALID_DELAY) _sleep_fn(_RCMD_INVALID_DELAY) # Do not include original error message in returned error _Fail("Executing command '%s' failed" % cmd) elif cmdresult.failed or cmdresult.fail_reason: _Fail("Restricted command '%s' failed: %s; output: %s", cmd, cmdresult.fail_reason, cmdresult.output) else: return cmdresult.output finally: if lock is not None: # Release lock at last lock.Close() lock = None def SetWatcherPause(until, _filename=pathutils.WATCHER_PAUSEFILE): """Creates or removes the watcher pause file. @type until: None or number @param until: Unix timestamp saying until when the watcher shouldn't run """ if until is None: logging.info("Received request to no longer pause watcher") utils.RemoveFile(_filename) else: logging.info("Received request to pause watcher until %s", until) if not ht.TNumber(until): _Fail("Duration must be numeric") utils.WriteFile(_filename, data="%d\n" % (until, ), mode=0o644) def ConfigureOVS(ovs_name, ovs_link): """Creates a OpenvSwitch on the node. This function sets up a OpenvSwitch on the node with given name nad connects it via a given eth device. @type ovs_name: string @param ovs_name: Name of the OpenvSwitch to create. @type ovs_link: None or string @param ovs_link: Ethernet device for outside connection (can be missing) """ # Initialize the OpenvSwitch result = utils.RunCmd(["ovs-vsctl", "--may-exist", "add-br", ovs_name]) if result.failed: _Fail("Failed to create openvswitch. Script return value: %s, output: '%s'" % (result.exit_code, result.output), log=True) # And connect it to a physical interface, if given if ovs_link: result = utils.RunCmd(["ovs-vsctl", "--may-exist", "add-port", ovs_name, ovs_link]) if result.failed: _Fail("Failed to connect openvswitch to interface %s. Script return" " value: %s, output: '%s'" % (ovs_link, result.exit_code, result.output), log=True) def GetFileInfo(file_path): """ Checks if a file exists and returns information related to it. Currently returned information: - file size: int, size in bytes @type file_path: string @param file_path: Name of file to examine. @rtype: tuple of bool, dict @return: Whether the file exists, and a dictionary of information about the file gathered by os.stat. """ try: stat_info = os.stat(file_path) values_dict = { constants.STAT_SIZE: stat_info.st_size, } return True, values_dict except IOError: return False, {} class HooksRunner(object): """Hook runner. This class is instantiated on the node side (ganeti-noded) and not on the master side. """ def __init__(self, hooks_base_dir=None): """Constructor for hooks runner. @type hooks_base_dir: str or None @param hooks_base_dir: if not None, this overrides the L{pathutils.HOOKS_BASE_DIR} (useful for unittests) """ if hooks_base_dir is None: hooks_base_dir = pathutils.HOOKS_BASE_DIR # yeah, _BASE_DIR is not valid for attributes, we use it like a # constant self._BASE_DIR = hooks_base_dir # pylint: disable=C0103 def RunLocalHooks(self, node_list, hpath, phase, env): """Check that the hooks will be run only locally and then run them. """ assert len(node_list) == 1 node = node_list[0] _, myself = ssconf.GetMasterAndMyself() assert node == myself results = self.RunHooks(hpath, phase, env) # Return values in the form expected by HooksMaster return {node: (None, False, results)} def RunHooks(self, hpath, phase, env): """Run the scripts in the hooks directory. @type hpath: str @param hpath: the path to the hooks directory which holds the scripts @type phase: str @param phase: either L{constants.HOOKS_PHASE_PRE} or L{constants.HOOKS_PHASE_POST} @type env: dict @param env: dictionary with the environment for the hook @rtype: list @return: list of 3-element tuples: - script path - script result, either L{constants.HKR_SUCCESS} or L{constants.HKR_FAIL} - output of the script @raise errors.ProgrammerError: for invalid input parameters """ if phase == constants.HOOKS_PHASE_PRE: suffix = "pre" elif phase == constants.HOOKS_PHASE_POST: suffix = "post" else: _Fail("Unknown hooks phase '%s'", phase) subdir = "%s-%s.d" % (hpath, suffix) dir_name = utils.PathJoin(self._BASE_DIR, subdir) results = [] if not os.path.isdir(dir_name): # for non-existing/non-dirs, we simply exit instead of logging a # warning at every operation return results runparts_results = utils.RunParts(dir_name, env=env, reset_env=True) for (relname, relstatus, runresult) in runparts_results: if relstatus == constants.RUNPARTS_SKIP: rrval = constants.HKR_SKIP output = "" elif relstatus == constants.RUNPARTS_ERR: rrval = constants.HKR_FAIL output = "Hook script execution error: %s" % runresult elif relstatus == constants.RUNPARTS_RUN: if runresult.failed: rrval = constants.HKR_FAIL else: rrval = constants.HKR_SUCCESS output = utils.SafeEncode(runresult.output.strip()) results.append(("%s/%s" % (subdir, relname), rrval, output)) return results class IAllocatorRunner(object): """IAllocator runner. This class is instantiated on the node side (ganeti-noded) and not on the master side. """ @staticmethod def Run(name, idata, ial_params): """Run an iallocator script. @type name: str @param name: the iallocator script name @type idata: str @param idata: the allocator input data @type ial_params: list @param ial_params: the iallocator parameters @rtype: tuple @return: two element tuple of: - status - either error message or stdout of allocator (for success) """ alloc_script = utils.FindFile(name, constants.IALLOCATOR_SEARCH_PATH, os.path.isfile) if alloc_script is None: _Fail("iallocator module '%s' not found in the search path", name) fd, fin_name = tempfile.mkstemp(prefix="ganeti-iallocator.") try: os.write(fd, idata.encode("utf-8")) os.close(fd) result = utils.RunCmd([alloc_script, fin_name] + ial_params) if result.failed: _Fail("iallocator module '%s' failed: %s, output '%s'", name, result.fail_reason, result.output) finally: os.unlink(fin_name) return result.stdout class DevCacheManager(object): """Simple class for managing a cache of block device information. """ _DEV_PREFIX = "/dev/" _ROOT_DIR = pathutils.BDEV_CACHE_DIR @classmethod def _ConvertPath(cls, dev_path): """Converts a /dev/name path to the cache file name. This replaces slashes with underscores and strips the /dev prefix. It then returns the full path to the cache file. @type dev_path: str @param dev_path: the C{/dev/} path name @rtype: str @return: the converted path name """ if dev_path.startswith(cls._DEV_PREFIX): dev_path = dev_path[len(cls._DEV_PREFIX):] dev_path = dev_path.replace("/", "_") fpath = utils.PathJoin(cls._ROOT_DIR, "bdev_%s" % dev_path) return fpath @classmethod def UpdateCache(cls, dev_path, owner, on_primary, iv_name): """Updates the cache information for a given device. @type dev_path: str @param dev_path: the pathname of the device @type owner: str @param owner: the owner (instance name) of the device @type on_primary: bool @param on_primary: whether this is the primary node nor not @type iv_name: str @param iv_name: the instance-visible name of the device, as in objects.Disk.iv_name @rtype: None """ if dev_path is None: logging.error("DevCacheManager.UpdateCache got a None dev_path") return fpath = cls._ConvertPath(dev_path) if on_primary: state = "primary" else: state = "secondary" if iv_name is None: iv_name = "not_visible" fdata = "%s %s %s\n" % (str(owner), state, iv_name) try: utils.WriteFile(fpath, data=fdata) except EnvironmentError as err: logging.exception("Can't update bdev cache for %s: %s", dev_path, err) @classmethod def RemoveCache(cls, dev_path): """Remove data for a dev_path. This is just a wrapper over L{utils.io.RemoveFile} with a converted path name and logging. @type dev_path: str @param dev_path: the pathname of the device @rtype: None """ if dev_path is None: logging.error("DevCacheManager.RemoveCache got a None dev_path") return fpath = cls._ConvertPath(dev_path) try: utils.RemoveFile(fpath) except EnvironmentError as err: logging.exception("Can't update bdev cache for %s: %s", dev_path, err) ganeti-3.1.0~rc2/lib/bootstrap.py000064400000000000000000001354601476477700300167520ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Functions to bootstrap a new cluster. """ import os import os.path import re import logging import time from ganeti.cmdlib import cluster import ganeti.rpc.node as rpc from ganeti import ssh from ganeti import utils from ganeti import errors from ganeti import config from ganeti import constants from ganeti import objects from ganeti import ssconf from ganeti import serializer from ganeti import hypervisor from ganeti.storage import drbd from ganeti.storage import filestorage from ganeti import netutils from ganeti import luxi from ganeti import jstore from ganeti import pathutils from ganeti import runtime from ganeti import vcluster # ec_id for InitConfig's temporary reservation manager _INITCONF_ECID = "initconfig-ecid" #: After how many seconds daemon must be responsive _DAEMON_READY_TIMEOUT = 10.0 def GenerateHmacKey(file_name): """Writes a new HMAC key. @type file_name: str @param file_name: Path to output file """ utils.WriteFile(file_name, data="%s\n" % utils.GenerateSecret(), mode=0o400, backup=True) # pylint: disable=R0913 def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_spice_cert, new_confd_hmac_key, new_cds, new_client_cert, master_name, rapi_cert_pem=None, spice_cert_pem=None, spice_cacert_pem=None, cds=None, nodecert_file=pathutils.NODED_CERT_FILE, clientcert_file=pathutils.NODED_CLIENT_CERT_FILE, rapicert_file=pathutils.RAPI_CERT_FILE, spicecert_file=pathutils.SPICE_CERT_FILE, spicecacert_file=pathutils.SPICE_CACERT_FILE, hmackey_file=pathutils.CONFD_HMAC_KEY, cds_file=pathutils.CLUSTER_DOMAIN_SECRET_FILE): """Updates the cluster certificates, keys and secrets. @type new_cluster_cert: bool @param new_cluster_cert: Whether to generate a new cluster certificate @type new_rapi_cert: bool @param new_rapi_cert: Whether to generate a new RAPI certificate @type new_spice_cert: bool @param new_spice_cert: Whether to generate a new SPICE certificate @type new_confd_hmac_key: bool @param new_confd_hmac_key: Whether to generate a new HMAC key @type new_cds: bool @param new_cds: Whether to generate a new cluster domain secret @type new_client_cert: bool @param new_client_cert: Whether to generate a new client certificate @type master_name: string @param master_name: FQDN of the master node @type rapi_cert_pem: string @param rapi_cert_pem: New RAPI certificate in PEM format @type spice_cert_pem: string @param spice_cert_pem: New SPICE certificate in PEM format @type spice_cacert_pem: string @param spice_cacert_pem: Certificate of the CA that signed the SPICE certificate, in PEM format @type cds: string @param cds: New cluster domain secret @type nodecert_file: string @param nodecert_file: optional override of the node cert file path @type rapicert_file: string @param rapicert_file: optional override of the rapi cert file path @type spicecert_file: string @param spicecert_file: optional override of the spice cert file path @type spicecacert_file: string @param spicecacert_file: optional override of the spice CA cert file path @type hmackey_file: string @param hmackey_file: optional override of the hmac key file path """ # pylint: disable=R0913 # noded SSL certificate utils.GenerateNewSslCert( new_cluster_cert, nodecert_file, 1, "Generating new cluster certificate at %s" % nodecert_file) # If the cluster certificate was renewed, the client cert has to be # renewed and resigned. if new_cluster_cert or new_client_cert: utils.GenerateNewClientSslCert(clientcert_file, nodecert_file, master_name) # confd HMAC key if new_confd_hmac_key or not os.path.exists(hmackey_file): logging.debug("Writing new confd HMAC key to %s", hmackey_file) GenerateHmacKey(hmackey_file) if rapi_cert_pem: # Assume rapi_pem contains a valid PEM-formatted certificate and key logging.debug("Writing RAPI certificate at %s", rapicert_file) utils.WriteFile(rapicert_file, data=rapi_cert_pem, backup=True) else: utils.GenerateNewSslCert( new_rapi_cert, rapicert_file, 1, "Generating new RAPI certificate at %s" % rapicert_file) # SPICE spice_cert_exists = os.path.exists(spicecert_file) spice_cacert_exists = os.path.exists(spicecacert_file) if spice_cert_pem: # spice_cert_pem implies also spice_cacert_pem logging.debug("Writing SPICE certificate at %s", spicecert_file) utils.WriteFile(spicecert_file, data=spice_cert_pem, backup=True) logging.debug("Writing SPICE CA certificate at %s", spicecacert_file) utils.WriteFile(spicecacert_file, data=spice_cacert_pem, backup=True) elif new_spice_cert or not spice_cert_exists: if spice_cert_exists: utils.CreateBackup(spicecert_file) if spice_cacert_exists: utils.CreateBackup(spicecacert_file) logging.debug("Generating new self-signed SPICE certificate at %s", spicecert_file) (_, cert_pem) = utils.GenerateSelfSignedSslCert(spicecert_file, 1) # Self-signed certificate -> the public certificate is also the CA public # certificate logging.debug("Writing the public certificate to %s", spicecert_file) utils.io.WriteFile(spicecacert_file, mode=0o400, data=cert_pem) # Cluster domain secret if cds: logging.debug("Writing cluster domain secret to %s", cds_file) utils.WriteFile(cds_file, data=cds, backup=True) elif new_cds or not os.path.exists(cds_file): logging.debug("Generating new cluster domain secret at %s", cds_file) GenerateHmacKey(cds_file) def _InitGanetiServerSetup(master_name, cfg): """Setup the necessary configuration for the initial node daemon. This creates the nodepass file containing the shared password for the cluster, generates the SSL certificate and starts the node daemon. @type master_name: str @param master_name: Name of the master node @type cfg: ConfigWriter @param cfg: the configuration writer """ # Generate cluster secrets GenerateClusterCrypto(True, False, False, False, False, False, master_name) # Add the master's SSL certificate digest to the configuration. master_uuid = cfg.GetMasterNode() master_digest = utils.GetCertificateDigest() cfg.AddNodeToCandidateCerts(master_uuid, master_digest) cfg.Update(cfg.GetClusterInfo(), logging.error) ssconf.WriteSsconfFiles(cfg.GetSsconfValues()) if not os.path.exists( os.path.join(pathutils.DATA_DIR, "%s%s" % (constants.SSCONF_FILEPREFIX, constants.SS_MASTER_CANDIDATES_CERTS))): raise errors.OpExecError("Ssconf file for master candidate certificates" " was not written.") if not os.path.exists(pathutils.NODED_CERT_FILE): raise errors.OpExecError("The server certficate was not created properly.") if not os.path.exists(pathutils.NODED_CLIENT_CERT_FILE): raise errors.OpExecError("The client certificate was not created" " properly.") # set up the inter-node password and certificate result = utils.RunCmd([pathutils.DAEMON_UTIL, "start", constants.NODED]) if result.failed: raise errors.OpExecError("Could not start the node daemon, command %s" " had exitcode %s and error %s" % (result.cmd, result.exit_code, result.output)) _WaitForNodeDaemon(master_name) def _WaitForNodeDaemon(node_name): """Wait for node daemon to become responsive. """ def _CheckNodeDaemon(): # Pylint bug # pylint: disable=E1101 result = rpc.BootstrapRunner().call_version([node_name])[node_name] if result.fail_msg: raise utils.RetryAgain() try: utils.Retry(_CheckNodeDaemon, 1.0, _DAEMON_READY_TIMEOUT) except utils.RetryTimeout: raise errors.OpExecError("Node daemon on %s didn't answer queries within" " %s seconds" % (node_name, _DAEMON_READY_TIMEOUT)) def _WaitForMasterDaemon(): """Wait for master daemon to become responsive. """ def _CheckMasterDaemon(): try: cl = luxi.Client() (cluster_name, ) = cl.QueryConfigValues(["cluster_name"]) except Exception: raise utils.RetryAgain() logging.debug("Received cluster name %s from master", cluster_name) try: utils.Retry(_CheckMasterDaemon, 1.0, _DAEMON_READY_TIMEOUT) except utils.RetryTimeout: raise errors.OpExecError("Master daemon didn't answer queries within" " %s seconds" % _DAEMON_READY_TIMEOUT) def _WaitForSshDaemon(hostname, port): """Wait for SSH daemon to become responsive. """ family = ssconf.SimpleStore().GetPrimaryIPFamily() hostip = netutils.GetHostname(name=hostname, family=family).ip def _CheckSshDaemon(): if netutils.TcpPing(hostip, port, timeout=1.0, live_port_needed=True): logging.debug("SSH daemon on %s:%s (IP address %s) has become" " responsive", hostname, port, hostip) else: raise utils.RetryAgain() try: utils.Retry(_CheckSshDaemon, 1.0, _DAEMON_READY_TIMEOUT) except utils.RetryTimeout: raise errors.OpExecError("SSH daemon on %s:%s (IP address %s) didn't" " become responsive within %s seconds" % (hostname, port, hostip, _DAEMON_READY_TIMEOUT)) def _InitFileStorageDir(file_storage_dir): """Initialize if needed the file storage. @param file_storage_dir: the user-supplied value @return: either empty string (if file storage was disabled at build time) or the normalized path to the storage directory """ file_storage_dir = os.path.normpath(file_storage_dir) if not os.path.isabs(file_storage_dir): raise errors.OpPrereqError("File storage directory '%s' is not an absolute" " path" % file_storage_dir, errors.ECODE_INVAL) if not os.path.exists(file_storage_dir): try: os.makedirs(file_storage_dir, 0o750) except OSError as err: raise errors.OpPrereqError("Cannot create file storage directory" " '%s': %s" % (file_storage_dir, err), errors.ECODE_ENVIRON) if not os.path.isdir(file_storage_dir): raise errors.OpPrereqError("The file storage directory '%s' is not" " a directory." % file_storage_dir, errors.ECODE_ENVIRON) return file_storage_dir def _PrepareFileBasedStorage( enabled_disk_templates, file_storage_dir, default_dir, file_disk_template, _storage_path_acceptance_fn, init_fn=_InitFileStorageDir, acceptance_fn=None): """Checks if a file-base storage type is enabled and inits the dir. @type enabled_disk_templates: list of string @param enabled_disk_templates: list of enabled disk templates @type file_storage_dir: string @param file_storage_dir: the file storage directory @type default_dir: string @param default_dir: default file storage directory when C{file_storage_dir} is 'None' @type file_disk_template: string @param file_disk_template: a disk template whose storage type is 'ST_FILE', 'ST_SHARED_FILE' or 'ST_GLUSTER' @type _storage_path_acceptance_fn: function @param _storage_path_acceptance_fn: checks whether the given file-based storage directory is acceptable @see: C{cluster.CheckFileBasedStoragePathVsEnabledDiskTemplates} for details @rtype: string @returns: the name of the actual file storage directory """ assert (file_disk_template in utils.storage.GetDiskTemplatesOfStorageTypes( constants.ST_FILE, constants.ST_SHARED_FILE, constants.ST_GLUSTER )) if file_storage_dir is None: file_storage_dir = default_dir if not acceptance_fn: acceptance_fn = \ lambda path: filestorage.CheckFileStoragePathAcceptance( path, exact_match_ok=True) _storage_path_acceptance_fn(logging.warning, file_storage_dir, enabled_disk_templates) file_storage_enabled = file_disk_template in enabled_disk_templates if file_storage_enabled: try: acceptance_fn(file_storage_dir) except errors.FileStoragePathError as e: raise errors.OpPrereqError(str(e)) result_file_storage_dir = init_fn(file_storage_dir) else: result_file_storage_dir = file_storage_dir return result_file_storage_dir def _PrepareFileStorage( enabled_disk_templates, file_storage_dir, init_fn=_InitFileStorageDir, acceptance_fn=None): """Checks if file storage is enabled and inits the dir. @see: C{_PrepareFileBasedStorage} """ return _PrepareFileBasedStorage( enabled_disk_templates, file_storage_dir, pathutils.DEFAULT_FILE_STORAGE_DIR, constants.DT_FILE, cluster.CheckFileStoragePathVsEnabledDiskTemplates, init_fn=init_fn, acceptance_fn=acceptance_fn) def _PrepareSharedFileStorage( enabled_disk_templates, file_storage_dir, init_fn=_InitFileStorageDir, acceptance_fn=None): """Checks if shared file storage is enabled and inits the dir. @see: C{_PrepareFileBasedStorage} """ return _PrepareFileBasedStorage( enabled_disk_templates, file_storage_dir, pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR, constants.DT_SHARED_FILE, cluster.CheckSharedFileStoragePathVsEnabledDiskTemplates, init_fn=init_fn, acceptance_fn=acceptance_fn) def _PrepareGlusterStorage( enabled_disk_templates, file_storage_dir, init_fn=_InitFileStorageDir, acceptance_fn=None): """Checks if gluster storage is enabled and inits the dir. @see: C{_PrepareFileBasedStorage} """ return _PrepareFileBasedStorage( enabled_disk_templates, file_storage_dir, pathutils.DEFAULT_GLUSTER_STORAGE_DIR, constants.DT_GLUSTER, cluster.CheckGlusterStoragePathVsEnabledDiskTemplates, init_fn=init_fn, acceptance_fn=acceptance_fn) def _InitCheckEnabledDiskTemplates(enabled_disk_templates): """Checks the sanity of the enabled disk templates. """ if not enabled_disk_templates: raise errors.OpPrereqError("Enabled disk templates list must contain at" " least one member", errors.ECODE_INVAL) invalid_disk_templates = \ set(enabled_disk_templates) - constants.DISK_TEMPLATES if invalid_disk_templates: raise errors.OpPrereqError("Enabled disk templates list contains invalid" " entries: %s" % invalid_disk_templates, errors.ECODE_INVAL) def _RestrictIpolicyToEnabledDiskTemplates(ipolicy, enabled_disk_templates): """Restricts the ipolicy's disk templates to the enabled ones. This function clears the ipolicy's list of allowed disk templates from the ones that are not enabled by the cluster. @type ipolicy: dict @param ipolicy: the instance policy @type enabled_disk_templates: list of string @param enabled_disk_templates: the list of cluster-wide enabled disk templates """ assert constants.IPOLICY_DTS in ipolicy allowed_disk_templates = ipolicy[constants.IPOLICY_DTS] restricted_disk_templates = list(set(allowed_disk_templates) .intersection(set(enabled_disk_templates))) ipolicy[constants.IPOLICY_DTS] = restricted_disk_templates def _InitCheckDrbdHelper(drbd_helper, drbd_enabled): """Checks the DRBD usermode helper. @type drbd_helper: string @param drbd_helper: name of the DRBD usermode helper that the system should use """ if not drbd_enabled: return if drbd_helper is not None: try: curr_helper = drbd.DRBD8.GetUsermodeHelper() except errors.BlockDeviceError as err: raise errors.OpPrereqError("Error while checking drbd helper" " (disable drbd with --enabled-disk-templates" " if you are not using drbd): %s" % str(err), errors.ECODE_ENVIRON) if drbd_helper != curr_helper: raise errors.OpPrereqError("Error: requiring %s as drbd helper but %s" " is the current helper" % (drbd_helper, curr_helper), errors.ECODE_INVAL) def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914 master_netmask, master_netdev, file_storage_dir, shared_file_storage_dir, gluster_storage_dir, candidate_pool_size, ssh_key_type, ssh_key_bits, secondary_ip=None, vg_name=None, beparams=None, nicparams=None, ndparams=None, hvparams=None, diskparams=None, enabled_hypervisors=None, modify_etc_hosts=True, modify_ssh_setup=True, maintain_node_health=False, drbd_helper=None, uid_pool=None, default_iallocator=None, default_iallocator_params=None, primary_ip_version=None, ipolicy=None, prealloc_wipe_disks=False, use_external_mip_script=False, hv_state=None, disk_state=None, enabled_disk_templates=None, install_image=None, zeroing_image=None, compression_tools=None, enabled_user_shutdown=False): """Initialise the cluster. @type candidate_pool_size: int @param candidate_pool_size: master candidate pool size @type enabled_disk_templates: list of string @param enabled_disk_templates: list of disk_templates to be used in this cluster @type enabled_user_shutdown: bool @param enabled_user_shutdown: whether user shutdown is enabled cluster wide """ # TODO: complete the docstring if config.ConfigWriter.IsCluster(): raise errors.OpPrereqError("Cluster is already initialised", errors.ECODE_STATE) data_dir = vcluster.AddNodePrefix(pathutils.DATA_DIR) queue_dir = vcluster.AddNodePrefix(pathutils.QUEUE_DIR) archive_dir = vcluster.AddNodePrefix(pathutils.JOB_QUEUE_ARCHIVE_DIR) for ddir in [queue_dir, data_dir, archive_dir]: if os.path.isdir(ddir): for entry in os.listdir(ddir): if not os.path.isdir(os.path.join(ddir, entry)): raise errors.OpPrereqError( "%s contains non-directory entries like %s. Remove left-overs of an" " old cluster before initialising a new one" % (ddir, entry), errors.ECODE_STATE) if not enabled_hypervisors: raise errors.OpPrereqError("Enabled hypervisors list must contain at" " least one member", errors.ECODE_INVAL) invalid_hvs = set(enabled_hypervisors) - constants.HYPER_TYPES if invalid_hvs: raise errors.OpPrereqError("Enabled hypervisors contains invalid" " entries: %s" % invalid_hvs, errors.ECODE_INVAL) _InitCheckEnabledDiskTemplates(enabled_disk_templates) try: ipcls = netutils.IPAddress.GetClassFromIpVersion(primary_ip_version) except errors.ProgrammerError: raise errors.OpPrereqError("Invalid primary ip version: %d." % primary_ip_version, errors.ECODE_INVAL) hostname = netutils.GetHostname(family=ipcls.family) if not ipcls.IsValid(hostname.ip): raise errors.OpPrereqError("This host's IP (%s) is not a valid IPv%d" " address." % (hostname.ip, primary_ip_version), errors.ECODE_INVAL) if ipcls.IsLoopback(hostname.ip): raise errors.OpPrereqError("This host's IP (%s) resolves to a loopback" " address. Please fix DNS or %s." % (hostname.ip, pathutils.ETC_HOSTS), errors.ECODE_ENVIRON) if not ipcls.Own(hostname.ip): raise errors.OpPrereqError("Inconsistency: this host's name resolves" " to %s,\nbut this ip address does not" " belong to this host" % hostname.ip, errors.ECODE_ENVIRON) clustername = netutils.GetHostname(name=cluster_name, family=ipcls.family) if netutils.TcpPing(clustername.ip, constants.DEFAULT_NODED_PORT, timeout=5): raise errors.OpPrereqError("Cluster IP already active", errors.ECODE_NOTUNIQUE) if not secondary_ip: if primary_ip_version == constants.IP6_VERSION: raise errors.OpPrereqError("When using a IPv6 primary address, a valid" " IPv4 address must be given as secondary", errors.ECODE_INVAL) secondary_ip = hostname.ip if not netutils.IP4Address.IsValid(secondary_ip): raise errors.OpPrereqError("Secondary IP address (%s) has to be a valid" " IPv4 address." % secondary_ip, errors.ECODE_INVAL) if not netutils.IP4Address.Own(secondary_ip): raise errors.OpPrereqError("You gave %s as secondary IP," " but it does not belong to this host." % secondary_ip, errors.ECODE_ENVIRON) if master_netmask is not None: if not ipcls.ValidateNetmask(master_netmask): raise errors.OpPrereqError("CIDR netmask (%s) not valid for IPv%s " % (master_netmask, primary_ip_version), errors.ECODE_INVAL) else: master_netmask = ipcls.iplen if vg_name: # Check if volume group is valid vgstatus = utils.CheckVolumeGroupSize(utils.ListVolumeGroups(), vg_name, constants.MIN_VG_SIZE) if vgstatus: raise errors.OpPrereqError("Error: %s" % vgstatus, errors.ECODE_INVAL) drbd_enabled = constants.DT_DRBD8 in enabled_disk_templates _InitCheckDrbdHelper(drbd_helper, drbd_enabled) logging.debug("Stopping daemons (if any are running)") result = utils.RunCmd([pathutils.DAEMON_UTIL, "stop-all"]) if result.failed: raise errors.OpExecError("Could not stop daemons, command %s" " had exitcode %s and error '%s'" % (result.cmd, result.exit_code, result.output)) file_storage_dir = _PrepareFileStorage(enabled_disk_templates, file_storage_dir) shared_file_storage_dir = _PrepareSharedFileStorage(enabled_disk_templates, shared_file_storage_dir) gluster_storage_dir = _PrepareGlusterStorage(enabled_disk_templates, gluster_storage_dir) if not re.match("^[0-9a-z]{2}:[0-9a-z]{2}:[0-9a-z]{2}$", mac_prefix): raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix, errors.ECODE_INVAL) if not nicparams.get('mode', None) == constants.NIC_MODE_OVS: # Do not do this check if mode=openvswitch, since the openvswitch is not # created yet result = utils.RunCmd(["ip", "link", "show", "dev", master_netdev]) if result.failed: raise errors.OpPrereqError("Invalid master netdev given (%s): '%s'" % (master_netdev, result.output.strip()), errors.ECODE_INVAL) dirs = [(pathutils.RUN_DIR, constants.RUN_DIRS_MODE)] utils.EnsureDirs(dirs) objects.UpgradeBeParams(beparams) utils.ForceDictType(beparams, constants.BES_PARAMETER_TYPES) utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES) objects.NIC.CheckParameterSyntax(nicparams) full_ipolicy = objects.FillIPolicy(constants.IPOLICY_DEFAULTS, ipolicy) _RestrictIpolicyToEnabledDiskTemplates(full_ipolicy, enabled_disk_templates) if ndparams is not None: utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES) else: ndparams = dict(constants.NDC_DEFAULTS) # This is ugly, as we modify the dict itself # FIXME: Make utils.ForceDictType pure functional or write a wrapper # around it if hv_state: for hvname, hvs_data in hv_state.items(): utils.ForceDictType(hvs_data, constants.HVSTS_PARAMETER_TYPES) hv_state[hvname] = objects.Cluster.SimpleFillHvState(hvs_data) else: hv_state = dict((hvname, constants.HVST_DEFAULTS) for hvname in enabled_hypervisors) # FIXME: disk_state has no default values yet if disk_state: for storage, ds_data in disk_state.items(): if storage not in constants.DS_VALID_TYPES: raise errors.OpPrereqError("Invalid storage type in disk state: %s" % storage, errors.ECODE_INVAL) for ds_name, state in ds_data.items(): utils.ForceDictType(state, constants.DSS_PARAMETER_TYPES) ds_data[ds_name] = objects.Cluster.SimpleFillDiskState(state) # hvparams is a mapping of hypervisor->hvparams dict for hv_name, hv_params in hvparams.items(): utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES) hv_class = hypervisor.GetHypervisor(hv_name) hv_class.CheckParameterSyntax(hv_params) # diskparams is a mapping of disk-template->diskparams dict for template, dt_params in diskparams.items(): param_keys = set(dt_params.keys()) default_param_keys = set(constants.DISK_DT_DEFAULTS[template].keys()) if param_keys > default_param_keys: unknown_params = param_keys - default_param_keys raise errors.OpPrereqError("Invalid parameters for disk template %s:" " %s" % (template, utils.CommaJoin(unknown_params)), errors.ECODE_INVAL) utils.ForceDictType(dt_params, constants.DISK_DT_TYPES) if template == constants.DT_DRBD8 and vg_name is not None: # The default METAVG value is equal to the VG name set at init time, # if provided dt_params[constants.DRBD_DEFAULT_METAVG] = vg_name try: utils.VerifyDictOptions(diskparams, constants.DISK_DT_DEFAULTS) except errors.OpPrereqError as err: raise errors.OpPrereqError("While verify diskparam options: %s" % err, errors.ECODE_INVAL) # set up ssh config and /etc/hosts rsa_sshkey = "" dsa_sshkey = "" if os.path.isfile(pathutils.SSH_HOST_RSA_PUB): sshline = utils.ReadFile(pathutils.SSH_HOST_RSA_PUB) rsa_sshkey = sshline.split(" ")[1] if os.path.isfile(pathutils.SSH_HOST_DSA_PUB): sshline = utils.ReadFile(pathutils.SSH_HOST_DSA_PUB) dsa_sshkey = sshline.split(" ")[1] if not rsa_sshkey and not dsa_sshkey: raise errors.OpPrereqError("Failed to find SSH public keys", errors.ECODE_ENVIRON) if modify_etc_hosts: utils.AddHostToEtcHosts(hostname.name, hostname.ip) if modify_ssh_setup: ssh.InitSSHSetup(ssh_key_type, ssh_key_bits) if default_iallocator is not None: alloc_script = utils.FindFile(default_iallocator, constants.IALLOCATOR_SEARCH_PATH, os.path.isfile) if alloc_script is None: raise errors.OpPrereqError("Invalid default iallocator script '%s'" " specified" % default_iallocator, errors.ECODE_INVAL) else: # default to htools if utils.FindFile(constants.IALLOC_HAIL, constants.IALLOCATOR_SEARCH_PATH, os.path.isfile): default_iallocator = constants.IALLOC_HAIL # check if we have all the users we need try: runtime.GetEnts() except errors.ConfigurationError as err: raise errors.OpPrereqError("Required system user/group missing: %s" % err, errors.ECODE_ENVIRON) candidate_certs = {} now = time.time() if compression_tools is not None: cluster.CheckCompressionTools(compression_tools) initial_dc_config = dict(active=True, interval=int(constants.MOND_TIME_INTERVAL * 1e6)) data_collectors = dict( (name, initial_dc_config.copy()) for name in constants.DATA_COLLECTOR_NAMES) # init of cluster config file cluster_config = objects.Cluster( serial_no=1, rsahostkeypub=rsa_sshkey, dsahostkeypub=dsa_sshkey, highest_used_port=(constants.FIRST_DRBD_PORT - 1), mac_prefix=mac_prefix, volume_group_name=vg_name, tcpudp_port_pool=set(), master_ip=clustername.ip, master_netmask=master_netmask, master_netdev=master_netdev, cluster_name=clustername.name, file_storage_dir=file_storage_dir, shared_file_storage_dir=shared_file_storage_dir, gluster_storage_dir=gluster_storage_dir, enabled_hypervisors=enabled_hypervisors, beparams={constants.PP_DEFAULT: beparams}, nicparams={constants.PP_DEFAULT: nicparams}, ndparams=ndparams, hvparams=hvparams, diskparams=diskparams, candidate_pool_size=candidate_pool_size, modify_etc_hosts=modify_etc_hosts, modify_ssh_setup=modify_ssh_setup, uid_pool=uid_pool, ctime=now, mtime=now, maintain_node_health=maintain_node_health, data_collectors=data_collectors, drbd_usermode_helper=drbd_helper, default_iallocator=default_iallocator, default_iallocator_params=default_iallocator_params, primary_ip_family=ipcls.family, prealloc_wipe_disks=prealloc_wipe_disks, use_external_mip_script=use_external_mip_script, ipolicy=full_ipolicy, hv_state_static=hv_state, disk_state_static=disk_state, enabled_disk_templates=enabled_disk_templates, candidate_certs=candidate_certs, osparams={}, osparams_private_cluster={}, install_image=install_image, zeroing_image=zeroing_image, compression_tools=compression_tools, enabled_user_shutdown=enabled_user_shutdown, ssh_key_type=ssh_key_type, ssh_key_bits=ssh_key_bits, ) master_node_config = objects.Node(name=hostname.name, primary_ip=hostname.ip, secondary_ip=secondary_ip, serial_no=1, master_candidate=True, offline=False, drained=False, ctime=now, mtime=now, ) InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config) cfg = config.ConfigWriter(offline=True) ssh.WriteKnownHostsFile(cfg, pathutils.SSH_KNOWN_HOSTS_FILE) cfg.Update(cfg.GetClusterInfo(), logging.error) ssconf.WriteSsconfFiles(cfg.GetSsconfValues()) master_uuid = cfg.GetMasterNode() if modify_ssh_setup: ssh.InitPubKeyFile(master_uuid, ssh_key_type) # set up the inter-node password and certificate _InitGanetiServerSetup(hostname.name, cfg) logging.debug("Starting daemons") result = utils.RunCmd([pathutils.DAEMON_UTIL, "start-all"]) if result.failed: raise errors.OpExecError("Could not start daemons, command %s" " had exitcode %s and error %s" % (result.cmd, result.exit_code, result.output)) _WaitForMasterDaemon() def InitConfig(version, cluster_config, master_node_config, cfg_file=pathutils.CLUSTER_CONF_FILE): """Create the initial cluster configuration. It will contain the current node, which will also be the master node, and no instances. @type version: int @param version: configuration version @type cluster_config: L{objects.Cluster} @param cluster_config: cluster configuration @type master_node_config: L{objects.Node} @param master_node_config: master node configuration @type cfg_file: string @param cfg_file: configuration file path """ uuid_generator = config.TemporaryReservationManager() cluster_config.uuid = uuid_generator.Generate([], utils.NewUUID, _INITCONF_ECID) master_node_config.uuid = uuid_generator.Generate([], utils.NewUUID, _INITCONF_ECID) cluster_config.master_node = master_node_config.uuid nodes = { master_node_config.uuid: master_node_config, } default_nodegroup = objects.NodeGroup( uuid=uuid_generator.Generate([], utils.NewUUID, _INITCONF_ECID), name=constants.INITIAL_NODE_GROUP_NAME, members=[master_node_config.uuid], diskparams={}, ) nodegroups = { default_nodegroup.uuid: default_nodegroup, } now = time.time() config_data = objects.ConfigData(version=version, cluster=cluster_config, nodegroups=nodegroups, nodes=nodes, instances={}, networks={}, disks={}, filters={}, serial_no=1, ctime=now, mtime=now) utils.WriteFile(cfg_file, data=serializer.Dump(config_data.ToDict()), mode=0o600) def FinalizeClusterDestroy(master_uuid): """Execute the last steps of cluster destroy This function shuts down all the daemons, completing the destroy begun in cmdlib.LUDestroyOpcode. """ livelock = utils.livelock.LiveLock("bootstrap_destroy") cfg = config.GetConfig(None, livelock) modify_ssh_setup = cfg.GetClusterInfo().modify_ssh_setup runner = rpc.BootstrapRunner() master_name = cfg.GetNodeName(master_uuid) master_params = cfg.GetMasterNetworkParameters() master_params.uuid = master_uuid ems = cfg.GetUseExternalMipScript() result = runner.call_node_deactivate_master_ip(master_name, master_params, ems) msg = result.fail_msg if msg: logging.warning("Could not disable the master IP: %s", msg) result = runner.call_node_stop_master(master_name) msg = result.fail_msg if msg: logging.warning("Could not disable the master role: %s", msg) result = runner.call_node_leave_cluster(master_name, modify_ssh_setup) msg = result.fail_msg if msg: logging.warning("Could not shutdown the node daemon and cleanup" " the node: %s", msg) def SetupNodeDaemon(opts, cluster_name, node, ssh_port): """Add a node to the cluster. This function must be called before the actual opcode, and will ssh to the remote node, copy the needed files, and start ganeti-noded, allowing the master to do the rest via normal rpc calls. @param cluster_name: the cluster name @param node: the name of the new node @param ssh_port: the SSH port of the new node """ data = { constants.NDS_CLUSTER_NAME: cluster_name, constants.NDS_NODE_DAEMON_CERTIFICATE: utils.ReadFile(pathutils.NODED_CERT_FILE), constants.NDS_HMAC: utils.ReadFile(pathutils.CONFD_HMAC_KEY), constants.NDS_SSCONF: ssconf.SimpleStore().ReadAll(), constants.NDS_START_NODE_DAEMON: True, constants.NDS_NODE_NAME: node, } ssh.RunSshCmdWithStdin(cluster_name, node, pathutils.NODE_DAEMON_SETUP, ssh_port, data, debug=opts.debug, verbose=opts.verbose, use_cluster_key=True, ask_key=opts.ssh_key_check, strict_host_check=opts.ssh_key_check, ensure_version=True) _WaitForSshDaemon(node, ssh_port) _WaitForNodeDaemon(node) def MasterFailover(no_voting=False): """Failover the master node. This checks that we are not already the master, and will cause the current master to cease being master, and the non-master to become new master. Note: The call to MasterFailover from lib/client/gnt_cluster.py checks that a majority of nodes are healthy and responding before calling this. If this function is called from somewhere else, the caller should also verify that a majority of nodes are healthy. @type no_voting: boolean @param no_voting: force the operation without remote nodes agreement (dangerous) @returns: the pair of an exit code and warnings to display """ sstore = ssconf.SimpleStore() old_master, new_master = ssconf.GetMasterAndMyself(sstore) node_names = sstore.GetNodeList() mc_list = sstore.GetMasterCandidates() if old_master == new_master: raise errors.OpPrereqError("This commands must be run on the node" " where you want the new master to be." " %s is already the master" % old_master, errors.ECODE_INVAL) if new_master not in mc_list: mc_no_master = [name for name in mc_list if name != old_master] raise errors.OpPrereqError("This node is not among the nodes marked" " as master candidates. Only these nodes" " can become masters. Current list of" " master candidates is:\n" "%s" % ("\n".join(mc_no_master)), errors.ECODE_STATE) if not no_voting: vote_list = _GatherMasterVotes(node_names) if vote_list: voted_master = vote_list[0][0] if voted_master != old_master: raise errors.OpPrereqError("I have a wrong configuration, I believe" " the master is %s but the other nodes" " voted %s. Please resync the configuration" " of this node." % (old_master, voted_master), errors.ECODE_STATE) # end checks rcode = 0 warnings = [] logging.info("Setting master to %s, old master: %s", new_master, old_master) try: # Forcefully start WConfd so that we can access the configuration result = utils.RunCmd([pathutils.DAEMON_UTIL, "start", constants.WCONFD, "--force-node", "--no-voting", "--yes-do-it"]) if result.failed: raise errors.OpPrereqError("Could not start the configuration daemon," " command %s had exitcode %s and error %s" % (result.cmd, result.exit_code, result.output), errors.ECODE_NOENT) # instantiate a real config writer, as we now know we have the # configuration data livelock = utils.livelock.LiveLock("bootstrap_failover") cfg = config.GetConfig(None, livelock, accept_foreign=True) old_master_node = cfg.GetNodeInfoByName(old_master) if old_master_node is None: raise errors.OpPrereqError("Could not find old master node '%s' in" " cluster configuration." % old_master, errors.ECODE_NOENT) cluster_info = cfg.GetClusterInfo() new_master_node = cfg.GetNodeInfoByName(new_master) if new_master_node is None: raise errors.OpPrereqError("Could not find new master node '%s' in" " cluster configuration." % new_master, errors.ECODE_NOENT) cluster_info.master_node = new_master_node.uuid # this will also regenerate the ssconf files, since we updated the # cluster info cfg.Update(cluster_info, logging.error) # if cfg.Update worked, then it means the old master daemon won't be # able now to write its own config file (we rely on locking in both # backend.UploadFile() and ConfigWriter._Write(); hence the next # step is to kill the old master logging.info("Stopping the master daemon on node %s", old_master) runner = rpc.BootstrapRunner() master_params = cfg.GetMasterNetworkParameters() master_params.uuid = old_master_node.uuid ems = cfg.GetUseExternalMipScript() result = runner.call_node_deactivate_master_ip(old_master, master_params, ems) msg = result.fail_msg if msg: warning = "Could not disable the master IP: %s" % (msg,) logging.warning("%s", warning) warnings.append(warning) result = runner.call_node_stop_master(old_master) msg = result.fail_msg if msg: warning = ("Could not disable the master role on the old master" " %s, please disable manually: %s" % (old_master, msg)) logging.error("%s", warning) warnings.append(warning) except errors.ConfigurationError as err: logging.error("Error while trying to set the new master: %s", str(err)) return 1, warnings finally: # stop WConfd again: result = utils.RunCmd([pathutils.DAEMON_UTIL, "stop", constants.WCONFD]) if result.failed: warning = ("Could not stop the configuration daemon," " command %s had exitcode %s and error %s" % (result.cmd, result.exit_code, result.output)) logging.error("%s", warning) rcode = 1 logging.info("Checking master IP non-reachability...") master_ip = sstore.GetMasterIP() total_timeout = 30 # Here we have a phase where no master should be running def _check_ip(expected): if netutils.TcpPing(master_ip, constants.DEFAULT_NODED_PORT) != expected: raise utils.RetryAgain() try: utils.Retry(_check_ip, (1, 1.5, 5), total_timeout, args=[False]) except utils.RetryTimeout: warning = ("The master IP is still reachable after %s seconds," " continuing but activating the master IP on the current" " node will probably fail" % total_timeout) logging.warning("%s", warning) warnings.append(warning) rcode = 1 if jstore.CheckDrainFlag(): logging.info("Undraining job queue") jstore.SetDrainFlag(False) logging.info("Starting the master daemons on the new master") result = rpc.BootstrapRunner().call_node_start_master_daemons(new_master, no_voting) msg = result.fail_msg if msg: logging.error("Could not start the master role on the new master" " %s, please check: %s", new_master, msg) rcode = 1 # Finally verify that the new master managed to set up the master IP # and warn if it didn't. try: utils.Retry(_check_ip, (1, 1.5, 5), total_timeout, args=[True]) except utils.RetryTimeout: warning = ("The master IP did not come up within %s seconds; the" " cluster should still be working and reachable via %s," " but not via the master IP address" % (total_timeout, new_master)) logging.warning("%s", warning) warnings.append(warning) rcode = 1 logging.info("Master failed over from %s to %s", old_master, new_master) return rcode, warnings def GetMaster(): """Returns the current master node. This is a separate function in bootstrap since it's needed by gnt-cluster, and instead of importing directly ssconf, it's better to abstract it in bootstrap, where we do use ssconf in other functions too. """ sstore = ssconf.SimpleStore() old_master, _ = ssconf.GetMasterAndMyself(sstore) return old_master def _GatherMasterVotes(node_names): """Check the agreement on who is the master. This function will return a list of (node, number of votes), ordered by the number of votes. Note that the sum of votes is the number of nodes this machine knows, whereas the number of entries in the list could be different (if some nodes vote for another master). @type node_names: list @param node_names: the list of nodes to query for master info @rtype: list @return: list of (node, votes) """ if not node_names: # no nodes return [] results = rpc.BootstrapRunner().call_master_node_name(node_names) if not isinstance(results, dict): # this should not happen (unless internal error in rpc) logging.critical("Can't complete rpc call, aborting master startup") return [(None, len(node_names))] votes = {} for (node_name, nres) in results.items(): msg = nres.fail_msg if msg: logging.warning("Error contacting node %s: %s", node_name, msg) continue node = nres.payload if not node: logging.warning(('Expected a Node, encountered a None. Skipping this' ' voting result.')) if node not in votes: votes[node] = 1 else: votes[node] += 1 vote_list = list(votes.items()) vote_list.sort(key=lambda x: x[1], reverse=True) return vote_list def MajorityHealthy(ignore_offline_nodes=False): """Check if the majority of nodes is healthy Gather master votes from all nodes known to this node; return True if a strict majority of nodes is reachable and has some opinion on which node is master. Note that this will not guarantee any node to win an election but it ensures that a standard master-failover is still possible. @return: tuple of (boolean, [str]); the first is if a majority of nodes are healthy, the second is a list of the node names that are not considered healthy. """ if ignore_offline_nodes: node_names = ssconf.SimpleStore().GetOnlineNodeList() else: node_names = ssconf.SimpleStore().GetNodeList() node_count = len(node_names) vote_list = _GatherMasterVotes(node_names) if not vote_list: logging.warning(('Voting list was None; cannot determine if a majority of ' 'nodes are healthy')) return (False, node_names) total_votes = sum([count for (node, count) in vote_list if node is not None]) majority_healthy = 2 * total_votes > node_count # The list of nodes that did not vote is calculated to provide useful # debugging information to the client. voting_nodes = [node for (node, _) in vote_list] nonvoting_nodes = [node for node in node_names if node not in voting_nodes] logging.info("Total %d nodes, %d votes: %s", node_count, total_votes, vote_list) return (majority_healthy, nonvoting_nodes) ganeti-3.1.0~rc2/lib/build/000075500000000000000000000000001476477700300154515ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/build/__init__.py000064400000000000000000000037001476477700300175620ustar00rootroot00000000000000# # # Copyright (C) 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module used during the Ganeti build process""" import os import importlib.util import importlib.machinery def LoadModule(filename): """Loads an external module by filename. Use this function with caution. Python will always write the compiled source to a file named "${filename}c". @type filename: string @param filename: Path to module """ (name, _) = os.path.splitext(filename) loader = importlib.machinery.SourceFileLoader(name, filename) spec = importlib.util.spec_from_file_location(name, filename, loader=loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) return module ganeti-3.1.0~rc2/lib/build/shell_example_lexer.py000064400000000000000000000055221476477700300220500ustar00rootroot00000000000000# # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Pygments lexer for our custom shell example sessions. The lexer support the following custom markup: - comments: # this is a comment - command lines: '$ ' at the beginning of a line denotes a command - variable input: %input% (works in both commands and screen output) - otherwise, regular text output from commands will be plain """ from pygments.lexer import RegexLexer, bygroups, include from pygments.token import Name, Text, Generic, Comment import sphinx class ShellExampleLexer(RegexLexer): name = "ShellExampleLexer" aliases = "shell-example" filenames = [] tokens = { "root": [ include("comments"), include("userinput"), # switch to state input on '$ ' at the start of the line (r"^\$ ", Text, "input"), (r"\s+", Text), (r"[^#%\s\\]+", Text), (r"\\", Text), ], "input": [ include("comments"), include("userinput"), (r"[^#%\s\\]+", Generic.Strong), (r"\\\n", Generic.Strong), (r"\\", Generic.Strong), # switch to prev state at non-escaped new-line (r"\n", Text, "#pop"), (r"\s+", Text), ], "comments": [ (r"#.*\n", Comment.Single), ], "userinput": [ (r"(\\)(%)", bygroups(None, Text)), (r"(%)([^%]*)(%)", bygroups(None, Name.Variable, None)), ], } def setup(app): version = tuple(map(int, sphinx.__version__.split('.'))) if version >= (2, 1, 0): app.add_lexer("shell-example", ShellExampleLexer) else: app.add_lexer("shell-example", ShellExampleLexer()) ganeti-3.1.0~rc2/lib/build/sphinx_ext.py000064400000000000000000000435771476477700300202340ustar00rootroot00000000000000# # # Copyright (C) 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Sphinx extension for building opcode documentation. """ # pylint: disable=C0413 # C0413: Wrong import position import re from io import StringIO import docutils.statemachine import docutils.nodes import docutils.utils import docutils.parsers.rst from docutils.parsers.rst import Directive import sphinx.errors import sphinx.roles import sphinx.addnodes orig_manpage_role = None from ganeti import _constants from ganeti import constants from ganeti import compat from ganeti import errors from ganeti import utils from ganeti import opcodes from ganeti import opcodes_base from ganeti import ht from ganeti import rapi from ganeti import luxi from ganeti import objects from ganeti import http from ganeti import pathutils import ganeti.rapi.rlib2 # pylint: disable=W0611 import ganeti.rapi.connector # pylint: disable=W0611 #: Regular expression for man page names _MAN_RE = re.compile(r"^(?P[-\w_]+)\((?P
\d+)\)$") _TAB_WIDTH = 2 RAPI_URI_ENCODE_RE = re.compile("[^_a-z0-9]+", re.I) class ReSTError(Exception): """Custom class for generating errors in Sphinx. """ def _GetCommonParamNames(): """Builds a list of parameters common to all opcodes. """ names = set(map(compat.fst, opcodes.OpCode.OP_PARAMS)) # The "depends" attribute should be listed names.remove(opcodes_base.DEPEND_ATTR) return names COMMON_PARAM_NAMES = _GetCommonParamNames() #: Namespace for evaluating expressions EVAL_NS = dict(compat=compat, constants=constants, utils=utils, errors=errors, rlib2=rapi.rlib2, luxi=luxi, rapi=rapi, objects=objects, http=http, pathutils=pathutils) # Constants documentation for man pages CV_ECODES_DOC = "ecodes" # We don't care about the leak of variables _, name and doc here. # pylint: disable=W0621 CV_ECODES_DOC_LIST = [(name, doc) for (_, name, doc) in constants.CV_ALL_ECODES] DOCUMENTED_CONSTANTS = { CV_ECODES_DOC: sorted(CV_ECODES_DOC_LIST, key=lambda tup: tup[0]), } class OpcodeError(sphinx.errors.SphinxError): category = "Opcode error" def _SplitOption(text): """Split simple option list. @type text: string @param text: Options, e.g. "foo, bar, baz" """ return [i.strip(",").strip() for i in text.split()] def _ParseAlias(text): """Parse simple assignment option. @type text: string @param text: Assignments, e.g. "foo=bar, hello=world" @rtype: dict """ result = {} for part in _SplitOption(text): if "=" not in part: raise OpcodeError("Invalid option format, missing equal sign") (name, value) = part.split("=", 1) result[name.strip()] = value.strip() return result def _BuildOpcodeParams(op_id, include, exclude, alias): """Build opcode parameter documentation. @type op_id: string @param op_id: Opcode ID """ op_cls = opcodes.OP_MAPPING[op_id] params_with_alias = \ utils.NiceSort([(alias.get(name, name), name, default, test, doc) for (name, default, test, doc) in op_cls.GetAllParams()], key=compat.fst) for (rapi_name, name, default, test, doc) in params_with_alias: # Hide common parameters if not explicitly included if (name in COMMON_PARAM_NAMES and (not include or name not in include)): continue if exclude is not None and name in exclude: continue if include is not None and name not in include: continue has_default = default is not None or default is not ht.NoDefault has_test = test is not None buf = StringIO() buf.write("``%s``" % (rapi_name,)) if has_default or has_test: buf.write(" (") if has_default: if default == "": buf.write("defaults to the empty string") else: buf.write("defaults to ``%s``" % (default,)) if has_test: buf.write(", ") if has_test: buf.write("must be ``%s``" % (test,)) buf.write(")") yield buf.getvalue() # Add text for line in doc.splitlines(): yield " %s" % line def _BuildOpcodeResult(op_id): """Build opcode result documentation. @type op_id: string @param op_id: Opcode ID """ op_cls = opcodes.OP_MAPPING[op_id] result_fn = getattr(op_cls, "OP_RESULT", None) if not result_fn: raise OpcodeError("Opcode '%s' has no result description" % op_id) return "``%s``" % result_fn class OpcodeParams(Directive): """Custom directive for opcode parameters. See also . """ has_content = False required_arguments = 1 optional_arguments = 0 final_argument_whitespace = False option_spec = dict(include=_SplitOption, exclude=_SplitOption, alias=_ParseAlias) def run(self): op_id = self.arguments[0] include = self.options.get("include", None) exclude = self.options.get("exclude", None) alias = self.options.get("alias", {}) path = op_id include_text = "\n\n".join(_BuildOpcodeParams(op_id, include, exclude, alias)) # Inject into state machine include_lines = docutils.statemachine.string2lines(include_text, _TAB_WIDTH, convert_whitespace=1) self.state_machine.insert_input(include_lines, path) return [] class OpcodeResult(Directive): """Custom directive for opcode result. See also . """ has_content = False required_arguments = 1 optional_arguments = 0 final_argument_whitespace = False def run(self): op_id = self.arguments[0] path = op_id include_text = _BuildOpcodeResult(op_id) # Inject into state machine include_lines = docutils.statemachine.string2lines(include_text, _TAB_WIDTH, convert_whitespace=1) self.state_machine.insert_input(include_lines, path) return [] def PythonEvalRole(role, rawtext, text, lineno, inliner, options={}, content=[]): """Custom role to evaluate Python expressions. The expression's result is included as a literal. """ # pylint: disable=W0102,W0613 # W0102: Dangerous default value as argument # W0613: Unused argument code = docutils.utils.unescape(text, restore_backslashes=True) try: result = eval(code, EVAL_NS) # pylint: disable=W0123 except Exception as err: # pylint: disable=W0703 msg = inliner.reporter.error("Failed to evaluate %r: %s" % (code, err), line=lineno) return ([inliner.problematic(rawtext, rawtext, msg)], [msg]) node = docutils.nodes.literal("", str(result), **options) return ([node], []) class PythonAssert(Directive): """Custom directive for writing assertions. The content must be a valid Python expression. If its result does not evaluate to C{True}, the assertion fails. """ has_content = True required_arguments = 0 optional_arguments = 0 final_argument_whitespace = False def run(self): # Handle combinations of Sphinx and docutils not providing the wanted method if hasattr(self, "assert_has_content"): self.assert_has_content() else: assert self.content code = "\n".join(self.content) try: result = eval(code, EVAL_NS) # pylint: disable=W0123 except Exception as err: raise self.error("Failed to evaluate %r: %s" % (code, err)) if not result: raise self.error("Assertion failed: %s" % (code, )) return [] def BuildQueryFields(fields): """Build query fields documentation. @type fields: dict (field name as key, field details as value) """ defs = [(fdef.name, fdef.doc) for (_, (fdef, _, _, _)) in utils.NiceSort(fields.items(), key=compat.fst)] return BuildValuesDoc(defs) def BuildValuesDoc(values): """Builds documentation for a list of values @type values: list of tuples in the form (value, documentation) """ for name, doc in values: assert len(doc.splitlines()) == 1 yield "``%s``" % (name,) yield " %s" % (doc,) def _ManPageNodeClass(*args, **kwargs): """Generates a pending XRef like a ":doc:`...`" reference. """ # Type for sphinx/environment.py:BuildEnvironment.resolve_references kwargs["reftype"] = "doc" # Force custom title kwargs["refexplicit"] = True kwargs["refdomain"] = "std" return sphinx.addnodes.pending_xref(*args, **kwargs) class _ManPageXRefRole(sphinx.roles.XRefRole): def __init__(self): """Initializes this class. """ sphinx.roles.XRefRole.__init__(self, nodeclass=_ManPageNodeClass, warn_dangling=True) assert not hasattr(self, "converted"), \ "Sphinx base class gained an attribute named 'converted'" self.converted = None def process_link(self, env, refnode, has_explicit_title, title, target): """Specialization for man page links. """ if has_explicit_title: raise ReSTError("Setting explicit title is not allowed for man pages") # Check format and extract name and section m = _MAN_RE.match(title) if not m: raise ReSTError("Man page reference '%s' does not match regular" " expression '%s'" % (title, _MAN_RE.pattern)) name = m.group("name") section = int(m.group("section")) wanted_section = _constants.MAN_PAGES.get(name, None) if not (wanted_section is None or wanted_section == section): raise ReSTError("Referenced man page '%s' has section number %s, but the" " reference uses section %s" % (name, wanted_section, section)) self.converted = bool(wanted_section is not None and env.app.config.enable_manpages) if self.converted: # Create link to known man page return (title, "man-%s" % name) else: # No changes return (title, target) def _ManPageRole(typ, rawtext, text, lineno, inliner, # pylint: disable=W0102 options={}, content=[]): """Custom role for man page references. Converts man pages to links if enabled during the build. """ xref = _ManPageXRefRole() assert ht.TNone(xref.converted) # Check if it's a known man page try: result = xref(typ, rawtext, text, lineno, inliner, options=options, content=content) except ReSTError as err: msg = inliner.reporter.error(str(err), line=lineno) return ([inliner.problematic(rawtext, rawtext, msg)], [msg]) assert ht.TBool(xref.converted) # Return if the conversion was successful (i.e. the man page was known and # conversion was enabled) if xref.converted: return result # Fallback if man page links are disabled or an unknown page is referenced return orig_manpage_role(typ, rawtext, text, lineno, inliner, options=options, content=content) def _EncodeRapiResourceLink(method, uri): """Encodes a RAPI resource URI for use as a link target. """ parts = [RAPI_URI_ENCODE_RE.sub("-", uri.lower()).strip("-")] if method is not None: parts.append(method.lower()) return "rapi-res-%s" % "+".join([p for p in parts if p]) def _MakeRapiResourceLink(method, uri): """Generates link target name for RAPI resource. """ if uri in ["/", "/2"]: # Don't link these return None elif uri == "/version": return _EncodeRapiResourceLink(method, uri) elif uri.startswith("/2/"): return _EncodeRapiResourceLink(method, uri[len("/2/"):]) else: raise ReSTError("Unhandled URI '%s'" % uri) def _GetHandlerMethods(handler): """Returns list of HTTP methods supported by handler class. @type handler: L{rapi.baserlib.ResourceBase} @param handler: Handler class @rtype: list of strings """ return sorted(m_attrs.method for m_attrs in rapi.baserlib.OPCODE_ATTRS # Only if handler supports method if hasattr(handler, m_attrs.method) or hasattr(handler, m_attrs.opcode)) def _DescribeHandlerAccess(handler, method): """Returns textual description of required RAPI permissions. @type handler: L{rapi.baserlib.ResourceBase} @param handler: Handler class @type method: string @param method: HTTP method (e.g. L{http.HTTP_GET}) @rtype: string """ access = rapi.baserlib.GetHandlerAccess(handler, method) if access: return utils.CommaJoin(sorted(access)) else: return "*(none)*" class _RapiHandlersForDocsHelper(object): @classmethod def Build(cls): """Returns dictionary of resource handlers. """ resources = \ rapi.connector.GetHandlers("[node_name]", "[instance_name]", "[group_name]", "[network_name]", "[job_id]", "[disk_index]", "[filter_uuid]", "[resource]", translate=cls._TranslateResourceUri) return resources @classmethod def _TranslateResourceUri(cls, *args): """Translates a resource URI for use in documentation. @see: L{rapi.connector.GetHandlers} """ return "".join(map(cls._UriPatternToString, args)) @staticmethod def _UriPatternToString(value): """Converts L{rapi.connector.UriPattern} to strings. """ if isinstance(value, rapi.connector.UriPattern): return value.content else: return value _RAPI_RESOURCES_FOR_DOCS = _RapiHandlersForDocsHelper.Build() def _BuildRapiAccessTable(res): """Build a table with access permissions needed for all RAPI resources. """ for (uri, handler) in utils.NiceSort(res.items(), key=compat.fst): reslink = _MakeRapiResourceLink(None, uri) if not reslink: # No link was generated continue yield ":ref:`%s <%s>`" % (uri, reslink) for method in _GetHandlerMethods(handler): yield (" | :ref:`%s <%s>`: %s" % (method, _MakeRapiResourceLink(method, uri), _DescribeHandlerAccess(handler, method))) class RapiAccessTable(Directive): """Custom directive to generate table of all RAPI resources. See also . """ has_content = False required_arguments = 0 optional_arguments = 0 final_argument_whitespace = False option_spec = {} def run(self): include_text = "\n".join(_BuildRapiAccessTable(_RAPI_RESOURCES_FOR_DOCS)) # Inject into state machine include_lines = docutils.statemachine.string2lines(include_text, _TAB_WIDTH, convert_whitespace=1) self.state_machine.insert_input(include_lines, self.__class__.__name__) return [] class RapiResourceDetails(Directive): """Custom directive for RAPI resource details. See also . """ has_content = False required_arguments = 1 optional_arguments = 0 final_argument_whitespace = False def run(self): uri = self.arguments[0] try: handler = _RAPI_RESOURCES_FOR_DOCS[uri] except KeyError: raise self.error("Unknown resource URI '%s'" % uri) lines = [ ".. list-table::", " :widths: 1 4", " :header-rows: 1", "", " * - Method", " - :ref:`Required permissions `", ] for method in _GetHandlerMethods(handler): lines.extend([ " * - :ref:`%s <%s>`" % (method, _MakeRapiResourceLink(method, uri)), " - %s" % _DescribeHandlerAccess(handler, method), ]) # Inject into state machine include_lines = \ docutils.statemachine.string2lines("\n".join(lines), _TAB_WIDTH, convert_whitespace=1) self.state_machine.insert_input(include_lines, self.__class__.__name__) return [] def setup(app): """Sphinx extension callback. """ global orig_manpage_role try: # Access to a protected member of a client class # pylint: disable=W0212 orig_manpage_role = docutils.parsers.rst.roles._roles["manpage"] except (AttributeError, ValueError, KeyError) as err: # Normally the "manpage" role is registered by sphinx/roles.py raise Exception("Can't find reST role named 'manpage': %s" % err) # TODO: Implement Sphinx directive for query fields app.add_directive("opcode_params", OpcodeParams) app.add_directive("opcode_result", OpcodeResult) app.add_directive("pyassert", PythonAssert) app.add_role("pyeval", PythonEvalRole) app.add_directive("rapi_access_table", RapiAccessTable) app.add_directive("rapi_resource_details", RapiResourceDetails) app.add_config_value("enable_manpages", False, True) app.add_role("manpage", _ManPageRole) ganeti-3.1.0~rc2/lib/cli.py000064400000000000000000002647061476477700300155120ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module dealing with command line parsing""" import sys import textwrap import os.path import time import logging import errno import itertools import shlex from io import StringIO from optparse import (OptionParser, TitledHelpFormatter) from ganeti import utils from ganeti import errors from ganeti import constants from ganeti import opcodes import ganeti.rpc.errors as rpcerr from ganeti import ssh from ganeti import compat from ganeti import netutils from ganeti import qlang from ganeti import objects from ganeti import pathutils from ganeti import serializer import ganeti.cli_opts # Import constants from ganeti.cli_opts import * # pylint: disable=W0401,W0614 from ganeti.runtime import (GetClient) __all__ = [ # Generic functions for CLI programs "ConfirmOperation", "CreateIPolicyFromOpts", "GenericMain", "GenericInstanceCreate", "GenericList", "GenericListFields", "GetClient", "GetOnlineNodes", "GetNodesSshPorts", "GetNodeUUIDs", "JobExecutor", "ParseTimespec", "RunWhileClusterStopped", "RunWhileDaemonsStopped", "SubmitOpCode", "SubmitOpCodeToDrainedQueue", "SubmitOrSend", # Formatting functions "ToStderr", "ToStdout", "ToStdoutAndLoginfo", "FormatError", "FormatQueryResult", "FormatParamsDictInfo", "FormatPolicyInfo", "PrintIPolicyCommand", "PrintGenericInfo", "GenerateTable", "AskUser", "FormatTimestamp", "FormatLogMessage", # Tags functions "ListTags", "AddTags", "RemoveTags", # command line options support infrastructure "ARGS_MANY_INSTANCES", "ARGS_MANY_NODES", "ARGS_MANY_GROUPS", "ARGS_MANY_NETWORKS", "ARGS_MANY_FILTERS", "ARGS_NONE", "ARGS_ONE_INSTANCE", "ARGS_ONE_NODE", "ARGS_ONE_GROUP", "ARGS_ONE_OS", "ARGS_ONE_NETWORK", "ARGS_ONE_FILTER", "ArgChoice", "ArgCommand", "ArgFile", "ArgGroup", "ArgHost", "ArgInstance", "ArgJobId", "ArgNetwork", "ArgNode", "ArgOs", "ArgExtStorage", "ArgFilter", "ArgSuggest", "ArgUnknown", "FixHvParams", "SplitNodeOption", "CalculateOSNames", "ParseFields", ] + ganeti.cli_opts.__all__ # Command line options # Query result status for clients (QR_NORMAL, QR_UNKNOWN, QR_INCOMPLETE) = range(3) #: Maximum batch size for ChooseJob _CHOOSE_BATCH = 25 # constants used to create InstancePolicy dictionary TISPECS_GROUP_TYPES = { constants.ISPECS_MIN: constants.VTYPE_INT, constants.ISPECS_MAX: constants.VTYPE_INT, } TISPECS_CLUSTER_TYPES = { constants.ISPECS_MIN: constants.VTYPE_INT, constants.ISPECS_MAX: constants.VTYPE_INT, constants.ISPECS_STD: constants.VTYPE_INT, } #: User-friendly names for query2 field types _QFT_NAMES = { constants.QFT_UNKNOWN: "Unknown", constants.QFT_TEXT: "Text", constants.QFT_BOOL: "Boolean", constants.QFT_NUMBER: "Number", constants.QFT_NUMBER_FLOAT: "Floating-point number", constants.QFT_UNIT: "Storage size", constants.QFT_TIMESTAMP: "Timestamp", constants.QFT_OTHER: "Custom", } class _Argument(object): def __init__(self, min=0, max=None): # pylint: disable=W0622 self.min = min self.max = max def __repr__(self): return ("<%s min=%s max=%s>" % (self.__class__.__name__, self.min, self.max)) class ArgSuggest(_Argument): """Suggesting argument. Value can be any of the ones passed to the constructor. """ # pylint: disable=W0622 def __init__(self, min=0, max=None, choices=None): _Argument.__init__(self, min=min, max=max) self.choices = choices def __repr__(self): return ("<%s min=%s max=%s choices=%r>" % (self.__class__.__name__, self.min, self.max, self.choices)) class ArgChoice(ArgSuggest): """Choice argument. Value can be any of the ones passed to the constructor. Like L{ArgSuggest}, but value must be one of the choices. """ class ArgUnknown(_Argument): """Unknown argument to program (e.g. determined at runtime). """ class ArgInstance(_Argument): """Instances argument. """ class ArgNode(_Argument): """Node argument. """ class ArgNetwork(_Argument): """Network argument. """ class ArgGroup(_Argument): """Node group argument. """ class ArgJobId(_Argument): """Job ID argument. """ class ArgFile(_Argument): """File path argument. """ class ArgCommand(_Argument): """Command argument. """ class ArgHost(_Argument): """Host argument. """ class ArgOs(_Argument): """OS argument. """ class ArgExtStorage(_Argument): """ExtStorage argument. """ class ArgFilter(_Argument): """Filter UUID argument. """ ARGS_NONE = [] ARGS_MANY_INSTANCES = [ArgInstance()] ARGS_MANY_NETWORKS = [ArgNetwork()] ARGS_MANY_NODES = [ArgNode()] ARGS_MANY_GROUPS = [ArgGroup()] ARGS_MANY_FILTERS = [ArgFilter()] ARGS_ONE_INSTANCE = [ArgInstance(min=1, max=1)] ARGS_ONE_NETWORK = [ArgNetwork(min=1, max=1)] ARGS_ONE_NODE = [ArgNode(min=1, max=1)] ARGS_ONE_GROUP = [ArgGroup(min=1, max=1)] ARGS_ONE_OS = [ArgOs(min=1, max=1)] ARGS_ONE_FILTER = [ArgFilter(min=1, max=1)] def _ExtractTagsObject(opts, args): """Extract the tag type object. Note that this function will modify its args parameter. """ if not hasattr(opts, "tag_type"): raise errors.ProgrammerError("tag_type not passed to _ExtractTagsObject") kind = opts.tag_type if kind == constants.TAG_CLUSTER: retval = kind, "" elif kind in (constants.TAG_NODEGROUP, constants.TAG_NODE, constants.TAG_NETWORK, constants.TAG_INSTANCE): if not args: raise errors.OpPrereqError("no arguments passed to the command", errors.ECODE_INVAL) name = args.pop(0) retval = kind, name else: raise errors.ProgrammerError("Unhandled tag type '%s'" % kind) return retval def _ExtendTags(opts, args): """Extend the args if a source file has been given. This function will extend the tags with the contents of the file passed in the 'tags_source' attribute of the opts parameter. A file named '-' will be replaced by stdin. """ fname = opts.tags_source if fname is None: return if fname == "-": new_fh = sys.stdin else: new_fh = open(fname, "r") new_data = [] try: # we don't use the nice 'new_data = [line.strip() for line in fh]' # because of python bug 1633941 while True: line = new_fh.readline() if not line: break new_data.append(line.strip()) finally: new_fh.close() args.extend(new_data) def ListTags(opts, args): """List the tags on a given object. This is a generic implementation that knows how to deal with all three cases of tag objects (cluster, node, instance). The opts argument is expected to contain a tag_type field denoting what object type we work on. """ kind, name = _ExtractTagsObject(opts, args) cl = GetClient() result = cl.QueryTags(kind, name) result = list(result) result.sort() for tag in result: ToStdout(tag) def AddTags(opts, args): """Add tags on a given object. This is a generic implementation that knows how to deal with all three cases of tag objects (cluster, node, instance). The opts argument is expected to contain a tag_type field denoting what object type we work on. """ kind, name = _ExtractTagsObject(opts, args) _ExtendTags(opts, args) if not args: raise errors.OpPrereqError("No tags to be added", errors.ECODE_INVAL) op = opcodes.OpTagsSet(kind=kind, name=name, tags=args) SubmitOrSend(op, opts) def RemoveTags(opts, args): """Remove tags from a given object. This is a generic implementation that knows how to deal with all three cases of tag objects (cluster, node, instance). The opts argument is expected to contain a tag_type field denoting what object type we work on. """ kind, name = _ExtractTagsObject(opts, args) _ExtendTags(opts, args) if not args: raise errors.OpPrereqError("No tags to be removed", errors.ECODE_INVAL) op = opcodes.OpTagsDel(kind=kind, name=name, tags=args) SubmitOrSend(op, opts) class _ShowUsage(Exception): """Exception class for L{_ParseArgs}. """ def __init__(self, exit_error): """Initializes instances of this class. @type exit_error: bool @param exit_error: Whether to report failure on exit """ Exception.__init__(self) self.exit_error = exit_error class _ShowVersion(Exception): """Exception class for L{_ParseArgs}. """ def _ParseArgs(binary, argv, commands, aliases, env_override): """Parser for the command line arguments. This function parses the arguments and returns the function which must be executed together with its (modified) arguments. @param binary: Script name @param argv: Command line arguments @param commands: Dictionary containing command definitions @param aliases: dictionary with command aliases {"alias": "target", ...} @param env_override: list of env variables allowed for default args @raise _ShowUsage: If usage description should be shown @raise _ShowVersion: If version should be shown """ assert not (env_override - set(commands)) assert not (set(aliases.keys()) & set(commands.keys())) if len(argv) > 1: cmd = argv[1] else: # No option or command given raise _ShowUsage(exit_error=True) if cmd == "--version": raise _ShowVersion() elif cmd == "--help": raise _ShowUsage(exit_error=False) elif not (cmd in commands or cmd in aliases): raise _ShowUsage(exit_error=True) # get command, unalias it, and look it up in commands if cmd in aliases: if aliases[cmd] not in commands: raise errors.ProgrammerError("Alias '%s' maps to non-existing" " command '%s'" % (cmd, aliases[cmd])) cmd = aliases[cmd] if cmd in env_override: args_env_name = ("%s_%s" % (binary.replace("-", "_"), cmd)).upper() env_args = os.environ.get(args_env_name) if env_args: argv = utils.InsertAtPos(argv, 2, shlex.split(env_args)) func, args_def, parser_opts, usage, description = commands[cmd] parser = OptionParser(option_list=parser_opts + COMMON_OPTS, description=description, formatter=TitledHelpFormatter(), usage="%%prog %s %s" % (cmd, usage)) parser.disable_interspersed_args() options, args = parser.parse_args(args=argv[2:]) if not _CheckArguments(cmd, args_def, args): return None, None, None return func, options, args def _FormatUsage(binary, commands): """Generates a nice description of all commands. @param binary: Script name @param commands: Dictionary containing command definitions """ # compute the max line length for cmd + usage mlen = min(60, max(map(len, commands))) yield "Usage: %s {command} [options...] [argument...]" % binary yield "%s --help to see details, or man %s" % (binary, binary) yield "" yield "Commands:" # and format a nice command list for (cmd, (_, _, _, _, help_text)) in sorted(commands.items()): help_lines = textwrap.wrap(help_text, 79 - 3 - mlen) yield " %-*s - %s" % (mlen, cmd, help_lines.pop(0)) for line in help_lines: yield " %-*s %s" % (mlen, "", line) yield "" def _CheckArguments(cmd, args_def, args): """Verifies the arguments using the argument definition. Algorithm: 1. Abort with error if values specified by user but none expected. 1. For each argument in definition 1. Keep running count of minimum number of values (min_count) 1. Keep running count of maximum number of values (max_count) 1. If it has an unlimited number of values 1. Abort with error if it's not the last argument in the definition 1. If last argument has limited number of values 1. Abort with error if number of values doesn't match or is too large 1. Abort with error if user didn't pass enough values (min_count) """ if args and not args_def: ToStderr("Error: Command %s expects no arguments", cmd) return False min_count = None max_count = None check_max = None last_idx = len(args_def) - 1 for idx, arg in enumerate(args_def): if min_count is None: min_count = arg.min elif arg.min is not None: min_count += arg.min if max_count is None: max_count = arg.max elif arg.max is not None: max_count += arg.max if idx == last_idx: check_max = (arg.max is not None) elif arg.max is None: raise errors.ProgrammerError("Only the last argument can have max=None") if check_max: # Command with exact number of arguments if (min_count is not None and max_count is not None and min_count == max_count and len(args) != min_count): ToStderr("Error: Command %s expects %d argument(s)", cmd, min_count) return False # Command with limited number of arguments if max_count is not None and len(args) > max_count: ToStderr("Error: Command %s expects only %d argument(s)", cmd, max_count) return False # Command with some required arguments if min_count is not None and len(args) < min_count: ToStderr("Error: Command %s expects at least %d argument(s)", cmd, min_count) return False return True def SplitNodeOption(value): """Splits the value of a --node option. """ if value and ":" in value: return value.split(":", 1) else: return (value, None) def CalculateOSNames(os_name, os_variants): """Calculates all the names an OS can be called, according to its variants. @type os_name: string @param os_name: base name of the os @type os_variants: list or None @param os_variants: list of supported variants @rtype: list @return: list of valid names """ if os_variants: return ["%s+%s" % (os_name, v) for v in os_variants] else: return [os_name] def ParseFields(selected, default): """Parses the values of "--field"-like options. @type selected: string or None @param selected: User-selected options @type default: list @param default: Default fields """ if selected is None: return default if selected.startswith("+"): return default + selected[1:].split(",") return selected.split(",") def AskUser(text, choices=None): """Ask the user a question. @param text: the question to ask @param choices: list with elements tuples (input_char, return_value, description); if not given, it will default to: [('y', True, 'Perform the operation'), ('n', False, 'Do no do the operation')]; note that the '?' char is reserved for help @return: one of the return values from the choices list; if input is not possible (i.e. not running with a tty, we return the last entry from the list """ if choices is None: choices = [("y", True, "Perform the operation"), ("n", False, "Do not perform the operation")] if not choices or not isinstance(choices, list): raise errors.ProgrammerError("Invalid choices argument to AskUser") for entry in choices: if not isinstance(entry, tuple) or len(entry) < 3 or entry[0] == "?": raise errors.ProgrammerError("Invalid choices element to AskUser") answer = choices[-1][1] new_text = [] for line in text.splitlines(): new_text.append(textwrap.fill(line, 70, replace_whitespace=False)) text = "\n".join(new_text) try: f = utils.OpenTTY() except IOError: return answer try: chars = [entry[0] for entry in choices] chars[-1] = "[%s]" % chars[-1] chars.append("?") maps = dict([(entry[0], entry[1]) for entry in choices]) while True: f.write(text) f.write("\n") f.write("/".join(chars)) f.write(": ") f.flush() line = f.readline(2).strip().lower() if line in maps: answer = maps[line] break elif line == "?": for entry in choices: f.write(" %s - %s\n" % (entry[0], entry[2])) f.write("\n") continue finally: f.close() return answer def SendJob(ops, cl=None): """Function to submit an opcode without waiting for the results. @type ops: list @param ops: list of opcodes @type cl: luxi.Client @param cl: the luxi client to use for communicating with the master; if None, a new client will be created """ if cl is None: cl = GetClient() job_id = cl.SubmitJob(ops) return job_id def GenericPollJob(job_id, cbs, report_cbs, cancel_fn=None, update_freq=constants.DEFAULT_WFJC_TIMEOUT): """Generic job-polling function. @type job_id: number @param job_id: Job ID @type cbs: Instance of L{JobPollCbBase} @param cbs: Data callbacks @type report_cbs: Instance of L{JobPollReportCbBase} @param report_cbs: Reporting callbacks @type cancel_fn: Function returning a boolean @param cancel_fn: Function to check if we should cancel the running job @type update_freq: int/long @param update_freq: number of seconds between each WFJC reports @return: the opresult of the job @raise errors.JobLost: If job can't be found @raise errors.JobCanceled: If job is canceled @raise errors.OpExecError: If job didn't succeed """ prev_job_info = None prev_logmsg_serial = None status = None should_cancel = False if update_freq <= 0: raise errors.ParameterError("Update frequency must be a positive number") while True: if cancel_fn: timer = 0 while timer < update_freq: result = cbs.WaitForJobChangeOnce(job_id, ["status"], prev_job_info, prev_logmsg_serial, timeout=constants.CLI_WFJC_FREQUENCY) should_cancel = cancel_fn() if should_cancel or not result or result != constants.JOB_NOTCHANGED: break timer += constants.CLI_WFJC_FREQUENCY else: result = cbs.WaitForJobChangeOnce(job_id, ["status"], prev_job_info, prev_logmsg_serial, timeout=update_freq) if not result: # job not found, go away! raise errors.JobLost("Job with id %s lost" % job_id) if should_cancel: logging.info("Job %s canceled because the client timed out.", job_id) cbs.CancelJob(job_id) raise errors.JobCanceled("Job was canceled") if result == constants.JOB_NOTCHANGED: report_cbs.ReportNotChanged(job_id, status) # Wait again continue # Split result, a tuple of (field values, log entries) (job_info, log_entries) = result (status, ) = job_info if log_entries: for log_entry in log_entries: (serial, timestamp, log_type, message) = log_entry report_cbs.ReportLogMessage(job_id, serial, timestamp, log_type, message) prev_logmsg_serial = serial if prev_logmsg_serial is None \ else max(prev_logmsg_serial, serial) # TODO: Handle canceled and archived jobs elif status in (constants.JOB_STATUS_SUCCESS, constants.JOB_STATUS_ERROR, constants.JOB_STATUS_CANCELING, constants.JOB_STATUS_CANCELED): break prev_job_info = job_info jobs = cbs.QueryJobs([job_id], ["status", "opstatus", "opresult"]) if not jobs: raise errors.JobLost("Job with id %s lost" % job_id) status, opstatus, result = jobs[0] if status == constants.JOB_STATUS_SUCCESS: return result if status in (constants.JOB_STATUS_CANCELING, constants.JOB_STATUS_CANCELED): raise errors.JobCanceled("Job was canceled") has_ok = False for idx, (status, msg) in enumerate(zip(opstatus, result)): if status == constants.OP_STATUS_SUCCESS: has_ok = True elif status == constants.OP_STATUS_ERROR: errors.MaybeRaise(msg) if has_ok: raise errors.OpExecError("partial failure (opcode %d): %s" % (idx, msg)) raise errors.OpExecError(str(msg)) # default failure mode raise errors.OpExecError(result) class JobPollCbBase(object): """Base class for L{GenericPollJob} callbacks. """ def __init__(self): """Initializes this class. """ def WaitForJobChangeOnce(self, job_id, fields, prev_job_info, prev_log_serial, timeout=constants.DEFAULT_WFJC_TIMEOUT): """Waits for changes on a job. """ raise NotImplementedError() def QueryJobs(self, job_ids, fields): """Returns the selected fields for the selected job IDs. @type job_ids: list of numbers @param job_ids: Job IDs @type fields: list of strings @param fields: Fields """ raise NotImplementedError() def CancelJob(self, job_id): """Cancels a currently running job. @type job_id: number @param job_id: The ID of the Job we want to cancel """ raise NotImplementedError() class JobPollReportCbBase(object): """Base class for L{GenericPollJob} reporting callbacks. """ def __init__(self): """Initializes this class. """ def ReportLogMessage(self, job_id, serial, timestamp, log_type, log_msg): """Handles a log message. """ raise NotImplementedError() def ReportNotChanged(self, job_id, status): """Called for if a job hasn't changed in a while. @type job_id: number @param job_id: Job ID @type status: string or None @param status: Job status if available """ raise NotImplementedError() class _LuxiJobPollCb(JobPollCbBase): def __init__(self, cl): """Initializes this class. """ JobPollCbBase.__init__(self) self.cl = cl def WaitForJobChangeOnce(self, job_id, fields, prev_job_info, prev_log_serial, timeout=constants.DEFAULT_WFJC_TIMEOUT): """Waits for changes on a job. """ return self.cl.WaitForJobChangeOnce(job_id, fields, prev_job_info, prev_log_serial, timeout=timeout) def QueryJobs(self, job_ids, fields): """Returns the selected fields for the selected job IDs. """ return self.cl.QueryJobs(job_ids, fields) def CancelJob(self, job_id): """Cancels a currently running job. """ return self.cl.CancelJob(job_id) class FeedbackFnJobPollReportCb(JobPollReportCbBase): def __init__(self, feedback_fn): """Initializes this class. """ JobPollReportCbBase.__init__(self) self.feedback_fn = feedback_fn assert callable(feedback_fn) def ReportLogMessage(self, job_id, serial, timestamp, log_type, log_msg): """Handles a log message. """ self.feedback_fn((timestamp, log_type, log_msg)) def ReportNotChanged(self, job_id, status): """Called if a job hasn't changed in a while. """ # Ignore class StdioJobPollReportCb(JobPollReportCbBase): def __init__(self): """Initializes this class. """ JobPollReportCbBase.__init__(self) self.notified_queued = False self.notified_waitlock = False def ReportLogMessage(self, job_id, serial, timestamp, log_type, log_msg): """Handles a log message. """ ToStdout("%s %s", time.ctime(utils.MergeTime(timestamp)), FormatLogMessage(log_type, log_msg)) def ReportNotChanged(self, job_id, status): """Called if a job hasn't changed in a while. """ if status is None: return if status == constants.JOB_STATUS_QUEUED and not self.notified_queued: ToStderr("Job %s is waiting in queue", job_id) self.notified_queued = True elif status == constants.JOB_STATUS_WAITING and not self.notified_waitlock: ToStderr("Job %s is trying to acquire all necessary locks", job_id) self.notified_waitlock = True def FormatLogMessage(log_type, log_msg): """Formats a job message according to its type. """ if log_type != constants.ELOG_MESSAGE: log_msg = str(log_msg) return utils.SafeEncode(log_msg) def PollJob(job_id, cl=None, feedback_fn=None, reporter=None, cancel_fn=None, update_freq=constants.DEFAULT_WFJC_TIMEOUT): """Function to poll for the result of a job. @type job_id: job identified @param job_id: the job to poll for results @type cl: luxi.Client @param cl: the luxi client to use for communicating with the master; if None, a new client will be created @type cancel_fn: Function returning a boolean @param cancel_fn: Function to check if we should cancel the running job @type update_freq: int/long @param update_freq: number of seconds between each WFJC report """ if cl is None: cl = GetClient() if reporter is None: if feedback_fn: reporter = FeedbackFnJobPollReportCb(feedback_fn) else: reporter = StdioJobPollReportCb() elif feedback_fn: raise errors.ProgrammerError("Can't specify reporter and feedback function") return GenericPollJob(job_id, _LuxiJobPollCb(cl), reporter, cancel_fn=cancel_fn, update_freq=update_freq) def SubmitOpCode(op, cl=None, feedback_fn=None, opts=None, reporter=None): """Legacy function to submit an opcode. This is just a simple wrapper over the construction of the processor instance. It should be extended to better handle feedback and interaction functions. """ if cl is None: cl = GetClient() SetGenericOpcodeOpts([op], opts) job_id = SendJob([op], cl=cl) if hasattr(opts, "print_jobid") and opts.print_jobid: ToStdout("%d" % job_id) op_results = PollJob(job_id, cl=cl, feedback_fn=feedback_fn, reporter=reporter) return op_results[0] def SubmitOpCodeToDrainedQueue(op): """Forcefully insert a job in the queue, even if it is drained. """ cl = GetClient() job_id = cl.SubmitJobToDrainedQueue([op]) op_results = PollJob(job_id, cl=cl) return op_results[0] def SubmitOrSend(op, opts, cl=None, feedback_fn=None): """Wrapper around SubmitOpCode or SendJob. This function will decide, based on the 'opts' parameter, whether to submit and wait for the result of the opcode (and return it), or whether to just send the job and print its identifier. It is used in order to simplify the implementation of the '--submit' option. It will also process the opcodes if we're sending the via SendJob (otherwise SubmitOpCode does it). """ if opts and opts.submit_only: job = [op] SetGenericOpcodeOpts(job, opts) job_id = SendJob(job, cl=cl) if opts.print_jobid: ToStdout("%d" % job_id) raise errors.JobSubmittedException(job_id) else: return SubmitOpCode(op, cl=cl, feedback_fn=feedback_fn, opts=opts) def _InitReasonTrail(op, opts): """Builds the first part of the reason trail Builds the initial part of the reason trail, adding the user provided reason (if it exists) and the name of the command starting the operation. @param op: the opcode the reason trail will be added to @param opts: the command line options selected by the user """ assert len(sys.argv) >= 2 trail = [] if opts.reason: trail.append((constants.OPCODE_REASON_SRC_USER, opts.reason, utils.EpochNano())) binary = os.path.basename(sys.argv[0]) source = "%s:%s" % (constants.OPCODE_REASON_SRC_CLIENT, binary) command = sys.argv[1] trail.append((source, command, utils.EpochNano())) op.reason = trail def SetGenericOpcodeOpts(opcode_list, options): """Processor for generic options. This function updates the given opcodes based on generic command line options (like debug, dry-run, etc.). @param opcode_list: list of opcodes @param options: command line options or None @return: None (in-place modification) """ if not options: return for op in opcode_list: op.debug_level = options.debug if hasattr(options, "dry_run"): op.dry_run = options.dry_run if getattr(options, "priority", None) is not None: op.priority = options.priority _InitReasonTrail(op, options) def FormatError(err): """Return a formatted error message for a given error. This function takes an exception instance and returns a tuple consisting of two values: first, the recommended exit code, and second, a string describing the error message (not newline-terminated). """ retcode = 1 obuf = StringIO() msg = str(err) if isinstance(err, errors.ConfigurationError): txt = "Corrupt configuration file: %s" % msg logging.error(txt) obuf.write(txt + "\n") obuf.write("Aborting.") retcode = 2 elif isinstance(err, errors.HooksAbort): obuf.write("Failure: hooks execution failed:\n") for node, script, out in err.args[0]: if out: obuf.write(" node: %s, script: %s, output: %s\n" % (node, script, out)) else: obuf.write(" node: %s, script: %s (no output)\n" % (node, script)) elif isinstance(err, errors.HooksFailure): obuf.write("Failure: hooks general failure: %s" % msg) elif isinstance(err, errors.ResolverError): this_host = netutils.Hostname.GetSysName() if err.args[0] == this_host: msg = "Failure: can't resolve my own hostname ('%s')" else: msg = "Failure: can't resolve hostname '%s'" obuf.write(msg % err.args[0]) elif isinstance(err, errors.OpPrereqError): if len(err.args) == 2: obuf.write("Failure: prerequisites not met for this" " operation:\nerror type: %s, error details:\n%s" % (err.args[1], err.args[0])) else: obuf.write("Failure: prerequisites not met for this" " operation:\n%s" % msg) elif isinstance(err, errors.OpExecError): obuf.write("Failure: command execution error:\n%s" % msg) elif isinstance(err, errors.TagError): obuf.write("Failure: invalid tag(s) given:\n%s" % msg) elif isinstance(err, errors.JobQueueDrainError): obuf.write("Failure: the job queue is marked for drain and doesn't" " accept new requests\n") elif isinstance(err, errors.JobQueueFull): obuf.write("Failure: the job queue is full and doesn't accept new" " job submissions until old jobs are archived\n") elif isinstance(err, errors.TypeEnforcementError): obuf.write("Parameter Error: %s" % msg) elif isinstance(err, errors.ParameterError): obuf.write("Failure: unknown/wrong parameter name '%s'" % msg) elif isinstance(err, rpcerr.NoMasterError): if err.args[0] == pathutils.MASTER_SOCKET: daemon = "the master daemon" elif err.args[0] == pathutils.QUERY_SOCKET: daemon = "the config daemon" else: daemon = "socket '%s'" % str(err.args[0]) obuf.write("Cannot communicate with %s.\nIs the process running" " and listening for connections?" % daemon) elif isinstance(err, rpcerr.TimeoutError): obuf.write("Timeout while talking to the master daemon. Jobs might have" " been submitted and will continue to run even if the call" " timed out. Useful commands in this situation are \"gnt-job" " list\", \"gnt-job cancel\" and \"gnt-job watch\". Error:\n") obuf.write(msg) elif isinstance(err, rpcerr.PermissionError): obuf.write("It seems you don't have permissions to connect to the" " master daemon.\nPlease retry as a different user.") elif isinstance(err, rpcerr.ProtocolError): obuf.write("Unhandled protocol error while talking to the master daemon:\n" "%s" % msg) elif isinstance(err, errors.JobLost): obuf.write("Error checking job status: %s" % msg) elif isinstance(err, errors.QueryFilterParseError): obuf.write("Error while parsing query filter: %s\n" % err.args[0]) obuf.write("\n".join(err.GetDetails())) elif isinstance(err, errors.GenericError): obuf.write("Unhandled Ganeti error: %s" % msg) elif isinstance(err, errors.JobSubmittedException): obuf.write("JobID: %s\n" % err.args[0]) retcode = 0 else: obuf.write("Unhandled exception: %s" % msg) return retcode, obuf.getvalue().rstrip("\n") def GenericMain(commands, override=None, aliases=None, env_override=frozenset()): """Generic main function for all the gnt-* commands. @param commands: a dictionary with a special structure, see the design doc for command line handling. @param override: if not None, we expect a dictionary with keys that will override command line options; this can be used to pass options from the scripts to generic functions @param aliases: dictionary with command aliases {'alias': 'target, ...} @param env_override: list of environment names which are allowed to submit default args for commands """ # save the program name and the entire command line for later logging if sys.argv: binary = os.path.basename(sys.argv[0]) if not binary: binary = sys.argv[0] if len(sys.argv) >= 2: logname = utils.ShellQuoteArgs([binary, sys.argv[1]]) else: logname = binary cmdline = utils.ShellQuoteArgs([binary] + sys.argv[1:]) else: binary = "" cmdline = "" if aliases is None: aliases = {} try: (func, options, args) = _ParseArgs(binary, sys.argv, commands, aliases, env_override) except _ShowVersion: ToStdout("%s (ganeti %s) %s", binary, constants.VCS_VERSION, constants.RELEASE_VERSION) return constants.EXIT_SUCCESS except _ShowUsage as err: for line in _FormatUsage(binary, commands): ToStdout(line) if err.exit_error: return constants.EXIT_FAILURE else: return constants.EXIT_SUCCESS except errors.ParameterError as err: result, err_msg = FormatError(err) ToStderr(err_msg) return 1 if func is None: # parse error return 1 if override is not None: for key, val in override.items(): setattr(options, key, val) utils.SetupLogging(pathutils.LOG_COMMANDS, logname, debug=options.debug, stderr_logging=True) logging.debug("Command line: %s", cmdline) try: result = func(options, args) except (errors.GenericError, rpcerr.ProtocolError, errors.JobSubmittedException) as err: result, err_msg = FormatError(err) logging.exception("Error during command processing") ToStderr(err_msg) except KeyboardInterrupt: result = constants.EXIT_FAILURE ToStderr("Aborted. Note that if the operation created any jobs, they" " might have been submitted and" " will continue to run in the background.") except IOError as err: if err.errno == errno.EPIPE: # our terminal went away, we'll exit sys.exit(constants.EXIT_FAILURE) else: raise return result def ParseNicOption(optvalue): """Parses the value of the --net option(s). """ try: nic_max = max(int(nidx[0]) + 1 for nidx in optvalue) except (TypeError, ValueError) as err: raise errors.OpPrereqError("Invalid NIC index passed: %s" % str(err), errors.ECODE_INVAL) nics = [{}] * nic_max for nidx, ndict in optvalue: nidx = int(nidx) if not isinstance(ndict, dict): raise errors.OpPrereqError("Invalid nic/%d value: expected dict," " got %s" % (nidx, ndict), errors.ECODE_INVAL) utils.ForceDictType(ndict, constants.INIC_PARAMS_TYPES) nics[nidx] = ndict return nics def FixHvParams(hvparams): # In Ganeti 2.8.4 the separator for the usb_devices hvparam was changed from # comma to space because commas cannot be accepted on the command line # (they already act as the separator between different hvparams). Still, # RAPI should be able to accept commas for backwards compatibility. # Therefore, we convert spaces into commas here, and we keep the old # parsing logic everywhere else. try: new_usb_devices = hvparams[constants.HV_USB_DEVICES].replace(" ", ",") hvparams[constants.HV_USB_DEVICES] = new_usb_devices except KeyError: #No usb_devices, no modification required pass def GenericInstanceCreate(mode, opts, args): """Add an instance to the cluster via either creation or import. @param mode: constants.INSTANCE_CREATE or constants.INSTANCE_IMPORT @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the new instance name @rtype: int @return: the desired exit code """ instance = args[0] forthcoming = opts.ensure_value("forthcoming", False) commit = opts.ensure_value("commit", False) if forthcoming and commit: raise errors.OpPrereqError("Creating an instance only forthcoming and" " commiting it are mutally exclusive", errors.ECODE_INVAL) (pnode, snode) = SplitNodeOption(opts.node) hypervisor = None hvparams = {} if opts.hypervisor: hypervisor, hvparams = opts.hypervisor if opts.nics: nics = ParseNicOption(opts.nics) elif opts.no_nics: # no nics nics = [] elif mode == constants.INSTANCE_CREATE: # default of one nic, all auto nics = [{}] else: # mode == import nics = [] if opts.disk_template == constants.DT_DISKLESS: if opts.disks or opts.sd_size is not None: raise errors.OpPrereqError("Diskless instance but disk" " information passed", errors.ECODE_INVAL) disks = [] else: if (not opts.disks and not opts.sd_size and mode == constants.INSTANCE_CREATE): raise errors.OpPrereqError("No disk information specified", errors.ECODE_INVAL) if opts.disks and opts.sd_size is not None: raise errors.OpPrereqError("Please use either the '--disk' or" " '-s' option", errors.ECODE_INVAL) if opts.sd_size is not None: opts.disks = [(0, {constants.IDISK_SIZE: opts.sd_size})] if opts.disks: try: disk_max = max(int(didx[0]) + 1 for didx in opts.disks) except ValueError as err: raise errors.OpPrereqError("Invalid disk index passed: %s" % str(err), errors.ECODE_INVAL) disks = [{}] * disk_max else: disks = [] for didx, ddict in opts.disks: didx = int(didx) if not isinstance(ddict, dict): msg = "Invalid disk/%d value: expected dict, got %s" % (didx, ddict) raise errors.OpPrereqError(msg, errors.ECODE_INVAL) elif constants.IDISK_SIZE in ddict: if constants.IDISK_ADOPT in ddict: raise errors.OpPrereqError("Only one of 'size' and 'adopt' allowed" " (disk %d)" % didx, errors.ECODE_INVAL) try: ddict[constants.IDISK_SIZE] = \ utils.ParseUnit(ddict[constants.IDISK_SIZE]) except ValueError as err: raise errors.OpPrereqError("Invalid disk size for disk %d: %s" % (didx, err), errors.ECODE_INVAL) elif constants.IDISK_ADOPT in ddict: if constants.IDISK_SPINDLES in ddict: raise errors.OpPrereqError("spindles is not a valid option when" " adopting a disk", errors.ECODE_INVAL) if mode == constants.INSTANCE_IMPORT: raise errors.OpPrereqError("Disk adoption not allowed for instance" " import", errors.ECODE_INVAL) ddict[constants.IDISK_SIZE] = 0 else: raise errors.OpPrereqError("Missing size or adoption source for" " disk %d" % didx, errors.ECODE_INVAL) if constants.IDISK_SPINDLES in ddict: ddict[constants.IDISK_SPINDLES] = int(ddict[constants.IDISK_SPINDLES]) disks[didx] = ddict if opts.tags is not None: tags = opts.tags.split(",") else: tags = [] utils.ForceDictType(opts.beparams, constants.BES_PARAMETER_COMPAT) utils.ForceDictType(hvparams, constants.HVS_PARAMETER_TYPES) FixHvParams(hvparams) osparams_private = opts.osparams_private or serializer.PrivateDict() osparams_secret = opts.osparams_secret or serializer.PrivateDict() helper_startup_timeout = opts.helper_startup_timeout helper_shutdown_timeout = opts.helper_shutdown_timeout if mode == constants.INSTANCE_CREATE: start = opts.start os_type = opts.os force_variant = opts.force_variant src_node = None src_path = None no_install = opts.no_install identify_defaults = False compress = constants.IEC_NONE if opts.instance_communication is None: instance_communication = False else: instance_communication = opts.instance_communication elif mode == constants.INSTANCE_IMPORT: if forthcoming: raise errors.OpPrereqError("forthcoming instances can only be created," " not imported") start = False os_type = None force_variant = False src_node = opts.src_node src_path = opts.src_dir no_install = None identify_defaults = opts.identify_defaults compress = opts.compress instance_communication = False else: raise errors.ProgrammerError("Invalid creation mode %s" % mode) op = opcodes.OpInstanceCreate( forthcoming=forthcoming, commit=commit, instance_name=instance, disks=disks, disk_template=opts.disk_template, group_name=opts.nodegroup, nics=nics, conflicts_check=opts.conflicts_check, pnode=pnode, snode=snode, ip_check=opts.ip_check, name_check=opts.name_check, wait_for_sync=opts.wait_for_sync, file_storage_dir=opts.file_storage_dir, file_driver=opts.file_driver, iallocator=opts.iallocator, hypervisor=hypervisor, hvparams=hvparams, beparams=opts.beparams, osparams=opts.osparams, osparams_private=osparams_private, osparams_secret=osparams_secret, mode=mode, opportunistic_locking=opts.opportunistic_locking, start=start, os_type=os_type, force_variant=force_variant, src_node=src_node, src_path=src_path, compress=compress, tags=tags, no_install=no_install, identify_defaults=identify_defaults, ignore_ipolicy=opts.ignore_ipolicy, instance_communication=instance_communication, helper_startup_timeout=helper_startup_timeout, helper_shutdown_timeout=helper_shutdown_timeout) SubmitOrSend(op, opts) return 0 class _RunWhileDaemonsStoppedHelper(object): """Helper class for L{RunWhileDaemonsStopped} to simplify state management """ def __init__(self, feedback_fn, cluster_name, master_node, online_nodes, ssh_ports, exclude_daemons, debug, verbose): """Initializes this class. @type feedback_fn: callable @param feedback_fn: Feedback function @type cluster_name: string @param cluster_name: Cluster name @type master_node: string @param master_node Master node name @type online_nodes: list @param online_nodes: List of names of online nodes @type ssh_ports: list @param ssh_ports: List of SSH ports of online nodes @type exclude_daemons: list of string @param exclude_daemons: list of daemons that will be restarted on master after all others are shutdown @type debug: boolean @param debug: show debug output @type verbose: boolesn @param verbose: show verbose output """ self.feedback_fn = feedback_fn self.cluster_name = cluster_name self.master_node = master_node self.online_nodes = online_nodes self.ssh_ports = dict(zip(online_nodes, ssh_ports)) self.ssh = ssh.SshRunner(self.cluster_name) self.nonmaster_nodes = [name for name in online_nodes if name != master_node] self.exclude_daemons = exclude_daemons self.debug = debug self.verbose = verbose assert self.master_node not in self.nonmaster_nodes def _RunCmd(self, node_name, cmd): """Runs a command on the local or a remote machine. @type node_name: string @param node_name: Machine name @type cmd: list @param cmd: Command """ if node_name is None or node_name == self.master_node: # No need to use SSH result = utils.RunCmd(cmd) else: result = self.ssh.Run(node_name, constants.SSH_LOGIN_USER, utils.ShellQuoteArgs(cmd), port=self.ssh_ports[node_name]) if result.failed: errmsg = ["Failed to run command %s" % result.cmd] if node_name: errmsg.append("on node %s" % node_name) errmsg.append(": exitcode %s and error %s" % (result.exit_code, result.output)) raise errors.OpExecError(" ".join(errmsg)) def Call(self, fn, *args): """Call function while all daemons are stopped. @type fn: callable @param fn: Function to be called """ # Pause watcher by acquiring an exclusive lock on watcher state file self.feedback_fn("Blocking watcher") watcher_block = utils.FileLock.Open(pathutils.WATCHER_LOCK_FILE) try: # TODO: Currently, this just blocks. There's no timeout. # TODO: Should it be a shared lock? watcher_block.Exclusive(blocking=True) # Stop master daemons, so that no new jobs can come in and all running # ones are finished self.feedback_fn("Stopping master daemons") self._RunCmd(None, [pathutils.DAEMON_UTIL, "stop-master"]) try: # Stop daemons on all nodes online_nodes = [self.master_node] + [n for n in self.online_nodes if n != self.master_node] for node_name in online_nodes: self.feedback_fn("Stopping daemons on %s" % node_name) self._RunCmd(node_name, [pathutils.DAEMON_UTIL, "stop-all"]) # Starting any daemons listed as exception if node_name == self.master_node: for daemon in self.exclude_daemons: self.feedback_fn("Starting daemon '%s' on %s" % (daemon, node_name)) self._RunCmd(node_name, [pathutils.DAEMON_UTIL, "start", daemon]) # All daemons are shut down now try: return fn(self, *args) except Exception as err: _, errmsg = FormatError(err) logging.exception("Caught exception") self.feedback_fn(errmsg) raise finally: # Start cluster again, master node last for node_name in self.nonmaster_nodes + [self.master_node]: # Stopping any daemons listed as exception. # This might look unnecessary, but it makes sure that daemon-util # starts all daemons in the right order. if node_name == self.master_node: self.exclude_daemons.reverse() for daemon in self.exclude_daemons: self.feedback_fn("Stopping daemon '%s' on %s" % (daemon, node_name)) self._RunCmd(node_name, [pathutils.DAEMON_UTIL, "stop", daemon]) self.feedback_fn("Starting daemons on %s" % node_name) self._RunCmd(node_name, [pathutils.DAEMON_UTIL, "start-all"]) finally: # Resume watcher watcher_block.Close() def RunWhileDaemonsStopped(feedback_fn, exclude_daemons, fn, *args, **kwargs): """Calls a function while all cluster daemons are stopped. @type feedback_fn: callable @param feedback_fn: Feedback function @type exclude_daemons: list of string @param exclude_daemons: list of daemons that stopped, but immediately restarted on the master to be available when calling 'fn'. If None, all daemons will be stopped and none will be started before calling 'fn'. @type fn: callable @param fn: Function to be called when daemons are stopped """ feedback_fn("Gathering cluster information") # This ensures we're running on the master daemon cl = GetClient() (cluster_name, master_node) = \ cl.QueryConfigValues(["cluster_name", "master_node"]) online_nodes = GetOnlineNodes([], cl=cl) ssh_ports = GetNodesSshPorts(online_nodes, cl) # Don't keep a reference to the client. The master daemon will go away. del cl assert master_node in online_nodes if exclude_daemons is None: exclude_daemons = [] debug = kwargs.get("debug", False) verbose = kwargs.get("verbose", False) return _RunWhileDaemonsStoppedHelper( feedback_fn, cluster_name, master_node, online_nodes, ssh_ports, exclude_daemons, debug, verbose).Call(fn, *args) def RunWhileClusterStopped(feedback_fn, fn, *args): """Calls a function while all cluster daemons are stopped. @type feedback_fn: callable @param feedback_fn: Feedback function @type fn: callable @param fn: Function to be called when daemons are stopped """ RunWhileDaemonsStopped(feedback_fn, None, fn, *args) def GenerateTable(headers, fields, separator, data, numfields=None, unitfields=None, units=None): """Prints a table with headers and different fields. @type headers: dict @param headers: dictionary mapping field names to headers for the table @type fields: list @param fields: the field names corresponding to each row in the data field @param separator: the separator to be used; if this is None, the default 'smart' algorithm is used which computes optimal field width, otherwise just the separator is used between each field @type data: list @param data: a list of lists, each sublist being one row to be output @type numfields: list @param numfields: a list with the fields that hold numeric values and thus should be right-aligned @type unitfields: list @param unitfields: a list with the fields that hold numeric values that should be formatted with the units field @type units: string or None @param units: the units we should use for formatting, or None for automatic choice (human-readable for non-separator usage, otherwise megabytes); this is a one-letter string """ if units is None: if separator: units = "m" else: units = "h" if numfields is None: numfields = [] if unitfields is None: unitfields = [] numfields = utils.FieldSet(*numfields) unitfields = utils.FieldSet(*unitfields) format_fields = [] for field in fields: if headers and field not in headers: # TODO: handle better unknown fields (either revert to old # style of raising exception, or deal more intelligently with # variable fields) headers[field] = field if separator is not None: format_fields.append("%s") elif numfields.Matches(field): format_fields.append("%*s") else: format_fields.append("%-*s") if separator is None: mlens = [0 for name in fields] format_str = " ".join(format_fields) else: format_str = separator.replace("%", "%%").join(format_fields) for row in data: if row is None: continue for idx, val in enumerate(row): if unitfields.Matches(fields[idx]): try: val = int(val) except (TypeError, ValueError): pass else: val = row[idx] = utils.FormatUnit(val, units) val = row[idx] = str(val) if separator is None: mlens[idx] = max(mlens[idx], len(val)) result = [] if headers: args = [] for idx, name in enumerate(fields): hdr = headers[name] if separator is None: mlens[idx] = max(mlens[idx], len(hdr)) args.append(mlens[idx]) args.append(hdr) result.append(format_str % tuple(args)) if separator is None: assert len(mlens) == len(fields) if fields and not numfields.Matches(fields[-1]): mlens[-1] = 0 for line in data: args = [] if line is None: line = ["-" for _ in fields] for idx in range(len(fields)): if separator is None: args.append(mlens[idx]) args.append(line[idx]) result.append(format_str % tuple(args)) return result def _FormatBool(value): """Formats a boolean value as a string. """ if value: return "Y" return "N" #: Default formatting for query results; (callback, align right) _DEFAULT_FORMAT_QUERY = { constants.QFT_TEXT: (str, False), constants.QFT_BOOL: (_FormatBool, False), constants.QFT_NUMBER: (str, True), constants.QFT_NUMBER_FLOAT: (str, True), constants.QFT_TIMESTAMP: (utils.FormatTime, False), constants.QFT_OTHER: (str, False), constants.QFT_UNKNOWN: (str, False), } def _GetColumnFormatter(fdef, override, unit): """Returns formatting function for a field. @type fdef: L{objects.QueryFieldDefinition} @type override: dict @param override: Dictionary for overriding field formatting functions, indexed by field name, contents like L{_DEFAULT_FORMAT_QUERY} @type unit: string @param unit: Unit used for formatting fields of type L{constants.QFT_UNIT} @rtype: tuple; (callable, bool) @return: Returns the function to format a value (takes one parameter) and a boolean for aligning the value on the right-hand side """ fmt = override.get(fdef.name, None) if fmt is not None: return fmt assert constants.QFT_UNIT not in _DEFAULT_FORMAT_QUERY if fdef.kind == constants.QFT_UNIT: # Can't keep this information in the static dictionary return (lambda value: utils.FormatUnit(value, unit), True) fmt = _DEFAULT_FORMAT_QUERY.get(fdef.kind, None) if fmt is not None: return fmt raise NotImplementedError("Can't format column type '%s'" % fdef.kind) class _QueryColumnFormatter(object): """Callable class for formatting fields of a query. """ def __init__(self, fn, status_fn, verbose): """Initializes this class. @type fn: callable @param fn: Formatting function @type status_fn: callable @param status_fn: Function to report fields' status @type verbose: boolean @param verbose: whether to use verbose field descriptions or not """ self._fn = fn self._status_fn = status_fn self._verbose = verbose def __call__(self, data): """Returns a field's string representation. """ (status, value) = data # Report status self._status_fn(status) if status == constants.RS_NORMAL: return self._fn(value) assert value is None, \ "Found value %r for abnormal status %s" % (value, status) return FormatResultError(status, self._verbose) def FormatResultError(status, verbose): """Formats result status other than L{constants.RS_NORMAL}. @param status: The result status @type verbose: boolean @param verbose: Whether to return the verbose text @return: Text of result status """ assert status != constants.RS_NORMAL, \ "FormatResultError called with status equal to constants.RS_NORMAL" try: (verbose_text, normal_text) = constants.RSS_DESCRIPTION[status] except KeyError: raise NotImplementedError("Unknown status %s" % status) else: if verbose: return verbose_text return normal_text def FormatQueryResult(result, unit=None, format_override=None, separator=None, header=False, verbose=False): """Formats data in L{objects.QueryResponse}. @type result: L{objects.QueryResponse} @param result: result of query operation @type unit: string @param unit: Unit used for formatting fields of type L{constants.QFT_UNIT}, see L{utils.text.FormatUnit} @type format_override: dict @param format_override: Dictionary for overriding field formatting functions, indexed by field name, contents like L{_DEFAULT_FORMAT_QUERY} @type separator: string or None @param separator: String used to separate fields @type header: bool @param header: Whether to output header row @type verbose: boolean @param verbose: whether to use verbose field descriptions or not """ if unit is None: if separator: unit = "m" else: unit = "h" if format_override is None: format_override = {} stats = dict.fromkeys(constants.RS_ALL, 0) def _RecordStatus(status): if status in stats: stats[status] += 1 columns = [] for fdef in result.fields: assert fdef.title and fdef.name (fn, align_right) = _GetColumnFormatter(fdef, format_override, unit) columns.append(TableColumn(fdef.title, _QueryColumnFormatter(fn, _RecordStatus, verbose), align_right)) table = FormatTable(result.data, columns, header, separator) # Collect statistics assert len(stats) == len(constants.RS_ALL) assert compat.all(count >= 0 for count in stats.values()) # Determine overall status. If there was no data, unknown fields must be # detected via the field definitions. if (stats[constants.RS_UNKNOWN] or (not result.data and _GetUnknownFields(result.fields))): status = QR_UNKNOWN elif compat.any(count > 0 for key, count in stats.items() if key != constants.RS_NORMAL): status = QR_INCOMPLETE else: status = QR_NORMAL return (status, table) def _GetUnknownFields(fdefs): """Returns list of unknown fields included in C{fdefs}. @type fdefs: list of L{objects.QueryFieldDefinition} """ return [fdef for fdef in fdefs if fdef.kind == constants.QFT_UNKNOWN] def _WarnUnknownFields(fdefs): """Prints a warning to stderr if a query included unknown fields. @type fdefs: list of L{objects.QueryFieldDefinition} """ unknown = _GetUnknownFields(fdefs) if unknown: ToStderr("Warning: Queried for unknown fields %s", utils.CommaJoin(fdef.name for fdef in unknown)) return True return False def GenericList(resource, fields, names, unit, separator, header, cl=None, format_override=None, verbose=False, force_filter=False, namefield=None, qfilter=None, isnumeric=False): """Generic implementation for listing all items of a resource. @param resource: One of L{constants.QR_VIA_LUXI} @type fields: list of strings @param fields: List of fields to query for @type names: list of strings @param names: Names of items to query for @type unit: string or None @param unit: Unit used for formatting fields of type L{constants.QFT_UNIT} or None for automatic choice (human-readable for non-separator usage, otherwise megabytes); this is a one-letter string @type separator: string or None @param separator: String used to separate fields @type header: bool @param header: Whether to show header row @type force_filter: bool @param force_filter: Whether to always treat names as filter @type format_override: dict @param format_override: Dictionary for overriding field formatting functions, indexed by field name, contents like L{_DEFAULT_FORMAT_QUERY} @type verbose: boolean @param verbose: whether to use verbose field descriptions or not @type namefield: string @param namefield: Name of field to use for simple filters (see L{qlang.MakeFilter} for details) @type qfilter: list or None @param qfilter: Query filter (in addition to names) @param isnumeric: bool @param isnumeric: Whether the namefield's type is numeric, and therefore any simple filters built by namefield should use integer values to reflect that """ if not names: names = None namefilter = qlang.MakeFilter(names, force_filter, namefield=namefield, isnumeric=isnumeric) if qfilter is None: qfilter = namefilter elif namefilter is not None: qfilter = [qlang.OP_AND, namefilter, qfilter] if cl is None: cl = GetClient() response = cl.Query(resource, fields, qfilter) found_unknown = _WarnUnknownFields(response.fields) (status, data) = FormatQueryResult(response, unit=unit, separator=separator, header=header, format_override=format_override, verbose=verbose) for line in data: ToStdout(line) assert ((found_unknown and status == QR_UNKNOWN) or (not found_unknown and status != QR_UNKNOWN)) if status == QR_UNKNOWN: return constants.EXIT_UNKNOWN_FIELD # TODO: Should the list command fail if not all data could be collected? return constants.EXIT_SUCCESS def _FieldDescValues(fdef): """Helper function for L{GenericListFields} to get query field description. @type fdef: L{objects.QueryFieldDefinition} @rtype: list """ return [ fdef.name, _QFT_NAMES.get(fdef.kind, fdef.kind), fdef.title, fdef.doc, ] def GenericListFields(resource, fields, separator, header, cl=None): """Generic implementation for listing fields for a resource. @param resource: One of L{constants.QR_VIA_LUXI} @type fields: list of strings @param fields: List of fields to query for @type separator: string or None @param separator: String used to separate fields @type header: bool @param header: Whether to show header row """ if cl is None: cl = GetClient() if not fields: fields = None response = cl.QueryFields(resource, fields) found_unknown = _WarnUnknownFields(response.fields) columns = [ TableColumn("Name", str, False), TableColumn("Type", str, False), TableColumn("Title", str, False), TableColumn("Description", str, False), ] rows = [_FieldDescValues(f) for f in response.fields] for line in FormatTable(rows, columns, header, separator): ToStdout(line) if found_unknown: return constants.EXIT_UNKNOWN_FIELD return constants.EXIT_SUCCESS class TableColumn(object): """Describes a column for L{FormatTable}. """ def __init__(self, title, fn, align_right): """Initializes this class. @type title: string @param title: Column title @type fn: callable @param fn: Formatting function @type align_right: bool @param align_right: Whether to align values on the right-hand side """ self.title = title self.format = fn self.align_right = align_right def _GetColFormatString(width, align_right): """Returns the format string for a field. """ if align_right: sign = "" else: sign = "-" return "%%%s%ss" % (sign, width) def FormatTable(rows, columns, header, separator): """Formats data as a table. @type rows: list of lists @param rows: Row data, one list per row @type columns: list of L{TableColumn} @param columns: Column descriptions @type header: bool @param header: Whether to show header row @type separator: string or None @param separator: String used to separate columns """ if header: data = [[col.title for col in columns]] colwidth = [len(col.title) for col in columns] else: data = [] colwidth = [0 for _ in columns] # Format row data for row in rows: assert len(row) == len(columns) formatted = [col.format(value) for value, col in zip(row, columns)] if separator is None: # Update column widths for idx, (oldwidth, value) in enumerate(zip(colwidth, formatted)): # Modifying a list's items while iterating is fine colwidth[idx] = max(oldwidth, len(value)) data.append(formatted) if separator is not None: # Return early if a separator is used return [separator.join(row) for row in data] if columns and not columns[-1].align_right: # Avoid unnecessary spaces at end of line colwidth[-1] = 0 # Build format string fmt = " ".join([_GetColFormatString(width, col.align_right) for col, width in zip(columns, colwidth)]) return [fmt % tuple(row) for row in data] def FormatTimestamp(ts): """Formats a given timestamp. @type ts: timestamp @param ts: a timeval-type timestamp, a tuple of seconds and microseconds @rtype: string @return: a string with the formatted timestamp """ if not isinstance(ts, (tuple, list)) or len(ts) != 2: return "?" (sec, usecs) = ts return utils.FormatTime(sec, usecs=usecs) def ParseTimespec(value): """Parse a time specification. The following suffixed will be recognized: - s: seconds - m: minutes - h: hours - d: day - w: weeks Without any suffix, the value will be taken to be in seconds. """ value = str(value) if not value: raise errors.OpPrereqError("Empty time specification passed", errors.ECODE_INVAL) suffix_map = { "s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, } if value[-1] not in suffix_map: try: value = int(value) except (TypeError, ValueError): raise errors.OpPrereqError("Invalid time specification '%s'" % value, errors.ECODE_INVAL) else: multiplier = suffix_map[value[-1]] value = value[:-1] if not value: # no data left after stripping the suffix raise errors.OpPrereqError("Invalid time specification (only" " suffix passed)", errors.ECODE_INVAL) try: value = int(value) * multiplier except (TypeError, ValueError): raise errors.OpPrereqError("Invalid time specification '%s'" % value, errors.ECODE_INVAL) return value def GetOnlineNodes(nodes, cl=None, nowarn=False, secondary_ips=False, filter_master=False, nodegroup=None): """Returns the names of online nodes. This function will also log a warning on stderr with the names of the online nodes. @param nodes: if not empty, use only this subset of nodes (minus the offline ones) @param cl: if not None, luxi client to use @type nowarn: boolean @param nowarn: by default, this function will output a note with the offline nodes that are skipped; if this parameter is True the note is not displayed @type secondary_ips: boolean @param secondary_ips: if True, return the secondary IPs instead of the names, useful for doing network traffic over the replication interface (if any) @type filter_master: boolean @param filter_master: if True, do not return the master node in the list (useful in coordination with secondary_ips where we cannot check our node name against the list) @type nodegroup: string @param nodegroup: If set, only return nodes in this node group """ if cl is None: cl = GetClient() qfilter = [] if nodes: qfilter.append(qlang.MakeSimpleFilter("name", nodes)) if nodegroup is not None: qfilter.append([qlang.OP_OR, [qlang.OP_EQUAL, "group", nodegroup], [qlang.OP_EQUAL, "group.uuid", nodegroup]]) if filter_master: qfilter.append([qlang.OP_NOT, [qlang.OP_TRUE, "master"]]) if qfilter: if len(qfilter) > 1: final_filter = [qlang.OP_AND] + qfilter else: assert len(qfilter) == 1 final_filter = qfilter[0] else: final_filter = None result = cl.Query(constants.QR_NODE, ["name", "offline", "sip"], final_filter) def _IsOffline(row): (_, (_, offline), _) = row return offline def _GetName(row): ((_, name), _, _) = row return name def _GetSip(row): (_, _, (_, sip)) = row return sip (offline, online) = compat.partition(result.data, _IsOffline) if offline and not nowarn: ToStderr("Note: skipping offline node(s): %s" % utils.CommaJoin(map(_GetName, offline))) if secondary_ips: fn = _GetSip else: fn = _GetName return [fn(node) for node in online] def GetNodesSshPorts(nodes, cl): """Retrieves SSH ports of given nodes. @param nodes: the names of nodes @type nodes: a list of strings @param cl: a client to use for the query @type cl: L{ganeti.luxi.Client} @return: the list of SSH ports corresponding to the nodes @rtype: a list of tuples """ return [t[0] for t in cl.QueryNodes(names=nodes, fields=["ndp/ssh_port"], use_locking=False)] def GetNodeUUIDs(nodes, cl): """Retrieves the UUIDs of given nodes. @param nodes: the names of nodes @type nodes: a list of string @param cl: a client to use for the query @type cl: L{ganeti.luxi.Client} @return: the list of UUIDs corresponding to the nodes @rtype: a list of tuples """ return [t[0] for t in cl.QueryNodes(names=nodes, fields=["uuid"], use_locking=False)] def _ToStream(stream, txt, *args): """Write a message to a stream, bypassing the logging system @type stream: file object @param stream: the file to which we should write @type txt: str @param txt: the message """ try: if args: args = tuple(args) stream.write(txt % args) else: stream.write(txt) stream.write("\n") stream.flush() except IOError as err: if err.errno == errno.EPIPE: # our terminal went away, we'll exit sys.exit(constants.EXIT_FAILURE) else: raise def ToStdout(txt, *args): """Write a message to stdout only, bypassing the logging system This is just a wrapper over _ToStream. @type txt: str @param txt: the message """ _ToStream(sys.stdout, txt, *args) def ToStdoutAndLoginfo(txt, *args): """Write a message to stdout and additionally log it at INFO level""" ToStdout(txt, *args) logging.info(txt, *args) def ToStderr(txt, *args): """Write a message to stderr only, bypassing the logging system This is just a wrapper over _ToStream. @type txt: str @param txt: the message """ _ToStream(sys.stderr, txt, *args) class JobExecutor(object): """Class which manages the submission and execution of multiple jobs. Note that instances of this class should not be reused between GetResults() calls. """ def __init__(self, cl=None, verbose=True, opts=None, feedback_fn=None): self.queue = [] if cl is None: cl = GetClient() self.cl = cl self.verbose = verbose self.jobs = [] self.opts = opts self.feedback_fn = feedback_fn self._counter = itertools.count() @staticmethod def _IfName(name, fmt): """Helper function for formatting name. """ if name: return fmt % name return "" def QueueJob(self, name, *ops): """Record a job for later submit. @type name: string @param name: a description of the job, will be used in WaitJobSet """ SetGenericOpcodeOpts(ops, self.opts) self.queue.append((next(self._counter), name, ops)) def AddJobId(self, name, status, job_id): """Adds a job ID to the internal queue. """ self.jobs.append((next(self._counter), status, job_id, name)) def SubmitPending(self, each=False): """Submit all pending jobs. """ if each: results = [] for (_, _, ops) in self.queue: # SubmitJob will remove the success status, but raise an exception if # the submission fails, so we'll notice that anyway. results.append([True, self.cl.SubmitJob(ops)[0]]) else: results = self.cl.SubmitManyJobs([ops for (_, _, ops) in self.queue]) for ((status, data), (idx, name, _)) in zip(results, self.queue): self.jobs.append((idx, status, data, name)) def _ChooseJob(self): """Choose a non-waiting/queued job to poll next. """ assert self.jobs, "_ChooseJob called with empty job list" result = self.cl.QueryJobs([i[2] for i in self.jobs[:_CHOOSE_BATCH]], ["status"]) assert result for job_data, status in zip(self.jobs, result): if (isinstance(status, list) and status and status[0] in (constants.JOB_STATUS_QUEUED, constants.JOB_STATUS_WAITING, constants.JOB_STATUS_CANCELING)): # job is still present and waiting continue # good candidate found (either running job or lost job) self.jobs.remove(job_data) return job_data # no job found return self.jobs.pop(0) def GetResults(self): """Wait for and return the results of all jobs. @rtype: list @return: list of tuples (success, job results), in the same order as the submitted jobs; if a job has failed, instead of the result there will be the error message """ if not self.jobs: self.SubmitPending() results = [] if self.verbose: ok_jobs = [row[2] for row in self.jobs if row[1]] if ok_jobs: ToStdout("Submitted jobs %s", utils.CommaJoin(ok_jobs)) # first, remove any non-submitted jobs self.jobs, failures = compat.partition(self.jobs, lambda x: x[1]) for idx, _, jid, name in failures: ToStderr("Failed to submit job%s: %s", self._IfName(name, " for %s"), jid) results.append((idx, False, jid)) while self.jobs: (idx, _, jid, name) = self._ChooseJob() ToStdout("Waiting for job %s%s ...", jid, self._IfName(name, " for %s")) try: job_result = PollJob(jid, cl=self.cl, feedback_fn=self.feedback_fn) success = True except errors.JobLost as err: _, job_result = FormatError(err) ToStderr("Job %s%s has been archived, cannot check its result", jid, self._IfName(name, " for %s")) success = False except (errors.GenericError, rpcerr.ProtocolError) as err: _, job_result = FormatError(err) success = False # the error message will always be shown, verbose or not ToStderr("Job %s%s has failed: %s", jid, self._IfName(name, " for %s"), job_result) results.append((idx, success, job_result)) # sort based on the index, then drop it results.sort() results = [i[1:] for i in results] return results def WaitOrShow(self, wait): """Wait for job results or only print the job IDs. @type wait: boolean @param wait: whether to wait or not """ if wait: return self.GetResults() else: if not self.jobs: self.SubmitPending() for _, status, result, name in self.jobs: if status: ToStdout("%s: %s", result, name) else: ToStderr("Failure for %s: %s", name, result) return [row[1:3] for row in self.jobs] def FormatParamsDictInfo(param_dict, actual, roman=False): """Formats a parameter dictionary. @type param_dict: dict @param param_dict: the own parameters @type actual: dict @param actual: the current parameter set (including defaults) @rtype: dict @return: dictionary where the value of each parameter is either a fully formatted string or a dictionary containing formatted strings """ ret = {} for (key, data) in actual.items(): if isinstance(data, dict) and data: ret[key] = FormatParamsDictInfo(param_dict.get(key, {}), data, roman) else: default_str = "default (%s)" % compat.TryToRoman(data, roman) ret[key] = str(compat.TryToRoman(param_dict.get(key, default_str), roman)) return ret def _FormatListInfoDefault(data, def_data): if data is not None: ret = utils.CommaJoin(data) else: ret = "default (%s)" % utils.CommaJoin(def_data) return ret def FormatPolicyInfo(custom_ipolicy, eff_ipolicy, iscluster, roman=False): """Formats an instance policy. @type custom_ipolicy: dict @param custom_ipolicy: own policy @type eff_ipolicy: dict @param eff_ipolicy: effective policy (including defaults); ignored for cluster @type iscluster: bool @param iscluster: the policy is at cluster level @type roman: bool @param roman: whether to print the values in roman numerals @rtype: list of pairs @return: formatted data, suitable for L{PrintGenericInfo} """ if iscluster: eff_ipolicy = custom_ipolicy minmax_out = [] custom_minmax = custom_ipolicy.get(constants.ISPECS_MINMAX) if custom_minmax: for (k, minmax) in enumerate(custom_minmax): minmax_out.append([ ("%s/%s" % (key, k), FormatParamsDictInfo(minmax[key], minmax[key], roman)) for key in constants.ISPECS_MINMAX_KEYS ]) else: for (k, minmax) in enumerate(eff_ipolicy[constants.ISPECS_MINMAX]): minmax_out.append([ ("%s/%s" % (key, k), FormatParamsDictInfo({}, minmax[key], roman)) for key in constants.ISPECS_MINMAX_KEYS ]) ret = [("bounds specs", minmax_out)] if iscluster: stdspecs = custom_ipolicy[constants.ISPECS_STD] ret.append( (constants.ISPECS_STD, FormatParamsDictInfo(stdspecs, stdspecs, roman)) ) ret.append( ("allowed disk templates", _FormatListInfoDefault(custom_ipolicy.get(constants.IPOLICY_DTS), eff_ipolicy[constants.IPOLICY_DTS])) ) to_roman = compat.TryToRoman ret.extend([ (key, str(to_roman(custom_ipolicy.get(key, "default (%s)" % eff_ipolicy[key]), roman))) for key in constants.IPOLICY_PARAMETERS ]) return ret def _PrintSpecsParameters(buf, specs): values = ("%s=%s" % (par, val) for (par, val) in sorted(specs.items())) buf.write(",".join(values)) def PrintIPolicyCommand(buf, ipolicy, isgroup): """Print the command option used to generate the given instance policy. Currently only the parts dealing with specs are supported. @type buf: StringIO @param buf: stream to write into @type ipolicy: dict @param ipolicy: instance policy @type isgroup: bool @param isgroup: whether the policy is at group level """ if not isgroup: stdspecs = ipolicy.get("std") if stdspecs: buf.write(" %s " % IPOLICY_STD_SPECS_STR) _PrintSpecsParameters(buf, stdspecs) minmaxes = ipolicy.get("minmax", []) first = True for minmax in minmaxes: minspecs = minmax.get("min") maxspecs = minmax.get("max") if minspecs and maxspecs: if first: buf.write(" %s " % IPOLICY_BOUNDS_SPECS_STR) first = False else: buf.write("//") buf.write("min:") _PrintSpecsParameters(buf, minspecs) buf.write("/max:") _PrintSpecsParameters(buf, maxspecs) def ConfirmOperation(names, list_type, text, extra=""): """Ask the user to confirm an operation on a list of list_type. This function is used to request confirmation for doing an operation on a given list of list_type. @type names: list @param names: the list of names that we display when we ask for confirmation @type list_type: str @param list_type: Human readable name for elements in the list (e.g. nodes) @type text: str @param text: the operation that the user should confirm @rtype: boolean @return: True or False depending on user's confirmation. """ count = len(names) msg = ("The %s will operate on %d %s.\n%s" "Do you want to continue?" % (text, count, list_type, extra)) affected = (("\nAffected %s:\n" % list_type) + "\n".join([" %s" % name for name in names])) choices = [("y", True, "Yes, execute the %s" % text), ("n", False, "No, abort the %s" % text)] if count > 20: choices.insert(1, ("v", "v", "View the list of affected %s" % list_type)) question = msg else: question = msg + affected choice = AskUser(question, choices) if choice == "v": choices.pop(1) choice = AskUser(msg + affected, choices) return choice def _MaybeParseUnit(elements): """Parses and returns an array of potential values with units. """ parsed = {} for k, v in elements.items(): if v == constants.VALUE_DEFAULT: parsed[k] = v else: parsed[k] = utils.ParseUnit(v) return parsed def _InitISpecsFromSplitOpts(ipolicy, ispecs_mem_size, ispecs_cpu_count, ispecs_disk_count, ispecs_disk_size, ispecs_nic_count, group_ipolicy, fill_all): try: if ispecs_mem_size: ispecs_mem_size = _MaybeParseUnit(ispecs_mem_size) if ispecs_disk_size: ispecs_disk_size = _MaybeParseUnit(ispecs_disk_size) except (TypeError, ValueError, errors.UnitParseError) as err: raise errors.OpPrereqError("Invalid disk (%s) or memory (%s) size" " in policy: %s" % (ispecs_disk_size, ispecs_mem_size, err), errors.ECODE_INVAL) # prepare ipolicy dict ispecs_transposed = { constants.ISPEC_MEM_SIZE: ispecs_mem_size, constants.ISPEC_CPU_COUNT: ispecs_cpu_count, constants.ISPEC_DISK_COUNT: ispecs_disk_count, constants.ISPEC_DISK_SIZE: ispecs_disk_size, constants.ISPEC_NIC_COUNT: ispecs_nic_count, } # first, check that the values given are correct if group_ipolicy: forced_type = TISPECS_GROUP_TYPES else: forced_type = TISPECS_CLUSTER_TYPES for specs in ispecs_transposed.values(): assert isinstance(specs, dict) utils.ForceDictType(specs, forced_type) # then transpose ispecs = { constants.ISPECS_MIN: {}, constants.ISPECS_MAX: {}, constants.ISPECS_STD: {}, } for (name, specs) in ispecs_transposed.items(): assert name in constants.ISPECS_PARAMETERS for key, val in specs.items(): # {min: .. ,max: .., std: ..} assert key in ispecs ispecs[key][name] = val minmax_out = {} for key in constants.ISPECS_MINMAX_KEYS: if fill_all: minmax_out[key] = \ objects.FillDict(constants.ISPECS_MINMAX_DEFAULTS[key], ispecs[key]) else: minmax_out[key] = ispecs[key] ipolicy[constants.ISPECS_MINMAX] = [minmax_out] if fill_all: ipolicy[constants.ISPECS_STD] = \ objects.FillDict(constants.IPOLICY_DEFAULTS[constants.ISPECS_STD], ispecs[constants.ISPECS_STD]) else: ipolicy[constants.ISPECS_STD] = ispecs[constants.ISPECS_STD] def _ParseSpecUnit(spec, keyname): ret = spec.copy() for k in [constants.ISPEC_DISK_SIZE, constants.ISPEC_MEM_SIZE]: if k in ret: try: ret[k] = utils.ParseUnit(ret[k]) except (TypeError, ValueError, errors.UnitParseError) as err: raise errors.OpPrereqError(("Invalid parameter %s (%s) in %s instance" " specs: %s" % (k, ret[k], keyname, err)), errors.ECODE_INVAL) return ret def _ParseISpec(spec, keyname, required): ret = _ParseSpecUnit(spec, keyname) utils.ForceDictType(ret, constants.ISPECS_PARAMETER_TYPES) missing = constants.ISPECS_PARAMETERS - frozenset(ret) if required and missing: raise errors.OpPrereqError("Missing parameters in ipolicy spec %s: %s" % (keyname, utils.CommaJoin(missing)), errors.ECODE_INVAL) return ret def _GetISpecsInAllowedValues(minmax_ispecs, allowed_values): ret = None if (minmax_ispecs and allowed_values and len(minmax_ispecs) == 1 and len(minmax_ispecs[0]) == 1): for (key, spec) in minmax_ispecs[0].items(): # This loop is executed exactly once if key in allowed_values and not spec: ret = key return ret def _InitISpecsFromFullOpts(ipolicy_out, minmax_ispecs, std_ispecs, group_ipolicy, allowed_values): found_allowed = _GetISpecsInAllowedValues(minmax_ispecs, allowed_values) if found_allowed is not None: ipolicy_out[constants.ISPECS_MINMAX] = found_allowed elif minmax_ispecs is not None: minmax_out = [] for mmpair in minmax_ispecs: mmpair_out = {} for (key, spec) in mmpair.items(): if key not in constants.ISPECS_MINMAX_KEYS: msg = "Invalid key in bounds instance specifications: %s" % key raise errors.OpPrereqError(msg, errors.ECODE_INVAL) mmpair_out[key] = _ParseISpec(spec, key, True) minmax_out.append(mmpair_out) ipolicy_out[constants.ISPECS_MINMAX] = minmax_out if std_ispecs is not None: assert not group_ipolicy # This is not an option for gnt-group ipolicy_out[constants.ISPECS_STD] = _ParseISpec(std_ispecs, "std", False) def CreateIPolicyFromOpts(ispecs_mem_size=None, ispecs_cpu_count=None, ispecs_disk_count=None, ispecs_disk_size=None, ispecs_nic_count=None, minmax_ispecs=None, std_ispecs=None, ipolicy_disk_templates=None, ipolicy_vcpu_ratio=None, ipolicy_spindle_ratio=None, group_ipolicy=False, allowed_values=None, fill_all=False): """Creation of instance policy based on command line options. @param fill_all: whether for cluster policies we should ensure that all values are filled """ assert not (fill_all and allowed_values) split_specs = (ispecs_mem_size or ispecs_cpu_count or ispecs_disk_count or ispecs_disk_size or ispecs_nic_count) if split_specs and (minmax_ispecs is not None or std_ispecs is not None): raise errors.OpPrereqError("A --specs-xxx option cannot be specified" " together with any --ipolicy-xxx-specs option", errors.ECODE_INVAL) ipolicy_out = objects.MakeEmptyIPolicy() if split_specs: assert fill_all _InitISpecsFromSplitOpts(ipolicy_out, ispecs_mem_size, ispecs_cpu_count, ispecs_disk_count, ispecs_disk_size, ispecs_nic_count, group_ipolicy, fill_all) elif minmax_ispecs is not None or std_ispecs is not None: _InitISpecsFromFullOpts(ipolicy_out, minmax_ispecs, std_ispecs, group_ipolicy, allowed_values) if ipolicy_disk_templates is not None: if allowed_values and ipolicy_disk_templates in allowed_values: ipolicy_out[constants.IPOLICY_DTS] = ipolicy_disk_templates else: ipolicy_out[constants.IPOLICY_DTS] = list(ipolicy_disk_templates) if ipolicy_vcpu_ratio is not None: ipolicy_out[constants.IPOLICY_VCPU_RATIO] = ipolicy_vcpu_ratio if ipolicy_spindle_ratio is not None: ipolicy_out[constants.IPOLICY_SPINDLE_RATIO] = ipolicy_spindle_ratio assert not (frozenset(ipolicy_out) - constants.IPOLICY_ALL_KEYS) if not group_ipolicy and fill_all: ipolicy_out = objects.FillIPolicy(constants.IPOLICY_DEFAULTS, ipolicy_out) return ipolicy_out def _NotAContainer(data): """ Checks whether the input is not a container data type. @rtype: bool """ return not isinstance(data, (list, dict, tuple)) def _GetAlignmentMapping(data): """ Returns info about alignment if present in an encoded ordered dictionary. @type data: list of tuple @param data: The encoded ordered dictionary, as defined in L{_SerializeGenericInfo}. @rtype: dict of any to int @return: The dictionary mapping alignment groups to the maximum length of the dictionary key found in the group. """ alignment_map = {} for entry in data: if len(entry) > 2: group_key = entry[2] key_length = len(entry[0]) if group_key in alignment_map: alignment_map[group_key] = max(alignment_map[group_key], key_length) else: alignment_map[group_key] = key_length return alignment_map def _SerializeGenericInfo(buf, data, level, afterkey=False): """Formatting core of L{PrintGenericInfo}. @param buf: (string) stream to accumulate the result into @param data: data to format @type level: int @param level: depth in the data hierarchy, used for indenting @type afterkey: bool @param afterkey: True when we are in the middle of a line after a key (used to properly add newlines or indentation) """ baseind = " " if isinstance(data, dict): if not data: buf.write("\n") else: if afterkey: buf.write("\n") doindent = True else: doindent = False for key in sorted(data): if doindent: buf.write(baseind * level) else: doindent = True buf.write(key) buf.write(": ") _SerializeGenericInfo(buf, data[key], level + 1, afterkey=True) elif isinstance(data, list) and len(data) > 0 and isinstance(data[0], tuple): # list of tuples (an ordered dictionary) # the tuples may have two or three members - key, value, and alignment group # if the alignment group is present, align all values sharing the same group if afterkey: buf.write("\n") doindent = True else: doindent = False alignment_mapping = _GetAlignmentMapping(data) for entry in data: key, val = entry[0:2] if doindent: buf.write(baseind * level) else: doindent = True buf.write(key) buf.write(": ") if len(entry) > 2: max_key_length = alignment_mapping[entry[2]] buf.write(" " * (max_key_length - len(key))) _SerializeGenericInfo(buf, val, level + 1, afterkey=True) elif isinstance(data, tuple) and all(map(_NotAContainer, data)): # tuples with simple content are serialized as inline lists buf.write("[%s]\n" % utils.CommaJoin(data)) elif isinstance(data, list) or isinstance(data, tuple): # lists and tuples if not data: buf.write("\n") else: if afterkey: buf.write("\n") doindent = True else: doindent = False for item in data: if doindent: buf.write(baseind * level) else: doindent = True buf.write("-") buf.write(baseind[1:]) _SerializeGenericInfo(buf, item, level + 1) else: # This branch should be only taken for strings, but it's practically # impossible to guarantee that no other types are produced somewhere buf.write(str(data)) buf.write("\n") def PrintGenericInfo(data): """Print information formatted according to the hierarchy. The output is a valid YAML string. @param data: the data to print. It's a hierarchical structure whose elements can be: - dictionaries, where keys are strings and values are of any of the types listed here - lists of tuples (key, value) or (key, value, alignment_group), where key is a string, value is of any of the types listed here, and alignment_group can be any hashable value; it's a way to encode ordered dictionaries; any entries sharing the same alignment group are aligned by appending whitespace before the value as needed - lists of any of the types listed here - strings """ buf = StringIO() _SerializeGenericInfo(buf, data, 0) ToStdout(buf.getvalue().rstrip("\n")) ganeti-3.1.0~rc2/lib/cli_opts.py000064400000000000000000002114161476477700300165450ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module containing Ganeti's command line parsing options""" import re from optparse import (Option, OptionValueError) import json import sys import textwrap from ganeti import utils from ganeti import errors from ganeti import constants from ganeti import compat from ganeti import pathutils from ganeti import serializer __all__ = [ "ABSOLUTE_OPT", "ADD_RESERVED_IPS_OPT", "ADD_UIDS_OPT", "ALL_OPT", "ALLOC_POLICY_OPT", "ALLOCATABLE_OPT", "ALLOW_FAILOVER_OPT", "AUTO_PROMOTE_OPT", "AUTO_REPLACE_OPT", "BACKEND_OPT", "BLK_OS_OPT", "CAPAB_MASTER_OPT", "CAPAB_VM_OPT", "CLEANUP_OPT", "cli_option", "CLUSTER_DOMAIN_SECRET_OPT", "COMMIT_OPT", "COMMON_CREATE_OPTS", "COMMON_OPTS", "COMPRESS_OPT", "COMPRESSION_TOOLS_OPT", "CONFIRM_OPT", "CP_SIZE_OPT", "DEBUG_OPT", "DEBUG_SIMERR_OPT", "DEFAULT_IALLOCATOR_OPT", "DEFAULT_IALLOCATOR_PARAMS_OPT", "DISK_OPT", "DISK_PARAMS_OPT", "DISK_STATE_OPT", "DISK_TEMPLATE_OPT", "DISKIDX_OPT", "DRAINED_OPT", "DRBD_HELPER_OPT", "DRY_RUN_OPT", "DST_NODE_OPT", "EARLY_RELEASE_OPT", "ENABLED_DATA_COLLECTORS_OPT", "ENABLED_DISK_TEMPLATES_OPT", "ENABLED_HV_OPT", "ENABLED_USER_SHUTDOWN_OPT", "ERROR_CODES_OPT", "EXT_PARAMS_OPT", "FAILURE_ONLY_OPT", "FIELDS_OPT", "FILESTORE_DIR_OPT", "FILESTORE_DRIVER_OPT", "FORCE_FAILOVER_OPT", "FORCE_FILTER_OPT", "FORCE_OPT", "FORCE_VARIANT_OPT", "FORTHCOMING_OPT", "GATEWAY6_OPT", "GATEWAY_OPT", "GLOBAL_FILEDIR_OPT", "GLOBAL_GLUSTER_FILEDIR_OPT", "GLOBAL_SHARED_FILEDIR_OPT", "HELPER_SHUTDOWN_TIMEOUT_OPT", "HELPER_STARTUP_TIMEOUT_OPT", "HID_OS_OPT", "NOHOTPLUG_OPT", "HV_STATE_OPT", "HVLIST_OPT", "HVOPTS_OPT", "HYPERVISOR_OPT", "IALLOCATOR_OPT", "IDENTIFY_DEFAULTS_OPT", "IGNORE_CONSIST_OPT", "IGNORE_ERRORS_OPT", "IGNORE_FAILURES_OPT", "IGNORE_HVVERSIONS_OPT", "IGNORE_IPOLICY_OPT", "IGNORE_OFFLINE_OPT", "IGNORE_REMOVE_FAILURES_OPT", "IGNORE_SECONDARIES_OPT", "IGNORE_SOFT_ERRORS_OPT", "IGNORE_SIZE_OPT", "INCLUDEDEFAULTS_OPT", "INSTALL_IMAGE_OPT", "INSTANCE_COMMUNICATION_NETWORK_OPT", "INSTANCE_COMMUNICATION_OPT", "INSTANCE_POLICY_OPTS", "INTERVAL_OPT", "IPOLICY_BOUNDS_SPECS_STR", "IPOLICY_DISK_TEMPLATES", "IPOLICY_SPINDLE_RATIO", "IPOLICY_STD_SPECS_OPT", "IPOLICY_STD_SPECS_STR", "IPOLICY_VCPU_RATIO", "LONG_SLEEP_OPT", "MAC_PREFIX_OPT", "MAINTAIN_NODE_HEALTH_OPT", "MASTER_NETDEV_OPT", "MASTER_NETMASK_OPT", "MAX_TRACK_OPT", "MC_OPT", "MIGRATION_MODE_OPT", "MODIFY_ETCHOSTS_OPT", "NET_OPT", "NETWORK6_OPT", "NETWORK_OPT", "NEW_CLUSTER_CERT_OPT", "NEW_CLUSTER_DOMAIN_SECRET_OPT", "NEW_CONFD_HMAC_KEY_OPT", "NEW_NODE_CERT_OPT", "NEW_PRIMARY_OPT", "NEW_RAPI_CERT_OPT", "NEW_SECONDARY_OPT", "NEW_SPICE_CERT_OPT", "NEW_SSH_KEY_OPT", "NIC_PARAMS_OPT", "NO_INSTALL_OPT", "NO_REMEMBER_OPT", "NOCONFLICTSCHECK_OPT", "NODE_FORCE_JOIN_OPT", "NODE_LIST_OPT", "NODE_PARAMS_OPT", "NODE_PLACEMENT_OPT", "NODE_POWERED_OPT", "NODEGROUP_OPT", "NODEGROUP_OPT_NAME", "NOHDR_OPT", "NOHVPARAMASSESS_OPT", "IPCHECK_OPT", "NOIPCHECK_OPT", "NAMECHECK_OPT", "NONAMECHECK_OPT", "NOMODIFY_ETCHOSTS_OPT", "NOMODIFY_SSH_SETUP_OPT", "NONICS_OPT", "NONLIVE_OPT", "NONPLUS1_OPT", "NORUNTIME_CHGS_OPT", "NOSHUTDOWN_OPT", "NOSSH_KEYCHECK_OPT", "NOSTART_OPT", "NOVOTING_OPT", "NWSYNC_OPT", "OFFLINE_INST_OPT", "OFFLINE_OPT", "ON_PRIMARY_OPT", "ON_SECONDARY_OPT", "ONLINE_INST_OPT", "OOB_TIMEOUT_OPT", "OPT_COMPL_ALL", "OPT_COMPL_INST_ADD_NODES", "OPT_COMPL_MANY_NODES", "OPT_COMPL_ONE_EXTSTORAGE", "OPT_COMPL_ONE_FILTER", "OPT_COMPL_ONE_IALLOCATOR", "OPT_COMPL_ONE_INSTANCE", "OPT_COMPL_ONE_NETWORK", "OPT_COMPL_ONE_NODE", "OPT_COMPL_ONE_NODEGROUP", "OPT_COMPL_ONE_OS", "OS_OPT", "OS_SIZE_OPT", "OSPARAMS_OPT", "OSPARAMS_PRIVATE_OPT", "OSPARAMS_SECRET_OPT", "POWER_DELAY_OPT", "PREALLOC_WIPE_DISKS_OPT", "PRIMARY_IP_VERSION_OPT", "PRIMARY_ONLY_OPT", "PRINT_JOBID_OPT", "PRIORITY_OPT", "RAPI_CERT_OPT", "READD_OPT", "REASON_OPT", "REBOOT_TYPE_OPT", "REMOVE_INSTANCE_OPT", "REMOVE_RESERVED_IPS_OPT", "REMOVE_UIDS_OPT", "RESERVED_LVS_OPT", "ROMAN_OPT", "RQL_OPT", "RUNTIME_MEM_OPT", "SECONDARY_IP_OPT", "SECONDARY_ONLY_OPT", "SELECT_OS_OPT", "SEP_OPT", "SEQUENTIAL_OPT", "SHOW_MACHINE_OPT", "SHOWCMD_OPT", "SHUTDOWN_TIMEOUT_OPT", "SINGLE_NODE_OPT", "SPECS_CPU_COUNT_OPT", "SPECS_DISK_COUNT_OPT", "SPECS_DISK_SIZE_OPT", "SPECS_MEM_SIZE_OPT", "SPECS_NIC_COUNT_OPT", "SPICE_CACERT_OPT", "SPICE_CERT_OPT", "SPLIT_ISPECS_OPTS", "SRC_DIR_OPT", "SRC_NODE_OPT", "SSH_KEY_BITS_OPT", "SSH_KEY_TYPE_OPT", "STARTUP_PAUSED_OPT", "STATIC_OPT", "SUBMIT_OPT", "SUBMIT_OPTS", "SYNC_OPT", "TAG_ADD_OPT", "TAG_SRC_OPT", "TIMEOUT_OPT", "TO_GROUP_OPT", "TRANSPORT_COMPRESSION_OPT", "UIDPOOL_OPT", "USE_EXTERNAL_MIP_SCRIPT", "USE_REPL_NET_OPT", "USEUNITS_OPT", "VERBOSE_OPT", "VERIFY_CLUTTER_OPT", "VG_NAME_OPT", "WFSYNC_OPT", "YES_DOIT_OPT", "ZERO_FREE_SPACE_OPT", "ZEROING_IMAGE_OPT", "ZEROING_TIMEOUT_FIXED_OPT", "ZEROING_TIMEOUT_PER_MIB_OPT", ] NO_PREFIX = "no_" UN_PREFIX = "-" #: Priorities (sorted) _PRIORITY_NAMES = [ ("low", constants.OP_PRIO_LOW), ("normal", constants.OP_PRIO_NORMAL), ("high", constants.OP_PRIO_HIGH), ] #: Priority dictionary for easier lookup # TODO: Replace this and _PRIORITY_NAMES with a single sorted dictionary once # we migrate to Python 2.6 _PRIONAME_TO_VALUE = dict(_PRIORITY_NAMES) def check_unit(option, opt, value): # pylint: disable=W0613 """OptParsers custom converter for units. """ try: return utils.ParseUnit(value) except errors.UnitParseError as err: raise OptionValueError("option %s: %s" % (opt, err)) def _SplitKeyVal(opt, data, parse_prefixes): """Convert a KeyVal string into a dict. This function will convert a key=val[,...] string into a dict. Empty values will be converted specially: keys which have the prefix 'no_' will have the value=False and the prefix stripped, keys with the prefix "-" will have value=None and the prefix stripped, and the others will have value=True. @type opt: string @param opt: a string holding the option name for which we process the data, used in building error messages @type data: string @param data: a string of the format key=val,key=val,... @type parse_prefixes: bool @param parse_prefixes: whether to handle prefixes specially @rtype: dict @return: {key=val, key=val} @raises errors.ParameterError: if there are duplicate keys """ kv_dict = {} if data: for elem in utils.UnescapeAndSplit(data, sep=","): if "=" in elem: key, val = elem.split("=", 1) elif parse_prefixes: if elem.startswith(NO_PREFIX): key, val = elem[len(NO_PREFIX):], False elif elem.startswith(UN_PREFIX): key, val = elem[len(UN_PREFIX):], None else: key, val = elem, True else: raise errors.ParameterError("Missing value for key '%s' in option %s" % (elem, opt)) if key in kv_dict: raise errors.ParameterError("Duplicate key '%s' in option %s" % (key, opt)) kv_dict[key] = val return kv_dict def _SplitIdentKeyVal(opt, value, parse_prefixes): """Helper function to parse "ident:key=val,key=val" options. @type opt: string @param opt: option name, used in error messages @type value: string @param value: expected to be in the format "ident:key=val,key=val,..." @type parse_prefixes: bool @param parse_prefixes: whether to handle prefixes specially (see L{_SplitKeyVal}) @rtype: tuple @return: (ident, {key=val, key=val}) @raises errors.ParameterError: in case of duplicates or other parsing errors """ if ":" not in value: ident, rest = value, "" else: ident, rest = value.split(":", 1) if parse_prefixes and ident.startswith(NO_PREFIX): if rest: msg = "Cannot pass options when removing parameter groups: %s" % value raise errors.ParameterError(msg) retval = (ident[len(NO_PREFIX):], False) elif (parse_prefixes and ident.startswith(UN_PREFIX) and (len(ident) <= len(UN_PREFIX) or not ident[len(UN_PREFIX)].isdigit())): if rest: msg = "Cannot pass options when removing parameter groups: %s" % value raise errors.ParameterError(msg) retval = (ident[len(UN_PREFIX):], None) else: kv_dict = _SplitKeyVal(opt, rest, parse_prefixes) retval = (ident, kv_dict) return retval def check_ident_key_val(option, opt, value): # pylint: disable=W0613 """Custom parser for ident:key=val,key=val options. This will store the parsed values as a tuple (ident, {key: val}). As such, multiple uses of this option via action=append is possible. """ return _SplitIdentKeyVal(opt, value, True) def check_key_val(option, opt, value): # pylint: disable=W0613 """Custom parser class for key=val,key=val options. This will store the parsed values as a dict {key: val}. """ return _SplitKeyVal(opt, value, True) def check_key_private_val(option, opt, value): # pylint: disable=W0613 """Custom parser class for private and secret key=val,key=val options. This will store the parsed values as a dict {key: val}. """ return serializer.PrivateDict(_SplitKeyVal(opt, value, True)) def _SplitListKeyVal(opt, value): retval = {} for elem in value.split("/"): if not elem: raise errors.ParameterError("Empty section in option '%s'" % opt) (ident, valdict) = _SplitIdentKeyVal(opt, elem, False) if ident in retval: msg = ("Duplicated parameter '%s' in parsing %s: %s" % (ident, opt, elem)) raise errors.ParameterError(msg) retval[ident] = valdict return retval def check_multilist_ident_key_val(_, opt, value): """Custom parser for "ident:key=val,key=val/ident:key=val//ident:.." options. @rtype: list of dictionary @return: [{ident: {key: val, key: val}, ident: {key: val}}, {ident:..}] """ retval = [] for line in value.split("//"): retval.append(_SplitListKeyVal(opt, line)) return retval def check_bool(option, opt, value): # pylint: disable=W0613 """Custom parser for yes/no options. This will store the parsed value as either True or False. """ value = value.lower() if value == constants.VALUE_FALSE or value == "no": return False elif value == constants.VALUE_TRUE or value == "yes": return True else: raise errors.ParameterError("Invalid boolean value '%s'" % value) def check_list(option, opt, value): # pylint: disable=W0613 """Custom parser for comma-separated lists. """ # we have to make this explicit check since "".split(",") is [""], # not an empty list :( if not value: return [] else: return utils.UnescapeAndSplit(value) def check_maybefloat(option, opt, value): # pylint: disable=W0613 """Custom parser for float numbers which might be also defaults. """ value = value.lower() if value == constants.VALUE_DEFAULT: return value else: return float(value) def check_json(option, opt, value): # pylint: disable=W0613 """Custom parser for JSON arguments. Takes a string containing JSON, returns a Python object. """ return json.loads(value) def check_filteraction(option, opt, value): # pylint: disable=W0613 """Custom parser for filter rule actions. Takes a string, returns an action as a Python object (list or string). The string "RATE_LIMIT n" becomes `["RATE_LIMIT", n]`. All other strings stay as they are. """ match = re.match(r"RATE_LIMIT\s+(\d+)", value) if match: n = int(match.group(1)) return ["RATE_LIMIT", n] else: return value # completion_suggestion is normally a list. Using numeric values not evaluating # to False for dynamic completion. (OPT_COMPL_MANY_NODES, OPT_COMPL_ONE_NODE, OPT_COMPL_ONE_INSTANCE, OPT_COMPL_ONE_OS, OPT_COMPL_ONE_EXTSTORAGE, OPT_COMPL_ONE_FILTER, OPT_COMPL_ONE_IALLOCATOR, OPT_COMPL_ONE_NETWORK, OPT_COMPL_INST_ADD_NODES, OPT_COMPL_ONE_NODEGROUP) = range(100, 110) OPT_COMPL_ALL = compat.UniqueFrozenset([ OPT_COMPL_MANY_NODES, OPT_COMPL_ONE_NODE, OPT_COMPL_ONE_INSTANCE, OPT_COMPL_ONE_OS, OPT_COMPL_ONE_EXTSTORAGE, OPT_COMPL_ONE_FILTER, OPT_COMPL_ONE_IALLOCATOR, OPT_COMPL_ONE_NETWORK, OPT_COMPL_INST_ADD_NODES, OPT_COMPL_ONE_NODEGROUP, ]) class CliOption(Option): """Custom option class for optparse. """ ATTRS = Option.ATTRS + [ "completion_suggest", ] TYPES = Option.TYPES + ( "multilistidentkeyval", "identkeyval", "keyval", "keyprivateval", "unit", "bool", "list", "maybefloat", "json", "filteraction", ) TYPE_CHECKER = Option.TYPE_CHECKER.copy() TYPE_CHECKER["multilistidentkeyval"] = check_multilist_ident_key_val TYPE_CHECKER["identkeyval"] = check_ident_key_val TYPE_CHECKER["keyval"] = check_key_val TYPE_CHECKER["keyprivateval"] = check_key_private_val TYPE_CHECKER["unit"] = check_unit TYPE_CHECKER["bool"] = check_bool TYPE_CHECKER["list"] = check_list TYPE_CHECKER["maybefloat"] = check_maybefloat TYPE_CHECKER["json"] = check_json TYPE_CHECKER["filteraction"] = check_filteraction # optparse.py sets make_option, so we do it for our own option class, too cli_option = CliOption # pylint: disable=C0103 _YORNO = "yes|no" DEBUG_OPT = cli_option("-d", "--debug", default=0, action="count", help="Increase debugging level") NOHDR_OPT = cli_option("--no-headers", default=False, action="store_true", dest="no_headers", help="Don't display column headers") SEP_OPT = cli_option("--separator", default=None, action="store", dest="separator", help=("Separator between output fields" " (defaults to one space)")) USEUNITS_OPT = cli_option("--units", default=None, dest="units", choices=("h", "m", "g", "t"), help="Specify units for output (one of h/m/g/t)") FIELDS_OPT = cli_option("-o", "--output", dest="output", action="store", type="string", metavar="FIELDS", help="Comma separated list of output fields") FORCE_OPT = cli_option("-f", "--force", dest="force", action="store_true", default=False, help="Force the operation") CONFIRM_OPT = cli_option("--yes", dest="confirm", action="store_true", default=False, help="Do not require confirmation") IGNORE_OFFLINE_OPT = cli_option("--ignore-offline", dest="ignore_offline", action="store_true", default=False, help=("Ignore offline nodes and do as much" " as possible")) IGNORE_SOFT_ERRORS_OPT = cli_option("--ignore-soft-errors", dest="ignore_soft_errors", action="store_true", default=False, help=("Tell htools to ignore any soft" " errors like N+1 violations")) TAG_ADD_OPT = cli_option("--tags", dest="tags", default=None, help="Comma-separated list of instance" " tags") TAG_SRC_OPT = cli_option("--from", dest="tags_source", default=None, help="File with tag names") SUBMIT_OPT = cli_option("--submit", dest="submit_only", default=False, action="store_true", help=("Submit the job and return the job ID, but" " don't wait for the job to finish")) PRINT_JOBID_OPT = cli_option("--print-jobid", dest="print_jobid", default=False, action="store_true", help=("Additionally print the job as first line" " on stdout (for scripting).")) SEQUENTIAL_OPT = cli_option("--sequential", dest="sequential", default=False, action="store_true", help=("Execute all resulting jobs sequentially")) SYNC_OPT = cli_option("--sync", dest="do_locking", default=False, action="store_true", help=("Grab locks while doing the queries" " in order to ensure more consistent results")) DRY_RUN_OPT = cli_option("--dry-run", default=False, action="store_true", help=("Do not execute the operation, just run the" " check steps and verify if it could be" " executed")) VERBOSE_OPT = cli_option("-v", "--verbose", default=False, action="store_true", help="Increase the verbosity of the operation") DEBUG_SIMERR_OPT = cli_option("--debug-simulate-errors", default=False, action="store_true", dest="simulate_errors", help="Debugging option that makes the operation" " treat most runtime checks as failed") NWSYNC_OPT = cli_option("--no-wait-for-sync", dest="wait_for_sync", default=True, action="store_false", help="Don't wait for sync (DANGEROUS!)") WFSYNC_OPT = cli_option("--wait-for-sync", dest="wait_for_sync", default=False, action="store_true", help="Wait for disks to sync") ONLINE_INST_OPT = cli_option("--online", dest="online_inst", action="store_true", default=False, help="Enable offline instance") OFFLINE_INST_OPT = cli_option("--offline", dest="offline_inst", action="store_true", default=False, help="Disable down instance") DISK_TEMPLATE_OPT = cli_option("-t", "--disk-template", dest="disk_template", help=("Custom disk setup (%s)" % utils.CommaJoin(constants.DISK_TEMPLATES)), default=None, metavar="TEMPL", choices=list(constants.DISK_TEMPLATES)) EXT_PARAMS_OPT = cli_option("-e", "--ext-params", dest="ext_params", default={}, type="keyval", help="Parameters for ExtStorage template" " conversions in the format:" " provider=prvdr[,param1=val1,param2=val2,...]") NONICS_OPT = cli_option("--no-nics", default=False, action="store_true", help="Do not create any network cards for" " the instance") FILESTORE_DIR_OPT = cli_option("--file-storage-dir", dest="file_storage_dir", help="Relative path under default cluster-wide" " file storage dir to store file-based disks", default=None, metavar="") FILESTORE_DRIVER_OPT = cli_option("--file-driver", dest="file_driver", help="Driver to use for image files", default=None, metavar="", choices=list(constants.FILE_DRIVER)) IALLOCATOR_OPT = cli_option("-I", "--iallocator", metavar="", help="Select nodes for the instance automatically" " using the iallocator plugin", default=None, type="string", completion_suggest=OPT_COMPL_ONE_IALLOCATOR) DEFAULT_IALLOCATOR_OPT = cli_option("-I", "--default-iallocator", metavar="", help="Set the default instance" " allocator plugin", default=None, type="string", completion_suggest=OPT_COMPL_ONE_IALLOCATOR) DEFAULT_IALLOCATOR_PARAMS_OPT = cli_option("--default-iallocator-params", dest="default_iallocator_params", help="iallocator template" " parameters, in the format" " template:option=value," " option=value,...", type="keyval", default=None) OS_OPT = cli_option("-o", "--os-type", dest="os", help="What OS to run", metavar="", completion_suggest=OPT_COMPL_ONE_OS) OSPARAMS_OPT = cli_option("-O", "--os-parameters", dest="osparams", type="keyval", default={}, help="OS parameters") OSPARAMS_PRIVATE_OPT = cli_option("--os-parameters-private", dest="osparams_private", type="keyprivateval", default=serializer.PrivateDict(), help="Private OS parameters" " (won't be logged)") OSPARAMS_SECRET_OPT = cli_option("--os-parameters-secret", dest="osparams_secret", type="keyprivateval", default=serializer.PrivateDict(), help="Secret OS parameters (won't be logged or" " saved; you must supply these for every" " operation.)") FORCE_VARIANT_OPT = cli_option("--force-variant", dest="force_variant", action="store_true", default=False, help="Force an unknown variant") NO_INSTALL_OPT = cli_option("--no-install", dest="no_install", action="store_true", default=False, help="Do not install the OS (will" " enable no-start)") NORUNTIME_CHGS_OPT = cli_option("--no-runtime-changes", dest="allow_runtime_chgs", default=True, action="store_false", help="Don't allow runtime changes") BACKEND_OPT = cli_option("-B", "--backend-parameters", dest="beparams", type="keyval", default={}, help="Backend parameters") HVOPTS_OPT = cli_option("-H", "--hypervisor-parameters", type="keyval", default={}, dest="hvparams", help="Hypervisor parameters") DISK_PARAMS_OPT = cli_option("-D", "--disk-parameters", dest="diskparams", help="Disk template parameters, in the format" " template:option=value,option=value,...", type="identkeyval", action="append", default=[]) SPECS_MEM_SIZE_OPT = cli_option("--specs-mem-size", dest="ispecs_mem_size", type="keyval", default={}, help="Memory size specs: list of key=value," " where key is one of min, max, std" " (in MB or using a unit)") SPECS_CPU_COUNT_OPT = cli_option("--specs-cpu-count", dest="ispecs_cpu_count", type="keyval", default={}, help="CPU count specs: list of key=value," " where key is one of min, max, std") SPECS_DISK_COUNT_OPT = cli_option("--specs-disk-count", dest="ispecs_disk_count", type="keyval", default={}, help="Disk count specs: list of key=value," " where key is one of min, max, std") SPECS_DISK_SIZE_OPT = cli_option("--specs-disk-size", dest="ispecs_disk_size", type="keyval", default={}, help="Disk size specs: list of key=value," " where key is one of min, max, std" " (in MB or using a unit)") SPECS_NIC_COUNT_OPT = cli_option("--specs-nic-count", dest="ispecs_nic_count", type="keyval", default={}, help="NIC count specs: list of key=value," " where key is one of min, max, std") IPOLICY_BOUNDS_SPECS_STR = "--ipolicy-bounds-specs" IPOLICY_BOUNDS_SPECS_OPT = cli_option(IPOLICY_BOUNDS_SPECS_STR, dest="ipolicy_bounds_specs", type="multilistidentkeyval", default=None, help="Complete instance specs limits") IPOLICY_STD_SPECS_STR = "--ipolicy-std-specs" IPOLICY_STD_SPECS_OPT = cli_option(IPOLICY_STD_SPECS_STR, dest="ipolicy_std_specs", type="keyval", default=None, help="Complete standard instance specs") IPOLICY_DISK_TEMPLATES = cli_option("--ipolicy-disk-templates", dest="ipolicy_disk_templates", type="list", default=None, help="Comma-separated list of" " enabled disk templates") IPOLICY_VCPU_RATIO = cli_option("--ipolicy-vcpu-ratio", dest="ipolicy_vcpu_ratio", type="maybefloat", default=None, help="The maximum allowed vcpu-to-cpu ratio") IPOLICY_SPINDLE_RATIO = cli_option("--ipolicy-spindle-ratio", dest="ipolicy_spindle_ratio", type="maybefloat", default=None, help=("The maximum allowed instances to" " spindle ratio")) HYPERVISOR_OPT = cli_option("-H", "--hypervisor-parameters", dest="hypervisor", help="Hypervisor and hypervisor options, in the" " format hypervisor:option=value,option=value,...", default=None, type="identkeyval") HVLIST_OPT = cli_option("-H", "--hypervisor-parameters", dest="hvparams", help="Hypervisor and hypervisor options, in the" " format hypervisor:option=value,option=value,...", default=[], action="append", type="identkeyval") IPCHECK_OPT = cli_option("--ip-check", dest="ip_check", default=False, action="store_true", help="Check that the instance's IP is alive (ping)") def WarnDeprecatedOption(option, opt_str, value, parser): """Callback for processing deprecated options. """ msg = textwrap.fill(option.help, subsequent_indent=" ") print("Warning: %s: %s" % (opt_str, msg), file=sys.stderr) NOIPCHECK_OPT = cli_option("--no-ip-check", action="callback", callback=WarnDeprecatedOption, help="This option is deprecated/without any" " effect. IP check is now disabled by default." " Use --ip-check for connection test.") NAMECHECK_OPT = cli_option("--name-check", dest="name_check", default=False, action="store_true", help="Check that the instance's name is" " resolvable") NONAMECHECK_OPT = cli_option("--no-name-check", action="callback", callback=WarnDeprecatedOption, help="This option is deprecated/without any" " effect. Name check is now disabled by default." " Use --name-check for resolving names.") NET_OPT = cli_option("--net", help="NIC parameters", default=[], dest="nics", action="append", type="identkeyval") DISK_OPT = cli_option("--disk", help="Disk parameters", default=[], dest="disks", action="append", type="identkeyval") DISKIDX_OPT = cli_option("--disks", dest="disks", default=None, help="Comma-separated list of disks" " indices to act on (e.g. 0,2) (optional," " defaults to all disks)") OS_SIZE_OPT = cli_option("-s", "--os-size", dest="sd_size", help="Enforces a single-disk configuration using the" " given disk size, in MiB unless a suffix is used", default=None, type="unit", metavar="") IGNORE_CONSIST_OPT = cli_option("--ignore-consistency", dest="ignore_consistency", action="store_true", default=False, help="Ignore the consistency of the disks on" " the secondary. The source node must be " "marked offline first for this to succeed.") IGNORE_HVVERSIONS_OPT = cli_option("--ignore-hvversions", dest="ignore_hvversions", action="store_true", default=False, help="Ignore incompatible hypervisor" " versions between source and target") ALLOW_FAILOVER_OPT = cli_option("--allow-failover", dest="allow_failover", action="store_true", default=False, help="If migration is not possible fallback to" " failover") FORCE_FAILOVER_OPT = cli_option("--force-failover", dest="force_failover", action="store_true", default=False, help="Do not use migration, always use" " failover") NONLIVE_OPT = cli_option("--non-live", dest="live", default=True, action="store_false", help="Do a non-live migration (this usually means" " freeze the instance, save the state, transfer and" " only then resume running on the secondary node)") MIGRATION_MODE_OPT = cli_option("--migration-mode", dest="migration_mode", default=None, choices=list(constants.HT_MIGRATION_MODES), help="Override default migration mode (choose" " either live or non-live") NODE_PLACEMENT_OPT = cli_option("-n", "--node", dest="node", help="Target node and optional secondary node", metavar="[:]", completion_suggest=OPT_COMPL_INST_ADD_NODES) NODE_LIST_OPT = cli_option("-n", "--node", dest="nodes", default=[], action="append", metavar="", help="Use only this node (can be used multiple" " times, if not given defaults to all nodes)", completion_suggest=OPT_COMPL_ONE_NODE) NODEGROUP_OPT_NAME = "--node-group" NODEGROUP_OPT = cli_option("-g", NODEGROUP_OPT_NAME, dest="nodegroup", help="Node group (name or uuid)", metavar="", default=None, type="string", completion_suggest=OPT_COMPL_ONE_NODEGROUP) SINGLE_NODE_OPT = cli_option("-n", "--node", dest="node", help="Target node", metavar="", completion_suggest=OPT_COMPL_ONE_NODE) NOSTART_OPT = cli_option("--no-start", dest="start", default=True, action="store_false", help="Don't start the instance after creation") FORTHCOMING_OPT = cli_option("--forthcoming", dest="forthcoming", action="store_true", default=False, help="Only reserve resources, but do not" " create the instance yet") COMMIT_OPT = cli_option("--commit", dest="commit", action="store_true", default=False, help="The instance is already reserved and should" " be committed now") SHOWCMD_OPT = cli_option("--show-cmd", dest="show_command", action="store_true", default=False, help="Show command instead of executing it") CLEANUP_OPT = cli_option("--cleanup", dest="cleanup", default=False, action="store_true", help="Instead of performing the migration/failover," " try to recover from a failed cleanup. This is safe" " to run even if the instance is healthy, but it" " will create extra replication traffic and " " disrupt briefly the replication (like during the" " migration/failover") STATIC_OPT = cli_option("-s", "--static", dest="static", action="store_true", default=False, help="Only show configuration data, not runtime data") ALL_OPT = cli_option("--all", dest="show_all", default=False, action="store_true", help="Show info on all instances on the cluster." " This can take a long time to run, use wisely") SELECT_OS_OPT = cli_option("--select-os", dest="select_os", action="store_true", default=False, help="Interactive OS reinstall, lists available" " OS templates for selection") IGNORE_FAILURES_OPT = cli_option("--ignore-failures", dest="ignore_failures", action="store_true", default=False, help="Remove the instance from the cluster" " configuration even if there are failures" " during the removal process") IGNORE_REMOVE_FAILURES_OPT = cli_option("--ignore-remove-failures", dest="ignore_remove_failures", action="store_true", default=False, help="Remove the instance from the" " cluster configuration even if there" " are failures during the removal" " process") REMOVE_INSTANCE_OPT = cli_option("--remove-instance", dest="remove_instance", action="store_true", default=False, help="Remove the instance from the cluster") DST_NODE_OPT = cli_option("-n", "--target-node", dest="dst_node", help="Specifies the new node for the instance", metavar="NODE", default=None, completion_suggest=OPT_COMPL_ONE_NODE) NEW_SECONDARY_OPT = cli_option("-n", "--new-secondary", dest="dst_node", help="Specifies the new secondary node", metavar="NODE", default=None, completion_suggest=OPT_COMPL_ONE_NODE) NEW_PRIMARY_OPT = cli_option("--new-primary", dest="new_primary_node", help="Specifies the new primary node", metavar="", default=None, completion_suggest=OPT_COMPL_ONE_NODE) ON_PRIMARY_OPT = cli_option("-p", "--on-primary", dest="on_primary", default=False, action="store_true", help="Replace the disk(s) on the primary" " node (applies only to internally mirrored" " disk templates, e.g. %s)" % utils.CommaJoin(constants.DTS_INT_MIRROR)) ON_SECONDARY_OPT = cli_option("-s", "--on-secondary", dest="on_secondary", default=False, action="store_true", help="Replace the disk(s) on the secondary" " node (applies only to internally mirrored" " disk templates, e.g. %s)" % utils.CommaJoin(constants.DTS_INT_MIRROR)) AUTO_PROMOTE_OPT = cli_option("--auto-promote", dest="auto_promote", default=False, action="store_true", help="Lock all nodes and auto-promote as needed" " to MC status") AUTO_REPLACE_OPT = cli_option("-a", "--auto", dest="auto", default=False, action="store_true", help="Automatically replace faulty disks" " (applies only to internally mirrored" " disk templates, e.g. %s)" % utils.CommaJoin(constants.DTS_INT_MIRROR)) IGNORE_SIZE_OPT = cli_option("--ignore-size", dest="ignore_size", default=False, action="store_true", help="Ignore current recorded size" " (useful for forcing activation when" " the recorded size is wrong)") SRC_NODE_OPT = cli_option("--src-node", dest="src_node", help="Source node", metavar="", completion_suggest=OPT_COMPL_ONE_NODE) SRC_DIR_OPT = cli_option("--src-dir", dest="src_dir", help="Source directory", metavar="") SECONDARY_IP_OPT = cli_option("-s", "--secondary-ip", dest="secondary_ip", help="Specify the secondary ip for the node", metavar="ADDRESS", default=None) READD_OPT = cli_option("--readd", dest="readd", default=False, action="store_true", help="Readd old node after replacing it") NOSSH_KEYCHECK_OPT = cli_option("--no-ssh-key-check", dest="ssh_key_check", default=True, action="store_false", help="Disable SSH key fingerprint checking") NODE_FORCE_JOIN_OPT = cli_option("--force-join", dest="force_join", default=False, action="store_true", help="Force the joining of a node") MC_OPT = cli_option("-C", "--master-candidate", dest="master_candidate", type="bool", default=None, metavar=_YORNO, help="Set the master_candidate flag on the node") OFFLINE_OPT = cli_option("-O", "--offline", dest="offline", metavar=_YORNO, type="bool", default=None, help=("Set the offline flag on the node" " (cluster does not communicate with offline" " nodes)")) DRAINED_OPT = cli_option("-D", "--drained", dest="drained", metavar=_YORNO, type="bool", default=None, help=("Set the drained flag on the node" " (excluded from allocation operations)")) CAPAB_MASTER_OPT = cli_option("--master-capable", dest="master_capable", type="bool", default=None, metavar=_YORNO, help="Set the master_capable flag on the node") CAPAB_VM_OPT = cli_option("--vm-capable", dest="vm_capable", type="bool", default=None, metavar=_YORNO, help="Set the vm_capable flag on the node") ALLOCATABLE_OPT = cli_option("--allocatable", dest="allocatable", type="bool", default=None, metavar=_YORNO, help="Set the allocatable flag on a volume") ENABLED_HV_OPT = cli_option("--enabled-hypervisors", dest="enabled_hypervisors", help="Comma-separated list of hypervisors", type="string", default=None) ENABLED_DISK_TEMPLATES_OPT = cli_option("--enabled-disk-templates", dest="enabled_disk_templates", help="Comma-separated list of " "disk templates", type="string", default=None) ENABLED_USER_SHUTDOWN_OPT = cli_option("--user-shutdown", default=None, dest="enabled_user_shutdown", help="Whether user shutdown is enabled", type="bool") NIC_PARAMS_OPT = cli_option("-N", "--nic-parameters", dest="nicparams", type="keyval", default={}, help="NIC parameters") CP_SIZE_OPT = cli_option("-C", "--candidate-pool-size", default=None, dest="candidate_pool_size", type="int", help="Set the candidate pool size") RQL_OPT = cli_option("--max-running-jobs", dest="max_running_jobs", type="int", help="Set the maximal number of jobs to " "run simultaneously") MAX_TRACK_OPT = cli_option("--max-tracked-jobs", dest="max_tracked_jobs", type="int", help="Set the maximal number of jobs to " "be tracked simultaneously for " "scheduling") COMPRESSION_TOOLS_OPT = \ cli_option("--compression-tools", dest="compression_tools", type="string", default=None, help="Comma-separated list of compression tools which are" " allowed to be used by Ganeti in various operations") VG_NAME_OPT = cli_option("--vg-name", dest="vg_name", help=("Enables LVM and specifies the volume group" " name (cluster-wide) for disk allocation" " [%s]" % constants.DEFAULT_VG), metavar="VG", default=None) YES_DOIT_OPT = cli_option("--yes-do-it", "--ya-rly", dest="yes_do_it", help="Destroy cluster", action="store_true") NOVOTING_OPT = cli_option("--no-voting", dest="no_voting", help="Skip node agreement check (dangerous)", action="store_true", default=False) MAC_PREFIX_OPT = cli_option("-m", "--mac-prefix", dest="mac_prefix", help="Specify the mac prefix for the instance IP" " addresses, in the format XX:XX:XX", metavar="PREFIX", default=None) MASTER_NETDEV_OPT = cli_option("--master-netdev", dest="master_netdev", help="Specify the node interface (cluster-wide)" " on which the master IP address will be added" " (cluster init default: %s)" % constants.DEFAULT_BRIDGE, metavar="NETDEV", default=None) MASTER_NETMASK_OPT = cli_option("--master-netmask", dest="master_netmask", help="Specify the netmask of the master IP", metavar="NETMASK", default=None) USE_EXTERNAL_MIP_SCRIPT = cli_option("--use-external-mip-script", dest="use_external_mip_script", help="Specify whether to run a" " user-provided script for the master" " IP address turnup and" " turndown operations", type="bool", metavar=_YORNO, default=None) GLOBAL_FILEDIR_OPT = cli_option("--file-storage-dir", dest="file_storage_dir", help="Specify the default directory (cluster-" "wide) for storing the file-based disks [%s]" % pathutils.DEFAULT_FILE_STORAGE_DIR, metavar="DIR", default=None) GLOBAL_SHARED_FILEDIR_OPT = cli_option( "--shared-file-storage-dir", dest="shared_file_storage_dir", help="Specify the default directory (cluster-wide) for storing the" " shared file-based disks [%s]" % pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR, metavar="SHAREDDIR", default=None) GLOBAL_GLUSTER_FILEDIR_OPT = cli_option( "--gluster-storage-dir", dest="gluster_storage_dir", help="Specify the default directory (cluster-wide) for mounting Gluster" " file systems [%s]" % pathutils.DEFAULT_GLUSTER_STORAGE_DIR, metavar="GLUSTERDIR", default=pathutils.DEFAULT_GLUSTER_STORAGE_DIR) NOMODIFY_ETCHOSTS_OPT = cli_option("--no-etc-hosts", dest="modify_etc_hosts", help="Don't modify %s" % pathutils.ETC_HOSTS, action="store_false", default=True) MODIFY_ETCHOSTS_OPT = \ cli_option("--modify-etc-hosts", dest="modify_etc_hosts", metavar=_YORNO, default=None, type="bool", help="Defines whether the cluster should autonomously modify" " and keep in sync the /etc/hosts file of the nodes") NOMODIFY_SSH_SETUP_OPT = cli_option("--no-ssh-init", dest="modify_ssh_setup", help="Don't initialize SSH keys", action="store_false", default=True) ERROR_CODES_OPT = cli_option("--error-codes", dest="error_codes", help="Enable parseable error messages", action="store_true", default=False) NONPLUS1_OPT = cli_option("--no-nplus1-mem", dest="skip_nplusone_mem", help="Skip N+1 memory redundancy tests", action="store_true", default=False) NOHVPARAMASSESS_OPT = cli_option("--no-hv-param-assessment", dest="skip_hvparam_assessment", help="Skip hypervisor parameter assessment", action="store_true", default=False) REBOOT_TYPE_OPT = cli_option("-t", "--type", dest="reboot_type", help="Type of reboot: soft/hard/full", default=constants.INSTANCE_REBOOT_HARD, metavar="", choices=list(constants.REBOOT_TYPES)) IGNORE_SECONDARIES_OPT = cli_option("--ignore-secondaries", dest="ignore_secondaries", default=False, action="store_true", help="Ignore errors from secondaries") NOSHUTDOWN_OPT = cli_option("--noshutdown", dest="shutdown", action="store_false", default=True, help="Don't shutdown the instance (unsafe)") TIMEOUT_OPT = cli_option("--timeout", dest="timeout", type="int", default=constants.DEFAULT_SHUTDOWN_TIMEOUT, help="Maximum time (in seconds) to wait") COMPRESS_OPT = cli_option("--compress", dest="compress", type="string", default=constants.IEC_NONE, help="The compression mode to use") TRANSPORT_COMPRESSION_OPT = \ cli_option("--transport-compression", dest="transport_compression", type="string", default=constants.IEC_NONE, help="The compression mode to use during transport") SHUTDOWN_TIMEOUT_OPT = cli_option("--shutdown-timeout", dest="shutdown_timeout", type="int", default=constants.DEFAULT_SHUTDOWN_TIMEOUT, help="Maximum time (in seconds) to wait for" " instance shutdown") INTERVAL_OPT = cli_option("--interval", dest="interval", type="int", default=None, help=("Number of seconds between repetions of the" " command")) EARLY_RELEASE_OPT = cli_option("--early-release", dest="early_release", default=False, action="store_true", help="Release the locks on the secondary" " node(s) early") NEW_CLUSTER_CERT_OPT = cli_option("--new-cluster-certificate", dest="new_cluster_cert", default=False, action="store_true", help="Generate a new cluster certificate") NEW_NODE_CERT_OPT = cli_option( "--new-node-certificates", dest="new_node_cert", default=False, action="store_true", help="Generate new node certificates (for all nodes)") NEW_SSH_KEY_OPT = cli_option( "--new-ssh-keys", dest="new_ssh_keys", default=False, action="store_true", help="Generate new node SSH keys (for all nodes)") RAPI_CERT_OPT = cli_option("--rapi-certificate", dest="rapi_cert", default=None, help="File containing new RAPI certificate") NEW_RAPI_CERT_OPT = cli_option("--new-rapi-certificate", dest="new_rapi_cert", default=None, action="store_true", help=("Generate a new self-signed RAPI" " certificate")) SPICE_CERT_OPT = cli_option("--spice-certificate", dest="spice_cert", default=None, help="File containing new SPICE certificate") SPICE_CACERT_OPT = cli_option("--spice-ca-certificate", dest="spice_cacert", default=None, help="File containing the certificate of the CA" " which signed the SPICE certificate") NEW_SPICE_CERT_OPT = cli_option("--new-spice-certificate", dest="new_spice_cert", default=None, action="store_true", help=("Generate a new self-signed SPICE" " certificate")) NEW_CONFD_HMAC_KEY_OPT = cli_option("--new-confd-hmac-key", dest="new_confd_hmac_key", default=False, action="store_true", help=("Create a new HMAC key for %s" % constants.CONFD)) CLUSTER_DOMAIN_SECRET_OPT = cli_option("--cluster-domain-secret", dest="cluster_domain_secret", default=None, help=("Load new new cluster domain" " secret from file")) NEW_CLUSTER_DOMAIN_SECRET_OPT = cli_option("--new-cluster-domain-secret", dest="new_cluster_domain_secret", default=False, action="store_true", help=("Create a new cluster domain" " secret")) USE_REPL_NET_OPT = cli_option("--use-replication-network", dest="use_replication_network", help="Whether to use the replication network" " for talking to the nodes", action="store_true", default=False) MAINTAIN_NODE_HEALTH_OPT = \ cli_option("--maintain-node-health", dest="maintain_node_health", metavar=_YORNO, default=None, type="bool", help="Configure the cluster to automatically maintain node" " health, by shutting down unknown instances, shutting down" " unknown DRBD devices, etc.") IDENTIFY_DEFAULTS_OPT = \ cli_option("--identify-defaults", dest="identify_defaults", default=False, action="store_true", help="Identify which saved instance parameters are equal to" " the current cluster defaults and set them as such, instead" " of marking them as overridden") UIDPOOL_OPT = cli_option("--uid-pool", default=None, action="store", dest="uid_pool", help=("A list of user-ids or user-id" " ranges separated by commas")) ADD_UIDS_OPT = cli_option("--add-uids", default=None, action="store", dest="add_uids", help=("A list of user-ids or user-id" " ranges separated by commas, to be" " added to the user-id pool")) REMOVE_UIDS_OPT = cli_option("--remove-uids", default=None, action="store", dest="remove_uids", help=("A list of user-ids or user-id" " ranges separated by commas, to be" " removed from the user-id pool")) RESERVED_LVS_OPT = cli_option("--reserved-lvs", default=None, action="store", dest="reserved_lvs", help=("A comma-separated list of reserved" " logical volumes names, that will be" " ignored by cluster verify")) ROMAN_OPT = cli_option("--roman", dest="roman_integers", default=False, action="store_true", help="Use roman numbers for positive integers") DRBD_HELPER_OPT = cli_option("--drbd-usermode-helper", dest="drbd_helper", action="store", default=None, help="Specifies usermode helper for DRBD") PRIMARY_IP_VERSION_OPT = \ cli_option("--primary-ip-version", default=constants.IP4_VERSION, action="store", dest="primary_ip_version", metavar="%d|%d" % (constants.IP4_VERSION, constants.IP6_VERSION), help="Cluster-wide IP version for primary IP") SHOW_MACHINE_OPT = cli_option("-M", "--show-machine-names", default=False, action="store_true", help="Show machine name for every line in output") FAILURE_ONLY_OPT = cli_option("--failure-only", default=False, action="store_true", help=("Hide successful results and show failures" " only (determined by the exit code)")) REASON_OPT = cli_option("--reason", default=[], help="The reason for executing the command") def _PriorityOptionCb(option, _, value, parser): """Callback for processing C{--priority} option. """ value = _PRIONAME_TO_VALUE[value] setattr(parser.values, option.dest, value) PRIORITY_OPT = cli_option("--priority", default=None, dest="priority", metavar="|".join(name for name, _ in _PRIORITY_NAMES), choices=list(_PRIONAME_TO_VALUE), action="callback", type="choice", callback=_PriorityOptionCb, help="Priority for opcode processing") OPPORTUNISTIC_OPT = cli_option("--opportunistic-locking", dest="opportunistic_locking", action="store_true", default=False, help="Opportunistically acquire locks") HID_OS_OPT = cli_option("--hidden", dest="hidden", type="bool", default=None, metavar=_YORNO, help="Sets the hidden flag on the OS") BLK_OS_OPT = cli_option("--blacklisted", dest="blacklisted", type="bool", default=None, metavar=_YORNO, help="Sets the blacklisted flag on the OS") PREALLOC_WIPE_DISKS_OPT = cli_option("--prealloc-wipe-disks", default=None, type="bool", metavar=_YORNO, dest="prealloc_wipe_disks", help=("Wipe disks prior to instance" " creation")) NODE_PARAMS_OPT = cli_option("--node-parameters", dest="ndparams", type="keyval", default=None, help="Node parameters") ALLOC_POLICY_OPT = cli_option("--alloc-policy", dest="alloc_policy", action="store", metavar="POLICY", default=None, help="Allocation policy for the node group") NODE_POWERED_OPT = cli_option("--node-powered", default=None, type="bool", metavar=_YORNO, dest="node_powered", help="Specify if the SoR for node is powered") OOB_TIMEOUT_OPT = cli_option("--oob-timeout", dest="oob_timeout", type="int", default=constants.OOB_TIMEOUT, help="Maximum time to wait for out-of-band helper") POWER_DELAY_OPT = cli_option("--power-delay", dest="power_delay", type="float", default=constants.OOB_POWER_DELAY, help="Time in seconds to wait between power-ons") FORCE_FILTER_OPT = cli_option("-F", "--filter", dest="force_filter", action="store_true", default=False, help=("Whether command argument should be treated" " as filter")) NO_REMEMBER_OPT = cli_option("--no-remember", dest="no_remember", action="store_true", default=False, help="Perform but do not record the change" " in the configuration") PRIMARY_ONLY_OPT = cli_option("-p", "--primary-only", default=False, action="store_true", help="Evacuate primary instances only") SECONDARY_ONLY_OPT = cli_option("-s", "--secondary-only", default=False, action="store_true", help="Evacuate secondary instances only" " (applies only to internally mirrored" " disk templates, e.g. %s)" % utils.CommaJoin(constants.DTS_INT_MIRROR)) STARTUP_PAUSED_OPT = cli_option("--paused", dest="startup_paused", action="store_true", default=False, help="Pause instance at startup") TO_GROUP_OPT = cli_option("--to", dest="to", metavar="", help="Destination node group (name or uuid)", default=None, action="append", completion_suggest=OPT_COMPL_ONE_NODEGROUP) IGNORE_ERRORS_OPT = cli_option("-I", "--ignore-errors", default=[], action="append", dest="ignore_errors", choices=list(constants.CV_ALL_ECODES_STRINGS), help="Error code to be ignored") DISK_STATE_OPT = cli_option("--disk-state", default=[], dest="disk_state", action="append", help=("Specify disk state information in the" " format" " storage_type/identifier:option=value,...;" " note this is unused for now"), type="identkeyval") HV_STATE_OPT = cli_option("--hypervisor-state", default=[], dest="hv_state", action="append", help=("Specify hypervisor state information in the" " format hypervisor:option=value,...;" " note this is unused for now"), type="identkeyval") IGNORE_IPOLICY_OPT = cli_option("--ignore-ipolicy", dest="ignore_ipolicy", action="store_true", default=False, help="Ignore instance policy violations") RUNTIME_MEM_OPT = cli_option("-m", "--runtime-memory", dest="runtime_mem", help="Sets the instance's runtime memory," " ballooning it up or down to the new value", default=None, type="unit", metavar="") ABSOLUTE_OPT = cli_option("--absolute", dest="absolute", action="store_true", default=False, help="Marks the grow as absolute instead of the" " (default) relative mode") NETWORK_OPT = cli_option("--network", action="store", default=None, dest="network", help="IP network in CIDR notation") GATEWAY_OPT = cli_option("--gateway", action="store", default=None, dest="gateway", help="IP address of the router (gateway)") ADD_RESERVED_IPS_OPT = cli_option("--add-reserved-ips", action="store", default=None, dest="add_reserved_ips", help="Comma-separated list of" " reserved IPs to add") REMOVE_RESERVED_IPS_OPT = cli_option("--remove-reserved-ips", action="store", default=None, dest="remove_reserved_ips", help="Comma-delimited list of" " reserved IPs to remove") NETWORK6_OPT = cli_option("--network6", action="store", default=None, dest="network6", help="IP network in CIDR notation") GATEWAY6_OPT = cli_option("--gateway6", action="store", default=None, dest="gateway6", help="IP6 address of the router (gateway)") NOCONFLICTSCHECK_OPT = cli_option("--no-conflicts-check", dest="conflicts_check", default=True, action="store_false", help="Don't check for conflicting IPs") INCLUDEDEFAULTS_OPT = cli_option("--include-defaults", dest="include_defaults", default=False, action="store_true", help="Include default values") NOHOTPLUG_OPT = cli_option("--no-hotplug", dest="hotplug", action="store_false", default=True, help="Hotplug supported devices (NICs and Disks)") INSTALL_IMAGE_OPT = \ cli_option("--install-image", dest="install_image", action="store", type="string", default=None, help="The OS image to use for running the OS scripts safely") INSTANCE_COMMUNICATION_OPT = \ cli_option("-c", "--communication", dest="instance_communication", help=constants.INSTANCE_COMMUNICATION_DOC, type="bool") INSTANCE_COMMUNICATION_NETWORK_OPT = \ cli_option("--instance-communication-network", dest="instance_communication_network", type="string", help="Set the network name for instance communication") ZEROING_IMAGE_OPT = \ cli_option("--zeroing-image", dest="zeroing_image", action="store", default=None, help="The OS image to use to zero instance disks") ZERO_FREE_SPACE_OPT = \ cli_option("--zero-free-space", dest="zero_free_space", action="store_true", default=False, help="Whether to zero the free space on the disks of the " "instance prior to the export") HELPER_STARTUP_TIMEOUT_OPT = \ cli_option("--helper-startup-timeout", dest="helper_startup_timeout", action="store", type="int", help="Startup timeout for the helper VM") HELPER_SHUTDOWN_TIMEOUT_OPT = \ cli_option("--helper-shutdown-timeout", dest="helper_shutdown_timeout", action="store", type="int", help="Shutdown timeout for the helper VM") ZEROING_TIMEOUT_FIXED_OPT = \ cli_option("--zeroing-timeout-fixed", dest="zeroing_timeout_fixed", action="store", type="int", help="The fixed amount of time to wait before assuming that the " "zeroing failed") ZEROING_TIMEOUT_PER_MIB_OPT = \ cli_option("--zeroing-timeout-per-mib", dest="zeroing_timeout_per_mib", action="store", type="float", help="The amount of time to wait per MiB of data to zero, in " "addition to the fixed timeout") ENABLED_DATA_COLLECTORS_OPT = \ cli_option("--enabled-data-collectors", dest="enabled_data_collectors", type="keyval", default={}, help="Deactivate or reactivate a data collector for reporting, " "in the format collector=bool, where collector is one of %s." % ", ".join(constants.DATA_COLLECTOR_NAMES)) VERIFY_CLUTTER_OPT = cli_option( "--verify-ssh-clutter", default=False, dest="verify_clutter", help="Verify that Ganeti did not clutter" " up the 'authorized_keys' file", action="store_true") LONG_SLEEP_OPT = cli_option( "--long-sleep", default=False, dest="long_sleep", help="Allow long shutdowns when backing up instances", action="store_true") SSH_KEY_TYPE_OPT = \ cli_option("--ssh-key-type", default=None, choices=list(constants.SSHK_ALL), dest="ssh_key_type", help="Type of SSH key deployed by Ganeti for cluster actions") SSH_KEY_BITS_OPT = \ cli_option("--ssh-key-bits", default=None, type="int", dest="ssh_key_bits", help="Length of SSH keys generated by Ganeti, in bits") #: Options provided by all commands COMMON_OPTS = [DEBUG_OPT, REASON_OPT] # options related to asynchronous job handling SUBMIT_OPTS = [ SUBMIT_OPT, PRINT_JOBID_OPT, ] # common options for creating instances. add and import then add their own # specific ones. COMMON_CREATE_OPTS = [ BACKEND_OPT, DISK_OPT, DISK_TEMPLATE_OPT, FILESTORE_DIR_OPT, FILESTORE_DRIVER_OPT, HYPERVISOR_OPT, IALLOCATOR_OPT, NET_OPT, NODE_PLACEMENT_OPT, NODEGROUP_OPT, IPCHECK_OPT, NOIPCHECK_OPT, NAMECHECK_OPT, NONAMECHECK_OPT, NOCONFLICTSCHECK_OPT, NONICS_OPT, NWSYNC_OPT, OSPARAMS_OPT, OSPARAMS_PRIVATE_OPT, OSPARAMS_SECRET_OPT, OS_SIZE_OPT, OPPORTUNISTIC_OPT, SUBMIT_OPT, PRINT_JOBID_OPT, TAG_ADD_OPT, DRY_RUN_OPT, PRIORITY_OPT, ] # common instance policy options INSTANCE_POLICY_OPTS = [ IPOLICY_BOUNDS_SPECS_OPT, IPOLICY_DISK_TEMPLATES, IPOLICY_VCPU_RATIO, IPOLICY_SPINDLE_RATIO, ] # instance policy split specs options SPLIT_ISPECS_OPTS = [ SPECS_CPU_COUNT_OPT, SPECS_DISK_COUNT_OPT, SPECS_DISK_SIZE_OPT, SPECS_MEM_SIZE_OPT, SPECS_NIC_COUNT_OPT, ] ganeti-3.1.0~rc2/lib/client/000075500000000000000000000000001476477700300156305ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/client/__init__.py000064400000000000000000000025461476477700300177500ustar00rootroot00000000000000# # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Common command line client code. """ ganeti-3.1.0~rc2/lib/client/base.py000064400000000000000000000054401476477700300171170ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utils for CLI commands""" from ganeti import cli from ganeti import constants from ganeti import ht def GetResult(cl, opts, result): """Waits for jobs and returns whether they have succeeded Some OpCodes return of list of jobs. This function can be used after issueing a given OpCode to look at the OpCode's result and, if it is of type L{ht.TJobIdListOnly}, then it will wait for the jobs to complete, otherwise just return L{constants.EXIT_SUCCESS}. @type cl: L{ganeti.luxi.Client} @param cl: client that was used to submit the OpCode, which will also be used to poll the jobs @param opts: CLI options @param result: result of the opcode which might contain job information, in which case the jobs will be polled, or simply the result of the opcode @rtype: int @return: L{constants.EXIT_SUCCESS} if all jobs completed successfully, L{constants.EXIT_FAILURE} otherwise """ if not ht.TJobIdListOnly(result): return constants.EXIT_SUCCESS jex = cli.JobExecutor(cl=cl, opts=opts) for (status, job_id) in result[constants.JOB_IDS_KEY]: jex.AddJobId(None, status, job_id) bad_jobs = [job_result for success, job_result in jex.GetResults() if not success] if len(bad_jobs) > 0: for job in bad_jobs: cli.ToStdout("Job failed, result is '%s'.", job) cli.ToStdout("%s job(s) failed.", bad_jobs) return constants.EXIT_FAILURE else: return constants.EXIT_SUCCESS ganeti-3.1.0~rc2/lib/client/gnt_backup.py000064400000000000000000000134211476477700300203200ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Backup related commands""" # pylint: disable=W0401,W0613,W0614,C0103 # W0401: Wildcard import ganeti.cli # W0613: Unused argument, since all functions follow the same API # W0614: Unused import %s from wildcard import (since we need cli) # C0103: Invalid name gnt-backup from ganeti.cli import * from ganeti import opcodes from ganeti import constants from ganeti import errors from ganeti import qlang _LIST_DEF_FIELDS = ["node", "export"] def PrintExportList(opts, args): """Prints a list of all the exported system images. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ selected_fields = ParseFields(opts.output, _LIST_DEF_FIELDS) qfilter = qlang.MakeSimpleFilter("node", opts.nodes) cl = GetClient() return GenericList(constants.QR_EXPORT, selected_fields, None, opts.units, opts.separator, not opts.no_headers, verbose=opts.verbose, qfilter=qfilter, cl=cl) def ListExportFields(opts, args): """List export fields. @param opts: the command line options selected by the user @type args: list @param args: fields to list, or empty for all @rtype: int @return: the desired exit code """ cl = GetClient() return GenericListFields(constants.QR_EXPORT, args, opts.separator, not opts.no_headers, cl=cl) def ExportInstance(opts, args): """Export an instance to an image in the cluster. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the name of the instance to be exported @rtype: int @return: the desired exit code """ ignore_remove_failures = opts.ignore_remove_failures if not opts.node: raise errors.OpPrereqError("Target node must be specified", errors.ECODE_INVAL) op = opcodes.OpBackupExport( instance_name=args[0], target_node=opts.node, compress=opts.transport_compression, shutdown=opts.shutdown, shutdown_timeout=opts.shutdown_timeout, remove_instance=opts.remove_instance, ignore_remove_failures=ignore_remove_failures, zero_free_space=opts.zero_free_space, zeroing_timeout_fixed=opts.zeroing_timeout_fixed, zeroing_timeout_per_mib=opts.zeroing_timeout_per_mib, long_sleep=opts.long_sleep ) SubmitOrSend(op, opts) return 0 def ImportInstance(opts, args): """Add an instance to the cluster. This is just a wrapper over GenericInstanceCreate. """ return GenericInstanceCreate(constants.INSTANCE_IMPORT, opts, args) def RemoveExport(opts, args): """Remove an export from the cluster. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the name of the instance whose backup should be removed @rtype: int @return: the desired exit code """ op = opcodes.OpBackupRemove(instance_name=args[0]) SubmitOrSend(op, opts) return 0 # this is defined separately due to readability only import_opts = [ IDENTIFY_DEFAULTS_OPT, SRC_DIR_OPT, SRC_NODE_OPT, COMPRESS_OPT, IGNORE_IPOLICY_OPT, HELPER_STARTUP_TIMEOUT_OPT, HELPER_SHUTDOWN_TIMEOUT_OPT, ] commands = { "list": ( PrintExportList, ARGS_NONE, [NODE_LIST_OPT, NOHDR_OPT, SEP_OPT, USEUNITS_OPT, FIELDS_OPT, VERBOSE_OPT], "", "Lists instance exports available in the ganeti cluster"), "list-fields": ( ListExportFields, [ArgUnknown()], [NOHDR_OPT, SEP_OPT], "[fields...]", "Lists all available fields for exports"), "export": ( ExportInstance, ARGS_ONE_INSTANCE, [FORCE_OPT, SINGLE_NODE_OPT, TRANSPORT_COMPRESSION_OPT, NOSHUTDOWN_OPT, SHUTDOWN_TIMEOUT_OPT, REMOVE_INSTANCE_OPT, IGNORE_REMOVE_FAILURES_OPT, DRY_RUN_OPT, PRIORITY_OPT, ZERO_FREE_SPACE_OPT, ZEROING_TIMEOUT_FIXED_OPT, ZEROING_TIMEOUT_PER_MIB_OPT, LONG_SLEEP_OPT] + SUBMIT_OPTS, "-n [opts...] ", "Exports an instance to an image"), "import": ( ImportInstance, ARGS_ONE_INSTANCE, COMMON_CREATE_OPTS + import_opts, "[...] -t disk-type -n node[:secondary-node] ", "Imports an instance from an exported image"), "remove": ( RemoveExport, [ArgUnknown(min=1, max=1)], [DRY_RUN_OPT, PRIORITY_OPT] + SUBMIT_OPTS, "", "Remove exports of named instance from the filesystem."), } def Main(): return GenericMain(commands) ganeti-3.1.0~rc2/lib/client/gnt_cluster.py000064400000000000000000002560561476477700300205510ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Cluster related commands""" # pylint: disable=W0401,W0613,W0614,C0103 # W0401: Wildcard import ganeti.cli # W0613: Unused argument, since all functions follow the same API # W0614: Unused import %s from wildcard import (since we need cli) # C0103: Invalid name gnt-cluster import itertools import os import time import tempfile from io import StringIO import OpenSSL from ganeti.cli import * from ganeti import bootstrap from ganeti import compat from ganeti import constants from ganeti import config from ganeti import errors from ganeti import netutils from ganeti import objects from ganeti import opcodes from ganeti import pathutils from ganeti import qlang from ganeti.rpc.node import RunWithRPC from ganeti import serializer from ganeti import ssconf from ganeti import ssh from ganeti import uidpool from ganeti import utils from ganeti.client import base ON_OPT = cli_option("--on", default=False, action="store_true", dest="on", help="Recover from an EPO") GROUPS_OPT = cli_option("--groups", default=False, action="store_true", dest="groups", help="Arguments are node groups instead of nodes") FORCE_FAILOVER = cli_option("--yes-do-it", dest="yes_do_it", help="Override interactive check for --no-voting", default=False, action="store_true") IGNORE_OFFLINE_NODES_FAILOVER = cli_option( "--ignore-offline-nodes", dest="ignore_offline_nodes", help="Ignores offline nodes for master failover voting", default=True) FORCE_DISTRIBUTION = cli_option("--yes-do-it", dest="yes_do_it", help="Unconditionally distribute the" " configuration, even if the queue" " is drained", default=False, action="store_true") TO_OPT = cli_option("--to", default=None, type="string", help="The Ganeti version to upgrade to") RESUME_OPT = cli_option("--resume", default=False, action="store_true", help="Resume any pending Ganeti upgrades") DATA_COLLECTOR_INTERVAL_OPT = cli_option( "--data-collector-interval", default={}, type="keyval", help="Set collection intervals in seconds of data collectors.") STRICT_OPT = cli_option("--no-strict", default=False, dest="no_strict", action="store_true", help="Do not run group verify in strict mode") _EPO_PING_INTERVAL = 30 # 30 seconds between pings _EPO_PING_TIMEOUT = 1 # 1 second _EPO_REACHABLE_TIMEOUT = 15 * 60 # 15 minutes def _InitEnabledDiskTemplates(opts): """Initialize the list of enabled disk templates. """ if opts.enabled_disk_templates: return opts.enabled_disk_templates.split(",") else: return constants.DEFAULT_ENABLED_DISK_TEMPLATES def _InitVgName(opts, enabled_disk_templates): """Initialize the volume group name. @type enabled_disk_templates: list of strings @param enabled_disk_templates: cluster-wide enabled disk templates """ vg_name = None if opts.vg_name is not None: vg_name = opts.vg_name if vg_name: if not utils.IsLvmEnabled(enabled_disk_templates): ToStdout("You specified a volume group with --vg-name, but you did not" " enable any disk template that uses lvm.") elif utils.IsLvmEnabled(enabled_disk_templates): raise errors.OpPrereqError( "LVM disk templates are enabled, but vg name not set.") elif utils.IsLvmEnabled(enabled_disk_templates): vg_name = constants.DEFAULT_VG return vg_name def _InitDrbdHelper(opts, enabled_disk_templates, feedback_fn=ToStdout): """Initialize the DRBD usermode helper. """ drbd_enabled = constants.DT_DRBD8 in enabled_disk_templates if not drbd_enabled and opts.drbd_helper is not None: feedback_fn("Note: You specified a DRBD usermode helper, while DRBD storage" " is not enabled.") if drbd_enabled: if opts.drbd_helper is None: return constants.DEFAULT_DRBD_HELPER if opts.drbd_helper == '': raise errors.OpPrereqError( "Unsetting the drbd usermode helper while enabling DRBD is not" " allowed.") return opts.drbd_helper @RunWithRPC def InitCluster(opts, args): """Initialize the cluster. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the desired cluster name @rtype: int @return: the desired exit code """ enabled_disk_templates = _InitEnabledDiskTemplates(opts) try: vg_name = _InitVgName(opts, enabled_disk_templates) drbd_helper = _InitDrbdHelper(opts, enabled_disk_templates) except errors.OpPrereqError as e: ToStderr(str(e)) return 1 master_netdev = opts.master_netdev if master_netdev is None: nic_mode = opts.nicparams.get(constants.NIC_MODE, None) if not nic_mode: # default case, use bridging master_netdev = constants.DEFAULT_BRIDGE elif nic_mode == constants.NIC_MODE_OVS: # default ovs is different from default bridge master_netdev = constants.DEFAULT_OVS opts.nicparams[constants.NIC_LINK] = constants.DEFAULT_OVS hvlist = opts.enabled_hypervisors if hvlist is None: hvlist = constants.DEFAULT_ENABLED_HYPERVISOR hvlist = hvlist.split(",") hvparams = dict(opts.hvparams) beparams = opts.beparams nicparams = opts.nicparams diskparams = dict(opts.diskparams) # check the disk template types here, as we cannot rely on the type check done # by the opcode parameter types diskparams_keys = set(diskparams.keys()) if diskparams_keys > constants.DISK_TEMPLATES: unknown = utils.NiceSort(diskparams_keys - constants.DISK_TEMPLATES) ToStderr("Disk templates unknown: %s" % utils.CommaJoin(unknown)) return 1 # prepare beparams dict beparams = objects.FillDict(constants.BEC_DEFAULTS, beparams) utils.ForceDictType(beparams, constants.BES_PARAMETER_COMPAT) # prepare nicparams dict nicparams = objects.FillDict(constants.NICC_DEFAULTS, nicparams) utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES) # prepare ndparams dict if opts.ndparams is None: ndparams = dict(constants.NDC_DEFAULTS) else: ndparams = objects.FillDict(constants.NDC_DEFAULTS, opts.ndparams) utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES) # prepare hvparams dict for hv in constants.HYPER_TYPES: if hv not in hvparams: hvparams[hv] = {} hvparams[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], hvparams[hv]) utils.ForceDictType(hvparams[hv], constants.HVS_PARAMETER_TYPES) # prepare diskparams dict for templ in constants.DISK_TEMPLATES: if templ not in diskparams: diskparams[templ] = {} diskparams[templ] = objects.FillDict(constants.DISK_DT_DEFAULTS[templ], diskparams[templ]) utils.ForceDictType(diskparams[templ], constants.DISK_DT_TYPES) # prepare ipolicy dict ipolicy = CreateIPolicyFromOpts( ispecs_mem_size=opts.ispecs_mem_size, ispecs_cpu_count=opts.ispecs_cpu_count, ispecs_disk_count=opts.ispecs_disk_count, ispecs_disk_size=opts.ispecs_disk_size, ispecs_nic_count=opts.ispecs_nic_count, minmax_ispecs=opts.ipolicy_bounds_specs, std_ispecs=opts.ipolicy_std_specs, ipolicy_disk_templates=opts.ipolicy_disk_templates, ipolicy_vcpu_ratio=opts.ipolicy_vcpu_ratio, ipolicy_spindle_ratio=opts.ipolicy_spindle_ratio, fill_all=True) if opts.candidate_pool_size is None: opts.candidate_pool_size = constants.MASTER_POOL_SIZE_DEFAULT if opts.mac_prefix is None: opts.mac_prefix = constants.DEFAULT_MAC_PREFIX uid_pool = opts.uid_pool if uid_pool is not None: uid_pool = uidpool.ParseUidPool(uid_pool) if opts.prealloc_wipe_disks is None: opts.prealloc_wipe_disks = False external_ip_setup_script = opts.use_external_mip_script if external_ip_setup_script is None: external_ip_setup_script = False try: primary_ip_version = int(opts.primary_ip_version) except (ValueError, TypeError) as err: ToStderr("Invalid primary ip version value: %s" % str(err)) return 1 master_netmask = opts.master_netmask try: if master_netmask is not None: master_netmask = int(master_netmask) except (ValueError, TypeError) as err: ToStderr("Invalid master netmask value: %s" % str(err)) return 1 if opts.disk_state: disk_state = utils.FlatToDict(opts.disk_state) else: disk_state = {} hv_state = dict(opts.hv_state) if opts.install_image: install_image = opts.install_image else: install_image = "" if opts.zeroing_image: zeroing_image = opts.zeroing_image else: zeroing_image = "" compression_tools = _GetCompressionTools(opts) default_ialloc_params = opts.default_iallocator_params enabled_user_shutdown = bool(opts.enabled_user_shutdown) if opts.ssh_key_type: ssh_key_type = opts.ssh_key_type else: ssh_key_type = constants.SSH_DEFAULT_KEY_TYPE ssh_key_bits = ssh.DetermineKeyBits(ssh_key_type, opts.ssh_key_bits, None, None) bootstrap.InitCluster(cluster_name=args[0], secondary_ip=opts.secondary_ip, vg_name=vg_name, mac_prefix=opts.mac_prefix, master_netmask=master_netmask, master_netdev=master_netdev, file_storage_dir=opts.file_storage_dir, shared_file_storage_dir=opts.shared_file_storage_dir, gluster_storage_dir=opts.gluster_storage_dir, enabled_hypervisors=hvlist, hvparams=hvparams, beparams=beparams, nicparams=nicparams, ndparams=ndparams, diskparams=diskparams, ipolicy=ipolicy, candidate_pool_size=opts.candidate_pool_size, modify_etc_hosts=opts.modify_etc_hosts, modify_ssh_setup=opts.modify_ssh_setup, maintain_node_health=opts.maintain_node_health, drbd_helper=drbd_helper, uid_pool=uid_pool, default_iallocator=opts.default_iallocator, default_iallocator_params=default_ialloc_params, primary_ip_version=primary_ip_version, prealloc_wipe_disks=opts.prealloc_wipe_disks, use_external_mip_script=external_ip_setup_script, hv_state=hv_state, disk_state=disk_state, enabled_disk_templates=enabled_disk_templates, install_image=install_image, zeroing_image=zeroing_image, compression_tools=compression_tools, enabled_user_shutdown=enabled_user_shutdown, ssh_key_type=ssh_key_type, ssh_key_bits=ssh_key_bits, ) op = opcodes.OpClusterPostInit() SubmitOpCode(op, opts=opts) return 0 @RunWithRPC def DestroyCluster(opts, args): """Destroy the cluster. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ if not opts.yes_do_it: ToStderr("Destroying a cluster is irreversible. If you really want" " destroy this cluster, supply the --yes-do-it option.") return 1 op = opcodes.OpClusterDestroy() master_uuid = SubmitOpCode(op, opts=opts) # if we reached this, the opcode didn't fail; we can proceed to # shutdown all the daemons bootstrap.FinalizeClusterDestroy(master_uuid) return 0 def RenameCluster(opts, args): """Rename the cluster. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the new cluster name @rtype: int @return: the desired exit code """ cl = GetClient() (cluster_name, ) = cl.QueryConfigValues(["cluster_name"]) new_name = args[0] if not opts.force: usertext = ("This will rename the cluster from '%s' to '%s'. If you are" " connected over the network to the cluster name, the" " operation is very dangerous as the IP address will be" " removed from the node and the change may not go through." " Continue?") % (cluster_name, new_name) if not AskUser(usertext): return 1 op = opcodes.OpClusterRename(name=new_name) result = SubmitOpCode(op, opts=opts, cl=cl) if result: ToStdout("Cluster renamed from '%s' to '%s'", cluster_name, result) return 0 def ActivateMasterIp(opts, args): """Activates the master IP. """ op = opcodes.OpClusterActivateMasterIp() SubmitOpCode(op) return 0 def DeactivateMasterIp(opts, args): """Deactivates the master IP. """ if not opts.confirm: usertext = ("This will disable the master IP. All the open connections to" " the master IP will be closed. To reach the master you will" " need to use its node IP." " Continue?") if not AskUser(usertext): return 1 op = opcodes.OpClusterDeactivateMasterIp() SubmitOpCode(op) return 0 def RedistributeConfig(opts, args): """Forces push of the cluster configuration. @param opts: the command line options selected by the user @type args: list @param args: empty list @rtype: int @return: the desired exit code """ op = opcodes.OpClusterRedistConf() if opts.yes_do_it: SubmitOpCodeToDrainedQueue(op) else: SubmitOrSend(op, opts) return 0 def ShowClusterVersion(opts, args): """Write version of ganeti software to the standard output. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ cl = GetClient() result = cl.QueryClusterInfo() ToStdout("Software version: %s", result["software_version"]) ToStdout("Internode protocol: %s", result["protocol_version"]) ToStdout("Configuration format: %s", result["config_version"]) ToStdout("OS api version: %s", result["os_api_version"]) ToStdout("Export interface: %s", result["export_version"]) ToStdout("VCS version: %s", result["vcs_version"]) return 0 def ShowClusterMaster(opts, args): """Write name of master node to the standard output. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ master = bootstrap.GetMaster() ToStdout(master) return 0 def _FormatGroupedParams(paramsdict, roman=False): """Format Grouped parameters (be, nic, disk) by group. @type paramsdict: dict of dicts @param paramsdict: {group: {param: value, ...}, ...} @rtype: dict of dicts @return: copy of the input dictionaries with strings as values """ ret = {} for (item, val) in paramsdict.items(): if isinstance(val, dict): ret[item] = _FormatGroupedParams(val, roman=roman) elif roman and isinstance(val, int): ret[item] = compat.TryToRoman(val) else: ret[item] = str(val) return ret def _FormatDataCollectors(paramsdict): """Format Grouped parameters (be, nic, disk) by group. @type paramsdict: dict of dicts @param paramsdict: response of QueryClusterInfo @rtype: dict of dicts @return: parameter grouped by data collector """ enabled = paramsdict[constants.DATA_COLLECTORS_ENABLED_NAME] interval = paramsdict[constants.DATA_COLLECTORS_INTERVAL_NAME] ret = {} for key in enabled: ret[key] = dict(active=enabled[key], interval="%.3fs" % (interval[key] / 1e6)) return ret def ShowClusterConfig(opts, args): """Shows cluster information. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ cl = GetClient() result = cl.QueryClusterInfo() if result["tags"]: tags = utils.CommaJoin(utils.NiceSort(result["tags"])) else: tags = "(none)" if result["reserved_lvs"]: reserved_lvs = utils.CommaJoin(result["reserved_lvs"]) else: reserved_lvs = "(none)" enabled_hv = result["enabled_hypervisors"] hvparams = dict((k, v) for k, v in result["hvparams"].items() if k in enabled_hv) info = [ ("Cluster name", result["name"]), ("Cluster UUID", result["uuid"]), ("Creation time", utils.FormatTime(result["ctime"])), ("Modification time", utils.FormatTime(result["mtime"])), ("Master node", result["master"]), ("Architecture (this node)", "%s (%s)" % (result["architecture"][0], result["architecture"][1])), ("Tags", tags), ("Default hypervisor", result["default_hypervisor"]), ("Enabled hypervisors", utils.CommaJoin(enabled_hv)), ("Hypervisor parameters", _FormatGroupedParams(hvparams, opts.roman_integers)), ("OS-specific hypervisor parameters", _FormatGroupedParams(result["os_hvp"], opts.roman_integers)), ("OS parameters", _FormatGroupedParams(result["osparams"], opts.roman_integers)), ("Hidden OSes", utils.CommaJoin(result["hidden_os"])), ("Blacklisted OSes", utils.CommaJoin(result["blacklisted_os"])), ("Cluster parameters", [ ("candidate pool size", compat.TryToRoman(result["candidate_pool_size"], convert=opts.roman_integers)), ("maximal number of jobs running simultaneously", compat.TryToRoman(result["max_running_jobs"], convert=opts.roman_integers)), ("maximal number of jobs simultaneously tracked by the scheduler", compat.TryToRoman(result["max_tracked_jobs"], convert=opts.roman_integers)), ("mac prefix", result["mac_prefix"]), ("master netdev", result["master_netdev"]), ("master netmask", compat.TryToRoman(result["master_netmask"], opts.roman_integers)), ("use external master IP address setup script", result["use_external_mip_script"]), ("lvm volume group", result["volume_group_name"]), ("lvm reserved volumes", reserved_lvs), ("drbd usermode helper", result["drbd_usermode_helper"]), ("file storage path", result["file_storage_dir"]), ("shared file storage path", result["shared_file_storage_dir"]), ("gluster storage path", result["gluster_storage_dir"]), ("maintenance of node health", result["maintain_node_health"]), ("uid pool", uidpool.FormatUidPool(result["uid_pool"])), ("default instance allocator", result["default_iallocator"]), ("default instance allocator parameters", result["default_iallocator_params"]), ("primary ip version", compat.TryToRoman(result["primary_ip_version"], opts.roman_integers)), ("preallocation wipe disks", result["prealloc_wipe_disks"]), ("OS search path", utils.CommaJoin(pathutils.OS_SEARCH_PATH)), ("ExtStorage Providers search path", utils.CommaJoin(pathutils.ES_SEARCH_PATH)), ("enabled disk templates", utils.CommaJoin(result["enabled_disk_templates"])), ("install image", result["install_image"]), ("instance communication network", result["instance_communication_network"]), ("zeroing image", result["zeroing_image"]), ("compression tools", result["compression_tools"]), ("enabled user shutdown", result["enabled_user_shutdown"]), ("modify ssh setup", result["modify_ssh_setup"]), ("ssh_key_type", result["ssh_key_type"]), ("ssh_key_bits", result["ssh_key_bits"]), ]), ("Default node parameters", _FormatGroupedParams(result["ndparams"], roman=opts.roman_integers)), ("Default instance parameters", _FormatGroupedParams(result["beparams"], roman=opts.roman_integers)), ("Default nic parameters", _FormatGroupedParams(result["nicparams"], roman=opts.roman_integers)), ("Default disk parameters", _FormatGroupedParams(result["diskparams"], roman=opts.roman_integers)), ("Instance policy - limits for instances", FormatPolicyInfo(result["ipolicy"], None, True, opts.roman_integers)), ("Data collectors", _FormatDataCollectors(result)), ] PrintGenericInfo(info) return 0 def ClusterCopyFile(opts, args): """Copy a file from master to some nodes. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the path of the file to be copied @rtype: int @return: the desired exit code """ filename = args[0] filename = os.path.abspath(filename) if not os.path.exists(filename): raise errors.OpPrereqError("No such filename '%s'" % filename, errors.ECODE_INVAL) cl = GetClient() qcl = GetClient() try: cluster_name = cl.QueryConfigValues(["cluster_name"])[0] results = GetOnlineNodes(nodes=opts.nodes, cl=qcl, filter_master=True, secondary_ips=opts.use_replication_network, nodegroup=opts.nodegroup) ports = GetNodesSshPorts(opts.nodes, qcl) finally: cl.Close() qcl.Close() srun = ssh.SshRunner(cluster_name) for (node, port) in zip(results, ports): if not srun.CopyFileToNode(node, port, filename): ToStderr("Copy of file %s to node %s:%d failed", filename, node, port) return 0 def RunClusterCommand(opts, args): """Run a command on some nodes. @param opts: the command line options selected by the user @type args: list @param args: should contain the command to be run and its arguments @rtype: int @return: the desired exit code """ cl = GetClient() qcl = GetClient() command = " ".join(args) nodes = GetOnlineNodes(nodes=opts.nodes, cl=qcl, nodegroup=opts.nodegroup) ports = GetNodesSshPorts(nodes, qcl) cluster_name, master_node = cl.QueryConfigValues(["cluster_name", "master_node"]) srun = ssh.SshRunner(cluster_name=cluster_name) # Make sure master node is at list end if master_node in nodes: nodes.remove(master_node) nodes.append(master_node) for (name, port) in zip(nodes, ports): result = srun.Run(name, constants.SSH_LOGIN_USER, command, port=port) if opts.failure_only and result.exit_code == constants.EXIT_SUCCESS: # Do not output anything for successful commands continue ToStdout("------------------------------------------------") if opts.show_machine_names: for line in result.output.splitlines(): ToStdout("%s: %s", name, line) else: ToStdout("node: %s", name) ToStdout("%s", result.output) ToStdout("return code = %s", result.exit_code) return 0 def VerifyCluster(opts, args): """Verify integrity of cluster, performing various test on nodes. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ skip_checks = [] if opts.skip_nplusone_mem: skip_checks.append(constants.VERIFY_NPLUSONE_MEM) if opts.skip_hvparam_assessment: skip_checks.append(constants.VERIFY_HVPARAM_ASSESSMENT) cl = GetClient() op = opcodes.OpClusterVerify(verbose=opts.verbose, error_codes=opts.error_codes, debug_simulate_errors=opts.simulate_errors, skip_checks=skip_checks, ignore_errors=opts.ignore_errors, group_name=opts.nodegroup, verify_clutter=opts.verify_clutter) result = SubmitOpCode(op, cl=cl, opts=opts) # Keep track of submitted jobs jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result[constants.JOB_IDS_KEY]: jex.AddJobId(None, status, job_id) results = jex.GetResults() bad_jobs = sum(1 for (job_success, _) in results if not job_success) bad_results = sum(1 for (_, op_res) in results if not (op_res and op_res[0])) if bad_jobs == 0 and bad_results == 0: rcode = constants.EXIT_SUCCESS else: rcode = constants.EXIT_FAILURE if bad_jobs > 0: ToStdout("%s job(s) failed while verifying the cluster.", bad_jobs) return rcode def VerifyDisks(opts, args): """Verify integrity of cluster disks. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ cl = GetClient() op = opcodes.OpClusterVerifyDisks(group_name=opts.nodegroup, is_strict=not opts.no_strict) result = SubmitOpCode(op, cl=cl, opts=opts) # Keep track of submitted jobs jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result[constants.JOB_IDS_KEY]: jex.AddJobId(None, status, job_id) retcode = constants.EXIT_SUCCESS for (status, result) in jex.GetResults(): if not status: ToStdout("Job failed: %s", result) continue ((bad_nodes, instances, missing), ) = result for node, text in bad_nodes.items(): ToStdout("Error gathering data on node %s: %s", node, utils.SafeEncode(text[-400:])) retcode = constants.EXIT_FAILURE ToStdout("You need to fix these nodes first before fixing instances") for iname in instances: if iname in missing: continue op = opcodes.OpInstanceActivateDisks(instance_name=iname) try: ToStdout("Activating disks for instance '%s'", iname) SubmitOpCode(op, opts=opts, cl=cl) except errors.GenericError as err: nret, msg = FormatError(err) retcode |= nret ToStderr("Error activating disks for instance %s: %s", iname, msg) if missing: for iname, ival in missing.items(): all_missing = compat.all(x[0] in bad_nodes for x in ival) if all_missing: ToStdout("Instance %s cannot be verified as it lives on" " broken nodes", iname) continue ToStdout("Instance %s has missing logical volumes:", iname) ival.sort() for node, vol in ival: if node in bad_nodes: ToStdout("\tbroken node %s /dev/%s", node, vol) else: ToStdout("\t%s /dev/%s", node, vol) ToStdout("You need to replace or recreate disks for all the above" " instances if this message persists after fixing broken nodes.") retcode = constants.EXIT_FAILURE elif not instances: ToStdout("No disks need to be activated.") return retcode def RepairDiskSizes(opts, args): """Verify sizes of cluster disks. @param opts: the command line options selected by the user @type args: list @param args: optional list of instances to restrict check to @rtype: int @return: the desired exit code """ op = opcodes.OpClusterRepairDiskSizes(instances=args) SubmitOpCode(op, opts=opts) @RunWithRPC def MasterFailover(opts, args): """Failover the master node. This command, when run on a non-master node, will cause the current master to cease being master, and the non-master to become new master. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ if opts.no_voting: # Don't ask for confirmation if the user provides the confirmation flag. if not opts.yes_do_it: usertext = ("This will perform the failover even if most other nodes" " are down, or if this node is outdated. This is dangerous" " as it can lead to a non-consistent cluster. Check the" " gnt-cluster(8) man page before proceeding. Continue?") if not AskUser(usertext): return 1 else: # Verify that a majority of nodes are still healthy (majority_healthy, unhealthy_nodes) = bootstrap.MajorityHealthy( opts.ignore_offline_nodes) if not majority_healthy: ToStderr("Master-failover with voting is only possible if the majority" " of nodes are still healthy; use the --no-voting option after" " ensuring by other means that you won't end up in a dual-master" " scenario. Unhealthy nodes: %s" % unhealthy_nodes) return 1 rvalue, msgs = bootstrap.MasterFailover(no_voting=opts.no_voting) for msg in msgs: ToStderr(msg) return rvalue def MasterPing(opts, args): """Checks if the master is alive. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ try: cl = GetClient() cl.QueryClusterInfo() return 0 except Exception: # pylint: disable=W0703 return 1 def SearchTags(opts, args): """Searches the tags on all the cluster. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the tag pattern @rtype: int @return: the desired exit code """ op = opcodes.OpTagsSearch(pattern=args[0]) result = SubmitOpCode(op, opts=opts) if not result: return 1 result = list(result) result.sort() for path, tag in result: ToStdout("%s %s", path, tag) def _ReadAndVerifyCert(cert_filename, verify_private_key=False): """Reads and verifies an X509 certificate. @type cert_filename: string @param cert_filename: the path of the file containing the certificate to verify encoded in PEM format @type verify_private_key: bool @param verify_private_key: whether to verify the private key in addition to the public certificate @rtype: string @return: a string containing the PEM-encoded certificate. """ try: pem = utils.ReadFile(cert_filename) except IOError as err: raise errors.X509CertError(cert_filename, "Unable to read certificate: %s" % str(err)) try: OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pem) except Exception as err: raise errors.X509CertError(cert_filename, "Unable to load certificate: %s" % str(err)) if verify_private_key: try: OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, pem) except Exception as err: raise errors.X509CertError(cert_filename, "Unable to load private key: %s" % str(err)) return pem # pylint: disable=R0913 def _RenewCrypto(new_cluster_cert, new_rapi_cert, # pylint: disable=R0911 rapi_cert_filename, new_spice_cert, spice_cert_filename, spice_cacert_filename, new_confd_hmac_key, new_cds, cds_filename, force, new_node_cert, new_ssh_keys, ssh_key_type, ssh_key_bits, verbose, debug): """Renews cluster certificates, keys and secrets. @type new_cluster_cert: bool @param new_cluster_cert: Whether to generate a new cluster certificate @type new_rapi_cert: bool @param new_rapi_cert: Whether to generate a new RAPI certificate @type rapi_cert_filename: string @param rapi_cert_filename: Path to file containing new RAPI certificate @type new_spice_cert: bool @param new_spice_cert: Whether to generate a new SPICE certificate @type spice_cert_filename: string @param spice_cert_filename: Path to file containing new SPICE certificate @type spice_cacert_filename: string @param spice_cacert_filename: Path to file containing the certificate of the CA that signed the SPICE certificate @type new_confd_hmac_key: bool @param new_confd_hmac_key: Whether to generate a new HMAC key @type new_cds: bool @param new_cds: Whether to generate a new cluster domain secret @type cds_filename: string @param cds_filename: Path to file containing new cluster domain secret @type force: bool @param force: Whether to ask user for confirmation @type new_node_cert: bool @param new_node_cert: Whether to generate new node certificates @type new_ssh_keys: bool @param new_ssh_keys: Whether to generate new node SSH keys @type ssh_key_type: One of L{constants.SSHK_ALL} @param ssh_key_type: The type of SSH key to be generated @type ssh_key_bits: int @param ssh_key_bits: The length of the key to be generated @type verbose: boolean @param verbose: Show verbose output @type debug: boolean @param debug: Show debug output """ ToStdout("Updating certificates now. Running \"gnt-cluster verify\" " " is recommended after this operation.") if new_rapi_cert and rapi_cert_filename: ToStderr("Only one of the --new-rapi-certificate and --rapi-certificate" " options can be specified at the same time.") return 1 if new_cds and cds_filename: ToStderr("Only one of the --new-cluster-domain-secret and" " --cluster-domain-secret options can be specified at" " the same time.") return 1 if new_spice_cert and (spice_cert_filename or spice_cacert_filename): ToStderr("When using --new-spice-certificate, the --spice-certificate" " and --spice-ca-certificate must not be used.") return 1 if bool(spice_cacert_filename) ^ bool(spice_cert_filename): ToStderr("Both --spice-certificate and --spice-ca-certificate must be" " specified.") return 1 rapi_cert_pem, spice_cert_pem, spice_cacert_pem = (None, None, None) try: if rapi_cert_filename: rapi_cert_pem = _ReadAndVerifyCert(rapi_cert_filename, True) if spice_cert_filename: spice_cert_pem = _ReadAndVerifyCert(spice_cert_filename, True) spice_cacert_pem = _ReadAndVerifyCert(spice_cacert_filename) except errors.X509CertError as err: ToStderr("Unable to load X509 certificate from %s: %s", err.args[0], err.args[1]) return 1 if cds_filename: try: cds = utils.ReadFile(cds_filename) except Exception as err: # pylint: disable=W0703 ToStderr("Can't load new cluster domain secret from %s: %s" % (cds_filename, str(err))) return 1 else: cds = None if not force: usertext = ("This requires all daemons on all nodes to be restarted and" " may take some time. Continue?") if not AskUser(usertext): return 1 def _RenewCryptoInner(ctx): ctx.feedback_fn("Updating certificates and keys") bootstrap.GenerateClusterCrypto(False, new_rapi_cert, new_spice_cert, new_confd_hmac_key, new_cds, False, None, rapi_cert_pem=rapi_cert_pem, spice_cert_pem=spice_cert_pem, spice_cacert_pem=spice_cacert_pem, cds=cds) files_to_copy = [] if new_rapi_cert or rapi_cert_pem: files_to_copy.append(pathutils.RAPI_CERT_FILE) if new_spice_cert or spice_cert_pem: files_to_copy.append(pathutils.SPICE_CERT_FILE) files_to_copy.append(pathutils.SPICE_CACERT_FILE) if new_confd_hmac_key: files_to_copy.append(pathutils.CONFD_HMAC_KEY) if new_cds or cds: files_to_copy.append(pathutils.CLUSTER_DOMAIN_SECRET_FILE) if files_to_copy: for node_name in ctx.nonmaster_nodes: port = ctx.ssh_ports[node_name] ctx.feedback_fn("Copying %s to %s:%d" % (", ".join(files_to_copy), node_name, port)) for file_name in files_to_copy: ctx.ssh.CopyFileToNode(node_name, port, file_name) def _RenewClientCerts(ctx): ctx.feedback_fn("Updating client SSL certificates.") cluster_name = ssconf.SimpleStore().GetClusterName() for node_name in ctx.nonmaster_nodes + [ctx.master_node]: ssh_port = ctx.ssh_ports[node_name] data = { constants.NDS_CLUSTER_NAME: cluster_name, constants.NDS_NODE_DAEMON_CERTIFICATE: utils.ReadFile(pathutils.NODED_CERT_FILE), constants.NDS_NODE_NAME: node_name, constants.NDS_ACTION: constants.CRYPTO_ACTION_CREATE, } ssh.RunSshCmdWithStdin( cluster_name, node_name, pathutils.SSL_UPDATE, ssh_port, data, debug=ctx.debug, verbose=ctx.verbose, use_cluster_key=True, ask_key=False, strict_host_check=True) # Create a temporary ssconf file using the master's client cert digest # and the 'bootstrap' keyword to enable distribution of all nodes' digests. master_digest = utils.GetCertificateDigest() ssconf_master_candidate_certs_filename = os.path.join( pathutils.DATA_DIR, "%s%s" % (constants.SSCONF_FILEPREFIX, constants.SS_MASTER_CANDIDATES_CERTS)) utils.WriteFile( ssconf_master_candidate_certs_filename, data="%s=%s" % (constants.CRYPTO_BOOTSTRAP, master_digest)) for node_name in ctx.nonmaster_nodes: port = ctx.ssh_ports[node_name] ctx.feedback_fn("Copying %s to %s:%d" % (ssconf_master_candidate_certs_filename, node_name, port)) ctx.ssh.CopyFileToNode(node_name, port, ssconf_master_candidate_certs_filename) # Write the boostrap entry to the config using wconfd. config_live_lock = utils.livelock.LiveLock("renew_crypto") cfg = config.GetConfig(None, config_live_lock) cfg.AddNodeToCandidateCerts(constants.CRYPTO_BOOTSTRAP, master_digest) cfg.Update(cfg.GetClusterInfo(), ctx.feedback_fn) def _RenewServerAndClientCerts(ctx): ctx.feedback_fn("Updating the cluster SSL certificate.") master_name = ssconf.SimpleStore().GetMasterNode() bootstrap.GenerateClusterCrypto(True, # cluster cert False, # rapi cert False, # spice cert False, # confd hmac key False, # cds True, # client cert master_name) for node_name in ctx.nonmaster_nodes: port = ctx.ssh_ports[node_name] server_cert = pathutils.NODED_CERT_FILE ctx.feedback_fn("Copying %s to %s:%d" % (server_cert, node_name, port)) ctx.ssh.CopyFileToNode(node_name, port, server_cert) _RenewClientCerts(ctx) if new_rapi_cert or new_spice_cert or new_confd_hmac_key or new_cds: RunWhileClusterStopped(ToStdout, _RenewCryptoInner) # If only node certificates are recreated, call _RenewClientCerts only. if new_node_cert and not new_cluster_cert: RunWhileDaemonsStopped(ToStdout, [constants.NODED, constants.WCONFD], _RenewClientCerts, verbose=verbose, debug=debug) # If the cluster certificate are renewed, the client certificates need # to be renewed too. if new_cluster_cert: RunWhileDaemonsStopped(ToStdout, [constants.NODED, constants.WCONFD], _RenewServerAndClientCerts, verbose=verbose, debug=debug) if new_node_cert or new_cluster_cert or new_ssh_keys: cl = GetClient() renew_op = opcodes.OpClusterRenewCrypto( node_certificates=new_node_cert or new_cluster_cert, renew_ssh_keys=new_ssh_keys, ssh_key_type=ssh_key_type, ssh_key_bits=ssh_key_bits) SubmitOpCode(renew_op, cl=cl) ToStdout("All requested certificates and keys have been replaced." " Running \"gnt-cluster verify\" now is recommended.") return 0 def _BuildGanetiPubKeys(options, pub_key_file=pathutils.SSH_PUB_KEYS, cl=None, get_online_nodes_fn=GetOnlineNodes, get_nodes_ssh_ports_fn=GetNodesSshPorts, get_node_uuids_fn=GetNodeUUIDs, homedir_fn=None): """Recreates the 'ganeti_pub_key' file by polling all nodes. """ if not cl: cl = GetClient() (cluster_name, master_node, modify_ssh_setup, ssh_key_type) = \ cl.QueryConfigValues(["cluster_name", "master_node", "modify_ssh_setup", "ssh_key_type"]) # In case Ganeti is not supposed to modify the SSH setup, simply exit and do # not update this file. if not modify_ssh_setup: return if os.path.exists(pub_key_file): utils.CreateBackup(pub_key_file) utils.RemoveFile(pub_key_file) ssh.ClearPubKeyFile(pub_key_file) online_nodes = get_online_nodes_fn([], cl=cl) ssh_ports = get_nodes_ssh_ports_fn(online_nodes + [master_node], cl) ssh_port_map = dict(zip(online_nodes + [master_node], ssh_ports)) node_uuids = get_node_uuids_fn(online_nodes + [master_node], cl) node_uuid_map = dict(zip(online_nodes + [master_node], node_uuids)) nonmaster_nodes = [name for name in online_nodes if name != master_node] _, pub_key_filename, _ = \ ssh.GetUserFiles(constants.SSH_LOGIN_USER, mkdir=False, dircheck=False, kind=ssh_key_type, _homedir_fn=homedir_fn) # get the key file of the master node pub_key = utils.ReadFile(pub_key_filename) ssh.AddPublicKey(node_uuid_map[master_node], pub_key, key_file=pub_key_file) # get the key files of all non-master nodes for node in nonmaster_nodes: pub_key = ssh.ReadRemoteSshPubKeys(pub_key_filename, node, cluster_name, ssh_port_map[node], options.ssh_key_check, options.ssh_key_check) ssh.AddPublicKey(node_uuid_map[node], pub_key, key_file=pub_key_file) def RenewCrypto(opts, args): """Renews cluster certificates, keys and secrets. """ if opts.new_ssh_keys: _BuildGanetiPubKeys(opts) return _RenewCrypto(opts.new_cluster_cert, opts.new_rapi_cert, opts.rapi_cert, opts.new_spice_cert, opts.spice_cert, opts.spice_cacert, opts.new_confd_hmac_key, opts.new_cluster_domain_secret, opts.cluster_domain_secret, opts.force, opts.new_node_cert, opts.new_ssh_keys, opts.ssh_key_type, opts.ssh_key_bits, opts.verbose, opts.debug > 0) def _GetEnabledDiskTemplates(opts): """Determine the list of enabled disk templates. """ if opts.enabled_disk_templates: return opts.enabled_disk_templates.split(",") else: return None def _GetVgName(opts, enabled_disk_templates): """Determine the volume group name. @type enabled_disk_templates: list of strings @param enabled_disk_templates: cluster-wide enabled disk-templates """ # consistency between vg name and enabled disk templates vg_name = None if opts.vg_name is not None: vg_name = opts.vg_name if enabled_disk_templates: if vg_name and not utils.IsLvmEnabled(enabled_disk_templates): ToStdout("You specified a volume group with --vg-name, but you did not" " enable any of the following lvm-based disk templates: %s" % utils.CommaJoin(constants.DTS_LVM)) return vg_name def _GetDrbdHelper(opts, enabled_disk_templates): """Determine the DRBD usermode helper. """ drbd_helper = opts.drbd_helper if enabled_disk_templates: drbd_enabled = constants.DT_DRBD8 in enabled_disk_templates if not drbd_enabled and opts.drbd_helper: ToStdout("You specified a DRBD usermode helper with " " --drbd-usermode-helper while DRBD is not enabled.") return drbd_helper def _GetCompressionTools(opts): """Determine the list of custom compression tools. """ if opts.compression_tools: return opts.compression_tools.split(",") elif opts.compression_tools is None: return None # To note the parameter was not provided else: return constants.IEC_DEFAULT_TOOLS # Resetting to default def SetClusterParams(opts, args): """Modify the cluster. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ if not (opts.vg_name is not None or opts.drbd_helper is not None or opts.enabled_hypervisors or opts.hvparams or opts.beparams or opts.nicparams or opts.ndparams or opts.diskparams or opts.candidate_pool_size is not None or opts.max_running_jobs is not None or opts.max_tracked_jobs is not None or opts.uid_pool is not None or opts.maintain_node_health is not None or opts.add_uids is not None or opts.remove_uids is not None or opts.default_iallocator is not None or opts.default_iallocator_params is not None or opts.reserved_lvs is not None or opts.mac_prefix is not None or opts.master_netdev is not None or opts.master_netmask is not None or opts.use_external_mip_script is not None or opts.prealloc_wipe_disks is not None or opts.hv_state or opts.enabled_disk_templates or opts.disk_state or opts.ipolicy_bounds_specs is not None or opts.ipolicy_std_specs is not None or opts.ipolicy_disk_templates is not None or opts.ipolicy_vcpu_ratio is not None or opts.ipolicy_spindle_ratio is not None or opts.modify_etc_hosts is not None or opts.file_storage_dir is not None or opts.install_image is not None or opts.instance_communication_network is not None or opts.zeroing_image is not None or opts.shared_file_storage_dir is not None or opts.compression_tools is not None or opts.shared_file_storage_dir is not None or opts.enabled_user_shutdown is not None or opts.data_collector_interval or opts.enabled_data_collectors): ToStderr("Please give at least one of the parameters.") return 1 enabled_disk_templates = _GetEnabledDiskTemplates(opts) vg_name = _GetVgName(opts, enabled_disk_templates) try: drbd_helper = _GetDrbdHelper(opts, enabled_disk_templates) except errors.OpPrereqError as e: ToStderr(str(e)) return 1 hvlist = opts.enabled_hypervisors if hvlist is not None: hvlist = hvlist.split(",") # a list of (name, dict) we can pass directly to dict() (or []) hvparams = dict(opts.hvparams) for hv_params in hvparams.values(): utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES) diskparams = dict(opts.diskparams) for dt_params in diskparams.values(): utils.ForceDictType(dt_params, constants.DISK_DT_TYPES) beparams = opts.beparams utils.ForceDictType(beparams, constants.BES_PARAMETER_COMPAT) nicparams = opts.nicparams utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES) ndparams = opts.ndparams if ndparams is not None: utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES) ipolicy = CreateIPolicyFromOpts( minmax_ispecs=opts.ipolicy_bounds_specs, std_ispecs=opts.ipolicy_std_specs, ipolicy_disk_templates=opts.ipolicy_disk_templates, ipolicy_vcpu_ratio=opts.ipolicy_vcpu_ratio, ipolicy_spindle_ratio=opts.ipolicy_spindle_ratio, ) mnh = opts.maintain_node_health uid_pool = opts.uid_pool if uid_pool is not None: uid_pool = uidpool.ParseUidPool(uid_pool) add_uids = opts.add_uids if add_uids is not None: add_uids = uidpool.ParseUidPool(add_uids) remove_uids = opts.remove_uids if remove_uids is not None: remove_uids = uidpool.ParseUidPool(remove_uids) if opts.reserved_lvs is not None: if opts.reserved_lvs == "": opts.reserved_lvs = [] else: opts.reserved_lvs = utils.UnescapeAndSplit(opts.reserved_lvs, sep=",") if opts.master_netmask is not None: try: opts.master_netmask = int(opts.master_netmask) except ValueError: ToStderr("The --master-netmask option expects an int parameter.") return 1 ext_ip_script = opts.use_external_mip_script if opts.disk_state: disk_state = utils.FlatToDict(opts.disk_state) else: disk_state = {} hv_state = dict(opts.hv_state) compression_tools = _GetCompressionTools(opts) enabled_data_collectors = dict( (k, v.lower().startswith("t")) for k, v in opts.enabled_data_collectors.items()) unrecognized_data_collectors = [ k for k in enabled_data_collectors if k not in constants.DATA_COLLECTOR_NAMES] if unrecognized_data_collectors: ToStderr("Data collector names not recognized: %s" % ", ".join(unrecognized_data_collectors)) try: data_collector_interval = dict( (k, int(1e6 * float(v))) for (k, v) in opts.data_collector_interval.items()) except ValueError: ToStderr("Can't transform all values to integers: {}".format( opts.data_collector_interval)) return 1 if any(v <= 0 for v in data_collector_interval): ToStderr("Some interval times where not above zero.") return 1 op = opcodes.OpClusterSetParams( vg_name=vg_name, drbd_helper=drbd_helper, enabled_hypervisors=hvlist, hvparams=hvparams, os_hvp=None, beparams=beparams, nicparams=nicparams, ndparams=ndparams, diskparams=diskparams, ipolicy=ipolicy, candidate_pool_size=opts.candidate_pool_size, max_running_jobs=opts.max_running_jobs, max_tracked_jobs=opts.max_tracked_jobs, maintain_node_health=mnh, modify_etc_hosts=opts.modify_etc_hosts, uid_pool=uid_pool, add_uids=add_uids, remove_uids=remove_uids, default_iallocator=opts.default_iallocator, default_iallocator_params=opts.default_iallocator_params, prealloc_wipe_disks=opts.prealloc_wipe_disks, mac_prefix=opts.mac_prefix, master_netdev=opts.master_netdev, master_netmask=opts.master_netmask, reserved_lvs=opts.reserved_lvs, use_external_mip_script=ext_ip_script, hv_state=hv_state, disk_state=disk_state, enabled_disk_templates=enabled_disk_templates, force=opts.force, file_storage_dir=opts.file_storage_dir, install_image=opts.install_image, instance_communication_network=opts.instance_communication_network, zeroing_image=opts.zeroing_image, shared_file_storage_dir=opts.shared_file_storage_dir, compression_tools=compression_tools, enabled_user_shutdown=opts.enabled_user_shutdown, enabled_data_collectors=enabled_data_collectors, data_collector_interval=data_collector_interval, ) return base.GetResult(None, opts, SubmitOrSend(op, opts)) def QueueOps(opts, args): """Queue operations. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the subcommand @rtype: int @return: the desired exit code """ command = args[0] client = GetClient() if command in ("drain", "undrain"): drain_flag = command == "drain" client.SetQueueDrainFlag(drain_flag) elif command == "info": result = client.QueryConfigValues(["drain_flag"]) if result[0]: val = "set" else: val = "unset" ToStdout("The drain flag is %s" % val) else: raise errors.OpPrereqError("Command '%s' is not valid." % command, errors.ECODE_INVAL) return 0 def _ShowWatcherPause(until): if until is None or until < time.time(): ToStdout("The watcher is not paused.") else: ToStdout("The watcher is paused until %s.", time.ctime(until)) def WatcherOps(opts, args): """Watcher operations. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the subcommand @rtype: int @return: the desired exit code """ command = args[0] client = GetClient() if command == "continue": client.SetWatcherPause(None) ToStdout("The watcher is no longer paused.") elif command == "pause": if len(args) < 2: raise errors.OpPrereqError("Missing pause duration", errors.ECODE_INVAL) result = client.SetWatcherPause(time.time() + ParseTimespec(args[1])) _ShowWatcherPause(result) elif command == "info": result = client.QueryConfigValues(["watcher_pause"]) _ShowWatcherPause(result[0]) else: raise errors.OpPrereqError("Command '%s' is not valid." % command, errors.ECODE_INVAL) return 0 def _OobPower(opts, node_list, power): """Puts the node in the list to desired power state. @param opts: The command line options selected by the user @param node_list: The list of nodes to operate on @param power: True if they should be powered on, False otherwise @return: The success of the operation (none failed) """ if power: command = constants.OOB_POWER_ON else: command = constants.OOB_POWER_OFF op = opcodes.OpOobCommand(node_names=node_list, command=command, ignore_status=True, timeout=opts.oob_timeout, power_delay=opts.power_delay) result = SubmitOpCode(op, opts=opts) errs = 0 for node_result in result: (node_tuple, data_tuple) = node_result (_, node_name) = node_tuple (data_status, _) = data_tuple if data_status != constants.RS_NORMAL: assert data_status != constants.RS_UNAVAIL errs += 1 ToStderr("There was a problem changing power for %s, please investigate", node_name) if errs > 0: return False return True def _InstanceStart(opts, inst_list, start, no_remember=False): """Puts the instances in the list to desired state. @param opts: The command line options selected by the user @param inst_list: The list of instances to operate on @param start: True if they should be started, False for shutdown @param no_remember: If the instance state should be remembered @return: The success of the operation (none failed) """ if start: opcls = opcodes.OpInstanceStartup text_submit, text_success, text_failed = ("startup", "started", "starting") else: opcls = compat.partial(opcodes.OpInstanceShutdown, timeout=opts.shutdown_timeout, no_remember=no_remember) text_submit, text_success, text_failed = ("shutdown", "stopped", "stopping") jex = JobExecutor(opts=opts) for inst in inst_list: ToStdout("Submit %s of instance %s", text_submit, inst) op = opcls(instance_name=inst) jex.QueueJob(inst, op) results = jex.GetResults() bad_cnt = len([1 for (success, _) in results if not success]) if bad_cnt == 0: ToStdout("All instances have been %s successfully", text_success) else: ToStderr("There were errors while %s instances:\n" "%d error(s) out of %d instance(s)", text_failed, bad_cnt, len(results)) return False return True class _RunWhenNodesReachableHelper(object): """Helper class to make shared internal state sharing easier. @ivar success: Indicates if all action_cb calls were successful """ def __init__(self, node_list, action_cb, node2ip, port, feedback_fn, _ping_fn=netutils.TcpPing, _sleep_fn=time.sleep): """Init the object. @param node_list: The list of nodes to be reachable @param action_cb: Callback called when a new host is reachable @type node2ip: dict @param node2ip: Node to ip mapping @param port: The port to use for the TCP ping @param feedback_fn: The function used for feedback @param _ping_fn: Function to check reachabilty (for unittest use only) @param _sleep_fn: Function to sleep (for unittest use only) """ self.down = set(node_list) self.up = set() self.node2ip = node2ip self.success = True self.action_cb = action_cb self.port = port self.feedback_fn = feedback_fn self._ping_fn = _ping_fn self._sleep_fn = _sleep_fn def __call__(self): """When called we run action_cb. @raises utils.RetryAgain: When there are still down nodes """ if not self.action_cb(self.up): self.success = False if self.down: raise utils.RetryAgain() else: return self.success def Wait(self, secs): """Checks if a host is up or waits remaining seconds. @param secs: The secs remaining """ start = time.time() for node in self.down: if self._ping_fn(self.node2ip[node], self.port, timeout=_EPO_PING_TIMEOUT, live_port_needed=True): self.feedback_fn("Node %s became available" % node) self.up.add(node) self.down -= self.up # If we have a node available there is the possibility to run the # action callback successfully, therefore we don't wait and return return self._sleep_fn(max(0.0, start + secs - time.time())) def _RunWhenNodesReachable(node_list, action_cb, interval): """Run action_cb when nodes become reachable. @param node_list: The list of nodes to be reachable @param action_cb: Callback called when a new host is reachable @param interval: The earliest time to retry """ client = GetClient() cluster_info = client.QueryClusterInfo() if cluster_info["primary_ip_version"] == constants.IP4_VERSION: family = netutils.IPAddress.family else: family = netutils.IP6Address.family node2ip = dict((node, netutils.GetHostname(node, family=family).ip) for node in node_list) port = netutils.GetDaemonPort(constants.NODED) helper = _RunWhenNodesReachableHelper(node_list, action_cb, node2ip, port, ToStdout) try: return utils.Retry(helper, interval, _EPO_REACHABLE_TIMEOUT, wait_fn=helper.Wait) except utils.RetryTimeout: ToStderr("Time exceeded while waiting for nodes to become reachable" " again:\n - %s", " - ".join(helper.down)) return False def _MaybeInstanceStartup(opts, inst_map, nodes_online, _instance_start_fn=_InstanceStart): """Start the instances conditional based on node_states. @param opts: The command line options selected by the user @param inst_map: A dict of inst -> nodes mapping @param nodes_online: A list of nodes online @param _instance_start_fn: Callback to start instances (unittest use only) @return: Success of the operation on all instances """ start_inst_list = [] for (inst, nodes) in inst_map.items(): if not (nodes - nodes_online): # All nodes the instance lives on are back online start_inst_list.append(inst) for inst in start_inst_list: del inst_map[inst] if start_inst_list: return _instance_start_fn(opts, start_inst_list, True) return True def _EpoOn(opts, full_node_list, node_list, inst_map): """Does the actual power on. @param opts: The command line options selected by the user @param full_node_list: All nodes to operate on (includes nodes not supporting OOB) @param node_list: The list of nodes to operate on (all need to support OOB) @param inst_map: A dict of inst -> nodes mapping @return: The desired exit status """ if node_list and not _OobPower(opts, node_list, False): ToStderr("Not all nodes seem to get back up, investigate and start" " manually if needed") # Wait for the nodes to be back up action_cb = compat.partial(_MaybeInstanceStartup, opts, dict(inst_map)) ToStdout("Waiting until all nodes are available again") if not _RunWhenNodesReachable(full_node_list, action_cb, _EPO_PING_INTERVAL): ToStderr("Please investigate and start stopped instances manually") return constants.EXIT_FAILURE return constants.EXIT_SUCCESS def _EpoOff(opts, node_list, inst_map): """Does the actual power off. @param opts: The command line options selected by the user @param node_list: The list of nodes to operate on (all need to support OOB) @param inst_map: A dict of inst -> nodes mapping @return: The desired exit status """ if not _InstanceStart(opts, list(inst_map), False, no_remember=True): ToStderr("Please investigate and stop instances manually before continuing") return constants.EXIT_FAILURE if not node_list: return constants.EXIT_SUCCESS if _OobPower(opts, node_list, False): return constants.EXIT_SUCCESS else: return constants.EXIT_FAILURE def Epo(opts, args, qcl=None, _on_fn=_EpoOn, _off_fn=_EpoOff, _confirm_fn=ConfirmOperation, _stdout_fn=ToStdout, _stderr_fn=ToStderr): """EPO operations. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the subcommand @rtype: int @return: the desired exit code """ if opts.groups and opts.show_all: _stderr_fn("Only one of --groups or --all are allowed") return constants.EXIT_FAILURE elif args and opts.show_all: _stderr_fn("Arguments in combination with --all are not allowed") return constants.EXIT_FAILURE if qcl is None: # Query client qcl = GetClient() if opts.groups: node_query_list = \ itertools.chain(*qcl.QueryGroups(args, ["node_list"], False)) else: node_query_list = args result = qcl.QueryNodes(node_query_list, ["name", "master", "pinst_list", "sinst_list", "powered", "offline"], False) all_nodes = [r[0] for r in result] node_list = [] inst_map = {} for (node, master, pinsts, sinsts, powered, offline) in result: if not offline: for inst in (pinsts + sinsts): if inst in inst_map: if not master: inst_map[inst].add(node) elif master: inst_map[inst] = set() else: inst_map[inst] = set([node]) if master and opts.on: # We ignore the master for turning on the machines, in fact we are # already operating on the master at this point :) continue elif master and not opts.show_all: _stderr_fn("%s is the master node, please do a master-failover to another" " node not affected by the EPO or use --all if you intend to" " shutdown the whole cluster", node) return constants.EXIT_FAILURE elif powered is None: _stdout_fn("Node %s does not support out-of-band handling, it can not be" " handled in a fully automated manner", node) elif powered == opts.on: _stdout_fn("Node %s is already in desired power state, skipping", node) elif not offline or (offline and powered): node_list.append(node) if not (opts.force or _confirm_fn(all_nodes, "nodes", "epo")): return constants.EXIT_FAILURE if opts.on: return _on_fn(opts, all_nodes, node_list, inst_map) else: return _off_fn(opts, node_list, inst_map) def _GetCreateCommand(info): buf = StringIO() buf.write("gnt-cluster init") PrintIPolicyCommand(buf, info["ipolicy"], False) buf.write(" ") buf.write(info["name"]) return buf.getvalue() def ShowCreateCommand(opts, args): """Shows the command that can be used to re-create the cluster. Currently it works only for ipolicy specs. """ cl = GetClient() result = cl.QueryClusterInfo() ToStdout(_GetCreateCommand(result)) def _RunCommandAndReport(cmd): """Run a command and report its output, iff it failed. @param cmd: the command to execute @type cmd: list @rtype: bool @return: False, if the execution failed. """ result = utils.RunCmd(cmd) if result.failed: ToStderr("Command %s failed: %s; Output %s" % (cmd, result.fail_reason, result.output)) return False return True def _VerifyCommand(cmd): """Verify that a given command succeeds on all online nodes. As this function is intended to run during upgrades, it is implemented in such a way that it still works, if all Ganeti daemons are down. @param cmd: a list of unquoted shell arguments @type cmd: list @rtype: list @return: the list of node names that are online where the command failed. """ command = utils.text.ShellQuoteArgs([str(val) for val in cmd]) return _VerifyCommandRaw(command) def _VerifyCommandRaw(command): """Verify that a given command succeeds on all online nodes. As this function is intended to run during upgrades, it is implemented in such a way that it still works, if all Ganeti daemons are down. @param cmd: a bare string to pass to SSH. The caller must do their own shell/ssh escaping. @type cmd: string @rtype: list @return: the list of node names that are online where the command failed. """ nodes = ssconf.SimpleStore().GetOnlineNodeList() master_node = ssconf.SimpleStore().GetMasterNode() cluster_name = ssconf.SimpleStore().GetClusterName() # If master node is in 'nodes', make sure master node is at list end if master_node in nodes: nodes.remove(master_node) nodes.append(master_node) failed = [] srun = ssh.SshRunner(cluster_name=cluster_name) for name in nodes: result = srun.Run(name, constants.SSH_LOGIN_USER, command) if result.exit_code != 0: failed.append(name) return failed def _VerifyVersionInstalled(versionstring): """Verify that the given version of ganeti is installed on all online nodes. Do nothing, if this is the case, otherwise print an appropriate message to stderr. @param versionstring: the version to check for @type versionstring: string @rtype: bool @return: True, if the version is installed on all online nodes """ badnodes = _VerifyCommand(["test", "-d", os.path.join(pathutils.PKGLIBDIR, versionstring)]) if badnodes: ToStderr("Ganeti version %s not installed on nodes %s" % (versionstring, ", ".join(badnodes))) return False return True def _GetRunning(): """Determine the list of running jobs. @rtype: list @return: the number of jobs still running """ cl = GetClient() qfilter = qlang.MakeSimpleFilter("status", frozenset([constants.JOB_STATUS_RUNNING])) return len(cl.Query(constants.QR_JOB, [], qfilter).data) def _SetGanetiVersionAndEnsure(versionstring): """Symlink the active version of ganeti to the given versionstring, and run the ensure-dirs script. @type versionstring: string @rtype: list @return: the list of nodes where the version change failed """ # Update symlinks to point at the new version. if constants.HAS_GNU_LN: link_lib_cmd = [ "ln", "-s", "-f", "-T", os.path.join(pathutils.PKGLIBDIR, versionstring), os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")] link_share_cmd = [ "ln", "-s", "-f", "-T", os.path.join(pathutils.SHAREDIR, versionstring), os.path.join(pathutils.SYSCONFDIR, "ganeti/share")] cmds = [link_lib_cmd, link_share_cmd] else: rm_lib_cmd = [ "rm", "-f", os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")] link_lib_cmd = [ "ln", "-s", "-f", os.path.join(pathutils.PKGLIBDIR, versionstring), os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")] rm_share_cmd = [ "rm", "-f", os.path.join(pathutils.SYSCONFDIR, "ganeti/share")] ln_share_cmd = [ "ln", "-s", "-f", os.path.join(pathutils.SHAREDIR, versionstring), os.path.join(pathutils.SYSCONFDIR, "ganeti/share")] cmds = [rm_lib_cmd, link_lib_cmd, rm_share_cmd, ln_share_cmd] # Run the ensure-dirs script to verify the new version is OK. cmds.append([pathutils.ENSURE_DIRS]) # Submit all commands to ssh, exiting on the first failure. # The command string is a single argument that's given to ssh to submit to # the remote shell, so it only needs enough escaping to satisfy the remote # shell, rather than the 2 levels of escaping usually required when using # ssh from the commandline. quoted_cmds = [utils.text.ShellQuoteArgs(cmd) for cmd in cmds] cmd = " && ".join(quoted_cmds) failed = _VerifyCommandRaw(cmd) return list(set(failed)) def _ExecuteCommands(fns): """Execute a list of functions, in reverse order. @type fns: list of functions. @param fns: the functions to be executed. """ for fn in reversed(fns): fn() def _GetConfigVersion(): """Determine the version the configuration file currently has. @rtype: tuple or None @return: (major, minor, revision) if the version can be determined, None otherwise """ config_data = serializer.LoadJson(utils.ReadFile(pathutils.CLUSTER_CONF_FILE)) try: config_version = config_data["version"] except KeyError: return None return utils.SplitVersion(config_version) def _ReadIntentToUpgrade(): """Read the file documenting the intent to upgrade the cluster. @rtype: (string, string) or (None, None) @return: (old version, version to upgrade to), if the file exists, and (None, None) otherwise. """ if not os.path.isfile(pathutils.INTENT_TO_UPGRADE): return (None, None) contentstring = utils.ReadFile(pathutils.INTENT_TO_UPGRADE) contents = utils.UnescapeAndSplit(contentstring) if len(contents) != 3: # file syntactically mal-formed return (None, None) return (contents[0], contents[1]) def _WriteIntentToUpgrade(version): """Write file documenting the intent to upgrade the cluster. @type version: string @param version: the version we intent to upgrade to """ utils.WriteFile(pathutils.INTENT_TO_UPGRADE, data=utils.EscapeAndJoin([constants.RELEASE_VERSION, version, "%d" % os.getpid()])) def _UpgradeBeforeConfigurationChange(versionstring): """ Carry out all the tasks necessary for an upgrade that happen before the configuration file, or Ganeti version, changes. @type versionstring: string @param versionstring: the version to upgrade to @rtype: (bool, list) @return: tuple of a bool indicating success and a list of rollback tasks """ rollback = [] ToStdoutAndLoginfo("Verifying %s present on all nodes", versionstring) if not _VerifyVersionInstalled(versionstring): return (False, rollback) _WriteIntentToUpgrade(versionstring) rollback.append( lambda: utils.RunCmd(["rm", "-f", pathutils.INTENT_TO_UPGRADE])) ToStdoutAndLoginfo("Draining queue") client = GetClient() client.SetQueueDrainFlag(True) rollback.append(lambda: GetClient().SetQueueDrainFlag(False)) if utils.SimpleRetry(0, _GetRunning, constants.UPGRADE_QUEUE_POLL_INTERVAL, constants.UPGRADE_QUEUE_DRAIN_TIMEOUT): ToStderr("Failed to completely empty the queue.") return (False, rollback) ToStdoutAndLoginfo("Pausing the watcher for one hour.") rollback.append(lambda: GetClient().SetWatcherPause(None)) GetClient().SetWatcherPause(time.time() + 60 * 60) ToStdoutAndLoginfo("Stopping daemons on master node.") if not _RunCommandAndReport([pathutils.DAEMON_UTIL, "stop-all"]): return (False, rollback) ToStdoutAndLoginfo("Stopping daemons everywhere.") rollback.append(lambda: _VerifyCommand([pathutils.DAEMON_UTIL, "start-all"])) badnodes = _VerifyCommand([pathutils.DAEMON_UTIL, "stop-all"]) if badnodes: ToStderr("Failed to stop daemons on %s." % (", ".join(badnodes),)) return (False, rollback) backuptar = os.path.join(pathutils.BACKUP_DIR, "ganeti%d.tar" % time.time()) ToStdoutAndLoginfo("Backing up configuration as %s", backuptar) if not _RunCommandAndReport(["mkdir", "-p", pathutils.BACKUP_DIR]): return (False, rollback) # Create the archive in a safe manner, as it contains sensitive # information. (_, tmp_name) = tempfile.mkstemp(prefix=backuptar, dir=pathutils.BACKUP_DIR) tar_cmd = ["tar", "-cf", tmp_name, "--exclude=queue/archive"] # Some distributions (e.g. Debian) may set EXPORT_DIR to a subdirectory of # DATA_DIR. Avoid backing up the EXPORT_DIR, as it might contain significant # amounts of data. if utils.IsBelowDir(pathutils.DATA_DIR, pathutils.EXPORT_DIR): tar_cmd.append("--exclude=%s" % os.path.relpath(pathutils.EXPORT_DIR, pathutils.DATA_DIR)) tar_cmd.append(pathutils.DATA_DIR) if not _RunCommandAndReport(tar_cmd): return (False, rollback) os.rename(tmp_name, backuptar) return (True, rollback) def _VersionSpecificDowngrade(): """ Perform any additional downrade tasks that are version specific and need to be done just after the configuration downgrade. This function needs to be idempotent, so that it can be redone if the downgrade procedure gets interrupted after changing the configuration. Note that this function has to be reset with every version bump. @return: True upon success """ ToStdoutAndLoginfo("Performing version-specific downgrade tasks.") return True def _SwitchVersionAndConfig(versionstring, downgrade): """ Switch to the new Ganeti version and change the configuration, in correct order. @type versionstring: string @param versionstring: the version to change to @type downgrade: bool @param downgrade: True, if the configuration should be downgraded @rtype: (bool, list) @return: tupe of a bool indicating success, and a list of additional rollback tasks """ rollback = [] if downgrade: ToStdoutAndLoginfo("Downgrading configuration") if not _RunCommandAndReport([pathutils.CFGUPGRADE, "--downgrade", "-f"]): return (False, rollback) # Note: version specific downgrades need to be done before switching # binaries, so that we still have the knowledgeable binary if the downgrade # process gets interrupted at this point. if not _VersionSpecificDowngrade(): return (False, rollback) # Configuration change is the point of no return. From then onwards, it is # safer to push through the up/dowgrade than to try to roll it back. ToStdoutAndLoginfo("Switching to version %s on all nodes", versionstring) rollback.append(lambda: _SetGanetiVersionAndEnsure(constants.DIR_VERSION)) badnodes = _SetGanetiVersionAndEnsure(versionstring) if badnodes: ToStderr("Failed to switch to Ganeti version %s on nodes %s" % (versionstring, ", ".join(badnodes))) if not downgrade: return (False, rollback) # Now that we have changed to the new version of Ganeti we should # not communicate over luxi any more, as luxi might have changed in # incompatible ways. Therefore, manually call the corresponding ganeti # commands using their canonical (version independent) path. if not downgrade: ToStdoutAndLoginfo("Upgrading configuration") if not _RunCommandAndReport([pathutils.CFGUPGRADE, "-f"]): return (False, rollback) return (True, rollback) def _UpgradeAfterConfigurationChange(oldversion): """ Carry out the upgrade actions necessary after switching to the new Ganeti version and updating the configuration. As this part is run at a time where the new version of Ganeti is already running, no communication should happen via luxi, as this is not a stable interface. Also, as the configuration change is the point of no return, all actions are pushed through, even if some of them fail. @param oldversion: the version the upgrade started from @type oldversion: string @rtype: int @return: the intended return value """ returnvalue = 0 ToStdoutAndLoginfo("Starting daemons everywhere.") badnodes = _VerifyCommand([pathutils.DAEMON_UTIL, "start-all"]) if badnodes: ToStderr("Warning: failed to start daemons on %s." % (", ".join(badnodes),)) returnvalue = 1 ToStdoutAndLoginfo("Redistributing the configuration.") if not _RunCommandAndReport(["gnt-cluster", "redist-conf", "--yes-do-it"]): returnvalue = 1 ToStdoutAndLoginfo("Restarting daemons everywhere.") badnodes = _VerifyCommand([pathutils.DAEMON_UTIL, "stop-all"]) badnodes.extend(_VerifyCommand([pathutils.DAEMON_UTIL, "start-all"])) if badnodes: ToStderr("Warning: failed to start daemons on %s." % (", ".join(list(set(badnodes))),)) returnvalue = 1 ToStdoutAndLoginfo("Undraining the queue.") if not _RunCommandAndReport(["gnt-cluster", "queue", "undrain"]): returnvalue = 1 _RunCommandAndReport(["rm", "-f", pathutils.INTENT_TO_UPGRADE]) ToStdoutAndLoginfo("Running post-upgrade hooks") if not _RunCommandAndReport([pathutils.POST_UPGRADE, oldversion]): returnvalue = 1 ToStdoutAndLoginfo("Unpausing the watcher.") if not _RunCommandAndReport(["gnt-cluster", "watcher", "continue"]): returnvalue = 1 ToStdoutAndLoginfo("Verifying cluster.") if not _RunCommandAndReport(["gnt-cluster", "verify"]): returnvalue = 1 return returnvalue def UpgradeGanetiCommand(opts, args): """Upgrade a cluster to a new ganeti version. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ if ((not opts.resume and opts.to is None) or (opts.resume and opts.to is not None)): ToStderr("Precisely one of the options --to and --resume" " has to be given") return 1 # If we're not told to resume, verify there is no upgrade # in progress. if not opts.resume: oldversion, versionstring = _ReadIntentToUpgrade() if versionstring is not None: # An upgrade is going on; verify whether the target matches if versionstring == opts.to: ToStderr("An upgrade is already in progress. Target version matches," " resuming.") opts.resume = True opts.to = None else: ToStderr("An upgrade from %s to %s is in progress; use --resume to" " finish it first" % (oldversion, versionstring)) return 1 utils.SetupLogging(pathutils.LOG_COMMANDS, 'gnt-cluster upgrade', debug=1) oldversion = constants.RELEASE_VERSION if opts.resume: ssconf.CheckMaster(False) oldversion, versionstring = _ReadIntentToUpgrade() if versionstring is None: return 0 version = utils.version.ParseVersion(versionstring) if version is None: return 1 configversion = _GetConfigVersion() if configversion is None: return 1 # If the upgrade we resume was an upgrade between compatible # versions (like 2.10.0 to 2.10.1), the correct configversion # does not guarantee that the config has been updated. # However, in the case of a compatible update with the configuration # not touched, we are running a different dirversion with the same # config version. config_already_modified = \ (utils.IsCorrectConfigVersion(version, configversion) and not (versionstring != constants.DIR_VERSION and configversion == (constants.CONFIG_MAJOR, constants.CONFIG_MINOR, constants.CONFIG_REVISION))) if not config_already_modified: # We have to start from the beginning; however, some daemons might have # already been stopped, so the only way to get into a well-defined state # is by starting all daemons again. _VerifyCommand([pathutils.DAEMON_UTIL, "start-all"]) else: versionstring = opts.to config_already_modified = False version = utils.version.ParseVersion(versionstring) if version is None: ToStderr("Could not parse version string %s" % versionstring) return 1 msg = utils.version.UpgradeRange(version) if msg is not None: ToStderr("Cannot upgrade to %s: %s" % (versionstring, msg)) return 1 if not config_already_modified: success, rollback = _UpgradeBeforeConfigurationChange(versionstring) if not success: _ExecuteCommands(rollback) return 1 else: rollback = [] downgrade = utils.version.ShouldCfgdowngrade(version) success, additionalrollback = \ _SwitchVersionAndConfig(versionstring, downgrade) if not success: rollback.extend(additionalrollback) _ExecuteCommands(rollback) return 1 return _UpgradeAfterConfigurationChange(oldversion) commands = { "init": ( InitCluster, [ArgHost(min=1, max=1)], [BACKEND_OPT, CP_SIZE_OPT, ENABLED_HV_OPT, GLOBAL_FILEDIR_OPT, HVLIST_OPT, MAC_PREFIX_OPT, MASTER_NETDEV_OPT, MASTER_NETMASK_OPT, NIC_PARAMS_OPT, NOMODIFY_ETCHOSTS_OPT, NOMODIFY_SSH_SETUP_OPT, SECONDARY_IP_OPT, VG_NAME_OPT, MAINTAIN_NODE_HEALTH_OPT, UIDPOOL_OPT, DRBD_HELPER_OPT, DEFAULT_IALLOCATOR_OPT, DEFAULT_IALLOCATOR_PARAMS_OPT, PRIMARY_IP_VERSION_OPT, PREALLOC_WIPE_DISKS_OPT, NODE_PARAMS_OPT, GLOBAL_SHARED_FILEDIR_OPT, USE_EXTERNAL_MIP_SCRIPT, DISK_PARAMS_OPT, HV_STATE_OPT, DISK_STATE_OPT, ENABLED_DISK_TEMPLATES_OPT, IPOLICY_STD_SPECS_OPT, GLOBAL_GLUSTER_FILEDIR_OPT, INSTALL_IMAGE_OPT, ZEROING_IMAGE_OPT, COMPRESSION_TOOLS_OPT, ENABLED_USER_SHUTDOWN_OPT, SSH_KEY_BITS_OPT, SSH_KEY_TYPE_OPT, ] + INSTANCE_POLICY_OPTS + SPLIT_ISPECS_OPTS, "[...] ", "Initialises a new cluster configuration"), "destroy": ( DestroyCluster, ARGS_NONE, [YES_DOIT_OPT], "", "Destroy cluster"), "rename": ( RenameCluster, [ArgHost(min=1, max=1)], [FORCE_OPT, DRY_RUN_OPT], "", "Renames the cluster"), "redist-conf": ( RedistributeConfig, ARGS_NONE, SUBMIT_OPTS + [DRY_RUN_OPT, PRIORITY_OPT, FORCE_DISTRIBUTION], "", "Forces a push of the configuration file and ssconf files" " to the nodes in the cluster"), "verify": ( VerifyCluster, ARGS_NONE, [VERBOSE_OPT, DEBUG_SIMERR_OPT, ERROR_CODES_OPT, NONPLUS1_OPT, NOHVPARAMASSESS_OPT, DRY_RUN_OPT, PRIORITY_OPT, NODEGROUP_OPT, IGNORE_ERRORS_OPT, VERIFY_CLUTTER_OPT], "", "Does a check on the cluster configuration"), "verify-disks": ( VerifyDisks, ARGS_NONE, [PRIORITY_OPT, NODEGROUP_OPT, STRICT_OPT], "", "Does a check on the cluster disk status"), "repair-disk-sizes": ( RepairDiskSizes, ARGS_MANY_INSTANCES, [DRY_RUN_OPT, PRIORITY_OPT], "[...]", "Updates mismatches in recorded disk sizes"), "master-failover": ( MasterFailover, ARGS_NONE, [NOVOTING_OPT, FORCE_FAILOVER, IGNORE_OFFLINE_NODES_FAILOVER], "", "Makes the current node the master"), "master-ping": ( MasterPing, ARGS_NONE, [], "", "Checks if the master is alive"), "version": ( ShowClusterVersion, ARGS_NONE, [], "", "Shows the cluster version"), "getmaster": ( ShowClusterMaster, ARGS_NONE, [], "", "Shows the cluster master"), "copyfile": ( ClusterCopyFile, [ArgFile(min=1, max=1)], [NODE_LIST_OPT, USE_REPL_NET_OPT, NODEGROUP_OPT], "[-n ...] ", "Copies a file to all (or only some) nodes"), "command": ( RunClusterCommand, [ArgCommand(min=1)], [NODE_LIST_OPT, NODEGROUP_OPT, SHOW_MACHINE_OPT, FAILURE_ONLY_OPT], "[-n ...] ", "Runs a command on all (or only some) nodes"), "info": ( ShowClusterConfig, ARGS_NONE, [ROMAN_OPT], "[--roman]", "Show cluster configuration"), "list-tags": ( ListTags, ARGS_NONE, [], "", "List the tags of the cluster"), "add-tags": ( AddTags, [ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, "...", "Add tags to the cluster"), "remove-tags": ( RemoveTags, [ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, "...", "Remove tags from the cluster"), "search-tags": ( SearchTags, [ArgUnknown(min=1, max=1)], [PRIORITY_OPT], "", "Searches the tags on all objects on" " the cluster for a given pattern (regex)"), "queue": ( QueueOps, [ArgChoice(min=1, max=1, choices=["drain", "undrain", "info"])], [], "drain|undrain|info", "Change queue properties"), "watcher": ( WatcherOps, [ArgChoice(min=1, max=1, choices=["pause", "continue", "info"]), ArgSuggest(min=0, max=1, choices=["30m", "1h", "4h"])], [], "{pause |continue|info}", "Change watcher properties"), "modify": ( SetClusterParams, ARGS_NONE, [FORCE_OPT, BACKEND_OPT, CP_SIZE_OPT, RQL_OPT, MAX_TRACK_OPT, INSTALL_IMAGE_OPT, INSTANCE_COMMUNICATION_NETWORK_OPT, ENABLED_HV_OPT, HVLIST_OPT, MAC_PREFIX_OPT, MASTER_NETDEV_OPT, MASTER_NETMASK_OPT, NIC_PARAMS_OPT, VG_NAME_OPT, MAINTAIN_NODE_HEALTH_OPT, UIDPOOL_OPT, ADD_UIDS_OPT, REMOVE_UIDS_OPT, DRBD_HELPER_OPT, DEFAULT_IALLOCATOR_OPT, DEFAULT_IALLOCATOR_PARAMS_OPT, RESERVED_LVS_OPT, DRY_RUN_OPT, PRIORITY_OPT, PREALLOC_WIPE_DISKS_OPT, NODE_PARAMS_OPT, USE_EXTERNAL_MIP_SCRIPT, DISK_PARAMS_OPT, HV_STATE_OPT, DISK_STATE_OPT] + SUBMIT_OPTS + [ENABLED_DISK_TEMPLATES_OPT, IPOLICY_STD_SPECS_OPT, MODIFY_ETCHOSTS_OPT, ENABLED_USER_SHUTDOWN_OPT] + INSTANCE_POLICY_OPTS + [GLOBAL_FILEDIR_OPT, GLOBAL_SHARED_FILEDIR_OPT, ZEROING_IMAGE_OPT, COMPRESSION_TOOLS_OPT] + [ENABLED_DATA_COLLECTORS_OPT, DATA_COLLECTOR_INTERVAL_OPT], "[...]", "Alters the parameters of the cluster"), "renew-crypto": ( RenewCrypto, ARGS_NONE, [NEW_CLUSTER_CERT_OPT, NEW_RAPI_CERT_OPT, RAPI_CERT_OPT, NEW_CONFD_HMAC_KEY_OPT, FORCE_OPT, NEW_CLUSTER_DOMAIN_SECRET_OPT, CLUSTER_DOMAIN_SECRET_OPT, NEW_SPICE_CERT_OPT, SPICE_CERT_OPT, SPICE_CACERT_OPT, NEW_NODE_CERT_OPT, NEW_SSH_KEY_OPT, NOSSH_KEYCHECK_OPT, VERBOSE_OPT, SSH_KEY_BITS_OPT, SSH_KEY_TYPE_OPT], "[...]", "Renews cluster certificates, keys and secrets"), "epo": ( Epo, [ArgUnknown()], [FORCE_OPT, ON_OPT, GROUPS_OPT, ALL_OPT, OOB_TIMEOUT_OPT, SHUTDOWN_TIMEOUT_OPT, POWER_DELAY_OPT], "[...] [args]", "Performs an emergency power-off on given args"), "activate-master-ip": ( ActivateMasterIp, ARGS_NONE, [], "", "Activates the master IP"), "deactivate-master-ip": ( DeactivateMasterIp, ARGS_NONE, [CONFIRM_OPT], "", "Deactivates the master IP"), "show-ispecs-cmd": ( ShowCreateCommand, ARGS_NONE, [], "", "Show the command line to re-create the cluster"), "upgrade": ( UpgradeGanetiCommand, ARGS_NONE, [TO_OPT, RESUME_OPT], "", "Upgrade (or downgrade) to a new Ganeti version"), } #: dictionary with aliases for commands aliases = { "masterfailover": "master-failover", "show": "info", } def Main(): return GenericMain(commands, override={"tag_type": constants.TAG_CLUSTER}, aliases=aliases) ganeti-3.1.0~rc2/lib/client/gnt_debug.py000064400000000000000000000654371476477700300201570ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Debugging commands""" # pylint: disable=W0401,W0614,C0103 # W0401: Wildcard import ganeti.cli # W0614: Unused import %s from wildcard import (since we need cli) # C0103: Invalid name gnt-backup import logging import socket import time import json from ganeti.cli import * from ganeti import cli from ganeti import constants from ganeti import opcodes from ganeti import utils from ganeti import errors from ganeti import compat from ganeti import ht from ganeti import metad from ganeti import wconfd #: Default fields for L{ListLocks} _LIST_LOCKS_DEF_FIELDS = [ "name", "mode", "owner", "pending", ] def Delay(opts, args): """Sleeps for a while @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the duration the sleep @rtype: int @return: the desired exit code """ delay = float(args[0]) op = opcodes.OpTestDelay(duration=delay, on_master=opts.on_master, on_nodes=opts.on_nodes, repeat=opts.repeat, interruptible=opts.interruptible, no_locks=opts.no_locks) SubmitOrSend(op, opts) return 0 def GenericOpCodes(opts, args): """Send any opcode to the master. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the path of the file with the opcode definition @rtype: int @return: the desired exit code """ cl = cli.GetClient() jex = cli.JobExecutor(cl=cl, verbose=opts.verbose, opts=opts) job_cnt = 0 op_cnt = 0 if opts.timing_stats: ToStdout("Loading...") for job_idx in range(opts.rep_job): for fname in args: op_data = json.loads(utils.ReadFile(fname)) op_list = [opcodes.OpCode.LoadOpCode(val) for val in op_data] op_list = op_list * opts.rep_op jex.QueueJob("file %s/%d" % (fname, job_idx), *op_list) op_cnt += len(op_list) job_cnt += 1 if opts.timing_stats: t1 = time.time() ToStdout("Submitting...") jex.SubmitPending(each=opts.each) if opts.timing_stats: t2 = time.time() ToStdout("Executing...") jex.GetResults() if opts.timing_stats: t3 = time.time() ToStdout("C:op %4d" % op_cnt) ToStdout("C:job %4d" % job_cnt) ToStdout("T:submit %4.4f" % (t2 - t1)) ToStdout("T:exec %4.4f" % (t3 - t2)) ToStdout("T:total %4.4f" % (t3 - t1)) return 0 def TestAllocator(opts, args): """Runs the test allocator opcode. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the iallocator name @rtype: int @return: the desired exit code """ try: disks = [{ constants.IDISK_SIZE: utils.ParseUnit(val), constants.IDISK_MODE: constants.DISK_RDWR, } for val in opts.disks.split(",")] except errors.UnitParseError as err: ToStderr("Invalid disks parameter '%s': %s", opts.disks, err) return 1 nics = [val.split("/") for val in opts.nics.split(",")] for row in nics: while len(row) < 3: row.append(None) for i in range(3): if row[i] == "": row[i] = None nic_dict = [{ constants.INIC_MAC: v[0], constants.INIC_IP: v[1], # The iallocator interface defines a "bridge" item "bridge": v[2], } for v in nics] if opts.tags is None: opts.tags = [] else: opts.tags = opts.tags.split(",") if opts.target_groups is None: target_groups = [] else: target_groups = opts.target_groups op = opcodes.OpTestAllocator(mode=opts.mode, name=args[0], instances=args, memory=opts.memory, disks=disks, disk_template=opts.disk_template, nics=nic_dict, os=opts.os, vcpus=opts.vcpus, tags=opts.tags, direction=opts.direction, iallocator=opts.iallocator, evac_mode=opts.evac_mode, target_groups=target_groups, spindle_use=opts.spindle_use, count=opts.count) result = SubmitOpCode(op, opts=opts) ToStdout("%s" % result) return 0 def _TestJobDependency(opts): """Tests job dependencies. """ ToStdout("Testing job dependencies") try: cl = cli.GetClient() SubmitOpCode(opcodes.OpTestDelay(duration=0, depends=[(-1, None)]), cl=cl) except errors.GenericError as err: if opts.debug: ToStdout("Ignoring error for 'wrong dependencies' test: %s", err) else: raise errors.OpExecError("Submitting plain opcode with relative job ID" " did not fail as expected") # TODO: Test dependencies on errors jobs = [ [opcodes.OpTestDelay(duration=1)], [opcodes.OpTestDelay(duration=1, depends=[(-1, [])])], [opcodes.OpTestDelay(duration=1, depends=[(-2, [constants.JOB_STATUS_SUCCESS])])], [opcodes.OpTestDelay(duration=1, depends=[])], [opcodes.OpTestDelay(duration=1, depends=[(-2, [constants.JOB_STATUS_SUCCESS])])], ] # Function for checking result check_fn = ht.TListOf(ht.TAnd(ht.TIsLength(2), ht.TItems([ht.TBool, ht.TOr(ht.TNonEmptyString, ht.TJobId)]))) cl = cli.GetClient() result = cl.SubmitManyJobs(jobs) if not check_fn(result): raise errors.OpExecError("Job submission doesn't match %s: %s" % (check_fn, result)) # Wait for jobs to finish jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result: jex.AddJobId(None, status, job_id) job_results = jex.GetResults() if not compat.all(row[0] for row in job_results): raise errors.OpExecError("At least one of the submitted jobs failed: %s" % job_results) # Get details about jobs data = cl.QueryJobs([job_id for (_, job_id) in result], ["id", "opexec", "ops"]) data_job_id = [job_id for (job_id, _, _) in data] data_opexec = [opexec for (_, opexec, _) in data] data_op = [[opcodes.OpCode.LoadOpCode(op) for op in ops] for (_, _, ops) in data] assert compat.all(not op.depends or len(op.depends) == 1 for ops in data_op for op in ops) # Check resolved job IDs in dependencies for (job_idx, res_jobdep) in [(1, data_job_id[0]), (2, data_job_id[0]), (4, data_job_id[2])]: if data_op[job_idx][0].depends[0][0] != res_jobdep: raise errors.OpExecError("Job %s's opcode doesn't depend on correct job" " ID (%s)" % (job_idx, res_jobdep)) # Check execution order if not (data_opexec[0] <= data_opexec[1] and data_opexec[0] <= data_opexec[2] and data_opexec[2] <= data_opexec[4]): raise errors.OpExecError("Jobs did not run in correct order: %s" % data) assert len(jobs) == 5 and compat.all(len(ops) == 1 for ops in jobs) ToStdout("Job dependency tests were successful") def _TestJobSubmission(opts): """Tests submitting jobs. """ ToStdout("Testing job submission") testdata = [ (0, 0, constants.OP_PRIO_LOWEST), (0, 0, constants.OP_PRIO_HIGHEST), ] for priority in (constants.OP_PRIO_SUBMIT_VALID | frozenset([constants.OP_PRIO_LOWEST, constants.OP_PRIO_HIGHEST])): for offset in [-1, +1]: testdata.extend([ (0, 0, priority + offset), (3, 0, priority + offset), (0, 3, priority + offset), (4, 2, priority + offset), ]) for before, after, failpriority in testdata: ops = [] ops.extend([opcodes.OpTestDelay(duration=0) for _ in range(before)]) ops.append(opcodes.OpTestDelay(duration=0, priority=failpriority)) ops.extend([opcodes.OpTestDelay(duration=0) for _ in range(after)]) try: cl = cli.GetClient() cl.SubmitJob(ops) except errors.GenericError as err: if opts.debug: ToStdout("Ignoring error for 'wrong priority' test: %s", err) else: raise errors.OpExecError("Submitting opcode with priority %s did not" " fail when it should (allowed are %s)" % (failpriority, constants.OP_PRIO_SUBMIT_VALID)) jobs = [ [opcodes.OpTestDelay(duration=0), opcodes.OpTestDelay(duration=0, dry_run=False), opcodes.OpTestDelay(duration=0, dry_run=True)], ops, ] try: cl = cli.GetClient() cl.SubmitManyJobs(jobs) except errors.GenericError as err: if opts.debug: ToStdout("Ignoring error for 'wrong priority' test: %s", err) else: raise errors.OpExecError("Submitting manyjobs with an incorrect one" " did not fail when it should.") ToStdout("Job submission tests were successful") class _JobQueueTestReporter(cli.StdioJobPollReportCb): def __init__(self): """Initializes this class. """ cli.StdioJobPollReportCb.__init__(self) self._expected_msgcount = 0 self._all_testmsgs = [] self._testmsgs = None self._job_id = None def GetTestMessages(self): """Returns all test log messages received so far. """ return self._all_testmsgs def GetJobId(self): """Returns the job ID. """ return self._job_id def ReportLogMessage(self, job_id, serial, timestamp, log_type, log_msg): """Handles a log message. """ if self._job_id is None: self._job_id = job_id elif self._job_id != job_id: raise errors.ProgrammerError("The same reporter instance was used for" " more than one job") if log_type == constants.ELOG_JQUEUE_TEST: (sockname, test, arg) = log_msg return self._ProcessTestMessage(job_id, sockname, test, arg) elif (log_type == constants.ELOG_MESSAGE and log_msg.startswith(constants.JQT_MSGPREFIX)): if self._testmsgs is None: raise errors.OpExecError("Received test message without a preceding" " start message") testmsg = log_msg[len(constants.JQT_MSGPREFIX):] self._testmsgs.append(testmsg) self._all_testmsgs.append(testmsg) return return cli.StdioJobPollReportCb.ReportLogMessage(self, job_id, serial, timestamp, log_type, log_msg) def _ProcessTestMessage(self, job_id, sockname, test, arg): """Handles a job queue test message. """ if test not in constants.JQT_ALL: raise errors.OpExecError("Received invalid test message %s" % test) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: sock.settimeout(30.0) logging.debug("Connecting to %s", sockname) sock.connect(sockname) logging.debug("Checking status") jobdetails = cli.GetClient().QueryJobs([job_id], ["status"])[0] if not jobdetails: raise errors.OpExecError("Can't find job %s" % job_id) status = jobdetails[0] logging.debug("Status of job %s is %s", job_id, status) if test == constants.JQT_EXPANDNAMES: if status != constants.JOB_STATUS_WAITING: raise errors.OpExecError("Job status while expanding names is '%s'," " not '%s' as expected" % (status, constants.JOB_STATUS_WAITING)) elif test in (constants.JQT_EXEC, constants.JQT_LOGMSG): if status != constants.JOB_STATUS_RUNNING: raise errors.OpExecError("Job status while executing opcode is '%s'," " not '%s' as expected" % (status, constants.JOB_STATUS_RUNNING)) if test == constants.JQT_STARTMSG: logging.debug("Expecting %s test messages", arg) self._testmsgs = [] elif test == constants.JQT_LOGMSG: if len(self._testmsgs) != arg: raise errors.OpExecError("Received %s test messages when %s are" " expected" % (len(self._testmsgs), arg)) finally: logging.debug("Closing socket") sock.close() def TestJobqueue(opts, _): """Runs a few tests on the job queue. """ _TestJobSubmission(opts) _TestJobDependency(opts) (TM_SUCCESS, TM_MULTISUCCESS, TM_FAIL, TM_PARTFAIL) = range(4) TM_ALL = compat.UniqueFrozenset([ TM_SUCCESS, TM_MULTISUCCESS, TM_FAIL, TM_PARTFAIL, ]) for mode in TM_ALL: test_messages = [ "Testing mode %s" % mode, "Hello World", "A", "", "B", "Foo|bar|baz", utils.TimestampForFilename(), ] fail = mode in (TM_FAIL, TM_PARTFAIL) if mode == TM_PARTFAIL: ToStdout("Testing partial job failure") ops = [ opcodes.OpTestJqueue(notify_waitlock=True, notify_exec=True, log_messages=test_messages, fail=False), opcodes.OpTestJqueue(notify_waitlock=True, notify_exec=True, log_messages=test_messages, fail=False), opcodes.OpTestJqueue(notify_waitlock=True, notify_exec=True, log_messages=test_messages, fail=True), opcodes.OpTestJqueue(notify_waitlock=True, notify_exec=True, log_messages=test_messages, fail=False), ] expect_messages = 3 * [test_messages] expect_opstatus = [ constants.OP_STATUS_SUCCESS, constants.OP_STATUS_SUCCESS, constants.OP_STATUS_ERROR, constants.OP_STATUS_ERROR, ] expect_resultlen = 2 elif mode == TM_MULTISUCCESS: ToStdout("Testing multiple successful opcodes") ops = [ opcodes.OpTestJqueue(notify_waitlock=True, notify_exec=True, log_messages=test_messages, fail=False), opcodes.OpTestJqueue(notify_waitlock=True, notify_exec=True, log_messages=test_messages, fail=False), ] expect_messages = 2 * [test_messages] expect_opstatus = [ constants.OP_STATUS_SUCCESS, constants.OP_STATUS_SUCCESS, ] expect_resultlen = 2 else: if mode == TM_SUCCESS: ToStdout("Testing job success") expect_opstatus = [constants.OP_STATUS_SUCCESS] elif mode == TM_FAIL: ToStdout("Testing job failure") expect_opstatus = [constants.OP_STATUS_ERROR] else: raise errors.ProgrammerError("Unknown test mode %s" % mode) ops = [ opcodes.OpTestJqueue(notify_waitlock=True, notify_exec=True, log_messages=test_messages, fail=fail), ] expect_messages = [test_messages] expect_resultlen = 1 cl = cli.GetClient() cli.SetGenericOpcodeOpts(ops, opts) # Send job to master daemon job_id = cli.SendJob(ops, cl=cl) reporter = _JobQueueTestReporter() results = None try: results = cli.PollJob(job_id, cl=cl, reporter=reporter) except errors.OpExecError as err: if not fail: raise ToStdout("Ignoring error for 'job fail' test: %s", err) else: if fail: raise errors.OpExecError("Job didn't fail when it should") # Check length of result if fail: if results is not None: raise errors.OpExecError("Received result from failed job") elif len(results) != expect_resultlen: raise errors.OpExecError("Received %s results (%s), expected %s" % (len(results), results, expect_resultlen)) # Check received log messages all_messages = [i for j in expect_messages for i in j] if reporter.GetTestMessages() != all_messages: raise errors.OpExecError("Received test messages don't match input" " (input %r, received %r)" % (all_messages, reporter.GetTestMessages())) # Check final status reported_job_id = reporter.GetJobId() if reported_job_id != job_id: raise errors.OpExecError("Reported job ID %s doesn't match" "submission job ID %s" % (reported_job_id, job_id)) jobdetails = cli.GetClient().QueryJobs([job_id], ["status", "opstatus"])[0] if not jobdetails: raise errors.OpExecError("Can't find job %s" % job_id) if fail: exp_status = constants.JOB_STATUS_ERROR else: exp_status = constants.JOB_STATUS_SUCCESS (final_status, final_opstatus) = jobdetails if final_status != exp_status: raise errors.OpExecError("Final job status is %s, not %s as expected" % (final_status, exp_status)) if len(final_opstatus) != len(ops): raise errors.OpExecError("Did not receive status for all opcodes (got %s," " expected %s)" % (len(final_opstatus), len(ops))) if final_opstatus != expect_opstatus: raise errors.OpExecError("Opcode status is %s, expected %s" % (final_opstatus, expect_opstatus)) ToStdout("Job queue test successful") return 0 def TestOsParams(opts, _): """Set secret os parameters. """ op = opcodes.OpTestOsParams(osparams_secret=opts.osparams_secret) SubmitOrSend(op, opts) return 0 def ListLocks(opts, args): # pylint: disable=W0613 """List all locks. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ selected_fields = ParseFields(opts.output, _LIST_LOCKS_DEF_FIELDS) def _DashIfNone(fn): def wrapper(value): if not value: return "-" return fn(value) return wrapper def _FormatPending(value): """Format pending acquires. """ return utils.CommaJoin("%s:%s" % (mode, ",".join(map(str, threads))) for mode, threads in value) # Format raw values fmtoverride = { "mode": (_DashIfNone(str), False), "owner": (_DashIfNone(",".join), False), "pending": (_DashIfNone(_FormatPending), False), } while True: ret = GenericList(constants.QR_LOCK, selected_fields, None, None, opts.separator, not opts.no_headers, format_override=fmtoverride, verbose=opts.verbose) if ret != constants.EXIT_SUCCESS: return ret if not opts.interval: break ToStdout("") time.sleep(opts.interval) return 0 def Metad(opts, args): # pylint: disable=W0613 """Send commands to Metad. @param opts: the command line options selected by the user @type args: list @param args: the command to send, followed by the command-specific arguments @rtype: int @return: the desired exit code """ if args[0] == "echo": if len(args) != 2: ToStderr("Command 'echo' takes only precisely argument.") return 1 result = metad.Client().Echo(args[1]) print("Answer: %s" % (result,)) else: ToStderr("Command '%s' not supported", args[0]) return 1 return 0 def Wconfd(opts, args): # pylint: disable=W0613 """Send commands to WConfD. @param opts: the command line options selected by the user @type args: list @param args: the command to send, followed by the command-specific arguments @rtype: int @return: the desired exit code """ if args[0] == "echo": if len(args) != 2: ToStderr("Command 'echo' takes only precisely argument.") return 1 result = wconfd.Client().Echo(args[1]) print("Answer: %s" % (result,)) elif args[0] == "cleanuplocks": if len(args) != 1: ToStderr("Command 'cleanuplocks' takes no arguments.") return 1 wconfd.Client().CleanupLocks() print("Stale locks cleaned up.") elif args[0] == "listlocks": if len(args) != 2: ToStderr("Command 'listlocks' takes precisely one argument.") return 1 wconfdcontext = (int(args[1]), utils.livelock.GuessLockfileFor("masterd_1")) result = wconfd.Client().ListLocks(wconfdcontext) print("Answer: %s" % (result,)) elif args[0] == "listalllocks": if len(args) != 1: ToStderr("Command 'listalllocks' takes no arguments.") return 1 result = wconfd.Client().ListAllLocks() print("Answer: %s" % (result,)) elif args[0] == "listalllocksowners": if len(args) != 1: ToStderr("Command 'listalllocks' takes no arguments.") return 1 result = wconfd.Client().ListAllLocksOwners() print("Answer: %s" % (result,)) elif args[0] == "flushconfig": if len(args) != 1: ToStderr("Command 'flushconfig' takes no arguments.") return 1 wconfd.Client().FlushConfig() print("Configuration flushed.") else: ToStderr("Command '%s' not supported", args[0]) return 1 return 0 commands = { "delay": ( Delay, [ArgUnknown(min=1, max=1)], [cli_option("--no-master", dest="on_master", default=True, action="store_false", help="Do not sleep in the master code"), cli_option("-n", dest="on_nodes", default=[], action="append", help="Select nodes to sleep on"), cli_option("-r", "--repeat", type="int", default="0", dest="repeat", help="Number of times to repeat the sleep"), cli_option("-i", "--interruptible", default=False, dest="interruptible", action="store_true", help="Allows the opcode to be interrupted by using a domain " "socket"), cli_option("-l", "--no-locks", default=False, dest="no_locks", action="store_true", help="Don't take locks while performing the delay"), DRY_RUN_OPT, PRIORITY_OPT] + SUBMIT_OPTS, "[...] ", "Executes a TestDelay OpCode"), "submit-job": ( GenericOpCodes, [ArgFile(min=1)], [VERBOSE_OPT, cli_option("--op-repeat", type="int", default="1", dest="rep_op", help="Repeat the opcode sequence this number of times"), cli_option("--job-repeat", type="int", default="1", dest="rep_job", help="Repeat the job this number of times"), cli_option("--timing-stats", default=False, action="store_true", help="Show timing stats"), cli_option("--each", default=False, action="store_true", help="Submit each job separately"), DRY_RUN_OPT, PRIORITY_OPT, ], "...", "Submits jobs built from json files" " containing a list of serialized opcodes"), "iallocator": ( TestAllocator, [ArgUnknown(min=1)], [cli_option("--dir", dest="direction", default=constants.IALLOCATOR_DIR_IN, choices=list(constants.VALID_IALLOCATOR_DIRECTIONS), help="Show allocator input (in) or allocator" " results (out)"), IALLOCATOR_OPT, cli_option("-m", "--mode", default="relocate", choices=list(constants.VALID_IALLOCATOR_MODES), help=("Request mode (one of %s)" % utils.CommaJoin(constants.VALID_IALLOCATOR_MODES))), cli_option("--memory", default=128, type="unit", help="Memory size for the instance (MiB)"), cli_option("--disks", default="4096,4096", help="Comma separated list of disk sizes (MiB)"), DISK_TEMPLATE_OPT, cli_option("--nics", default="00:11:22:33:44:55", help="Comma separated list of nics, each nic" " definition is of form mac/ip/bridge, if" " missing values are replace by None"), OS_OPT, cli_option("-p", "--vcpus", default=1, type="int", help="Select number of VCPUs for the instance"), cli_option("--tags", default=None, help="Comma separated list of tags"), cli_option("--evac-mode", default=constants.NODE_EVAC_ALL, choices=list(constants.NODE_EVAC_MODES), help=("Node evacuation mode (one of %s)" % utils.CommaJoin(constants.NODE_EVAC_MODES))), cli_option("--target-groups", help="Target groups for relocation", default=[], action="append"), cli_option("--spindle-use", help="How many spindles to use", default=1, type="int"), cli_option("--count", help="How many instances to allocate", default=2, type="int"), DRY_RUN_OPT, PRIORITY_OPT, ], "{...} ", "Executes a TestAllocator OpCode"), "test-jobqueue": ( TestJobqueue, ARGS_NONE, [PRIORITY_OPT], "", "Test a few aspects of the job queue"), "test-osparams": ( TestOsParams, ARGS_NONE, [OSPARAMS_SECRET_OPT] + SUBMIT_OPTS, "[--os-parameters-secret ]", "Test secret os parameter transmission"), "locks": ( ListLocks, ARGS_NONE, [NOHDR_OPT, SEP_OPT, FIELDS_OPT, INTERVAL_OPT, VERBOSE_OPT], "[--interval N]", "Show a list of locks in the master daemon"), "wconfd": ( Wconfd, [ArgUnknown(min=1)], [], " ", "Directly talk to WConfD"), "metad": ( Metad, [ArgUnknown(min=1)], [], " ", "Directly talk to Metad"), } #: dictionary with aliases for commands aliases = { "allocator": "iallocator", } def Main(): return GenericMain(commands, aliases=aliases) ganeti-3.1.0~rc2/lib/client/gnt_filter.py000064400000000000000000000157721476477700300203530ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Job filter rule commands""" # pylint: disable=W0401,W0614 # W0401: Wildcard import ganeti.cli # W0614: Unused import %s from wildcard import (since we need cli) from ganeti.cli import * from ganeti import constants from ganeti import utils #: default list of fields for L{ListFilters} _LIST_DEF_FIELDS = ["uuid", "watermark", "priority", "predicates", "action", "reason_trail"] def AddFilter(opts, args): """Add a job filter rule. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ assert args == [] reason = [] if opts.reason: reason = [(constants.OPCODE_REASON_SRC_USER, opts.reason, utils.EpochNano())] cl = GetClient() result = cl.ReplaceFilter(None, opts.priority, opts.predicates, opts.action, reason) print(result) # Prints the UUID of the replaced/created filter def ListFilters(opts, args): """List job filter rules and their properties. @param opts: the command line options selected by the user @type args: list @param args: filters to list, or empty for all @rtype: int @return: the desired exit code """ desired_fields = ParseFields(opts.output, _LIST_DEF_FIELDS) cl = GetClient() return GenericList(constants.QR_FILTER, desired_fields, args, None, opts.separator, not opts.no_headers, verbose=opts.verbose, cl=cl, namefield="uuid") def ListFilterFields(opts, args): """List filter rule fields. @param opts: the command line options selected by the user @type args: list @param args: fields to list, or empty for all @rtype: int @return: the desired exit code """ cl = GetClient() return GenericListFields(constants.QR_FILTER, args, opts.separator, not opts.no_headers, cl=cl) def ReplaceFilter(opts, args): """Replaces a job filter rule with the given UUID, or creates it, if it doesn't exist already. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the UUID of the filter @rtype: int @return: the desired exit code """ (uuid,) = args reason = [] if opts.reason: reason = [(constants.OPCODE_REASON_SRC_USER, opts.reason, utils.EpochNano())] cl = GetClient() result = cl.ReplaceFilter(uuid, priority=opts.priority, predicates=opts.predicates, action=opts.action, reason=reason) print(result) # Prints the UUID of the replaced/created filter return 0 def ShowFilter(_, args): """Show filter rule details. @type args: list @param args: should either be an empty list, in which case we show information about all filters, or should contain a list of filter UUIDs to be queried for information @rtype: int @return: the desired exit code """ cl = GetClient() result = cl.QueryFilters(fields=["uuid", "watermark", "priority", "predicates", "action", "reason_trail"], uuids=args) for (uuid, watermark, priority, predicates, action, reason_trail) in result: ToStdout("UUID: %s", uuid) ToStdout(" Watermark: %s", watermark) ToStdout(" Priority: %s", priority) ToStdout(" Predicates: %s", predicates) ToStdout(" Action: %s", action) ToStdout(" Reason trail: %s", reason_trail) return 0 def DeleteFilter(_, args): """Remove a job filter rule. @type args: list @param args: a list of length 1 with the UUID of the filter to remove @rtype: int @return: the desired exit code """ (uuid,) = args cl = GetClient() result = cl.DeleteFilter(uuid) assert result is None return 0 FILTER_PRIORITY_OPT = \ cli_option("--priority", dest="priority", action="store", default=0, type="int", help="Priority for filter processing") FILTER_PREDICATES_OPT = \ cli_option("--predicates", dest="predicates", action="store", default=[], type="json", help="List of predicates in the Ganeti query language," " given as a JSON list.") FILTER_ACTION_OPT = \ cli_option("--action", dest="action", action="store", default="CONTINUE", type="filteraction", help="The effect of the filter. Can be one of 'ACCEPT'," " 'PAUSE', 'REJECT', 'CONTINUE' and '[RATE_LIMIT, n]'," " where n is a positive integer.") commands = { "add": ( AddFilter, ARGS_NONE, [FILTER_PRIORITY_OPT, FILTER_PREDICATES_OPT, FILTER_ACTION_OPT], "", "Adds a new filter rule"), "list": ( ListFilters, ARGS_MANY_FILTERS, [NOHDR_OPT, SEP_OPT, FIELDS_OPT, VERBOSE_OPT], "[...]", "Lists the job filter rules. The available fields can be shown" " using the \"list-fields\" command (see the man page for details)." " The default list is (in order): %s." % utils.CommaJoin(_LIST_DEF_FIELDS)), "list-fields": ( ListFilterFields, [ArgUnknown()], [NOHDR_OPT, SEP_OPT], "[...]", "Lists all available fields for filters"), "info": ( ShowFilter, ARGS_MANY_FILTERS, [], "[...]", "Shows information about the filter(s)"), "replace": ( ReplaceFilter, ARGS_ONE_FILTER, [FILTER_PRIORITY_OPT, FILTER_PREDICATES_OPT, FILTER_ACTION_OPT], "", "Replaces a filter"), "delete": ( DeleteFilter, ARGS_ONE_FILTER, [], "", "Removes a filter"), } def Main(): return GenericMain(commands) ganeti-3.1.0~rc2/lib/client/gnt_group.py000064400000000000000000000306241476477700300202130ustar00rootroot00000000000000# # # Copyright (C) 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Node group related commands""" # pylint: disable=W0401,W0614 # W0401: Wildcard import ganeti.cli # W0614: Unused import %s from wildcard import (since we need cli) from io import StringIO from ganeti.cli import * from ganeti import constants from ganeti import opcodes from ganeti import utils from ganeti import compat from ganeti.client import base #: default list of fields for L{ListGroups} _LIST_DEF_FIELDS = ["name", "node_cnt", "pinst_cnt", "alloc_policy", "ndparams"] _ENV_OVERRIDE = compat.UniqueFrozenset(["list"]) def AddGroup(opts, args): """Add a node group to the cluster. @param opts: the command line options selected by the user @type args: list @param args: a list of length 1 with the name of the group to create @rtype: int @return: the desired exit code """ ipolicy = CreateIPolicyFromOpts( minmax_ispecs=opts.ipolicy_bounds_specs, ipolicy_vcpu_ratio=opts.ipolicy_vcpu_ratio, ipolicy_spindle_ratio=opts.ipolicy_spindle_ratio, ipolicy_disk_templates=opts.ipolicy_disk_templates, group_ipolicy=True) (group_name,) = args diskparams = dict(opts.diskparams) if opts.disk_state: disk_state = utils.FlatToDict(opts.disk_state) else: disk_state = {} hv_state = dict(opts.hv_state) op = opcodes.OpGroupAdd(group_name=group_name, ndparams=opts.ndparams, alloc_policy=opts.alloc_policy, diskparams=diskparams, ipolicy=ipolicy, hv_state=hv_state, disk_state=disk_state) return base.GetResult(None, opts, SubmitOrSend(op, opts)) def AssignNodes(opts, args): """Assign nodes to a group. @param opts: the command line options selected by the user @type args: list @param args: args[0]: group to assign nodes to; args[1:]: nodes to assign @rtype: int @return: the desired exit code """ group_name = args[0] node_names = args[1:] op = opcodes.OpGroupAssignNodes(group_name=group_name, nodes=node_names, force=opts.force) SubmitOrSend(op, opts) def _FmtDict(data): """Format dict data into command-line format. @param data: The input dict to be formatted @return: The formatted dict """ if not data: return "(empty)" return utils.CommaJoin(["%s=%s" % (key, value) for key, value in data.items()]) def ListGroups(opts, args): """List node groups and their properties. @param opts: the command line options selected by the user @type args: list @param args: groups to list, or empty for all @rtype: int @return: the desired exit code """ desired_fields = ParseFields(opts.output, _LIST_DEF_FIELDS) fmtoverride = { "node_list": (",".join, False), "pinst_list": (",".join, False), "ndparams": (_FmtDict, False), } cl = GetClient() return GenericList(constants.QR_GROUP, desired_fields, args, None, opts.separator, not opts.no_headers, format_override=fmtoverride, verbose=opts.verbose, force_filter=opts.force_filter, cl=cl) def ListGroupFields(opts, args): """List node fields. @param opts: the command line options selected by the user @type args: list @param args: fields to list, or empty for all @rtype: int @return: the desired exit code """ cl = GetClient() return GenericListFields(constants.QR_GROUP, args, opts.separator, not opts.no_headers, cl=cl) def SetGroupParams(opts, args): """Modifies a node group's parameters. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the node group name @rtype: int @return: the desired exit code """ allmods = [opts.ndparams, opts.alloc_policy, opts.diskparams, opts.hv_state, opts.disk_state, opts.ipolicy_bounds_specs, opts.ipolicy_vcpu_ratio, opts.ipolicy_spindle_ratio, opts.diskparams, opts.ipolicy_disk_templates] if allmods.count(None) == len(allmods): ToStderr("Please give at least one of the parameters.") return 1 if opts.disk_state: disk_state = utils.FlatToDict(opts.disk_state) else: disk_state = {} hv_state = dict(opts.hv_state) diskparams = dict(opts.diskparams) # create ipolicy object ipolicy = CreateIPolicyFromOpts( minmax_ispecs=opts.ipolicy_bounds_specs, ipolicy_disk_templates=opts.ipolicy_disk_templates, ipolicy_vcpu_ratio=opts.ipolicy_vcpu_ratio, ipolicy_spindle_ratio=opts.ipolicy_spindle_ratio, group_ipolicy=True, allowed_values=[constants.VALUE_DEFAULT]) op = opcodes.OpGroupSetParams(group_name=args[0], ndparams=opts.ndparams, alloc_policy=opts.alloc_policy, hv_state=hv_state, disk_state=disk_state, diskparams=diskparams, ipolicy=ipolicy) result = SubmitOrSend(op, opts) if result: ToStdout("Modified node group %s", args[0]) for param, data in result: ToStdout(" - %-5s -> %s", param, data) return 0 def RemoveGroup(opts, args): """Remove a node group from the cluster. @param opts: the command line options selected by the user @type args: list @param args: a list of length 1 with the name of the group to remove @rtype: int @return: the desired exit code """ (group_name,) = args op = opcodes.OpGroupRemove(group_name=group_name) SubmitOrSend(op, opts) def RenameGroup(opts, args): """Rename a node group. @param opts: the command line options selected by the user @type args: list @param args: a list of length 2, [old_name, new_name] @rtype: int @return: the desired exit code """ group_name, new_name = args op = opcodes.OpGroupRename(group_name=group_name, new_name=new_name) SubmitOrSend(op, opts) def EvacuateGroup(opts, args): """Evacuate a node group. """ (group_name, ) = args cl = GetClient() op = opcodes.OpGroupEvacuate(group_name=group_name, iallocator=opts.iallocator, target_groups=opts.to, early_release=opts.early_release, sequential=opts.sequential, force_failover=opts.force_failover) result = SubmitOrSend(op, opts, cl=cl) # Keep track of submitted jobs jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result[constants.JOB_IDS_KEY]: jex.AddJobId(None, status, job_id) results = jex.GetResults() bad_cnt = len([row for row in results if not row[0]]) if bad_cnt == 0: ToStdout("All instances evacuated successfully.") rcode = constants.EXIT_SUCCESS else: ToStdout("There were %s errors during the evacuation.", bad_cnt) rcode = constants.EXIT_FAILURE return rcode def _FormatGroupInfo(group): (name, ndparams, custom_ndparams, diskparams, custom_diskparams, ipolicy, custom_ipolicy) = group return [ ("Node group", name), ("Node parameters", FormatParamsDictInfo(custom_ndparams, ndparams)), ("Disk parameters", FormatParamsDictInfo(custom_diskparams, diskparams)), ("Instance policy", FormatPolicyInfo(custom_ipolicy, ipolicy, False)), ] def GroupInfo(_, args): """Shows info about node group. """ cl = GetClient() selected_fields = ["name", "ndparams", "custom_ndparams", "diskparams", "custom_diskparams", "ipolicy", "custom_ipolicy"] result = cl.QueryGroups(names=args, fields=selected_fields, use_locking=False) PrintGenericInfo([ _FormatGroupInfo(group) for group in result ]) def _GetCreateCommand(group): (name, ipolicy) = group buf = StringIO() buf.write("gnt-group add") PrintIPolicyCommand(buf, ipolicy, True) buf.write(" ") buf.write(name) return buf.getvalue() def ShowCreateCommand(opts, args): """Shows the command that can be used to re-create a node group. Currently it works only for ipolicy specs. """ cl = GetClient() selected_fields = ["name"] if opts.include_defaults: selected_fields += ["ipolicy"] else: selected_fields += ["custom_ipolicy"] result = cl.QueryGroups(names=args, fields=selected_fields, use_locking=False) for group in result: ToStdout(_GetCreateCommand(group)) commands = { "add": ( AddGroup, ARGS_ONE_GROUP, [DRY_RUN_OPT, ALLOC_POLICY_OPT, NODE_PARAMS_OPT, DISK_PARAMS_OPT, HV_STATE_OPT, DISK_STATE_OPT, PRIORITY_OPT] + SUBMIT_OPTS + INSTANCE_POLICY_OPTS, "", "Add a new node group to the cluster"), "assign-nodes": ( AssignNodes, ARGS_ONE_GROUP + ARGS_MANY_NODES, [DRY_RUN_OPT, FORCE_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ...", "Assign nodes to a group"), "list": ( ListGroups, ARGS_MANY_GROUPS, [NOHDR_OPT, SEP_OPT, FIELDS_OPT, VERBOSE_OPT, FORCE_FILTER_OPT], "[...]", "Lists the node groups in the cluster. The available fields can be shown" " using the \"list-fields\" command (see the man page for details)." " The default list is (in order): %s." % utils.CommaJoin(_LIST_DEF_FIELDS)), "list-fields": ( ListGroupFields, [ArgUnknown()], [NOHDR_OPT, SEP_OPT], "[fields...]", "Lists all available fields for node groups"), "modify": ( SetGroupParams, ARGS_ONE_GROUP, [DRY_RUN_OPT] + SUBMIT_OPTS + [ALLOC_POLICY_OPT, NODE_PARAMS_OPT, HV_STATE_OPT, DISK_STATE_OPT, DISK_PARAMS_OPT, PRIORITY_OPT] + INSTANCE_POLICY_OPTS, "", "Alters the parameters of a node group"), "remove": ( RemoveGroup, ARGS_ONE_GROUP, [DRY_RUN_OPT, PRIORITY_OPT] + SUBMIT_OPTS, "[--dry-run] ", "Remove an (empty) node group from the cluster"), "rename": ( RenameGroup, [ArgGroup(min=2, max=2)], [DRY_RUN_OPT] + SUBMIT_OPTS + [PRIORITY_OPT], "[--dry-run] ", "Rename a node group"), "evacuate": ( EvacuateGroup, [ArgGroup(min=1, max=1)], [TO_GROUP_OPT, IALLOCATOR_OPT, EARLY_RELEASE_OPT, SEQUENTIAL_OPT, FORCE_FAILOVER_OPT] + SUBMIT_OPTS, "[-I ] [--to ] ", "Evacuate all instances within a group"), "list-tags": ( ListTags, ARGS_ONE_GROUP, [], "", "List the tags of the given group"), "add-tags": ( AddTags, [ArgGroup(min=1, max=1), ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ...", "Add tags to the given group"), "remove-tags": ( RemoveTags, [ArgGroup(min=1, max=1), ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ...", "Remove tags from the given group"), "info": ( GroupInfo, ARGS_MANY_GROUPS, [], "[...]", "Show group information"), "show-ispecs-cmd": ( ShowCreateCommand, ARGS_MANY_GROUPS, [INCLUDEDEFAULTS_OPT], "[--include-defaults] [...]", "Show the command line to re-create a group"), } def Main(): return GenericMain(commands, override={"tag_type": constants.TAG_NODEGROUP}, env_override=_ENV_OVERRIDE) ganeti-3.1.0~rc2/lib/client/gnt_instance.py000064400000000000000000001725551476477700300206750ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Instance related commands""" # pylint: disable=W0401,W0614,C0103 # W0401: Wildcard import ganeti.cli # W0614: Unused import %s from wildcard import (since we need cli) # C0103: Invalid name gnt-instance import copy import itertools import json import logging from ganeti.cli import * from ganeti import opcodes from ganeti import constants from ganeti import compat from ganeti import utils from ganeti import errors from ganeti import netutils from ganeti import ssh from ganeti import objects from ganeti import ht _EXPAND_CLUSTER = "cluster" _EXPAND_NODES_BOTH = "nodes" _EXPAND_NODES_PRI = "nodes-pri" _EXPAND_NODES_SEC = "nodes-sec" _EXPAND_NODES_BOTH_BY_TAGS = "nodes-by-tags" _EXPAND_NODES_PRI_BY_TAGS = "nodes-pri-by-tags" _EXPAND_NODES_SEC_BY_TAGS = "nodes-sec-by-tags" _EXPAND_INSTANCES = "instances" _EXPAND_INSTANCES_BY_TAGS = "instances-by-tags" _EXPAND_NODES_TAGS_MODES = compat.UniqueFrozenset([ _EXPAND_NODES_BOTH_BY_TAGS, _EXPAND_NODES_PRI_BY_TAGS, _EXPAND_NODES_SEC_BY_TAGS, ]) #: default list of options for L{ListInstances} _LIST_DEF_FIELDS = [ "name", "hypervisor", "os", "pnode", "status", "oper_ram", ] _MISSING = object() _ENV_OVERRIDE = compat.UniqueFrozenset(["list"]) _INST_DATA_VAL = ht.TListOf(ht.TDict) def _ExpandMultiNames(mode, names, client=None): """Expand the given names using the passed mode. For _EXPAND_CLUSTER, all instances will be returned. For _EXPAND_NODES_PRI/SEC, all instances having those nodes as primary/secondary will be returned. For _EXPAND_NODES_BOTH, all instances having those nodes as either primary or secondary will be returned. For _EXPAND_INSTANCES, the given instances will be returned. @param mode: one of L{_EXPAND_CLUSTER}, L{_EXPAND_NODES_BOTH}, L{_EXPAND_NODES_PRI}, L{_EXPAND_NODES_SEC} or L{_EXPAND_INSTANCES} @param names: a list of names; for cluster, it must be empty, and for node and instance it must be a list of valid item names (short names are valid as usual, e.g. node1 instead of node1.example.com) @rtype: list @return: the list of names after the expansion @raise errors.ProgrammerError: for unknown selection type @raise errors.OpPrereqError: for invalid input parameters """ if client is None: client = GetClient() if mode == _EXPAND_CLUSTER: if names: raise errors.OpPrereqError("Cluster filter mode takes no arguments", errors.ECODE_INVAL) idata = client.QueryInstances([], ["name"], False) inames = [row[0] for row in idata] elif (mode in _EXPAND_NODES_TAGS_MODES or mode in (_EXPAND_NODES_BOTH, _EXPAND_NODES_PRI, _EXPAND_NODES_SEC)): if mode in _EXPAND_NODES_TAGS_MODES: if not names: raise errors.OpPrereqError("No node tags passed", errors.ECODE_INVAL) ndata = client.QueryNodes([], ["name", "pinst_list", "sinst_list", "tags"], False) ndata = [row for row in ndata if set(row[3]).intersection(names)] else: if not names: raise errors.OpPrereqError("No node names passed", errors.ECODE_INVAL) ndata = client.QueryNodes(names, ["name", "pinst_list", "sinst_list"], False) ipri = [row[1] for row in ndata] pri_names = list(itertools.chain(*ipri)) isec = [row[2] for row in ndata] sec_names = list(itertools.chain(*isec)) if mode in (_EXPAND_NODES_BOTH, _EXPAND_NODES_BOTH_BY_TAGS): inames = pri_names + sec_names elif mode in (_EXPAND_NODES_PRI, _EXPAND_NODES_PRI_BY_TAGS): inames = pri_names elif mode in (_EXPAND_NODES_SEC, _EXPAND_NODES_SEC_BY_TAGS): inames = sec_names else: raise errors.ProgrammerError("Unhandled shutdown type") elif mode == _EXPAND_INSTANCES: if not names: raise errors.OpPrereqError("No instance names passed", errors.ECODE_INVAL) idata = client.QueryInstances(names, ["name"], False) inames = [row[0] for row in idata] elif mode == _EXPAND_INSTANCES_BY_TAGS: if not names: raise errors.OpPrereqError("No instance tags passed", errors.ECODE_INVAL) idata = client.QueryInstances([], ["name", "tags"], False) inames = [row[0] for row in idata if set(row[1]).intersection(names)] else: raise errors.OpPrereqError("Unknown mode '%s'" % mode, errors.ECODE_INVAL) return inames def _EnsureInstancesExist(client, names): """Check for and ensure the given instance names exist. This function will raise an OpPrereqError in case they don't exist. Otherwise it will exit cleanly. @type client: L{ganeti.luxi.Client} @param client: the client to use for the query @type names: list @param names: the list of instance names to query @raise errors.OpPrereqError: in case any instance is missing """ # TODO: change LUInstanceQuery to that it actually returns None # instead of raising an exception, or devise a better mechanism result = client.QueryInstances(names, ["name"], False) for orig_name, row in zip(names, result): if row[0] is None: raise errors.OpPrereqError("Instance '%s' does not exist" % orig_name, errors.ECODE_NOENT) def GenericManyOps(operation, fn): """Generic multi-instance operations. The will return a wrapper that processes the options and arguments given, and uses the passed function to build the opcode needed for the specific operation. Thus all the generic loop/confirmation code is abstracted into this function. """ def realfn(opts, args): if opts.multi_mode is None: opts.multi_mode = _EXPAND_INSTANCES cl = GetClient() inames = _ExpandMultiNames(opts.multi_mode, args, client=cl) if not inames: if opts.multi_mode == _EXPAND_CLUSTER: ToStdout("Cluster is empty, no instances to shutdown") return 0 raise errors.OpPrereqError("Selection filter does not match" " any instances", errors.ECODE_INVAL) multi_on = opts.multi_mode != _EXPAND_INSTANCES or len(inames) > 1 if not (opts.force_multi or not multi_on or ConfirmOperation(inames, "instances", operation)): return 1 jex = JobExecutor(verbose=multi_on, cl=cl, opts=opts) for name in inames: op = fn(name, opts) jex.QueueJob(name, op) results = jex.WaitOrShow(not opts.submit_only) rcode = compat.all(row[0] for row in results) return int(not rcode) return realfn def ListInstances(opts, args): """List instances and their properties. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ selected_fields = ParseFields(opts.output, _LIST_DEF_FIELDS) fmtoverride = dict.fromkeys(["tags", "disk.sizes", "nic.macs", "nic.ips", "nic.modes", "nic.links", "nic.bridges", "nic.networks", "snodes", "snodes.group", "snodes.group.uuid"], (lambda value: ",".join(str(item) for item in value), False)) cl = GetClient() return GenericList(constants.QR_INSTANCE, selected_fields, args, opts.units, opts.separator, not opts.no_headers, format_override=fmtoverride, verbose=opts.verbose, force_filter=opts.force_filter, cl=cl) def ListInstanceFields(opts, args): """List instance fields. @param opts: the command line options selected by the user @type args: list @param args: fields to list, or empty for all @rtype: int @return: the desired exit code """ return GenericListFields(constants.QR_INSTANCE, args, opts.separator, not opts.no_headers) def AddInstance(opts, args): """Add an instance to the cluster. This is just a wrapper over L{GenericInstanceCreate}. """ return GenericInstanceCreate(constants.INSTANCE_CREATE, opts, args) def BatchCreate(opts, args): """Create instances using a definition file. This function reads a json file with L{opcodes.OpInstanceCreate} serialisations. @param opts: the command line options selected by the user @type args: list @param args: should contain one element, the json filename @rtype: int @return: the desired exit code """ (json_filename,) = args cl = GetClient() try: instance_data = json.loads(utils.ReadFile(json_filename)) except Exception as err: # pylint: disable=W0703 ToStderr("Can't parse the instance definition file: %s" % str(err)) return 1 if not _INST_DATA_VAL(instance_data): ToStderr("The instance definition file is not %s" % _INST_DATA_VAL) return 1 instances = [] possible_params = set(opcodes.OpInstanceCreate.GetAllSlots()) for (idx, inst) in enumerate(instance_data): unknown = set(inst.keys()) - possible_params if unknown: # TODO: Suggest closest match for more user friendly experience raise errors.OpPrereqError("Unknown fields in definition %s: %s" % (idx, utils.CommaJoin(unknown)), errors.ECODE_INVAL) op = opcodes.OpInstanceCreate(**inst) op.Validate(False) instances.append(op) op = opcodes.OpInstanceMultiAlloc(iallocator=opts.iallocator, instances=instances) result = SubmitOrSend(op, opts, cl=cl) # Keep track of submitted jobs jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result[constants.JOB_IDS_KEY]: jex.AddJobId(None, status, job_id) results = jex.GetResults() bad_cnt = len([row for row in results if not row[0]]) if bad_cnt == 0: ToStdout("All instances created successfully.") rcode = constants.EXIT_SUCCESS else: ToStdout("There were %s errors during the creation.", bad_cnt) rcode = constants.EXIT_FAILURE return rcode def ReinstallInstance(opts, args): """Reinstall an instance. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the name of the instance to be reinstalled @rtype: int @return: the desired exit code """ # first, compute the desired name list if opts.multi_mode is None: opts.multi_mode = _EXPAND_INSTANCES inames = _ExpandMultiNames(opts.multi_mode, args) if not inames: raise errors.OpPrereqError("Selection filter does not match any instances", errors.ECODE_INVAL) # second, if requested, ask for an OS if opts.select_os is True: op = opcodes.OpOsDiagnose(output_fields=["name", "variants"], names=[]) result = SubmitOpCode(op, opts=opts) if not result: ToStdout("Can't get the OS list") return 1 ToStdout("Available OS templates:") number = 0 choices = [] for (name, variants) in result: for entry in CalculateOSNames(name, variants): ToStdout("%3s: %s", number, entry) choices.append(("%s" % number, entry, entry)) number += 1 choices.append(("x", "exit", "Exit gnt-instance reinstall")) selected = AskUser("Enter OS template number (or x to abort):", choices) if selected == "exit": ToStderr("User aborted reinstall, exiting") return 1 os_name = selected os_msg = "change the OS to '%s'" % selected else: os_name = opts.os if opts.os is not None: os_msg = "change the OS to '%s'" % os_name else: os_msg = "keep the same OS" # third, get confirmation: multi-reinstall requires --force-multi, # single-reinstall either --force or --force-multi (--force-multi is # a stronger --force) multi_on = opts.multi_mode != _EXPAND_INSTANCES or len(inames) > 1 if multi_on: warn_msg = ("Note: this will remove *all* data for the" " below instances! It will %s.\n" % os_msg) if not (opts.force_multi or ConfirmOperation(inames, "instances", "reinstall", extra=warn_msg)): return 1 else: if not (opts.force or opts.force_multi): usertext = ("This will reinstall the instance '%s' (and %s) which" " removes all data. Continue?") % (inames[0], os_msg) if not AskUser(usertext): return 1 jex = JobExecutor(verbose=multi_on, opts=opts) for instance_name in inames: op = opcodes.OpInstanceReinstall(instance_name=instance_name, os_type=os_name, force_variant=opts.force_variant, osparams=opts.osparams, osparams_private=opts.osparams_private, osparams_secret=opts.osparams_secret) jex.QueueJob(instance_name, op) results = jex.WaitOrShow(not opts.submit_only) if compat.all(map(compat.fst, results)): return constants.EXIT_SUCCESS else: return constants.EXIT_FAILURE def RemoveInstance(opts, args): """Remove an instance. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the name of the instance to be removed @rtype: int @return: the desired exit code """ instance_name = args[0] force = opts.force cl = GetClient() if not force: _EnsureInstancesExist(cl, [instance_name]) usertext = ("This will remove the volumes of the instance %s" " (including mirrors), thus removing all the data" " of the instance. Continue?") % instance_name if not AskUser(usertext): return 1 op = opcodes.OpInstanceRemove(instance_name=instance_name, ignore_failures=opts.ignore_failures, shutdown_timeout=opts.shutdown_timeout) SubmitOrSend(op, opts, cl=cl) return 0 def RenameInstance(opts, args): """Rename an instance. @param opts: the command line options selected by the user @type args: list @param args: should contain two elements, the old and the new instance names @rtype: int @return: the desired exit code """ op = opcodes.OpInstanceRename(instance_name=args[0], new_name=args[1], ip_check=opts.ip_check, name_check=opts.name_check) result = SubmitOrSend(op, opts) if result: ToStdout("Instance '%s' renamed to '%s'", args[0], result) return 0 def ActivateDisks(opts, args): """Activate an instance's disks. This serves two purposes: - it allows (as long as the instance is not running) mounting the disks and modifying them from the node - it repairs inactive secondary drbds @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the instance name @rtype: int @return: the desired exit code """ instance_name = args[0] op = opcodes.OpInstanceActivateDisks(instance_name=instance_name, ignore_size=opts.ignore_size, wait_for_sync=opts.wait_for_sync) disks_info = SubmitOrSend(op, opts) for host, iname, nname in disks_info: ToStdout("%s:%s:%s", host, iname, nname) return 0 def DeactivateDisks(opts, args): """Deactivate an instance's disks. This function takes the instance name, looks for its primary node and the tries to shutdown its block devices on that node. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the instance name @rtype: int @return: the desired exit code """ instance_name = args[0] op = opcodes.OpInstanceDeactivateDisks(instance_name=instance_name, force=opts.force) SubmitOrSend(op, opts) return 0 def RecreateDisks(opts, args): """Recreate an instance's disks. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the instance name @rtype: int @return: the desired exit code """ instance_name = args[0] disks = [] if opts.disks: for didx, ddict in opts.disks: didx = int(didx) if not ht.TDict(ddict): msg = "Invalid disk/%d value: expected dict, got %s" % (didx, ddict) raise errors.OpPrereqError(msg, errors.ECODE_INVAL) if constants.IDISK_SIZE in ddict: try: ddict[constants.IDISK_SIZE] = \ utils.ParseUnit(ddict[constants.IDISK_SIZE]) except ValueError as err: raise errors.OpPrereqError("Invalid disk size for disk %d: %s" % (didx, err), errors.ECODE_INVAL) if constants.IDISK_SPINDLES in ddict: try: ddict[constants.IDISK_SPINDLES] = \ int(ddict[constants.IDISK_SPINDLES]) except ValueError as err: raise errors.OpPrereqError("Invalid spindles for disk %d: %s" % (didx, err), errors.ECODE_INVAL) disks.append((didx, ddict)) # TODO: Verify modifyable parameters (already done in # LUInstanceRecreateDisks, but it'd be nice to have in the client) if opts.node: if opts.iallocator: msg = "At most one of either --nodes or --iallocator can be passed" raise errors.OpPrereqError(msg, errors.ECODE_INVAL) pnode, snode = SplitNodeOption(opts.node) nodes = [pnode] if snode is not None: nodes.append(snode) else: nodes = [] op = opcodes.OpInstanceRecreateDisks(instance_name=instance_name, disks=disks, nodes=nodes, iallocator=opts.iallocator) SubmitOrSend(op, opts) return 0 def GrowDisk(opts, args): """Grow an instance's disks. @param opts: the command line options selected by the user @type args: list @param args: should contain three elements, the target instance name, the target disk id, and the target growth @rtype: int @return: the desired exit code """ instance = args[0] disk = args[1] try: disk = int(disk) except (TypeError, ValueError) as err: raise errors.OpPrereqError("Invalid disk index: %s" % str(err), errors.ECODE_INVAL) try: amount = utils.ParseUnit(args[2]) except errors.UnitParseError: raise errors.OpPrereqError("Can't parse the given amount '%s'" % args[2], errors.ECODE_INVAL) op = opcodes.OpInstanceGrowDisk(instance_name=instance, disk=disk, amount=amount, wait_for_sync=opts.wait_for_sync, absolute=opts.absolute, ignore_ipolicy=opts.ignore_ipolicy ) SubmitOrSend(op, opts) return 0 def _StartupInstance(name, opts): """Startup instances. This returns the opcode to start an instance, and its decorator will wrap this into a loop starting all desired instances. @param name: the name of the instance to act on @param opts: the command line options selected by the user @return: the opcode needed for the operation """ op = opcodes.OpInstanceStartup(instance_name=name, force=opts.force, ignore_offline_nodes=opts.ignore_offline, no_remember=opts.no_remember, startup_paused=opts.startup_paused) # do not add these parameters to the opcode unless they're defined if opts.hvparams: op.hvparams = opts.hvparams if opts.beparams: op.beparams = opts.beparams return op def _RebootInstance(name, opts): """Reboot instance(s). This returns the opcode to reboot an instance, and its decorator will wrap this into a loop rebooting all desired instances. @param name: the name of the instance to act on @param opts: the command line options selected by the user @return: the opcode needed for the operation """ return opcodes.OpInstanceReboot(instance_name=name, reboot_type=opts.reboot_type, ignore_secondaries=opts.ignore_secondaries, shutdown_timeout=opts.shutdown_timeout) def _ShutdownInstance(name, opts): """Shutdown an instance. This returns the opcode to shutdown an instance, and its decorator will wrap this into a loop shutting down all desired instances. @param name: the name of the instance to act on @param opts: the command line options selected by the user @return: the opcode needed for the operation """ return opcodes.OpInstanceShutdown(instance_name=name, force=opts.force, timeout=opts.timeout, ignore_offline_nodes=opts.ignore_offline, no_remember=opts.no_remember) def ReplaceDisks(opts, args): """Replace the disks of an instance @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the instance name @rtype: int @return: the desired exit code """ new_2ndary = opts.dst_node iallocator = opts.iallocator if opts.disks is None: disks = [] else: try: disks = [int(i) for i in opts.disks.split(",")] except (TypeError, ValueError) as err: raise errors.OpPrereqError("Invalid disk index passed: %s" % str(err), errors.ECODE_INVAL) cnt = [opts.on_primary, opts.on_secondary, opts.auto, new_2ndary is not None, iallocator is not None].count(True) if cnt != 1: raise errors.OpPrereqError("One and only one of the -p, -s, -a, -n and -I" " options must be passed", errors.ECODE_INVAL) elif opts.on_primary: mode = constants.REPLACE_DISK_PRI elif opts.on_secondary: mode = constants.REPLACE_DISK_SEC elif opts.auto: mode = constants.REPLACE_DISK_AUTO if disks: raise errors.OpPrereqError("Cannot specify disks when using automatic" " mode", errors.ECODE_INVAL) elif new_2ndary is not None or iallocator is not None: # replace secondary mode = constants.REPLACE_DISK_CHG op = opcodes.OpInstanceReplaceDisks(instance_name=args[0], disks=disks, remote_node=new_2ndary, mode=mode, iallocator=iallocator, early_release=opts.early_release, ignore_ipolicy=opts.ignore_ipolicy) SubmitOrSend(op, opts) return 0 def FailoverInstance(opts, args): """Failover an instance. The failover is done by shutting it down on its present node and starting it on the secondary. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the instance name @rtype: int @return: the desired exit code """ cl = GetClient() instance_name = args[0] ignore_consistency = opts.ignore_consistency force = opts.force iallocator = opts.iallocator target_node = opts.dst_node if iallocator and target_node: raise errors.OpPrereqError("Specify either an iallocator (-I), or a target" " node (-n) but not both", errors.ECODE_INVAL) if not force: _EnsureInstancesExist(cl, [instance_name]) usertext = ("Failover will happen to image %s." " This requires a shutdown of the instance. Continue?" % (instance_name,)) if not AskUser(usertext): return 1 if ignore_consistency: usertext = ("To failover instance %s, the source node must be marked" " offline first. Is this already the case?") % instance_name if not AskUser(usertext): return 1 op = opcodes.OpInstanceFailover(instance_name=instance_name, ignore_consistency=ignore_consistency, shutdown_timeout=opts.shutdown_timeout, iallocator=iallocator, target_node=target_node, ignore_ipolicy=opts.ignore_ipolicy) SubmitOrSend(op, opts, cl=cl) return 0 def MigrateInstance(opts, args): """Migrate an instance. The migrate is done without shutdown. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the instance name @rtype: int @return: the desired exit code """ cl = GetClient() instance_name = args[0] force = opts.force iallocator = opts.iallocator target_node = opts.dst_node if iallocator and target_node: raise errors.OpPrereqError("Specify either an iallocator (-I), or a target" " node (-n) but not both", errors.ECODE_INVAL) if not force: _EnsureInstancesExist(cl, [instance_name]) if opts.cleanup: usertext = ("Instance %s will be recovered from a failed migration." " Note that the migration procedure (including cleanup)" % (instance_name,)) else: usertext = ("Instance %s will be migrated. Note that migration" % (instance_name,)) usertext += (" might impact the instance if anything goes wrong" " (e.g. due to bugs in the hypervisor). Continue?") if not AskUser(usertext): return 1 # this should be removed once --non-live is deprecated if not opts.live and opts.migration_mode is not None: raise errors.OpPrereqError("Only one of the --non-live and " "--migration-mode options can be passed", errors.ECODE_INVAL) if not opts.live: # --non-live passed mode = constants.HT_MIGRATION_NONLIVE else: mode = opts.migration_mode op = opcodes.OpInstanceMigrate(instance_name=instance_name, mode=mode, cleanup=opts.cleanup, iallocator=iallocator, target_node=target_node, allow_failover=opts.allow_failover, allow_runtime_changes=opts.allow_runtime_chgs, ignore_ipolicy=opts.ignore_ipolicy, ignore_hvversions=opts.ignore_hvversions) SubmitOrSend(op, cl=cl, opts=opts) return 0 def MoveInstance(opts, args): """Move an instance. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the instance name @rtype: int @return: the desired exit code """ cl = GetClient() instance_name = args[0] force = opts.force if not force: usertext = ("Instance %s will be moved." " This requires a shutdown of the instance. Continue?" % (instance_name,)) if not AskUser(usertext): return 1 op = opcodes.OpInstanceMove(instance_name=instance_name, target_node=opts.node, compress=opts.compress, shutdown_timeout=opts.shutdown_timeout, ignore_consistency=opts.ignore_consistency, ignore_ipolicy=opts.ignore_ipolicy) SubmitOrSend(op, opts, cl=cl) return 0 def ConnectToInstanceConsole(opts, args): """Connect to the console of an instance. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the instance name @rtype: int @return: the desired exit code """ instance_name = args[0] cl = GetClient() try: cluster_name = cl.QueryConfigValues(["cluster_name"])[0] idata = cl.QueryInstances([instance_name], ["console", "oper_state"], False) if not idata: raise errors.OpPrereqError("Instance '%s' does not exist" % instance_name, errors.ECODE_NOENT) finally: # Ensure client connection is closed while external commands are run cl.Close() del cl (console_data, oper_state) = idata[0] if not console_data: if oper_state: # Instance is running raise errors.OpExecError("Console information for instance %s is" " unavailable" % instance_name) else: raise errors.OpExecError("Instance %s is not running, can't get console" % instance_name) return _DoConsole(objects.InstanceConsole.FromDict(console_data), opts.show_command, cluster_name) def _DoConsole(console, show_command, cluster_name, feedback_fn=ToStdout, _runcmd_fn=utils.RunCmd): """Acts based on the result of L{opcodes.OpInstanceConsole}. @type console: L{objects.InstanceConsole} @param console: Console object @type show_command: bool @param show_command: Whether to just display commands @type cluster_name: string @param cluster_name: Cluster name as retrieved from master daemon """ console.Validate() if console.kind == constants.CONS_MESSAGE: feedback_fn(console.message) elif console.kind == constants.CONS_VNC: feedback_fn("Instance %s has VNC listening on %s:%s (display %s)," " URL ", console.instance, console.host, console.port, console.display, console.host, console.port) elif console.kind == constants.CONS_SPICE: feedback_fn("Instance %s has SPICE listening on %s:%s", console.instance, console.host, console.port) elif console.kind == constants.CONS_SSH: # Convert to string if not already one if isinstance(console.command, str): cmd = console.command else: cmd = utils.ShellQuoteArgs(console.command) srun = ssh.SshRunner(cluster_name=cluster_name) ssh_cmd = srun.BuildCmd(console.host, console.user, cmd, port=console.port, batch=True, quiet=False, tty=True) if show_command: feedback_fn(utils.ShellQuoteArgs(ssh_cmd)) else: result = _runcmd_fn(ssh_cmd, interactive=True) if result.failed: logging.error("Console command \"%s\" failed with reason '%s' and" " output %r", result.cmd, result.fail_reason, result.output) raise errors.OpExecError("Connection to console of instance %s failed," " please check cluster configuration" % console.instance) else: raise errors.GenericError("Unknown console type '%s'" % console.kind) return constants.EXIT_SUCCESS def _FormatDiskDetails(dev_type, dev, roman): """Formats the logical_id of a disk. """ if dev_type == constants.DT_DRBD8: drbd_info = dev["drbd_info"] data = [ ("nodeA", "%s, minor=%s" % (drbd_info["primary_node"], compat.TryToRoman(drbd_info["primary_minor"], convert=roman))), ("nodeB", "%s, minor=%s" % (drbd_info["secondary_node"], compat.TryToRoman(drbd_info["secondary_minor"], convert=roman))), ("port", str(compat.TryToRoman(drbd_info["port"], convert=roman))), ] elif dev_type == constants.DT_PLAIN: vg_name, lv_name = dev["logical_id"] data = ["%s/%s" % (vg_name, lv_name)] else: data = [str(dev["logical_id"])] return data def _FormatBlockDevInfo(idx, top_level, dev, roman): """Show block device information. This is only used by L{ShowInstanceConfig}, but it's too big to be left for an inline definition. @type idx: int @param idx: the index of the current disk @type top_level: boolean @param top_level: if this a top-level disk? @type dev: dict @param dev: dictionary with disk information @type roman: boolean @param roman: whether to try to use roman integers @return: a list of either strings, tuples or lists (which should be formatted at a higher indent level) """ def helper(dtype, status): """Format one line for physical device status. @type dtype: str @param dtype: a constant from the L{constants.DTS_BLOCK} set @type status: tuple @param status: a tuple as returned from L{backend.FindBlockDevice} @return: the string representing the status """ if not status: return "not active" txt = "" (path, major, minor, syncp, estt, degr, ldisk_status) = status if major is None: major_string = "N/A" else: major_string = str(compat.TryToRoman(major, convert=roman)) if minor is None: minor_string = "N/A" else: minor_string = str(compat.TryToRoman(minor, convert=roman)) txt += ("%s (%s:%s)" % (path, major_string, minor_string)) if dtype in (constants.DT_DRBD8, ): if syncp is not None: sync_text = "*RECOVERING* %5.2f%%," % syncp if estt: sync_text += " ETA %ss" % compat.TryToRoman(estt, convert=roman) else: sync_text += " ETA unknown" else: sync_text = "in sync" if degr: degr_text = "*DEGRADED*" else: degr_text = "ok" if ldisk_status == constants.LDS_FAULTY: ldisk_text = " *MISSING DISK*" elif ldisk_status == constants.LDS_UNKNOWN: ldisk_text = " *UNCERTAIN STATE*" else: ldisk_text = "" txt += (" %s, status %s%s" % (sync_text, degr_text, ldisk_text)) elif dtype == constants.DT_PLAIN: if ldisk_status == constants.LDS_FAULTY: ldisk_text = " *FAILED* (failed drive?)" else: ldisk_text = "" txt += ldisk_text return txt # the header if top_level: if dev["iv_name"] is not None: txt = dev["iv_name"] else: txt = "disk %s" % compat.TryToRoman(idx, convert=roman) else: txt = "child %s" % compat.TryToRoman(idx, convert=roman) if isinstance(dev["size"], int): nice_size = utils.FormatUnit(dev["size"], "h", roman) else: nice_size = str(dev["size"]) data = [(txt, "%s, size %s" % (dev["dev_type"], nice_size))] if top_level: if dev["spindles"] is not None: data.append(("spindles", dev["spindles"])) data.append(("access mode", dev["mode"])) if dev["logical_id"] is not None: try: l_id = _FormatDiskDetails(dev["dev_type"], dev, roman) except ValueError: l_id = [str(dev["logical_id"])] if len(l_id) == 1: data.append(("logical_id", l_id[0])) else: data.extend(l_id) if dev["pstatus"]: data.append(("on primary", helper(dev["dev_type"], dev["pstatus"]))) if dev["sstatus"]: data.append(("on secondary", helper(dev["dev_type"], dev["sstatus"]))) data.append(("name", dev["name"])) data.append(("UUID", dev["uuid"])) if dev["children"]: data.append(("child devices", [ _FormatBlockDevInfo(c_idx, False, child, roman) for c_idx, child in enumerate(dev["children"]) ])) return data def _FormatInstanceNicInfo(idx, nic, roman=False): """Helper function for L{_FormatInstanceInfo()}""" (name, uuid, ip, mac, mode, link, vlan, _, netinfo) = nic network_name = None if netinfo: network_name = netinfo["name"] return [ ("nic/%s" % str(compat.TryToRoman(idx, roman)), ""), ("MAC", str(mac)), ("IP", str(ip)), ("mode", str(mode)), ("link", str(link)), ("vlan", str(compat.TryToRoman(vlan, roman))), ("network", str(network_name)), ("UUID", str(uuid)), ("name", str(name)), ] def _FormatInstanceNodesInfo(instance): """Helper function for L{_FormatInstanceInfo()}""" pgroup = ("%s (UUID %s)" % (instance["pnode_group_name"], instance["pnode_group_uuid"])) secs = utils.CommaJoin(("%s (group %s, group UUID %s)" % (name, group_name, group_uuid)) for (name, group_name, group_uuid) in zip(instance["snodes"], instance["snodes_group_names"], instance["snodes_group_uuids"])) return [ [ ("primary", instance["pnode"]), ("group", pgroup), ], [("secondaries", secs)], ] def _GetVncConsoleInfo(instance): """Helper function for L{_FormatInstanceInfo()}""" vnc_bind_address = instance["hv_actual"].get(constants.HV_VNC_BIND_ADDRESS, None) if vnc_bind_address: port = instance["network_port"] display = int(port) - constants.VNC_BASE_PORT if display > 0 and vnc_bind_address == constants.IP4_ADDRESS_ANY: vnc_console_port = "%s:%s (display %s)" % (instance["pnode"], port, display) elif display > 0 and netutils.IP4Address.IsValid(vnc_bind_address): vnc_console_port = ("%s:%s (node %s) (display %s)" % (vnc_bind_address, port, instance["pnode"], display)) else: # vnc bind address is a file vnc_console_port = "%s:%s" % (instance["pnode"], vnc_bind_address) ret = "vnc to %s" % vnc_console_port else: ret = None return ret def _FormatInstanceInfo(instance, roman_integers): """Format instance information for L{cli.PrintGenericInfo()}""" istate = "configured to be %s" % instance["config_state"] if instance["run_state"]: istate += ", actual state is %s" % instance["run_state"] info = [ ("Instance name", instance["name"]), ("UUID", instance["uuid"]), ("Serial number", str(compat.TryToRoman(instance["serial_no"], convert=roman_integers))), ("Creation time", utils.FormatTime(instance["ctime"])), ("Modification time", utils.FormatTime(instance["mtime"])), ("State", istate), ("Nodes", _FormatInstanceNodesInfo(instance)), ("Operating system", instance["os"]), ("Operating system parameters", FormatParamsDictInfo(instance["os_instance"], instance["os_actual"], roman_integers)), ] if "network_port" in instance: info.append(("Allocated network port", str(compat.TryToRoman(instance["network_port"], convert=roman_integers)))) info.append(("Hypervisor", instance["hypervisor"])) console = _GetVncConsoleInfo(instance) if console: info.append(("console connection", console)) # deprecated "memory" value, kept for one version for compatibility # TODO(ganeti 2.7) remove. be_actual = copy.deepcopy(instance["be_actual"]) be_actual["memory"] = be_actual[constants.BE_MAXMEM] info.extend([ ("Hypervisor parameters", FormatParamsDictInfo(instance["hv_instance"], instance["hv_actual"], roman_integers)), ("Back-end parameters", FormatParamsDictInfo(instance["be_instance"], be_actual, roman_integers)), ("NICs", [ _FormatInstanceNicInfo(idx, nic, roman_integers) for (idx, nic) in enumerate(instance["nics"]) ]), ("Disk template", instance["disk_template"]), ("Disks", [ _FormatBlockDevInfo(idx, True, device, roman_integers) for (idx, device) in enumerate(instance["disks"]) ]), ]) return info def ShowInstanceConfig(opts, args): """Compute instance run-time status. @param opts: the command line options selected by the user @type args: list @param args: either an empty list, and then we query all instances, or should contain a list of instance names @rtype: int @return: the desired exit code """ if not args and not opts.show_all: ToStderr("No instance selected." " Please pass in --all if you want to query all instances.\n" "Note that this can take a long time on a big cluster.") return 1 elif args and opts.show_all: ToStderr("Cannot use --all if you specify instance names.") return 1 retcode = 0 op = opcodes.OpInstanceQueryData(instances=args, static=opts.static, use_locking=not opts.static) result = SubmitOpCode(op, opts=opts) if not result: ToStdout("No instances.") return 1 PrintGenericInfo([ _FormatInstanceInfo(instance, opts.roman_integers) for instance in result.values() ]) return retcode def _ConvertNicDiskModifications(mods): """Converts NIC/disk modifications from CLI to opcode. When L{opcodes.OpInstanceSetParams} was changed to support adding/removing disks at arbitrary indices, its parameter format changed. This function converts legacy requests (e.g. "--net add" or "--disk add:size=4G") to the newer format and adds support for new-style requests (e.g. "--new 4:add"). @type mods: list of tuples @param mods: Modifications as given by command line parser @rtype: list of tuples @return: Modifications as understood by L{opcodes.OpInstanceSetParams} """ result = [] for (identifier, params) in mods: if identifier == constants.DDM_ADD: # Add item as last item (legacy interface) action = constants.DDM_ADD identifier = -1 elif identifier == constants.DDM_ATTACH: # Attach item as last item (legacy interface) action = constants.DDM_ATTACH identifier = -1 elif identifier == constants.DDM_REMOVE: # Remove last item (legacy interface) action = constants.DDM_REMOVE identifier = -1 elif identifier == constants.DDM_DETACH: # Detach last item (legacy interface) action = constants.DDM_DETACH identifier = -1 else: # Modifications and adding/attaching/removing/detaching at arbitrary # indices add = params.pop(constants.DDM_ADD, _MISSING) attach = params.pop(constants.DDM_ATTACH, _MISSING) remove = params.pop(constants.DDM_REMOVE, _MISSING) detach = params.pop(constants.DDM_DETACH, _MISSING) modify = params.pop(constants.DDM_MODIFY, _MISSING) # Check if the user has requested more than one operation and raise an # exception. If no operations have been given, default to modify. action = constants.DDM_MODIFY ops = { constants.DDM_ADD: add, constants.DDM_ATTACH: attach, constants.DDM_REMOVE: remove, constants.DDM_DETACH: detach, constants.DDM_MODIFY: modify, } count = 0 for op, param in ops.items(): if param is not _MISSING: count += 1 action = op if count > 1: raise errors.OpPrereqError( "Cannot do more than one of the following operations at the" " same time: %s" % ", ".join(ops.keys()), errors.ECODE_INVAL) assert not (constants.DDMS_VALUES_WITH_MODIFY & set(params.keys())) if action in (constants.DDM_REMOVE, constants.DDM_DETACH) and params: raise errors.OpPrereqError("Not accepting parameters on removal/detach", errors.ECODE_INVAL) result.append((action, identifier, params)) return result def _ParseExtStorageParams(params): """Parses the disk params for ExtStorage conversions. """ if params: if constants.IDISK_PROVIDER not in params: raise errors.OpPrereqError("Missing required parameter '%s' when" " converting to an ExtStorage disk template" % constants.IDISK_PROVIDER, errors.ECODE_INVAL) else: for param in params: if (param != constants.IDISK_PROVIDER and param in constants.IDISK_PARAMS): raise errors.OpPrereqError("Invalid parameter '%s' when converting" " to an ExtStorage template (it is not" " allowed modifying existing disk" " parameters)" % param, errors.ECODE_INVAL) return params def _ParseDiskSizes(mods): """Parses disk sizes in parameters. """ for (action, _, params) in mods: if params and constants.IDISK_SPINDLES in params: params[constants.IDISK_SPINDLES] = \ int(params[constants.IDISK_SPINDLES]) if params and constants.IDISK_SIZE in params: params[constants.IDISK_SIZE] = \ utils.ParseUnit(params[constants.IDISK_SIZE]) elif action == constants.DDM_ADD: raise errors.OpPrereqError("Missing required parameter 'size'", errors.ECODE_INVAL) return mods def SetInstanceParams(opts, args): """Modifies an instance. All parameters take effect only at the next restart of the instance. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the instance name @rtype: int @return: the desired exit code """ if not (opts.nics or opts.disks or opts.disk_template or opts.hvparams or opts.beparams or opts.os or opts.osparams or opts.osparams_private or opts.offline_inst or opts.online_inst or opts.runtime_mem or opts.new_primary_node or opts.instance_communication is not None): ToStderr("Please give at least one of the parameters.") return 1 for param in opts.beparams: if isinstance(opts.beparams[param], str): if opts.beparams[param].lower() == "default": opts.beparams[param] = constants.VALUE_DEFAULT utils.ForceDictType(opts.beparams, constants.BES_PARAMETER_COMPAT, allowed_values=[constants.VALUE_DEFAULT]) for param in opts.hvparams: if isinstance(opts.hvparams[param], str): if opts.hvparams[param].lower() == "default": opts.hvparams[param] = constants.VALUE_DEFAULT utils.ForceDictType(opts.hvparams, constants.HVS_PARAMETER_TYPES, allowed_values=[constants.VALUE_DEFAULT]) FixHvParams(opts.hvparams) nics = _ConvertNicDiskModifications(opts.nics) for action, _, __ in nics: if action == constants.DDM_MODIFY and opts.hotplug and not opts.force: usertext = ("You are about to hot-modify a NIC. This will be done" " by removing the existing NIC and then adding a new one." " Network connection might be lost. Continue?") if not AskUser(usertext): return 1 disks = _ParseDiskSizes(_ConvertNicDiskModifications(opts.disks)) # verify the user provided parameters for disk template conversions if opts.disk_template: if (opts.ext_params and opts.disk_template != constants.DT_EXT): ToStderr("Specifying ExtStorage parameters requires converting" " to the '%s' disk template" % constants.DT_EXT) return 1 elif (not opts.ext_params and opts.disk_template == constants.DT_EXT): ToStderr("Provider option is missing, use either the" " '--ext-params' or '-e' option") return 1 if ((opts.file_driver or opts.file_storage_dir) and not opts.disk_template in constants.DTS_FILEBASED): ToStderr("Specifying file-based configuration arguments requires" " converting to a file-based disk template") return 1 ext_params = _ParseExtStorageParams(opts.ext_params) if opts.offline_inst: offline = True elif opts.online_inst: offline = False else: offline = None instance_comm = opts.instance_communication op = opcodes.OpInstanceSetParams(instance_name=args[0], nics=nics, disks=disks, hotplug=opts.hotplug, disk_template=opts.disk_template, ext_params=ext_params, file_driver=opts.file_driver, file_storage_dir=opts.file_storage_dir, remote_node=opts.node, iallocator=opts.iallocator, pnode=opts.new_primary_node, hvparams=opts.hvparams, beparams=opts.beparams, runtime_mem=opts.runtime_mem, os_name=opts.os, osparams=opts.osparams, osparams_private=opts.osparams_private, force_variant=opts.force_variant, force=opts.force, wait_for_sync=opts.wait_for_sync, offline=offline, conflicts_check=opts.conflicts_check, ignore_ipolicy=opts.ignore_ipolicy, instance_communication=instance_comm) # even if here we process the result, we allow submit only result = SubmitOrSend(op, opts) if result: ToStdout("Modified instance %s", args[0]) for param, data in result: ToStdout(" - %-5s -> %s", param, data) ToStdout("Please don't forget that most parameters take effect" " only at the next (re)start of the instance initiated by" " ganeti; restarting from within the instance will" " not be enough.") if opts.hvparams: ToStdout("Note that changing hypervisor parameters without performing a" " restart might lead to a crash while performing a live" " migration. This will be addressed in future Ganeti versions.") return 0 def ChangeGroup(opts, args): """Moves an instance to another group. """ (instance_name, ) = args cl = GetClient() op = opcodes.OpInstanceChangeGroup(instance_name=instance_name, iallocator=opts.iallocator, target_groups=opts.to, early_release=opts.early_release) result = SubmitOrSend(op, opts, cl=cl) # Keep track of submitted jobs jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result[constants.JOB_IDS_KEY]: jex.AddJobId(None, status, job_id) results = jex.GetResults() bad_cnt = len([row for row in results if not row[0]]) if bad_cnt == 0: ToStdout("Instance '%s' changed group successfully.", instance_name) rcode = constants.EXIT_SUCCESS else: ToStdout("There were %s errors while changing group of instance '%s'.", bad_cnt, instance_name) rcode = constants.EXIT_FAILURE return rcode # multi-instance selection options m_force_multi = cli_option("--force-multiple", dest="force_multi", help="Do not ask for confirmation when more than" " one instance is affected", action="store_true", default=False) m_pri_node_opt = cli_option("--primary", dest="multi_mode", help="Filter by nodes (primary only)", const=_EXPAND_NODES_PRI, action="store_const") m_sec_node_opt = cli_option("--secondary", dest="multi_mode", help="Filter by nodes (secondary only)", const=_EXPAND_NODES_SEC, action="store_const") m_node_opt = cli_option("--node", dest="multi_mode", help="Filter by nodes (primary and secondary)", const=_EXPAND_NODES_BOTH, action="store_const") m_clust_opt = cli_option("--all", dest="multi_mode", help="Select all instances in the cluster", const=_EXPAND_CLUSTER, action="store_const") m_inst_opt = cli_option("--instance", dest="multi_mode", help="Filter by instance name [default]", const=_EXPAND_INSTANCES, action="store_const") m_node_tags_opt = cli_option("--node-tags", dest="multi_mode", help="Filter by node tag", const=_EXPAND_NODES_BOTH_BY_TAGS, action="store_const") m_pri_node_tags_opt = cli_option("--pri-node-tags", dest="multi_mode", help="Filter by primary node tag", const=_EXPAND_NODES_PRI_BY_TAGS, action="store_const") m_sec_node_tags_opt = cli_option("--sec-node-tags", dest="multi_mode", help="Filter by secondary node tag", const=_EXPAND_NODES_SEC_BY_TAGS, action="store_const") m_inst_tags_opt = cli_option("--tags", dest="multi_mode", help="Filter by instance tag", const=_EXPAND_INSTANCES_BY_TAGS, action="store_const") # this is defined separately due to readability only add_opts = [ FORTHCOMING_OPT, COMMIT_OPT, NOSTART_OPT, OS_OPT, FORCE_VARIANT_OPT, NO_INSTALL_OPT, IGNORE_IPOLICY_OPT, INSTANCE_COMMUNICATION_OPT, HELPER_STARTUP_TIMEOUT_OPT, HELPER_SHUTDOWN_TIMEOUT_OPT, ] commands = { "add": ( AddInstance, [ArgHost(min=1, max=1)], COMMON_CREATE_OPTS + add_opts, "[...] -t disk-type -n node[:secondary-node] -o os-type ", "Creates and adds a new instance to the cluster"), "batch-create": ( BatchCreate, [ArgFile(min=1, max=1)], [DRY_RUN_OPT, PRIORITY_OPT, IALLOCATOR_OPT] + SUBMIT_OPTS, "", "Create a bunch of instances based on specs in the file."), "console": ( ConnectToInstanceConsole, ARGS_ONE_INSTANCE, [SHOWCMD_OPT, PRIORITY_OPT], "[--show-cmd] ", "Opens a console on the specified instance"), "failover": ( FailoverInstance, ARGS_ONE_INSTANCE, [FORCE_OPT, IGNORE_CONSIST_OPT] + SUBMIT_OPTS + [SHUTDOWN_TIMEOUT_OPT, DRY_RUN_OPT, PRIORITY_OPT, DST_NODE_OPT, IALLOCATOR_OPT, IGNORE_IPOLICY_OPT, CLEANUP_OPT], "[-f] ", "Stops the instance, changes its primary node and" " (if it was originally running) starts it on the new node" " (the secondary for mirrored instances or any node" " for shared storage)."), "migrate": ( MigrateInstance, ARGS_ONE_INSTANCE, [FORCE_OPT, NONLIVE_OPT, MIGRATION_MODE_OPT, CLEANUP_OPT, DRY_RUN_OPT, PRIORITY_OPT, DST_NODE_OPT, IALLOCATOR_OPT, ALLOW_FAILOVER_OPT, IGNORE_IPOLICY_OPT, IGNORE_HVVERSIONS_OPT, NORUNTIME_CHGS_OPT] + SUBMIT_OPTS, "[-f] ", "Migrate instance to its secondary node" " (only for mirrored instances)"), "move": ( MoveInstance, ARGS_ONE_INSTANCE, [FORCE_OPT] + SUBMIT_OPTS + [SINGLE_NODE_OPT, COMPRESS_OPT, SHUTDOWN_TIMEOUT_OPT, DRY_RUN_OPT, PRIORITY_OPT, IGNORE_CONSIST_OPT, IGNORE_IPOLICY_OPT], "[-f] ", "Move instance to an arbitrary node" " (only for instances of type file and lv)"), "info": ( ShowInstanceConfig, ARGS_MANY_INSTANCES, [STATIC_OPT, ALL_OPT, ROMAN_OPT, PRIORITY_OPT], "[-s] {--all | ...}", "Show information on the specified instance(s)"), "list": ( ListInstances, ARGS_MANY_INSTANCES, [NOHDR_OPT, SEP_OPT, USEUNITS_OPT, FIELDS_OPT, VERBOSE_OPT, FORCE_FILTER_OPT], "[...]", "Lists the instances and their status. The available fields can be shown" " using the \"list-fields\" command (see the man page for details)." " The default field list is (in order): %s." % utils.CommaJoin(_LIST_DEF_FIELDS), ), "list-fields": ( ListInstanceFields, [ArgUnknown()], [NOHDR_OPT, SEP_OPT], "[fields...]", "Lists all available fields for instances"), "reinstall": ( ReinstallInstance, [ArgInstance()], [FORCE_OPT, OS_OPT, FORCE_VARIANT_OPT, m_force_multi, m_node_opt, m_pri_node_opt, m_sec_node_opt, m_clust_opt, m_inst_opt, m_node_tags_opt, m_pri_node_tags_opt, m_sec_node_tags_opt, m_inst_tags_opt, SELECT_OS_OPT] + SUBMIT_OPTS + [DRY_RUN_OPT, PRIORITY_OPT, OSPARAMS_OPT, OSPARAMS_PRIVATE_OPT, OSPARAMS_SECRET_OPT], "[-f] ", "Reinstall a stopped instance"), "remove": ( RemoveInstance, ARGS_ONE_INSTANCE, [FORCE_OPT, SHUTDOWN_TIMEOUT_OPT, IGNORE_FAILURES_OPT] + SUBMIT_OPTS + [DRY_RUN_OPT, PRIORITY_OPT], "[-f] ", "Shuts down the instance and removes it"), "rename": ( RenameInstance, [ArgInstance(min=1, max=1), ArgHost(min=1, max=1)], [IPCHECK_OPT, NAMECHECK_OPT] + SUBMIT_OPTS + [DRY_RUN_OPT, PRIORITY_OPT], " ", "Rename the instance"), "replace-disks": ( ReplaceDisks, ARGS_ONE_INSTANCE, [AUTO_REPLACE_OPT, DISKIDX_OPT, IALLOCATOR_OPT, EARLY_RELEASE_OPT, NEW_SECONDARY_OPT, ON_PRIMARY_OPT, ON_SECONDARY_OPT] + SUBMIT_OPTS + [DRY_RUN_OPT, PRIORITY_OPT, IGNORE_IPOLICY_OPT], "[-s|-p|-a|-n NODE|-I NAME] ", "Replaces disks for the instance"), "modify": ( SetInstanceParams, ARGS_ONE_INSTANCE, [BACKEND_OPT, DISK_OPT, FORCE_OPT, HVOPTS_OPT, NET_OPT] + SUBMIT_OPTS + [DISK_TEMPLATE_OPT, SINGLE_NODE_OPT, IALLOCATOR_OPT, OS_OPT, FORCE_VARIANT_OPT, OSPARAMS_OPT, OSPARAMS_PRIVATE_OPT, DRY_RUN_OPT, PRIORITY_OPT, NWSYNC_OPT, OFFLINE_INST_OPT, ONLINE_INST_OPT, IGNORE_IPOLICY_OPT, RUNTIME_MEM_OPT, NOCONFLICTSCHECK_OPT, NEW_PRIMARY_OPT, NOHOTPLUG_OPT, INSTANCE_COMMUNICATION_OPT, EXT_PARAMS_OPT, FILESTORE_DRIVER_OPT, FILESTORE_DIR_OPT], "", "Alters the parameters of an instance"), "shutdown": ( GenericManyOps("shutdown", _ShutdownInstance), [ArgInstance()], [FORCE_OPT, m_node_opt, m_pri_node_opt, m_sec_node_opt, m_clust_opt, m_node_tags_opt, m_pri_node_tags_opt, m_sec_node_tags_opt, m_inst_tags_opt, m_inst_opt, m_force_multi, TIMEOUT_OPT] + SUBMIT_OPTS + [DRY_RUN_OPT, PRIORITY_OPT, IGNORE_OFFLINE_OPT, NO_REMEMBER_OPT], "...", "Stops one or more instances"), "startup": ( GenericManyOps("startup", _StartupInstance), [ArgInstance()], [FORCE_OPT, m_force_multi, m_node_opt, m_pri_node_opt, m_sec_node_opt, m_node_tags_opt, m_pri_node_tags_opt, m_sec_node_tags_opt, m_inst_tags_opt, m_clust_opt, m_inst_opt] + SUBMIT_OPTS + [HVOPTS_OPT, BACKEND_OPT, DRY_RUN_OPT, PRIORITY_OPT, IGNORE_OFFLINE_OPT, NO_REMEMBER_OPT, STARTUP_PAUSED_OPT], "...", "Starts one or more instances"), "reboot": ( GenericManyOps("reboot", _RebootInstance), [ArgInstance()], [m_force_multi, REBOOT_TYPE_OPT, IGNORE_SECONDARIES_OPT, m_node_opt, m_pri_node_opt, m_sec_node_opt, m_clust_opt, m_inst_opt] + SUBMIT_OPTS + [m_node_tags_opt, m_pri_node_tags_opt, m_sec_node_tags_opt, m_inst_tags_opt, SHUTDOWN_TIMEOUT_OPT, DRY_RUN_OPT, PRIORITY_OPT], "...", "Reboots one or more instances"), "activate-disks": ( ActivateDisks, ARGS_ONE_INSTANCE, SUBMIT_OPTS + [IGNORE_SIZE_OPT, PRIORITY_OPT, WFSYNC_OPT], "", "Activate an instance's disks"), "deactivate-disks": ( DeactivateDisks, ARGS_ONE_INSTANCE, [FORCE_OPT] + SUBMIT_OPTS + [DRY_RUN_OPT, PRIORITY_OPT], "[-f] ", "Deactivate an instance's disks"), "recreate-disks": ( RecreateDisks, ARGS_ONE_INSTANCE, SUBMIT_OPTS + [DISK_OPT, NODE_PLACEMENT_OPT, DRY_RUN_OPT, PRIORITY_OPT, IALLOCATOR_OPT], "", "Recreate an instance's disks"), "grow-disk": ( GrowDisk, [ArgInstance(min=1, max=1), ArgUnknown(min=1, max=1), ArgUnknown(min=1, max=1)], SUBMIT_OPTS + [NWSYNC_OPT, DRY_RUN_OPT, PRIORITY_OPT, ABSOLUTE_OPT, IGNORE_IPOLICY_OPT], " ", "Grow an instance's disk"), "change-group": ( ChangeGroup, ARGS_ONE_INSTANCE, [TO_GROUP_OPT, IALLOCATOR_OPT, EARLY_RELEASE_OPT, PRIORITY_OPT] + SUBMIT_OPTS, "[-I ] [--to ] ", "Change group of instance"), "list-tags": ( ListTags, ARGS_ONE_INSTANCE, [], "", "List the tags of the given instance"), "add-tags": ( AddTags, [ArgInstance(min=1, max=1), ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ...", "Add tags to the given instance"), "remove-tags": ( RemoveTags, [ArgInstance(min=1, max=1), ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ...", "Remove tags from given instance"), } #: dictionary with aliases for commands aliases = { "start": "startup", "stop": "shutdown", "show": "info", } def Main(): return GenericMain(commands, aliases=aliases, override={"tag_type": constants.TAG_INSTANCE}, env_override=_ENV_OVERRIDE) ganeti-3.1.0~rc2/lib/client/gnt_job.py000064400000000000000000000421401476477700300176250ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Job related commands""" # pylint: disable=W0401,W0613,W0614,C0103 # W0401: Wildcard import ganeti.cli # W0613: Unused argument, since all functions follow the same API # W0614: Unused import %s from wildcard import (since we need cli) # C0103: Invalid name gnt-job from ganeti.cli import * from ganeti import constants from ganeti import errors from ganeti import utils from ganeti import cli from ganeti import qlang #: default list of fields for L{ListJobs} _LIST_DEF_FIELDS = ["id", "status", "summary"] #: map converting the job status contants to user-visible #: names _USER_JOB_STATUS = { constants.JOB_STATUS_QUEUED: "queued", constants.JOB_STATUS_WAITING: "waiting", constants.JOB_STATUS_CANCELING: "canceling", constants.JOB_STATUS_RUNNING: "running", constants.JOB_STATUS_CANCELED: "canceled", constants.JOB_STATUS_SUCCESS: "success", constants.JOB_STATUS_ERROR: "error", } def _FormatStatus(value): """Formats a job status. """ try: return _USER_JOB_STATUS[value] except KeyError: raise errors.ProgrammerError("Unknown job status code '%s'" % value) def _FormatSummary(value): """Formats a job's summary. """ return ','.join(value) _JOB_LIST_FORMAT = { "status": (_FormatStatus, False), "summary": (_FormatSummary, False), } _JOB_LIST_FORMAT.update(dict.fromkeys(["opstart", "opexec", "opend"], (lambda value: [FormatTimestamp(v) for v in value], None))) def _ParseJobIds(args): """Parses a list of string job IDs into integers. @param args: list of strings @return: list of integers @raise OpPrereqError: in case of invalid values """ try: return [int(a) for a in args] except (ValueError, TypeError) as err: raise errors.OpPrereqError("Invalid job ID passed: %s" % err, errors.ECODE_INVAL) def ListJobs(opts, args): """List the jobs @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ selected_fields = ParseFields(opts.output, _LIST_DEF_FIELDS) if opts.archived and "archived" not in selected_fields: selected_fields.append("archived") qfilter = qlang.MakeSimpleFilter("status", opts.status_filter) cl = GetClient() return GenericList(constants.QR_JOB, selected_fields, args, None, opts.separator, not opts.no_headers, format_override=_JOB_LIST_FORMAT, verbose=opts.verbose, force_filter=opts.force_filter, namefield="id", qfilter=qfilter, isnumeric=True, cl=cl) def ListJobFields(opts, args): """List job fields. @param opts: the command line options selected by the user @type args: list @param args: fields to list, or empty for all @rtype: int @return: the desired exit code """ cl = GetClient() return GenericListFields(constants.QR_JOB, args, opts.separator, not opts.no_headers, cl=cl) def ArchiveJobs(opts, args): """Archive jobs. @param opts: the command line options selected by the user @type args: list @param args: should contain the job IDs to be archived @rtype: int @return: the desired exit code """ client = GetClient() rcode = 0 for job_id in args: if not client.ArchiveJob(job_id): ToStderr("Failed to archive job with ID '%s'", job_id) rcode = 1 return rcode def AutoArchiveJobs(opts, args): """Archive jobs based on age. This will archive jobs based on their age, or all jobs if a 'all' is passed. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the age as a time spec that can be parsed by L{ganeti.cli.ParseTimespec} or the keyword I{all}, which will cause all jobs to be archived @rtype: int @return: the desired exit code """ client = GetClient() age = args[0] if age == "all": age = -1 else: age = ParseTimespec(age) (archived_count, jobs_left) = client.AutoArchiveJobs(age) ToStdout("Archived %s jobs, %s unchecked left", archived_count, jobs_left) return 0 def _MultiJobAction(opts, args, cl, stdout_fn, ask_fn, question, action_fn): """Applies a function to multiple jobs. @param opts: Command line options @type args: list @param args: Job IDs @rtype: int @return: Exit code """ if cl is None: cl = GetClient() if stdout_fn is None: stdout_fn = ToStdout if ask_fn is None: ask_fn = AskUser result = constants.EXIT_SUCCESS if bool(args) ^ (opts.status_filter is None): raise errors.OpPrereqError("Either a status filter or job ID(s) must be" " specified and never both", errors.ECODE_INVAL) if opts.status_filter is not None: response = cl.Query(constants.QR_JOB, ["id", "status", "summary"], qlang.MakeSimpleFilter("status", opts.status_filter)) jobs = [i for ((_, i), _, _) in response.data] if not jobs: raise errors.OpPrereqError("No jobs with the requested status have been" " found", errors.ECODE_STATE) if not opts.force: (_, table) = FormatQueryResult(response, header=True, format_override=_JOB_LIST_FORMAT) for line in table: stdout_fn(line) if not ask_fn(question): return constants.EXIT_CONFIRMATION else: jobs = args for job_id in jobs: (success, msg) = action_fn(cl, job_id) if not success: result = constants.EXIT_FAILURE stdout_fn(msg) return result def CancelJobs(opts, args, cl=None, _stdout_fn=ToStdout, _ask_fn=AskUser): """Cancel not-yet-started jobs. @param opts: the command line options selected by the user @type args: list @param args: should contain the job IDs to be cancelled @rtype: int @return: the desired exit code """ if opts.kill: action_name = "KILL" if not opts.yes_do_it: raise errors.OpPrereqError("The --kill option must be confirmed" " with --yes-do-it", errors.ECODE_INVAL) else: action_name = "Cancel" return _MultiJobAction(opts, args, cl, _stdout_fn, _ask_fn, "%s job(s) listed above?" % action_name, lambda cl, job_id: cl.CancelJob(job_id, kill=opts.kill)) def ChangePriority(opts, args): """Change priority of jobs. @param opts: Command line options @type args: list @param args: Job IDs @rtype: int @return: Exit code """ if opts.priority is None: ToStderr("--priority option must be given.") return constants.EXIT_FAILURE return _MultiJobAction(opts, args, None, None, None, "Change priority of job(s) listed above?", lambda cl, job_id: cl.ChangeJobPriority(job_id, opts.priority)) def _ListOpcodeTimestamp(name, ts, container): """ Adds the opcode timestamp to the given container. """ if isinstance(ts, (tuple, list)): container.append((name, FormatTimestamp(ts), "opcode_timestamp")) else: container.append((name, "N/A", "opcode_timestamp")) def _CalcDelta(from_ts, to_ts): """ Calculates the delta between two timestamps. """ return to_ts[0] - from_ts[0] + (to_ts[1] - from_ts[1]) / 1000000.0 def _ListJobTimestamp(name, ts, container, prior_ts=None): """ Adds the job timestamp to the given container. @param prior_ts: The timestamp used to calculate the amount of time that passed since the given timestamp. """ if ts is not None: delta = "" if prior_ts is not None: delta = " (delta %.6fs)" % _CalcDelta(prior_ts, ts) output = "%s%s" % (FormatTimestamp(ts), delta) container.append((name, output, "job_timestamp")) else: container.append((name, "unknown (%s)" % str(ts), "job_timestamp")) def ShowJobs(opts, args): """Show detailed information about jobs. @param opts: the command line options selected by the user @type args: list @param args: should contain the job IDs to be queried @rtype: int @return: the desired exit code """ selected_fields = [ "id", "status", "ops", "opresult", "opstatus", "oplog", "opstart", "opexec", "opend", "received_ts", "start_ts", "end_ts", ] qfilter = qlang.MakeSimpleFilter("id", _ParseJobIds(args)) cl = GetClient() result = cl.Query(constants.QR_JOB, selected_fields, qfilter).data job_info_container = [] for entry in result: ((_, job_id), (rs_status, status), (_, ops), (_, opresult), (_, opstatus), (_, oplog), (_, opstart), (_, opexec), (_, opend), (_, recv_ts), (_, start_ts), (_, end_ts)) = entry # Detect non-normal results if rs_status != constants.RS_NORMAL: job_info_container.append("Job ID %s not found" % job_id) continue # Container for produced data job_info = [("Job ID", job_id)] if status in _USER_JOB_STATUS: status = _USER_JOB_STATUS[status] else: raise errors.ProgrammerError("Unknown job status code '%s'" % status) job_info.append(("Status", status)) _ListJobTimestamp("Received", recv_ts, job_info) _ListJobTimestamp("Processing start", start_ts, job_info, prior_ts=recv_ts) _ListJobTimestamp("Processing end", end_ts, job_info, prior_ts=start_ts) if end_ts is not None and recv_ts is not None: job_info.append(("Total processing time", "%.6f seconds" % _CalcDelta(recv_ts, end_ts))) else: job_info.append(("Total processing time", "N/A")) opcode_container = [] for (opcode, result, status, log, s_ts, x_ts, e_ts) in \ zip(ops, opresult, opstatus, oplog, opstart, opexec, opend): opcode_info = [] opcode_info.append(("Opcode", opcode["OP_ID"])) opcode_info.append(("Status", status)) _ListOpcodeTimestamp("Processing start", s_ts, opcode_info) _ListOpcodeTimestamp("Execution start", x_ts, opcode_info) _ListOpcodeTimestamp("Processing end", e_ts, opcode_info) opcode_info.append(("Input fields", opcode)) opcode_info.append(("Result", result)) exec_log_container = [] for serial, log_ts, log_type, log_msg in log: time_txt = FormatTimestamp(log_ts) encoded = FormatLogMessage(log_type, log_msg) # Arranged in this curious way to preserve the brevity for multiple # logs. This content cannot be exposed as a 4-tuple, as time contains # the colon, causing some YAML parsers to fail. exec_log_info = [ ("Time", time_txt), ("Content", (serial, log_type, encoded,)), ] exec_log_container.append(exec_log_info) opcode_info.append(("Execution log", exec_log_container)) opcode_container.append(opcode_info) job_info.append(("Opcodes", opcode_container)) job_info_container.append(job_info) PrintGenericInfo(job_info_container) return 0 def WatchJob(opts, args): """Follow a job and print its output as it arrives. @param opts: the command line options selected by the user @type args: list @param args: Contains the job ID @rtype: int @return: the desired exit code """ job_id = args[0] msg = ("Output from job %s follows" % job_id) ToStdout(msg) ToStdout("-" * len(msg)) retcode = 0 try: cli.PollJob(job_id) except errors.GenericError as err: (retcode, job_result) = cli.FormatError(err) ToStderr("Job %s failed: %s", job_id, job_result) return retcode def WaitJob(opts, args): """Wait for a job to finish, not producing any output. @param opts: the command line options selected by the user @type args: list @param args: Contains the job ID @rtype: int @return: the desired exit code """ job_id = args[0] retcode = 0 try: cli.PollJob(job_id, feedback_fn=lambda _: None) except errors.GenericError as err: (retcode, job_result) = cli.FormatError(err) ToStderr("Job %s failed: %s", job_id, job_result) return retcode _KILL_OPT = \ cli_option("--kill", default=False, action="store_true", dest="kill", help="Kill running jobs with SIGKILL") _YES_DOIT_OPT = cli_option("--yes-do-it", "--ya-rly", dest="yes_do_it", help="Really use --kill", action="store_true") _PENDING_OPT = \ cli_option("--pending", default=None, action="store_const", dest="status_filter", const=constants.JOBS_PENDING, help="Select jobs pending execution or being cancelled") _RUNNING_OPT = \ cli_option("--running", default=None, action="store_const", dest="status_filter", const=frozenset([ constants.JOB_STATUS_RUNNING, ]), help="Show jobs currently running only") _ERROR_OPT = \ cli_option("--error", default=None, action="store_const", dest="status_filter", const=frozenset([ constants.JOB_STATUS_ERROR, ]), help="Show failed jobs only") _FINISHED_OPT = \ cli_option("--finished", default=None, action="store_const", dest="status_filter", const=constants.JOBS_FINALIZED, help="Show finished jobs only") _ARCHIVED_OPT = \ cli_option("--archived", default=False, action="store_true", dest="archived", help="Include archived jobs in list (slow and expensive)") _QUEUED_OPT = \ cli_option("--queued", default=None, action="store_const", dest="status_filter", const=frozenset([ constants.JOB_STATUS_QUEUED, ]), help="Select queued jobs only") _WAITING_OPT = \ cli_option("--waiting", default=None, action="store_const", dest="status_filter", const=frozenset([ constants.JOB_STATUS_WAITING, ]), help="Select waiting jobs only") commands = { "list": ( ListJobs, [ArgJobId()], [NOHDR_OPT, SEP_OPT, FIELDS_OPT, VERBOSE_OPT, FORCE_FILTER_OPT, _PENDING_OPT, _RUNNING_OPT, _ERROR_OPT, _FINISHED_OPT, _ARCHIVED_OPT], "[...]", "Lists the jobs and their status. The available fields can be shown" " using the \"list-fields\" command (see the man page for details)." " The default field list is (in order): %s." % utils.CommaJoin(_LIST_DEF_FIELDS)), "list-fields": ( ListJobFields, [ArgUnknown()], [NOHDR_OPT, SEP_OPT], "[...]", "Lists all available fields for jobs"), "archive": ( ArchiveJobs, [ArgJobId(min=1)], [], " [...]", "Archive specified jobs"), "autoarchive": ( AutoArchiveJobs, [ArgSuggest(min=1, max=1, choices=["1d", "1w", "4w", "all"])], [], "", "Auto archive jobs older than the given age"), "cancel": ( CancelJobs, [ArgJobId()], [FORCE_OPT, _KILL_OPT, _PENDING_OPT, _QUEUED_OPT, _WAITING_OPT, _YES_DOIT_OPT], "{[--force] [--kill --yes-do-it] {--pending | --queued | --waiting} |" " [...]}", "Cancel jobs"), "info": ( ShowJobs, [ArgJobId(min=1)], [], " [...]", "Show detailed information about the specified jobs"), "wait": ( WaitJob, [ArgJobId(min=1, max=1)], [], "", "Wait for a job to finish"), "watch": ( WatchJob, [ArgJobId(min=1, max=1)], [], "", "Follows a job and prints its output as it arrives"), "change-priority": ( ChangePriority, [ArgJobId()], [PRIORITY_OPT, FORCE_OPT, _PENDING_OPT, _QUEUED_OPT, _WAITING_OPT], "--priority {[--force] {--pending | --queued | --waiting} |" " [...]}", "Change the priority of jobs"), } #: dictionary with aliases for commands aliases = { "show": "info", } def Main(): return GenericMain(commands, aliases=aliases) ganeti-3.1.0~rc2/lib/client/gnt_network.py000064400000000000000000000317511476477700300205520ustar00rootroot00000000000000# # # Copyright (C) 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """IP pool related commands""" # pylint: disable=W0401,W0614 # W0401: Wildcard import ganeti.cli # W0614: Unused import %s from wildcard import (since we need cli) import textwrap import itertools from ganeti.cli import * from ganeti import constants from ganeti import opcodes from ganeti import utils from ganeti import errors from ganeti import objects #: default list of fields for L{ListNetworks} _LIST_DEF_FIELDS = ["name", "network", "gateway", "mac_prefix", "group_list", "tags"] def _HandleReservedIPs(ips): if ips is None: return None elif not ips: return [] else: return utils.UnescapeAndSplit(ips, sep=",") def AddNetwork(opts, args): """Add a network to the cluster. @param opts: the command line options selected by the user @type args: list @param args: a list of length 1 with the network name to create @rtype: int @return: the desired exit code """ (network_name, ) = args if opts.network is None: raise errors.OpPrereqError("The --network option must be given", errors.ECODE_INVAL) if opts.tags is not None: tags = opts.tags.split(",") else: tags = [] reserved_ips = _HandleReservedIPs(opts.add_reserved_ips) op = opcodes.OpNetworkAdd(network_name=network_name, gateway=opts.gateway, network=opts.network, gateway6=opts.gateway6, network6=opts.network6, mac_prefix=opts.mac_prefix, add_reserved_ips=reserved_ips, conflicts_check=opts.conflicts_check, tags=tags) SubmitOrSend(op, opts) def _GetDefaultGroups(cl, groups): """Gets list of groups to operate on. If C{groups} doesn't contain groups, a list of all groups in the cluster is returned. @type cl: L{luxi.Client} @type groups: list @rtype: list """ if groups: return groups return list(itertools.chain(*cl.QueryGroups([], ["uuid"], False))) def ConnectNetwork(opts, args): """Map a network to a node group. @param opts: the command line options selected by the user @type args: list @param args: Network, mode, physlink and node groups @rtype: int @return: the desired exit code """ cl = GetClient() network = args[0] nicparams = objects.FillDict(constants.NICC_DEFAULTS, opts.nicparams) groups = _GetDefaultGroups(cl, args[1:]) # TODO: Change logic to support "--submit" for group in groups: op = opcodes.OpNetworkConnect(group_name=group, network_name=network, network_mode=nicparams[constants.NIC_MODE], network_link=nicparams[constants.NIC_LINK], network_vlan=nicparams[constants.NIC_VLAN], conflicts_check=opts.conflicts_check) SubmitOpCode(op, opts=opts, cl=cl) def DisconnectNetwork(opts, args): """Unmap a network from a node group. @param opts: the command line options selected by the user @type args: list @param args: Network and node groups @rtype: int @return: the desired exit code """ cl = GetClient() (network, ) = args[:1] groups = _GetDefaultGroups(cl, args[1:]) # TODO: Change logic to support "--submit" for group in groups: op = opcodes.OpNetworkDisconnect(group_name=group, network_name=network) SubmitOpCode(op, opts=opts, cl=cl) def ListNetworks(opts, args): """List Ip pools and their properties. @param opts: the command line options selected by the user @type args: list @param args: networks to list, or empty for all @rtype: int @return: the desired exit code """ desired_fields = ParseFields(opts.output, _LIST_DEF_FIELDS) fmtoverride = { "group_list": (lambda data: utils.CommaJoin("%s (%s, %s, %s)" % (name, mode, link, vlan) for (name, mode, link, vlan) in data), False), "inst_list": (",".join, False), "tags": (",".join, False), } cl = GetClient() return GenericList(constants.QR_NETWORK, desired_fields, args, None, opts.separator, not opts.no_headers, verbose=opts.verbose, format_override=fmtoverride, cl=cl) def ListNetworkFields(opts, args): """List network fields. @param opts: the command line options selected by the user @type args: list @param args: fields to list, or empty for all @rtype: int @return: the desired exit code """ cl = GetClient() return GenericListFields(constants.QR_NETWORK, args, opts.separator, not opts.no_headers, cl=cl) def ShowNetworkConfig(_, args): """Show network information. @type args: list @param args: should either be an empty list, in which case we show information about all nodes, or should contain a list of networks (names or UUIDs) to be queried for information @rtype: int @return: the desired exit code """ cl = GetClient() result = cl.QueryNetworks(fields=["name", "network", "gateway", "network6", "gateway6", "mac_prefix", "free_count", "reserved_count", "map", "group_list", "inst_list", "external_reservations", "serial_no", "uuid"], names=args, use_locking=False) for (name, network, gateway, network6, gateway6, mac_prefix, free_count, reserved_count, mapping, group_list, instances, ext_res, serial, uuid) in result: size = free_count + reserved_count ToStdout("Network name: %s", name) ToStdout("UUID: %s", uuid) ToStdout("Serial number: %d", serial) ToStdout(" Subnet: %s", network) ToStdout(" Gateway: %s", gateway) ToStdout(" IPv6 Subnet: %s", network6) ToStdout(" IPv6 Gateway: %s", gateway6) ToStdout(" Mac Prefix: %s", mac_prefix) ToStdout(" Size: %d", size) ToStdout(" Free: %d (%.2f%%)", free_count, 100 * float(free_count) / float(size)) ToStdout(" Usage map:") lenmapping = len(mapping) idx = 0 while idx < lenmapping: line = mapping[idx: idx + 64] ToStdout(" %s %s %d", str(idx).rjust(4), line.ljust(64), idx + 63) idx += 64 ToStdout(" (X) used (.) free") if ext_res: ToStdout(" externally reserved IPs:") for line in textwrap.wrap(ext_res, width=64): ToStdout(" %s" % line) if group_list: ToStdout(" connected to node groups:") for group, nic_mode, nic_link, nic_vlan in group_list: ToStdout(" %s (mode:%s link:%s vlan:%s)", group, nic_mode, nic_link, nic_vlan) else: ToStdout(" not connected to any node group") if instances: ToStdout(" used by %d instances:", len(instances)) for name in instances: (ips, networks) = cl.QueryInstances([name], ["nic.ips", "nic.networks"], use_locking=False)[0] l = lambda value: ", ".join(str(idx) + ":" + str(ip) for idx, (ip, net) in enumerate(value) if net == uuid) ToStdout(" %s: %s", name, l(zip(ips, networks))) else: ToStdout(" not used by any instances") def SetNetworkParams(opts, args): """Modifies an IP address pool's parameters. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the node group name @rtype: int @return: the desired exit code """ # TODO: add "network": opts.network, all_changes = { "gateway": opts.gateway, "add_reserved_ips": _HandleReservedIPs(opts.add_reserved_ips), "remove_reserved_ips": _HandleReservedIPs(opts.remove_reserved_ips), "mac_prefix": opts.mac_prefix, "gateway6": opts.gateway6, "network6": opts.network6, } if list(all_changes.values()).count(None) == len(all_changes): ToStderr("Please give at least one of the parameters.") return 1 op = opcodes.OpNetworkSetParams(network_name=args[0], **all_changes) # TODO: add feedback to user, e.g. list the modifications SubmitOrSend(op, opts) def RemoveNetwork(opts, args): """Remove an IP address pool from the cluster. @param opts: the command line options selected by the user @type args: list @param args: a list of length 1 with the id of the IP address pool to remove @rtype: int @return: the desired exit code """ (network_name,) = args op = opcodes.OpNetworkRemove(network_name=network_name, force=opts.force) SubmitOrSend(op, opts) def RenameNetwork(opts, args): """Rename a network. @param opts: the command line options selected by the user @type args: list @param args: a list of length 2, [old_name, new_name] @rtype: int @return: the desired exit code """ network_name, new_name = args op = opcodes.OpNetworkRename(network_name=network_name, new_name=new_name) SubmitOrSend(op, opts) commands = { "add": ( AddNetwork, ARGS_ONE_NETWORK, [DRY_RUN_OPT, NETWORK_OPT, GATEWAY_OPT, ADD_RESERVED_IPS_OPT, MAC_PREFIX_OPT, NETWORK6_OPT, GATEWAY6_OPT, NOCONFLICTSCHECK_OPT, TAG_ADD_OPT, PRIORITY_OPT] + SUBMIT_OPTS, "", "Add a new IP network to the cluster"), "list": ( ListNetworks, ARGS_MANY_NETWORKS, [NOHDR_OPT, SEP_OPT, FIELDS_OPT, VERBOSE_OPT], "[...]", "Lists the IP networks in the cluster. The available fields can be shown" " using the \"list-fields\" command (see the man page for details)." " The default list is (in order): %s." % utils.CommaJoin(_LIST_DEF_FIELDS)), "list-fields": ( ListNetworkFields, [ArgUnknown()], [NOHDR_OPT, SEP_OPT], "[...]", "Lists all available fields for networks"), "info": ( ShowNetworkConfig, ARGS_MANY_NETWORKS, [], "[...]", "Show information about the network(s)"), "modify": ( SetNetworkParams, ARGS_ONE_NETWORK, [DRY_RUN_OPT] + SUBMIT_OPTS + [ADD_RESERVED_IPS_OPT, REMOVE_RESERVED_IPS_OPT, GATEWAY_OPT, MAC_PREFIX_OPT, NETWORK6_OPT, GATEWAY6_OPT, PRIORITY_OPT], "", "Alters the parameters of a network"), "connect": ( ConnectNetwork, [ArgNetwork(min=1, max=1), ArgGroup()], [NOCONFLICTSCHECK_OPT, PRIORITY_OPT, NIC_PARAMS_OPT], " [...]", "Map a given network to the specified node group" " with given mode and link (netparams)"), "disconnect": ( DisconnectNetwork, [ArgNetwork(min=1, max=1), ArgGroup()], [PRIORITY_OPT], " [...]", "Unmap a given network from a specified node group"), "remove": ( RemoveNetwork, ARGS_ONE_NETWORK, [FORCE_OPT, DRY_RUN_OPT] + SUBMIT_OPTS + [PRIORITY_OPT], "[--dry-run] ", "Remove an (empty) network from the cluster"), "rename": ( RenameNetwork, [ArgGroup(min=2, max=2)], [DRY_RUN_OPT] + SUBMIT_OPTS + [PRIORITY_OPT], "[--dry-run] ", "Rename a network"), "list-tags": ( ListTags, ARGS_ONE_NETWORK, [], "", "List the tags of the given network"), "add-tags": ( AddTags, [ArgNetwork(min=1, max=1), ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ...", "Add tags to the given network"), "remove-tags": ( RemoveTags, [ArgNetwork(min=1, max=1), ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ...", "Remove tags from given network"), } def Main(): return GenericMain(commands, override={"tag_type": constants.TAG_NETWORK}) ganeti-3.1.0~rc2/lib/client/gnt_node.py000064400000000000000000001221111476477700300177750ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Node related commands""" # pylint: disable=W0401,W0613,W0614,C0103 # W0401: Wildcard import ganeti.cli # W0613: Unused argument, since all functions follow the same API # W0614: Unused import %s from wildcard import (since we need cli) # C0103: Invalid name gnt-node import itertools import errno from ganeti.cli import * from ganeti import cli from ganeti import bootstrap from ganeti import opcodes from ganeti import utils from ganeti import constants from ganeti import errors from ganeti import netutils from ganeti import pathutils from ganeti.rpc.node import RunWithRPC from ganeti import ssh from ganeti import compat from ganeti import confd from ganeti.confd import client as confd_client #: default list of field for L{ListNodes} _LIST_DEF_FIELDS = [ "name", "dtotal", "dfree", "mtotal", "mnode", "mfree", "pinst_cnt", "sinst_cnt", ] #: Default field list for L{ListVolumes} _LIST_VOL_DEF_FIELDS = ["node", "phys", "vg", "name", "size", "instance"] #: default list of field for L{ListStorage} _LIST_STOR_DEF_FIELDS = [ constants.SF_NODE, constants.SF_TYPE, constants.SF_NAME, constants.SF_SIZE, constants.SF_USED, constants.SF_FREE, constants.SF_ALLOCATABLE, ] #: default list of power commands _LIST_POWER_COMMANDS = ["on", "off", "cycle", "status"] #: headers (and full field list) for L{ListStorage} _LIST_STOR_HEADERS = { constants.SF_NODE: "Node", constants.SF_TYPE: "Type", constants.SF_NAME: "Name", constants.SF_SIZE: "Size", constants.SF_USED: "Used", constants.SF_FREE: "Free", constants.SF_ALLOCATABLE: "Allocatable", } #: User-facing storage unit types _USER_STORAGE_TYPE = { constants.ST_FILE: "file", constants.ST_LVM_PV: "lvm-pv", constants.ST_LVM_VG: "lvm-vg", constants.ST_SHARED_FILE: "sharedfile", constants.ST_GLUSTER: "gluster", } _STORAGE_TYPE_OPT = \ cli_option("-t", "--storage-type", dest="user_storage_type", choices=list(_USER_STORAGE_TYPE), default=None, metavar="STORAGE_TYPE", help=("Storage type (%s)" % utils.CommaJoin(list(_USER_STORAGE_TYPE)))) _REPAIRABLE_STORAGE_TYPES = \ [st for st, so in constants.VALID_STORAGE_OPERATIONS.items() if constants.SO_FIX_CONSISTENCY in so] _MODIFIABLE_STORAGE_TYPES = list(constants.MODIFIABLE_STORAGE_FIELDS) _OOB_COMMAND_ASK = compat.UniqueFrozenset([ constants.OOB_POWER_OFF, constants.OOB_POWER_CYCLE, ]) _ENV_OVERRIDE = compat.UniqueFrozenset(["list"]) NONODE_SETUP_OPT = cli_option("--no-node-setup", default=True, action="store_false", dest="node_setup", help=("Do not make initial SSH setup on remote" " node (needs to be done manually)")) IGNORE_STATUS_OPT = cli_option("--ignore-status", default=False, action="store_true", dest="ignore_status", help=("Ignore the Node(s) offline status" " (potentially DANGEROUS)")) def ConvertStorageType(user_storage_type): """Converts a user storage type to its internal name. """ try: return _USER_STORAGE_TYPE[user_storage_type] except KeyError: raise errors.OpPrereqError("Unknown storage type: %s" % user_storage_type, errors.ECODE_INVAL) def _TryReadFile(path): """Tries to read a file. If the file is not found, C{None} is returned. @type path: string @param path: Filename @rtype: None or string @todo: Consider adding a generic ENOENT wrapper """ try: return utils.ReadFile(path) except EnvironmentError as err: if err.errno == errno.ENOENT: return None else: raise def _ReadSshKeys(keyfiles, _tostderr_fn=ToStderr): """Reads the DSA SSH keys according to C{keyfiles}. @type keyfiles: dict @param keyfiles: Dictionary with keys of L{constants.SSHK_ALL} and two-values tuples (private and public key file) @rtype: list @return: List of three-values tuples (L{constants.SSHK_ALL}, private and public key as strings) """ result = [] for (kind, (private_file, public_file)) in keyfiles.items(): private_key = _TryReadFile(private_file) public_key = _TryReadFile(public_file) if public_key and private_key: result.append((kind, private_key, public_key)) elif public_key or private_key: _tostderr_fn("Couldn't find a complete set of keys for kind '%s';" " files '%s' and '%s'", kind, private_file, public_file) return result def _SetupSSH(options, cluster_name, node, ssh_port, cl): """Configures a destination node's SSH daemon. @param options: Command line options @type cluster_name @param cluster_name: Cluster name @type node: string @param node: Destination node name @type ssh_port: int @param ssh_port: Destination node ssh port @param cl: luxi client """ # Retrieve the list of master and master candidates candidate_filter = ["|", ["=", "role", "M"], ["=", "role", "C"]] result = cl.Query(constants.QR_NODE, ["uuid"], candidate_filter) if len(result.data) < 1: raise errors.OpPrereqError("No master or master candidate node is found.") candidates = [uuid for ((_, uuid),) in result.data] candidate_keys = ssh.QueryPubKeyFile(candidates) if options.force_join: ToStderr("The \"--force-join\" option is no longer supported and will be" " ignored.") host_keys = _ReadSshKeys(constants.SSH_DAEMON_KEYFILES) (_, root_keyfiles) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=False, dircheck=False) dsa_root_keyfiles = dict((kind, value) for (kind, value) in root_keyfiles.items() if kind == constants.SSHK_DSA) root_keys = _ReadSshKeys(dsa_root_keyfiles) (_, cert_pem) = \ utils.ExtractX509Certificate(utils.ReadFile(pathutils.NODED_CERT_FILE)) (ssh_key_type, ssh_key_bits) = \ cl.QueryConfigValues(["ssh_key_type", "ssh_key_bits"]) data = { constants.SSHS_CLUSTER_NAME: cluster_name, constants.SSHS_NODE_DAEMON_CERTIFICATE: cert_pem, constants.SSHS_SSH_HOST_KEY: host_keys, constants.SSHS_SSH_ROOT_KEY: root_keys, constants.SSHS_SSH_AUTHORIZED_KEYS: candidate_keys, constants.SSHS_SSH_KEY_TYPE: ssh_key_type, constants.SSHS_SSH_KEY_BITS: ssh_key_bits, } ssh.RunSshCmdWithStdin(cluster_name, node, pathutils.PREPARE_NODE_JOIN, ssh_port, data, debug=options.debug, verbose=options.verbose, use_cluster_key=False, ask_key=options.ssh_key_check, strict_host_check=options.ssh_key_check) (_, pub_keyfile) = root_keyfiles[ssh_key_type] pub_key = ssh.ReadRemoteSshPubKeys(pub_keyfile, node, cluster_name, ssh_port, options.ssh_key_check, options.ssh_key_check) # Unfortunately, we have to add the key with the node name rather than # the node's UUID here, because at this point, we do not have a UUID yet. # The entry will be corrected in noded later. ssh.AddPublicKey(node, pub_key) @RunWithRPC def AddNode(opts, args): """Add a node to the cluster. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the new node name @rtype: int @return: the desired exit code """ cl = GetClient() node = netutils.GetHostname(name=args[0]).name readd = opts.readd # Retrieve relevant parameters of the node group. ssh_port = None try: # Passing [] to QueryGroups means query the default group: node_groups = [opts.nodegroup] if opts.nodegroup is not None else [] output = cl.QueryGroups(names=node_groups, fields=["ndp/ssh_port"], use_locking=False) (ssh_port, ) = output[0] except (errors.OpPrereqError, errors.OpExecError): pass try: output = cl.QueryNodes(names=[node], fields=["name", "sip", "master", "ndp/ssh_port"], use_locking=False) if len(output) == 0: node_exists = "" sip = None else: node_exists, sip, is_master, ssh_port = output[0] except (errors.OpPrereqError, errors.OpExecError): node_exists = "" sip = None if readd: if not node_exists: ToStderr("Node %s not in the cluster" " - please retry without '--readd'", node) return 1 if is_master: ToStderr("Node %s is the master, cannot readd", node) return 1 else: if node_exists: ToStderr("Node %s already in the cluster (as %s)" " - please retry with '--readd'", node, node_exists) return 1 sip = opts.secondary_ip # read the cluster name from the master (cluster_name, ) = cl.QueryConfigValues(["cluster_name"]) if not opts.node_setup: ToStdout("-- WARNING -- \n" "The option --no-node-setup is disabled. Whether or not the\n" "SSH setup is manipulated while adding a node is determined\n" "by the 'modify_ssh_setup' value in the cluster-wide\n" "configuration instead.\n") (modify_ssh_setup, ) = \ cl.QueryConfigValues(["modify_ssh_setup"]) if modify_ssh_setup: ToStderr("-- WARNING -- \n" "Performing this operation is going to perform the following\n" "changes to the target machine (%s) and the current cluster\n" "nodes:\n" "* A new SSH daemon key pair is generated on the target machine.\n" "* The public SSH keys of all master candidates of the cluster\n" " are added to the target machine's 'authorized_keys' file.\n" "* In case the target machine is a master candidate, its newly\n" " generated public SSH key will be distributed to all other\n" " cluster nodes.\n", node) if modify_ssh_setup: _SetupSSH(opts, cluster_name, node, ssh_port, cl) bootstrap.SetupNodeDaemon(opts, cluster_name, node, ssh_port) if opts.disk_state: disk_state = utils.FlatToDict(opts.disk_state) else: disk_state = {} hv_state = dict(opts.hv_state) op = opcodes.OpNodeAdd(node_name=args[0], secondary_ip=sip, readd=opts.readd, group=opts.nodegroup, vm_capable=opts.vm_capable, ndparams=opts.ndparams, master_capable=opts.master_capable, disk_state=disk_state, hv_state=hv_state, node_setup=modify_ssh_setup) SubmitOpCode(op, opts=opts) def ListNodes(opts, args): """List nodes and their properties. @param opts: the command line options selected by the user @type args: list @param args: nodes to list, or empty for all @rtype: int @return: the desired exit code """ selected_fields = ParseFields(opts.output, _LIST_DEF_FIELDS) fmtoverride = dict.fromkeys(["pinst_list", "sinst_list", "tags"], (",".join, False)) cl = GetClient() return GenericList(constants.QR_NODE, selected_fields, args, opts.units, opts.separator, not opts.no_headers, format_override=fmtoverride, verbose=opts.verbose, force_filter=opts.force_filter, cl=cl) def ListNodeFields(opts, args): """List node fields. @param opts: the command line options selected by the user @type args: list @param args: fields to list, or empty for all @rtype: int @return: the desired exit code """ cl = GetClient() return GenericListFields(constants.QR_NODE, args, opts.separator, not opts.no_headers, cl=cl) def EvacuateNode(opts, args): """Relocate all secondary instance from a node. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ if opts.dst_node is not None: ToStderr("New secondary node given (disabling iallocator), hence evacuating" " secondary instances only.") opts.secondary_only = True opts.primary_only = False if opts.secondary_only and opts.primary_only: raise errors.OpPrereqError("Only one of the --primary-only and" " --secondary-only options can be passed", errors.ECODE_INVAL) elif opts.primary_only: mode = constants.NODE_EVAC_PRI elif opts.secondary_only: mode = constants.NODE_EVAC_SEC else: mode = constants.NODE_EVAC_ALL # Determine affected instances fields = [] if not opts.secondary_only: fields.append("pinst_list") if not opts.primary_only: fields.append("sinst_list") cl = GetClient() qcl = GetClient() result = qcl.QueryNodes(names=args, fields=fields, use_locking=False) qcl.Close() instances = set(itertools.chain(*itertools.chain(*itertools.chain(result)))) if not instances: # No instances to evacuate ToStderr("No instances to evacuate on node(s) %s, exiting.", utils.CommaJoin(args)) return constants.EXIT_SUCCESS if not (opts.force or AskUser("Relocate instance(s) %s from node(s) %s?" % (utils.CommaJoin(utils.NiceSort(instances)), utils.CommaJoin(args)))): return constants.EXIT_CONFIRMATION # Evacuate node op = opcodes.OpNodeEvacuate(node_name=args[0], mode=mode, remote_node=opts.dst_node, iallocator=opts.iallocator, early_release=opts.early_release, ignore_soft_errors=opts.ignore_soft_errors) result = SubmitOrSend(op, opts, cl=cl) # Keep track of submitted jobs jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result[constants.JOB_IDS_KEY]: jex.AddJobId(None, status, job_id) results = jex.GetResults() bad_cnt = len([row for row in results if not row[0]]) if bad_cnt == 0: ToStdout("All instances evacuated successfully.") rcode = constants.EXIT_SUCCESS else: ToStdout("There were %s errors during the evacuation.", bad_cnt) rcode = constants.EXIT_FAILURE return rcode def FailoverNode(opts, args): """Failover all primary instance on a node. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ cl = GetClient() force = opts.force selected_fields = ["name", "pinst_list"] # these fields are static data anyway, so it doesn't matter, but # locking=True should be safer qcl = GetClient() result = qcl.QueryNodes(names=args, fields=selected_fields, use_locking=False) qcl.Close() node, pinst = result[0] if not pinst: ToStderr("No primary instances on node %s, exiting.", node) return 0 pinst = utils.NiceSort(pinst) retcode = 0 if not force and not AskUser("Fail over instance(s) %s?" % (",".join("'%s'" % name for name in pinst))): return 2 jex = JobExecutor(cl=cl, opts=opts) for iname in pinst: op = opcodes.OpInstanceFailover(instance_name=iname, ignore_consistency=opts.ignore_consistency, iallocator=opts.iallocator) jex.QueueJob(iname, op) results = jex.GetResults() bad_cnt = len([row for row in results if not row[0]]) if bad_cnt == 0: ToStdout("All %d instance(s) failed over successfully.", len(results)) else: ToStdout("There were errors during the failover:\n" "%d error(s) out of %d instance(s).", bad_cnt, len(results)) return retcode def MigrateNode(opts, args): """Migrate all primary instance on a node. """ cl = GetClient() force = opts.force selected_fields = ["name", "pinst_list"] qcl = GetClient() result = qcl.QueryNodes(names=args, fields=selected_fields, use_locking=False) qcl.Close() ((node, pinst), ) = result if not pinst: ToStdout("No primary instances on node %s, exiting." % node) return 0 pinst = utils.NiceSort(pinst) if not (force or AskUser("Migrate instance(s) %s?" % utils.CommaJoin(utils.NiceSort(pinst)))): return constants.EXIT_CONFIRMATION # this should be removed once --non-live is deprecated if not opts.live and opts.migration_mode is not None: raise errors.OpPrereqError("Only one of the --non-live and " "--migration-mode options can be passed", errors.ECODE_INVAL) if not opts.live: # --non-live passed mode = constants.HT_MIGRATION_NONLIVE else: mode = opts.migration_mode op = opcodes.OpNodeMigrate(node_name=args[0], mode=mode, iallocator=opts.iallocator, target_node=opts.dst_node, allow_runtime_changes=opts.allow_runtime_chgs, ignore_ipolicy=opts.ignore_ipolicy) result = SubmitOrSend(op, opts, cl=cl) # Keep track of submitted jobs jex = JobExecutor(cl=cl, opts=opts) for (status, job_id) in result[constants.JOB_IDS_KEY]: jex.AddJobId(None, status, job_id) results = jex.GetResults() bad_cnt = len([row for row in results if not row[0]]) if bad_cnt == 0: ToStdout("All instances migrated successfully.") rcode = constants.EXIT_SUCCESS else: ToStdout("There were %s errors during the node migration.", bad_cnt) rcode = constants.EXIT_FAILURE return rcode def _FormatNodeInfo(node_info): """Format node information for L{cli.PrintGenericInfo()}. """ (name, primary_ip, secondary_ip, pinst, sinst, is_mc, drained, offline, master_capable, vm_capable, powered, ndparams, ndparams_custom) = node_info info = [ ("Node name", name), ("primary ip", primary_ip), ("secondary ip", secondary_ip), ("master candidate", is_mc), ("drained", drained), ("offline", offline), ] if powered is not None: info.append(("powered", powered)) info.extend([ ("master_capable", master_capable), ("vm_capable", vm_capable), ]) if vm_capable: info.extend([ ("primary for instances", [iname for iname in utils.NiceSort(pinst)]), ("secondary for instances", [iname for iname in utils.NiceSort(sinst)]), ]) info.append(("node parameters", FormatParamsDictInfo(ndparams_custom, ndparams))) return info def ShowNodeConfig(opts, args): """Show node information. @param opts: the command line options selected by the user @type args: list @param args: should either be an empty list, in which case we show information about all nodes, or should contain a list of nodes to be queried for information @rtype: int @return: the desired exit code """ cl = GetClient() result = cl.QueryNodes(fields=["name", "pip", "sip", "pinst_list", "sinst_list", "master_candidate", "drained", "offline", "master_capable", "vm_capable", "powered", "ndparams", "custom_ndparams"], names=args, use_locking=False) PrintGenericInfo([ _FormatNodeInfo(node_info) for node_info in result ]) return 0 def RemoveNode(opts, args): """Remove a node from the cluster. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the name of the node to be removed @rtype: int @return: the desired exit code """ op = opcodes.OpNodeRemove(node_name=args[0]) SubmitOpCode(op, opts=opts) return 0 def PowercycleNode(opts, args): """Remove a node from the cluster. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the name of the node to be removed @rtype: int @return: the desired exit code """ node = args[0] if (not opts.confirm and not AskUser("Are you sure you want to hard powercycle node %s?" % node)): return 2 op = opcodes.OpNodePowercycle(node_name=node, force=opts.force) result = SubmitOrSend(op, opts) if result: ToStderr(result) return 0 def PowerNode(opts, args): """Change/ask power state of a node. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the name of the node to be removed @rtype: int @return: the desired exit code """ command = args.pop(0) if opts.no_headers: headers = None else: headers = {"node": "Node", "status": "Status"} if command not in _LIST_POWER_COMMANDS: ToStderr("power subcommand %s not supported." % command) return constants.EXIT_FAILURE oob_command = "power-%s" % command if oob_command in _OOB_COMMAND_ASK: if not args: ToStderr("Please provide at least one node for this command") return constants.EXIT_FAILURE elif not opts.force and not ConfirmOperation(args, "nodes", "power %s" % command): return constants.EXIT_FAILURE assert len(args) > 0 opcodelist = [] if not opts.ignore_status and oob_command == constants.OOB_POWER_OFF: # TODO: This is a little ugly as we can't catch and revert for node in args: opcodelist.append(opcodes.OpNodeSetParams(node_name=node, offline=True, auto_promote=opts.auto_promote)) opcodelist.append(opcodes.OpOobCommand(node_names=args, command=oob_command, ignore_status=opts.ignore_status, timeout=opts.oob_timeout, power_delay=opts.power_delay)) cli.SetGenericOpcodeOpts(opcodelist, opts) job_id = cli.SendJob(opcodelist) # We just want the OOB Opcode status # If it fails PollJob gives us the error message in it result = cli.PollJob(job_id)[-1] errs = 0 data = [] for node_result in result: (node_tuple, data_tuple) = node_result (_, node_name) = node_tuple (data_status, data_node) = data_tuple if data_status == constants.RS_NORMAL: if oob_command == constants.OOB_POWER_STATUS: if data_node[constants.OOB_POWER_STATUS_POWERED]: text = "powered" else: text = "unpowered" data.append([node_name, text]) else: # We don't expect data here, so we just say, it was successfully invoked data.append([node_name, "invoked"]) else: errs += 1 data.append([node_name, cli.FormatResultError(data_status, True)]) data = GenerateTable(separator=opts.separator, headers=headers, fields=["node", "status"], data=data) for line in data: ToStdout(line) if errs: return constants.EXIT_FAILURE else: return constants.EXIT_SUCCESS def Health(opts, args): """Show health of a node using OOB. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the name of the node to be removed @rtype: int @return: the desired exit code """ op = opcodes.OpOobCommand(node_names=args, command=constants.OOB_HEALTH, timeout=opts.oob_timeout) result = SubmitOpCode(op, opts=opts) if opts.no_headers: headers = None else: headers = {"node": "Node", "status": "Status"} errs = 0 data = [] for node_result in result: (node_tuple, data_tuple) = node_result (_, node_name) = node_tuple (data_status, data_node) = data_tuple if data_status == constants.RS_NORMAL: data.append([node_name, "%s=%s" % tuple(data_node[0])]) for item, status in data_node[1:]: data.append(["", "%s=%s" % (item, status)]) else: errs += 1 data.append([node_name, cli.FormatResultError(data_status, True)]) data = GenerateTable(separator=opts.separator, headers=headers, fields=["node", "status"], data=data) for line in data: ToStdout(line) if errs: return constants.EXIT_FAILURE else: return constants.EXIT_SUCCESS def ListVolumes(opts, args): """List logical volumes on node(s). @param opts: the command line options selected by the user @type args: list @param args: should either be an empty list, in which case we list data for all nodes, or contain a list of nodes to display data only for those @rtype: int @return: the desired exit code """ selected_fields = ParseFields(opts.output, _LIST_VOL_DEF_FIELDS) op = opcodes.OpNodeQueryvols(nodes=args, output_fields=selected_fields) output = SubmitOpCode(op, opts=opts) if not opts.no_headers: headers = {"node": "Node", "phys": "PhysDev", "vg": "VG", "name": "Name", "size": "Size", "instance": "Instance"} else: headers = None unitfields = ["size"] numfields = ["size"] data = GenerateTable(separator=opts.separator, headers=headers, fields=selected_fields, unitfields=unitfields, numfields=numfields, data=output, units=opts.units) for line in data: ToStdout(line) return 0 def ListStorage(opts, args): """List physical volumes on node(s). @param opts: the command line options selected by the user @type args: list @param args: should either be an empty list, in which case we list data for all nodes, or contain a list of nodes to display data only for those @rtype: int @return: the desired exit code """ selected_fields = ParseFields(opts.output, _LIST_STOR_DEF_FIELDS) op = opcodes.OpNodeQueryStorage(nodes=args, storage_type=opts.user_storage_type, output_fields=selected_fields) output = SubmitOpCode(op, opts=opts) if not opts.no_headers: headers = { constants.SF_NODE: "Node", constants.SF_TYPE: "Type", constants.SF_NAME: "Name", constants.SF_SIZE: "Size", constants.SF_USED: "Used", constants.SF_FREE: "Free", constants.SF_ALLOCATABLE: "Allocatable", } else: headers = None unitfields = [constants.SF_SIZE, constants.SF_USED, constants.SF_FREE] numfields = [constants.SF_SIZE, constants.SF_USED, constants.SF_FREE] # change raw values to nicer strings for row in output: for idx, field in enumerate(selected_fields): val = row[idx] if field == constants.SF_ALLOCATABLE: if val: val = "Y" else: val = "N" row[idx] = str(val) data = GenerateTable(separator=opts.separator, headers=headers, fields=selected_fields, unitfields=unitfields, numfields=numfields, data=output, units=opts.units) for line in data: ToStdout(line) return 0 def ModifyStorage(opts, args): """Modify storage volume on a node. @param opts: the command line options selected by the user @type args: list @param args: should contain 3 items: node name, storage type and volume name @rtype: int @return: the desired exit code """ (node_name, user_storage_type, volume_name) = args storage_type = ConvertStorageType(user_storage_type) changes = {} if opts.allocatable is not None: changes[constants.SF_ALLOCATABLE] = opts.allocatable if changes: op = opcodes.OpNodeModifyStorage(node_name=node_name, storage_type=storage_type, name=volume_name, changes=changes) SubmitOrSend(op, opts) else: ToStderr("No changes to perform, exiting.") def RepairStorage(opts, args): """Repairs a storage volume on a node. @param opts: the command line options selected by the user @type args: list @param args: should contain 3 items: node name, storage type and volume name @rtype: int @return: the desired exit code """ (node_name, user_storage_type, volume_name) = args storage_type = ConvertStorageType(user_storage_type) op = opcodes.OpRepairNodeStorage(node_name=node_name, storage_type=storage_type, name=volume_name, ignore_consistency=opts.ignore_consistency) SubmitOrSend(op, opts) def SetNodeParams(opts, args): """Modifies a node. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the node name @rtype: int @return: the desired exit code """ all_changes = [opts.master_candidate, opts.drained, opts.offline, opts.master_capable, opts.vm_capable, opts.secondary_ip, opts.ndparams] if (all_changes.count(None) == len(all_changes) and not (opts.hv_state or opts.disk_state)): ToStderr("Please give at least one of the parameters.") return 1 if opts.disk_state: disk_state = utils.FlatToDict(opts.disk_state) else: disk_state = {} # Comparing explicitly to false to distinguish between a parameter # modification that doesn't set the node online (where the value will be None) # and modifying the node to bring it online. if opts.offline is False and not opts.force: usertext = ("You are setting this node online manually. If the" " configuration has changed, this can cause issues such as" " split brain. To safely bring a node back online, please use" " --readd instead. If you are confident that the configuration" " hasn't changed, continue?") if not AskUser(usertext): return 1 hv_state = dict(opts.hv_state) op = opcodes.OpNodeSetParams(node_name=args[0], master_candidate=opts.master_candidate, offline=opts.offline, drained=opts.drained, master_capable=opts.master_capable, vm_capable=opts.vm_capable, secondary_ip=opts.secondary_ip, force=opts.force, ndparams=opts.ndparams, auto_promote=opts.auto_promote, powered=opts.node_powered, hv_state=hv_state, disk_state=disk_state) # even if here we process the result, we allow submit only result = SubmitOrSend(op, opts) if result: ToStdout("Modified node %s", args[0]) for param, data in result: ToStdout(" - %-5s -> %s", param, data) return 0 def RestrictedCommand(opts, args): """Runs a remote command on node(s). @param opts: Command line options selected by user @type args: list @param args: Command line arguments @rtype: int @return: Exit code """ cl = GetClient() if len(args) > 1 or opts.nodegroup: # Expand node names nodes = GetOnlineNodes(nodes=args[1:], cl=cl, nodegroup=opts.nodegroup) else: raise errors.OpPrereqError("Node group or node names must be given", errors.ECODE_INVAL) op = opcodes.OpRestrictedCommand(command=args[0], nodes=nodes, use_locking=opts.do_locking) result = SubmitOrSend(op, opts, cl=cl) exit_code = constants.EXIT_SUCCESS for (node, (status, text)) in zip(nodes, result): ToStdout("------------------------------------------------") if status: if opts.show_machine_names: for line in text.splitlines(): ToStdout("%s: %s", node, line) else: ToStdout("Node: %s", node) ToStdout(text) else: exit_code = constants.EXIT_FAILURE ToStdout(text) return exit_code class ReplyStatus(object): """Class holding a reply status for synchronous confd clients. """ def __init__(self): self.failure = True self.answer = False def ListDrbd(opts, args): """Modifies a node. @param opts: the command line options selected by the user @type args: list @param args: should contain only one element, the node name @rtype: int @return: the desired exit code """ if len(args) != 1: ToStderr("Please give one (and only one) node.") return constants.EXIT_FAILURE status = ReplyStatus() def ListDrbdConfdCallback(reply): """Callback for confd queries""" if reply.type == confd_client.UPCALL_REPLY: answer = reply.server_reply.answer reqtype = reply.orig_request.type if reqtype == constants.CONFD_REQ_NODE_DRBD: if reply.server_reply.status != constants.CONFD_REPL_STATUS_OK: ToStderr("Query gave non-ok status '%s': %s" % (reply.server_reply.status, reply.server_reply.answer)) status.failure = True return if not confd.HTNodeDrbd(answer): ToStderr("Invalid response from server: expected %s, got %s", confd.HTNodeDrbd, answer) status.failure = True else: status.failure = False status.answer = answer else: ToStderr("Unexpected reply %s!?", reqtype) status.failure = True node = args[0] hmac = utils.ReadFile(pathutils.CONFD_HMAC_KEY) filter_callback = confd_client.ConfdFilterCallback(ListDrbdConfdCallback) counting_callback = confd_client.ConfdCountingCallback(filter_callback) cf_client = confd_client.ConfdClient(hmac, [constants.IP4_ADDRESS_LOCALHOST], counting_callback) req = confd_client.ConfdClientRequest(type=constants.CONFD_REQ_NODE_DRBD, query=node) def DoConfdRequestReply(req): counting_callback.RegisterQuery(req.rsalt) cf_client.SendRequest(req, async_=False) while not counting_callback.AllAnswered(): if not cf_client.ReceiveReply(): ToStderr("Did not receive all expected confd replies") break DoConfdRequestReply(req) if status.failure: return constants.EXIT_FAILURE fields = ["node", "minor", "instance", "disk", "role", "peer"] if opts.no_headers: headers = None else: headers = {"node": "Node", "minor": "Minor", "instance": "Instance", "disk": "Disk", "role": "Role", "peer": "PeerNode"} data = GenerateTable(separator=opts.separator, headers=headers, fields=fields, data=sorted(status.answer), numfields=["minor"]) for line in data: ToStdout(line) return constants.EXIT_SUCCESS commands = { "add": ( AddNode, [ArgHost(min=1, max=1)], [SECONDARY_IP_OPT, READD_OPT, NOSSH_KEYCHECK_OPT, NODE_FORCE_JOIN_OPT, NONODE_SETUP_OPT, VERBOSE_OPT, NODEGROUP_OPT, PRIORITY_OPT, CAPAB_MASTER_OPT, CAPAB_VM_OPT, NODE_PARAMS_OPT, HV_STATE_OPT, DISK_STATE_OPT], "[-s ip] [--readd] [--no-ssh-key-check] [--force-join]" " [--no-node-setup] [--verbose] [--network] ", "Add a node to the cluster"), "evacuate": ( EvacuateNode, ARGS_ONE_NODE, [FORCE_OPT, IALLOCATOR_OPT, IGNORE_SOFT_ERRORS_OPT, NEW_SECONDARY_OPT, EARLY_RELEASE_OPT, PRIORITY_OPT, PRIMARY_ONLY_OPT, SECONDARY_ONLY_OPT] + SUBMIT_OPTS, "[-f] {-I | -n } [-p | -s] [options...] ", "Relocate the primary and/or secondary instances from a node"), "failover": ( FailoverNode, ARGS_ONE_NODE, [FORCE_OPT, IGNORE_CONSIST_OPT, IALLOCATOR_OPT, PRIORITY_OPT], "[-f] ", "Stops the primary instances on a node and start them on their" " secondary node (only for instances with drbd disk template)"), "migrate": ( MigrateNode, ARGS_ONE_NODE, [FORCE_OPT, NONLIVE_OPT, MIGRATION_MODE_OPT, DST_NODE_OPT, IALLOCATOR_OPT, PRIORITY_OPT, IGNORE_IPOLICY_OPT, NORUNTIME_CHGS_OPT] + SUBMIT_OPTS, "[-f] ", "Migrate all the primary instance on a node away from it" " (only for instances of type drbd)"), "info": ( ShowNodeConfig, ARGS_MANY_NODES, [], "[...]", "Show information about the node(s)"), "list": ( ListNodes, ARGS_MANY_NODES, [NOHDR_OPT, SEP_OPT, USEUNITS_OPT, FIELDS_OPT, VERBOSE_OPT, FORCE_FILTER_OPT], "[...]", "Lists the nodes in the cluster. The available fields can be shown using" " the \"list-fields\" command (see the man page for details)." " The default field list is (in order): %s." % utils.CommaJoin(_LIST_DEF_FIELDS)), "list-fields": ( ListNodeFields, [ArgUnknown()], [NOHDR_OPT, SEP_OPT], "[fields...]", "Lists all available fields for nodes"), "modify": ( SetNodeParams, ARGS_ONE_NODE, [FORCE_OPT] + SUBMIT_OPTS + [MC_OPT, DRAINED_OPT, OFFLINE_OPT, CAPAB_MASTER_OPT, CAPAB_VM_OPT, SECONDARY_IP_OPT, AUTO_PROMOTE_OPT, DRY_RUN_OPT, PRIORITY_OPT, NODE_PARAMS_OPT, NODE_POWERED_OPT, HV_STATE_OPT, DISK_STATE_OPT], "", "Alters the parameters of a node"), "powercycle": ( PowercycleNode, ARGS_ONE_NODE, [FORCE_OPT, CONFIRM_OPT, DRY_RUN_OPT, PRIORITY_OPT] + SUBMIT_OPTS, "", "Tries to forcefully powercycle a node"), "power": ( PowerNode, [ArgChoice(min=1, max=1, choices=_LIST_POWER_COMMANDS), ArgNode()], SUBMIT_OPTS + [AUTO_PROMOTE_OPT, PRIORITY_OPT, IGNORE_STATUS_OPT, FORCE_OPT, NOHDR_OPT, SEP_OPT, OOB_TIMEOUT_OPT, POWER_DELAY_OPT], "on|off|cycle|status [...]", "Change power state of node by calling out-of-band helper."), "remove": ( RemoveNode, ARGS_ONE_NODE, [DRY_RUN_OPT, PRIORITY_OPT], "", "Removes a node from the cluster"), "volumes": ( ListVolumes, [ArgNode()], [NOHDR_OPT, SEP_OPT, USEUNITS_OPT, FIELDS_OPT, PRIORITY_OPT], "[...]", "List logical volumes on node(s)"), "list-storage": ( ListStorage, ARGS_MANY_NODES, [NOHDR_OPT, SEP_OPT, USEUNITS_OPT, FIELDS_OPT, _STORAGE_TYPE_OPT, PRIORITY_OPT], "[...]", "List physical volumes on node(s). The available" " fields are (see the man page for details): %s." % (utils.CommaJoin(_LIST_STOR_HEADERS))), "modify-storage": ( ModifyStorage, [ArgNode(min=1, max=1), ArgChoice(min=1, max=1, choices=_MODIFIABLE_STORAGE_TYPES), ArgFile(min=1, max=1)], [ALLOCATABLE_OPT, DRY_RUN_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ", "Modify storage volume on a node"), "repair-storage": ( RepairStorage, [ArgNode(min=1, max=1), ArgChoice(min=1, max=1, choices=_REPAIRABLE_STORAGE_TYPES), ArgFile(min=1, max=1)], [IGNORE_CONSIST_OPT, DRY_RUN_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ", "Repairs a storage volume on a node"), "list-tags": ( ListTags, ARGS_ONE_NODE, [], "", "List the tags of the given node"), "add-tags": ( AddTags, [ArgNode(min=1, max=1), ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ...", "Add tags to the given node"), "remove-tags": ( RemoveTags, [ArgNode(min=1, max=1), ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT] + SUBMIT_OPTS, " ...", "Remove tags from the given node"), "health": ( Health, ARGS_MANY_NODES, [NOHDR_OPT, SEP_OPT, PRIORITY_OPT, OOB_TIMEOUT_OPT], "[...]", "List health of node(s) using out-of-band"), "list-drbd": ( ListDrbd, ARGS_ONE_NODE, [NOHDR_OPT, SEP_OPT], "", "Query the list of used DRBD minors on the given node"), "restricted-command": ( RestrictedCommand, [ArgUnknown(min=1, max=1)] + ARGS_MANY_NODES, [SYNC_OPT, PRIORITY_OPT] + SUBMIT_OPTS + [SHOW_MACHINE_OPT, NODEGROUP_OPT], " ...", "Executes a restricted command on node(s)"), } #: dictionary with aliases for commands aliases = { "show": "info", } def Main(): return GenericMain(commands, aliases=aliases, override={"tag_type": constants.TAG_NODE}, env_override=_ENV_OVERRIDE) ganeti-3.1.0~rc2/lib/client/gnt_os.py000064400000000000000000000240301476477700300174720ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """OS scripts related commands""" # pylint: disable=W0401,W0613,W0614,C0103 # W0401: Wildcard import ganeti.cli # W0613: Unused argument, since all functions follow the same API # W0614: Unused import %s from wildcard import (since we need cli) # C0103: Invalid name gnt-os from ganeti.cli import * from ganeti import constants from ganeti import opcodes from ganeti import utils def ListOS(opts, args): """List the valid OSes in the cluster. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ op = opcodes.OpOsDiagnose(output_fields=["name", "variants"], names=[]) result = SubmitOpCode(op, opts=opts) if not opts.no_headers: headers = {"name": "Name"} else: headers = None os_names = [] for (name, variants) in result: os_names.extend([[n] for n in CalculateOSNames(name, variants)]) data = GenerateTable(separator=None, headers=headers, fields=["name"], data=os_names, units=None) for line in data: ToStdout(line) return 0 def ShowOSInfo(opts, args): """List detailed information about OSes in the cluster. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ op = opcodes.OpOsDiagnose(output_fields=["name", "valid", "variants", "parameters", "api_versions", "blacklisted", "hidden", "os_hvp", "osparams", "trusted"], names=[]) result = SubmitOpCode(op, opts=opts) if result is None: ToStderr("Can't get the OS list") return 1 do_filter = bool(args) total_os_hvp = {} total_osparams = {} for (name, valid, variants, parameters, api_versions, blk, hid, os_hvp, osparams, trusted) in result: total_os_hvp.update(os_hvp) total_osparams.update(osparams) if do_filter: if name not in args: continue else: args.remove(name) ToStdout("%s:", name) ToStdout(" - valid: %s", valid) ToStdout(" - hidden: %s", hid) ToStdout(" - blacklisted: %s", blk) if valid: ToStdout(" - API versions:") for version in sorted(api_versions): ToStdout(" - %s", version) ToStdout(" - variants:") for vname in variants: ToStdout(" - %s", vname) ToStdout(" - parameters:") for pname, pdesc in parameters: ToStdout(" - %s: %s", pname, pdesc) ToStdout(" - trusted: %s", trusted) ToStdout("") if args: all_names = set(total_os_hvp) | set(total_osparams) for name in args: if not name in all_names: ToStdout("%s: ", name) else: info = [ (name, [ ("OS-specific hypervisor parameters", total_os_hvp.get(name, {})), ("OS parameters", total_osparams.get(name, {})), ]), ] PrintGenericInfo(info) ToStdout("") return 0 def _OsStatus(status, diagnose): """Beautifier function for OS status. @type status: boolean @param status: is the OS valid @type diagnose: string @param diagnose: the error message for invalid OSes @rtype: string @return: a formatted status """ if status: return "valid" else: return "invalid - %s" % diagnose def DiagnoseOS(opts, args): """Analyse all OSes on this cluster. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ op = opcodes.OpOsDiagnose(output_fields=["name", "valid", "variants", "node_status", "hidden", "blacklisted"], names=[]) result = SubmitOpCode(op, opts=opts) if result is None: ToStderr("Can't get the OS list") return 1 has_bad = False for os_name, _, os_variants, node_data, hid, blk in result: nodes_valid = {} nodes_bad = {} nodes_hidden = {} for node_name, node_info in node_data.items(): nodes_hidden[node_name] = [] if node_info: # at least one entry in the per-node list (fo_path, fo_status, fo_msg, fo_variants, fo_params, fo_api, fo_trusted) = node_info.pop(0) fo_msg = "%s (path: %s)" % (_OsStatus(fo_status, fo_msg), fo_path) if fo_api: max_os_api = max(fo_api) fo_msg += " [API versions: %s]" % utils.CommaJoin(fo_api) else: max_os_api = 0 fo_msg += " [no API versions declared]" if max_os_api >= constants.OS_API_V15: if fo_variants: fo_msg += " [variants: %s]" % utils.CommaJoin(fo_variants) else: fo_msg += " [no variants]" if max_os_api >= constants.OS_API_V20: if fo_params: fo_msg += (" [parameters: %s]" % utils.CommaJoin([v[0] for v in fo_params])) else: fo_msg += " [no parameters]" if fo_trusted: fo_msg += " [trusted]" else: fo_msg += " [untrusted]" if fo_status: nodes_valid[node_name] = fo_msg else: nodes_bad[node_name] = fo_msg for hpath, hstatus, hmsg, _, _, _ in node_info: nodes_hidden[node_name].append(" [hidden] path: %s, status: %s" % (hpath, _OsStatus(hstatus, hmsg))) else: nodes_bad[node_name] = "OS not found" # TODO: Shouldn't the global status be calculated by the LU? if nodes_valid and not nodes_bad: status = "valid" elif not nodes_valid and nodes_bad: status = "invalid" has_bad = True else: status = "partial valid" has_bad = True st_msg = "OS: %s [global status: %s]" % (os_name, status) if hid: st_msg += " [hidden]" if blk: st_msg += " [blacklisted]" ToStdout(st_msg) if os_variants: ToStdout(" Variants: [%s]" % utils.CommaJoin(os_variants)) for msg_map in (nodes_valid, nodes_bad): map_k = utils.NiceSort(msg_map) for node_name in map_k: ToStdout(" Node: %s, status: %s", node_name, msg_map[node_name]) for msg in nodes_hidden[node_name]: ToStdout(msg) ToStdout("") return int(has_bad) def ModifyOS(opts, args): """Modify OS parameters for one OS. @param opts: the command line options selected by the user @type args: list @param args: should be a list with one entry @rtype: int @return: the desired exit code """ # We have to disable pylint for this assignment because of a Pylint bug: # Even though there is no `os` in scope, it claims # Redefining name 'os' from outer scope # It is supposed to come from `from ganeti.cli import *`, but that doesn't # export `os` since it has `__all__` set. os = args[0] # pylint: disable=W0621 if opts.hvparams: os_hvp = {os: dict(opts.hvparams)} else: os_hvp = None if opts.osparams: osp = {os: opts.osparams} else: osp = None if opts.osparams_private: osp_private = {os: opts.osparams_private} else: osp_private = None if opts.hidden is not None: if opts.hidden: ohid = [(constants.DDM_ADD, os)] else: ohid = [(constants.DDM_REMOVE, os)] else: ohid = None if opts.blacklisted is not None: if opts.blacklisted: oblk = [(constants.DDM_ADD, os)] else: oblk = [(constants.DDM_REMOVE, os)] else: oblk = None if not (os_hvp or osp or osp_private or ohid or oblk): ToStderr("At least one of OS parameters or hypervisor parameters" " must be passed") return 1 op = opcodes.OpClusterSetParams(os_hvp=os_hvp, osparams=osp, osparams_private_cluster=osp_private, hidden_os=ohid, blacklisted_os=oblk) SubmitOrSend(op, opts) return 0 commands = { "list": ( ListOS, ARGS_NONE, [NOHDR_OPT, PRIORITY_OPT], "", "Lists all valid operating systems on the cluster"), "diagnose": ( DiagnoseOS, ARGS_NONE, [PRIORITY_OPT], "", "Diagnose all operating systems"), "info": ( ShowOSInfo, [ArgOs()], [PRIORITY_OPT], "", "Show detailed information about " "operating systems"), "modify": ( ModifyOS, ARGS_ONE_OS, [HVLIST_OPT, OSPARAMS_OPT, OSPARAMS_PRIVATE_OPT, DRY_RUN_OPT, PRIORITY_OPT, HID_OS_OPT, BLK_OS_OPT] + SUBMIT_OPTS, "", "Modify the OS parameters"), } #: dictionary with aliases for commands aliases = { "show": "info", } def Main(): return GenericMain(commands, aliases=aliases) ganeti-3.1.0~rc2/lib/client/gnt_storage.py000064400000000000000000000141431476477700300205210ustar00rootroot00000000000000# # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """External Storage related commands""" # pylint: disable=W0401,W0613,W0614,C0103 # W0401: Wildcard import ganeti.cli # W0613: Unused argument, since all functions follow the same API # W0614: Unused import %s from wildcard import (since we need cli) # C0103: Invalid name gnt-storage from ganeti.cli import * from ganeti import opcodes from ganeti import utils def ShowExtStorageInfo(opts, args): """List detailed information about ExtStorage providers. @param opts: the command line options selected by the user @type args: list @param args: empty list or list of ExtStorage providers' names @rtype: int @return: the desired exit code """ op = opcodes.OpExtStorageDiagnose(output_fields=["name", "nodegroup_status", "parameters"], names=[]) result = SubmitOpCode(op, opts=opts) if not result: ToStderr("Can't get the ExtStorage providers list") return 1 do_filter = bool(args) for (name, nodegroup_data, parameters) in result: if do_filter: if name not in args: continue else: args.remove(name) nodegroups_valid = [] for nodegroup_name, nodegroup_status in nodegroup_data.items(): if nodegroup_status: nodegroups_valid.append(nodegroup_name) ToStdout("%s:", name) if nodegroups_valid != []: ToStdout(" - Valid for nodegroups:") for ndgrp in utils.NiceSort(nodegroups_valid): ToStdout(" %s", ndgrp) ToStdout(" - Supported parameters:") for pname, pdesc in parameters: ToStdout(" %s: %s", pname, pdesc) else: ToStdout(" - Invalid for all nodegroups") ToStdout("") if args: for name in args: ToStdout("%s: Not Found", name) ToStdout("") return 0 def _ExtStorageStatus(status, diagnose): """Beautifier function for ExtStorage status. @type status: boolean @param status: is the ExtStorage provider valid @type diagnose: string @param diagnose: the error message for invalid ExtStorages @rtype: string @return: a formatted status """ if status: return "valid" else: return "invalid - %s" % diagnose def DiagnoseExtStorage(opts, args): """Analyse all ExtStorage providers. @param opts: the command line options selected by the user @type args: list @param args: should be an empty list @rtype: int @return: the desired exit code """ op = opcodes.OpExtStorageDiagnose(output_fields=["name", "node_status", "nodegroup_status"], names=[]) result = SubmitOpCode(op, opts=opts) if not result: ToStderr("Can't get the list of ExtStorage providers") return 1 for provider_name, node_data, nodegroup_data in result: nodes_valid = {} nodes_bad = {} nodegroups_valid = {} nodegroups_bad = {} # Per node diagnose for node_name, node_info in node_data.items(): if node_info: # at least one entry in the per-node list (fo_path, fo_status, fo_msg, fo_params) = node_info.pop(0) fo_msg = "%s (path: %s)" % (_ExtStorageStatus(fo_status, fo_msg), fo_path) if fo_params: fo_msg += (" [parameters: %s]" % utils.CommaJoin([v[0] for v in fo_params])) else: fo_msg += " [no parameters]" if fo_status: nodes_valid[node_name] = fo_msg else: nodes_bad[node_name] = fo_msg else: nodes_bad[node_name] = "ExtStorage provider not found" # Per nodegroup diagnose for nodegroup_name, nodegroup_status in nodegroup_data.items(): status = nodegroup_status if status: nodegroups_valid[nodegroup_name] = "valid" else: nodegroups_bad[nodegroup_name] = "invalid" def _OutputPerNodegroupStatus(msg_map): map_k = utils.NiceSort(msg_map) for nodegroup in map_k: ToStdout(" For nodegroup: %s --> %s", nodegroup, msg_map[nodegroup]) def _OutputPerNodeStatus(msg_map): map_k = utils.NiceSort(msg_map) for node_name in map_k: ToStdout(" Node: %s, status: %s", node_name, msg_map[node_name]) # Print the output st_msg = "Provider: %s" % provider_name ToStdout(st_msg) ToStdout("---") _OutputPerNodeStatus(nodes_valid) _OutputPerNodeStatus(nodes_bad) ToStdout(" --") _OutputPerNodegroupStatus(nodegroups_valid) _OutputPerNodegroupStatus(nodegroups_bad) ToStdout("") return 0 commands = { "diagnose": ( DiagnoseExtStorage, ARGS_NONE, [PRIORITY_OPT], "", "Diagnose all ExtStorage providers"), "info": ( ShowExtStorageInfo, [ArgOs()], [PRIORITY_OPT], "", "Show info about ExtStorage providers"), } def Main(): return GenericMain(commands) ganeti-3.1.0~rc2/lib/cmdlib/000075500000000000000000000000001476477700300156045ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/cmdlib/__init__.py000064400000000000000000000075601476477700300177250ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing the master-side code. This file only imports all LU's (and other classes) in order to re-export them to clients of cmdlib. """ from ganeti.cmdlib.base import \ LogicalUnit, \ NoHooksLU, \ ResultWithJobs from ganeti.cmdlib.cluster import \ LUClusterActivateMasterIp, \ LUClusterDeactivateMasterIp, \ LUClusterConfigQuery, \ LUClusterDestroy, \ LUClusterPostInit, \ LUClusterQuery, \ LUClusterRedistConf, \ LUClusterRename, \ LUClusterRepairDiskSizes, \ LUClusterSetParams, \ LUClusterRenewCrypto from ganeti.cmdlib.cluster.verify import \ LUClusterVerify, \ LUClusterVerifyConfig, \ LUClusterVerifyGroup, \ LUClusterVerifyDisks from ganeti.cmdlib.group import \ LUGroupAdd, \ LUGroupAssignNodes, \ LUGroupSetParams, \ LUGroupRemove, \ LUGroupRename, \ LUGroupEvacuate, \ LUGroupVerifyDisks from ganeti.cmdlib.node import \ LUNodeAdd, \ LUNodeSetParams, \ LUNodePowercycle, \ LUNodeEvacuate, \ LUNodeMigrate, \ LUNodeModifyStorage, \ LUNodeQueryvols, \ LUNodeQueryStorage, \ LUNodeRemove, \ LURepairNodeStorage from ganeti.cmdlib.instance import \ LUInstanceRename, \ LUInstanceRemove, \ LUInstanceMove, \ LUInstanceMultiAlloc, \ LUInstanceChangeGroup from ganeti.cmdlib.instance_create import \ LUInstanceCreate from ganeti.cmdlib.instance_storage import \ LUInstanceRecreateDisks, \ LUInstanceGrowDisk, \ LUInstanceReplaceDisks, \ LUInstanceActivateDisks, \ LUInstanceDeactivateDisks from ganeti.cmdlib.instance_migration import \ LUInstanceFailover, \ LUInstanceMigrate from ganeti.cmdlib.instance_operation import \ LUInstanceStartup, \ LUInstanceShutdown, \ LUInstanceReinstall, \ LUInstanceReboot, \ LUInstanceConsole from ganeti.cmdlib.instance_set_params import \ LUInstanceSetParams from ganeti.cmdlib.instance_query import \ LUInstanceQueryData from ganeti.cmdlib.backup import \ LUBackupPrepare, \ LUBackupExport, \ LUBackupRemove from ganeti.cmdlib.query import \ LUQuery, \ LUQueryFields from ganeti.cmdlib.operating_system import \ LUOsDiagnose from ganeti.cmdlib.tags import \ LUTagsGet, \ LUTagsSearch, \ LUTagsSet, \ LUTagsDel from ganeti.cmdlib.network import \ LUNetworkAdd, \ LUNetworkRemove, \ LUNetworkRename, \ LUNetworkSetParams, \ LUNetworkConnect, \ LUNetworkDisconnect from ganeti.cmdlib.misc import \ LUOobCommand, \ LUExtStorageDiagnose, \ LURestrictedCommand from ganeti.cmdlib.test import \ LUTestOsParams, \ LUTestDelay, \ LUTestJqueue, \ LUTestAllocator ganeti-3.1.0~rc2/lib/cmdlib/backup.py000064400000000000000000000530111476477700300174230ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with backup operations.""" import logging import OpenSSL from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import masterd from ganeti import utils from ganeti.cmdlib.base import NoHooksLU, LogicalUnit from ganeti.cmdlib.common import CheckNodeOnline, ExpandNodeUuidAndName from ganeti.cmdlib.instance_helpervm import RunWithHelperVM from ganeti.cmdlib.instance_storage import StartInstanceDisks, \ ShutdownInstanceDisks from ganeti.cmdlib.instance_utils import GetClusterDomainSecret, \ BuildInstanceHookEnvByObject, CheckNodeNotDrained, RemoveInstance, \ CheckCompressionTool class LUBackupPrepare(NoHooksLU): """Prepares an instance for an export and returns useful information. """ REQ_BGL = False def ExpandNames(self): self._ExpandAndLockInstance() def CheckPrereq(self): """Check prerequisites. """ self.instance = self.cfg.GetInstanceInfoByName(self.op.instance_name) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name CheckNodeOnline(self, self.instance.primary_node) self._cds = GetClusterDomainSecret() def Exec(self, feedback_fn): """Prepares an instance for an export. """ if self.op.mode == constants.EXPORT_MODE_REMOTE: salt = utils.GenerateSecret(8) feedback_fn("Generating X509 certificate on %s" % self.cfg.GetNodeName(self.instance.primary_node)) result = self.rpc.call_x509_cert_create(self.instance.primary_node, constants.RIE_CERT_VALIDITY) result.Raise("Can't create X509 key and certificate on %s" % self.cfg.GetNodeName(result.node)) (name, cert_pem) = result.payload cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_pem) return { "handshake": masterd.instance.ComputeRemoteExportHandshake(self._cds), "x509_key_name": (name, utils.Sha1Hmac(self._cds, name, salt=salt), salt), "x509_ca": utils.SignX509Certificate(cert, self._cds, salt), } return None class LUBackupExport(LogicalUnit): """Export an instance to an image in the cluster. """ HPATH = "instance-export" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def CheckArguments(self): """Check the arguments. """ self.x509_key_name = self.op.x509_key_name self.dest_x509_ca_pem = self.op.destination_x509_ca if self.op.mode == constants.EXPORT_MODE_REMOTE: if not self.x509_key_name: raise errors.OpPrereqError("Missing X509 key name for encryption", errors.ECODE_INVAL) if not self.dest_x509_ca_pem: raise errors.OpPrereqError("Missing destination X509 CA", errors.ECODE_INVAL) if self.op.zero_free_space and not self.op.compress: raise errors.OpPrereqError("Zeroing free space does not make sense " "unless compression is used") if self.op.zero_free_space and not self.op.shutdown: raise errors.OpPrereqError("Unless the instance is shut down, zeroing " "cannot be used.") def ExpandNames(self): self._ExpandAndLockInstance() # In case we are zeroing, a node lock is required as we will be creating and # destroying a disk - allocations should be stopped, but not on the entire # cluster if self.op.zero_free_space: self.recalculate_locks = {locking.LEVEL_NODE: constants.LOCKS_REPLACE} self._LockInstancesNodes(primary_only=True) # Lock all nodes for local exports if self.op.mode == constants.EXPORT_MODE_LOCAL: (self.op.target_node_uuid, self.op.target_node) = \ ExpandNodeUuidAndName(self.cfg, self.op.target_node_uuid, self.op.target_node) # FIXME: lock only instance primary and destination node # # Sad but true, for now we have do lock all nodes, as we don't know where # the previous export might be, and in this LU we search for it and # remove it from its current node. In the future we could fix this by: # - making a tasklet to search (share-lock all), then create the # new one, then one to remove, after # - removing the removal operation altogether self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET def DeclareLocks(self, level): """Last minute lock declaration.""" # All nodes are locked anyway, so nothing to do here. def BuildHooksEnv(self): """Build hooks env. This will run on the master, primary node and target node. """ env = { "EXPORT_MODE": self.op.mode, "EXPORT_NODE": self.op.target_node, "EXPORT_DO_SHUTDOWN": self.op.shutdown, "SHUTDOWN_TIMEOUT": self.op.shutdown_timeout, # TODO: Generic function for boolean env variables "REMOVE_INSTANCE": str(bool(self.op.remove_instance)), } env.update(BuildInstanceHookEnvByObject( self, self.instance, secondary_nodes=self.secondary_nodes, disks=self.inst_disks)) return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode(), self.instance.primary_node] if self.op.mode == constants.EXPORT_MODE_LOCAL: nl.append(self.op.target_node_uuid) return (nl, nl) def CheckPrereq(self): """Check prerequisites. This checks that the instance and node names are valid. """ self.instance = self.cfg.GetInstanceInfoByName(self.op.instance_name) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name CheckNodeOnline(self, self.instance.primary_node) if (self.op.remove_instance and self.instance.admin_state == constants.ADMINST_UP and not self.op.shutdown): raise errors.OpPrereqError("Can not remove instance without shutting it" " down before", errors.ECODE_STATE) if self.op.mode == constants.EXPORT_MODE_LOCAL: self.dst_node = self.cfg.GetNodeInfo(self.op.target_node_uuid) assert self.dst_node is not None CheckNodeOnline(self, self.dst_node.uuid) CheckNodeNotDrained(self, self.dst_node.uuid) self._cds = None self.dest_disk_info = None self.dest_x509_ca = None elif self.op.mode == constants.EXPORT_MODE_REMOTE: self.dst_node = None if len(self.op.target_node) != len(self.instance.disks): raise errors.OpPrereqError(("Received destination information for %s" " disks, but instance %s has %s disks") % (len(self.op.target_node), self.op.instance_name, len(self.instance.disks)), errors.ECODE_INVAL) cds = GetClusterDomainSecret() # Check X509 key name try: (key_name, hmac_digest, hmac_salt) = self.x509_key_name except (TypeError, ValueError) as err: raise errors.OpPrereqError("Invalid data for X509 key name: %s" % err, errors.ECODE_INVAL) if not utils.VerifySha1Hmac(cds, key_name, hmac_digest, salt=hmac_salt): raise errors.OpPrereqError("HMAC for X509 key name is wrong", errors.ECODE_INVAL) # Load and verify CA try: (cert, _) = utils.LoadSignedX509Certificate(self.dest_x509_ca_pem, cds) except OpenSSL.crypto.Error as err: raise errors.OpPrereqError("Unable to load destination X509 CA (%s)" % (err, ), errors.ECODE_INVAL) (errcode, msg) = utils.VerifyX509Certificate(cert, None, None) if errcode is not None: raise errors.OpPrereqError("Invalid destination X509 CA (%s)" % (msg, ), errors.ECODE_INVAL) self.dest_x509_ca = cert # Verify target information disk_info = [] for idx, disk_data in enumerate(self.op.target_node): try: (host, port, magic) = \ masterd.instance.CheckRemoteExportDiskInfo(cds, idx, disk_data) except errors.GenericError as err: raise errors.OpPrereqError("Target info for disk %s: %s" % (idx, err), errors.ECODE_INVAL) disk_info.append((host, port, magic)) assert len(disk_info) == len(self.op.target_node) self.dest_disk_info = disk_info else: raise errors.ProgrammerError("Unhandled export mode %r" % self.op.mode) # Check prerequisites for zeroing if self.op.zero_free_space: # Check that user shutdown detection has been enabled hvparams = self.cfg.GetClusterInfo().FillHV(self.instance) if self.instance.hypervisor == constants.HT_KVM and \ not hvparams.get(constants.HV_KVM_USER_SHUTDOWN, False): raise errors.OpPrereqError("Instance shutdown detection must be " "enabled for zeroing to work", errors.ECODE_INVAL) # Check that the instance is set to boot from the disk if constants.HV_BOOT_ORDER in hvparams and \ hvparams[constants.HV_BOOT_ORDER] != constants.HT_BO_DISK: raise errors.OpPrereqError("Booting from disk must be set for zeroing " "to work", errors.ECODE_INVAL) # Check that the zeroing image is set if not self.cfg.GetZeroingImage(): raise errors.OpPrereqError("A zeroing image must be set for zeroing to" " work", errors.ECODE_INVAL) if self.op.zeroing_timeout_fixed is None: self.op.zeroing_timeout_fixed = constants.HELPER_VM_STARTUP if self.op.zeroing_timeout_per_mib is None: self.op.zeroing_timeout_per_mib = constants.ZEROING_TIMEOUT_PER_MIB else: if (self.op.zeroing_timeout_fixed is not None or self.op.zeroing_timeout_per_mib is not None): raise errors.OpPrereqError("Zeroing timeout options can only be used" " only with the --zero-free-space option", errors.ECODE_INVAL) if self.op.long_sleep and not self.op.shutdown: raise errors.OpPrereqError("The long sleep option only makes sense when" " the instance can be shut down.", errors.ECODE_INVAL) self.secondary_nodes = \ self.cfg.GetInstanceSecondaryNodes(self.instance.uuid) self.inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) # Check if the compression tool is whitelisted CheckCompressionTool(self, self.op.compress) def _CleanupExports(self, feedback_fn): """Removes exports of current instance from all other nodes. If an instance in a cluster with nodes A..D was exported to node C, its exports will be removed from the nodes A, B and D. """ assert self.op.mode != constants.EXPORT_MODE_REMOTE node_uuids = self.cfg.GetNodeList() node_uuids.remove(self.dst_node.uuid) # on one-node clusters nodelist will be empty after the removal # if we proceed the backup would be removed because OpBackupQuery # substitutes an empty list with the full cluster node list. iname = self.instance.name if node_uuids: feedback_fn("Removing old exports for instance %s" % iname) exportlist = self.rpc.call_export_list(node_uuids) for node_uuid in exportlist: if exportlist[node_uuid].fail_msg: continue if iname in exportlist[node_uuid].payload: msg = self.rpc.call_export_remove(node_uuid, iname).fail_msg if msg: self.LogWarning("Could not remove older export for instance %s" " on node %s: %s", iname, self.cfg.GetNodeName(node_uuid), msg) def _InstanceDiskSizeSum(self): """Calculates the size of all the disks of the instance used in this LU. @rtype: int @return: Size of the disks in MiB """ inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) return sum([d.size for d in inst_disks]) def ZeroFreeSpace(self, feedback_fn): """Zeroes the free space on a shutdown instance. @type feedback_fn: function @param feedback_fn: Function used to log progress """ assert self.op.zeroing_timeout_fixed is not None assert self.op.zeroing_timeout_per_mib is not None zeroing_image = self.cfg.GetZeroingImage() # Calculate the sum prior to adding the temporary disk instance_disks_size_sum = self._InstanceDiskSizeSum() timeout = self.op.zeroing_timeout_fixed + \ self.op.zeroing_timeout_per_mib * instance_disks_size_sum RunWithHelperVM(self, self.instance, zeroing_image, self.op.shutdown_timeout, timeout, log_prefix="Zeroing free disk space", feedback_fn=feedback_fn) def StartInstance(self, feedback_fn, src_node_uuid): """Send the node instructions to start the instance. @raise errors.OpExecError: If the instance didn't start up. """ assert self.instance.disks_active feedback_fn("Starting instance %s" % self.instance.name) result = self.rpc.call_instance_start(src_node_uuid, (self.instance, None, None), False, self.op.reason) msg = result.fail_msg if msg: feedback_fn("Failed to start instance: %s" % msg) ShutdownInstanceDisks(self, self.instance) raise errors.OpExecError("Could not start instance: %s" % msg) def TrySnapshot(self): """Returns true if there is a reason to prefer a snapshot.""" return (not self.op.remove_instance and self.instance.admin_state == constants.ADMINST_UP) def DoReboot(self): """Returns true iff the instance needs to be started after transfer.""" return (self.op.shutdown and self.instance.admin_state == constants.ADMINST_UP and not self.op.remove_instance) def Exec(self, feedback_fn): """Export an instance to an image in the cluster. """ assert self.op.mode in constants.EXPORT_MODES src_node_uuid = self.instance.primary_node if self.op.shutdown: # shutdown the instance, but not the disks feedback_fn("Shutting down instance %s" % self.instance.name) result = self.rpc.call_instance_shutdown(src_node_uuid, self.instance, self.op.shutdown_timeout, self.op.reason) # TODO: Maybe ignore failures if ignore_remove_failures is set result.Raise("Could not shutdown instance %s on" " node %s" % (self.instance.name, self.cfg.GetNodeName(src_node_uuid))) if self.op.zero_free_space: self.ZeroFreeSpace(feedback_fn) activate_disks = not self.instance.disks_active if activate_disks: # Activate the instance disks if we're exporting a stopped instance feedback_fn("Activating disks for %s" % self.instance.name) StartInstanceDisks(self, self.instance, None) self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) try: helper = masterd.instance.ExportInstanceHelper(self, feedback_fn, self.instance) snapshots_available = False if self.TrySnapshot(): snapshots_available = helper.CreateSnapshots() if not snapshots_available: if not self.op.shutdown: raise errors.OpExecError( "Not all disks could be snapshotted, and you requested a live " "export; aborting" ) if not self.op.long_sleep: raise errors.OpExecError( "Not all disks could be snapshotted, and you did not allow the " "instance to remain offline for a longer time through the " "--long-sleep option; aborting" ) try: if self.DoReboot() and snapshots_available: self.StartInstance(feedback_fn, src_node_uuid) if self.op.mode == constants.EXPORT_MODE_LOCAL: (fin_resu, dresults) = helper.LocalExport(self.dst_node, self.op.compress) elif self.op.mode == constants.EXPORT_MODE_REMOTE: connect_timeout = constants.RIE_CONNECT_TIMEOUT timeouts = masterd.instance.ImportExportTimeouts(connect_timeout) (key_name, _, _) = self.x509_key_name dest_ca_pem = \ OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, self.dest_x509_ca) (fin_resu, dresults) = helper.RemoteExport(self.dest_disk_info, key_name, dest_ca_pem, self.op.compress, timeouts) if self.DoReboot() and not snapshots_available: self.StartInstance(feedback_fn, src_node_uuid) finally: helper.Cleanup() # Check for backwards compatibility assert len(dresults) == len(self.instance.disks) assert compat.all(isinstance(i, bool) for i in dresults), \ "Not all results are boolean: %r" % dresults finally: if activate_disks: feedback_fn("Deactivating disks for %s" % self.instance.name) ShutdownInstanceDisks(self, self.instance) if not (compat.all(dresults) and fin_resu): failures = [] if not fin_resu: failures.append("export finalization") if not compat.all(dresults): fdsk = utils.CommaJoin(idx for (idx, dsk) in enumerate(dresults) if not dsk) failures.append("disk export: disk(s) %s" % fdsk) raise errors.OpExecError("Export failed, errors in %s" % utils.CommaJoin(failures)) # At this point, the export was successful, we can cleanup/finish # Remove instance if requested if self.op.remove_instance: feedback_fn("Removing instance %s" % self.instance.name) RemoveInstance(self, feedback_fn, self.instance, self.op.ignore_remove_failures) if self.op.mode == constants.EXPORT_MODE_LOCAL: self._CleanupExports(feedback_fn) return fin_resu, dresults class LUBackupRemove(NoHooksLU): """Remove exports related to the named instance. """ REQ_BGL = False def ExpandNames(self): self.needed_locks = { # We need all nodes to be locked in order for RemoveExport to work, but # we don't need to lock the instance itself, as nothing will happen to it # (and we can remove exports also for a removed instance) locking.LEVEL_NODE: locking.ALL_SET, } def Exec(self, feedback_fn): """Remove any export. """ (_, inst_name) = self.cfg.ExpandInstanceName(self.op.instance_name) # If the instance was not found we'll try with the name that was passed in. # This will only work if it was an FQDN, though. fqdn_warn = False if not inst_name: fqdn_warn = True inst_name = self.op.instance_name locked_nodes = self.owned_locks(locking.LEVEL_NODE) exportlist = self.rpc.call_export_list(locked_nodes) found = False for node_uuid in exportlist: msg = exportlist[node_uuid].fail_msg if msg: self.LogWarning("Failed to query node %s (continuing): %s", self.cfg.GetNodeName(node_uuid), msg) continue if inst_name in exportlist[node_uuid].payload: found = True result = self.rpc.call_export_remove(node_uuid, inst_name) msg = result.fail_msg if msg: logging.error("Could not remove export for instance %s" " on node %s: %s", inst_name, self.cfg.GetNodeName(node_uuid), msg) if fqdn_warn and not found: feedback_fn("Export not found. If trying to remove an export belonging" " to a deleted instance please use its Fully Qualified" " Domain Name.") ganeti-3.1.0~rc2/lib/cmdlib/base.py000064400000000000000000000566141476477700300171040ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Base classes and functions for cmdlib.""" import logging from ganeti import errors from ganeti import constants from ganeti import locking from ganeti import query from ganeti import utils from ganeti.cmdlib.common import ExpandInstanceUuidAndName class ResultWithJobs(object): """Data container for LU results with jobs. Instances of this class returned from L{LogicalUnit.Exec} will be recognized by L{mcpu._ProcessResult}. The latter will then submit the jobs contained in the C{jobs} attribute and include the job IDs in the opcode result. """ def __init__(self, jobs, **kwargs): """Initializes this class. Additional return values can be specified as keyword arguments. @type jobs: list of lists of L{opcode.OpCode} @param jobs: A list of lists of opcode objects """ self.jobs = jobs self.other = kwargs class LUWConfdClient(object): """Wrapper class for wconfd client calls from LUs. Correctly updates the cache of the LU's owned locks when leaving. Also transparently adds the context for resource requests. """ def __init__(self, lu): self.lu = lu def TryUpdateLocks(self, req): self.lu.wconfd.Client().TryUpdateLocks(self.lu.wconfdcontext, req) self.lu.wconfdlocks = \ self.lu.wconfd.Client().ListLocks(self.lu.wconfdcontext) def DownGradeLocksLevel(self, level): self.lu.wconfd.Client().DownGradeLocksLevel(self.lu.wconfdcontext, level) self.lu.wconfdlocks = \ self.lu.wconfd.Client().ListLocks(self.lu.wconfdcontext) def FreeLocksLevel(self, level): self.lu.wconfd.Client().FreeLocksLevel(self.lu.wconfdcontext, level) self.lu.wconfdlocks = \ self.lu.wconfd.Client().ListLocks(self.lu.wconfdcontext) class LogicalUnit(object): # pylint: disable=R0902 """Logical Unit base class. Subclasses must follow these rules: - implement ExpandNames - implement CheckPrereq (except when tasklets are used) - implement Exec (except when tasklets are used) - implement BuildHooksEnv - implement BuildHooksNodes - redefine HPATH and HTYPE - optionally redefine their run requirements: REQ_BGL: the LU needs to hold the Big Ganeti Lock exclusively Note that all commands require root permissions. @ivar dry_run_result: the value (if any) that will be returned to the caller in dry-run mode (signalled by opcode dry_run parameter) """ # This class has more than 20 instance variables, but at most have sensible # defaults and are used in a declartive way, this is not a problem. HPATH = None HTYPE = None REQ_BGL = True def __init__(self, processor, op, cfg, rpc_runner, wconfdcontext, wconfd): """Constructor for LogicalUnit. This needs to be overridden in derived classes in order to check op validity. @type wconfdcontext: (int, string) @param wconfdcontext: the identity of the logical unit to represent itself to wconfd when asking for resources; it is given as job id and livelock file. @param wconfd: the wconfd class to use; dependency injection to allow testability. """ self.proc = processor self.op = op self.cfg = cfg self.wconfdlocks = [] self.wconfdcontext = wconfdcontext self.rpc = rpc_runner self.wconfd = wconfd # wconfd module to use, for testing # Dictionaries used to declare locking needs to mcpu self.needed_locks = None self.share_locks = dict.fromkeys(locking.LEVELS, 0) self.opportunistic_locks = dict.fromkeys(locking.LEVELS, False) self.opportunistic_locks_count = dict.fromkeys(locking.LEVELS, 1) self.dont_collate_locks = dict.fromkeys(locking.LEVELS, False) self.add_locks = {} # Used to force good behavior when calling helper functions self.recalculate_locks = {} # logging self.Log = processor.Log # pylint: disable=C0103 self.LogWarning = processor.LogWarning # pylint: disable=C0103 self.LogInfo = processor.LogInfo # pylint: disable=C0103 self.LogStep = processor.LogStep # pylint: disable=C0103 # support for dry-run self.dry_run_result = None # support for generic debug attribute if (not hasattr(self.op, "debug_level") or not isinstance(self.op.debug_level, int)): self.op.debug_level = 0 # Tasklets self.tasklets = None # Validate opcode parameters and set defaults self.op.Validate(True) self.CheckArguments() def WConfdClient(self): return LUWConfdClient(self) def owned_locks(self, level): """Return the list of locks owned by the LU at a given level. This method assumes that is field wconfdlocks is set correctly by mcpu. """ levelprefix = "%s/" % (locking.LEVEL_NAMES[level],) locks = set([lock[0][len(levelprefix):] for lock in self.wconfdlocks if lock[0].startswith(levelprefix)]) expand_fns = { locking.LEVEL_CLUSTER: (lambda: [locking.BGL]), locking.LEVEL_INSTANCE: lambda: self.cfg.GetInstanceNames(self.cfg.GetInstanceList()), locking.LEVEL_NODEGROUP: self.cfg.GetNodeGroupList, locking.LEVEL_NODE: self.cfg.GetNodeList, locking.LEVEL_NODE_RES: self.cfg.GetNodeList, locking.LEVEL_NETWORK: self.cfg.GetNetworkList, } if locking.LOCKSET_NAME in locks: return expand_fns[level]() else: return locks def release_request(self, level, names): """Return a request to release the specified locks of the given level. Correctly break up the group lock to do so. """ levelprefix = "%s/" % (locking.LEVEL_NAMES[level],) release = [[levelprefix + lock, "release"] for lock in names] # if we break up the set-lock, make sure we ask for the rest of it. setlock = levelprefix + locking.LOCKSET_NAME if [setlock, "exclusive"] in self.wconfdlocks: owned = self.owned_locks(level) request = [[levelprefix + lock, "exclusive"] for lock in owned if lock not in names] elif [setlock, "shared"] in self.wconfdlocks: owned = self.owned_locks(level) request = [[levelprefix + lock, "shared"] for lock in owned if lock not in names] else: request = [] return release + [[setlock, "release"]] + request def CheckArguments(self): """Check syntactic validity for the opcode arguments. This method is for doing a simple syntactic check and ensure validity of opcode parameters, without any cluster-related checks. While the same can be accomplished in ExpandNames and/or CheckPrereq, doing these separate is better because: - ExpandNames is left as as purely a lock-related function - CheckPrereq is run after we have acquired locks (and possible waited for them) The function is allowed to change the self.op attribute so that later methods can no longer worry about missing parameters. """ pass def ExpandNames(self): """Expand names for this LU. This method is called before starting to execute the opcode, and it should update all the parameters of the opcode to their canonical form (e.g. a short node name must be fully expanded after this method has successfully completed). This way locking, hooks, logging, etc. can work correctly. LUs which implement this method must also populate the self.needed_locks member, as a dict with lock levels as keys, and a list of needed lock names as values. Rules: - use an empty dict if you don't need any lock - if you don't need any lock at a particular level omit that level (note that in this case C{DeclareLocks} won't be called at all for that level) - if you need locks at a level, but you can't calculate it in this function, initialise that level with an empty list and do further processing in L{LogicalUnit.DeclareLocks} (see that function's docstring) - don't put anything for the BGL level - if you want all locks at a level use L{locking.ALL_SET} as a value If you need to share locks (rather than acquire them exclusively) at one level you can modify self.share_locks, setting a true value (usually 1) for that level. By default locks are not shared. This function can also define a list of tasklets, which then will be executed in order instead of the usual LU-level CheckPrereq and Exec functions, if those are not defined by the LU. Examples:: # Acquire all nodes and one instance self.needed_locks = { locking.LEVEL_NODE: locking.ALL_SET, locking.LEVEL_INSTANCE: ['instance1.example.com'], } # Acquire just two nodes self.needed_locks = { locking.LEVEL_NODE: ['node1-uuid', 'node2-uuid'], } # Acquire no locks self.needed_locks = {} # No, you can't leave it to the default value None """ # The implementation of this method is mandatory only if the new LU is # concurrent, so that old LUs don't need to be changed all at the same # time. if self.REQ_BGL: self.needed_locks = {} # Exclusive LUs don't need locks. else: raise NotImplementedError def DeclareLocks(self, level): """Declare LU locking needs for a level While most LUs can just declare their locking needs at ExpandNames time, sometimes there's the need to calculate some locks after having acquired the ones before. This function is called just before acquiring locks at a particular level, but after acquiring the ones at lower levels, and permits such calculations. It can be used to modify self.needed_locks, and by default it does nothing. This function is only called if you have something already set in self.needed_locks for the level. @param level: Locking level which is going to be locked @type level: member of L{ganeti.locking.LEVELS} """ def CheckPrereq(self): """Check prerequisites for this LU. This method should check that the prerequisites for the execution of this LU are fulfilled. It can do internode communication, but it should be idempotent - no cluster or system changes are allowed. The method should raise errors.OpPrereqError in case something is not fulfilled. Its return value is ignored. This method should also update all the parameters of the opcode to their canonical form if it hasn't been done by ExpandNames before. """ if self.tasklets is not None: for (idx, tl) in enumerate(self.tasklets): logging.debug("Checking prerequisites for tasklet %s/%s", idx + 1, len(self.tasklets)) tl.CheckPrereq() else: pass def Exec(self, feedback_fn): """Execute the LU. This method should implement the actual work. It should raise errors.OpExecError for failures that are somewhat dealt with in code, or expected. """ if self.tasklets is not None: for (idx, tl) in enumerate(self.tasklets): logging.debug("Executing tasklet %s/%s", idx + 1, len(self.tasklets)) tl.Exec(feedback_fn) else: raise NotImplementedError def PrepareRetry(self, _feedback_fn): """Prepare the LU to run again. This method is called if the Exec failed for temporarily lacking resources. It is expected to change the state of the LU so that it can be tried again, and also change its locking policy to acquire more resources to have a better chance of suceeding in the retry. """ # pylint: disable=R0201 raise errors.OpRetryNotSupportedError() def BuildHooksEnv(self): """Build hooks environment for this LU. @rtype: dict @return: Dictionary containing the environment that will be used for running the hooks for this LU. The keys of the dict must not be prefixed with "GANETI_"--that'll be added by the hooks runner. The hooks runner will extend the environment with additional variables. If no environment should be defined, an empty dictionary should be returned (not C{None}). @note: If the C{HPATH} attribute of the LU class is C{None}, this function will not be called. """ raise NotImplementedError def BuildHooksNodes(self): """Build list of nodes to run LU's hooks. @rtype: tuple; (list, list) @return: Tuple containing a list of node UUIDs on which the hook should run before the execution and a list of node UUIDs on which the hook should run after the execution. No nodes should be returned as an empty list (and not None). @note: If the C{HPATH} attribute of the LU class is C{None}, this function will not be called. """ raise NotImplementedError def PreparePostHookNodes(self, post_hook_node_uuids): """Extend list of nodes to run the post LU hook. This method allows LUs to change the list of node UUIDs on which the post hook should run after the LU has been executed but before the post hook is run. @type post_hook_node_uuids: list @param post_hook_node_uuids: The initial list of node UUIDs to run the post hook on, as returned by L{BuildHooksNodes}. @rtype: list @return: list of node UUIDs on which the post hook should run. The default implementation returns the passed in C{post_hook_node_uuids}, but custom implementations can choose to alter the list. """ # For consistency with HooksCallBack we ignore the "could be a function" # warning # pylint: disable=R0201 return post_hook_node_uuids def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result): """Notify the LU about the results of its hooks. This method is called every time a hooks phase is executed, and notifies the Logical Unit about the hooks' result. The LU can then use it to alter its result based on the hooks. By default the method does nothing and the previous result is passed back unchanged but any LU can define it if it wants to use the local cluster hook-scripts somehow. @param phase: one of L{constants.HOOKS_PHASE_POST} or L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase @param hook_results: the results of the multi-node hooks rpc call @param feedback_fn: function used send feedback back to the caller @param lu_result: the previous Exec result this LU had, or None in the PRE phase @return: the new Exec result, based on the previous result and hook results """ # API must be kept, thus we ignore the unused argument and could # be a function warnings # pylint: disable=W0613,R0201 return lu_result def _ExpandAndLockInstance(self, allow_forthcoming=False): """Helper function to expand and lock an instance. Many LUs that work on an instance take its name in self.op.instance_name and need to expand it and then declare the expanded name for locking. This function does it, and then updates self.op.instance_name to the expanded name. It also initializes needed_locks as a dict, if this hasn't been done before. @param allow_forthcoming: if True, do not insist that the intsance be real; the default behaviour is to raise a prerequisite error if the specified instance is forthcoming. """ if self.needed_locks is None: self.needed_locks = {} else: assert locking.LEVEL_INSTANCE not in self.needed_locks, \ "_ExpandAndLockInstance called with instance-level locks set" (self.op.instance_uuid, self.op.instance_name) = \ ExpandInstanceUuidAndName(self.cfg, self.op.instance_uuid, self.op.instance_name) self.needed_locks[locking.LEVEL_INSTANCE] = self.op.instance_name if not allow_forthcoming: if self.cfg.GetInstanceInfo(self.op.instance_uuid).forthcoming: raise errors.OpPrereqError( "forthcoming instances not supported for this operation") def _LockInstancesNodes(self, primary_only=False, level=locking.LEVEL_NODE): """Helper function to declare instances' nodes for locking. This function should be called after locking one or more instances to lock their nodes. Its effect is populating self.needed_locks[locking.LEVEL_NODE] with all primary or secondary nodes for instances already locked and present in self.needed_locks[locking.LEVEL_INSTANCE]. It should be called from DeclareLocks, and for safety only works if self.recalculate_locks[locking.LEVEL_NODE] is set. In the future it may grow parameters to just lock some instance's nodes, or to just lock primaries or secondary nodes, if needed. If should be called in DeclareLocks in a way similar to:: if level == locking.LEVEL_NODE: self._LockInstancesNodes() @type primary_only: boolean @param primary_only: only lock primary nodes of locked instances @param level: Which lock level to use for locking nodes """ assert level in self.recalculate_locks, \ "_LockInstancesNodes helper function called with no nodes to recalculate" # TODO: check if we're really been called with the instance locks held # For now we'll replace self.needed_locks[locking.LEVEL_NODE], but in the # future we might want to have different behaviors depending on the value # of self.recalculate_locks[locking.LEVEL_NODE] wanted_node_uuids = [] locked_i = self.owned_locks(locking.LEVEL_INSTANCE) for _, instance in self.cfg.GetMultiInstanceInfoByName(locked_i): wanted_node_uuids.append(instance.primary_node) if not primary_only: wanted_node_uuids.extend( self.cfg.GetInstanceSecondaryNodes(instance.uuid)) if self.recalculate_locks[level] == constants.LOCKS_REPLACE: self.needed_locks[level] = wanted_node_uuids elif self.recalculate_locks[level] == constants.LOCKS_APPEND: self.needed_locks[level].extend(wanted_node_uuids) else: raise errors.ProgrammerError("Unknown recalculation mode") del self.recalculate_locks[level] def AssertReleasedLocks(self, level): """Raise AssertionError if the LU holds some locks of the given level. """ assert not self.owned_locks(level) class NoHooksLU(LogicalUnit): # pylint: disable=W0223 """Simple LU which runs no hooks. This LU is intended as a parent for other LogicalUnits which will run no hooks, in order to reduce duplicate code. """ HPATH = None HTYPE = None def BuildHooksEnv(self): """Empty BuildHooksEnv for NoHooksLu. This just raises an error. """ raise AssertionError("BuildHooksEnv called for NoHooksLUs") def BuildHooksNodes(self): """Empty BuildHooksNodes for NoHooksLU. """ raise AssertionError("BuildHooksNodes called for NoHooksLU") def PreparePostHookNodes(self, post_hook_node_uuids): """Empty PreparePostHookNodes for NoHooksLU. """ raise AssertionError("PreparePostHookNodes called for NoHooksLU") class Tasklet(object): """Tasklet base class. Tasklets are subcomponents for LUs. LUs can consist entirely of tasklets or they can mix legacy code with tasklets. Locking needs to be done in the LU, tasklets know nothing about locks. Subclasses must follow these rules: - Implement CheckPrereq - Implement Exec """ def __init__(self, lu): self.lu = lu # Shortcuts self.cfg = lu.cfg self.rpc = lu.rpc def CheckPrereq(self): """Check prerequisites for this tasklets. This method should check whether the prerequisites for the execution of this tasklet are fulfilled. It can do internode communication, but it should be idempotent - no cluster or system changes are allowed. The method should raise errors.OpPrereqError in case something is not fulfilled. Its return value is ignored. This method should also update all parameters to their canonical form if it hasn't been done before. """ pass def Exec(self, feedback_fn): """Execute the tasklet. This method should implement the actual work. It should raise errors.OpExecError for failures that are somewhat dealt with in code, or expected. """ raise NotImplementedError class QueryBase(object): """Base for query utility classes. """ #: Attribute holding field definitions FIELDS = None #: Field to sort by SORT_FIELD = "name" def __init__(self, qfilter, fields, use_locking): """Initializes this class. """ self.use_locking = use_locking self.query = query.Query(self.FIELDS, fields, qfilter=qfilter, namefield=self.SORT_FIELD) self.requested_data = self.query.RequestedData() self.names = self.query.RequestedNames() # Sort only if no names were requested self.sort_by_name = not self.names self.do_locking = None self.wanted = None def _GetNames(self, lu, all_names, lock_level): """Helper function to determine names asked for in the query. """ if self.do_locking: names = lu.owned_locks(lock_level) else: names = all_names if self.wanted == locking.ALL_SET: assert not self.names # caller didn't specify names, so ordering is not important return utils.NiceSort(names) # caller specified names and we must keep the same order assert self.names missing = set(self.wanted).difference(names) if missing: raise errors.OpExecError("Some items were removed before retrieving" " their data: %s" % missing) # Return expanded names return self.wanted def ExpandNames(self, lu): """Expand names for this query. See L{LogicalUnit.ExpandNames}. """ raise NotImplementedError() def DeclareLocks(self, lu, level): """Declare locks for this query. See L{LogicalUnit.DeclareLocks}. """ raise NotImplementedError() def _GetQueryData(self, lu): """Collects all data for this query. @return: Query data object """ raise NotImplementedError() def NewStyleQuery(self, lu): """Collect data and execute query. """ return query.GetQueryResponse(self.query, self._GetQueryData(lu), sort_by_name=self.sort_by_name) def OldStyleQuery(self, lu): """Collect data and execute query. """ return self.query.OldStyleQuery(self._GetQueryData(lu), sort_by_name=self.sort_by_name) ganeti-3.1.0~rc2/lib/cmdlib/cluster/000075500000000000000000000000001476477700300172655ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/cmdlib/cluster/__init__.py000064400000000000000000002074761476477700300214160ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with the cluster.""" import copy import itertools import logging import operator import os import re import time from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import hypervisor from ganeti import locking from ganeti import masterd from ganeti import netutils from ganeti import objects from ganeti import opcodes from ganeti import pathutils from ganeti import query import ganeti.rpc.node as rpc from ganeti import runtime from ganeti import ssh from ganeti import uidpool from ganeti import utils from ganeti import vcluster from ganeti.cmdlib.base import NoHooksLU, QueryBase, LogicalUnit, \ ResultWithJobs from ganeti.cmdlib.common import ShareAll, RunPostHook, \ ComputeAncillaryFiles, RedistributeAncillaryFiles, UploadHelper, \ GetWantedInstances, MergeAndVerifyHvState, MergeAndVerifyDiskState, \ GetUpdatedIPolicy, ComputeNewInstanceViolations, GetUpdatedParams, \ CheckOSParams, CheckHVParams, AdjustCandidatePool, CheckNodePVs, \ ComputeIPolicyInstanceViolation, AnnotateDiskParams, SupportsOob, \ CheckIpolicyVsDiskTemplates, CheckDiskAccessModeValidity, \ CheckDiskAccessModeConsistency, GetClientCertDigest, \ AddInstanceCommunicationNetworkOp, ConnectInstanceCommunicationNetworkOp, \ CheckImageValidity, EnsureKvmdOnNodes import ganeti.masterd.instance class LUClusterRenewCrypto(NoHooksLU): """Renew the cluster's crypto tokens. """ _MAX_NUM_RETRIES = 3 REQ_BGL = False def ExpandNames(self): self.needed_locks = { locking.LEVEL_NODE: locking.ALL_SET, } self.share_locks = ShareAll() self.share_locks[locking.LEVEL_NODE] = 0 def CheckPrereq(self): """Check prerequisites. Notably the compatibility of specified key bits and key type. """ cluster_info = self.cfg.GetClusterInfo() self.ssh_key_type = self.op.ssh_key_type if self.ssh_key_type is None: self.ssh_key_type = cluster_info.ssh_key_type self.ssh_key_bits = ssh.DetermineKeyBits(self.ssh_key_type, self.op.ssh_key_bits, cluster_info.ssh_key_type, cluster_info.ssh_key_bits) def _RenewNodeSslCertificates(self, feedback_fn): """Renews the nodes' SSL certificates. Note that most of this operation is done in gnt_cluster.py, this LU only takes care of the renewal of the client SSL certificates. """ master_uuid = self.cfg.GetMasterNode() cluster = self.cfg.GetClusterInfo() logging.debug("Renewing the master's SSL node certificate." " Master's UUID: %s.", master_uuid) # mapping node UUIDs to client certificate digests digest_map = {} master_digest = utils.GetCertificateDigest( cert_filename=pathutils.NODED_CLIENT_CERT_FILE) digest_map[master_uuid] = master_digest logging.debug("Adding the master's SSL node certificate digest to the" " configuration. Master's UUID: %s, Digest: %s", master_uuid, master_digest) node_errors = {} nodes = self.cfg.GetAllNodesInfo() logging.debug("Renewing non-master nodes' node certificates.") for (node_uuid, node_info) in nodes.items(): if node_info.offline: logging.info("* Skipping offline node %s", node_info.name) continue if node_uuid != master_uuid: logging.debug("Adding certificate digest of node '%s'.", node_uuid) last_exception = None for i in range(self._MAX_NUM_RETRIES): try: if node_info.master_candidate: node_digest = GetClientCertDigest(self, node_uuid) digest_map[node_uuid] = node_digest logging.debug("Added the node's certificate to candidate" " certificate list. Current list: %s.", str(cluster.candidate_certs)) break except errors.OpExecError as e: last_exception = e logging.error("Could not fetch a non-master node's SSL node" " certificate at attempt no. %s. The node's UUID" " is %s, and the error was: %s.", str(i), node_uuid, e) else: if last_exception: node_errors[node_uuid] = last_exception if node_errors: msg = ("Some nodes' SSL client certificates could not be fetched." " Please make sure those nodes are reachable and rerun" " the operation. The affected nodes and their errors are:\n") for uuid, e in node_errors.items(): msg += "Node %s: %s\n" % (uuid, e) feedback_fn(msg) self.cfg.SetCandidateCerts(digest_map) def _RenewSshKeys(self, feedback_fn): """Renew all nodes' SSH keys. @type feedback_fn: function @param feedback_fn: logging function, see L{ganeti.cmdlist.base.LogicalUnit} """ master_uuid = self.cfg.GetMasterNode() nodes = self.cfg.GetAllNodesInfo() nodes_uuid_names = [(node_uuid, node_info.name) for (node_uuid, node_info) in nodes.items() if not node_info.offline] node_names = [name for (_, name) in nodes_uuid_names] node_uuids = [uuid for (uuid, _) in nodes_uuid_names] potential_master_candidates = self.cfg.GetPotentialMasterCandidates() master_candidate_uuids = self.cfg.GetMasterCandidateUuids() cluster_info = self.cfg.GetClusterInfo() result = self.rpc.call_node_ssh_keys_renew( [master_uuid], node_uuids, node_names, master_candidate_uuids, potential_master_candidates, cluster_info.ssh_key_type, # Old key type self.ssh_key_type, # New key type self.ssh_key_bits) # New key bits result[master_uuid].Raise("Could not renew the SSH keys of all nodes") # After the keys have been successfully swapped, time to commit the change # in key type cluster_info.ssh_key_type = self.ssh_key_type cluster_info.ssh_key_bits = self.ssh_key_bits self.cfg.Update(cluster_info, feedback_fn) def Exec(self, feedback_fn): if self.op.node_certificates: feedback_fn("Renewing Node SSL certificates") self._RenewNodeSslCertificates(feedback_fn) if self.op.renew_ssh_keys: if self.cfg.GetClusterInfo().modify_ssh_setup: feedback_fn("Renewing SSH keys") self._RenewSshKeys(feedback_fn) else: feedback_fn("Cannot renew SSH keys if the cluster is configured to not" " modify the SSH setup.") class LUClusterActivateMasterIp(NoHooksLU): """Activate the master IP on the master node. """ def Exec(self, feedback_fn): """Activate the master IP. """ master_params = self.cfg.GetMasterNetworkParameters() ems = self.cfg.GetUseExternalMipScript() result = self.rpc.call_node_activate_master_ip(master_params.uuid, master_params, ems) result.Raise("Could not activate the master IP") class LUClusterDeactivateMasterIp(NoHooksLU): """Deactivate the master IP on the master node. """ def Exec(self, feedback_fn): """Deactivate the master IP. """ master_params = self.cfg.GetMasterNetworkParameters() ems = self.cfg.GetUseExternalMipScript() result = self.rpc.call_node_deactivate_master_ip(master_params.uuid, master_params, ems) result.Raise("Could not deactivate the master IP") class LUClusterConfigQuery(NoHooksLU): """Return configuration values. """ REQ_BGL = False def CheckArguments(self): self.cq = ClusterQuery(None, self.op.output_fields, False) def ExpandNames(self): self.cq.ExpandNames(self) def DeclareLocks(self, level): self.cq.DeclareLocks(self, level) def Exec(self, feedback_fn): result = self.cq.OldStyleQuery(self) assert len(result) == 1 return result[0] class LUClusterDestroy(LogicalUnit): """Logical unit for destroying the cluster. """ HPATH = "cluster-destroy" HTYPE = constants.HTYPE_CLUSTER # Read by the job queue to detect when the cluster is gone and job files will # never be available. # FIXME: This variable should be removed together with the Python job queue. clusterHasBeenDestroyed = False def BuildHooksEnv(self): """Build hooks env. """ return { "OP_TARGET": self.cfg.GetClusterName(), } def BuildHooksNodes(self): """Build hooks nodes. """ return ([], []) def CheckPrereq(self): """Check prerequisites. This checks whether the cluster is empty. Any errors are signaled by raising errors.OpPrereqError. """ master = self.cfg.GetMasterNode() nodelist = self.cfg.GetNodeList() if len(nodelist) != 1 or nodelist[0] != master: raise errors.OpPrereqError("There are still %d node(s) in" " this cluster." % (len(nodelist) - 1), errors.ECODE_INVAL) instancelist = self.cfg.GetInstanceList() if instancelist: raise errors.OpPrereqError("There are still %d instance(s) in" " this cluster." % len(instancelist), errors.ECODE_INVAL) def Exec(self, feedback_fn): """Destroys the cluster. """ master_params = self.cfg.GetMasterNetworkParameters() # Run post hooks on master node before it's removed RunPostHook(self, self.cfg.GetNodeName(master_params.uuid)) ems = self.cfg.GetUseExternalMipScript() result = self.rpc.call_node_deactivate_master_ip(master_params.uuid, master_params, ems) result.Warn("Error disabling the master IP address", self.LogWarning) self.wconfd.Client().PrepareClusterDestruction(self.wconfdcontext) # signal to the job queue that the cluster is gone LUClusterDestroy.clusterHasBeenDestroyed = True return master_params.uuid class LUClusterPostInit(LogicalUnit): """Logical unit for running hooks after cluster initialization. """ HPATH = "cluster-init" HTYPE = constants.HTYPE_CLUSTER def CheckArguments(self): self.master_uuid = self.cfg.GetMasterNode() self.master_ndparams = self.cfg.GetNdParams(self.cfg.GetMasterNodeInfo()) # TODO: When Issue 584 is solved, and None is properly parsed when used # as a default value, ndparams.get(.., None) can be changed to # ndparams[..] to access the values directly # OpenvSwitch: Warn user if link is missing if (self.master_ndparams[constants.ND_OVS] and not self.master_ndparams.get(constants.ND_OVS_LINK, None)): self.LogInfo("No physical interface for OpenvSwitch was given." " OpenvSwitch will not have an outside connection. This" " might not be what you want.") def BuildHooksEnv(self): """Build hooks env. """ return { "OP_TARGET": self.cfg.GetClusterName(), } def BuildHooksNodes(self): """Build hooks nodes. """ return ([], [self.cfg.GetMasterNode()]) def Exec(self, feedback_fn): """Create and configure Open vSwitch """ if self.master_ndparams[constants.ND_OVS]: result = self.rpc.call_node_configure_ovs( self.master_uuid, self.master_ndparams[constants.ND_OVS_NAME], self.master_ndparams.get(constants.ND_OVS_LINK, None)) result.Raise("Could not successully configure Open vSwitch") return True class ClusterQuery(QueryBase): FIELDS = query.CLUSTER_FIELDS #: Do not sort (there is only one item) SORT_FIELD = None def ExpandNames(self, lu): lu.needed_locks = {} # The following variables interact with _QueryBase._GetNames self.wanted = locking.ALL_SET self.do_locking = self.use_locking if self.do_locking: raise errors.OpPrereqError("Can not use locking for cluster queries", errors.ECODE_INVAL) def DeclareLocks(self, lu, level): pass def _GetQueryData(self, lu): """Computes the list of nodes and their attributes. """ if query.CQ_CONFIG in self.requested_data: cluster = lu.cfg.GetClusterInfo() nodes = lu.cfg.GetAllNodesInfo() else: cluster = NotImplemented nodes = NotImplemented if query.CQ_QUEUE_DRAINED in self.requested_data: drain_flag = os.path.exists(pathutils.JOB_QUEUE_DRAIN_FILE) else: drain_flag = NotImplemented if query.CQ_WATCHER_PAUSE in self.requested_data: master_node_uuid = lu.cfg.GetMasterNode() result = lu.rpc.call_get_watcher_pause(master_node_uuid) result.Raise("Can't retrieve watcher pause from master node '%s'" % lu.cfg.GetMasterNodeName()) watcher_pause = result.payload else: watcher_pause = NotImplemented return query.ClusterQueryData(cluster, nodes, drain_flag, watcher_pause) class LUClusterQuery(NoHooksLU): """Query cluster configuration. """ REQ_BGL = False def ExpandNames(self): self.needed_locks = {} def Exec(self, feedback_fn): """Return cluster config. """ cluster = self.cfg.GetClusterInfo() os_hvp = {} # Filter just for enabled hypervisors for os_name, hv_dict in cluster.os_hvp.items(): os_hvp[os_name] = {} for hv_name, hv_params in hv_dict.items(): if hv_name in cluster.enabled_hypervisors: os_hvp[os_name][hv_name] = hv_params # Convert ip_family to ip_version primary_ip_version = constants.IP4_VERSION if cluster.primary_ip_family == netutils.IP6Address.family: primary_ip_version = constants.IP6_VERSION result = { "software_version": constants.RELEASE_VERSION, "protocol_version": constants.PROTOCOL_VERSION, "config_version": constants.CONFIG_VERSION, "os_api_version": max(constants.OS_API_VERSIONS), "export_version": constants.EXPORT_VERSION, "vcs_version": constants.VCS_VERSION, "architecture": runtime.GetArchInfo(), "name": cluster.cluster_name, "master": self.cfg.GetMasterNodeName(), "default_hypervisor": cluster.primary_hypervisor, "enabled_hypervisors": cluster.enabled_hypervisors, "hvparams": dict([(hypervisor_name, cluster.hvparams[hypervisor_name]) for hypervisor_name in cluster.enabled_hypervisors]), "os_hvp": os_hvp, "beparams": cluster.beparams, "osparams": cluster.osparams, "ipolicy": cluster.ipolicy, "nicparams": cluster.nicparams, "ndparams": cluster.ndparams, "diskparams": cluster.diskparams, "candidate_pool_size": cluster.candidate_pool_size, "max_running_jobs": cluster.max_running_jobs, "max_tracked_jobs": cluster.max_tracked_jobs, "mac_prefix": cluster.mac_prefix, "master_netdev": cluster.master_netdev, "master_netmask": cluster.master_netmask, "use_external_mip_script": cluster.use_external_mip_script, "volume_group_name": cluster.volume_group_name, "drbd_usermode_helper": cluster.drbd_usermode_helper, "file_storage_dir": cluster.file_storage_dir, "shared_file_storage_dir": cluster.shared_file_storage_dir, "maintain_node_health": cluster.maintain_node_health, "ctime": cluster.ctime, "mtime": cluster.mtime, "uuid": cluster.uuid, "tags": list(cluster.GetTags()), "uid_pool": cluster.uid_pool, "default_iallocator": cluster.default_iallocator, "default_iallocator_params": cluster.default_iallocator_params, "reserved_lvs": cluster.reserved_lvs, "primary_ip_version": primary_ip_version, "prealloc_wipe_disks": cluster.prealloc_wipe_disks, "hidden_os": cluster.hidden_os, "blacklisted_os": cluster.blacklisted_os, "enabled_disk_templates": cluster.enabled_disk_templates, "install_image": cluster.install_image, "instance_communication_network": cluster.instance_communication_network, "compression_tools": cluster.compression_tools, "enabled_user_shutdown": cluster.enabled_user_shutdown, } return result class LUClusterRedistConf(NoHooksLU): """Force the redistribution of cluster configuration. This is a very simple LU. """ REQ_BGL = False def ExpandNames(self): self.needed_locks = { locking.LEVEL_NODE: locking.ALL_SET, } self.share_locks = ShareAll() def Exec(self, feedback_fn): """Redistribute the configuration. """ self.cfg.Update(self.cfg.GetClusterInfo(), feedback_fn) RedistributeAncillaryFiles(self) class LUClusterRename(LogicalUnit): """Rename the cluster. """ HPATH = "cluster-rename" HTYPE = constants.HTYPE_CLUSTER def BuildHooksEnv(self): """Build hooks env. """ return { "OP_TARGET": self.cfg.GetClusterName(), "NEW_NAME": self.op.name, } def BuildHooksNodes(self): """Build hooks nodes. """ return ([self.cfg.GetMasterNode()], self.cfg.GetNodeList()) def CheckPrereq(self): """Verify that the passed name is a valid one. """ hostname = netutils.GetHostname(name=self.op.name, family=self.cfg.GetPrimaryIPFamily()) new_name = hostname.name self.ip = new_ip = hostname.ip old_name = self.cfg.GetClusterName() old_ip = self.cfg.GetMasterIP() if new_name == old_name and new_ip == old_ip: raise errors.OpPrereqError("Neither the name nor the IP address of the" " cluster has changed", errors.ECODE_INVAL) if new_ip != old_ip: if netutils.TcpPing(new_ip, constants.DEFAULT_NODED_PORT): raise errors.OpPrereqError("The given cluster IP address (%s) is" " reachable on the network" % new_ip, errors.ECODE_NOTUNIQUE) self.op.name = new_name def Exec(self, feedback_fn): """Rename the cluster. """ clustername = self.op.name new_ip = self.ip # shutdown the master IP master_params = self.cfg.GetMasterNetworkParameters() ems = self.cfg.GetUseExternalMipScript() result = self.rpc.call_node_deactivate_master_ip(master_params.uuid, master_params, ems) result.Raise("Could not disable the master role") try: cluster = self.cfg.GetClusterInfo() cluster.cluster_name = clustername cluster.master_ip = new_ip self.cfg.Update(cluster, feedback_fn) # update the known hosts file ssh.WriteKnownHostsFile(self.cfg, pathutils.SSH_KNOWN_HOSTS_FILE) node_list = self.cfg.GetOnlineNodeList() try: node_list.remove(master_params.uuid) except ValueError: pass UploadHelper(self, node_list, pathutils.SSH_KNOWN_HOSTS_FILE) finally: master_params.ip = new_ip result = self.rpc.call_node_activate_master_ip(master_params.uuid, master_params, ems) result.Warn("Could not re-enable the master role on the master," " please restart manually", self.LogWarning) return clustername class LUClusterRepairDiskSizes(NoHooksLU): """Verifies the cluster disks sizes. """ REQ_BGL = False def ExpandNames(self): if self.op.instances: (_, self.wanted_names) = GetWantedInstances(self, self.op.instances) # Not getting the node allocation lock as only a specific set of # instances (and their nodes) is going to be acquired self.needed_locks = { locking.LEVEL_NODE_RES: [], locking.LEVEL_INSTANCE: self.wanted_names, } self.recalculate_locks[locking.LEVEL_NODE_RES] = constants.LOCKS_REPLACE else: self.wanted_names = None self.needed_locks = { locking.LEVEL_NODE_RES: locking.ALL_SET, locking.LEVEL_INSTANCE: locking.ALL_SET, } self.share_locks = { locking.LEVEL_NODE_RES: 1, locking.LEVEL_INSTANCE: 0, } def DeclareLocks(self, level): if level == locking.LEVEL_NODE_RES and self.wanted_names is not None: self._LockInstancesNodes(primary_only=True, level=level) def CheckPrereq(self): """Check prerequisites. This only checks the optional instance list against the existing names. """ if self.wanted_names is None: self.wanted_names = self.owned_locks(locking.LEVEL_INSTANCE) self.wanted_instances = [ info for (_, info) in self.cfg.GetMultiInstanceInfoByName(self.wanted_names) ] def _EnsureChildSizes(self, disk): """Ensure children of the disk have the needed disk size. This is valid mainly for DRBD8 and fixes an issue where the children have smaller disk size. @param disk: an L{ganeti.objects.Disk} object """ if disk.dev_type == constants.DT_DRBD8: assert disk.children, "Empty children for DRBD8?" fchild = disk.children[0] mismatch = fchild.size < disk.size if mismatch: self.LogInfo("Child disk has size %d, parent %d, fixing", fchild.size, disk.size) fchild.size = disk.size # and we recurse on this child only, not on the metadev return self._EnsureChildSizes(fchild) or mismatch else: return False def Exec(self, feedback_fn): """Verify the size of cluster disks. """ # TODO: check child disks too # TODO: check differences in size between primary/secondary nodes per_node_disks = {} for instance in self.wanted_instances: pnode = instance.primary_node if pnode not in per_node_disks: per_node_disks[pnode] = [] for idx, disk in enumerate(self.cfg.GetInstanceDisks(instance.uuid)): per_node_disks[pnode].append((instance, idx, disk)) assert not (frozenset(per_node_disks) - frozenset(self.owned_locks(locking.LEVEL_NODE_RES))), \ "Not owning correct locks" assert not self.owned_locks(locking.LEVEL_NODE) es_flags = rpc.GetExclusiveStorageForNodes(self.cfg, list(per_node_disks)) changed = [] for node_uuid, dskl in per_node_disks.items(): if not dskl: # no disks on the node continue newl = [([v[2].Copy()], v[0]) for v in dskl] node_name = self.cfg.GetNodeName(node_uuid) result = self.rpc.call_blockdev_getdimensions(node_uuid, newl) if result.fail_msg: self.LogWarning("Failure in blockdev_getdimensions call to node" " %s, ignoring", node_name) continue if len(result.payload) != len(dskl): logging.warning("Invalid result from node %s: len(dksl)=%d," " result.payload=%s", node_name, len(dskl), result.payload) self.LogWarning("Invalid result from node %s, ignoring node results", node_name) continue for ((instance, idx, disk), dimensions) in zip(dskl, result.payload): if dimensions is None: self.LogWarning("Disk %d of instance %s did not return size" " information, ignoring", idx, instance.name) continue if not isinstance(dimensions, (tuple, list)): self.LogWarning("Disk %d of instance %s did not return valid" " dimension information, ignoring", idx, instance.name) continue (size, spindles) = dimensions if not isinstance(size, int): self.LogWarning("Disk %d of instance %s did not return valid" " size information, ignoring", idx, instance.name) continue size = size >> 20 if size != disk.size: self.LogInfo("Disk %d of instance %s has mismatched size," " correcting: recorded %d, actual %d", idx, instance.name, disk.size, size) disk.size = size self.cfg.Update(disk, feedback_fn) changed.append((instance.name, idx, "size", size)) if es_flags[node_uuid]: if spindles is None: self.LogWarning("Disk %d of instance %s did not return valid" " spindles information, ignoring", idx, instance.name) elif disk.spindles is None or disk.spindles != spindles: self.LogInfo("Disk %d of instance %s has mismatched spindles," " correcting: recorded %s, actual %s", idx, instance.name, disk.spindles, spindles) disk.spindles = spindles self.cfg.Update(disk, feedback_fn) changed.append((instance.name, idx, "spindles", disk.spindles)) if self._EnsureChildSizes(disk): self.cfg.Update(disk, feedback_fn) changed.append((instance.name, idx, "size", disk.size)) return changed def _ValidateNetmask(cfg, netmask): """Checks if a netmask is valid. @type cfg: L{config.ConfigWriter} @param cfg: cluster configuration @type netmask: int @param netmask: netmask to be verified @raise errors.OpPrereqError: if the validation fails """ ip_family = cfg.GetPrimaryIPFamily() try: ipcls = netutils.IPAddress.GetClassFromIpFamily(ip_family) except errors.ProgrammerError: raise errors.OpPrereqError("Invalid primary ip family: %s." % ip_family, errors.ECODE_INVAL) if not ipcls.ValidateNetmask(netmask): raise errors.OpPrereqError("CIDR netmask (%s) not valid" % (netmask), errors.ECODE_INVAL) def CheckFileBasedStoragePathVsEnabledDiskTemplates( logging_warn_fn, file_storage_dir, enabled_disk_templates, file_disk_template): """Checks whether the given file-based storage directory is acceptable. Note: This function is public, because it is also used in bootstrap.py. @type logging_warn_fn: function @param logging_warn_fn: function which accepts a string and logs it @type file_storage_dir: string @param file_storage_dir: the directory to be used for file-based instances @type enabled_disk_templates: list of string @param enabled_disk_templates: the list of enabled disk templates @type file_disk_template: string @param file_disk_template: the file-based disk template for which the path should be checked """ assert (file_disk_template in utils.storage.GetDiskTemplatesOfStorageTypes( constants.ST_FILE, constants.ST_SHARED_FILE, constants.ST_GLUSTER )) file_storage_enabled = file_disk_template in enabled_disk_templates if file_storage_dir is not None: if file_storage_dir == "": if file_storage_enabled: raise errors.OpPrereqError( "Unsetting the '%s' storage directory while having '%s' storage" " enabled is not permitted." % (file_disk_template, file_disk_template), errors.ECODE_INVAL) else: if not file_storage_enabled: logging_warn_fn( "Specified a %s storage directory, although %s storage is not" " enabled." % (file_disk_template, file_disk_template)) else: raise errors.ProgrammerError("Received %s storage dir with value" " 'None'." % file_disk_template) def CheckFileStoragePathVsEnabledDiskTemplates( logging_warn_fn, file_storage_dir, enabled_disk_templates): """Checks whether the given file storage directory is acceptable. @see: C{CheckFileBasedStoragePathVsEnabledDiskTemplates} """ CheckFileBasedStoragePathVsEnabledDiskTemplates( logging_warn_fn, file_storage_dir, enabled_disk_templates, constants.DT_FILE) def CheckSharedFileStoragePathVsEnabledDiskTemplates( logging_warn_fn, file_storage_dir, enabled_disk_templates): """Checks whether the given shared file storage directory is acceptable. @see: C{CheckFileBasedStoragePathVsEnabledDiskTemplates} """ CheckFileBasedStoragePathVsEnabledDiskTemplates( logging_warn_fn, file_storage_dir, enabled_disk_templates, constants.DT_SHARED_FILE) def CheckGlusterStoragePathVsEnabledDiskTemplates( logging_warn_fn, file_storage_dir, enabled_disk_templates): """Checks whether the given gluster storage directory is acceptable. @see: C{CheckFileBasedStoragePathVsEnabledDiskTemplates} """ CheckFileBasedStoragePathVsEnabledDiskTemplates( logging_warn_fn, file_storage_dir, enabled_disk_templates, constants.DT_GLUSTER) def CheckCompressionTools(tools): """Check whether the provided compression tools look like executables. @type tools: list of string @param tools: The tools provided as opcode input """ regex = re.compile('^[-_a-zA-Z0-9]+$') illegal_tools = [t for t in tools if not regex.match(t)] if illegal_tools: raise errors.OpPrereqError( "The tools '%s' contain illegal characters: only alphanumeric values," " dashes, and underscores are allowed" % ", ".join(illegal_tools), errors.ECODE_INVAL ) if constants.IEC_GZIP not in tools: raise errors.OpPrereqError("For compatibility reasons, the %s utility must" " be present among the compression tools" % constants.IEC_GZIP, errors.ECODE_INVAL) if constants.IEC_NONE in tools: raise errors.OpPrereqError("%s is a reserved value used for no compression," " and cannot be used as the name of a tool" % constants.IEC_NONE, errors.ECODE_INVAL) class LUClusterSetParams(LogicalUnit): """Change the parameters of the cluster. """ HPATH = "cluster-modify" HTYPE = constants.HTYPE_CLUSTER REQ_BGL = False def CheckArguments(self): """Check parameters """ if self.op.uid_pool: uidpool.CheckUidPool(self.op.uid_pool) if self.op.add_uids: uidpool.CheckUidPool(self.op.add_uids) if self.op.remove_uids: uidpool.CheckUidPool(self.op.remove_uids) if self.op.mac_prefix: self.op.mac_prefix = \ utils.NormalizeAndValidateThreeOctetMacPrefix(self.op.mac_prefix) if self.op.master_netmask is not None: _ValidateNetmask(self.cfg, self.op.master_netmask) if self.op.diskparams: for dt_params in self.op.diskparams.values(): utils.ForceDictType(dt_params, constants.DISK_DT_TYPES) try: utils.VerifyDictOptions(self.op.diskparams, constants.DISK_DT_DEFAULTS) CheckDiskAccessModeValidity(self.op.diskparams) except errors.OpPrereqError as err: raise errors.OpPrereqError("While verify diskparams options: %s" % err, errors.ECODE_INVAL) if self.op.install_image is not None: CheckImageValidity(self.op.install_image, "Install image must be an absolute path or a URL") def ExpandNames(self): # FIXME: in the future maybe other cluster params won't require checking on # all nodes to be modified. # FIXME: This opcode changes cluster-wide settings. Is acquiring all # resource locks the right thing, shouldn't it be the BGL instead? self.needed_locks = { locking.LEVEL_NODE: locking.ALL_SET, locking.LEVEL_INSTANCE: locking.ALL_SET, locking.LEVEL_NODEGROUP: locking.ALL_SET, } self.share_locks = ShareAll() def BuildHooksEnv(self): """Build hooks env. """ return { "OP_TARGET": self.cfg.GetClusterName(), "NEW_VG_NAME": self.op.vg_name, } def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() return ([mn], [mn]) def _CheckVgName(self, node_uuids, enabled_disk_templates, new_enabled_disk_templates): """Check the consistency of the vg name on all nodes and in case it gets unset whether there are instances still using it. """ lvm_is_enabled = utils.IsLvmEnabled(enabled_disk_templates) lvm_gets_enabled = utils.LvmGetsEnabled(enabled_disk_templates, new_enabled_disk_templates) current_vg_name = self.cfg.GetVGName() if self.op.vg_name == '': if lvm_is_enabled: raise errors.OpPrereqError("Cannot unset volume group if lvm-based" " disk templates are or get enabled.", errors.ECODE_INVAL) if self.op.vg_name is None: if current_vg_name is None and lvm_is_enabled: raise errors.OpPrereqError("Please specify a volume group when" " enabling lvm-based disk-templates.", errors.ECODE_INVAL) if self.op.vg_name is not None and not self.op.vg_name: if self.cfg.DisksOfType(constants.DT_PLAIN): raise errors.OpPrereqError("Cannot disable lvm storage while lvm-based" " instances exist", errors.ECODE_INVAL) if (self.op.vg_name is not None and lvm_is_enabled) or \ (self.cfg.GetVGName() is not None and lvm_gets_enabled): self._CheckVgNameOnNodes(node_uuids) def _CheckVgNameOnNodes(self, node_uuids): """Check the status of the volume group on each node. """ vglist = self.rpc.call_vg_list(node_uuids) for node_uuid in node_uuids: msg = vglist[node_uuid].fail_msg if msg: # ignoring down node self.LogWarning("Error while gathering data on node %s" " (ignoring node): %s", self.cfg.GetNodeName(node_uuid), msg) continue vgstatus = utils.CheckVolumeGroupSize(vglist[node_uuid].payload, self.op.vg_name, constants.MIN_VG_SIZE) if vgstatus: raise errors.OpPrereqError("Error on node '%s': %s" % (self.cfg.GetNodeName(node_uuid), vgstatus), errors.ECODE_ENVIRON) @staticmethod def _GetDiskTemplateSetsInner(op_enabled_disk_templates, old_enabled_disk_templates): """Computes three sets of disk templates. @see: C{_GetDiskTemplateSets} for more details. """ enabled_disk_templates = None new_enabled_disk_templates = [] disabled_disk_templates = [] if op_enabled_disk_templates: enabled_disk_templates = op_enabled_disk_templates new_enabled_disk_templates = \ list(set(enabled_disk_templates) - set(old_enabled_disk_templates)) disabled_disk_templates = \ list(set(old_enabled_disk_templates) - set(enabled_disk_templates)) else: enabled_disk_templates = old_enabled_disk_templates return (enabled_disk_templates, new_enabled_disk_templates, disabled_disk_templates) def _GetDiskTemplateSets(self, cluster): """Computes three sets of disk templates. The three sets are: - disk templates that will be enabled after this operation (no matter if they were enabled before or not) - disk templates that get enabled by this operation (thus haven't been enabled before.) - disk templates that get disabled by this operation """ return self._GetDiskTemplateSetsInner(self.op.enabled_disk_templates, cluster.enabled_disk_templates) def _CheckIpolicy(self, cluster, enabled_disk_templates): """Checks the ipolicy. @type cluster: C{objects.Cluster} @param cluster: the cluster's configuration @type enabled_disk_templates: list of string @param enabled_disk_templates: list of (possibly newly) enabled disk templates """ # FIXME: write unit tests for this if self.op.ipolicy: self.new_ipolicy = GetUpdatedIPolicy(cluster.ipolicy, self.op.ipolicy, group_policy=False) CheckIpolicyVsDiskTemplates(self.new_ipolicy, enabled_disk_templates) all_instances = self.cfg.GetAllInstancesInfo().values() violations = set() for group in self.cfg.GetAllNodeGroupsInfo().values(): instances = frozenset( inst for inst in all_instances if compat.any(nuuid in group.members for nuuid in self.cfg.GetInstanceNodes(inst.uuid))) new_ipolicy = objects.FillIPolicy(self.new_ipolicy, group.ipolicy) ipol = masterd.instance.CalculateGroupIPolicy(cluster, group) new = ComputeNewInstanceViolations(ipol, new_ipolicy, instances, self.cfg) if new: violations.update(new) if violations: self.LogWarning("After the ipolicy change the following instances" " violate them: %s", utils.CommaJoin(utils.NiceSort(violations))) else: CheckIpolicyVsDiskTemplates(cluster.ipolicy, enabled_disk_templates) def _CheckDrbdHelperOnNodes(self, drbd_helper, node_uuids): """Checks whether the set DRBD helper actually exists on the nodes. @type drbd_helper: string @param drbd_helper: path of the drbd usermode helper binary @type node_uuids: list of strings @param node_uuids: list of node UUIDs to check for the helper """ # checks given drbd helper on all nodes helpers = self.rpc.call_drbd_helper(node_uuids) for (_, ninfo) in self.cfg.GetMultiNodeInfo(node_uuids): if ninfo.offline: self.LogInfo("Not checking drbd helper on offline node %s", ninfo.name) continue msg = helpers[ninfo.uuid].fail_msg if msg: raise errors.OpPrereqError("Error checking drbd helper on node" " '%s': %s" % (ninfo.name, msg), errors.ECODE_ENVIRON) node_helper = helpers[ninfo.uuid].payload if node_helper != drbd_helper: raise errors.OpPrereqError("Error on node '%s': drbd helper is %s" % (ninfo.name, node_helper), errors.ECODE_ENVIRON) def _CheckDrbdHelper(self, node_uuids, drbd_enabled, drbd_gets_enabled): """Check the DRBD usermode helper. @type node_uuids: list of strings @param node_uuids: a list of nodes' UUIDs @type drbd_enabled: boolean @param drbd_enabled: whether DRBD will be enabled after this operation (no matter if it was disabled before or not) @type drbd_gets_enabled: boolen @param drbd_gets_enabled: true if DRBD was disabled before this operation, but will be enabled afterwards """ if self.op.drbd_helper == '': if drbd_enabled: raise errors.OpPrereqError("Cannot disable drbd helper while" " DRBD is enabled.", errors.ECODE_STATE) if self.cfg.DisksOfType(constants.DT_DRBD8): raise errors.OpPrereqError("Cannot disable drbd helper while" " drbd-based instances exist", errors.ECODE_INVAL) else: if self.op.drbd_helper is not None and drbd_enabled: self._CheckDrbdHelperOnNodes(self.op.drbd_helper, node_uuids) else: if drbd_gets_enabled: current_drbd_helper = self.cfg.GetClusterInfo().drbd_usermode_helper if current_drbd_helper is not None: self._CheckDrbdHelperOnNodes(current_drbd_helper, node_uuids) else: raise errors.OpPrereqError("Cannot enable DRBD without a" " DRBD usermode helper set.", errors.ECODE_STATE) def _CheckInstancesOfDisabledDiskTemplates( self, disabled_disk_templates): """Check whether we try to disable a disk template that is in use. @type disabled_disk_templates: list of string @param disabled_disk_templates: list of disk templates that are going to be disabled by this operation """ for disk_template in disabled_disk_templates: disks_with_type = self.cfg.DisksOfType(disk_template) if disks_with_type: disk_desc = [] for disk in disks_with_type: instance_uuid = self.cfg.GetInstanceForDisk(disk.uuid) instance = self.cfg.GetInstanceInfo(instance_uuid) if instance: instance_desc = "on " + instance.name else: instance_desc = "detached" disk_desc.append("%s (%s)" % (disk, instance_desc)) raise errors.OpPrereqError( "Cannot disable disk template '%s', because there is at least one" " disk using it:\n * %s" % (disk_template, "\n * ".join(disk_desc)), errors.ECODE_STATE) if constants.DT_DISKLESS in disabled_disk_templates: instances = self.cfg.GetAllInstancesInfo() for inst in instances.values(): if not inst.disks: raise errors.OpPrereqError( "Cannot disable disk template 'diskless', because there is at" " least one instance using it:\n * %s" % inst.name, errors.ECODE_STATE) @staticmethod def _CheckInstanceCommunicationNetwork(network, warning_fn): """Check whether an existing network is configured for instance communication. Checks whether an existing network is configured with the parameters that are advisable for instance communication, and otherwise issue security warnings. @type network: L{ganeti.objects.Network} @param network: L{ganeti.objects.Network} object whose configuration is being checked @type warning_fn: function @param warning_fn: function used to print warnings @rtype: None @return: None """ def _MaybeWarn(err, val, default): if val != default: warning_fn("Supplied instance communication network '%s' %s '%s'," " this might pose a security risk (default is '%s').", network.name, err, val, default) if network.network is None: raise errors.OpPrereqError("Supplied instance communication network '%s'" " must have an IPv4 network address.", network.name) _MaybeWarn("has an IPv4 gateway", network.gateway, None) _MaybeWarn("has a non-standard IPv4 network address", network.network, constants.INSTANCE_COMMUNICATION_NETWORK4) _MaybeWarn("has an IPv6 gateway", network.gateway6, None) _MaybeWarn("has a non-standard IPv6 network address", network.network6, constants.INSTANCE_COMMUNICATION_NETWORK6) _MaybeWarn("has a non-standard MAC prefix", network.mac_prefix, constants.INSTANCE_COMMUNICATION_MAC_PREFIX) def CheckPrereq(self): """Check prerequisites. This checks whether the given params don't conflict and if the given volume group is valid. """ node_uuids = self.owned_locks(locking.LEVEL_NODE) self.cluster = cluster = self.cfg.GetClusterInfo() vm_capable_node_uuids = [node.uuid for node in self.cfg.GetAllNodesInfo().values() if node.uuid in node_uuids and node.vm_capable] (enabled_disk_templates, new_enabled_disk_templates, disabled_disk_templates) = self._GetDiskTemplateSets(cluster) self._CheckInstancesOfDisabledDiskTemplates(disabled_disk_templates) self._CheckVgName(vm_capable_node_uuids, enabled_disk_templates, new_enabled_disk_templates) if self.op.file_storage_dir is not None: CheckFileStoragePathVsEnabledDiskTemplates( self.LogWarning, self.op.file_storage_dir, enabled_disk_templates) if self.op.shared_file_storage_dir is not None: CheckSharedFileStoragePathVsEnabledDiskTemplates( self.LogWarning, self.op.shared_file_storage_dir, enabled_disk_templates) drbd_enabled = constants.DT_DRBD8 in enabled_disk_templates drbd_gets_enabled = constants.DT_DRBD8 in new_enabled_disk_templates self._CheckDrbdHelper(vm_capable_node_uuids, drbd_enabled, drbd_gets_enabled) if (self.op.diskparams is not None and constants.DT_DRBD8 in self.op.diskparams): self.LogWarning("Changing DRBD parameters only affects devices created " "in the future, not existing ones") self.LogWarning("You need to shutdown and start (not reboot!) existing " "instances to adopt the changes") self.LogWarning("Alternatively you can swap the secondary node by " "running `gnt-instance replace-disks --new-secondary " "$instance`") # validate params changes if self.op.beparams: objects.UpgradeBeParams(self.op.beparams) utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES) self.new_beparams = cluster.SimpleFillBE(self.op.beparams) if self.op.ndparams: utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES) self.new_ndparams = cluster.SimpleFillND(self.op.ndparams) # TODO: we need a more general way to handle resetting # cluster-level parameters to default values if self.new_ndparams["oob_program"] == "": self.new_ndparams["oob_program"] = \ constants.NDC_DEFAULTS[constants.ND_OOB_PROGRAM] if self.op.hv_state: new_hv_state = MergeAndVerifyHvState(self.op.hv_state, self.cluster.hv_state_static) self.new_hv_state = dict((hv, cluster.SimpleFillHvState(values)) for hv, values in new_hv_state.items()) if self.op.disk_state: new_disk_state = MergeAndVerifyDiskState(self.op.disk_state, self.cluster.disk_state_static) self.new_disk_state = \ dict((storage, dict((name, cluster.SimpleFillDiskState(values)) for name, values in svalues.items())) for storage, svalues in new_disk_state.items()) self._CheckIpolicy(cluster, enabled_disk_templates) if self.op.nicparams: utils.ForceDictType(self.op.nicparams, constants.NICS_PARAMETER_TYPES) self.new_nicparams = cluster.SimpleFillNIC(self.op.nicparams) objects.NIC.CheckParameterSyntax(self.new_nicparams) nic_errors = [] # check all instances for consistency for instance in self.cfg.GetAllInstancesInfo().values(): for nic_idx, nic in enumerate(instance.nics): params_copy = copy.deepcopy(nic.nicparams) params_filled = objects.FillDict(self.new_nicparams, params_copy) # check parameter syntax try: objects.NIC.CheckParameterSyntax(params_filled) except errors.ConfigurationError as err: nic_errors.append("Instance %s, nic/%d: %s" % (instance.name, nic_idx, err)) # if we're moving instances to routed, check that they have an ip target_mode = params_filled[constants.NIC_MODE] if target_mode == constants.NIC_MODE_ROUTED and not nic.ip: nic_errors.append("Instance %s, nic/%d: routed NIC with no ip" " address" % (instance.name, nic_idx)) if nic_errors: raise errors.OpPrereqError("Cannot apply the change, errors:\n%s" % "\n".join(nic_errors), errors.ECODE_INVAL) # hypervisor list/parameters self.new_hvparams = new_hvp = objects.FillDict(cluster.hvparams, {}) if self.op.hvparams: for hv_name, hv_dict in self.op.hvparams.items(): if hv_name not in self.new_hvparams: self.new_hvparams[hv_name] = hv_dict else: self.new_hvparams[hv_name].update(hv_dict) # disk template parameters self.new_diskparams = objects.FillDict(cluster.diskparams, {}) if self.op.diskparams: for dt_name, dt_params in self.op.diskparams.items(): if dt_name not in self.new_diskparams: self.new_diskparams[dt_name] = dt_params else: self.new_diskparams[dt_name].update(dt_params) CheckDiskAccessModeConsistency(self.op.diskparams, self.cfg) # os hypervisor parameters self.new_os_hvp = objects.FillDict(cluster.os_hvp, {}) if self.op.os_hvp: for os_name, hvs in self.op.os_hvp.items(): if os_name not in self.new_os_hvp: self.new_os_hvp[os_name] = hvs else: for hv_name, hv_dict in hvs.items(): if hv_dict is None: # Delete if it exists self.new_os_hvp[os_name].pop(hv_name, None) elif hv_name not in self.new_os_hvp[os_name]: self.new_os_hvp[os_name][hv_name] = hv_dict else: self.new_os_hvp[os_name][hv_name].update(hv_dict) # Cleanup any OS that has an empty hypervisor parameter list, as we don't # need them in the cluster config anymore. for os_name, hvs in list(self.new_os_hvp.items()): if not hvs: self.new_os_hvp.pop(os_name, None) # os parameters self._BuildOSParams(cluster) # changes to the hypervisor list if self.op.enabled_hypervisors is not None: for hv in self.op.enabled_hypervisors: # if the hypervisor doesn't already exist in the cluster # hvparams, we initialize it to empty, and then (in both # cases) we make sure to fill the defaults, as we might not # have a complete defaults list if the hypervisor wasn't # enabled before if hv not in new_hvp: new_hvp[hv] = {} new_hvp[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], new_hvp[hv]) utils.ForceDictType(new_hvp[hv], constants.HVS_PARAMETER_TYPES) if self.op.hvparams or self.op.enabled_hypervisors is not None: # either the enabled list has changed, or the parameters have, validate for hv_name, hv_params in self.new_hvparams.items(): if ((self.op.hvparams and hv_name in self.op.hvparams) or (self.op.enabled_hypervisors and hv_name in self.op.enabled_hypervisors)): # either this is a new hypervisor, or its parameters have changed hv_class = hypervisor.GetHypervisorClass(hv_name) utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES) hv_class.CheckParameterSyntax(hv_params) CheckHVParams(self, node_uuids, hv_name, hv_params) if self.op.os_hvp: # no need to check any newly-enabled hypervisors, since the # defaults have already been checked in the above code-block for os_name, os_hvp in self.new_os_hvp.items(): for hv_name, hv_params in os_hvp.items(): utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES) # we need to fill in the new os_hvp on top of the actual hv_p cluster_defaults = self.new_hvparams.get(hv_name, {}) new_osp = objects.FillDict(cluster_defaults, hv_params) hv_class = hypervisor.GetHypervisorClass(hv_name) hv_class.CheckParameterSyntax(new_osp) CheckHVParams(self, node_uuids, hv_name, new_osp) if self.op.default_iallocator: alloc_script = utils.FindFile(self.op.default_iallocator, constants.IALLOCATOR_SEARCH_PATH, os.path.isfile) if alloc_script is None: raise errors.OpPrereqError("Invalid default iallocator script '%s'" " specified" % self.op.default_iallocator, errors.ECODE_INVAL) if self.op.instance_communication_network: network_name = self.op.instance_communication_network try: network_uuid = self.cfg.LookupNetwork(network_name) except errors.OpPrereqError: network_uuid = None if network_uuid is not None: network = self.cfg.GetNetwork(network_uuid) self._CheckInstanceCommunicationNetwork(network, self.LogWarning) if self.op.compression_tools: CheckCompressionTools(self.op.compression_tools) def _BuildOSParams(self, cluster): "Calculate the new OS parameters for this operation." def _GetNewParams(source, new_params): "Wrapper around GetUpdatedParams." if new_params is None: return source result = objects.FillDict(source, {}) # deep copy of source for os_name in new_params: result[os_name] = GetUpdatedParams(result.get(os_name, {}), new_params[os_name], use_none=True) if not result[os_name]: del result[os_name] # we removed all parameters return result self.new_osp = _GetNewParams(cluster.osparams, self.op.osparams) self.new_osp_private = _GetNewParams(cluster.osparams_private_cluster, self.op.osparams_private_cluster) # Remove os validity check changed_oses = (set(self.new_osp.keys()) | set(self.new_osp_private.keys())) for os_name in changed_oses: os_params = cluster.SimpleFillOS( os_name, self.new_osp.get(os_name, {}), os_params_private=self.new_osp_private.get(os_name, {}) ) # check the parameter validity (remote check) CheckOSParams(self, False, [self.cfg.GetMasterNode()], os_name, os_params, False) def _SetVgName(self, feedback_fn): """Determines and sets the new volume group name. """ if self.op.vg_name is not None: new_volume = self.op.vg_name if not new_volume: new_volume = None if new_volume != self.cfg.GetVGName(): self.cfg.SetVGName(new_volume) else: feedback_fn("Cluster LVM configuration already in desired" " state, not changing") def _SetFileStorageDir(self, feedback_fn): """Set the file storage directory. """ if self.op.file_storage_dir is not None: if self.cluster.file_storage_dir == self.op.file_storage_dir: feedback_fn("Global file storage dir already set to value '%s'" % self.cluster.file_storage_dir) else: self.cluster.file_storage_dir = self.op.file_storage_dir def _SetSharedFileStorageDir(self, feedback_fn): """Set the shared file storage directory. """ if self.op.shared_file_storage_dir is not None: if self.cluster.shared_file_storage_dir == \ self.op.shared_file_storage_dir: feedback_fn("Global shared file storage dir already set to value '%s'" % self.cluster.shared_file_storage_dir) else: self.cluster.shared_file_storage_dir = self.op.shared_file_storage_dir def _SetDrbdHelper(self, feedback_fn): """Set the DRBD usermode helper. """ if self.op.drbd_helper is not None: if not constants.DT_DRBD8 in self.cluster.enabled_disk_templates: feedback_fn("Note that you specified a drbd user helper, but did not" " enable the drbd disk template.") new_helper = self.op.drbd_helper if not new_helper: new_helper = None if new_helper != self.cfg.GetDRBDHelper(): self.cfg.SetDRBDHelper(new_helper) else: feedback_fn("Cluster DRBD helper already in desired state," " not changing") @staticmethod def _EnsureInstanceCommunicationNetwork(cfg, network_name): """Ensure that the instance communication network exists and is connected to all groups. The instance communication network given by L{network_name} it is created, if necessary, via the opcode 'OpNetworkAdd'. Also, the instance communication network is connected to all existing node groups, if necessary, via the opcode 'OpNetworkConnect'. @type cfg: L{config.ConfigWriter} @param cfg: cluster configuration @type network_name: string @param network_name: instance communication network name @rtype: L{ganeti.cmdlib.ResultWithJobs} or L{None} @return: L{ganeti.cmdlib.ResultWithJobs} if the instance communication needs to be created or it needs to be connected to a group, otherwise L{None} """ jobs = [] try: network_uuid = cfg.LookupNetwork(network_name) network_exists = True except errors.OpPrereqError: network_exists = False if not network_exists: jobs.append(AddInstanceCommunicationNetworkOp(network_name)) for group_uuid in cfg.GetNodeGroupList(): group = cfg.GetNodeGroup(group_uuid) if network_exists: network_connected = network_uuid in group.networks else: # The network was created asynchronously by the previous # opcode and, therefore, we don't have access to its # network_uuid. As a result, we assume that the network is # not connected to any group yet. network_connected = False if not network_connected: op = ConnectInstanceCommunicationNetworkOp(group_uuid, network_name) jobs.append(op) if jobs: return ResultWithJobs([jobs]) else: return None @staticmethod def _ModifyInstanceCommunicationNetwork(cfg, network_name, feedback_fn): """Update the instance communication network stored in the cluster configuration. Compares the user-supplied instance communication network against the one stored in the Ganeti cluster configuration. If there is a change, the instance communication network may be possibly created and connected to all groups (see L{LUClusterSetParams._EnsureInstanceCommunicationNetwork}). @type cfg: L{config.ConfigWriter} @param cfg: cluster configuration @type network_name: string @param network_name: instance communication network name @type feedback_fn: function @param feedback_fn: see L{ganeti.cmdlist.base.LogicalUnit} @rtype: L{LUClusterSetParams._EnsureInstanceCommunicationNetwork} or L{None} @return: see L{LUClusterSetParams._EnsureInstanceCommunicationNetwork} """ config_network_name = cfg.GetInstanceCommunicationNetwork() if network_name == config_network_name: feedback_fn("Instance communication network already is '%s', nothing to" " do." % network_name) else: try: cfg.LookupNetwork(config_network_name) feedback_fn("Previous instance communication network '%s'" " should be removed manually." % config_network_name) except errors.OpPrereqError: pass if network_name: feedback_fn("Changing instance communication network to '%s', only new" " instances will be affected." % network_name) else: feedback_fn("Disabling instance communication network, only new" " instances will be affected.") cfg.SetInstanceCommunicationNetwork(network_name) if network_name: return LUClusterSetParams._EnsureInstanceCommunicationNetwork( cfg, network_name) else: return None def Exec(self, feedback_fn): """Change the parameters of the cluster. """ # re-read the fresh configuration self.cluster = self.cfg.GetClusterInfo() if self.op.enabled_disk_templates: self.cluster.enabled_disk_templates = \ list(self.op.enabled_disk_templates) # save the changes self.cfg.Update(self.cluster, feedback_fn) self._SetVgName(feedback_fn) self.cluster = self.cfg.GetClusterInfo() self._SetFileStorageDir(feedback_fn) self._SetSharedFileStorageDir(feedback_fn) self.cfg.Update(self.cluster, feedback_fn) self._SetDrbdHelper(feedback_fn) # re-read the fresh configuration again self.cluster = self.cfg.GetClusterInfo() ensure_kvmd = False stop_kvmd_silently = not ( constants.HT_KVM in self.cluster.enabled_hypervisors or (self.op.enabled_hypervisors is not None and constants.HT_KVM in self.op.enabled_hypervisors)) active = constants.DATA_COLLECTOR_STATE_ACTIVE if self.op.enabled_data_collectors is not None: for name, val in self.op.enabled_data_collectors.items(): self.cluster.data_collectors[name][active] = val if self.op.data_collector_interval: internal = constants.DATA_COLLECTOR_PARAMETER_INTERVAL for name, val in self.op.data_collector_interval.items(): self.cluster.data_collectors[name][internal] = int(val) if self.op.hvparams: self.cluster.hvparams = self.new_hvparams if self.op.os_hvp: self.cluster.os_hvp = self.new_os_hvp if self.op.enabled_hypervisors is not None: self.cluster.hvparams = self.new_hvparams self.cluster.enabled_hypervisors = self.op.enabled_hypervisors ensure_kvmd = True if self.op.beparams: self.cluster.beparams[constants.PP_DEFAULT] = self.new_beparams if self.op.nicparams: self.cluster.nicparams[constants.PP_DEFAULT] = self.new_nicparams if self.op.ipolicy: self.cluster.ipolicy = self.new_ipolicy if self.op.osparams: self.cluster.osparams = self.new_osp if self.op.osparams_private_cluster: self.cluster.osparams_private_cluster = self.new_osp_private if self.op.ndparams: self.cluster.ndparams = self.new_ndparams if self.op.diskparams: self.cluster.diskparams = self.new_diskparams if self.op.hv_state: self.cluster.hv_state_static = self.new_hv_state if self.op.disk_state: self.cluster.disk_state_static = self.new_disk_state if self.op.candidate_pool_size is not None: self.cluster.candidate_pool_size = self.op.candidate_pool_size # we need to update the pool size here, otherwise the save will fail master_node = self.cfg.GetMasterNode() potential_master_candidates = self.cfg.GetPotentialMasterCandidates() modify_ssh_setup = self.cfg.GetClusterInfo().modify_ssh_setup AdjustCandidatePool( self, [], master_node, potential_master_candidates, feedback_fn, modify_ssh_setup) if self.op.max_running_jobs is not None: self.cluster.max_running_jobs = self.op.max_running_jobs if self.op.max_tracked_jobs is not None: self.cluster.max_tracked_jobs = self.op.max_tracked_jobs if self.op.maintain_node_health is not None: self.cluster.maintain_node_health = self.op.maintain_node_health if self.op.modify_etc_hosts is not None: self.cluster.modify_etc_hosts = self.op.modify_etc_hosts if self.op.prealloc_wipe_disks is not None: self.cluster.prealloc_wipe_disks = self.op.prealloc_wipe_disks if self.op.add_uids is not None: uidpool.AddToUidPool(self.cluster.uid_pool, self.op.add_uids) if self.op.remove_uids is not None: uidpool.RemoveFromUidPool(self.cluster.uid_pool, self.op.remove_uids) if self.op.uid_pool is not None: self.cluster.uid_pool = self.op.uid_pool if self.op.default_iallocator is not None: self.cluster.default_iallocator = self.op.default_iallocator if self.op.default_iallocator_params is not None: self.cluster.default_iallocator_params = self.op.default_iallocator_params if self.op.reserved_lvs is not None: self.cluster.reserved_lvs = self.op.reserved_lvs if self.op.use_external_mip_script is not None: self.cluster.use_external_mip_script = self.op.use_external_mip_script if self.op.enabled_user_shutdown is not None and \ self.cluster.enabled_user_shutdown != self.op.enabled_user_shutdown: self.cluster.enabled_user_shutdown = self.op.enabled_user_shutdown ensure_kvmd = True def helper_os(aname, mods, desc): desc += " OS list" lst = getattr(self.cluster, aname) for key, val in mods: if key == constants.DDM_ADD: if val in lst: feedback_fn("OS %s already in %s, ignoring" % (val, desc)) else: lst.append(val) elif key == constants.DDM_REMOVE: if val in lst: lst.remove(val) else: feedback_fn("OS %s not found in %s, ignoring" % (val, desc)) else: raise errors.ProgrammerError("Invalid modification '%s'" % key) if self.op.hidden_os: helper_os("hidden_os", self.op.hidden_os, "hidden") if self.op.blacklisted_os: helper_os("blacklisted_os", self.op.blacklisted_os, "blacklisted") if self.op.mac_prefix: self.cluster.mac_prefix = self.op.mac_prefix if self.op.master_netdev: master_params = self.cfg.GetMasterNetworkParameters() ems = self.cfg.GetUseExternalMipScript() feedback_fn("Shutting down master ip on the current netdev (%s)" % self.cluster.master_netdev) result = self.rpc.call_node_deactivate_master_ip(master_params.uuid, master_params, ems) if not self.op.force: result.Raise("Could not disable the master ip") else: if result.fail_msg: msg = ("Could not disable the master ip (continuing anyway): %s" % result.fail_msg) feedback_fn(msg) feedback_fn("Changing master_netdev from %s to %s" % (master_params.netdev, self.op.master_netdev)) self.cluster.master_netdev = self.op.master_netdev if self.op.master_netmask: master_params = self.cfg.GetMasterNetworkParameters() feedback_fn("Changing master IP netmask to %s" % self.op.master_netmask) result = self.rpc.call_node_change_master_netmask( master_params.uuid, master_params.netmask, self.op.master_netmask, master_params.ip, master_params.netdev) result.Warn("Could not change the master IP netmask", feedback_fn) self.cluster.master_netmask = self.op.master_netmask if self.op.install_image: self.cluster.install_image = self.op.install_image if self.op.zeroing_image is not None: CheckImageValidity(self.op.zeroing_image, "Zeroing image must be an absolute path or a URL") self.cluster.zeroing_image = self.op.zeroing_image self.cfg.Update(self.cluster, feedback_fn) if self.op.master_netdev: master_params = self.cfg.GetMasterNetworkParameters() feedback_fn("Starting the master ip on the new master netdev (%s)" % self.op.master_netdev) ems = self.cfg.GetUseExternalMipScript() result = self.rpc.call_node_activate_master_ip(master_params.uuid, master_params, ems) result.Warn("Could not re-enable the master ip on the master," " please restart manually", self.LogWarning) # Even though 'self.op.enabled_user_shutdown' is being tested # above, the RPCs can only be done after 'self.cfg.Update' because # this will update the cluster object and sync 'Ssconf', and kvmd # uses 'Ssconf'. if ensure_kvmd: EnsureKvmdOnNodes(self, feedback_fn, silent_stop=stop_kvmd_silently) if self.op.compression_tools is not None: self.cfg.SetCompressionTools(self.op.compression_tools) network_name = self.op.instance_communication_network if network_name is not None: return self._ModifyInstanceCommunicationNetwork(self.cfg, network_name, feedback_fn) else: return None ganeti-3.1.0~rc2/lib/cmdlib/cluster/verify.py000064400000000000000000002642501476477700300211540ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units for cluster verification.""" import itertools import logging import operator import re import time import ganeti.masterd.instance import ganeti.rpc.node as rpc from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import pathutils from ganeti import utils from ganeti import vcluster from ganeti import hypervisor from ganeti import opcodes from ganeti.cmdlib.base import LogicalUnit, NoHooksLU, ResultWithJobs from ganeti.cmdlib.common import ShareAll, ComputeAncillaryFiles, \ CheckNodePVs, ComputeIPolicyInstanceViolation, AnnotateDiskParams, \ SupportsOob def _GetAllHypervisorParameters(cluster, instances): """Compute the set of all hypervisor parameters. @type cluster: L{objects.Cluster} @param cluster: the cluster object @param instances: list of L{objects.Instance} @param instances: additional instances from which to obtain parameters @rtype: list of (origin, hypervisor, parameters) @return: a list with all parameters found, indicating the hypervisor they apply to, and the origin (can be "cluster", "os X", or "instance Y") """ hvp_data = [] for hv_name in cluster.enabled_hypervisors: hvp_data.append(("cluster", hv_name, cluster.GetHVDefaults(hv_name))) for os_name, os_hvp in cluster.os_hvp.items(): for hv_name, hv_params in os_hvp.items(): if hv_params: full_params = cluster.GetHVDefaults(hv_name, os_name=os_name) hvp_data.append(("os %s" % os_name, hv_name, full_params)) # TODO: collapse identical parameter values in a single one for instance in instances: if instance.hvparams: hvp_data.append(("instance %s" % instance.name, instance.hypervisor, cluster.FillHV(instance))) return hvp_data class _VerifyErrors(object): """Mix-in for cluster/group verify LUs. It provides _Error and _ErrorIf, and updates the self.bad boolean. (Expects self.op and self._feedback_fn to be available.) """ ETYPE_ERROR = constants.CV_ERROR ETYPE_WARNING = constants.CV_WARNING def _ErrorMsgList(self, error_descriptor, object_name, message_list, log_type=ETYPE_ERROR): """Format multiple error messages. Based on the opcode's error_codes parameter, either format a parseable error code, or a simpler error string. This must be called only from Exec and functions called from Exec. @type error_descriptor: tuple (string, string, string) @param error_descriptor: triplet describing the error (object_type, code, description) @type object_name: string @param object_name: name of object (instance, node ..) the error relates to @type message_list: list of strings @param message_list: body of error messages @type log_type: string @param log_type: log message type (WARNING, ERROR ..) """ # Called with empty list - nothing to do if not message_list: return object_type, error_code, _ = error_descriptor # If the error code is in the list of ignored errors, demote the error to a # warning if error_code in self.op.ignore_errors: # pylint: disable=E1101 log_type = self.ETYPE_WARNING prefixed_list = [] if self.op.error_codes: # This is a mix-in. pylint: disable=E1101 for msg in message_list: prefixed_list.append(" - %s:%s:%s:%s:%s" % ( log_type, error_code, object_type, object_name, msg)) else: if not object_name: object_name = "" for msg in message_list: prefixed_list.append(" - %s: %s %s: %s" % ( log_type, object_type, object_name, msg)) # Report messages via the feedback_fn # pylint: disable=E1101 self._feedback_fn(constants.ELOG_MESSAGE_LIST, prefixed_list) # do not mark the operation as failed for WARN cases only if log_type == self.ETYPE_ERROR: self.bad = True def _ErrorMsg(self, error_descriptor, object_name, message, log_type=ETYPE_ERROR): """Log a single error message. """ self._ErrorMsgList(error_descriptor, object_name, [message], log_type) # TODO: Replace this method with a cleaner interface, get rid of the if # condition as it only rarely saves lines, but makes things less readable. def _ErrorIf(self, cond, *args, **kwargs): """Log an error message if the passed condition is True. """ if (bool(cond) or self.op.debug_simulate_errors): # pylint: disable=E1101 self._Error(*args, **kwargs) # TODO: Replace this method with a cleaner interface def _Error(self, ecode, item, message, *args, **kwargs): """Log an error message if the passed condition is True. """ #TODO: Remove 'code' argument in favour of using log_type log_type = kwargs.get('code', self.ETYPE_ERROR) if args: message = message % args self._ErrorMsgList(ecode, item, [message], log_type=log_type) class LUClusterVerify(NoHooksLU): """Submits all jobs necessary to verify the cluster. """ REQ_BGL = False def ExpandNames(self): self.needed_locks = {} def Exec(self, feedback_fn): jobs = [] if self.op.group_name: groups = [self.op.group_name] depends_fn = lambda: None else: groups = self.cfg.GetNodeGroupList() # Verify global configuration jobs.append([ opcodes.OpClusterVerifyConfig(ignore_errors=self.op.ignore_errors), ]) # Always depend on global verification depends_fn = lambda: [(-len(jobs), [])] jobs.extend( [opcodes.OpClusterVerifyGroup(group_name=group, ignore_errors=self.op.ignore_errors, depends=depends_fn(), verify_clutter=self.op.verify_clutter)] for group in groups) # Fix up all parameters for op in itertools.chain(*jobs): op.debug_simulate_errors = self.op.debug_simulate_errors op.verbose = self.op.verbose op.error_codes = self.op.error_codes try: op.skip_checks = self.op.skip_checks except AttributeError: assert not isinstance(op, opcodes.OpClusterVerifyGroup) return ResultWithJobs(jobs) class LUClusterVerifyDisks(NoHooksLU): """Verifies the cluster disks status. """ REQ_BGL = False def ExpandNames(self): self.share_locks = ShareAll() if self.op.group_name: self.needed_locks = { locking.LEVEL_NODEGROUP: [self.cfg.LookupNodeGroup(self.op.group_name)] } else: self.needed_locks = { locking.LEVEL_NODEGROUP: locking.ALL_SET, } def Exec(self, feedback_fn): group_names = self.owned_locks(locking.LEVEL_NODEGROUP) instances = self.cfg.GetInstanceList() only_ext = compat.all( self.cfg.GetInstanceDiskTemplate(i) == constants.DT_EXT for i in instances) # We skip current NodeGroup verification if there are only external storage # devices. Currently we provide an interface for external storage provider # for disk verification implementations, however current ExtStorageDevice # does not provide an API for this yet. # # This check needs to be revisited if ES_ACTION_VERIFY on ExtStorageDevice # is implemented. if only_ext: logging.info("All instances have ext storage, skipping verify disks.") return ResultWithJobs([]) else: # Submit one instance of L{opcodes.OpGroupVerifyDisks} per node group return ResultWithJobs( [[opcodes.OpGroupVerifyDisks(group_name=group, is_strict=self.op.is_strict)] for group in group_names]) class LUClusterVerifyConfig(NoHooksLU, _VerifyErrors): """Verifies the cluster config. """ REQ_BGL = False def _VerifyHVP(self, hvp_data): """Verifies locally the syntax of the hypervisor parameters. """ for item, hv_name, hv_params in hvp_data: msg = ("hypervisor %s parameters syntax check (source %s): %%s" % (item, hv_name)) try: hv_class = hypervisor.GetHypervisorClass(hv_name) utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES) hv_class.CheckParameterSyntax(hv_params) except errors.GenericError as err: self._ErrorIf(True, constants.CV_ECLUSTERCFG, None, msg % str(err)) def ExpandNames(self): self.needed_locks = dict.fromkeys(locking.LEVELS, locking.ALL_SET) self.share_locks = ShareAll() def CheckPrereq(self): """Check prerequisites. """ # Retrieve all information self.all_group_info = self.cfg.GetAllNodeGroupsInfo() self.all_node_info = self.cfg.GetAllNodesInfo() self.all_inst_info = self.cfg.GetAllInstancesInfo() def Exec(self, feedback_fn): """Verify integrity of cluster, performing various test on nodes. """ self.bad = False self._feedback_fn = feedback_fn feedback_fn("* Verifying cluster config") msg_list = self.cfg.VerifyConfig() self._ErrorMsgList(constants.CV_ECLUSTERCFG, None, msg_list) feedback_fn("* Verifying cluster certificate files") for cert_filename in pathutils.ALL_CERT_FILES: (errcode, msg) = utils.VerifyCertificate(cert_filename) self._ErrorIf(errcode, constants.CV_ECLUSTERCERT, None, msg, code=errcode) self._ErrorIf(not utils.CanRead(constants.LUXID_USER, pathutils.NODED_CERT_FILE), constants.CV_ECLUSTERCERT, None, pathutils.NODED_CERT_FILE + " must be accessible by the " + constants.LUXID_USER + " user") feedback_fn("* Verifying hypervisor parameters") instances_info = list(self.all_inst_info.values()) self._VerifyHVP(_GetAllHypervisorParameters(self.cfg.GetClusterInfo(), instances_info)) feedback_fn("* Verifying all nodes belong to an existing group") # We do this verification here because, should this bogus circumstance # occur, it would never be caught by VerifyGroup, which only acts on # nodes/instances reachable from existing node groups. dangling_nodes = set(node for node in self.all_node_info.values() if node.group not in self.all_group_info) dangling_instances = {} no_node_instances = [] for inst in self.all_inst_info.values(): if inst.primary_node in [node.uuid for node in dangling_nodes]: dangling_instances.setdefault(inst.primary_node, []).append(inst) elif inst.primary_node not in self.all_node_info: no_node_instances.append(inst) pretty_dangling = [ "%s (%s)" % (node.name, utils.CommaJoin(inst.name for inst in dangling_instances.get(node.uuid, []))) for node in dangling_nodes] self._ErrorIf(bool(dangling_nodes), constants.CV_ECLUSTERDANGLINGNODES, None, "the following nodes (and their instances) belong to a non" " existing group: %s", utils.CommaJoin(pretty_dangling)) self._ErrorIf(bool(no_node_instances), constants.CV_ECLUSTERDANGLINGINST, None, "the following instances have a non-existing primary-node:" " %s", utils.CommaJoin(inst.name for inst in no_node_instances)) return not self.bad class LUClusterVerifyGroup(LogicalUnit, _VerifyErrors): """Verifies the status of a node group. """ HPATH = "cluster-verify" HTYPE = constants.HTYPE_CLUSTER REQ_BGL = False _HOOKS_INDENT_RE = re.compile("^", re.M) class NodeImage(object): """A class representing the logical and physical status of a node. @type uuid: string @ivar uuid: the node UUID to which this object refers @ivar volumes: a structure as returned from L{ganeti.backend.GetVolumeList} (runtime) @ivar instances: a list of running instances (runtime) @ivar pinst: list of configured primary instances (config) @ivar sinst: list of configured secondary instances (config) @ivar sbp: dictionary of {primary-node: list of instances} for all instances for which this node is secondary (config) @ivar mfree: free memory, as reported by hypervisor (runtime) @ivar dfree: free disk, as reported by the node (runtime) @ivar offline: the offline status (config) @type rpc_fail: boolean @ivar rpc_fail: whether the RPC verify call was successfull (overall, not whether the individual keys were correct) (runtime) @type lvm_fail: boolean @ivar lvm_fail: whether the RPC call didn't return valid LVM data @type hyp_fail: boolean @ivar hyp_fail: whether the RPC call didn't return the instance list @type ghost: boolean @ivar ghost: whether this is a known node or not (config) @type os_fail: boolean @ivar os_fail: whether the RPC call didn't return valid OS data @type oslist: list @ivar oslist: list of OSes as diagnosed by DiagnoseOS @type vm_capable: boolean @ivar vm_capable: whether the node can host instances @type pv_min: float @ivar pv_min: size in MiB of the smallest PVs @type pv_max: float @ivar pv_max: size in MiB of the biggest PVs """ def __init__(self, offline=False, uuid=None, vm_capable=True): self.uuid = uuid self.volumes = {} self.instances = [] self.pinst = [] self.sinst = [] self.sbp = {} self.mfree = 0 self.dfree = 0 self.offline = offline self.vm_capable = vm_capable self.rpc_fail = False self.lvm_fail = False self.hyp_fail = False self.ghost = False self.os_fail = False self.oslist = {} self.pv_min = None self.pv_max = None def ExpandNames(self): # This raises errors.OpPrereqError on its own: self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name) # Get instances in node group; this is unsafe and needs verification later inst_uuids = \ self.cfg.GetNodeGroupInstances(self.group_uuid, primary_only=True) self.needed_locks = { locking.LEVEL_INSTANCE: self.cfg.GetInstanceNames(inst_uuids), locking.LEVEL_NODEGROUP: [self.group_uuid], locking.LEVEL_NODE: [], } self.share_locks = ShareAll() def DeclareLocks(self, level): if level == locking.LEVEL_NODE: # Get members of node group; this is unsafe and needs verification later nodes = set(self.cfg.GetNodeGroup(self.group_uuid).members) # In Exec(), we warn about mirrored instances that have primary and # secondary living in separate node groups. To fully verify that # volumes for these instances are healthy, we will need to do an # extra call to their secondaries. We ensure here those nodes will # be locked. for inst_name in self.owned_locks(locking.LEVEL_INSTANCE): # Important: access only the instances whose lock is owned instance = self.cfg.GetInstanceInfoByName(inst_name) disks = self.cfg.GetInstanceDisks(instance.uuid) if utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR): nodes.update(self.cfg.GetInstanceSecondaryNodes(instance.uuid)) self.needed_locks[locking.LEVEL_NODE] = nodes def CheckPrereq(self): assert self.group_uuid in self.owned_locks(locking.LEVEL_NODEGROUP) self.group_info = self.cfg.GetNodeGroup(self.group_uuid) group_node_uuids = set(self.group_info.members) group_inst_uuids = \ self.cfg.GetNodeGroupInstances(self.group_uuid, primary_only=True) unlocked_node_uuids = \ group_node_uuids.difference(self.owned_locks(locking.LEVEL_NODE)) unlocked_inst_uuids = \ group_inst_uuids.difference( [self.cfg.GetInstanceInfoByName(name).uuid for name in self.owned_locks(locking.LEVEL_INSTANCE)]) if unlocked_node_uuids: raise errors.OpPrereqError( "Missing lock for nodes: %s" % utils.CommaJoin(self.cfg.GetNodeNames(unlocked_node_uuids)), errors.ECODE_STATE) if unlocked_inst_uuids: raise errors.OpPrereqError( "Missing lock for instances: %s" % utils.CommaJoin(self.cfg.GetInstanceNames(unlocked_inst_uuids)), errors.ECODE_STATE) self.all_node_info = self.cfg.GetAllNodesInfo() self.all_inst_info = self.cfg.GetAllInstancesInfo() self.all_disks_info = self.cfg.GetAllDisksInfo() self.my_node_uuids = group_node_uuids self.my_node_info = dict((node_uuid, self.all_node_info[node_uuid]) for node_uuid in group_node_uuids) self.my_inst_uuids = group_inst_uuids self.my_inst_info = dict((inst_uuid, self.all_inst_info[inst_uuid]) for inst_uuid in group_inst_uuids) # We detect here the nodes that will need the extra RPC calls for verifying # split LV volumes; they should be locked. extra_lv_nodes = {} for inst in self.my_inst_info.values(): disks = self.cfg.GetInstanceDisks(inst.uuid) if utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR): inst_nodes = self.cfg.GetInstanceNodes(inst.uuid) for nuuid in inst_nodes: if self.all_node_info[nuuid].group != self.group_uuid: if nuuid in extra_lv_nodes: extra_lv_nodes[nuuid].append(inst.name) else: extra_lv_nodes[nuuid] = [inst.name] extra_lv_nodes_set = set(extra_lv_nodes) unlocked_lv_nodes = \ extra_lv_nodes_set.difference(self.owned_locks(locking.LEVEL_NODE)) if unlocked_lv_nodes: node_strings = ['%s: [%s]' % ( self.cfg.GetNodeName(node), utils.CommaJoin(extra_lv_nodes[node])) for node in unlocked_lv_nodes] raise errors.OpPrereqError("Missing node locks for LV check: %s" % utils.CommaJoin(node_strings), errors.ECODE_STATE) self.extra_lv_nodes = list(extra_lv_nodes_set) def _VerifyNode(self, ninfo, nresult): """Perform some basic validation on data returned from a node. - check the result data structure is well formed and has all the mandatory fields - check ganeti version @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the results from the node @rtype: boolean @return: whether overall this call was successful (and we can expect reasonable values in the respose) """ # main result, nresult should be a non-empty dict test = not nresult or not isinstance(nresult, dict) self._ErrorIf(test, constants.CV_ENODERPC, ninfo.name, "unable to verify node: no data returned") if test: return False # compares ganeti version local_version = constants.PROTOCOL_VERSION remote_version = nresult.get("version", None) test = not (remote_version and isinstance(remote_version, (list, tuple)) and len(remote_version) == 2) self._ErrorIf(test, constants.CV_ENODERPC, ninfo.name, "connection to node returned invalid data") if test: return False test = local_version != remote_version[0] self._ErrorIf(test, constants.CV_ENODEVERSION, ninfo.name, "incompatible protocol versions: master %s," " node %s", local_version, remote_version[0]) if test: return False # node seems compatible, we can actually try to look into its results # full package version self._ErrorIf(constants.RELEASE_VERSION != remote_version[1], constants.CV_ENODEVERSION, ninfo.name, "software version mismatch: master %s, node %s", constants.RELEASE_VERSION, remote_version[1], code=self.ETYPE_WARNING) hyp_result = nresult.get(constants.NV_HYPERVISOR, None) if ninfo.vm_capable and isinstance(hyp_result, dict): for hv_name, hv_result in hyp_result.items(): test = hv_result is not None self._ErrorIf(test, constants.CV_ENODEHV, ninfo.name, "hypervisor %s verify failure: '%s'", hv_name, hv_result) hvp_result = nresult.get(constants.NV_HVPARAMS, None) if ninfo.vm_capable and isinstance(hvp_result, list): for item, hv_name, hv_result in hvp_result: self._ErrorIf(True, constants.CV_ENODEHV, ninfo.name, "hypervisor %s parameter verify failure (source %s): %s", hv_name, item, hv_result) test = nresult.get(constants.NV_NODESETUP, ["Missing NODESETUP results"]) self._ErrorIf(test, constants.CV_ENODESETUP, ninfo.name, "node setup error: %s", "; ".join(test)) return True def _VerifyNodeTime(self, ninfo, nresult, nvinfo_starttime, nvinfo_endtime): """Check the node time. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @param nvinfo_starttime: the start time of the RPC call @param nvinfo_endtime: the end time of the RPC call """ ntime = nresult.get(constants.NV_TIME, None) try: ntime_merged = utils.MergeTime(ntime) except (ValueError, TypeError): self._ErrorIf(True, constants.CV_ENODETIME, ninfo.name, "Node returned invalid time") return if ntime_merged < (nvinfo_starttime - constants.NODE_MAX_CLOCK_SKEW): ntime_diff = "%.01fs" % abs(nvinfo_starttime - ntime_merged) elif ntime_merged > (nvinfo_endtime + constants.NODE_MAX_CLOCK_SKEW): ntime_diff = "%.01fs" % abs(ntime_merged - nvinfo_endtime) else: ntime_diff = None self._ErrorIf(ntime_diff is not None, constants.CV_ENODETIME, ninfo.name, "Node time diverges by at least %s from master node time", ntime_diff) def _UpdateVerifyNodeLVM(self, ninfo, nresult, vg_name, nimg): """Check the node LVM results and update info for cross-node checks. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @param vg_name: the configured VG name @type nimg: L{NodeImage} @param nimg: node image """ if vg_name is None: return # checks vg existence and size > 20G vglist = nresult.get(constants.NV_VGLIST, None) test = not vglist self._ErrorIf(test, constants.CV_ENODELVM, ninfo.name, "unable to check volume groups") if not test: vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name, constants.MIN_VG_SIZE) self._ErrorIf(vgstatus, constants.CV_ENODELVM, ninfo.name, vgstatus) # Check PVs (errmsgs, pvminmax) = CheckNodePVs(nresult, self._exclusive_storage) for em in errmsgs: self._Error(constants.CV_ENODELVM, ninfo.name, em) if pvminmax is not None: (nimg.pv_min, nimg.pv_max) = pvminmax def _VerifyGroupDRBDVersion(self, node_verify_infos): """Check cross-node DRBD version consistency. @type node_verify_infos: dict @param node_verify_infos: infos about nodes as returned from the node_verify call. """ node_versions = {} for node_uuid, ndata in node_verify_infos.items(): nresult = ndata.payload if nresult: version = nresult.get(constants.NV_DRBDVERSION, None) if version: node_versions[node_uuid] = version if len(set(node_versions.values())) > 1: for node_uuid, version in sorted(node_versions.items()): msg = "DRBD version mismatch: %s" % version self._Error(constants.CV_ENODEDRBDHELPER, node_uuid, msg, code=self.ETYPE_WARNING) def _VerifyGroupLVM(self, node_image, vg_name): """Check cross-node consistency in LVM. @type node_image: dict @param node_image: info about nodes, mapping from node to names to L{NodeImage} objects @param vg_name: the configured VG name """ if vg_name is None: return # Only exclusive storage needs this kind of checks if not self._exclusive_storage: return # exclusive_storage wants all PVs to have the same size (approximately), # if the smallest and the biggest ones are okay, everything is fine. # pv_min is None iff pv_max is None vals = [ni for ni in node_image.values() if ni.pv_min is not None] if not vals: return (pvmin, minnode_uuid) = min((ni.pv_min, ni.uuid) for ni in vals) (pvmax, maxnode_uuid) = max((ni.pv_max, ni.uuid) for ni in vals) bad = utils.LvmExclusiveTestBadPvSizes(pvmin, pvmax) self._ErrorIf(bad, constants.CV_EGROUPDIFFERENTPVSIZE, self.group_info.name, "PV sizes differ too much in the group; smallest (%s MB) is" " on %s, biggest (%s MB) is on %s", pvmin, self.cfg.GetNodeName(minnode_uuid), pvmax, self.cfg.GetNodeName(maxnode_uuid)) def _VerifyNodeBridges(self, ninfo, nresult, bridges): """Check the node bridges. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @param bridges: the expected list of bridges """ if not bridges: return missing = nresult.get(constants.NV_BRIDGES, None) test = not isinstance(missing, list) self._ErrorIf(test, constants.CV_ENODENET, ninfo.name, "did not return valid bridge information") if not test: self._ErrorIf(bool(missing), constants.CV_ENODENET, ninfo.name, "missing bridges: %s" % utils.CommaJoin(sorted(missing))) def _VerifyNodeUserScripts(self, ninfo, nresult): """Check the results of user scripts presence and executability on the node @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node """ test = not constants.NV_USERSCRIPTS in nresult self._ErrorIf(test, constants.CV_ENODEUSERSCRIPTS, ninfo.name, "did not return user scripts information") broken_scripts = nresult.get(constants.NV_USERSCRIPTS, None) if not test: self._ErrorIf(broken_scripts, constants.CV_ENODEUSERSCRIPTS, ninfo.name, "user scripts not present or not executable: %s" % utils.CommaJoin(sorted(broken_scripts))) def _VerifyNodeNetwork(self, ninfo, nresult): """Check the node network connectivity results. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node """ test = constants.NV_NODELIST not in nresult self._ErrorIf(test, constants.CV_ENODESSH, ninfo.name, "node hasn't returned node ssh connectivity data") if not test: if nresult[constants.NV_NODELIST]: for a_node, a_msg in nresult[constants.NV_NODELIST].items(): self._ErrorIf(True, constants.CV_ENODESSH, ninfo.name, "ssh communication with node '%s': %s", a_node, a_msg) if constants.NV_NODENETTEST not in nresult: self._ErrorMsg(constants.CV_ENODENET, ninfo.name, "node hasn't returned node tcp connectivity data") elif nresult[constants.NV_NODENETTEST]: nlist = utils.NiceSort(nresult[constants.NV_NODENETTEST]) msglist = [] for node in nlist: msglist.append("tcp communication with node '%s': %s" % (node, nresult[constants.NV_NODENETTEST][node])) self._ErrorMsgList(constants.CV_ENODENET, ninfo.name, msglist) if constants.NV_MASTERIP not in nresult: self._ErrorMsg(constants.CV_ENODENET, ninfo.name, "node hasn't returned node master IP reachability data") elif nresult[constants.NV_MASTERIP] is False: # be explicit, could be None if ninfo.uuid == self.master_node: msg = "the master node cannot reach the master IP (not configured?)" else: msg = "cannot reach the master IP" self._ErrorMsg(constants.CV_ENODENET, ninfo.name, msg) def _VerifyInstance(self, instance, node_image, diskstatus): """Verify an instance. This function checks to see if the required block devices are available on the instance's node, and that the nodes are in the correct state. """ pnode_uuid = instance.primary_node pnode_img = node_image[pnode_uuid] groupinfo = self.cfg.GetAllNodeGroupsInfo() node_vol_should = {} self.cfg.GetInstanceLVsByNode(instance.uuid, lvmap=node_vol_should) cluster = self.cfg.GetClusterInfo() ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(cluster, self.group_info) err = ComputeIPolicyInstanceViolation(ipolicy, instance, self.cfg) self._ErrorIf(err, constants.CV_EINSTANCEPOLICY, instance.name, utils.CommaJoin(err), code=self.ETYPE_WARNING) for node_uuid in node_vol_should: n_img = node_image[node_uuid] if n_img.offline or n_img.rpc_fail or n_img.lvm_fail: # ignore missing volumes on offline or broken nodes continue for volume in node_vol_should[node_uuid]: test = volume not in n_img.volumes self._ErrorIf(test, constants.CV_EINSTANCEMISSINGDISK, instance.name, "volume %s missing on node %s", volume, self.cfg.GetNodeName(node_uuid)) if instance.admin_state == constants.ADMINST_UP: test = instance.uuid not in pnode_img.instances and not pnode_img.offline self._ErrorIf(test, constants.CV_EINSTANCEDOWN, instance.name, "instance not running on its primary node %s", self.cfg.GetNodeName(pnode_uuid)) self._ErrorIf(pnode_img.offline, constants.CV_EINSTANCEBADNODE, instance.name, "instance is marked as running and lives on" " offline node %s", self.cfg.GetNodeName(pnode_uuid)) diskdata = [(nname, success, status, idx) for (nname, disks) in diskstatus.items() for idx, (success, status) in enumerate(disks)] for nname, success, bdev_status, idx in diskdata: # the 'ghost node' construction in Exec() ensures that we have a # node here snode = node_image[nname] bad_snode = snode.ghost or snode.offline self._ErrorIf(instance.disks_active and not success and not bad_snode, constants.CV_EINSTANCEFAULTYDISK, instance.name, "couldn't retrieve status for disk/%s on %s: %s", idx, self.cfg.GetNodeName(nname), bdev_status) if instance.disks_active and success and bdev_status.is_degraded: msg = "disk/%s on %s is degraded" % (idx, self.cfg.GetNodeName(nname)) code = self.ETYPE_ERROR accepted_lds = [constants.LDS_OKAY, constants.LDS_SYNC] if bdev_status.ldisk_status in accepted_lds: code = self.ETYPE_WARNING msg += "; local disk state is '%s'" % \ constants.LDS_NAMES[bdev_status.ldisk_status] self._Error(constants.CV_EINSTANCEFAULTYDISK, instance.name, msg, code=code) self._ErrorIf(pnode_img.rpc_fail and not pnode_img.offline, constants.CV_ENODERPC, self.cfg.GetNodeName(pnode_uuid), "instance %s, connection to primary node failed", instance.name) secondary_nodes = self.cfg.GetInstanceSecondaryNodes(instance.uuid) self._ErrorIf(len(secondary_nodes) > 1, constants.CV_EINSTANCELAYOUT, instance.name, "instance has multiple secondary nodes: %s", utils.CommaJoin(secondary_nodes), code=self.ETYPE_WARNING) inst_nodes = self.cfg.GetInstanceNodes(instance.uuid) es_flags = rpc.GetExclusiveStorageForNodes(self.cfg, inst_nodes) disks = self.cfg.GetInstanceDisks(instance.uuid) if any(es_flags.values()): if not utils.AllDiskOfType(disks, constants.DTS_EXCL_STORAGE): # Disk template not compatible with exclusive_storage: no instance # node should have the flag set es_nodes = [n for (n, es) in es_flags.items() if es] unsupported = [d.dev_type for d in disks if d.dev_type not in constants.DTS_EXCL_STORAGE] self._Error(constants.CV_EINSTANCEUNSUITABLENODE, instance.name, "instance uses disk types %s, which are not supported on" " nodes that have exclusive storage set: %s", utils.CommaJoin(unsupported), utils.CommaJoin(self.cfg.GetNodeNames(es_nodes))) for (idx, disk) in enumerate(disks): self._ErrorIf(disk.spindles is None, constants.CV_EINSTANCEMISSINGCFGPARAMETER, instance.name, "number of spindles not configured for disk %s while" " exclusive storage is enabled, try running" " gnt-cluster repair-disk-sizes", idx) if utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR): instance_nodes = utils.NiceSort(inst_nodes) instance_groups = {} for node_uuid in instance_nodes: instance_groups.setdefault(self.all_node_info[node_uuid].group, []).append(node_uuid) pretty_list = [ "%s (group %s)" % (utils.CommaJoin(self.cfg.GetNodeNames(nodes)), groupinfo[group].name) # Sort so that we always list the primary node first. for group, nodes in sorted(instance_groups.items(), key=lambda g_n: pnode_uuid in g_n[1], reverse=True)] self._ErrorIf(len(instance_groups) > 1, constants.CV_EINSTANCESPLITGROUPS, instance.name, "instance has primary and secondary nodes in" " different groups: %s", utils.CommaJoin(pretty_list), code=self.ETYPE_WARNING) inst_nodes_offline = [] for snode in secondary_nodes: s_img = node_image[snode] self._ErrorIf(s_img.rpc_fail and not s_img.offline, constants.CV_ENODERPC, self.cfg.GetNodeName(snode), "instance %s, connection to secondary node failed", instance.name) if s_img.offline: inst_nodes_offline.append(snode) # warn that the instance lives on offline nodes self._ErrorIf(inst_nodes_offline, constants.CV_EINSTANCEBADNODE, instance.name, "instance has offline secondary node(s) %s", utils.CommaJoin(self.cfg.GetNodeNames(inst_nodes_offline))) # ... or ghost/non-vm_capable nodes for node_uuid in inst_nodes: self._ErrorIf(node_image[node_uuid].ghost, constants.CV_EINSTANCEBADNODE, instance.name, "instance lives on ghost node %s", self.cfg.GetNodeName(node_uuid)) self._ErrorIf(not node_image[node_uuid].vm_capable, constants.CV_EINSTANCEBADNODE, instance.name, "instance lives on non-vm_capable node %s", self.cfg.GetNodeName(node_uuid)) def _VerifyOrphanVolumes(self, vg_name, node_vol_should, node_image, reserved): """Verify if there are any unknown volumes in the cluster. The .os, .swap and backup volumes are ignored. All other volumes are reported as unknown. @type vg_name: string @param vg_name: the name of the Ganeti-administered volume group @type node_vol_should: dict @param node_vol_should: mapping of node UUIDs to expected LVs on each node @type node_image: dict @param node_image: mapping of node UUIDs to L{NodeImage} objects @type reserved: L{ganeti.utils.FieldSet} @param reserved: a FieldSet of reserved volume names """ for node_uuid, n_img in node_image.items(): if (n_img.offline or n_img.rpc_fail or n_img.lvm_fail or self.all_node_info[node_uuid].group != self.group_uuid): # skip non-healthy nodes continue for volume in n_img.volumes: # skip volumes not belonging to the ganeti-administered volume group if volume.split('/')[0] != vg_name: continue test = ((node_uuid not in node_vol_should or volume not in node_vol_should[node_uuid]) and not reserved.Matches(volume)) self._ErrorIf(test, constants.CV_ENODEORPHANLV, self.cfg.GetNodeName(node_uuid), "volume %s is unknown", volume, code=_VerifyErrors.ETYPE_WARNING) def _VerifyNPlusOneMemory(self, node_image, all_insts): """Verify N+1 Memory Resilience. Check that if one single node dies we can still start all the instances it was primary for. """ cluster_info = self.cfg.GetClusterInfo() for node_uuid, n_img in node_image.items(): # This code checks that every node which is now listed as # secondary has enough memory to host all instances it is # supposed to should a single other node in the cluster fail. # FIXME: not ready for failover to an arbitrary node # FIXME: does not support file-backed instances # WARNING: we currently take into account down instances as well # as up ones, considering that even if they're down someone # might want to start them even in the event of a node failure. if n_img.offline or \ self.all_node_info[node_uuid].group != self.group_uuid: # we're skipping nodes marked offline and nodes in other groups from # the N+1 warning, since most likely we don't have good memory # information from them; we already list instances living on such # nodes, and that's enough warning continue #TODO(dynmem): also consider ballooning out other instances for prinode, inst_uuids in n_img.sbp.items(): needed_mem = 0 for inst_uuid in inst_uuids: bep = cluster_info.FillBE(all_insts[inst_uuid]) if bep[constants.BE_AUTO_BALANCE]: needed_mem += bep[constants.BE_MINMEM] test = n_img.mfree < needed_mem self._ErrorIf(test, constants.CV_ENODEN1, self.cfg.GetNodeName(node_uuid), "not enough memory to accomodate instance failovers" " should node %s fail (%dMiB needed, %dMiB available)", self.cfg.GetNodeName(prinode), needed_mem, n_img.mfree) def _CertError(self, *args): """Helper function for _VerifyClientCertificates.""" self._Error(constants.CV_ECLUSTERCLIENTCERT, None, *args) self._cert_error_found = True def _VerifyClientCertificates(self, nodes, all_nvinfo): """Verifies the consistency of the client certificates. This includes several aspects: - the individual validation of all nodes' certificates - the consistency of the master candidate certificate map - the consistency of the master candidate certificate map with the certificates that the master candidates are actually using. @param nodes: the list of nodes to consider in this verification @param all_nvinfo: the map of results of the verify_node call to all nodes """ rebuild_certs_msg = ( "To rebuild node certificates, please run" " 'gnt-cluster renew-crypto --new-node-certificates'.") self._cert_error_found = False candidate_certs = self.cfg.GetClusterInfo().candidate_certs if not candidate_certs: self._CertError( "The cluster's list of master candidate certificates is empty." " This may be because you just updated the cluster. " + rebuild_certs_msg) return if len(candidate_certs) != len(set(candidate_certs.values())): self._CertError( "There are at least two master candidates configured to use the same" " certificate.") # collect the client certificate for node in nodes: if node.offline: continue nresult = all_nvinfo[node.uuid] if nresult.fail_msg or not nresult.payload: continue (errcode, msg) = nresult.payload.get(constants.NV_CLIENT_CERT, None) if errcode is not None: self._CertError( "Client certificate of node '%s' failed validation: %s (code '%s')", node.uuid, msg, errcode) if not errcode: digest = msg if node.master_candidate: if node.uuid in candidate_certs: if digest != candidate_certs[node.uuid]: self._CertError( "Client certificate digest of master candidate '%s' does not" " match its entry in the cluster's map of master candidate" " certificates. Expected: %s Got: %s", node.uuid, digest, candidate_certs[node.uuid]) else: self._CertError( "The master candidate '%s' does not have an entry in the" " map of candidate certificates.", node.uuid) if digest in candidate_certs.values(): self._CertError( "Master candidate '%s' is using a certificate of another node.", node.uuid) else: if node.uuid in candidate_certs: self._CertError( "Node '%s' is not a master candidate, but still listed in the" " map of master candidate certificates.", node.uuid) if (node.uuid not in candidate_certs and digest in candidate_certs.values()): self._CertError( "Node '%s' is not a master candidate and is incorrectly using a" " certificate of another node which is master candidate.", node.uuid) if self._cert_error_found: self._CertError(rebuild_certs_msg) def _VerifySshSetup(self, nodes, all_nvinfo): """Evaluates the verification results of the SSH setup and clutter test. @param nodes: List of L{objects.Node} objects @param all_nvinfo: RPC results """ for node in nodes: if not node.offline: nresult = all_nvinfo[node.uuid] if nresult.fail_msg or not nresult.payload: self._ErrorIf(True, constants.CV_ENODESSH, node.name, "Could not verify the SSH setup of this node.") return for ssh_test in [constants.NV_SSH_SETUP, constants.NV_SSH_CLUTTER]: result = nresult.payload.get(ssh_test, None) error_msg = "" if isinstance(result, list): error_msg = " ".join(result) self._ErrorIf(result, constants.CV_ENODESSH, None, error_msg) def _VerifyFiles(self, nodes, master_node_uuid, all_nvinfo, filemap): """Verifies file checksums collected from all nodes. @param nodes: List of L{objects.Node} objects @param master_node_uuid: UUID of master node @param all_nvinfo: RPC results """ (files_all, files_opt, files_mc, files_vm) = filemap files2nodefn = [ (files_all, None), (files_mc, lambda node: (node.master_candidate or node.uuid == master_node_uuid)), (files_vm, lambda node: node.vm_capable), ] # Build mapping from filename to list of nodes which should have the file nodefiles = {} for (files, fn) in files2nodefn: if fn is None: filenodes = nodes else: filenodes = [n for n in nodes if fn(n)] nodefiles.update((filename, frozenset(fn.uuid for fn in filenodes)) for filename in files) assert set(nodefiles) == (files_all | files_mc | files_vm) fileinfo = dict((filename, {}) for filename in nodefiles) ignore_nodes = set() for node in nodes: if node.offline: ignore_nodes.add(node.uuid) continue nresult = all_nvinfo[node.uuid] if nresult.fail_msg or not nresult.payload: node_files = None else: fingerprints = nresult.payload.get(constants.NV_FILELIST, {}) node_files = dict((vcluster.LocalizeVirtualPath(key), value) for (key, value) in fingerprints.items()) del fingerprints test = not (node_files and isinstance(node_files, dict)) self._ErrorIf(test, constants.CV_ENODEFILECHECK, node.name, "Node did not return file checksum data") if test: ignore_nodes.add(node.uuid) continue # Build per-checksum mapping from filename to nodes having it for (filename, checksum) in node_files.items(): assert filename in nodefiles fileinfo[filename].setdefault(checksum, set()).add(node.uuid) for (filename, checksums) in fileinfo.items(): assert compat.all(len(i) > 10 for i in checksums), "Invalid checksum" # Nodes having the file with_file = frozenset(node_uuid for node_uuids in fileinfo[filename].values() for node_uuid in node_uuids) - ignore_nodes expected_nodes = nodefiles[filename] - ignore_nodes # Nodes missing file missing_file = expected_nodes - with_file if filename in files_opt: # All or no nodes self._ErrorIf(missing_file and missing_file != expected_nodes, constants.CV_ECLUSTERFILECHECK, None, "File %s is optional, but it must exist on all or no" " nodes (not found on %s)", filename, utils.CommaJoin(utils.NiceSort( self.cfg.GetNodeName(n) for n in missing_file))) else: self._ErrorIf(missing_file, constants.CV_ECLUSTERFILECHECK, None, "File %s is missing from node(s) %s", filename, utils.CommaJoin(utils.NiceSort( self.cfg.GetNodeName(n) for n in missing_file))) # Warn if a node has a file it shouldn't unexpected = with_file - expected_nodes self._ErrorIf(unexpected, constants.CV_ECLUSTERFILECHECK, None, "File %s should not exist on node(s) %s", filename, utils.CommaJoin(utils.NiceSort( self.cfg.GetNodeName(n) for n in unexpected))) # See if there are multiple versions of the file test = len(checksums) > 1 if test: variants = ["variant %s on %s" % (idx + 1, utils.CommaJoin(utils.NiceSort( self.cfg.GetNodeName(n) for n in node_uuids))) for (idx, (checksum, node_uuids)) in enumerate(sorted(checksums.items()))] else: variants = [] self._ErrorIf(test, constants.CV_ECLUSTERFILECHECK, None, "File %s found with %s different checksums (%s)", filename, len(checksums), "; ".join(variants)) def _VerifyNodeDrbdHelper(self, ninfo, nresult, drbd_helper): """Verify the drbd helper. """ if drbd_helper: helper_result = nresult.get(constants.NV_DRBDHELPER, None) test = (helper_result is None) self._ErrorIf(test, constants.CV_ENODEDRBDHELPER, ninfo.name, "no drbd usermode helper returned") if helper_result: status, payload = helper_result test = not status self._ErrorIf(test, constants.CV_ENODEDRBDHELPER, ninfo.name, "drbd usermode helper check unsuccessful: %s", payload) test = status and (payload != drbd_helper) self._ErrorIf(test, constants.CV_ENODEDRBDHELPER, ninfo.name, "wrong drbd usermode helper: %s", payload) @staticmethod def _ComputeDrbdMinors(ninfo, instanceinfo, disks_info, drbd_map, error_if): """Gives the DRBD information in a map for a node. @type ninfo: L{objects.Node} @param ninfo: the node to check @param instanceinfo: the dict of instances @param disks_info: the dict of disks @param drbd_map: the DRBD map as returned by L{ganeti.config.ConfigWriter.ComputeDRBDMap} @type error_if: callable like L{_ErrorIf} @param error_if: The error reporting function @return: dict from minor number to (disk_uuid, instance_uuid, active) """ node_drbd = {} for minor, disk_uuid in drbd_map[ninfo.uuid].items(): test = disk_uuid not in disks_info error_if(test, constants.CV_ECLUSTERCFG, None, "ghost disk '%s' in temporary DRBD map", disk_uuid) # ghost disk should not be active, but otherwise we # don't give double warnings (both ghost disk and # unallocated minor in use) if test: node_drbd[minor] = (disk_uuid, None, False) else: disk_active = False disk_instance = None for (inst_uuid, inst) in instanceinfo.items(): if disk_uuid in inst.disks: disk_active = inst.disks_active disk_instance = inst_uuid break node_drbd[minor] = (disk_uuid, disk_instance, disk_active) return node_drbd def _VerifyNodeDrbd(self, ninfo, nresult, instanceinfo, disks_info, drbd_helper, drbd_map): """Verifies and the node DRBD status. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @param instanceinfo: the dict of instances @param disks_info: the dict of disks @param drbd_helper: the configured DRBD usermode helper @param drbd_map: the DRBD map as returned by L{ganeti.config.ConfigWriter.ComputeDRBDMap} """ self._VerifyNodeDrbdHelper(ninfo, nresult, drbd_helper) # compute the DRBD minors node_drbd = self._ComputeDrbdMinors(ninfo, instanceinfo, disks_info, drbd_map, self._ErrorIf) # and now check them used_minors = nresult.get(constants.NV_DRBDLIST, []) test = not isinstance(used_minors, (tuple, list)) self._ErrorIf(test, constants.CV_ENODEDRBD, ninfo.name, "cannot parse drbd status file: %s", str(used_minors)) if test: # we cannot check drbd status return for minor, (disk_uuid, inst_uuid, must_exist) in node_drbd.items(): test = minor not in used_minors and must_exist if inst_uuid is not None: attached = "(attached in instance '%s')" % \ self.cfg.GetInstanceName(inst_uuid) else: attached = "(detached)" self._ErrorIf(test, constants.CV_ENODEDRBD, ninfo.name, "drbd minor %d of disk %s %s is not active", minor, disk_uuid, attached) for minor in used_minors: test = minor not in node_drbd self._ErrorIf(test, constants.CV_ENODEDRBD, ninfo.name, "unallocated drbd minor %d is in use", minor) def _UpdateNodeOS(self, ninfo, nresult, nimg): """Builds the node OS structures. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @param nimg: the node image object """ remote_os = nresult.get(constants.NV_OSLIST, None) test = (not isinstance(remote_os, list) or not compat.all(isinstance(v, list) and len(v) == 8 for v in remote_os)) self._ErrorIf(test, constants.CV_ENODEOS, ninfo.name, "node hasn't returned valid OS data") nimg.os_fail = test if test: return os_dict = {} for (name, os_path, status, diagnose, variants, parameters, api_ver, trusted) in nresult[constants.NV_OSLIST]: if name not in os_dict: os_dict[name] = [] # parameters is a list of lists instead of list of tuples due to # JSON lacking a real tuple type, fix it: parameters = [tuple(v) for v in parameters] os_dict[name].append((os_path, status, diagnose, set(variants), set(parameters), set(api_ver), trusted)) nimg.oslist = os_dict def _VerifyNodeOS(self, ninfo, nimg, base): """Verifies the node OS list. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nimg: the node image object @param base: the 'template' node we match against (e.g. from the master) """ assert not nimg.os_fail, "Entered _VerifyNodeOS with failed OS rpc?" beautify_params = lambda l: ["%s: %s" % (k, v) for (k, v) in l] for os_name, os_data in nimg.oslist.items(): assert os_data, "Empty OS status for OS %s?!" % os_name f_path, f_status, f_diag, f_var, f_param, f_api, f_trusted = os_data[0] self._ErrorIf(not f_status, constants.CV_ENODEOS, ninfo.name, "Invalid OS %s (located at %s): %s", os_name, f_path, f_diag) self._ErrorIf(len(os_data) > 1, constants.CV_ENODEOS, ninfo.name, "OS '%s' has multiple entries" " (first one shadows the rest): %s", os_name, utils.CommaJoin([v[0] for v in os_data])) # comparisons with the 'base' image test = os_name not in base.oslist self._ErrorIf(test, constants.CV_ENODEOS, ninfo.name, "Extra OS %s not present on reference node (%s)", os_name, self.cfg.GetNodeName(base.uuid)) if test: continue assert base.oslist[os_name], "Base node has empty OS status?" _, b_status, _, b_var, b_param, b_api, b_trusted = base.oslist[os_name][0] if not b_status: # base OS is invalid, skipping continue for kind, a, b in [("API version", f_api, b_api), ("variants list", f_var, b_var), ("parameters", beautify_params(f_param), beautify_params(b_param))]: self._ErrorIf(a != b, constants.CV_ENODEOS, ninfo.name, "OS %s for %s differs from reference node %s:" " [%s] vs. [%s]", kind, os_name, self.cfg.GetNodeName(base.uuid), utils.CommaJoin(sorted(a)), utils.CommaJoin(sorted(b))) for kind, a, b in [("trusted", f_trusted, b_trusted)]: self._ErrorIf(a != b, constants.CV_ENODEOS, ninfo.name, "OS %s for %s differs from reference node %s:" " %s vs. %s", kind, os_name, self.cfg.GetNodeName(base.uuid), a, b) # check any missing OSes missing = set(base.oslist).difference(nimg.oslist) self._ErrorIf(missing, constants.CV_ENODEOS, ninfo.name, "OSes present on reference node %s" " but missing on this node: %s", self.cfg.GetNodeName(base.uuid), utils.CommaJoin(missing)) def _VerifyAcceptedFileStoragePaths(self, ninfo, nresult, is_master): """Verifies paths in L{pathutils.FILE_STORAGE_PATHS_FILE}. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @type is_master: bool @param is_master: Whether node is the master node """ cluster = self.cfg.GetClusterInfo() if (is_master and (cluster.IsFileStorageEnabled() or cluster.IsSharedFileStorageEnabled())): try: fspaths = nresult[constants.NV_ACCEPTED_STORAGE_PATHS] except KeyError: # This should never happen self._ErrorIf(True, constants.CV_ENODEFILESTORAGEPATHS, ninfo.name, "Node did not return forbidden file storage paths") else: self._ErrorIf(fspaths, constants.CV_ENODEFILESTORAGEPATHS, ninfo.name, "Found forbidden file storage paths: %s", utils.CommaJoin(fspaths)) else: self._ErrorIf(constants.NV_ACCEPTED_STORAGE_PATHS in nresult, constants.CV_ENODEFILESTORAGEPATHS, ninfo.name, "Node should not have returned forbidden file storage" " paths") def _VerifyStoragePaths(self, ninfo, nresult, file_disk_template, verify_key, error_key): """Verifies (file) storage paths. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @type file_disk_template: string @param file_disk_template: file-based disk template, whose directory is supposed to be verified @type verify_key: string @param verify_key: key for the verification map of this file verification step @param error_key: error key to be added to the verification results in case something goes wrong in this verification step """ assert (file_disk_template in utils.storage.GetDiskTemplatesOfStorageTypes( constants.ST_FILE, constants.ST_SHARED_FILE, constants.ST_GLUSTER )) cluster = self.cfg.GetClusterInfo() if cluster.IsDiskTemplateEnabled(file_disk_template): self._ErrorIf( verify_key in nresult, error_key, ninfo.name, "The configured %s storage path is unusable: %s" % (file_disk_template, nresult.get(verify_key))) def _VerifyFileStoragePaths(self, ninfo, nresult): """Verifies (file) storage paths. @see: C{_VerifyStoragePaths} """ self._VerifyStoragePaths( ninfo, nresult, constants.DT_FILE, constants.NV_FILE_STORAGE_PATH, constants.CV_ENODEFILESTORAGEPATHUNUSABLE) def _VerifySharedFileStoragePaths(self, ninfo, nresult): """Verifies (file) storage paths. @see: C{_VerifyStoragePaths} """ self._VerifyStoragePaths( ninfo, nresult, constants.DT_SHARED_FILE, constants.NV_SHARED_FILE_STORAGE_PATH, constants.CV_ENODESHAREDFILESTORAGEPATHUNUSABLE) def _VerifyGlusterStoragePaths(self, ninfo, nresult): """Verifies (file) storage paths. @see: C{_VerifyStoragePaths} """ self._VerifyStoragePaths( ninfo, nresult, constants.DT_GLUSTER, constants.NV_GLUSTER_STORAGE_PATH, constants.CV_ENODEGLUSTERSTORAGEPATHUNUSABLE) def _VerifyOob(self, ninfo, nresult): """Verifies out of band functionality of a node. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node """ # We just have to verify the paths on master and/or master candidates # as the oob helper is invoked on the master if ((ninfo.master_candidate or ninfo.master_capable) and constants.NV_OOB_PATHS in nresult): for path_result in nresult[constants.NV_OOB_PATHS]: self._ErrorIf(path_result, constants.CV_ENODEOOBPATH, ninfo.name, path_result) def _UpdateNodeVolumes(self, ninfo, nresult, nimg, vg_name): """Verifies and updates the node volume data. This function will update a L{NodeImage}'s internal structures with data from the remote call. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @param nimg: the node image object @param vg_name: the configured VG name """ nimg.lvm_fail = True lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data") if vg_name is None: pass elif isinstance(lvdata, str): self._ErrorIf(True, constants.CV_ENODELVM, ninfo.name, "LVM problem on node: %s", utils.SafeEncode(lvdata)) elif not isinstance(lvdata, dict): self._ErrorIf(True, constants.CV_ENODELVM, ninfo.name, "rpc call to node failed (lvlist)") else: nimg.volumes = lvdata nimg.lvm_fail = False def _UpdateNodeInstances(self, ninfo, nresult, nimg): """Verifies and updates the node instance list. If the listing was successful, then updates this node's instance list. Otherwise, it marks the RPC call as failed for the instance list key. @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @param nimg: the node image object """ idata = nresult.get(constants.NV_INSTANCELIST, None) test = not isinstance(idata, list) self._ErrorIf(test, constants.CV_ENODEHV, ninfo.name, "rpc call to node failed (instancelist): %s", utils.SafeEncode(str(idata))) if test: nimg.hyp_fail = True else: nimg.instances = [uuid for (uuid, _) in self.cfg.GetMultiInstanceInfoByName(idata)] def _UpdateNodeInfo(self, ninfo, nresult, nimg, vg_name): """Verifies and computes a node information map @type ninfo: L{objects.Node} @param ninfo: the node to check @param nresult: the remote results for the node @param nimg: the node image object @param vg_name: the configured VG name """ # try to read free memory (from the hypervisor) hv_info = nresult.get(constants.NV_HVINFO, None) test = not isinstance(hv_info, dict) or "memory_free" not in hv_info self._ErrorIf(test, constants.CV_ENODEHV, ninfo.name, "rpc call to node failed (hvinfo)") if not test: try: nimg.mfree = int(hv_info["memory_free"]) except (ValueError, TypeError): self._ErrorIf(True, constants.CV_ENODERPC, ninfo.name, "node returned invalid nodeinfo, check hypervisor") # FIXME: devise a free space model for file based instances as well if vg_name is not None: test = (constants.NV_VGLIST not in nresult or vg_name not in nresult[constants.NV_VGLIST]) self._ErrorIf(test, constants.CV_ENODELVM, ninfo.name, "node didn't return data for the volume group '%s'" " - it is either missing or broken", vg_name) if not test: try: nimg.dfree = int(nresult[constants.NV_VGLIST][vg_name]) except (ValueError, TypeError): self._ErrorIf(True, constants.CV_ENODERPC, ninfo.name, "node returned invalid LVM info, check LVM status") def _CollectDiskInfo(self, node_uuids, node_image, instanceinfo): """Gets per-disk status information for all instances. @type node_uuids: list of strings @param node_uuids: Node UUIDs @type node_image: dict of (UUID, L{objects.Node}) @param node_image: Node objects @type instanceinfo: dict of (UUID, L{objects.Instance}) @param instanceinfo: Instance objects @rtype: {instance: {node: [(succes, payload)]}} @return: a dictionary of per-instance dictionaries with nodes as keys and disk information as values; the disk information is a list of tuples (success, payload) """ node_disks = {} node_disks_dev_inst_only = {} diskless_instances = set() nodisk_instances = set() for nuuid in node_uuids: node_inst_uuids = list(itertools.chain(node_image[nuuid].pinst, node_image[nuuid].sinst)) diskless_instances.update(uuid for uuid in node_inst_uuids if not instanceinfo[uuid].disks) disks = [(inst_uuid, disk) for inst_uuid in node_inst_uuids for disk in self.cfg.GetInstanceDisks(inst_uuid)] if not disks: nodisk_instances.update(uuid for uuid in node_inst_uuids if instanceinfo[uuid].disks) # No need to collect data continue node_disks[nuuid] = disks # _AnnotateDiskParams makes already copies of the disks dev_inst_only = [] for (inst_uuid, dev) in disks: (anno_disk,) = AnnotateDiskParams(instanceinfo[inst_uuid], [dev], self.cfg) dev_inst_only.append((anno_disk, instanceinfo[inst_uuid])) node_disks_dev_inst_only[nuuid] = dev_inst_only assert len(node_disks) == len(node_disks_dev_inst_only) # Collect data from all nodes with disks result = self.rpc.call_blockdev_getmirrorstatus_multi( list(node_disks), node_disks_dev_inst_only) assert len(result) == len(node_disks) instdisk = {} for (nuuid, nres) in result.items(): node = self.cfg.GetNodeInfo(nuuid) disks = node_disks[node.uuid] if nres.offline: # No data from this node data = len(disks) * [(False, "node offline")] else: msg = nres.fail_msg self._ErrorIf(msg, constants.CV_ENODERPC, node.name, "while getting disk information: %s", msg) if msg: # No data from this node data = len(disks) * [(False, msg)] else: data = [] for idx, i in enumerate(nres.payload): if isinstance(i, (tuple, list)) and len(i) == 2: data.append(i) else: logging.warning("Invalid result from node %s, entry %d: %s", node.name, idx, i) data.append((False, "Invalid result from the remote node")) for ((inst_uuid, _), status) in zip(disks, data): instdisk.setdefault(inst_uuid, {}).setdefault(node.uuid, []) \ .append(status) # Add empty entries for diskless instances. for inst_uuid in diskless_instances: assert inst_uuid not in instdisk instdisk[inst_uuid] = {} # ...and disk-full instances that happen to have no disks for inst_uuid in nodisk_instances: assert inst_uuid not in instdisk instdisk[inst_uuid] = {} assert compat.all(len(statuses) == len(instanceinfo[inst].disks) and len(nuuids) <= len( self.cfg.GetInstanceNodes(instanceinfo[inst].uuid)) and compat.all(isinstance(s, (tuple, list)) and len(s) == 2 for s in statuses) for inst, nuuids in instdisk.items() for nuuid, statuses in nuuids.items()) if __debug__: instdisk_keys = set(instdisk) instanceinfo_keys = set(instanceinfo) assert instdisk_keys == instanceinfo_keys, \ ("instdisk keys (%s) do not match instanceinfo keys (%s)" % (instdisk_keys, instanceinfo_keys)) return instdisk @staticmethod def _SshNodeSelector(group_uuid, all_nodes): """Create endless iterators for all potential SSH check hosts. """ nodes = [node for node in all_nodes if (node.group != group_uuid and not node.offline)] keyfunc = operator.attrgetter("group") nodes_by_group = itertools.groupby(sorted(nodes, key=keyfunc), keyfunc) node_names = (sorted(n.name for n in names) for _, names in nodes_by_group) return [itertools.cycle(ns) for ns in node_names] @classmethod def _SelectSshCheckNodes(cls, group_nodes, group_uuid, all_nodes): """Choose which nodes should talk to which other nodes. We will make nodes contact all nodes in their group, and one node from every other group. @rtype: tuple of (string, dict of strings to list of strings, string) @return: a tuple containing the list of all online nodes, a dictionary mapping node names to additional nodes of other node groups to which connectivity should be tested, and a list of all online master candidates @warning: This algorithm has a known issue if one node group is much smaller than others (e.g. just one node). In such a case all other nodes will talk to the single node. """ online_nodes = sorted(node.name for node in group_nodes if not node.offline) online_mcs = sorted(node.name for node in group_nodes if (node.master_candidate and not node.offline)) sel = cls._SshNodeSelector(group_uuid, all_nodes) return (online_nodes, dict((name, sorted([next(i) for i in sel])) for name in online_nodes), online_mcs) def _PrepareSshSetupCheck(self): """Prepare the input data for the SSH setup verification. """ all_nodes_info = self.cfg.GetAllNodesInfo() potential_master_candidates = self.cfg.GetPotentialMasterCandidates() node_status = [ (uuid, node_info.name, node_info.master_candidate, node_info.name in potential_master_candidates, not node_info.offline) for (uuid, node_info) in all_nodes_info.items()] return node_status def BuildHooksEnv(self): """Build hooks env. Cluster-Verify hooks just ran in the post phase and their failure makes the output be logged in the verify output and the verification to fail. """ env = { "CLUSTER_TAGS": " ".join(self.cfg.GetClusterInfo().GetTags()), } env.update(("NODE_TAGS_%s" % node.name, " ".join(node.GetTags())) for node in self.my_node_info.values()) return env def BuildHooksNodes(self): """Build hooks nodes. """ return ([], list(self.my_node_info)) def _AssessHypervisorParameters(self): """Check clusterwide hypervisor parameters for suboptimal values """ self._feedback_fn("* Assessing cluster hypervisor parameters") cluster = self.cfg.GetClusterInfo() for hv_name in cluster.enabled_hypervisors: msg = ("hypervisor %s parameter assessment: %%s" % hv_name) hv_params = cluster.GetHVDefaults(hv_name) try: hv_class = hypervisor.GetHypervisorClass(hv_name) utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES) warnings = hv_class.AssessParameters(hv_params) except errors.GenericError as err: self._ErrorIf(True, constants.CV_ECLUSTERCFG, None, msg % str(err)) for warning in warnings: self._feedback_fn(" - %s" % warning) if warnings: self._feedback_fn(" - Please refer to the gnt-instance man page for " "detailed information on the usage of each " "hypervisor parameter") @staticmethod def _VerifyOtherNotes(feedback_fn, i_non_redundant, i_non_a_balanced, i_offline, n_offline, n_drained): feedback_fn("* Other Notes") if i_non_redundant: feedback_fn(" - NOTICE: %d non-redundant instance(s) found." % len(i_non_redundant)) if i_non_a_balanced: feedback_fn(" - NOTICE: %d non-auto-balanced instance(s) found." % len(i_non_a_balanced)) if i_offline: feedback_fn(" - NOTICE: %d offline instance(s) found." % i_offline) if n_offline: feedback_fn(" - NOTICE: %d offline node(s) found." % n_offline) if n_drained: feedback_fn(" - NOTICE: %d drained node(s) found." % n_drained) def _VerifyExclusionTags(self, nodename, pinst, ctags): """Verify that all instances have different exclusion tags. @type nodename: string @param nodename: the name of the node for which the check is done @type pinst: list of string @param pinst: list of UUIDs of those instances having the given node as primary node @type ctags: list of string @param ctags: tags of the cluster """ exclusion_prefixes = utils.GetExclusionPrefixes(ctags) tags_seen = set([]) conflicting_tags = set([]) for iuuid in pinst: allitags = self.my_inst_info[iuuid].tags if allitags is None: allitags = [] itags = set([tag for tag in allitags if utils.IsGoodTag(exclusion_prefixes, tag)]) conflicts = itags.intersection(tags_seen) if len(conflicts) > 0: conflicting_tags = conflicting_tags.union(conflicts) tags_seen = tags_seen.union(itags) self._ErrorIf(len(conflicting_tags) > 0, constants.CV_EEXTAGS, nodename, "Tags where there is more than one instance: %s", list(conflicting_tags), code=constants.CV_WARNING) def Exec(self, feedback_fn): # pylint: disable=R0915 """Verify integrity of the node group, performing various test on nodes. """ # This method has too many local variables. pylint: disable=R0914 feedback_fn("* Verifying group '%s'" % self.group_info.name) if not self.my_node_uuids: # empty node group feedback_fn("* Empty node group, skipping verification") return True self.bad = False verbose = self.op.verbose self._feedback_fn = feedback_fn vg_name = self.cfg.GetVGName() drbd_helper = self.cfg.GetDRBDHelper() cluster = self.cfg.GetClusterInfo() hypervisors = cluster.enabled_hypervisors node_data_list = list(self.my_node_info.values()) i_non_redundant = [] # Non redundant instances i_non_a_balanced = [] # Non auto-balanced instances i_offline = 0 # Count of offline instances n_offline = 0 # Count of offline nodes n_drained = 0 # Count of nodes being drained node_vol_should = {} # FIXME: verify OS list # File verification filemap = ComputeAncillaryFiles(cluster, False) # do local checksums master_node_uuid = self.master_node = self.cfg.GetMasterNode() master_ip = self.cfg.GetMasterIP() online_master_candidates = sorted( node.name for node in node_data_list if (node.master_candidate and not node.offline)) feedback_fn("* Gathering data (%d nodes)" % len(self.my_node_uuids)) user_scripts = [] if self.cfg.GetUseExternalMipScript(): user_scripts.append(pathutils.EXTERNAL_MASTER_SETUP_SCRIPT) online_nodes = [(node.name, node.primary_ip, node.secondary_ip) for node in node_data_list if not node.offline] node_nettest_params = (online_nodes, online_master_candidates) node_verify_param = { constants.NV_FILELIST: [vcluster.MakeVirtualPath(f) for f in utils.UniqueSequence(filename for files in filemap for filename in files)], constants.NV_NODELIST: self._SelectSshCheckNodes(node_data_list, self.group_uuid, list(self.all_node_info.values())), constants.NV_HYPERVISOR: hypervisors, constants.NV_HVPARAMS: _GetAllHypervisorParameters(cluster, list(self.all_inst_info.values())), constants.NV_NODENETTEST: node_nettest_params, constants.NV_INSTANCELIST: hypervisors, constants.NV_VERSION: None, constants.NV_HVINFO: self.cfg.GetHypervisorType(), constants.NV_NODESETUP: None, constants.NV_TIME: None, constants.NV_MASTERIP: (self.cfg.GetMasterNodeName(), master_ip, online_master_candidates), constants.NV_OSLIST: None, constants.NV_NONVMNODES: self.cfg.GetNonVmCapableNodeNameList(), constants.NV_USERSCRIPTS: user_scripts, constants.NV_CLIENT_CERT: None, } if self.cfg.GetClusterInfo().modify_ssh_setup: node_verify_param[constants.NV_SSH_SETUP] = \ (self._PrepareSshSetupCheck(), self.cfg.GetClusterInfo().ssh_key_type) if self.op.verify_clutter: node_verify_param[constants.NV_SSH_CLUTTER] = True if vg_name is not None: node_verify_param[constants.NV_VGLIST] = None node_verify_param[constants.NV_LVLIST] = vg_name node_verify_param[constants.NV_PVLIST] = [vg_name] if cluster.IsDiskTemplateEnabled(constants.DT_DRBD8): if drbd_helper: node_verify_param[constants.NV_DRBDVERSION] = None node_verify_param[constants.NV_DRBDLIST] = None node_verify_param[constants.NV_DRBDHELPER] = drbd_helper if cluster.IsFileStorageEnabled() or \ cluster.IsSharedFileStorageEnabled(): # Load file storage paths only from master node node_verify_param[constants.NV_ACCEPTED_STORAGE_PATHS] = \ self.cfg.GetMasterNodeName() if cluster.IsFileStorageEnabled(): node_verify_param[constants.NV_FILE_STORAGE_PATH] = \ cluster.file_storage_dir if cluster.IsSharedFileStorageEnabled(): node_verify_param[constants.NV_SHARED_FILE_STORAGE_PATH] = \ cluster.shared_file_storage_dir # bridge checks # FIXME: this needs to be changed per node-group, not cluster-wide bridges = set() default_nicpp = cluster.nicparams[constants.PP_DEFAULT] if default_nicpp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: bridges.add(default_nicpp[constants.NIC_LINK]) for inst_uuid in self.my_inst_info.values(): for nic in inst_uuid.nics: full_nic = cluster.SimpleFillNIC(nic.nicparams) if full_nic[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: bridges.add(full_nic[constants.NIC_LINK]) if bridges: node_verify_param[constants.NV_BRIDGES] = list(bridges) # Build our expected cluster state node_image = dict((node.uuid, self.NodeImage(offline=node.offline, uuid=node.uuid, vm_capable=node.vm_capable)) for node in node_data_list) # Gather OOB paths oob_paths = [] for node in self.all_node_info.values(): path = SupportsOob(self.cfg, node) if path and path not in oob_paths: oob_paths.append(path) if oob_paths: node_verify_param[constants.NV_OOB_PATHS] = oob_paths for inst_uuid in self.my_inst_uuids: instance = self.my_inst_info[inst_uuid] if instance.admin_state == constants.ADMINST_OFFLINE: i_offline += 1 inst_nodes = self.cfg.GetInstanceNodes(instance.uuid) for nuuid in inst_nodes: if nuuid not in node_image: gnode = self.NodeImage(uuid=nuuid) gnode.ghost = (nuuid not in self.all_node_info) node_image[nuuid] = gnode self.cfg.GetInstanceLVsByNode(instance.uuid, lvmap=node_vol_should) pnode = instance.primary_node node_image[pnode].pinst.append(instance.uuid) for snode in self.cfg.GetInstanceSecondaryNodes(instance.uuid): nimg = node_image[snode] nimg.sinst.append(instance.uuid) if pnode not in nimg.sbp: nimg.sbp[pnode] = [] nimg.sbp[pnode].append(instance.uuid) es_flags = rpc.GetExclusiveStorageForNodes(self.cfg, list(self.my_node_info)) # The value of exclusive_storage should be the same across the group, so if # it's True for at least a node, we act as if it were set for all the nodes self._exclusive_storage = compat.any(es_flags.values()) if self._exclusive_storage: node_verify_param[constants.NV_EXCLUSIVEPVS] = True # At this point, we have the in-memory data structures complete, # except for the runtime information, which we'll gather next # NOTE: Here we lock the configuration for the duration of RPC calls, # which means that the cluster configuration changes are blocked during # this period. # This is something that should be done only exceptionally and only for # justified cases! # In this case, we need the lock as we can only verify the integrity of # configuration files on MCs only if we know nobody else is modifying it. # FIXME: The check for integrity of config.data should be moved to # WConfD, which is the only one who can otherwise ensure nobody # will modify the configuration during the check. with self.cfg.GetConfigManager(shared=True, forcelock=True): feedback_fn("* Gathering information about nodes (%s nodes)" % len(self.my_node_uuids)) # Force the configuration to be fully distributed before doing any tests self.cfg.FlushConfigGroup(self.group_uuid) # Due to the way our RPC system works, exact response times cannot be # guaranteed (e.g. a broken node could run into a timeout). By keeping # the time before and after executing the request, we can at least have # a time window. nvinfo_starttime = time.time() # Get lock on the configuration so that nobody modifies it concurrently. # Otherwise it can be modified by other jobs, failing the consistency # test. # NOTE: This is an exceptional situation, we should otherwise avoid # locking the configuration for something but very fast, pure operations. cluster_name = self.cfg.GetClusterName() hvparams = self.cfg.GetClusterInfo().hvparams all_nvinfo = self.rpc.call_node_verify(self.my_node_uuids, node_verify_param, cluster_name, hvparams) nvinfo_endtime = time.time() if self.extra_lv_nodes and vg_name is not None: feedback_fn("* Gathering information about extra nodes (%s nodes)" % len(self.extra_lv_nodes)) extra_lv_nvinfo = \ self.rpc.call_node_verify(self.extra_lv_nodes, {constants.NV_LVLIST: vg_name}, self.cfg.GetClusterName(), self.cfg.GetClusterInfo().hvparams) else: extra_lv_nvinfo = {} # If not all nodes are being checked, we need to make sure the master # node and a non-checked vm_capable node are in the list. absent_node_uuids = set(self.all_node_info).difference(self.my_node_info) if absent_node_uuids: vf_nvinfo = all_nvinfo.copy() vf_node_info = list(self.my_node_info.values()) additional_node_uuids = [] if master_node_uuid not in self.my_node_info: additional_node_uuids.append(master_node_uuid) vf_node_info.append(self.all_node_info[master_node_uuid]) # Add the first vm_capable node we find which is not included, # excluding the master node (which we already have) for node_uuid in absent_node_uuids: nodeinfo = self.all_node_info[node_uuid] if (nodeinfo.vm_capable and not nodeinfo.offline and node_uuid != master_node_uuid): additional_node_uuids.append(node_uuid) vf_node_info.append(self.all_node_info[node_uuid]) break key = constants.NV_FILELIST feedback_fn("* Gathering information about the master node") vf_nvinfo.update(self.rpc.call_node_verify( additional_node_uuids, {key: node_verify_param[key]}, self.cfg.GetClusterName(), self.cfg.GetClusterInfo().hvparams)) else: vf_nvinfo = all_nvinfo vf_node_info = list(self.my_node_info.values()) all_drbd_map = self.cfg.ComputeDRBDMap() feedback_fn("* Gathering disk information (%s nodes)" % len(self.my_node_uuids)) instdisk = self._CollectDiskInfo(list(self.my_node_info), node_image, self.my_inst_info) feedback_fn("* Verifying configuration file consistency") self._VerifyClientCertificates(list(self.my_node_info.values()), all_nvinfo) if self.cfg.GetClusterInfo().modify_ssh_setup: self._VerifySshSetup(list(self.my_node_info.values()), all_nvinfo) self._VerifyFiles(vf_node_info, master_node_uuid, vf_nvinfo, filemap) feedback_fn("* Verifying node status") refos_img = None for node_i in node_data_list: nimg = node_image[node_i.uuid] if node_i.offline: if verbose: feedback_fn("* Skipping offline node %s" % (node_i.name,)) n_offline += 1 continue if node_i.uuid == master_node_uuid: ntype = "master" elif node_i.master_candidate: ntype = "master candidate" elif node_i.drained: ntype = "drained" n_drained += 1 else: ntype = "regular" if verbose: feedback_fn("* Verifying node %s (%s)" % (node_i.name, ntype)) msg = all_nvinfo[node_i.uuid].fail_msg self._ErrorIf(msg, constants.CV_ENODERPC, node_i.name, "while contacting node: %s", msg) if msg: nimg.rpc_fail = True continue nresult = all_nvinfo[node_i.uuid].payload nimg.call_ok = self._VerifyNode(node_i, nresult) self._VerifyNodeTime(node_i, nresult, nvinfo_starttime, nvinfo_endtime) self._VerifyNodeNetwork(node_i, nresult) self._VerifyNodeUserScripts(node_i, nresult) self._VerifyOob(node_i, nresult) self._VerifyAcceptedFileStoragePaths(node_i, nresult, node_i.uuid == master_node_uuid) self._VerifyFileStoragePaths(node_i, nresult) self._VerifySharedFileStoragePaths(node_i, nresult) self._VerifyGlusterStoragePaths(node_i, nresult) if nimg.vm_capable: self._UpdateVerifyNodeLVM(node_i, nresult, vg_name, nimg) if constants.DT_DRBD8 in cluster.enabled_disk_templates: self._VerifyNodeDrbd(node_i, nresult, self.all_inst_info, self.all_disks_info, drbd_helper, all_drbd_map) if (constants.DT_PLAIN in cluster.enabled_disk_templates) or \ (constants.DT_DRBD8 in cluster.enabled_disk_templates): self._UpdateNodeVolumes(node_i, nresult, nimg, vg_name) self._UpdateNodeInstances(node_i, nresult, nimg) self._UpdateNodeInfo(node_i, nresult, nimg, vg_name) self._UpdateNodeOS(node_i, nresult, nimg) if not nimg.os_fail: if refos_img is None: refos_img = nimg self._VerifyNodeOS(node_i, nimg, refos_img) self._VerifyNodeBridges(node_i, nresult, bridges) # Check whether all running instances are primary for the node. (This # can no longer be done from _VerifyInstance below, since some of the # wrong instances could be from other node groups.) non_primary_inst_uuids = set(nimg.instances).difference(nimg.pinst) for inst_uuid in non_primary_inst_uuids: test = inst_uuid in self.all_inst_info self._ErrorIf(test, constants.CV_EINSTANCEWRONGNODE, self.cfg.GetInstanceName(inst_uuid), "instance should not run on node %s", node_i.name) self._ErrorIf(not test, constants.CV_ENODEORPHANINSTANCE, node_i.name, "node is running unknown instance %s", inst_uuid) self._VerifyExclusionTags(node_i.name, nimg.pinst, cluster.tags) self._VerifyGroupDRBDVersion(all_nvinfo) self._VerifyGroupLVM(node_image, vg_name) for node_uuid, result in extra_lv_nvinfo.items(): self._UpdateNodeVolumes(self.all_node_info[node_uuid], result.payload, node_image[node_uuid], vg_name) feedback_fn("* Verifying instance status") for inst_uuid in self.my_inst_uuids: instance = self.my_inst_info[inst_uuid] if verbose: feedback_fn("* Verifying instance %s" % instance.name) self._VerifyInstance(instance, node_image, instdisk[inst_uuid]) # If the instance is not fully redundant we cannot survive losing its # primary node, so we are not N+1 compliant. inst_disks = self.cfg.GetInstanceDisks(instance.uuid) if not utils.AllDiskOfType(inst_disks, constants.DTS_MIRRORED): i_non_redundant.append(instance) if not cluster.FillBE(instance)[constants.BE_AUTO_BALANCE]: i_non_a_balanced.append(instance) feedback_fn("* Verifying orphan volumes") reserved = utils.FieldSet(*cluster.reserved_lvs) # We will get spurious "unknown volume" warnings if any node of this group # is secondary for an instance whose primary is in another group. To avoid # them, we find these instances and add their volumes to node_vol_should. for instance in self.all_inst_info.values(): for secondary in self.cfg.GetInstanceSecondaryNodes(instance.uuid): if (secondary in self.my_node_info and instance.uuid not in self.my_inst_info): self.cfg.GetInstanceLVsByNode(instance.uuid, lvmap=node_vol_should) break self._VerifyOrphanVolumes(vg_name, node_vol_should, node_image, reserved) if constants.VERIFY_NPLUSONE_MEM not in self.op.skip_checks: feedback_fn("* Verifying N+1 Memory redundancy") self._VerifyNPlusOneMemory(node_image, self.my_inst_info) if constants.VERIFY_HVPARAM_ASSESSMENT not in self.op.skip_checks: self._AssessHypervisorParameters() self._VerifyOtherNotes(feedback_fn, i_non_redundant, i_non_a_balanced, i_offline, n_offline, n_drained) return not self.bad def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result): """Analyze the post-hooks' result This method analyses the hook result, handles it, and sends some nicely-formatted feedback back to the user. @param phase: one of L{constants.HOOKS_PHASE_POST} or L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase @param hooks_results: the results of the multi-node hooks rpc call @param feedback_fn: function used send feedback back to the caller @param lu_result: previous Exec result @return: the new Exec result, based on the previous result and hook results """ # We only really run POST phase hooks, only for non-empty groups, # and are only interested in their results if not self.my_node_uuids: # empty node group pass elif phase == constants.HOOKS_PHASE_POST: # Used to change hooks' output to proper indentation feedback_fn("* Hooks Results") assert hooks_results, "invalid result from hooks" for node_name in hooks_results: res = hooks_results[node_name] msg = res.fail_msg test = msg and not res.offline self._ErrorIf(test, constants.CV_ENODEHOOKS, node_name, "Communication failure in hooks execution: %s", msg) if test: lu_result = False continue if res.offline: # No need to investigate payload if node is offline continue for script, hkr, output in res.payload: test = hkr == constants.HKR_FAIL self._ErrorIf(test, constants.CV_ENODEHOOKS, node_name, "Script %s failed, output:", script) if test: output = self._HOOKS_INDENT_RE.sub(" ", output) feedback_fn("%s" % output) lu_result = False return lu_result ganeti-3.1.0~rc2/lib/cmdlib/common.py000064400000000000000000001601001476477700300174440ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Common functions used by multiple logical units.""" import copy import math import os import urllib.request, urllib.error, urllib.parse from ganeti import constants from ganeti import errors from ganeti import hypervisor from ganeti import locking from ganeti import objects from ganeti import opcodes from ganeti import pathutils import ganeti.rpc.node as rpc from ganeti.serializer import Private from ganeti import ssconf from ganeti import utils # States of instance INSTANCE_DOWN = [constants.ADMINST_DOWN] INSTANCE_ONLINE = [constants.ADMINST_DOWN, constants.ADMINST_UP] INSTANCE_NOT_RUNNING = [constants.ADMINST_DOWN, constants.ADMINST_OFFLINE] def _ExpandItemName(expand_fn, name, kind): """Expand an item name. @param expand_fn: the function to use for expansion @param name: requested item name @param kind: text description ('Node' or 'Instance') @return: the result of the expand_fn, if successful @raise errors.OpPrereqError: if the item is not found """ (uuid, full_name) = expand_fn(name) if uuid is None or full_name is None: raise errors.OpPrereqError("%s '%s' not known" % (kind, name), errors.ECODE_NOENT) return (uuid, full_name) def ExpandInstanceUuidAndName(cfg, expected_uuid, name): """Wrapper over L{_ExpandItemName} for instance.""" (uuid, full_name) = _ExpandItemName(cfg.ExpandInstanceName, name, "Instance") if expected_uuid is not None and uuid != expected_uuid: raise errors.OpPrereqError( "The instances UUID '%s' does not match the expected UUID '%s' for" " instance '%s'. Maybe the instance changed since you submitted this" " job." % (uuid, expected_uuid, full_name), errors.ECODE_NOTUNIQUE) return (uuid, full_name) def ExpandNodeUuidAndName(cfg, expected_uuid, name): """Expand a short node name into the node UUID and full name. @type cfg: L{config.ConfigWriter} @param cfg: The cluster configuration @type expected_uuid: string @param expected_uuid: expected UUID for the node (or None if there is no expectation). If it does not match, a L{errors.OpPrereqError} is raised. @type name: string @param name: the short node name """ (uuid, full_name) = _ExpandItemName(cfg.ExpandNodeName, name, "Node") if expected_uuid is not None and uuid != expected_uuid: raise errors.OpPrereqError( "The nodes UUID '%s' does not match the expected UUID '%s' for node" " '%s'. Maybe the node changed since you submitted this job." % (uuid, expected_uuid, full_name), errors.ECODE_NOTUNIQUE) return (uuid, full_name) def ShareAll(): """Returns a dict declaring all lock levels shared. """ return dict.fromkeys(locking.LEVELS, 1) def CheckNodeGroupInstances(cfg, group_uuid, owned_instance_names): """Checks if the instances in a node group are still correct. @type cfg: L{config.ConfigWriter} @param cfg: The cluster configuration @type group_uuid: string @param group_uuid: Node group UUID @type owned_instance_names: set or frozenset @param owned_instance_names: List of currently owned instances """ wanted_instances = frozenset(cfg.GetInstanceNames( cfg.GetNodeGroupInstances(group_uuid))) if owned_instance_names != wanted_instances: group_name = cfg.GetNodeGroup(group_uuid).name raise errors.OpPrereqError("Instances in node group '%s' changed since" " locks were acquired, wanted '%s', have '%s';" " retry the operation" % (group_name, utils.CommaJoin(wanted_instances), utils.CommaJoin(owned_instance_names)), errors.ECODE_STATE) return wanted_instances def GetWantedNodes(lu, short_node_names): """Returns list of checked and expanded node names. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type short_node_names: list @param short_node_names: list of node names or None for all nodes @rtype: tuple of lists @return: tupe with (list of node UUIDs, list of node names) @raise errors.ProgrammerError: if the nodes parameter is wrong type """ if short_node_names: node_uuids = [ExpandNodeUuidAndName(lu.cfg, None, name)[0] for name in short_node_names] else: node_uuids = lu.cfg.GetNodeList() return (node_uuids, [lu.cfg.GetNodeName(uuid) for uuid in node_uuids]) def GetWantedInstances(lu, short_inst_names): """Returns list of checked and expanded instance names. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type short_inst_names: list @param short_inst_names: list of instance names or None for all instances @rtype: tuple of lists @return: tuple of (instance UUIDs, instance names) @raise errors.OpPrereqError: if the instances parameter is wrong type @raise errors.OpPrereqError: if any of the passed instances is not found """ if short_inst_names: inst_uuids = [ExpandInstanceUuidAndName(lu.cfg, None, name)[0] for name in short_inst_names] else: inst_uuids = lu.cfg.GetInstanceList() return (inst_uuids, [lu.cfg.GetInstanceName(uuid) for uuid in inst_uuids]) def RunPostHook(lu, node_name): """Runs the post-hook for an opcode on a single node. """ hm = lu.proc.BuildHooksManager(lu) try: hm.RunPhase(constants.HOOKS_PHASE_POST, node_names=[node_name]) except Exception as err: # pylint: disable=W0703 lu.LogWarning("Errors occurred running hooks on %s: %s", node_name, err) def RedistributeAncillaryFiles(lu): """Distribute additional files which are part of the cluster configuration. ConfigWriter takes care of distributing the config and ssconf files, but there are more files which should be distributed to all nodes. This function makes sure those are copied. """ # Gather target nodes cluster = lu.cfg.GetClusterInfo() master_info = lu.cfg.GetMasterNodeInfo() online_node_uuids = lu.cfg.GetOnlineNodeList() online_node_uuid_set = frozenset(online_node_uuids) vm_node_uuids = list(online_node_uuid_set.intersection( lu.cfg.GetVmCapableNodeList())) # Never distribute to master node for node_uuids in [online_node_uuids, vm_node_uuids]: if master_info.uuid in node_uuids: node_uuids.remove(master_info.uuid) # Gather file lists (files_all, _, files_mc, files_vm) = \ ComputeAncillaryFiles(cluster, True) # Never re-distribute configuration file from here assert not (pathutils.CLUSTER_CONF_FILE in files_all or pathutils.CLUSTER_CONF_FILE in files_vm) assert not files_mc, "Master candidates not handled in this function" filemap = [ (online_node_uuids, files_all), (vm_node_uuids, files_vm), ] # Upload the files for (node_uuids, files) in filemap: for fname in files: UploadHelper(lu, node_uuids, fname) def ComputeAncillaryFiles(cluster, redist): """Compute files external to Ganeti which need to be consistent. @type redist: boolean @param redist: Whether to include files which need to be redistributed """ # Compute files for all nodes files_all = set([ pathutils.SSH_KNOWN_HOSTS_FILE, pathutils.CONFD_HMAC_KEY, pathutils.CLUSTER_DOMAIN_SECRET_FILE, pathutils.SPICE_CERT_FILE, pathutils.SPICE_CACERT_FILE, pathutils.RAPI_USERS_FILE, ]) if redist: # we need to ship at least the RAPI certificate files_all.add(pathutils.RAPI_CERT_FILE) else: files_all.update(pathutils.ALL_CERT_FILES) files_all.update(ssconf.SimpleStore().GetFileList()) if cluster.modify_etc_hosts: files_all.add(pathutils.ETC_HOSTS) if cluster.use_external_mip_script: files_all.add(pathutils.EXTERNAL_MASTER_SETUP_SCRIPT) # Files which are optional, these must: # - be present in one other category as well # - either exist or not exist on all nodes of that category (mc, vm all) files_opt = set([ pathutils.RAPI_USERS_FILE, ]) # Files which should only be on master candidates files_mc = set() if not redist: files_mc.add(pathutils.CLUSTER_CONF_FILE) # File storage if (not redist and (cluster.IsFileStorageEnabled() or cluster.IsSharedFileStorageEnabled())): files_all.add(pathutils.FILE_STORAGE_PATHS_FILE) files_opt.add(pathutils.FILE_STORAGE_PATHS_FILE) # Files which should only be on VM-capable nodes files_vm = set( filename for hv_name in cluster.enabled_hypervisors for filename in hypervisor.GetHypervisorClass(hv_name).GetAncillaryFiles()[0]) files_opt |= set( filename for hv_name in cluster.enabled_hypervisors for filename in hypervisor.GetHypervisorClass(hv_name).GetAncillaryFiles()[1]) # Filenames in each category must be unique all_files_set = files_all | files_mc | files_vm assert (len(all_files_set) == sum(map(len, [files_all, files_mc, files_vm]))), \ "Found file listed in more than one file list" # Optional files must be present in one other category assert all_files_set.issuperset(files_opt), \ "Optional file not in a different required list" # This one file should never ever be re-distributed via RPC assert not (redist and pathutils.FILE_STORAGE_PATHS_FILE in all_files_set) return (files_all, files_opt, files_mc, files_vm) def UploadHelper(lu, node_uuids, fname): """Helper for uploading a file and showing warnings. """ if os.path.exists(fname): result = lu.rpc.call_upload_file(node_uuids, fname) for to_node_uuids, to_result in result.items(): msg = to_result.fail_msg if msg: msg = ("Copy of file %s to node %s failed: %s" % (fname, lu.cfg.GetNodeName(to_node_uuids), msg)) lu.LogWarning(msg) def MergeAndVerifyHvState(op_input, obj_input): """Combines the hv state from an opcode with the one of the object @param op_input: The input dict from the opcode @param obj_input: The input dict from the objects @return: The verified and updated dict """ if op_input: invalid_hvs = set(op_input) - constants.HYPER_TYPES if invalid_hvs: raise errors.OpPrereqError("Invalid hypervisor(s) in hypervisor state:" " %s" % utils.CommaJoin(invalid_hvs), errors.ECODE_INVAL) if obj_input is None: obj_input = {} type_check = constants.HVSTS_PARAMETER_TYPES return _UpdateAndVerifySubDict(obj_input, op_input, type_check) return None def MergeAndVerifyDiskState(op_input, obj_input): """Combines the disk state from an opcode with the one of the object @param op_input: The input dict from the opcode @param obj_input: The input dict from the objects @return: The verified and updated dict """ if op_input: invalid_dst = set(op_input) - constants.DS_VALID_TYPES if invalid_dst: raise errors.OpPrereqError("Invalid storage type(s) in disk state: %s" % utils.CommaJoin(invalid_dst), errors.ECODE_INVAL) type_check = constants.DSS_PARAMETER_TYPES if obj_input is None: obj_input = {} return dict((key, _UpdateAndVerifySubDict(obj_input.get(key, {}), value, type_check)) for key, value in op_input.items()) return None def CheckOSParams(lu, required, node_uuids, osname, osparams, force_variant): """OS parameters validation. @type lu: L{LogicalUnit} @param lu: the logical unit for which we check @type required: boolean @param required: whether the validation should fail if the OS is not found @type node_uuids: list @param node_uuids: the list of nodes on which we should check @type osname: string @param osname: the name of the OS we should use @type osparams: dict @param osparams: the parameters which we need to check @raise errors.OpPrereqError: if the parameters are not valid """ node_uuids = _FilterVmNodes(lu, node_uuids) # Last chance to unwrap private elements. for key in osparams: if isinstance(osparams[key], Private): osparams[key] = osparams[key].Get() if osname: result = lu.rpc.call_os_validate(node_uuids, required, osname, [constants.OS_VALIDATE_PARAMETERS], osparams, force_variant) for node_uuid, nres in result.items(): # we don't check for offline cases since this should be run only # against the master node and/or an instance's nodes nres.Raise("OS Parameters validation failed on node %s" % lu.cfg.GetNodeName(node_uuid)) if not nres.payload: lu.LogInfo("OS %s not found on node %s, validation skipped", osname, lu.cfg.GetNodeName(node_uuid)) def CheckImageValidity(image, error_message): """Checks if a given image description is either a valid file path or a URL. @type image: string @param image: An absolute path or URL, the assumed location of a disk image. @type error_message: string @param error_message: The error message to show if the image is not valid. @raise errors.OpPrereqError: If the validation fails. """ if image is not None and not (utils.IsUrl(image) or os.path.isabs(image)): raise errors.OpPrereqError(error_message) def CheckOSImage(op): """Checks if the OS image in the OS parameters of an opcode is valid. This function can also be used in LUs as they carry an opcode. @type op: L{opcodes.OpCode} @param op: opcode containing the OS params @rtype: string or NoneType @return: None if the OS parameters in the opcode do not contain the OS image, otherwise the OS image value contained in the OS parameters @raise errors.OpPrereqError: if OS image is not a URL or an absolute path """ os_image = objects.GetOSImage(op.osparams) CheckImageValidity(os_image, "OS image must be an absolute path or a URL") return os_image def CheckHVParams(lu, node_uuids, hvname, hvparams): """Hypervisor parameter validation. This function abstracts the hypervisor parameter validation to be used in both instance create and instance modify. @type lu: L{LogicalUnit} @param lu: the logical unit for which we check @type node_uuids: list @param node_uuids: the list of nodes on which we should check @type hvname: string @param hvname: the name of the hypervisor we should use @type hvparams: dict @param hvparams: the parameters which we need to check @raise errors.OpPrereqError: if the parameters are not valid """ node_uuids = _FilterVmNodes(lu, node_uuids) cluster = lu.cfg.GetClusterInfo() hvfull = objects.FillDict(cluster.hvparams.get(hvname, {}), hvparams) hvinfo = lu.rpc.call_hypervisor_validate_params(node_uuids, hvname, hvfull) for node_uuid in node_uuids: info = hvinfo[node_uuid] if info.offline: continue info.Raise("Hypervisor parameter validation failed on node %s" % lu.cfg.GetNodeName(node_uuid)) def AddMasterCandidateSshKey( lu, master_node, node, potential_master_candidates, feedback_fn): ssh_result = lu.rpc.call_node_ssh_key_add( [master_node], node.uuid, node.name, potential_master_candidates, True, # add node's key to all node's 'authorized_keys' True, # all nodes are potential master candidates False) # do not update the node's public keys ssh_result[master_node].Raise( "Could not update the SSH setup of node '%s' after promotion" " (UUID: %s)." % (node.name, node.uuid)) WarnAboutFailedSshUpdates(ssh_result, master_node, feedback_fn) def AdjustCandidatePool( lu, exceptions, master_node, potential_master_candidates, feedback_fn, modify_ssh_setup): """Adjust the candidate pool after node operations. @type master_node: string @param master_node: name of the master node @type potential_master_candidates: list of string @param potential_master_candidates: list of node names of potential master candidates @type feedback_fn: function @param feedback_fn: function emitting user-visible output @type modify_ssh_setup: boolean @param modify_ssh_setup: whether or not the ssh setup can be modified. """ mod_list = lu.cfg.MaintainCandidatePool(exceptions) if mod_list: lu.LogInfo("Promoted nodes to master candidate role: %s", utils.CommaJoin(node.name for node in mod_list)) for node in mod_list: AddNodeCertToCandidateCerts(lu, lu.cfg, node.uuid) if modify_ssh_setup: AddMasterCandidateSshKey( lu, master_node, node, potential_master_candidates, feedback_fn) mc_now, mc_max, _ = lu.cfg.GetMasterCandidateStats(exceptions) if mc_now > mc_max: lu.LogInfo("Note: more nodes are candidates (%d) than desired (%d)" % (mc_now, mc_max)) def CheckNodePVs(nresult, exclusive_storage): """Check node PVs. """ pvlist_dict = nresult.get(constants.NV_PVLIST, None) if pvlist_dict is None: return (["Can't get PV list from node"], None) pvlist = [objects.LvmPvInfo.FromDict(d) for d in pvlist_dict] errlist = [] # check that ':' is not present in PV names, since it's a # special character for lvcreate (denotes the range of PEs to # use on the PV) for pv in pvlist: if ":" in pv.name: errlist.append("Invalid character ':' in PV '%s' of VG '%s'" % (pv.name, pv.vg_name)) es_pvinfo = None if exclusive_storage: (errmsgs, es_pvinfo) = utils.LvmExclusiveCheckNodePvs(pvlist) errlist.extend(errmsgs) shared_pvs = nresult.get(constants.NV_EXCLUSIVEPVS, None) if shared_pvs: for (pvname, lvlist) in shared_pvs: # TODO: Check that LVs are really unrelated (snapshots, DRBD meta...) errlist.append("PV %s is shared among unrelated LVs (%s)" % (pvname, utils.CommaJoin(lvlist))) return (errlist, es_pvinfo) def _ComputeMinMaxSpec(name, qualifier, ispecs, value): """Computes if value is in the desired range. @param name: name of the parameter for which we perform the check @param qualifier: a qualifier used in the error message (e.g. 'disk/1', not just 'disk') @param ispecs: dictionary containing min and max values @param value: actual value that we want to use @return: None or an error string """ if value in [None, constants.VALUE_AUTO]: return None max_v = ispecs[constants.ISPECS_MAX].get(name, value) min_v = ispecs[constants.ISPECS_MIN].get(name, value) if value > max_v or min_v > value: if qualifier: fqn = "%s/%s" % (name, qualifier) else: fqn = name return ("%s value %s is not in range [%s, %s]" % (fqn, value, min_v, max_v)) return None def ComputeIPolicySpecViolation(ipolicy, mem_size, cpu_count, disk_count, nic_count, disk_sizes, spindle_use, disk_types, _compute_fn=_ComputeMinMaxSpec): """Verifies ipolicy against provided specs. @type ipolicy: dict @param ipolicy: The ipolicy @type mem_size: int @param mem_size: The memory size @type cpu_count: int @param cpu_count: Used cpu cores @type disk_count: int @param disk_count: Number of disks used @type nic_count: int @param nic_count: Number of nics used @type disk_sizes: list of ints @param disk_sizes: Disk sizes of used disk (len must match C{disk_count}) @type spindle_use: int @param spindle_use: The number of spindles this instance uses @type disk_types: list of strings @param disk_types: The disk template of the instance @param _compute_fn: The compute function (unittest only) @return: A list of violations, or an empty list of no violations are found """ assert disk_count == len(disk_sizes) assert isinstance(disk_types, list) assert disk_count == len(disk_types) test_settings = [ (constants.ISPEC_MEM_SIZE, "", mem_size), (constants.ISPEC_CPU_COUNT, "", cpu_count), (constants.ISPEC_NIC_COUNT, "", nic_count), (constants.ISPEC_SPINDLE_USE, "", spindle_use), ] + [(constants.ISPEC_DISK_SIZE, str(idx), d) for idx, d in enumerate(disk_sizes)] allowed_dts = set(ipolicy[constants.IPOLICY_DTS]) ret = [] if disk_count != 0: # This check doesn't make sense for diskless instances test_settings.append((constants.ISPEC_DISK_COUNT, "", disk_count)) elif constants.DT_DISKLESS not in allowed_dts: ret.append("Disk template %s is not allowed (allowed templates %s)" % (constants.DT_DISKLESS, utils.CommaJoin(allowed_dts))) forbidden_dts = set(disk_types) - allowed_dts if forbidden_dts: ret.append("Disk template %s is not allowed (allowed templates: %s)" % (utils.CommaJoin(forbidden_dts), utils.CommaJoin(allowed_dts))) min_errs = None for minmax in ipolicy[constants.ISPECS_MINMAX]: errs = [err for err in (_compute_fn(name, qualifier, minmax, value) for (name, qualifier, value) in test_settings) if err] if min_errs is None or len(errs) < len(min_errs): min_errs = errs assert min_errs is not None return ret + min_errs def ComputeIPolicyDiskSizesViolation(ipolicy, disk_sizes, disks, _compute_fn=_ComputeMinMaxSpec): """Verifies ipolicy against provided disk sizes. No other specs except the disk sizes, the number of disks and the disk template are checked. @type ipolicy: dict @param ipolicy: The ipolicy @type disk_sizes: list of ints @param disk_sizes: Disk sizes of used disk (len must match C{disk_count}) @type disks: list of L{Disk} @param disks: The Disk objects of the instance @param _compute_fn: The compute function (unittest only) @return: A list of violations, or an empty list of no violations are found """ if len(disk_sizes) != len(disks): return [constants.ISPEC_DISK_COUNT] dev_types = [d.dev_type for d in disks] return ComputeIPolicySpecViolation(ipolicy, # mem_size, cpu_count, disk_count None, None, len(disk_sizes), None, disk_sizes, # nic_count, disk_sizes None, # spindle_use dev_types, _compute_fn=_compute_fn) def ComputeIPolicyInstanceViolation(ipolicy, instance, cfg, _compute_fn=ComputeIPolicySpecViolation): """Compute if instance meets the specs of ipolicy. @type ipolicy: dict @param ipolicy: The ipolicy to verify against @type instance: L{objects.Instance} @param instance: The instance to verify @type cfg: L{config.ConfigWriter} @param cfg: Cluster configuration @param _compute_fn: The function to verify ipolicy (unittest only) @see: L{ComputeIPolicySpecViolation} """ ret = [] be_full = cfg.GetClusterInfo().FillBE(instance) mem_size = be_full[constants.BE_MAXMEM] cpu_count = be_full[constants.BE_VCPUS] inst_nodes = cfg.GetInstanceNodes(instance.uuid) es_flags = rpc.GetExclusiveStorageForNodes(cfg, inst_nodes) disks = cfg.GetInstanceDisks(instance.uuid) if any(es_flags.values()): # With exclusive storage use the actual spindles try: spindle_use = sum([disk.spindles for disk in disks]) except TypeError: ret.append("Number of spindles not configured for disks of instance %s" " while exclusive storage is enabled, try running gnt-cluster" " repair-disk-sizes" % instance.name) # _ComputeMinMaxSpec ignores 'None's spindle_use = None else: spindle_use = be_full[constants.BE_SPINDLE_USE] disk_count = len(disks) disk_sizes = [disk.size for disk in disks] nic_count = len(instance.nics) disk_types = [d.dev_type for d in disks] return ret + _compute_fn(ipolicy, mem_size, cpu_count, disk_count, nic_count, disk_sizes, spindle_use, disk_types) def _ComputeViolatingInstances(ipolicy, instances, cfg): """Computes a set of instances who violates given ipolicy. @param ipolicy: The ipolicy to verify @type instances: L{objects.Instance} @param instances: List of instances to verify @type cfg: L{config.ConfigWriter} @param cfg: Cluster configuration @return: A frozenset of instance names violating the ipolicy """ return frozenset([inst.name for inst in instances if ComputeIPolicyInstanceViolation(ipolicy, inst, cfg)]) def ComputeNewInstanceViolations(old_ipolicy, new_ipolicy, instances, cfg): """Computes a set of any instances that would violate the new ipolicy. @param old_ipolicy: The current (still in-place) ipolicy @param new_ipolicy: The new (to become) ipolicy @param instances: List of instances to verify @type cfg: L{config.ConfigWriter} @param cfg: Cluster configuration @return: A list of instances which violates the new ipolicy but did not before """ return (_ComputeViolatingInstances(new_ipolicy, instances, cfg) - _ComputeViolatingInstances(old_ipolicy, instances, cfg)) def GetUpdatedParams(old_params, update_dict, use_default=True, use_none=False): """Return the new version of a parameter dictionary. @type old_params: dict @param old_params: old parameters @type update_dict: dict @param update_dict: dict containing new parameter values, or constants.VALUE_DEFAULT to reset the parameter to its default value @param use_default: boolean @type use_default: whether to recognise L{constants.VALUE_DEFAULT} values as 'to be deleted' values @param use_none: boolean @type use_none: whether to recognise C{None} values as 'to be deleted' values @rtype: dict @return: the new parameter dictionary """ params_copy = copy.deepcopy(old_params) for key, val in update_dict.items(): if ((use_default and val == constants.VALUE_DEFAULT) or (use_none and val is None)): try: del params_copy[key] except KeyError: pass else: params_copy[key] = val return params_copy def GetUpdatedIPolicy(old_ipolicy, new_ipolicy, group_policy=False): """Return the new version of an instance policy. @param group_policy: whether this policy applies to a group and thus we should support removal of policy entries """ ipolicy = copy.deepcopy(old_ipolicy) for key, value in new_ipolicy.items(): if key not in constants.IPOLICY_ALL_KEYS: raise errors.OpPrereqError("Invalid key in new ipolicy: %s" % key, errors.ECODE_INVAL) if (not value or value == [constants.VALUE_DEFAULT] or value == constants.VALUE_DEFAULT): if group_policy: if key in ipolicy: del ipolicy[key] else: raise errors.OpPrereqError("Can't unset ipolicy attribute '%s'" " on the cluster'" % key, errors.ECODE_INVAL) else: if key in constants.IPOLICY_PARAMETERS: # FIXME: we assume all such values are float try: ipolicy[key] = float(value) except (TypeError, ValueError) as err: raise errors.OpPrereqError("Invalid value for attribute" " '%s': '%s', error: %s" % (key, value, err), errors.ECODE_INVAL) elif key == constants.ISPECS_MINMAX: for minmax in value: for k in minmax: utils.ForceDictType(minmax[k], constants.ISPECS_PARAMETER_TYPES) ipolicy[key] = value elif key == constants.ISPECS_STD: if group_policy: msg = "%s cannot appear in group instance specs" % key raise errors.OpPrereqError(msg, errors.ECODE_INVAL) ipolicy[key] = GetUpdatedParams(old_ipolicy.get(key, {}), value, use_none=False, use_default=False) utils.ForceDictType(ipolicy[key], constants.ISPECS_PARAMETER_TYPES) else: # FIXME: we assume all others are lists; this should be redone # in a nicer way ipolicy[key] = list(value) try: objects.InstancePolicy.CheckParameterSyntax(ipolicy, not group_policy) except errors.ConfigurationError as err: raise errors.OpPrereqError("Invalid instance policy: %s" % err, errors.ECODE_INVAL) return ipolicy def AnnotateDiskParams(instance, devs, cfg): """Little helper wrapper to the rpc annotation method. @param instance: The instance object @type devs: List of L{objects.Disk} @param devs: The root devices (not any of its children!) @param cfg: The config object @returns The annotated disk copies @see L{ganeti.rpc.node.AnnotateDiskParams} """ return rpc.AnnotateDiskParams(devs, cfg.GetInstanceDiskParams(instance)) def SupportsOob(cfg, node): """Tells if node supports OOB. @type cfg: L{config.ConfigWriter} @param cfg: The cluster configuration @type node: L{objects.Node} @param node: The node @return: The OOB script if supported or an empty string otherwise """ return cfg.GetNdParams(node)[constants.ND_OOB_PROGRAM] def _UpdateAndVerifySubDict(base, updates, type_check): """Updates and verifies a dict with sub dicts of the same type. @param base: The dict with the old data @param updates: The dict with the new data @param type_check: Dict suitable to ForceDictType to verify correct types @returns: A new dict with updated and verified values """ def fn(old, value): new = GetUpdatedParams(old, value) utils.ForceDictType(new, type_check) return new ret = copy.deepcopy(base) ret.update(dict((key, fn(base.get(key, {}), value)) for key, value in updates.items())) return ret def _FilterVmNodes(lu, node_uuids): """Filters out non-vm_capable nodes from a list. @type lu: L{LogicalUnit} @param lu: the logical unit for which we check @type node_uuids: list @param node_uuids: the list of nodes on which we should check @rtype: list @return: the list of vm-capable nodes """ vm_nodes = frozenset(lu.cfg.GetNonVmCapableNodeList()) return [uuid for uuid in node_uuids if uuid not in vm_nodes] def GetDefaultIAllocator(cfg, ialloc): """Decides on which iallocator to use. @type cfg: L{config.ConfigWriter} @param cfg: Cluster configuration object @type ialloc: string or None @param ialloc: Iallocator specified in opcode @rtype: string @return: Iallocator name """ if not ialloc: # Use default iallocator ialloc = cfg.GetDefaultIAllocator() if not ialloc: raise errors.OpPrereqError("No iallocator was specified, neither in the" " opcode nor as a cluster-wide default", errors.ECODE_INVAL) return ialloc def CheckInstancesNodeGroups(cfg, instances, owned_groups, owned_node_uuids, cur_group_uuid): """Checks if node groups for locked instances are still correct. @type cfg: L{config.ConfigWriter} @param cfg: Cluster configuration @type instances: dict; string as key, L{objects.Instance} as value @param instances: Dictionary, instance UUID as key, instance object as value @type owned_groups: iterable of string @param owned_groups: List of owned groups @type owned_node_uuids: iterable of string @param owned_node_uuids: List of owned nodes @type cur_group_uuid: string or None @param cur_group_uuid: Optional group UUID to check against instance's groups """ for (uuid, inst) in instances.items(): inst_nodes = cfg.GetInstanceNodes(inst.uuid) assert owned_node_uuids.issuperset(inst_nodes), \ "Instance %s's nodes changed while we kept the lock" % inst.name inst_groups = CheckInstanceNodeGroups(cfg, uuid, owned_groups) assert cur_group_uuid is None or cur_group_uuid in inst_groups, \ "Instance %s has no node in group %s" % (inst.name, cur_group_uuid) def CheckInstanceNodeGroups(cfg, inst_uuid, owned_groups, primary_only=False): """Checks if the owned node groups are still correct for an instance. @type cfg: L{config.ConfigWriter} @param cfg: The cluster configuration @type inst_uuid: string @param inst_uuid: Instance UUID @type owned_groups: set or frozenset @param owned_groups: List of currently owned node groups @type primary_only: boolean @param primary_only: Whether to check node groups for only the primary node """ inst_groups = cfg.GetInstanceNodeGroups(inst_uuid, primary_only) if not owned_groups.issuperset(inst_groups): raise errors.OpPrereqError("Instance %s's node groups changed since" " locks were acquired, current groups are" " are '%s', owning groups '%s'; retry the" " operation" % (cfg.GetInstanceName(inst_uuid), utils.CommaJoin(inst_groups), utils.CommaJoin(owned_groups)), errors.ECODE_STATE) return inst_groups def LoadNodeEvacResult(lu, alloc_result, early_release, use_nodes): """Unpacks the result of change-group and node-evacuate iallocator requests. Iallocator modes L{constants.IALLOCATOR_MODE_NODE_EVAC} and L{constants.IALLOCATOR_MODE_CHG_GROUP}. @type lu: L{LogicalUnit} @param lu: Logical unit instance @type alloc_result: tuple/list @param alloc_result: Result from iallocator @type early_release: bool @param early_release: Whether to release locks early if possible @type use_nodes: bool @param use_nodes: Whether to display node names instead of groups """ (moved, failed, jobs) = alloc_result if failed: failreason = utils.CommaJoin("%s (%s)" % (name, reason) for (name, reason) in failed) lu.LogWarning("Unable to evacuate instances %s", failreason) raise errors.OpExecError("Unable to evacuate instances %s" % failreason) if moved: lu.LogInfo("Instances to be moved: %s", utils.CommaJoin( "%s (to %s)" % (name, _NodeEvacDest(use_nodes, group, node_names)) for (name, group, node_names) in moved)) return [ [ _SetOpEarlyRelease(early_release, opcodes.OpCode.LoadOpCode(o)) for o in ops ] for ops in jobs ] def _NodeEvacDest(use_nodes, group, node_names): """Returns group or nodes depending on caller's choice. """ if use_nodes: return utils.CommaJoin(node_names) else: return group def _SetOpEarlyRelease(early_release, op): """Sets C{early_release} flag on opcodes if available. """ try: op.early_release = early_release except AttributeError: assert not isinstance(op, opcodes.OpInstanceReplaceDisks) return op def MapInstanceLvsToNodes(cfg, instances): """Creates a map from (node, volume) to instance name. @type cfg: L{config.ConfigWriter} @param cfg: The cluster configuration @type instances: list of L{objects.Instance} @rtype: dict; tuple of (node uuid, volume name) as key, L{objects.Instance} object as value """ return dict( ((node_uuid, vol), inst) for inst in instances for (node_uuid, vols) in cfg.GetInstanceLVsByNode(inst.uuid).items() for vol in vols) def CheckParamsNotGlobal(params, glob_pars, kind, bad_levels, good_levels): """Make sure that none of the given paramters is global. If a global parameter is found, an L{errors.OpPrereqError} exception is raised. This is used to avoid setting global parameters for individual nodes. @type params: dictionary @param params: Parameters to check @type glob_pars: dictionary @param glob_pars: Forbidden parameters @type kind: string @param kind: Kind of parameters (e.g. "node") @type bad_levels: string @param bad_levels: Level(s) at which the parameters are forbidden (e.g. "instance") @type good_levels: strings @param good_levels: Level(s) at which the parameters are allowed (e.g. "cluster or group") """ used_globals = glob_pars.intersection(params) if used_globals: msg = ("The following %s parameters are global and cannot" " be customized at %s level, please modify them at" " %s level: %s" % (kind, bad_levels, good_levels, utils.CommaJoin(used_globals))) raise errors.OpPrereqError(msg, errors.ECODE_INVAL) def IsExclusiveStorageEnabledNode(cfg, node): """Whether exclusive_storage is in effect for the given node. @type cfg: L{config.ConfigWriter} @param cfg: The cluster configuration @type node: L{objects.Node} @param node: The node @rtype: bool @return: The effective value of exclusive_storage """ return cfg.GetNdParams(node)[constants.ND_EXCLUSIVE_STORAGE] def IsInstanceRunning(lu, instance, prereq=True): """Given an instance object, checks if the instance is running. This function asks the backend whether the instance is running and user shutdown instances are considered not to be running. @type lu: L{LogicalUnit} @param lu: LU on behalf of which we make the check @type instance: L{objects.Instance} @param instance: instance to check whether it is running @rtype: bool @return: 'True' if the instance is running, 'False' otherwise """ hvparams = lu.cfg.GetClusterInfo().FillHV(instance) result = lu.rpc.call_instance_info(instance.primary_node, instance.name, instance.hypervisor, hvparams) # TODO: This 'prepreq=True' is a problem if this function is called # within the 'Exec' method of a LU. result.Raise("Can't retrieve instance information for instance '%s'" % instance.name, prereq=prereq, ecode=errors.ECODE_ENVIRON) return result.payload and \ "state" in result.payload and \ (result.payload["state"] != hypervisor.hv_base.HvInstanceState.SHUTDOWN) def CheckInstanceState(lu, instance, req_states, msg=None): """Ensure that an instance is in one of the required states. @param lu: the LU on behalf of which we make the check @param instance: the instance to check @param msg: if passed, should be a message to replace the default one @raise errors.OpPrereqError: if the instance is not in the required state """ if msg is None: msg = ("can't use instance from outside %s states" % utils.CommaJoin(req_states)) if instance.admin_state not in req_states: raise errors.OpPrereqError("Instance '%s' is marked to be %s, %s" % (instance.name, instance.admin_state, msg), errors.ECODE_STATE) if constants.ADMINST_UP not in req_states: pnode_uuid = instance.primary_node # Replicating the offline check if not lu.cfg.GetNodeInfo(pnode_uuid).offline: if IsInstanceRunning(lu, instance): raise errors.OpPrereqError("Instance %s is running, %s" % (instance.name, msg), errors.ECODE_STATE) else: lu.LogWarning("Primary node offline, ignoring check that instance" " is down") def CheckIAllocatorOrNode(lu, iallocator_slot, node_slot): """Check the sanity of iallocator and node arguments and use the cluster-wide iallocator if appropriate. Check that at most one of (iallocator, node) is specified. If none is specified, or the iallocator is L{constants.DEFAULT_IALLOCATOR_SHORTCUT}, then the LU's opcode's iallocator slot is filled with the cluster-wide default iallocator. @type iallocator_slot: string @param iallocator_slot: the name of the opcode iallocator slot @type node_slot: string @param node_slot: the name of the opcode target node slot """ node = getattr(lu.op, node_slot, None) ialloc = getattr(lu.op, iallocator_slot, None) if node == []: node = None if node is not None and ialloc is not None: raise errors.OpPrereqError("Do not specify both, iallocator and node", errors.ECODE_INVAL) elif ((node is None and ialloc is None) or ialloc == constants.DEFAULT_IALLOCATOR_SHORTCUT): default_iallocator = lu.cfg.GetDefaultIAllocator() if default_iallocator: setattr(lu.op, iallocator_slot, default_iallocator) else: raise errors.OpPrereqError("No iallocator or node given and no" " cluster-wide default iallocator found;" " please specify either an iallocator or a" " node, or set a cluster-wide default" " iallocator", errors.ECODE_INVAL) def FindFaultyInstanceDisks(cfg, rpc_runner, instance, node_uuid, prereq): faulty = [] disks = cfg.GetInstanceDisks(instance.uuid) result = rpc_runner.call_blockdev_getmirrorstatus( node_uuid, (disks, instance)) result.Raise("Failed to get disk status from node %s" % cfg.GetNodeName(node_uuid), prereq=prereq, ecode=errors.ECODE_ENVIRON) for idx, bdev_status in enumerate(result.payload): if bdev_status and bdev_status.ldisk_status == constants.LDS_FAULTY: faulty.append(idx) return faulty def CheckNodeOnline(lu, node_uuid, msg=None): """Ensure that a given node is online. @param lu: the LU on behalf of which we make the check @param node_uuid: the node to check @param msg: if passed, should be a message to replace the default one @raise errors.OpPrereqError: if the node is offline """ if msg is None: msg = "Can't use offline node" if lu.cfg.GetNodeInfo(node_uuid).offline: raise errors.OpPrereqError("%s: %s" % (msg, lu.cfg.GetNodeName(node_uuid)), errors.ECODE_STATE) def CheckDiskTemplateEnabled(cluster, disk_template): """Helper function to check if a disk template is enabled. @type cluster: C{objects.Cluster} @param cluster: the cluster's configuration @type disk_template: str @param disk_template: the disk template to be checked """ assert disk_template is not None if disk_template not in constants.DISK_TEMPLATES: raise errors.OpPrereqError("'%s' is not a valid disk template." " Valid disk templates are: %s" % (disk_template, ",".join(constants.DISK_TEMPLATES))) if not disk_template in cluster.enabled_disk_templates: raise errors.OpPrereqError("Disk template '%s' is not enabled in cluster." " Enabled disk templates are: %s" % (disk_template, ",".join(cluster.enabled_disk_templates))) def CheckStorageTypeEnabled(cluster, storage_type): """Helper function to check if a storage type is enabled. @type cluster: C{objects.Cluster} @param cluster: the cluster's configuration @type storage_type: str @param storage_type: the storage type to be checked """ assert storage_type is not None assert storage_type in constants.STORAGE_TYPES # special case for lvm-pv, because it cannot be enabled # via disk templates if storage_type == constants.ST_LVM_PV: CheckStorageTypeEnabled(cluster, constants.ST_LVM_VG) else: possible_disk_templates = \ utils.storage.GetDiskTemplatesOfStorageTypes(storage_type) for disk_template in possible_disk_templates: if disk_template in cluster.enabled_disk_templates: return raise errors.OpPrereqError("No disk template of storage type '%s' is" " enabled in this cluster. Enabled disk" " templates are: %s" % (storage_type, ",".join(cluster.enabled_disk_templates))) def CheckIpolicyVsDiskTemplates(ipolicy, enabled_disk_templates): """Checks ipolicy disk templates against enabled disk tempaltes. @type ipolicy: dict @param ipolicy: the new ipolicy @type enabled_disk_templates: list of string @param enabled_disk_templates: list of enabled disk templates on the cluster @raises errors.OpPrereqError: if there is at least one allowed disk template that is not also enabled. """ assert constants.IPOLICY_DTS in ipolicy allowed_disk_templates = ipolicy[constants.IPOLICY_DTS] not_enabled = set(allowed_disk_templates) - set(enabled_disk_templates) if not_enabled: raise errors.OpPrereqError("The following disk templates are allowed" " by the ipolicy, but not enabled on the" " cluster: %s" % utils.CommaJoin(not_enabled), errors.ECODE_INVAL) def CheckDiskAccessModeValidity(parameters): """Checks if the access parameter is legal. @see: L{CheckDiskAccessModeConsistency} for cluster consistency checks. @raise errors.OpPrereqError: if the check fails. """ for disk_template in parameters: access = parameters[disk_template].get(constants.LDP_ACCESS, constants.DISK_KERNELSPACE) if access not in constants.DISK_VALID_ACCESS_MODES: valid_vals_str = utils.CommaJoin(constants.DISK_VALID_ACCESS_MODES) raise errors.OpPrereqError("Invalid value of '{d}:{a}': '{v}' (expected" " one of {o})".format(d=disk_template, a=constants.LDP_ACCESS, v=access, o=valid_vals_str)) def CheckDiskAccessModeConsistency(parameters, cfg, group=None): """Checks if the access param is consistent with the cluster configuration. @note: requires a configuration lock to run. @param parameters: the parameters to validate @param cfg: the cfg object of the cluster @param group: if set, only check for consistency within this group. @raise errors.OpPrereqError: if the LU attempts to change the access parameter to an invalid value, such as "pink bunny". @raise errors.OpPrereqError: if the LU attempts to change the access parameter to an inconsistent value, such as asking for RBD userspace access to the chroot hypervisor. """ CheckDiskAccessModeValidity(parameters) for disk_template in parameters: access = parameters[disk_template].get(constants.LDP_ACCESS, constants.DISK_KERNELSPACE) if disk_template not in constants.DTS_HAVE_ACCESS: continue #Check the combination of instance hypervisor, disk template and access #protocol is sane. inst_uuids = cfg.GetNodeGroupInstances(group) if group else \ cfg.GetInstanceList() for entry in inst_uuids: inst = cfg.GetInstanceInfo(entry) disks = cfg.GetInstanceDisks(entry) for disk in disks: if disk.dev_type != disk_template: continue hv = inst.hypervisor if not IsValidDiskAccessModeCombination(hv, disk.dev_type, access): raise errors.OpPrereqError("Instance {i}: cannot use '{a}' access" " setting with {h} hypervisor and {d} disk" " type.".format(i=inst.name, a=access, h=hv, d=disk.dev_type)) def IsValidDiskAccessModeCombination(hv, disk_template, mode): """Checks if an hypervisor can read a disk template with given mode. @param hv: the hypervisor that will access the data @param disk_template: the disk template the data is stored as @param mode: how the hypervisor should access the data @return: True if the hypervisor can read a given read disk_template in the specified mode. """ if mode == constants.DISK_KERNELSPACE: return True if (hv == constants.HT_KVM and disk_template in constants.DTS_HAVE_ACCESS and mode == constants.DISK_USERSPACE): return True # Everything else: return False def AddNodeCertToCandidateCerts(lu, cfg, node_uuid): """Add the node's client SSL certificate digest to the candidate certs. @type lu: L{LogicalUnit} @param lu: the logical unit @type cfg: L{ConfigWriter} @param cfg: the configuration client to use @type node_uuid: string @param node_uuid: the node's UUID """ result = lu.rpc.call_node_crypto_tokens( node_uuid, [(constants.CRYPTO_TYPE_SSL_DIGEST, constants.CRYPTO_ACTION_GET, None)]) result.Raise("Could not retrieve the node's (uuid %s) SSL digest." % node_uuid) ((crypto_type, digest), ) = result.payload assert crypto_type == constants.CRYPTO_TYPE_SSL_DIGEST cfg.AddNodeToCandidateCerts(node_uuid, digest) def RemoveNodeCertFromCandidateCerts(cfg, node_uuid): """Removes the node's certificate from the candidate certificates list. @type cfg: C{config.ConfigWriter} @param cfg: the cluster's configuration @type node_uuid: string @param node_uuid: the node's UUID """ cfg.RemoveNodeFromCandidateCerts(node_uuid) def GetClientCertDigest(lu, node_uuid, filename=None): """Get the client SSL certificate digest for the node. @type node_uuid: string @param node_uuid: the node's UUID @type filename: string @param filename: the certificate's filename @rtype: string @return: the digest of the newly created certificate """ options = {} if filename: options[constants.CRYPTO_OPTION_CERT_FILE] = filename result = lu.rpc.call_node_crypto_tokens( node_uuid, [(constants.CRYPTO_TYPE_SSL_DIGEST, constants.CRYPTO_ACTION_GET, options)]) result.Raise("Could not fetch the node's (uuid %s) SSL client" " certificate." % node_uuid) ((crypto_type, new_digest), ) = result.payload assert crypto_type == constants.CRYPTO_TYPE_SSL_DIGEST return new_digest def AddInstanceCommunicationNetworkOp(network): """Create an OpCode that adds the instance communication network. This OpCode contains the configuration necessary for the instance communication network. @type network: string @param network: name or UUID of the instance communication network @rtype: L{ganeti.opcodes.OpCode} @return: OpCode that creates the instance communication network """ return opcodes.OpNetworkAdd( network_name=network, gateway=None, network=constants.INSTANCE_COMMUNICATION_NETWORK4, gateway6=None, network6=constants.INSTANCE_COMMUNICATION_NETWORK6, mac_prefix=constants.INSTANCE_COMMUNICATION_MAC_PREFIX, add_reserved_ips=None, conflicts_check=True, tags=[]) def ConnectInstanceCommunicationNetworkOp(group_uuid, network): """Create an OpCode that connects a group to the instance communication network. This OpCode contains the configuration necessary for the instance communication network. @type group_uuid: string @param group_uuid: UUID of the group to connect @type network: string @param network: name or UUID of the network to connect to, i.e., the instance communication network @rtype: L{ganeti.opcodes.OpCode} @return: OpCode that connects the group to the instance communication network """ return opcodes.OpNetworkConnect( group_name=group_uuid, network_name=network, network_mode=constants.INSTANCE_COMMUNICATION_NETWORK_MODE, network_link=constants.INSTANCE_COMMUNICATION_NETWORK_LINK, conflicts_check=True) def DetermineImageSize(lu, image, node_uuid): """Determines the size of the specified image. @type image: string @param image: absolute filepath or URL of the image @type node_uuid: string @param node_uuid: if L{image} is a filepath, this is the UUID of the node where the image is located @rtype: int @return: size of the image in MB, rounded up @raise OpExecError: if the image does not exist """ # Check if we are dealing with a URL first class _HeadRequest(urllib.request.Request): def get_method(self): return "HEAD" if utils.IsUrl(image): try: response = urllib.request.urlopen(_HeadRequest(image)) except urllib.error.URLError: raise errors.OpExecError("Could not retrieve image from given url '%s'" % image) content_length_str = response.info().getheader('content-length') if not content_length_str: raise errors.OpExecError("Could not determine image size from given url" " '%s'" % image) byte_size = int(content_length_str) else: # We end up here if a file path is used result = lu.rpc.call_get_file_info(node_uuid, image) result.Raise("Could not determine size of file '%s'" % image) success, attributes = result.payload if not success: raise errors.OpExecError("Could not open file '%s'" % image) byte_size = attributes[constants.STAT_SIZE] # Finally, the conversion return math.ceil(byte_size / 1024. / 1024.) def EnsureKvmdOnNodes(lu, feedback_fn, nodes=None, silent_stop=False): """Ensure KVM daemon is running on nodes with KVM instances. If user shutdown is enabled in the cluster: - The KVM daemon will be started on VM capable nodes containing KVM instances. - The KVM daemon will be stopped on non VM capable nodes. If user shutdown is disabled in the cluster: - The KVM daemon will be stopped on all nodes Issues a warning for each failed RPC call. @type lu: L{LogicalUnit} @param lu: logical unit on whose behalf we execute @type feedback_fn: callable @param feedback_fn: feedback function @type nodes: list of string @param nodes: if supplied, it overrides the node uuids to start/stop; this is used mainly for optimization @type silent_stop: bool @param silent_stop: if we should suppress warnings in case KVM daemon is already stopped """ cluster = lu.cfg.GetClusterInfo() # Either use the passed nodes or consider all cluster nodes if nodes is not None: node_uuids = set(nodes) else: node_uuids = lu.cfg.GetNodeList() # Determine in which nodes should the KVM daemon be started/stopped if constants.HT_KVM in cluster.enabled_hypervisors and \ cluster.enabled_user_shutdown: start_nodes = [] stop_nodes = [] for node_uuid in node_uuids: if lu.cfg.GetNodeInfo(node_uuid).vm_capable: start_nodes.append(node_uuid) else: stop_nodes.append(node_uuid) else: start_nodes = [] stop_nodes = node_uuids # Start KVM where necessary if start_nodes: results = lu.rpc.call_node_ensure_daemon(start_nodes, constants.KVMD, True) for node_uuid in start_nodes: results[node_uuid].Warn("Failed to start KVM daemon on node '%s'" % lu.cfg.GetNodeName(node_uuid), feedback_fn) # Stop KVM where necessary if stop_nodes: results = lu.rpc.call_node_ensure_daemon(stop_nodes, constants.KVMD, False) if not silent_stop: for node_uuid in stop_nodes: results[node_uuid].Warn("Failed to stop KVM daemon on node '%s'" % lu.cfg.GetNodeName(node_uuid), feedback_fn) def WarnAboutFailedSshUpdates(result, master_uuid, feedback_fn): node_errors = result[master_uuid].payload if node_errors: feedback_fn("Some nodes' SSH key files could not be updated:") for node_name, error_msg in node_errors: feedback_fn("%s: %s" % (node_name, error_msg)) ganeti-3.1.0~rc2/lib/cmdlib/group.py000064400000000000000000001053531476477700300173210ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with node groups.""" import itertools import logging from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import objects from ganeti import opcodes from ganeti import utils from ganeti.masterd import iallocator from ganeti.cmdlib.base import LogicalUnit, NoHooksLU, ResultWithJobs from ganeti.cmdlib.common import MergeAndVerifyHvState, \ MergeAndVerifyDiskState, GetWantedNodes, GetUpdatedParams, \ CheckNodeGroupInstances, GetUpdatedIPolicy, \ ComputeNewInstanceViolations, GetDefaultIAllocator, ShareAll, \ CheckInstancesNodeGroups, LoadNodeEvacResult, MapInstanceLvsToNodes, \ CheckIpolicyVsDiskTemplates, CheckDiskAccessModeValidity, \ CheckDiskAccessModeConsistency, ConnectInstanceCommunicationNetworkOp import ganeti.masterd.instance class LUGroupAdd(LogicalUnit): """Logical unit for creating node groups. """ HPATH = "group-add" HTYPE = constants.HTYPE_GROUP REQ_BGL = False def ExpandNames(self): # We need the new group's UUID here so that we can create and acquire the # corresponding lock. Later, in Exec(), we'll indicate to cfg.AddNodeGroup # that it should not check whether the UUID exists in the configuration. self.group_uuid = self.cfg.GenerateUniqueID(self.proc.GetECId()) self.needed_locks = {} self.add_locks[locking.LEVEL_NODEGROUP] = self.group_uuid def _CheckIpolicy(self): """Checks the group's ipolicy for consistency and validity. """ if self.op.ipolicy: cluster = self.cfg.GetClusterInfo() full_ipolicy = cluster.SimpleFillIPolicy(self.op.ipolicy) try: objects.InstancePolicy.CheckParameterSyntax(full_ipolicy, False) except errors.ConfigurationError as err: raise errors.OpPrereqError("Invalid instance policy: %s" % err, errors.ECODE_INVAL) CheckIpolicyVsDiskTemplates(full_ipolicy, cluster.enabled_disk_templates) def CheckPrereq(self): """Check prerequisites. This checks that the given group name is not an existing node group already. """ try: existing_uuid = self.cfg.LookupNodeGroup(self.op.group_name) except errors.OpPrereqError: pass else: raise errors.OpPrereqError("Desired group name '%s' already exists as a" " node group (UUID: %s)" % (self.op.group_name, existing_uuid), errors.ECODE_EXISTS) if self.op.ndparams: utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES) if self.op.hv_state: self.new_hv_state = MergeAndVerifyHvState(self.op.hv_state, None) else: self.new_hv_state = None if self.op.disk_state: self.new_disk_state = MergeAndVerifyDiskState(self.op.disk_state, None) else: self.new_disk_state = None if self.op.diskparams: for templ in constants.DISK_TEMPLATES: if templ in self.op.diskparams: utils.ForceDictType(self.op.diskparams[templ], constants.DISK_DT_TYPES) self.new_diskparams = self.op.diskparams try: utils.VerifyDictOptions(self.new_diskparams, constants.DISK_DT_DEFAULTS) except errors.OpPrereqError as err: raise errors.OpPrereqError("While verify diskparams options: %s" % err, errors.ECODE_INVAL) else: self.new_diskparams = {} self._CheckIpolicy() def BuildHooksEnv(self): """Build hooks env. """ return { "GROUP_NAME": self.op.group_name, } def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() return ([mn], [mn]) @staticmethod def _ConnectInstanceCommunicationNetwork(cfg, group_uuid, network_name): """Connect a node group to the instance communication network. The group is connected to the instance communication network via the Opcode 'OpNetworkConnect'. @type cfg: L{ganeti.config.ConfigWriter} @param cfg: Ganeti configuration @type group_uuid: string @param group_uuid: UUID of the group to connect @type network_name: string @param network_name: name of the network to connect to @rtype: L{ganeti.cmdlib.ResultWithJobs} or L{None} @return: L{ganeti.cmdlib.ResultWithJobs} if the group needs to be connected, otherwise (the group is already connected) L{None} """ try: cfg.LookupNetwork(network_name) network_exists = True except errors.OpPrereqError: network_exists = False if network_exists: op = ConnectInstanceCommunicationNetworkOp(group_uuid, network_name) return ResultWithJobs([[op]]) else: return None def Exec(self, feedback_fn): """Add the node group to the cluster. """ group_obj = objects.NodeGroup(name=self.op.group_name, members=[], uuid=self.group_uuid, alloc_policy=self.op.alloc_policy, ndparams=self.op.ndparams, diskparams=self.new_diskparams, ipolicy=self.op.ipolicy, hv_state_static=self.new_hv_state, disk_state_static=self.new_disk_state) self.cfg.AddNodeGroup(group_obj, self.proc.GetECId(), check_uuid=False) network_name = self.cfg.GetClusterInfo().instance_communication_network if network_name: return self._ConnectInstanceCommunicationNetwork(self.cfg, self.group_uuid, network_name) class LUGroupAssignNodes(NoHooksLU): """Logical unit for assigning nodes to groups. """ REQ_BGL = False def ExpandNames(self): # These raise errors.OpPrereqError on their own: self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name) (self.op.node_uuids, self.op.nodes) = GetWantedNodes(self, self.op.nodes) # We want to lock all the affected nodes and groups. We have readily # available the list of nodes, and the *destination* group. To gather the # list of "source" groups, we need to fetch node information later on. self.needed_locks = { locking.LEVEL_NODEGROUP: set([self.group_uuid]), locking.LEVEL_NODE: self.op.node_uuids, } def DeclareLocks(self, level): if level == locking.LEVEL_NODEGROUP: assert len(self.needed_locks[locking.LEVEL_NODEGROUP]) == 1 # Try to get all affected nodes' groups without having the group or node # lock yet. Needs verification later in the code flow. groups = self.cfg.GetNodeGroupsFromNodes(self.op.node_uuids) self.needed_locks[locking.LEVEL_NODEGROUP].update(groups) def CheckPrereq(self): """Check prerequisites. """ assert self.needed_locks[locking.LEVEL_NODEGROUP] assert (frozenset(self.owned_locks(locking.LEVEL_NODE)) == frozenset(self.op.node_uuids)) expected_locks = (set([self.group_uuid]) | self.cfg.GetNodeGroupsFromNodes(self.op.node_uuids)) actual_locks = self.owned_locks(locking.LEVEL_NODEGROUP) if actual_locks != expected_locks: raise errors.OpExecError("Nodes changed groups since locks were acquired," " current groups are '%s', used to be '%s'" % (utils.CommaJoin(expected_locks), utils.CommaJoin(actual_locks))) self.node_data = self.cfg.GetAllNodesInfo() self.group = self.cfg.GetNodeGroup(self.group_uuid) instance_data = self.cfg.GetAllInstancesInfo() if self.group is None: raise errors.OpExecError("Could not retrieve group '%s' (UUID: %s)" % (self.op.group_name, self.group_uuid)) (new_splits, previous_splits) = \ self.CheckAssignmentForSplitInstances([(uuid, self.group_uuid) for uuid in self.op.node_uuids], self.node_data, instance_data) if new_splits: fmt_new_splits = utils.CommaJoin(utils.NiceSort( self.cfg.GetInstanceNames(new_splits))) if not self.op.force: raise errors.OpExecError("The following instances get split by this" " change and --force was not given: %s" % fmt_new_splits) else: self.LogWarning("This operation will split the following instances: %s", fmt_new_splits) if previous_splits: self.LogWarning("In addition, these already-split instances continue" " to be split across groups: %s", utils.CommaJoin(utils.NiceSort( self.cfg.GetInstanceNames(previous_splits)))) def Exec(self, feedback_fn): """Assign nodes to a new group. """ mods = [(node_uuid, self.group_uuid) for node_uuid in self.op.node_uuids] self.cfg.AssignGroupNodes(mods) def CheckAssignmentForSplitInstances(self, changes, node_data, instance_data): """Check for split instances after a node assignment. This method considers a series of node assignments as an atomic operation, and returns information about split instances after applying the set of changes. In particular, it returns information about newly split instances, and instances that were already split, and remain so after the change. Only disks whose template is listed in constants.DTS_INT_MIRROR are considered. @type changes: list of (node_uuid, new_group_uuid) pairs. @param changes: list of node assignments to consider. @param node_data: a dict with data for all nodes @param instance_data: a dict with all instances to consider @rtype: a two-tuple @return: a list of instances that were previously okay and result split as a consequence of this change, and a list of instances that were previously split and this change does not fix. """ changed_nodes = dict((uuid, group) for uuid, group in changes if node_data[uuid].group != group) all_split_instances = set() previously_split_instances = set() for inst in instance_data.values(): inst_disks = self.cfg.GetInstanceDisks(inst.uuid) if not utils.AnyDiskOfType(inst_disks, constants.DTS_INT_MIRROR): continue inst_nodes = self.cfg.GetInstanceNodes(inst.uuid) if len(set(node_data[node_uuid].group for node_uuid in inst_nodes)) > 1: previously_split_instances.add(inst.uuid) if len(set(changed_nodes.get(node_uuid, node_data[node_uuid].group) for node_uuid in inst_nodes)) > 1: all_split_instances.add(inst.uuid) return (list(all_split_instances - previously_split_instances), list(previously_split_instances & all_split_instances)) class LUGroupSetParams(LogicalUnit): """Modifies the parameters of a node group. """ HPATH = "group-modify" HTYPE = constants.HTYPE_GROUP REQ_BGL = False def CheckArguments(self): all_changes = [ self.op.ndparams, self.op.diskparams, self.op.alloc_policy, self.op.hv_state, self.op.disk_state, self.op.ipolicy, ] if all_changes.count(None) == len(all_changes): raise errors.OpPrereqError("Please pass at least one modification", errors.ECODE_INVAL) if self.op.diskparams: CheckDiskAccessModeValidity(self.op.diskparams) def ExpandNames(self): # This raises errors.OpPrereqError on its own: self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name) self.needed_locks = { locking.LEVEL_INSTANCE: [], locking.LEVEL_NODEGROUP: [self.group_uuid], } self.share_locks[locking.LEVEL_INSTANCE] = 1 def DeclareLocks(self, level): if level == locking.LEVEL_INSTANCE: assert not self.needed_locks[locking.LEVEL_INSTANCE] # Lock instances optimistically, needs verification once group lock has # been acquired self.needed_locks[locking.LEVEL_INSTANCE] = \ self.cfg.GetInstanceNames( self.cfg.GetNodeGroupInstances(self.group_uuid)) @staticmethod def _UpdateAndVerifyDiskParams(old, new): """Updates and verifies disk parameters. """ new_params = GetUpdatedParams(old, new) utils.ForceDictType(new_params, constants.DISK_DT_TYPES) return new_params def _CheckIpolicy(self, cluster, owned_instance_names): """Sanity checks for the ipolicy. @type cluster: C{objects.Cluster} @param cluster: the cluster's configuration @type owned_instance_names: list of string @param owned_instance_names: list of instances """ if self.op.ipolicy: self.new_ipolicy = GetUpdatedIPolicy(self.group.ipolicy, self.op.ipolicy, group_policy=True) new_ipolicy = cluster.SimpleFillIPolicy(self.new_ipolicy) CheckIpolicyVsDiskTemplates(new_ipolicy, cluster.enabled_disk_templates) instances = \ dict(self.cfg.GetMultiInstanceInfoByName(owned_instance_names)) gmi = ganeti.masterd.instance violations = \ ComputeNewInstanceViolations(gmi.CalculateGroupIPolicy(cluster, self.group), new_ipolicy, list(instances.values()), self.cfg) if violations: self.LogWarning("After the ipolicy change the following instances" " violate them: %s", utils.CommaJoin(violations)) def CheckPrereq(self): """Check prerequisites. """ owned_instance_names = frozenset(self.owned_locks(locking.LEVEL_INSTANCE)) # Check if locked instances are still correct CheckNodeGroupInstances(self.cfg, self.group_uuid, owned_instance_names) self.group = self.cfg.GetNodeGroup(self.group_uuid) cluster = self.cfg.GetClusterInfo() if self.group is None: raise errors.OpExecError("Could not retrieve group '%s' (UUID: %s)" % (self.op.group_name, self.group_uuid)) if self.op.ndparams: new_ndparams = GetUpdatedParams(self.group.ndparams, self.op.ndparams) utils.ForceDictType(new_ndparams, constants.NDS_PARAMETER_TYPES) self.new_ndparams = new_ndparams if self.op.diskparams: diskparams = self.group.diskparams uavdp = self._UpdateAndVerifyDiskParams # For each disktemplate subdict update and verify the values new_diskparams = dict((dt, uavdp(diskparams.get(dt, {}), self.op.diskparams[dt])) for dt in constants.DISK_TEMPLATES if dt in self.op.diskparams) # As we've all subdicts of diskparams ready, lets merge the actual # dict with all updated subdicts self.new_diskparams = objects.FillDict(diskparams, new_diskparams) try: utils.VerifyDictOptions(self.new_diskparams, constants.DISK_DT_DEFAULTS) CheckDiskAccessModeConsistency(self.new_diskparams, self.cfg, group=self.group) except errors.OpPrereqError as err: raise errors.OpPrereqError("While verify diskparams options: %s" % err, errors.ECODE_INVAL) if self.op.hv_state: self.new_hv_state = MergeAndVerifyHvState(self.op.hv_state, self.group.hv_state_static) if self.op.disk_state: self.new_disk_state = \ MergeAndVerifyDiskState(self.op.disk_state, self.group.disk_state_static) self._CheckIpolicy(cluster, owned_instance_names) def BuildHooksEnv(self): """Build hooks env. """ return { "GROUP_NAME": self.op.group_name, "NEW_ALLOC_POLICY": self.op.alloc_policy, } def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() return ([mn], [mn]) def Exec(self, feedback_fn): """Modifies the node group. """ result = [] if self.op.ndparams: self.group.ndparams = self.new_ndparams result.append(("ndparams", str(self.group.ndparams))) if self.op.diskparams: self.group.diskparams = self.new_diskparams result.append(("diskparams", str(self.group.diskparams))) if self.op.alloc_policy: self.group.alloc_policy = self.op.alloc_policy if self.op.hv_state: self.group.hv_state_static = self.new_hv_state if self.op.disk_state: self.group.disk_state_static = self.new_disk_state if self.op.ipolicy: self.group.ipolicy = self.new_ipolicy self.cfg.Update(self.group, feedback_fn) return result class LUGroupRemove(LogicalUnit): HPATH = "group-remove" HTYPE = constants.HTYPE_GROUP REQ_BGL = False def ExpandNames(self): # This will raises errors.OpPrereqError on its own: self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name) self.needed_locks = { locking.LEVEL_NODEGROUP: [self.group_uuid], } def CheckPrereq(self): """Check prerequisites. This checks that the given group name exists as a node group, that is empty (i.e., contains no nodes), and that is not the last group of the cluster. """ # Verify that the group is empty. group_nodes = [node.uuid for node in self.cfg.GetAllNodesInfo().values() if node.group == self.group_uuid] if group_nodes: raise errors.OpPrereqError("Group '%s' not empty, has the following" " nodes: %s" % (self.op.group_name, utils.CommaJoin(utils.NiceSort(group_nodes))), errors.ECODE_STATE) # Verify the cluster would not be left group-less. if len(self.cfg.GetNodeGroupList()) == 1: raise errors.OpPrereqError("Group '%s' is the only group, cannot be" " removed" % self.op.group_name, errors.ECODE_STATE) def BuildHooksEnv(self): """Build hooks env. """ return { "GROUP_NAME": self.op.group_name, } def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() return ([mn], [mn]) def Exec(self, feedback_fn): """Remove the node group. """ try: self.cfg.RemoveNodeGroup(self.group_uuid) except errors.ConfigurationError: raise errors.OpExecError("Group '%s' with UUID %s disappeared" % (self.op.group_name, self.group_uuid)) class LUGroupRename(LogicalUnit): HPATH = "group-rename" HTYPE = constants.HTYPE_GROUP REQ_BGL = False def ExpandNames(self): # This raises errors.OpPrereqError on its own: self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name) self.needed_locks = { locking.LEVEL_NODEGROUP: [self.group_uuid], } def CheckPrereq(self): """Check prerequisites. Ensures requested new name is not yet used. """ try: new_name_uuid = self.cfg.LookupNodeGroup(self.op.new_name) except errors.OpPrereqError: pass else: raise errors.OpPrereqError("Desired new name '%s' clashes with existing" " node group (UUID: %s)" % (self.op.new_name, new_name_uuid), errors.ECODE_EXISTS) def BuildHooksEnv(self): """Build hooks env. """ return { "OLD_NAME": self.op.group_name, "NEW_NAME": self.op.new_name, } def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() all_nodes = self.cfg.GetAllNodesInfo() all_nodes.pop(mn, None) run_nodes = [mn] run_nodes.extend(node.uuid for node in all_nodes.values() if node.group == self.group_uuid) return (run_nodes, run_nodes) def Exec(self, feedback_fn): """Rename the node group. """ group = self.cfg.GetNodeGroup(self.group_uuid) if group is None: raise errors.OpExecError("Could not retrieve group '%s' (UUID: %s)" % (self.op.group_name, self.group_uuid)) group.name = self.op.new_name self.cfg.Update(group, feedback_fn) return self.op.new_name class LUGroupEvacuate(LogicalUnit): HPATH = "group-evacuate" HTYPE = constants.HTYPE_GROUP REQ_BGL = False def ExpandNames(self): # This raises errors.OpPrereqError on its own: self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name) if self.op.target_groups: self.req_target_uuids = [self.cfg.LookupNodeGroup(g) for g in self.op.target_groups] else: self.req_target_uuids = [] if self.group_uuid in self.req_target_uuids: raise errors.OpPrereqError("Group to be evacuated (%s) can not be used" " as a target group (targets are %s)" % (self.group_uuid, utils.CommaJoin(self.req_target_uuids)), errors.ECODE_INVAL) self.op.iallocator = GetDefaultIAllocator(self.cfg, self.op.iallocator) self.share_locks = ShareAll() self.needed_locks = { locking.LEVEL_INSTANCE: [], locking.LEVEL_NODEGROUP: [], locking.LEVEL_NODE: [], } def DeclareLocks(self, level): if level == locking.LEVEL_INSTANCE: assert not self.needed_locks[locking.LEVEL_INSTANCE] # Lock instances optimistically, needs verification once node and group # locks have been acquired self.needed_locks[locking.LEVEL_INSTANCE] = \ self.cfg.GetInstanceNames( self.cfg.GetNodeGroupInstances(self.group_uuid)) elif level == locking.LEVEL_NODEGROUP: assert not self.needed_locks[locking.LEVEL_NODEGROUP] if self.req_target_uuids: lock_groups = set([self.group_uuid] + self.req_target_uuids) # Lock all groups used by instances optimistically; this requires going # via the node before it's locked, requiring verification later on lock_groups.update(group_uuid for instance_name in self.owned_locks(locking.LEVEL_INSTANCE) for group_uuid in self.cfg.GetInstanceNodeGroups( self.cfg.GetInstanceInfoByName(instance_name) .uuid)) else: # No target groups, need to lock all of them lock_groups = locking.ALL_SET self.needed_locks[locking.LEVEL_NODEGROUP] = lock_groups elif level == locking.LEVEL_NODE: # This will only lock the nodes in the group to be evacuated which # contain actual instances self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_APPEND self._LockInstancesNodes() # Lock all nodes in group to be evacuated and target groups owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP)) assert self.group_uuid in owned_groups member_node_uuids = [node_uuid for group in owned_groups for node_uuid in self.cfg.GetNodeGroup(group).members] self.needed_locks[locking.LEVEL_NODE].extend(member_node_uuids) def CheckPrereq(self): owned_instance_names = frozenset(self.owned_locks(locking.LEVEL_INSTANCE)) owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP)) owned_node_uuids = frozenset(self.owned_locks(locking.LEVEL_NODE)) assert owned_groups.issuperset(self.req_target_uuids) assert self.group_uuid in owned_groups # Check if locked instances are still correct CheckNodeGroupInstances(self.cfg, self.group_uuid, owned_instance_names) # Get instance information self.instances = \ dict(self.cfg.GetMultiInstanceInfoByName(owned_instance_names)) # Check if node groups for locked instances are still correct CheckInstancesNodeGroups(self.cfg, self.instances, owned_groups, owned_node_uuids, self.group_uuid) if self.req_target_uuids: # User requested specific target groups self.target_uuids = self.req_target_uuids else: # All groups except the one to be evacuated are potential targets self.target_uuids = [group_uuid for group_uuid in owned_groups if group_uuid != self.group_uuid] if not self.target_uuids: raise errors.OpPrereqError("There are no possible target groups", errors.ECODE_INVAL) def BuildHooksEnv(self): """Build hooks env. """ return { "GROUP_NAME": self.op.group_name, "TARGET_GROUPS": " ".join(self.target_uuids), } def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() assert self.group_uuid in self.owned_locks(locking.LEVEL_NODEGROUP) run_nodes = [mn] + self.cfg.GetNodeGroup(self.group_uuid).members return (run_nodes, run_nodes) @staticmethod def _MigrateToFailover(op): """Return an equivalent failover opcode for a migrate one. If the argument is not a failover opcode, return it unchanged. """ if not isinstance(op, opcodes.OpInstanceMigrate): return op else: return opcodes.OpInstanceFailover( instance_name=op.instance_name, instance_uuid=getattr(op, "instance_uuid", None), target_node=getattr(op, "target_node", None), target_node_uuid=getattr(op, "target_node_uuid", None), ignore_ipolicy=op.ignore_ipolicy, cleanup=op.cleanup) def Exec(self, feedback_fn): inst_names = list(self.owned_locks(locking.LEVEL_INSTANCE)) assert self.group_uuid not in self.target_uuids req = iallocator.IAReqGroupChange(instances=inst_names, target_groups=self.target_uuids) ial = iallocator.IAllocator(self.cfg, self.rpc, req) ial.Run(self.op.iallocator) if not ial.success: raise errors.OpPrereqError("Can't compute group evacuation using" " iallocator '%s': %s" % (self.op.iallocator, ial.info), errors.ECODE_NORES) jobs = LoadNodeEvacResult(self, ial.result, self.op.early_release, False) self.LogInfo("Iallocator returned %s job(s) for evacuating node group %s", len(jobs), self.op.group_name) if self.op.force_failover: self.LogInfo("Will insist on failovers") jobs = [[self._MigrateToFailover(op) for op in job] for job in jobs] if self.op.sequential: self.LogInfo("Jobs will be submitted to run sequentially") for job in jobs[1:]: for op in job: op.depends = [(-1, ["error", "success"])] return ResultWithJobs(jobs) class LUGroupVerifyDisks(NoHooksLU): """Verifies the status of all disks in a node group. """ REQ_BGL = False def ExpandNames(self): # Raises errors.OpPrereqError on its own if group can't be found self.group_uuid = self.cfg.LookupNodeGroup(self.op.group_name) self.share_locks = ShareAll() self.needed_locks = { locking.LEVEL_INSTANCE: [], locking.LEVEL_NODEGROUP: [], locking.LEVEL_NODE: [], } self.dont_collate_locks[locking.LEVEL_NODEGROUP] = True self.dont_collate_locks[locking.LEVEL_NODE] = True # If run in strict mode, require locks for all nodes in the node group # so we can verify all the disks. In non-strict mode, just verify the # nodes that are available for locking. if not self.op.is_strict: self.opportunistic_locks[locking.LEVEL_NODE] = True self.opportunistic_locks[locking.LEVEL_INSTANCE] = True def DeclareLocks(self, level): if level == locking.LEVEL_INSTANCE: assert not self.needed_locks[locking.LEVEL_INSTANCE] # Lock instances optimistically, needs verification once node and group # locks have been acquired self.needed_locks[locking.LEVEL_INSTANCE] = \ self.cfg.GetInstanceNames( self.cfg.GetNodeGroupInstances(self.group_uuid)) elif level == locking.LEVEL_NODEGROUP: assert not self.needed_locks[locking.LEVEL_NODEGROUP] self.needed_locks[locking.LEVEL_NODEGROUP] = \ set([self.group_uuid] + # Lock all groups used by instances optimistically; this requires # going via the node before it's locked, requiring verification # later on [group_uuid for instance_name in self.owned_locks(locking.LEVEL_INSTANCE) for group_uuid in self.cfg.GetInstanceNodeGroups( self.cfg.GetInstanceInfoByName(instance_name).uuid)]) elif level == locking.LEVEL_NODE: # This will only lock the nodes in the group to be verified which contain # actual instances self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_APPEND self._LockInstancesNodes() # Lock all nodes in group to be verified assert self.group_uuid in self.owned_locks(locking.LEVEL_NODEGROUP) member_node_uuids = self.cfg.GetNodeGroup(self.group_uuid).members self.needed_locks[locking.LEVEL_NODE].extend(member_node_uuids) def CheckPrereq(self): owned_inst_names = frozenset(self.owned_locks(locking.LEVEL_INSTANCE)) owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP)) owned_node_uuids = frozenset(self.owned_locks(locking.LEVEL_NODE)) assert self.group_uuid in owned_groups if self.op.is_strict: # Check if locked instances are still correct CheckNodeGroupInstances(self.cfg, self.group_uuid, owned_inst_names) # Get instance information self.instances = dict(self.cfg.GetMultiInstanceInfoByName(owned_inst_names)) # Check if node groups for locked instances are still correct CheckInstancesNodeGroups(self.cfg, self.instances, owned_groups, owned_node_uuids, self.group_uuid) def _VerifyInstanceLvs(self, node_errors, offline_disk_instance_names, missing_disks): node_lv_to_inst = MapInstanceLvsToNodes( self.cfg, [inst for inst in self.instances.values() if inst.disks_active]) if node_lv_to_inst: node_uuids = utils.NiceSort(set(self.owned_locks(locking.LEVEL_NODE)) & set(self.cfg.GetVmCapableNodeList())) node_lvs = self.rpc.call_lv_list(node_uuids, []) for (node_uuid, node_res) in node_lvs.items(): if node_res.offline: continue msg = node_res.fail_msg if msg: logging.warning("Error enumerating LVs on node %s: %s", self.cfg.GetNodeName(node_uuid), msg) node_errors[node_uuid] = msg continue for lv_name, (_, _, lv_online) in node_res.payload.items(): inst = node_lv_to_inst.pop((node_uuid, lv_name), None) if not lv_online and inst is not None: offline_disk_instance_names.add(inst.name) # any leftover items in nv_dict are missing LVs, let's arrange the data # better for key, inst in node_lv_to_inst.items(): missing_disks.setdefault(inst.name, []).append(list(key)) def _VerifyDrbdStates(self, node_errors, offline_disk_instance_names): node_to_inst = {} owned_node_uuids = set(self.owned_locks(locking.LEVEL_NODE)) for inst in self.instances.values(): disks = self.cfg.GetInstanceDisks(inst.uuid) if not (inst.disks_active and utils.AnyDiskOfType(disks, [constants.DT_DRBD8])): continue secondary_nodes = self.cfg.GetInstanceSecondaryNodes(inst.uuid) for node_uuid in itertools.chain([inst.primary_node], secondary_nodes): if not node_uuid in owned_node_uuids: logging.info("Node %s is not locked, skipping check.", node_uuid) continue node_to_inst.setdefault(node_uuid, []).append(inst) for (node_uuid, insts) in node_to_inst.items(): node_disks = [(self.cfg.GetInstanceDisks(inst.uuid), inst) for inst in insts] node_res = self.rpc.call_drbd_needs_activation(node_uuid, node_disks) msg = node_res.fail_msg if msg: logging.warning("Error getting DRBD status on node %s: %s", self.cfg.GetNodeName(node_uuid), msg) node_errors[node_uuid] = msg continue faulty_disk_uuids = set(node_res.payload) for inst in self.instances.values(): disks = self.cfg.GetInstanceDisks(inst.uuid) inst_disk_uuids = set([disk.uuid for disk in disks]) if inst_disk_uuids.intersection(faulty_disk_uuids): offline_disk_instance_names.add(inst.name) def Exec(self, feedback_fn): """Verify integrity of cluster disks. @rtype: tuple of three items @return: a tuple of (dict of node-to-node_error, list of instances which need activate-disks, dict of instance: (node, volume) for missing volumes """ node_errors = {} offline_disk_instance_names = set() missing_disks = {} self._VerifyInstanceLvs(node_errors, offline_disk_instance_names, missing_disks) self._VerifyDrbdStates(node_errors, offline_disk_instance_names) return (node_errors, list(offline_disk_instance_names), missing_disks) ganeti-3.1.0~rc2/lib/cmdlib/instance.py000064400000000000000000000760231476477700300177720ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with instances.""" import logging import os from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import locking from ganeti.masterd import iallocator from ganeti import masterd from ganeti import netutils from ganeti import objects from ganeti import utils from ganeti.cmdlib.base import NoHooksLU, LogicalUnit, ResultWithJobs from ganeti.cmdlib.common import \ INSTANCE_NOT_RUNNING, CheckNodeOnline, \ ShareAll, GetDefaultIAllocator, CheckInstanceNodeGroups, \ LoadNodeEvacResult, \ ExpandInstanceUuidAndName, \ CheckInstanceState, ExpandNodeUuidAndName, \ CheckDiskTemplateEnabled from ganeti.cmdlib.instance_storage import CreateDisks, \ ComputeDisks, \ StartInstanceDisks, ShutdownInstanceDisks, \ AssembleInstanceDisks from ganeti.cmdlib.instance_utils import \ BuildInstanceHookEnvByObject,\ CheckNodeNotDrained, RemoveInstance, CopyLockList, \ CheckNodeVmCapable, CheckTargetNodeIPolicy, \ GetInstanceInfoText, RemoveDisks, CheckNodeFreeMemory, \ CheckInstanceBridgesExist, \ CheckInstanceExistence, \ CheckHostnameSane, CheckOpportunisticLocking, ComputeFullBeParams, \ ComputeNics, CreateInstanceAllocRequest import ganeti.masterd.instance class LUInstanceRename(LogicalUnit): """Rename an instance. """ HPATH = "instance-rename" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def CheckArguments(self): """Check arguments. """ if self.op.ip_check and not self.op.name_check: # TODO: make the ip check more flexible and not depend on the name check raise errors.OpPrereqError("IP address check requires a name check", errors.ECODE_INVAL) self._new_name_resolved = False def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ env = BuildInstanceHookEnvByObject(self, self.instance) env["INSTANCE_NEW_NAME"] = self.op.new_name return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] + \ list(self.cfg.GetInstanceNodes(self.instance.uuid)) return (nl, nl) def _PerformChecksAndResolveNewName(self): """Checks and resolves the new name, storing the FQDN, if permitted. """ if self._new_name_resolved or not self.op.name_check: return hostname = CheckHostnameSane(self, self.op.new_name) self.op.new_name = hostname.name if (self.op.ip_check and netutils.TcpPing(hostname.ip, constants.DEFAULT_NODED_PORT)): raise errors.OpPrereqError("IP %s of instance %s already in use" % (hostname.ip, self.op.new_name), errors.ECODE_NOTUNIQUE) self._new_name_resolved = True def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster and is not running. """ (self.op.instance_uuid, self.op.instance_name) = \ ExpandInstanceUuidAndName(self.cfg, self.op.instance_uuid, self.op.instance_name) instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert instance is not None # It should actually not happen that an instance is running with a disabled # disk template, but in case it does, the renaming of file-based instances # will fail horribly. Thus, we test it before. for disk in self.cfg.GetInstanceDisks(instance.uuid): if (disk.dev_type in constants.DTS_FILEBASED and self.op.new_name != instance.name): # TODO: when disks are separate objects, this should check for disk # types, not disk templates. CheckDiskTemplateEnabled(self.cfg.GetClusterInfo(), disk.dev_type) CheckNodeOnline(self, instance.primary_node) CheckInstanceState(self, instance, INSTANCE_NOT_RUNNING, msg="cannot rename") self.instance = instance self._PerformChecksAndResolveNewName() if self.op.new_name != instance.name: CheckInstanceExistence(self, self.op.new_name) def ExpandNames(self): self._ExpandAndLockInstance(allow_forthcoming=True) # Note that this call might not resolve anything if name checks have been # disabled in the opcode. In this case, we might have a renaming collision # if a shortened name and a full name are used simultaneously, as we will # have two different locks. However, at that point the user has taken away # the tools necessary to detect this issue. self._PerformChecksAndResolveNewName() # Used to prevent instance namespace collisions. if self.op.new_name != self.op.instance_name: CheckInstanceExistence(self, self.op.new_name) self.add_locks[locking.LEVEL_INSTANCE] = self.op.new_name def Exec(self, feedback_fn): """Rename the instance. """ old_name = self.instance.name rename_file_storage = False disks = self.cfg.GetInstanceDisks(self.instance.uuid) renamed_storage = [d for d in disks if (d.dev_type in constants.DTS_FILEBASED and d.dev_type != constants.DT_GLUSTER)] if renamed_storage and self.op.new_name != self.instance.name: disks = self.cfg.GetInstanceDisks(self.instance.uuid) old_file_storage_dir = os.path.dirname(disks[0].logical_id[1]) rename_file_storage = True self.cfg.RenameInstance(self.instance.uuid, self.op.new_name) # Assert that we have both the locks needed assert old_name in self.owned_locks(locking.LEVEL_INSTANCE) assert self.op.new_name in self.owned_locks(locking.LEVEL_INSTANCE) # re-read the instance from the configuration after rename renamed_inst = self.cfg.GetInstanceInfo(self.instance.uuid) disks = self.cfg.GetInstanceDisks(renamed_inst.uuid) if self.instance.forthcoming: return renamed_inst.name if rename_file_storage: new_file_storage_dir = os.path.dirname(disks[0].logical_id[1]) result = self.rpc.call_file_storage_dir_rename(renamed_inst.primary_node, old_file_storage_dir, new_file_storage_dir) result.Raise("Could not rename on node %s directory '%s' to '%s'" " (but the instance has been renamed in Ganeti)" % (self.cfg.GetNodeName(renamed_inst.primary_node), old_file_storage_dir, new_file_storage_dir)) StartInstanceDisks(self, renamed_inst, None) renamed_inst = self.cfg.GetInstanceInfo(renamed_inst.uuid) # update info on disks info = GetInstanceInfoText(renamed_inst) for (idx, disk) in enumerate(disks): for node_uuid in self.cfg.GetInstanceNodes(renamed_inst.uuid): result = self.rpc.call_blockdev_setinfo(node_uuid, (disk, renamed_inst), info) result.Warn("Error setting info on node %s for disk %s" % (self.cfg.GetNodeName(node_uuid), idx), self.LogWarning) try: result = self.rpc.call_instance_run_rename(renamed_inst.primary_node, renamed_inst, old_name, self.op.debug_level) result.Warn("Could not run OS rename script for instance %s on node %s" " (but the instance has been renamed in Ganeti)" % (renamed_inst.name, self.cfg.GetNodeName(renamed_inst.primary_node)), self.LogWarning) finally: ShutdownInstanceDisks(self, renamed_inst) return renamed_inst.name class LUInstanceRemove(LogicalUnit): """Remove an instance. """ HPATH = "instance-remove" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def ExpandNames(self): self._ExpandAndLockInstance(allow_forthcoming=True) self.needed_locks[locking.LEVEL_NODE] = [] self.needed_locks[locking.LEVEL_NODE_RES] = [] self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE self.dont_collate_locks[locking.LEVEL_NODE] = True self.dont_collate_locks[locking.LEVEL_NODE_RES] = True def DeclareLocks(self, level): if level == locking.LEVEL_NODE: self._LockInstancesNodes() elif level == locking.LEVEL_NODE_RES: # Copy node locks self.needed_locks[locking.LEVEL_NODE_RES] = \ CopyLockList(self.needed_locks[locking.LEVEL_NODE]) def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ env = BuildInstanceHookEnvByObject(self, self.instance, secondary_nodes=self.secondary_nodes, disks=self.inst_disks) env["SHUTDOWN_TIMEOUT"] = self.op.shutdown_timeout return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] nl_post = list(self.cfg.GetInstanceNodes(self.instance.uuid)) + nl return (nl, nl_post) def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name self.secondary_nodes = \ self.cfg.GetInstanceSecondaryNodes(self.instance.uuid) self.inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) def Exec(self, feedback_fn): """Remove the instance. """ assert (self.owned_locks(locking.LEVEL_NODE) == self.owned_locks(locking.LEVEL_NODE_RES)) assert not (set(self.cfg.GetInstanceNodes(self.instance.uuid)) - self.owned_locks(locking.LEVEL_NODE)), \ "Not owning correct locks" if not self.instance.forthcoming: logging.info("Shutting down instance %s on node %s", self.instance.name, self.cfg.GetNodeName(self.instance.primary_node)) result = self.rpc.call_instance_shutdown(self.instance.primary_node, self.instance, self.op.shutdown_timeout, self.op.reason) if self.op.ignore_failures: result.Warn("Warning: can't shutdown instance", feedback_fn) else: result.Raise("Could not shutdown instance %s on node %s" % (self.instance.name, self.cfg.GetNodeName(self.instance.primary_node))) else: logging.info("Instance %s on node %s is forthcoming; not shutting down", self.instance.name, self.cfg.GetNodeName(self.instance.primary_node)) RemoveInstance(self, feedback_fn, self.instance, self.op.ignore_failures) class LUInstanceMove(LogicalUnit): """Move an instance by data-copying. This LU is only used if the instance needs to be moved by copying the data from one node in the cluster to another. The instance is shut down and the data is copied to the new node and the configuration change is propagated, then the instance is started again. See also: L{LUInstanceFailover} for moving an instance on shared storage (no copying required). L{LUInstanceMigrate} for the live migration of an instance (no shutdown required). """ HPATH = "instance-move" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def ExpandNames(self): self._ExpandAndLockInstance() (self.op.target_node_uuid, self.op.target_node) = \ ExpandNodeUuidAndName(self.cfg, self.op.target_node_uuid, self.op.target_node) self.needed_locks[locking.LEVEL_NODE] = [self.op.target_node_uuid] self.needed_locks[locking.LEVEL_NODE_RES] = [] self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_APPEND def DeclareLocks(self, level): if level == locking.LEVEL_NODE: self._LockInstancesNodes(primary_only=True) elif level == locking.LEVEL_NODE_RES: # Copy node locks self.needed_locks[locking.LEVEL_NODE_RES] = \ CopyLockList(self.needed_locks[locking.LEVEL_NODE]) def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and target nodes of the instance. """ env = { "TARGET_NODE": self.op.target_node, "SHUTDOWN_TIMEOUT": self.op.shutdown_timeout, } env.update(BuildInstanceHookEnvByObject(self, self.instance)) return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [ self.cfg.GetMasterNode(), self.instance.primary_node, self.op.target_node_uuid, ] return (nl, nl) def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name disks = self.cfg.GetInstanceDisks(self.instance.uuid) for idx, dsk in enumerate(disks): if dsk.dev_type not in constants.DTS_COPYABLE: raise errors.OpPrereqError("Instance disk %d has disk type %s and is" " not suitable for copying" % (idx, dsk.dev_type), errors.ECODE_STATE) target_node = self.cfg.GetNodeInfo(self.op.target_node_uuid) assert target_node is not None, \ "Cannot retrieve locked node %s" % self.op.target_node self.target_node_uuid = target_node.uuid if target_node.uuid == self.instance.primary_node: raise errors.OpPrereqError("Instance %s is already on the node %s" % (self.instance.name, target_node.name), errors.ECODE_STATE) cluster = self.cfg.GetClusterInfo() bep = cluster.FillBE(self.instance) CheckNodeOnline(self, target_node.uuid) CheckNodeNotDrained(self, target_node.uuid) CheckNodeVmCapable(self, target_node.uuid) group_info = self.cfg.GetNodeGroup(target_node.group) ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(cluster, group_info) CheckTargetNodeIPolicy(self, ipolicy, self.instance, target_node, self.cfg, ignore=self.op.ignore_ipolicy) if self.instance.admin_state == constants.ADMINST_UP: # check memory requirements on the target node CheckNodeFreeMemory( self, target_node.uuid, "failing over instance %s" % self.instance.name, bep[constants.BE_MAXMEM], self.instance.hypervisor, cluster.hvparams[self.instance.hypervisor]) else: self.LogInfo("Not checking memory on the secondary node as" " instance will not be started") # check bridge existance CheckInstanceBridgesExist(self, self.instance, node_uuid=target_node.uuid) def Exec(self, feedback_fn): """Move an instance. The move is done by shutting it down on its present node, copying the data over (slow) and starting it on the new node. """ source_node = self.cfg.GetNodeInfo(self.instance.primary_node) target_node = self.cfg.GetNodeInfo(self.target_node_uuid) self.LogInfo("Shutting down instance %s on source node %s", self.instance.name, source_node.name) assert (self.owned_locks(locking.LEVEL_NODE) == self.owned_locks(locking.LEVEL_NODE_RES)) result = self.rpc.call_instance_shutdown(source_node.uuid, self.instance, self.op.shutdown_timeout, self.op.reason) if self.op.ignore_consistency: result.Warn("Could not shutdown instance %s on node %s. Proceeding" " anyway. Please make sure node %s is down. Error details" % (self.instance.name, source_node.name, source_node.name), self.LogWarning) else: result.Raise("Could not shutdown instance %s on node %s" % (self.instance.name, source_node.name)) # create the target disks try: CreateDisks(self, self.instance, target_node_uuid=target_node.uuid) except errors.OpExecError: self.LogWarning("Device creation failed") for disk_uuid in self.instance.disks: self.cfg.ReleaseDRBDMinors(disk_uuid) raise errs = [] transfers = [] # activate, get path, create transfer jobs disks = self.cfg.GetInstanceDisks(self.instance.uuid) for idx, disk in enumerate(disks): # FIXME: pass debug option from opcode to backend dt = masterd.instance.DiskTransfer("disk/%s" % idx, constants.IEIO_RAW_DISK, (disk, self.instance), constants.IEIO_RAW_DISK, (disk, self.instance), None) transfers.append(dt) self.cfg.Update(disk, feedback_fn) import_result = \ masterd.instance.TransferInstanceData(self, feedback_fn, source_node.uuid, target_node.uuid, target_node.secondary_ip, self.op.compress, self.instance, transfers) if not compat.all(import_result): errs.append("Failed to transfer instance data") if errs: self.LogWarning("Some disks failed to copy, aborting") try: RemoveDisks(self, self.instance, target_node_uuid=target_node.uuid) finally: for disk_uuid in self.instance.disks: self.cfg.ReleaseDRBDMinors(disk_uuid) raise errors.OpExecError("Errors during disk copy: %s" % (",".join(errs),)) self.instance.primary_node = target_node.uuid self.cfg.Update(self.instance, feedback_fn) for disk in disks: self.cfg.SetDiskNodes(disk.uuid, [target_node.uuid]) self.LogInfo("Removing the disks on the original node") RemoveDisks(self, self.instance, target_node_uuid=source_node.uuid) # Only start the instance if it's marked as up if self.instance.admin_state == constants.ADMINST_UP: self.LogInfo("Starting instance %s on node %s", self.instance.name, target_node.name) disks_ok, _, _ = AssembleInstanceDisks(self, self.instance, ignore_secondaries=True) if not disks_ok: ShutdownInstanceDisks(self, self.instance) raise errors.OpExecError("Can't activate the instance's disks") result = self.rpc.call_instance_start(target_node.uuid, (self.instance, None, None), False, self.op.reason) msg = result.fail_msg if msg: ShutdownInstanceDisks(self, self.instance) raise errors.OpExecError("Could not start instance %s on node %s: %s" % (self.instance.name, target_node.name, msg)) class LUInstanceMultiAlloc(NoHooksLU): """Allocates multiple instances at the same time. """ REQ_BGL = False def CheckArguments(self): """Check arguments. """ nodes = [] for inst in self.op.instances: if inst.iallocator is not None: raise errors.OpPrereqError("iallocator are not allowed to be set on" " instance objects", errors.ECODE_INVAL) nodes.append(bool(inst.pnode)) if inst.disk_template in constants.DTS_INT_MIRROR: nodes.append(bool(inst.snode)) has_nodes = compat.any(nodes) if compat.all(nodes) ^ has_nodes: raise errors.OpPrereqError("There are instance objects providing" " pnode/snode while others do not", errors.ECODE_INVAL) if not has_nodes and self.op.iallocator is None: default_iallocator = self.cfg.GetDefaultIAllocator() if default_iallocator: self.op.iallocator = default_iallocator else: raise errors.OpPrereqError("No iallocator or nodes on the instances" " given and no cluster-wide default" " iallocator found; please specify either" " an iallocator or nodes on the instances" " or set a cluster-wide default iallocator", errors.ECODE_INVAL) CheckOpportunisticLocking(self.op) dups = utils.FindDuplicates([op.instance_name for op in self.op.instances]) if dups: raise errors.OpPrereqError("There are duplicate instance names: %s" % utils.CommaJoin(dups), errors.ECODE_INVAL) def ExpandNames(self): """Calculate the locks. """ self.share_locks = ShareAll() self.needed_locks = {} if self.op.iallocator: self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET self.needed_locks[locking.LEVEL_NODE_RES] = locking.ALL_SET if self.op.opportunistic_locking: self.opportunistic_locks[locking.LEVEL_NODE] = True self.opportunistic_locks[locking.LEVEL_NODE_RES] = True else: nodeslist = [] for inst in self.op.instances: (inst.pnode_uuid, inst.pnode) = \ ExpandNodeUuidAndName(self.cfg, inst.pnode_uuid, inst.pnode) nodeslist.append(inst.pnode_uuid) if inst.snode is not None: (inst.snode_uuid, inst.snode) = \ ExpandNodeUuidAndName(self.cfg, inst.snode_uuid, inst.snode) nodeslist.append(inst.snode_uuid) self.needed_locks[locking.LEVEL_NODE] = nodeslist # Lock resources of instance's primary and secondary nodes (copy to # prevent accidential modification) self.needed_locks[locking.LEVEL_NODE_RES] = list(nodeslist) def CheckPrereq(self): """Check prerequisite. """ if self.op.iallocator: cluster = self.cfg.GetClusterInfo() default_vg = self.cfg.GetVGName() ec_id = self.proc.GetECId() if self.op.opportunistic_locking: # Only consider nodes for which a lock is held node_whitelist = self.cfg.GetNodeNames( set(self.owned_locks(locking.LEVEL_NODE)) & set(self.owned_locks(locking.LEVEL_NODE_RES))) else: node_whitelist = None insts = [CreateInstanceAllocRequest(op, ComputeDisks(op.disks, op.disk_template, default_vg), ComputeNics(op, cluster, None, self.cfg, ec_id), ComputeFullBeParams(op, cluster), node_whitelist) for op in self.op.instances] req = iallocator.IAReqMultiInstanceAlloc(instances=insts) ial = iallocator.IAllocator(self.cfg, self.rpc, req) ial.Run(self.op.iallocator) if not ial.success: raise errors.OpPrereqError("Can't compute nodes using" " iallocator '%s': %s" % (self.op.iallocator, ial.info), errors.ECODE_NORES) self.ia_result = ial.result if self.op.dry_run: self.dry_run_result = objects.FillDict(self._ConstructPartialResult(), { constants.JOB_IDS_KEY: [], }) def _ConstructPartialResult(self): """Contructs the partial result. """ if self.op.iallocator: (allocatable, failed_insts) = self.ia_result # pylint: disable=W0633 allocatable_insts = [i[0] for i in allocatable] else: allocatable_insts = [op.instance_name for op in self.op.instances] failed_insts = [] return { constants.ALLOCATABLE_KEY: allocatable_insts, constants.FAILED_KEY: failed_insts, } def Exec(self, feedback_fn): """Executes the opcode. """ jobs = [] if self.op.iallocator: op2inst = dict((op.instance_name, op) for op in self.op.instances) (allocatable, failed) = self.ia_result # pylint: disable=W0633 for (name, node_names) in allocatable: op = op2inst.pop(name) (op.pnode_uuid, op.pnode) = \ ExpandNodeUuidAndName(self.cfg, None, node_names[0]) if len(node_names) > 1: (op.snode_uuid, op.snode) = \ ExpandNodeUuidAndName(self.cfg, None, node_names[1]) jobs.append([op]) missing = set(op2inst.keys()) - set(failed) assert not missing, \ "Iallocator did return incomplete result: %s" % \ utils.CommaJoin(missing) else: jobs.extend([op] for op in self.op.instances) return ResultWithJobs(jobs, **self._ConstructPartialResult()) class LUInstanceChangeGroup(LogicalUnit): HPATH = "instance-change-group" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def ExpandNames(self): self.share_locks = ShareAll() self.needed_locks = { locking.LEVEL_NODEGROUP: [], locking.LEVEL_NODE: [], } self._ExpandAndLockInstance() if self.op.target_groups: self.req_target_uuids = [self.cfg.LookupNodeGroup(g) for g in self.op.target_groups] else: self.req_target_uuids = None self.op.iallocator = GetDefaultIAllocator(self.cfg, self.op.iallocator) def DeclareLocks(self, level): if level == locking.LEVEL_NODEGROUP: assert not self.needed_locks[locking.LEVEL_NODEGROUP] if self.req_target_uuids: lock_groups = set(self.req_target_uuids) # Lock all groups used by instance optimistically; this requires going # via the node before it's locked, requiring verification later on instance_groups = self.cfg.GetInstanceNodeGroups(self.op.instance_uuid) lock_groups.update(instance_groups) else: # No target groups, need to lock all of them lock_groups = locking.ALL_SET self.needed_locks[locking.LEVEL_NODEGROUP] = lock_groups elif level == locking.LEVEL_NODE: if self.req_target_uuids: # Lock all nodes used by instances self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_APPEND self._LockInstancesNodes() # Lock all nodes in all potential target groups lock_groups = (frozenset(self.owned_locks(locking.LEVEL_NODEGROUP)) | self.cfg.GetInstanceNodeGroups(self.op.instance_uuid)) member_nodes = [node_uuid for group in lock_groups for node_uuid in self.cfg.GetNodeGroup(group).members] self.needed_locks[locking.LEVEL_NODE].extend(member_nodes) else: # Lock all nodes as all groups are potential targets self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET def CheckPrereq(self): owned_instance_names = frozenset(self.owned_locks(locking.LEVEL_INSTANCE)) owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP)) owned_nodes = frozenset(self.owned_locks(locking.LEVEL_NODE)) assert (self.req_target_uuids is None or owned_groups.issuperset(self.req_target_uuids)) assert owned_instance_names == set([self.op.instance_name]) # Get instance information self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) # Check if node groups for locked instance are still correct instance_all_nodes = self.cfg.GetInstanceNodes(self.instance.uuid) assert owned_nodes.issuperset(instance_all_nodes), \ ("Instance %s's nodes changed while we kept the lock" % self.op.instance_name) inst_groups = CheckInstanceNodeGroups(self.cfg, self.op.instance_uuid, owned_groups) if self.req_target_uuids: # User requested specific target groups self.target_uuids = frozenset(self.req_target_uuids) else: # All groups except those used by the instance are potential targets self.target_uuids = owned_groups - inst_groups conflicting_groups = self.target_uuids & inst_groups if conflicting_groups: raise errors.OpPrereqError("Can't use group(s) '%s' as targets, they are" " used by the instance '%s'" % (utils.CommaJoin(conflicting_groups), self.op.instance_name), errors.ECODE_INVAL) if not self.target_uuids: raise errors.OpPrereqError("There are no possible target groups", errors.ECODE_INVAL) def BuildHooksEnv(self): """Build hooks env. """ assert self.target_uuids env = { "TARGET_GROUPS": " ".join(self.target_uuids), } env.update(BuildInstanceHookEnvByObject(self, self.instance)) return env def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() return ([mn], [mn]) def Exec(self, feedback_fn): instances = list(self.owned_locks(locking.LEVEL_INSTANCE)) assert instances == [self.op.instance_name], "Instance not locked" req = iallocator.IAReqGroupChange(instances=instances, target_groups=list(self.target_uuids)) ial = iallocator.IAllocator(self.cfg, self.rpc, req) ial.Run(self.op.iallocator) if not ial.success: raise errors.OpPrereqError("Can't compute solution for changing group of" " instance '%s' using iallocator '%s': %s" % (self.op.instance_name, self.op.iallocator, ial.info), errors.ECODE_NORES) jobs = LoadNodeEvacResult(self, ial.result, self.op.early_release, False) self.LogInfo("Iallocator returned %s job(s) for changing group of" " instance '%s'", len(jobs), self.op.instance_name) return ResultWithJobs(jobs) ganeti-3.1.0~rc2/lib/cmdlib/instance_create.py000064400000000000000000002054471476477700300213210ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical unit for creating a single instance.""" import logging import os import OpenSSL from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import hypervisor from ganeti import locking from ganeti.masterd import iallocator from ganeti import masterd from ganeti import netutils from ganeti import objects from ganeti import pathutils from ganeti import utils from ganeti import serializer from ganeti.cmdlib.base import LogicalUnit from ganeti.cmdlib.common import \ CheckNodeOnline, \ CheckParamsNotGlobal, \ IsExclusiveStorageEnabledNode, CheckHVParams, CheckOSParams, \ ExpandNodeUuidAndName, \ IsValidDiskAccessModeCombination, \ CheckDiskTemplateEnabled, CheckIAllocatorOrNode, CheckOSImage from ganeti.cmdlib.instance_helpervm import RunWithHelperVM from ganeti.cmdlib.instance_storage import CalculateFileStorageDir, \ CheckNodesFreeDiskPerVG, CheckRADOSFreeSpace, CheckSpindlesExclusiveStorage, \ ComputeDiskSizePerVG, CreateDisks, \ GenerateDiskTemplate, CommitDisks, \ WaitForSync, ComputeDisks, \ ImageDisks, WipeDisks from ganeti.cmdlib.instance_utils import \ CheckNodeNotDrained, CopyLockList, \ ReleaseLocks, CheckNodeVmCapable, \ RemoveDisks, CheckNodeFreeMemory, \ UpdateMetadata, CheckForConflictingIp, \ ComputeInstanceCommunicationNIC, \ ComputeIPolicyInstanceSpecViolation, \ CheckHostnameSane, CheckOpportunisticLocking, \ ComputeFullBeParams, ComputeNics, GetClusterDomainSecret, \ CheckInstanceExistence, CreateInstanceAllocRequest, BuildInstanceHookEnv, \ NICListToTuple, CheckNicsBridgesExist, CheckCompressionTool import ganeti.masterd.instance def _ValidateTrunkVLAN(vlan): if not compat.all(vl.isdigit() for vl in vlan[1:].split(':')): raise errors.OpPrereqError("Specified VLAN parameter is invalid" " : %s" % vlan, errors.ECODE_INVAL) class LUInstanceCreate(LogicalUnit): """Create an instance. """ HPATH = "instance-add" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def _CheckDiskTemplateValid(self): """Checks validity of disk template. """ cluster = self.cfg.GetClusterInfo() if self.op.disk_template is None: # FIXME: It would be better to take the default disk template from the # ipolicy, but for the ipolicy we need the primary node, which we get from # the iallocator, which wants the disk template as input. To solve this # chicken-and-egg problem, it should be possible to specify just a node # group from the iallocator and take the ipolicy from that. self.op.disk_template = cluster.enabled_disk_templates[0] CheckDiskTemplateEnabled(cluster, self.op.disk_template) def _CheckDiskArguments(self): """Checks validity of disk-related arguments. """ # check that disk's names are unique and valid utils.ValidateDeviceNames("disk", self.op.disks) self._CheckDiskTemplateValid() # check disks. parameter names and consistent adopt/no-adopt strategy has_adopt = has_no_adopt = False for disk in self.op.disks: if self.op.disk_template != constants.DT_EXT: utils.ForceDictType(disk, constants.IDISK_PARAMS_TYPES) if constants.IDISK_ADOPT in disk: # pylint: disable=R0102 has_adopt = True else: has_no_adopt = True if has_adopt and has_no_adopt: raise errors.OpPrereqError("Either all disks are adopted or none is", errors.ECODE_INVAL) if has_adopt: if self.op.disk_template not in constants.DTS_MAY_ADOPT: raise errors.OpPrereqError("Disk adoption is not supported for the" " '%s' disk template" % self.op.disk_template, errors.ECODE_INVAL) if self.op.iallocator is not None: raise errors.OpPrereqError("Disk adoption not allowed with an" " iallocator script", errors.ECODE_INVAL) if self.op.mode == constants.INSTANCE_IMPORT: raise errors.OpPrereqError("Disk adoption not allowed for" " instance import", errors.ECODE_INVAL) else: if self.op.disk_template in constants.DTS_MUST_ADOPT: raise errors.OpPrereqError("Disk template %s requires disk adoption," " but no 'adopt' parameter given" % self.op.disk_template, errors.ECODE_INVAL) self.adopt_disks = has_adopt def _CheckVLANArguments(self): """ Check validity of VLANs if given """ for nic in self.op.nics: vlan = nic.get(constants.INIC_VLAN, None) if vlan: if vlan[0] == ".": # vlan starting with dot means single untagged vlan, # might be followed by trunk (:) if not vlan[1:].isdigit(): _ValidateTrunkVLAN(vlan) elif vlan[0] == ":": # Trunk - tagged only _ValidateTrunkVLAN(vlan) elif vlan.isdigit(): # This is the simplest case. No dots, only single digit # -> Create untagged access port, dot needs to be added nic[constants.INIC_VLAN] = "." + vlan else: raise errors.OpPrereqError("Specified VLAN parameter is invalid" " : %s" % vlan, errors.ECODE_INVAL) def CheckArguments(self): """Check arguments. """ if self.op.forthcoming and self.op.commit: raise errors.OpPrereqError("Forthcoming generation and commiting are" " mutually exclusive", errors.ECODE_INVAL) # do not require name_check to ease forward/backward compatibility # for tools if self.op.no_install and self.op.start: self.LogInfo("No-installation mode selected, disabling startup") self.op.start = False # validate/normalize the instance name self.op.instance_name = \ netutils.Hostname.GetNormalizedName(self.op.instance_name) if self.op.ip_check and not self.op.name_check: # TODO: make the ip check more flexible and not depend on the name check raise errors.OpPrereqError("Cannot do IP address check without a name" " check", errors.ECODE_INVAL) # instance name verification if self.op.name_check: self.hostname = CheckHostnameSane(self, self.op.instance_name) self.op.instance_name = self.hostname.name # used in CheckPrereq for ip ping check self.check_ip = self.hostname.ip else: self.check_ip = None # add NIC for instance communication if self.op.instance_communication: nic_name = ComputeInstanceCommunicationNIC(self.op.instance_name) for nic in self.op.nics: if nic.get(constants.INIC_NAME, None) == nic_name: break else: self.op.nics.append({constants.INIC_NAME: nic_name, constants.INIC_MAC: constants.VALUE_GENERATE, constants.INIC_IP: constants.NIC_IP_POOL, constants.INIC_NETWORK: self.cfg.GetInstanceCommunicationNetwork()}) # timeouts for unsafe OS installs if self.op.helper_startup_timeout is None: self.op.helper_startup_timeout = constants.HELPER_VM_STARTUP if self.op.helper_shutdown_timeout is None: self.op.helper_shutdown_timeout = constants.HELPER_VM_SHUTDOWN # check nics' parameter names for nic in self.op.nics: utils.ForceDictType(nic, constants.INIC_PARAMS_TYPES) # check that NIC's parameters names are unique and valid utils.ValidateDeviceNames("NIC", self.op.nics) self._CheckVLANArguments() self._CheckDiskArguments() assert self.op.disk_template is not None # file storage checks if (self.op.file_driver and not self.op.file_driver in constants.FILE_DRIVER): raise errors.OpPrereqError("Invalid file driver name '%s'" % self.op.file_driver, errors.ECODE_INVAL) # set default file_driver if unset and required if (not self.op.file_driver and self.op.disk_template in constants.DTS_FILEBASED): self.op.file_driver = constants.FD_DEFAULT ### Node/iallocator related checks CheckIAllocatorOrNode(self, "iallocator", "pnode") if self.op.pnode is not None: if self.op.disk_template in constants.DTS_INT_MIRROR: if self.op.snode is None: raise errors.OpPrereqError("The networked disk templates need" " a mirror node", errors.ECODE_INVAL) elif self.op.snode: self.LogWarning("Secondary node will be ignored on non-mirrored disk" " template") self.op.snode = None CheckOpportunisticLocking(self.op) if self.op.mode == constants.INSTANCE_IMPORT: # On import force_variant must be True, because if we forced it at # initial install, our only chance when importing it back is that it # works again! self.op.force_variant = True if self.op.no_install: self.LogInfo("No-installation mode has no effect during import") if objects.GetOSImage(self.op.osparams): self.LogInfo("OS image has no effect during import") elif self.op.mode == constants.INSTANCE_CREATE: os_image = CheckOSImage(self.op) if self.op.os_type is None and os_image is None: raise errors.OpPrereqError("No guest OS or OS image specified", errors.ECODE_INVAL) if self.op.os_type is not None \ and self.op.os_type in self.cfg.GetClusterInfo().blacklisted_os: raise errors.OpPrereqError("Guest OS '%s' is not allowed for" " installation" % self.op.os_type, errors.ECODE_STATE) elif self.op.mode == constants.INSTANCE_REMOTE_IMPORT: if objects.GetOSImage(self.op.osparams): self.LogInfo("OS image has no effect during import") self._cds = GetClusterDomainSecret() # Check handshake to ensure both clusters have the same domain secret src_handshake = self.op.source_handshake if not src_handshake: raise errors.OpPrereqError("Missing source handshake", errors.ECODE_INVAL) errmsg = masterd.instance.CheckRemoteExportHandshake(self._cds, src_handshake) if errmsg: raise errors.OpPrereqError("Invalid handshake: %s" % errmsg, errors.ECODE_INVAL) # Load and check source CA self.source_x509_ca_pem = self.op.source_x509_ca if not self.source_x509_ca_pem: raise errors.OpPrereqError("Missing source X509 CA", errors.ECODE_INVAL) try: (cert, _) = utils.LoadSignedX509Certificate(self.source_x509_ca_pem, self._cds) except OpenSSL.crypto.Error as err: raise errors.OpPrereqError("Unable to load source X509 CA (%s)" % (err, ), errors.ECODE_INVAL) (errcode, msg) = utils.VerifyX509Certificate(cert, None, None) if errcode is not None: raise errors.OpPrereqError("Invalid source X509 CA (%s)" % (msg, ), errors.ECODE_INVAL) self.source_x509_ca = cert src_instance_name = self.op.source_instance_name if not src_instance_name: raise errors.OpPrereqError("Missing source instance name", errors.ECODE_INVAL) self.source_instance_name = \ netutils.GetHostname(name=src_instance_name).name else: raise errors.OpPrereqError("Invalid instance creation mode %r" % self.op.mode, errors.ECODE_INVAL) def ExpandNames(self): """ExpandNames for CreateInstance. Figure out the right locks for instance creation. """ self.needed_locks = {} if self.op.commit: (uuid, name) = self.cfg.ExpandInstanceName(self.op.instance_name) if name is None: raise errors.OpPrereqError("Instance %s unknown" % self.op.instance_name, errors.ECODE_INVAL) self.op.instance_name = name if not self.cfg.GetInstanceInfo(uuid).forthcoming: raise errors.OpPrereqError("Instance %s (with uuid %s) not forthcoming" " but --commit was passed." % (name, uuid), errors.ECODE_STATE) logging.debug("Verified that instance %s with uuid %s is forthcoming", name, uuid) else: # this is just a preventive check, but someone might still add this # instance in the meantime; we check again in CheckPrereq CheckInstanceExistence(self, self.op.instance_name) self.add_locks[locking.LEVEL_INSTANCE] = self.op.instance_name if self.op.commit: (uuid, _) = self.cfg.ExpandInstanceName(self.op.instance_name) self.needed_locks[locking.LEVEL_NODE] = self.cfg.GetInstanceNodes(uuid) logging.debug("Forthcoming instance %s resides on %s", uuid, self.needed_locks[locking.LEVEL_NODE]) elif self.op.iallocator: # TODO: Find a solution to not lock all nodes in the cluster, e.g. by # specifying a group on instance creation and then selecting nodes from # that group self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET if self.op.opportunistic_locking: self.opportunistic_locks[locking.LEVEL_NODE] = True self.opportunistic_locks[locking.LEVEL_NODE_RES] = True if self.op.disk_template == constants.DT_DRBD8: self.opportunistic_locks_count[locking.LEVEL_NODE] = 2 self.opportunistic_locks_count[locking.LEVEL_NODE_RES] = 2 else: (self.op.pnode_uuid, self.op.pnode) = \ ExpandNodeUuidAndName(self.cfg, self.op.pnode_uuid, self.op.pnode) nodelist = [self.op.pnode_uuid] if self.op.snode is not None: (self.op.snode_uuid, self.op.snode) = \ ExpandNodeUuidAndName(self.cfg, self.op.snode_uuid, self.op.snode) nodelist.append(self.op.snode_uuid) self.needed_locks[locking.LEVEL_NODE] = nodelist # in case of import lock the source node too if self.op.mode == constants.INSTANCE_IMPORT: src_node = self.op.src_node src_path = self.op.src_path if src_path is None: self.op.src_path = src_path = self.op.instance_name if src_node is None: self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET self.op.src_node = None if os.path.isabs(src_path): raise errors.OpPrereqError("Importing an instance from a path" " requires a source node option", errors.ECODE_INVAL) else: (self.op.src_node_uuid, self.op.src_node) = (_, src_node) = \ ExpandNodeUuidAndName(self.cfg, self.op.src_node_uuid, src_node) if self.needed_locks[locking.LEVEL_NODE] is not locking.ALL_SET: self.needed_locks[locking.LEVEL_NODE].append(self.op.src_node_uuid) if not os.path.isabs(src_path): self.op.src_path = \ utils.PathJoin(pathutils.EXPORT_DIR, src_path) self.needed_locks[locking.LEVEL_NODE_RES] = \ CopyLockList(self.needed_locks[locking.LEVEL_NODE]) # Optimistically acquire shared group locks (we're reading the # configuration). We can't just call GetInstanceNodeGroups, because the # instance doesn't exist yet. Therefore we lock all node groups of all # nodes we have. if self.needed_locks[locking.LEVEL_NODE] == locking.ALL_SET: # In the case we lock all nodes for opportunistic allocation, we have no # choice than to lock all groups, because they're allocated before nodes. # This is sad, but true. At least we release all those we don't need in # CheckPrereq later. self.needed_locks[locking.LEVEL_NODEGROUP] = locking.ALL_SET else: self.needed_locks[locking.LEVEL_NODEGROUP] = \ list(self.cfg.GetNodeGroupsFromNodes( self.needed_locks[locking.LEVEL_NODE])) self.share_locks[locking.LEVEL_NODEGROUP] = 1 def DeclareLocks(self, level): if level == locking.LEVEL_NODE_RES: if self.op.opportunistic_locking: self.needed_locks[locking.LEVEL_NODE_RES] = \ CopyLockList(list(self.owned_locks(locking.LEVEL_NODE))) def _RunAllocator(self): """Run the allocator based on input opcode. """ if self.op.opportunistic_locking: # Only consider nodes for which a lock is held node_name_whitelist = self.cfg.GetNodeNames( set(self.owned_locks(locking.LEVEL_NODE)) & set(self.owned_locks(locking.LEVEL_NODE_RES))) logging.debug("Trying to allocate on nodes %s", node_name_whitelist) else: node_name_whitelist = None req = CreateInstanceAllocRequest(self.op, self.disks, self.nics, self.be_full, node_name_whitelist) ial = iallocator.IAllocator(self.cfg, self.rpc, req) ial.Run(self.op.iallocator) if not ial.success: # When opportunistic locks are used only a temporary failure is generated if self.op.opportunistic_locking: ecode = errors.ECODE_TEMP_NORES self.LogInfo("IAllocator '%s' failed on opportunistically acquired" " nodes: %s", self.op.iallocator, ial.info) else: ecode = errors.ECODE_NORES raise errors.OpPrereqError("Can't compute nodes using" " iallocator '%s': %s" % (self.op.iallocator, ial.info), ecode) (self.op.pnode_uuid, self.op.pnode) = ExpandNodeUuidAndName( self.cfg, None, ial.result[0]) # pylint: disable=E1136 self.LogInfo("Selected nodes for instance %s via iallocator %s: %s", self.op.instance_name, self.op.iallocator, utils.CommaJoin(ial.result)) assert req.RequiredNodes() in (1, 2), "Wrong node count from iallocator" if req.RequiredNodes() == 2: (self.op.snode_uuid, self.op.snode) = ExpandNodeUuidAndName( self.cfg, None, ial.result[1]) # pylint: disable=E1136 def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ env = { "ADD_MODE": self.op.mode, } if self.op.mode == constants.INSTANCE_IMPORT: env["SRC_NODE"] = self.op.src_node env["SRC_PATH"] = self.op.src_path env["SRC_IMAGES"] = self.src_images env.update(BuildInstanceHookEnv( name=self.op.instance_name, primary_node_name=self.op.pnode, secondary_node_names=self.cfg.GetNodeNames(self.secondaries), status=self.op.start, os_type=self.op.os_type, minmem=self.be_full[constants.BE_MINMEM], maxmem=self.be_full[constants.BE_MAXMEM], vcpus=self.be_full[constants.BE_VCPUS], nics=NICListToTuple(self, self.nics), disk_template=self.op.disk_template, # Note that self.disks here is not a list with objects.Disk # but with dicts as returned by ComputeDisks. disks=self.disks, bep=self.be_full, hvp=self.hv_full, hypervisor_name=self.op.hypervisor, tags=self.op.tags, )) return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode(), self.op.pnode_uuid] + self.secondaries return nl, nl def _ReadExportInfo(self): """Reads the export information from disk. It will override the opcode source node and path with the actual information, if these two were not specified before. @return: the export information """ assert self.op.mode == constants.INSTANCE_IMPORT if self.op.src_node_uuid is None: locked_nodes = self.owned_locks(locking.LEVEL_NODE) exp_list = self.rpc.call_export_list(locked_nodes) found = False for node_uuid in exp_list: if exp_list[node_uuid].fail_msg: continue if self.op.src_path in exp_list[node_uuid].payload: found = True self.op.src_node = self.cfg.GetNodeInfo(node_uuid).name self.op.src_node_uuid = node_uuid self.op.src_path = utils.PathJoin(pathutils.EXPORT_DIR, self.op.src_path) break if not found: raise errors.OpPrereqError("No export found for relative path %s" % self.op.src_path, errors.ECODE_INVAL) CheckNodeOnline(self, self.op.src_node_uuid) result = self.rpc.call_export_info(self.op.src_node_uuid, self.op.src_path) result.Raise("No export or invalid export found in dir %s" % self.op.src_path) export_info = objects.SerializableConfigParser.Loads(str(result.payload)) if not export_info.has_section(constants.INISECT_EXP): raise errors.ProgrammerError("Corrupted export config", errors.ECODE_ENVIRON) ei_version = export_info.get(constants.INISECT_EXP, "version") if int(ei_version) != constants.EXPORT_VERSION: raise errors.OpPrereqError("Wrong export version %s (wanted %d)" % (ei_version, constants.EXPORT_VERSION), errors.ECODE_ENVIRON) return export_info def _ReadExportParams(self, einfo): """Use export parameters as defaults. In case the opcode doesn't specify (as in override) some instance parameters, then try to use them from the export information, if that declares them. """ self.op.os_type = einfo.get(constants.INISECT_EXP, "os") if not self.op.disks: disks = [] # TODO: import the disk iv_name too for idx in range(constants.MAX_DISKS): if einfo.has_option(constants.INISECT_INS, "disk%d_size" % idx): disk_sz = einfo.getint(constants.INISECT_INS, "disk%d_size" % idx) disk_name = einfo.get(constants.INISECT_INS, "disk%d_name" % idx) disk = { constants.IDISK_SIZE: disk_sz, constants.IDISK_NAME: disk_name } disks.append(disk) self.op.disks = disks if not disks and self.op.disk_template != constants.DT_DISKLESS: raise errors.OpPrereqError("No disk info specified and the export" " is missing the disk information", errors.ECODE_INVAL) if not self.op.nics: nics = [] for idx in range(constants.MAX_NICS): if einfo.has_option(constants.INISECT_INS, "nic%d_mac" % idx): ndict = {} for name in [constants.INIC_IP, constants.INIC_MAC, constants.INIC_NAME]: nic_param_name = "nic%d_%s" % (idx, name) if einfo.has_option(constants.INISECT_INS, nic_param_name): v = einfo.get(constants.INISECT_INS, "nic%d_%s" % (idx, name)) ndict[name] = v network = einfo.get(constants.INISECT_INS, "nic%d_%s" % (idx, constants.INIC_NETWORK)) # in case network is given link and mode are inherited # from nodegroup's netparams and thus should not be passed here if network: ndict[constants.INIC_NETWORK] = network else: for name in list(constants.NICS_PARAMETERS): v = einfo.get(constants.INISECT_INS, "nic%d_%s" % (idx, name)) ndict[name] = v nics.append(ndict) else: break self.op.nics = nics if not self.op.tags and einfo.has_option(constants.INISECT_INS, "tags"): self.op.tags = einfo.get(constants.INISECT_INS, "tags").split() if (self.op.hypervisor is None and einfo.has_option(constants.INISECT_INS, "hypervisor")): self.op.hypervisor = einfo.get(constants.INISECT_INS, "hypervisor") if einfo.has_section(constants.INISECT_HYP): # use the export parameters but do not override the ones # specified by the user for name, value in einfo.items(constants.INISECT_HYP): if name not in self.op.hvparams: self.op.hvparams[name] = value if einfo.has_section(constants.INISECT_BEP): # use the parameters, without overriding for name, value in einfo.items(constants.INISECT_BEP): if name not in self.op.beparams: self.op.beparams[name] = value # Compatibility for the old "memory" be param if name == constants.BE_MEMORY: if constants.BE_MAXMEM not in self.op.beparams: self.op.beparams[constants.BE_MAXMEM] = value if constants.BE_MINMEM not in self.op.beparams: self.op.beparams[constants.BE_MINMEM] = value else: # try to read the parameters old style, from the main section for name in constants.BES_PARAMETERS: if (name not in self.op.beparams and einfo.has_option(constants.INISECT_INS, name)): self.op.beparams[name] = einfo.get(constants.INISECT_INS, name) if einfo.has_section(constants.INISECT_OSP): # use the parameters, without overriding for name, value in einfo.items(constants.INISECT_OSP): if name not in self.op.osparams: self.op.osparams[name] = value if einfo.has_section(constants.INISECT_OSP_PRIVATE): # use the parameters, without overriding for name, value in einfo.items(constants.INISECT_OSP_PRIVATE): if name not in self.op.osparams_private: self.op.osparams_private[name] = serializer.Private(value, descr=name) def _RevertToDefaults(self, cluster): """Revert the instance parameters to the default values. """ # hvparams hv_defs = cluster.SimpleFillHV(self.op.hypervisor, self.op.os_type, {}) for name in list(self.op.hvparams): if name in hv_defs and hv_defs[name] == self.op.hvparams[name]: del self.op.hvparams[name] # beparams be_defs = cluster.SimpleFillBE({}) for name in list(self.op.beparams): if name in be_defs and be_defs[name] == self.op.beparams[name]: del self.op.beparams[name] # nic params nic_defs = cluster.SimpleFillNIC({}) for nic in self.op.nics: for name in constants.NICS_PARAMETERS: if name in nic and name in nic_defs and nic[name] == nic_defs[name]: del nic[name] # osparams os_defs = cluster.SimpleFillOS(self.op.os_type, {}) for name in list(self.op.osparams): if name in os_defs and os_defs[name] == self.op.osparams[name]: del self.op.osparams[name] os_defs_ = cluster.SimpleFillOS(self.op.os_type, {}, os_params_private={}) for name in list(self.op.osparams_private): if name in os_defs_ and os_defs_[name] == self.op.osparams_private[name]: del self.op.osparams_private[name] def _GetNodesFromForthcomingInstance(self): """Set nodes as in the forthcoming instance """ (uuid, name) = self.cfg.ExpandInstanceName(self.op.instance_name) inst = self.cfg.GetInstanceInfo(uuid) self.op.pnode_uuid = inst.primary_node self.op.pnode = self.cfg.GetNodeName(inst.primary_node) sec_nodes = self.cfg.GetInstanceSecondaryNodes(uuid) node_names = [self.op.pnode] if sec_nodes: self.op.snode_uuid = sec_nodes[0] self.op.snode = self.cfg.GetNodeName(sec_nodes[0]) node_names.append(self.op.snode) self.LogInfo("Nodes of instance %s: %s", name, node_names) def CheckPrereq(self): # pylint: disable=R0914 """Check prerequisites. """ owned_nodes = frozenset(self.owned_locks(locking.LEVEL_NODE)) if self.op.commit: # Check that the instance is still on the cluster, forthcoming, and # still resides on the nodes we acquired. (uuid, name) = self.cfg.ExpandInstanceName(self.op.instance_name) if uuid is None: raise errors.OpPrereqError("Instance %s disappeared from the cluster" " while waiting for locks" % (self.op.instance_name,), errors.ECODE_STATE) if not self.cfg.GetInstanceInfo(uuid).forthcoming: raise errors.OpPrereqError("Instance %s (with uuid %s) is no longer" " forthcoming" % (name, uuid), errors.ECODE_STATE) required_nodes = self.cfg.GetInstanceNodes(uuid) if not owned_nodes.issuperset(required_nodes): raise errors.OpPrereqError("Forthcoming instance %s nodes changed" " since locks were acquired; retry the" " operation" % self.op.instance_name, errors.ECODE_STATE) else: CheckInstanceExistence(self, self.op.instance_name) # Check that the optimistically acquired groups are correct wrt the # acquired nodes owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP)) cur_groups = list(self.cfg.GetNodeGroupsFromNodes(owned_nodes)) if not owned_groups.issuperset(cur_groups): raise errors.OpPrereqError("New instance %s's node groups changed since" " locks were acquired, current groups are" " are '%s', owning groups '%s'; retry the" " operation" % (self.op.instance_name, utils.CommaJoin(cur_groups), utils.CommaJoin(owned_groups)), errors.ECODE_STATE) self.instance_file_storage_dir = CalculateFileStorageDir( self.op.disk_template, self.cfg, self.op.instance_name, self.op.file_storage_dir) if self.op.mode == constants.INSTANCE_IMPORT: export_info = self._ReadExportInfo() self._ReadExportParams(export_info) self._old_instance_name = export_info.get(constants.INISECT_INS, "name") else: self._old_instance_name = None if (not self.cfg.GetVGName() and self.op.disk_template not in constants.DTS_NOT_LVM): raise errors.OpPrereqError("Cluster does not support lvm-based" " instances", errors.ECODE_STATE) if (self.op.hypervisor is None or self.op.hypervisor == constants.VALUE_AUTO): self.op.hypervisor = self.cfg.GetHypervisorType() cluster = self.cfg.GetClusterInfo() enabled_hvs = cluster.enabled_hypervisors if self.op.hypervisor not in enabled_hvs: raise errors.OpPrereqError("Selected hypervisor (%s) not enabled in the" " cluster (%s)" % (self.op.hypervisor, ",".join(enabled_hvs)), errors.ECODE_STATE) # Check tag validity for tag in self.op.tags: objects.TaggableObject.ValidateTag(tag) # check hypervisor parameter syntax (locally) utils.ForceDictType(self.op.hvparams, constants.HVS_PARAMETER_TYPES) filled_hvp = cluster.SimpleFillHV(self.op.hypervisor, self.op.os_type, self.op.hvparams) hv_type = hypervisor.GetHypervisorClass(self.op.hypervisor) hv_type.CheckParameterSyntax(filled_hvp) self.hv_full = filled_hvp # check that we don't specify global parameters on an instance CheckParamsNotGlobal(self.op.hvparams, constants.HVC_GLOBALS, "hypervisor", "instance", "cluster") # fill and remember the beparams dict self.be_full = ComputeFullBeParams(self.op, cluster) # build os parameters if self.op.osparams_private is None: self.op.osparams_private = serializer.PrivateDict() if self.op.osparams_secret is None: self.op.osparams_secret = serializer.PrivateDict() self.os_full = cluster.SimpleFillOS( self.op.os_type, self.op.osparams, os_params_private=self.op.osparams_private, os_params_secret=self.op.osparams_secret ) # now that hvp/bep are in final format, let's reset to defaults, # if told to do so if self.op.identify_defaults: self._RevertToDefaults(cluster) # NIC buildup self.nics = ComputeNics(self.op, cluster, self.check_ip, self.cfg, self.proc.GetECId()) # disk checks/pre-build default_vg = self.cfg.GetVGName() self.disks = ComputeDisks(self.op.disks, self.op.disk_template, default_vg) if self.op.mode == constants.INSTANCE_IMPORT: disk_images = [] for idx in range(len(self.disks)): option = "disk%d_dump" % idx if export_info.has_option(constants.INISECT_INS, option): # FIXME: are the old os-es, disk sizes, etc. useful? export_name = export_info.get(constants.INISECT_INS, option) image = utils.PathJoin(self.op.src_path, export_name) disk_images.append(image) else: disk_images.append(False) self.src_images = disk_images if self.op.instance_name == self._old_instance_name: for idx, nic in enumerate(self.nics): if nic.mac == constants.VALUE_AUTO: nic_mac_ini = "nic%d_mac" % idx nic.mac = export_info.get(constants.INISECT_INS, nic_mac_ini) # ENDIF: self.op.mode == constants.INSTANCE_IMPORT # ip ping checks (we use the same ip that was resolved in ExpandNames) if self.op.ip_check: if netutils.TcpPing(self.check_ip, constants.DEFAULT_NODED_PORT): raise errors.OpPrereqError("IP %s of instance %s already in use" % (self.check_ip, self.op.instance_name), errors.ECODE_NOTUNIQUE) #### mac address generation # By generating here the mac address both the allocator and the hooks get # the real final mac address rather than the 'auto' or 'generate' value. # There is a race condition between the generation and the instance object # creation, which means that we know the mac is valid now, but we're not # sure it will be when we actually add the instance. If things go bad # adding the instance will abort because of a duplicate mac, and the # creation job will fail. for nic in self.nics: if nic.mac in (constants.VALUE_AUTO, constants.VALUE_GENERATE): nic.mac = self.cfg.GenerateMAC(nic.network, self.proc.GetECId()) #### allocator run if self.op.iallocator is not None: if self.op.commit: self._GetNodesFromForthcomingInstance() else: self._RunAllocator() # Release all unneeded node locks keep_locks = [uuid for uuid in [self.op.pnode_uuid, self.op.snode_uuid, self.op.src_node_uuid] if uuid] ReleaseLocks(self, locking.LEVEL_NODE, keep=keep_locks) ReleaseLocks(self, locking.LEVEL_NODE_RES, keep=keep_locks) # Release all unneeded group locks ReleaseLocks(self, locking.LEVEL_NODEGROUP, keep=self.cfg.GetNodeGroupsFromNodes(keep_locks)) assert (self.owned_locks(locking.LEVEL_NODE) == self.owned_locks(locking.LEVEL_NODE_RES)), \ ("Node locks differ from node resource locks (%s vs %s)" % (self.owned_locks(locking.LEVEL_NODE), self.owned_locks(locking.LEVEL_NODE_RES))) #### node related checks # check primary node self.pnode = pnode = self.cfg.GetNodeInfo(self.op.pnode_uuid) assert self.pnode is not None, \ "Cannot retrieve locked node %s" % self.op.pnode_uuid if pnode.offline: raise errors.OpPrereqError("Cannot use offline primary node '%s'" % pnode.name, errors.ECODE_STATE) if pnode.drained: raise errors.OpPrereqError("Cannot use drained primary node '%s'" % pnode.name, errors.ECODE_STATE) if not pnode.vm_capable: raise errors.OpPrereqError("Cannot use non-vm_capable primary node" " '%s'" % pnode.name, errors.ECODE_STATE) self.secondaries = [] # Fill in any IPs from IP pools. This must happen here, because we need to # know the nic's primary node, as specified by the iallocator for idx, nic in enumerate(self.nics): net_uuid = nic.network if net_uuid is not None: nobj = self.cfg.GetNetwork(net_uuid) netparams = self.cfg.GetGroupNetParams(net_uuid, self.pnode.uuid) if netparams is None: raise errors.OpPrereqError("No netparams found for network" " %s. Probably not connected to" " node's %s nodegroup" % (nobj.name, self.pnode.name), errors.ECODE_INVAL) self.LogInfo("NIC/%d inherits netparams %s" % (idx, list(netparams.values()))) nic.nicparams = dict(netparams) if nic.ip is not None: if nic.ip.lower() == constants.NIC_IP_POOL: try: nic.ip = self.cfg.GenerateIp(net_uuid, self.proc.GetECId()) except errors.ReservationError: raise errors.OpPrereqError("Unable to get a free IP for NIC %d" " from the address pool" % idx, errors.ECODE_STATE) self.LogInfo("Chose IP %s from network %s", nic.ip, nobj.name) else: try: self.cfg.ReserveIp(net_uuid, nic.ip, self.proc.GetECId(), check=self.op.conflicts_check) except errors.ReservationError: raise errors.OpPrereqError("IP address %s already in use" " or does not belong to network %s" % (nic.ip, nobj.name), errors.ECODE_NOTUNIQUE) # net is None, ip None or given elif self.op.conflicts_check: CheckForConflictingIp(self, nic.ip, self.pnode.uuid) # mirror node verification if self.op.disk_template in constants.DTS_INT_MIRROR: if self.op.snode_uuid == pnode.uuid: raise errors.OpPrereqError("The secondary node cannot be the" " primary node", errors.ECODE_INVAL) CheckNodeOnline(self, self.op.snode_uuid) CheckNodeNotDrained(self, self.op.snode_uuid) CheckNodeVmCapable(self, self.op.snode_uuid) self.secondaries.append(self.op.snode_uuid) snode = self.cfg.GetNodeInfo(self.op.snode_uuid) if pnode.group != snode.group: self.LogWarning("The primary and secondary nodes are in two" " different node groups; the disk parameters" " from the first disk's node group will be" " used") nodes = [pnode] if self.op.disk_template in constants.DTS_INT_MIRROR: nodes.append(snode) has_es = lambda n: IsExclusiveStorageEnabledNode(self.cfg, n) excl_stor = compat.any(map(has_es, nodes)) if excl_stor and not self.op.disk_template in constants.DTS_EXCL_STORAGE: raise errors.OpPrereqError("Disk template %s not supported with" " exclusive storage" % self.op.disk_template, errors.ECODE_STATE) for disk in self.disks: CheckSpindlesExclusiveStorage(disk, excl_stor, True) node_uuids = [pnode.uuid] + self.secondaries if not self.adopt_disks: if self.op.disk_template == constants.DT_RBD: # _CheckRADOSFreeSpace() is just a placeholder. # Any function that checks prerequisites can be placed here. # Check if there is enough space on the RADOS cluster. CheckRADOSFreeSpace() elif self.op.disk_template == constants.DT_EXT: # FIXME: Function that checks prereqs if needed pass elif self.op.disk_template in constants.DTS_LVM: # Check lv size requirements, if not adopting req_sizes = ComputeDiskSizePerVG(self.op.disk_template, self.disks) CheckNodesFreeDiskPerVG(self, node_uuids, req_sizes) else: # FIXME: add checks for other, non-adopting, non-lvm disk templates pass elif self.op.disk_template == constants.DT_PLAIN: # Check the adoption data all_lvs = set(["%s/%s" % (disk[constants.IDISK_VG], disk[constants.IDISK_ADOPT]) for disk in self.disks]) if len(all_lvs) != len(self.disks): raise errors.OpPrereqError("Duplicate volume names given for adoption", errors.ECODE_INVAL) for lv_name in all_lvs: try: # FIXME: lv_name here is "vg/lv" need to ensure that other calls # to ReserveLV uses the same syntax self.cfg.ReserveLV(lv_name, self.proc.GetECId()) except errors.ReservationError: raise errors.OpPrereqError("LV named %s used by another instance" % lv_name, errors.ECODE_NOTUNIQUE) vg_names = self.rpc.call_vg_list([pnode.uuid])[pnode.uuid] vg_names.Raise("Cannot get VG information from node %s" % pnode.name, prereq=True) node_lvs = self.rpc.call_lv_list([pnode.uuid], list(vg_names.payload))[pnode.uuid] node_lvs.Raise("Cannot get LV information from node %s" % pnode.name, prereq=True) node_lvs = node_lvs.payload delta = all_lvs.difference(node_lvs) if delta: raise errors.OpPrereqError("Missing logical volume(s): %s" % utils.CommaJoin(delta), errors.ECODE_INVAL) online_lvs = [lv for lv in all_lvs if node_lvs[lv][2]] if online_lvs: raise errors.OpPrereqError("Online logical volumes found, cannot" " adopt: %s" % utils.CommaJoin(online_lvs), errors.ECODE_STATE) # update the size of disk based on what is found for dsk in self.disks: dsk[constants.IDISK_SIZE] = \ int(float(node_lvs["%s/%s" % (dsk[constants.IDISK_VG], dsk[constants.IDISK_ADOPT])][0])) elif self.op.disk_template == constants.DT_BLOCK: # Normalize and de-duplicate device paths all_disks = set([os.path.abspath(disk[constants.IDISK_ADOPT]) for disk in self.disks]) if len(all_disks) != len(self.disks): raise errors.OpPrereqError("Duplicate disk names given for adoption", errors.ECODE_INVAL) baddisks = [d for d in all_disks if not d.startswith(constants.ADOPTABLE_BLOCKDEV_ROOT)] if baddisks: raise errors.OpPrereqError("Device node(s) %s lie outside %s and" " cannot be adopted" % (utils.CommaJoin(baddisks), constants.ADOPTABLE_BLOCKDEV_ROOT), errors.ECODE_INVAL) node_disks = self.rpc.call_bdev_sizes([pnode.uuid], list(all_disks))[pnode.uuid] node_disks.Raise("Cannot get block device information from node %s" % pnode.name, prereq=True) node_disks = node_disks.payload delta = all_disks.difference(node_disks) if delta: raise errors.OpPrereqError("Missing block device(s): %s" % utils.CommaJoin(delta), errors.ECODE_INVAL) for dsk in self.disks: dsk[constants.IDISK_SIZE] = \ int(float(node_disks[dsk[constants.IDISK_ADOPT]])) # Check disk access param to be compatible with specified hypervisor node_info = self.cfg.GetNodeInfo(self.op.pnode_uuid) node_group = self.cfg.GetNodeGroup(node_info.group) group_disk_params = self.cfg.GetGroupDiskParams(node_group) group_access_type = group_disk_params[self.op.disk_template].get( constants.RBD_ACCESS, constants.DISK_KERNELSPACE ) for dsk in self.disks: access_type = dsk.get(constants.IDISK_ACCESS, group_access_type) if not IsValidDiskAccessModeCombination(self.op.hypervisor, self.op.disk_template, access_type): raise errors.OpPrereqError("Selected hypervisor (%s) cannot be" " used with %s disk access param" % (self.op.hypervisor, access_type), errors.ECODE_STATE) # Verify instance specs spindle_use = self.be_full.get(constants.BE_SPINDLE_USE, None) ispec = { constants.ISPEC_MEM_SIZE: self.be_full.get(constants.BE_MAXMEM, None), constants.ISPEC_CPU_COUNT: self.be_full.get(constants.BE_VCPUS, None), constants.ISPEC_DISK_COUNT: len(self.disks), constants.ISPEC_DISK_SIZE: [disk[constants.IDISK_SIZE] for disk in self.disks], constants.ISPEC_NIC_COUNT: len(self.nics), constants.ISPEC_SPINDLE_USE: spindle_use, } group_info = self.cfg.GetNodeGroup(pnode.group) ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(cluster, group_info) disk_types = [self.op.disk_template] * len(self.disks) res = ComputeIPolicyInstanceSpecViolation(ipolicy, ispec, disk_types) if not self.op.ignore_ipolicy and res: msg = ("Instance allocation to group %s (%s) violates policy: %s" % (pnode.group, group_info.name, utils.CommaJoin(res))) raise errors.OpPrereqError(msg, errors.ECODE_INVAL) CheckHVParams(self, node_uuids, self.op.hypervisor, self.op.hvparams) CheckOSParams(self, True, node_uuids, self.op.os_type, self.os_full, self.op.force_variant) CheckNicsBridgesExist(self, self.nics, self.pnode.uuid) CheckCompressionTool(self, self.op.compress) #TODO: _CheckExtParams (remotely) # Check parameters for extstorage # memory check on primary node #TODO(dynmem): use MINMEM for checking if self.op.start: hvfull = objects.FillDict(cluster.hvparams.get(self.op.hypervisor, {}), self.op.hvparams) CheckNodeFreeMemory(self, self.pnode.uuid, "creating instance %s" % self.op.instance_name, self.be_full[constants.BE_MAXMEM], self.op.hypervisor, hvfull) self.dry_run_result = list(node_uuids) def _RemoveDegradedDisks(self, feedback_fn, disk_abort, instance): """Removes degraded disks and instance. It optionally checks whether disks are degraded. If the disks are degraded, they are removed and the instance is also removed from the configuration. If L{disk_abort} is True, then the disks are considered degraded and removed, and the instance is removed from the configuration. If L{disk_abort} is False, then it first checks whether disks are degraded and, if so, it removes the disks and the instance is removed from the configuration. @type feedback_fn: callable @param feedback_fn: function used send feedback back to the caller @type disk_abort: boolean @param disk_abort: True if disks are degraded, False to first check if disks are degraded @type instance: L{objects.Instance} @param instance: instance containing the disks to check @rtype: NoneType @return: None @raise errors.OpPrereqError: if disks are degraded """ disk_info = self.cfg.GetInstanceDisks(instance.uuid) if disk_abort: pass elif self.op.wait_for_sync: disk_abort = not WaitForSync(self, instance) elif utils.AnyDiskOfType(disk_info, constants.DTS_INT_MIRROR): # make sure the disks are not degraded (still sync-ing is ok) feedback_fn("* checking mirrors status") disk_abort = not WaitForSync(self, instance, oneshot=True) else: disk_abort = False if disk_abort: RemoveDisks(self, instance) for disk_uuid in instance.disks: self.cfg.RemoveInstanceDisk(instance.uuid, disk_uuid) self.cfg.RemoveInstance(instance.uuid) raise errors.OpExecError("There are some degraded disks for" " this instance") def RunOsScripts(self, feedback_fn, iobj): """Run OS scripts If necessary, disks are paused. It handles instance create, import, and remote import. @type feedback_fn: callable @param feedback_fn: function used send feedback back to the caller @type iobj: L{objects.Instance} @param iobj: instance object """ if not iobj.disks: return if self.adopt_disks: return disks = self.cfg.GetInstanceDisks(iobj.uuid) if self.op.mode == constants.INSTANCE_CREATE: os_image = objects.GetOSImage(self.op.osparams) if os_image is None and not self.op.no_install: pause_sync = (not self.op.wait_for_sync and utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR)) if pause_sync: feedback_fn("* pausing disk sync to install instance OS") result = self.rpc.call_blockdev_pause_resume_sync(self.pnode.uuid, (disks, iobj), True) for idx, success in enumerate(result.payload): if not success: logging.warn("pause-sync of instance %s for disk %d failed", self.op.instance_name, idx) feedback_fn("* running the instance OS create scripts...") # FIXME: pass debug option from opcode to backend os_add_result = \ self.rpc.call_instance_os_add(self.pnode.uuid, (iobj, self.op.osparams_secret), False, self.op.debug_level) if pause_sync: feedback_fn("* resuming disk sync") result = self.rpc.call_blockdev_pause_resume_sync(self.pnode.uuid, (disks, iobj), False) for idx, success in enumerate(result.payload): if not success: logging.warn("resume-sync of instance %s for disk %d failed", self.op.instance_name, idx) os_add_result.Raise("Could not add os for instance %s" " on node %s" % (self.op.instance_name, self.pnode.name)) else: if self.op.mode == constants.INSTANCE_IMPORT: feedback_fn("* running the instance OS import scripts...") transfers = [] for idx, image in enumerate(self.src_images): if not image: continue if iobj.os: dst_io = constants.IEIO_SCRIPT dst_ioargs = ((disks[idx], iobj), idx) else: dst_io = constants.IEIO_RAW_DISK dst_ioargs = (disks[idx], iobj) # FIXME: pass debug option from opcode to backend dt = masterd.instance.DiskTransfer("disk/%s" % idx, constants.IEIO_FILE, (image, ), dst_io, dst_ioargs, None) transfers.append(dt) import_result = \ masterd.instance.TransferInstanceData(self, feedback_fn, self.op.src_node_uuid, self.pnode.uuid, self.pnode.secondary_ip, self.op.compress, iobj, transfers) if not compat.all(import_result): self.LogWarning("Some disks for instance %s on node %s were not" " imported successfully" % (self.op.instance_name, self.pnode.name)) rename_from = self._old_instance_name elif self.op.mode == constants.INSTANCE_REMOTE_IMPORT: feedback_fn("* preparing remote import...") # The source cluster will stop the instance before attempting to make # a connection. In some cases stopping an instance can take a long # time, hence the shutdown timeout is added to the connection # timeout. connect_timeout = (constants.RIE_CONNECT_TIMEOUT + self.op.source_shutdown_timeout) timeouts = masterd.instance.ImportExportTimeouts(connect_timeout) assert iobj.primary_node == self.pnode.uuid disk_results = \ masterd.instance.RemoteImport(self, feedback_fn, iobj, self.pnode, self.source_x509_ca, self._cds, self.op.compress, timeouts) if not compat.all(disk_results): # TODO: Should the instance still be started, even if some disks # failed to import (valid for local imports, too)? self.LogWarning("Some disks for instance %s on node %s were not" " imported successfully" % (self.op.instance_name, self.pnode.name)) rename_from = self.source_instance_name else: # also checked in the prereq part raise errors.ProgrammerError("Unknown OS initialization mode '%s'" % self.op.mode) assert iobj.name == self.op.instance_name # Run rename script on newly imported instance if iobj.os: feedback_fn("Running rename script for %s" % self.op.instance_name) result = self.rpc.call_instance_run_rename(self.pnode.uuid, iobj, rename_from, self.op.debug_level) result.Warn("Failed to run rename script for %s on node %s" % (self.op.instance_name, self.pnode.name), self.LogWarning) def GetOsInstallPackageEnvironment(self, instance, script): """Returns the OS scripts environment for the helper VM @type instance: L{objects.Instance} @param instance: instance for which the OS scripts are run @type script: string @param script: script to run (e.g., constants.OS_SCRIPT_CREATE_UNTRUSTED) @rtype: dict of string to string @return: OS scripts environment for the helper VM """ env = {"OS_SCRIPT": script} # We pass only the instance's disks, not the helper VM's disks. if instance.hypervisor == constants.HT_KVM: prefix = "/dev/vd" elif instance.hypervisor in [constants.HT_XEN_PVM, constants.HT_XEN_HVM]: prefix = "/dev/xvd" else: raise errors.OpExecError("Cannot run OS scripts in a virtualized" " environment for hypervisor '%s'" % instance.hypervisor) num_disks = len(self.cfg.GetInstanceDisks(instance.uuid)) for idx, disk_label in enumerate(utils.GetDiskLabels(prefix, num_disks + 1, start=1)): env["DISK_%d_PATH" % idx] = disk_label return env def UpdateInstanceOsInstallPackage(self, feedback_fn, instance, override_env): """Updates the OS parameter 'os-install-package' for an instance. The OS install package is an archive containing an OS definition and a file containing the environment variables needed to run the OS scripts. The OS install package is served by the metadata daemon to the instances, so the OS scripts can run inside the virtualized environment. @type feedback_fn: callable @param feedback_fn: function used send feedback back to the caller @type instance: L{objects.Instance} @param instance: instance for which the OS parameter 'os-install-package' is updated @type override_env: dict of string to string @param override_env: if supplied, it overrides the environment of the export OS scripts archive """ if "os-install-package" in instance.osparams: feedback_fn("Using OS install package '%s'" % instance.osparams["os-install-package"]) else: result = self.rpc.call_os_export(instance.primary_node, instance, override_env) result.Raise("Could not export OS '%s'" % instance.os) instance.osparams["os-install-package"] = result.payload feedback_fn("Created OS install package '%s'" % result.payload) def RunOsScriptsVirtualized(self, feedback_fn, instance): """Runs the OS scripts inside a safe virtualized environment. The virtualized environment reuses the instance and temporarily creates a disk onto which the image of the helper VM is dumped. The temporary disk is used to boot the helper VM. The OS scripts are passed to the helper VM through the metadata daemon and the OS install package. @type feedback_fn: callable @param feedback_fn: function used send feedback back to the caller @type instance: L{objects.Instance} @param instance: instance for which the OS scripts must be run inside the virtualized environment """ install_image = self.cfg.GetInstallImage() if not install_image: raise errors.OpExecError("Cannot create install instance because an" " install image has not been specified") env = self.GetOsInstallPackageEnvironment( instance, constants.OS_SCRIPT_CREATE_UNTRUSTED) self.UpdateInstanceOsInstallPackage(feedback_fn, instance, env) UpdateMetadata(feedback_fn, self.rpc, instance, osparams_private=self.op.osparams_private, osparams_secret=self.op.osparams_secret) RunWithHelperVM(self, instance, install_image, self.op.helper_startup_timeout, self.op.helper_shutdown_timeout, log_prefix="Running OS create script", feedback_fn=feedback_fn) def Exec(self, feedback_fn): """Create and add the instance to the cluster. """ assert not (self.owned_locks(locking.LEVEL_NODE_RES) - self.owned_locks(locking.LEVEL_NODE)), \ "Node locks differ from node resource locks" ht_kind = self.op.hypervisor if ht_kind in constants.HTS_REQ_PORT: network_port = self.cfg.AllocatePort() else: network_port = None if self.op.commit: (instance_uuid, _) = self.cfg.ExpandInstanceName(self.op.instance_name) else: instance_uuid = self.cfg.GenerateUniqueID(self.proc.GetECId()) # This is ugly but we got a chicken-egg problem here # We can only take the group disk parameters, as the instance # has no disks yet (we are generating them right here). nodegroup = self.cfg.GetNodeGroup(self.pnode.group) if self.op.commit: disks = self.cfg.GetInstanceDisks(instance_uuid) CommitDisks(disks) else: disks = GenerateDiskTemplate(self, self.op.disk_template, instance_uuid, self.pnode.uuid, self.secondaries, self.disks, self.instance_file_storage_dir, self.op.file_driver, 0, feedback_fn, self.cfg.GetGroupDiskParams(nodegroup), forthcoming=self.op.forthcoming) if self.op.os_type is None: os_type = "" else: os_type = self.op.os_type iobj = objects.Instance(name=self.op.instance_name, uuid=instance_uuid, os=os_type, primary_node=self.pnode.uuid, nics=self.nics, disks=[], disk_template=self.op.disk_template, disks_active=False, admin_state=constants.ADMINST_DOWN, admin_state_source=constants.ADMIN_SOURCE, network_port=network_port, beparams=self.op.beparams, hvparams=self.op.hvparams, hypervisor=self.op.hypervisor, osparams=self.op.osparams, osparams_private=self.op.osparams_private, forthcoming=self.op.forthcoming, ) if self.op.tags: for tag in self.op.tags: iobj.AddTag(tag) if self.adopt_disks: if self.op.disk_template == constants.DT_PLAIN: # rename LVs to the newly-generated names; we need to construct # 'fake' LV disks with the old data, plus the new unique_id tmp_disks = [objects.Disk.FromDict(v.ToDict()) for v in disks] rename_to = [] for t_dsk, a_dsk in zip(tmp_disks, self.disks): rename_to.append(t_dsk.logical_id) t_dsk.logical_id = (t_dsk.logical_id[0], a_dsk[constants.IDISK_ADOPT]) result = self.rpc.call_blockdev_rename(self.pnode.uuid, list(zip(tmp_disks, rename_to))) result.Raise("Failed to rename adoped LVs") elif self.op.forthcoming: feedback_fn("Instance is forthcoming, not creating disks") else: feedback_fn("* creating instance disks...") try: CreateDisks(self, iobj, disks=disks) except errors.OpExecError: self.LogWarning("Device creation failed") for disk in disks: self.cfg.ReleaseDRBDMinors(disk.uuid) raise feedback_fn("adding instance %s to cluster config" % self.op.instance_name) self.cfg.AddInstance(iobj, self.proc.GetECId(), replace=self.op.commit) feedback_fn("adding disks to cluster config") for disk in disks: self.cfg.AddInstanceDisk(iobj.uuid, disk, replace=self.op.commit) if self.op.forthcoming: feedback_fn("Instance is forthcoming; not creating the actual instance") return self.cfg.GetNodeNames(list(self.cfg.GetInstanceNodes(iobj.uuid))) # re-read the instance from the configuration iobj = self.cfg.GetInstanceInfo(iobj.uuid) if self.op.mode == constants.INSTANCE_IMPORT: # Release unused nodes ReleaseLocks(self, locking.LEVEL_NODE, keep=[self.op.src_node_uuid]) else: # Release all nodes ReleaseLocks(self, locking.LEVEL_NODE) # Wipe disks disk_abort = False if not self.adopt_disks and self.cfg.GetClusterInfo().prealloc_wipe_disks: feedback_fn("* wiping instance disks...") try: WipeDisks(self, iobj) except errors.OpExecError as err: logging.exception("Wiping disks failed") self.LogWarning("Wiping instance disks failed (%s)", err) disk_abort = True self._RemoveDegradedDisks(feedback_fn, disk_abort, iobj) # Image disks os_image = objects.GetOSImage(iobj.osparams) disk_abort = False if not self.adopt_disks and os_image is not None: feedback_fn("* imaging instance disks...") try: ImageDisks(self, iobj, os_image) except errors.OpExecError as err: logging.exception("Imaging disks failed") self.LogWarning("Imaging instance disks failed (%s)", err) disk_abort = True self._RemoveDegradedDisks(feedback_fn, disk_abort, iobj) # instance disks are now active iobj.disks_active = True # Release all node resource locks ReleaseLocks(self, locking.LEVEL_NODE_RES) if iobj.os: result = self.rpc.call_os_diagnose([iobj.primary_node])[iobj.primary_node] result.Raise("Failed to get OS '%s'" % iobj.os) trusted = None for (name, _, _, _, _, _, _, os_trusted) in result.payload: if name == objects.OS.GetName(iobj.os): trusted = os_trusted break if trusted is None: raise errors.OpPrereqError("OS '%s' is not available in node '%s'" % (iobj.os, iobj.primary_node)) elif trusted: self.RunOsScripts(feedback_fn, iobj) else: self.RunOsScriptsVirtualized(feedback_fn, iobj) # Instance is modified by 'RunOsScriptsVirtualized', # therefore, it must be retrieved once again from the # configuration, otherwise there will be a config object # version mismatch. iobj = self.cfg.GetInstanceInfo(iobj.uuid) # Update instance metadata so that it can be reached from the # metadata service. UpdateMetadata(feedback_fn, self.rpc, iobj, osparams_private=self.op.osparams_private, osparams_secret=self.op.osparams_secret) assert not self.owned_locks(locking.LEVEL_NODE_RES) if self.op.start: iobj.admin_state = constants.ADMINST_UP self.cfg.Update(iobj, feedback_fn) logging.info("Starting instance %s on node %s", self.op.instance_name, self.pnode.name) feedback_fn("* starting instance...") result = self.rpc.call_instance_start(self.pnode.uuid, (iobj, None, None), False, self.op.reason) result.Raise("Could not start instance") return self.cfg.GetNodeNames(list(self.cfg.GetInstanceNodes(iobj.uuid))) def PrepareRetry(self, feedback_fn): # A temporary lack of resources can only happen if opportunistic locking # is used. assert self.op.opportunistic_locking logging.info("Opportunistic locking did not suceed, falling back to" " full lock allocation") feedback_fn("* falling back to full lock allocation") self.op.opportunistic_locking = False ganeti-3.1.0~rc2/lib/cmdlib/instance_helpervm.py000064400000000000000000000150441476477700300216700ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Functions for running helper virtual machines to perform tasks on instances. """ import contextlib from ganeti import constants from ganeti import errors from ganeti.utils import retry from ganeti.cmdlib.common import IsInstanceRunning, DetermineImageSize from ganeti.cmdlib.instance_storage import StartInstanceDisks, \ TemporaryDisk, ImageDisks @contextlib.contextmanager def HelperVM(lu, instance, vm_image, startup_timeout, vm_timeout, log_prefix=None, feedback_fn=None): """Runs a given helper VM for a given instance. @type lu: L{LogicalUnit} @param lu: the lu on whose behalf we execute @type instance: L{objects.Instance} @param instance: the instance definition @type vm_image: string @param vm_image: the name of the helper VM image to dump on a temporary disk @type startup_timeout: int @param startup_timeout: how long to wait for the helper VM to start up @type vm_timeout: int @param vm_timeout: how long to wait for the helper VM to finish its work @type log_prefix: string @param log_prefix: a prefix for all log messages @type feedback_fn: function @param feedback_fn: Function used to log progress """ if log_prefix: add_prefix = lambda msg: "%s: %s" % (log_prefix, msg) else: add_prefix = lambda msg: msg if feedback_fn is not None: log_feedback = lambda msg: feedback_fn(add_prefix(msg)) else: log_feedback = lambda _: None try: disk_size = DetermineImageSize(lu, vm_image, instance.primary_node) except errors.OpExecError as err: raise errors.OpExecError("Could not create temporary disk: %s", err) with TemporaryDisk(lu, instance, [(constants.DT_PLAIN, constants.DISK_RDWR, disk_size)], log_feedback): log_feedback("Activating helper VM's temporary disks") StartInstanceDisks(lu, instance, False) log_feedback("Imaging temporary disks with image %s" % (vm_image, )) ImageDisks(lu, instance, vm_image) log_feedback("Starting helper VM") result = lu.rpc.call_instance_start(instance.primary_node, (instance, [], []), False, lu.op.reason) result.Raise(add_prefix("Could not start helper VM with image %s" % (vm_image, ))) # First wait for the instance to start up running_check = lambda: IsInstanceRunning(lu, instance, prereq=False) instance_up = retry.SimpleRetry(True, running_check, 5.0, startup_timeout) if not instance_up: raise errors.OpExecError(add_prefix("Could not boot instance using" " image %s" % (vm_image, ))) log_feedback("Helper VM is up") def cleanup(): log_feedback("Waiting for helper VM to finish") # Then for it to be finished, detected by its shutdown instance_up = retry.SimpleRetry(False, running_check, 20.0, vm_timeout) if instance_up: lu.LogWarning(add_prefix("Helper VM has not finished within the" " timeout; shutting it down forcibly")) return \ lu.rpc.call_instance_shutdown(instance.primary_node, instance, constants.DEFAULT_SHUTDOWN_TIMEOUT, lu.op.reason) else: return None # Run the inner block and handle possible errors try: yield except Exception: # if the cleanup failed for some reason, log it and just re-raise result = cleanup() if result: result.Warn(add_prefix("Could not shut down helper VM with image" " %s within timeout" % (vm_image, ))) log_feedback("Error running helper VM with image %s" % (vm_image, )) raise else: result = cleanup() # if the cleanup failed for some reason, throw an exception if result: result.Raise(add_prefix("Could not shut down helper VM with image %s" " within timeout" % (vm_image, ))) raise errors.OpExecError("Error running helper VM with image %s" % (vm_image, )) log_feedback("Helper VM execution completed") def RunWithHelperVM(lu, instance, vm_image, startup_timeout, vm_timeout, log_prefix=None, feedback_fn=None): """Runs a given helper VM for a given instance. @type lu: L{LogicalUnit} @param lu: the lu on whose behalf we execute @type instance: L{objects.Instance} @param instance: the instance definition @type vm_image: string @param vm_image: the name of the helper VM image to dump on a temporary disk @type startup_timeout: int @param startup_timeout: how long to wait for the helper VM to start up @type vm_timeout: int @param vm_timeout: how long to wait for the helper VM to finish its work @type log_prefix: string @param log_prefix: a prefix for all log messages @type feedback_fn: function @param feedback_fn: Function used to log progress """ with HelperVM(lu, instance, vm_image, startup_timeout, vm_timeout, log_prefix=log_prefix, feedback_fn=feedback_fn): pass ganeti-3.1.0~rc2/lib/cmdlib/instance_migration.py000064400000000000000000001370151476477700300220420ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with instance migration an failover.""" import logging import time from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import hypervisor from ganeti.masterd import iallocator from ganeti import utils from ganeti.cmdlib.base import LogicalUnit, Tasklet from ganeti.cmdlib.common import ExpandInstanceUuidAndName, \ CheckIAllocatorOrNode, ExpandNodeUuidAndName from ganeti.cmdlib.instance_storage import CheckDiskConsistency, \ ExpandCheckDisks, ShutdownInstanceDisks, AssembleInstanceDisks from ganeti.cmdlib.instance_utils import BuildInstanceHookEnvByObject, \ CheckTargetNodeIPolicy, ReleaseLocks, CheckNodeNotDrained, \ CopyLockList, CheckNodeFreeMemory, CheckInstanceBridgesExist import ganeti.masterd.instance def _ExpandNamesForMigration(lu): """Expands names for use with L{TLMigrateInstance}. @type lu: L{LogicalUnit} """ if lu.op.target_node is not None: (lu.op.target_node_uuid, lu.op.target_node) = \ ExpandNodeUuidAndName(lu.cfg, lu.op.target_node_uuid, lu.op.target_node) lu.needed_locks[locking.LEVEL_NODE] = [] lu.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE lu.dont_collate_locks[locking.LEVEL_NODE] = True lu.needed_locks[locking.LEVEL_NODE_RES] = [] lu.recalculate_locks[locking.LEVEL_NODE_RES] = constants.LOCKS_REPLACE lu.dont_collate_locks[locking.LEVEL_NODE_RES] = True def _DeclareLocksForMigration(lu, level): """Declares locks for L{TLMigrateInstance}. @type lu: L{LogicalUnit} @param level: Lock level """ if level == locking.LEVEL_NODE: assert lu.op.instance_name in lu.owned_locks(locking.LEVEL_INSTANCE) instance = lu.cfg.GetInstanceInfo(lu.op.instance_uuid) disks = lu.cfg.GetInstanceDisks(instance.uuid) if utils.AnyDiskOfType(disks, constants.DTS_EXT_MIRROR): if lu.op.target_node is None: lu.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET else: lu.needed_locks[locking.LEVEL_NODE] = [instance.primary_node, lu.op.target_node_uuid] else: lu._LockInstancesNodes() # pylint: disable=W0212 assert (lu.needed_locks[locking.LEVEL_NODE] or lu.needed_locks[locking.LEVEL_NODE] is locking.ALL_SET) elif level == locking.LEVEL_NODE_RES: # Copy node locks lu.needed_locks[locking.LEVEL_NODE_RES] = \ CopyLockList(lu.needed_locks[locking.LEVEL_NODE]) class LUInstanceFailover(LogicalUnit): """Failover an instance. This is migration by shutting the instance down, but with the disks of the instance already available on the new node. See also: L{LUInstanceMove} for moving an instance by copying the data. L{LUInstanceMigrate} for the live migration of an instance (no shutdown required). """ HPATH = "instance-failover" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def CheckArguments(self): """Check the arguments. """ self.iallocator = getattr(self.op, "iallocator", None) self.target_node = getattr(self.op, "target_node", None) def ExpandNames(self): self._ExpandAndLockInstance(allow_forthcoming=True) _ExpandNamesForMigration(self) self._migrater = \ TLMigrateInstance(self, self.op.instance_uuid, self.op.instance_name, self.op.cleanup, True, False, self.op.ignore_consistency, True, self.op.shutdown_timeout, self.op.ignore_ipolicy, True) self.tasklets = [self._migrater] def DeclareLocks(self, level): _DeclareLocksForMigration(self, level) def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ instance = self._migrater.instance source_node_uuid = instance.primary_node target_node_uuid = self._migrater.target_node_uuid env = { "IGNORE_CONSISTENCY": self.op.ignore_consistency, "SHUTDOWN_TIMEOUT": self.op.shutdown_timeout, "OLD_PRIMARY": self.cfg.GetNodeName(source_node_uuid), "NEW_PRIMARY": self.cfg.GetNodeName(target_node_uuid), "FAILOVER_CLEANUP": self.op.cleanup, } disks = self.cfg.GetInstanceDisks(instance.uuid) if utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR): secondary_nodes = self.cfg.GetInstanceSecondaryNodes(instance.uuid) env["OLD_SECONDARY"] = self.cfg.GetNodeName(secondary_nodes[0]) env["NEW_SECONDARY"] = self.cfg.GetNodeName(source_node_uuid) else: env["OLD_SECONDARY"] = env["NEW_SECONDARY"] = "" env.update(BuildInstanceHookEnvByObject(self, instance)) return env def BuildHooksNodes(self): """Build hooks nodes. """ instance = self._migrater.instance secondary_nodes = self.cfg.GetInstanceSecondaryNodes(instance.uuid) nl = [self.cfg.GetMasterNode()] + list(secondary_nodes) nl.append(self._migrater.target_node_uuid) return (nl, nl + [instance.primary_node]) class LUInstanceMigrate(LogicalUnit): """Migrate an instance. This is migration without shutting down (live migration) and the disks are already available on the new node. See also: L{LUInstanceMove} for moving an instance by copying the data. L{LUInstanceFailover} for the migration of an instance where a shutdown is required. """ HPATH = "instance-migrate" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def ExpandNames(self): self._ExpandAndLockInstance() _ExpandNamesForMigration(self) self._migrater = \ TLMigrateInstance(self, self.op.instance_uuid, self.op.instance_name, self.op.cleanup, False, self.op.allow_failover, False, self.op.allow_runtime_changes, constants.DEFAULT_SHUTDOWN_TIMEOUT, self.op.ignore_ipolicy, self.op.ignore_hvversions) self.tasklets = [self._migrater] def DeclareLocks(self, level): _DeclareLocksForMigration(self, level) def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ instance = self._migrater.instance source_node_uuid = instance.primary_node target_node_uuid = self._migrater.target_node_uuid env = BuildInstanceHookEnvByObject(self, instance) env.update({ "MIGRATE_LIVE": self._migrater.live, "MIGRATE_CLEANUP": self.op.cleanup, "OLD_PRIMARY": self.cfg.GetNodeName(source_node_uuid), "NEW_PRIMARY": self.cfg.GetNodeName(target_node_uuid), "ALLOW_RUNTIME_CHANGES": self.op.allow_runtime_changes, }) disks = self.cfg.GetInstanceDisks(instance.uuid) if utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR): secondary_nodes = self.cfg.GetInstanceSecondaryNodes(instance.uuid) env["OLD_SECONDARY"] = self.cfg.GetNodeName(secondary_nodes[0]) env["NEW_SECONDARY"] = self.cfg.GetNodeName(source_node_uuid) else: env["OLD_SECONDARY"] = env["NEW_SECONDARY"] = "" return env def BuildHooksNodes(self): """Build hooks nodes. """ instance = self._migrater.instance secondary_nodes = self.cfg.GetInstanceSecondaryNodes(instance.uuid) snode_uuids = list(secondary_nodes) nl = [self.cfg.GetMasterNode(), instance.primary_node] + snode_uuids nl.append(self._migrater.target_node_uuid) return (nl, nl) class TLMigrateInstance(Tasklet): """Tasklet class for instance migration. @type live: boolean @ivar live: whether the migration will be done live or non-live; this variable is initalized only after CheckPrereq has run @type cleanup: boolean @ivar cleanup: Wheater we cleanup from a failed migration @type iallocator: string @ivar iallocator: The iallocator used to determine target_node @type target_node_uuid: string @ivar target_node_uuid: If given, the target node UUID to reallocate the instance to @type failover: boolean @ivar failover: Whether operation results in failover or migration @type fallback: boolean @ivar fallback: Whether fallback to failover is allowed if migration not possible @type ignore_consistency: boolean @ivar ignore_consistency: Wheter we should ignore consistency between source and target node @type shutdown_timeout: int @ivar shutdown_timeout: In case of failover timeout of the shutdown @type ignore_ipolicy: bool @ivar ignore_ipolicy: If true, we can ignore instance policy when migrating @type ignore_hvversions: bool @ivar ignore_hvversions: If true, accept incompatible hypervisor versions """ # Constants _MIGRATION_POLL_INTERVAL = 1 # seconds _MIGRATION_FEEDBACK_INTERVAL = 10 # seconds def __init__(self, lu, instance_uuid, instance_name, cleanup, failover, fallback, ignore_consistency, allow_runtime_changes, shutdown_timeout, ignore_ipolicy, ignore_hvversions): """Initializes this class. """ Tasklet.__init__(self, lu) # Parameters self.instance_uuid = instance_uuid self.instance_name = instance_name self.cleanup = cleanup self.live = False # will be overridden later self.failover = failover self.fallback = fallback self.ignore_consistency = ignore_consistency self.shutdown_timeout = shutdown_timeout self.ignore_ipolicy = ignore_ipolicy self.allow_runtime_changes = allow_runtime_changes self.ignore_hvversions = ignore_hvversions def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ (self.instance_uuid, self.instance_name) = \ ExpandInstanceUuidAndName(self.lu.cfg, self.instance_uuid, self.instance_name) self.instance = self.cfg.GetInstanceInfo(self.instance_uuid) assert self.instance is not None cluster = self.cfg.GetClusterInfo() if (not self.cleanup and not self.instance.admin_state == constants.ADMINST_UP and not self.failover and self.fallback): self.lu.LogInfo("Instance is marked down or offline, fallback allowed," " switching to failover") self.failover = True disks = self.cfg.GetInstanceDisks(self.instance.uuid) if not utils.AllDiskOfType(disks, constants.DTS_MIRRORED): if self.failover: text = "failovers" else: text = "migrations" invalid_disks = set(d.dev_type for d in disks if d.dev_type not in constants.DTS_MIRRORED) raise errors.OpPrereqError("Instance's disk layout '%s' does not allow" " %s" % (utils.CommaJoin(invalid_disks), text), errors.ECODE_STATE) # TODO allow heterogeneous disk types if all are mirrored in some way. if utils.AllDiskOfType(disks, constants.DTS_EXT_MIRROR): CheckIAllocatorOrNode(self.lu, "iallocator", "target_node") if self.lu.op.iallocator: self._RunAllocator() else: # We set set self.target_node_uuid as it is required by # BuildHooksEnv self.target_node_uuid = self.lu.op.target_node_uuid # Check that the target node is correct in terms of instance policy nodeinfo = self.cfg.GetNodeInfo(self.target_node_uuid) group_info = self.cfg.GetNodeGroup(nodeinfo.group) ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(cluster, group_info) CheckTargetNodeIPolicy(self.lu, ipolicy, self.instance, nodeinfo, self.cfg, ignore=self.ignore_ipolicy) # self.target_node is already populated, either directly or by the # iallocator run target_node_uuid = self.target_node_uuid if self.target_node_uuid == self.instance.primary_node: raise errors.OpPrereqError( "Cannot migrate instance %s to its primary (%s)" % (self.instance.name, self.cfg.GetNodeName(self.instance.primary_node)), errors.ECODE_STATE) if len(self.lu.tasklets) == 1: # It is safe to release locks only when we're the only tasklet # in the LU ReleaseLocks(self.lu, locking.LEVEL_NODE, keep=[self.instance.primary_node, self.target_node_uuid]) elif utils.AllDiskOfType(disks, constants.DTS_INT_MIRROR): templates = [d.dev_type for d in disks] secondary_node_uuids = \ self.cfg.GetInstanceSecondaryNodes(self.instance.uuid) if not secondary_node_uuids: raise errors.ConfigurationError("No secondary node but using" " %s disk types" % utils.CommaJoin(set(templates))) self.target_node_uuid = target_node_uuid = secondary_node_uuids[0] if self.lu.op.iallocator or \ (self.lu.op.target_node_uuid and self.lu.op.target_node_uuid != target_node_uuid): if self.failover: text = "failed over" else: text = "migrated" raise errors.OpPrereqError("Instances with disk types %s cannot" " be %s to arbitrary nodes" " (neither an iallocator nor a target" " node can be passed)" % (utils.CommaJoin(set(templates)), text), errors.ECODE_INVAL) nodeinfo = self.cfg.GetNodeInfo(target_node_uuid) group_info = self.cfg.GetNodeGroup(nodeinfo.group) ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(cluster, group_info) CheckTargetNodeIPolicy(self.lu, ipolicy, self.instance, nodeinfo, self.cfg, ignore=self.ignore_ipolicy) else: raise errors.OpPrereqError("Instance mixes internal and external " "mirroring. This is not currently supported.") i_be = cluster.FillBE(self.instance) # check memory requirements on the secondary node if (not self.cleanup and (not self.failover or self.instance.admin_state == constants.ADMINST_UP)): self.tgt_free_mem = CheckNodeFreeMemory( self.lu, target_node_uuid, "migrating instance %s" % self.instance.name, i_be[constants.BE_MINMEM], self.instance.hypervisor, self.cfg.GetClusterInfo().hvparams[self.instance.hypervisor]) else: self.lu.LogInfo("Not checking memory on the secondary node as" " instance will not be started") # check if failover must be forced instead of migration if (not self.cleanup and not self.failover and i_be[constants.BE_ALWAYS_FAILOVER]): self.lu.LogInfo("Instance configured to always failover; fallback" " to failover") self.failover = True # check bridge existance CheckInstanceBridgesExist(self.lu, self.instance, node_uuid=target_node_uuid) if not self.cleanup: CheckNodeNotDrained(self.lu, target_node_uuid) if not self.failover: result = self.rpc.call_instance_migratable(self.instance.primary_node, self.instance) if result.fail_msg and self.fallback: self.lu.LogInfo("Can't migrate, instance offline, fallback to" " failover") self.failover = True else: result.Raise("Can't migrate, please use failover", prereq=True, ecode=errors.ECODE_STATE) assert not (self.failover and self.cleanup) if not self.failover: if self.lu.op.live is not None and self.lu.op.mode is not None: raise errors.OpPrereqError("Only one of the 'live' and 'mode'" " parameters are accepted", errors.ECODE_INVAL) if self.lu.op.live is not None: if self.lu.op.live: self.lu.op.mode = constants.HT_MIGRATION_LIVE else: self.lu.op.mode = constants.HT_MIGRATION_NONLIVE # reset the 'live' parameter to None so that repeated # invocations of CheckPrereq do not raise an exception self.lu.op.live = None elif self.lu.op.mode is None: # read the default value from the hypervisor i_hv = cluster.FillHV(self.instance, skip_globals=False) self.lu.op.mode = i_hv[constants.HV_MIGRATION_MODE] self.live = self.lu.op.mode == constants.HT_MIGRATION_LIVE else: # Failover is never live self.live = False if not (self.failover or self.cleanup): remote_info = self.rpc.call_instance_info( self.instance.primary_node, self.instance.name, self.instance.hypervisor, cluster.hvparams[self.instance.hypervisor]) remote_info.Raise("Error checking instance on node %s" % self.cfg.GetNodeName(self.instance.primary_node), prereq=True) instance_running = bool(remote_info.payload) if instance_running: self.current_mem = int(remote_info.payload["memory"]) def _RunAllocator(self): """Run the allocator based on input opcode. """ # FIXME: add a self.ignore_ipolicy option req = iallocator.IAReqRelocate( inst_uuid=self.instance_uuid, relocate_from_node_uuids=[self.instance.primary_node]) ial = iallocator.IAllocator(self.cfg, self.rpc, req) ial.Run(self.lu.op.iallocator) if not ial.success: raise errors.OpPrereqError("Can't compute nodes using" " iallocator '%s': %s" % (self.lu.op.iallocator, ial.info), errors.ECODE_NORES) self.target_node_uuid = self.cfg.GetNodeInfoByName( ial.result[0]).uuid # pylint: disable=E1136 self.lu.LogInfo("Selected nodes for instance %s via iallocator %s: %s", self.instance_name, self.lu.op.iallocator, utils.CommaJoin(ial.result)) def _WaitUntilSync(self): """Poll with custom rpc for disk sync. This uses our own step-based rpc call. """ self.feedback_fn("* wait until resync is done") all_done = False disks = self.cfg.GetInstanceDisks(self.instance.uuid) while not all_done: all_done = True result = self.rpc.call_drbd_wait_sync(self.all_node_uuids, (disks, self.instance)) min_percent = 100 for node_uuid, nres in result.items(): nres.Raise("Cannot resync disks on node %s" % self.cfg.GetNodeName(node_uuid)) node_done, node_percent = nres.payload all_done = all_done and node_done if node_percent is not None: min_percent = min(min_percent, node_percent) if not all_done: if min_percent < 100: self.feedback_fn(" - progress: %.1f%%" % min_percent) time.sleep(2) def _OpenInstanceDisks(self, node_uuid, exclusive): """Open instance disks. """ if exclusive: mode = "in exclusive mode" else: mode = "in shared mode" node_name = self.cfg.GetNodeName(node_uuid) self.feedback_fn("* opening instance disks on node %s %s" % (node_name, mode)) disks = self.cfg.GetInstanceDisks(self.instance.uuid) result = self.rpc.call_blockdev_open(node_uuid, self.instance.name, (disks, self.instance), exclusive) result.Raise("Cannot open instance disks on node %s" % node_name) def _CloseInstanceDisks(self, node_uuid): """Close instance disks. """ node_name = self.cfg.GetNodeName(node_uuid) self.feedback_fn("* closing instance disks on node %s" % node_name) disks = self.cfg.GetInstanceDisks(self.instance.uuid) result = self.rpc.call_blockdev_close(node_uuid, self.instance.name, (disks, self.instance)) msg = result.fail_msg if msg: if result.offline or self.ignore_consistency: self.lu.LogWarning("Could not close instance disks on node %s," " proceeding anyway" % node_name) else: raise errors.OpExecError("Cannot close instance disks on node %s: %s" % (node_name, msg)) def _GoStandalone(self): """Disconnect from the network. """ self.feedback_fn("* changing into standalone mode") disks = self.cfg.GetInstanceDisks(self.instance.uuid) result = self.rpc.call_drbd_disconnect_net( self.all_node_uuids, (disks, self.instance)) for node_uuid, nres in result.items(): nres.Raise("Cannot disconnect disks node %s" % self.cfg.GetNodeName(node_uuid)) def _GoReconnect(self, multimaster): """Reconnect to the network. """ if multimaster: msg = "dual-master" else: msg = "single-master" self.feedback_fn("* changing disks into %s mode" % msg) disks = self.cfg.GetInstanceDisks(self.instance.uuid) result = self.rpc.call_drbd_attach_net(self.all_node_uuids, (disks, self.instance), multimaster) for node_uuid, nres in result.items(): nres.Raise("Cannot change disks config on node %s" % self.cfg.GetNodeName(node_uuid)) def _FindInstanceLocations(self, name): """Returns a list of nodes that have the given instance running @type name: string @param name: instance name string to search for @return: list of strings, node uuids """ self.feedback_fn("* checking where the instance actually runs (if this" " hangs, the hypervisor might be in a bad state)") cluster_hvparams = self.cfg.GetClusterInfo().hvparams online_node_uuids = self.cfg.GetOnlineNodeList() instance_list = self.rpc.call_instance_list( online_node_uuids, [self.instance.hypervisor], cluster_hvparams) # Verify each result and raise an exception if failed for node_uuid, result in instance_list.items(): result.Raise("Can't contact node %s" % self.cfg.GetNodeName(node_uuid)) # Xen renames the instance during migration, unfortunately we don't have # a nicer way of identifying that it's the same instance. This is an awful # leaking abstraction. # # xl: (in tools/libxl/xl_cmdimpl.c migrate_domain() & migrate_receive()) # source dom name target dom name # during copy: $DOM $DOM--incoming # finalize migrate: $DOM--migratedaway $DOM # finished: $DOM variants = [name, name + '--incoming', name + '--migratedaway'] node_uuids = [node for node, data in instance_list.items() if any(var in data.payload for var in variants)] self.feedback_fn("* instance running on: %s" % ','.join( self.cfg.GetNodeName(uuid) for uuid in node_uuids)) return node_uuids def _ExecCleanup(self): """Try to cleanup after a failed migration. The cleanup is done by: - check that the instance is running only on one node - try 'aborting' migration if it is running on two nodes - update the config if needed - change disks on its secondary node to secondary - wait until disks are fully synchronized - disconnect from the network - change disks into single-master mode - wait again until disks are fully synchronized """ instance_locations = self._FindInstanceLocations(self.instance.name) runningon_source = self.source_node_uuid in instance_locations runningon_target = self.target_node_uuid in instance_locations if runningon_source and runningon_target: # If we have an instance on both the source and the destination, we know # that instance migration was interrupted in the middle, we can try to # do effectively the same as when aborting an interrupted migration. self.feedback_fn("Trying to cleanup after failed migration") result = self.rpc.call_migration_info( self.source_node_uuid, self.instance) if result.fail_msg: raise errors.OpExecError( "Failed fetching source migration information from %s: %s" % (self.cfg.GetNodeName(self.source_node_uuid), result.fail_msg)) self.migration_info = result.payload abort_results = self._AbortMigration() if abort_results[0].fail_msg or abort_results[1].fail_msg: raise errors.OpExecError( "Instance migration cleanup failed: %s" % ','.join([ abort_results[0].fail_msg, abort_results[1].fail_msg])) # AbortMigration() should have fixed instance locations, so query again instance_locations = self._FindInstanceLocations(self.instance.name) runningon_source = self.source_node_uuid in instance_locations runningon_target = self.target_node_uuid in instance_locations # Abort didn't work, manual intervention required if runningon_source and runningon_target: raise errors.OpExecError("Instance seems to be running on two nodes," " or the hypervisor is confused; you will have" " to ensure manually that it runs only on one" " and restart this operation") if not (runningon_source or runningon_target): if len(instance_locations) == 1: # The instance is running on a differrent node than expected, let's # adopt it as if it was running on the secondary self.target_node_uuid = instance_locations[0] self.feedback_fn("* instance running on unexpected node (%s)," " updating as the new secondary" % self.cfg.GetNodeName(self.target_node_uuid)) runningon_target = True else: raise errors.OpExecError("Instance does not seem to be running at all;" " in this case it's safer to repair by" " running 'gnt-instance stop' to ensure disk" " shutdown, and then restarting it") if runningon_target: # the migration has actually succeeded, we need to update the config self.feedback_fn("* instance running on secondary node (%s)," " updating config" % self.cfg.GetNodeName(self.target_node_uuid)) self.cfg.SetInstancePrimaryNode(self.instance.uuid, self.target_node_uuid) demoted_node_uuid = self.source_node_uuid else: self.feedback_fn("* instance confirmed to be running on its" " primary node (%s)" % self.cfg.GetNodeName(self.source_node_uuid)) demoted_node_uuid = self.target_node_uuid disks = self.cfg.GetInstanceDisks(self.instance.uuid) # TODO: Cleanup code duplication of _RevertDiskStatus() self._CloseInstanceDisks(demoted_node_uuid) if utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR): try: self._WaitUntilSync() except errors.OpExecError: # we ignore here errors, since if the device is standalone, it # won't be able to sync pass self._GoStandalone() self._GoReconnect(False) self._WaitUntilSync() elif utils.AnyDiskOfType(disks, constants.DTS_EXT_MIRROR): self._OpenInstanceDisks(self.instance.primary_node, True) self.feedback_fn("* done") def _RevertDiskStatus(self): """Try to revert the disk status after a failed migration. """ disks = self.cfg.GetInstanceDisks(self.instance.uuid) self._CloseInstanceDisks(self.target_node_uuid) unmap_types = (constants.DT_RBD, constants.DT_EXT) if utils.AnyDiskOfType(disks, unmap_types): # If the instance's disk template is `rbd' or `ext' and there was an # unsuccessful migration, unmap the device from the target node. unmap_disks = [d for d in disks if d.dev_type in unmap_types] disks = ExpandCheckDisks(unmap_disks, unmap_disks) self.feedback_fn("* unmapping instance's disks %s from %s" % (utils.CommaJoin(d.name for d in unmap_disks), self.cfg.GetNodeName(self.target_node_uuid))) for disk in disks: result = self.rpc.call_blockdev_shutdown(self.target_node_uuid, (disk, self.instance)) msg = result.fail_msg if msg: logging.error("Migration failed and I couldn't unmap the block device" " %s on target node %s: %s", disk.iv_name, self.cfg.GetNodeName(self.target_node_uuid), msg) logging.error("You need to unmap the device %s manually on %s", disk.iv_name, self.cfg.GetNodeName(self.target_node_uuid)) if utils.AllDiskOfType(disks, constants.DTS_EXT_MIRROR): self._OpenInstanceDisks(self.source_node_uuid, True) return try: self._GoStandalone() self._GoReconnect(False) self._WaitUntilSync() except errors.OpExecError as err: self.lu.LogWarning("Migration failed and I can't reconnect the drives," " please try to recover the instance manually;" " error '%s'" % str(err)) def _AbortMigration(self): """Call the hypervisor code to abort a started migration. @return: tuple of rpc call results """ src_result = self.rpc.call_instance_finalize_migration_dst( self.target_node_uuid, self.instance, self.migration_info, False) abort_msg = src_result.fail_msg if abort_msg: logging.error("Aborting migration failed on target node %s: %s", self.cfg.GetNodeName(self.target_node_uuid), abort_msg) # Don't raise an exception here, as we stil have to try to revert the # disk status, even if this step failed. dst_result = self.rpc.call_instance_finalize_migration_src( self.source_node_uuid, self.instance, False, self.live) abort_msg = dst_result.fail_msg if abort_msg: logging.error("Aborting migration failed on source node %s: %s", self.cfg.GetNodeName(self.source_node_uuid), abort_msg) return src_result, dst_result def _ExecMigration(self): """Migrate an instance. The migrate is done by: - change the disks into dual-master mode - wait until disks are fully synchronized again - migrate the instance - change disks on the new secondary node (the old primary) to secondary - wait until disks are fully synchronized - change disks into single-master mode """ # Check for hypervisor version mismatch and warn the user. hvspecs = [(self.instance.hypervisor, self.cfg.GetClusterInfo().hvparams[self.instance.hypervisor])] nodeinfo = self.rpc.call_node_info( [self.source_node_uuid, self.target_node_uuid], None, hvspecs) for ninfo in nodeinfo.values(): ninfo.Raise("Unable to retrieve node information from node '%s'" % ninfo.node) (_, _, (src_info, )) = nodeinfo[self.source_node_uuid].payload (_, _, (dst_info, )) = nodeinfo[self.target_node_uuid].payload if ((constants.HV_NODEINFO_KEY_VERSION in src_info) and (constants.HV_NODEINFO_KEY_VERSION in dst_info)): src_version = src_info[constants.HV_NODEINFO_KEY_VERSION] dst_version = dst_info[constants.HV_NODEINFO_KEY_VERSION] if src_version != dst_version: self.feedback_fn("* warning: hypervisor version mismatch between" " source (%s) and target (%s) node" % (src_version, dst_version)) hv = hypervisor.GetHypervisorClass(self.instance.hypervisor) if hv.VersionsSafeForMigration(src_version, dst_version): self.feedback_fn(" migrating from hypervisor version %s to %s should" " be safe" % (src_version, dst_version)) else: self.feedback_fn(" migrating from hypervisor version %s to %s is" " likely unsupported" % (src_version, dst_version)) if self.ignore_hvversions: self.feedback_fn(" continuing anyway (told to ignore version" " mismatch)") else: raise errors.OpExecError("Unsupported migration between hypervisor" " versions (%s to %s)" % (src_version, dst_version)) self.feedback_fn("* checking disk consistency between source and target") for (idx, dev) in enumerate(self.cfg.GetInstanceDisks(self.instance.uuid)): if not CheckDiskConsistency(self.lu, self.instance, dev, self.target_node_uuid, False): raise errors.OpExecError("Disk %s is degraded or not fully" " synchronized on target node," " aborting migration" % idx) if self.current_mem > self.tgt_free_mem: if not self.allow_runtime_changes: raise errors.OpExecError("Memory ballooning not allowed and not enough" " free memory to fit instance %s on target" " node %s (have %dMB, need %dMB)" % (self.instance.name, self.cfg.GetNodeName(self.target_node_uuid), self.tgt_free_mem, self.current_mem)) self.feedback_fn("* setting instance memory to %s" % self.tgt_free_mem) rpcres = self.rpc.call_instance_balloon_memory(self.instance.primary_node, self.instance, self.tgt_free_mem) rpcres.Raise("Cannot modify instance runtime memory") # First get the migration information from the remote node result = self.rpc.call_migration_info(self.source_node_uuid, self.instance) msg = result.fail_msg if msg: log_err = ("Failed fetching source migration information from %s: %s" % (self.cfg.GetNodeName(self.source_node_uuid), msg)) logging.error(log_err) raise errors.OpExecError(log_err) self.migration_info = migration_info = result.payload disks = self.cfg.GetInstanceDisks(self.instance.uuid) self._CloseInstanceDisks(self.target_node_uuid) if utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR): # Then switch the disks to master/master mode self._GoStandalone() self._GoReconnect(True) self._WaitUntilSync() self._OpenInstanceDisks(self.source_node_uuid, False) self._OpenInstanceDisks(self.target_node_uuid, False) self.feedback_fn("* preparing %s to accept the instance" % self.cfg.GetNodeName(self.target_node_uuid)) result = self.rpc.call_accept_instance(self.target_node_uuid, self.instance, migration_info, self.nodes_ip[self.target_node_uuid]) msg = result.fail_msg if msg: logging.error("Instance pre-migration failed, trying to revert" " disk status: %s", msg) self.feedback_fn("Pre-migration failed, aborting") self._AbortMigration() self._RevertDiskStatus() raise errors.OpExecError("Could not pre-migrate instance %s: %s" % (self.instance.name, msg)) self.feedback_fn("* migrating instance to %s" % self.cfg.GetNodeName(self.target_node_uuid)) cluster = self.cfg.GetClusterInfo() result = self.rpc.call_instance_migrate( self.source_node_uuid, cluster.cluster_name, self.instance, self.nodes_ip[self.target_node_uuid], self.live) msg = result.fail_msg if msg: logging.error("Instance migration failed, trying to revert" " disk status: %s", msg) self.feedback_fn("Migration failed, aborting") self._AbortMigration() self._RevertDiskStatus() raise errors.OpExecError("Could not migrate instance %s: %s" % (self.instance.name, msg)) self.feedback_fn("* starting memory transfer") last_feedback = time.time() while True: result = self.rpc.call_instance_get_migration_status( self.source_node_uuid, self.instance) msg = result.fail_msg ms = result.payload # MigrationStatus instance if msg or (ms.status in constants.HV_MIGRATION_FAILED_STATUSES): logging.error("Instance migration failed, trying to revert" " disk status: %s", msg) self.feedback_fn("Migration failed, aborting") self._AbortMigration() self._RevertDiskStatus() if not msg: msg = "hypervisor returned failure" raise errors.OpExecError("Could not migrate instance %s: %s" % (self.instance.name, msg)) if ms.postcopy_status == constants.HV_KVM_MIGRATION_POSTCOPY_ACTIVE: self.feedback_fn("* memory transfer has switched to postcopy") if ms.status not in constants.HV_KVM_MIGRATION_ACTIVE_STATUSES: self.feedback_fn("* memory transfer complete") break if (utils.TimeoutExpired(last_feedback, self._MIGRATION_FEEDBACK_INTERVAL) and ms.transferred_ram is not None): mem_progress = 100 * float(ms.transferred_ram) / float(ms.total_ram) self.feedback_fn("* memory transfer progress: %.2f %%" % mem_progress) last_feedback = time.time() time.sleep(self._MIGRATION_POLL_INTERVAL) # Always call finalize on both source and target, they should compose # a single operation, consisting of (potentially) parallel steps, that # should be always attempted/retried together (like in _AbortMigration) # without setting any expecetations in what order they execute. result_src = self.rpc.call_instance_finalize_migration_src( self.source_node_uuid, self.instance, True, self.live) result_dst = self.rpc.call_instance_finalize_migration_dst( self.target_node_uuid, self.instance, migration_info, True) err_msg = [] if result_src.fail_msg: logging.error("Instance migration succeeded, but finalization failed" " on the source node: %s", result_src.fail_msg) err_msg.append(self.cfg.GetNodeName(self.source_node_uuid) + ': ' + result_src.fail_msg) if result_dst.fail_msg: logging.error("Instance migration succeeded, but finalization failed" " on the target node: %s", result_dst.fail_msg) err_msg.append(self.cfg.GetNodeName(self.target_node_uuid) + ': ' + result_dst.fail_msg) if err_msg: raise errors.OpExecError( "Could not finalize instance migration: %s" % ' '.join(err_msg)) # Update instance location only after finalize completed. This way, if # either finalize fails, the config still stores the old primary location, # so we can know which instance to delete if we need to (manually) clean up. self.cfg.SetInstancePrimaryNode(self.instance.uuid, self.target_node_uuid) self.instance = self.cfg.GetInstanceInfo(self.instance_uuid) self._CloseInstanceDisks(self.source_node_uuid) disks = self.cfg.GetInstanceDisks(self.instance_uuid) if utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR): self._WaitUntilSync() self._GoStandalone() self._GoReconnect(False) self._WaitUntilSync() elif utils.AnyDiskOfType(disks, constants.DTS_EXT_MIRROR): self._OpenInstanceDisks(self.target_node_uuid, True) # If the instance's disk template is `rbd' or `ext' and there was a # successful migration, unmap the device from the source node. unmap_types = (constants.DT_RBD, constants.DT_EXT) if utils.AnyDiskOfType(disks, unmap_types): unmap_disks = [d for d in disks if d.dev_type in unmap_types] disks = ExpandCheckDisks(unmap_disks, unmap_disks) self.feedback_fn("* unmapping instance's disks %s from %s" % (utils.CommaJoin(d.name for d in unmap_disks), self.cfg.GetNodeName(self.source_node_uuid))) for disk in disks: result = self.rpc.call_blockdev_shutdown(self.source_node_uuid, (disk, self.instance)) msg = result.fail_msg if msg: logging.error("Migration was successful, but couldn't unmap the" " block device %s on source node %s: %s", disk.iv_name, self.cfg.GetNodeName(self.source_node_uuid), msg) logging.error("You need to unmap the device %s manually on %s", disk.iv_name, self.cfg.GetNodeName(self.source_node_uuid)) self.feedback_fn("* done") def _ExecFailover(self): """Failover an instance. The failover is done by shutting it down on its present node and starting it on the secondary. """ if self.instance.forthcoming: self.feedback_fn("Instance is forthcoming, just updating the" " configuration") self.cfg.SetInstancePrimaryNode(self.instance.uuid, self.target_node_uuid) return primary_node = self.cfg.GetNodeInfo(self.instance.primary_node) source_node_uuid = self.instance.primary_node if self.instance.disks_active: self.feedback_fn("* checking disk consistency between source and target") inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) for (idx, dev) in enumerate(inst_disks): # for drbd, these are drbd over lvm if not CheckDiskConsistency(self.lu, self.instance, dev, self.target_node_uuid, False): if primary_node.offline: self.feedback_fn("Node %s is offline, ignoring degraded disk %s on" " target node %s" % (primary_node.name, idx, self.cfg.GetNodeName(self.target_node_uuid))) elif not self.ignore_consistency: raise errors.OpExecError("Disk %s is degraded on target node," " aborting failover" % idx) else: self.feedback_fn("* not checking disk consistency as instance is not" " running") self.feedback_fn("* shutting down instance on source node") logging.info("Shutting down instance %s on node %s", self.instance.name, self.cfg.GetNodeName(source_node_uuid)) result = self.rpc.call_instance_shutdown(source_node_uuid, self.instance, self.shutdown_timeout, self.lu.op.reason) msg = result.fail_msg if msg: if self.ignore_consistency or primary_node.offline: self.lu.LogWarning("Could not shutdown instance %s on node %s," " proceeding anyway; please make sure node" " %s is down; error details: %s", self.instance.name, self.cfg.GetNodeName(source_node_uuid), self.cfg.GetNodeName(source_node_uuid), msg) else: raise errors.OpExecError("Could not shutdown instance %s on" " node %s: %s" % (self.instance.name, self.cfg.GetNodeName(source_node_uuid), msg)) disk_template = self.cfg.GetInstanceDiskTemplate(self.instance.uuid) if disk_template in constants.DTS_EXT_MIRROR: self._CloseInstanceDisks(source_node_uuid) self.feedback_fn("* deactivating the instance's disks on source node") if not ShutdownInstanceDisks(self.lu, self.instance, ignore_primary=True): raise errors.OpExecError("Can't shut down the instance's disks") self.cfg.SetInstancePrimaryNode(self.instance.uuid, self.target_node_uuid) self.instance = self.cfg.GetInstanceInfo(self.instance_uuid) # Only start the instance if it's marked as up if self.instance.admin_state == constants.ADMINST_UP: self.feedback_fn("* activating the instance's disks on target node %s" % self.cfg.GetNodeName(self.target_node_uuid)) logging.info("Starting instance %s on node %s", self.instance.name, self.cfg.GetNodeName(self.target_node_uuid)) disks_ok, _, _ = AssembleInstanceDisks(self.lu, self.instance, ignore_secondaries=True) if not disks_ok: ShutdownInstanceDisks(self.lu, self.instance) raise errors.OpExecError("Can't activate the instance's disks") self.feedback_fn("* starting the instance on the target node %s" % self.cfg.GetNodeName(self.target_node_uuid)) result = self.rpc.call_instance_start(self.target_node_uuid, (self.instance, None, None), False, self.lu.op.reason) msg = result.fail_msg if msg: ShutdownInstanceDisks(self.lu, self.instance) raise errors.OpExecError("Could not start instance %s on node %s: %s" % (self.instance.name, self.cfg.GetNodeName(self.target_node_uuid), msg)) def Exec(self, feedback_fn): """Perform the migration. """ self.feedback_fn = feedback_fn self.source_node_uuid = self.instance.primary_node # FIXME: if we implement migrate-to-any in DRBD, this needs fixing disks = self.cfg.GetInstanceDisks(self.instance.uuid) # TODO allow mixed disks if utils.AllDiskOfType(disks, constants.DTS_INT_MIRROR): secondary_nodes = self.cfg.GetInstanceSecondaryNodes(self.instance.uuid) self.target_node_uuid = secondary_nodes[0] # Otherwise self.target_node has been populated either # directly, or through an iallocator. self.all_node_uuids = [self.source_node_uuid, self.target_node_uuid] self.nodes_ip = dict((uuid, node.secondary_ip) for (uuid, node) in self.cfg.GetMultiNodeInfo(self.all_node_uuids)) if self.failover: feedback_fn("Failover instance %s" % self.instance.name) self._ExecFailover() else: feedback_fn("Migrating instance %s" % self.instance.name) if self.cleanup: return self._ExecCleanup() else: return self._ExecMigration() ganeti-3.1.0~rc2/lib/cmdlib/instance_operation.py000064400000000000000000000533431476477700300220520ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with instance operations (start/stop/...). Those operations have in common that they affect the operating system in a running instance directly. """ import logging from ganeti import constants from ganeti import errors from ganeti import hypervisor from ganeti import locking from ganeti import objects from ganeti import utils from ganeti.cmdlib.base import LogicalUnit, NoHooksLU from ganeti.cmdlib.common import INSTANCE_ONLINE, INSTANCE_DOWN, \ CheckHVParams, CheckInstanceState, CheckNodeOnline, GetUpdatedParams, \ CheckOSParams, CheckOSImage, ShareAll from ganeti.cmdlib.instance_storage import StartInstanceDisks, \ ShutdownInstanceDisks, ImageDisks from ganeti.cmdlib.instance_utils import BuildInstanceHookEnvByObject, \ CheckInstanceBridgesExist, CheckNodeFreeMemory, UpdateMetadata from ganeti.hypervisor import hv_base def _IsInstanceUserDown(cluster, instance, instance_info): hvparams = cluster.FillHV(instance, skip_globals=True) return instance_info and \ "state" in instance_info and \ hv_base.HvInstanceState.IsShutdown(instance_info["state"]) and \ (instance.hypervisor != constants.HT_KVM or hvparams[constants.HV_KVM_USER_SHUTDOWN]) class LUInstanceStartup(LogicalUnit): """Starts an instance. """ HPATH = "instance-start" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def CheckArguments(self): # extra beparams if self.op.beparams: # fill the beparams dict objects.UpgradeBeParams(self.op.beparams) utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES) def ExpandNames(self): self._ExpandAndLockInstance() self.recalculate_locks[locking.LEVEL_NODE_RES] = constants.LOCKS_REPLACE def DeclareLocks(self, level): if level == locking.LEVEL_NODE_RES: self._LockInstancesNodes(primary_only=True, level=locking.LEVEL_NODE_RES) def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ env = { "FORCE": self.op.force, } env.update(BuildInstanceHookEnvByObject(self, self.instance)) return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] + \ list(self.cfg.GetInstanceNodes(self.instance.uuid)) return (nl, nl) def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name cluster = self.cfg.GetClusterInfo() # extra hvparams if self.op.hvparams: # check hypervisor parameter syntax (locally) utils.ForceDictType(self.op.hvparams, constants.HVS_PARAMETER_TYPES) filled_hvp = cluster.FillHV(self.instance) filled_hvp.update(self.op.hvparams) hv_type = hypervisor.GetHypervisorClass(self.instance.hypervisor) hv_type.CheckParameterSyntax(filled_hvp) CheckHVParams(self, self.cfg.GetInstanceNodes(self.instance.uuid), self.instance.hypervisor, filled_hvp) CheckInstanceState(self, self.instance, INSTANCE_ONLINE) self.primary_offline = \ self.cfg.GetNodeInfo(self.instance.primary_node).offline if self.primary_offline and self.op.ignore_offline_nodes: self.LogWarning("Ignoring offline primary node") if self.op.hvparams or self.op.beparams: self.LogWarning("Overridden parameters are ignored") else: CheckNodeOnline(self, self.instance.primary_node) bep = self.cfg.GetClusterInfo().FillBE(self.instance) bep.update(self.op.beparams) # check bridges existence CheckInstanceBridgesExist(self, self.instance) remote_info = self.rpc.call_instance_info( self.instance.primary_node, self.instance.name, self.instance.hypervisor, cluster.hvparams[self.instance.hypervisor]) remote_info.Raise("Error checking node %s" % self.cfg.GetNodeName(self.instance.primary_node), prereq=True, ecode=errors.ECODE_ENVIRON) self.requires_cleanup = False if remote_info.payload: if _IsInstanceUserDown(self.cfg.GetClusterInfo(), self.instance, remote_info.payload): self.requires_cleanup = True else: # not running already CheckNodeFreeMemory( self, self.instance.primary_node, "starting instance %s" % self.instance.name, bep[constants.BE_MINMEM], self.instance.hypervisor, self.cfg.GetClusterInfo().hvparams[self.instance.hypervisor]) def Exec(self, feedback_fn): """Start the instance. """ if not self.op.no_remember: self.instance = self.cfg.MarkInstanceUp(self.instance.uuid) if self.primary_offline: assert self.op.ignore_offline_nodes self.LogInfo("Primary node offline, marked instance as started") else: if self.requires_cleanup: result = self.rpc.call_instance_shutdown( self.instance.primary_node, self.instance, self.op.shutdown_timeout, self.op.reason) result.Raise("Could not shutdown instance '%s'" % self.instance.name) ShutdownInstanceDisks(self, self.instance) StartInstanceDisks(self, self.instance, self.op.force) self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) result = \ self.rpc.call_instance_start(self.instance.primary_node, (self.instance, self.op.hvparams, self.op.beparams), self.op.startup_paused, self.op.reason) if result.fail_msg: ShutdownInstanceDisks(self, self.instance) result.Raise("Could not start instance '%s'" % self.instance.name) class LUInstanceShutdown(LogicalUnit): """Shutdown an instance. """ HPATH = "instance-stop" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def ExpandNames(self): self._ExpandAndLockInstance() def CheckArguments(self): """Check arguments. """ if self.op.no_remember and self.op.admin_state_source is not None: self.LogWarning("Parameter 'admin_state_source' has no effect if used" " with parameter 'no_remember'") if self.op.admin_state_source is None: self.op.admin_state_source = constants.ADMIN_SOURCE def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ env = BuildInstanceHookEnvByObject(self, self.instance) env["TIMEOUT"] = self.op.timeout return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] + \ list(self.cfg.GetInstanceNodes(self.instance.uuid)) return (nl, nl) def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name if self.op.force: self.LogWarning("Ignoring offline instance check") else: CheckInstanceState(self, self.instance, INSTANCE_ONLINE) self.primary_offline = \ self.cfg.GetNodeInfo(self.instance.primary_node).offline if self.primary_offline and self.op.ignore_offline_nodes: self.LogWarning("Ignoring offline primary node") else: CheckNodeOnline(self, self.instance.primary_node) if self.op.admin_state_source == constants.USER_SOURCE: cluster = self.cfg.GetClusterInfo() result = self.rpc.call_instance_info( self.instance.primary_node, self.instance.name, self.instance.hypervisor, cluster.hvparams[self.instance.hypervisor]) result.Raise("Error checking instance '%s'" % self.instance.name, prereq=True) if not _IsInstanceUserDown(cluster, self.instance, result.payload): raise errors.OpPrereqError("Instance '%s' was not shutdown by the user" % self.instance.name) def Exec(self, feedback_fn): """Shutdown the instance. """ # If the instance is offline we shouldn't mark it as down, as that # resets the offline flag. if not self.op.no_remember and self.instance.admin_state in INSTANCE_ONLINE: self.instance = self.cfg.MarkInstanceDown(self.instance.uuid) if self.op.admin_state_source == constants.ADMIN_SOURCE: self.cfg.MarkInstanceDown(self.instance.uuid) elif self.op.admin_state_source == constants.USER_SOURCE: self.cfg.MarkInstanceUserDown(self.instance.uuid) if self.primary_offline: assert self.op.ignore_offline_nodes self.LogInfo("Primary node offline, marked instance as stopped") else: result = self.rpc.call_instance_shutdown( self.instance.primary_node, self.instance, self.op.timeout, self.op.reason) result.Raise("Could not shutdown instance '%s'" % self.instance.name) ShutdownInstanceDisks(self, self.instance) class LUInstanceReinstall(LogicalUnit): """Reinstall an instance. """ HPATH = "instance-reinstall" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def CheckArguments(self): CheckOSImage(self.op) def ExpandNames(self): self._ExpandAndLockInstance() def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ return BuildInstanceHookEnvByObject(self, self.instance) def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] + \ list(self.cfg.GetInstanceNodes(self.instance.uuid)) return (nl, nl) def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster and is not running. """ instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name CheckNodeOnline(self, instance.primary_node, "Instance primary node" " offline, cannot reinstall") if not instance.disks: raise errors.OpPrereqError("Instance '%s' has no disks" % self.op.instance_name, errors.ECODE_INVAL) CheckInstanceState(self, instance, INSTANCE_DOWN, msg="cannot reinstall") # Handle OS parameters self._MergeValidateOsParams(instance) self.instance = instance def _MergeValidateOsParams(self, instance): "Handle the OS parameter merging and validation for the target instance." node_uuids = list(self.cfg.GetInstanceNodes(instance.uuid)) self.op.osparams = self.op.osparams or {} self.op.osparams_private = self.op.osparams_private or {} self.op.osparams_secret = self.op.osparams_secret or {} # Handle the use of 'default' values. params_public = GetUpdatedParams(instance.osparams, self.op.osparams) params_private = GetUpdatedParams(instance.osparams_private, self.op.osparams_private) params_secret = self.op.osparams_secret # Handle OS parameters if self.op.os_type is not None: instance_os = self.op.os_type else: instance_os = instance.os cluster = self.cfg.GetClusterInfo() self.osparams = cluster.SimpleFillOS( instance_os, params_public, os_params_private=params_private, os_params_secret=params_secret ) self.osparams_private = params_private self.osparams_secret = params_secret CheckOSParams(self, True, node_uuids, instance_os, self.osparams, self.op.force_variant) def _ReinstallOSScripts(self, instance, osparams, debug_level): """Reinstall OS scripts on an instance. @type instance: L{objects.Instance} @param instance: instance of which the OS scripts should run @type osparams: L{dict} @param osparams: OS parameters @type debug_level: non-negative int @param debug_level: debug level @rtype: NoneType @return: None @raise errors.OpExecError: in case of failure """ self.LogInfo("Running instance OS create scripts...") result = self.rpc.call_instance_os_add(instance.primary_node, (instance, osparams), True, debug_level) result.Raise("Could not install OS for instance '%s' on node '%s'" % (instance.name, self.cfg.GetNodeName(instance.primary_node))) def Exec(self, feedback_fn): """Reinstall the instance. """ os_image = objects.GetOSImage(self.op.osparams) if os_image is not None: feedback_fn("Using OS image '%s'" % os_image) else: os_image = objects.GetOSImage(self.instance.osparams) os_type = self.op.os_type if os_type is not None: feedback_fn("Changing OS scripts to '%s'..." % os_type) self.instance.os = os_type self.cfg.Update(self.instance, feedback_fn) else: os_type = self.instance.os if not os_image and not os_type: self.LogInfo("No OS scripts or OS image specified or found in the" " instance's configuration, nothing to install") else: if self.op.osparams is not None: self.instance.osparams = self.op.osparams if self.op.osparams_private is not None: self.instance.osparams_private = self.op.osparams_private self.cfg.Update(self.instance, feedback_fn) StartInstanceDisks(self, self.instance, None) self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) try: if os_image: ImageDisks(self, self.instance, os_image) if os_type: self._ReinstallOSScripts(self.instance, self.osparams, self.op.debug_level) UpdateMetadata(feedback_fn, self.rpc, self.instance, osparams_public=self.osparams, osparams_private=self.osparams_private, osparams_secret=self.osparams_secret) finally: ShutdownInstanceDisks(self, self.instance) class LUInstanceReboot(LogicalUnit): """Reboot an instance. """ HPATH = "instance-reboot" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def ExpandNames(self): self._ExpandAndLockInstance() def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ env = { "IGNORE_SECONDARIES": self.op.ignore_secondaries, "REBOOT_TYPE": self.op.reboot_type, "SHUTDOWN_TIMEOUT": self.op.shutdown_timeout, } env.update(BuildInstanceHookEnvByObject(self, self.instance)) return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] + \ list(self.cfg.GetInstanceNodes(self.instance.uuid)) return (nl, nl) def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name CheckInstanceState(self, self.instance, INSTANCE_ONLINE) CheckNodeOnline(self, self.instance.primary_node) # check bridges existence CheckInstanceBridgesExist(self, self.instance) def Exec(self, feedback_fn): """Reboot the instance. """ cluster = self.cfg.GetClusterInfo() remote_info = self.rpc.call_instance_info( self.instance.primary_node, self.instance.name, self.instance.hypervisor, cluster.hvparams[self.instance.hypervisor]) remote_info.Raise("Error checking node %s" % self.cfg.GetNodeName(self.instance.primary_node)) instance_running = bool(remote_info.payload) current_node_uuid = self.instance.primary_node if instance_running and \ self.op.reboot_type in [constants.INSTANCE_REBOOT_SOFT, constants.INSTANCE_REBOOT_HARD]: result = self.rpc.call_instance_reboot(current_node_uuid, self.instance, self.op.reboot_type, self.op.shutdown_timeout, self.op.reason) result.Raise("Could not reboot instance") else: if instance_running: result = self.rpc.call_instance_shutdown(current_node_uuid, self.instance, self.op.shutdown_timeout, self.op.reason) result.Raise("Could not shutdown instance for full reboot") ShutdownInstanceDisks(self, self.instance) self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) else: self.LogInfo("Instance %s was already stopped, starting now", self.instance.name) StartInstanceDisks(self, self.instance, self.op.ignore_secondaries) self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) result = self.rpc.call_instance_start(current_node_uuid, (self.instance, None, None), False, self.op.reason) msg = result.fail_msg if msg: ShutdownInstanceDisks(self, self.instance) self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) raise errors.OpExecError("Could not start instance for" " full reboot: %s" % msg) self.cfg.MarkInstanceUp(self.instance.uuid) def GetInstanceConsole(cluster, instance, primary_node, node_group): """Returns console information for an instance. @type cluster: L{objects.Cluster} @type instance: L{objects.Instance} @type primary_node: L{objects.Node} @type node_group: L{objects.NodeGroup} @rtype: dict """ hyper = hypervisor.GetHypervisorClass(instance.hypervisor) # beparams and hvparams are passed separately, to avoid editing the # instance and then saving the defaults in the instance itself. hvparams = cluster.FillHV(instance) beparams = cluster.FillBE(instance) console = hyper.GetInstanceConsole(instance, primary_node, node_group, hvparams, beparams) assert console.instance == instance.name console.Validate() return console.ToDict() class LUInstanceConsole(NoHooksLU): """Connect to an instance's console. This is somewhat special in that it returns the command line that you need to run on the master node in order to connect to the console. """ REQ_BGL = False def ExpandNames(self): self.share_locks = ShareAll() self._ExpandAndLockInstance() def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name CheckNodeOnline(self, self.instance.primary_node) def Exec(self, feedback_fn): """Connect to the console of an instance """ node_uuid = self.instance.primary_node cluster_hvparams = self.cfg.GetClusterInfo().hvparams node_insts = self.rpc.call_instance_list( [node_uuid], [self.instance.hypervisor], cluster_hvparams)[node_uuid] node_insts.Raise("Can't get node information from %s" % self.cfg.GetNodeName(node_uuid)) if self.instance.name not in node_insts.payload: if self.instance.admin_state == constants.ADMINST_UP: state = constants.INSTST_ERRORDOWN elif self.instance.admin_state == constants.ADMINST_DOWN: state = constants.INSTST_ADMINDOWN else: state = constants.INSTST_ADMINOFFLINE raise errors.OpExecError("Instance %s is not running (state %s)" % (self.instance.name, state)) logging.debug("Connecting to console of %s on %s", self.instance.name, self.cfg.GetNodeName(node_uuid)) node = self.cfg.GetNodeInfo(self.instance.primary_node) group = self.cfg.GetNodeGroup(node.group) return GetInstanceConsole(self.cfg.GetClusterInfo(), self.instance, node, group) ganeti-3.1.0~rc2/lib/cmdlib/instance_query.py000064400000000000000000000264711476477700300212210ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units for querying instances.""" import itertools from ganeti import constants from ganeti import locking from ganeti import utils from ganeti.cmdlib.base import NoHooksLU from ganeti.cmdlib.common import ShareAll, GetWantedInstances, \ CheckInstancesNodeGroups, AnnotateDiskParams from ganeti.cmdlib.instance_utils import NICListToTuple from ganeti.hypervisor import hv_base class LUInstanceQueryData(NoHooksLU): """Query runtime instance data. """ REQ_BGL = False def ExpandNames(self): self.needed_locks = {} # Use locking if requested or when non-static information is wanted if not (self.op.static or self.op.use_locking): self.LogWarning("Non-static data requested, locks need to be acquired") self.op.use_locking = True if self.op.instances or not self.op.use_locking: # Expand instance names right here (_, self.wanted_names) = GetWantedInstances(self, self.op.instances) else: # Will use acquired locks self.wanted_names = None if self.op.use_locking: self.share_locks = ShareAll() if self.wanted_names is None: self.needed_locks[locking.LEVEL_INSTANCE] = locking.ALL_SET else: self.needed_locks[locking.LEVEL_INSTANCE] = self.wanted_names self.needed_locks[locking.LEVEL_NODEGROUP] = [] self.needed_locks[locking.LEVEL_NODE] = [] self.needed_locks[locking.LEVEL_NETWORK] = [] self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE self.dont_collate_locks[locking.LEVEL_NODEGROUP] = True self.dont_collate_locks[locking.LEVEL_NODE] = True self.dont_collate_locks[locking.LEVEL_NETWORK] = True def DeclareLocks(self, level): if self.op.use_locking: owned_instances = dict(self.cfg.GetMultiInstanceInfoByName( self.owned_locks(locking.LEVEL_INSTANCE))) if level == locking.LEVEL_NODEGROUP: # Lock all groups used by instances optimistically; this requires going # via the node before it's locked, requiring verification later on self.needed_locks[locking.LEVEL_NODEGROUP] = \ frozenset(group_uuid for instance_uuid in owned_instances for group_uuid in self.cfg.GetInstanceNodeGroups(instance_uuid)) elif level == locking.LEVEL_NODE: self._LockInstancesNodes() elif level == locking.LEVEL_NETWORK: self.needed_locks[locking.LEVEL_NETWORK] = \ frozenset(net_uuid for instance_uuid in owned_instances.keys() for net_uuid in self.cfg.GetInstanceNetworks(instance_uuid)) def CheckPrereq(self): """Check prerequisites. This only checks the optional instance list against the existing names. """ owned_instances = frozenset(self.owned_locks(locking.LEVEL_INSTANCE)) owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP)) owned_node_uuids = frozenset(self.owned_locks(locking.LEVEL_NODE)) owned_networks = frozenset(self.owned_locks(locking.LEVEL_NETWORK)) if self.wanted_names is None: assert self.op.use_locking, "Locking was not used" self.wanted_names = owned_instances instances = dict(self.cfg.GetMultiInstanceInfoByName(self.wanted_names)) if self.op.use_locking: CheckInstancesNodeGroups(self.cfg, instances, owned_groups, owned_node_uuids, None) else: assert not (owned_instances or owned_groups or owned_node_uuids or owned_networks) self.wanted_instances = list(instances.values()) def _ComputeBlockdevStatus(self, node_uuid, instance, dev): """Returns the status of a block device """ if self.op.static or not node_uuid: return None result = self.rpc.call_blockdev_find(node_uuid, (dev, instance)) if result.offline: return None result.Raise("Can't compute disk status for %s" % instance.name) status = result.payload if status is None: return None return (status.dev_path, status.major, status.minor, status.sync_percent, status.estimated_time, status.is_degraded, status.ldisk_status) def _ComputeDiskStatus(self, instance, node_uuid2name_fn, dev): """Compute block device status. """ (anno_dev,) = AnnotateDiskParams(instance, [dev], self.cfg) return self._ComputeDiskStatusInner(instance, None, node_uuid2name_fn, anno_dev) def _ComputeDiskStatusInner(self, instance, snode_uuid, node_uuid2name_fn, dev): """Compute block device status. @attention: The device has to be annotated already. """ drbd_info = None output_logical_id = dev.logical_id if dev.dev_type in constants.DTS_DRBD: # we change the snode then (otherwise we use the one passed in) if dev.logical_id[0] == instance.primary_node: snode_uuid = dev.logical_id[1] snode_minor = dev.logical_id[4] pnode_minor = dev.logical_id[3] else: snode_uuid = dev.logical_id[0] snode_minor = dev.logical_id[3] pnode_minor = dev.logical_id[4] drbd_info = { "primary_node": node_uuid2name_fn(instance.primary_node), "primary_minor": pnode_minor, "secondary_node": node_uuid2name_fn(snode_uuid), "secondary_minor": snode_minor, "port": dev.logical_id[2], } # replace the secret present at the end of the ids with None output_logical_id = dev.logical_id[:-1] + (None,) dev_pstatus = self._ComputeBlockdevStatus(instance.primary_node, instance, dev) dev_sstatus = self._ComputeBlockdevStatus(snode_uuid, instance, dev) if dev.children: dev_children = [ self._ComputeDiskStatusInner(instance, snode_uuid, node_uuid2name_fn, d) for d in dev.children ] else: dev_children = [] return { "iv_name": dev.iv_name, "dev_type": dev.dev_type, "logical_id": output_logical_id, "drbd_info": drbd_info, "pstatus": dev_pstatus, "sstatus": dev_sstatus, "children": dev_children, "mode": dev.mode, "size": dev.size, "spindles": dev.spindles, "name": dev.name, "uuid": dev.uuid, } def Exec(self, feedback_fn): """Gather and return data""" result = {} cluster = self.cfg.GetClusterInfo() node_uuids = itertools.chain(*(self.cfg.GetInstanceNodes(i.uuid) for i in self.wanted_instances)) nodes = dict(self.cfg.GetMultiNodeInfo(node_uuids)) groups = dict(self.cfg.GetMultiNodeGroupInfo(node.group for node in nodes.values())) for instance in self.wanted_instances: pnode = nodes[instance.primary_node] hvparams = cluster.FillHV(instance, skip_globals=True) if self.op.static or pnode.offline: remote_state = None if pnode.offline: self.LogWarning("Primary node %s is marked offline, returning static" " information only for instance %s" % (pnode.name, instance.name)) else: remote_info = self.rpc.call_instance_info( instance.primary_node, instance.name, instance.hypervisor, cluster.hvparams[instance.hypervisor]) remote_info.Raise("Error checking node %s" % pnode.name) remote_info = remote_info.payload allow_userdown = \ cluster.enabled_user_shutdown and \ (instance.hypervisor != constants.HT_KVM or hvparams[constants.HV_KVM_USER_SHUTDOWN]) if remote_info and "state" in remote_info: if hv_base.HvInstanceState.IsShutdown(remote_info["state"]): if allow_userdown: remote_state = "user down" else: remote_state = "down" else: remote_state = "up" else: if instance.admin_state == constants.ADMINST_UP: remote_state = "down" elif instance.admin_state == constants.ADMINST_DOWN: if instance.admin_state_source == constants.USER_SOURCE: remote_state = "user down" else: remote_state = "down" else: remote_state = "offline" group2name_fn = lambda uuid: groups[uuid].name node_uuid2name_fn = lambda uuid: nodes[uuid].name disk_objects = self.cfg.GetInstanceDisks(instance.uuid) output_disks = [self._ComputeDiskStatus(instance, node_uuid2name_fn, d) for d in disk_objects] secondary_nodes = self.cfg.GetInstanceSecondaryNodes(instance.uuid) snodes_group_uuids = [nodes[snode_uuid].group for snode_uuid in secondary_nodes] result[instance.name] = { "name": instance.name, "config_state": instance.admin_state, "run_state": remote_state, "pnode": pnode.name, "pnode_group_uuid": pnode.group, "pnode_group_name": group2name_fn(pnode.group), "snodes": [node_uuid2name_fn(n) for n in secondary_nodes], "snodes_group_uuids": snodes_group_uuids, "snodes_group_names": [group2name_fn(u) for u in snodes_group_uuids], "os": instance.os, # this happens to be the same format used for hooks "nics": NICListToTuple(self, instance.nics), "disk_template": utils.GetDiskTemplate(disk_objects), "disks": output_disks, "hypervisor": instance.hypervisor, "network_port": instance.network_port, "hv_instance": instance.hvparams, "hv_actual": hvparams, "be_instance": instance.beparams, "be_actual": cluster.FillBE(instance), "os_instance": instance.osparams, "os_actual": cluster.SimpleFillOS(instance.os, instance.osparams), "serial_no": instance.serial_no, "mtime": instance.mtime, "ctime": instance.ctime, "uuid": instance.uuid, } return result ganeti-3.1.0~rc2/lib/cmdlib/instance_set_params.py000064400000000000000000002443551476477700300222150ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical unit setting parameters of a single instance.""" import copy import logging import os from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import ht from ganeti import hypervisor from ganeti import locking from ganeti.masterd import iallocator from ganeti import netutils from ganeti import objects from ganeti import utils import ganeti.rpc.node as rpc from ganeti.cmdlib.base import LogicalUnit from ganeti.cmdlib.common import INSTANCE_DOWN, \ INSTANCE_NOT_RUNNING, CheckNodeOnline, \ CheckParamsNotGlobal, \ IsExclusiveStorageEnabledNode, CheckHVParams, CheckOSParams, \ GetUpdatedParams, CheckInstanceState, ExpandNodeUuidAndName, \ IsValidDiskAccessModeCombination, AnnotateDiskParams, \ CheckIAllocatorOrNode from ganeti.cmdlib.instance_storage import CalculateFileStorageDir, \ CheckDiskExtProvider, CheckNodesFreeDiskPerVG, CheckRADOSFreeSpace, \ CheckSpindlesExclusiveStorage, ComputeDiskSizePerVG, ComputeDisksInfo, \ CreateDisks, CreateSingleBlockDev, GenerateDiskTemplate, \ IsExclusiveStorageEnabledNodeUuid, ShutdownInstanceDisks, \ WaitForSync, WipeOrCleanupDisks, AssembleInstanceDisks from ganeti.cmdlib.instance_utils import BuildInstanceHookEnvByObject, \ NICToTuple, CheckNodeNotDrained, CopyLockList, \ ReleaseLocks, CheckNodeVmCapable, CheckTargetNodeIPolicy, \ GetInstanceInfoText, RemoveDisks, CheckNodeFreeMemory, \ UpdateMetadata, CheckForConflictingIp, \ PrepareContainerMods, ComputeInstanceCommunicationNIC, \ ApplyContainerMods, ComputeIPolicyInstanceSpecViolation, \ CheckNodesPhysicalCPUs import ganeti.masterd.instance class InstNicModPrivate(object): """Data structure for network interface modifications. Used by L{LUInstanceSetParams}. """ def __init__(self): self.params = None self.filled = None class LUInstanceSetParams(LogicalUnit): """Modifies an instances's parameters. """ HPATH = "instance-modify" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def GenericGetDiskInfo(self, uuid=None, name=None): """Find a disk object using the provided params. Accept arguments as keywords and use the GetDiskInfo/GetDiskInfoByName config functions to retrieve the disk info based on these arguments. In case of an error, raise the appropriate exceptions. """ if uuid: disk = self.cfg.GetDiskInfo(uuid) if disk is None: raise errors.OpPrereqError("No disk was found with this UUID: %s" % uuid, errors.ECODE_INVAL) elif name: disk = self.cfg.GetDiskInfoByName(name) if disk is None: raise errors.OpPrereqError("No disk was found with this name: %s" % name, errors.ECODE_INVAL) else: raise errors.ProgrammerError("No disk UUID or name was given") return disk @staticmethod def _UpgradeDiskNicMods(kind, mods, verify_fn): assert ht.TList(mods) assert not mods or len(mods[0]) in (2, 3) if mods and len(mods[0]) == 2: result = [] addremove = 0 for op, params in mods: if op in (constants.DDM_ADD, constants.DDM_ATTACH, constants.DDM_REMOVE, constants.DDM_DETACH): result.append((op, -1, params)) addremove += 1 if addremove > 1: raise errors.OpPrereqError("Only one %s add/attach/remove/detach " "operation is supported at a time" % kind, errors.ECODE_INVAL) else: result.append((constants.DDM_MODIFY, op, params)) assert verify_fn(result) else: result = mods return result @staticmethod def _CheckMods(kind, mods, key_types, item_fn): """Ensures requested disk/NIC modifications are valid. Note that the 'attach' action needs a way to refer to the UUID of the disk, since the disk name is not unique cluster-wide. However, the UUID of the disk is not settable but rather generated by Ganeti automatically, therefore it cannot be passed as an IDISK parameter. For this reason, this function will override the checks to accept uuid parameters solely for the attach action. """ # Create a key_types copy with the 'uuid' as a valid key type. key_types_attach = key_types.copy() key_types_attach['uuid'] = 'string' for (op, _, params) in mods: assert ht.TDict(params) # If 'key_types' is an empty dict, we assume we have an # 'ext' template and thus do not ForceDictType if key_types: utils.ForceDictType(params, (key_types if op != constants.DDM_ATTACH else key_types_attach)) if op not in (constants.DDM_ADD, constants.DDM_ATTACH, constants.DDM_MODIFY, constants.DDM_REMOVE, constants.DDM_DETACH): raise errors.ProgrammerError("Unhandled operation '%s'" % op) if op in (constants.DDM_REMOVE, constants.DDM_DETACH): if params: raise errors.OpPrereqError("No settings should be passed when" " removing or detaching a %s" % kind, errors.ECODE_INVAL) item_fn(op, params) def _VerifyDiskModification(self, op, params, excl_stor, group_access_types): """Verifies a disk modification. """ disk_type = params.get( constants.IDISK_TYPE, self.cfg.GetInstanceDiskTemplate(self.instance.uuid)) if op == constants.DDM_ADD: params[constants.IDISK_TYPE] = disk_type if disk_type == constants.DT_DISKLESS: raise errors.OpPrereqError( "Must specify disk type on diskless instance", errors.ECODE_INVAL) if disk_type != constants.DT_EXT: utils.ForceDictType(params, constants.IDISK_PARAMS_TYPES) mode = params.setdefault(constants.IDISK_MODE, constants.DISK_RDWR) if mode not in constants.DISK_ACCESS_SET: raise errors.OpPrereqError("Invalid disk access mode '%s'" % mode, errors.ECODE_INVAL) size = params.get(constants.IDISK_SIZE, None) if size is None: raise errors.OpPrereqError("Required disk parameter '%s' missing" % constants.IDISK_SIZE, errors.ECODE_INVAL) size = int(size) params[constants.IDISK_SIZE] = size name = params.get(constants.IDISK_NAME, None) if name is not None and name.lower() == constants.VALUE_NONE: params[constants.IDISK_NAME] = None # These checks are necessary when adding and attaching disks if op in (constants.DDM_ADD, constants.DDM_ATTACH): CheckSpindlesExclusiveStorage(params, excl_stor, True) # If the disk is added we need to check for ext provider if op == constants.DDM_ADD: CheckDiskExtProvider(params, disk_type) # Make sure we do not add syncing disks to instances with inactive disks if not self.op.wait_for_sync and not self.instance.disks_active: raise errors.OpPrereqError("Can't %s a disk to an instance with" " deactivated disks and --no-wait-for-sync" " given" % op, errors.ECODE_INVAL) # Check disk access param (only for specific disks) if disk_type in constants.DTS_HAVE_ACCESS: access_type = params.get(constants.IDISK_ACCESS, group_access_types[disk_type]) if not IsValidDiskAccessModeCombination(self.instance.hypervisor, disk_type, access_type): raise errors.OpPrereqError("Selected hypervisor (%s) cannot be" " used with %s disk access param" % (self.instance.hypervisor, access_type), errors.ECODE_STATE) if op == constants.DDM_ATTACH: if len(params) != 1 or ('uuid' not in params and constants.IDISK_NAME not in params): raise errors.OpPrereqError("Only one argument is permitted in %s op," " either %s or uuid" % (constants.DDM_ATTACH, constants.IDISK_NAME, ), errors.ECODE_INVAL) self._CheckAttachDisk(params) elif op == constants.DDM_MODIFY: if constants.IDISK_SIZE in params: raise errors.OpPrereqError("Disk size change not possible, use" " grow-disk", errors.ECODE_INVAL) disk_info = self.cfg.GetInstanceDisks(self.instance.uuid) # Disk modification supports changing only the disk name and mode. # Changing arbitrary parameters is allowed only for ext disk template", if not utils.AllDiskOfType(disk_info, [constants.DT_EXT]): utils.ForceDictType(params, constants.MODIFIABLE_IDISK_PARAMS_TYPES) else: # We have to check that the 'access' and 'disk_provider' parameters # cannot be modified for param in [constants.IDISK_ACCESS, constants.IDISK_PROVIDER]: if param in params: raise errors.OpPrereqError("Disk '%s' parameter change is" " not possible" % param, errors.ECODE_INVAL) name = params.get(constants.IDISK_NAME, None) if name is not None and name.lower() == constants.VALUE_NONE: params[constants.IDISK_NAME] = None if op == constants.DDM_REMOVE and not self.op.hotplug: CheckInstanceState(self, self.instance, INSTANCE_NOT_RUNNING, msg="can't remove volume from a running instance" " without using hotplug") @staticmethod def _VerifyNicModification(op, params): """Verifies a network interface modification. """ if op in (constants.DDM_ADD, constants.DDM_MODIFY): ip = params.get(constants.INIC_IP, None) name = params.get(constants.INIC_NAME, None) req_net = params.get(constants.INIC_NETWORK, None) link = params.get(constants.NIC_LINK, None) mode = params.get(constants.NIC_MODE, None) if name is not None and name.lower() == constants.VALUE_NONE: params[constants.INIC_NAME] = None if req_net is not None: if req_net.lower() == constants.VALUE_NONE: params[constants.INIC_NETWORK] = None req_net = None elif link is not None or mode is not None: raise errors.OpPrereqError("If network is given" " mode or link should not", errors.ECODE_INVAL) if op == constants.DDM_ADD: macaddr = params.get(constants.INIC_MAC, None) if macaddr is None: params[constants.INIC_MAC] = constants.VALUE_AUTO if ip is not None: if ip.lower() == constants.VALUE_NONE: params[constants.INIC_IP] = None else: if ip.lower() == constants.NIC_IP_POOL: if op == constants.DDM_ADD and req_net is None: raise errors.OpPrereqError("If ip=pool, parameter network" " cannot be none", errors.ECODE_INVAL) else: if not netutils.IPAddress.IsValid(ip): raise errors.OpPrereqError("Invalid IP address '%s'" % ip, errors.ECODE_INVAL) if constants.INIC_MAC in params: macaddr = params[constants.INIC_MAC] if macaddr not in (constants.VALUE_AUTO, constants.VALUE_GENERATE): macaddr = utils.NormalizeAndValidateMac(macaddr) if op == constants.DDM_MODIFY and macaddr == constants.VALUE_AUTO: raise errors.OpPrereqError("'auto' is not a valid MAC address when" " modifying an existing NIC", errors.ECODE_INVAL) def _LookupDiskIndex(self, idx): """Looks up uuid or name of disk if necessary.""" try: return int(idx) except ValueError: pass for i, d in enumerate(self.cfg.GetInstanceDisks(self.instance.uuid)): if d.name == idx or d.uuid == idx: return i raise errors.OpPrereqError("Lookup of disk %r failed" % idx) def _LookupDiskMods(self): """Looks up uuid or name of disk if necessary.""" return [(op, self._LookupDiskIndex(idx), params) for op, idx, params in self.op.disks] def CheckArguments(self): if not (self.op.nics or self.op.disks or self.op.disk_template or self.op.hvparams or self.op.beparams or self.op.os_name or self.op.osparams or self.op.offline is not None or self.op.runtime_mem or self.op.pnode or self.op.osparams_private or self.op.instance_communication is not None): raise errors.OpPrereqError("No changes submitted", errors.ECODE_INVAL) if self.op.hvparams: CheckParamsNotGlobal(self.op.hvparams, constants.HVC_GLOBALS, "hypervisor", "instance", "cluster") self.op.disks = self._UpgradeDiskNicMods( "disk", self.op.disks, ht.TSetParamsMods(ht.TIDiskParams)) self.op.nics = self._UpgradeDiskNicMods( "NIC", self.op.nics, ht.TSetParamsMods(ht.TINicParams)) # Check disk template modifications if self.op.disk_template: if self.op.disks: raise errors.OpPrereqError("Disk template conversion and other disk" " changes not supported at the same time", errors.ECODE_INVAL) # mirrored template node checks if self.op.disk_template in constants.DTS_INT_MIRROR: CheckIAllocatorOrNode(self, "iallocator", "remote_node") elif self.op.remote_node: self.LogWarning("Changing the disk template to a non-mirrored one," " the secondary node will be ignored") # the secondary node must be cleared in order to be ignored, otherwise # the operation will fail, in the GenerateDiskTemplate method self.op.remote_node = None # file-based template checks if self.op.disk_template in constants.DTS_FILEBASED: self._FillFileDriver() # Check NIC modifications self._CheckMods("NIC", self.op.nics, constants.INIC_PARAMS_TYPES, self._VerifyNicModification) if self.op.pnode: (self.op.pnode_uuid, self.op.pnode) = \ ExpandNodeUuidAndName(self.cfg, self.op.pnode_uuid, self.op.pnode) def _CheckAttachDisk(self, params): """Check if disk can be attached to an instance. Check if the disk and instance have the same template. Also, check if the disk nodes are visible from the instance. """ uuid = params.get("uuid", None) name = params.get(constants.IDISK_NAME, None) disk = self.GenericGetDiskInfo(uuid, name) instance_template = self.cfg.GetInstanceDiskTemplate(self.instance.uuid) if (disk.dev_type != instance_template and instance_template != constants.DT_DISKLESS): raise errors.OpPrereqError("Instance has '%s' template while disk has" " '%s' template" % (instance_template, disk.dev_type), errors.ECODE_INVAL) instance_nodes = self.cfg.GetInstanceNodes(self.instance.uuid) # Make sure we do not attach disks to instances on wrong nodes. If the # instance is diskless, that instance is associated only to the primary # node, whereas the disk can be associated to two nodes in the case of DRBD, # hence, we have a subset check here. if disk.nodes and not set(instance_nodes).issubset(set(disk.nodes)): raise errors.OpPrereqError("Disk nodes are %s while the instance's nodes" " are %s" % (disk.nodes, instance_nodes), errors.ECODE_INVAL) # Make sure a DRBD disk has the same primary node as the instance where it # will be attached to. disk_primary = disk.GetPrimaryNode(self.instance.primary_node) if self.instance.primary_node != disk_primary: raise errors.OpExecError("The disks' primary node is %s whereas the " "instance's primary node is %s." % (disk_primary, self.instance.primary_node)) def ExpandNames(self): self._ExpandAndLockInstance() self.needed_locks[locking.LEVEL_NODEGROUP] = [] # Can't even acquire node locks in shared mode as upcoming changes in # Ganeti 2.6 will start to modify the node object on disk conversion self.needed_locks[locking.LEVEL_NODE] = [] self.needed_locks[locking.LEVEL_NODE_RES] = [] self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE # Look node group to look up the ipolicy self.share_locks[locking.LEVEL_NODEGROUP] = 1 self.dont_collate_locks[locking.LEVEL_NODEGROUP] = True self.dont_collate_locks[locking.LEVEL_NODE] = True self.dont_collate_locks[locking.LEVEL_NODE_RES] = True def DeclareLocks(self, level): if level == locking.LEVEL_NODEGROUP: assert not self.needed_locks[locking.LEVEL_NODEGROUP] # Acquire locks for the instance's nodegroups optimistically. Needs # to be verified in CheckPrereq self.needed_locks[locking.LEVEL_NODEGROUP] = \ self.cfg.GetInstanceNodeGroups(self.op.instance_uuid) elif level == locking.LEVEL_NODE: self._LockInstancesNodes() if self.op.disk_template and self.op.remote_node: (self.op.remote_node_uuid, self.op.remote_node) = \ ExpandNodeUuidAndName(self.cfg, self.op.remote_node_uuid, self.op.remote_node) self.needed_locks[locking.LEVEL_NODE].append(self.op.remote_node_uuid) elif self.op.disk_template in constants.DTS_INT_MIRROR: # If we have to find the secondary node for a conversion to DRBD, # close node locks to the whole node group. self.needed_locks[locking.LEVEL_NODE] = \ list(self.cfg.GetNodeGroupMembersByNodes( self.needed_locks[locking.LEVEL_NODE])) elif level == locking.LEVEL_NODE_RES and self.op.disk_template: # Copy node locks self.needed_locks[locking.LEVEL_NODE_RES] = \ CopyLockList(self.needed_locks[locking.LEVEL_NODE]) def BuildHooksEnv(self): """Build hooks env. This runs on the master, primary and secondaries. """ args = {} if constants.BE_MINMEM in self.be_new: args["minmem"] = self.be_new[constants.BE_MINMEM] if constants.BE_MAXMEM in self.be_new: args["maxmem"] = self.be_new[constants.BE_MAXMEM] if constants.BE_VCPUS in self.be_new: args["vcpus"] = self.be_new[constants.BE_VCPUS] # TODO: export disk changes. Note: _BuildInstanceHookEnv* don't export disk # information at all. if self._new_nics is not None: nics = [] for nic in self._new_nics: n = copy.deepcopy(nic) nicparams = self.cluster.SimpleFillNIC(n.nicparams) n.nicparams = nicparams nics.append(NICToTuple(self, n)) args["nics"] = nics env = BuildInstanceHookEnvByObject(self, self.instance, override=args) if self.op.disk_template: env["NEW_DISK_TEMPLATE"] = self.op.disk_template if self.op.runtime_mem: env["RUNTIME_MEMORY"] = self.op.runtime_mem return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] + \ list(self.cfg.GetInstanceNodes(self.instance.uuid)) return (nl, nl) def _PrepareNicModification(self, params, private, old_ip, old_net_uuid, old_params, cluster, pnode_uuid): update_params_dict = dict([(key, params[key]) for key in constants.NICS_PARAMETERS if key in params]) req_link = update_params_dict.get(constants.NIC_LINK, None) req_mode = update_params_dict.get(constants.NIC_MODE, None) new_net_uuid = None new_net_uuid_or_name = params.get(constants.INIC_NETWORK, old_net_uuid) if new_net_uuid_or_name: new_net_uuid = self.cfg.LookupNetwork(new_net_uuid_or_name) new_net_obj = self.cfg.GetNetwork(new_net_uuid) if old_net_uuid: old_net_obj = self.cfg.GetNetwork(old_net_uuid) if new_net_uuid: netparams = self.cfg.GetGroupNetParams(new_net_uuid, pnode_uuid) if not netparams: raise errors.OpPrereqError("No netparams found for the network" " %s, probably not connected" % new_net_obj.name, errors.ECODE_INVAL) new_params = dict(netparams) else: new_params = GetUpdatedParams(old_params, update_params_dict) utils.ForceDictType(new_params, constants.NICS_PARAMETER_TYPES) new_filled_params = cluster.SimpleFillNIC(new_params) objects.NIC.CheckParameterSyntax(new_filled_params) new_mode = new_filled_params[constants.NIC_MODE] if new_mode == constants.NIC_MODE_BRIDGED: bridge = new_filled_params[constants.NIC_LINK] msg = self.rpc.call_bridges_exist(pnode_uuid, [bridge]).fail_msg if msg: msg = "Error checking bridges on node '%s': %s" % \ (self.cfg.GetNodeName(pnode_uuid), msg) if self.op.force: self.warn.append(msg) else: raise errors.OpPrereqError(msg, errors.ECODE_ENVIRON) elif new_mode == constants.NIC_MODE_ROUTED: ip = params.get(constants.INIC_IP, old_ip) if ip is None and not new_net_uuid: raise errors.OpPrereqError("Cannot set the NIC IP address to None" " on a routed NIC if not attached to a" " network", errors.ECODE_INVAL) elif new_mode == constants.NIC_MODE_OVS: # TODO: check OVS link self.LogInfo("OVS links are currently not checked for correctness") if constants.INIC_MAC in params: mac = params[constants.INIC_MAC] if mac is None: raise errors.OpPrereqError("Cannot unset the NIC MAC address", errors.ECODE_INVAL) elif mac in (constants.VALUE_AUTO, constants.VALUE_GENERATE): # otherwise generate the MAC address params[constants.INIC_MAC] = \ self.cfg.GenerateMAC(new_net_uuid, self.proc.GetECId()) else: # or validate/reserve the current one try: self.cfg.ReserveMAC(mac, self.proc.GetECId()) except errors.ReservationError: raise errors.OpPrereqError("MAC address '%s' already in use" " in cluster" % mac, errors.ECODE_NOTUNIQUE) elif new_net_uuid != old_net_uuid: def get_net_prefix(net_uuid): mac_prefix = None if net_uuid: nobj = self.cfg.GetNetwork(net_uuid) mac_prefix = nobj.mac_prefix return mac_prefix new_prefix = get_net_prefix(new_net_uuid) old_prefix = get_net_prefix(old_net_uuid) if old_prefix != new_prefix: params[constants.INIC_MAC] = \ self.cfg.GenerateMAC(new_net_uuid, self.proc.GetECId()) # if there is a change in (ip, network) tuple new_ip = params.get(constants.INIC_IP, old_ip) if (new_ip, new_net_uuid) != (old_ip, old_net_uuid): if new_ip: # if IP is pool then require a network and generate one IP if new_ip.lower() == constants.NIC_IP_POOL: if new_net_uuid: try: new_ip = self.cfg.GenerateIp(new_net_uuid, self.proc.GetECId()) except errors.ReservationError: raise errors.OpPrereqError("Unable to get a free IP" " from the address pool", errors.ECODE_STATE) self.LogInfo("Chose IP %s from network %s", new_ip, new_net_obj.name) params[constants.INIC_IP] = new_ip else: raise errors.OpPrereqError("ip=pool, but no network found", errors.ECODE_INVAL) # Reserve new IP if in the new network if any elif new_net_uuid: try: self.cfg.ReserveIp(new_net_uuid, new_ip, self.proc.GetECId(), check=self.op.conflicts_check) self.LogInfo("Reserving IP %s in network %s", new_ip, new_net_obj.name) except errors.ReservationError: raise errors.OpPrereqError("IP %s not available in network %s" % (new_ip, new_net_obj.name), errors.ECODE_NOTUNIQUE) # new network is None so check if new IP is a conflicting IP elif self.op.conflicts_check: CheckForConflictingIp(self, new_ip, pnode_uuid) # release old IP if old network is not None if old_ip and old_net_uuid: try: self.cfg.ReleaseIp(old_net_uuid, old_ip, self.proc.GetECId()) except errors.AddressPoolError: logging.warning("Release IP %s not contained in network %s", old_ip, old_net_obj.name) # there are no changes in (ip, network) tuple and old network is not None elif (old_net_uuid is not None and (req_link is not None or req_mode is not None)): raise errors.OpPrereqError("Not allowed to change link or mode of" " a NIC that is connected to a network", errors.ECODE_INVAL) private.params = new_params private.filled = new_filled_params def _PreCheckDiskTemplate(self, pnode_info): """CheckPrereq checks related to a new disk template.""" # Arguments are passed to avoid configuration lookups pnode_uuid = self.instance.primary_node # TODO make sure heterogeneous disk types can be converted. disk_template = self.cfg.GetInstanceDiskTemplate(self.instance.uuid) if disk_template == constants.DT_MIXED: raise errors.OpPrereqError( "Conversion from mixed is not yet supported.") inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) if utils.AnyDiskOfType(inst_disks, constants.DTS_NOT_CONVERTIBLE_FROM): raise errors.OpPrereqError( "Conversion from the '%s' disk template is not supported" % self.cfg.GetInstanceDiskTemplate(self.instance.uuid), errors.ECODE_INVAL) elif self.op.disk_template in constants.DTS_NOT_CONVERTIBLE_TO: raise errors.OpPrereqError("Conversion to the '%s' disk template is" " not supported" % self.op.disk_template, errors.ECODE_INVAL) if (self.op.disk_template != constants.DT_EXT and utils.AllDiskOfType(inst_disks, [self.op.disk_template])): raise errors.OpPrereqError("Instance already has disk template %s" % self.op.disk_template, errors.ECODE_INVAL) if not self.cluster.IsDiskTemplateEnabled(self.op.disk_template): enabled_dts = utils.CommaJoin(self.cluster.enabled_disk_templates) raise errors.OpPrereqError("Disk template '%s' is not enabled for this" " cluster (enabled templates: %s)" % (self.op.disk_template, enabled_dts), errors.ECODE_STATE) default_vg = self.cfg.GetVGName() if (not default_vg and self.op.disk_template not in constants.DTS_NOT_LVM): raise errors.OpPrereqError("Disk template conversions to lvm-based" " instances are not supported by the cluster", errors.ECODE_STATE) CheckInstanceState(self, self.instance, INSTANCE_DOWN, msg="cannot change disk template") # compute new disks' information self.disks_info = ComputeDisksInfo(inst_disks, self.op.disk_template, default_vg, self.op.ext_params) # mirror node verification if self.op.disk_template in constants.DTS_INT_MIRROR \ and self.op.remote_node_uuid: if self.op.remote_node_uuid == pnode_uuid: raise errors.OpPrereqError("Given new secondary node %s is the same" " as the primary node of the instance" % self.op.remote_node, errors.ECODE_STATE) CheckNodeOnline(self, self.op.remote_node_uuid) CheckNodeNotDrained(self, self.op.remote_node_uuid) CheckNodeVmCapable(self, self.op.remote_node_uuid) snode_info = self.cfg.GetNodeInfo(self.op.remote_node_uuid) snode_group = self.cfg.GetNodeGroup(snode_info.group) ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(self.cluster, snode_group) CheckTargetNodeIPolicy(self, ipolicy, self.instance, snode_info, self.cfg, ignore=self.op.ignore_ipolicy) if pnode_info.group != snode_info.group: self.LogWarning("The primary and secondary nodes are in two" " different node groups; the disk parameters" " from the first disk's node group will be" " used") # check that the template is in the primary node group's allowed templates pnode_group = self.cfg.GetNodeGroup(pnode_info.group) ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(self.cluster, pnode_group) allowed_dts = ipolicy[constants.IPOLICY_DTS] if self.op.disk_template not in allowed_dts: raise errors.OpPrereqError("Disk template '%s' in not allowed (allowed" " templates: %s)" % (self.op.disk_template, utils.CommaJoin(allowed_dts)), errors.ECODE_STATE) if not self.op.disk_template in constants.DTS_EXCL_STORAGE: # Make sure none of the nodes require exclusive storage nodes = [pnode_info] if self.op.disk_template in constants.DTS_INT_MIRROR \ and self.op.remote_node_uuid: assert snode_info nodes.append(snode_info) has_es = lambda n: IsExclusiveStorageEnabledNode(self.cfg, n) if compat.any(map(has_es, nodes)): errmsg = ("Cannot convert disk template from %s to %s when exclusive" " storage is enabled" % ( self.cfg.GetInstanceDiskTemplate(self.instance.uuid), self.op.disk_template)) raise errors.OpPrereqError(errmsg, errors.ECODE_STATE) # TODO remove setting the disk template after DiskSetParams exists. # node capacity checks if (self.op.disk_template == constants.DT_PLAIN and utils.AllDiskOfType(inst_disks, [constants.DT_DRBD8])): # we ensure that no capacity checks will be made for conversions from # the 'drbd' to the 'plain' disk template pass elif (self.op.disk_template == constants.DT_DRBD8 and utils.AllDiskOfType(inst_disks, [constants.DT_PLAIN])): # for conversions from the 'plain' to the 'drbd' disk template, check # only the remote node's capacity if self.op.remote_node_uuid: req_sizes = ComputeDiskSizePerVG(self.op.disk_template, self.disks_info) CheckNodesFreeDiskPerVG(self, [self.op.remote_node_uuid], req_sizes) elif self.op.disk_template in constants.DTS_LVM: # rest lvm-based capacity checks node_uuids = [pnode_uuid] if self.op.remote_node_uuid: node_uuids.append(self.op.remote_node_uuid) req_sizes = ComputeDiskSizePerVG(self.op.disk_template, self.disks_info) CheckNodesFreeDiskPerVG(self, node_uuids, req_sizes) elif self.op.disk_template == constants.DT_RBD: # CheckRADOSFreeSpace() is simply a placeholder CheckRADOSFreeSpace() elif self.op.disk_template == constants.DT_EXT: # FIXME: Capacity checks for extstorage template, if exists pass else: # FIXME: Checks about other non lvm-based disk templates pass def _PreCheckDisks(self, ispec): """CheckPrereq checks related to disk changes. @type ispec: dict @param ispec: instance specs to be updated with the new disks """ self.diskparams = self.cfg.GetInstanceDiskParams(self.instance) inst_nodes = self.cfg.GetInstanceNodes(self.instance.uuid) excl_stor = compat.any( list(rpc.GetExclusiveStorageForNodes(self.cfg, inst_nodes).values()) ) # Get the group access type node_info = self.cfg.GetNodeInfo(self.instance.primary_node) node_group = self.cfg.GetNodeGroup(node_info.group) group_disk_params = self.cfg.GetGroupDiskParams(node_group) group_access_types = dict( (dt, group_disk_params[dt].get( constants.RBD_ACCESS, constants.DISK_KERNELSPACE)) for dt in constants.DISK_TEMPLATES) # Check disk modifications. This is done here and not in CheckArguments # (as with NICs), because we need to know the instance's disk template ver_fn = lambda op, par: self._VerifyDiskModification(op, par, excl_stor, group_access_types) # Don't enforce param types here in case it's an ext disk added. The check # happens inside _VerifyDiskModification. self._CheckMods("disk", self.op.disks, {}, ver_fn) self.diskmod = PrepareContainerMods(self.op.disks, None) def _PrepareDiskMod(_, disk, params, __): disk.name = params.get(constants.IDISK_NAME, None) # Verify disk changes (operating on a copy) inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) disks = copy.deepcopy(inst_disks) ApplyContainerMods("disk", disks, None, self.diskmod, None, None, _PrepareDiskMod, None, None) utils.ValidateDeviceNames("disk", disks) if len(disks) > constants.MAX_DISKS: raise errors.OpPrereqError("Instance has too many disks (%d), cannot add" " more" % constants.MAX_DISKS, errors.ECODE_STATE) disk_sizes = [disk.size for disk in inst_disks] disk_sizes.extend(params["size"] for (op, idx, params, private) in self.diskmod if op == constants.DDM_ADD) ispec[constants.ISPEC_DISK_COUNT] = len(disk_sizes) ispec[constants.ISPEC_DISK_SIZE] = disk_sizes # either --online or --offline was passed if self.op.offline is not None: if self.op.offline: msg = "can't change to offline without being down first" else: msg = "can't change to online (down) without being offline first" CheckInstanceState(self, self.instance, INSTANCE_NOT_RUNNING, msg=msg) @staticmethod def _InstanceCommunicationDDM(cfg, instance_communication, instance): """Create a NIC mod that adds or removes the instance communication NIC to a running instance. The NICS are dynamically created using the Dynamic Device Modification (DDM). This function produces a NIC modification (mod) that inserts an additional NIC meant for instance communication in or removes an existing instance communication NIC from a running instance, using DDM. @type cfg: L{config.ConfigWriter} @param cfg: cluster configuration @type instance_communication: boolean @param instance_communication: whether instance communication is enabled or disabled @type instance: L{objects.Instance} @param instance: instance to which the NIC mod will be applied to @rtype: (L{constants.DDM_ADD}, -1, parameters) or (L{constants.DDM_REMOVE}, -1, parameters) or L{None} @return: DDM mod containing an action to add or remove the NIC, or None if nothing needs to be done """ nic_name = ComputeInstanceCommunicationNIC(instance.name) instance_communication_nic = None for nic in instance.nics: if nic.name == nic_name: instance_communication_nic = nic break if instance_communication and not instance_communication_nic: action = constants.DDM_ADD params = {constants.INIC_NAME: nic_name, constants.INIC_MAC: constants.VALUE_GENERATE, constants.INIC_IP: constants.NIC_IP_POOL, constants.INIC_NETWORK: cfg.GetInstanceCommunicationNetwork()} elif not instance_communication and instance_communication_nic: action = constants.DDM_REMOVE params = None else: action = None params = None if action is not None: return (action, -1, params) else: return None def _GetInstanceInfo(self, cluster_hvparams): pnode_uuid = self.instance.primary_node instance_info = self.rpc.call_instance_info( pnode_uuid, self.instance.name, self.instance.hypervisor, cluster_hvparams) return instance_info def _CheckHotplug(self): if self.op.hotplug: result = self.rpc.call_hotplug_supported(self.instance.primary_node, self.instance) if result.fail_msg: self.LogWarning(result.fail_msg) self.op.hotplug = False self.LogInfo("Modification will take place without hotplugging.") def _PrepareNicCommunication(self): # add or remove NIC for instance communication if self.op.instance_communication is not None: mod = self._InstanceCommunicationDDM(self.cfg, self.op.instance_communication, self.instance) if mod is not None: self.op.nics.append(mod) self.nicmod = PrepareContainerMods(self.op.nics, InstNicModPrivate) def _ProcessHVParams(self, node_uuids): if self.op.hvparams: hv_type = self.instance.hypervisor i_hvdict = GetUpdatedParams(self.instance.hvparams, self.op.hvparams) utils.ForceDictType(i_hvdict, constants.HVS_PARAMETER_TYPES) hv_new = self.cluster.SimpleFillHV(hv_type, self.instance.os, i_hvdict) # local check hypervisor.GetHypervisorClass(hv_type).CheckParameterSyntax(hv_new) CheckHVParams(self, node_uuids, self.instance.hypervisor, hv_new) self.hv_proposed = self.hv_new = hv_new # the new actual values self.hv_inst = i_hvdict # the new dict (without defaults) else: self.hv_proposed = self.cluster.SimpleFillHV(self.instance.hypervisor, self.instance.os, self.instance.hvparams) self.hv_new = self.hv_inst = {} def _ProcessBeParams(self): if self.op.beparams: i_bedict = GetUpdatedParams(self.instance.beparams, self.op.beparams, use_none=True) objects.UpgradeBeParams(i_bedict) utils.ForceDictType(i_bedict, constants.BES_PARAMETER_TYPES) be_new = self.cluster.SimpleFillBE(i_bedict) self.be_proposed = self.be_new = be_new # the new actual values self.be_inst = i_bedict # the new dict (without defaults) else: self.be_new = self.be_inst = {} self.be_proposed = self.cluster.SimpleFillBE(self.instance.beparams) return self.cluster.FillBE(self.instance) def _ValidateCpuParams(self): # CPU param validation -- checking every time a parameter is # changed to cover all cases where either CPU mask or vcpus have # changed if (constants.BE_VCPUS in self.be_proposed and constants.HV_CPU_MASK in self.hv_proposed): cpu_list = \ utils.ParseMultiCpuMask(self.hv_proposed[constants.HV_CPU_MASK]) # Verify mask is consistent with number of vCPUs. Can skip this # test if only 1 entry in the CPU mask, which means same mask # is applied to all vCPUs. if (len(cpu_list) > 1 and len(cpu_list) != self.be_proposed[constants.BE_VCPUS]): raise errors.OpPrereqError("Number of vCPUs [%d] does not match the" " CPU mask [%s]" % (self.be_proposed[constants.BE_VCPUS], self.hv_proposed[constants.HV_CPU_MASK]), errors.ECODE_INVAL) # Only perform this test if a new CPU mask is given if constants.HV_CPU_MASK in self.hv_new and cpu_list: # Calculate the largest CPU number requested max_requested_cpu = max(map(max, cpu_list)) # Check that all of the instance's nodes have enough physical CPUs to # satisfy the requested CPU mask hvspecs = [(self.instance.hypervisor, self.cfg.GetClusterInfo() .hvparams[self.instance.hypervisor])] CheckNodesPhysicalCPUs(self, self.cfg.GetInstanceNodes(self.instance.uuid), max_requested_cpu + 1, hvspecs) def _ProcessOsParams(self, node_uuids): # osparams processing instance_os = (self.op.os_name if self.op.os_name and not self.op.force else self.instance.os) if self.op.osparams or self.op.osparams_private: public_parms = self.op.osparams or {} private_parms = self.op.osparams_private or {} dupe_keys = utils.GetRepeatedKeys(public_parms, private_parms) if dupe_keys: raise errors.OpPrereqError("OS parameters repeated multiple times: %s" % utils.CommaJoin(dupe_keys)) self.os_inst = GetUpdatedParams(self.instance.osparams, public_parms) self.os_inst_private = GetUpdatedParams(self.instance.osparams_private, private_parms) CheckOSParams(self, True, node_uuids, instance_os, objects.FillDict(self.os_inst, self.os_inst_private), self.op.force_variant) else: self.os_inst = {} self.os_inst_private = {} def _ProcessMem(self, cluster_hvparams, be_old, pnode_uuid): #TODO(dynmem): do the appropriate check involving MINMEM if (constants.BE_MAXMEM in self.op.beparams and not self.op.force and self.be_new[constants.BE_MAXMEM] > be_old[constants.BE_MAXMEM]): mem_check_list = [pnode_uuid] if self.be_new[constants.BE_AUTO_BALANCE]: # either we changed auto_balance to yes or it was from before mem_check_list.extend( self.cfg.GetInstanceSecondaryNodes(self.instance.uuid)) instance_info = self._GetInstanceInfo(cluster_hvparams) hvspecs = [(self.instance.hypervisor, cluster_hvparams)] nodeinfo = self.rpc.call_node_info(mem_check_list, None, hvspecs) pninfo = nodeinfo[pnode_uuid] msg = pninfo.fail_msg if msg: # Assume the primary node is unreachable and go ahead self.warn.append("Can't get info from primary node %s: %s" % (self.cfg.GetNodeName(pnode_uuid), msg)) else: (_, _, (pnhvinfo, )) = pninfo.payload if not isinstance(pnhvinfo.get("memory_free", None), int): self.warn.append("Node data from primary node %s doesn't contain" " free memory information" % self.cfg.GetNodeName(pnode_uuid)) elif instance_info.fail_msg: self.warn.append("Can't get instance runtime information: %s" % instance_info.fail_msg) else: if instance_info.payload: current_mem = int(instance_info.payload["memory"]) else: # Assume instance not running # (there is a slight race condition here, but it's not very # probable, and we have no other way to check) # TODO: Describe race condition current_mem = 0 #TODO(dynmem): do the appropriate check involving MINMEM miss_mem = (self.be_new[constants.BE_MAXMEM] - current_mem - pnhvinfo["memory_free"]) if miss_mem > 0: raise errors.OpPrereqError("This change will prevent the instance" " from starting, due to %d MB of memory" " missing on its primary node" % miss_mem, errors.ECODE_NORES) if self.be_new[constants.BE_AUTO_BALANCE]: secondary_nodes = \ self.cfg.GetInstanceSecondaryNodes(self.instance.uuid) for node_uuid, nres in nodeinfo.items(): if node_uuid not in secondary_nodes: continue nres.Raise("Can't get info from secondary node %s" % self.cfg.GetNodeName(node_uuid), prereq=True, ecode=errors.ECODE_STATE) (_, _, (nhvinfo, )) = nres.payload if not isinstance(nhvinfo.get("memory_free", None), int): raise errors.OpPrereqError("Secondary node %s didn't return free" " memory information" % self.cfg.GetNodeName(node_uuid), errors.ECODE_STATE) #TODO(dynmem): do the appropriate check involving MINMEM elif self.be_new[constants.BE_MAXMEM] > nhvinfo["memory_free"]: raise errors.OpPrereqError("This change will prevent the instance" " from failover to its secondary node" " %s, due to not enough memory" % self.cfg.GetNodeName(node_uuid), errors.ECODE_STATE) if self.op.runtime_mem: remote_info = self.rpc.call_instance_info( self.instance.primary_node, self.instance.name, self.instance.hypervisor, cluster_hvparams) remote_info.Raise("Error checking node %s" % self.cfg.GetNodeName(self.instance.primary_node), prereq=True) if not remote_info.payload: # not running already raise errors.OpPrereqError("Instance %s is not running" % self.instance.name, errors.ECODE_STATE) current_memory = remote_info.payload["memory"] if (not self.op.force and (self.op.runtime_mem > self.be_proposed[constants.BE_MAXMEM] or self.op.runtime_mem < self.be_proposed[constants.BE_MINMEM])): raise errors.OpPrereqError("Instance %s must have memory between %d" " and %d MB of memory unless --force is" " given" % (self.instance.name, self.be_proposed[constants.BE_MINMEM], self.be_proposed[constants.BE_MAXMEM]), errors.ECODE_INVAL) delta = self.op.runtime_mem - current_memory if delta > 0: CheckNodeFreeMemory( self, self.instance.primary_node, "ballooning memory for instance %s" % self.instance.name, delta, self.instance.hypervisor, self.cfg.GetClusterInfo().hvparams[self.instance.hypervisor]) def CheckPrereq(self): """Check prerequisites. This only checks the instance list against the existing names. """ assert self.op.instance_name in self.owned_locks(locking.LEVEL_INSTANCE) self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) self.cluster = self.cfg.GetClusterInfo() cluster_hvparams = self.cluster.hvparams[self.instance.hypervisor] self.op.disks = self._LookupDiskMods() assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name self.warn = [] if (self.op.pnode_uuid is not None and self.op.pnode_uuid != self.instance.primary_node and not self.op.force): instance_info = self._GetInstanceInfo(cluster_hvparams) if instance_info.fail_msg: self.warn.append("Can't get instance runtime information: %s" % instance_info.fail_msg) elif instance_info.payload: raise errors.OpPrereqError( "Instance is still running on %s" % self.cfg.GetNodeName(self.instance.primary_node), errors.ECODE_STATE) pnode_uuid = self.instance.primary_node assert pnode_uuid in self.owned_locks(locking.LEVEL_NODE) node_uuids = list(self.cfg.GetInstanceNodes(self.instance.uuid)) pnode_info = self.cfg.GetNodeInfo(pnode_uuid) assert pnode_info.group in self.owned_locks(locking.LEVEL_NODEGROUP) group_info = self.cfg.GetNodeGroup(pnode_info.group) # dictionary with instance information after the modification ispec = {} if self.instance.admin_state == constants.ADMINST_UP: self._CheckHotplug() else: self.op.hotplug = False self._PrepareNicCommunication() # disks processing assert not (self.op.disk_template and self.op.disks), \ "Can't modify disk template and apply disk changes at the same time" if self.op.disk_template: self._PreCheckDiskTemplate(pnode_info) self._PreCheckDisks(ispec) self._ProcessHVParams(node_uuids) be_old = self._ProcessBeParams() self._ValidateCpuParams() self._ProcessOsParams(node_uuids) self._ProcessMem(cluster_hvparams, be_old, pnode_uuid) # make self.cluster visible in the functions below cluster = self.cluster def _PrepareNicCreate(_, params, private): self._PrepareNicModification(params, private, None, None, {}, cluster, pnode_uuid) return (None, None) def _PrepareNicAttach(_, __, ___): raise errors.OpPrereqError("Attach operation is not supported for NICs", errors.ECODE_INVAL) def _PrepareNicMod(_, nic, params, private): self._PrepareNicModification(params, private, nic.ip, nic.network, nic.nicparams, cluster, pnode_uuid) return None def _PrepareNicRemove(_, params, __): ip = params.ip net = params.network if net is not None and ip is not None: self.cfg.ReleaseIp(net, ip, self.proc.GetECId()) def _PrepareNicDetach(_, __, ___): raise errors.OpPrereqError("Detach operation is not supported for NICs", errors.ECODE_INVAL) # Verify NIC changes (operating on copy) nics = [nic.Copy() for nic in self.instance.nics] ApplyContainerMods("NIC", nics, None, self.nicmod, _PrepareNicCreate, _PrepareNicAttach, _PrepareNicMod, _PrepareNicRemove, _PrepareNicDetach) if len(nics) > constants.MAX_NICS: raise errors.OpPrereqError("Instance has too many network interfaces" " (%d), cannot add more" % constants.MAX_NICS, errors.ECODE_STATE) # Pre-compute NIC changes (necessary to use result in hooks) self._nic_chgdesc = [] if self.nicmod: # Operate on copies as this is still in prereq nics = [nic.Copy() for nic in self.instance.nics] ApplyContainerMods("NIC", nics, self._nic_chgdesc, self.nicmod, self._CreateNewNic, None, self._ApplyNicMods, self._RemoveNic, None) # Verify that NIC names are unique and valid utils.ValidateDeviceNames("NIC", nics) self._new_nics = nics ispec[constants.ISPEC_NIC_COUNT] = len(self._new_nics) else: self._new_nics = None ispec[constants.ISPEC_NIC_COUNT] = len(self.instance.nics) if not self.op.ignore_ipolicy: ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(self.cluster, group_info) # Fill ispec with backend parameters ispec[constants.ISPEC_SPINDLE_USE] = \ self.be_new.get(constants.BE_SPINDLE_USE, None) ispec[constants.ISPEC_CPU_COUNT] = self.be_new.get(constants.BE_VCPUS, None) # Copy ispec to verify parameters with min/max values separately if self.op.disk_template: count = ispec[constants.ISPEC_DISK_COUNT] new_disk_types = [self.op.disk_template] * count else: old_disks = self.cfg.GetInstanceDisks(self.instance.uuid) add_disk_count = ispec[constants.ISPEC_DISK_COUNT] - len(old_disks) dev_type = self.cfg.GetInstanceDiskTemplate(self.instance.uuid) if dev_type == constants.DT_DISKLESS and add_disk_count != 0: raise errors.ProgrammerError( "Conversion from diskless instance not possible and should have" " been caught") new_disk_types = ([d.dev_type for d in old_disks] + [dev_type] * add_disk_count) ispec_max = ispec.copy() ispec_max[constants.ISPEC_MEM_SIZE] = \ self.be_new.get(constants.BE_MAXMEM, None) res_max = ComputeIPolicyInstanceSpecViolation(ipolicy, ispec_max, new_disk_types) ispec_min = ispec.copy() ispec_min[constants.ISPEC_MEM_SIZE] = \ self.be_new.get(constants.BE_MINMEM, None) res_min = ComputeIPolicyInstanceSpecViolation(ipolicy, ispec_min, new_disk_types) if res_max or res_min: # FIXME: Improve error message by including information about whether # the upper or lower limit of the parameter fails the ipolicy. msg = ("Instance allocation to group %s (%s) violates policy: %s" % (group_info, group_info.name, utils.CommaJoin(set(res_max + res_min)))) raise errors.OpPrereqError(msg, errors.ECODE_INVAL) def _ConvertInstanceDisks(self, feedback_fn): """Converts the disks of an instance to another type. This function converts the disks of an instance. It supports conversions among all the available disk types except conversions between the LVM-based disk types, that use their separate code path. Also, this method does not support conversions that include the 'diskless' template and those targeting the 'blockdev' template. @type feedback_fn: callable @param feedback_fn: function used to send feedback back to the caller @rtype: NoneType @return: None @raise errors.OpPrereqError: in case of failure """ template_info = self.op.disk_template if self.op.disk_template == constants.DT_EXT: template_info = ":".join([self.op.disk_template, self.op.ext_params["provider"]]) old_template = self.cfg.GetInstanceDiskTemplate(self.instance.uuid) feedback_fn("Converting disk template from '%s' to '%s'" % (old_template, template_info)) assert not (old_template in constants.DTS_NOT_CONVERTIBLE_FROM or self.op.disk_template in constants.DTS_NOT_CONVERTIBLE_TO), \ ("Unsupported disk template conversion from '%s' to '%s'" % (old_template, self.op.disk_template)) pnode_uuid = self.instance.primary_node snode_uuid = [] if self.op.remote_node_uuid: snode_uuid = [self.op.remote_node_uuid] old_disks = self.cfg.GetInstanceDisks(self.instance.uuid) feedback_fn("Generating new '%s' disk template..." % template_info) file_storage_dir = CalculateFileStorageDir( self.op.disk_template, self.cfg, self.instance.name, file_storage_dir=self.op.file_storage_dir) new_disks = GenerateDiskTemplate(self, self.op.disk_template, self.instance.uuid, pnode_uuid, snode_uuid, self.disks_info, file_storage_dir, self.op.file_driver, 0, feedback_fn, self.diskparams) # Create the new block devices for the instance. feedback_fn("Creating new empty disks of type '%s'..." % template_info) try: CreateDisks(self, self.instance, disk_template=self.op.disk_template, disks=new_disks) except errors.OpExecError: self.LogWarning("Device creation failed") for disk in new_disks: self.cfg.ReleaseDRBDMinors(disk.uuid) raise # Transfer the data from the old to the newly created disks of the instance. feedback_fn("Populating the new empty disks of type '%s'..." % template_info) for idx, (old, new) in enumerate(zip(old_disks, new_disks)): feedback_fn(" - copying data from disk %s (%s), size %s" % (idx, old.dev_type, utils.FormatUnit(new.size, "h"))) if old.dev_type == constants.DT_DRBD8: old = old.children[0] result = self.rpc.call_blockdev_convert(pnode_uuid, (old, self.instance), (new, self.instance)) msg = result.fail_msg if msg: # A disk failed to copy. Abort the conversion operation and rollback # the modifications to the previous state. The instance will remain # intact. if self.op.disk_template == constants.DT_DRBD8: new = new.children[0] self.Log(" - ERROR: Could not copy disk '%s' to '%s'" % (old.logical_id[1], new.logical_id[1])) try: self.LogInfo("Some disks failed to copy") self.LogInfo("The instance will not be affected, aborting operation") self.LogInfo("Removing newly created disks of type '%s'..." % template_info) RemoveDisks(self, self.instance, disks=new_disks) self.LogInfo("Newly created disks removed successfully") finally: for disk in new_disks: self.cfg.ReleaseDRBDMinors(disk.uuid) result.Raise("Error while converting the instance's template") # In case of DRBD disk, return its port to the pool for disk in old_disks: if disk.dev_type == constants.DT_DRBD8: tcp_port = disk.logical_id[2] self.cfg.AddTcpUdpPort(tcp_port) # Remove old disks from the instance. feedback_fn("Detaching old disks (%s) from the instance and removing" " them from cluster config" % old_template) for old_disk in old_disks: self.cfg.RemoveInstanceDisk(self.instance.uuid, old_disk.uuid) # Attach the new disks to the instance. feedback_fn("Adding new disks (%s) to cluster config and attaching" " them to the instance" % template_info) for (idx, new_disk) in enumerate(new_disks): self.cfg.AddInstanceDisk(self.instance.uuid, new_disk, idx=idx) # Re-read the instance from the configuration. self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) # Release node locks while waiting for sync and disks removal. ReleaseLocks(self, locking.LEVEL_NODE) disk_abort = not WaitForSync(self, self.instance, oneshot=not self.op.wait_for_sync) if disk_abort: raise errors.OpExecError("There are some degraded disks for" " this instance, please cleanup manually") feedback_fn("Removing old block devices of type '%s'..." % old_template) RemoveDisks(self, self.instance, disks=old_disks) # Node resource locks will be released by the caller. def _ConvertPlainToDrbd(self, feedback_fn): """Converts an instance from plain to drbd. """ feedback_fn("Converting disk template from 'plain' to 'drbd'") if not self.op.remote_node_uuid: feedback_fn("Using %s to choose new secondary" % self.op.iallocator) req = iallocator.IAReqInstanceAllocateSecondary( name=self.op.instance_name) ial = iallocator.IAllocator(self.cfg, self.rpc, req) ial.Run(self.op.iallocator) if not ial.success: raise errors.OpPrereqError("Can's find secondary node using" " iallocator %s: %s" % (self.op.iallocator, ial.info), errors.ECODE_NORES) feedback_fn("%s choose %s as new secondary" % (self.op.iallocator, ial.result)) self.op.remote_node = ial.result self.op.remote_node_uuid = self.cfg.GetNodeInfoByName(ial.result).uuid pnode_uuid = self.instance.primary_node snode_uuid = self.op.remote_node_uuid old_disks = self.cfg.GetInstanceDisks(self.instance.uuid) assert utils.AnyDiskOfType(old_disks, [constants.DT_PLAIN]) new_disks = GenerateDiskTemplate(self, self.op.disk_template, self.instance.uuid, pnode_uuid, [snode_uuid], self.disks_info, None, None, 0, feedback_fn, self.diskparams) anno_disks = rpc.AnnotateDiskParams(new_disks, self.diskparams) p_excl_stor = IsExclusiveStorageEnabledNodeUuid(self.cfg, pnode_uuid) s_excl_stor = IsExclusiveStorageEnabledNodeUuid(self.cfg, snode_uuid) info = GetInstanceInfoText(self.instance) feedback_fn("Creating additional volumes...") # first, create the missing data and meta devices for disk in anno_disks: # unfortunately this is... not too nice CreateSingleBlockDev(self, pnode_uuid, self.instance, disk.children[1], info, True, p_excl_stor) for child in disk.children: CreateSingleBlockDev(self, snode_uuid, self.instance, child, info, True, s_excl_stor) # at this stage, all new LVs have been created, we can rename the # old ones feedback_fn("Renaming original volumes...") rename_list = [(o, n.children[0].logical_id) for (o, n) in zip(old_disks, new_disks)] result = self.rpc.call_blockdev_rename(pnode_uuid, rename_list) result.Raise("Failed to rename original LVs") feedback_fn("Initializing DRBD devices...") # all child devices are in place, we can now create the DRBD devices try: for disk in anno_disks: for (node_uuid, excl_stor) in [(pnode_uuid, p_excl_stor), (snode_uuid, s_excl_stor)]: f_create = node_uuid == pnode_uuid CreateSingleBlockDev(self, node_uuid, self.instance, disk, info, f_create, excl_stor) except errors.GenericError as e: feedback_fn("Initializing of DRBD devices failed;" " renaming back original volumes...") rename_back_list = [(n.children[0], o.logical_id) for (n, o) in zip(new_disks, old_disks)] result = self.rpc.call_blockdev_rename(pnode_uuid, rename_back_list) result.Raise("Failed to rename LVs back after error %s" % str(e)) raise # Remove the old disks from the instance for old_disk in old_disks: self.cfg.RemoveInstanceDisk(self.instance.uuid, old_disk.uuid) # Attach the new disks to the instance for (idx, new_disk) in enumerate(new_disks): self.cfg.AddInstanceDisk(self.instance.uuid, new_disk, idx=idx) # re-read the instance from the configuration self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) # Release node locks while waiting for sync ReleaseLocks(self, locking.LEVEL_NODE) # disks are created, waiting for sync disk_abort = not WaitForSync(self, self.instance, oneshot=not self.op.wait_for_sync) if disk_abort: raise errors.OpExecError("There are some degraded disks for" " this instance, please cleanup manually") # Node resource locks will be released by caller def _ConvertDrbdToPlain(self, feedback_fn): """Converts an instance from drbd to plain. """ secondary_nodes = self.cfg.GetInstanceSecondaryNodes(self.instance.uuid) disks = self.cfg.GetInstanceDisks(self.instance.uuid) assert len(secondary_nodes) == 1 assert utils.AnyDiskOfType(disks, [constants.DT_DRBD8]) feedback_fn("Converting disk template from 'drbd' to 'plain'") old_disks = AnnotateDiskParams(self.instance, disks, self.cfg) new_disks = [d.children[0] for d in disks] # copy over size, mode and name and set the correct nodes for parent, child in zip(old_disks, new_disks): child.size = parent.size child.mode = parent.mode child.name = parent.name child.nodes = [self.instance.primary_node] # this is a DRBD disk, return its port to the pool for disk in old_disks: tcp_port = disk.logical_id[2] self.cfg.AddTcpUdpPort(tcp_port) # Remove the old disks from the instance for old_disk in old_disks: self.cfg.RemoveInstanceDisk(self.instance.uuid, old_disk.uuid) # Attach the new disks to the instance for (idx, new_disk) in enumerate(new_disks): self.cfg.AddInstanceDisk(self.instance.uuid, new_disk, idx=idx) # re-read the instance from the configuration self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) # Release locks in case removing disks takes a while ReleaseLocks(self, locking.LEVEL_NODE) feedback_fn("Removing volumes on the secondary node...") RemoveDisks(self, self.instance, disks=old_disks, target_node_uuid=secondary_nodes[0]) feedback_fn("Removing unneeded volumes on the primary node...") meta_disks = [] for idx, disk in enumerate(old_disks): meta_disks.append(disk.children[1]) RemoveDisks(self, self.instance, disks=meta_disks) def _HotplugDevice(self, action, dev_type, device, extra, seq): self.LogInfo("Trying to hotplug device...") msg = "hotplug:" result = self.rpc.call_hotplug_device(self.instance.primary_node, self.instance, action, dev_type, (device, self.instance), extra, seq) if result.fail_msg: self.LogWarning("Could not hotplug device: %s" % result.fail_msg) self.LogInfo("Continuing execution..") msg += "failed" else: self.LogInfo("Hotplug done.") msg += "done" return msg def _FillFileDriver(self): if not self.op.file_driver: self.op.file_driver = constants.FD_DEFAULT elif self.op.file_driver not in constants.FILE_DRIVER: raise errors.OpPrereqError("Invalid file driver name '%s'" % self.op.file_driver, errors.ECODE_INVAL) def _GenerateDiskTemplateWrapper(self, idx, disk_type, params): file_path = CalculateFileStorageDir( disk_type, self.cfg, self.instance.name, file_storage_dir=self.op.file_storage_dir) self._FillFileDriver() secondary_nodes = self.cfg.GetInstanceSecondaryNodes(self.instance.uuid) return \ GenerateDiskTemplate(self, disk_type, self.instance.uuid, self.instance.primary_node, secondary_nodes, [params], file_path, self.op.file_driver, idx, self.Log, self.diskparams)[0] def _CreateNewDisk(self, idx, params, _): """Creates a new disk. """ # add a new disk disk_template = self.cfg.GetInstanceDiskTemplate(self.instance.uuid) disk = self._GenerateDiskTemplateWrapper(idx, disk_template, params) new_disks = CreateDisks(self, self.instance, disks=[disk]) self.cfg.AddInstanceDisk(self.instance.uuid, disk, idx) # re-read the instance from the configuration self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) if self.cluster.prealloc_wipe_disks: # Wipe new disk WipeOrCleanupDisks(self, self.instance, disks=[(idx, disk, 0)], cleanup=new_disks) changes = [ ("disk/%d" % idx, "add:size=%s,mode=%s" % (disk.size, disk.mode)), ] if self.op.hotplug: result = self.rpc.call_blockdev_assemble(self.instance.primary_node, (disk, self.instance), self.instance, True, idx) if result.fail_msg: changes.append(("disk/%d" % idx, "assemble:failed")) self.LogWarning("Can't assemble newly created disk %d: %s", idx, result.fail_msg) else: _, link_name, uri = result.payload msg = self._HotplugDevice(constants.HOTPLUG_ACTION_ADD, constants.HOTPLUG_TARGET_DISK, disk, (link_name, uri), idx) changes.append(("disk/%d" % idx, msg)) return (disk, changes) def _PostAddDisk(self, _, disk): if not WaitForSync(self, self.instance, disks=[disk], oneshot=not self.op.wait_for_sync): raise errors.OpExecError("Failed to sync disks of %s" % self.instance.name) # the disk is active at this point, so deactivate it if the instance disks # are supposed to be inactive if not self.instance.disks_active: ShutdownInstanceDisks(self, self.instance, disks=[disk]) def _AttachDisk(self, idx, params, _): """Attaches an existing disk to an instance. """ uuid = params.get("uuid", None) name = params.get(constants.IDISK_NAME, None) disk = self.GenericGetDiskInfo(uuid, name) # Rename disk before attaching (if disk is filebased) if disk.dev_type in constants.DTS_INSTANCE_DEPENDENT_PATH: # Add disk size/mode, else GenerateDiskTemplate will not work. params[constants.IDISK_SIZE] = disk.size params[constants.IDISK_MODE] = str(disk.mode) dummy_disk = self._GenerateDiskTemplateWrapper(idx, disk.dev_type, params) new_logical_id = dummy_disk.logical_id result = self.rpc.call_blockdev_rename(self.instance.primary_node, [(disk, new_logical_id)]) result.Raise("Failed before attach") self.cfg.SetDiskLogicalID(disk.uuid, new_logical_id) disk.logical_id = new_logical_id # Attach disk to instance self.cfg.AttachInstanceDisk(self.instance.uuid, disk.uuid, idx) # re-read the instance from the configuration self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) changes = [ ("disk/%d" % idx, "attach:size=%s,mode=%s" % (disk.size, disk.mode)), ] disks_ok, _, payloads = AssembleInstanceDisks(self, self.instance, disks=[disk]) if not disks_ok: changes.append(("disk/%d" % idx, "assemble:failed")) return disk, changes if self.op.hotplug: _, link_name, uri = payloads[0] msg = self._HotplugDevice(constants.HOTPLUG_ACTION_ADD, constants.HOTPLUG_TARGET_DISK, disk, (link_name, uri), idx) changes.append(("disk/%d" % idx, msg)) return (disk, changes) def _ModifyDisk(self, idx, disk, params, _): """Modifies a disk. """ changes = [] if constants.IDISK_MODE in params: disk.mode = params.get(constants.IDISK_MODE) changes.append(("disk.mode/%d" % idx, disk.mode)) if constants.IDISK_NAME in params: disk.name = params.get(constants.IDISK_NAME) changes.append(("disk.name/%d" % idx, disk.name)) # Modify arbitrary params in case instance template is ext for key, value in params.items(): if (key not in constants.MODIFIABLE_IDISK_PARAMS and disk.dev_type == constants.DT_EXT): # stolen from GetUpdatedParams: default means reset/delete if value.lower() == constants.VALUE_DEFAULT: try: del disk.params[key] except KeyError: pass else: disk.params[key] = value changes.append(("disk.params:%s/%d" % (key, idx), value)) # Update disk object self.cfg.Update(disk, self.feedback_fn) return changes def _RemoveDisk(self, idx, root, _): """Removes a disk. """ hotmsg = "" if self.op.hotplug: hotmsg = self._HotplugDevice(constants.HOTPLUG_ACTION_REMOVE, constants.HOTPLUG_TARGET_DISK, root, None, idx) ShutdownInstanceDisks(self, self.instance, [root]) RemoveDisks(self, self.instance, disks=[root]) # if this is a DRBD disk, return its port to the pool if root.dev_type in constants.DTS_DRBD: self.cfg.AddTcpUdpPort(root.logical_id[2]) # Remove disk from config self.cfg.RemoveInstanceDisk(self.instance.uuid, root.uuid) # re-read the instance from the configuration self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) return hotmsg def _DetachDisk(self, idx, root, _): """Detaches a disk from an instance. """ hotmsg = "" if self.op.hotplug: hotmsg = self._HotplugDevice(constants.HOTPLUG_ACTION_REMOVE, constants.HOTPLUG_TARGET_DISK, root, None, idx) # Always shutdown the disk before detaching. ShutdownInstanceDisks(self, self.instance, [root]) # Rename detached disk. # # Transform logical_id from: # // # to # / if root.dev_type in (constants.DT_FILE, constants.DT_SHARED_FILE): file_driver = root.logical_id[0] instance_path, disk_name = os.path.split(root.logical_id[1]) new_path = os.path.join(os.path.dirname(instance_path), disk_name) new_logical_id = (file_driver, new_path) result = self.rpc.call_blockdev_rename(self.instance.primary_node, [(root, new_logical_id)]) result.Raise("Failed before detach") # Update logical_id self.cfg.SetDiskLogicalID(root.uuid, new_logical_id) # Remove disk from config self.cfg.DetachInstanceDisk(self.instance.uuid, root.uuid) # re-read the instance from the configuration self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) return hotmsg def _CreateNewNic(self, idx, params, private): """Creates data structure for a new network interface. """ mac = params[constants.INIC_MAC] ip = params.get(constants.INIC_IP, None) net = params.get(constants.INIC_NETWORK, None) name = params.get(constants.INIC_NAME, None) net_uuid = self.cfg.LookupNetwork(net) #TODO: not private.filled?? can a nic have no nicparams?? nicparams = private.filled nobj = objects.NIC(mac=mac, ip=ip, network=net_uuid, name=name, nicparams=nicparams) nobj.uuid = self.cfg.GenerateUniqueID(self.proc.GetECId()) changes = [ ("nic.%d" % idx, "add:mac=%s,ip=%s,mode=%s,link=%s,network=%s" % (mac, ip, private.filled[constants.NIC_MODE], private.filled[constants.NIC_LINK], net)), ] if self.op.hotplug: msg = self._HotplugDevice(constants.HOTPLUG_ACTION_ADD, constants.HOTPLUG_TARGET_NIC, nobj, None, idx) changes.append(("nic.%d" % idx, msg)) return (nobj, changes) def _ApplyNicMods(self, idx, nic, params, private): """Modifies a network interface. """ changes = [] for key in [constants.INIC_MAC, constants.INIC_IP, constants.INIC_NAME]: if key in params: changes.append(("nic.%s/%d" % (key, idx), params[key])) setattr(nic, key, params[key]) new_net = params.get(constants.INIC_NETWORK, nic.network) new_net_uuid = self.cfg.LookupNetwork(new_net) if new_net_uuid != nic.network: changes.append(("nic.network/%d" % idx, new_net)) nic.network = new_net_uuid if private.filled: nic.nicparams = private.filled for (key, val) in nic.nicparams.items(): changes.append(("nic.%s/%d" % (key, idx), val)) if self.op.hotplug: msg = self._HotplugDevice(constants.HOTPLUG_ACTION_MODIFY, constants.HOTPLUG_TARGET_NIC, nic, None, idx) changes.append(("nic/%d" % idx, msg)) return changes def _RemoveNic(self, idx, nic, _): if self.op.hotplug: return self._HotplugDevice(constants.HOTPLUG_ACTION_REMOVE, constants.HOTPLUG_TARGET_NIC, nic, None, idx) def Exec(self, feedback_fn): """Modifies an instance. All parameters take effect only at the next restart of the instance. """ self.feedback_fn = feedback_fn # Process here the warnings from CheckPrereq, as we don't have a # feedback_fn there. # TODO: Replace with self.LogWarning for warn in self.warn: feedback_fn("WARNING: %s" % warn) assert ((self.op.disk_template is None) ^ bool(self.owned_locks(locking.LEVEL_NODE_RES))), \ "Not owning any node resource locks" result = [] # New primary node if self.op.pnode_uuid: self.instance.primary_node = self.op.pnode_uuid # runtime memory if self.op.runtime_mem: rpcres = self.rpc.call_instance_balloon_memory(self.instance.primary_node, self.instance, self.op.runtime_mem) rpcres.Raise("Cannot modify instance runtime memory") result.append(("runtime_memory", self.op.runtime_mem)) # Apply disk changes inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) ApplyContainerMods("disk", inst_disks, result, self.diskmod, self._CreateNewDisk, self._AttachDisk, self._ModifyDisk, self._RemoveDisk, self._DetachDisk, post_add_fn=self._PostAddDisk) if self.op.disk_template: if __debug__: check_nodes = set(self.cfg.GetInstanceNodes(self.instance.uuid)) if self.op.remote_node_uuid: check_nodes.add(self.op.remote_node_uuid) for level in [locking.LEVEL_NODE, locking.LEVEL_NODE_RES]: owned = self.owned_locks(level) assert not (check_nodes - owned), \ ("Not owning the correct locks, owning %r, expected at least %r" % (owned, check_nodes)) r_shut = ShutdownInstanceDisks(self, self.instance) if not r_shut: raise errors.OpExecError("Cannot shutdown instance disks, unable to" " proceed with disk template conversion") #TODO make heterogeneous conversions work mode = (self.cfg.GetInstanceDiskTemplate(self.instance.uuid), self.op.disk_template) try: if mode in self._DISK_CONVERSIONS: self._DISK_CONVERSIONS[mode](self, feedback_fn) else: self._ConvertInstanceDisks(feedback_fn) except: for disk in inst_disks: self.cfg.ReleaseDRBDMinors(disk.uuid) raise result.append(("disk_template", self.op.disk_template)) disk_info = self.cfg.GetInstanceDisks(self.instance.uuid) assert utils.AllDiskOfType(disk_info, [self.op.disk_template]), \ ("Expected disk template '%s', found '%s'" % (self.op.disk_template, self.cfg.GetInstanceDiskTemplate(self.instance.uuid))) # Release node and resource locks if there are any (they might already have # been released during disk conversion) ReleaseLocks(self, locking.LEVEL_NODE) ReleaseLocks(self, locking.LEVEL_NODE_RES) # Apply NIC changes if self._new_nics is not None: self.instance.nics = self._new_nics result.extend(self._nic_chgdesc) # hvparams changes if self.op.hvparams: self.instance.hvparams = self.hv_inst for key, val in self.op.hvparams.items(): result.append(("hv/%s" % key, val)) # beparams changes if self.op.beparams: self.instance.beparams = self.be_inst for key, val in self.op.beparams.items(): result.append(("be/%s" % key, val)) # OS change if self.op.os_name: self.instance.os = self.op.os_name # osparams changes if self.op.osparams: self.instance.osparams = self.os_inst for key, val in self.op.osparams.items(): result.append(("os/%s" % key, val)) if self.op.osparams_private: self.instance.osparams_private = self.os_inst_private for key, val in self.op.osparams_private.items(): # Show the Private(...) blurb. result.append(("os_private/%s" % key, repr(val))) self.cfg.Update(self.instance, feedback_fn, self.proc.GetECId()) if self.op.offline is None: # Ignore pass elif self.op.offline: # Mark instance as offline self.instance = self.cfg.MarkInstanceOffline(self.instance.uuid) result.append(("admin_state", constants.ADMINST_OFFLINE)) else: # Mark instance as online, but stopped self.instance = self.cfg.MarkInstanceDown(self.instance.uuid) result.append(("admin_state", constants.ADMINST_DOWN)) UpdateMetadata(feedback_fn, self.rpc, self.instance) assert not (self.owned_locks(locking.LEVEL_NODE_RES) or self.owned_locks(locking.LEVEL_NODE)), \ "All node locks should have been released by now" return result _DISK_CONVERSIONS = { (constants.DT_PLAIN, constants.DT_DRBD8): _ConvertPlainToDrbd, (constants.DT_DRBD8, constants.DT_PLAIN): _ConvertDrbdToPlain, } ganeti-3.1.0~rc2/lib/cmdlib/instance_storage.py000064400000000000000000003603071476477700300215170ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with storage of instances.""" import itertools import logging import os import time from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import ht from ganeti import locking from ganeti.masterd import iallocator from ganeti import objects from ganeti import utils import ganeti.rpc.node as rpc from ganeti.cmdlib.base import LogicalUnit, NoHooksLU, Tasklet from ganeti.cmdlib.common import INSTANCE_DOWN, INSTANCE_NOT_RUNNING, \ AnnotateDiskParams, CheckIAllocatorOrNode, ExpandNodeUuidAndName, \ ComputeIPolicyDiskSizesViolation, \ CheckNodeOnline, CheckInstanceNodeGroups, CheckInstanceState, \ IsExclusiveStorageEnabledNode, FindFaultyInstanceDisks, GetWantedNodes, \ CheckDiskTemplateEnabled, IsInstanceRunning from ganeti.cmdlib.instance_utils import GetInstanceInfoText, \ CopyLockList, ReleaseLocks, CheckNodeVmCapable, \ BuildInstanceHookEnvByObject, CheckNodeNotDrained, CheckTargetNodeIPolicy import ganeti.masterd.instance _DISK_TEMPLATE_NAME_PREFIX = { constants.DT_PLAIN: "", constants.DT_RBD: ".rbd", constants.DT_EXT: ".ext", constants.DT_FILE: ".file", constants.DT_SHARED_FILE: ".sharedfile", } def CreateSingleBlockDev(lu, node_uuid, instance, device, info, force_open, excl_stor): """Create a single block device on a given node. This will not recurse over children of the device, so they must be created in advance. @param lu: the lu on whose behalf we execute @param node_uuid: the node on which to create the device @type instance: L{objects.Instance} @param instance: the instance which owns the device @type device: L{objects.Disk} @param device: the device to create @param info: the extra 'metadata' we should attach to the device (this will be represented as a LVM tag) @type force_open: boolean @param force_open: this parameter will be passes to the L{backend.BlockdevCreate} function where it specifies whether we run on primary or not, and it affects both the child assembly and the device own Open() execution @type excl_stor: boolean @param excl_stor: Whether exclusive_storage is active for the node """ result = lu.rpc.call_blockdev_create(node_uuid, (device, instance), device.size, instance.name, force_open, info, excl_stor) result.Raise("Can't create block device %s on" " node %s for instance %s" % (device, lu.cfg.GetNodeName(node_uuid), instance.name)) def _CreateBlockDevInner(lu, node_uuid, instance, device, force_create, info, force_open, excl_stor): """Create a tree of block devices on a given node. If this device type has to be created on secondaries, create it and all its children. If not, just recurse to children keeping the same 'force' value. @attention: The device has to be annotated already. @param lu: the lu on whose behalf we execute @param node_uuid: the node on which to create the device @type instance: L{objects.Instance} @param instance: the instance which owns the device @type device: L{objects.Disk} @param device: the device to create @type force_create: boolean @param force_create: whether to force creation of this device; this will be change to True whenever we find a device which has CreateOnSecondary() attribute @param info: the extra 'metadata' we should attach to the device (this will be represented as a LVM tag) @type force_open: boolean @param force_open: this parameter will be passes to the L{backend.BlockdevCreate} function where it specifies whether we run on primary or not, and it affects both the child assembly and the device own Open() execution @type excl_stor: boolean @param excl_stor: Whether exclusive_storage is active for the node @return: list of created devices """ created_devices = [] try: if device.CreateOnSecondary(): force_create = True if device.children: for child in device.children: devs = _CreateBlockDevInner(lu, node_uuid, instance, child, force_create, info, force_open, excl_stor) created_devices.extend(devs) if not force_create: return created_devices CreateSingleBlockDev(lu, node_uuid, instance, device, info, force_open, excl_stor) # The device has been completely created, so there is no point in keeping # its subdevices in the list. We just add the device itself instead. created_devices = [(node_uuid, device)] return created_devices except errors.DeviceCreationError as e: e.created_devices.extend(created_devices) raise e except errors.OpExecError as e: raise errors.DeviceCreationError(str(e), created_devices) def IsExclusiveStorageEnabledNodeUuid(cfg, node_uuid): """Whether exclusive_storage is in effect for the given node. @type cfg: L{config.ConfigWriter} @param cfg: The cluster configuration @type node_uuid: string @param node_uuid: The node UUID @rtype: bool @return: The effective value of exclusive_storage @raise errors.OpPrereqError: if no node exists with the given name """ ni = cfg.GetNodeInfo(node_uuid) if ni is None: raise errors.OpPrereqError("Invalid node UUID %s" % node_uuid, errors.ECODE_NOENT) return IsExclusiveStorageEnabledNode(cfg, ni) def _CreateBlockDev(lu, node_uuid, instance, device, force_create, info, force_open): """Wrapper around L{_CreateBlockDevInner}. This method annotates the root device first. """ (disk,) = AnnotateDiskParams(instance, [device], lu.cfg) excl_stor = IsExclusiveStorageEnabledNodeUuid(lu.cfg, node_uuid) return _CreateBlockDevInner(lu, node_uuid, instance, disk, force_create, info, force_open, excl_stor) def _UndoCreateDisks(lu, disks_created, instance): """Undo the work performed by L{CreateDisks}. This function is called in case of an error to undo the work of L{CreateDisks}. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @param disks_created: the result returned by L{CreateDisks} @type instance: L{objects.Instance} @param instance: the instance for which disks were created """ for (node_uuid, disk) in disks_created: result = lu.rpc.call_blockdev_remove(node_uuid, (disk, instance)) result.Warn("Failed to remove newly-created disk %s on node %s" % (disk, lu.cfg.GetNodeName(node_uuid)), logging.warning) def CreateDisks(lu, instance, disk_template=None, to_skip=None, target_node_uuid=None, disks=None): """Create all disks for an instance. This abstracts away some work from AddInstance. Since the instance may not have been saved to the config file yet, this function can not query the config file for the instance's disks; in that case they need to be passed as an argument. This function is also used by the disk template conversion mechanism to create the new disks of the instance. Since the instance will have the old template at the time we create the new disks, the new template must be passed as an extra argument. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type instance: L{objects.Instance} @param instance: the instance whose disks we should create @type disk_template: string @param disk_template: if provided, overrides the instance's disk_template @type to_skip: list @param to_skip: list of indices to skip @type target_node_uuid: string @param target_node_uuid: if passed, overrides the target node for creation @type disks: list of {objects.Disk} @param disks: the disks to create; if not specified, all the disks of the instance are created @return: information about the created disks, to be used to call L{_UndoCreateDisks} @raise errors.OpPrereqError: in case of error """ info = GetInstanceInfoText(instance) if disks is None: disks = lu.cfg.GetInstanceDisks(instance.uuid) if target_node_uuid is None: pnode_uuid = instance.primary_node # We cannot use config's 'GetInstanceNodes' here as 'CreateDisks' # is used by 'LUInstanceCreate' and the instance object is not # stored in the config yet. all_node_uuids = [] for disk in disks: all_node_uuids.extend(disk.all_nodes) all_node_uuids = set(all_node_uuids) # ensure that primary node is always the first all_node_uuids.discard(pnode_uuid) all_node_uuids = [pnode_uuid] + list(all_node_uuids) else: pnode_uuid = target_node_uuid all_node_uuids = [pnode_uuid] if disk_template is None: disk_template = utils.GetDiskTemplate(disks) if disk_template == constants.DT_MIXED: raise errors.OpExecError("Creating disk for '%s' instances " "only possible with explicit disk template." % (constants.DT_MIXED,)) CheckDiskTemplateEnabled(lu.cfg.GetClusterInfo(), disk_template) if disk_template in constants.DTS_FILEBASED: file_storage_dir = os.path.dirname(disks[0].logical_id[1]) result = lu.rpc.call_file_storage_dir_create(pnode_uuid, file_storage_dir) result.Raise("Failed to create directory '%s' on" " node %s" % (file_storage_dir, lu.cfg.GetNodeName(pnode_uuid))) disks_created = [] for idx, device in enumerate(disks): if to_skip and idx in to_skip: continue logging.info("Creating disk %s for instance '%s'", idx, instance.name) for node_uuid in all_node_uuids: f_create = node_uuid == pnode_uuid try: _CreateBlockDev(lu, node_uuid, instance, device, f_create, info, f_create) disks_created.append((node_uuid, device)) except errors.DeviceCreationError as e: logging.warning("Creating disk %s for instance '%s' failed", idx, instance.name) disks_created.extend(e.created_devices) _UndoCreateDisks(lu, disks_created, instance) raise errors.OpExecError(e.message) return disks_created def ComputeDiskSizePerVG(disk_template, disks): """Compute disk size requirements in the volume group """ def _compute(disks, payload): """Universal algorithm. """ vgs = {} for disk in disks: vg_name = disk[constants.IDISK_VG] vgs[vg_name] = \ vgs.get(vg_name, 0) + disk[constants.IDISK_SIZE] + payload return vgs # Required free disk space as a function of disk and swap space req_size_dict = { constants.DT_DISKLESS: {}, constants.DT_PLAIN: _compute(disks, 0), # 128 MB are added for drbd metadata for each disk constants.DT_DRBD8: _compute(disks, constants.DRBD_META_SIZE), constants.DT_FILE: {}, constants.DT_SHARED_FILE: {}, constants.DT_GLUSTER: {}, } if disk_template not in req_size_dict: raise errors.ProgrammerError("Disk template '%s' size requirement" " is unknown" % disk_template) return req_size_dict[disk_template] def ComputeDisks(disks, disk_template, default_vg): """Computes the instance disks. @type disks: list of dictionaries @param disks: The disks' input dictionary @type disk_template: string @param disk_template: The disk template of the instance @type default_vg: string @param default_vg: The default_vg to assume @return: The computed disks """ new_disks = [] for disk in disks: mode = disk.get(constants.IDISK_MODE, constants.DISK_RDWR) if mode not in constants.DISK_ACCESS_SET: raise errors.OpPrereqError("Invalid disk access mode '%s'" % mode, errors.ECODE_INVAL) size = disk.get(constants.IDISK_SIZE, None) if size is None: raise errors.OpPrereqError("Missing disk size", errors.ECODE_INVAL) try: size = int(size) except (TypeError, ValueError): raise errors.OpPrereqError("Invalid disk size '%s'" % size, errors.ECODE_INVAL) CheckDiskExtProvider(disk, disk_template) data_vg = disk.get(constants.IDISK_VG, default_vg) name = disk.get(constants.IDISK_NAME, None) if name is not None and name.lower() == constants.VALUE_NONE: name = None new_disk = { constants.IDISK_SIZE: size, constants.IDISK_MODE: mode, constants.IDISK_VG: data_vg, constants.IDISK_NAME: name, constants.IDISK_TYPE: disk_template, } for key in [ constants.IDISK_METAVG, constants.IDISK_ADOPT, constants.IDISK_SPINDLES, ]: if key in disk: new_disk[key] = disk[key] # Add IDISK_ACCESS parameter for disk templates that support it if (disk_template in constants.DTS_HAVE_ACCESS and constants.IDISK_ACCESS in disk): new_disk[constants.IDISK_ACCESS] = disk[constants.IDISK_ACCESS] # For extstorage, demand the `provider' option and add any # additional parameters (ext-params) to the dict if disk_template == constants.DT_EXT: new_disk[constants.IDISK_PROVIDER] = disk[constants.IDISK_PROVIDER] for key in disk: if key not in constants.IDISK_PARAMS: new_disk[key] = disk[key] new_disks.append(new_disk) return new_disks def ComputeDisksInfo(disks, disk_template, default_vg, ext_params): """Computes the new instance's disks for the template conversion. This method is used by the disks template conversion mechanism. Using the 'ComputeDisks' method as an auxiliary method computes the disks that will be used for generating the new disk template of the instance. It computes the size, mode, and name parameters from the instance's current disks, such as the volume group and the access parameters for the templates that support them. For conversions targeting an extstorage template, the mandatory provider's name or any user-provided extstorage parameters will also be included in the result. @type disks: list of {objects.Disk} @param disks: The current disks of the instance @type disk_template: string @param disk_template: The disk template of the instance @type default_vg: string @param default_vg: The default volume group to assume @type ext_params: dict @param ext_params: The extstorage parameters @rtype: list of dictionaries @return: The computed disks' information for the new template """ # Ensure 'ext_params' does not violate existing disks' params for key in ext_params: if key != constants.IDISK_PROVIDER: assert key not in constants.IDISK_PARAMS, \ "Invalid extstorage parameter '%s'" % key # Prepare the disks argument for the 'ComputeDisks' method. inst_disks = [dict((key, value) for key, value in disk.items() if key in constants.IDISK_PARAMS) for disk in map(objects.Disk.ToDict, disks)] # Update disks with the user-provided 'ext_params'. for disk in inst_disks: disk.update(ext_params) # Compute the new disks' information. new_disks = ComputeDisks(inst_disks, disk_template, default_vg) # Add missing parameters to the previously computed disks. for disk, new_disk in zip(disks, new_disks): # Conversions between ExtStorage templates allowed only for different # providers. if (disk.dev_type == disk_template and disk_template == constants.DT_EXT): provider = new_disk[constants.IDISK_PROVIDER] if provider == disk.params[constants.IDISK_PROVIDER]: raise errors.OpPrereqError("Not converting, '%s' of type ExtStorage" " already using provider '%s'" % (disk.iv_name, provider), errors.ECODE_INVAL) # Add IDISK_ACCESS parameter for conversions between disk templates that # support it. if (disk_template in constants.DTS_HAVE_ACCESS and constants.IDISK_ACCESS in disk.params): new_disk[constants.IDISK_ACCESS] = disk.params[constants.IDISK_ACCESS] # For LVM-based conversions (plain <-> drbd) use the same volume group. if disk_template in constants.DTS_LVM: if disk.dev_type == constants.DT_PLAIN: new_disk[constants.IDISK_VG] = disk.logical_id[0] elif disk.dev_type == constants.DT_DRBD8: new_disk[constants.IDISK_VG] = disk.children[0].logical_id[0] return new_disks def CalculateFileStorageDir(disk_type, cfg, instance_name, file_storage_dir=None): """Calculate final instance file storage dir. @type disk_type: disk template @param disk_type: L{constants.DT_FILE}, L{constants.DT_SHARED_FILE}, or L{constants.DT_GLUSTER} @type cfg: ConfigWriter @param cfg: the configuration that is to be used. @type file_storage_dir: path @param file_storage_dir: the path below the configured base. @type instance_name: string @param instance_name: name of the instance this disk is for. @rtype: string @return: The file storage directory for the instance """ # file storage dir calculation/check instance_file_storage_dir = None if disk_type in constants.DTS_FILEBASED: # build the full file storage dir path joinargs = [] cfg_storage = None if disk_type == constants.DT_FILE: cfg_storage = cfg.GetFileStorageDir() elif disk_type == constants.DT_SHARED_FILE: cfg_storage = cfg.GetSharedFileStorageDir() elif disk_type == constants.DT_GLUSTER: cfg_storage = cfg.GetGlusterStorageDir() if not cfg_storage: raise errors.OpPrereqError( "Cluster file storage dir for {tpl} storage type not defined".format( tpl=repr(disk_type) ), errors.ECODE_STATE) joinargs.append(cfg_storage) if file_storage_dir is not None: joinargs.append(file_storage_dir) if disk_type != constants.DT_GLUSTER: joinargs.append(instance_name) if len(joinargs) > 1: instance_file_storage_dir = utils.PathJoin(*joinargs) else: instance_file_storage_dir = joinargs[0] return instance_file_storage_dir def CheckRADOSFreeSpace(): """Compute disk size requirements inside the RADOS cluster. """ # For the RADOS cluster we assume there is always enough space. pass def _GenerateDRBD8Branch(lu, primary_uuid, secondary_uuid, size, vgnames, names, iv_name, forthcoming=False): """Generate a drbd8 device complete with its children. """ assert len(vgnames) == len(names) == 2 port = lu.cfg.AllocatePort() shared_secret = lu.cfg.GenerateDRBDSecret(lu.proc.GetECId()) dev_data = objects.Disk(dev_type=constants.DT_PLAIN, size=size, logical_id=(vgnames[0], names[0]), nodes=[primary_uuid, secondary_uuid], params={}, forthcoming=forthcoming) dev_data.uuid = lu.cfg.GenerateUniqueID(lu.proc.GetECId()) dev_meta = objects.Disk(dev_type=constants.DT_PLAIN, size=constants.DRBD_META_SIZE, logical_id=(vgnames[1], names[1]), nodes=[primary_uuid, secondary_uuid], params={}, forthcoming=forthcoming) dev_meta.uuid = lu.cfg.GenerateUniqueID(lu.proc.GetECId()) drbd_uuid = lu.cfg.GenerateUniqueID(lu.proc.GetECId()) minors = lu.cfg.AllocateDRBDMinor([primary_uuid, secondary_uuid], drbd_uuid) assert len(minors) == 2 drbd_dev = objects.Disk(dev_type=constants.DT_DRBD8, size=size, logical_id=(primary_uuid, secondary_uuid, port, minors[0], minors[1], shared_secret), children=[dev_data, dev_meta], nodes=[primary_uuid, secondary_uuid], iv_name=iv_name, params={}, forthcoming=forthcoming) drbd_dev.uuid = drbd_uuid return drbd_dev def GenerateDiskTemplate( lu, template_name, instance_uuid, primary_node_uuid, secondary_node_uuids, disk_info, file_storage_dir, file_driver, base_index, feedback_fn, full_disk_params, forthcoming=False): """Generate the entire disk layout for a given template type. """ vgname = lu.cfg.GetVGName() disk_count = len(disk_info) disks = [] CheckDiskTemplateEnabled(lu.cfg.GetClusterInfo(), template_name) if template_name == constants.DT_DISKLESS: pass elif template_name == constants.DT_DRBD8: if len(secondary_node_uuids) != 1: raise errors.ProgrammerError("Wrong template configuration") remote_node_uuid = secondary_node_uuids[0] drbd_params = objects.Disk.ComputeLDParams(template_name, full_disk_params)[0] drbd_default_metavg = drbd_params[constants.LDP_DEFAULT_METAVG] names = [] for lv_prefix in _GenerateUniqueNames(lu, [".disk%d" % (base_index + i) for i in range(disk_count)]): names.append(lv_prefix + "_data") names.append(lv_prefix + "_meta") for idx, disk in enumerate(disk_info): disk_index = idx + base_index data_vg = disk.get(constants.IDISK_VG, vgname) meta_vg = disk.get(constants.IDISK_METAVG, drbd_default_metavg) disk_dev = _GenerateDRBD8Branch(lu, primary_node_uuid, remote_node_uuid, disk[constants.IDISK_SIZE], [data_vg, meta_vg], names[idx * 2:idx * 2 + 2], "disk/%d" % disk_index, forthcoming=forthcoming) disk_dev.mode = disk[constants.IDISK_MODE] disk_dev.name = disk.get(constants.IDISK_NAME, None) disk_dev.dev_type = template_name disks.append(disk_dev) else: if secondary_node_uuids: raise errors.ProgrammerError("Wrong template configuration") name_prefix = _DISK_TEMPLATE_NAME_PREFIX.get(template_name, None) if name_prefix is None: names = None else: names = _GenerateUniqueNames(lu, ["%s.disk%s" % (name_prefix, base_index + i) for i in range(disk_count)]) disk_nodes = [] if template_name == constants.DT_PLAIN: def logical_id_fn(idx, _, disk): vg = disk.get(constants.IDISK_VG, vgname) return (vg, names[idx]) disk_nodes = [primary_node_uuid] elif template_name == constants.DT_GLUSTER: logical_id_fn = lambda _1, disk_index, _2: \ (file_driver, "ganeti/%s.%d" % (instance_uuid, disk_index)) elif template_name in constants.DTS_FILEBASED: # Gluster handled above logical_id_fn = \ lambda idx, disk_index, disk: (file_driver, "%s/%s" % (file_storage_dir, names[idx])) if template_name == constants.DT_FILE: disk_nodes = [primary_node_uuid] elif template_name == constants.DT_BLOCK: logical_id_fn = \ lambda idx, disk_index, disk: (constants.BLOCKDEV_DRIVER_MANUAL, disk[constants.IDISK_ADOPT]) elif template_name == constants.DT_RBD: logical_id_fn = lambda idx, _, disk: ("rbd", names[idx]) elif template_name == constants.DT_EXT: def logical_id_fn(idx, _, disk): provider = disk.get(constants.IDISK_PROVIDER, None) if provider is None: raise errors.ProgrammerError("Disk template is %s, but '%s' is" " not found", constants.DT_EXT, constants.IDISK_PROVIDER) return (provider, names[idx]) else: raise errors.ProgrammerError("Unknown disk template '%s'" % template_name) dev_type = template_name for idx, disk in enumerate(disk_info): params = {} # Only for the Ext template add disk_info to params if template_name == constants.DT_EXT: params[constants.IDISK_PROVIDER] = disk[constants.IDISK_PROVIDER] for key in disk: if key not in constants.IDISK_PARAMS: params[key] = disk[key] # Add IDISK_ACCESS param to disk params if (template_name in constants.DTS_HAVE_ACCESS and constants.IDISK_ACCESS in disk): params[constants.IDISK_ACCESS] = disk[constants.IDISK_ACCESS] disk_index = idx + base_index size = disk[constants.IDISK_SIZE] feedback_fn("* disk %s, size %s" % (disk_index, utils.FormatUnit(size, "h"))) disk_dev = objects.Disk(dev_type=dev_type, size=size, logical_id=logical_id_fn(idx, disk_index, disk), iv_name="disk/%d" % disk_index, mode=disk[constants.IDISK_MODE], params=params, nodes=disk_nodes, spindles=disk.get(constants.IDISK_SPINDLES), forthcoming=forthcoming) disk_dev.name = disk.get(constants.IDISK_NAME, None) disk_dev.uuid = lu.cfg.GenerateUniqueID(lu.proc.GetECId()) disks.append(disk_dev) return disks def CommitDisks(disks): """Recursively remove the forthcoming flag """ for disk in disks: disk.forthcoming = False CommitDisks(disk.children) def CheckSpindlesExclusiveStorage(diskdict, es_flag, required): """Check the presence of the spindle options with exclusive_storage. @type diskdict: dict @param diskdict: disk parameters @type es_flag: bool @param es_flag: the effective value of the exlusive_storage flag @type required: bool @param required: whether spindles are required or just optional @raise errors.OpPrereqError when spindles are given and they should not """ if (not es_flag and constants.IDISK_SPINDLES in diskdict and diskdict[constants.IDISK_SPINDLES] is not None): raise errors.OpPrereqError("Spindles in instance disks cannot be specified" " when exclusive storage is not active", errors.ECODE_INVAL) if (es_flag and required and (constants.IDISK_SPINDLES not in diskdict or diskdict[constants.IDISK_SPINDLES] is None)): raise errors.OpPrereqError("You must specify spindles in instance disks" " when exclusive storage is active", errors.ECODE_INVAL) def CheckDiskExtProvider(diskdict, disk_template): """Check that the given disk should or should not have the provider param. @type diskdict: dict @param diskdict: disk parameters @type disk_template: string @param disk_template: the desired template of this disk @raise errors.OpPrereqError: when the parameter is used in the wrong way """ ext_provider = diskdict.get(constants.IDISK_PROVIDER, None) if ext_provider and disk_template != constants.DT_EXT: raise errors.OpPrereqError("The '%s' option is only valid for the %s" " disk template, not %s" % (constants.IDISK_PROVIDER, constants.DT_EXT, disk_template), errors.ECODE_INVAL) if ext_provider is None and disk_template == constants.DT_EXT: raise errors.OpPrereqError("Missing provider for template '%s'" % constants.DT_EXT, errors.ECODE_INVAL) class LUInstanceRecreateDisks(LogicalUnit): """Recreate an instance's missing disks. """ HPATH = "instance-recreate-disks" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False _MODIFYABLE = compat.UniqueFrozenset([ constants.IDISK_SIZE, constants.IDISK_MODE, constants.IDISK_SPINDLES, ]) # New or changed disk parameters may have different semantics assert constants.IDISK_PARAMS == (_MODIFYABLE | frozenset([ constants.IDISK_ADOPT, # TODO: Implement support changing VG while recreating constants.IDISK_VG, constants.IDISK_METAVG, constants.IDISK_PROVIDER, constants.IDISK_NAME, constants.IDISK_ACCESS, constants.IDISK_TYPE, ])) def _RunAllocator(self): """Run the allocator based on input opcode. """ be_full = self.cfg.GetClusterInfo().FillBE(self.instance) # FIXME # The allocator should actually run in "relocate" mode, but current # allocators don't support relocating all the nodes of an instance at # the same time. As a workaround we use "allocate" mode, but this is # suboptimal for two reasons: # - The instance name passed to the allocator is present in the list of # existing instances, so there could be a conflict within the # internal structures of the allocator. This doesn't happen with the # current allocators, but it's a liability. # - The allocator counts the resources used by the instance twice: once # because the instance exists already, and once because it tries to # allocate a new instance. # The allocator could choose some of the nodes on which the instance is # running, but that's not a problem. If the instance nodes are broken, # they should be already be marked as drained or offline, and hence # skipped by the allocator. If instance disks have been lost for other # reasons, then recreating the disks on the same nodes should be fine. spindle_use = be_full[constants.BE_SPINDLE_USE] disk_template = self.cfg.GetInstanceDiskTemplate(self.instance.uuid) disks = [{ constants.IDISK_SIZE: d.size, constants.IDISK_MODE: d.mode, constants.IDISK_SPINDLES: d.spindles, constants.IDISK_TYPE: d.dev_type } for d in self.cfg.GetInstanceDisks(self.instance.uuid)] req = iallocator.IAReqInstanceAlloc(name=self.op.instance_name, disk_template=disk_template, group_name=None, tags=list(self.instance.GetTags()), os=self.instance.os, nics=[{}], vcpus=be_full[constants.BE_VCPUS], memory=be_full[constants.BE_MAXMEM], spindle_use=spindle_use, disks=disks, hypervisor=self.instance.hypervisor, node_whitelist=None) ial = iallocator.IAllocator(self.cfg, self.rpc, req) ial.Run(self.op.iallocator) assert req.RequiredNodes() == \ len(self.cfg.GetInstanceNodes(self.instance.uuid)) if not ial.success: raise errors.OpPrereqError("Can't compute nodes using iallocator '%s':" " %s" % (self.op.iallocator, ial.info), errors.ECODE_NORES) (self.op.node_uuids, self.op.nodes) = GetWantedNodes(self, ial.result) self.LogInfo("Selected nodes for instance %s via iallocator %s: %s", self.op.instance_name, self.op.iallocator, utils.CommaJoin(self.op.nodes)) def CheckArguments(self): if self.op.disks and ht.TNonNegativeInt(self.op.disks[0]): # Normalize and convert deprecated list of disk indices self.op.disks = [(idx, {}) for idx in sorted(frozenset(self.op.disks))] duplicates = utils.FindDuplicates(map(compat.fst, self.op.disks)) if duplicates: raise errors.OpPrereqError("Some disks have been specified more than" " once: %s" % utils.CommaJoin(duplicates), errors.ECODE_INVAL) # We don't want _CheckIAllocatorOrNode selecting the default iallocator # when neither iallocator nor nodes are specified if self.op.iallocator or self.op.nodes: CheckIAllocatorOrNode(self, "iallocator", "nodes") for (idx, params) in self.op.disks: utils.ForceDictType(params, constants.IDISK_PARAMS_TYPES) unsupported = frozenset(params) - self._MODIFYABLE if unsupported: raise errors.OpPrereqError("Parameters for disk %s try to change" " unmodifyable parameter(s): %s" % (idx, utils.CommaJoin(unsupported)), errors.ECODE_INVAL) def ExpandNames(self): self._ExpandAndLockInstance() self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_APPEND if self.op.nodes: (self.op.node_uuids, self.op.nodes) = GetWantedNodes(self, self.op.nodes) self.needed_locks[locking.LEVEL_NODE] = list(self.op.node_uuids) else: self.needed_locks[locking.LEVEL_NODE] = [] if self.op.iallocator: # iallocator will select a new node in the same group self.needed_locks[locking.LEVEL_NODEGROUP] = [] self.needed_locks[locking.LEVEL_NODE_RES] = [] self.dont_collate_locks[locking.LEVEL_NODEGROUP] = True self.dont_collate_locks[locking.LEVEL_NODE] = True self.dont_collate_locks[locking.LEVEL_NODE_RES] = True def DeclareLocks(self, level): if level == locking.LEVEL_NODEGROUP: assert self.op.iallocator is not None assert not self.op.nodes assert not self.needed_locks[locking.LEVEL_NODEGROUP] self.share_locks[locking.LEVEL_NODEGROUP] = 1 # Lock the primary group used by the instance optimistically; this # requires going via the node before it's locked, requiring # verification later on self.needed_locks[locking.LEVEL_NODEGROUP] = \ self.cfg.GetInstanceNodeGroups(self.op.instance_uuid, primary_only=True) elif level == locking.LEVEL_NODE: # If an allocator is used, then we lock all the nodes in the current # instance group, as we don't know yet which ones will be selected; # if we replace the nodes without using an allocator, locks are # already declared in ExpandNames; otherwise, we need to lock all the # instance nodes for disk re-creation if self.op.iallocator: assert not self.op.nodes assert not self.needed_locks[locking.LEVEL_NODE] assert len(self.owned_locks(locking.LEVEL_NODEGROUP)) == 1 # Lock member nodes of the group of the primary node for group_uuid in self.owned_locks(locking.LEVEL_NODEGROUP): self.needed_locks[locking.LEVEL_NODE].extend( self.cfg.GetNodeGroup(group_uuid).members) elif not self.op.nodes: self._LockInstancesNodes(primary_only=False) elif level == locking.LEVEL_NODE_RES: # Copy node locks self.needed_locks[locking.LEVEL_NODE_RES] = \ CopyLockList(self.needed_locks[locking.LEVEL_NODE]) def BuildHooksEnv(self): """Build hooks env. This runs on master, primary and secondary nodes of the instance. """ return BuildInstanceHookEnvByObject(self, self.instance) def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] + \ list(self.cfg.GetInstanceNodes(self.instance.uuid)) return (nl, nl) def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster and is not running. """ instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name if self.op.node_uuids: inst_nodes = self.cfg.GetInstanceNodes(instance.uuid) if len(self.op.node_uuids) != len(inst_nodes): raise errors.OpPrereqError("Instance %s currently has %d nodes, but" " %d replacement nodes were specified" % (instance.name, len(inst_nodes), len(self.op.node_uuids)), errors.ECODE_INVAL) disks = self.cfg.GetInstanceDisks(instance.uuid) assert (not utils.AnyDiskOfType(disks, [constants.DT_DRBD8]) or len(self.op.node_uuids) == 2) assert (not utils.AnyDiskOfType(disks, [constants.DT_PLAIN]) or len(self.op.node_uuids) == 1) primary_node = self.op.node_uuids[0] else: primary_node = instance.primary_node if not self.op.iallocator: CheckNodeOnline(self, primary_node) if not instance.disks: raise errors.OpPrereqError("Instance '%s' has no disks" % self.op.instance_name, errors.ECODE_INVAL) # Verify if node group locks are still correct owned_groups = self.owned_locks(locking.LEVEL_NODEGROUP) if owned_groups: # Node group locks are acquired only for the primary node (and only # when the allocator is used) CheckInstanceNodeGroups(self.cfg, instance.uuid, owned_groups, primary_only=True) # if we replace nodes *and* the old primary is offline, we don't # check the instance state old_pnode = self.cfg.GetNodeInfo(instance.primary_node) if not ((self.op.iallocator or self.op.node_uuids) and old_pnode.offline): CheckInstanceState(self, instance, INSTANCE_NOT_RUNNING, msg="cannot recreate disks") if self.op.disks: self.disks = dict(self.op.disks) else: self.disks = dict((idx, {}) for idx in range(len(instance.disks))) maxidx = max(self.disks.keys()) if maxidx >= len(instance.disks): raise errors.OpPrereqError("Invalid disk index '%s'" % maxidx, errors.ECODE_INVAL) if ((self.op.node_uuids or self.op.iallocator) and sorted(self.disks.keys()) != list(range(len(instance.disks)))): raise errors.OpPrereqError("Can't recreate disks partially and" " change the nodes at the same time", errors.ECODE_INVAL) self.instance = instance if self.op.iallocator: self._RunAllocator() # Release unneeded node and node resource locks ReleaseLocks(self, locking.LEVEL_NODE, keep=self.op.node_uuids) ReleaseLocks(self, locking.LEVEL_NODE_RES, keep=self.op.node_uuids) if self.op.node_uuids: node_uuids = self.op.node_uuids else: node_uuids = self.cfg.GetInstanceNodes(instance.uuid) excl_stor = compat.any( rpc.GetExclusiveStorageForNodes(self.cfg, node_uuids).values() ) for new_params in self.disks.values(): CheckSpindlesExclusiveStorage(new_params, excl_stor, False) def Exec(self, feedback_fn): """Recreate the disks. """ assert (self.owned_locks(locking.LEVEL_NODE) == self.owned_locks(locking.LEVEL_NODE_RES)) to_skip = [] mods = [] # keeps track of needed changes inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) for idx, disk in enumerate(inst_disks): try: changes = self.disks[idx] except KeyError: # Disk should not be recreated to_skip.append(idx) continue # update secondaries for disks, if needed if self.op.node_uuids and disk.dev_type == constants.DT_DRBD8: # need to update the nodes and minors assert len(self.op.node_uuids) == 2 assert len(disk.logical_id) == 6 # otherwise disk internals # have changed (_, _, old_port, _, _, old_secret) = disk.logical_id new_minors = self.cfg.AllocateDRBDMinor(self.op.node_uuids, disk.uuid) new_id = (self.op.node_uuids[0], self.op.node_uuids[1], old_port, new_minors[0], new_minors[1], old_secret) assert len(disk.logical_id) == len(new_id) else: new_id = None mods.append((idx, new_id, changes)) # now that we have passed all asserts above, we can apply the mods # in a single run (to avoid partial changes) for idx, new_id, changes in mods: disk = inst_disks[idx] if new_id is not None: assert disk.dev_type == constants.DT_DRBD8 disk.logical_id = new_id if changes: disk.Update(size=changes.get(constants.IDISK_SIZE, None), mode=changes.get(constants.IDISK_MODE, None), spindles=changes.get(constants.IDISK_SPINDLES, None)) self.cfg.Update(disk, feedback_fn) # change primary node, if needed if self.op.node_uuids: self.LogWarning("Changing the instance's nodes, you will have to" " remove any disks left on the older nodes manually") self.instance.primary_node = self.op.node_uuids[0] self.cfg.Update(self.instance, feedback_fn) for disk in inst_disks: self.cfg.SetDiskNodes(disk.uuid, self.op.node_uuids) # All touched nodes must be locked mylocks = self.owned_locks(locking.LEVEL_NODE) inst_nodes = self.cfg.GetInstanceNodes(self.instance.uuid) assert mylocks.issuperset(frozenset(inst_nodes)) new_disks = CreateDisks(self, self.instance, to_skip=to_skip) # TODO: Release node locks before wiping, or explain why it's not possible inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) if self.cfg.GetClusterInfo().prealloc_wipe_disks: wipedisks = [(idx, disk, 0) for (idx, disk) in enumerate(inst_disks) if idx not in to_skip] WipeOrCleanupDisks(self, self.instance, disks=wipedisks, cleanup=new_disks) def _PerformNodeInfoCall(lu, node_uuids, vg): """Prepares the input and performs a node info call. @type lu: C{LogicalUnit} @param lu: a logical unit from which we get configuration data @type node_uuids: list of string @param node_uuids: list of node UUIDs to perform the call for @type vg: string @param vg: the volume group's name """ lvm_storage_units = [(constants.ST_LVM_VG, vg)] storage_units = rpc.PrepareStorageUnitsForNodes(lu.cfg, lvm_storage_units, node_uuids) hvname = lu.cfg.GetHypervisorType() hvparams = lu.cfg.GetClusterInfo().hvparams nodeinfo = lu.rpc.call_node_info(node_uuids, storage_units, [(hvname, hvparams[hvname])]) return nodeinfo def _CheckVgCapacityForNode(node_name, node_info, vg, requested): """Checks the vg capacity for a given node. @type node_info: tuple (_, list of dicts, _) @param node_info: the result of the node info call for one node @type node_name: string @param node_name: the name of the node @type vg: string @param vg: volume group name @type requested: int @param requested: the amount of disk in MiB to check for @raise errors.OpPrereqError: if the node doesn't have enough disk, or we cannot check the node """ (_, space_info, _) = node_info lvm_vg_info = utils.storage.LookupSpaceInfoByStorageType( space_info, constants.ST_LVM_VG) if not lvm_vg_info: raise errors.OpPrereqError("Can't retrieve storage information for LVM", errors.ECODE_ENVIRON) vg_free = lvm_vg_info.get("storage_free", None) if not isinstance(vg_free, int): raise errors.OpPrereqError("Can't compute free disk space on node" " %s for vg %s, result was '%s'" % (node_name, vg, vg_free), errors.ECODE_ENVIRON) if requested > vg_free: raise errors.OpPrereqError("Not enough disk space on target node %s" " vg %s: required %d MiB, available %d MiB" % (node_name, vg, requested, vg_free), errors.ECODE_NORES) def _CheckNodesFreeDiskOnVG(lu, node_uuids, vg, requested): """Checks if nodes have enough free disk space in the specified VG. This function checks if all given nodes have the needed amount of free disk. In case any node has less disk or we cannot get the information from the node, this function raises an OpPrereqError exception. @type lu: C{LogicalUnit} @param lu: a logical unit from which we get configuration data @type node_uuids: C{list} @param node_uuids: the list of node UUIDs to check @type vg: C{str} @param vg: the volume group to check @type requested: C{int} @param requested: the amount of disk in MiB to check for @raise errors.OpPrereqError: if the node doesn't have enough disk, or we cannot check the node """ nodeinfo = _PerformNodeInfoCall(lu, node_uuids, vg) for node_uuid in node_uuids: node_name = lu.cfg.GetNodeName(node_uuid) info = nodeinfo[node_uuid] info.Raise("Cannot get current information from node %s" % node_name, prereq=True, ecode=errors.ECODE_ENVIRON) _CheckVgCapacityForNode(node_name, info.payload, vg, requested) def CheckNodesFreeDiskPerVG(lu, node_uuids, req_sizes): """Checks if nodes have enough free disk space in all the VGs. This function checks if all given nodes have the needed amount of free disk. In case any node has less disk or we cannot get the information from the node, this function raises an OpPrereqError exception. @type lu: C{LogicalUnit} @param lu: a logical unit from which we get configuration data @type node_uuids: C{list} @param node_uuids: the list of node UUIDs to check @type req_sizes: C{dict} @param req_sizes: the hash of vg and corresponding amount of disk in MiB to check for @raise errors.OpPrereqError: if the node doesn't have enough disk, or we cannot check the node """ for vg, req_size in req_sizes.items(): _CheckNodesFreeDiskOnVG(lu, node_uuids, vg, req_size) def _DiskSizeInBytesToMebibytes(lu, size): """Converts a disk size in bytes to mebibytes. Warns and rounds up if the size isn't an even multiple of 1 MiB. """ (mib, remainder) = divmod(size, 1024 * 1024) if remainder != 0: lu.LogWarning("Disk size is not an even multiple of 1 MiB; rounding up" " to not overwrite existing data (%s bytes will not be" " wiped)", (1024 * 1024) - remainder) mib += 1 return mib def _CalcEta(time_taken, written, total_size): """Calculates the ETA based on size written and total size. @param time_taken: The time taken so far @param written: amount written so far @param total_size: The total size of data to be written @return: The remaining time in seconds """ avg_time = time_taken / float(written) return (total_size - written) * avg_time def WipeDisks(lu, instance, disks=None): """Wipes instance disks. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type instance: L{objects.Instance} @param instance: the instance whose disks we should create @type disks: None or list of tuple of (number, L{objects.Disk}, number) @param disks: Disk details; tuple contains disk index, disk object and the start offset """ node_uuid = instance.primary_node node_name = lu.cfg.GetNodeName(node_uuid) if disks is None: inst_disks = lu.cfg.GetInstanceDisks(instance.uuid) disks = [(idx, disk, 0) for (idx, disk) in enumerate(inst_disks)] logging.info("Pausing synchronization of disks of instance '%s'", instance.name) result = lu.rpc.call_blockdev_pause_resume_sync(node_uuid, ([d[1] for d in disks], instance), True) result.Raise("Failed to pause disk synchronization on node '%s'" % node_name) for idx, success in enumerate(result.payload): if not success: logging.warn("Pausing synchronization of disk %s of instance '%s'" " failed", idx, instance.name) try: for (idx, device, offset) in disks: # The wipe size is MIN_WIPE_CHUNK_PERCENT % of the instance disk but # MAX_WIPE_CHUNK at max. Truncating to integer to avoid rounding errors. wipe_chunk_size = \ int(min(constants.MAX_WIPE_CHUNK, device.size / 100.0 * constants.MIN_WIPE_CHUNK_PERCENT)) size = device.size last_output = 0 start_time = time.time() if offset == 0: info_text = "" else: info_text = (" (from %s to %s)" % (utils.FormatUnit(offset, "h"), utils.FormatUnit(size, "h"))) lu.LogInfo("* Wiping disk %s%s", idx, info_text) logging.info("Wiping disk %d for instance %s on node %s using" " chunk size %s", idx, instance.name, node_name, wipe_chunk_size) while offset < size: wipe_size = min(wipe_chunk_size, size - offset) logging.debug("Wiping disk %d, offset %s, chunk %s", idx, offset, wipe_size) result = lu.rpc.call_blockdev_wipe(node_uuid, (device, instance), offset, wipe_size) result.Raise("Could not wipe disk %d at offset %d for size %d" % (idx, offset, wipe_size)) now = time.time() offset += wipe_size if now - last_output >= 60: eta = _CalcEta(now - start_time, offset, size) lu.LogInfo(" - done: %.1f%% ETA: %s", offset / float(size) * 100, utils.FormatSeconds(eta)) last_output = now finally: logging.info("Resuming synchronization of disks for instance '%s'", instance.name) result = lu.rpc.call_blockdev_pause_resume_sync(node_uuid, ([d[1] for d in disks], instance), False) if result.fail_msg: lu.LogWarning("Failed to resume disk synchronization on node '%s': %s", node_name, result.fail_msg) else: for idx, success in enumerate(result.payload): if not success: lu.LogWarning("Resuming synchronization of disk %s of instance '%s'" " failed", idx, instance.name) def ImageDisks(lu, instance, image, disks=None): """Dumps an image onto an instance disk. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type instance: L{objects.Instance} @param instance: the instance whose disks we should create @type image: string @param image: the image whose disks we should create @type disks: None or list of ints @param disks: disk indices """ node_uuid = instance.primary_node node_name = lu.cfg.GetNodeName(node_uuid) inst_disks = lu.cfg.GetInstanceDisks(instance.uuid) if disks is None: disks = [(0, inst_disks[0])] else: disks = [(idx, inst_disks[idx]) for idx in disks] logging.info("Pausing synchronization of disks of instance '%s'", instance.name) result = lu.rpc.call_blockdev_pause_resume_sync(node_uuid, ([d[1] for d in disks], instance), True) result.Raise("Failed to pause disk synchronization on node '%s'" % node_name) for idx, success in enumerate(result.payload): if not success: logging.warn("Pausing synchronization of disk %s of instance '%s'" " failed", idx, instance.name) try: for (idx, device) in disks: lu.LogInfo("Imaging disk '%d' for instance '%s' on node '%s'", idx, instance.name, node_name) result = lu.rpc.call_blockdev_image(node_uuid, (device, instance), image, device.size) result.Raise("Could not image disk '%d' for instance '%s' on node '%s'" % (idx, instance.name, node_name)) finally: logging.info("Resuming synchronization of disks for instance '%s'", instance.name) result = lu.rpc.call_blockdev_pause_resume_sync(node_uuid, ([d[1] for d in disks], instance), False) if result.fail_msg: lu.LogWarning("Failed to resume disk synchronization for instance '%s' on" " node '%s'", node_name, result.fail_msg) else: for idx, success in enumerate(result.payload): if not success: lu.LogWarning("Failed to resume synchronization of disk '%d' of" " instance '%s'", idx, instance.name) def WipeOrCleanupDisks(lu, instance, disks=None, cleanup=None): """Wrapper for L{WipeDisks} that handles errors. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type instance: L{objects.Instance} @param instance: the instance whose disks we should wipe @param disks: see L{WipeDisks} @param cleanup: the result returned by L{CreateDisks}, used for cleanup in case of error @raise errors.OpPrereqError: in case of failure """ try: WipeDisks(lu, instance, disks=disks) except errors.OpExecError: logging.warning("Wiping disks for instance '%s' failed", instance.name) _UndoCreateDisks(lu, cleanup, instance) raise def ExpandCheckDisks(instance_disks, disks): """Return the instance disks selected by the disks list @type disks: list of L{objects.Disk} or None @param disks: selected disks @rtype: list of L{objects.Disk} @return: selected instance disks to act on """ if disks is None: return instance_disks else: inst_disks_uuids = [d.uuid for d in instance_disks] disks_uuids = [d.uuid for d in disks] if not set(disks_uuids).issubset(inst_disks_uuids): raise errors.ProgrammerError("Can only act on disks belonging to the" " target instance: expected a subset of %s," " got %s" % (inst_disks_uuids, disks_uuids)) return disks def WaitForSync(lu, instance, disks=None, oneshot=False): """Sleep and poll for an instance's disk to sync. """ inst_disks = lu.cfg.GetInstanceDisks(instance.uuid) if not inst_disks or disks is not None and not disks: return True disks = [d for d in ExpandCheckDisks(inst_disks, disks) if d.dev_type in constants.DTS_INT_MIRROR] if not oneshot: lu.LogInfo("Waiting for instance %s to sync disks", instance.name) node_uuid = instance.primary_node node_name = lu.cfg.GetNodeName(node_uuid) # TODO: Convert to utils.Retry retries = 0 degr_retries = 10 # in seconds, as we sleep 1 second each time while True: max_time = 0 done = True cumul_degraded = False rstats = lu.rpc.call_blockdev_getmirrorstatus(node_uuid, (disks, instance)) msg = rstats.fail_msg if msg: lu.LogWarning("Can't get any data from node %s: %s", node_name, msg) retries += 1 if retries >= 10: raise errors.RemoteError("Can't contact node %s for mirror data," " aborting." % node_name) time.sleep(6) continue rstats = rstats.payload retries = 0 for i, mstat in enumerate(rstats): if mstat is None: lu.LogWarning("Can't compute data for node %s/%s", node_name, disks[i].iv_name) continue cumul_degraded = (cumul_degraded or (mstat.is_degraded and mstat.sync_percent is None)) if mstat.sync_percent is not None: done = False if mstat.estimated_time is not None: rem_time = ("%s remaining (estimated)" % utils.FormatSeconds(mstat.estimated_time)) max_time = mstat.estimated_time else: rem_time = "no time estimate" max_time = 5 # sleep at least a bit between retries lu.LogInfo("- device %s: %5.2f%% done, %s", disks[i].iv_name, mstat.sync_percent, rem_time) # if we're done but degraded, let's do a few small retries, to # make sure we see a stable and not transient situation; therefore # we force restart of the loop if (done or oneshot) and cumul_degraded and degr_retries > 0: logging.info("Degraded disks found, %d retries left", degr_retries) degr_retries -= 1 time.sleep(1) continue if done or oneshot: break time.sleep(min(60, max_time)) if done: lu.LogInfo("Instance %s's disks are in sync", instance.name) return not cumul_degraded def ShutdownInstanceDisks(lu, instance, disks=None, ignore_primary=False): """Shutdown block devices of an instance. This does the shutdown on all nodes of the instance. If the ignore_primary is false, errors on the primary node are ignored. Modifies the configuration of the instance, so the caller should re-read the instance configuration, if needed. """ all_result = True if disks is None: # only mark instance disks as inactive if all disks are affected lu.cfg.MarkInstanceDisksInactive(instance.uuid) inst_disks = lu.cfg.GetInstanceDisks(instance.uuid) disks = ExpandCheckDisks(inst_disks, disks) for disk in disks: for node_uuid, top_disk in disk.ComputeNodeTree(instance.primary_node): result = lu.rpc.call_blockdev_shutdown(node_uuid, (top_disk, instance)) msg = result.fail_msg if msg: lu.LogWarning("Could not shutdown block device %s on node %s: %s", disk.iv_name, lu.cfg.GetNodeName(node_uuid), msg) if ((node_uuid == instance.primary_node and not ignore_primary) or (node_uuid != instance.primary_node and not result.offline)): all_result = False return all_result def _SafeShutdownInstanceDisks(lu, instance, disks=None, req_states=None): """Shutdown block devices of an instance. This function checks if an instance is running, before calling _ShutdownInstanceDisks. """ if req_states is None: req_states = INSTANCE_DOWN CheckInstanceState(lu, instance, req_states, msg="cannot shutdown disks") ShutdownInstanceDisks(lu, instance, disks=disks) def AssembleInstanceDisks(lu, instance, disks=None, ignore_secondaries=False, ignore_size=False): """Prepare the block devices for an instance. This sets up the block devices on all nodes. Modifies the configuration of the instance, so the caller should re-read the instance configuration, if needed. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type instance: L{objects.Instance} @param instance: the instance for whose disks we assemble @type disks: list of L{objects.Disk} or None @param disks: which disks to assemble (or all, if None) @type ignore_secondaries: boolean @param ignore_secondaries: if true, errors on secondary nodes won't result in an error return from the function @type ignore_size: boolean @param ignore_size: if true, the current known size of the disk will not be used during the disk activation, useful for cases when the size is wrong @return: False if the operation failed, otherwise a list of (host, instance_visible_name, node_visible_name) with the mapping from node devices to instance devices, as well as the payloads of the RPC calls """ device_info = [] disks_ok = True payloads = [] if disks is None: # only mark instance disks as active if all disks are affected instance = lu.cfg.MarkInstanceDisksActive(instance.uuid) inst_disks = lu.cfg.GetInstanceDisks(instance.uuid) disks = ExpandCheckDisks(inst_disks, disks) # With the two passes mechanism we try to reduce the window of # opportunity for the race condition of switching DRBD to primary # before handshaking occured, but we do not eliminate it # The proper fix would be to wait (with some limits) until the # connection has been made and drbd transitions from WFConnection # into any other network-connected state (Connected, SyncTarget, # SyncSource, etc.) # 1st pass, assemble on all nodes in secondary mode for idx, inst_disk in enumerate(disks): for node_uuid, node_disk in inst_disk.ComputeNodeTree( instance.primary_node): if ignore_size: node_disk = node_disk.Copy() node_disk.UnsetSize() result = lu.rpc.call_blockdev_assemble(node_uuid, (node_disk, instance), instance, False, idx) msg = result.fail_msg if msg: secondary_nodes = lu.cfg.GetInstanceSecondaryNodes(instance.uuid) is_offline_secondary = (node_uuid in secondary_nodes and result.offline) lu.LogWarning("Could not prepare block device %s on node %s" " (is_primary=False, pass=1): %s", inst_disk.iv_name, lu.cfg.GetNodeName(node_uuid), msg) if not (ignore_secondaries or is_offline_secondary): disks_ok = False # FIXME: race condition on drbd migration to primary # 2nd pass, do only the primary node for idx, inst_disk in enumerate(disks): dev_path = None for node_uuid, node_disk in inst_disk.ComputeNodeTree( instance.primary_node): if node_uuid != instance.primary_node: continue if ignore_size: node_disk = node_disk.Copy() node_disk.UnsetSize() result = lu.rpc.call_blockdev_assemble(node_uuid, (node_disk, instance), instance, True, idx) payloads.append(result.payload) msg = result.fail_msg if msg: lu.LogWarning("Could not prepare block device %s on node %s" " (is_primary=True, pass=2): %s", inst_disk.iv_name, lu.cfg.GetNodeName(node_uuid), msg) disks_ok = False else: dev_path, _, __ = result.payload device_info.append((lu.cfg.GetNodeName(instance.primary_node), inst_disk.iv_name, dev_path)) if not disks_ok: lu.cfg.MarkInstanceDisksInactive(instance.uuid) return disks_ok, device_info, payloads def StartInstanceDisks(lu, instance, force): """Start the disks of an instance. Modifies the configuration of the instance, so the caller should re-read the instance configuration, if needed. """ disks_ok, _, _ = AssembleInstanceDisks(lu, instance, ignore_secondaries=force) if not disks_ok: ShutdownInstanceDisks(lu, instance) if force is not None and not force: lu.LogWarning("", hint=("If the message above refers to a secondary node," " you can retry the operation using '--force'")) raise errors.OpExecError("Disk consistency error") class LUInstanceGrowDisk(LogicalUnit): """Grow a disk of an instance. """ HPATH = "disk-grow" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def ExpandNames(self): self._ExpandAndLockInstance() self.needed_locks[locking.LEVEL_NODE] = [] self.needed_locks[locking.LEVEL_NODE_RES] = [] self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE self.recalculate_locks[locking.LEVEL_NODE_RES] = constants.LOCKS_REPLACE self.dont_collate_locks[locking.LEVEL_NODE] = True self.dont_collate_locks[locking.LEVEL_NODE_RES] = True def DeclareLocks(self, level): if level == locking.LEVEL_NODE: self._LockInstancesNodes() elif level == locking.LEVEL_NODE_RES: # Copy node locks self.needed_locks[locking.LEVEL_NODE_RES] = \ CopyLockList(self.needed_locks[locking.LEVEL_NODE]) def BuildHooksEnv(self): """Build hooks env. This runs on the master, the primary and all the secondaries. """ env = { "DISK": self.op.disk, "AMOUNT": self.op.amount, "ABSOLUTE": self.op.absolute, } env.update(BuildInstanceHookEnvByObject(self, self.instance)) return env def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] + \ list(self.cfg.GetInstanceNodes(self.instance.uuid)) return (nl, nl) def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name node_uuids = list(self.cfg.GetInstanceNodes(self.instance.uuid)) for node_uuid in node_uuids: CheckNodeOnline(self, node_uuid) self.node_es_flags = rpc.GetExclusiveStorageForNodes(self.cfg, node_uuids) self.disk = self.cfg.GetDiskInfo(self.instance.FindDisk(self.op.disk)) if self.disk.dev_type not in constants.DTS_GROWABLE: raise errors.OpPrereqError( "Instance's disk layout %s does not support" " growing" % self.disk.dev_type, errors.ECODE_INVAL) if self.op.absolute: self.target = self.op.amount self.delta = self.target - self.disk.size if self.delta < 0: raise errors.OpPrereqError("Requested size (%s) is smaller than " "current disk size (%s)" % (utils.FormatUnit(self.target, "h"), utils.FormatUnit(self.disk.size, "h")), errors.ECODE_STATE) else: self.delta = self.op.amount self.target = self.disk.size + self.delta if self.delta < 0: raise errors.OpPrereqError("Requested increment (%s) is negative" % utils.FormatUnit(self.delta, "h"), errors.ECODE_INVAL) self._CheckDiskSpace(node_uuids, self.disk.ComputeGrowth(self.delta)) self._CheckIPolicy(self.target) def _CheckDiskSpace(self, node_uuids, req_vgspace): template = self.disk.dev_type if (template not in constants.DTS_NO_FREE_SPACE_CHECK and not any(self.node_es_flags.values())): # TODO: check the free disk space for file, when that feature will be # supported # With exclusive storage we need to do something smarter than just looking # at free space, which, in the end, is basically a dry run. So we rely on # the dry run performed in Exec() instead. CheckNodesFreeDiskPerVG(self, node_uuids, req_vgspace) def _CheckIPolicy(self, target_size): cluster = self.cfg.GetClusterInfo() group_uuid = list(self.cfg.GetInstanceNodeGroups(self.op.instance_uuid, primary_only=True))[0] group_info = self.cfg.GetNodeGroup(group_uuid) ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(cluster, group_info) disks = self.cfg.GetInstanceDisks(self.op.instance_uuid) disk_sizes = [disk.size if disk.uuid != self.disk.uuid else target_size for disk in disks] # The ipolicy checker below ignores None, so we only give it the disk size res = ComputeIPolicyDiskSizesViolation(ipolicy, disk_sizes, disks) if res: msg = ("Growing disk %s violates policy: %s" % (self.op.disk, utils.CommaJoin(res))) if self.op.ignore_ipolicy: self.LogWarning(msg) else: raise errors.OpPrereqError(msg, errors.ECODE_INVAL) def Exec(self, feedback_fn): """Execute disk grow. """ assert set([self.instance.name]) == self.owned_locks(locking.LEVEL_INSTANCE) assert (self.owned_locks(locking.LEVEL_NODE) == self.owned_locks(locking.LEVEL_NODE_RES)) wipe_disks = self.cfg.GetClusterInfo().prealloc_wipe_disks disks_ok, _, _ = AssembleInstanceDisks(self, self.instance, disks=[self.disk]) if not disks_ok: raise errors.OpExecError("Cannot activate block device to grow") feedback_fn("Growing disk %s of instance '%s' by %s to %s" % (self.op.disk, self.instance.name, utils.FormatUnit(self.delta, "h"), utils.FormatUnit(self.target, "h"))) # First run all grow ops in dry-run mode inst_nodes = self.cfg.GetInstanceNodes(self.instance.uuid) for node_uuid in inst_nodes: result = self.rpc.call_blockdev_grow(node_uuid, (self.disk, self.instance), self.delta, True, True, self.node_es_flags[node_uuid]) result.Raise("Dry-run grow request failed to node %s" % self.cfg.GetNodeName(node_uuid)) if wipe_disks: # Get disk size from primary node for wiping result = self.rpc.call_blockdev_getdimensions( self.instance.primary_node, [([self.disk], self.instance)]) result.Raise("Failed to retrieve disk size from node '%s'" % self.instance.primary_node) (disk_dimensions, ) = result.payload if disk_dimensions is None: raise errors.OpExecError("Failed to retrieve disk size from primary" " node '%s'" % self.instance.primary_node) (disk_size_in_bytes, _) = disk_dimensions old_disk_size = _DiskSizeInBytesToMebibytes(self, disk_size_in_bytes) assert old_disk_size >= self.disk.size, \ ("Retrieved disk size too small (got %s, should be at least %s)" % (old_disk_size, self.disk.size)) else: old_disk_size = None # We know that (as far as we can test) operations across different # nodes will succeed, time to run it for real on the backing storage for node_uuid in inst_nodes: result = self.rpc.call_blockdev_grow(node_uuid, (self.disk, self.instance), self.delta, False, True, self.node_es_flags[node_uuid]) result.Raise("Grow request failed to node %s" % self.cfg.GetNodeName(node_uuid)) # And now execute it for logical storage, on the primary node node_uuid = self.instance.primary_node result = self.rpc.call_blockdev_grow(node_uuid, (self.disk, self.instance), self.delta, False, False, self.node_es_flags[node_uuid]) result.Raise("Grow request failed to node %s" % self.cfg.GetNodeName(node_uuid)) self.disk.RecordGrow(self.delta) self.cfg.Update(self.instance, feedback_fn) self.cfg.Update(self.disk, feedback_fn) # Changes have been recorded, release node lock ReleaseLocks(self, locking.LEVEL_NODE) # Downgrade lock while waiting for sync self.WConfdClient().DownGradeLocksLevel( locking.LEVEL_NAMES[locking.LEVEL_INSTANCE]) assert wipe_disks ^ (old_disk_size is None) if wipe_disks: inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) assert inst_disks[self.op.disk].ToDict() == self.disk.ToDict() # Wipe newly added disk space WipeDisks(self, self.instance, disks=[(self.op.disk, self.disk, old_disk_size)]) if self.op.wait_for_sync: disk_abort = not WaitForSync(self, self.instance, disks=[self.disk]) if disk_abort: self.LogWarning("Disk syncing has not returned a good status; check" " the instance") if not self.instance.disks_active: _SafeShutdownInstanceDisks(self, self.instance, disks=[self.disk]) elif not self.instance.disks_active: self.LogWarning("Not shutting down the disk even if the instance is" " not supposed to be running because no wait for" " sync mode was requested") assert self.owned_locks(locking.LEVEL_NODE_RES) assert set([self.instance.name]) == self.owned_locks(locking.LEVEL_INSTANCE) # Notifify the hv about the change if the instance is running if IsInstanceRunning(self, self.instance): feedback_fn("Try to notify the Hypervisor about the disk change") new_size = self.target * (1024 ** 2) result = self.rpc.call_resize_disk(self.instance.primary_node, self.instance, self.disk.ToDict(), new_size) if result.fail_msg: self.LogWarning(f"Disk resize in hypervisor failed: {result.fail_msg}") class LUInstanceReplaceDisks(LogicalUnit): """Replace the disks of an instance. """ HPATH = "mirrors-replace" HTYPE = constants.HTYPE_INSTANCE REQ_BGL = False def CheckArguments(self): """Check arguments. """ if self.op.mode == constants.REPLACE_DISK_CHG: if self.op.remote_node is None and self.op.iallocator is None: raise errors.OpPrereqError("When changing the secondary either an" " iallocator script must be used or the" " new node given", errors.ECODE_INVAL) else: CheckIAllocatorOrNode(self, "iallocator", "remote_node") elif self.op.remote_node is not None or self.op.iallocator is not None: # Not replacing the secondary raise errors.OpPrereqError("The iallocator and new node options can" " only be used when changing the" " secondary node", errors.ECODE_INVAL) def ExpandNames(self): self._ExpandAndLockInstance(allow_forthcoming=True) assert locking.LEVEL_NODE not in self.needed_locks assert locking.LEVEL_NODE_RES not in self.needed_locks assert locking.LEVEL_NODEGROUP not in self.needed_locks assert self.op.iallocator is None or self.op.remote_node is None, \ "Conflicting options" if self.op.remote_node is not None: (self.op.remote_node_uuid, self.op.remote_node) = \ ExpandNodeUuidAndName(self.cfg, self.op.remote_node_uuid, self.op.remote_node) # Warning: do not remove the locking of the new secondary here # unless DRBD8Dev.AddChildren is changed to work in parallel; # currently it doesn't since parallel invocations of # FindUnusedMinor will conflict self.needed_locks[locking.LEVEL_NODE] = [self.op.remote_node_uuid] self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_APPEND else: self.needed_locks[locking.LEVEL_NODE] = [] self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE if self.op.iallocator is not None: # iallocator will select a new node in the same group self.needed_locks[locking.LEVEL_NODEGROUP] = [] self.needed_locks[locking.LEVEL_NODE_RES] = [] self.dont_collate_locks[locking.LEVEL_NODEGROUP] = True self.dont_collate_locks[locking.LEVEL_NODE] = True self.dont_collate_locks[locking.LEVEL_NODE_RES] = True self.replacer = TLReplaceDisks(self, self.op.instance_uuid, self.op.instance_name, self.op.mode, self.op.iallocator, self.op.remote_node_uuid, self.op.disks, self.op.early_release, self.op.ignore_ipolicy) self.tasklets = [self.replacer] def DeclareLocks(self, level): if level == locking.LEVEL_NODEGROUP: assert self.op.remote_node_uuid is None assert self.op.iallocator is not None assert not self.needed_locks[locking.LEVEL_NODEGROUP] self.share_locks[locking.LEVEL_NODEGROUP] = 1 # Lock all groups used by instance optimistically; this requires going # via the node before it's locked, requiring verification later on self.needed_locks[locking.LEVEL_NODEGROUP] = \ self.cfg.GetInstanceNodeGroups(self.op.instance_uuid) elif level == locking.LEVEL_NODE: if self.op.iallocator is not None: assert self.op.remote_node_uuid is None assert not self.needed_locks[locking.LEVEL_NODE] # Lock member nodes of all locked groups self.needed_locks[locking.LEVEL_NODE] = \ [node_uuid for group_uuid in self.owned_locks(locking.LEVEL_NODEGROUP) for node_uuid in self.cfg.GetNodeGroup(group_uuid).members] else: self._LockInstancesNodes() elif level == locking.LEVEL_NODE_RES: # Reuse node locks self.needed_locks[locking.LEVEL_NODE_RES] = \ self.needed_locks[locking.LEVEL_NODE] def BuildHooksEnv(self): """Build hooks env. This runs on the master, the primary and all the secondaries. """ instance = self.replacer.instance secondary_nodes = self.cfg.GetInstanceSecondaryNodes(instance.uuid) env = { "MODE": self.op.mode, "NEW_SECONDARY": self.op.remote_node, "OLD_SECONDARY": self.cfg.GetNodeName(secondary_nodes[0]), } env.update(BuildInstanceHookEnvByObject(self, instance)) return env def BuildHooksNodes(self): """Build hooks nodes. """ instance = self.replacer.instance nl = [ self.cfg.GetMasterNode(), instance.primary_node, ] if self.op.remote_node_uuid is not None: nl.append(self.op.remote_node_uuid) return nl, nl def CheckPrereq(self): """Check prerequisites. """ # Verify if node group locks are still correct owned_groups = self.owned_locks(locking.LEVEL_NODEGROUP) if owned_groups: CheckInstanceNodeGroups(self.cfg, self.op.instance_uuid, owned_groups) return LogicalUnit.CheckPrereq(self) class LUInstanceActivateDisks(NoHooksLU): """Bring up an instance's disks. """ REQ_BGL = False def ExpandNames(self): self._ExpandAndLockInstance() self.needed_locks[locking.LEVEL_NODE] = [] self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE def DeclareLocks(self, level): if level == locking.LEVEL_NODE: self._LockInstancesNodes() def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name CheckNodeOnline(self, self.instance.primary_node) def Exec(self, feedback_fn): """Activate the disks. """ disks_ok, disks_info, _ = AssembleInstanceDisks( self, self.instance, ignore_size=self.op.ignore_size) if not disks_ok: raise errors.OpExecError("Cannot activate block devices") if self.op.wait_for_sync: if not WaitForSync(self, self.instance): self.cfg.MarkInstanceDisksInactive(self.instance.uuid) raise errors.OpExecError("Some disks of the instance are degraded!") return disks_info class LUInstanceDeactivateDisks(NoHooksLU): """Shutdown an instance's disks. """ REQ_BGL = False def ExpandNames(self): self._ExpandAndLockInstance() self.needed_locks[locking.LEVEL_NODE] = [] self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE def DeclareLocks(self, level): if level == locking.LEVEL_NODE: self._LockInstancesNodes() def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.op.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name def Exec(self, feedback_fn): """Deactivate the disks """ if self.op.force: ShutdownInstanceDisks(self, self.instance) else: _SafeShutdownInstanceDisks(self, self.instance) def _CheckDiskConsistencyInner(lu, instance, dev, node_uuid, on_primary, ldisk=False): """Check that mirrors are not degraded. @attention: The device has to be annotated already. The ldisk parameter, if True, will change the test from the is_degraded attribute (which represents overall non-ok status for the device(s)) to the ldisk (representing the local storage status). """ result = True if on_primary or dev.AssembleOnSecondary(): rstats = lu.rpc.call_blockdev_find(node_uuid, (dev, instance)) msg = rstats.fail_msg if msg: lu.LogWarning("Can't find disk on node %s: %s", lu.cfg.GetNodeName(node_uuid), msg) result = False elif not rstats.payload: lu.LogWarning("Can't find disk on node %s", lu.cfg.GetNodeName(node_uuid)) result = False else: if ldisk: result = result and rstats.payload.ldisk_status == constants.LDS_OKAY else: result = result and not rstats.payload.is_degraded if dev.children: for child in dev.children: result = result and _CheckDiskConsistencyInner(lu, instance, child, node_uuid, on_primary) return result def CheckDiskConsistency(lu, instance, dev, node_uuid, on_primary, ldisk=False): """Wrapper around L{_CheckDiskConsistencyInner}. """ (disk,) = AnnotateDiskParams(instance, [dev], lu.cfg) return _CheckDiskConsistencyInner(lu, instance, disk, node_uuid, on_primary, ldisk=ldisk) def _BlockdevFind(lu, node_uuid, dev, instance): """Wrapper around call_blockdev_find to annotate diskparams. @param lu: A reference to the lu object @param node_uuid: The node to call out @param dev: The device to find @param instance: The instance object the device belongs to @returns The result of the rpc call """ (disk,) = AnnotateDiskParams(instance, [dev], lu.cfg) return lu.rpc.call_blockdev_find(node_uuid, (disk, instance)) def _GenerateUniqueNames(lu, exts): """Generate a suitable LV name. This will generate a logical volume name for the given instance. """ results = [] for val in exts: new_id = lu.cfg.GenerateUniqueID(lu.proc.GetECId()) results.append("%s%s" % (new_id, val)) return results class TLReplaceDisks(Tasklet): """Replaces disks for an instance. Note: Locking is not within the scope of this class. """ def __init__(self, lu, instance_uuid, instance_name, mode, iallocator_name, remote_node_uuid, disks, early_release, ignore_ipolicy): """Initializes this class. """ Tasklet.__init__(self, lu) # Parameters self.instance_uuid = instance_uuid self.instance_name = instance_name self.mode = mode self.iallocator_name = iallocator_name self.remote_node_uuid = remote_node_uuid self.disks = disks self.early_release = early_release self.ignore_ipolicy = ignore_ipolicy # Runtime data self.instance = None self.new_node_uuid = None self.target_node_uuid = None self.other_node_uuid = None self.remote_node_info = None self.node_secondary_ip = None @staticmethod def _RunAllocator(lu, iallocator_name, instance_uuid, relocate_from_node_uuids): """Compute a new secondary node using an IAllocator. """ req = iallocator.IAReqRelocate( inst_uuid=instance_uuid, relocate_from_node_uuids=list(relocate_from_node_uuids)) ial = iallocator.IAllocator(lu.cfg, lu.rpc, req) ial.Run(iallocator_name) if not ial.success: raise errors.OpPrereqError("Can't compute nodes using iallocator '%s':" " %s" % (iallocator_name, ial.info), errors.ECODE_NORES) remote_node_name = ial.result[0] # pylint: disable=E1136 remote_node = lu.cfg.GetNodeInfoByName(remote_node_name) if remote_node is None: raise errors.OpPrereqError("Node %s not found in configuration" % remote_node_name, errors.ECODE_NOENT) lu.LogInfo("Selected new secondary for instance '%s': %s", instance_uuid, remote_node_name) return remote_node.uuid def _FindFaultyDisks(self, node_uuid): """Wrapper for L{FindFaultyInstanceDisks}. """ return FindFaultyInstanceDisks(self.cfg, self.rpc, self.instance, node_uuid, True) def _CheckDisksActivated(self, instance): """Checks if the instance disks are activated. @param instance: The instance to check disks @return: True if they are activated, False otherwise """ node_uuids = self.cfg.GetInstanceNodes(instance.uuid) for idx, dev in enumerate(self.cfg.GetInstanceDisks(instance.uuid)): for node_uuid in node_uuids: self.lu.LogInfo("Checking disk/%d on %s", idx, self.cfg.GetNodeName(node_uuid)) result = _BlockdevFind(self, node_uuid, dev, instance) if result.offline: continue elif result.fail_msg or not result.payload: return False return True def CheckPrereq(self): """Check prerequisites. This checks that the instance is in the cluster. """ self.instance = self.cfg.GetInstanceInfo(self.instance_uuid) assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.instance_name secondary_nodes = self.cfg.GetInstanceSecondaryNodes(self.instance.uuid) if len(secondary_nodes) != 1: raise errors.OpPrereqError("The instance has a strange layout," " expected one secondary but found %d" % len(secondary_nodes), errors.ECODE_FAULT) secondary_node_uuid = secondary_nodes[0] if self.iallocator_name is None: remote_node_uuid = self.remote_node_uuid else: remote_node_uuid = self._RunAllocator(self.lu, self.iallocator_name, self.instance.uuid, secondary_nodes) if remote_node_uuid is None: self.remote_node_info = None else: assert remote_node_uuid in self.lu.owned_locks(locking.LEVEL_NODE), \ "Remote node '%s' is not locked" % remote_node_uuid self.remote_node_info = self.cfg.GetNodeInfo(remote_node_uuid) assert self.remote_node_info is not None, \ "Cannot retrieve locked node %s" % remote_node_uuid if remote_node_uuid == self.instance.primary_node: raise errors.OpPrereqError("The specified node is the primary node of" " the instance", errors.ECODE_INVAL) if remote_node_uuid == secondary_node_uuid: raise errors.OpPrereqError("The specified node is already the" " secondary node of the instance", errors.ECODE_INVAL) if self.disks and self.mode in (constants.REPLACE_DISK_AUTO, constants.REPLACE_DISK_CHG): raise errors.OpPrereqError("Cannot specify disks to be replaced", errors.ECODE_INVAL) if self.mode == constants.REPLACE_DISK_AUTO: if not self._CheckDisksActivated(self.instance): raise errors.OpPrereqError("Please run activate-disks on instance %s" " first" % self.instance_name, errors.ECODE_STATE) faulty_primary = self._FindFaultyDisks(self.instance.primary_node) faulty_secondary = self._FindFaultyDisks(secondary_node_uuid) if faulty_primary and faulty_secondary: raise errors.OpPrereqError("Instance %s has faulty disks on more than" " one node and can not be repaired" " automatically" % self.instance_name, errors.ECODE_STATE) if faulty_primary: self.disks = faulty_primary self.target_node_uuid = self.instance.primary_node self.other_node_uuid = secondary_node_uuid check_nodes = [self.target_node_uuid, self.other_node_uuid] elif faulty_secondary: self.disks = faulty_secondary self.target_node_uuid = secondary_node_uuid self.other_node_uuid = self.instance.primary_node check_nodes = [self.target_node_uuid, self.other_node_uuid] else: self.disks = [] check_nodes = [] else: # Non-automatic modes if self.mode == constants.REPLACE_DISK_PRI: self.target_node_uuid = self.instance.primary_node self.other_node_uuid = secondary_node_uuid check_nodes = [self.target_node_uuid, self.other_node_uuid] elif self.mode == constants.REPLACE_DISK_SEC: self.target_node_uuid = secondary_node_uuid self.other_node_uuid = self.instance.primary_node check_nodes = [self.target_node_uuid, self.other_node_uuid] elif self.mode == constants.REPLACE_DISK_CHG: self.new_node_uuid = remote_node_uuid self.other_node_uuid = self.instance.primary_node self.target_node_uuid = secondary_node_uuid check_nodes = [self.new_node_uuid, self.other_node_uuid] CheckNodeNotDrained(self.lu, remote_node_uuid) CheckNodeVmCapable(self.lu, remote_node_uuid) old_node_info = self.cfg.GetNodeInfo(secondary_node_uuid) assert old_node_info is not None if old_node_info.offline and not self.early_release: # doesn't make sense to delay the release self.early_release = True self.lu.LogInfo("Old secondary %s is offline, automatically enabling" " early-release mode", secondary_node_uuid) else: raise errors.ProgrammerError("Unhandled disk replace mode (%s)" % self.mode) # If not specified all disks should be replaced if not self.disks: self.disks = list(range(len(self.instance.disks))) disks = self.cfg.GetInstanceDisks(self.instance.uuid) if (not disks or not utils.AllDiskOfType(disks, [constants.DT_DRBD8])): raise errors.OpPrereqError("Can only run replace disks for DRBD8-based" " instances", errors.ECODE_INVAL) # TODO: This is ugly, but right now we can't distinguish between internal # submitted opcode and external one. We should fix that. if self.remote_node_info: # We change the node, lets verify it still meets instance policy new_group_info = self.cfg.GetNodeGroup(self.remote_node_info.group) cluster = self.cfg.GetClusterInfo() ipolicy = ganeti.masterd.instance.CalculateGroupIPolicy(cluster, new_group_info) CheckTargetNodeIPolicy(self.lu, ipolicy, self.instance, self.remote_node_info, self.cfg, ignore=self.ignore_ipolicy) for node_uuid in check_nodes: CheckNodeOnline(self.lu, node_uuid) touched_nodes = frozenset(node_uuid for node_uuid in [self.new_node_uuid, self.other_node_uuid, self.target_node_uuid] if node_uuid is not None) # Release unneeded node and node resource locks ReleaseLocks(self.lu, locking.LEVEL_NODE, keep=touched_nodes) ReleaseLocks(self.lu, locking.LEVEL_NODE_RES, keep=touched_nodes) # Release any owned node group ReleaseLocks(self.lu, locking.LEVEL_NODEGROUP) # Check whether disks are valid for disk_idx in self.disks: self.instance.FindDisk(disk_idx) # Get secondary node IP addresses self.node_secondary_ip = dict((uuid, node.secondary_ip) for (uuid, node) in self.cfg.GetMultiNodeInfo(touched_nodes)) def Exec(self, feedback_fn): """Execute disk replacement. This dispatches the disk replacement to the appropriate handler. """ if __debug__: # Verify owned locks before starting operation owned_nodes = self.lu.owned_locks(locking.LEVEL_NODE) assert set(owned_nodes) == set(self.node_secondary_ip), \ ("Incorrect node locks, owning %s, expected %s" % (owned_nodes, list(self.node_secondary_ip))) assert (self.lu.owned_locks(locking.LEVEL_NODE) == self.lu.owned_locks(locking.LEVEL_NODE_RES)) owned_instances = self.lu.owned_locks(locking.LEVEL_INSTANCE) assert list(owned_instances) == [self.instance_name], \ "Instance '%s' not locked" % self.instance_name if not self.disks: feedback_fn("No disks need replacement for instance '%s'" % self.instance.name) return feedback_fn("Replacing disk(s) %s for instance '%s'" % (utils.CommaJoin(self.disks), self.instance.name)) feedback_fn("Current primary node: %s" % self.cfg.GetNodeName(self.instance.primary_node)) secondary_nodes = self.cfg.GetInstanceSecondaryNodes(self.instance.uuid) feedback_fn("Current secondary node: %s" % utils.CommaJoin(self.cfg.GetNodeNames(secondary_nodes))) activate_disks = not self.instance.disks_active # Activate the instance disks if we're replacing them on a down instance # that is real (forthcoming instances currently only have forthcoming # disks). if activate_disks and not self.instance.forthcoming: StartInstanceDisks(self.lu, self.instance, True) # Re-read the instance object modified by the previous call self.instance = self.cfg.GetInstanceInfo(self.instance.uuid) try: # Should we replace the secondary node? if self.new_node_uuid is not None: fn = self._ExecDrbd8Secondary else: fn = self._ExecDrbd8DiskOnly result = fn(feedback_fn) finally: # Deactivate the instance disks if we're replacing them on a # down instance if activate_disks and not self.instance.forthcoming: _SafeShutdownInstanceDisks(self.lu, self.instance, req_states=INSTANCE_NOT_RUNNING) self.lu.AssertReleasedLocks(locking.LEVEL_NODE) if __debug__: # Verify owned locks owned_nodes = self.lu.owned_locks(locking.LEVEL_NODE_RES) nodes = frozenset(self.node_secondary_ip) assert ((self.early_release and not owned_nodes) or (not self.early_release and not (set(owned_nodes) - nodes))), \ ("Not owning the correct locks, early_release=%s, owned=%r," " nodes=%r" % (self.early_release, owned_nodes, nodes)) return result def _CheckVolumeGroup(self, node_uuids): self.lu.LogInfo("Checking volume groups") vgname = self.cfg.GetVGName() # Make sure volume group exists on all involved nodes results = self.rpc.call_vg_list(node_uuids) if not results: raise errors.OpExecError("Can't list volume groups on the nodes") for node_uuid in node_uuids: res = results[node_uuid] res.Raise("Error checking node %s" % self.cfg.GetNodeName(node_uuid)) if vgname not in res.payload: raise errors.OpExecError("Volume group '%s' not found on node %s" % (vgname, self.cfg.GetNodeName(node_uuid))) def _CheckDisksExistence(self, node_uuids): # Check disk existence for idx, dev in enumerate(self.cfg.GetInstanceDisks(self.instance.uuid)): if idx not in self.disks: continue for node_uuid in node_uuids: self.lu.LogInfo("Checking disk/%d on %s", idx, self.cfg.GetNodeName(node_uuid)) result = _BlockdevFind(self, node_uuid, dev, self.instance) msg = result.fail_msg if msg or not result.payload: if not msg: msg = "disk not found" if not self._CheckDisksActivated(self.instance): extra_hint = ("\nDisks seem to be not properly activated. Try" " running activate-disks on the instance before" " using replace-disks.") else: extra_hint = "" raise errors.OpExecError("Can't find disk/%d on node %s: %s%s" % (idx, self.cfg.GetNodeName(node_uuid), msg, extra_hint)) def _CheckDisksConsistency(self, node_uuid, on_primary, ldisk): for idx, dev in enumerate(self.cfg.GetInstanceDisks(self.instance.uuid)): if idx not in self.disks: continue self.lu.LogInfo("Checking disk/%d consistency on node %s" % (idx, self.cfg.GetNodeName(node_uuid))) if not CheckDiskConsistency(self.lu, self.instance, dev, node_uuid, on_primary, ldisk=ldisk): raise errors.OpExecError("Node %s has degraded storage, unsafe to" " replace disks for instance %s" % (self.cfg.GetNodeName(node_uuid), self.instance.name)) def _CreateNewStorage(self, node_uuid): """Create new storage on the primary or secondary node. This is only used for same-node replaces, not for changing the secondary node, hence we don't want to modify the existing disk. """ iv_names = {} inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) disks = AnnotateDiskParams(self.instance, inst_disks, self.cfg) for idx, dev in enumerate(disks): if idx not in self.disks: continue self.lu.LogInfo("Adding storage on %s for disk/%d", self.cfg.GetNodeName(node_uuid), idx) lv_names = [".disk%d_%s" % (idx, suffix) for suffix in ["data", "meta"]] names = _GenerateUniqueNames(self.lu, lv_names) (data_disk, meta_disk) = dev.children vg_data = data_disk.logical_id[0] lv_data = objects.Disk(dev_type=constants.DT_PLAIN, size=dev.size, logical_id=(vg_data, names[0]), params=data_disk.params) vg_meta = meta_disk.logical_id[0] lv_meta = objects.Disk(dev_type=constants.DT_PLAIN, size=constants.DRBD_META_SIZE, logical_id=(vg_meta, names[1]), params=meta_disk.params) new_lvs = [lv_data, lv_meta] old_lvs = [child.Copy() for child in dev.children] iv_names[dev.iv_name] = (dev, old_lvs, new_lvs) excl_stor = IsExclusiveStorageEnabledNodeUuid(self.lu.cfg, node_uuid) # we pass force_create=True to force the LVM creation for new_lv in new_lvs: try: _CreateBlockDevInner(self.lu, node_uuid, self.instance, new_lv, True, GetInstanceInfoText(self.instance), False, excl_stor) except errors.DeviceCreationError as e: raise errors.OpExecError("Can't create block device: %s" % e.message) return iv_names def _CheckDevices(self, node_uuid, iv_names): for name, (dev, _, _) in iv_names.items(): result = _BlockdevFind(self, node_uuid, dev, self.instance) msg = result.fail_msg if msg or not result.payload: if not msg: msg = "disk not found" raise errors.OpExecError("Can't find DRBD device %s: %s" % (name, msg)) if result.payload.is_degraded: raise errors.OpExecError("DRBD device %s is degraded!" % name) def _RemoveOldStorage(self, node_uuid, iv_names): for name, (_, old_lvs, _) in iv_names.items(): self.lu.LogInfo("Remove logical volumes for %s", name) for lv in old_lvs: msg = self.rpc.call_blockdev_remove(node_uuid, (lv, self.instance)) \ .fail_msg if msg: self.lu.LogWarning("Can't remove old LV: %s", msg, hint="remove unused LVs manually") def _ExecDrbd8DiskOnly(self, feedback_fn): # pylint: disable=W0613 """Replace a disk on the primary or secondary for DRBD 8. The algorithm for replace is quite complicated: 1. for each disk to be replaced: 1. create new LVs on the target node with unique names 1. detach old LVs from the drbd device 1. rename old LVs to name_replaced. 1. rename new LVs to old LVs 1. attach the new LVs (with the old names now) to the drbd device 1. wait for sync across all devices 1. for each modified disk: 1. remove old LVs (which have the name name_replaces.) Failures are not very well handled. """ steps_total = 6 if self.instance.forthcoming: feedback_fn("Instance forthcoming, not touching disks") return # Step: check device activation self.lu.LogStep(1, steps_total, "Check device existence") self._CheckDisksExistence([self.other_node_uuid, self.target_node_uuid]) self._CheckVolumeGroup([self.target_node_uuid, self.other_node_uuid]) # Step: check other node consistency self.lu.LogStep(2, steps_total, "Check peer consistency") self._CheckDisksConsistency( self.other_node_uuid, self.other_node_uuid == self.instance.primary_node, False) # Step: create new storage self.lu.LogStep(3, steps_total, "Allocate new storage") iv_names = self._CreateNewStorage(self.target_node_uuid) # Step: for each lv, detach+rename*2+attach self.lu.LogStep(4, steps_total, "Changing drbd configuration") for dev, old_lvs, new_lvs in iv_names.values(): self.lu.LogInfo("Detaching %s drbd from local storage", dev.iv_name) result = self.rpc.call_blockdev_removechildren(self.target_node_uuid, (dev, self.instance), (old_lvs, self.instance)) result.Raise("Can't detach drbd from local storage on node" " %s for device %s" % (self.cfg.GetNodeName(self.target_node_uuid), dev.iv_name)) #dev.children = [] #cfg.Update(instance) # ok, we created the new LVs, so now we know we have the needed # storage; as such, we proceed on the target node to rename # old_lv to _old, and new_lv to old_lv; note that we rename LVs # using the assumption that logical_id == unique_id on that node # FIXME(iustin): use a better name for the replaced LVs temp_suffix = int(time.time()) ren_fn = lambda d, suff: (d.logical_id[0], d.logical_id[1] + "_replaced-%s" % suff) # Build the rename list based on what LVs exist on the node rename_old_to_new = [] for to_ren in old_lvs: result = self.rpc.call_blockdev_find(self.target_node_uuid, (to_ren, self.instance)) if not result.fail_msg and result.payload: # device exists rename_old_to_new.append((to_ren, ren_fn(to_ren, temp_suffix))) self.lu.LogInfo("Renaming the old LVs on the target node") result = self.rpc.call_blockdev_rename(self.target_node_uuid, rename_old_to_new) result.Raise("Can't rename old LVs on node %s" % self.cfg.GetNodeName(self.target_node_uuid)) # Now we rename the new LVs to the old LVs self.lu.LogInfo("Renaming the new LVs on the target node") rename_new_to_old = [(new, old.logical_id) for old, new in zip(old_lvs, new_lvs)] result = self.rpc.call_blockdev_rename(self.target_node_uuid, rename_new_to_old) result.Raise("Can't rename new LVs on node %s" % self.cfg.GetNodeName(self.target_node_uuid)) # Intermediate steps of in memory modifications for old, new in zip(old_lvs, new_lvs): new.logical_id = old.logical_id # We need to modify old_lvs so that removal later removes the # right LVs, not the newly added ones; note that old_lvs is a # copy here for disk in old_lvs: disk.logical_id = ren_fn(disk, temp_suffix) # Now that the new lvs have the old name, we can add them to the device self.lu.LogInfo("Adding new mirror component on %s", self.cfg.GetNodeName(self.target_node_uuid)) result = self.rpc.call_blockdev_addchildren(self.target_node_uuid, (dev, self.instance), (new_lvs, self.instance)) msg = result.fail_msg if msg: for new_lv in new_lvs: msg2 = self.rpc.call_blockdev_remove(self.target_node_uuid, (new_lv, self.instance)).fail_msg if msg2: self.lu.LogWarning("Can't rollback device %s: %s", dev, msg2, hint=("cleanup manually the unused logical" "volumes")) raise errors.OpExecError("Can't add local storage to drbd: %s" % msg) cstep = itertools.count(5) if self.early_release: self.lu.LogStep(next(cstep), steps_total, "Removing old storage") self._RemoveOldStorage(self.target_node_uuid, iv_names) # TODO: Check if releasing locks early still makes sense ReleaseLocks(self.lu, locking.LEVEL_NODE_RES) else: # Release all resource locks except those used by the instance ReleaseLocks(self.lu, locking.LEVEL_NODE_RES, keep=list(self.node_secondary_ip)) # Release all node locks while waiting for sync ReleaseLocks(self.lu, locking.LEVEL_NODE) # TODO: Can the instance lock be downgraded here? Take the optional disk # shutdown in the caller into consideration. # Wait for sync # This can fail as the old devices are degraded and _WaitForSync # does a combined result over all disks, so we don't check its return value self.lu.LogStep(next(cstep), steps_total, "Sync devices") WaitForSync(self.lu, self.instance) # Check all devices manually self._CheckDevices(self.instance.primary_node, iv_names) # Step: remove old storage if not self.early_release: self.lu.LogStep(next(cstep), steps_total, "Removing old storage") self._RemoveOldStorage(self.target_node_uuid, iv_names) def _UpdateDisksSecondary(self, iv_names, feedback_fn): """Update the configuration of disks to have a new secondary. @param iv_names: iterable of triples for all volumes of the instance. The first component has to be the device and the third the logical id. @param feedback_fn: function to used send feedback back to the caller of the OpCode """ self.lu.LogInfo("Updating instance configuration") for dev, _, new_logical_id in iv_names.values(): dev.logical_id = new_logical_id self.cfg.Update(dev, feedback_fn) self.cfg.SetDiskNodes(dev.uuid, [self.instance.primary_node, self.new_node_uuid]) self.cfg.Update(self.instance, feedback_fn) def _ExecDrbd8Secondary(self, feedback_fn): """Replace the secondary node for DRBD 8. The algorithm for replace is quite complicated: - for all disks of the instance: - create new LVs on the new node with same names - shutdown the drbd device on the old secondary - disconnect the drbd network on the primary - create the drbd device on the new secondary - network attach the drbd on the primary, using an artifice: the drbd code for Attach() will connect to the network if it finds a device which is connected to the good local disks but not network enabled - wait for sync across all devices - remove all disks from the old secondary Failures are not very well handled. """ if self.instance.forthcoming: feedback_fn("Instance fortcoming, will only update the configuration") inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) minors = self.cfg.AllocateDRBDMinor([self.new_node_uuid for _ in inst_disks], self.instance.uuid) logging.debug("Allocated minors %r", minors) iv_names = {} for idx, (dev, new_minor) in enumerate(zip(inst_disks, minors)): (o_node1, _, o_port, o_minor1, o_minor2, o_secret) = \ dev.logical_id if self.instance.primary_node == o_node1: p_minor = o_minor1 else: p_minor = o_minor2 new_net_id = (self.instance.primary_node, self.new_node_uuid, o_port, p_minor, new_minor, o_secret) iv_names[idx] = (dev, dev.children, new_net_id) logging.debug("Allocated new_minor: %s, new_logical_id: %s", new_minor, new_net_id) self._UpdateDisksSecondary(iv_names, feedback_fn) ReleaseLocks(self.lu, locking.LEVEL_NODE) return steps_total = 6 pnode = self.instance.primary_node # Step: check device activation self.lu.LogStep(1, steps_total, "Check device existence") self._CheckDisksExistence([self.instance.primary_node]) self._CheckVolumeGroup([self.instance.primary_node]) # Step: check other node consistency self.lu.LogStep(2, steps_total, "Check peer consistency") self._CheckDisksConsistency(self.instance.primary_node, True, True) # Step: create new storage self.lu.LogStep(3, steps_total, "Allocate new storage") inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) disks = AnnotateDiskParams(self.instance, inst_disks, self.cfg) excl_stor = IsExclusiveStorageEnabledNodeUuid(self.lu.cfg, self.new_node_uuid) for idx, dev in enumerate(disks): self.lu.LogInfo("Adding new local storage on %s for disk/%d" % (self.cfg.GetNodeName(self.new_node_uuid), idx)) # we pass force_create=True to force LVM creation for new_lv in dev.children: try: _CreateBlockDevInner(self.lu, self.new_node_uuid, self.instance, new_lv, True, GetInstanceInfoText(self.instance), False, excl_stor) except errors.DeviceCreationError as e: raise errors.OpExecError("Can't create block device: %s" % e.message) # Step 4: dbrd minors and drbd setups changes # after this, we must manually remove the drbd minors on both the # error and the success paths self.lu.LogStep(4, steps_total, "Changing drbd configuration") minors = [] for disk in inst_disks: minor = self.cfg.AllocateDRBDMinor([self.new_node_uuid], disk.uuid) minors.append(minor[0]) logging.debug("Allocated minors %r", minors) iv_names = {} for idx, (dev, new_minor) in enumerate(zip(inst_disks, minors)): self.lu.LogInfo("activating a new drbd on %s for disk/%d" % (self.cfg.GetNodeName(self.new_node_uuid), idx)) # create new devices on new_node; note that we create two IDs: # one without port, so the drbd will be activated without # networking information on the new node at this stage, and one # with network, for the latter activation in step 4 (o_node1, o_node2, o_port, o_minor1, o_minor2, o_secret) = dev.logical_id if self.instance.primary_node == o_node1: p_minor = o_minor1 else: assert self.instance.primary_node == o_node2, "Three-node instance?" p_minor = o_minor2 new_alone_id = (self.instance.primary_node, self.new_node_uuid, None, p_minor, new_minor, o_secret) new_net_id = (self.instance.primary_node, self.new_node_uuid, o_port, p_minor, new_minor, o_secret) iv_names[idx] = (dev, dev.children, new_net_id) logging.debug("Allocated new_minor: %s, new_logical_id: %s", new_minor, new_net_id) new_drbd = objects.Disk(dev_type=constants.DT_DRBD8, logical_id=new_alone_id, children=dev.children, size=dev.size, params={}) (anno_new_drbd,) = AnnotateDiskParams(self.instance, [new_drbd], self.cfg) try: CreateSingleBlockDev(self.lu, self.new_node_uuid, self.instance, anno_new_drbd, GetInstanceInfoText(self.instance), False, excl_stor) except errors.GenericError: for disk in inst_disks: self.cfg.ReleaseDRBDMinors(disk.uuid) raise # We have new devices, shutdown the drbd on the old secondary for idx, dev in enumerate(inst_disks): self.lu.LogInfo("Shutting down drbd for disk/%d on old node", idx) msg = self.rpc.call_blockdev_shutdown(self.target_node_uuid, (dev, self.instance)).fail_msg if msg: self.lu.LogWarning("Failed to shutdown drbd for disk/%d on old" "node: %s" % (idx, msg), hint=("Please cleanup this device manually as" " soon as possible")) self.lu.LogInfo("Detaching primary drbds from the network (=> standalone)") result = self.rpc.call_drbd_disconnect_net( [pnode], (inst_disks, self.instance))[pnode] msg = result.fail_msg if msg: # detaches didn't succeed (unlikely) for disk in inst_disks: self.cfg.ReleaseDRBDMinors(disk.uuid) raise errors.OpExecError("Can't detach the disks from the network on" " old node: %s" % (msg,)) # if we managed to detach at least one, we update all the disks of # the instance to point to the new secondary self._UpdateDisksSecondary(iv_names, feedback_fn) # Release all node locks (the configuration has been updated) ReleaseLocks(self.lu, locking.LEVEL_NODE) # and now perform the drbd attach self.lu.LogInfo("Attaching primary drbds to new secondary" " (standalone => connected)") inst_disks = self.cfg.GetInstanceDisks(self.instance.uuid) result = self.rpc.call_drbd_attach_net([self.instance.primary_node, self.new_node_uuid], (inst_disks, self.instance), False) for to_node, to_result in result.items(): msg = to_result.fail_msg if msg: raise errors.OpExecError( "Can't attach drbd disks on node %s: %s (please do a gnt-instance " "info %s to see the status of disks)" % (self.cfg.GetNodeName(to_node), msg, self.instance.name)) cstep = itertools.count(5) if self.early_release: self.lu.LogStep(next(cstep), steps_total, "Removing old storage") self._RemoveOldStorage(self.target_node_uuid, iv_names) # TODO: Check if releasing locks early still makes sense ReleaseLocks(self.lu, locking.LEVEL_NODE_RES) else: # Release all resource locks except those used by the instance ReleaseLocks(self.lu, locking.LEVEL_NODE_RES, keep=list(self.node_secondary_ip)) # TODO: Can the instance lock be downgraded here? Take the optional disk # shutdown in the caller into consideration. # Wait for sync # This can fail as the old devices are degraded and _WaitForSync # does a combined result over all disks, so we don't check its return value self.lu.LogStep(next(cstep), steps_total, "Sync devices") WaitForSync(self.lu, self.instance) # Check all devices manually self._CheckDevices(self.instance.primary_node, iv_names) # Step: remove old storage if not self.early_release: self.lu.LogStep(next(cstep), steps_total, "Removing old storage") self._RemoveOldStorage(self.target_node_uuid, iv_names) class TemporaryDisk(object): """ Creates a new temporary bootable disk, and makes sure it is destroyed. Is a context manager, and should be used with the ``with`` statement as such. The disk is guaranteed to be created at index 0, shifting any other disks of the instance by one place, and allowing the instance to be booted with the content of the disk. """ def __init__(self, lu, instance, disks, feedback_fn, shutdown_timeout=constants.DEFAULT_SHUTDOWN_TIMEOUT): """ Constructor storing arguments until used later. @type lu: L{ganeti.cmdlib.base.LogicalUnit} @param lu: The LU within which this disk is created. @type instance: L{ganeti.objects.Instance} @param instance: The instance to which the disk should be added @type disks: list of triples (disk template, disk access mode, int) @param disks: disk specification, which is a list of triples containing the disk template (e.g., L{constants.DT_PLAIN}), the disk access mode (i.e., L{constants.DISK_RDONLY} or L{constants.DISK_RDWR}), and size in MiB. @type feedback_fn: function @param feedback_fn: Function used to log progress """ self._lu = lu self._instance = instance self._disks = disks self._feedback_fn = feedback_fn self._shutdown_timeout = shutdown_timeout def _EnsureInstanceDiskState(self): """ Ensures that the instance is down, and its disks inactive. All the operations related to the creation and destruction of disks require that the instance is down and that the disks are inactive. This function is invoked to make it so. """ # The instance needs to be down before any of these actions occur # Whether it is must be checked manually through a RPC - configuration # reflects only the desired state self._feedback_fn("Shutting down instance") result = self._lu.rpc.call_instance_shutdown(self._instance.primary_node, self._instance, self._shutdown_timeout, self._lu.op.reason) result.Raise("Shutdown of instance '%s' while removing temporary disk " "failed" % self._instance.name) # Disks need to be deactivated prior to being removed # The disks_active configuration entry should match the actual state if self._instance.disks_active: self._feedback_fn("Deactivating disks") ShutdownInstanceDisks(self._lu, self._instance) def __enter__(self): """ Context manager entry function, creating the disk. @rtype: L{ganeti.objects.Disk} @return: The disk object created. """ self._EnsureInstanceDiskState() new_disks = [] # The iv_name of the disk intentionally diverges from Ganeti's standards, as # this disk should be very temporary and its presence should be reported. # With the special iv_name, gnt-cluster verify detects the disk and warns # the user of its presence. Removing the disk restores the instance to its # proper state, despite an error that appears when the removal is performed. for idx, (disk_template, disk_access, disk_size) in enumerate(self._disks): new_disk = objects.Disk() new_disk.dev_type = disk_template new_disk.mode = disk_access new_disk.uuid = self._lu.cfg.GenerateUniqueID(self._lu.proc.GetECId()) new_disk.logical_id = (self._lu.cfg.GetVGName(), new_disk.uuid) new_disk.params = {} new_disk.size = disk_size new_disks.append(new_disk) self._feedback_fn("Attempting to create temporary disk") self._undoing_info = CreateDisks(self._lu, self._instance, disks=new_disks) for idx, new_disk in enumerate(new_disks): self._lu.cfg.AddInstanceDisk(self._instance.uuid, new_disk, idx=idx) self._instance = self._lu.cfg.GetInstanceInfo(self._instance.uuid) self._feedback_fn("Temporary disk created") self._new_disks = new_disks return new_disks def __exit__(self, exc_type, _value, _traceback): """ Context manager exit function, destroying the disk. """ if exc_type: self._feedback_fn("Exception raised, cleaning up temporary disk") else: self._feedback_fn("Regular cleanup of temporary disk") try: self._EnsureInstanceDiskState() _UndoCreateDisks(self._lu, self._undoing_info, self._instance) for disk in self._new_disks: self._lu.cfg.RemoveInstanceDisk(self._instance.uuid, disk.uuid) self._instance = self._lu.cfg.GetInstanceInfo(self._instance.uuid) self._feedback_fn("Temporary disk removed") except: self._feedback_fn("Disk cleanup failed; it will have to be removed " "manually") raise ganeti-3.1.0~rc2/lib/cmdlib/instance_utils.py000064400000000000000000001261341476477700300212110ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility function mainly, but not only used by instance LU's.""" import logging import os from ganeti import constants from ganeti import errors from ganeti import ht from ganeti import locking from ganeti.masterd import iallocator from ganeti import netutils from ganeti import objects from ganeti import pathutils from ganeti import utils from ganeti.cmdlib.common import AnnotateDiskParams, \ ComputeIPolicyInstanceViolation, CheckDiskTemplateEnabled, \ ComputeIPolicySpecViolation #: Type description for changes as returned by L{ApplyContainerMods}'s #: callbacks _TApplyContModsCbChanges = \ ht.TMaybeListOf(ht.TAnd(ht.TIsLength(2), ht.TItems([ ht.TNonEmptyString, ht.TAny, ]))) def BuildInstanceHookEnv(name, primary_node_name, secondary_node_names, os_type, status, minmem, maxmem, vcpus, nics, disk_template, disks, bep, hvp, hypervisor_name, tags): """Builds instance related env variables for hooks This builds the hook environment from individual variables. @type name: string @param name: the name of the instance @type primary_node_name: string @param primary_node_name: the name of the instance's primary node @type secondary_node_names: list @param secondary_node_names: list of secondary nodes as strings @type os_type: string @param os_type: the name of the instance's OS @type status: string @param status: the desired status of the instance @type minmem: string @param minmem: the minimum memory size of the instance @type maxmem: string @param maxmem: the maximum memory size of the instance @type vcpus: string @param vcpus: the count of VCPUs the instance has @type nics: list @param nics: list of tuples (name, uuid, ip, mac, mode, link, vlan, net, netinfo) representing the NICs the instance has @type disk_template: string @param disk_template: the disk template of the instance @type disks: list @param disks: list of disks (either objects.Disk or dict) @type bep: dict @param bep: the backend parameters for the instance @type hvp: dict @param hvp: the hypervisor parameters for the instance @type hypervisor_name: string @param hypervisor_name: the hypervisor for the instance @type tags: list @param tags: list of instance tags as strings @rtype: dict @return: the hook environment for this instance """ env = { "OP_TARGET": name, "INSTANCE_NAME": name, "INSTANCE_PRIMARY": primary_node_name, "INSTANCE_SECONDARIES": " ".join(secondary_node_names), "INSTANCE_OS_TYPE": os_type, "INSTANCE_STATUS": status, "INSTANCE_MINMEM": minmem, "INSTANCE_MAXMEM": maxmem, # TODO(2.9) remove deprecated "memory" value "INSTANCE_MEMORY": maxmem, "INSTANCE_VCPUS": vcpus, "INSTANCE_DISK_TEMPLATE": disk_template, "INSTANCE_HYPERVISOR": hypervisor_name, } if nics: nic_count = len(nics) for idx, (name, uuid, ip, mac, mode, link, vlan, net, netinfo) \ in enumerate(nics): if ip is None: ip = "" if name: env["INSTANCE_NIC%d_NAME" % idx] = name env["INSTANCE_NIC%d_UUID" % idx] = uuid env["INSTANCE_NIC%d_IP" % idx] = ip env["INSTANCE_NIC%d_MAC" % idx] = mac env["INSTANCE_NIC%d_MODE" % idx] = mode env["INSTANCE_NIC%d_LINK" % idx] = link env["INSTANCE_NIC%d_VLAN" % idx] = vlan if netinfo: nobj = objects.Network.FromDict(netinfo) env.update(nobj.HooksDict("INSTANCE_NIC%d_" % idx)) elif net: # FIXME: broken network reference: the instance NIC specifies a # network, but the relevant network entry was not in the config. This # should be made impossible. env["INSTANCE_NIC%d_NETWORK_NAME" % idx] = net if mode == constants.NIC_MODE_BRIDGED or \ mode == constants.NIC_MODE_OVS: env["INSTANCE_NIC%d_BRIDGE" % idx] = link else: nic_count = 0 env["INSTANCE_NIC_COUNT"] = nic_count if disks: disk_count = len(disks) for idx, disk in enumerate(disks): env.update(BuildDiskEnv(idx, disk)) else: disk_count = 0 env["INSTANCE_DISK_COUNT"] = disk_count if not tags: tags = [] env["INSTANCE_TAGS"] = " ".join(tags) for source, kind in [(bep, "BE"), (hvp, "HV")]: for key, value in source.items(): env["INSTANCE_%s_%s" % (kind, key)] = value return env def BuildInstanceHookEnvByObject(lu, instance, secondary_nodes=None, disks=None, override=None): """Builds instance related env variables for hooks from an object. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type instance: L{objects.Instance} @param instance: the instance for which we should build the environment @type override: dict @param override: dictionary with key/values that will override our values @rtype: dict @return: the hook environment dictionary """ cluster = lu.cfg.GetClusterInfo() bep = cluster.FillBE(instance) hvp = cluster.FillHV(instance) # Override secondary_nodes if secondary_nodes is None: secondary_nodes = lu.cfg.GetInstanceSecondaryNodes(instance.uuid) # Override disks if disks is None: disks = lu.cfg.GetInstanceDisks(instance.uuid) disk_template = utils.GetDiskTemplate(disks) args = { "name": instance.name, "primary_node_name": lu.cfg.GetNodeName(instance.primary_node), "secondary_node_names": lu.cfg.GetNodeNames(secondary_nodes), "os_type": instance.os, "status": instance.admin_state, "maxmem": bep[constants.BE_MAXMEM], "minmem": bep[constants.BE_MINMEM], "vcpus": bep[constants.BE_VCPUS], "nics": NICListToTuple(lu, instance.nics), "disk_template": disk_template, "disks": disks, "bep": bep, "hvp": hvp, "hypervisor_name": instance.hypervisor, "tags": instance.tags, } if override: args.update(override) return BuildInstanceHookEnv(**args) def GetClusterDomainSecret(): """Reads the cluster domain secret. """ return utils.ReadOneLineFile(pathutils.CLUSTER_DOMAIN_SECRET_FILE, strict=True) def CheckNodeNotDrained(lu, node_uuid): """Ensure that a given node is not drained. @param lu: the LU on behalf of which we make the check @param node_uuid: the node to check @raise errors.OpPrereqError: if the node is drained """ node = lu.cfg.GetNodeInfo(node_uuid) if node.drained: raise errors.OpPrereqError("Can't use drained node %s" % node.name, errors.ECODE_STATE) def CheckNodeVmCapable(lu, node_uuid): """Ensure that a given node is vm capable. @param lu: the LU on behalf of which we make the check @param node_uuid: the node to check @raise errors.OpPrereqError: if the node is not vm capable """ if not lu.cfg.GetNodeInfo(node_uuid).vm_capable: raise errors.OpPrereqError("Can't use non-vm_capable node %s" % node_uuid, errors.ECODE_STATE) def RemoveInstance(lu, feedback_fn, instance, ignore_failures): """Utility function to remove an instance. """ logging.info("Removing block devices for instance %s", instance.name) if not RemoveDisks(lu, instance, ignore_failures=ignore_failures): if not ignore_failures: raise errors.OpExecError("Can't remove instance's disks") feedback_fn("Warning: can't remove instance's disks") logging.info("Removing instance's disks") for disk in instance.disks: lu.cfg.RemoveInstanceDisk(instance.uuid, disk) logging.info("Removing instance %s out of cluster config", instance.name) lu.cfg.RemoveInstance(instance.uuid) def _StoragePathsRemoved(removed, disks): """Returns an iterable of all storage paths to be removed. A storage path is removed if no disks are contained in it anymore. @type removed: list of L{objects.Disk} @param removed: The disks that are being removed @type disks: list of L{objects.Disk} @param disks: All disks attached to the instance @rtype: list of file paths @returns: the storage directories that need to be removed """ remaining_storage_dirs = set() for disk in disks: if (disk not in removed and disk.dev_type in (constants.DT_FILE, constants.DT_SHARED_FILE)): remaining_storage_dirs.add(os.path.dirname(disk.logical_id[1])) deleted_storage_dirs = set() for disk in removed: if disk.dev_type in (constants.DT_FILE, constants.DT_SHARED_FILE): deleted_storage_dirs.add(os.path.dirname(disk.logical_id[1])) return deleted_storage_dirs - remaining_storage_dirs def RemoveDisks(lu, instance, disks=None, target_node_uuid=None, ignore_failures=False): """Remove all or a subset of disks for an instance. This abstracts away some work from `AddInstance()` and `RemoveInstance()`. Note that in case some of the devices couldn't be removed, the removal will continue with the other ones. This function is also used by the disk template conversion mechanism to remove the old block devices of the instance. Since the instance has changed its template at the time we remove the original disks, we must specify the template of the disks we are about to remove as an argument. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type instance: L{objects.Instance} @param instance: the instance whose disks we should remove @type disks: list of L{objects.Disk} @param disks: the disks to remove; if not specified, all the disks of the instance are removed @type target_node_uuid: string @param target_node_uuid: used to override the node on which to remove the disks @rtype: boolean @return: the success of the removal """ logging.info("Removing block devices for instance %s", instance.name) all_result = True ports_to_release = set() all_disks = lu.cfg.GetInstanceDisks(instance.uuid) if disks is None: disks = all_disks anno_disks = AnnotateDiskParams(instance, disks, lu.cfg) uuid_idx_map = {} for (idx, device) in enumerate(all_disks): uuid_idx_map[device.uuid] = idx for (idx, device) in enumerate(anno_disks): if target_node_uuid: edata = [(target_node_uuid, device)] else: edata = device.ComputeNodeTree(instance.primary_node) for node_uuid, disk in edata: result = lu.rpc.call_blockdev_remove(node_uuid, (disk, instance)) if result.fail_msg: lu.LogWarning("Could not remove disk %s on node %s," " continuing anyway: %s", uuid_idx_map.get(device.uuid), lu.cfg.GetNodeName(node_uuid), result.fail_msg) if not (result.offline and node_uuid != instance.primary_node): all_result = False # if this is a DRBD disk, return its port to the pool if device.dev_type in constants.DTS_DRBD: ports_to_release.add(device.logical_id[2]) if all_result or ignore_failures: for port in ports_to_release: lu.cfg.AddTcpUdpPort(port) for d in disks: CheckDiskTemplateEnabled(lu.cfg.GetClusterInfo(), d.dev_type) if target_node_uuid: tgt = target_node_uuid else: tgt = instance.primary_node obsolete_storage_paths = _StoragePathsRemoved(disks, all_disks) for file_storage_dir in obsolete_storage_paths: result = lu.rpc.call_file_storage_dir_remove(tgt, file_storage_dir) if result.fail_msg: lu.LogWarning("Could not remove directory '%s' on node %s: %s", file_storage_dir, lu.cfg.GetNodeName(tgt), result.fail_msg) all_result = False return all_result def NICToTuple(lu, nic): """Build a tupple of nic information. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type nic: L{objects.NIC} @param nic: nic to convert to hooks tuple """ cluster = lu.cfg.GetClusterInfo() filled_params = cluster.SimpleFillNIC(nic.nicparams) mode = filled_params[constants.NIC_MODE] link = filled_params[constants.NIC_LINK] vlan = filled_params[constants.NIC_VLAN] netinfo = None if nic.network: nobj = lu.cfg.GetNetwork(nic.network) netinfo = objects.Network.ToDict(nobj) return (nic.name, nic.uuid, nic.ip, nic.mac, mode, link, vlan, nic.network, netinfo) def NICListToTuple(lu, nics): """Build a list of nic information tuples. This list is suitable to be passed to _BuildInstanceHookEnv or as a return value in LUInstanceQueryData. @type lu: L{LogicalUnit} @param lu: the logical unit on whose behalf we execute @type nics: list of L{objects.NIC} @param nics: list of nics to convert to hooks tuples """ hooks_nics = [] for nic in nics: hooks_nics.append(NICToTuple(lu, nic)) return hooks_nics def CopyLockList(names): """Makes a copy of a list of lock names. Handles L{locking.ALL_SET} correctly. """ if names == locking.ALL_SET: return locking.ALL_SET else: return names[:] def ReleaseLocks(lu, level, names=None, keep=None): """Releases locks owned by an LU. @type lu: L{LogicalUnit} @param level: Lock level @type names: list or None @param names: Names of locks to release @type keep: list or None @param keep: Names of locks to retain """ logging.debug("Lu %s ReleaseLocks %s names=%s, keep=%s", lu.wconfdcontext, level, names, keep) assert not (keep is not None and names is not None), \ "Only one of the 'names' and the 'keep' parameters can be given" if names is not None: should_release = names.__contains__ elif keep: should_release = lambda name: name not in keep else: should_release = None levelname = locking.LEVEL_NAMES[level] owned = lu.owned_locks(level) if not owned: # Not owning any lock at this level, do nothing pass elif should_release: retain = [] release = [] # Determine which locks to release for name in owned: if should_release(name): release.append(name) else: retain.append(name) assert len(lu.owned_locks(level)) == (len(retain) + len(release)) # Release just some locks lu.WConfdClient().TryUpdateLocks( lu.release_request(level, release)) assert frozenset(lu.owned_locks(level)) == frozenset(retain) else: lu.WConfdClient().FreeLocksLevel(levelname) def _ComputeIPolicyNodeViolation(ipolicy, instance, current_group, target_group, cfg, _compute_fn=ComputeIPolicyInstanceViolation): """Compute if instance meets the specs of the new target group. @param ipolicy: The ipolicy to verify @param instance: The instance object to verify @param current_group: The current group of the instance @param target_group: The new group of the instance @type cfg: L{config.ConfigWriter} @param cfg: Cluster configuration @param _compute_fn: The function to verify ipolicy (unittest only) @see: L{ganeti.cmdlib.common.ComputeIPolicySpecViolation} """ if current_group == target_group: return [] else: return _compute_fn(ipolicy, instance, cfg) def CheckTargetNodeIPolicy(lu, ipolicy, instance, node, cfg, ignore=False, _compute_fn=_ComputeIPolicyNodeViolation): """Checks that the target node is correct in terms of instance policy. @param ipolicy: The ipolicy to verify @param instance: The instance object to verify @param node: The new node to relocate @type cfg: L{config.ConfigWriter} @param cfg: Cluster configuration @param ignore: Ignore violations of the ipolicy @param _compute_fn: The function to verify ipolicy (unittest only) @see: L{ganeti.cmdlib.common.ComputeIPolicySpecViolation} """ primary_node = lu.cfg.GetNodeInfo(instance.primary_node) res = _compute_fn(ipolicy, instance, primary_node.group, node.group, cfg) if res: msg = ("Instance does not meet target node group's (%s) instance" " policy: %s") % (node.group, utils.CommaJoin(res)) if ignore: lu.LogWarning(msg) else: raise errors.OpPrereqError(msg, errors.ECODE_INVAL) def GetInstanceInfoText(instance): """Compute that text that should be added to the disk's metadata. """ return "originstname+%s" % instance.name def CheckNodeFreeMemory(lu, node_uuid, reason, requested, hvname, hvparams): """Checks if a node has enough free memory. This function checks if a given node has the needed amount of free memory. In case the node has less memory or we cannot get the information from the node, this function raises an OpPrereqError exception. @type lu: C{LogicalUnit} @param lu: a logical unit from which we get configuration data @type node_uuid: C{str} @param node_uuid: the node to check @type reason: C{str} @param reason: string to use in the error message @type requested: C{int} @param requested: the amount of memory in MiB to check for @type hvname: string @param hvname: the hypervisor's name @type hvparams: dict of strings @param hvparams: the hypervisor's parameters @rtype: integer @return: node current free memory @raise errors.OpPrereqError: if the node doesn't have enough memory, or we cannot check the node """ node_name = lu.cfg.GetNodeName(node_uuid) nodeinfo = lu.rpc.call_node_info([node_uuid], None, [(hvname, hvparams)]) nodeinfo[node_uuid].Raise("Can't get data from node %s" % node_name, prereq=True, ecode=errors.ECODE_ENVIRON) (_, _, (hv_info, )) = nodeinfo[node_uuid].payload free_mem = hv_info.get("memory_free", None) if not isinstance(free_mem, int): raise errors.OpPrereqError("Can't compute free memory on node %s, result" " was '%s'" % (node_name, free_mem), errors.ECODE_ENVIRON) if requested > free_mem: raise errors.OpPrereqError("Not enough memory on node %s for %s:" " needed %s MiB, available %s MiB" % (node_name, reason, requested, free_mem), errors.ECODE_NORES) return free_mem def CheckInstanceBridgesExist(lu, instance, node_uuid=None): """Check that the brigdes needed by an instance exist. """ if node_uuid is None: node_uuid = instance.primary_node CheckNicsBridgesExist(lu, instance.nics, node_uuid) def CheckNicsBridgesExist(lu, nics, node_uuid): """Check that the brigdes needed by a list of nics exist. """ cluster = lu.cfg.GetClusterInfo() paramslist = [cluster.SimpleFillNIC(nic.nicparams) for nic in nics] brlist = [params[constants.NIC_LINK] for params in paramslist if params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED] if brlist: result = lu.rpc.call_bridges_exist(node_uuid, brlist) result.Raise("Error checking bridges on destination node '%s'" % lu.cfg.GetNodeName(node_uuid), prereq=True, ecode=errors.ECODE_ENVIRON) def UpdateMetadata(feedback_fn, rpc, instance, osparams_public=None, osparams_private=None, osparams_secret=None): """Updates instance metadata on the metadata daemon on the instance's primary node. If the daemon isn't available (not compiled), do nothing. In case the RPC fails, this function simply issues a warning and proceeds normally. @type feedback_fn: callable @param feedback_fn: function used send feedback back to the caller @type rpc: L{rpc.node.RpcRunner} @param rpc: RPC runner @type instance: L{objects.Instance} @param instance: instance for which the metadata should be updated @type osparams_public: NoneType or dict @param osparams_public: public OS parameters used to override those defined in L{instance} @type osparams_private: NoneType or dict @param osparams_private: private OS parameters used to override those defined in L{instance} @type osparams_secret: NoneType or dict @param osparams_secret: secret OS parameters used to override those defined in L{instance} @rtype: NoneType @return: None """ if not constants.ENABLE_METAD: return data = instance.ToDict() if osparams_public is not None: data["osparams_public"] = osparams_public if osparams_private is not None: data["osparams_private"] = osparams_private if osparams_secret is not None: data["osparams_secret"] = osparams_secret else: data["osparams_secret"] = {} result = rpc.call_instance_metadata_modify(instance.primary_node, data) result.Warn("Could not update metadata for instance '%s'" % instance.name, feedback_fn) def CheckCompressionTool(lu, compression_tool): """ Checks if the provided compression tool is allowed to be used. @type compression_tool: string @param compression_tool: Compression tool to use for importing or exporting the instance @rtype: NoneType @return: None @raise errors.OpPrereqError: If the tool is not enabled by Ganeti or whitelisted """ allowed_tools = lu.cfg.GetCompressionTools() if (compression_tool != constants.IEC_NONE and compression_tool not in allowed_tools): raise errors.OpPrereqError( "Compression tool not allowed, tools allowed are [%s]" % ", ".join(allowed_tools), errors.ECODE_INVAL ) def BuildDiskLogicalIDEnv(idx, disk): """Helper method to create hooks env related to disk's logical_id @type idx: integer @param idx: The index of the disk @type disk: L{objects.Disk} @param disk: The disk object """ if disk.dev_type == constants.DT_PLAIN: vg, name = disk.logical_id ret = { "INSTANCE_DISK%d_VG" % idx: vg, "INSTANCE_DISK%d_ID" % idx: name } elif disk.dev_type in (constants.DT_FILE, constants.DT_SHARED_FILE): file_driver, name = disk.logical_id ret = { "INSTANCE_DISK%d_DRIVER" % idx: file_driver, "INSTANCE_DISK%d_ID" % idx: name } elif disk.dev_type == constants.DT_BLOCK: block_driver, adopt = disk.logical_id ret = { "INSTANCE_DISK%d_DRIVER" % idx: block_driver, "INSTANCE_DISK%d_ID" % idx: adopt } elif disk.dev_type == constants.DT_RBD: rbd, name = disk.logical_id ret = { "INSTANCE_DISK%d_DRIVER" % idx: rbd, "INSTANCE_DISK%d_ID" % idx: name } elif disk.dev_type == constants.DT_EXT: provider, name = disk.logical_id ret = { "INSTANCE_DISK%d_PROVIDER" % idx: provider, "INSTANCE_DISK%d_ID" % idx: name } elif disk.dev_type == constants.DT_DRBD8: pnode, snode, port, pmin, smin, _ = disk.logical_id data, meta = disk.children data_vg, data_name = data.logical_id meta_vg, meta_name = meta.logical_id ret = { "INSTANCE_DISK%d_PNODE" % idx: pnode, "INSTANCE_DISK%d_SNODE" % idx: snode, "INSTANCE_DISK%d_PORT" % idx: port, "INSTANCE_DISK%d_PMINOR" % idx: pmin, "INSTANCE_DISK%d_SMINOR" % idx: smin, "INSTANCE_DISK%d_DATA_VG" % idx: data_vg, "INSTANCE_DISK%d_DATA_ID" % idx: data_name, "INSTANCE_DISK%d_META_VG" % idx: meta_vg, "INSTANCE_DISK%d_META_ID" % idx: meta_name, } elif disk.dev_type == constants.DT_GLUSTER: file_driver, name = disk.logical_id ret = { "INSTANCE_DISK%d_DRIVER" % idx: file_driver, "INSTANCE_DISK%d_ID" % idx: name } elif disk.dev_type == constants.DT_DISKLESS: ret = {} else: ret = {} ret.update({ "INSTANCE_DISK%d_DEV_TYPE" % idx: disk.dev_type }) return ret def BuildDiskEnv(idx, disk): """Helper method to create disk's hooks env @type idx: integer @param idx: The index of the disk @type disk: L{objects.Disk} or dict @param disk: The disk object or a simple dict in case of LUInstanceCreate """ ret = {} # In case of LUInstanceCreate this runs in CheckPrereq where lu.disks # is a list of dicts i.e the result of ComputeDisks if isinstance(disk, dict): uuid = disk.get("uuid", "") name = disk.get(constants.IDISK_NAME, "") size = disk.get(constants.IDISK_SIZE, "") mode = disk.get(constants.IDISK_MODE, "") elif isinstance(disk, objects.Disk): uuid = disk.uuid name = disk.name size = disk.size mode = disk.mode ret.update(BuildDiskLogicalIDEnv(idx, disk)) # only name is optional here if name: ret["INSTANCE_DISK%d_NAME" % idx] = name ret["INSTANCE_DISK%d_UUID" % idx] = uuid ret["INSTANCE_DISK%d_SIZE" % idx] = size ret["INSTANCE_DISK%d_MODE" % idx] = mode return ret def CheckInstanceExistence(lu, instance_name): """Raises an error if an instance with the given name exists already. @type instance_name: string @param instance_name: The name of the instance. To be used in the locking phase. """ if instance_name in \ [inst.name for inst in lu.cfg.GetAllInstancesInfo().values()]: raise errors.OpPrereqError("Instance '%s' is already in the cluster" % instance_name, errors.ECODE_EXISTS) def CheckForConflictingIp(lu, ip, node_uuid): """In case of conflicting IP address raise error. @type ip: string @param ip: IP address @type node_uuid: string @param node_uuid: node UUID """ (conf_net, _) = lu.cfg.CheckIPInNodeGroup(ip, node_uuid) if conf_net is not None: raise errors.OpPrereqError(("The requested IP address (%s) belongs to" " network %s, but the target NIC does not." % (ip, conf_net)), errors.ECODE_STATE) return (None, None) def ComputeIPolicyInstanceSpecViolation( ipolicy, instance_spec, disk_types, _compute_fn=ComputeIPolicySpecViolation): """Compute if instance specs meets the specs of ipolicy. @type ipolicy: dict @param ipolicy: The ipolicy to verify against @param instance_spec: dict @param instance_spec: The instance spec to verify @type disk_types: list of strings @param disk_types: the disk templates of the instance @param _compute_fn: The function to verify ipolicy (unittest only) @see: L{ComputeIPolicySpecViolation} """ mem_size = instance_spec.get(constants.ISPEC_MEM_SIZE, None) cpu_count = instance_spec.get(constants.ISPEC_CPU_COUNT, None) disk_count = instance_spec.get(constants.ISPEC_DISK_COUNT, 0) disk_sizes = instance_spec.get(constants.ISPEC_DISK_SIZE, []) nic_count = instance_spec.get(constants.ISPEC_NIC_COUNT, 0) spindle_use = instance_spec.get(constants.ISPEC_SPINDLE_USE, None) return _compute_fn(ipolicy, mem_size, cpu_count, disk_count, nic_count, disk_sizes, spindle_use, disk_types) def ComputeInstanceCommunicationNIC(instance_name): """Compute the name of the instance NIC used by instance communication. With instance communication, a new NIC is added to the instance. This NIC has a special name that identities it as being part of instance communication, and not just a normal NIC. This function generates the name of the NIC based on a prefix and the instance name @type instance_name: string @param instance_name: name of the instance the NIC belongs to @rtype: string @return: name of the NIC """ return constants.INSTANCE_COMMUNICATION_NIC_PREFIX + instance_name def PrepareContainerMods(mods, private_fn): """Prepares a list of container modifications by adding a private data field. @type mods: list of tuples; (operation, index, parameters) @param mods: List of modifications @type private_fn: callable or None @param private_fn: Callable for constructing a private data field for a modification @rtype: list """ if private_fn is None: fn = lambda: None else: fn = private_fn return [(op, idx, params, fn()) for (op, idx, params) in mods] def ApplyContainerMods(kind, container, chgdesc, mods, create_fn, attach_fn, modify_fn, remove_fn, detach_fn, post_add_fn=None): """Applies descriptions in C{mods} to C{container}. @type kind: string @param kind: One-word item description @type container: list @param container: Container to modify @type chgdesc: None or list @param chgdesc: List of applied changes @type mods: list @param mods: Modifications as returned by L{PrepareContainerMods} @type create_fn: callable @param create_fn: Callback for creating a new item (L{constants.DDM_ADD}); receives absolute item index, parameters and private data object as added by L{PrepareContainerMods}, returns tuple containing new item and changes as list @type attach_fn: callable @param attach_fn: Callback for attaching an existing item to a container (L{constants.DDM_ATTACH}); receives absolute item index and item UUID or name, returns tuple containing new item and changes as list @type modify_fn: callable @param modify_fn: Callback for modifying an existing item (L{constants.DDM_MODIFY}); receives absolute item index, item, parameters and private data object as added by L{PrepareContainerMods}, returns changes as list @type remove_fn: callable @param remove_fn: Callback on removing item; receives absolute item index, item and private data object as added by L{PrepareContainerMods} @type detach_fn: callable @param detach_fn: Callback on detaching item; receives absolute item index, item and private data object as added by L{PrepareContainerMods} @type post_add_fn: callable @param post_add_fn: Callable for post-processing a newly created item after it has been put into the container. It receives the index of the new item and the new item as parameters. """ for (op, identifier, params, private) in mods: changes = None if op == constants.DDM_ADD: addidx = GetIndexFromIdentifier(identifier, kind, container) if create_fn is None: item = params else: (item, changes) = create_fn(addidx, params, private) InsertItemToIndex(identifier, item, container) if post_add_fn is not None: post_add_fn(addidx, item) elif op == constants.DDM_ATTACH: addidx = GetIndexFromIdentifier(identifier, kind, container) if attach_fn is None: item = params else: (item, changes) = attach_fn(addidx, params, private) InsertItemToIndex(identifier, item, container) if post_add_fn is not None: post_add_fn(addidx, item) else: # Retrieve existing item (absidx, item) = GetItemFromContainer(identifier, kind, container) if op == constants.DDM_REMOVE: assert not params changes = [("%s/%s" % (kind, absidx), "remove")] if remove_fn is not None: msg = remove_fn(absidx, item, private) if msg: changes.append(("%s/%s" % (kind, absidx), msg)) assert container[absidx] == item del container[absidx] elif op == constants.DDM_DETACH: assert not params changes = [("%s/%s" % (kind, absidx), "detach")] if detach_fn is not None: msg = detach_fn(absidx, item, private) if msg: changes.append(("%s/%s" % (kind, absidx), msg)) assert container[absidx] == item del container[absidx] elif op == constants.DDM_MODIFY: if modify_fn is not None: changes = modify_fn(absidx, item, params, private) else: raise errors.ProgrammerError("Unhandled operation '%s'" % op) assert _TApplyContModsCbChanges(changes) if not (chgdesc is None or changes is None): chgdesc.extend(changes) def GetItemFromContainer(identifier, kind, container): """Return the item refered by the identifier. @type identifier: string @param identifier: Item index or name or UUID @type kind: string @param kind: One-word item description @type container: list @param container: Container to get the item from """ # Index try: idx = int(identifier) if idx == -1: # Append absidx = len(container) - 1 elif idx < 0: raise IndexError("Not accepting negative indices other than -1") elif idx > len(container): raise IndexError("Got %s index %s, but there are only %s" % (kind, idx, len(container))) else: absidx = idx return (absidx, container[idx]) except ValueError: pass for idx, item in enumerate(container): if item.uuid == identifier or item.name == identifier: return (idx, item) raise errors.OpPrereqError("Cannot find %s with identifier %s" % (kind, identifier), errors.ECODE_NOENT) def GetIndexFromIdentifier(identifier, kind, container): """Check if the identifier represents a valid container index and return it. Used in "add" and "attach" actions. @type identifier: string @param identifier: Item index or name or UUID @type kind: string @param kind: Type of item, e.g. "disk", "nic" @type container: list @param container: Container to calculate the index from """ try: idx = int(identifier) except ValueError: raise errors.OpPrereqError("Only positive integer or -1 is accepted", errors.ECODE_INVAL) if idx == -1: return len(container) else: if idx < 0: raise IndexError("Not accepting negative indices other than -1") elif idx > len(container): raise IndexError("Got %s index %s, but there are only %s" % (kind, idx, len(container))) return idx def InsertItemToIndex(identifier, item, container): """Insert an item to the provided index of a container. Used in "add" and "attach" actions. @type identifier: string @param identifier: Item index @type item: object @param item: The item to be inserted @type container: list @param container: Container to insert the item to """ try: idx = int(identifier) except ValueError: raise errors.OpPrereqError("Only positive integer or -1 is accepted", errors.ECODE_INVAL) if idx == -1: container.append(item) else: assert idx >= 0 assert idx <= len(container) # list.insert does so before the specified index container.insert(idx, item) def CheckNodesPhysicalCPUs(lu, node_uuids, requested, hypervisor_specs): """Checks if nodes have enough physical CPUs This function checks if all given nodes have the needed number of physical CPUs. In case any node has less CPUs or we cannot get the information from the node, this function raises an OpPrereqError exception. @type lu: C{LogicalUnit} @param lu: a logical unit from which we get configuration data @type node_uuids: C{list} @param node_uuids: the list of node UUIDs to check @type requested: C{int} @param requested: the minimum acceptable number of physical CPUs @type hypervisor_specs: list of pairs (string, dict of strings) @param hypervisor_specs: list of hypervisor specifications in pairs (hypervisor_name, hvparams) @raise errors.OpPrereqError: if the node doesn't have enough CPUs, or we cannot check the node """ nodeinfo = lu.rpc.call_node_info(node_uuids, None, hypervisor_specs) for node_uuid in node_uuids: info = nodeinfo[node_uuid] node_name = lu.cfg.GetNodeName(node_uuid) info.Raise("Cannot get current information from node %s" % node_name, prereq=True, ecode=errors.ECODE_ENVIRON) (_, _, (hv_info, )) = info.payload num_cpus = hv_info.get("cpu_total", None) if not isinstance(num_cpus, int): raise errors.OpPrereqError("Can't compute the number of physical CPUs" " on node %s, result was '%s'" % (node_name, num_cpus), errors.ECODE_ENVIRON) if requested > num_cpus: raise errors.OpPrereqError("Node %s has %s physical CPUs, but %s are " "required" % (node_name, num_cpus, requested), errors.ECODE_NORES) def CheckHostnameSane(lu, name): """Ensures that a given hostname resolves to a 'sane' name. The given name is required to be a prefix of the resolved hostname, to prevent accidental mismatches. @param lu: the logical unit on behalf of which we're checking @param name: the name we should resolve and check @return: the resolved hostname object """ hostname = netutils.GetHostname(name=name) if hostname.name != name: lu.LogInfo("Resolved given name '%s' to '%s'", name, hostname.name) if not utils.MatchNameComponent(name, [hostname.name]): raise errors.OpPrereqError(("Resolved hostname '%s' does not look the" " same as given hostname '%s'") % (hostname.name, name), errors.ECODE_INVAL) return hostname def CheckOpportunisticLocking(op): """Generate error if opportunistic locking is not possible. """ if op.opportunistic_locking and not op.iallocator: raise errors.OpPrereqError("Opportunistic locking is only available in" " combination with an instance allocator", errors.ECODE_INVAL) def CreateInstanceAllocRequest(op, disks, nics, beparams, node_name_whitelist): """Wrapper around IAReqInstanceAlloc. @param op: The instance opcode @param disks: The computed disks @param nics: The computed nics @param beparams: The full filled beparams @param node_name_whitelist: List of nodes which should appear as online to the allocator (unless the node is already marked offline) @returns: A filled L{iallocator.IAReqInstanceAlloc} """ spindle_use = beparams[constants.BE_SPINDLE_USE] return iallocator.IAReqInstanceAlloc(name=op.instance_name, disk_template=op.disk_template, group_name=op.group_name, tags=op.tags, os=op.os_type, vcpus=beparams[constants.BE_VCPUS], memory=beparams[constants.BE_MAXMEM], spindle_use=spindle_use, disks=disks, nics=[n.ToDict() for n in nics], hypervisor=op.hypervisor, node_whitelist=node_name_whitelist) def ComputeFullBeParams(op, cluster): """Computes the full beparams. @param op: The instance opcode @param cluster: The cluster config object @return: The fully filled beparams """ default_beparams = cluster.beparams[constants.PP_DEFAULT] for param, value in op.beparams.items(): if value == constants.VALUE_AUTO: op.beparams[param] = default_beparams[param] objects.UpgradeBeParams(op.beparams) utils.ForceDictType(op.beparams, constants.BES_PARAMETER_TYPES) return cluster.SimpleFillBE(op.beparams) def ComputeNics(op, cluster, default_ip, cfg, ec_id): """Computes the nics. @param op: The instance opcode @param cluster: Cluster configuration object @param default_ip: The default ip to assign @param cfg: An instance of the configuration object @param ec_id: Execution context ID @returns: The build up nics """ nics = [] for nic in op.nics: nic_mode_req = nic.get(constants.INIC_MODE, None) nic_mode = nic_mode_req if nic_mode is None or nic_mode == constants.VALUE_AUTO: nic_mode = cluster.nicparams[constants.PP_DEFAULT][constants.NIC_MODE] net = nic.get(constants.INIC_NETWORK, None) link = nic.get(constants.NIC_LINK, None) ip = nic.get(constants.INIC_IP, None) vlan = nic.get(constants.INIC_VLAN, None) if net is None or net.lower() == constants.VALUE_NONE: net = None else: if nic_mode_req is not None or link is not None: raise errors.OpPrereqError("If network is given, no mode or link" " is allowed to be passed", errors.ECODE_INVAL) # ip validity checks if ip is None or ip.lower() == constants.VALUE_NONE: nic_ip = None elif ip.lower() == constants.VALUE_AUTO: if not op.name_check: raise errors.OpPrereqError("IP address set to auto but the name checks" " are not enabled", errors.ECODE_INVAL) nic_ip = default_ip else: # We defer pool operations until later, so that the iallocator has # filled in the instance's node(s) dimara if ip.lower() == constants.NIC_IP_POOL: if net is None: raise errors.OpPrereqError("if ip=pool, parameter network" " must be passed too", errors.ECODE_INVAL) elif not netutils.IPAddress.IsValid(ip): raise errors.OpPrereqError("Invalid IP address '%s'" % ip, errors.ECODE_INVAL) nic_ip = ip # TODO: check the ip address for uniqueness if nic_mode == constants.NIC_MODE_ROUTED and not nic_ip and not net: raise errors.OpPrereqError("Routed nic mode requires an ip address" " if not attached to a network", errors.ECODE_INVAL) # MAC address verification mac = nic.get(constants.INIC_MAC, constants.VALUE_AUTO) if mac not in (constants.VALUE_AUTO, constants.VALUE_GENERATE): mac = utils.NormalizeAndValidateMac(mac) try: # TODO: We need to factor this out cfg.ReserveMAC(mac, ec_id) except errors.ReservationError: raise errors.OpPrereqError("MAC address %s already in use" " in cluster" % mac, errors.ECODE_NOTUNIQUE) # Build nic parameters nicparams = {} if nic_mode_req: nicparams[constants.NIC_MODE] = nic_mode if link: nicparams[constants.NIC_LINK] = link if vlan: nicparams[constants.NIC_VLAN] = vlan check_params = cluster.SimpleFillNIC(nicparams) objects.NIC.CheckParameterSyntax(check_params) net_uuid = cfg.LookupNetwork(net) name = nic.get(constants.INIC_NAME, None) if name is not None and name.lower() == constants.VALUE_NONE: name = None nic_obj = objects.NIC(mac=mac, ip=nic_ip, name=name, network=net_uuid, nicparams=nicparams) nic_obj.uuid = cfg.GenerateUniqueID(ec_id) nics.append(nic_obj) return nics ganeti-3.1.0~rc2/lib/cmdlib/misc.py000064400000000000000000000341731476477700300171210ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Miscellaneous logical units that don't fit into any category.""" import logging import time from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import qlang from ganeti import query from ganeti import utils from ganeti.cmdlib.base import NoHooksLU, QueryBase from ganeti.cmdlib.common import GetWantedNodes, SupportsOob class LUOobCommand(NoHooksLU): """Logical unit for OOB handling. """ REQ_BGL = False _SKIP_MASTER = (constants.OOB_POWER_OFF, constants.OOB_POWER_CYCLE) def ExpandNames(self): """Gather locks we need. """ if self.op.node_names: (self.op.node_uuids, self.op.node_names) = \ GetWantedNodes(self, self.op.node_names) lock_node_uuids = self.op.node_uuids else: lock_node_uuids = locking.ALL_SET self.needed_locks = { locking.LEVEL_NODE: lock_node_uuids, } def CheckPrereq(self): """Check prerequisites. This checks: - the node exists in the configuration - OOB is supported Any errors are signaled by raising errors.OpPrereqError. """ self.nodes = [] self.master_node_uuid = self.cfg.GetMasterNode() master_node_obj = self.cfg.GetNodeInfo(self.master_node_uuid) assert self.op.power_delay >= 0.0 if self.op.node_uuids: if (self.op.command in self._SKIP_MASTER and master_node_obj.uuid in self.op.node_uuids): master_oob_handler = SupportsOob(self.cfg, master_node_obj) if master_oob_handler: additional_text = ("run '%s %s %s' if you want to operate on the" " master regardless") % (master_oob_handler, self.op.command, master_node_obj.name) else: additional_text = "it does not support out-of-band operations" raise errors.OpPrereqError(("Operating on the master node %s is not" " allowed for %s; %s") % (master_node_obj.name, self.op.command, additional_text), errors.ECODE_INVAL) else: self.op.node_uuids = self.cfg.GetNodeList() if self.op.command in self._SKIP_MASTER: self.op.node_uuids.remove(master_node_obj.uuid) if self.op.command in self._SKIP_MASTER: assert master_node_obj.uuid not in self.op.node_uuids for node_uuid in self.op.node_uuids: node = self.cfg.GetNodeInfo(node_uuid) if node is None: raise errors.OpPrereqError("Node %s not found" % node_uuid, errors.ECODE_NOENT) self.nodes.append(node) if (not self.op.ignore_status and (self.op.command == constants.OOB_POWER_OFF and not node.offline)): raise errors.OpPrereqError(("Cannot power off node %s because it is" " not marked offline") % node.name, errors.ECODE_STATE) def Exec(self, feedback_fn): """Execute OOB and return result if we expect any. """ ret = [] for idx, node in enumerate(utils.NiceSort(self.nodes, key=lambda node: node.name)): node_entry = [(constants.RS_NORMAL, node.name)] ret.append(node_entry) oob_program = SupportsOob(self.cfg, node) if not oob_program: node_entry.append((constants.RS_UNAVAIL, None)) continue logging.info("Executing out-of-band command '%s' using '%s' on %s", self.op.command, oob_program, node.name) result = self.rpc.call_run_oob(self.master_node_uuid, oob_program, self.op.command, node.name, self.op.timeout) if result.fail_msg: self.LogWarning("Out-of-band RPC failed on node '%s': %s", node.name, result.fail_msg) node_entry.append((constants.RS_NODATA, None)) continue try: self._CheckPayload(result) except errors.OpExecError as err: self.LogWarning("Payload returned by node '%s' is not valid: %s", node.name, err) node_entry.append((constants.RS_NODATA, None)) else: if self.op.command == constants.OOB_HEALTH: # For health we should log important events for item, status in result.payload: if status in [constants.OOB_STATUS_WARNING, constants.OOB_STATUS_CRITICAL]: self.LogWarning("Item '%s' on node '%s' has status '%s'", item, node.name, status) if self.op.command == constants.OOB_POWER_ON: node.powered = True elif self.op.command == constants.OOB_POWER_OFF: node.powered = False elif self.op.command == constants.OOB_POWER_STATUS: powered = result.payload[constants.OOB_POWER_STATUS_POWERED] if powered != node.powered: logging.warning(("Recorded power state (%s) of node '%s' does not" " match actual power state (%s)"), node.powered, node.name, powered) # For configuration changing commands we should update the node if self.op.command in (constants.OOB_POWER_ON, constants.OOB_POWER_OFF): self.cfg.Update(node, feedback_fn) node_entry.append((constants.RS_NORMAL, result.payload)) if (self.op.command == constants.OOB_POWER_ON and idx < len(self.nodes) - 1): time.sleep(self.op.power_delay) return ret def _CheckPayload(self, result): """Checks if the payload is valid. @param result: RPC result @raises errors.OpExecError: If payload is not valid """ errs = [] if self.op.command == constants.OOB_HEALTH: if not isinstance(result.payload, list): errs.append("command 'health' is expected to return a list but got %s" % type(result.payload)) else: for item, status in result.payload: if status not in constants.OOB_STATUSES: errs.append("health item '%s' has invalid status '%s'" % (item, status)) if self.op.command == constants.OOB_POWER_STATUS: if not isinstance(result.payload, dict): errs.append("power-status is expected to return a dict but got %s" % type(result.payload)) if self.op.command in [ constants.OOB_POWER_ON, constants.OOB_POWER_OFF, constants.OOB_POWER_CYCLE, ]: if result.payload is not None: errs.append("%s is expected to not return payload but got '%s'" % (self.op.command, result.payload)) if errs: raise errors.OpExecError("Check of out-of-band payload failed due to %s" % utils.CommaJoin(errs)) class ExtStorageQuery(QueryBase): FIELDS = query.EXTSTORAGE_FIELDS def ExpandNames(self, lu): # Lock all nodes in shared mode # Temporary removal of locks, should be reverted later # TODO: reintroduce locks when they are lighter-weight lu.needed_locks = {} #self.share_locks[locking.LEVEL_NODE] = 1 #self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET # The following variables interact with _QueryBase._GetNames if self.names: self.wanted = [lu.cfg.GetNodeInfoByName(name).uuid for name in self.names] else: self.wanted = locking.ALL_SET self.do_locking = self.use_locking def DeclareLocks(self, lu, level): pass @staticmethod def _DiagnoseByProvider(rlist): """Remaps a per-node return list into an a per-provider per-node dictionary @param rlist: a map with node uuids as keys and ExtStorage objects as values @rtype: dict @return: a dictionary with extstorage providers as keys and as value another map, with node uuids as keys and tuples of (path, status, diagnose, parameters) as values, eg:: {"provider1": {"node_uuid1": [(/usr/lib/..., True, "", [])] "node_uuid2": [(/srv/..., False, "missing file")] "node_uuid3": [(/srv/..., True, "", [])] } """ all_es = {} # we build here the list of nodes that didn't fail the RPC (at RPC # level), so that nodes with a non-responding node daemon don't # make all OSes invalid good_nodes = [node_uuid for node_uuid in rlist if not rlist[node_uuid].fail_msg] for node_uuid, nr in rlist.items(): if nr.fail_msg or not nr.payload: continue for (name, path, status, diagnose, params) in nr.payload: if name not in all_es: # build a list of nodes for this os containing empty lists # for each node in node_list all_es[name] = {} for nuuid in good_nodes: all_es[name][nuuid] = [] # convert params from [name, help] to (name, help) params = [tuple(v) for v in params] all_es[name][node_uuid].append((path, status, diagnose, params)) return all_es def _GetQueryData(self, lu): """Computes the list of nodes and their attributes. """ valid_nodes = [node.uuid for node in lu.cfg.GetAllNodesInfo().values() if not node.offline and node.vm_capable] pol = self._DiagnoseByProvider(lu.rpc.call_extstorage_diagnose(valid_nodes)) data = {} nodegroup_list = lu.cfg.GetNodeGroupList() for (es_name, es_data) in pol.items(): # For every provider compute the nodegroup validity. # To do this we need to check the validity of each node in es_data # and then construct the corresponding nodegroup dict: # { nodegroup1: status # nodegroup2: status # } ndgrp_data = {} for nodegroup in nodegroup_list: ndgrp = lu.cfg.GetNodeGroup(nodegroup) nodegroup_nodes = ndgrp.members nodegroup_name = ndgrp.name node_statuses = [] for node in nodegroup_nodes: if node in valid_nodes: if es_data[node] != []: node_status = es_data[node][0][1] node_statuses.append(node_status) else: node_statuses.append(False) if False in node_statuses: ndgrp_data[nodegroup_name] = False else: ndgrp_data[nodegroup_name] = True # Compute the provider's parameters parameters = set() for idx, esl in enumerate(es_data.values()): valid = bool(esl and esl[0][1]) if not valid: break node_params = esl[0][3] if idx == 0: # First entry parameters.update(node_params) else: # Filter out inconsistent values parameters.intersection_update(node_params) params = list(parameters) # Now fill all the info for this provider info = query.ExtStorageInfo(name=es_name, node_status=es_data, nodegroup_status=ndgrp_data, parameters=params) data[es_name] = info # Prepare data in requested order return [data[name] for name in self._GetNames(lu, list(pol), None) if name in data] class LUExtStorageDiagnose(NoHooksLU): """Logical unit for ExtStorage diagnose/query. """ REQ_BGL = False def CheckArguments(self): self.eq = ExtStorageQuery(qlang.MakeSimpleFilter("name", self.op.names), self.op.output_fields, False) def ExpandNames(self): self.eq.ExpandNames(self) def Exec(self, feedback_fn): return self.eq.OldStyleQuery(self) class LURestrictedCommand(NoHooksLU): """Logical unit for executing restricted commands. """ REQ_BGL = False def ExpandNames(self): if self.op.nodes: (self.op.node_uuids, self.op.nodes) = GetWantedNodes(self, self.op.nodes) self.needed_locks = { locking.LEVEL_NODE: self.op.node_uuids, } self.share_locks = { locking.LEVEL_NODE: not self.op.use_locking, } def CheckPrereq(self): """Check prerequisites. """ def Exec(self, feedback_fn): """Execute restricted command and return output. """ owned_nodes = frozenset(self.owned_locks(locking.LEVEL_NODE)) # Check if correct locks are held assert set(self.op.node_uuids).issubset(owned_nodes) rpcres = self.rpc.call_restricted_command(self.op.node_uuids, self.op.command) result = [] for node_uuid in self.op.node_uuids: nres = rpcres[node_uuid] if nres.fail_msg: msg = ("Command '%s' on node '%s' failed: %s" % (self.op.command, self.cfg.GetNodeName(node_uuid), nres.fail_msg)) result.append((False, msg)) else: result.append((True, nres.payload)) return result ganeti-3.1.0~rc2/lib/cmdlib/network.py000064400000000000000000000524631476477700300176610ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with networks.""" from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import network from ganeti import objects from ganeti import utils from ganeti.cmdlib.base import LogicalUnit from ganeti.cmdlib.common import CheckNodeGroupInstances def _BuildNetworkHookEnv(name, subnet, gateway, network6, gateway6, mac_prefix, tags): """Builds network related env variables for hooks This builds the hook environment from individual variables. @type name: string @param name: the name of the network @type subnet: string @param subnet: the ipv4 subnet @type gateway: string @param gateway: the ipv4 gateway @type network6: string @param network6: the ipv6 subnet @type gateway6: string @param gateway6: the ipv6 gateway @type mac_prefix: string @param mac_prefix: the mac_prefix @type tags: list @param tags: the tags of the network """ env = {} if name: env["NETWORK_NAME"] = name if subnet: env["NETWORK_SUBNET"] = subnet if gateway: env["NETWORK_GATEWAY"] = gateway if network6: env["NETWORK_SUBNET6"] = network6 if gateway6: env["NETWORK_GATEWAY6"] = gateway6 if mac_prefix: env["NETWORK_MAC_PREFIX"] = mac_prefix if tags: env["NETWORK_TAGS"] = " ".join(tags) return env class LUNetworkAdd(LogicalUnit): """Logical unit for creating networks. """ HPATH = "network-add" HTYPE = constants.HTYPE_NETWORK REQ_BGL = False def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() return ([mn], [mn]) def CheckArguments(self): if self.op.mac_prefix: self.op.mac_prefix = \ utils.NormalizeAndValidateThreeOctetMacPrefix(self.op.mac_prefix) def ExpandNames(self): self.network_uuid = self.cfg.GenerateUniqueID(self.proc.GetECId()) if self.op.conflicts_check: self.share_locks[locking.LEVEL_NODE] = 1 self.needed_locks = { locking.LEVEL_NODE: locking.ALL_SET, } else: self.needed_locks = {} self.add_locks[locking.LEVEL_NETWORK] = self.network_uuid def CheckPrereq(self): if self.op.network is None: raise errors.OpPrereqError("Network must be given", errors.ECODE_INVAL) try: existing_uuid = self.cfg.LookupNetwork(self.op.network_name) except errors.OpPrereqError: pass else: raise errors.OpPrereqError("Desired network name '%s' already exists as a" " network (UUID: %s)" % (self.op.network_name, existing_uuid), errors.ECODE_EXISTS) # Check tag validity for tag in self.op.tags: objects.TaggableObject.ValidateTag(tag) def BuildHooksEnv(self): """Build hooks env. """ args = { "name": self.op.network_name, "subnet": self.op.network, "gateway": self.op.gateway, "network6": self.op.network6, "gateway6": self.op.gateway6, "mac_prefix": self.op.mac_prefix, "tags": self.op.tags, } return _BuildNetworkHookEnv(**args) def Exec(self, feedback_fn): """Add the ip pool to the cluster. """ nobj = objects.Network(name=self.op.network_name, network=self.op.network, gateway=self.op.gateway, network6=self.op.network6, gateway6=self.op.gateway6, mac_prefix=self.op.mac_prefix, uuid=self.network_uuid) # Initialize the associated address pool try: pool = network.AddressPool.InitializeNetwork(nobj) except errors.AddressPoolError as err: raise errors.OpExecError("Cannot create IP address pool for network" " '%s': %s" % (self.op.network_name, err)) # Check if we need to reserve the nodes and the cluster master IP # These may not be allocated to any instances in routed mode, as # they wouldn't function anyway. if self.op.conflicts_check: for node in self.cfg.GetAllNodesInfo().values(): for ip in [node.primary_ip, node.secondary_ip]: try: if pool.Contains(ip): pool.Reserve(ip, external=True) self.LogInfo("Reserved IP address of node '%s' (%s)", node.name, ip) except errors.AddressPoolError as err: self.LogWarning("Cannot reserve IP address '%s' of node '%s': %s", ip, node.name, err) master_ip = self.cfg.GetClusterInfo().master_ip try: if pool.Contains(master_ip): pool.Reserve(master_ip, external=True) self.LogInfo("Reserved cluster master IP address (%s)", master_ip) except errors.AddressPoolError as err: self.LogWarning("Cannot reserve cluster master IP address (%s): %s", master_ip, err) if self.op.add_reserved_ips: for ip in self.op.add_reserved_ips: try: pool.Reserve(ip, external=True) except errors.AddressPoolError as err: raise errors.OpExecError("Cannot reserve IP address '%s': %s" % (ip, err)) if self.op.tags: for tag in self.op.tags: nobj.AddTag(tag) self.cfg.AddNetwork(nobj, self.proc.GetECId(), check_uuid=False) class LUNetworkRemove(LogicalUnit): HPATH = "network-remove" HTYPE = constants.HTYPE_NETWORK REQ_BGL = False def ExpandNames(self): self.network_uuid = self.cfg.LookupNetwork(self.op.network_name) self.share_locks[locking.LEVEL_NODEGROUP] = 1 self.needed_locks = { locking.LEVEL_NETWORK: [self.network_uuid], locking.LEVEL_NODEGROUP: locking.ALL_SET, } def CheckPrereq(self): """Check prerequisites. This checks that the given network name exists as a network, that is empty (i.e., contains no nodes), and that is not the last group of the cluster. """ # Verify that the network is not conncted. node_groups = [group.name for group in self.cfg.GetAllNodeGroupsInfo().values() if self.network_uuid in group.networks] if node_groups: self.LogWarning("Network '%s' is connected to the following" " node groups: %s" % (self.op.network_name, utils.CommaJoin(utils.NiceSort(node_groups)))) raise errors.OpPrereqError("Network still connected", errors.ECODE_STATE) def BuildHooksEnv(self): """Build hooks env. """ return { "NETWORK_NAME": self.op.network_name, } def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() return ([mn], [mn]) def Exec(self, feedback_fn): """Remove the network. """ try: self.cfg.RemoveNetwork(self.network_uuid) except errors.ConfigurationError: raise errors.OpExecError("Network '%s' with UUID %s disappeared" % (self.op.network_name, self.network_uuid)) class LUNetworkRename(LogicalUnit): HPATH = "network-rename" HTYPE = constants.HTYPE_NETWORK REQ_BGL = False def ExpandNames(self): self.network_uuid = self.cfg.LookupNetwork(self.op.network_name) self.needed_locks = { locking.LEVEL_NETWORK: [self.network_uuid], } def CheckPrereq(self): """Check prerequisites. Ensures requested new name is not yet used. """ try: new_name_uuid = self.cfg.LookupNetwork(self.op.new_name) except errors.OpPrereqError: pass else: raise errors.OpPrereqError("Desired new name '%s' clashes with existing" " network (UUID: %s)" % (self.op.new_name, new_name_uuid), errors.ECODE_EXISTS) def BuildHooksEnv(self): """Build hooks env. """ return { "OLD_NAME": self.op.network_name, "NEW_NAME": self.op.new_name, } def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() all_nodes = self.cfg.GetAllNodesInfo() all_nodes.pop(mn, None) run_nodes = [mn] for group in self.cfg.GetAllNodeGroupsInfo().values(): if self.network_uuid in group.networks: run_nodes.extend(node.uuid for node in all_nodes.values() if node.group == group.uuid) return (run_nodes, run_nodes) def Exec(self, feedback_fn): """Rename the network. """ network = self.cfg.GetNetwork(self.network_uuid) if network is None: raise errors.OpExecError("Could not retrieve network '%s' (UUID: %s)" % (self.op.network_name, self.network_uuid)) network.name = self.op.new_name self.cfg.Update(network, feedback_fn) return self.op.new_name class LUNetworkSetParams(LogicalUnit): """Modifies the parameters of a network. """ HPATH = "network-modify" HTYPE = constants.HTYPE_NETWORK REQ_BGL = False def CheckArguments(self): if (self.op.gateway and (self.op.add_reserved_ips or self.op.remove_reserved_ips)): raise errors.OpPrereqError("Cannot modify gateway and reserved ips" " at once", errors.ECODE_INVAL) def ExpandNames(self): self.network_uuid = self.cfg.LookupNetwork(self.op.network_name) self.needed_locks = { locking.LEVEL_NETWORK: [self.network_uuid], } def CheckPrereq(self): """Check prerequisites. """ self.network = self.cfg.GetNetwork(self.network_uuid) self.gateway = self.network.gateway self.mac_prefix = self.network.mac_prefix self.network6 = self.network.network6 self.gateway6 = self.network.gateway6 self.tags = self.network.tags self.pool = network.AddressPool(self.network) if self.op.gateway: if self.op.gateway == constants.VALUE_NONE: self.gateway = None else: self.gateway = self.op.gateway if self.pool.IsReserved(self.gateway): raise errors.OpPrereqError("Gateway IP address '%s' is already" " reserved" % self.gateway, errors.ECODE_STATE) if self.op.mac_prefix: if self.op.mac_prefix == constants.VALUE_NONE: self.mac_prefix = None else: self.mac_prefix = \ utils.NormalizeAndValidateThreeOctetMacPrefix(self.op.mac_prefix) if self.op.gateway6: if self.op.gateway6 == constants.VALUE_NONE: self.gateway6 = None else: self.gateway6 = self.op.gateway6 if self.op.network6: if self.op.network6 == constants.VALUE_NONE: self.network6 = None else: self.network6 = self.op.network6 def BuildHooksEnv(self): """Build hooks env. """ args = { "name": self.op.network_name, "subnet": self.network.network, "gateway": self.gateway, "network6": self.network6, "gateway6": self.gateway6, "mac_prefix": self.mac_prefix, "tags": self.tags, } return _BuildNetworkHookEnv(**args) def BuildHooksNodes(self): """Build hooks nodes. """ mn = self.cfg.GetMasterNode() return ([mn], [mn]) def Exec(self, feedback_fn): """Modifies the network. """ #TODO: reserve/release via temporary reservation manager # extend cfg.ReserveIp/ReleaseIp with the external flag if self.op.gateway: if self.gateway == self.network.gateway: self.LogWarning("Gateway is already %s", self.gateway) else: if self.gateway: self.pool.Reserve(self.gateway, external=True) if self.network.gateway: self.pool.Release(self.network.gateway, external=True) self.network.gateway = self.gateway if self.op.add_reserved_ips: for ip in self.op.add_reserved_ips: try: self.pool.Reserve(ip, external=True) except errors.AddressPoolError as err: self.LogWarning("Cannot reserve IP address %s: %s", ip, err) if self.op.remove_reserved_ips: for ip in self.op.remove_reserved_ips: if ip == self.network.gateway: self.LogWarning("Cannot unreserve Gateway's IP") continue try: self.pool.Release(ip, external=True) except errors.AddressPoolError as err: self.LogWarning("Cannot release IP address %s: %s", ip, err) if self.op.mac_prefix: self.network.mac_prefix = self.mac_prefix if self.op.network6: self.network.network6 = self.network6 if self.op.gateway6: self.network.gateway6 = self.gateway6 self.pool.Validate() self.cfg.Update(self.network, feedback_fn) def _FmtNetworkConflict(details): """Utility for L{_NetworkConflictCheck}. """ return utils.CommaJoin("nic%s/%s" % (idx, ipaddr) for (idx, ipaddr) in details) def _NetworkConflictCheck(lu, check_fn, action, instances): """Checks for network interface conflicts with a network. @type lu: L{LogicalUnit} @type check_fn: callable receiving one parameter (L{objects.NIC}) and returning boolean @param check_fn: Function checking for conflict @type action: string @param action: Part of error message (see code) @param instances: the instances to check @type instances: list of instance objects @raise errors.OpPrereqError: If conflicting IP addresses are found. """ conflicts = [] for instance in instances: instconflicts = [(idx, nic.ip) for (idx, nic) in enumerate(instance.nics) if check_fn(nic)] if instconflicts: conflicts.append((instance.name, instconflicts)) if conflicts: lu.LogWarning("IP addresses from network '%s', which is about to %s" " node group '%s', are in use: %s" % (lu.network_name, action, lu.group.name, utils.CommaJoin(("%s: %s" % (name, _FmtNetworkConflict(details))) for (name, details) in conflicts))) raise errors.OpPrereqError("Conflicting IP addresses found; " " remove/modify the corresponding network" " interfaces", errors.ECODE_STATE) class LUNetworkConnect(LogicalUnit): """Connect a network to a nodegroup """ HPATH = "network-connect" HTYPE = constants.HTYPE_NETWORK REQ_BGL = False def ExpandNames(self): self.network_name = self.op.network_name self.group_name = self.op.group_name self.network_mode = self.op.network_mode self.network_link = self.op.network_link self.network_vlan = self.op.network_vlan self.network_uuid = self.cfg.LookupNetwork(self.network_name) self.group_uuid = self.cfg.LookupNodeGroup(self.group_name) self.needed_locks = { locking.LEVEL_INSTANCE: [], locking.LEVEL_NODEGROUP: [self.group_uuid], } self.share_locks[locking.LEVEL_INSTANCE] = 1 if self.op.conflicts_check: self.needed_locks[locking.LEVEL_NETWORK] = [self.network_uuid] self.share_locks[locking.LEVEL_NETWORK] = 1 def DeclareLocks(self, level): if level == locking.LEVEL_INSTANCE: assert not self.needed_locks[locking.LEVEL_INSTANCE] # Lock instances optimistically, needs verification once group lock has # been acquired if self.op.conflicts_check: self.needed_locks[locking.LEVEL_INSTANCE] = \ self.cfg.GetInstanceNames( self.cfg.GetNodeGroupInstances(self.group_uuid)) def BuildHooksEnv(self): ret = { "GROUP_NAME": self.group_name, "GROUP_NETWORK_MODE": self.network_mode, "GROUP_NETWORK_LINK": self.network_link, "GROUP_NETWORK_VLAN": self.network_vlan, } return ret def BuildHooksNodes(self): node_uuids = self.cfg.GetNodeGroup(self.group_uuid).members return (node_uuids, node_uuids) def CheckPrereq(self): owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP)) assert self.group_uuid in owned_groups # Check if locked instances are still correct owned_instance_names = frozenset(self.owned_locks(locking.LEVEL_INSTANCE)) if self.op.conflicts_check: CheckNodeGroupInstances(self.cfg, self.group_uuid, owned_instance_names) self.netparams = { constants.NIC_MODE: self.network_mode, constants.NIC_LINK: self.network_link, constants.NIC_VLAN: self.network_vlan, } objects.NIC.CheckParameterSyntax(self.netparams) self.group = self.cfg.GetNodeGroup(self.group_uuid) #if self.network_mode == constants.NIC_MODE_BRIDGED: # _CheckNodeGroupBridgesExist(self, self.network_link, self.group_uuid) self.connected = False if self.network_uuid in self.group.networks: self.LogWarning("Network '%s' is already mapped to group '%s'" % (self.network_name, self.group.name)) self.connected = True # check only if not already connected elif self.op.conflicts_check: pool = network.AddressPool(self.cfg.GetNetwork(self.network_uuid)) _NetworkConflictCheck( self, lambda nic: pool.Contains(nic.ip), "connect to", [instance_info for (_, instance_info) in self.cfg.GetMultiInstanceInfoByName(owned_instance_names)]) def Exec(self, feedback_fn): # Connect the network and update the group only if not already connected if not self.connected: self.group.networks[self.network_uuid] = self.netparams self.cfg.Update(self.group, feedback_fn) class LUNetworkDisconnect(LogicalUnit): """Disconnect a network to a nodegroup """ HPATH = "network-disconnect" HTYPE = constants.HTYPE_NETWORK REQ_BGL = False def ExpandNames(self): self.network_name = self.op.network_name self.group_name = self.op.group_name self.network_uuid = self.cfg.LookupNetwork(self.network_name) self.group_uuid = self.cfg.LookupNodeGroup(self.group_name) self.needed_locks = { locking.LEVEL_INSTANCE: [], locking.LEVEL_NODEGROUP: [self.group_uuid], } self.share_locks[locking.LEVEL_INSTANCE] = 1 def DeclareLocks(self, level): if level == locking.LEVEL_INSTANCE: assert not self.needed_locks[locking.LEVEL_INSTANCE] # Lock instances optimistically, needs verification once group lock has # been acquired self.needed_locks[locking.LEVEL_INSTANCE] = \ self.cfg.GetInstanceNames( self.cfg.GetNodeGroupInstances(self.group_uuid)) def BuildHooksEnv(self): ret = { "GROUP_NAME": self.group_name, } if self.connected: ret.update({ "GROUP_NETWORK_MODE": self.netparams[constants.NIC_MODE], "GROUP_NETWORK_LINK": self.netparams[constants.NIC_LINK], "GROUP_NETWORK_VLAN": self.netparams[constants.NIC_VLAN], }) return ret def BuildHooksNodes(self): nodes = self.cfg.GetNodeGroup(self.group_uuid).members return (nodes, nodes) def CheckPrereq(self): owned_groups = frozenset(self.owned_locks(locking.LEVEL_NODEGROUP)) assert self.group_uuid in owned_groups # Check if locked instances are still correct owned_instances = frozenset(self.owned_locks(locking.LEVEL_INSTANCE)) CheckNodeGroupInstances(self.cfg, self.group_uuid, owned_instances) self.group = self.cfg.GetNodeGroup(self.group_uuid) self.connected = True if self.network_uuid not in self.group.networks: self.LogWarning("Network '%s' is not mapped to group '%s'", self.network_name, self.group.name) self.connected = False # We need this check only if network is not already connected else: _NetworkConflictCheck( self, lambda nic: nic.network == self.network_uuid, "disconnect from", [instance_info for (_, instance_info) in self.cfg.GetMultiInstanceInfoByName(owned_instances)]) self.netparams = self.group.networks.get(self.network_uuid) def Exec(self, feedback_fn): # Disconnect the network and update the group only if network is connected if self.connected: del self.group.networks[self.network_uuid] self.cfg.Update(self.group, feedback_fn) ganeti-3.1.0~rc2/lib/cmdlib/node.py000064400000000000000000001763761476477700300171270ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with nodes.""" import logging import operator from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import netutils from ganeti import objects from ganeti import opcodes import ganeti.rpc.node as rpc from ganeti import utils from ganeti.masterd import iallocator from ganeti.cmdlib.base import LogicalUnit, NoHooksLU, ResultWithJobs from ganeti.cmdlib.common import CheckParamsNotGlobal, \ MergeAndVerifyHvState, MergeAndVerifyDiskState, \ IsExclusiveStorageEnabledNode, CheckNodePVs, \ RedistributeAncillaryFiles, ExpandNodeUuidAndName, ShareAll, SupportsOob, \ CheckInstanceState, INSTANCE_DOWN, GetUpdatedParams, \ AdjustCandidatePool, CheckIAllocatorOrNode, LoadNodeEvacResult, \ GetWantedNodes, MapInstanceLvsToNodes, RunPostHook, \ FindFaultyInstanceDisks, CheckStorageTypeEnabled, GetClientCertDigest, \ AddNodeCertToCandidateCerts, RemoveNodeCertFromCandidateCerts, \ EnsureKvmdOnNodes, WarnAboutFailedSshUpdates, AddMasterCandidateSshKey def _DecideSelfPromotion(lu, exceptions=None): """Decide whether I should promote myself as a master candidate. """ cp_size = lu.cfg.GetClusterInfo().candidate_pool_size mc_now, mc_should, _ = lu.cfg.GetMasterCandidateStats(exceptions) # the new node will increase mc_max with one, so: mc_should = min(mc_should + 1, cp_size) return mc_now < mc_should def _CheckNodeHasSecondaryIP(lu, node, secondary_ip, prereq): """Ensure that a node has the given secondary ip. @type lu: L{LogicalUnit} @param lu: the LU on behalf of which we make the check @type node: L{objects.Node} @param node: the node to check @type secondary_ip: string @param secondary_ip: the ip to check @type prereq: boolean @param prereq: whether to throw a prerequisite or an execute error @raise errors.OpPrereqError: if the node doesn't have the ip, and prereq=True @raise errors.OpExecError: if the node doesn't have the ip, and prereq=False """ # this can be called with a new node, which has no UUID yet, so perform the # RPC call using its name result = lu.rpc.call_node_has_ip_address(node.name, secondary_ip) result.Raise("Failure checking secondary ip on node %s" % node.name, prereq=prereq, ecode=errors.ECODE_ENVIRON) if not result.payload: msg = ("Node claims it doesn't have the secondary ip you gave (%s)," " please fix and re-run this command" % secondary_ip) if prereq: raise errors.OpPrereqError(msg, errors.ECODE_ENVIRON) else: raise errors.OpExecError(msg) class LUNodeAdd(LogicalUnit): """Logical unit for adding node to the cluster. """ HPATH = "node-add" HTYPE = constants.HTYPE_NODE _NFLAGS = ["master_capable", "vm_capable"] def CheckArguments(self): self.primary_ip_family = self.cfg.GetPrimaryIPFamily() # validate/normalize the node name self.hostname = netutils.GetHostname(name=self.op.node_name, family=self.primary_ip_family) self.op.node_name = self.hostname.name if self.op.readd and self.op.node_name == self.cfg.GetMasterNodeName(): raise errors.OpPrereqError("Cannot readd the master node", errors.ECODE_STATE) if self.op.readd and self.op.group: raise errors.OpPrereqError("Cannot pass a node group when a node is" " being readded", errors.ECODE_INVAL) def BuildHooksEnv(self): """Build hooks env. This will run on all nodes before, and on all nodes + the new node after. """ return { "OP_TARGET": self.op.node_name, "NODE_NAME": self.op.node_name, "NODE_PIP": self.op.primary_ip, "NODE_SIP": self.op.secondary_ip, "MASTER_CAPABLE": str(self.op.master_capable), "VM_CAPABLE": str(self.op.vm_capable), } def BuildHooksNodes(self): """Build hooks nodes. """ hook_nodes = self.cfg.GetNodeList() new_node_info = self.cfg.GetNodeInfoByName(self.op.node_name) if new_node_info is not None: # Exclude added node hook_nodes = list(set(hook_nodes) - set([new_node_info.uuid])) # add the new node as post hook node by name; it does not have an UUID yet return (hook_nodes, hook_nodes) def PreparePostHookNodes(self, post_hook_node_uuids): return post_hook_node_uuids + [self.new_node.uuid] def CheckPrereq(self): """Check prerequisites. This checks: - the new node is not already in the config - it is resolvable - its parameters (single/dual homed) matches the cluster Any errors are signaled by raising errors.OpPrereqError. """ node_name = self.hostname.name self.op.primary_ip = self.hostname.ip if self.op.secondary_ip is None: if self.primary_ip_family == netutils.IP6Address.family: raise errors.OpPrereqError("When using a IPv6 primary address, a valid" " IPv4 address must be given as secondary", errors.ECODE_INVAL) self.op.secondary_ip = self.op.primary_ip secondary_ip = self.op.secondary_ip if not netutils.IP4Address.IsValid(secondary_ip): raise errors.OpPrereqError("Secondary IP (%s) needs to be a valid IPv4" " address" % secondary_ip, errors.ECODE_INVAL) existing_node_info = self.cfg.GetNodeInfoByName(node_name) if not self.op.readd and existing_node_info is not None: raise errors.OpPrereqError("Node %s is already in the configuration" % node_name, errors.ECODE_EXISTS) elif self.op.readd and existing_node_info is None: raise errors.OpPrereqError("Node %s is not in the configuration" % node_name, errors.ECODE_NOENT) self.changed_primary_ip = False for existing_node in self.cfg.GetAllNodesInfo().values(): if self.op.readd and node_name == existing_node.name: if existing_node.secondary_ip != secondary_ip: raise errors.OpPrereqError("Readded node doesn't have the same IP" " address configuration as before", errors.ECODE_INVAL) if existing_node.primary_ip != self.op.primary_ip: self.changed_primary_ip = True continue if (existing_node.primary_ip == self.op.primary_ip or existing_node.secondary_ip == self.op.primary_ip or existing_node.primary_ip == secondary_ip or existing_node.secondary_ip == secondary_ip): raise errors.OpPrereqError("New node ip address(es) conflict with" " existing node %s" % existing_node.name, errors.ECODE_NOTUNIQUE) # After this 'if' block, None is no longer a valid value for the # _capable op attributes if self.op.readd: assert existing_node_info is not None, \ "Can't retrieve locked node %s" % node_name for attr in self._NFLAGS: if getattr(self.op, attr) is None: setattr(self.op, attr, getattr(existing_node_info, attr)) else: for attr in self._NFLAGS: if getattr(self.op, attr) is None: setattr(self.op, attr, True) if self.op.readd and not self.op.vm_capable: pri, sec = self.cfg.GetNodeInstances(existing_node_info.uuid) if pri or sec: raise errors.OpPrereqError("Node %s being re-added with vm_capable" " flag set to false, but it already holds" " instances" % node_name, errors.ECODE_STATE) # check that the type of the node (single versus dual homed) is the # same as for the master myself = self.cfg.GetMasterNodeInfo() master_singlehomed = myself.secondary_ip == myself.primary_ip newbie_singlehomed = secondary_ip == self.op.primary_ip if master_singlehomed != newbie_singlehomed: if master_singlehomed: raise errors.OpPrereqError("The master has no secondary ip but the" " new node has one", errors.ECODE_INVAL) else: raise errors.OpPrereqError("The master has a secondary ip but the" " new node doesn't have one", errors.ECODE_INVAL) # checks reachability if not netutils.TcpPing(self.op.primary_ip, constants.DEFAULT_NODED_PORT): raise errors.OpPrereqError("Node not reachable by ping", errors.ECODE_ENVIRON) if not newbie_singlehomed: # check reachability from my secondary ip to newbie's secondary ip if not netutils.TcpPing(secondary_ip, constants.DEFAULT_NODED_PORT, source=myself.secondary_ip): raise errors.OpPrereqError("Node secondary ip not reachable by TCP" " based ping to node daemon port", errors.ECODE_ENVIRON) if self.op.readd: exceptions = [existing_node_info.uuid] else: exceptions = [] if self.op.master_capable: self.master_candidate = _DecideSelfPromotion(self, exceptions=exceptions) else: self.master_candidate = False self.node_group = None if self.op.readd: self.new_node = existing_node_info self.node_group = existing_node_info.group else: self.node_group = self.cfg.LookupNodeGroup(self.op.group) self.new_node = objects.Node(name=node_name, primary_ip=self.op.primary_ip, secondary_ip=secondary_ip, master_candidate=self.master_candidate, offline=False, drained=False, group=self.node_group, ndparams={}) if self.op.ndparams: utils.ForceDictType(self.op.ndparams, constants.NDS_PARAMETER_TYPES) CheckParamsNotGlobal(self.op.ndparams, constants.NDC_GLOBALS, "node", "node", "cluster or group") if self.op.hv_state: self.new_hv_state = MergeAndVerifyHvState(self.op.hv_state, None) if self.op.disk_state: self.new_disk_state = MergeAndVerifyDiskState(self.op.disk_state, None) # TODO: If we need to have multiple DnsOnlyRunner we probably should make # it a property on the base class. rpcrunner = rpc.DnsOnlyRunner() result = rpcrunner.call_version([node_name])[node_name] result.Raise("Can't get version information from node %s" % node_name, prereq=True, ecode=errors.ECODE_ENVIRON) if constants.PROTOCOL_VERSION == result.payload: logging.info("Communication to node %s fine, sw version %s match", node_name, result.payload) else: raise errors.OpPrereqError("Version mismatch master version %s," " node version %s" % (constants.PROTOCOL_VERSION, result.payload), errors.ECODE_ENVIRON) vg_name = self.cfg.GetVGName() if vg_name is not None: vparams = {constants.NV_PVLIST: [vg_name]} excl_stor = IsExclusiveStorageEnabledNode(self.cfg, self.new_node) cname = self.cfg.GetClusterName() result = rpcrunner.call_node_verify_light( [node_name], vparams, cname, self.cfg.GetClusterInfo().hvparams, )[node_name] (errmsgs, _) = CheckNodePVs(result.payload, excl_stor) if errmsgs: raise errors.OpPrereqError("Checks on node PVs failed: %s" % "; ".join(errmsgs), errors.ECODE_ENVIRON) def _InitOpenVSwitch(self): filled_ndparams = self.cfg.GetClusterInfo().FillND( self.new_node, self.cfg.GetNodeGroup(self.new_node.group)) ovs = filled_ndparams.get(constants.ND_OVS, None) ovs_name = filled_ndparams.get(constants.ND_OVS_NAME, None) ovs_link = filled_ndparams.get(constants.ND_OVS_LINK, None) if ovs: if not ovs_link: self.LogInfo("No physical interface for OpenvSwitch was given." " OpenvSwitch will not have an outside connection. This" " might not be what you want.") result = self.rpc.call_node_configure_ovs( self.new_node.name, ovs_name, ovs_link) result.Raise("Failed to initialize OpenVSwitch on new node") def _SshUpdate(self, new_node_uuid, new_node_name, is_master_candidate, is_potential_master_candidate, rpcrunner, readd, feedback_fn): """Update the SSH setup of all nodes after adding a new node. @type readd: boolean @param readd: whether or not this node is readded """ potential_master_candidates = self.cfg.GetPotentialMasterCandidates() master_node = self.cfg.GetMasterNode() if readd: # clear previous keys master_candidate_uuids = self.cfg.GetMasterCandidateUuids() remove_result = rpcrunner.call_node_ssh_key_remove( [master_node], new_node_uuid, new_node_name, master_candidate_uuids, potential_master_candidates, True, # from authorized keys True, # from public keys False, # clear authorized keys True, # clear public keys True) # it's a readd remove_result[master_node].Raise( "Could not remove SSH keys of node %s before readding," " (UUID: %s)." % (new_node_name, new_node_uuid)) WarnAboutFailedSshUpdates(remove_result, master_node, feedback_fn) result = rpcrunner.call_node_ssh_key_add( [master_node], new_node_uuid, new_node_name, potential_master_candidates, is_master_candidate, is_potential_master_candidate, is_potential_master_candidate) result[master_node].Raise("Could not update the node's SSH setup.") WarnAboutFailedSshUpdates(result, master_node, feedback_fn) def Exec(self, feedback_fn): """Adds the new node to the cluster. """ assert locking.BGL in self.owned_locks(locking.LEVEL_CLUSTER), \ "Not owning BGL" # We adding a new node so we assume it's powered self.new_node.powered = True # for re-adds, reset the offline/drained/master-candidate flags; # we need to reset here, otherwise offline would prevent RPC calls # later in the procedure; this also means that if the re-add # fails, we are left with a non-offlined, broken node if self.op.readd: self.new_node.offline = False self.new_node.drained = False self.LogInfo("Readding a node, the offline/drained flags were reset") # if we demote the node, we do cleanup later in the procedure self.new_node.master_candidate = self.master_candidate if self.changed_primary_ip: self.new_node.primary_ip = self.op.primary_ip # copy the master/vm_capable flags for attr in self._NFLAGS: setattr(self.new_node, attr, getattr(self.op, attr)) # notify the user about any possible mc promotion if self.new_node.master_candidate: self.LogInfo("Node will be a master candidate") if self.op.ndparams: self.new_node.ndparams = self.op.ndparams else: self.new_node.ndparams = {} if self.op.hv_state: self.new_node.hv_state_static = self.new_hv_state if self.op.disk_state: self.new_node.disk_state_static = self.new_disk_state # Add node to our /etc/hosts, and add key to known_hosts if self.cfg.GetClusterInfo().modify_etc_hosts: master_node = self.cfg.GetMasterNode() result = self.rpc.call_etc_hosts_modify( master_node, constants.ETC_HOSTS_ADD, self.hostname.name, self.hostname.ip) result.Raise("Can't update hosts file with new host data") if self.new_node.secondary_ip != self.new_node.primary_ip: _CheckNodeHasSecondaryIP(self, self.new_node, self.new_node.secondary_ip, False) node_verifier_uuids = [self.cfg.GetMasterNode()] node_verify_param = { constants.NV_NODELIST: ([self.new_node.name], {}, []), # TODO: do a node-net-test as well? } result = self.rpc.call_node_verify( node_verifier_uuids, node_verify_param, self.cfg.GetClusterName(), self.cfg.GetClusterInfo().hvparams) for verifier in node_verifier_uuids: result[verifier].Raise("Cannot communicate with node %s" % verifier) nl_payload = result[verifier].payload[constants.NV_NODELIST] if nl_payload: for failed in nl_payload: feedback_fn("ssh/hostname verification failed" " (checking from %s): %s" % (verifier, nl_payload[failed])) raise errors.OpExecError("ssh/hostname verification failed") self._InitOpenVSwitch() if self.op.readd: RedistributeAncillaryFiles(self) # make sure we redistribute the config self.cfg.Update(self.new_node, feedback_fn) # and make sure the new node will not have old files around if not self.new_node.master_candidate: result = self.rpc.call_node_demote_from_mc(self.new_node.uuid) result.Warn("Node failed to demote itself from master candidate status", self.LogWarning) else: self.cfg.AddNode(self.new_node, self.proc.GetECId()) RedistributeAncillaryFiles(self) # We create a new certificate even if the node is readded digest = GetClientCertDigest(self, self.new_node.uuid) if self.new_node.master_candidate: self.cfg.AddNodeToCandidateCerts(self.new_node.uuid, digest) else: self.cfg.RemoveNodeFromCandidateCerts(self.new_node.uuid, warn_fn=None) # Ensure, that kvmd is in the expected state on the added node. EnsureKvmdOnNodes(self, feedback_fn, nodes=[self.new_node.uuid], silent_stop=True) # Update SSH setup of all nodes if self.op.node_setup: # FIXME: so far, all nodes are considered potential master candidates self._SshUpdate(self.new_node.uuid, self.new_node.name, self.new_node.master_candidate, True, self.rpc, self.op.readd, feedback_fn) class LUNodeSetParams(LogicalUnit): """Modifies the parameters of a node. @cvar _F2R: a dictionary from tuples of flags (mc, drained, offline) to the node role (as _ROLE_*) @cvar _R2F: a dictionary from node role to tuples of flags @cvar _FLAGS: a list of attribute names corresponding to the flags """ HPATH = "node-modify" HTYPE = constants.HTYPE_NODE REQ_BGL = False (_ROLE_CANDIDATE, _ROLE_DRAINED, _ROLE_OFFLINE, _ROLE_REGULAR) = range(4) _F2R = { (True, False, False): _ROLE_CANDIDATE, (False, True, False): _ROLE_DRAINED, (False, False, True): _ROLE_OFFLINE, (False, False, False): _ROLE_REGULAR, } _R2F = dict((v, k) for k, v in _F2R.items()) _FLAGS = ["master_candidate", "drained", "offline"] def CheckArguments(self): (self.op.node_uuid, self.op.node_name) = \ ExpandNodeUuidAndName(self.cfg, self.op.node_uuid, self.op.node_name) all_mods = [self.op.offline, self.op.master_candidate, self.op.drained, self.op.master_capable, self.op.vm_capable, self.op.secondary_ip, self.op.ndparams, self.op.hv_state, self.op.disk_state] if all_mods.count(None) == len(all_mods): raise errors.OpPrereqError("Please pass at least one modification", errors.ECODE_INVAL) if all_mods.count(True) > 1: raise errors.OpPrereqError("Can't set the node into more than one" " state at the same time", errors.ECODE_INVAL) # Boolean value that tells us whether we might be demoting from MC self.might_demote = (self.op.master_candidate is False or self.op.offline is True or self.op.drained is True or self.op.master_capable is False) if self.op.secondary_ip: if not netutils.IP4Address.IsValid(self.op.secondary_ip): raise errors.OpPrereqError("Secondary IP (%s) needs to be a valid IPv4" " address" % self.op.secondary_ip, errors.ECODE_INVAL) self.lock_all = self.op.auto_promote and self.might_demote self.lock_instances = self.op.secondary_ip is not None def _InstanceFilter(self, instance): """Filter for getting affected instances. """ disks = self.cfg.GetInstanceDisks(instance.uuid) any_mirrored = utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR) return (any_mirrored and self.op.node_uuid in self.cfg.GetInstanceNodes(instance.uuid)) def ExpandNames(self): if self.lock_all: self.needed_locks = { locking.LEVEL_NODE: locking.ALL_SET, } else: self.needed_locks = { locking.LEVEL_NODE: self.op.node_uuid, } # Since modifying a node can have severe effects on currently running # operations the resource lock is at least acquired in shared mode self.needed_locks[locking.LEVEL_NODE_RES] = \ self.needed_locks[locking.LEVEL_NODE] # Get all locks except nodes in shared mode; they are not used for anything # but read-only access self.share_locks = ShareAll() self.share_locks[locking.LEVEL_NODE] = 0 self.share_locks[locking.LEVEL_NODE_RES] = 0 if self.lock_instances: self.needed_locks[locking.LEVEL_INSTANCE] = \ self.cfg.GetInstanceNames( list(self.cfg.GetInstancesInfoByFilter(self._InstanceFilter))) def BuildHooksEnv(self): """Build hooks env. This runs on the master node. """ return { "OP_TARGET": self.op.node_name, "MASTER_CANDIDATE": str(self.op.master_candidate), "OFFLINE": str(self.op.offline), "DRAINED": str(self.op.drained), "MASTER_CAPABLE": str(self.op.master_capable), "VM_CAPABLE": str(self.op.vm_capable), } def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode(), self.op.node_uuid] return (nl, nl) def CheckPrereq(self): """Check prerequisites. This only checks the instance list against the existing names. """ node = self.cfg.GetNodeInfo(self.op.node_uuid) if self.lock_instances: affected_instances = \ self.cfg.GetInstancesInfoByFilter(self._InstanceFilter) # Verify instance locks owned_instance_names = self.owned_locks(locking.LEVEL_INSTANCE) wanted_instance_names = frozenset([inst.name for inst in affected_instances.values()]) if wanted_instance_names - owned_instance_names: raise errors.OpPrereqError("Instances affected by changing node %s's" " secondary IP address have changed since" " locks were acquired, wanted '%s', have" " '%s'; retry the operation" % (node.name, utils.CommaJoin(wanted_instance_names), utils.CommaJoin(owned_instance_names)), errors.ECODE_STATE) else: affected_instances = None if (self.op.master_candidate is not None or self.op.drained is not None or self.op.offline is not None): # we can't change the master's node flags if node.uuid == self.cfg.GetMasterNode(): raise errors.OpPrereqError("The master role can be changed" " only via master-failover", errors.ECODE_INVAL) if self.op.master_candidate and not node.master_capable: raise errors.OpPrereqError("Node %s is not master capable, cannot make" " it a master candidate" % node.name, errors.ECODE_STATE) if self.op.vm_capable is False: (ipri, isec) = self.cfg.GetNodeInstances(node.uuid) if ipri or isec: raise errors.OpPrereqError("Node %s hosts instances, cannot unset" " the vm_capable flag" % node.name, errors.ECODE_STATE) if node.master_candidate and self.might_demote and not self.lock_all: assert not self.op.auto_promote, "auto_promote set but lock_all not" # check if after removing the current node, we're missing master # candidates (mc_remaining, mc_should, _) = \ self.cfg.GetMasterCandidateStats(exceptions=[node.uuid]) if mc_remaining < mc_should: raise errors.OpPrereqError("Not enough master candidates, please" " pass auto promote option to allow" " promotion (--auto-promote or RAPI" " auto_promote=True)", errors.ECODE_STATE) self.old_flags = old_flags = (node.master_candidate, node.drained, node.offline) assert old_flags in self._F2R, "Un-handled old flags %s" % str(old_flags) self.old_role = old_role = self._F2R[old_flags] # Check for ineffective changes for attr in self._FLAGS: if getattr(self.op, attr) is False and getattr(node, attr) is False: self.LogInfo("Ignoring request to unset flag %s, already unset", attr) setattr(self.op, attr, None) # Past this point, any flag change to False means a transition # away from the respective state, as only real changes are kept # TODO: We might query the real power state if it supports OOB if SupportsOob(self.cfg, node): if self.op.offline is False and not (node.powered or self.op.powered is True): raise errors.OpPrereqError(("Node %s needs to be turned on before its" " offline status can be reset") % self.op.node_name, errors.ECODE_STATE) elif self.op.powered is not None: raise errors.OpPrereqError(("Unable to change powered state for node %s" " as it does not support out-of-band" " handling") % self.op.node_name, errors.ECODE_STATE) # If we're being deofflined/drained, we'll MC ourself if needed if (self.op.drained is False or self.op.offline is False or (self.op.master_capable and not node.master_capable)): if _DecideSelfPromotion(self): self.op.master_candidate = True self.LogInfo("Auto-promoting node to master candidate") # If we're no longer master capable, we'll demote ourselves from MC if self.op.master_capable is False and node.master_candidate: if self.op.node_uuid == self.cfg.GetMasterNode(): raise errors.OpPrereqError("Master must remain master capable", errors.ECODE_STATE) self.LogInfo("Demoting from master candidate") self.op.master_candidate = False # Compute new role assert [getattr(self.op, attr) for attr in self._FLAGS].count(True) <= 1 if self.op.master_candidate: new_role = self._ROLE_CANDIDATE elif self.op.drained: new_role = self._ROLE_DRAINED elif self.op.offline: new_role = self._ROLE_OFFLINE elif False in [self.op.master_candidate, self.op.drained, self.op.offline]: # False is still in new flags, which means we're un-setting (the # only) True flag new_role = self._ROLE_REGULAR else: # no new flags, nothing, keep old role new_role = old_role self.new_role = new_role if old_role == self._ROLE_OFFLINE and new_role != old_role: # Trying to transition out of offline status result = self.rpc.call_version([node.uuid])[node.uuid] if result.fail_msg: raise errors.OpPrereqError("Node %s is being de-offlined but fails" " to report its version: %s" % (node.name, result.fail_msg), errors.ECODE_STATE) else: self.LogWarning("Transitioning node from offline to online state" " without using re-add. Please make sure the node" " is healthy!") # When changing the secondary ip, verify if this is a single-homed to # multi-homed transition or vice versa, and apply the relevant # restrictions. if self.op.secondary_ip: # Ok even without locking, because this can't be changed by any LU master = self.cfg.GetMasterNodeInfo() master_singlehomed = master.secondary_ip == master.primary_ip if master_singlehomed and self.op.secondary_ip != node.primary_ip: if self.op.force and node.uuid == master.uuid: self.LogWarning("Transitioning from single-homed to multi-homed" " cluster; all nodes will require a secondary IP" " address") else: raise errors.OpPrereqError("Changing the secondary ip on a" " single-homed cluster requires the" " --force option to be passed, and the" " target node to be the master", errors.ECODE_INVAL) elif not master_singlehomed and self.op.secondary_ip == node.primary_ip: if self.op.force and node.uuid == master.uuid: self.LogWarning("Transitioning from multi-homed to single-homed" " cluster; secondary IP addresses will have to be" " removed") else: raise errors.OpPrereqError("Cannot set the secondary IP to be the" " same as the primary IP on a multi-homed" " cluster, unless the --force option is" " passed, and the target node is the" " master", errors.ECODE_INVAL) assert not (set([inst.name for inst in affected_instances.values()]) - self.owned_locks(locking.LEVEL_INSTANCE)) if node.offline: if affected_instances: msg = ("Cannot change secondary IP address: offline node has" " instances (%s) configured to use it" % utils.CommaJoin( [inst.name for inst in affected_instances.values()])) raise errors.OpPrereqError(msg, errors.ECODE_STATE) else: # On online nodes, check that no instances are running, and that # the node has the new ip and we can reach it. for instance in affected_instances.values(): CheckInstanceState(self, instance, INSTANCE_DOWN, msg="cannot change secondary ip") _CheckNodeHasSecondaryIP(self, node, self.op.secondary_ip, True) if master.uuid != node.uuid: # check reachability from master secondary ip to new secondary ip if not netutils.TcpPing(self.op.secondary_ip, constants.DEFAULT_NODED_PORT, source=master.secondary_ip): raise errors.OpPrereqError("Node secondary ip not reachable by TCP" " based ping to node daemon port", errors.ECODE_ENVIRON) if self.op.ndparams: new_ndparams = GetUpdatedParams(node.ndparams, self.op.ndparams) utils.ForceDictType(new_ndparams, constants.NDS_PARAMETER_TYPES) CheckParamsNotGlobal(self.op.ndparams, constants.NDC_GLOBALS, "node", "node", "cluster or group") self.new_ndparams = new_ndparams if self.op.hv_state: self.new_hv_state = MergeAndVerifyHvState(self.op.hv_state, node.hv_state_static) if self.op.disk_state: self.new_disk_state = \ MergeAndVerifyDiskState(self.op.disk_state, node.disk_state_static) def Exec(self, feedback_fn): """Modifies a node. """ node = self.cfg.GetNodeInfo(self.op.node_uuid) result = [] if self.op.ndparams: node.ndparams = self.new_ndparams if self.op.powered is not None: node.powered = self.op.powered if self.op.hv_state: node.hv_state_static = self.new_hv_state if self.op.disk_state: node.disk_state_static = self.new_disk_state for attr in ["master_capable", "vm_capable"]: val = getattr(self.op, attr) if val is not None: setattr(node, attr, val) result.append((attr, str(val))) if self.op.secondary_ip: node.secondary_ip = self.op.secondary_ip result.append(("secondary_ip", self.op.secondary_ip)) # this will trigger configuration file update, if needed self.cfg.Update(node, feedback_fn) master_node = self.cfg.GetMasterNode() potential_master_candidates = self.cfg.GetPotentialMasterCandidates() modify_ssh_setup = self.cfg.GetClusterInfo().modify_ssh_setup if self.new_role != self.old_role: new_flags = self._R2F[self.new_role] for of, nf, desc in zip(self.old_flags, new_flags, self._FLAGS): if of != nf: result.append((desc, str(nf))) (node.master_candidate, node.drained, node.offline) = new_flags self.cfg.Update(node, feedback_fn) # Tell the node to demote itself, if no longer MC and not offline. # This must be done only after the configuration is updated so that # it's ensured the node won't receive any further configuration updates. if self.old_role == self._ROLE_CANDIDATE and \ self.new_role != self._ROLE_OFFLINE: msg = self.rpc.call_node_demote_from_mc(node.name).fail_msg if msg: self.LogWarning("Node failed to demote itself: %s", msg) # we locked all nodes, we adjust the CP before updating this node if self.lock_all: AdjustCandidatePool( self, [node.uuid], master_node, potential_master_candidates, feedback_fn, modify_ssh_setup) # if node gets promoted, grant RPC priviledges if self.new_role == self._ROLE_CANDIDATE: AddNodeCertToCandidateCerts(self, self.cfg, node.uuid) # if node is demoted, revoke RPC priviledges if self.old_role == self._ROLE_CANDIDATE: RemoveNodeCertFromCandidateCerts(self.cfg, node.uuid) # KVM configuration never changes here, so disable warnings if KVM disabled. silent_stop = constants.HT_KVM not in \ self.cfg.GetClusterInfo().enabled_hypervisors EnsureKvmdOnNodes(self, feedback_fn, nodes=[node.uuid], silent_stop=silent_stop) # this will trigger job queue propagation or cleanup if the mc # flag changed if [self.old_role, self.new_role].count(self._ROLE_CANDIDATE) == 1: if modify_ssh_setup: if self.old_role == self._ROLE_CANDIDATE: master_candidate_uuids = self.cfg.GetMasterCandidateUuids() ssh_result = self.rpc.call_node_ssh_key_remove( [master_node], node.uuid, node.name, master_candidate_uuids, potential_master_candidates, True, # remove node's key from all nodes' authorized_keys file False, # currently, all nodes are potential master candidates False, # do not clear node's 'authorized_keys' False, # do not clear node's 'ganeti_pub_keys' False) # no readd ssh_result[master_node].Raise( "Could not adjust the SSH setup after demoting node '%s'" " (UUID: %s)." % (node.name, node.uuid)) WarnAboutFailedSshUpdates(ssh_result, master_node, feedback_fn) if self.new_role == self._ROLE_CANDIDATE: AddMasterCandidateSshKey( self, master_node, node, potential_master_candidates, feedback_fn) return result class LUNodePowercycle(NoHooksLU): """Powercycles a node. """ REQ_BGL = False def CheckArguments(self): (self.op.node_uuid, self.op.node_name) = \ ExpandNodeUuidAndName(self.cfg, self.op.node_uuid, self.op.node_name) if self.op.node_uuid == self.cfg.GetMasterNode() and not self.op.force: raise errors.OpPrereqError("The node is the master and the force" " parameter was not set", errors.ECODE_INVAL) def ExpandNames(self): """Locking for PowercycleNode. This is a last-resort option and shouldn't block on other jobs. Therefore, we grab no locks. """ self.needed_locks = {} def Exec(self, feedback_fn): """Reboots a node. """ default_hypervisor = self.cfg.GetHypervisorType() hvparams = self.cfg.GetClusterInfo().hvparams[default_hypervisor] result = self.rpc.call_node_powercycle(self.op.node_uuid, default_hypervisor, hvparams) result.Raise("Failed to schedule the reboot") return result.payload def _GetNodeInstancesInner(cfg, fn): return [i for i in cfg.GetAllInstancesInfo().values() if fn(i)] def _GetNodePrimaryInstances(cfg, node_uuid): """Returns primary instances on a node. """ return _GetNodeInstancesInner(cfg, lambda inst: node_uuid == inst.primary_node) def _GetNodeSecondaryInstances(cfg, node_uuid): """Returns secondary instances on a node. """ return _GetNodeInstancesInner(cfg, lambda inst: node_uuid in cfg.GetInstanceSecondaryNodes(inst.uuid)) def _GetNodeInstances(cfg, node_uuid): """Returns a list of all primary and secondary instances on a node. """ return _GetNodeInstancesInner(cfg, lambda inst: node_uuid in cfg.GetInstanceNodes(inst.uuid)) class LUNodeEvacuate(NoHooksLU): """Evacuates instances off a list of nodes. """ REQ_BGL = False def CheckArguments(self): CheckIAllocatorOrNode(self, "iallocator", "remote_node") def ExpandNames(self): (self.op.node_uuid, self.op.node_name) = \ ExpandNodeUuidAndName(self.cfg, self.op.node_uuid, self.op.node_name) if self.op.remote_node is not None: (self.op.remote_node_uuid, self.op.remote_node) = \ ExpandNodeUuidAndName(self.cfg, self.op.remote_node_uuid, self.op.remote_node) assert self.op.remote_node if self.op.node_uuid == self.op.remote_node_uuid: raise errors.OpPrereqError("Can not use evacuated node as a new" " secondary node", errors.ECODE_INVAL) if self.op.mode != constants.NODE_EVAC_SEC: raise errors.OpPrereqError("Without the use of an iallocator only" " secondary instances can be evacuated", errors.ECODE_INVAL) # Declare locks self.share_locks = ShareAll() self.needed_locks = { locking.LEVEL_INSTANCE: [], locking.LEVEL_NODEGROUP: [], locking.LEVEL_NODE: [], } # Determine nodes (via group) optimistically, needs verification once locks # have been acquired self.lock_nodes = self._DetermineNodes() def _DetermineNodes(self): """Gets the list of node UUIDs to operate on. """ if self.op.remote_node is None: # Iallocator will choose any node(s) in the same group group_nodes = self.cfg.GetNodeGroupMembersByNodes([self.op.node_uuid]) else: group_nodes = frozenset([self.op.remote_node_uuid]) # Determine nodes to be locked return set([self.op.node_uuid]) | group_nodes def _DetermineInstances(self): """Builds list of instances to operate on. """ assert self.op.mode in constants.NODE_EVAC_MODES if self.op.mode == constants.NODE_EVAC_PRI: # Primary instances only inst_fn = _GetNodePrimaryInstances assert self.op.remote_node is None, \ "Evacuating primary instances requires iallocator" elif self.op.mode == constants.NODE_EVAC_SEC: # Secondary instances only inst_fn = _GetNodeSecondaryInstances else: # All instances assert self.op.mode == constants.NODE_EVAC_ALL inst_fn = _GetNodeInstances # TODO: In 2.6, change the iallocator interface to take an evacuation mode # per instance raise errors.OpPrereqError("Due to an issue with the iallocator" " interface it is not possible to evacuate" " all instances at once; specify explicitly" " whether to evacuate primary or secondary" " instances", errors.ECODE_INVAL) return inst_fn(self.cfg, self.op.node_uuid) def DeclareLocks(self, level): if level == locking.LEVEL_INSTANCE: # Lock instances optimistically, needs verification once node and group # locks have been acquired self.needed_locks[locking.LEVEL_INSTANCE] = \ set(i.name for i in self._DetermineInstances()) elif level == locking.LEVEL_NODEGROUP: # Lock node groups for all potential target nodes optimistically, needs # verification once nodes have been acquired self.needed_locks[locking.LEVEL_NODEGROUP] = \ self.cfg.GetNodeGroupsFromNodes(self.lock_nodes) elif level == locking.LEVEL_NODE: self.needed_locks[locking.LEVEL_NODE] = self.lock_nodes def CheckPrereq(self): # Verify locks owned_instance_names = self.owned_locks(locking.LEVEL_INSTANCE) owned_nodes = self.owned_locks(locking.LEVEL_NODE) owned_groups = self.owned_locks(locking.LEVEL_NODEGROUP) need_nodes = self._DetermineNodes() if not owned_nodes.issuperset(need_nodes): raise errors.OpPrereqError("Nodes in same group as '%s' changed since" " locks were acquired, current nodes are" " are '%s', used to be '%s'; retry the" " operation" % (self.op.node_name, utils.CommaJoin(need_nodes), utils.CommaJoin(owned_nodes)), errors.ECODE_STATE) wanted_groups = self.cfg.GetNodeGroupsFromNodes(owned_nodes) if owned_groups != wanted_groups: raise errors.OpExecError("Node groups changed since locks were acquired," " current groups are '%s', used to be '%s';" " retry the operation" % (utils.CommaJoin(wanted_groups), utils.CommaJoin(owned_groups))) # Determine affected instances self.instances = self._DetermineInstances() self.instance_names = [i.name for i in self.instances] if set(self.instance_names) != owned_instance_names: raise errors.OpExecError("Instances on node '%s' changed since locks" " were acquired, current instances are '%s'," " used to be '%s'; retry the operation" % (self.op.node_name, utils.CommaJoin(self.instance_names), utils.CommaJoin(owned_instance_names))) if self.instance_names: self.LogInfo("Evacuating instances from node '%s': %s", self.op.node_name, utils.CommaJoin(utils.NiceSort(self.instance_names))) else: self.LogInfo("No instances to evacuate from node '%s'", self.op.node_name) if self.op.remote_node is not None: for i in self.instances: if i.primary_node == self.op.remote_node_uuid: raise errors.OpPrereqError("Node %s is the primary node of" " instance %s, cannot use it as" " secondary" % (self.op.remote_node, i.name), errors.ECODE_INVAL) def Exec(self, feedback_fn): assert (self.op.iallocator is not None) ^ (self.op.remote_node is not None) if not self.instance_names: # No instances to evacuate jobs = [] elif self.op.iallocator is not None: # TODO: Implement relocation to other group req = iallocator.IAReqNodeEvac( evac_mode=self.op.mode, instances=list(self.instance_names), ignore_soft_errors=self.op.ignore_soft_errors) ial = iallocator.IAllocator(self.cfg, self.rpc, req) ial.Run(self.op.iallocator) if not ial.success: raise errors.OpPrereqError("Can't compute node evacuation using" " iallocator '%s': %s" % (self.op.iallocator, ial.info), errors.ECODE_NORES) jobs = LoadNodeEvacResult(self, ial.result, self.op.early_release, True) elif self.op.remote_node is not None: assert self.op.mode == constants.NODE_EVAC_SEC jobs = [ [opcodes.OpInstanceReplaceDisks(instance_name=instance_name, remote_node=self.op.remote_node, disks=[], mode=constants.REPLACE_DISK_CHG, early_release=self.op.early_release)] for instance_name in self.instance_names] else: raise errors.ProgrammerError("No iallocator or remote node") return ResultWithJobs(jobs) class LUNodeMigrate(LogicalUnit): """Migrate all instances from a node. """ HPATH = "node-migrate" HTYPE = constants.HTYPE_NODE REQ_BGL = False def CheckArguments(self): pass def ExpandNames(self): (self.op.node_uuid, self.op.node_name) = \ ExpandNodeUuidAndName(self.cfg, self.op.node_uuid, self.op.node_name) self.share_locks = ShareAll() self.needed_locks = { locking.LEVEL_NODE: [self.op.node_uuid], } def BuildHooksEnv(self): """Build hooks env. This runs on the master, the primary and all the secondaries. """ return { "NODE_NAME": self.op.node_name, "ALLOW_RUNTIME_CHANGES": self.op.allow_runtime_changes, } def BuildHooksNodes(self): """Build hooks nodes. """ nl = [self.cfg.GetMasterNode()] return (nl, nl) def CheckPrereq(self): pass def Exec(self, feedback_fn): # Prepare jobs for migration instances jobs = [ [opcodes.OpInstanceMigrate( instance_name=inst.name, mode=self.op.mode, live=self.op.live, iallocator=self.op.iallocator, target_node=self.op.target_node, allow_runtime_changes=self.op.allow_runtime_changes, ignore_ipolicy=self.op.ignore_ipolicy)] for inst in _GetNodePrimaryInstances(self.cfg, self.op.node_uuid)] # TODO: Run iallocator in this opcode and pass correct placement options to # OpInstanceMigrate. Since other jobs can modify the cluster between # running the iallocator and the actual migration, a good consistency model # will have to be found. assert (frozenset(self.owned_locks(locking.LEVEL_NODE)) == frozenset([self.op.node_uuid])) return ResultWithJobs(jobs) def _GetStorageTypeArgs(cfg, storage_type): """Returns the arguments for a storage type. """ # Special case for file storage if storage_type == constants.ST_FILE: return [[cfg.GetFileStorageDir()]] elif storage_type == constants.ST_SHARED_FILE: return [[cfg.GetSharedFileStorageDir()]] elif storage_type == constants.ST_GLUSTER: return [[cfg.GetGlusterStorageDir()]] else: return [] class LUNodeModifyStorage(NoHooksLU): """Logical unit for modifying a storage volume on a node. """ REQ_BGL = False def CheckArguments(self): (self.op.node_uuid, self.op.node_name) = \ ExpandNodeUuidAndName(self.cfg, self.op.node_uuid, self.op.node_name) storage_type = self.op.storage_type try: modifiable = constants.MODIFIABLE_STORAGE_FIELDS[storage_type] except KeyError: raise errors.OpPrereqError("Storage units of type '%s' can not be" " modified" % storage_type, errors.ECODE_INVAL) diff = set(self.op.changes.keys()) - modifiable if diff: raise errors.OpPrereqError("The following fields can not be modified for" " storage units of type '%s': %r" % (storage_type, list(diff)), errors.ECODE_INVAL) def CheckPrereq(self): """Check prerequisites. """ CheckStorageTypeEnabled(self.cfg.GetClusterInfo(), self.op.storage_type) def ExpandNames(self): self.needed_locks = { locking.LEVEL_NODE: self.op.node_uuid, } def Exec(self, feedback_fn): """Computes the list of nodes and their attributes. """ st_args = _GetStorageTypeArgs(self.cfg, self.op.storage_type) result = self.rpc.call_storage_modify(self.op.node_uuid, self.op.storage_type, st_args, self.op.name, self.op.changes) result.Raise("Failed to modify storage unit '%s' on %s" % (self.op.name, self.op.node_name)) def _CheckOutputFields(fields, selected): """Checks whether all selected fields are valid according to fields. @type fields: L{utils.FieldSet} @param fields: fields set @type selected: L{utils.FieldSet} @param selected: fields set """ delta = fields.NonMatching(selected) if delta: raise errors.OpPrereqError("Unknown output fields selected: %s" % ",".join(delta), errors.ECODE_INVAL) class LUNodeQueryvols(NoHooksLU): """Logical unit for getting volumes on node(s). """ REQ_BGL = False def CheckArguments(self): _CheckOutputFields(utils.FieldSet(constants.VF_NODE, constants.VF_PHYS, constants.VF_VG, constants.VF_NAME, constants.VF_SIZE, constants.VF_INSTANCE), self.op.output_fields) def ExpandNames(self): self.share_locks = ShareAll() if self.op.nodes: self.needed_locks = { locking.LEVEL_NODE: GetWantedNodes(self, self.op.nodes)[0], } else: self.needed_locks = { locking.LEVEL_NODE: locking.ALL_SET, } def Exec(self, feedback_fn): """Computes the list of nodes and their attributes. """ node_uuids = self.owned_locks(locking.LEVEL_NODE) volumes = self.rpc.call_node_volumes(node_uuids) ilist = self.cfg.GetAllInstancesInfo() vol2inst = MapInstanceLvsToNodes(self.cfg, list(ilist.values())) output = [] for node_uuid in node_uuids: nresult = volumes[node_uuid] if nresult.offline: continue msg = nresult.fail_msg if msg: self.LogWarning("Can't compute volume data on node %s: %s", self.cfg.GetNodeName(node_uuid), msg) continue node_vols = sorted(nresult.payload, key=operator.itemgetter(constants.VF_DEV)) for vol in node_vols: node_output = [] for field in self.op.output_fields: if field == constants.VF_NODE: val = self.cfg.GetNodeName(node_uuid) elif field == constants.VF_PHYS: val = vol[constants.VF_DEV] elif field == constants.VF_VG: val = vol[constants.VF_VG] elif field == constants.VF_NAME: val = vol[constants.VF_NAME] elif field == constants.VF_SIZE: val = int(float(vol[constants.VF_SIZE])) elif field == constants.VF_INSTANCE: inst = vol2inst.get((node_uuid, vol[constants.VF_VG] + "/" + vol[constants.VF_NAME]), None) if inst is not None: val = inst.name else: val = "-" else: raise errors.ParameterError(field) node_output.append(str(val)) output.append(node_output) return output class LUNodeQueryStorage(NoHooksLU): """Logical unit for getting information on storage units on node(s). """ REQ_BGL = False def CheckArguments(self): _CheckOutputFields(utils.FieldSet(*constants.VALID_STORAGE_FIELDS), self.op.output_fields) def ExpandNames(self): self.share_locks = ShareAll() if self.op.nodes: self.needed_locks = { locking.LEVEL_NODE: GetWantedNodes(self, self.op.nodes)[0], } else: self.needed_locks = { locking.LEVEL_NODE: locking.ALL_SET, } def _DetermineStorageType(self): """Determines the default storage type of the cluster. """ enabled_disk_templates = self.cfg.GetClusterInfo().enabled_disk_templates default_storage_type = \ constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[enabled_disk_templates[0]] return default_storage_type def CheckPrereq(self): """Check prerequisites. """ if self.op.storage_type: CheckStorageTypeEnabled(self.cfg.GetClusterInfo(), self.op.storage_type) self.storage_type = self.op.storage_type else: self.storage_type = self._DetermineStorageType() supported_storage_types = constants.STS_REPORT_NODE_STORAGE if self.storage_type not in supported_storage_types: raise errors.OpPrereqError( "Storage reporting for storage type '%s' is not supported. Please" " use the --storage-type option to specify one of the supported" " storage types (%s) or set the default disk template to one that" " supports storage reporting." % (self.storage_type, utils.CommaJoin(supported_storage_types))) def Exec(self, feedback_fn): """Computes the list of nodes and their attributes. """ if self.op.storage_type: self.storage_type = self.op.storage_type else: self.storage_type = self._DetermineStorageType() self.node_uuids = self.owned_locks(locking.LEVEL_NODE) # Always get name to sort by if constants.SF_NAME in self.op.output_fields: fields = self.op.output_fields[:] else: fields = [constants.SF_NAME] + self.op.output_fields # Never ask for node or type as it's only known to the LU for extra in [constants.SF_NODE, constants.SF_TYPE]: while extra in fields: fields.remove(extra) field_idx = dict([(name, idx) for (idx, name) in enumerate(fields)]) name_idx = field_idx[constants.SF_NAME] st_args = _GetStorageTypeArgs(self.cfg, self.storage_type) data = self.rpc.call_storage_list(self.node_uuids, self.storage_type, st_args, self.op.name, fields) result = [] for node_uuid in utils.NiceSort(self.node_uuids): node_name = self.cfg.GetNodeName(node_uuid) nresult = data[node_uuid] if nresult.offline: continue msg = nresult.fail_msg if msg: self.LogWarning("Can't get storage data from node %s: %s", node_name, msg) continue rows = dict([(row[name_idx], row) for row in nresult.payload]) for name in utils.NiceSort(rows): row = rows[name] out = [] for field in self.op.output_fields: if field == constants.SF_NODE: val = node_name elif field == constants.SF_TYPE: val = self.storage_type elif field in field_idx: val = row[field_idx[field]] else: raise errors.ParameterError(field) out.append(val) result.append(out) return result class LUNodeRemove(LogicalUnit): """Logical unit for removing a node. """ HPATH = "node-remove" HTYPE = constants.HTYPE_NODE def BuildHooksEnv(self): """Build hooks env. """ return { "OP_TARGET": self.op.node_name, "NODE_NAME": self.op.node_name, } def BuildHooksNodes(self): """Build hooks nodes. This doesn't run on the target node in the pre phase as a failed node would then be impossible to remove. """ all_nodes = self.cfg.GetNodeList() try: all_nodes.remove(self.op.node_uuid) except ValueError: pass return (all_nodes, all_nodes) def CheckPrereq(self): """Check prerequisites. This checks: - the node exists in the configuration - it does not have primary or secondary instances - it's not the master Any errors are signaled by raising errors.OpPrereqError. """ (self.op.node_uuid, self.op.node_name) = \ ExpandNodeUuidAndName(self.cfg, self.op.node_uuid, self.op.node_name) node = self.cfg.GetNodeInfo(self.op.node_uuid) assert node is not None masternode = self.cfg.GetMasterNode() if node.uuid == masternode: raise errors.OpPrereqError("Node is the master node, failover to another" " node is required", errors.ECODE_INVAL) for _, instance in self.cfg.GetAllInstancesInfo().items(): if node.uuid in self.cfg.GetInstanceNodes(instance.uuid): raise errors.OpPrereqError("Instance %s is still running on the node," " please remove first" % instance.name, errors.ECODE_INVAL) self.op.node_name = node.name self.node = node def Exec(self, feedback_fn): """Removes the node from the cluster. """ logging.info("Stopping the node daemon and removing configs from node %s", self.node.name) modify_ssh_setup = self.cfg.GetClusterInfo().modify_ssh_setup assert locking.BGL in self.owned_locks(locking.LEVEL_CLUSTER), \ "Not owning BGL" master_node = self.cfg.GetMasterNode() potential_master_candidates = self.cfg.GetPotentialMasterCandidates() if modify_ssh_setup: # retrieve the list of potential master candidates before the node is # removed potential_master_candidate = \ self.op.node_name in potential_master_candidates master_candidate_uuids = self.cfg.GetMasterCandidateUuids() result = self.rpc.call_node_ssh_key_remove( [master_node], self.node.uuid, self.op.node_name, master_candidate_uuids, potential_master_candidates, self.node.master_candidate, # from_authorized_keys potential_master_candidate, # from_public_keys True, # clear node's 'authorized_keys' True, # clear node's 'ganeti_public_keys' False) # no readd result[master_node].Raise( "Could not remove the SSH key of node '%s' (UUID: %s)." % (self.op.node_name, self.node.uuid)) WarnAboutFailedSshUpdates(result, master_node, feedback_fn) # Promote nodes to master candidate as needed AdjustCandidatePool( self, [self.node.uuid], master_node, potential_master_candidates, feedback_fn, modify_ssh_setup) self.cfg.RemoveNode(self.node.uuid) # Run post hooks on the node before it's removed RunPostHook(self, self.node.name) # we have to call this by name rather than by UUID, as the node is no longer # in the config result = self.rpc.call_node_leave_cluster(self.node.name, modify_ssh_setup) msg = result.fail_msg if msg: self.LogWarning("Errors encountered on the remote node while leaving" " the cluster: %s", msg) cluster = self.cfg.GetClusterInfo() # Remove node from candidate certificate list if self.node.master_candidate: self.cfg.RemoveNodeFromCandidateCerts(self.node.uuid) # Remove node from our /etc/hosts if cluster.modify_etc_hosts: master_node_uuid = self.cfg.GetMasterNode() result = self.rpc.call_etc_hosts_modify(master_node_uuid, constants.ETC_HOSTS_REMOVE, self.node.name, None) result.Raise("Can't update hosts file with new host data") RedistributeAncillaryFiles(self) class LURepairNodeStorage(NoHooksLU): """Repairs the volume group on a node. """ REQ_BGL = False def CheckArguments(self): (self.op.node_uuid, self.op.node_name) = \ ExpandNodeUuidAndName(self.cfg, self.op.node_uuid, self.op.node_name) storage_type = self.op.storage_type if (constants.SO_FIX_CONSISTENCY not in constants.VALID_STORAGE_OPERATIONS.get(storage_type, [])): raise errors.OpPrereqError("Storage units of type '%s' can not be" " repaired" % storage_type, errors.ECODE_INVAL) def ExpandNames(self): self.needed_locks = { locking.LEVEL_NODE: [self.op.node_uuid], } def _CheckFaultyDisks(self, instance, node_uuid): """Ensure faulty disks abort the opcode or at least warn.""" try: if FindFaultyInstanceDisks(self.cfg, self.rpc, instance, node_uuid, True): raise errors.OpPrereqError("Instance '%s' has faulty disks on" " node '%s'" % (instance.name, self.cfg.GetNodeName(node_uuid)), errors.ECODE_STATE) except errors.OpPrereqError as err: if self.op.ignore_consistency: self.LogWarning(str(err.args[0])) else: raise def CheckPrereq(self): """Check prerequisites. """ CheckStorageTypeEnabled(self.cfg.GetClusterInfo(), self.op.storage_type) # Check whether any instance on this node has faulty disks for inst in _GetNodeInstances(self.cfg, self.op.node_uuid): if not inst.disks_active: continue check_nodes = set(self.cfg.GetInstanceNodes(inst.uuid)) check_nodes.discard(self.op.node_uuid) for inst_node_uuid in check_nodes: self._CheckFaultyDisks(inst, inst_node_uuid) def Exec(self, feedback_fn): feedback_fn("Repairing storage unit '%s' on %s ..." % (self.op.name, self.op.node_name)) st_args = _GetStorageTypeArgs(self.cfg, self.op.storage_type) result = self.rpc.call_storage_execute(self.op.node_uuid, self.op.storage_type, st_args, self.op.name, constants.SO_FIX_CONSISTENCY) result.Raise("Failed to repair storage unit '%s' on %s" % (self.op.name, self.op.node_name)) ganeti-3.1.0~rc2/lib/cmdlib/operating_system.py000064400000000000000000000156511476477700300215620ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with OS.""" from ganeti import locking from ganeti import qlang from ganeti import query from ganeti.cmdlib.base import QueryBase, NoHooksLU class OsQuery(QueryBase): FIELDS = query.OS_FIELDS def ExpandNames(self, lu): # Lock all nodes in shared mode # Temporary removal of locks, should be reverted later # TODO: reintroduce locks when they are lighter-weight lu.needed_locks = {} #self.share_locks[locking.LEVEL_NODE] = 1 #self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET # The following variables interact with _QueryBase._GetNames if self.names: self.wanted = self.names else: self.wanted = locking.ALL_SET self.do_locking = self.use_locking def DeclareLocks(self, lu, level): pass @staticmethod def _DiagnoseByOS(rlist): """Remaps a per-node return list into a per-os per-node dictionary @param rlist: a map with node names as keys and OS objects as values @rtype: dict @return: a dictionary with osnames as keys and as value another map, with node UUIDs as keys and tuples of (path, status, diagnose, variants, parameters, api_versions) as values, eg:: {"debian-etch": {"node1-uuid": [(/usr/lib/..., True, "", [], []), (/srv/..., False, "invalid api")], "node2-uuid": [(/srv/..., True, "", [], [])]} } """ all_os = {} # we build here the list of nodes that didn't fail the RPC (at RPC # level), so that nodes with a non-responding node daemon don't # make all OSes invalid good_node_uuids = [node_uuid for node_uuid in rlist if not rlist[node_uuid].fail_msg] for node_uuid, nr in rlist.items(): if nr.fail_msg or not nr.payload: continue for (name, path, status, diagnose, variants, params, api_versions, trusted) in nr.payload: if name not in all_os: # build a list of nodes for this os containing empty lists # for each node in node_list all_os[name] = {} for nuuid in good_node_uuids: all_os[name][nuuid] = [] # convert params from [name, help] to (name, help) params = [tuple(v) for v in params] all_os[name][node_uuid].append((path, status, diagnose, variants, params, api_versions, trusted)) return all_os def _GetQueryData(self, lu): """Computes the list of nodes and their attributes. """ valid_node_uuids = [node.uuid for node in lu.cfg.GetAllNodesInfo().values() if not node.offline and node.vm_capable] pol = self._DiagnoseByOS(lu.rpc.call_os_diagnose(valid_node_uuids)) cluster = lu.cfg.GetClusterInfo() data = {} for (os_name, os_data) in pol.items(): info = query.OsInfo(name=os_name, valid=True, node_status=os_data, hidden=(os_name in cluster.hidden_os), blacklisted=(os_name in cluster.blacklisted_os), os_hvp={}, osparams={}) variants = set() parameters = set() api_versions = set() trusted = True for idx, osl in enumerate(os_data.values()): info.valid = bool(info.valid and osl and osl[0][1]) if not info.valid: break (node_variants, node_params, node_api, node_trusted) = osl[0][3:7] if idx == 0: # First entry variants.update(node_variants) parameters.update(node_params) api_versions.update(node_api) else: # Filter out inconsistent values variants.intersection_update(node_variants) parameters.intersection_update(node_params) api_versions.intersection_update(node_api) if not node_trusted: trusted = False info.variants = list(variants) info.parameters = list(parameters) info.api_versions = list(api_versions) info.trusted = trusted for variant in variants: name = "+".join([os_name, variant]) if name in cluster.os_hvp: info.os_hvp[name] = cluster.os_hvp.get(name) if name in cluster.osparams: info.osparams[name] = cluster.osparams.get(name) data[os_name] = info # Prepare data in requested order return [data[name] for name in self._GetNames(lu, list(pol), None) if name in data] class LUOsDiagnose(NoHooksLU): """Logical unit for OS diagnose/query. """ REQ_BGL = False @staticmethod def _BuildFilter(fields, names): """Builds a filter for querying OSes. """ name_filter = qlang.MakeSimpleFilter("name", names) # Legacy behaviour: Hide hidden, blacklisted or invalid OSes if the # respective field is not requested status_filter = [[qlang.OP_NOT, [qlang.OP_TRUE, fname]] for fname in ["hidden", "blacklisted"] if fname not in fields] if "valid" not in fields: status_filter.append([qlang.OP_TRUE, "valid"]) if status_filter: status_filter.insert(0, qlang.OP_AND) else: status_filter = None if name_filter and status_filter: return [qlang.OP_AND, name_filter, status_filter] elif name_filter: return name_filter else: return status_filter def CheckArguments(self): self.oq = OsQuery(self._BuildFilter(self.op.output_fields, self.op.names), self.op.output_fields, False) def ExpandNames(self): self.oq.ExpandNames(self) def Exec(self, feedback_fn): return self.oq.OldStyleQuery(self) ganeti-3.1.0~rc2/lib/cmdlib/query.py000064400000000000000000000057361476477700300173360ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units for queries.""" from ganeti import constants from ganeti import errors from ganeti import query from ganeti.cmdlib.base import NoHooksLU from ganeti.cmdlib.cluster import ClusterQuery from ganeti.cmdlib.misc import ExtStorageQuery from ganeti.cmdlib.operating_system import OsQuery #: Query type implementations _QUERY_IMPL = { constants.QR_CLUSTER: ClusterQuery, constants.QR_OS: OsQuery, constants.QR_EXTSTORAGE: ExtStorageQuery, } assert set(_QUERY_IMPL.keys()) == constants.QR_VIA_OP def _GetQueryImplementation(name): """Returns the implementation for a query type. @param name: Query type, must be one of L{constants.QR_VIA_OP} """ try: return _QUERY_IMPL[name] except KeyError: raise errors.OpPrereqError("Unknown query resource '%s'" % name, errors.ECODE_INVAL) class LUQuery(NoHooksLU): """Query for resources/items of a certain kind. """ REQ_BGL = False def CheckArguments(self): qcls = _GetQueryImplementation(self.op.what) self.impl = qcls(self.op.qfilter, self.op.fields, self.op.use_locking) def ExpandNames(self): self.impl.ExpandNames(self) def DeclareLocks(self, level): self.impl.DeclareLocks(self, level) def Exec(self, feedback_fn): return self.impl.NewStyleQuery(self) class LUQueryFields(NoHooksLU): """Query for resources/items of a certain kind. """ REQ_BGL = False def CheckArguments(self): self.qcls = _GetQueryImplementation(self.op.what) def ExpandNames(self): self.needed_locks = {} def Exec(self, feedback_fn): return query.QueryFields(self.qcls.FIELDS, self.op.fields) ganeti-3.1.0~rc2/lib/cmdlib/tags.py000064400000000000000000000156761476477700300171330ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Logical units dealing with tags.""" import re from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import objects from ganeti import utils from ganeti.cmdlib.base import NoHooksLU from ganeti.cmdlib.common import ExpandNodeUuidAndName, \ ExpandInstanceUuidAndName, ShareAll class TagsLU(NoHooksLU): # pylint: disable=W0223 """Generic tags LU. This is an abstract class which is the parent of all the other tags LUs. """ def ExpandNames(self): self.group_uuid = None self.needed_locks = {} if self.op.kind == constants.TAG_NODE: (self.node_uuid, _) = \ ExpandNodeUuidAndName(self.cfg, None, self.op.name) lock_level = locking.LEVEL_NODE lock_name = self.node_uuid elif self.op.kind == constants.TAG_INSTANCE: (self.inst_uuid, inst_name) = \ ExpandInstanceUuidAndName(self.cfg, None, self.op.name) lock_level = locking.LEVEL_INSTANCE lock_name = inst_name elif self.op.kind == constants.TAG_NODEGROUP: self.group_uuid = self.cfg.LookupNodeGroup(self.op.name) lock_level = locking.LEVEL_NODEGROUP lock_name = self.group_uuid elif self.op.kind == constants.TAG_NETWORK: self.network_uuid = self.cfg.LookupNetwork(self.op.name) lock_level = locking.LEVEL_NETWORK lock_name = self.network_uuid else: lock_level = None lock_name = None if lock_level and getattr(self.op, "use_locking", True): self.needed_locks[lock_level] = lock_name # FIXME: Acquire BGL for cluster tag operations (as of this writing it's # not possible to acquire the BGL based on opcode parameters) def CheckPrereq(self): """Check prerequisites. """ if self.op.kind == constants.TAG_CLUSTER: self.target = self.cfg.GetClusterInfo() elif self.op.kind == constants.TAG_NODE: self.target = self.cfg.GetNodeInfo(self.node_uuid) elif self.op.kind == constants.TAG_INSTANCE: self.target = self.cfg.GetInstanceInfo(self.inst_uuid) elif self.op.kind == constants.TAG_NODEGROUP: self.target = self.cfg.GetNodeGroup(self.group_uuid) elif self.op.kind == constants.TAG_NETWORK: self.target = self.cfg.GetNetwork(self.network_uuid) else: raise errors.OpPrereqError("Wrong tag type requested (%s)" % str(self.op.kind), errors.ECODE_INVAL) class LUTagsGet(TagsLU): """Returns the tags of a given object. """ REQ_BGL = False def ExpandNames(self): TagsLU.ExpandNames(self) # Share locks as this is only a read operation self.share_locks = ShareAll() def Exec(self, feedback_fn): """Returns the tag list. """ return list(self.target.GetTags()) class LUTagsSearch(NoHooksLU): """Searches the tags for a given pattern. """ REQ_BGL = False def ExpandNames(self): self.needed_locks = {} def CheckPrereq(self): """Check prerequisites. This checks the pattern passed for validity by compiling it. """ try: self.re = re.compile(self.op.pattern) except re.error as err: raise errors.OpPrereqError("Invalid search pattern '%s': %s" % (self.op.pattern, err), errors.ECODE_INVAL) @staticmethod def _ExtendTagTargets(targets, object_type_name, object_info_dict): return targets.extend(("/%s/%s" % (object_type_name, o.name), o) for o in object_info_dict.values()) def Exec(self, feedback_fn): """Returns the tag list. """ tgts = [("/cluster", self.cfg.GetClusterInfo())] LUTagsSearch._ExtendTagTargets(tgts, "instances", self.cfg.GetAllInstancesInfo()) LUTagsSearch._ExtendTagTargets(tgts, "nodes", self.cfg.GetAllNodesInfo()) LUTagsSearch._ExtendTagTargets(tgts, "nodegroup", self.cfg.GetAllNodeGroupsInfo()) LUTagsSearch._ExtendTagTargets(tgts, "network", self.cfg.GetAllNetworksInfo()) results = [] for path, target in tgts: for tag in target.GetTags(): if self.re.search(tag): results.append((path, tag)) return results class LUTagsSet(TagsLU): """Sets a tag on a given object. """ REQ_BGL = False def CheckPrereq(self): """Check prerequisites. This checks the type and length of the tag name and value. """ TagsLU.CheckPrereq(self) for tag in self.op.tags: objects.TaggableObject.ValidateTag(tag) def Exec(self, feedback_fn): """Sets the tag. """ try: for tag in self.op.tags: self.target.AddTag(tag) except errors.TagError as err: raise errors.OpExecError("Error while setting tag: %s" % str(err)) self.cfg.Update(self.target, feedback_fn) class LUTagsDel(TagsLU): """Delete a list of tags from a given object. """ REQ_BGL = False def CheckPrereq(self): """Check prerequisites. This checks that we have the given tag. """ TagsLU.CheckPrereq(self) for tag in self.op.tags: objects.TaggableObject.ValidateTag(tag) del_tags = frozenset(self.op.tags) cur_tags = self.target.GetTags() diff_tags = del_tags - cur_tags if diff_tags: diff_names = ("'%s'" % i for i in sorted(diff_tags)) raise errors.OpPrereqError("Tag(s) %s not found" % (utils.CommaJoin(diff_names), ), errors.ECODE_NOENT) def Exec(self, feedback_fn): """Remove the tag from the object. """ for tag in self.op.tags: self.target.RemoveTag(tag) self.cfg.Update(self.target, feedback_fn) ganeti-3.1.0~rc2/lib/cmdlib/test.py000064400000000000000000000375441476477700300171520ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Test logical units.""" import logging import shutil import socket import tempfile import time from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import utils from ganeti.masterd import iallocator from ganeti.cmdlib.base import NoHooksLU from ganeti.cmdlib.common import ExpandInstanceUuidAndName, GetWantedNodes, \ GetWantedInstances class TestSocketWrapper(object): """ Utility class that opens a domain socket and cleans up as needed. """ def __init__(self): """ Constructor cleaning up variables to be used. """ self.tmpdir = None self.sock = None def Create(self, max_connections=1): """ Creates a bound and ready socket, cleaning up in case of failure. @type max_connections: int @param max_connections: The number of max connections allowed for the socket. @rtype: tuple of socket, string @return: The socket object and the path to reach it with. """ # Using a temporary directory as there's no easy way to create temporary # sockets without writing a custom loop around tempfile.mktemp and # socket.bind self.tmpdir = tempfile.mkdtemp() try: tmpsock = utils.PathJoin(self.tmpdir, "sock") logging.debug("Creating temporary socket at %s", tmpsock) self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: self.sock.bind(tmpsock) self.sock.listen(max_connections) except: self.sock.close() raise except: shutil.rmtree(self.tmpdir) raise return self.sock, tmpsock def Destroy(self): """ Destroys the socket and performs all necessary cleanup. """ if self.tmpdir is None or self.sock is None: raise Exception("A socket must be created successfully before attempting " "its destruction") try: self.sock.close() finally: shutil.rmtree(self.tmpdir) class LUTestDelay(NoHooksLU): """Sleep for a specified amount of time. This LU sleeps on the master and/or nodes for a specified amount of time. """ REQ_BGL = False def ExpandNames(self): """Expand names and set required locks. This expands the node list, if any. """ self.needed_locks = {} if self.op.duration <= 0: raise errors.OpPrereqError("Duration must be greater than zero") if not self.op.no_locks and (self.op.on_nodes or self.op.on_master): self.needed_locks[locking.LEVEL_NODE] = [] self.op.on_node_uuids = [] if self.op.on_nodes: # _GetWantedNodes can be used here, but is not always appropriate to use # this way in ExpandNames. Check LogicalUnit.ExpandNames docstring for # more information. (self.op.on_node_uuids, self.op.on_nodes) = \ GetWantedNodes(self, self.op.on_nodes) master_uuid = self.cfg.GetMasterNode() if self.op.on_master and master_uuid not in self.op.on_node_uuids: self.op.on_node_uuids.append(master_uuid) self.needed_locks = {} self.needed_locks[locking.LEVEL_NODE] = self.op.on_node_uuids def _InterruptibleDelay(self): """Delays but provides the mechanisms necessary to interrupt the delay as needed. """ socket_wrapper = TestSocketWrapper() sock, path = socket_wrapper.Create() self.Log(constants.ELOG_DELAY_TEST, (path,)) try: sock.settimeout(self.op.duration) start = time.time() (conn, _) = sock.accept() except socket.timeout: # If we timed out, all is well return False finally: # Destroys the original socket, but the new connection is still usable socket_wrapper.Destroy() try: # Change to remaining time time_to_go = self.op.duration - (time.time() - start) self.Log(constants.ELOG_MESSAGE, "Received connection, time to go is %d" % time_to_go) if time_to_go < 0: time_to_go = 0 # pylint: disable=E1101 # Instance of '_socketobject' has no ... member conn.settimeout(time_to_go) conn.recv(1) except socket.timeout: # A second timeout can occur if no data is sent return False finally: conn.close() self.Log(constants.ELOG_MESSAGE, "Interrupted, time spent waiting: %d" % (time.time() - start)) # Reaching this point means we were interrupted return True def _UninterruptibleDelay(self): """Delays without allowing interruptions. """ if self.op.on_node_uuids: result = self.rpc.call_test_delay(self.op.on_node_uuids, self.op.duration) for node_uuid, node_result in result.items(): node_result.Raise("Failure during rpc call to node %s" % self.cfg.GetNodeName(node_uuid)) else: if not utils.TestDelay(self.op.duration)[0]: raise errors.OpExecError("Error during master delay test") def _TestDelay(self): """Do the actual sleep. @rtype: bool @return: Whether the delay was interrupted """ if self.op.interruptible: return self._InterruptibleDelay() else: self._UninterruptibleDelay() return False def Exec(self, feedback_fn): """Execute the test delay opcode, with the wanted repetitions. """ if self.op.repeat == 0: i = self._TestDelay() else: top_value = self.op.repeat - 1 for i in range(self.op.repeat): self.LogInfo("Test delay iteration %d/%d", i, top_value) # Break in case of interruption if self._TestDelay(): break class LUTestJqueue(NoHooksLU): """Utility LU to test some aspects of the job queue. """ REQ_BGL = False # Must be lower than default timeout for WaitForJobChange to see whether it # notices changed jobs _CLIENT_CONNECT_TIMEOUT = 20.0 _CLIENT_CONFIRM_TIMEOUT = 60.0 @classmethod def _NotifyUsingSocket(cls, cb, errcls): """Opens a Unix socket and waits for another program to connect. @type cb: callable @param cb: Callback to send socket name to client @type errcls: class @param errcls: Exception class to use for errors """ # Using a temporary directory as there's no easy way to create temporary # sockets without writing a custom loop around tempfile.mktemp and # socket.bind socket_wrapper = TestSocketWrapper() sock, path = socket_wrapper.Create() cb(path) try: sock.settimeout(cls._CLIENT_CONNECT_TIMEOUT) (conn, _) = sock.accept() except socket.error as err: raise errcls("Client didn't connect in time (%s)" % err) finally: socket_wrapper.Destroy() # Wait for client to close try: try: # pylint: disable=E1101 # Instance of '_socketobject' has no ... member conn.settimeout(cls._CLIENT_CONFIRM_TIMEOUT) conn.recv(1) except socket.error as err: raise errcls("Client failed to confirm notification (%s)" % err) finally: conn.close() def _SendNotification(self, test, arg, sockname): """Sends a notification to the client. @type test: string @param test: Test name @param arg: Test argument (depends on test) @type sockname: string @param sockname: Socket path """ self.Log(constants.ELOG_JQUEUE_TEST, (sockname, test, arg)) def _Notify(self, prereq, test, arg): """Notifies the client of a test. @type prereq: bool @param prereq: Whether this is a prereq-phase test @type test: string @param test: Test name @param arg: Test argument (depends on test) """ if prereq: errcls = errors.OpPrereqError else: errcls = errors.OpExecError return self._NotifyUsingSocket(compat.partial(self._SendNotification, test, arg), errcls) def CheckArguments(self): self.checkargs_calls = getattr(self, "checkargs_calls", 0) + 1 self.expandnames_calls = 0 def ExpandNames(self): checkargs_calls = getattr(self, "checkargs_calls", 0) if checkargs_calls < 1: raise errors.ProgrammerError("CheckArguments was not called") self.expandnames_calls += 1 if self.op.notify_waitlock: self._Notify(True, constants.JQT_EXPANDNAMES, None) self.LogInfo("Expanding names") # Get lock on master node (just to get a lock, not for a particular reason) self.needed_locks = { locking.LEVEL_NODE: self.cfg.GetMasterNode(), } def Exec(self, feedback_fn): if self.expandnames_calls < 1: raise errors.ProgrammerError("ExpandNames was not called") if self.op.notify_exec: self._Notify(False, constants.JQT_EXEC, None) self.LogInfo("Executing") if self.op.log_messages: self._Notify(False, constants.JQT_STARTMSG, len(self.op.log_messages)) for idx, msg in enumerate(self.op.log_messages): self.LogInfo("Sending log message %s", idx + 1) feedback_fn(constants.JQT_MSGPREFIX + msg) # Report how many test messages have been sent self._Notify(False, constants.JQT_LOGMSG, idx + 1) if self.op.fail: raise errors.OpExecError("Opcode failure was requested") return True class LUTestOsParams(NoHooksLU): """Utility LU to test secret OS parameter transmission. """ REQ_BGL = False def ExpandNames(self): self.needed_locks = {} def Exec(self, feedback_fn): if self.op.osparams_secret: msg = "Secret OS parameters: %s" % self.op.osparams_secret.Unprivate() feedback_fn(msg) else: raise errors.OpExecError("Opcode needs secret parameters") class LUTestAllocator(NoHooksLU): """Run allocator tests. This LU runs the allocator tests """ def CheckPrereq(self): """Check prerequisites. This checks the opcode parameters depending on the director and mode test. """ if self.op.mode in (constants.IALLOCATOR_MODE_ALLOC, constants.IALLOCATOR_MODE_MULTI_ALLOC): (self.inst_uuid, iname) = self.cfg.ExpandInstanceName(self.op.name) if iname is not None: raise errors.OpPrereqError("Instance '%s' already in the cluster" % iname, errors.ECODE_EXISTS) for row in self.op.disks: if (not isinstance(row, dict) or constants.IDISK_SIZE not in row or not isinstance(row[constants.IDISK_SIZE], int) or constants.IDISK_MODE not in row or row[constants.IDISK_MODE] not in constants.DISK_ACCESS_SET): raise errors.OpPrereqError("Invalid contents of the 'disks'" " parameter", errors.ECODE_INVAL) if self.op.hypervisor is None: self.op.hypervisor = self.cfg.GetHypervisorType() elif self.op.mode == constants.IALLOCATOR_MODE_RELOC: (self.inst_uuid, self.op.name) = ExpandInstanceUuidAndName(self.cfg, None, self.op.name) self.relocate_from_node_uuids = \ list(self.cfg.GetInstanceSecondaryNodes(self.inst_uuid)) elif self.op.mode in (constants.IALLOCATOR_MODE_CHG_GROUP, constants.IALLOCATOR_MODE_NODE_EVAC): if not self.op.instances: raise errors.OpPrereqError("Missing instances", errors.ECODE_INVAL) (_, self.op.instances) = GetWantedInstances(self, self.op.instances) else: raise errors.OpPrereqError("Invalid test allocator mode '%s'" % self.op.mode, errors.ECODE_INVAL) if self.op.direction == constants.IALLOCATOR_DIR_OUT: if self.op.iallocator is None: raise errors.OpPrereqError("Missing allocator name", errors.ECODE_INVAL) def Exec(self, feedback_fn): """Run the allocator test. """ if self.op.mode == constants.IALLOCATOR_MODE_ALLOC: req = iallocator.IAReqInstanceAlloc(name=self.op.name, memory=self.op.memory, disks=self.op.disks, disk_template=self.op.disk_template, group_name=self.op.group_name, os=self.op.os, tags=self.op.tags, nics=self.op.nics, vcpus=self.op.vcpus, spindle_use=self.op.spindle_use, hypervisor=self.op.hypervisor, node_whitelist=None) elif self.op.mode == constants.IALLOCATOR_MODE_RELOC: req = iallocator.IAReqRelocate( inst_uuid=self.inst_uuid, relocate_from_node_uuids=list(self.relocate_from_node_uuids)) elif self.op.mode == constants.IALLOCATOR_MODE_CHG_GROUP: req = iallocator.IAReqGroupChange(instances=self.op.instances, target_groups=self.op.target_groups) elif self.op.mode == constants.IALLOCATOR_MODE_NODE_EVAC: req = iallocator.IAReqNodeEvac(instances=self.op.instances, evac_mode=self.op.evac_mode, ignore_soft_errors=False) elif self.op.mode == constants.IALLOCATOR_MODE_MULTI_ALLOC: disk_template = self.op.disk_template insts = [iallocator.IAReqInstanceAlloc(name="%s%s" % (self.op.name, idx), memory=self.op.memory, disks=self.op.disks, disk_template=disk_template, group_name=self.op.group_name, os=self.op.os, tags=self.op.tags, nics=self.op.nics, vcpus=self.op.vcpus, spindle_use=self.op.spindle_use, hypervisor=self.op.hypervisor, node_whitelist=None) for idx in range(self.op.count)] req = iallocator.IAReqMultiInstanceAlloc(instances=insts) else: raise errors.ProgrammerError("Uncatched mode %s in" " LUTestAllocator.Exec", self.op.mode) ial = iallocator.IAllocator(self.cfg, self.rpc, req) if self.op.direction == constants.IALLOCATOR_DIR_IN: result = ial.in_text else: ial.Run(self.op.iallocator, validate=False) result = ial.out_text return result ganeti-3.1.0~rc2/lib/compat.py000064400000000000000000000117241476477700300162140ustar00rootroot00000000000000# # # Copyright (C) 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module containing backported language/library functionality. """ import itertools import operator try: # pylint: disable=F0401 import functools except ImportError: functools = None try: # pylint: disable=F0401 import roman except ImportError: roman = None def _all(seq): """Returns True if all elements in the iterable are True. """ for _ in itertools.filterfalse(bool, seq): return False return True def _any(seq): """Returns True if any element of the iterable are True. """ for _ in filter(bool, seq): return True return False try: # pylint: disable=E0601 # pylint: disable=W0622 all = all except NameError: all = _all try: # pylint: disable=E0601 # pylint: disable=W0622 any = any except NameError: any = _any def partition(seq, pred=bool): # pylint: disable=W0622 """Partition a list in two, based on the given predicate. """ return (list(filter(pred, seq)), list(itertools.filterfalse(pred, seq))) # Even though we're using Python's built-in "partial" function if available, # this one is always defined for testing. def _partial(func, *args, **keywords): # pylint: disable=W0622 """Decorator with partial application of arguments and keywords. This function was copied from Python's documentation. """ def newfunc(*fargs, **fkeywords): newkeywords = keywords.copy() newkeywords.update(fkeywords) return func(*(args + fargs), **newkeywords) newfunc.func = func newfunc.args = args newfunc.keywords = keywords return newfunc if functools is None: partial = _partial else: partial = functools.partial def RomanOrRounded(value, rounding, convert=True): """Try to round the value to the closest integer and return it as a roman numeral. If the conversion is disabled, or if the roman module could not be loaded, round the value to the specified level and return it. @type value: number @param value: value to convert @type rounding: integer @param rounding: how many decimal digits the number should be rounded to @type convert: boolean @param convert: if False, don't try conversion at all @rtype: string @return: roman numeral for val, or formatted string representing val if conversion didn't succeed """ def _FormatOutput(val, r): format_string = "%0." + str(r) + "f" return format_string % val if roman is not None and convert: try: return roman.toRoman(round(value, 0)) except roman.RomanError: return _FormatOutput(value, rounding) return _FormatOutput(value, rounding) def TryToRoman(val, convert=True): """Try to convert a value to roman numerals If the roman module could be loaded convert the given value to a roman numeral. Gracefully fail back to leaving the value untouched. @type val: integer @param val: value to convert @type convert: boolean @param convert: if False, don't try conversion at all @rtype: string or typeof(val) @return: roman numeral for val, or val if conversion didn't succeed """ if roman is not None and convert: try: return roman.toRoman(val) except roman.RomanError: return val else: return val def UniqueFrozenset(seq): """Makes C{frozenset} from sequence after checking for duplicate elements. @raise ValueError: When there are duplicate elements """ if isinstance(seq, (list, tuple)): items = seq else: items = list(seq) result = frozenset(items) if len(items) != len(result): raise ValueError("Duplicate values found") return result #: returns the first element of a list-like value fst = operator.itemgetter(0) #: returns the second element of a list-like value snd = operator.itemgetter(1) ganeti-3.1.0~rc2/lib/confd/000075500000000000000000000000001476477700300154435ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/confd/__init__.py000064400000000000000000000046171476477700300175640ustar00rootroot00000000000000# # # Copyright (C) 2009, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Ganeti confd client/server library """ from ganeti import constants from ganeti import errors from ganeti import ht _FOURCC_LEN = 4 #: Items in the individual rows of the NodeDrbd query _HTNodeDrbdItems = [ht.TString, ht.TInt, ht.TString, ht.TString, ht.TString, ht.TString] #: Type for the (top-level) result of NodeDrbd query HTNodeDrbd = ht.TListOf(ht.TAnd(ht.TList, ht.TIsLength(len(_HTNodeDrbdItems)), ht.TItems(_HTNodeDrbdItems))) def PackMagic(payload): """Prepend the confd magic fourcc to a payload. """ return b"".join([constants.CONFD_MAGIC_FOURCC_BYTES, payload]) def UnpackMagic(payload): """Unpack and check the confd magic fourcc from a payload. """ if len(payload) < _FOURCC_LEN: raise errors.ConfdMagicError("UDP payload too short to contain the" " fourcc code") magic_number = payload[:_FOURCC_LEN] if magic_number != constants.CONFD_MAGIC_FOURCC_BYTES: raise errors.ConfdMagicError("UDP payload contains an unkown fourcc") return payload[_FOURCC_LEN:] ganeti-3.1.0~rc2/lib/confd/client.py000064400000000000000000000540621476477700300173020ustar00rootroot00000000000000# # # Copyright (C) 2009, 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Ganeti confd client Clients can use the confd client library to send requests to a group of master candidates running confd. The expected usage is through the asyncore framework, by sending queries, and asynchronously receiving replies through a callback. This way the client library doesn't ever need to "wait" on a particular answer, and can proceed even if some udp packets are lost. It's up to the user to reschedule queries if they haven't received responses and they need them. Example usage:: client = ConfdClient(...) # includes callback specification req = confd_client.ConfdClientRequest(type=constants.CONFD_REQ_PING) client.SendRequest(req) # then make sure your client calls asyncore.loop() or daemon.Mainloop.Run() # ... wait ... # And your callback will be called by asyncore, when your query gets a # response, or when it expires. You can use the provided ConfdFilterCallback to act as a filter, only passing "newer" answer to your callback, and filtering out outdated ones, or ones confirming what you already got. """ # pylint: disable=E0203 # E0203: Access to member %r before its definition, since we use # objects.py which doesn't explicitly initialise its members import time import random from ganeti import utils from ganeti import constants from ganeti import objects from ganeti import serializer from ganeti import daemon # contains AsyncUDPSocket from ganeti import errors from ganeti import confd from ganeti import ssconf from ganeti import compat from ganeti import netutils from ganeti import pathutils class ConfdAsyncUDPClient(daemon.AsyncUDPSocket): """Confd udp asyncore client This is kept separate from the main ConfdClient to make sure it's easy to implement a non-asyncore based client library. """ def __init__(self, client, family): """Constructor for ConfdAsyncUDPClient @type client: L{ConfdClient} @param client: client library, to pass the datagrams to """ daemon.AsyncUDPSocket.__init__(self, family) self.client = client # this method is overriding a daemon.AsyncUDPSocket method def handle_datagram(self, payload, ip, port): self.client.HandleResponse(payload, ip, port) class _Request(object): """Request status structure. @ivar request: the request data @ivar args: any extra arguments for the callback @ivar expiry: the expiry timestamp of the request @ivar sent: the set of contacted peers @ivar rcvd: the set of peers who replied """ def __init__(self, request, args, expiry, sent): self.request = request self.args = args self.expiry = expiry self.sent = frozenset(sent) self.rcvd = set() class ConfdClient(object): """Send queries to confd, and get back answers. Since the confd model works by querying multiple master candidates, and getting back answers, this is an asynchronous library. It can either work through asyncore or with your own handling. @type _requests: dict @ivar _requests: dictionary indexes by salt, which contains data about the outstanding requests; the values are objects of type L{_Request} """ def __init__(self, hmac_key, peers, callback, port=None, logger=None): """Constructor for ConfdClient @type hmac_key: string @param hmac_key: hmac key to talk to confd @type peers: list @param peers: list of peer nodes @type callback: f(L{ConfdUpcallPayload}) @param callback: function to call when getting answers @type port: integer @param port: confd port (default: use GetDaemonPort) @type logger: logging.Logger @param logger: optional logger for internal conditions """ if not callable(callback): raise errors.ProgrammerError("callback must be callable") self.UpdatePeerList(peers) self._SetPeersAddressFamily() self._hmac_key = hmac_key self._socket = ConfdAsyncUDPClient(self, self._family) self._callback = callback self._confd_port = port self._logger = logger self._requests = {} if self._confd_port is None: self._confd_port = netutils.GetDaemonPort(constants.CONFD) def UpdatePeerList(self, peers): """Update the list of peers @type peers: list @param peers: list of peer nodes """ # we are actually called from init, so: # pylint: disable=W0201 if not isinstance(peers, list): raise errors.ProgrammerError("peers must be a list") # make a copy of peers, since we're going to shuffle the list, later self._peers = list(peers) def _PackRequest(self, request, now=None): """Prepare a request to be sent on the wire. This function puts a proper salt in a confd request, puts the proper salt, and adds the correct magic number. """ if now is None: now = time.time() tstamp = "%d" % now req = serializer.DumpSignedJson(request.ToDict(), self._hmac_key, tstamp) return confd.PackMagic(req) def _UnpackReply(self, payload): in_payload = confd.UnpackMagic(payload) (dict_answer, salt) = serializer.LoadSignedJson(in_payload, self._hmac_key) answer = objects.ConfdReply.FromDict(dict_answer) return answer, salt def ExpireRequests(self): """Delete all the expired requests. """ now = time.time() for rsalt, rq in list(self._requests.items()): if now >= rq.expiry: del self._requests[rsalt] client_reply = ConfdUpcallPayload(salt=rsalt, type=UPCALL_EXPIRE, orig_request=rq.request, extra_args=rq.args, client=self, ) self._callback(client_reply) def SendRequest(self, request, args=None, coverage=0, async_=True): """Send a confd request to some MCs @type request: L{objects.ConfdRequest} @param request: the request to send @type args: tuple @param args: additional callback arguments @type coverage: integer @param coverage: number of remote nodes to contact; if default (0), it will use a reasonable default (L{ganeti.constants.CONFD_DEFAULT_REQ_COVERAGE}), if -1 is passed, it will use the maximum number of peers, otherwise the number passed in will be used @type async_: boolean @param async_: handle the write asynchronously """ if coverage == 0: coverage = min(len(self._peers), constants.CONFD_DEFAULT_REQ_COVERAGE) elif coverage == -1: coverage = len(self._peers) if coverage > len(self._peers): raise errors.ConfdClientError("Not enough MCs known to provide the" " desired coverage") if not request.rsalt: raise errors.ConfdClientError("Missing request rsalt") self.ExpireRequests() if request.rsalt in self._requests: raise errors.ConfdClientError("Duplicate request rsalt") if request.type not in constants.CONFD_REQS: raise errors.ConfdClientError("Invalid request type") random.shuffle(self._peers) targets = self._peers[:coverage] now = time.time() payload = self._PackRequest(request, now=now) for target in targets: try: self._socket.enqueue_send(target, self._confd_port, payload) except errors.UdpDataSizeError: raise errors.ConfdClientError("Request too big") expire_time = now + constants.CONFD_CLIENT_EXPIRE_TIMEOUT self._requests[request.rsalt] = _Request(request, args, expire_time, targets) if not async_: self.FlushSendQueue() def HandleResponse(self, payload, ip, port): """Asynchronous handler for a confd reply Call the relevant callback associated to the current request. """ try: try: answer, salt = self._UnpackReply(payload) except (errors.SignatureError, errors.ConfdMagicError) as err: if self._logger: self._logger.debug("Discarding broken package: %s" % err) return try: rq = self._requests[salt] except KeyError: if self._logger: self._logger.debug("Discarding unknown (expired?) reply: %s" % err) return rq.rcvd.add(ip) client_reply = ConfdUpcallPayload(salt=salt, type=UPCALL_REPLY, server_reply=answer, orig_request=rq.request, server_ip=ip, server_port=port, extra_args=rq.args, client=self, ) self._callback(client_reply) finally: self.ExpireRequests() def FlushSendQueue(self): """Send out all pending requests. Can be used for synchronous client use. """ while self._socket.writable(): self._socket.handle_write() def ReceiveReply(self, timeout=1): """Receive one reply. @type timeout: float @param timeout: how long to wait for the reply @rtype: boolean @return: True if some data has been handled, False otherwise """ return self._socket.process_next_packet(timeout=timeout) @staticmethod def _NeededReplies(peer_cnt): """Compute the minimum safe number of replies for a query. The algorithm is designed to work well for both small and big number of peers: - for less than three, we require all responses - for less than five, we allow one miss - otherwise, half the number plus one This guarantees that we progress monotonically: 1->1, 2->2, 3->2, 4->2, 5->3, 6->3, 7->4, etc. @type peer_cnt: int @param peer_cnt: the number of peers contacted @rtype: int @return: the number of replies which should give a safe coverage """ if peer_cnt < 3: return peer_cnt elif peer_cnt < 5: return peer_cnt - 1 else: return int(peer_cnt / 2) + 1 def WaitForReply(self, salt, timeout=constants.CONFD_CLIENT_EXPIRE_TIMEOUT): """Wait for replies to a given request. This method will wait until either the timeout expires or a minimum number (computed using L{_NeededReplies}) of replies are received for the given salt. It is useful when doing synchronous calls to this library. @param salt: the salt of the request we want responses for @param timeout: the maximum timeout (should be less or equal to L{ganeti.constants.CONFD_CLIENT_EXPIRE_TIMEOUT} @rtype: tuple @return: a tuple of (timed_out, sent_cnt, recv_cnt); if the request is unknown, timed_out will be true and the counters will be zero """ def _CheckResponse(): if salt not in self._requests: # expired? if self._logger: self._logger.debug("Discarding unknown/expired request: %s" % salt) return MISSING rq = self._requests[salt] if len(rq.rcvd) >= expected: # already got all replies return (False, len(rq.sent), len(rq.rcvd)) # else wait, using default timeout self.ReceiveReply() raise utils.RetryAgain() MISSING = (True, 0, 0) if salt not in self._requests: return MISSING # extend the expire time with the current timeout, so that we # don't get the request expired from under us rq = self._requests[salt] rq.expiry += timeout sent = len(rq.sent) expected = self._NeededReplies(sent) try: return utils.Retry(_CheckResponse, 0, timeout) except utils.RetryTimeout: if salt in self._requests: rq = self._requests[salt] return (True, len(rq.sent), len(rq.rcvd)) else: return MISSING def _SetPeersAddressFamily(self): if not self._peers: raise errors.ConfdClientError("Peer list empty") try: peer = self._peers[0] self._family = netutils.IPAddress.GetAddressFamily(peer) for peer in self._peers[1:]: if netutils.IPAddress.GetAddressFamily(peer) != self._family: raise errors.ConfdClientError("Peers must be of same address family") except errors.IPAddressError: raise errors.ConfdClientError("Peer address %s invalid" % peer) # UPCALL_REPLY: server reply upcall # has all ConfdUpcallPayload fields populated UPCALL_REPLY = 1 # UPCALL_EXPIRE: internal library request expire # has only salt, type, orig_request and extra_args UPCALL_EXPIRE = 2 CONFD_UPCALL_TYPES = compat.UniqueFrozenset([ UPCALL_REPLY, UPCALL_EXPIRE, ]) class ConfdUpcallPayload(objects.ConfigObject): """Callback argument for confd replies @type salt: string @ivar salt: salt associated with the query @type type: one of confd.client.CONFD_UPCALL_TYPES @ivar type: upcall type (server reply, expired request, ...) @type orig_request: L{objects.ConfdRequest} @ivar orig_request: original request @type server_reply: L{objects.ConfdReply} @ivar server_reply: server reply @type server_ip: string @ivar server_ip: answering server ip address @type server_port: int @ivar server_port: answering server port @type extra_args: any @ivar extra_args: 'args' argument of the SendRequest function @type client: L{ConfdClient} @ivar client: current confd client instance """ __slots__ = [ "salt", "type", "orig_request", "server_reply", "server_ip", "server_port", "extra_args", "client", ] class ConfdClientRequest(objects.ConfdRequest): """This is the client-side version of ConfdRequest. This version of the class helps creating requests, on the client side, by filling in some default values. """ def __init__(self, **kwargs): objects.ConfdRequest.__init__(self, **kwargs) if not self.rsalt: self.rsalt = utils.NewUUID() if not self.protocol: self.protocol = constants.CONFD_PROTOCOL_VERSION if self.type not in constants.CONFD_REQS: raise errors.ConfdClientError("Invalid request type") class ConfdFilterCallback(object): """Callback that calls another callback, but filters duplicate results. @ivar consistent: a dictionary indexed by salt; for each salt, if all responses ware identical, this will be True; this is the expected state on a healthy cluster; on inconsistent or partitioned clusters, this might be False, if we see answers with the same serial but different contents """ def __init__(self, callback, logger=None): """Constructor for ConfdFilterCallback @type callback: f(L{ConfdUpcallPayload}) @param callback: function to call when getting answers @type logger: logging.Logger @param logger: optional logger for internal conditions """ if not callable(callback): raise errors.ProgrammerError("callback must be callable") self._callback = callback self._logger = logger # answers contains a dict of salt -> answer self._answers = {} self.consistent = {} def _LogFilter(self, salt, new_reply, old_reply): if not self._logger: return if new_reply.serial > old_reply.serial: self._logger.debug("Filtering confirming answer, with newer" " serial for query %s" % salt) elif new_reply.serial == old_reply.serial: if new_reply.answer != old_reply.answer: self._logger.warning("Got incoherent answers for query %s" " (serial: %s)" % (salt, new_reply.serial)) else: self._logger.debug("Filtering confirming answer, with same" " serial for query %s" % salt) else: self._logger.debug("Filtering outdated answer for query %s" " serial: (%d < %d)" % (salt, old_reply.serial, new_reply.serial)) def _HandleExpire(self, up): # if we have no answer we have received none, before the expiration. if up.salt in self._answers: del self._answers[up.salt] if up.salt in self.consistent: del self.consistent[up.salt] def _HandleReply(self, up): """Handle a single confd reply, and decide whether to filter it. @rtype: boolean @return: True if the reply should be filtered, False if it should be passed on to the up-callback """ filter_upcall = False salt = up.salt if salt not in self.consistent: self.consistent[salt] = True if salt not in self._answers: # first answer for a query (don't filter, and record) self._answers[salt] = up.server_reply elif up.server_reply.serial > self._answers[salt].serial: # newer answer (record, and compare contents) old_answer = self._answers[salt] self._answers[salt] = up.server_reply if up.server_reply.answer == old_answer.answer: # same content (filter) (version upgrade was unrelated) filter_upcall = True self._LogFilter(salt, up.server_reply, old_answer) # else: different content, pass up a second answer else: # older or same-version answer (duplicate or outdated, filter) if (up.server_reply.serial == self._answers[salt].serial and up.server_reply.answer != self._answers[salt].answer): self.consistent[salt] = False filter_upcall = True self._LogFilter(salt, up.server_reply, self._answers[salt]) return filter_upcall def __call__(self, up): """Filtering callback @type up: L{ConfdUpcallPayload} @param up: upper callback """ filter_upcall = False if up.type == UPCALL_REPLY: filter_upcall = self._HandleReply(up) elif up.type == UPCALL_EXPIRE: self._HandleExpire(up) if not filter_upcall: self._callback(up) class ConfdCountingCallback(object): """Callback that calls another callback, and counts the answers """ def __init__(self, callback, logger=None): """Constructor for ConfdCountingCallback @type callback: f(L{ConfdUpcallPayload}) @param callback: function to call when getting answers @type logger: logging.Logger @param logger: optional logger for internal conditions """ if not callable(callback): raise errors.ProgrammerError("callback must be callable") self._callback = callback self._logger = logger # answers contains a dict of salt -> count self._answers = {} def RegisterQuery(self, salt): if salt in self._answers: raise errors.ProgrammerError("query already registered") self._answers[salt] = 0 def AllAnswered(self): """Have all the registered queries received at least an answer? """ return compat.all(self._answers.values()) def _HandleExpire(self, up): # if we have no answer we have received none, before the expiration. if up.salt in self._answers: del self._answers[up.salt] def _HandleReply(self, up): """Handle a single confd reply, and decide whether to filter it. @rtype: boolean @return: True if the reply should be filtered, False if it should be passed on to the up-callback """ if up.salt in self._answers: self._answers[up.salt] += 1 def __call__(self, up): """Filtering callback @type up: L{ConfdUpcallPayload} @param up: upper callback """ if up.type == UPCALL_REPLY: self._HandleReply(up) elif up.type == UPCALL_EXPIRE: self._HandleExpire(up) self._callback(up) class StoreResultCallback(object): """Callback that simply stores the most recent answer. @ivar _answers: dict of salt to (have_answer, reply) """ _NO_KEY = (False, None) def __init__(self): """Constructor for StoreResultCallback """ # answers contains a dict of salt -> best result self._answers = {} def GetResponse(self, salt): """Return the best match for a salt """ return self._answers.get(salt, self._NO_KEY) def _HandleExpire(self, up): """Expiration handler. """ if up.salt in self._answers and self._answers[up.salt] == self._NO_KEY: del self._answers[up.salt] def _HandleReply(self, up): """Handle a single confd reply, and decide whether to filter it. """ self._answers[up.salt] = (True, up) def __call__(self, up): """Filtering callback @type up: L{ConfdUpcallPayload} @param up: upper callback """ if up.type == UPCALL_REPLY: self._HandleReply(up) elif up.type == UPCALL_EXPIRE: self._HandleExpire(up) def GetConfdClient(callback): """Return a client configured using the given callback. This is handy to abstract the MC list and HMAC key reading. @attention: This should only be called on nodes which are part of a cluster, since it depends on a valid (ganeti) data directory; for code running outside of a cluster, you need to create the client manually """ ss = ssconf.SimpleStore() mc_file = ss.KeyToFilename(constants.SS_MASTER_CANDIDATES_IPS) mc_list = utils.ReadFile(mc_file).splitlines() hmac_key = utils.ReadFile(pathutils.CONFD_HMAC_KEY) return ConfdClient(hmac_key, mc_list, callback) ganeti-3.1.0~rc2/lib/config/000075500000000000000000000000001476477700300156175ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/config/__init__.py000064400000000000000000003310721476477700300177360ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Configuration management for Ganeti This module provides the interface to the Ganeti cluster configuration. The configuration data is stored on every node but is updated on the master only. After each update, the master distributes the data to the other nodes. Currently, the data storage format is JSON. YAML was slow and consuming too much memory. """ # TODO: Break up this file into multiple chunks - Wconfd RPC calls, local config # manipulations, grouped by object they operate on (cluster/instance/disk) # pylint: disable=C0302 # pylint: disable=R0904 # R0904: Too many public methods import copy import os import random import logging import time import threading import itertools from ganeti.config.temporary_reservations import TemporaryReservationManager from ganeti.config.utils import ConfigSync, ConfigManager from ganeti.config.verify import (VerifyType, VerifyNic, VerifyIpolicy, ValidateConfig) from ganeti import errors from ganeti import utils from ganeti import constants import ganeti.wconfd as wc from ganeti import objects from ganeti import serializer from ganeti import uidpool from ganeti import netutils from ganeti import runtime from ganeti import pathutils from ganeti import network def GetWConfdContext(ec_id, livelock): """Prepare a context for communication with WConfd. WConfd needs to know the identity of each caller to properly manage locks and detect job death. This helper function prepares the identity object given a job ID (optional) and a livelock file. @type ec_id: int, or None @param ec_id: the job ID or None, if the caller isn't a job @type livelock: L{ganeti.utils.livelock.LiveLock} @param livelock: a livelock object holding the lockfile needed for WConfd @return: the WConfd context """ if ec_id is None: return (threading.current_thread().name, livelock.GetPath(), os.getpid()) else: return (ec_id, livelock.GetPath(), os.getpid()) def GetConfig(ec_id, livelock, **kwargs): """A utility function for constructing instances of ConfigWriter. It prepares a WConfd context and uses it to create a ConfigWriter instance. @type ec_id: int, or None @param ec_id: the job ID or None, if the caller isn't a job @type livelock: L{ganeti.utils.livelock.LiveLock} @param livelock: a livelock object holding the lockfile needed for WConfd @type kwargs: dict @param kwargs: Any additional arguments for the ConfigWriter constructor @rtype: L{ConfigWriter} @return: the ConfigWriter context """ kwargs['wconfdcontext'] = GetWConfdContext(ec_id, livelock) # if the config is to be opened in the accept_foreign mode, we should # also tell the RPC client not to check for the master node accept_foreign = kwargs.get('accept_foreign', False) kwargs['wconfd'] = wc.Client(allow_non_master=accept_foreign) return ConfigWriter(**kwargs) # job id used for resource management at config upgrade time _UPGRADE_CONFIG_JID = "jid-cfg-upgrade" def _MatchNameComponentIgnoreCase(short_name, names): """Wrapper around L{utils.text.MatchNameComponent}. """ return utils.MatchNameComponent(short_name, names, case_sensitive=False) def _CheckInstanceDiskIvNames(disks): """Checks if instance's disks' C{iv_name} attributes are in order. @type disks: list of L{objects.Disk} @param disks: List of disks @rtype: list of tuples; (int, string, string) @return: List of wrongly named disks, each tuple contains disk index, expected and actual name """ result = [] for (idx, disk) in enumerate(disks): exp_iv_name = "disk/%s" % idx if disk.iv_name != exp_iv_name: result.append((idx, exp_iv_name, disk.iv_name)) return result class ConfigWriter(object): """The interface to the cluster configuration. WARNING: The class is no longer thread-safe! Each thread must construct a separate instance. @ivar _all_rms: a list of all temporary reservation managers Currently the class fulfills 3 main functions: 1. lock the configuration for access (monitor) 2. reload and write the config if necessary (bridge) 3. provide convenient access methods to config data (facade) """ def __init__(self, cfg_file=None, offline=False, _getents=runtime.GetEnts, accept_foreign=False, wconfdcontext=None, wconfd=None): self.write_count = 0 self._config_data = None self._SetConfigData(None) self._offline = offline if cfg_file is None: self._cfg_file = pathutils.CLUSTER_CONF_FILE else: self._cfg_file = cfg_file self._getents = _getents self._temporary_ids = TemporaryReservationManager() self._all_rms = [self._temporary_ids] # Note: in order to prevent errors when resolving our name later, # we compute it here once and reuse it; it's # better to raise an error before starting to modify the config # file than after it was modified self._my_hostname = netutils.Hostname.GetSysName() self._cfg_id = None self._wconfdcontext = wconfdcontext self._wconfd = wconfd self._accept_foreign = accept_foreign self._lock_count = 0 self._lock_current_shared = None self._lock_forced = False def _ConfigData(self): return self._config_data def OutDate(self): self._config_data = None def _SetConfigData(self, cfg): self._config_data = cfg def _GetWConfdContext(self): return self._wconfdcontext # this method needs to be static, so that we can call it on the class @staticmethod def IsCluster(): """Check if the cluster is configured. """ return os.path.exists(pathutils.CLUSTER_CONF_FILE) def _UnlockedGetNdParams(self, node): nodegroup = self._UnlockedGetNodeGroup(node.group) return self._ConfigData().cluster.FillND(node, nodegroup) @ConfigSync(shared=1) def GetNdParams(self, node): """Get the node params populated with cluster defaults. @type node: L{objects.Node} @param node: The node we want to know the params for @return: A dict with the filled in node params """ return self._UnlockedGetNdParams(node) @ConfigSync(shared=1) def GetNdGroupParams(self, nodegroup): """Get the node groups params populated with cluster defaults. @type nodegroup: L{objects.NodeGroup} @param nodegroup: The node group we want to know the params for @return: A dict with the filled in node group params """ return self._UnlockedGetNdGroupParams(nodegroup) def _UnlockedGetNdGroupParams(self, group): """Get the ndparams of the group. @type group: L{objects.NodeGroup} @param group: The group we want to know the params for @rtype: dict of str to int @return: A dict with the filled in node group params """ return self._ConfigData().cluster.FillNDGroup(group) @ConfigSync(shared=1) def GetGroupSshPorts(self): """Get a map of group UUIDs to SSH ports. @rtype: dict of str to int @return: a dict mapping the UUIDs to the SSH ports """ port_map = {} for uuid, group in self._config_data.nodegroups.items(): ndparams = self._UnlockedGetNdGroupParams(group) port = ndparams.get(constants.ND_SSH_PORT) port_map[uuid] = port return port_map @ConfigSync(shared=1) def GetInstanceDiskParams(self, instance): """Get the disk params populated with inherit chain. @type instance: L{objects.Instance} @param instance: The instance we want to know the params for @return: A dict with the filled in disk params """ node = self._UnlockedGetNodeInfo(instance.primary_node) nodegroup = self._UnlockedGetNodeGroup(node.group) return self._UnlockedGetGroupDiskParams(nodegroup) def _UnlockedGetInstanceDisks(self, inst_uuid): """Return the disks' info for the given instance @type inst_uuid: string @param inst_uuid: The UUID of the instance we want to know the disks for @rtype: List of L{objects.Disk} @return: A list with all the disks' info """ instance = self._UnlockedGetInstanceInfo(inst_uuid) if instance is None: raise errors.ConfigurationError("Unknown instance '%s'" % inst_uuid) return [self._UnlockedGetDiskInfo(disk_uuid) for disk_uuid in instance.disks] @ConfigSync(shared=1) def GetInstanceDisks(self, inst_uuid): """Return the disks' info for the given instance This is a simple wrapper over L{_UnlockedGetInstanceDisks}. """ return self._UnlockedGetInstanceDisks(inst_uuid) def AddInstanceDisk(self, inst_uuid, disk, idx=None, replace=False): """Add a disk to the config and attach it to instance.""" if not isinstance(disk, objects.Disk): raise errors.ProgrammerError("Invalid type passed to AddInstanceDisk") disk.UpgradeConfig() utils.SimpleRetry(True, self._wconfd.AddInstanceDisk, 0.1, 30, args=[inst_uuid, disk.ToDict(), idx, replace]) self.OutDate() def AttachInstanceDisk(self, inst_uuid, disk_uuid, idx=None): """Attach an existing disk to an instance.""" utils.SimpleRetry(True, self._wconfd.AttachInstanceDisk, 0.1, 30, args=[inst_uuid, disk_uuid, idx]) self.OutDate() def _UnlockedRemoveDisk(self, disk_uuid): """Remove the disk from the configuration. @type disk_uuid: string @param disk_uuid: The UUID of the disk object """ if disk_uuid not in self._ConfigData().disks: raise errors.ConfigurationError("Disk %s doesn't exist" % disk_uuid) # Disk must not be attached anywhere for inst in self._ConfigData().instances.values(): if disk_uuid in inst.disks: raise errors.ReservationError("Cannot remove disk %s. Disk is" " attached to instance %s" % (disk_uuid, inst.name)) # Remove disk from config file del self._ConfigData().disks[disk_uuid] self._ConfigData().cluster.serial_no += 1 def RemoveInstanceDisk(self, inst_uuid, disk_uuid): """Detach a disk from an instance and remove it from the config.""" utils.SimpleRetry(True, self._wconfd.RemoveInstanceDisk, 0.1, 30, args=[inst_uuid, disk_uuid]) self.OutDate() def DetachInstanceDisk(self, inst_uuid, disk_uuid): """Detach a disk from an instance.""" utils.SimpleRetry(True, self._wconfd.DetachInstanceDisk, 0.1, 30, args=[inst_uuid, disk_uuid]) self.OutDate() def _UnlockedGetDiskInfo(self, disk_uuid): """Returns information about a disk. It takes the information from the configuration file. @param disk_uuid: UUID of the disk @rtype: L{objects.Disk} @return: the disk object """ if disk_uuid not in self._ConfigData().disks: return None return self._ConfigData().disks[disk_uuid] @ConfigSync(shared=1) def GetDiskInfo(self, disk_uuid): """Returns information about a disk. This is a simple wrapper over L{_UnlockedGetDiskInfo}. """ return self._UnlockedGetDiskInfo(disk_uuid) def _UnlockedGetDiskInfoByName(self, disk_name): """Return information about a named disk. Return disk information from the configuration file, searching with the name of the disk. @param disk_name: Name of the disk @rtype: L{objects.Disk} @return: the disk object """ disk = None count = 0 for d in self._ConfigData().disks.values(): if d.name == disk_name: count += 1 disk = d if count > 1: raise errors.ConfigurationError("There are %s disks with this name: %s" % (count, disk_name)) return disk @ConfigSync(shared=1) def GetDiskInfoByName(self, disk_name): """Return information about a named disk. This is a simple wrapper over L{_UnlockedGetDiskInfoByName}. """ return self._UnlockedGetDiskInfoByName(disk_name) def _UnlockedGetDiskList(self): """Get the list of disks. @return: array of disks, ex. ['disk2-uuid', 'disk1-uuid'] """ return list(self._ConfigData().disks) @ConfigSync(shared=1) def GetAllDisksInfo(self): """Get the configuration of all disks. This is a simple wrapper over L{_UnlockedGetAllDisksInfo}. """ return self._UnlockedGetAllDisksInfo() def _UnlockedGetAllDisksInfo(self): """Get the configuration of all disks. @rtype: dict @return: dict of (disk, disk_info), where disk_info is what would GetDiskInfo return for the node """ my_dict = dict([(disk_uuid, self._UnlockedGetDiskInfo(disk_uuid)) for disk_uuid in self._UnlockedGetDiskList()]) return my_dict def _AllInstanceNodes(self, inst_uuid): """Compute the set of all disk-related nodes for an instance. This abstracts away some work from '_UnlockedGetInstanceNodes' and '_UnlockedGetInstanceSecondaryNodes'. @type inst_uuid: string @param inst_uuid: The UUID of the instance we want to get nodes for @rtype: set of strings @return: A set of names for all the nodes of the instance """ instance = self._UnlockedGetInstanceInfo(inst_uuid) if instance is None: raise errors.ConfigurationError("Unknown instance '%s'" % inst_uuid) instance_disks = self._UnlockedGetInstanceDisks(inst_uuid) all_nodes = [] for disk in instance_disks: all_nodes.extend(disk.all_nodes) return (set(all_nodes), instance) def _UnlockedGetInstanceNodes(self, inst_uuid): """Get all disk-related nodes for an instance. For non-DRBD instances, this will contain only the instance's primary node, whereas for DRBD instances, it will contain both the primary and the secondaries. @type inst_uuid: string @param inst_uuid: The UUID of the instance we want to get nodes for @rtype: list of strings @return: A list of names for all the nodes of the instance """ (all_nodes, instance) = self._AllInstanceNodes(inst_uuid) # ensure that primary node is always the first all_nodes.discard(instance.primary_node) return (instance.primary_node, ) + tuple(all_nodes) @ConfigSync(shared=1) def GetInstanceNodes(self, inst_uuid): """Get all disk-related nodes for an instance. This is just a wrapper over L{_UnlockedGetInstanceNodes} """ return self._UnlockedGetInstanceNodes(inst_uuid) def _UnlockedGetInstanceSecondaryNodes(self, inst_uuid): """Get the list of secondary nodes. @type inst_uuid: string @param inst_uuid: The UUID of the instance we want to get nodes for @rtype: list of strings @return: A tuple of names for all the secondary nodes of the instance """ (all_nodes, instance) = self._AllInstanceNodes(inst_uuid) all_nodes.discard(instance.primary_node) return tuple(all_nodes) @ConfigSync(shared=1) def GetInstanceSecondaryNodes(self, inst_uuid): """Get the list of secondary nodes. This is a simple wrapper over L{_UnlockedGetInstanceSecondaryNodes}. """ return self._UnlockedGetInstanceSecondaryNodes(inst_uuid) def _UnlockedGetInstanceLVsByNode(self, inst_uuid, lvmap=None): """Provide a mapping of node to LVs a given instance owns. @type inst_uuid: string @param inst_uuid: The UUID of the instance we want to compute the LVsByNode for @type lvmap: dict @param lvmap: Optional dictionary to receive the 'node' : ['lv', ...] data. @rtype: dict or None @return: None if lvmap arg is given, otherwise, a dictionary of the form { 'node_uuid' : ['volume1', 'volume2', ...], ... }; volumeN is of the form "vg_name/lv_name", compatible with GetVolumeList() """ def _MapLVsByNode(lvmap, devices, node_uuid): """Recursive helper function.""" if not node_uuid in lvmap: lvmap[node_uuid] = [] for dev in devices: if dev.dev_type == constants.DT_PLAIN: if not dev.forthcoming: lvmap[node_uuid].append(dev.logical_id[0] + "/" + dev.logical_id[1]) elif dev.dev_type in constants.DTS_DRBD: if dev.children: _MapLVsByNode(lvmap, dev.children, dev.logical_id[0]) _MapLVsByNode(lvmap, dev.children, dev.logical_id[1]) elif dev.children: _MapLVsByNode(lvmap, dev.children, node_uuid) instance = self._UnlockedGetInstanceInfo(inst_uuid) if instance is None: raise errors.ConfigurationError("Unknown instance '%s'" % inst_uuid) if lvmap is None: lvmap = {} ret = lvmap else: ret = None _MapLVsByNode(lvmap, self._UnlockedGetInstanceDisks(instance.uuid), instance.primary_node) return ret @ConfigSync(shared=1) def GetInstanceLVsByNode(self, inst_uuid, lvmap=None): """Provide a mapping of node to LVs a given instance owns. This is a simple wrapper over L{_UnlockedGetInstanceLVsByNode} """ return self._UnlockedGetInstanceLVsByNode(inst_uuid, lvmap=lvmap) @ConfigSync(shared=1) def GetGroupDiskParams(self, group): """Get the disk params populated with inherit chain. @type group: L{objects.NodeGroup} @param group: The group we want to know the params for @return: A dict with the filled in disk params """ return self._UnlockedGetGroupDiskParams(group) def _UnlockedGetGroupDiskParams(self, group): """Get the disk params populated with inherit chain down to node-group. @type group: L{objects.NodeGroup} @param group: The group we want to know the params for @return: A dict with the filled in disk params """ data = self._ConfigData().cluster.SimpleFillDP(group.diskparams) assert isinstance(data, dict), "Not a dictionary: " + str(data) return data @ConfigSync(shared=1) def GetPotentialMasterCandidates(self): """Gets the list of node names of potential master candidates. @rtype: list of str @return: list of node names of potential master candidates """ # FIXME: Note that currently potential master candidates are nodes # but this definition will be extended once RAPI-unmodifiable # parameters are introduced. nodes = self._UnlockedGetAllNodesInfo() return [node_info.name for node_info in nodes.values()] def GenerateMAC(self, net_uuid, _ec_id): """Generate a MAC for an instance. This should check the current instances for duplicates. """ return self._wconfd.GenerateMAC(self._GetWConfdContext(), net_uuid) def ReserveMAC(self, mac, _ec_id): """Reserve a MAC for an instance. This only checks instances managed by this cluster, it does not check for potential collisions elsewhere. """ self._wconfd.ReserveMAC(self._GetWConfdContext(), mac) @ConfigSync(shared=1) def CommitTemporaryIps(self, _ec_id): """Tell WConfD to commit all temporary ids""" self._wconfd.CommitTemporaryIps(self._GetWConfdContext()) def ReleaseIp(self, net_uuid, address, _ec_id): """Give a specific IP address back to an IP pool. The IP address is returned to the IP pool and marked as reserved. """ if net_uuid: if self._offline: raise errors.ProgrammerError("Can't call ReleaseIp in offline mode") self._wconfd.ReleaseIp(self._GetWConfdContext(), net_uuid, address) def GenerateIp(self, net_uuid, _ec_id): """Find a free IPv4 address for an instance. """ if self._offline: raise errors.ProgrammerError("Can't call GenerateIp in offline mode") return self._wconfd.GenerateIp(self._GetWConfdContext(), net_uuid) def ReserveIp(self, net_uuid, address, _ec_id, check=True): """Reserve a given IPv4 address for use by an instance. """ if self._offline: raise errors.ProgrammerError("Can't call ReserveIp in offline mode") return self._wconfd.ReserveIp(self._GetWConfdContext(), net_uuid, address, check) def ReserveLV(self, lv_name, _ec_id): """Reserve an VG/LV pair for an instance. @type lv_name: string @param lv_name: the logical volume name to reserve """ return self._wconfd.ReserveLV(self._GetWConfdContext(), lv_name) def GenerateDRBDSecret(self, _ec_id): """Generate a DRBD secret. This checks the current disks for duplicates. """ return self._wconfd.GenerateDRBDSecret(self._GetWConfdContext()) # FIXME: After _AllIDs is removed, move it to config_mock.py def _AllLVs(self): """Compute the list of all LVs. """ lvnames = set() for instance in self._ConfigData().instances.values(): node_data = self._UnlockedGetInstanceLVsByNode(instance.uuid) for lv_list in node_data.values(): lvnames.update(lv_list) return lvnames def _AllNICs(self): """Compute the list of all NICs. """ nics = [] for instance in self._ConfigData().instances.values(): nics.extend(instance.nics) return nics def _AllIDs(self, include_temporary): """Compute the list of all UUIDs and names we have. @type include_temporary: boolean @param include_temporary: whether to include the _temporary_ids set @rtype: set @return: a set of IDs """ existing = set() if include_temporary: existing.update(self._temporary_ids.GetReserved()) existing.update(self._AllLVs()) existing.update(self._ConfigData().instances) existing.update(self._ConfigData().nodes) existing.update([i.uuid for i in self._AllUUIDObjects() if i.uuid]) return existing def _GenerateUniqueID(self, ec_id): """Generate an unique UUID. This checks the current node, instances and disk names for duplicates. @rtype: string @return: the unique id """ existing = self._AllIDs(include_temporary=False) return self._temporary_ids.Generate(existing, utils.NewUUID, ec_id) @ConfigSync(shared=1) def GenerateUniqueID(self, ec_id): """Generate an unique ID. This is just a wrapper over the unlocked version. @type ec_id: string @param ec_id: unique id for the job to reserve the id to """ return self._GenerateUniqueID(ec_id) def _AllMACs(self): """Return all MACs present in the config. @rtype: list @return: the list of all MACs """ result = [] for instance in self._ConfigData().instances.values(): for nic in instance.nics: result.append(nic.mac) return result def _AllDRBDSecrets(self): """Return all DRBD secrets present in the config. @rtype: list @return: the list of all DRBD secrets """ def helper(disk, result): """Recursively gather secrets from this disk.""" if disk.dev_type == constants.DT_DRBD8: result.append(disk.logical_id[5]) if disk.children: for child in disk.children: helper(child, result) result = [] for disk in self._ConfigData().disks.values(): helper(disk, result) return result @staticmethod def _VerifyDisks(data, result): """Per-disk verification checks Extends L{result} with diagnostic information about the disks. @type data: see L{_ConfigData} @param data: configuration data @type result: list of strings @param result: list containing diagnostic messages """ for disk_uuid in data.disks: disk = data.disks[disk_uuid] result.extend(["disk %s error: %s" % (disk.uuid, msg) for msg in disk.Verify()]) if disk.uuid != disk_uuid: result.append("disk '%s' is indexed by wrong UUID '%s'" % (disk.name, disk_uuid)) def _UnlockedVerifyConfig(self): """Verify function. @rtype: list @return: a list of error messages; a non-empty list signifies configuration errors """ # pylint: disable=R0914 result = [] seen_macs = [] ports = {} data = self._ConfigData() cluster = data.cluster # First call WConfd to perform its checks, if we're not offline if not self._offline: try: self._wconfd.VerifyConfig() except errors.ConfigVerifyError as err: try: for msg in err.args[1]: result.append(msg) except IndexError: pass # check cluster parameters VerifyType("cluster", "beparams", cluster.SimpleFillBE({}), constants.BES_PARAMETER_TYPES, result.append) VerifyType("cluster", "nicparams", cluster.SimpleFillNIC({}), constants.NICS_PARAMETER_TYPES, result.append) VerifyNic("cluster", cluster.SimpleFillNIC({}), result.append) VerifyType("cluster", "ndparams", cluster.SimpleFillND({}), constants.NDS_PARAMETER_TYPES, result.append) VerifyIpolicy("cluster", cluster.ipolicy, True, result.append) for disk_template in cluster.diskparams: if disk_template not in constants.DTS_HAVE_ACCESS: continue access = cluster.diskparams[disk_template].get(constants.LDP_ACCESS, constants.DISK_KERNELSPACE) if access not in constants.DISK_VALID_ACCESS_MODES: result.append( "Invalid value of '%s:%s': '%s' (expected one of %s)" % ( disk_template, constants.LDP_ACCESS, access, utils.CommaJoin(constants.DISK_VALID_ACCESS_MODES) ) ) self._VerifyDisks(data, result) # per-instance checks for instance_uuid in data.instances: instance = data.instances[instance_uuid] if instance.uuid != instance_uuid: result.append("instance '%s' is indexed by wrong UUID '%s'" % (instance.name, instance_uuid)) if instance.primary_node not in data.nodes: result.append("instance '%s' has invalid primary node '%s'" % (instance.name, instance.primary_node)) for snode in self._UnlockedGetInstanceSecondaryNodes(instance.uuid): if snode not in data.nodes: result.append("instance '%s' has invalid secondary node '%s'" % (instance.name, snode)) for idx, nic in enumerate(instance.nics): if nic.mac in seen_macs: result.append("instance '%s' has NIC %d mac %s duplicate" % (instance.name, idx, nic.mac)) else: seen_macs.append(nic.mac) if nic.nicparams: filled = cluster.SimpleFillNIC(nic.nicparams) owner = "instance %s nic %d" % (instance.name, idx) VerifyType(owner, "nicparams", filled, constants.NICS_PARAMETER_TYPES, result.append) VerifyNic(owner, filled, result.append) # parameter checks if instance.beparams: VerifyType("instance %s" % instance.name, "beparams", cluster.FillBE(instance), constants.BES_PARAMETER_TYPES, result.append) # check that disks exists for disk_uuid in instance.disks: if disk_uuid not in data.disks: result.append("Instance '%s' has invalid disk '%s'" % (instance.name, disk_uuid)) instance_disks = self._UnlockedGetInstanceDisks(instance.uuid) # gather the drbd ports for duplicate checks for (idx, dsk) in enumerate(instance_disks): if dsk.dev_type in constants.DTS_DRBD: tcp_port = dsk.logical_id[2] if tcp_port not in ports: ports[tcp_port] = [] ports[tcp_port].append((instance.name, "drbd disk %s" % idx)) # gather network port reservation net_port = getattr(instance, "network_port", None) if net_port is not None: if net_port not in ports: ports[net_port] = [] ports[net_port].append((instance.name, "network port")) wrong_names = _CheckInstanceDiskIvNames(instance_disks) if wrong_names: tmp = "; ".join(("name of disk %s should be '%s', but is '%s'" % (idx, exp_name, actual_name)) for (idx, exp_name, actual_name) in wrong_names) result.append("Instance '%s' has wrongly named disks: %s" % (instance.name, tmp)) # cluster-wide pool of free ports for free_port in cluster.tcpudp_port_pool: if free_port not in ports: ports[free_port] = [] ports[free_port].append(("cluster", "port marked as free")) # compute tcp/udp duplicate ports keys = list(ports) keys.sort() for pnum in keys: pdata = ports[pnum] if len(pdata) > 1: txt = utils.CommaJoin(["%s/%s" % val for val in pdata]) result.append("tcp/udp port %s has duplicates: %s" % (pnum, txt)) # highest used tcp port check if keys: if keys[-1] > cluster.highest_used_port: result.append("Highest used port mismatch, saved %s, computed %s" % (cluster.highest_used_port, keys[-1])) if not data.nodes[cluster.master_node].master_candidate: result.append("Master node is not a master candidate") # master candidate checks mc_now, mc_max, _ = self._UnlockedGetMasterCandidateStats() if mc_now < mc_max: result.append("Not enough master candidates: actual %d, target %d" % (mc_now, mc_max)) # node checks for node_uuid, node in data.nodes.items(): if node.uuid != node_uuid: result.append("Node '%s' is indexed by wrong UUID '%s'" % (node.name, node_uuid)) if [node.master_candidate, node.drained, node.offline].count(True) > 1: result.append("Node %s state is invalid: master_candidate=%s," " drain=%s, offline=%s" % (node.name, node.master_candidate, node.drained, node.offline)) if node.group not in data.nodegroups: result.append("Node '%s' has invalid group '%s'" % (node.name, node.group)) else: VerifyType("node %s" % node.name, "ndparams", cluster.FillND(node, data.nodegroups[node.group]), constants.NDS_PARAMETER_TYPES, result.append) used_globals = constants.NDC_GLOBALS.intersection(node.ndparams) if used_globals: result.append("Node '%s' has some global parameters set: %s" % (node.name, utils.CommaJoin(used_globals))) # nodegroups checks nodegroups_names = set() for nodegroup_uuid in data.nodegroups: nodegroup = data.nodegroups[nodegroup_uuid] if nodegroup.uuid != nodegroup_uuid: result.append("node group '%s' (uuid: '%s') indexed by wrong uuid '%s'" % (nodegroup.name, nodegroup.uuid, nodegroup_uuid)) if utils.UUID_RE.match(nodegroup.name.lower()): result.append("node group '%s' (uuid: '%s') has uuid-like name" % (nodegroup.name, nodegroup.uuid)) if nodegroup.name in nodegroups_names: result.append("duplicate node group name '%s'" % nodegroup.name) else: nodegroups_names.add(nodegroup.name) group_name = "group %s" % nodegroup.name VerifyIpolicy(group_name, cluster.SimpleFillIPolicy(nodegroup.ipolicy), False, result.append) if nodegroup.ndparams: VerifyType(group_name, "ndparams", cluster.SimpleFillND(nodegroup.ndparams), constants.NDS_PARAMETER_TYPES, result.append) # drbd minors check # FIXME: The check for DRBD map needs to be implemented in WConfd # IP checks default_nicparams = cluster.nicparams[constants.PP_DEFAULT] ips = {} def _AddIpAddress(ip, name): ips.setdefault(ip, []).append(name) _AddIpAddress(cluster.master_ip, "cluster_ip") for node in data.nodes.values(): _AddIpAddress(node.primary_ip, "node:%s/primary" % node.name) if node.secondary_ip != node.primary_ip: _AddIpAddress(node.secondary_ip, "node:%s/secondary" % node.name) for instance in data.instances.values(): for idx, nic in enumerate(instance.nics): if nic.ip is None: continue nicparams = objects.FillDict(default_nicparams, nic.nicparams) nic_mode = nicparams[constants.NIC_MODE] nic_link = nicparams[constants.NIC_LINK] if nic_mode == constants.NIC_MODE_BRIDGED: link = "bridge:%s" % nic_link elif nic_mode == constants.NIC_MODE_ROUTED: link = "route:%s" % nic_link elif nic_mode == constants.NIC_MODE_OVS: link = "ovs:%s" % nic_link else: raise errors.ProgrammerError("NIC mode '%s' not handled" % nic_mode) _AddIpAddress("%s/%s/%s" % (link, nic.ip, nic.network), "instance:%s/nic:%d" % (instance.name, idx)) for ip, owners in ips.items(): if len(owners) > 1: result.append("IP address %s is used by multiple owners: %s" % (ip, utils.CommaJoin(owners))) return result @ConfigSync(shared=1) def VerifyConfigAndLog(self, feedback_fn=None): """A simple wrapper around L{_UnlockedVerifyConfigAndLog}""" return self._UnlockedVerifyConfigAndLog(feedback_fn=feedback_fn) def _UnlockedVerifyConfigAndLog(self, feedback_fn=None): """Verify the configuration and log any errors. The errors get logged as critical errors and also to the feedback function, if given. @param feedback_fn: Callable feedback function @rtype: list @return: a list of error messages; a non-empty list signifies configuration errors """ assert feedback_fn is None or callable(feedback_fn) # Warn on config errors, but don't abort the save - the # configuration has already been modified, and we can't revert; # the best we can do is to warn the user and save as is, leaving # recovery to the user config_errors = self._UnlockedVerifyConfig() if config_errors: errmsg = ("Configuration data is not consistent: %s" % (utils.CommaJoin(config_errors))) logging.critical(errmsg) if feedback_fn: feedback_fn(errmsg) return config_errors @ConfigSync(shared=1) def VerifyConfig(self): """Verify function. This is just a wrapper over L{_UnlockedVerifyConfig}. @rtype: list @return: a list of error messages; a non-empty list signifies configuration errors """ return self._UnlockedVerifyConfig() def AddTcpUdpPort(self, port): """Adds a new port to the available port pool.""" utils.SimpleRetry(True, self._wconfd.AddTcpUdpPort, 0.1, 30, args=[port]) self.OutDate() @ConfigSync(shared=1) def GetPortList(self): """Returns a copy of the current port list. """ return self._ConfigData().cluster.tcpudp_port_pool.copy() def AllocatePort(self): """Allocate a port.""" def WithRetry(): port = self._wconfd.AllocatePort() self.OutDate() if port is None: raise utils.RetryAgain() else: return port return utils.Retry(WithRetry, 0.1, 30) @ConfigSync(shared=1) def ComputeDRBDMap(self): """Compute the used DRBD minor/nodes. This is just a wrapper over a call to WConfd. @return: dictionary of node_uuid: dict of minor: instance_uuid; the returned dict will have all the nodes in it (even if with an empty list). """ if self._offline: raise errors.ProgrammerError("Can't call ComputeDRBDMap in offline mode") else: return dict((k, dict(v)) for (k, v) in self._wconfd.ComputeDRBDMap()) def AllocateDRBDMinor(self, node_uuids, disk_uuid): """Allocate a drbd minor. This is just a wrapper over a call to WConfd. The free minor will be automatically computed from the existing devices. A node can not be given multiple times. The result is the list of minors, in the same order as the passed nodes. @type node_uuids: list of strings @param node_uuids: the nodes in which we allocate minors @type disk_uuid: string @param disk_uuid: the disk for which we allocate minors @rtype: list of ints @return: A list of minors in the same order as the passed nodes """ assert isinstance(disk_uuid, str), \ "Invalid argument '%s' passed to AllocateDRBDMinor" % disk_uuid if self._offline: raise errors.ProgrammerError("Can't call AllocateDRBDMinor" " in offline mode") result = self._wconfd.AllocateDRBDMinor(disk_uuid, node_uuids) logging.debug("Request to allocate drbd minors, input: %s, returning %s", node_uuids, result) return result def ReleaseDRBDMinors(self, disk_uuid): """Release temporary drbd minors allocated for a given disk. This is just a wrapper over a call to WConfd. @type disk_uuid: string @param disk_uuid: the disk for which temporary minors should be released """ assert isinstance(disk_uuid, str), \ "Invalid argument passed to ReleaseDRBDMinors" # in offline mode we allow the calls to release DRBD minors, # because then nothing can be allocated anyway; # this is useful for testing if not self._offline: self._wconfd.ReleaseDRBDMinors(disk_uuid) @ConfigSync(shared=1) def GetInstanceDiskTemplate(self, inst_uuid): """Return the disk template of an instance. This corresponds to the currently attached disks. If no disks are attached, it is L{constants.DT_DISKLESS}, if homogeneous disk types are attached, that type is returned, if that isn't the case, L{constants.DT_MIXED} is returned. @type inst_uuid: str @param inst_uuid: The uuid of the instance. """ return utils.GetDiskTemplate(self._UnlockedGetInstanceDisks(inst_uuid)) @ConfigSync(shared=1) def GetConfigVersion(self): """Get the configuration version. @return: Config version """ return self._ConfigData().version @ConfigSync(shared=1) def GetClusterName(self): """Get cluster name. @return: Cluster name """ return self._ConfigData().cluster.cluster_name @ConfigSync(shared=1) def GetMasterNode(self): """Get the UUID of the master node for this cluster. @return: Master node UUID """ return self._ConfigData().cluster.master_node @ConfigSync(shared=1) def GetMasterNodeName(self): """Get the hostname of the master node for this cluster. @return: Master node hostname """ return self._UnlockedGetNodeName(self._ConfigData().cluster.master_node) @ConfigSync(shared=1) def GetMasterNodeInfo(self): """Get the master node information for this cluster. @rtype: objects.Node @return: Master node L{objects.Node} object """ return self._UnlockedGetNodeInfo(self._ConfigData().cluster.master_node) @ConfigSync(shared=1) def GetMasterIP(self): """Get the IP of the master node for this cluster. @return: Master IP """ return self._ConfigData().cluster.master_ip @ConfigSync(shared=1) def GetMasterNetdev(self): """Get the master network device for this cluster. """ return self._ConfigData().cluster.master_netdev @ConfigSync(shared=1) def GetMasterNetmask(self): """Get the netmask of the master node for this cluster. """ return self._ConfigData().cluster.master_netmask @ConfigSync(shared=1) def GetUseExternalMipScript(self): """Get flag representing whether to use the external master IP setup script. """ return self._ConfigData().cluster.use_external_mip_script @ConfigSync(shared=1) def GetFileStorageDir(self): """Get the file storage dir for this cluster. """ return self._ConfigData().cluster.file_storage_dir @ConfigSync(shared=1) def GetSharedFileStorageDir(self): """Get the shared file storage dir for this cluster. """ return self._ConfigData().cluster.shared_file_storage_dir @ConfigSync(shared=1) def GetGlusterStorageDir(self): """Get the Gluster storage dir for this cluster. """ return self._ConfigData().cluster.gluster_storage_dir @ConfigSync(shared=1) def GetHypervisorType(self): """Get the hypervisor type for this cluster. """ return self._ConfigData().cluster.enabled_hypervisors[0] @ConfigSync(shared=1) def GetRsaHostKey(self): """Return the rsa hostkey from the config. @rtype: string @return: the rsa hostkey """ return self._ConfigData().cluster.rsahostkeypub @ConfigSync(shared=1) def GetDsaHostKey(self): """Return the dsa hostkey from the config. @rtype: string @return: the dsa hostkey """ return self._ConfigData().cluster.dsahostkeypub @ConfigSync(shared=1) def GetDefaultIAllocator(self): """Get the default instance allocator for this cluster. """ return self._ConfigData().cluster.default_iallocator @ConfigSync(shared=1) def GetDefaultIAllocatorParameters(self): """Get the default instance allocator parameters for this cluster. @rtype: dict @return: dict of iallocator parameters """ return self._ConfigData().cluster.default_iallocator_params @ConfigSync(shared=1) def GetPrimaryIPFamily(self): """Get cluster primary ip family. @return: primary ip family """ return self._ConfigData().cluster.primary_ip_family @ConfigSync(shared=1) def GetMasterNetworkParameters(self): """Get network parameters of the master node. @rtype: L{object.MasterNetworkParameters} @return: network parameters of the master node """ cluster = self._ConfigData().cluster result = objects.MasterNetworkParameters( uuid=cluster.master_node, ip=cluster.master_ip, netmask=cluster.master_netmask, netdev=cluster.master_netdev, ip_family=cluster.primary_ip_family) return result @ConfigSync(shared=1) def GetInstallImage(self): """Get the install image location @rtype: string @return: location of the install image """ return self._ConfigData().cluster.install_image @ConfigSync() def SetInstallImage(self, install_image): """Set the install image location @type install_image: string @param install_image: location of the install image """ self._ConfigData().cluster.install_image = install_image @ConfigSync(shared=1) def GetInstanceCommunicationNetwork(self): """Get cluster instance communication network @rtype: string @return: instance communication network, which is the name of the network used for instance communication """ return self._ConfigData().cluster.instance_communication_network @ConfigSync() def SetInstanceCommunicationNetwork(self, network_name): """Set cluster instance communication network @type network_name: string @param network_name: instance communication network, which is the name of the network used for instance communication """ self._ConfigData().cluster.instance_communication_network = network_name @ConfigSync(shared=1) def GetZeroingImage(self): """Get the zeroing image location @rtype: string @return: the location of the zeroing image """ return self._config_data.cluster.zeroing_image @ConfigSync(shared=1) def GetCompressionTools(self): """Get cluster compression tools @rtype: list of string @return: a list of tools that are cleared for use in this cluster for the purpose of compressing data """ return self._ConfigData().cluster.compression_tools @ConfigSync() def SetCompressionTools(self, tools): """Set cluster compression tools @type tools: list of string @param tools: a list of tools that are cleared for use in this cluster for the purpose of compressing data """ self._ConfigData().cluster.compression_tools = tools @ConfigSync() def AddNodeGroup(self, group, ec_id, check_uuid=True): """Add a node group to the configuration. This method calls group.UpgradeConfig() to fill any missing attributes according to their default values. @type group: L{objects.NodeGroup} @param group: the NodeGroup object to add @type ec_id: string @param ec_id: unique id for the job to use when creating a missing UUID @type check_uuid: bool @param check_uuid: add an UUID to the group if it doesn't have one or, if it does, ensure that it does not exist in the configuration already """ self._UnlockedAddNodeGroup(group, ec_id, check_uuid) def _UnlockedAddNodeGroup(self, group, ec_id, check_uuid): """Add a node group to the configuration. """ logging.info("Adding node group %s to configuration", group.name) # Some code might need to add a node group with a pre-populated UUID # generated with ConfigWriter.GenerateUniqueID(). We allow them to bypass # the "does this UUID" exist already check. if check_uuid: self._EnsureUUID(group, ec_id) try: existing_uuid = self._UnlockedLookupNodeGroup(group.name) except errors.OpPrereqError: pass else: raise errors.OpPrereqError("Desired group name '%s' already exists as a" " node group (UUID: %s)" % (group.name, existing_uuid), errors.ECODE_EXISTS) group.serial_no = 1 group.ctime = group.mtime = time.time() group.UpgradeConfig() self._ConfigData().nodegroups[group.uuid] = group self._ConfigData().cluster.serial_no += 1 @ConfigSync() def RemoveNodeGroup(self, group_uuid): """Remove a node group from the configuration. @type group_uuid: string @param group_uuid: the UUID of the node group to remove """ logging.info("Removing node group %s from configuration", group_uuid) if group_uuid not in self._ConfigData().nodegroups: raise errors.ConfigurationError("Unknown node group '%s'" % group_uuid) assert len(self._ConfigData().nodegroups) != 1, \ "Group '%s' is the only group, cannot be removed" % group_uuid del self._ConfigData().nodegroups[group_uuid] self._ConfigData().cluster.serial_no += 1 def _UnlockedLookupNodeGroup(self, target): """Lookup a node group's UUID. @type target: string or None @param target: group name or UUID or None to look for the default @rtype: string @return: nodegroup UUID @raises errors.OpPrereqError: when the target group cannot be found """ if target is None: if len(self._ConfigData().nodegroups) != 1: raise errors.OpPrereqError("More than one node group exists. Target" " group must be specified explicitly.") else: return list(self._ConfigData().nodegroups)[0] if target in self._ConfigData().nodegroups: return target for nodegroup in self._ConfigData().nodegroups.values(): if nodegroup.name == target: return nodegroup.uuid raise errors.OpPrereqError("Node group '%s' not found" % target, errors.ECODE_NOENT) @ConfigSync(shared=1) def LookupNodeGroup(self, target): """Lookup a node group's UUID. This function is just a wrapper over L{_UnlockedLookupNodeGroup}. @type target: string or None @param target: group name or UUID or None to look for the default @rtype: string @return: nodegroup UUID """ return self._UnlockedLookupNodeGroup(target) def _UnlockedGetNodeGroup(self, uuid): """Lookup a node group. @type uuid: string @param uuid: group UUID @rtype: L{objects.NodeGroup} or None @return: nodegroup object, or None if not found """ if uuid not in self._ConfigData().nodegroups: return None return self._ConfigData().nodegroups[uuid] @ConfigSync(shared=1) def GetNodeGroup(self, uuid): """Lookup a node group. @type uuid: string @param uuid: group UUID @rtype: L{objects.NodeGroup} or None @return: nodegroup object, or None if not found """ return self._UnlockedGetNodeGroup(uuid) def _UnlockedGetAllNodeGroupsInfo(self): """Get the configuration of all node groups. """ return dict(self._ConfigData().nodegroups) @ConfigSync(shared=1) def GetAllNodeGroupsInfo(self): """Get the configuration of all node groups. """ return self._UnlockedGetAllNodeGroupsInfo() @ConfigSync(shared=1) def GetAllNodeGroupsInfoDict(self): """Get the configuration of all node groups expressed as a dictionary of dictionaries. """ return dict((uuid, ng.ToDict()) for (uuid, ng) in self._UnlockedGetAllNodeGroupsInfo().items()) @ConfigSync(shared=1) def GetNodeGroupList(self): """Get a list of node groups. """ return list(self._ConfigData().nodegroups) @ConfigSync(shared=1) def GetNodeGroupMembersByNodes(self, nodes): """Get nodes which are member in the same nodegroups as the given nodes. """ ngfn = lambda node_uuid: self._UnlockedGetNodeInfo(node_uuid).group return frozenset(member_uuid for node_uuid in nodes for member_uuid in self._UnlockedGetNodeGroup(ngfn(node_uuid)).members) @ConfigSync(shared=1) def GetMultiNodeGroupInfo(self, group_uuids): """Get the configuration of multiple node groups. @param group_uuids: List of node group UUIDs @rtype: list @return: List of tuples of (group_uuid, group_info) """ return [(uuid, self._UnlockedGetNodeGroup(uuid)) for uuid in group_uuids] def AddInstance(self, instance, _ec_id, replace=False): """Add an instance to the config. This should be used after creating a new instance. @type instance: L{objects.Instance} @param instance: the instance object @type replace: bool @param replace: if true, expect the instance to be present and replace rather than add. """ if not isinstance(instance, objects.Instance): raise errors.ProgrammerError("Invalid type passed to AddInstance") instance.serial_no = 1 utils.SimpleRetry(True, self._wconfd.AddInstance, 0.1, 30, args=[instance.ToDict(), self._GetWConfdContext(), replace]) self.OutDate() def _EnsureUUID(self, item, ec_id): """Ensures a given object has a valid UUID. @param item: the instance or node to be checked @param ec_id: the execution context id for the uuid reservation """ if not item.uuid: item.uuid = self._GenerateUniqueID(ec_id) else: self._CheckUniqueUUID(item, include_temporary=True) def _CheckUniqueUUID(self, item, include_temporary): """Checks that the UUID of the given object is unique. @param item: the instance or node to be checked @param include_temporary: whether temporarily generated UUID's should be included in the check. If the UUID of the item to be checked is a temporarily generated one, this has to be C{False}. """ if not item.uuid: raise errors.ConfigurationError("'%s' must have an UUID" % (item.name,)) if item.uuid in self._AllIDs(include_temporary=include_temporary): raise errors.ConfigurationError("Cannot add '%s': UUID %s already" " in use" % (item.name, item.uuid)) def _CheckUUIDpresent(self, item): """Checks that an object with the given UUID exists. @param item: the instance or other UUID possessing object to verify that its UUID is present """ if not item.uuid: raise errors.ConfigurationError("'%s' must have an UUID" % (item.name,)) if item.uuid not in self._AllIDs(include_temporary=False): raise errors.ConfigurationError("Cannot replace '%s': UUID %s not present" % (item.name, item.uuid)) def _SetInstanceStatus(self, inst_uuid, status, disks_active, admin_state_source): """Set the instance's status to a given value. @rtype: L{objects.Instance} @return: the updated instance object """ def WithRetry(): result = self._wconfd.SetInstanceStatus(inst_uuid, status, disks_active, admin_state_source) self.OutDate() if result is None: raise utils.RetryAgain() else: return result return objects.Instance.FromDict(utils.Retry(WithRetry, 0.1, 30)) def MarkInstanceUp(self, inst_uuid): """Mark the instance status to up in the config. This also sets the instance disks active flag. @rtype: L{objects.Instance} @return: the updated instance object """ return self._SetInstanceStatus(inst_uuid, constants.ADMINST_UP, True, constants.ADMIN_SOURCE) def MarkInstanceOffline(self, inst_uuid): """Mark the instance status to down in the config. This also clears the instance disks active flag. @rtype: L{objects.Instance} @return: the updated instance object """ return self._SetInstanceStatus(inst_uuid, constants.ADMINST_OFFLINE, False, constants.ADMIN_SOURCE) def RemoveInstance(self, inst_uuid): """Remove the instance from the configuration. """ utils.SimpleRetry(True, self._wconfd.RemoveInstance, 0.1, 30, args=[inst_uuid]) self.OutDate() @ConfigSync() def RenameInstance(self, inst_uuid, new_name): """Rename an instance. This needs to be done in ConfigWriter and not by RemoveInstance combined with AddInstance as only we can guarantee an atomic rename. """ if inst_uuid not in self._ConfigData().instances: raise errors.ConfigurationError("Unknown instance '%s'" % inst_uuid) inst = self._ConfigData().instances[inst_uuid] inst.name = new_name instance_disks = self._UnlockedGetInstanceDisks(inst_uuid) for (_, disk) in enumerate(instance_disks): if disk.dev_type in [constants.DT_FILE, constants.DT_SHARED_FILE]: # rename the file paths in logical and physical id file_storage_dir = os.path.dirname(os.path.dirname(disk.logical_id[1])) disk.logical_id = (disk.logical_id[0], utils.PathJoin(file_storage_dir, inst.name, os.path.basename(disk.logical_id[1]))) # Force update of ssconf files self._ConfigData().cluster.serial_no += 1 def MarkInstanceDown(self, inst_uuid): """Mark the status of an instance to down in the configuration. This does not touch the instance disks active flag, as shut down instances can still have active disks. @rtype: L{objects.Instance} @return: the updated instance object """ return self._SetInstanceStatus(inst_uuid, constants.ADMINST_DOWN, None, constants.ADMIN_SOURCE) def MarkInstanceUserDown(self, inst_uuid): """Mark the status of an instance to user down in the configuration. This does not touch the instance disks active flag, as user shut down instances can still have active disks. """ self._SetInstanceStatus(inst_uuid, constants.ADMINST_DOWN, None, constants.USER_SOURCE) def MarkInstanceDisksActive(self, inst_uuid): """Mark the status of instance disks active. @rtype: L{objects.Instance} @return: the updated instance object """ return self._SetInstanceStatus(inst_uuid, None, True, None) def MarkInstanceDisksInactive(self, inst_uuid): """Mark the status of instance disks inactive. @rtype: L{objects.Instance} @return: the updated instance object """ return self._SetInstanceStatus(inst_uuid, None, False, None) def _UnlockedGetInstanceList(self): """Get the list of instances. This function is for internal use, when the config lock is already held. """ return list(self._ConfigData().instances) @ConfigSync(shared=1) def GetInstanceList(self): """Get the list of instances. @return: array of instances, ex. ['instance2-uuid', 'instance1-uuid'] """ return self._UnlockedGetInstanceList() def ExpandInstanceName(self, short_name): """Attempt to expand an incomplete instance name. """ # Locking is done in L{ConfigWriter.GetAllInstancesInfo} all_insts = self.GetAllInstancesInfo().values() expanded_name = _MatchNameComponentIgnoreCase( short_name, [inst.name for inst in all_insts]) if expanded_name is not None: # there has to be exactly one instance with that name inst = [n for n in all_insts if n.name == expanded_name][0] return (inst.uuid, inst.name) else: return (None, None) def _UnlockedGetInstanceInfo(self, inst_uuid): """Returns information about an instance. This function is for internal use, when the config lock is already held. """ if inst_uuid not in self._ConfigData().instances: return None return self._ConfigData().instances[inst_uuid] @ConfigSync(shared=1) def GetInstanceInfo(self, inst_uuid): """Returns information about an instance. It takes the information from the configuration file. Other information of an instance are taken from the live systems. @param inst_uuid: UUID of the instance @rtype: L{objects.Instance} @return: the instance object """ return self._UnlockedGetInstanceInfo(inst_uuid) @ConfigSync(shared=1) def GetInstanceNodeGroups(self, inst_uuid, primary_only=False): """Returns set of node group UUIDs for instance's nodes. @rtype: frozenset """ instance = self._UnlockedGetInstanceInfo(inst_uuid) if not instance: raise errors.ConfigurationError("Unknown instance '%s'" % inst_uuid) if primary_only: nodes = [instance.primary_node] else: nodes = self._UnlockedGetInstanceNodes(instance.uuid) return frozenset(self._UnlockedGetNodeInfo(node_uuid).group for node_uuid in nodes) @ConfigSync(shared=1) def GetInstanceNetworks(self, inst_uuid): """Returns set of network UUIDs for instance's nics. @rtype: frozenset """ instance = self._UnlockedGetInstanceInfo(inst_uuid) if not instance: raise errors.ConfigurationError("Unknown instance '%s'" % inst_uuid) networks = set() for nic in instance.nics: if nic.network: networks.add(nic.network) return frozenset(networks) @ConfigSync(shared=1) def GetMultiInstanceInfo(self, inst_uuids): """Get the configuration of multiple instances. @param inst_uuids: list of instance UUIDs @rtype: list @return: list of tuples (instance UUID, instance_info), where instance_info is what would GetInstanceInfo return for the node, while keeping the original order """ return [(uuid, self._UnlockedGetInstanceInfo(uuid)) for uuid in inst_uuids] @ConfigSync(shared=1) def GetMultiInstanceInfoByName(self, inst_names): """Get the configuration of multiple instances. @param inst_names: list of instance names @rtype: list @return: list of tuples (instance, instance_info), where instance_info is what would GetInstanceInfo return for the node, while keeping the original order """ result = [] for name in inst_names: instance = self._UnlockedGetInstanceInfoByName(name) if instance: result.append((instance.uuid, instance)) else: raise errors.ConfigurationError("Instance data of instance '%s'" " not found." % name) return result @ConfigSync(shared=1) def GetAllInstancesInfo(self): """Get the configuration of all instances. @rtype: dict @return: dict of (instance, instance_info), where instance_info is what would GetInstanceInfo return for the node """ return self._UnlockedGetAllInstancesInfo() def _UnlockedGetAllInstancesInfo(self): my_dict = dict([(inst_uuid, self._UnlockedGetInstanceInfo(inst_uuid)) for inst_uuid in self._UnlockedGetInstanceList()]) return my_dict @ConfigSync(shared=1) def GetInstancesInfoByFilter(self, filter_fn): """Get instance configuration with a filter. @type filter_fn: callable @param filter_fn: Filter function receiving instance object as parameter, returning boolean. Important: this function is called while the configuration locks is held. It must not do any complex work or call functions potentially leading to a deadlock. Ideally it doesn't call any other functions and just compares instance attributes. """ return dict((uuid, inst) for (uuid, inst) in self._ConfigData().instances.items() if filter_fn(inst)) @ConfigSync(shared=1) def GetInstanceInfoByName(self, inst_name): """Get the L{objects.Instance} object for a named instance. @param inst_name: name of the instance to get information for @type inst_name: string @return: the corresponding L{objects.Instance} instance or None if no information is available """ return self._UnlockedGetInstanceInfoByName(inst_name) def _UnlockedGetInstanceInfoByName(self, inst_name): for inst in self._UnlockedGetAllInstancesInfo().values(): if inst.name == inst_name: return inst return None def _UnlockedGetInstanceName(self, inst_uuid): inst_info = self._UnlockedGetInstanceInfo(inst_uuid) if inst_info is None: raise errors.OpExecError("Unknown instance: %s" % inst_uuid) return inst_info.name @ConfigSync(shared=1) def GetInstanceName(self, inst_uuid): """Gets the instance name for the passed instance. @param inst_uuid: instance UUID to get name for @type inst_uuid: string @rtype: string @return: instance name """ return self._UnlockedGetInstanceName(inst_uuid) @ConfigSync(shared=1) def GetInstanceNames(self, inst_uuids): """Gets the instance names for the passed list of nodes. @param inst_uuids: list of instance UUIDs to get names for @type inst_uuids: list of strings @rtype: list of strings @return: list of instance names """ return self._UnlockedGetInstanceNames(inst_uuids) def SetInstancePrimaryNode(self, inst_uuid, target_node_uuid): """Sets the primary node of an existing instance @param inst_uuid: instance UUID @type inst_uuid: string @param target_node_uuid: the new primary node UUID @type target_node_uuid: string """ utils.SimpleRetry(True, self._wconfd.SetInstancePrimaryNode, 0.1, 30, args=[inst_uuid, target_node_uuid]) self.OutDate() @ConfigSync() def SetDiskNodes(self, disk_uuid, nodes): """Sets the nodes of an existing disk @param disk_uuid: disk UUID @type disk_uuid: string @param nodes: the new nodes for the disk @type nodes: list of node uuids """ self._UnlockedGetDiskInfo(disk_uuid).nodes = nodes @ConfigSync() def SetDiskLogicalID(self, disk_uuid, logical_id): """Sets the logical_id of an existing disk @param disk_uuid: disk UUID @type disk_uuid: string @param logical_id: the new logical_id for the disk @type logical_id: tuple """ disk = self._UnlockedGetDiskInfo(disk_uuid) if disk is None: raise errors.ConfigurationError("Unknown disk UUID '%s'" % disk_uuid) if len(disk.logical_id) != len(logical_id): raise errors.ProgrammerError("Logical ID format mismatch\n" "Existing logical ID: %s\n" "New logical ID: %s", disk.logical_id, logical_id) disk.logical_id = logical_id def _UnlockedGetInstanceNames(self, inst_uuids): return [self._UnlockedGetInstanceName(uuid) for uuid in inst_uuids] def _UnlockedAddNode(self, node, ec_id): """Add a node to the configuration. @type node: L{objects.Node} @param node: a Node instance """ logging.info("Adding node %s to configuration", node.name) self._EnsureUUID(node, ec_id) node.serial_no = 1 node.ctime = node.mtime = time.time() self._UnlockedAddNodeToGroup(node.uuid, node.group) assert node.uuid in self._ConfigData().nodegroups[node.group].members self._ConfigData().nodes[node.uuid] = node self._ConfigData().cluster.serial_no += 1 @ConfigSync() def AddNode(self, node, ec_id): """Add a node to the configuration. @type node: L{objects.Node} @param node: a Node instance """ self._UnlockedAddNode(node, ec_id) @ConfigSync() def RemoveNode(self, node_uuid): """Remove a node from the configuration. """ logging.info("Removing node %s from configuration", node_uuid) if node_uuid not in self._ConfigData().nodes: raise errors.ConfigurationError("Unknown node '%s'" % node_uuid) self._UnlockedRemoveNodeFromGroup(self._ConfigData().nodes[node_uuid]) del self._ConfigData().nodes[node_uuid] self._ConfigData().cluster.serial_no += 1 def ExpandNodeName(self, short_name): """Attempt to expand an incomplete node name into a node UUID. """ # Locking is done in L{ConfigWriter.GetAllNodesInfo} all_nodes = self.GetAllNodesInfo().values() expanded_name = _MatchNameComponentIgnoreCase( short_name, [node.name for node in all_nodes]) if expanded_name is not None: # there has to be exactly one node with that name node = [n for n in all_nodes if n.name == expanded_name][0] return (node.uuid, node.name) else: return (None, None) def _UnlockedGetNodeInfo(self, node_uuid): """Get the configuration of a node, as stored in the config. This function is for internal use, when the config lock is already held. @param node_uuid: the node UUID @rtype: L{objects.Node} @return: the node object """ if node_uuid not in self._ConfigData().nodes: return None return self._ConfigData().nodes[node_uuid] @ConfigSync(shared=1) def GetNodeInfo(self, node_uuid): """Get the configuration of a node, as stored in the config. This is just a locked wrapper over L{_UnlockedGetNodeInfo}. @param node_uuid: the node UUID @rtype: L{objects.Node} @return: the node object """ return self._UnlockedGetNodeInfo(node_uuid) @ConfigSync(shared=1) def GetNodeInstances(self, node_uuid): """Get the instances of a node, as stored in the config. @param node_uuid: the node UUID @rtype: (list, list) @return: a tuple with two lists: the primary and the secondary instances """ pri = [] sec = [] for inst in self._ConfigData().instances.values(): if inst.primary_node == node_uuid: pri.append(inst.uuid) if node_uuid in self._UnlockedGetInstanceSecondaryNodes(inst.uuid): sec.append(inst.uuid) return (pri, sec) @ConfigSync(shared=1) def GetNodeGroupInstances(self, uuid, primary_only=False): """Get the instances of a node group. @param uuid: Node group UUID @param primary_only: Whether to only consider primary nodes @rtype: frozenset @return: List of instance UUIDs in node group """ if primary_only: nodes_fn = lambda inst: [inst.primary_node] else: nodes_fn = lambda inst: self._UnlockedGetInstanceNodes(inst.uuid) return frozenset(inst.uuid for inst in self._ConfigData().instances.values() for node_uuid in nodes_fn(inst) if self._UnlockedGetNodeInfo(node_uuid).group == uuid) def _UnlockedGetHvparamsString(self, hvname): """Return the string representation of the list of hyervisor parameters of the given hypervisor. @see: C{GetHvparams} """ result = "" hvparams = self._ConfigData().cluster.hvparams[hvname] for key in hvparams: result += "%s=%s\n" % (key, hvparams[key]) return result @ConfigSync(shared=1) def GetHvparamsString(self, hvname): """Return the hypervisor parameters of the given hypervisor. @type hvname: string @param hvname: name of a hypervisor @rtype: string @return: string containing key-value-pairs, one pair on each line; format: KEY=VALUE """ return self._UnlockedGetHvparamsString(hvname) def _UnlockedGetNodeList(self): """Return the list of nodes which are in the configuration. This function is for internal use, when the config lock is already held. @rtype: list """ return list(self._ConfigData().nodes) @ConfigSync(shared=1) def GetNodeList(self): """Return the list of nodes which are in the configuration. """ return self._UnlockedGetNodeList() def _UnlockedGetOnlineNodeList(self): """Return the list of nodes which are online. """ all_nodes = [self._UnlockedGetNodeInfo(node) for node in self._UnlockedGetNodeList()] return [node.uuid for node in all_nodes if not node.offline] @ConfigSync(shared=1) def GetOnlineNodeList(self): """Return the list of nodes which are online. """ return self._UnlockedGetOnlineNodeList() @ConfigSync(shared=1) def GetVmCapableNodeList(self): """Return the list of nodes which are not vm capable. """ all_nodes = [self._UnlockedGetNodeInfo(node) for node in self._UnlockedGetNodeList()] return [node.uuid for node in all_nodes if node.vm_capable] @ConfigSync(shared=1) def GetNonVmCapableNodeList(self): """Return the list of nodes' uuids which are not vm capable. """ all_nodes = [self._UnlockedGetNodeInfo(node) for node in self._UnlockedGetNodeList()] return [node.uuid for node in all_nodes if not node.vm_capable] @ConfigSync(shared=1) def GetNonVmCapableNodeNameList(self): """Return the list of nodes' names which are not vm capable. """ all_nodes = [self._UnlockedGetNodeInfo(node) for node in self._UnlockedGetNodeList()] return [node.name for node in all_nodes if not node.vm_capable] @ConfigSync(shared=1) def GetMultiNodeInfo(self, node_uuids): """Get the configuration of multiple nodes. @param node_uuids: list of node UUIDs @rtype: list @return: list of tuples of (node, node_info), where node_info is what would GetNodeInfo return for the node, in the original order """ return [(uuid, self._UnlockedGetNodeInfo(uuid)) for uuid in node_uuids] def _UnlockedGetAllNodesInfo(self): """Gets configuration of all nodes. @note: See L{GetAllNodesInfo} """ return dict([(node_uuid, self._UnlockedGetNodeInfo(node_uuid)) for node_uuid in self._UnlockedGetNodeList()]) @ConfigSync(shared=1) def GetAllNodesInfo(self): """Get the configuration of all nodes. @rtype: dict @return: dict of (node, node_info), where node_info is what would GetNodeInfo return for the node """ return self._UnlockedGetAllNodesInfo() def _UnlockedGetNodeInfoByName(self, node_name): for node in self._UnlockedGetAllNodesInfo().values(): if node.name == node_name: return node return None @ConfigSync(shared=1) def GetNodeInfoByName(self, node_name): """Get the L{objects.Node} object for a named node. @param node_name: name of the node to get information for @type node_name: string @return: the corresponding L{objects.Node} instance or None if no information is available """ return self._UnlockedGetNodeInfoByName(node_name) @ConfigSync(shared=1) def GetNodeGroupInfoByName(self, nodegroup_name): """Get the L{objects.NodeGroup} object for a named node group. @param nodegroup_name: name of the node group to get information for @type nodegroup_name: string @return: the corresponding L{objects.NodeGroup} instance or None if no information is available """ for nodegroup in self._UnlockedGetAllNodeGroupsInfo().values(): if nodegroup.name == nodegroup_name: return nodegroup return None def _UnlockedGetNodeName(self, node_spec): if isinstance(node_spec, objects.Node): return node_spec.name elif isinstance(node_spec, str): node_info = self._UnlockedGetNodeInfo(node_spec) if node_info is None: raise errors.OpExecError("Unknown node: %s" % node_spec) return node_info.name else: raise errors.ProgrammerError("Can't handle node spec '%s'" % node_spec) @ConfigSync(shared=1) def GetNodeName(self, node_spec): """Gets the node name for the passed node. @param node_spec: node to get names for @type node_spec: either node UUID or a L{objects.Node} object @rtype: string @return: node name """ return self._UnlockedGetNodeName(node_spec) def _UnlockedGetNodeNames(self, node_specs): return [self._UnlockedGetNodeName(node_spec) for node_spec in node_specs] @ConfigSync(shared=1) def GetNodeNames(self, node_specs): """Gets the node names for the passed list of nodes. @param node_specs: list of nodes to get names for @type node_specs: list of either node UUIDs or L{objects.Node} objects @rtype: list of strings @return: list of node names """ return self._UnlockedGetNodeNames(node_specs) @ConfigSync(shared=1) def GetNodeGroupsFromNodes(self, node_uuids): """Returns groups for a list of nodes. @type node_uuids: list of string @param node_uuids: List of node UUIDs @rtype: frozenset """ return frozenset(self._UnlockedGetNodeInfo(uuid).group for uuid in node_uuids) def _UnlockedGetMasterCandidateUuids(self): """Get the list of UUIDs of master candidates. @rtype: list of strings @return: list of UUIDs of all master candidates. """ return [node.uuid for node in self._ConfigData().nodes.values() if node.master_candidate] @ConfigSync(shared=1) def GetMasterCandidateUuids(self): """Get the list of UUIDs of master candidates. @rtype: list of strings @return: list of UUIDs of all master candidates. """ return self._UnlockedGetMasterCandidateUuids() def _UnlockedGetMasterCandidateStats(self, exceptions=None): """Get the number of current and maximum desired and possible candidates. @type exceptions: list @param exceptions: if passed, list of nodes that should be ignored @rtype: tuple @return: tuple of (current, desired and possible, possible) """ mc_now = mc_should = mc_max = 0 for node in self._ConfigData().nodes.values(): if exceptions and node.uuid in exceptions: continue if not (node.offline or node.drained) and node.master_capable: mc_max += 1 if node.master_candidate: mc_now += 1 pool_size = self._ConfigData().cluster.candidate_pool_size mc_should = mc_max if pool_size is None else min(mc_max, pool_size) return (mc_now, mc_should, mc_max) @ConfigSync(shared=1) def GetMasterCandidateStats(self, exceptions=None): """Get the number of current and maximum possible candidates. This is just a wrapper over L{_UnlockedGetMasterCandidateStats}. @type exceptions: list @param exceptions: if passed, list of nodes that should be ignored @rtype: tuple @return: tuple of (current, max) """ return self._UnlockedGetMasterCandidateStats(exceptions) @ConfigSync() def MaintainCandidatePool(self, exception_node_uuids): """Try to grow the candidate pool to the desired size. @type exception_node_uuids: list @param exception_node_uuids: if passed, list of nodes that should be ignored @rtype: list @return: list with the adjusted nodes (L{objects.Node} instances) """ mc_now, mc_max, _ = self._UnlockedGetMasterCandidateStats( exception_node_uuids) mod_list = [] if mc_now < mc_max: node_list = list(self._ConfigData().nodes) random.shuffle(node_list) for uuid in node_list: if mc_now >= mc_max: break node = self._ConfigData().nodes[uuid] if (node.master_candidate or node.offline or node.drained or node.uuid in exception_node_uuids or not node.master_capable): continue mod_list.append(node) node.master_candidate = True node.serial_no += 1 mc_now += 1 if mc_now != mc_max: # this should not happen logging.warning("Warning: MaintainCandidatePool didn't manage to" " fill the candidate pool (%d/%d)", mc_now, mc_max) if mod_list: self._ConfigData().cluster.serial_no += 1 return mod_list def _UnlockedAddNodeToGroup(self, node_uuid, nodegroup_uuid): """Add a given node to the specified group. """ if nodegroup_uuid not in self._ConfigData().nodegroups: # This can happen if a node group gets deleted between its lookup and # when we're adding the first node to it, since we don't keep a lock in # the meantime. It's ok though, as we'll fail cleanly if the node group # is not found anymore. raise errors.OpExecError("Unknown node group: %s" % nodegroup_uuid) if node_uuid not in self._ConfigData().nodegroups[nodegroup_uuid].members: self._ConfigData().nodegroups[nodegroup_uuid].members.append(node_uuid) def _UnlockedRemoveNodeFromGroup(self, node): """Remove a given node from its group. """ nodegroup = node.group if nodegroup not in self._ConfigData().nodegroups: logging.warning("Warning: node '%s' has unknown node group '%s'" " (while being removed from it)", node.uuid, nodegroup) nodegroup_obj = self._ConfigData().nodegroups[nodegroup] if node.uuid not in nodegroup_obj.members: logging.warning("Warning: node '%s' not a member of its node group '%s'" " (while being removed from it)", node.uuid, nodegroup) else: nodegroup_obj.members.remove(node.uuid) @ConfigSync() def AssignGroupNodes(self, mods): """Changes the group of a number of nodes. @type mods: list of tuples; (node name, new group UUID) @param mods: Node membership modifications """ groups = self._ConfigData().nodegroups nodes = self._ConfigData().nodes resmod = [] # Try to resolve UUIDs first for (node_uuid, new_group_uuid) in mods: try: node = nodes[node_uuid] except KeyError: raise errors.ConfigurationError("Unable to find node '%s'" % node_uuid) if node.group == new_group_uuid: # Node is being assigned to its current group logging.debug("Node '%s' was assigned to its current group (%s)", node_uuid, node.group) continue # Try to find current group of node try: old_group = groups[node.group] except KeyError: raise errors.ConfigurationError("Unable to find old group '%s'" % node.group) # Try to find new group for node try: new_group = groups[new_group_uuid] except KeyError: raise errors.ConfigurationError("Unable to find new group '%s'" % new_group_uuid) assert node.uuid in old_group.members, \ ("Inconsistent configuration: node '%s' not listed in members for its" " old group '%s'" % (node.uuid, old_group.uuid)) assert node.uuid not in new_group.members, \ ("Inconsistent configuration: node '%s' already listed in members for" " its new group '%s'" % (node.uuid, new_group.uuid)) resmod.append((node, old_group, new_group)) # Apply changes for (node, old_group, new_group) in resmod: assert node.uuid != new_group.uuid and old_group.uuid != new_group.uuid, \ "Assigning to current group is not possible" node.group = new_group.uuid # Update members of involved groups if node.uuid in old_group.members: old_group.members.remove(node.uuid) if node.uuid not in new_group.members: new_group.members.append(node.uuid) # Update timestamps and serials (only once per node/group object) now = time.time() for obj in frozenset(itertools.chain(*resmod)): obj.serial_no += 1 obj.mtime = now # Force ssconf update self._ConfigData().cluster.serial_no += 1 def _BumpSerialNo(self): """Bump up the serial number of the config. """ self._ConfigData().serial_no += 1 self._ConfigData().mtime = time.time() def _AllUUIDObjects(self): """Returns all objects with uuid attributes. """ return (list(self._ConfigData().instances.values()) + list(self._ConfigData().nodes.values()) + list(self._ConfigData().nodegroups.values()) + list(self._ConfigData().networks.values()) + list(self._ConfigData().disks.values()) + self._AllNICs() + [self._ConfigData().cluster]) def GetConfigManager(self, shared=False, forcelock=False): """Returns a ConfigManager, which is suitable to perform a synchronized block of configuration operations. WARNING: This blocks all other configuration operations, so anything that runs inside the block should be very fast, preferably not using any IO. """ return ConfigManager(self, shared=shared, forcelock=forcelock) def _AddLockCount(self, count): self._lock_count += count return self._lock_count def _LockCount(self): return self._lock_count def _OpenConfig(self, shared, force=False): """Read the config data from WConfd or disk. """ if self._AddLockCount(1) > 1: if self._lock_current_shared and not shared: self._AddLockCount(-1) raise errors.ConfigurationError("Can't request an exclusive" " configuration lock while holding" " shared") elif not force or self._lock_forced or not shared or self._offline: return # we already have the lock, do nothing else: self._lock_current_shared = shared if force: self._lock_forced = True # Read the configuration data. If offline, read the file directly. # If online, call WConfd. if self._offline: try: raw_data = utils.ReadFile(self._cfg_file) data_dict = serializer.Load(raw_data) # Make sure the configuration has the right version ValidateConfig(data_dict) data = objects.ConfigData.FromDict(data_dict) except errors.ConfigVersionMismatch: raise except Exception as err: raise errors.ConfigurationError(err) self._cfg_id = utils.GetFileID(path=self._cfg_file) if (not hasattr(data, "cluster") or not hasattr(data.cluster, "rsahostkeypub")): raise errors.ConfigurationError("Incomplete configuration" " (missing cluster.rsahostkeypub)") if not data.cluster.master_node in data.nodes: msg = ("The configuration denotes node %s as master, but does not" " contain information about this node" % data.cluster.master_node) raise errors.ConfigurationError(msg) master_info = data.nodes[data.cluster.master_node] if master_info.name != self._my_hostname and not self._accept_foreign: msg = ("The configuration denotes node %s as master, while my" " hostname is %s; opening a foreign configuration is only" " possible in accept_foreign mode" % (master_info.name, self._my_hostname)) raise errors.ConfigurationError(msg) self._SetConfigData(data) # Upgrade configuration if needed self._UpgradeConfig(saveafter=True) else: if shared and not force: if self._config_data is None: logging.debug("Requesting config, as I have no up-to-date copy") dict_data = self._wconfd.ReadConfig() logging.debug("Configuration received") else: dict_data = None else: # poll until we acquire the lock while True: logging.debug("Receiving config from WConfd.LockConfig [shared=%s]", bool(shared)) dict_data = \ self._wconfd.LockConfig(self._GetWConfdContext(), bool(shared)) if dict_data is not None: logging.debug("Received config from WConfd.LockConfig") break time.sleep(random.random()) try: if dict_data is not None: self._SetConfigData(objects.ConfigData.FromDict(dict_data)) self._UpgradeConfig() except Exception as err: raise errors.ConfigurationError(err) def _CloseConfig(self, save): """Release resources relating the config data. """ if self._AddLockCount(-1) > 0: return # we still have the lock, do nothing if save: try: logging.debug("Writing configuration and unlocking it") self._WriteConfig(releaselock=True) logging.debug("Configuration write, unlock finished") except Exception as err: logging.critical("Can't write the configuration: %s", str(err)) raise elif not self._offline and \ not (self._lock_current_shared and not self._lock_forced): logging.debug("Unlocking configuration without writing") self._wconfd.UnlockConfig(self._GetWConfdContext()) self._lock_forced = False # TODO: To WConfd def _UpgradeConfig(self, saveafter=False): """Run any upgrade steps. This method performs both in-object upgrades and also update some data elements that need uniqueness across the whole configuration or interact with other objects. @warning: if 'saveafter' is 'True', this function will call L{_WriteConfig()} so it needs to be called only from a "safe" place. """ # Keep a copy of the persistent part of _config_data to check for changes # Serialization doesn't guarantee order in dictionaries if saveafter: oldconf = copy.deepcopy(self._ConfigData().ToDict()) else: oldconf = None # In-object upgrades self._ConfigData().UpgradeConfig() for item in self._AllUUIDObjects(): if item.uuid is None: item.uuid = self._GenerateUniqueID(_UPGRADE_CONFIG_JID) if not self._ConfigData().nodegroups: default_nodegroup_name = constants.INITIAL_NODE_GROUP_NAME default_nodegroup = objects.NodeGroup(name=default_nodegroup_name, members=[]) self._UnlockedAddNodeGroup(default_nodegroup, _UPGRADE_CONFIG_JID, True) for node in self._ConfigData().nodes.values(): if not node.group: node.group = self._UnlockedLookupNodeGroup(None) # This is technically *not* an upgrade, but needs to be done both when # nodegroups are being added, and upon normally loading the config, # because the members list of a node group is discarded upon # serializing/deserializing the object. self._UnlockedAddNodeToGroup(node.uuid, node.group) if saveafter: modified = (oldconf != self._ConfigData().ToDict()) else: modified = True # can't prove it didn't change, but doesn't matter if modified and saveafter: self._WriteConfig() self._UnlockedDropECReservations(_UPGRADE_CONFIG_JID) else: if self._offline: self._UnlockedVerifyConfigAndLog() def _WriteConfig(self, destination=None, releaselock=False): """Write the configuration data to persistent storage. """ if destination is None: destination = self._cfg_file # Save the configuration data. If offline, write the file directly. # If online, call WConfd. if self._offline: self._BumpSerialNo() txt = serializer.DumpJson( self._ConfigData().ToDict(_with_private=True), private_encoder=serializer.EncodeWithPrivateFields ) getents = self._getents() try: fd = utils.SafeWriteFile(destination, self._cfg_id, data=txt, close=False, gid=getents.confd_gid, mode=0o640) except errors.LockError: raise errors.ConfigurationError("The configuration file has been" " modified since the last write, cannot" " update") try: self._cfg_id = utils.GetFileID(fd=fd) finally: os.close(fd) else: try: if releaselock: res = self._wconfd.WriteConfigAndUnlock(self._GetWConfdContext(), self._ConfigData().ToDict()) if not res: logging.warning("WriteConfigAndUnlock indicates we already have" " released the lock; assuming this was just a retry" " and the initial call succeeded") else: self._wconfd.WriteConfig(self._GetWConfdContext(), self._ConfigData().ToDict()) except errors.LockError: raise errors.ConfigurationError("The configuration file has been" " modified since the last write, cannot" " update") self.write_count += 1 def _GetAllHvparamsStrings(self, hypervisors): """Get the hvparams of all given hypervisors from the config. @type hypervisors: list of string @param hypervisors: list of hypervisor names @rtype: dict of strings @returns: dictionary mapping the hypervisor name to a string representation of the hypervisor's hvparams """ hvparams = {} for hv in hypervisors: hvparams[hv] = self._UnlockedGetHvparamsString(hv) return hvparams @staticmethod def _ExtendByAllHvparamsStrings(ssconf_values, all_hvparams): """Extends the ssconf_values dictionary by hvparams. @type ssconf_values: dict of strings @param ssconf_values: dictionary mapping ssconf_keys to strings representing the content of ssconf files @type all_hvparams: dict of strings @param all_hvparams: dictionary mapping hypervisor names to a string representation of their hvparams @rtype: same as ssconf_values @returns: the ssconf_values dictionary extended by hvparams """ for hv in all_hvparams: ssconf_key = constants.SS_HVPARAMS_PREF + hv ssconf_values[ssconf_key] = all_hvparams[hv] return ssconf_values def _UnlockedGetSshPortMap(self, node_infos): node_ports = dict([(node.name, self._UnlockedGetNdParams(node).get( constants.ND_SSH_PORT)) for node in node_infos]) return node_ports def _UnlockedGetSsconfValues(self): """Return the values needed by ssconf. @rtype: dict @return: a dictionary with keys the ssconf names and values their associated value """ fn = "\n".join instance_names = utils.NiceSort( [inst.name for inst in self._UnlockedGetAllInstancesInfo().values()]) node_infos = list(self._UnlockedGetAllNodesInfo().values()) node_names = [node.name for node in node_infos] node_pri_ips = ["%s %s" % (ninfo.name, ninfo.primary_ip) for ninfo in node_infos] node_snd_ips = ["%s %s" % (ninfo.name, ninfo.secondary_ip) for ninfo in node_infos] node_vm_capable = ["%s=%s" % (ninfo.name, str(ninfo.vm_capable)) for ninfo in node_infos] instance_data = fn(instance_names) off_data = fn(node.name for node in node_infos if node.offline) on_data = fn(node.name for node in node_infos if not node.offline) mc_data = fn(node.name for node in node_infos if node.master_candidate) mc_ips_data = fn(node.primary_ip for node in node_infos if node.master_candidate) node_data = fn(node_names) node_pri_ips_data = fn(node_pri_ips) node_snd_ips_data = fn(node_snd_ips) node_vm_capable_data = fn(node_vm_capable) cluster = self._ConfigData().cluster cluster_tags = fn(cluster.GetTags()) master_candidates_certs = fn("%s=%s" % (mc_uuid, mc_cert) for mc_uuid, mc_cert in cluster.candidate_certs.items()) hypervisor_list = fn(cluster.enabled_hypervisors) all_hvparams = self._GetAllHvparamsStrings(constants.HYPER_TYPES) uid_pool = uidpool.FormatUidPool(cluster.uid_pool, separator="\n") nodegroups = ["%s %s" % (nodegroup.uuid, nodegroup.name) for nodegroup in self._ConfigData().nodegroups.values()] nodegroups_data = fn(utils.NiceSort(nodegroups)) networks = ["%s %s" % (net.uuid, net.name) for net in self._ConfigData().networks.values()] networks_data = fn(utils.NiceSort(networks)) ssh_ports = fn("%s=%s" % (node_name, port) for node_name, port in self._UnlockedGetSshPortMap(node_infos).items()) ssconf_values = { constants.SS_CLUSTER_NAME: cluster.cluster_name, constants.SS_CLUSTER_TAGS: cluster_tags, constants.SS_FILE_STORAGE_DIR: cluster.file_storage_dir, constants.SS_SHARED_FILE_STORAGE_DIR: cluster.shared_file_storage_dir, constants.SS_GLUSTER_STORAGE_DIR: cluster.gluster_storage_dir, constants.SS_MASTER_CANDIDATES: mc_data, constants.SS_MASTER_CANDIDATES_IPS: mc_ips_data, constants.SS_MASTER_CANDIDATES_CERTS: master_candidates_certs, constants.SS_MASTER_IP: cluster.master_ip, constants.SS_MASTER_NETDEV: cluster.master_netdev, constants.SS_MASTER_NETMASK: str(cluster.master_netmask), constants.SS_MASTER_NODE: self._UnlockedGetNodeName(cluster.master_node), constants.SS_NODE_LIST: node_data, constants.SS_NODE_PRIMARY_IPS: node_pri_ips_data, constants.SS_NODE_SECONDARY_IPS: node_snd_ips_data, constants.SS_NODE_VM_CAPABLE: node_vm_capable_data, constants.SS_OFFLINE_NODES: off_data, constants.SS_ONLINE_NODES: on_data, constants.SS_PRIMARY_IP_FAMILY: str(cluster.primary_ip_family), constants.SS_INSTANCE_LIST: instance_data, constants.SS_RELEASE_VERSION: constants.RELEASE_VERSION, constants.SS_HYPERVISOR_LIST: hypervisor_list, constants.SS_MAINTAIN_NODE_HEALTH: str(cluster.maintain_node_health), constants.SS_UID_POOL: uid_pool, constants.SS_NODEGROUPS: nodegroups_data, constants.SS_NETWORKS: networks_data, constants.SS_ENABLED_USER_SHUTDOWN: str(cluster.enabled_user_shutdown), constants.SS_SSH_PORTS: ssh_ports, } ssconf_values = self._ExtendByAllHvparamsStrings(ssconf_values, all_hvparams) bad_values = [(k, v) for k, v in ssconf_values.items() if not isinstance(v, str)] if bad_values: err = utils.CommaJoin("%s=%s" % (k, v) for k, v in bad_values) raise errors.ConfigurationError("Some ssconf key(s) have non-string" " values: %s" % err) return ssconf_values @ConfigSync(shared=1) def GetSsconfValues(self): """Wrapper using lock around _UnlockedGetSsconf(). """ return self._UnlockedGetSsconfValues() @ConfigSync(shared=1) def GetVGName(self): """Return the volume group name. """ return self._ConfigData().cluster.volume_group_name @ConfigSync() def SetVGName(self, vg_name): """Set the volume group name. """ self._ConfigData().cluster.volume_group_name = vg_name self._ConfigData().cluster.serial_no += 1 @ConfigSync(shared=1) def GetDRBDHelper(self): """Return DRBD usermode helper. """ return self._ConfigData().cluster.drbd_usermode_helper @ConfigSync() def SetDRBDHelper(self, drbd_helper): """Set DRBD usermode helper. """ self._ConfigData().cluster.drbd_usermode_helper = drbd_helper self._ConfigData().cluster.serial_no += 1 @ConfigSync(shared=1) def GetMACPrefix(self): """Return the mac prefix. """ return self._ConfigData().cluster.mac_prefix @ConfigSync(shared=1) def GetClusterInfo(self): """Returns information about the cluster @rtype: L{objects.Cluster} @return: the cluster object """ return self._ConfigData().cluster @ConfigSync(shared=1) def DisksOfType(self, dev_type): """Check if in there is at disk of the given type in the configuration. """ return self._ConfigData().DisksOfType(dev_type) @ConfigSync(shared=1) def GetDetachedConfig(self): """Returns a detached version of a ConfigManager, which represents a read-only snapshot of the configuration at this particular time. """ return DetachedConfig(self._ConfigData()) def Update(self, target, feedback_fn, ec_id=None): """Notify function to be called after updates. This function must be called when an object (as returned by GetInstanceInfo, GetNodeInfo, GetCluster) has been updated and the caller wants the modifications saved to the backing store. Note that all modified objects will be saved, but the target argument is the one the caller wants to ensure that it's saved. @param target: an instance of either L{objects.Cluster}, L{objects.Node} or L{objects.Instance} which is existing in the cluster @param feedback_fn: Callable feedback function """ update_function = None if isinstance(target, objects.Cluster): if self._offline: self.UpdateOfflineCluster(target, feedback_fn) return else: update_function = self._wconfd.UpdateCluster elif isinstance(target, objects.Node): update_function = self._wconfd.UpdateNode elif isinstance(target, objects.Instance): update_function = self._wconfd.UpdateInstance elif isinstance(target, objects.NodeGroup): update_function = self._wconfd.UpdateNodeGroup elif isinstance(target, objects.Network): update_function = self._wconfd.UpdateNetwork elif isinstance(target, objects.Disk): update_function = self._wconfd.UpdateDisk else: raise errors.ProgrammerError("Invalid object type (%s) passed to" " ConfigWriter.Update" % type(target)) def WithRetry(): result = update_function(target.ToDict()) self.OutDate() if result is None: raise utils.RetryAgain() else: return result vals = utils.Retry(WithRetry, 0.1, 30) self.OutDate() target.serial_no = vals[0] target.mtime = float(vals[1]) if ec_id is not None: # Commit all ips reserved by OpInstanceSetParams and OpGroupSetParams # FIXME: After RemoveInstance is moved to WConfd, use its internal # functions from TempRes module. self.CommitTemporaryIps(ec_id) # Just verify the configuration with our feedback function. # It will get written automatically by the decorator. self.VerifyConfigAndLog(feedback_fn=feedback_fn) @ConfigSync() def UpdateOfflineCluster(self, target, feedback_fn): self._ConfigData().cluster = target target.serial_no += 1 target.mtime = time.time() self.VerifyConfigAndLog(feedback_fn=feedback_fn) def _UnlockedDropECReservations(self, _ec_id): """Drop per-execution-context reservations """ # FIXME: Remove the following two lines after all reservations are moved to # wconfd. for rm in self._all_rms: rm.DropECReservations(_ec_id) if not self._offline: self._wconfd.DropAllReservations(self._GetWConfdContext()) def DropECReservations(self, ec_id): self._UnlockedDropECReservations(ec_id) @ConfigSync(shared=1) def GetAllNetworksInfo(self): """Get configuration info of all the networks. """ return dict(self._ConfigData().networks) def _UnlockedGetNetworkList(self): """Get the list of networks. This function is for internal use, when the config lock is already held. """ return list(self._ConfigData().networks) @ConfigSync(shared=1) def GetNetworkList(self): """Get the list of networks. @return: array of networks, ex. ["main", "vlan100", "200] """ return self._UnlockedGetNetworkList() @ConfigSync(shared=1) def GetNetworkNames(self): """Get a list of network names """ names = [net.name for net in self._ConfigData().networks.values()] return names def _UnlockedGetNetwork(self, uuid): """Returns information about a network. This function is for internal use, when the config lock is already held. """ if uuid not in self._ConfigData().networks: return None return self._ConfigData().networks[uuid] @ConfigSync(shared=1) def GetNetwork(self, uuid): """Returns information about a network. It takes the information from the configuration file. @param uuid: UUID of the network @rtype: L{objects.Network} @return: the network object """ return self._UnlockedGetNetwork(uuid) @ConfigSync() def AddNetwork(self, net, ec_id, check_uuid=True): """Add a network to the configuration. @type net: L{objects.Network} @param net: the Network object to add @type ec_id: string @param ec_id: unique id for the job to use when creating a missing UUID """ self._UnlockedAddNetwork(net, ec_id, check_uuid) def _UnlockedAddNetwork(self, net, ec_id, check_uuid): """Add a network to the configuration. """ logging.info("Adding network %s to configuration", net.name) if check_uuid: self._EnsureUUID(net, ec_id) net.serial_no = 1 net.ctime = net.mtime = time.time() self._ConfigData().networks[net.uuid] = net self._ConfigData().cluster.serial_no += 1 def _UnlockedLookupNetwork(self, target): """Lookup a network's UUID. @type target: string @param target: network name or UUID @rtype: string @return: network UUID @raises errors.OpPrereqError: when the target network cannot be found """ if target is None: return None if target in self._ConfigData().networks: return target for net in self._ConfigData().networks.values(): if net.name == target: return net.uuid raise errors.OpPrereqError("Network '%s' not found" % target, errors.ECODE_NOENT) @ConfigSync(shared=1) def LookupNetwork(self, target): """Lookup a network's UUID. This function is just a wrapper over L{_UnlockedLookupNetwork}. @type target: string @param target: network name or UUID @rtype: string @return: network UUID """ return self._UnlockedLookupNetwork(target) @ConfigSync() def RemoveNetwork(self, network_uuid): """Remove a network from the configuration. @type network_uuid: string @param network_uuid: the UUID of the network to remove """ logging.info("Removing network %s from configuration", network_uuid) if network_uuid not in self._ConfigData().networks: raise errors.ConfigurationError("Unknown network '%s'" % network_uuid) del self._ConfigData().networks[network_uuid] self._ConfigData().cluster.serial_no += 1 def _UnlockedGetGroupNetParams(self, net_uuid, node_uuid): """Get the netparams (mode, link) of a network. Get a network's netparams for a given node. @type net_uuid: string @param net_uuid: network uuid @type node_uuid: string @param node_uuid: node UUID @rtype: dict or None @return: netparams """ node_info = self._UnlockedGetNodeInfo(node_uuid) nodegroup_info = self._UnlockedGetNodeGroup(node_info.group) netparams = nodegroup_info.networks.get(net_uuid, None) return netparams @ConfigSync(shared=1) def GetGroupNetParams(self, net_uuid, node_uuid): """Locking wrapper of _UnlockedGetGroupNetParams() """ return self._UnlockedGetGroupNetParams(net_uuid, node_uuid) @ConfigSync(shared=1) def CheckIPInNodeGroup(self, ip, node_uuid): """Check IP uniqueness in nodegroup. Check networks that are connected in the node's node group if ip is contained in any of them. Used when creating/adding a NIC to ensure uniqueness among nodegroups. @type ip: string @param ip: ip address @type node_uuid: string @param node_uuid: node UUID @rtype: (string, dict) or (None, None) @return: (network name, netparams) """ if ip is None: return (None, None) node_info = self._UnlockedGetNodeInfo(node_uuid) nodegroup_info = self._UnlockedGetNodeGroup(node_info.group) for net_uuid in nodegroup_info.networks: net_info = self._UnlockedGetNetwork(net_uuid) pool = network.AddressPool(net_info) if pool.Contains(ip): return (net_info.name, nodegroup_info.networks[net_uuid]) return (None, None) @ConfigSync(shared=1) def GetCandidateCerts(self): """Returns the candidate certificate map. """ return self._ConfigData().cluster.candidate_certs @ConfigSync() def SetCandidateCerts(self, certs): """Replaces the master candidate cert list with the new values. @type certs: dict of string to string @param certs: map of node UUIDs to SSL client certificate digests. """ self._ConfigData().cluster.candidate_certs = certs @ConfigSync() def AddNodeToCandidateCerts(self, node_uuid, cert_digest, info_fn=logging.info, warn_fn=logging.warn): """Adds an entry to the candidate certificate map. @type node_uuid: string @param node_uuid: the node's UUID @type cert_digest: string @param cert_digest: the digest of the node's client SSL certificate @type info_fn: function @param info_fn: logging function for information messages @type warn_fn: function @param warn_fn: logging function for warning messages """ cluster = self._ConfigData().cluster if node_uuid in cluster.candidate_certs: old_cert_digest = cluster.candidate_certs[node_uuid] if old_cert_digest == cert_digest: if info_fn is not None: info_fn("Certificate digest for node %s already in config." "Not doing anything." % node_uuid) return else: if warn_fn is not None: warn_fn("Overriding differing certificate digest for node %s" % node_uuid) cluster.candidate_certs[node_uuid] = cert_digest @ConfigSync() def RemoveNodeFromCandidateCerts(self, node_uuid, warn_fn=logging.warn): """Removes the entry of the given node in the certificate map. @type node_uuid: string @param node_uuid: the node's UUID @type warn_fn: function @param warn_fn: logging function for warning messages """ cluster = self._ConfigData().cluster if node_uuid not in cluster.candidate_certs: if warn_fn is not None: warn_fn("Cannot remove certifcate for node %s, because it's not" " in the candidate map." % node_uuid) return del cluster.candidate_certs[node_uuid] def FlushConfig(self): """Force the distribution of configuration to master candidates. It is not necessary to hold a lock for this operation, it is handled internally by WConfd. """ if not self._offline: self._wconfd.FlushConfig() def FlushConfigGroup(self, uuid): """Force the distribution of configuration to master candidates of a group. It is not necessary to hold a lock for this operation, it is handled internally by WConfd. """ if not self._offline: self._wconfd.FlushConfigGroup(uuid) @ConfigSync(shared=1) def GetAllDiskInfo(self): """Get the configuration of all disks. @rtype: dict @return: dict of (disk, disk_info), where disk_info is what would GetDiskInfo return for disk """ return self._UnlockedGetAllDiskInfo() def _UnlockedGetAllDiskInfo(self): return dict((disk_uuid, self._UnlockedGetDiskInfo(disk_uuid)) for disk_uuid in self._UnlockedGetDiskList()) @ConfigSync(shared=1) def GetInstanceForDisk(self, disk_uuid): """Returns the instance the disk is currently attached to. @type disk_uuid: string @param disk_uuid: the identifier of the disk in question. @rtype: string @return: uuid of instance the disk is attached to. """ for inst_uuid, inst_info in self._UnlockedGetAllInstancesInfo().items(): if disk_uuid in inst_info.disks: return inst_uuid class DetachedConfig(ConfigWriter): """Read-only snapshot of the config.""" def __init__(self, config_data): super(DetachedConfig, self).__init__(self, offline=True) self._SetConfigData(config_data) @staticmethod def _WriteCallError(): raise errors.ProgrammerError("DetachedConfig supports only read-only" " operations") def _OpenConfig(self, shared, force=None): if not shared: DetachedConfig._WriteCallError() def _CloseConfig(self, save): if save: DetachedConfig._WriteCallError() ganeti-3.1.0~rc2/lib/config/temporary_reservations.py000064400000000000000000000065351476477700300230300ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Reserve resources, so that jobs can't take them. """ from ganeti import errors class TemporaryReservationManager(object): """A temporary resource reservation manager. This is used to reserve resources in a job, before using them, making sure other jobs cannot get them in the meantime. """ def __init__(self): self._ec_reserved = {} def Reserved(self, resource): for holder_reserved in self._ec_reserved.values(): if resource in holder_reserved: return True return False def Reserve(self, ec_id, resource): if self.Reserved(resource): raise errors.ReservationError("Duplicate reservation for resource '%s'" % str(resource)) if ec_id not in self._ec_reserved: self._ec_reserved[ec_id] = set([resource]) else: self._ec_reserved[ec_id].add(resource) def DropECReservations(self, ec_id): if ec_id in self._ec_reserved: del self._ec_reserved[ec_id] def GetReserved(self): all_reserved = set() for holder_reserved in self._ec_reserved.values(): all_reserved.update(holder_reserved) return all_reserved def GetECReserved(self, ec_id): """ Used when you want to retrieve all reservations for a specific execution context. E.g when commiting reserved IPs for a specific network. """ ec_reserved = set() if ec_id in self._ec_reserved: ec_reserved.update(self._ec_reserved[ec_id]) return ec_reserved def Generate(self, existing, generate_one_fn, ec_id): """Generate a new resource of this type """ assert callable(generate_one_fn) all_elems = self.GetReserved() all_elems.update(existing) retries = 64 while retries > 0: new_resource = generate_one_fn() if new_resource is not None and new_resource not in all_elems: break else: raise errors.ConfigurationError("Not able generate new resource" " (last tried: %s)" % new_resource) self.Reserve(ec_id, new_resource) return new_resource ganeti-3.1.0~rc2/lib/config/utils.py000064400000000000000000000055341476477700300173400ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utilities used by the config module.""" import logging def ConfigSync(shared=0): """Configuration synchronization decorator. """ def wrap(fn): def sync_function(*args, **kwargs): with args[0].GetConfigManager(shared): return fn(*args, **kwargs) return sync_function return wrap class ConfigManager(object): """Locks the configuration and exposes it to be read or modified. """ def __init__(self, config_writer, shared=False, forcelock=False): assert hasattr(config_writer, '_ConfigData'), \ "invalid argument: Not a ConfigWriter" self._config_writer = config_writer self._shared = shared self._forcelock = forcelock def __enter__(self): try: self._config_writer._OpenConfig(# pylint: disable=W0212 self._shared, force=self._forcelock) except Exception: logging.debug("Opening configuration failed") try: self._config_writer._CloseConfig(False) # pylint: disable=W0212 except Exception: # pylint: disable=W0703 logging.debug("Closing configuration failed as well") raise def __exit__(self, exc_type, exc_value, traceback): # save the configuration, if this was a write opreration that succeeded if exc_type is not None: logging.debug("Configuration operation failed," " the changes will not be saved") # pylint: disable=W0212 self._config_writer._CloseConfig(not self._shared and exc_type is None) return False ganeti-3.1.0~rc2/lib/config/verify.py000064400000000000000000000122411476477700300174750ustar00rootroot00000000000000# # # Copyright (C) 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Verification helpers for the configuration object.""" from ganeti import constants from ganeti import errors from ganeti import objects from ganeti import utils def ValidateConfig(data): """Verifies that a configuration dict looks valid. This only verifies the version of the configuration. @raise errors.ConfigurationError: if the version differs from what we expect """ if data['version'] != constants.CONFIG_VERSION: raise errors.ConfigVersionMismatch(constants.CONFIG_VERSION, data['version']) def VerifyType(owner, attr, value, template, callback): """Checks if an attribute has correct form. @type owner: str @param owner: name of the object containing the attribute @type attr: str @param attr: name of the attribute @type value: dict @param value: actual value of the attribute @type template: dict @param template: expected types of the keys @type callback: callable @param callback: will be called if there is an error """ try: utils.ForceDictType(value, template) except errors.GenericError as err: return callback("%s has invalid %s: %s" % (owner, attr, err)) def VerifyNic(owner, params, callback): """Checks if a NIC has correct form. @type owner: str @param owner: name of the object containing the attribute @type params: dict @param params: actual value of the NIC parameters @type callback: callable @param callback: will be called if there is an error """ try: objects.NIC.CheckParameterSyntax(params) except errors.ConfigurationError as err: callback("%s has invalid nicparams: %s" % (owner, err)) def VerifyIpolicy(owner, ipolicy, iscluster, callback): """Checks if an ipolicy has correct form. @type owner: str @param owner: name of the object containing the attribute @type ipolicy: dict @param ipolicy: actual value of the ipolicy parameters @type iscluster: bool @param iscluster: True iff the owner is the cluster @type callback: callable @param callback: will be called if there is an error """ try: objects.InstancePolicy.CheckParameterSyntax(ipolicy, iscluster) except errors.ConfigurationError as err: callback("%s has invalid instance policy: %s" % (owner, err)) for key, value in ipolicy.items(): if key == constants.ISPECS_MINMAX: for i, val in enumerate(value): VerifyIspecs(owner, "ipolicy/%s[%s]" % (key, i), val, callback) elif key == constants.ISPECS_STD: VerifyType(owner, "ipolicy/" + key, value, constants.ISPECS_PARAMETER_TYPES, callback) else: # FIXME: assuming list type if key in constants.IPOLICY_PARAMETERS: exp_type = float # if the value is int, it can be converted into float convertible_types = [int] else: exp_type = list convertible_types = [] # Try to convert from allowed types, if necessary. if any(isinstance(value, ct) for ct in convertible_types): try: value = exp_type(value) ipolicy[key] = value except ValueError: pass if not isinstance(value, exp_type): callback("%s has invalid instance policy: for %s," " expecting %s, got %s" % (owner, key, exp_type.__name__, type(value))) def VerifyIspecs(owner, parentkey, params, callback): """Checks if an ispec has correct form. @type owner: str @param owner: name of the object containing the attribute @type parentkey: str @param parentkey: the root name of the key @type params: dict @param params: actual value of the ispec parameters @type callback: callable @param callback: will be called if there is an error """ for (key, value) in params.items(): fullkey = "/".join([parentkey, key]) VerifyType(owner, fullkey, value, constants.ISPECS_PARAMETER_TYPES, callback) ganeti-3.1.0~rc2/lib/constants.py000064400000000000000000000054761476477700300167540ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module holding different constants.""" # pylint: disable=W0401,W0614 # # The modules 'ganeti._constants' and 'ganeti._vcsversion' are meant # to be re-exported but pylint complains because the imported names # are not actually used in this module. import re import socket from ganeti._constants import * from ganeti._vcsversion import * from ganeti import compat from ganeti import pathutils ALLOCATABLE_KEY = "allocatable" FAILED_KEY = "failed" DAEMONS_LOGFILES = \ dict((daemon, pathutils.GetLogFilename(DAEMONS_LOGBASE[daemon])) for daemon in DAEMONS_LOGBASE) DAEMONS_EXTRA_LOGFILES = \ dict((daemon, dict((extra, pathutils.GetLogFilename(DAEMONS_EXTRA_LOGBASE[daemon][extra])) for extra in DAEMONS_EXTRA_LOGBASE[daemon])) for daemon in DAEMONS_EXTRA_LOGBASE) IE_MAGIC_RE = re.compile(r"^[-_.a-zA-Z0-9]{5,100}$") # External script validation mask EXT_PLUGIN_MASK = re.compile("^[a-zA-Z0-9_-]+$") JOB_ID_TEMPLATE = r"\d+" JOB_FILE_RE = re.compile(r"^job-(%s)$" % JOB_ID_TEMPLATE) # HVC_DEFAULTS contains one value 'HV_VNC_PASSWORD_FILE' which is not # a constant because it depends on an environment variable that is # used for VClusters. Therefore, it cannot be automatically generated # by Haskell at compilation time (given that this environment variable # might be different at runtime). HVC_DEFAULTS[HT_XEN_HVM][HV_VNC_PASSWORD_FILE] = pathutils.VNC_PASSWORD_FILE # Do not re-export imported modules del re, socket, pathutils, compat ganeti-3.1.0~rc2/lib/daemon.py000064400000000000000000000515361476477700300162010ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module with helper classes and functions for daemons""" from __future__ import print_function import asyncore import collections import os import signal import logging import sched import time import socket import select import sys from ganeti import utils from ganeti import constants from ganeti import errors from ganeti import netutils from ganeti import ssconf from ganeti import runtime from ganeti import compat class GanetiBaseAsyncoreDispatcher(asyncore.dispatcher): """Base Ganeti Asyncore Dispacher """ # this method is overriding an asyncore.dispatcher method def handle_error(self): """Log an error in handling any request, and proceed. """ logging.exception("Error while handling asyncore request") # this method is overriding an asyncore.dispatcher method def writable(self): """Most of the time we don't want to check for writability. """ return False class AsyncUDPSocket(GanetiBaseAsyncoreDispatcher): """An improved asyncore udp socket. """ def __init__(self, family): """Constructor for AsyncUDPSocket """ GanetiBaseAsyncoreDispatcher.__init__(self) self._out_queue = [] self._family = family self.create_socket(family, socket.SOCK_DGRAM) # this method is overriding an asyncore.dispatcher method def handle_connect(self): # Python thinks that the first udp message from a source qualifies as a # "connect" and further ones are part of the same connection. We beg to # differ and treat all messages equally. pass # this method is overriding an asyncore.dispatcher method def handle_read(self): recv_result = utils.IgnoreSignals(self.socket.recvfrom, constants.MAX_UDP_DATA_SIZE) if recv_result is not None: payload, address = recv_result if self._family == socket.AF_INET6: # we ignore 'flow info' and 'scope id' as we don't need them ip, port, _, _ = address else: ip, port = address self.handle_datagram(payload, ip, port) def handle_datagram(self, payload, ip, port): """Handle an already read udp datagram """ raise NotImplementedError # this method is overriding an asyncore.dispatcher method def writable(self): # We should check whether we can write to the socket only if we have # something scheduled to be written return bool(self._out_queue) # this method is overriding an asyncore.dispatcher method def handle_write(self): if not self._out_queue: logging.error("handle_write called with empty output queue") return (ip, port, payload) = self._out_queue[0] utils.IgnoreSignals(self.socket.sendto, payload, 0, (ip, port)) self._out_queue.pop(0) def enqueue_send(self, ip, port, payload): """Enqueue a datagram to be sent when possible """ if isinstance(payload, str): payload = payload.encode("utf-8") if len(payload) > constants.MAX_UDP_DATA_SIZE: raise errors.UdpDataSizeError("Packet too big: %s > %s" % (len(payload), constants.MAX_UDP_DATA_SIZE)) self._out_queue.append((ip, port, payload)) def process_next_packet(self, timeout=0): """Process the next datagram, waiting for it if necessary. @type timeout: float @param timeout: how long to wait for data @rtype: boolean @return: True if some data has been handled, False otherwise """ result = utils.WaitForFdCondition(self.socket, select.POLLIN, timeout) if result is not None and result & select.POLLIN: self.handle_read() return True else: return False class AsyncAwaker(GanetiBaseAsyncoreDispatcher): """A way to notify the asyncore loop that something is going on. If an asyncore daemon is multithreaded when a thread tries to push some data to a socket, the main loop handling asynchronous requests might be sleeping waiting on a select(). To avoid this it can create an instance of the AsyncAwaker, which other threads can use to wake it up. """ def __init__(self, signal_fn=None): """Constructor for AsyncAwaker @type signal_fn: function @param signal_fn: function to call when awaken """ GanetiBaseAsyncoreDispatcher.__init__(self) assert signal_fn is None or callable(signal_fn) (self.in_socket, self.out_socket) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) self.in_socket.setblocking(0) self.in_socket.shutdown(socket.SHUT_WR) self.out_socket.shutdown(socket.SHUT_RD) self.set_socket(self.in_socket) self.need_signal = True self.signal_fn = signal_fn self.connected = True # this method is overriding an asyncore.dispatcher method def handle_read(self): utils.IgnoreSignals(self.recv, 4096) if self.signal_fn: self.signal_fn() self.need_signal = True # this method is overriding an asyncore.dispatcher method def close(self): asyncore.dispatcher.close(self) self.out_socket.close() def signal(self): """Signal the asyncore main loop. Any data we send here will be ignored, but it will cause the select() call to return. """ # Yes, there is a race condition here. No, we don't care, at worst we're # sending more than one wakeup token, which doesn't harm at all. if self.need_signal: self.need_signal = False self.out_socket.send(b"\x00") class _ShutdownCheck(object): """Logic for L{Mainloop} shutdown. """ def __init__(self, fn): """Initializes this class. @type fn: callable @param fn: Function returning C{None} if mainloop can be stopped or a duration in seconds after which the function should be called again @see: L{Mainloop.Run} """ assert callable(fn) self._fn = fn self._defer = None def CanShutdown(self): """Checks whether mainloop can be stopped. @rtype: bool """ if self._defer and self._defer.Remaining() > 0: # A deferred check has already been scheduled return False # Ask mainloop driver whether we can stop or should check again timeout = self._fn() if timeout is None: # Yes, can stop mainloop return True # Schedule another check in the future self._defer = utils.RunningTimeout(timeout, True) return False class Mainloop(object): """Generic mainloop for daemons """ _SHUTDOWN_TIMEOUT_PRIORITY = -(sys.maxsize - 1) def __init__(self): """Constructs a new Mainloop instance. """ self._signal_wait = [] self.awaker = AsyncAwaker() # Resolve uid/gids used runtime.GetEnts() @utils.SignalHandled([signal.SIGCHLD]) @utils.SignalHandled([signal.SIGTERM]) @utils.SignalHandled([signal.SIGINT]) def Run(self, shutdown_wait_fn=None, signal_handlers=None): """Runs the mainloop. @type shutdown_wait_fn: callable @param shutdown_wait_fn: Function to check whether loop can be terminated; B{important}: function must be idempotent and must return either None for shutting down or a timeout for another call @type signal_handlers: dict @param signal_handlers: signal->L{utils.SignalHandler} passed by decorator """ assert isinstance(signal_handlers, dict) and \ len(signal_handlers) > 0, \ "Broken SignalHandled decorator" _sig_notify = lambda _, __: self.awaker.signal() for handler in signal_handlers.values(): handler.SetHandlerFn(_sig_notify) # Counter for received signals shutdown_signals = 0 # Logic to wait for shutdown shutdown_waiter = None # Start actual main loop while True: if shutdown_signals == 1 and shutdown_wait_fn is not None: if shutdown_waiter is None: shutdown_waiter = _ShutdownCheck(shutdown_wait_fn) # Let mainloop driver decide if we can already abort if shutdown_waiter.CanShutdown(): break # Re-evaluate in a second timeout = 1.0 elif shutdown_signals >= 1: # Abort loop if more than one signal has been sent or no callback has # been given break else: # Wait forever on I/O events timeout = None asyncore.loop(count=1, timeout=timeout, use_poll=True) # Check whether a signal was raised for (sig, handler) in signal_handlers.items(): if handler.called: self._CallSignalWaiters(sig) if sig in (signal.SIGTERM, signal.SIGINT): logging.info("Received signal %s asking for shutdown", sig) shutdown_signals += 1 handler.Clear() def _CallSignalWaiters(self, signum): """Calls all signal waiters for a certain signal. @type signum: int @param signum: Signal number """ for owner in self._signal_wait: owner.OnSignal(signum) def RegisterSignal(self, owner): """Registers a receiver for signal notifications The receiver must support a "OnSignal(self, signum)" function. @type owner: instance @param owner: Receiver """ self._signal_wait.append(owner) def _VerifyDaemonUser(daemon_name): """Verifies the process uid matches the configured uid. This method verifies that a daemon is started as the user it is intended to be run @param daemon_name: The name of daemon to be started @return: A tuple with the first item indicating success or not, the second item current uid and third with expected uid """ getents = runtime.GetEnts() running_uid = os.getuid() daemon_uids = { constants.MASTERD: getents.masterd_uid, constants.RAPI: getents.rapi_uid, constants.NODED: getents.noded_uid, constants.CONFD: getents.confd_uid, } assert daemon_name in daemon_uids, "Invalid daemon %s" % daemon_name return (daemon_uids[daemon_name] == running_uid, running_uid, daemon_uids[daemon_name]) def _BeautifyError(err): """Try to format an error better. Since we're dealing with daemon startup errors, in many cases this will be due to socket error and such, so we try to format these cases better. @param err: an exception object @rtype: string @return: the formatted error description """ try: if isinstance(err, socket.error): return "Socket-related error: %s (errno=%s)" % (err.args[1], err.args[0]) elif isinstance(err, EnvironmentError): if err.filename is None: return "%s (errno=%s)" % (err.strerror, err.errno) else: return "%s (file %s) (errno=%s)" % (err.strerror, err.filename, err.errno) else: return str(err) except Exception: # pylint: disable=W0703 logging.exception("Error while handling existing error %s", err) return "%s" % str(err) def _HandleSigHup(reopen_fn, signum, frame): # pylint: disable=W0613 """Handler for SIGHUP. @param reopen_fn: List of callback functions for reopening log files """ logging.info("Reopening log files after receiving SIGHUP") for fn in reopen_fn: if fn: fn() def GenericMain(daemon_name, optionparser, check_fn, prepare_fn, exec_fn, multithreaded=False, console_logging=False, default_ssl_cert=None, default_ssl_key=None, warn_breach=False): """Shared main function for daemons. @type daemon_name: string @param daemon_name: daemon name @type optionparser: optparse.OptionParser @param optionparser: initialized optionparser with daemon-specific options (common -f -d options will be handled by this module) @type check_fn: function which accepts (options, args) @param check_fn: function that checks start conditions and exits if they're not met @type prepare_fn: function which accepts (options, args) @param prepare_fn: function that is run before forking, or None; it's result will be passed as the third parameter to exec_fn, or if None was passed in, we will just pass None to exec_fn @type exec_fn: function which accepts (options, args, prepare_results) @param exec_fn: function that's executed with the daemon's pid file held, and runs the daemon itself. @type multithreaded: bool @param multithreaded: Whether the daemon uses threads @type console_logging: boolean @param console_logging: if True, the daemon will fall back to the system console if logging fails @type default_ssl_cert: string @param default_ssl_cert: Default SSL certificate path @type default_ssl_key: string @param default_ssl_key: Default SSL key path @type warn_breach: bool @param warn_breach: issue a warning at daemon launch time, before daemonizing, about the possibility of breaking parameter privacy invariants through the otherwise helpful debug logging. """ optionparser.add_option("-f", "--foreground", dest="fork", help="Don't detach from the current terminal", default=True, action="store_false") optionparser.add_option("-d", "--debug", dest="debug", help="Enable some debug messages", default=False, action="store_true") optionparser.add_option("--syslog", dest="syslog", help="Enable logging to syslog (except debug" " messages); one of 'no', 'yes' or 'only' [%s]" % constants.SYSLOG_USAGE, default=constants.SYSLOG_USAGE, choices=["no", "yes", "only"]) family = ssconf.SimpleStore().GetPrimaryIPFamily() # family will default to AF_INET if there is no ssconf file (e.g. when # upgrading a cluster from 2.2 -> 2.3. This is intended, as Ganeti clusters # <= 2.2 can not be AF_INET6 if daemon_name in constants.DAEMONS_PORTS: default_bind_address = constants.IP4_ADDRESS_ANY if family == netutils.IP6Address.family: default_bind_address = constants.IP6_ADDRESS_ANY default_port = netutils.GetDaemonPort(daemon_name) # For networked daemons we allow choosing the port and bind address optionparser.add_option("-p", "--port", dest="port", help="Network port (default: %s)" % default_port, default=default_port, type="int") optionparser.add_option("-b", "--bind", dest="bind_address", help=("Bind address (default: '%s')" % default_bind_address), default=default_bind_address, metavar="ADDRESS") optionparser.add_option("-i", "--interface", dest="bind_interface", help=("Bind interface"), metavar="INTERFACE") if default_ssl_key is not None and default_ssl_cert is not None: optionparser.add_option("--no-ssl", dest="ssl", help="Do not secure HTTP protocol with SSL", default=True, action="store_false") optionparser.add_option("-K", "--ssl-key", dest="ssl_key", help=("SSL key path (default: %s)" % default_ssl_key), default=default_ssl_key, type="string", metavar="SSL_KEY_PATH") optionparser.add_option("-C", "--ssl-cert", dest="ssl_cert", help=("SSL certificate path (default: %s)" % default_ssl_cert), default=default_ssl_cert, type="string", metavar="SSL_CERT_PATH") # Disable the use of fork(2) if the daemon uses threads if multithreaded: utils.DisableFork() options, args = optionparser.parse_args() if getattr(options, "bind_interface", None) is not None: if options.bind_address != default_bind_address: msg = ("Can't specify both, bind address (%s) and bind interface (%s)" % (options.bind_address, options.bind_interface)) print(msg, file=sys.stderr) sys.exit(constants.EXIT_FAILURE) interface_ip_addresses = \ netutils.GetInterfaceIpAddresses(options.bind_interface) if family == netutils.IP6Address.family: if_addresses = interface_ip_addresses[constants.IP6_VERSION] else: if_addresses = interface_ip_addresses[constants.IP4_VERSION] if len(if_addresses) < 1: msg = "Failed to find IP for interface %s" % options.bind_interace print(msg, file=sys.stderr) sys.exit(constants.EXIT_FAILURE) options.bind_address = if_addresses[0] if getattr(options, "ssl", False): ssl_paths = { "certificate": options.ssl_cert, "key": options.ssl_key, } for name, path in ssl_paths.items(): if not os.path.isfile(path): print("SSL %s file '%s' was not found" % (name, path), file=sys.stderr) sys.exit(constants.EXIT_FAILURE) # TODO: By initiating http.HttpSslParams here we would only read the files # once and have a proper validation (isfile returns False on directories) # at the same time. result, running_uid, expected_uid = _VerifyDaemonUser(daemon_name) if not result: msg = ("%s started using wrong user ID (%d), expected %d" % (daemon_name, running_uid, expected_uid)) print(msg, file=sys.stderr) sys.exit(constants.EXIT_FAILURE) if check_fn is not None: check_fn(options, args) log_filename = constants.DAEMONS_LOGFILES[daemon_name] # node-daemon logging in lib/http/server.py, _HandleServerRequestInner if options.debug and warn_breach: sys.stderr.write(constants.DEBUG_MODE_CONFIDENTIALITY_WARNING % daemon_name) if options.fork: # Newer GnuTLS versions (>= 3.3.0) use a library constructor for # initialization and open /dev/urandom on library load time, way before we # fork(). Closing /dev/urandom causes subsequent ganeti.http.client # requests to fail and the process to receive a SIGABRT. As we cannot # reliably detect GnuTLS's socket, we work our way around this by keeping # all fds referring to /dev/urandom open. noclose_fds = [] for fd in os.listdir("/proc/self/fd"): try: if os.readlink(os.path.join("/proc/self/fd", fd)) == "/dev/urandom": noclose_fds.append(int(fd)) except EnvironmentError: # The fd might have disappeared (although it shouldn't as we're running # single-threaded). continue utils.CloseFDs(noclose_fds=noclose_fds) (wpipe, stdio_reopen_fn) = utils.Daemonize(logfile=log_filename) else: (wpipe, stdio_reopen_fn) = (None, None) log_reopen_fn = \ utils.SetupLogging(log_filename, daemon_name, debug=options.debug, stderr_logging=not options.fork, multithreaded=multithreaded, syslog=options.syslog, console_logging=console_logging) # Reopen log file(s) on SIGHUP signal.signal(signal.SIGHUP, compat.partial(_HandleSigHup, [log_reopen_fn, stdio_reopen_fn])) try: utils.WritePidFile(utils.DaemonPidFileName(daemon_name)) except errors.PidFileLockError as err: print("Error while locking PID file:\n%s" % err, file=sys.stderr) sys.exit(constants.EXIT_FAILURE) try: try: logging.info("%s daemon startup", daemon_name) if callable(prepare_fn): prep_results = prepare_fn(options, args) else: prep_results = None except Exception as err: utils.WriteErrorToFD(wpipe, _BeautifyError(err)) raise if wpipe is not None: # we're done with the preparation phase, we close the pipe to # let the parent know it's safe to exit os.close(wpipe) exec_fn(options, args, prep_results) finally: utils.RemoveFile(utils.DaemonPidFileName(daemon_name)) ganeti-3.1.0~rc2/lib/errors.py000064400000000000000000000303401476477700300162400ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Ganeti exception handling. """ from ganeti import constants ECODE_RESOLVER = constants.ERRORS_ECODE_RESOLVER ECODE_NORES = constants.ERRORS_ECODE_NORES ECODE_TEMP_NORES = constants.ERRORS_ECODE_TEMP_NORES ECODE_INVAL = constants.ERRORS_ECODE_INVAL ECODE_STATE = constants.ERRORS_ECODE_STATE ECODE_NOENT = constants.ERRORS_ECODE_NOENT ECODE_EXISTS = constants.ERRORS_ECODE_EXISTS ECODE_NOTUNIQUE = constants.ERRORS_ECODE_NOTUNIQUE ECODE_FAULT = constants.ERRORS_ECODE_FAULT ECODE_ENVIRON = constants.ERRORS_ECODE_ENVIRON ECODE_ALL = constants.ERRORS_ECODE_ALL class GenericError(Exception): """Base exception for Ganeti. """ class LockError(GenericError): """Lock error exception. This signifies problems in the locking subsystem. """ class PidFileLockError(LockError): """PID file is already locked by another process. """ class HypervisorError(GenericError): """Hypervisor-related exception. This is raised in case we can't communicate with the hypervisor properly. """ class HotplugError(HypervisorError): """Hotplug-related exception. This is raised in case a hotplug action fails or is not supported. It is currently used only by KVM hypervisor. """ class ProgrammerError(GenericError): """Programming-related error. This is raised in cases we determine that the calling conventions have been violated, meaning we got some desynchronisation between parts of our code. It signifies a real programming bug. """ class BlockDeviceError(GenericError): """Block-device related exception. This is raised in case we can't setup the instance's block devices properly. """ class ConfigurationError(GenericError): """Configuration related exception. Things like having an instance with a primary node that doesn't exist in the config or such raise this exception. """ class ConfigVersionMismatch(ConfigurationError): """Version mismatch in the configuration file. The error has two arguments: the expected and the actual found version. """ class ConfigVerifyError(ConfigurationError): """Error reported by configuration verification The error has two arguments: the main error message and a list of errors found. """ class AddressPoolError(GenericError): """Errors related to IP address pools. """ class ReservationError(GenericError): """Errors reserving a resource. """ class RemoteError(GenericError): """Programming-related error on remote call. This is raised when an unhandled error occurs in a call to a remote node. It usually signifies a real programming bug. """ class SignatureError(GenericError): """Error authenticating a remote message. This is raised when the hmac signature on a message doesn't verify correctly to the message itself. It can happen because of network unreliability or because of spurious traffic. """ class ParameterError(GenericError): """A passed parameter to a command is invalid. This is raised when the parameter passed to a request function is invalid. Correct code should have verified this before passing the request structure. The argument to this exception should be the parameter name. """ class ResultValidationError(GenericError): """The iallocation results fails validation. """ class OpPrereqError(GenericError): """Prerequisites for the OpCode are not fulfilled. This exception has two arguments: an error message, and one of the ECODE_* codes. """ class OpExecError(GenericError): """Error during OpCode execution. """ class OpResultError(GenericError): """Issue with OpCode result. """ class OpRetryNotSupportedError(GenericError): """This opcode does not support retries """ class DeviceCreationError(GenericError): """Error during the creation of a device. This exception should contain the list of the devices actually created up to now, in the form of pairs (node, device) """ def __init__(self, message, created_devices): GenericError.__init__(self) self.message = message self.created_devices = created_devices def __str__(self): return self.message class OpCodeUnknown(GenericError): """Unknown opcode submitted. This signifies a mismatch between the definitions on the client and server side. """ class JobLost(GenericError): """Submitted job lost. The job was submitted but it cannot be found in the current job list. """ class JobCanceled(GenericError): """Submitted job was canceled. The job that was submitted has transitioned to a canceling or canceled state. """ class JobFileCorrupted(GenericError): """Job file could not be properly decoded/restored. """ class ResolverError(GenericError): """Host name cannot be resolved. This is not a normal situation for Ganeti, as we rely on having a working resolver. The non-resolvable hostname is available as the first element of the args tuple; the other two elements of the tuple are the first two args of the socket.gaierror exception (error code and description). """ class HooksFailure(GenericError): """A generic hook failure. This signifies usually a setup misconfiguration. """ class HooksAbort(HooksFailure): """A required hook has failed. This caused an abort of the operation in the initial phase. This exception always has an attribute args which is a list of tuples of: - node: the source node on which this hooks has failed - script: the name of the script which aborted the run """ class UnitParseError(GenericError): """Unable to parse size unit. """ class ParseError(GenericError): """Generic parse error. Raised when unable to parse user input. """ class TypeEnforcementError(GenericError): """Unable to enforce data type. """ class X509CertError(GenericError): """Invalid X509 certificate. This error has two arguments: the certificate filename and the error cause. """ class TagError(GenericError): """Generic tag error. The argument to this exception will show the exact error. """ class CommandError(GenericError): """External command error. """ class StorageError(GenericError): """Storage-related exception. """ class InotifyError(GenericError): """Error raised when there is a failure setting up an inotify watcher. """ class QuitGanetiException(Exception): """Signal Ganeti that it must quit. This is not necessarily an error (and thus not a subclass of GenericError), but it's an exceptional circumstance and it is thus treated. This exception should be instantiated with two values. The first one will specify the return code to the caller, and the second one will be the returned result (either as an error or as a normal result). Usually only the leave cluster rpc call should return status True (as there it's expected we quit), every other call will return status False (as a critical error was encountered). Examples:: # Return a result of "True" to the caller, but quit ganeti afterwards raise QuitGanetiException(True, None) # Send an error to the caller, and quit ganeti raise QuitGanetiException(False, "Fatal safety violation, shutting down") """ class JobQueueError(GenericError): """Job queue error. """ class JobQueueDrainError(JobQueueError): """Job queue is marked for drain error. This is raised when a job submission attempt is made but the queue is marked for drain. """ class JobQueueFull(JobQueueError): """Job queue full error. Raised when job queue size reached its hard limit. """ class ConfdMagicError(GenericError): """A magic fourcc error in Ganeti confd. Errors processing the fourcc in ganeti confd datagrams. """ class ConfdClientError(GenericError): """A magic fourcc error in Ganeti confd. Errors in the confd client library. """ class UdpDataSizeError(GenericError): """UDP payload too big. """ class NoCtypesError(GenericError): """python ctypes module is not found in the system. """ class IPAddressError(GenericError): """Generic IP address error. """ class LuxiError(GenericError): """LUXI error. """ class QueryFilterParseError(ParseError): """Error while parsing query filter. This exception must be instantiated with two values. The first one is a string with an error description, the second one is an instance of a subclass of C{pyparsing.ParseBaseException} (used to display the exact error location). """ def GetDetails(self): """Returns a list of strings with details about the error. """ try: (_, inner) = self.args except IndexError: return None return [str(inner.line), (" " * (inner.column - 1)) + "^", str(inner)] class RapiTestResult(GenericError): """Exception containing results from RAPI test utilities. """ class FileStoragePathError(GenericError): """Error from file storage path validation. """ class SshUpdateError(GenericError): """Error from updating the SSH setup. """ class JobSubmittedException(Exception): """Job was submitted, client should exit. This exception has one argument, the ID of the job that was submitted. The handler should print this ID. This is not an error, just a structured way to exit from clients. """ # errors should be added above def GetErrorClass(name): """Return the class of an exception. Given the class name, return the class itself. @type name: str @param name: the exception name @rtype: class @return: the actual class, or None if not found """ item = globals().get(name, None) if item is not None: if not (isinstance(item, type(Exception)) and issubclass(item, GenericError)): item = None return item def EncodeException(err): """Encodes an exception into a format that L{MaybeRaise} will recognise. The passed L{err} argument will be formatted as a tuple (exception name, arguments) that the MaybeRaise function will recognise. @type err: GenericError child @param err: usually a child of GenericError (but any exception will be accepted) @rtype: tuple @return: tuple of (exception name, exception arguments) """ return (err.__class__.__name__, err.args) def GetEncodedError(result): """If this looks like an encoded Ganeti exception, return it. This function tries to parse the passed argument and if it looks like an encoding done by EncodeException, it will return the class object and arguments. """ tlt = (tuple, list) if (isinstance(result, tlt) and len(result) == 2 and isinstance(result[1], tlt)): # custom ganeti errors errcls = GetErrorClass(result[0]) if errcls: return (errcls, tuple(result[1])) return None def MaybeRaise(result): """If this looks like an encoded Ganeti exception, raise it. This function tries to parse the passed argument and if it looks like an encoding done by EncodeException, it will re-raise it. """ error = GetEncodedError(result) if error: (errcls, args) = error raise errcls(*args) ganeti-3.1.0~rc2/lib/hooksmaster.py000064400000000000000000000250411476477700300172650ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing the logic for running hooks. """ from ganeti import constants from ganeti import errors from ganeti import utils from ganeti import compat from ganeti import pathutils def _RpcResultsToHooksResults(rpc_results): """Function to convert RPC results to the format expected by HooksMaster. @type rpc_results: dict(node: L{rpc.RpcResult}) @param rpc_results: RPC results @rtype: dict(node: (fail_msg, offline, hooks_results)) @return: RPC results unpacked according to the format expected by L({hooksmaster.HooksMaster} """ return dict((node, (rpc_res.fail_msg, rpc_res.offline, rpc_res.payload)) for (node, rpc_res) in rpc_results.items()) class HooksMaster(object): def __init__(self, opcode, hooks_path, nodes, hooks_execution_fn, hooks_results_adapt_fn, build_env_fn, prepare_post_nodes_fn, log_fn, htype=None, cluster_name=None, master_name=None): """Base class for hooks masters. This class invokes the execution of hooks according to the behaviour specified by its parameters. @type opcode: string @param opcode: opcode of the operation to which the hooks are tied @type hooks_path: string @param hooks_path: prefix of the hooks directories @type nodes: 2-tuple of lists @param nodes: 2-tuple of lists containing nodes on which pre-hooks must be run and nodes on which post-hooks must be run @type hooks_execution_fn: function that accepts the following parameters: (node_list, hooks_path, phase, environment) @param hooks_execution_fn: function that will execute the hooks; can be None, indicating that no conversion is necessary. @type hooks_results_adapt_fn: function @param hooks_results_adapt_fn: function that will adapt the return value of hooks_execution_fn to the format expected by RunPhase @type build_env_fn: function that returns a dictionary having strings as keys @param build_env_fn: function that builds the environment for the hooks @type prepare_post_nodes_fn: function that take a list of node UUIDs and returns a list of node UUIDs @param prepare_post_nodes_fn: function that is invoked right before executing post hooks and can change the list of node UUIDs to run the post hooks on @type log_fn: function that accepts a string @param log_fn: logging function @type htype: string or None @param htype: None or one of L{constants.HTYPE_CLUSTER}, L{constants.HTYPE_NODE}, L{constants.HTYPE_INSTANCE} @type cluster_name: string @param cluster_name: name of the cluster @type master_name: string @param master_name: name of the master """ self.opcode = opcode self.hooks_path = hooks_path self.hooks_execution_fn = hooks_execution_fn self.hooks_results_adapt_fn = hooks_results_adapt_fn self.build_env_fn = build_env_fn self.prepare_post_nodes_fn = prepare_post_nodes_fn self.log_fn = log_fn self.htype = htype self.cluster_name = cluster_name self.master_name = master_name self.pre_env = self._BuildEnv(constants.HOOKS_PHASE_PRE) (self.pre_nodes, self.post_nodes) = nodes def _BuildEnv(self, phase): """Compute the environment and the target nodes. Based on the opcode and the current node list, this builds the environment for the hooks and the target node list for the run. """ if phase == constants.HOOKS_PHASE_PRE: prefix = "GANETI_" elif phase == constants.HOOKS_PHASE_POST: prefix = "GANETI_POST_" else: raise AssertionError("Unknown phase '%s'" % phase) env = {} if self.hooks_path is not None: phase_env = self.build_env_fn() if phase_env: assert not compat.any(key.upper().startswith(prefix) for key in phase_env) env.update(("%s%s" % (prefix, key), value) for (key, value) in phase_env.items()) if phase == constants.HOOKS_PHASE_PRE: assert compat.all((key.startswith("GANETI_") and not key.startswith("GANETI_POST_")) for key in env) elif phase == constants.HOOKS_PHASE_POST: assert compat.all(key.startswith("GANETI_POST_") for key in env) assert isinstance(self.pre_env, dict) # Merge with pre-phase environment assert not compat.any(key.startswith("GANETI_POST_") for key in self.pre_env) env.update(self.pre_env) else: raise AssertionError("Unknown phase '%s'" % phase) return env def _RunWrapper(self, node_list, hpath, phase, phase_env): """Simple wrapper over self.callfn. This method fixes the environment before executing the hooks. """ env = { "PATH": constants.HOOKS_PATH, "GANETI_HOOKS_VERSION": constants.HOOKS_VERSION, "GANETI_OP_CODE": self.opcode, "GANETI_DATA_DIR": pathutils.DATA_DIR, "GANETI_HOOKS_PHASE": phase, "GANETI_HOOKS_PATH": hpath, } if self.htype: env["GANETI_OBJECT_TYPE"] = self.htype if self.cluster_name is not None: env["GANETI_CLUSTER"] = self.cluster_name if self.master_name is not None: env["GANETI_MASTER"] = self.master_name if phase_env: env = utils.algo.JoinDisjointDicts(env, phase_env) # Convert everything to strings env = dict([(str(key), str(val)) for key, val in env.items()]) assert compat.all(key == "PATH" or key.startswith("GANETI_") for key in env) return self.hooks_execution_fn(node_list, hpath, phase, env) def RunPhase(self, phase, node_names=None): """Run all the scripts for a phase. This is the main function of the HookMaster. It executes self.hooks_execution_fn, and after running self.hooks_results_adapt_fn on its results it expects them to be in the form {node_name: (fail_msg, [(script, result, output), ...]}). @param phase: one of L{constants.HOOKS_PHASE_POST} or L{constants.HOOKS_PHASE_PRE}; it denotes the hooks phase @param node_names: overrides the predefined list of nodes for the given phase @return: the processed results of the hooks multi-node rpc call @raise errors.HooksFailure: on communication failure to the nodes @raise errors.HooksAbort: on failure of one of the hooks """ if phase == constants.HOOKS_PHASE_PRE: if node_names is None: node_names = self.pre_nodes env = self.pre_env elif phase == constants.HOOKS_PHASE_POST: if node_names is None: node_names = self.post_nodes if node_names is not None and self.prepare_post_nodes_fn is not None: node_names = frozenset(self.prepare_post_nodes_fn(list(node_names))) env = self._BuildEnv(phase) else: raise AssertionError("Unknown phase '%s'" % phase) if not node_names: # empty node list, we should not attempt to run this as either # we're in the cluster init phase and the rpc client part can't # even attempt to run, or this LU doesn't do hooks at all return results = self._RunWrapper(node_names, self.hooks_path, phase, env) if not results: msg = "Communication Failure" if phase == constants.HOOKS_PHASE_PRE: raise errors.HooksFailure(msg) else: self.log_fn(msg) return results converted_res = results if self.hooks_results_adapt_fn: converted_res = self.hooks_results_adapt_fn(results) errs = [] for node_name, (fail_msg, offline, hooks_results) in converted_res.items(): if offline: continue if fail_msg: self.log_fn("Communication failure to node %s: %s", node_name, fail_msg) continue for script, hkr, output in hooks_results: if hkr == constants.HKR_FAIL: if phase == constants.HOOKS_PHASE_PRE: errs.append((node_name, script, output)) else: if not output: output = "(no output)" self.log_fn("On %s script %s failed, output: %s" % (node_name, script, output)) if errs and phase == constants.HOOKS_PHASE_PRE: raise errors.HooksAbort(errs) return results def RunConfigUpdate(self): """Run the special configuration update hook This is a special hook that runs only on the master after each top-level LI if the configuration has been updated. """ phase = constants.HOOKS_PHASE_POST hpath = constants.HOOKS_NAME_CFGUPDATE nodes = [self.master_name] self._RunWrapper(nodes, hpath, phase, self.pre_env) @staticmethod def BuildFromLu(hooks_execution_fn, lu): if lu.HPATH is None: nodes = (None, None) else: hooks_nodes = lu.BuildHooksNodes() if len(hooks_nodes) != 2: raise errors.ProgrammerError( "LogicalUnit.BuildHooksNodes must return a 2-tuple") nodes = (frozenset(hooks_nodes[0]), frozenset(hooks_nodes[1])) master_name = cluster_name = None if lu.cfg: master_name = lu.cfg.GetMasterNodeName() cluster_name = lu.cfg.GetClusterName() return HooksMaster(lu.op.OP_ID, lu.HPATH, nodes, hooks_execution_fn, _RpcResultsToHooksResults, lu.BuildHooksEnv, lu.PreparePostHookNodes, lu.LogWarning, lu.HTYPE, cluster_name, master_name) ganeti-3.1.0~rc2/lib/ht.py000064400000000000000000000451071476477700300153460ustar00rootroot00000000000000# # # Copyright (C) 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing the parameter types code.""" import re import operator import ipaddress from ganeti import compat from ganeti import utils from ganeti import constants from ganeti import objects from ganeti.serializer import Private _PAREN_RE = re.compile("^[a-zA-Z0-9_-]+$") def Parens(text): """Enclose text in parens if necessary. @param text: Text """ text = str(text) if _PAREN_RE.match(text): return text else: return "(%s)" % text class _WrapperBase(object): __slots__ = [ "_fn", "_text", ] def __init__(self, text, fn): """Initializes this class. @param text: Description @param fn: Wrapped function """ assert text.strip() self._text = text self._fn = fn def __call__(self, *args): return self._fn(*args) class _DescWrapper(_WrapperBase): """Wrapper class for description text. """ def __str__(self): return self._text def __repr__(self): return "<%s %r>" % (self._text, self._fn) class _CommentWrapper(_WrapperBase): """Wrapper class for comment. """ def __str__(self): return "%s [%s]" % (self._fn, self._text) def WithDesc(text): """Builds wrapper class with description text. @type text: string @param text: Description text @return: Callable class """ assert text[0] == text[0].upper() return compat.partial(_DescWrapper, text) def Comment(text): """Builds wrapper for adding comment to description text. @type text: string @param text: Comment text @return: Callable class """ assert not frozenset(text).intersection("[]") return compat.partial(_CommentWrapper, text) def CombinationDesc(op, args, fn): """Build description for combinating operator. @type op: string @param op: Operator as text (e.g. "and") @type args: list @param args: Operator arguments @type fn: callable @param fn: Wrapped function """ # Some type descriptions are rather long. If "None" is listed at the # end or somewhere in between it is easily missed. Therefore it should # be at the beginning, e.g. "None or (long description)". if __debug__ and TNone in args and args.index(TNone) > 0: raise Exception("TNone must be listed first") if len(args) == 1: descr = str(args[0]) else: descr = (" %s " % op).join(Parens(i) for i in args) return WithDesc(descr)(fn) # Modifiable default values; need to define these here before the # actual LUs @WithDesc(str([])) def EmptyList(): """Returns an empty list. """ return [] @WithDesc(str({})) def EmptyDict(): """Returns an empty dict. """ return {} #: The without-default default value NoDefault = object() # Some basic types @WithDesc("Anything") def TAny(_): """Accepts any value. """ return True @WithDesc("NotNone") def TNotNone(val): """Checks if the given value is not None. """ return val is not None @WithDesc("None") def TNone(val): """Checks if the given value is None. """ return val is None @WithDesc("ValueNone") def TValueNone(val): """Checks if the given value is L{constants.VALUE_NONE}. """ return val == constants.VALUE_NONE @WithDesc("Boolean") def TBool(val): """Checks if the given value is a boolean. """ return isinstance(val, bool) @WithDesc("Integer") def TInt(val): """Checks if the given value is an integer. """ # For backwards compatibility with older Python versions, boolean values are # also integers and should be excluded in this test. # # >>> (isinstance(False, int), isinstance(True, int)) # (True, True) return isinstance(val, int) and not isinstance(val, bool) @WithDesc("Float") def TFloat(val): """Checks if the given value is a float. """ return isinstance(val, float) @WithDesc("String") def TString(val): """Checks if the given value is a string. """ return isinstance(val, str) @WithDesc("EvalToTrue") def TTrue(val): """Checks if a given value evaluates to a boolean True value. """ return bool(val) def TElemOf(target_list): """Builds a function that checks if a given value is a member of a list. """ def fn(val): return val in target_list return WithDesc("OneOf %s" % (utils.CommaJoin(sorted(target_list)), ))(fn) # Container types @WithDesc("List") def TList(val): """Checks if the given value is a list. """ return isinstance(val, list) @WithDesc("Tuple") def TTuple(val): """Checks if the given value is a tuple. """ return isinstance(val, tuple) @WithDesc("Dictionary") def TDict(val): """Checks if the given value is a dictionary. Note that L{PrivateDict}s subclass dict and pass this check. """ return isinstance(val, dict) def TIsLength(size): """Check is the given container is of the given size. """ def fn(container): return len(container) == size return WithDesc("Length %s" % (size, ))(fn) # Combinator types def TAnd(*args): """Combine multiple functions using an AND operation. """ def fn(val): return compat.all(t(val) for t in args) return CombinationDesc("and", args, fn) def TOr(*args): """Combine multiple functions using an OR operation. """ def fn(val): return compat.any(t(val) for t in args) return CombinationDesc("or", args, fn) def TMap(fn, test): """Checks that a modified version of the argument passes the given test. """ return WithDesc("Result of %s must be %s" % (Parens(fn), Parens(test)))(lambda val: test(fn(val))) def TRegex(pobj): """Checks whether a string matches a specific regular expression. @param pobj: Compiled regular expression as returned by C{re.compile} """ desc = WithDesc("String matching regex \"%s\"" % pobj.pattern.encode("unicode_escape")) return desc(TAnd(TString, pobj.match)) def TMaybe(test): """Wrap a test in a TOr(TNone, test). This makes it easier to define TMaybe* types. """ return TOr(TNone, test) def TMaybeValueNone(test): """Used for unsetting values. """ return TMaybe(TOr(TValueNone, test)) # Type aliases #: a non-empty string TNonEmptyString = WithDesc("NonEmptyString")(TAnd(TString, TTrue)) #: a maybe non-empty string TMaybeString = TMaybe(TNonEmptyString) #: a maybe boolean (bool or none) TMaybeBool = TMaybe(TBool) #: Maybe a dictionary (dict or None) TMaybeDict = TMaybe(TDict) #: Maybe a list (list or None) TMaybeList = TMaybe(TList) #: a non-negative number (value > 0) # val_type should be TInt, TDouble (== TFloat), or TNumber def TNonNegative(val_type): return WithDesc("EqualOrGreaterThanZero")(TAnd(val_type, lambda v: v >= 0)) #: a positive number (value >= 0) # val_type should be TInt, TDouble (== TFloat), or TNumber def TPositive(val_type): return WithDesc("GreaterThanZero")(TAnd(val_type, lambda v: v > 0)) #: a non-negative integer (value >= 0) TNonNegativeInt = TNonNegative(TInt) #: a positive integer (value > 0) TPositiveInt = TPositive(TInt) #: a maybe positive integer (positive integer or None) TMaybePositiveInt = TMaybe(TPositiveInt) #: a negative integer (value < 0) TNegativeInt = \ TAnd(TInt, WithDesc("LessThanZero")(compat.partial(operator.gt, 0))) #: a positive float TNonNegativeFloat = \ TAnd(TFloat, WithDesc("EqualOrGreaterThanZero")(lambda v: v >= 0.0)) #: Job ID TJobId = WithDesc("JobId")(TOr(TNonNegativeInt, TRegex(re.compile("^%s$" % constants.JOB_ID_TEMPLATE)))) #: Double (== Float) TDouble = TFloat #: Number TNumber = TOr(TInt, TFloat) #: Relative job ID TRelativeJobId = WithDesc("RelativeJobId")(TNegativeInt) def TInstanceOf(cls): """Checks if a given value is an instance of C{cls}. @type cls: class @param cls: Class object """ name = "%s.%s" % (cls.__module__, cls.__name__) desc = WithDesc("Instance of %s" % (Parens(name), )) return desc(lambda val: isinstance(val, cls)) def TPrivate(val_type): """Checks if a given value is an instance of Private. """ def fn(val): return isinstance(val, Private) and val_type(val.Get()) desc = WithDesc("Private %s" % Parens(val_type)) return desc(fn) def TSecret(val_type): """Checks if a given value is an instance of Private. However, the type is named Secret in the Haskell equivalent. """ def fn(val): return isinstance(val, Private) and val_type(val.Get()) desc = WithDesc("Private %s" % Parens(val_type)) return desc(fn) def TListOf(my_type): """Checks if a given value is a list with all elements of the same type. """ desc = WithDesc("List of %s" % (Parens(my_type), )) return desc(TAnd(TList, lambda lst: compat.all(my_type(v) for v in lst))) TMaybeListOf = lambda item_type: TMaybe(TListOf(item_type)) def TTupleOf(*val_types): """Checks if a given value is a list with the proper size and its elements match the given types. """ desc = WithDesc("Tuple of %s" % Parens(', '.join(str(v) for v in val_types))) return desc(TAnd(TOr(TTuple, TList), TIsLength(len(val_types)), TItems(val_types))) def TSetOf(val_type): """Checks if a given value is a list with all elements of the same type and eliminates duplicated elements. """ desc = WithDesc("Set of %s" % (Parens(val_type), )) return desc(lambda st: TListOf(val_type)(list(set(st)))) def TDictOf(key_type, val_type): """Checks a dict type for the type of its key/values. """ desc = WithDesc("Dictionary with keys of %s and values of %s" % (Parens(key_type), Parens(val_type))) def fn(container): return (compat.all(key_type(v) for v in container.keys()) and compat.all(val_type(v) for v in container.values())) return desc(TAnd(TDict, fn)) def _TStrictDictCheck(require_all, exclusive, items, val): """Helper function for L{TStrictDict}. """ notfound_fn = lambda _: not exclusive if require_all and not frozenset(val).issuperset(items): # Requires items not found in value return False return compat.all(items.get(key, notfound_fn)(value) for (key, value) in val.items()) def TStrictDict(require_all, exclusive, items): """Strict dictionary check with specific keys. @type require_all: boolean @param require_all: Whether all keys in L{items} are required @type exclusive: boolean @param exclusive: Whether only keys listed in L{items} should be accepted @type items: dictionary @param items: Mapping from key (string) to verification function """ descparts = ["Dictionary containing"] if exclusive: descparts.append(" none but the") if require_all: descparts.append(" required") if len(items) == 1: descparts.append(" key ") else: descparts.append(" keys ") descparts.append(utils.CommaJoin("\"%s\" (value %s)" % (key, value) for (key, value) in items.items())) desc = WithDesc("".join(descparts)) return desc(TAnd(TDict, compat.partial(_TStrictDictCheck, require_all, exclusive, items))) def TItems(items): """Checks individual items of a container. If the verified value and the list of expected items differ in length, this check considers only as many items as are contained in the shorter list. Use L{TIsLength} to enforce a certain length. @type items: list @param items: List of checks """ assert items, "Need items" text = ["Item", "item"] desc = WithDesc(utils.CommaJoin("%s %s is %s" % (text[int(idx > 0)], idx, Parens(check)) for (idx, check) in enumerate(items))) return desc(lambda value: compat.all(check(i) for (check, i) in zip(items, value))) TMaxValue = lambda max: WithDesc('Less than %s' % max)(lambda val: val < max) TAllocPolicy = TElemOf(constants.VALID_ALLOC_POLICIES) TCVErrorCode = TElemOf(constants.CV_ALL_ECODES_STRINGS) TQueryResultCode = TElemOf(constants.RS_ALL) TExportTarget = TOr(TNonEmptyString, TList) TExportMode = TElemOf(constants.EXPORT_MODES) TDiskIndex = TAnd(TNonNegativeInt, TMaxValue(constants.MAX_DISKS)) TReplaceDisksMode = TElemOf(constants.REPLACE_MODES) TDiskTemplate = TElemOf(constants.DISK_TEMPLATES) TEvacMode = TElemOf(constants.NODE_EVAC_MODES) TIAllocatorTestDir = TElemOf(constants.VALID_IALLOCATOR_DIRECTIONS) TIAllocatorMode = TElemOf(constants.VALID_IALLOCATOR_MODES) TImportExportCompression = TElemOf(constants.IEC_ALL) TAdminStateSource = TElemOf(constants.ADMIN_STATE_SOURCES) def TSetParamsMods(fn): """Generates a check for modification lists. """ # Old format # TODO: Remove in version 2.11 including support in LUInstanceSetParams old_mod_item_fn = \ TAnd(TIsLength(2), TItems([TOr(TElemOf(constants.DDMS_VALUES), TNonNegativeInt), fn])) # New format, supporting adding/removing disks/NICs at arbitrary indices mod_item_fn = \ TAnd(TIsLength(3), TItems([ TElemOf(constants.DDMS_VALUES_WITH_MODIFY), Comment("Device index, can be negative, e.g. -1 for last disk") (TOr(TInt, TString)), fn, ])) return TOr(Comment("Recommended")(TListOf(mod_item_fn)), Comment("Deprecated")(TListOf(old_mod_item_fn))) TINicParams = \ Comment("NIC parameters")(TDictOf(TElemOf(constants.INIC_PARAMS), TMaybe(TString))) TIDiskParams = \ Comment("Disk parameters")(TDictOf(TNonEmptyString, TOr(TNonEmptyString, TInt))) THypervisor = TElemOf(constants.HYPER_TYPES) TMigrationMode = TElemOf(constants.HT_MIGRATION_MODES) TNICMode = TElemOf(constants.NIC_VALID_MODES) TInstCreateMode = TElemOf(constants.INSTANCE_CREATE_MODES) TRebootType = TElemOf(constants.REBOOT_TYPES) TFileDriver = TElemOf(constants.FILE_DRIVER) TOobCommand = TElemOf(constants.OOB_COMMANDS) # FIXME: adjust this after all queries are in haskell TQueryTypeOp = TElemOf(set(constants.QR_VIA_OP) .union(set(constants.QR_VIA_LUXI))) TDiskParams = \ Comment("Disk parameters")(TDictOf(TNonEmptyString, TOr(TNonEmptyString, TInt))) TDiskChanges = \ TAnd(TIsLength(2), TItems([Comment("Disk index")(TNonNegativeInt), Comment("Parameters")(TDiskParams)])) TRecreateDisksInfo = TOr(TListOf(TNonNegativeInt), TListOf(TDiskChanges)) def TStorageType(val): """Builds a function that checks if a given value is a valid storage type. """ return (val in constants.STORAGE_TYPES) TTagKind = TElemOf(constants.VALID_TAG_TYPES) TDdmSimple = TElemOf(constants.DDMS_VALUES) TVerifyOptionalChecks = TElemOf(constants.VERIFY_OPTIONAL_CHECKS) TSshKeyType = TElemOf(constants.SSHK_ALL) @WithDesc("IPv4 network") def _CheckCIDRNetNotation(value): """Ensure a given CIDR notation type is valid. """ try: ipaddress.IPv4Network(value) except ipaddress.AddressValueError: return False return True @WithDesc("IPv4 address") def _CheckCIDRAddrNotation(value): """Ensure a given CIDR notation type is valid. """ try: ipaddress.IPv4Address(value) except ipaddress.AddressValueError: return False return True @WithDesc("IPv6 address") def _CheckCIDR6AddrNotation(value): """Ensure a given CIDR notation type is valid. """ try: ipaddress.IPv6Address(value) except ipaddress.AddressValueError: return False return True @WithDesc("IPv6 network") def _CheckCIDR6NetNotation(value): """Ensure a given CIDR notation type is valid. """ try: ipaddress.IPv6Network(value) except ipaddress.AddressValueError: return False return True TIPv4Address = TAnd(TString, _CheckCIDRAddrNotation) TIPv6Address = TAnd(TString, _CheckCIDR6AddrNotation) TIPv4Network = TAnd(TString, _CheckCIDRNetNotation) TIPv6Network = TAnd(TString, _CheckCIDR6NetNotation) def TObject(val_type): return TDictOf(TAny, val_type) def TObjectCheck(obj, fields_types): """Helper to generate type checks for objects. @param obj: The object to generate type checks @param fields_types: The fields and their types as a dict @return: A ht type check function """ assert set(obj.GetAllSlots()) == set(fields_types.keys()), \ "%s != %s" % (set(obj.GetAllSlots()), set(fields_types.keys())) return TStrictDict(True, True, fields_types) TQueryFieldDef = \ TObjectCheck(objects.QueryFieldDefinition, { "name": TNonEmptyString, "title": TNonEmptyString, "kind": TElemOf(constants.QFT_ALL), "doc": TNonEmptyString }) TQueryRow = \ TListOf(TAnd(TIsLength(2), TItems([TElemOf(constants.RS_ALL), TAny]))) TQueryResult = TListOf(TQueryRow) TQueryResponse = \ TObjectCheck(objects.QueryResponse, { "fields": TListOf(TQueryFieldDef), "data": TQueryResult }) TQueryFieldsResponse = \ TObjectCheck(objects.QueryFieldsResponse, { "fields": TListOf(TQueryFieldDef) }) TJobIdListItem = \ TAnd(TIsLength(2), TItems([Comment("success")(TBool), Comment("Job ID if successful, error message" " otherwise")(TOr(TString, TJobId))])) TJobIdList = TListOf(TJobIdListItem) TJobIdListOnly = TStrictDict(True, True, { constants.JOB_IDS_KEY: Comment("List of submitted jobs")(TJobIdList) }) TInstanceMultiAllocResponse = \ TStrictDict(True, True, { constants.JOB_IDS_KEY: Comment("List of submitted jobs")(TJobIdList), constants.ALLOCATABLE_KEY: TListOf(TNonEmptyString), constants.FAILED_KEY: TListOf(TNonEmptyString) }) ganeti-3.1.0~rc2/lib/http/000075500000000000000000000000001476477700300153315ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/http/__init__.py000064400000000000000000000734051476477700300174530ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008, 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """HTTP module. """ import errno import email import logging import select import socket from io import StringIO import OpenSSL from ganeti import constants from ganeti import utils HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION HTTP_OK = 200 HTTP_NO_CONTENT = 204 HTTP_NOT_MODIFIED = 304 HTTP_0_9 = "HTTP/0.9" HTTP_1_0 = "HTTP/1.0" HTTP_1_1 = "HTTP/1.1" HTTP_GET = "GET" HTTP_HEAD = "HEAD" HTTP_POST = "POST" HTTP_PUT = "PUT" HTTP_DELETE = "DELETE" HTTP_ETAG = "ETag" HTTP_HOST = "Host" HTTP_SERVER = "Server" HTTP_DATE = "Date" HTTP_USER_AGENT = "User-Agent" HTTP_CONTENT_TYPE = "Content-Type" HTTP_CONTENT_LENGTH = "Content-Length" HTTP_CONNECTION = "Connection" HTTP_KEEP_ALIVE = "Keep-Alive" HTTP_WWW_AUTHENTICATE = "WWW-Authenticate" HTTP_AUTHORIZATION = "Authorization" HTTP_AUTHENTICATION_INFO = "Authentication-Info" HTTP_ALLOW = "Allow" HTTP_APP_OCTET_STREAM = "application/octet-stream" HTTP_APP_JSON = "application/json" _SSL_UNEXPECTED_EOF = "Unexpected EOF" _SSL_SHUTDOWN_DURING_INIT = ('SSL routines', 'SSL_shutdown', 'shutdown while in init') # Socket operations (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_SHUTDOWN, SOCKOP_HANDSHAKE) = range(4) # send/receive quantum SOCK_BUF_SIZE = 32768 # OpenSSL.SSL.ConnectionType was deprecated in pyopenssl-19.1.0: try: SSL_CONN_TYPE = OpenSSL.SSL.Connection except AttributeError: SSL_CONN_TYPE = OpenSSL.SSL.ConnectionType class HttpError(Exception): """Internal exception for HTTP errors. This should only be used for internal error reporting. """ class HttpConnectionClosed(Exception): """Internal exception for a closed connection. This should only be used for internal error reporting. Only use it if there's no other way to report this condition. """ class HttpSessionHandshakeUnexpectedEOF(HttpError): """Internal exception for errors during SSL handshake. This should only be used for internal error reporting. """ class HttpSocketTimeout(Exception): """Internal exception for socket timeouts. This should only be used for internal error reporting. """ class HttpException(Exception): code = None message = None def __init__(self, message=None, headers=None): Exception.__init__(self) self.message = message self.headers = headers class HttpBadRequest(HttpException): """400 Bad Request RFC2616, 10.4.1: The request could not be understood by the server due to malformed syntax. The client SHOULD NOT repeat the request without modifications. """ code = 400 class HttpUnauthorized(HttpException): """401 Unauthorized RFC2616, section 10.4.2: The request requires user authentication. The response MUST include a WWW-Authenticate header field (section 14.47) containing a challenge applicable to the requested resource. """ code = 401 class HttpForbidden(HttpException): """403 Forbidden RFC2616, 10.4.4: The server understood the request, but is refusing to fulfill it. Authorization will not help and the request SHOULD NOT be repeated. """ code = 403 class HttpNotFound(HttpException): """404 Not Found RFC2616, 10.4.5: The server has not found anything matching the Request-URI. No indication is given of whether the condition is temporary or permanent. """ code = 404 class HttpMethodNotAllowed(HttpException): """405 Method Not Allowed RFC2616, 10.4.6: The method specified in the Request-Line is not allowed for the resource identified by the Request-URI. The response MUST include an Allow header containing a list of valid methods for the requested resource. """ code = 405 class HttpNotAcceptable(HttpException): """406 Not Acceptable RFC2616, 10.4.7: The resource identified by the request is only capable of generating response entities which have content characteristics not acceptable according to the accept headers sent in the request. """ code = 406 class HttpRequestTimeout(HttpException): """408 Request Timeout RFC2616, 10.4.9: The client did not produce a request within the time that the server was prepared to wait. The client MAY repeat the request without modifications at any later time. """ code = 408 class HttpConflict(HttpException): """409 Conflict RFC2616, 10.4.10: The request could not be completed due to a conflict with the current state of the resource. This code is only allowed in situations where it is expected that the user might be able to resolve the conflict and resubmit the request. """ code = 409 class HttpGone(HttpException): """410 Gone RFC2616, 10.4.11: The requested resource is no longer available at the server and no forwarding address is known. This condition is expected to be considered permanent. """ code = 410 class HttpLengthRequired(HttpException): """411 Length Required RFC2616, 10.4.12: The server refuses to accept the request without a defined Content-Length. The client MAY repeat the request if it adds a valid Content-Length header field containing the length of the message-body in the request message. """ code = 411 class HttpPreconditionFailed(HttpException): """412 Precondition Failed RFC2616, 10.4.13: The precondition given in one or more of the request-header fields evaluated to false when it was tested on the server. """ code = 412 class HttpUnsupportedMediaType(HttpException): """415 Unsupported Media Type RFC2616, 10.4.16: The server is refusing to service the request because the entity of the request is in a format not supported by the requested resource for the requested method. """ code = 415 class HttpInternalServerError(HttpException): """500 Internal Server Error RFC2616, 10.5.1: The server encountered an unexpected condition which prevented it from fulfilling the request. """ code = 500 class HttpNotImplemented(HttpException): """501 Not Implemented RFC2616, 10.5.2: The server does not support the functionality required to fulfill the request. """ code = 501 class HttpBadGateway(HttpException): """502 Bad Gateway RFC2616, 10.5.3: The server, while acting as a gateway or proxy, received an invalid response from the upstream server it accessed in attempting to fulfill the request. """ code = 502 class HttpServiceUnavailable(HttpException): """503 Service Unavailable RFC2616, 10.5.4: The server is currently unable to handle the request due to a temporary overloading or maintenance of the server. """ code = 503 class HttpGatewayTimeout(HttpException): """504 Gateway Timeout RFC2616, 10.5.5: The server, while acting as a gateway or proxy, did not receive a timely response from the upstream server specified by the URI (e.g. HTTP, FTP, LDAP) or some other auxiliary server (e.g. DNS) it needed to access in attempting to complete the request. """ code = 504 class HttpVersionNotSupported(HttpException): """505 HTTP Version Not Supported RFC2616, 10.5.6: The server does not support, or refuses to support, the HTTP protocol version that was used in the request message. """ code = 505 def ParseHeaders(buf): """Parses HTTP headers. @note: This is just a trivial wrapper around C{email.message_from_file} """ return email.message_from_file(buf) def SocketOperation(sock, op, arg1, timeout): """Wrapper around socket functions. This function abstracts error handling for socket operations, especially for the complicated interaction with OpenSSL. @type sock: socket @param sock: Socket for the operation @type op: int @param op: Operation to execute (SOCKOP_* constants) @type arg1: any @param arg1: Parameter for function (if needed) @type timeout: None or float @param timeout: Timeout in seconds or None @return: Return value of socket function """ # TODO: event_poll/event_check/override if op in (SOCKOP_SEND, SOCKOP_HANDSHAKE): event_poll = select.POLLOUT elif op == SOCKOP_RECV: event_poll = select.POLLIN elif op == SOCKOP_SHUTDOWN: event_poll = None # The timeout is only used when OpenSSL requests polling for a condition. # It is not advisable to have no timeout for shutdown. assert timeout else: raise AssertionError("Invalid socket operation") # Handshake is only supported by SSL sockets if (op == SOCKOP_HANDSHAKE and not isinstance(sock, SSL_CONN_TYPE)): return # No override by default event_override = 0 while True: # Poll only for certain operations and when asked for by an override if event_override or op in (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_HANDSHAKE): if event_override: wait_for_event = event_override else: wait_for_event = event_poll event = utils.WaitForFdCondition(sock, wait_for_event, timeout) if event is None: raise HttpSocketTimeout() if event & (select.POLLNVAL | select.POLLHUP | select.POLLERR): # Let the socket functions handle these break if not event & wait_for_event: continue # Reset override event_override = 0 try: try: if op == SOCKOP_SEND: # Non-SSL sockets expect bytes if isinstance(sock, socket.socket): data = arg1.encode("utf-8") # Use sendall to avoid partial writes that could cause desync with # our caller, as len(data) != len(arg1) in the general case sock.sendall(data) return len(arg1) return sock.send(arg1) elif op == SOCKOP_RECV: return sock.recv(arg1) elif op == SOCKOP_SHUTDOWN: if isinstance(sock, SSL_CONN_TYPE): # PyOpenSSL's shutdown() doesn't take arguments return sock.shutdown() else: return sock.shutdown(arg1) elif op == SOCKOP_HANDSHAKE: return sock.do_handshake() except OpenSSL.SSL.WantWriteError: # OpenSSL wants to write, poll for POLLOUT event_override = select.POLLOUT continue except OpenSSL.SSL.WantReadError: # OpenSSL wants to read, poll for POLLIN event_override = select.POLLIN | select.POLLPRI continue except OpenSSL.SSL.WantX509LookupError: continue except OpenSSL.SSL.ZeroReturnError as err: # SSL Connection has been closed. In SSL 3.0 and TLS 1.0, this only # occurs if a closure alert has occurred in the protocol, i.e. the # connection has been closed cleanly. Note that this does not # necessarily mean that the transport layer (e.g. a socket) has been # closed. if op == SOCKOP_SEND: # Can happen during a renegotiation raise HttpConnectionClosed(err.args) elif op == SOCKOP_RECV: return "" # SSL_shutdown shouldn't return SSL_ERROR_ZERO_RETURN raise socket.error(err.args) except OpenSSL.SSL.SysCallError as err: if op == SOCKOP_SEND: # arg1 is the data when writing if err.args and err.args[0] == -1 and arg1 == "": # errors when writing empty strings are expected # and can be ignored return 0 if err.args == (-1, _SSL_UNEXPECTED_EOF): if op == SOCKOP_RECV: return "" elif op == SOCKOP_HANDSHAKE: # Can happen if peer disconnects directly after the connection is # opened. raise HttpSessionHandshakeUnexpectedEOF(err.args) raise socket.error(err.args) except OpenSSL.SSL.Error as err: if err.args[0] == [_SSL_SHUTDOWN_DURING_INIT]: host, port = sock.getpeername() logging.warning("OpenSSL: unexpected shutdown while in init from" " %s:%d" % (host, port)) break raise socket.error(err.args) except socket.error as err: if err.args and err.args[0] == errno.EAGAIN: # Ignore EAGAIN continue raise def ShutdownConnection(sock, close_timeout, write_timeout, msgreader, force): """Closes the connection. @type sock: socket @param sock: Socket to be shut down @type close_timeout: float @param close_timeout: How long to wait for the peer to close the connection @type write_timeout: float @param write_timeout: Write timeout for shutdown @type msgreader: http.HttpMessageReader @param msgreader: Request message reader, used to determine whether peer should close connection @type force: bool @param force: Whether to forcibly close the connection without waiting for peer """ #print(msgreader.peer_will_close, force) if msgreader and msgreader.peer_will_close and not force: # Wait for peer to close try: # Check whether it's actually closed if not SocketOperation(sock, SOCKOP_RECV, 1, close_timeout): return except (socket.error, HttpError, HttpSocketTimeout): # Ignore errors at this stage pass # Close the connection from our side try: # We don't care about the return value, see NOTES in SSL_shutdown(3). SocketOperation(sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR, write_timeout) except HttpSocketTimeout: raise HttpError("Timeout while shutting down connection") except socket.error as err: # Ignore ENOTCONN if not (err.args and err.args[0] == errno.ENOTCONN): raise HttpError("Error while shutting down connection: %s" % err) def Handshake(sock, write_timeout): """Shakes peer's hands. @type sock: socket @param sock: Socket to be shut down @type write_timeout: float @param write_timeout: Write timeout for handshake """ try: return SocketOperation(sock, SOCKOP_HANDSHAKE, None, write_timeout) except HttpSocketTimeout: raise HttpError("Timeout during SSL handshake") except socket.error as err: raise HttpError("Error in SSL handshake: %s" % err) class HttpSslParams(object): """Data class for SSL key and certificate. """ def __init__(self, ssl_key_path, ssl_cert_path, ssl_chain_path=None): """Initializes this class. @type ssl_key_path: string @param ssl_key_path: Path to file containing SSL key in PEM format @type ssl_cert_path: string @param ssl_cert_path: Path to file containing SSL certificate in PEM format """ self.ssl_key_pem = utils.ReadFile(ssl_key_path) self.ssl_cert_pem = utils.ReadFile(ssl_cert_path) self.ssl_cert_path = ssl_cert_path self.ssl_chain_path = ssl_chain_path def GetCertificateDigest(self): return utils.GetCertificateDigest(cert_filename=self.ssl_cert_path) def GetCertificateFilename(self): return self.ssl_cert_path def GetChain(self): return self.ssl_chain_path def GetKey(self): return OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, self.ssl_key_pem) def GetCertificate(self): return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, self.ssl_cert_pem) class HttpBase(object): """Base class for HTTP server and client. """ def __init__(self): self.using_ssl = None self._ssl_params = None self._ssl_key = None self._ssl_cert = None def _CreateSocket(self, ssl_params, ssl_verify_peer, family, ssl_verify_callback): """Creates a TCP socket and initializes SSL if needed. @type ssl_params: HttpSslParams @param ssl_params: SSL key and certificate @type ssl_verify_peer: bool @param ssl_verify_peer: Whether to require client certificate and compare it with our certificate @type family: int @param family: socket.AF_INET | socket.AF_INET6 """ assert family in (socket.AF_INET, socket.AF_INET6) if ssl_verify_peer: assert ssl_verify_callback is not None self._ssl_params = ssl_params sock = socket.socket(family, socket.SOCK_STREAM) # Should we enable SSL? self.using_ssl = ssl_params is not None if not self.using_ssl: return sock self._ssl_key = ssl_params.GetKey() self._ssl_cert = ssl_params.GetCertificate() self._ssl_chain = ssl_params.GetChain() ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2) if self._ssl_chain: ctx.use_certificate_chain_file(self._ssl_chain) ciphers = self.GetSslCiphers() logging.debug("Setting SSL cipher string %s", ciphers) ctx.set_cipher_list(ciphers) ctx.use_privatekey(self._ssl_key) ctx.use_certificate(self._ssl_cert) ctx.check_privatekey() logging.debug("Certificate digest: %s.", ssl_params.GetCertificateDigest()) logging.debug("Certificate filename: %s.", ssl_params.GetCertificateFilename()) if ssl_verify_peer: ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, ssl_verify_callback) # Also add our certificate as a trusted CA to be sent to the client. # This is required at least for GnuTLS clients to work. try: # This will fail for PyOpenssl versions before 0.10 ctx.add_client_ca(self._ssl_cert) except AttributeError: # Fall back to letting OpenSSL read the certificate file directly. ctx.load_client_ca(ssl_params.ssl_cert_path) return OpenSSL.SSL.Connection(ctx, sock) def GetSslCiphers(self): # pylint: disable=R0201 """Returns the ciphers string for SSL. """ return constants.OPENSSL_CIPHERS def _SSLVerifyCallback(self, conn, cert, errnum, errdepth, ok): """Verify the certificate provided by the peer We only compare fingerprints. The client must use the same certificate as we do on our side. """ # some parameters are unused, but this is the API # pylint: disable=W0613 assert self._ssl_params, "SSL not initialized" return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and self._ssl_cert.digest("md5") == cert.digest("md5")) class HttpMessage(object): """Data structure for HTTP message. """ def __init__(self): self.start_line = None self.headers = None self.body = None class HttpClientToServerStartLine(object): """Data structure for HTTP request start line. """ def __init__(self, method, path, version): self.method = method self.path = path self.version = version def __str__(self): return "%s %s %s" % (self.method, self.path, self.version) class HttpServerToClientStartLine(object): """Data structure for HTTP response start line. """ def __init__(self, version, code, reason): self.version = version self.code = code self.reason = reason def __str__(self): return "%s %s %s" % (self.version, self.code, self.reason) class HttpMessageWriter(object): """Writes an HTTP message to a socket. """ def __init__(self, sock, msg, write_timeout): """Initializes this class and writes an HTTP message to a socket. @type sock: socket @param sock: Socket to be written to @type msg: http.HttpMessage @param msg: HTTP message to be written @type write_timeout: float @param write_timeout: Write timeout for socket """ self._msg = msg self._PrepareMessage() buf = self._FormatMessage() pos = 0 end = len(buf) while pos < end: # Send only SOCK_BUF_SIZE bytes at a time data = buf[pos:(pos + SOCK_BUF_SIZE)] sent = SocketOperation(sock, SOCKOP_SEND, data, write_timeout) # Remove sent bytes pos += sent assert pos == end, "Message wasn't sent completely" def _PrepareMessage(self): """Prepares the HTTP message by setting mandatory headers. """ # RFC2616, section 4.3: "The presence of a message-body in a request is # signaled by the inclusion of a Content-Length or Transfer-Encoding header # field in the request's message-headers." if self._msg.body: self._msg.headers[HTTP_CONTENT_LENGTH] = len(self._msg.body) def _FormatMessage(self): """Serializes the HTTP message into a string. """ buf = StringIO() # Add start line buf.write(str(self._msg.start_line)) buf.write("\r\n") # Add headers if self._msg.start_line.version != HTTP_0_9: for name, value in self._msg.headers.items(): buf.write("%s: %s\r\n" % (name, value)) buf.write("\r\n") # Add message body if needed if self.HasMessageBody(): buf.write(self._msg.body.decode()) elif self._msg.body: logging.warning("Ignoring message body") return buf.getvalue() def HasMessageBody(self): """Checks whether the HTTP message contains a body. Can be overridden by subclasses. """ return bool(self._msg.body) class HttpMessageReader(object): """Reads HTTP message from socket. """ # Length limits START_LINE_LENGTH_MAX = None HEADER_LENGTH_MAX = None # Parser state machine PS_START_LINE = "start-line" PS_HEADERS = "headers" PS_BODY = "entity-body" PS_COMPLETE = "complete" def __init__(self, sock, msg, read_timeout): """Reads an HTTP message from a socket. @type sock: socket @param sock: Socket to be read from @type msg: http.HttpMessage @param msg: Object for the read message @type read_timeout: float @param read_timeout: Read timeout for socket """ self.sock = sock self.msg = msg self.start_line_buffer = None self.header_buffer = StringIO() self.body_buffer = StringIO() self.parser_status = self.PS_START_LINE self.content_length = None self.peer_will_close = None buf = "" eof = False while self.parser_status != self.PS_COMPLETE: # TODO: Don't read more than necessary (Content-Length), otherwise # data might be lost and/or an error could occur data = SocketOperation(sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout) if data: buf += data.decode() else: eof = True # Do some parsing and error checking while more data arrives buf = self._ContinueParsing(buf, eof) # Must be done only after the buffer has been evaluated # TODO: Content-Length < len(data read) and connection closed if (eof and self.parser_status in (self.PS_START_LINE, self.PS_HEADERS)): raise HttpError("Connection closed prematurely") # Parse rest buf = self._ContinueParsing(buf, True) assert self.parser_status == self.PS_COMPLETE assert not buf, "Parser didn't read full response" # Body is complete msg.body = self.body_buffer.getvalue() def _ContinueParsing(self, buf, eof): """Main function for HTTP message state machine. @type buf: string @param buf: Receive buffer @type eof: bool @param eof: Whether we've reached EOF on the socket @rtype: string @return: Updated receive buffer """ # TODO: Use offset instead of slicing when possible if self.parser_status == self.PS_START_LINE: # Expect start line while True: idx = buf.find("\r\n") # RFC2616, section 4.1: "In the interest of robustness, servers SHOULD # ignore any empty line(s) received where a Request-Line is expected. # In other words, if the server is reading the protocol stream at the # beginning of a message and receives a CRLF first, it should ignore # the CRLF." if idx == 0: # TODO: Limit number of CRLFs/empty lines for safety? buf = buf[2:] continue if idx > 0: self.start_line_buffer = buf[:idx] self._CheckStartLineLength(len(self.start_line_buffer)) # Remove status line, including CRLF buf = buf[idx + 2:] self.msg.start_line = self.ParseStartLine(self.start_line_buffer) self.parser_status = self.PS_HEADERS else: # Check whether incoming data is getting too large, otherwise we just # fill our read buffer. self._CheckStartLineLength(len(buf)) break # TODO: Handle messages without headers if self.parser_status == self.PS_HEADERS: # Wait for header end idx = buf.find("\r\n\r\n") if idx >= 0: self.header_buffer.write(buf[:idx + 2]) self._CheckHeaderLength(self.header_buffer.tell()) # Remove headers, including CRLF buf = buf[idx + 4:] self._ParseHeaders() self.parser_status = self.PS_BODY else: # Check whether incoming data is getting too large, otherwise we just # fill our read buffer. self._CheckHeaderLength(len(buf)) if self.parser_status == self.PS_BODY: # TODO: Implement max size for body_buffer self.body_buffer.write(buf) buf = "" # Check whether we've read everything # # RFC2616, section 4.4: "When a message-body is included with a message, # the transfer-length of that body is determined by one of the following # [...] 5. By the server closing the connection. (Closing the connection # cannot be used to indicate the end of a request body, since that would # leave no possibility for the server to send back a response.)" # # TODO: Error when buffer length > Content-Length header if (eof or self.content_length is None or (self.content_length is not None and self.body_buffer.tell() >= self.content_length)): self.parser_status = self.PS_COMPLETE return buf def _CheckStartLineLength(self, length): """Limits the start line buffer size. @type length: int @param length: Buffer size """ if (self.START_LINE_LENGTH_MAX is not None and length > self.START_LINE_LENGTH_MAX): raise HttpError("Start line longer than %d chars" % self.START_LINE_LENGTH_MAX) def _CheckHeaderLength(self, length): """Limits the header buffer size. @type length: int @param length: Buffer size """ if (self.HEADER_LENGTH_MAX is not None and length > self.HEADER_LENGTH_MAX): raise HttpError("Headers longer than %d chars" % self.HEADER_LENGTH_MAX) def ParseStartLine(self, start_line): """Parses the start line of a message. Must be overridden by subclass. @type start_line: string @param start_line: Start line string """ raise NotImplementedError() def _WillPeerCloseConnection(self): """Evaluate whether peer will close the connection. @rtype: bool @return: Whether peer will close the connection """ # RFC2616, section 14.10: "HTTP/1.1 defines the "close" connection option # for the sender to signal that the connection will be closed after # completion of the response. For example, # # Connection: close # # in either the request or the response header fields indicates that the # connection SHOULD NOT be considered `persistent' (section 8.1) after the # current request/response is complete." hdr_connection = self.msg.headers.get(HTTP_CONNECTION, None) if hdr_connection: hdr_connection = hdr_connection.lower() # An HTTP/1.1 server is assumed to stay open unless explicitly closed. if self.msg.start_line.version == HTTP_1_1: return (hdr_connection and "close" in hdr_connection) # Some HTTP/1.0 implementations have support for persistent connections, # using rules different than HTTP/1.1. # For older HTTP, Keep-Alive indicates persistent connection. if self.msg.headers.get(HTTP_KEEP_ALIVE): return False # At least Akamai returns a "Connection: Keep-Alive" header, which was # supposed to be sent by the client. if hdr_connection and "keep-alive" in hdr_connection: return False return True def _ParseHeaders(self): """Parses the headers. This function also adjusts internal variables based on header values. RFC2616, section 4.3: The presence of a message-body in a request is signaled by the inclusion of a Content-Length or Transfer-Encoding header field in the request's message-headers. """ # Parse headers self.header_buffer.seek(0, 0) self.msg.headers = ParseHeaders(self.header_buffer) self.peer_will_close = self._WillPeerCloseConnection() # Do we have a Content-Length header? hdr_content_length = self.msg.headers.get(HTTP_CONTENT_LENGTH, None) if hdr_content_length: try: self.content_length = int(hdr_content_length) except (TypeError, ValueError): self.content_length = None if self.content_length is not None and self.content_length < 0: self.content_length = None # if the connection remains open and a content-length was not provided, # then assume that the connection WILL close. if self.content_length is None: self.peer_will_close = True ganeti-3.1.0~rc2/lib/http/auth.py000064400000000000000000000237771476477700300166640ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """HTTP authentication module. """ import logging import re import base64 import binascii from io import StringIO from hashlib import md5 from ganeti import compat from ganeti import http from ganeti import utils # Digest types from RFC2617 HTTP_BASIC_AUTH = "Basic" HTTP_DIGEST_AUTH = "Digest" # Not exactly as described in RFC2616, section 2.2, but good enough _NOQUOTE = re.compile(r"^[-_a-z0-9]+$", re.I) def _FormatAuthHeader(scheme, params): """Formats WWW-Authentication header value as per RFC2617, section 1.2 @type scheme: str @param scheme: Authentication scheme @type params: dict @param params: Additional parameters @rtype: str @return: Formatted header value """ buf = StringIO() buf.write(scheme) for name, value in params.items(): buf.write(" ") buf.write(name) buf.write("=") if _NOQUOTE.match(value): buf.write(value) else: buf.write("\"") # TODO: Better quoting buf.write(value.replace("\"", "\\\"")) buf.write("\"") return buf.getvalue() class HttpServerRequestAuthentication(object): # Default authentication realm AUTH_REALM = "Unspecified" # Schemes for passwords _CLEARTEXT_SCHEME = "{CLEARTEXT}" _HA1_SCHEME = "{HA1}" def GetAuthRealm(self, req): """Returns the authentication realm for a request. May be overridden by a subclass, which then can return different realms for different paths. @type req: L{http.server._HttpServerRequest} @param req: HTTP request context @rtype: string @return: Authentication realm """ # today we don't have per-request filtering, but we might want to # add it in the future # pylint: disable=W0613 return self.AUTH_REALM def AuthenticationRequired(self, req): """Determines whether authentication is required for a request. To enable authentication, override this function in a subclass and return C{True}. L{AUTH_REALM} must be set. @type req: L{http.server._HttpServerRequest} @param req: HTTP request context """ # Unused argument, method could be a function # pylint: disable=W0613,R0201 return False def PreHandleRequest(self, req): """Called before a request is handled. @type req: L{http.server._HttpServerRequest} @param req: HTTP request context """ # Authentication not required, and no credentials given? if not (self.AuthenticationRequired(req) or (req.request_headers and http.HTTP_AUTHORIZATION in req.request_headers)): return realm = self.GetAuthRealm(req) if not realm: raise AssertionError("No authentication realm") # Check "Authorization" header if self._CheckAuthorization(req): # User successfully authenticated return # Send 401 Unauthorized response params = { "realm": realm, } # TODO: Support for Digest authentication (RFC2617, section 3). # TODO: Support for more than one WWW-Authenticate header with the same # response (RFC2617, section 4.6). headers = { http.HTTP_WWW_AUTHENTICATE: _FormatAuthHeader(HTTP_BASIC_AUTH, params), } raise http.HttpUnauthorized(headers=headers) def _CheckAuthorization(self, req): """Checks 'Authorization' header sent by client. @type req: L{http.server._HttpServerRequest} @param req: HTTP request context @rtype: bool @return: Whether user is allowed to execute request """ credentials = req.request_headers.get(http.HTTP_AUTHORIZATION, None) if not credentials: return False # Extract scheme parts = credentials.strip().split(None, 2) if len(parts) < 1: # Missing scheme return False # RFC2617, section 1.2: "[...] It uses an extensible, case-insensitive # token to identify the authentication scheme [...]" scheme = parts[0].lower() if scheme == HTTP_BASIC_AUTH.lower(): # Do basic authentication if len(parts) < 2: raise http.HttpBadRequest(message=("Basic authentication requires" " credentials")) return self._CheckBasicAuthorization(req, parts[1]) elif scheme == HTTP_DIGEST_AUTH.lower(): # TODO: Implement digest authentication # RFC2617, section 3.3: "Note that the HTTP server does not actually need # to know the user's cleartext password. As long as H(A1) is available to # the server, the validity of an Authorization header may be verified." pass # Unsupported authentication scheme return False def _CheckBasicAuthorization(self, req, in_data): """Checks credentials sent for basic authentication. @type req: L{http.server._HttpServerRequest} @param req: HTTP request context @type in_data: str @param in_data: Username and password encoded as Base64 @rtype: bool @return: Whether user is allowed to execute request """ try: creds = base64.b64decode(in_data.encode("ascii")).decode("ascii") except (TypeError, binascii.Error, UnicodeError): logging.exception("Error when decoding Basic authentication credentials") return False if ":" not in creds: return False (user, password) = creds.split(":", 1) return self.Authenticate(req, user, password) def Authenticate(self, req, user, password): """Checks the password for a user. This function MUST be overridden by a subclass. """ raise NotImplementedError() def VerifyBasicAuthPassword(self, req, username, password, expected): """Checks the password for basic authentication. As long as they don't start with an opening brace ("E{lb}"), old passwords are supported. A new scheme uses H(A1) from RFC2617, where H is MD5 and A1 consists of the username, the authentication realm and the actual password. @type req: L{http.server._HttpServerRequest} @param req: HTTP request context @type username: string @param username: Username from HTTP headers @type password: string @param password: Password from HTTP headers @type expected: string @param expected: Expected password with optional scheme prefix (e.g. from users file) """ # Backwards compatibility for old-style passwords without a scheme if not expected.startswith("{"): expected = self._CLEARTEXT_SCHEME + expected # Check again, just to be sure if not expected.startswith("{"): raise AssertionError("Invalid scheme") scheme_end_idx = expected.find("}", 1) # Ensure scheme has a length of at least one character if scheme_end_idx <= 1: logging.warning("Invalid scheme in password for user '%s'", username) return False scheme = expected[:scheme_end_idx + 1].upper() expected_password = expected[scheme_end_idx + 1:] # Good old plain text password if scheme == self._CLEARTEXT_SCHEME: return password == expected_password # H(A1) as described in RFC2617 if scheme == self._HA1_SCHEME: realm = self.GetAuthRealm(req) if not realm: # There can not be a valid password for this case raise AssertionError("No authentication realm") digest = "%s:%s:%s" % (username, realm, password) expha1 = md5(digest.encode("ascii")) return (expected_password.lower() == expha1.hexdigest().lower()) logging.warning("Unknown scheme '%s' in password for user '%s'", scheme, username) return False class PasswordFileUser(object): """Data structure for users from password file. """ def __init__(self, name, password, options): self.name = name self.password = password self.options = options def ParsePasswordFile(contents): """Parses the contents of a password file. Lines in the password file are of the following format:: [options] Fields are separated by whitespace. Username and password are mandatory, options are optional and separated by comma (','). Empty lines and comments ('#') are ignored. @type contents: str @param contents: Contents of password file @rtype: dict @return: Dictionary containing L{PasswordFileUser} instances """ users = {} for line in utils.FilterEmptyLinesAndComments(contents): parts = line.split(None, 2) if len(parts) < 2: # Invalid line # TODO: Return line number from FilterEmptyLinesAndComments logging.warning("Ignoring non-comment line with less than two fields") continue name = parts[0] password = parts[1] # Extract options options = [] if len(parts) >= 3: for part in parts[2].split(","): options.append(part.strip()) else: logging.warning("Ignoring values for user '%s': %s", name, parts[3:]) users[name] = PasswordFileUser(name, password, options) return users ganeti-3.1.0~rc2/lib/http/client.py000064400000000000000000000267531476477700300171760ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008, 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """HTTP client module. """ import logging import threading from io import BytesIO import pycurl from ganeti import http from ganeti import compat from ganeti import netutils from ganeti import locking class HttpClientRequest(object): def __init__(self, host, port, method, path, headers=None, post_data=None, read_timeout=None, curl_config_fn=None, nicename=None, completion_cb=None): """Describes an HTTP request. @type host: string @param host: Hostname @type port: int @param port: Port @type method: string @param method: Method name @type path: string @param path: Request path @type headers: list or None @param headers: Additional headers to send, list of strings @type post_data: string or None @param post_data: Additional data to send @type read_timeout: int @param read_timeout: if passed, it will be used as the read timeout while reading the response from the server @type curl_config_fn: callable @param curl_config_fn: Function to configure cURL object before request @type nicename: string @param nicename: Name, presentable to a user, to describe this request (no whitespace) @type completion_cb: callable accepting this request object as a single parameter @param completion_cb: Callback for request completion """ assert path.startswith("/"), "Path must start with slash (/)" assert curl_config_fn is None or callable(curl_config_fn) assert completion_cb is None or callable(completion_cb) # Request attributes self.host = host self.port = port self.method = method self.path = path self.read_timeout = read_timeout self.curl_config_fn = curl_config_fn self.nicename = nicename self.completion_cb = completion_cb if post_data is None: self.post_data = "" else: self.post_data = post_data if headers is None: self.headers = [] elif isinstance(headers, dict): # Support for old interface self.headers = ["%s: %s" % (name, value) for name, value in headers.items()] else: self.headers = headers # Response status self.success = None self.error = None # Response attributes self.resp_status_code = None self.resp_body = None def __repr__(self): status = ["%s.%s" % (self.__class__.__module__, self.__class__.__name__), "%s:%s" % (self.host, self.port), self.method, self.path] return "<%s at %#x>" % (" ".join(status), id(self)) @property def url(self): """Returns the full URL for this requests. """ if netutils.IPAddress.IsValid(self.host): address = netutils.FormatAddress((self.host, self.port)) else: address = "%s:%s" % (self.host, self.port) # TODO: Support for non-SSL requests return "https://%s%s" % (address, self.path) def _StartRequest(curl, req): """Starts a request on a cURL object. @type curl: pycurl.Curl @param curl: cURL object @type req: L{HttpClientRequest} @param req: HTTP request """ logging.debug("Starting request %r", req) url = req.url method = req.method post_data = req.post_data headers = req.headers # Buffer for response resp_buffer = BytesIO() # Configure client for request curl.setopt(pycurl.VERBOSE, False) curl.setopt(pycurl.NOSIGNAL, True) curl.setopt(pycurl.USERAGENT, http.HTTP_GANETI_VERSION) curl.setopt(pycurl.PROXY, "") curl.setopt(pycurl.CUSTOMREQUEST, method) curl.setopt(pycurl.URL, url) curl.setopt(pycurl.POSTFIELDS, post_data) curl.setopt(pycurl.HTTPHEADER, headers) if req.read_timeout is None: curl.setopt(pycurl.TIMEOUT, 0) else: curl.setopt(pycurl.TIMEOUT, int(req.read_timeout)) # Disable SSL session ID caching (pycurl >= 7.16.0) if hasattr(pycurl, "SSL_SESSIONID_CACHE"): curl.setopt(pycurl.SSL_SESSIONID_CACHE, False) curl.setopt(pycurl.WRITEFUNCTION, resp_buffer.write) # Pass cURL object to external config function if req.curl_config_fn: req.curl_config_fn(curl) return _PendingRequest(curl, req, resp_buffer.getvalue) class _PendingRequest(object): def __init__(self, curl, req, resp_buffer_read): """Initializes this class. @type curl: pycurl.Curl @param curl: cURL object @type req: L{HttpClientRequest} @param req: HTTP request @type resp_buffer_read: callable @param resp_buffer_read: Function to read response body """ assert req.success is None self._curl = curl self._req = req self._resp_buffer_read = resp_buffer_read def GetCurlHandle(self): """Returns the cURL object. """ return self._curl def GetCurrentRequest(self): """Returns the current request. """ return self._req def Done(self, errmsg): """Finishes a request. @type errmsg: string or None @param errmsg: Error message if request failed """ curl = self._curl req = self._req assert req.success is None, "Request has already been finalized" try: # LOCAL_* options added in pycurl 7.21.5 from_str = "from %s:%s " % ( curl.getinfo(pycurl.LOCAL_IP), curl.getinfo(pycurl.LOCAL_PORT) ) except AttributeError: from_str = "" logging.debug("Request %s%s finished, errmsg=%s", from_str, req, errmsg) req.success = not bool(errmsg) req.error = errmsg # Get HTTP response code req.resp_status_code = curl.getinfo(pycurl.RESPONSE_CODE) req.resp_body = self._resp_buffer_read().decode("utf-8") # Ensure no potentially large variables are referenced curl.setopt(pycurl.POSTFIELDS, "") curl.setopt(pycurl.WRITEFUNCTION, lambda _: None) if req.completion_cb: req.completion_cb(req) class _NoOpRequestMonitor(object): # pylint: disable=W0232 """No-op request monitor. """ @staticmethod def acquire(*args, **kwargs): pass release = acquire Disable = acquire class _PendingRequestMonitor(object): _LOCK = "_lock" def __init__(self, owner, pending_fn): """Initializes this class. """ self._owner = owner self._pending_fn = pending_fn # The lock monitor runs in another thread, hence locking is necessary self._lock = locking.SharedLock("PendingHttpRequests") self.acquire = self._lock.acquire self.release = self._lock.release @locking.ssynchronized(_LOCK) def Disable(self): """Disable monitor. """ self._pending_fn = None @locking.ssynchronized(_LOCK, shared=1) def GetLockInfo(self, requested): # pylint: disable=W0613 """Retrieves information about pending requests. @type requested: set @param requested: Requested information, see C{query.LQ_*} """ # No need to sort here, that's being done by the lock manager and query # library. There are no priorities for requests, hence all show up as # one item under "pending". result = [] if self._pending_fn: owner_name = self._owner.name for client in self._pending_fn(): req = client.GetCurrentRequest() if req: if req.nicename is None: name = "%s%s" % (req.host, req.path) else: name = req.nicename result.append(("rpc/%s" % name, None, [owner_name], None)) return result def _ProcessCurlRequests(multi, requests): """cURL request processor. This generator yields a tuple once for every completed request, successful or not. The first value in the tuple is the handle, the second an error message or C{None} for successful requests. @type multi: C{pycurl.CurlMulti} @param multi: cURL multi object @type requests: sequence @param requests: cURL request handles """ for curl in requests: multi.add_handle(curl) while True: (ret, active) = multi.perform() assert ret in (pycurl.E_MULTI_OK, pycurl.E_CALL_MULTI_PERFORM) if ret == pycurl.E_CALL_MULTI_PERFORM: # cURL wants to be called again continue while True: (remaining_messages, successful, failed) = multi.info_read() for curl in successful: multi.remove_handle(curl) yield (curl, None) for curl, errnum, errmsg in failed: multi.remove_handle(curl) yield (curl, "Error %s: %s" % (errnum, errmsg)) if remaining_messages == 0: break if active == 0: # No active handles anymore break # Wait for I/O. The I/O timeout shouldn't be too long so that HTTP # timeouts, which are only evaluated in multi.perform, aren't # unnecessarily delayed. multi.select(1.0) def ProcessRequests(requests, lock_monitor_cb=None, _curl=pycurl.Curl, _curl_multi=pycurl.CurlMulti, _curl_process=_ProcessCurlRequests): """Processes any number of HTTP client requests. @type requests: list of L{HttpClientRequest} @param requests: List of all requests @param lock_monitor_cb: Callable for registering with lock monitor """ assert compat.all((req.error is None and req.success is None and req.resp_status_code is None and req.resp_body is None) for req in requests) # Prepare all requests curl_to_client = \ dict((client.GetCurlHandle(), client) for client in [_StartRequest(_curl(), req) for req in requests]) assert len(curl_to_client) == len(requests) if lock_monitor_cb: monitor = _PendingRequestMonitor(threading.current_thread(), curl_to_client.values) lock_monitor_cb(monitor) else: monitor = _NoOpRequestMonitor # Process all requests and act based on the returned values for (curl, msg) in _curl_process(_curl_multi(), list(curl_to_client)): monitor.acquire(shared=0) try: curl_to_client.pop(curl).Done(msg) finally: monitor.release() assert not curl_to_client, "Not all requests were processed" # Don't try to read information anymore as all requests have been processed monitor.Disable() assert compat.all(req.error is not None or (req.success and req.resp_status_code is not None and req.resp_body is not None) for req in requests) ganeti-3.1.0~rc2/lib/http/server.py000064400000000000000000000503011476477700300172100ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008, 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """HTTP server module. """ import html import logging import os import socket import time import signal import asyncore from http.server import BaseHTTPRequestHandler from ganeti import http from ganeti import utils from ganeti import netutils from ganeti import compat from ganeti import errors WEEKDAYNAME = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] MONTHNAME = [None, "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] # Default error message DEFAULT_ERROR_CONTENT_TYPE = "text/html" DEFAULT_ERROR_MESSAGE = """\ Error response

Error response

Error code %(code)d.

Message: %(message)s.

Error code explanation: %(code)s = %(explain)s. """ def _DateTimeHeader(gmnow=None): """Return the current date and time formatted for a message header. The time MUST be in the GMT timezone. """ if gmnow is None: gmnow = time.gmtime() (year, month, day, hh, mm, ss, wd, _, _) = gmnow return ("%s, %02d %3s %4d %02d:%02d:%02d GMT" % (WEEKDAYNAME[wd], day, MONTHNAME[month], year, hh, mm, ss)) class _HttpServerRequest(object): """Data structure for HTTP request on server side. """ def __init__(self, method, path, headers, body, sock): # Request attributes self.request_method = method self.request_path = path self.request_headers = headers self.request_body = body self.request_sock = sock # Response attributes self.resp_headers = {} # Private data for request handler (useful in combination with # authentication) self.private = None def __repr__(self): status = ["%s.%s" % (self.__class__.__module__, self.__class__.__name__), self.request_method, self.request_path, "headers=%r" % str(self.request_headers), "body=%r" % (self.request_body, )] return "<%s at %#x>" % (" ".join(status), id(self)) class _HttpServerToClientMessageWriter(http.HttpMessageWriter): """Writes an HTTP response to client. """ def __init__(self, sock, request_msg, response_msg, write_timeout): """Writes the response to the client. @type sock: socket @param sock: Target socket @type request_msg: http.HttpMessage @param request_msg: Request message, required to determine whether response may have a message body @type response_msg: http.HttpMessage @param response_msg: Response message @type write_timeout: float @param write_timeout: Write timeout for socket """ self._request_msg = request_msg self._response_msg = response_msg http.HttpMessageWriter.__init__(self, sock, response_msg, write_timeout) def HasMessageBody(self): """Logic to detect whether response should contain a message body. """ if self._request_msg.start_line: request_method = self._request_msg.start_line.method else: request_method = None response_code = self._response_msg.start_line.code # RFC2616, section 4.3: "A message-body MUST NOT be included in a request # if the specification of the request method (section 5.1.1) does not allow # sending an entity-body in requests" # # RFC2616, section 9.4: "The HEAD method is identical to GET except that # the server MUST NOT return a message-body in the response." # # RFC2616, section 10.2.5: "The 204 response MUST NOT include a # message-body [...]" # # RFC2616, section 10.3.5: "The 304 response MUST NOT contain a # message-body, [...]" return (http.HttpMessageWriter.HasMessageBody(self) and request_method != http.HTTP_HEAD and response_code >= http.HTTP_OK and response_code not in (http.HTTP_NO_CONTENT, http.HTTP_NOT_MODIFIED)) class _HttpClientToServerMessageReader(http.HttpMessageReader): """Reads an HTTP request sent by client. """ # Length limits START_LINE_LENGTH_MAX = 8192 HEADER_LENGTH_MAX = 4096 def ParseStartLine(self, start_line): """Parses the start line sent by client. Example: "GET /index.html HTTP/1.1" @type start_line: string @param start_line: Start line """ # Empty lines are skipped when reading assert start_line logging.debug("HTTP request: %s", start_line) words = start_line.split() if len(words) == 3: [method, path, version] = words if version[:5] != "HTTP/": raise http.HttpBadRequest("Bad request version (%r)" % version) try: base_version_number = version.split("/", 1)[1] version_number = base_version_number.split(".") # RFC 2145 section 3.1 says there can be only one "." and # - major and minor numbers MUST be treated as # separate integers; # - HTTP/2.4 is a lower version than HTTP/2.13, which in # turn is lower than HTTP/12.3; # - Leading zeros MUST be ignored by recipients. if len(version_number) != 2: raise http.HttpBadRequest("Bad request version (%r)" % version) version_number = (int(version_number[0]), int(version_number[1])) except (ValueError, IndexError): raise http.HttpBadRequest("Bad request version (%r)" % version) if version_number >= (2, 0): raise http.HttpVersionNotSupported("Invalid HTTP Version (%s)" % base_version_number) elif len(words) == 2: version = http.HTTP_0_9 [method, path] = words if method != http.HTTP_GET: raise http.HttpBadRequest("Bad HTTP/0.9 request type (%r)" % method) else: raise http.HttpBadRequest("Bad request syntax (%r)" % start_line) return http.HttpClientToServerStartLine(method, path, version) def _HandleServerRequestInner(handler, req_msg, reader): """Calls the handler function for the current request. """ handler_context = _HttpServerRequest(req_msg.start_line.method, req_msg.start_line.path, req_msg.headers, req_msg.body, reader.sock) logging.debug("Handling request %r", handler_context) try: try: # Authentication, etc. handler.PreHandleRequest(handler_context) # Call actual request handler result = handler.HandleRequest(handler_context) except (http.HttpException, errors.RapiTestResult, KeyboardInterrupt, SystemExit): raise except Exception as err: logging.exception("Caught exception") raise http.HttpInternalServerError(message=str(err)) except: logging.exception("Unknown exception") raise http.HttpInternalServerError(message="Unknown error") if not isinstance(result, (str, bytes)): raise http.HttpError("Handler function didn't return string type") return (http.HTTP_OK, handler_context.resp_headers, result) finally: # No reason to keep this any longer, even for exceptions handler_context.private = None class HttpResponder(object): # The default request version. This only affects responses up until # the point where the request line is parsed, so it mainly decides what # the client gets back when sending a malformed request line. # Most web servers default to HTTP 0.9, i.e. don't send a status line. default_request_version = http.HTTP_0_9 responses = BaseHTTPRequestHandler.responses def __init__(self, handler): """Initializes this class. """ self._handler = handler def __call__(self, fn): """Handles a request. @type fn: callable @param fn: Callback for retrieving HTTP request, must return a tuple containing request message (L{http.HttpMessage}) and C{None} or the message reader (L{_HttpClientToServerMessageReader}) """ response_msg = http.HttpMessage() response_msg.start_line = \ http.HttpServerToClientStartLine(version=self.default_request_version, code=None, reason=None) force_close = True try: (request_msg, req_msg_reader) = fn() response_msg.start_line.version = request_msg.start_line.version # RFC2616, 14.23: All Internet-based HTTP/1.1 servers MUST respond # with a 400 (Bad Request) status code to any HTTP/1.1 request # message which lacks a Host header field. if (request_msg.start_line.version == http.HTTP_1_1 and not (request_msg.headers and http.HTTP_HOST in request_msg.headers)): raise http.HttpBadRequest(message="Missing Host header") (response_msg.start_line.code, response_msg.headers, response_msg.body) = \ _HandleServerRequestInner(self._handler, request_msg, req_msg_reader) except http.HttpException as err: self._SetError(self.responses, self._handler, response_msg, err) request_msg = http.HttpMessage() req_msg_reader = None else: # Only wait for client to close if we didn't have any exception. force_close = False return (request_msg, req_msg_reader, force_close, self._Finalize(self.responses, response_msg)) @staticmethod def _SetError(responses, handler, response_msg, err): """Sets the response code and body from a HttpException. @type err: HttpException @param err: Exception instance """ try: (shortmsg, longmsg) = responses[err.code] except KeyError: shortmsg = longmsg = "Unknown" if err.message: message = err.message else: message = shortmsg values = { "code": err.code, "message": html.escape(message), "explain": longmsg, } (content_type, body) = handler.FormatErrorMessage(values) headers = { http.HTTP_CONTENT_TYPE: content_type, } if err.headers: headers.update(err.headers) response_msg.start_line.code = err.code response_msg.headers = headers response_msg.body = body @staticmethod def _Finalize(responses, msg): assert msg.start_line.reason is None if not msg.headers: msg.headers = {} msg.headers.update({ # TODO: Keep-alive is not supported http.HTTP_CONNECTION: "close", http.HTTP_DATE: _DateTimeHeader(), http.HTTP_SERVER: http.HTTP_GANETI_VERSION, }) # Get response reason based on code try: code_desc = responses[msg.start_line.code] except KeyError: reason = "" else: (reason, _) = code_desc msg.start_line.reason = reason return msg class HttpServerRequestExecutor(object): """Implements server side of HTTP. This class implements the server side of HTTP. It's based on code of Python's BaseHTTPServer, from both version 2.4 and 3k. It does not support non-ASCII character encodings. Keep-alive connections are not supported. """ # Timeouts in seconds for socket layer WRITE_TIMEOUT = 10 READ_TIMEOUT = 10 CLOSE_TIMEOUT = 1 def __init__(self, server, handler, sock, client_addr): """Initializes this class. """ responder = HttpResponder(handler) # Disable Python's timeout sock.settimeout(None) # Operate in non-blocking mode sock.setblocking(0) request_msg_reader = None force_close = True logging.debug("Connection from %s:%s", client_addr[0], client_addr[1]) try: # Block for closing connection try: # Do the secret SSL handshake if server.using_ssl: sock.set_accept_state() try: http.Handshake(sock, self.WRITE_TIMEOUT) except http.HttpSessionHandshakeUnexpectedEOF: logging.debug("Unexpected EOF from %s:%s", client_addr[0], client_addr[1]) # Ignore rest return (request_msg, request_msg_reader, force_close, response_msg) = \ responder(compat.partial(self._ReadRequest, sock, self.READ_TIMEOUT)) if response_msg: # HttpMessage.start_line can be of different types # Instance of 'HttpClientToServerStartLine' has no 'code' member # pylint: disable=E1103,E1101 logging.info("%s:%s %s %s", client_addr[0], client_addr[1], request_msg.start_line, response_msg.start_line.code) self._SendResponse(sock, request_msg, response_msg, self.WRITE_TIMEOUT) finally: http.ShutdownConnection(sock, self.CLOSE_TIMEOUT, self.WRITE_TIMEOUT, request_msg_reader, force_close) sock.close() finally: logging.debug("Disconnected %s:%s", client_addr[0], client_addr[1]) @staticmethod def _ReadRequest(sock, timeout): """Reads a request sent by client. """ msg = http.HttpMessage() try: reader = _HttpClientToServerMessageReader(sock, msg, timeout) except http.HttpSocketTimeout: raise http.HttpError("Timeout while reading request") except socket.error as err: raise http.HttpError("Error reading request: %s" % err) return (msg, reader) @staticmethod def _SendResponse(sock, req_msg, msg, timeout): """Sends the response to the client. """ try: _HttpServerToClientMessageWriter(sock, req_msg, msg, timeout) except http.HttpSocketTimeout: raise http.HttpError("Timeout while sending response") except socket.error as err: raise http.HttpError("Error sending response: %s" % err) class HttpServer(http.HttpBase, asyncore.dispatcher): """Generic HTTP server class """ def __init__(self, mainloop, local_address, port, max_clients, handler, ssl_params=None, ssl_verify_peer=False, request_executor_class=None, ssl_verify_callback=None): """Initializes the HTTP server @type mainloop: ganeti.daemon.Mainloop @param mainloop: Mainloop used to poll for I/O events @type local_address: string @param local_address: Local IP address to bind to @type port: int @param port: TCP port to listen on @type max_clients: int @param max_clients: maximum number of client connections open simultaneously. @type handler: HttpServerHandler @param handler: Request handler object @type ssl_params: HttpSslParams @param ssl_params: SSL key and certificate @type ssl_verify_peer: bool @param ssl_verify_peer: Whether to require client certificate and compare it with our certificate @type request_executor_class: class @param request_executor_class: a class derived from the HttpServerRequestExecutor class """ http.HttpBase.__init__(self) asyncore.dispatcher.__init__(self) if request_executor_class is None: self.request_executor = HttpServerRequestExecutor else: self.request_executor = request_executor_class self.mainloop = mainloop self.local_address = local_address self.port = port self.handler = handler family = netutils.IPAddress.GetAddressFamily(local_address) self.socket = self._CreateSocket(ssl_params, ssl_verify_peer, family, ssl_verify_callback) # Allow port to be reused self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._children = [] self.set_socket(self.socket) self.accepting = True self.max_clients = max_clients mainloop.RegisterSignal(self) def Start(self): self.socket.bind((self.local_address, self.port)) self.socket.listen(1024) def Stop(self): self.socket.close() def handle_accept(self): self._IncomingConnection() def OnSignal(self, signum): if signum == signal.SIGCHLD: self._CollectChildren(True) def _CollectChildren(self, quick): """Checks whether any child processes are done @type quick: bool @param quick: Whether to only use non-blocking functions """ if not quick: # Don't wait for other processes if it should be a quick check while len(self._children) > self.max_clients: try: # Waiting without a timeout brings us into a potential DoS situation. # As soon as too many children run, we'll not respond to new # requests. The real solution would be to add a timeout for children # and killing them after some time. pid, _ = os.waitpid(0, 0) except os.error: pid = None if pid and pid in self._children: self._children.remove(pid) for child in self._children: try: pid, _ = os.waitpid(child, os.WNOHANG) except os.error: pid = None if pid and pid in self._children: self._children.remove(pid) def _IncomingConnection(self): """Called for each incoming connection """ # pylint: disable=W0212 t_start = time.time() (connection, client_addr) = self.socket.accept() self._CollectChildren(False) try: pid = os.fork() except OSError: logging.exception("Failed to fork on request from %s:%s", client_addr[0], client_addr[1]) # Immediately close the connection. No SSL handshake has been done. try: connection.close() except socket.error: pass return if pid == 0: # Child process try: # The client shouldn't keep the listening socket open. If the parent # process is restarted, it would fail when there's already something # listening (in this case its own child from a previous run) on the # same port. try: self.socket.close() except socket.error: pass self.socket = None # In case the handler code uses temporary files utils.ResetTempfileModule() t_setup = time.time() self.request_executor(self, self.handler, connection, client_addr) t_end = time.time() logging.debug("Request from %s:%s executed in: %.4f [setup: %.4f] " "[workers: %d]", client_addr[0], client_addr[1], t_end - t_start, t_setup - t_start, len(self._children)) except Exception: # pylint: disable=W0703 logging.exception("Error while handling request from %s:%s", client_addr[0], client_addr[1]) os._exit(1) os._exit(0) else: self._children.append(pid) class HttpServerHandler(object): """Base class for handling HTTP server requests. Users of this class must subclass it and override the L{HandleRequest} function. """ def PreHandleRequest(self, req): """Called before handling a request. Can be overridden by a subclass. """ def HandleRequest(self, req): """Handles a request. Must be overridden by subclass. """ raise NotImplementedError() @staticmethod def FormatErrorMessage(values): """Formats the body of an error message. @type values: dict @param values: dictionary with keys C{code}, C{message} and C{explain}. @rtype: tuple; (string, string) @return: Content-type and response body """ return (DEFAULT_ERROR_CONTENT_TYPE, DEFAULT_ERROR_MESSAGE % values) ganeti-3.1.0~rc2/lib/hypervisor/000075500000000000000000000000001476477700300165645ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/hypervisor/__init__.py000064400000000000000000000051071476477700300207000ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Virtualization interface abstraction """ from ganeti import constants from ganeti import errors from ganeti.hypervisor import hv_fake from ganeti.hypervisor import hv_xen from ganeti.hypervisor import hv_kvm from ganeti.hypervisor import hv_chroot from ganeti.hypervisor import hv_lxc _HYPERVISOR_MAP = { constants.HT_XEN_PVM: hv_xen.XenPvmHypervisor, constants.HT_XEN_HVM: hv_xen.XenHvmHypervisor, constants.HT_FAKE: hv_fake.FakeHypervisor, constants.HT_KVM: hv_kvm.KVMHypervisor, constants.HT_CHROOT: hv_chroot.ChrootManager, constants.HT_LXC: hv_lxc.LXCHypervisor, } def GetHypervisorClass(ht_kind): """Return a Hypervisor class. This function returns the hypervisor class corresponding to the given hypervisor name. @type ht_kind: string @param ht_kind: The requested hypervisor type """ if ht_kind not in _HYPERVISOR_MAP: raise errors.HypervisorError("Unknown hypervisor type '%s'" % ht_kind) cls = _HYPERVISOR_MAP[ht_kind] return cls def GetHypervisor(ht_kind): """Return a Hypervisor instance. This is a wrapper over L{GetHypervisorClass} which returns an instance of the class. @type ht_kind: string @param ht_kind: The requested hypervisor type """ cls = GetHypervisorClass(ht_kind) return cls() ganeti-3.1.0~rc2/lib/hypervisor/hv_base.py000064400000000000000000000660571476477700300205630ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Base class for all hypervisors The syntax for the _CHECK variables and the contents of the PARAMETERS dict is the same, see the docstring for L{BaseHypervisor.PARAMETERS}. @var _FILE_CHECK: stub for file checks, without the required flag @var _DIR_CHECK: stub for directory checks, without the required flag @var REQ_FILE_CHECK: mandatory file parameter @var OPT_FILE_CHECK: optional file parameter @var REQ_DIR_CHECK: mandatory directory parametr @var OPT_DIR_CHECK: optional directory parameter @var NO_CHECK: parameter without any checks at all @var REQUIRED_CHECK: parameter required to exist (and non-false), but without other checks; beware that this can't be used for boolean parameters, where you should use NO_CHECK or a custom checker """ import os import re import logging from ganeti import constants from ganeti import errors from ganeti import objects from ganeti import utils def _IsCpuMaskWellFormed(cpu_mask): """Verifies if the given single CPU mask is valid The single CPU mask should be in the form "a,b,c,d", where each letter is a positive number or range. """ try: cpu_list = utils.ParseCpuMask(cpu_mask) except errors.ParseError: return False return isinstance(cpu_list, list) and len(cpu_list) > 0 def _IsMultiCpuMaskWellFormed(cpu_mask): """Verifies if the given multiple CPU mask is valid A valid multiple CPU mask is in the form "a:b:c:d", where each letter is a single CPU mask. """ try: utils.ParseMultiCpuMask(cpu_mask) except errors.ParseError: return False return True # Read the BaseHypervisor.PARAMETERS docstring for the syntax of the # _CHECK values # must be a file _FILE_CHECK = (utils.IsNormAbsPath, "must be an absolute normalized path", os.path.isfile, "not found or not a file") # must be a file or a URL _FILE_OR_URL_CHECK = (lambda x: utils.IsNormAbsPath(x) or utils.IsUrl(x), "must be an absolute normalized path or a URL", lambda x: os.path.isfile(x) or utils.IsUrl(x), "not found or not a file or URL") # must be a directory _DIR_CHECK = (utils.IsNormAbsPath, "must be an absolute normalized path", os.path.isdir, "not found or not a directory") # CPU mask must be well-formed # TODO: implement node level check for the CPU mask _CPU_MASK_CHECK = (_IsCpuMaskWellFormed, "CPU mask definition is not well-formed", None, None) # Multiple CPU mask must be well-formed _MULTI_CPU_MASK_CHECK = (_IsMultiCpuMaskWellFormed, "Multiple CPU mask definition is not well-formed", None, None) # Check for validity of port number _NET_PORT_CHECK = (lambda x: 0 < x < 65535, "invalid port number", None, None) # Check if number of queues is in safe range _VIRTIO_NET_QUEUES_CHECK = (lambda x: 0 < x < 9, "invalid number of queues", None, None) # Check that an integer is non negative _NONNEGATIVE_INT_CHECK = (lambda x: x >= 0, "cannot be negative", None, None) # nice wrappers for users REQ_FILE_CHECK = (True, ) + _FILE_CHECK OPT_FILE_CHECK = (False, ) + _FILE_CHECK REQ_FILE_OR_URL_CHECK = (True, ) + _FILE_OR_URL_CHECK OPT_FILE_OR_URL_CHECK = (False, ) + _FILE_OR_URL_CHECK REQ_DIR_CHECK = (True, ) + _DIR_CHECK OPT_DIR_CHECK = (False, ) + _DIR_CHECK REQ_NET_PORT_CHECK = (True, ) + _NET_PORT_CHECK OPT_NET_PORT_CHECK = (False, ) + _NET_PORT_CHECK REQ_VIRTIO_NET_QUEUES_CHECK = (True, ) + _VIRTIO_NET_QUEUES_CHECK OPT_VIRTIO_NET_QUEUES_CHECK = (False, ) + _VIRTIO_NET_QUEUES_CHECK REQ_CPU_MASK_CHECK = (True, ) + _CPU_MASK_CHECK OPT_CPU_MASK_CHECK = (False, ) + _CPU_MASK_CHECK REQ_MULTI_CPU_MASK_CHECK = (True, ) + _MULTI_CPU_MASK_CHECK OPT_MULTI_CPU_MASK_CHECK = (False, ) + _MULTI_CPU_MASK_CHECK REQ_NONNEGATIVE_INT_CHECK = (True, ) + _NONNEGATIVE_INT_CHECK OPT_NONNEGATIVE_INT_CHECK = (False, ) + _NONNEGATIVE_INT_CHECK # no checks at all NO_CHECK = (False, None, None, None, None) # required, but no other checks REQUIRED_CHECK = (True, None, None, None, None) # migration type MIGRATION_MODE_CHECK = (True, lambda x: x in constants.HT_MIGRATION_MODES, "invalid migration mode", None, None) def ParamInSet(required, my_set): """Builds parameter checker for set membership. @type required: boolean @param required: whether this is a required parameter @type my_set: tuple, list or set @param my_set: allowed values set """ fn = lambda x: x in my_set err = ("The value must be one of: %s" % utils.CommaJoin(my_set)) return (required, fn, err, None, None) def GenerateTapName(): """Generate a TAP network interface name for a NIC. This helper function generates a special TAP network interface name for NICs that are meant to be used in instance communication. This function checks the existing TAP interfaces in order to find a unique name for the new TAP network interface. The TAP network interface names are of the form 'gnt.com.%d', where '%d' is a unique number within the node. @rtype: string @return: TAP network interface name, or the empty string if the NIC is not used in instance communication """ result = utils.RunCmd(["ip", "link", "show"]) if result.failed: raise errors.HypervisorError("Failed to list TUN/TAP interfaces") idxs = set() for line in result.output.splitlines()[0::2]: parts = line.split(": ") if len(parts) < 2: raise errors.HypervisorError("Failed to parse TUN/TAP interfaces") r = re.match(r"gnt\.com\.([0-9]+)", parts[1]) if r is not None: idxs.add(int(r.group(1))) if idxs: idx = max(idxs) + 1 else: idx = 0 return "gnt.com.%d" % idx def ConfigureNIC(cmd, instance, seq, nic, tap): """Run the network configuration script for a specified NIC @type cmd: string @param cmd: command to run @type instance: instance object @param instance: instance we're acting on @type seq: int @param seq: nic sequence number @type nic: nic object @param nic: nic we're acting on @type tap: str @param tap: the host's tap interface this NIC corresponds to """ env = { "PATH": "%s:/sbin:/usr/sbin" % os.environ["PATH"], "INSTANCE": instance.name, "MAC": nic.mac, "MODE": nic.nicparams[constants.NIC_MODE], "INTERFACE": tap, "INTERFACE_INDEX": str(seq), "INTERFACE_UUID": nic.uuid, "TAGS": " ".join(instance.GetTags()), } if nic.ip: env["IP"] = nic.ip if nic.name: env["INTERFACE_NAME"] = nic.name if nic.nicparams[constants.NIC_LINK]: env["LINK"] = nic.nicparams[constants.NIC_LINK] if constants.NIC_VLAN in nic.nicparams: env["VLAN"] = nic.nicparams[constants.NIC_VLAN] if nic.network: n = objects.Network.FromDict(nic.netinfo) env.update(n.HooksDict()) if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: env["BRIDGE"] = nic.nicparams[constants.NIC_LINK] result = utils.RunCmd(cmd, env=env) if result.failed: raise errors.HypervisorError("Failed to configure interface %s: %s;" " network configuration script output: %s" % (tap, result.fail_reason, result.output)) class HvInstanceState(object): RUNNING = 0 SHUTDOWN = 1 @staticmethod def IsRunning(s): return s == HvInstanceState.RUNNING @staticmethod def IsShutdown(s): return s == HvInstanceState.SHUTDOWN class BaseHypervisor(object): """Abstract virtualisation technology interface The goal is that all aspects of the virtualisation technology are abstracted away from the rest of code. @cvar PARAMETERS: a dict of parameter name: check type; the check type is a five-tuple containing: - the required flag (boolean) - a function to check for syntax, that will be used in L{CheckParameterSyntax}, in the master daemon process - an error message for the above function - a function to check for parameter validity on the remote node, in the L{ValidateParameters} function - an error message for the above function @type CAN_MIGRATE: boolean @cvar CAN_MIGRATE: whether this hypervisor can do migration (either live or non-live) """ PARAMETERS = {} ANCILLARY_FILES = [] ANCILLARY_FILES_OPT = [] CAN_MIGRATE = False def StartInstance(self, instance, block_devices, startup_paused): """Start an instance. @type instance: L{objects.Instance} @param instance: instance to start @type block_devices: list of tuples (disk_object, link_name, drive_uri) @param block_devices: blockdevices assigned to this instance @type startup_paused: bool @param startup_paused: if instance should be paused at startup """ raise NotImplementedError def VerifyInstance(self, instance): # pylint: disable=R0201,W0613 """Verify if running instance (config) is in correct state. @type instance: L{objects.Instance} @param instance: instance to verify @return: bool, if instance in correct state """ return True def RestoreInstance(self, instance, block_devices): """Fixup running instance's (config) state. @type instance: L{objects.Instance} @param instance: instance to restore @type block_devices: list of tuples (disk_object, link_name, drive_uri) @param block_devices: blockdevices assigned to this instance """ pass def StopInstance(self, instance, force=False, retry=False, name=None, timeout=None): """Stop an instance @type instance: L{objects.Instance} @param instance: instance to stop @type force: boolean @param force: whether to do a "hard" stop (destroy) @type retry: boolean @param retry: whether this is just a retry call @type name: string or None @param name: if this parameter is passed, the the instance object should not be used (will be passed as None), and the shutdown must be done by name only @type timeout: int or None @param timeout: if the parameter is not None, a soft shutdown operation will be killed after the specified number of seconds. A hard (forced) shutdown cannot have a timeout @raise errors.HypervisorError: when a parameter is not valid or the instance failed to be stopped """ raise NotImplementedError def CleanupInstance(self, instance_name): """Cleanup after a stopped instance This is an optional method, used by hypervisors that need to cleanup after an instance has been stopped. @type instance_name: string @param instance_name: instance name to cleanup after """ pass def RebootInstance(self, instance): """Reboot an instance.""" raise NotImplementedError def ListInstances(self, hvparams=None): """Get the list of running instances.""" raise NotImplementedError def GetInstanceInfo(self, instance_name, hvparams=None): """Get instance properties. @type instance_name: string @param instance_name: the instance name @type hvparams: dict of strings @param hvparams: hvparams to be used with this instance @rtype: (string, string, int, int, HvInstanceState, int) @return: tuple (name, id, memory, vcpus, state, times) """ raise NotImplementedError def GetAllInstancesInfo(self, hvparams=None): """Get properties of all instances. @type hvparams: dict of strings @param hvparams: hypervisor parameter @rtype: (string, string, int, int, HvInstanceState, int) @return: list of tuples (name, id, memory, vcpus, state, times) """ raise NotImplementedError def GetNodeInfo(self, hvparams=None): """Return information about the node. @type hvparams: dict of strings @param hvparams: hypervisor parameters @return: a dict with at least the following keys (memory values in MiB): - memory_total: the total memory size on the node - memory_free: the available memory on the node for instances - memory_dom0: the memory used by the node itself, if available - cpu_total: total number of CPUs - cpu_dom0: number of CPUs used by the node OS - cpu_nodes: number of NUMA domains - cpu_sockets: number of physical CPU sockets """ raise NotImplementedError @classmethod def GetInstanceConsole(cls, instance, primary_node, node_group, hvparams, beparams): """Return information for connecting to the console of an instance. """ raise NotImplementedError @classmethod def GetAncillaryFiles(cls): """Return a list of ancillary files to be copied to all nodes as ancillary configuration files. @rtype: (list of absolute paths, list of absolute paths) @return: (all files, optional files) """ # By default we return a member variable, so that if an hypervisor has just # a static list of files it doesn't have to override this function. assert set(cls.ANCILLARY_FILES).issuperset(cls.ANCILLARY_FILES_OPT), \ "Optional ancillary files must be a subset of ancillary files" return (cls.ANCILLARY_FILES, cls.ANCILLARY_FILES_OPT) def Verify(self, hvparams=None): """Verify the hypervisor. @type hvparams: dict of strings @param hvparams: hypervisor parameters to be verified against @return: Problem description if something is wrong, C{None} otherwise """ raise NotImplementedError @staticmethod def VersionsSafeForMigration(src, target): """Decide if migration between those version is likely to suceed. Given two versions of a hypervisor, give a guess whether live migration from the one version to the other version is likely to succeed. The current """ if src == target: return True return False def MigrationInfo(self, instance): # pylint: disable=R0201,W0613 """Get instance information to perform a migration. By default assume no information is needed. @type instance: L{objects.Instance} @param instance: instance to be migrated @rtype: string/data (opaque) @return: instance migration information - serialized form """ return "" def AcceptInstance(self, instance, info, target): """Prepare to accept an instance. By default assume no preparation is needed. @type instance: L{objects.Instance} @param instance: instance to be accepted @type info: string/data (opaque) @param info: migration information, from the source node @type target: string @param target: target host (usually ip), on this node """ pass def BalloonInstanceMemory(self, instance, mem): """Balloon an instance memory to a certain value. @type instance: L{objects.Instance} @param instance: instance to be accepted @type mem: int @param mem: actual memory size to use for instance runtime """ raise NotImplementedError def FinalizeMigrationDst(self, instance, info, success): """Finalize the instance migration on the target node. Should finalize or revert any preparation done to accept the instance. Since by default we do no preparation, we also don't have anything to do @type instance: L{objects.Instance} @param instance: instance whose migration is being finalized @type info: string/data (opaque) @param info: migration information, from the source node @type success: boolean @param success: whether the migration was a success or a failure """ pass def MigrateInstance(self, cluster_name, instance, target, live): """Migrate an instance. @type cluster_name: string @param cluster_name: name of the cluster @type instance: L{objects.Instance} @param instance: the instance to be migrated @type target: string @param target: hostname (usually ip) of the target node @type live: boolean @param live: whether to do a live or non-live migration """ raise NotImplementedError def FinalizeMigrationSource(self, instance, success, live): """Finalize the instance migration on the source node. @type instance: L{objects.Instance} @param instance: the instance that was migrated @type success: bool @param success: whether the migration succeeded or not @type live: bool @param live: whether the user requested a live migration or not """ pass def GetMigrationStatus(self, instance): """Get the migration status @type instance: L{objects.Instance} @param instance: the instance that is being migrated @rtype: L{objects.MigrationStatus} @return: the status of the current migration (one of L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional progress info that can be retrieved from the hypervisor """ raise NotImplementedError def _InstanceStartupMemory(self, instance): """Get the correct startup memory for an instance This function calculates how much memory an instance should be started with, making sure it's a value between the minimum and the maximum memory, but also trying to use no more than the current free memory on the node. @type instance: L{objects.Instance} @param instance: the instance that is being started @rtype: integer @return: memory the instance should be started with """ free_memory = self.GetNodeInfo(hvparams=instance.hvparams)["memory_free"] max_start_mem = min(instance.beparams[constants.BE_MAXMEM], free_memory) start_mem = max(instance.beparams[constants.BE_MINMEM], max_start_mem) return start_mem @classmethod def _IsParamValueUnspecified(cls, param_value): """Check if the parameter value is a kind of value meaning unspecified. This function checks if the parameter value is a kind of value meaning unspecified. @type param_value: any @param param_value: the parameter value that needs to be checked @rtype: bool @return: True if the parameter value is a kind of value meaning unspecified, False otherwise """ return param_value is None \ or isinstance(param_value, str) and param_value == "" @classmethod def CheckParameterSyntax(cls, hvparams): """Check the given parameters for validity. This should check the passed set of parameters for validity. Classes should extend, not replace, this function. @type hvparams: dict @param hvparams: dictionary with parameter names/value @raise errors.HypervisorError: when a parameter is not valid """ for key in hvparams: if key not in cls.PARAMETERS: raise errors.HypervisorError("Parameter '%s' is not supported" % key) # cheap tests that run on the master, should not access the world for name, (required, check_fn, errstr, _, _) in cls.PARAMETERS.items(): if name not in hvparams: raise errors.HypervisorError("Parameter '%s' is missing" % name) value = hvparams[name] if not required and cls._IsParamValueUnspecified(value): continue if cls._IsParamValueUnspecified(value): raise errors.HypervisorError("Parameter '%s' is required but" " is currently not defined" % (name, )) if check_fn is not None and not check_fn(value): raise errors.HypervisorError("Parameter '%s' fails syntax" " check: %s (current value: '%s')" % (name, errstr, value)) @classmethod def ValidateParameters(cls, hvparams): """Check the given parameters for validity. This should check the passed set of parameters for validity. Classes should extend, not replace, this function. @type hvparams: dict @param hvparams: dictionary with parameter names/value @raise errors.HypervisorError: when a parameter is not valid """ for name, (required, _, _, check_fn, errstr) in cls.PARAMETERS.items(): value = hvparams[name] if not required and cls._IsParamValueUnspecified(value): continue if check_fn is not None and not check_fn(value): raise errors.HypervisorError("Parameter '%s' fails" " validation: %s (current value: '%s')" % (name, errstr, value)) @classmethod def AssessParameters(cls, hvparams): """Check the given parameters for uncommon/suboptimal values This should check the passed set of parameters for suboptimal values. @type hvparams: dict @param hvparams: dictionary with parameter names/value """ return [] @classmethod def PowercycleNode(cls, hvparams=None): """Hard powercycle a node using hypervisor specific methods. This method should hard powercycle the node, using whatever methods the hypervisor provides. Note that this means that all instances running on the node must be stopped too. @type hvparams: dict of strings @param hvparams: hypervisor params to be used on this node """ raise NotImplementedError @staticmethod def GetLinuxNodeInfo(meminfo="/proc/meminfo", cpuinfo="/proc/cpuinfo"): """For linux systems, return actual OS information. This is an abstraction for all non-hypervisor-based classes, where the node actually sees all the memory and CPUs via the /proc interface and standard commands. The other case if for example xen, where you only see the hardware resources via xen-specific tools. @param meminfo: name of the file containing meminfo @type meminfo: string @param cpuinfo: name of the file containing cpuinfo @type cpuinfo: string @return: a dict with the following keys (values in MiB): - memory_total: the total memory size on the node - memory_free: the available memory on the node for instances - memory_dom0: the memory used by the node itself, if available - cpu_total: total number of CPUs - cpu_dom0: number of CPUs used by the node OS - cpu_nodes: number of NUMA domains - cpu_sockets: number of physical CPU sockets """ try: data = utils.ReadFile(meminfo).splitlines() except EnvironmentError as err: raise errors.HypervisorError("Failed to list node info: %s" % (err,)) result = {} sum_free = 0 try: for line in data: splitfields = line.split(":", 1) if len(splitfields) > 1: key = splitfields[0].strip() val = splitfields[1].strip() if key == "MemTotal": result["memory_total"] = int(val.split()[0]) // 1024 elif key == "MemAvailable": result["memory_free"] = int(val.split()[0]) // 1024 except (ValueError, TypeError) as err: raise errors.HypervisorError("Failed to compute memory usage: %s" % (err,)) result["memory_dom0"] = result["memory_total"] - result["memory_free"] cpu_total = 0 try: fh = open(cpuinfo) try: cpu_total = len(re.findall(r"(?m)^processor\s*:\s*[0-9]+\s*$", fh.read())) finally: fh.close() except EnvironmentError as err: raise errors.HypervisorError("Failed to list node info: %s" % (err,)) result["cpu_total"] = cpu_total # We assume that the node OS can access all the CPUs result["cpu_dom0"] = cpu_total # FIXME: export correct data here result["cpu_nodes"] = 1 result["cpu_sockets"] = 1 return result @classmethod def LinuxPowercycle(cls): """Linux-specific powercycle method. """ try: fd = os.open("/proc/sysrq-trigger", os.O_WRONLY) try: os.write(fd, b"b") finally: fd.close() except OSError: logging.exception("Can't open the sysrq-trigger file") result = utils.RunCmd(["reboot", "-n", "-f"]) if not result: logging.error("Can't run shutdown: %s", result.output) @staticmethod def _FormatVerifyResults(msgs): """Formats the verification results, given a list of errors. @param msgs: list of errors, possibly empty @return: overall problem description if something is wrong, C{None} otherwise """ if msgs: return "; ".join(msgs) else: return None def ResizeDisk(self, instance, disk, new_size): """Notify the hypervisor about a disk change. @type disk: L{objects.Disk} @param disk: the disk to be changed @type new_size: int @param new_size: the new size in bytes @raise errors.HotplugError: if disk resize is not supported """ raise errors.HotplugError("Disk resize is not supported by this hypervisor") # pylint: disable=R0201,W0613 def HotAddDevice(self, instance, dev_type, device, extra, seq): """Hot-add a device. """ raise errors.HotplugError("Hotplug is not supported by this hypervisor") # pylint: disable=R0201,W0613 def HotDelDevice(self, instance, dev_type, device, extra, seq): """Hot-del a device. """ raise errors.HotplugError("Hotplug is not supported by this hypervisor") # pylint: disable=R0201,W0613 def HotModDevice(self, instance, dev_type, device, extra, seq): """Hot-mod a device. """ raise errors.HotplugError("Hotplug is not supported by this hypervisor") # pylint: disable=R0201,W0613 def VerifyHotplugSupport(self, instance, action, dev_type): """Verifies that hotplug is supported. Given the target device and hotplug action checks if hotplug is actually supported. @type instance: L{objects.Instance} @param instance: the instance object @type action: string @param action: one of the supported hotplug commands @type dev_type: string @param dev_type: one of the supported device types to hotplug @raise errors.HotplugError: if hotplugging is not supported """ raise errors.HotplugError("Hotplug is not supported.") def HotplugSupported(self, instance): """Checks if hotplug is supported. By default is not. Currently only KVM hypervisor supports it. """ raise errors.HotplugError("Hotplug is not supported by this hypervisor") ganeti-3.1.0~rc2/lib/hypervisor/hv_chroot.py000064400000000000000000000271461476477700300211430ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Chroot manager hypervisor """ import os import os.path import time import logging from ganeti import constants from ganeti import errors # pylint: disable=W0611 from ganeti import utils from ganeti import objects from ganeti import pathutils from ganeti.hypervisor import hv_base from ganeti.errors import HypervisorError class ChrootManager(hv_base.BaseHypervisor): """Chroot manager. This not-really hypervisor allows ganeti to manage chroots. It has special behaviour and requirements on the OS definition and the node environemnt: - the start and stop of the chroot environment are done via a script called ganeti-chroot located in the root directory of the first drive, which should be created by the OS definition - this script must accept the start and stop argument and, on shutdown, it should cleanly shutdown the daemons/processes using the chroot - the daemons run in chroot should only bind to the instance IP (to which the OS create script has access via the instance name) - since some daemons in the node could be listening on the wildcard address, some ports might be unavailable - the instance listing will show no memory usage - on shutdown, the chroot manager will try to find all mountpoints under the root dir of the instance and unmount them - instance alive check is based on whether any process is using the chroot """ _ROOT_DIR = pathutils.RUN_DIR + "/chroot-hypervisor" PARAMETERS = { constants.HV_INIT_SCRIPT: (True, utils.IsNormAbsPath, "must be an absolute normalized path", None, None), } def __init__(self): hv_base.BaseHypervisor.__init__(self) utils.EnsureDirs([(self._ROOT_DIR, constants.RUN_DIRS_MODE)]) @staticmethod def _IsDirLive(path): """Check if a directory looks like a live chroot. """ if not os.path.ismount(path): return False result = utils.RunCmd(["fuser", "-m", path]) return not result.failed @staticmethod def _GetMountSubdirs(path): """Return the list of mountpoints under a given path. """ result = [] for _, mountpoint, _, _ in utils.GetMounts(): if (mountpoint.startswith(path) and mountpoint != path): result.append(mountpoint) result.sort(key=lambda x: x.count("/"), reverse=True) return result @classmethod def _InstanceDir(cls, instance_name): """Return the root directory for an instance. """ return utils.PathJoin(cls._ROOT_DIR, instance_name) def ListInstances(self, hvparams=None): """Get the list of running instances. """ return [name for name in os.listdir(self._ROOT_DIR) if self._IsDirLive(utils.PathJoin(self._ROOT_DIR, name))] def GetInstanceInfo(self, instance_name, hvparams=None): """Get instance properties. @type instance_name: string @param instance_name: the instance name @type hvparams: dict of strings @param hvparams: hvparams to be used with this instance @return: (name, id, memory, vcpus, stat, times) """ dir_name = self._InstanceDir(instance_name) if not self._IsDirLive(dir_name): return None return (instance_name, 0, 0, 0, hv_base.HvInstanceState.RUNNING, 0) def GetAllInstancesInfo(self, hvparams=None): """Get properties of all instances. @type hvparams: dict of strings @param hvparams: hypervisor parameter @return: [(name, id, memory, vcpus, stat, times),...] """ data = [] for file_name in os.listdir(self._ROOT_DIR): path = utils.PathJoin(self._ROOT_DIR, file_name) if self._IsDirLive(path): data.append((file_name, 0, 0, 0, 0, 0)) return data def StartInstance(self, instance, block_devices, startup_paused): """Start an instance. For the chroot manager, we try to mount the block device and execute '/ganeti-chroot start'. """ root_dir = self._InstanceDir(instance.name) if not os.path.exists(root_dir): try: os.mkdir(root_dir) except IOError as err: raise HypervisorError("Failed to start instance %s: %s" % (instance.name, err)) if not os.path.isdir(root_dir): raise HypervisorError("Needed path %s is not a directory" % root_dir) if not os.path.ismount(root_dir): if not block_devices: raise HypervisorError("The chroot manager needs at least one disk") sda_dev_path = block_devices[0][1] result = utils.RunCmd(["mount", sda_dev_path, root_dir]) if result.failed: raise HypervisorError("Can't mount the chroot dir: %s" % result.output) init_script = instance.hvparams[constants.HV_INIT_SCRIPT] result = utils.RunCmd(["chroot", root_dir, init_script, "start"]) if result.failed: raise HypervisorError("Can't run the chroot start script: %s" % result.output) def StopInstance(self, instance, force=False, retry=False, name=None, timeout=None): """Stop an instance. This method has complicated cleanup tests, as we must: - try to kill all leftover processes - try to unmount any additional sub-mountpoints - finally unmount the instance dir """ assert(timeout is None or force is not None) if name is None: name = instance.name root_dir = self._InstanceDir(name) if not os.path.exists(root_dir) or not self._IsDirLive(root_dir): return timeout_cmd = [] if timeout is not None: timeout_cmd.extend(["timeout", str(timeout)]) # Run the chroot stop script only once if not retry and not force: result = utils.RunCmd(timeout_cmd.extend(["chroot", root_dir, "/ganeti-chroot", "stop"])) if result.failed: raise HypervisorError("Can't run the chroot stop script: %s" % result.output) if not force: utils.RunCmd(["fuser", "-k", "-TERM", "-m", root_dir]) else: utils.RunCmd(["fuser", "-k", "-KILL", "-m", root_dir]) # 2 seconds at most should be enough for KILL to take action time.sleep(2) if self._IsDirLive(root_dir): if force: raise HypervisorError("Can't stop the processes using the chroot") return def CleanupInstance(self, instance_name): """Cleanup after a stopped instance """ root_dir = self._InstanceDir(instance_name) if not os.path.exists(root_dir): return if self._IsDirLive(root_dir): raise HypervisorError("Processes are still using the chroot") for mpath in self._GetMountSubdirs(root_dir): utils.RunCmd(["umount", mpath]) result = utils.RunCmd(["umount", root_dir]) if result.failed: msg = ("Processes still alive in the chroot: %s" % utils.RunCmd("fuser -vm %s" % root_dir).output) logging.error(msg) raise HypervisorError("Can't umount the chroot dir: %s (%s)" % (result.output, msg)) def RebootInstance(self, instance): """Reboot an instance. This is not (yet) implemented for the chroot manager. """ raise HypervisorError("The chroot manager doesn't implement the" " reboot functionality") def BalloonInstanceMemory(self, instance, mem): """Balloon an instance memory to a certain value. @type instance: L{objects.Instance} @param instance: instance to be accepted @type mem: int @param mem: actual memory size to use for instance runtime """ # Currently chroots don't have memory limits pass def GetNodeInfo(self, hvparams=None): """Return information about the node. See L{BaseHypervisor.GetLinuxNodeInfo}. """ return self.GetLinuxNodeInfo() @classmethod def GetInstanceConsole(cls, instance, primary_node, # pylint: disable=W0221 node_group, hvparams, beparams, root_dir=None): """Return information for connecting to the console of an instance. """ if root_dir is None: root_dir = cls._InstanceDir(instance.name) if not os.path.ismount(root_dir): raise HypervisorError("Instance %s is not running" % instance.name) ndparams = node_group.FillND(primary_node) return objects.InstanceConsole(instance=instance.name, kind=constants.CONS_SSH, host=primary_node.name, port=ndparams.get(constants.ND_SSH_PORT), user=constants.SSH_CONSOLE_USER, command=["chroot", root_dir, "/bin/sh"]) def Verify(self, hvparams=None): """Verify the hypervisor. For the chroot manager, it just checks the existence of the base dir. @type hvparams: dict of strings @param hvparams: hypervisor parameters to be verified against, not used in for chroot @return: Problem description if something is wrong, C{None} otherwise """ if os.path.exists(self._ROOT_DIR): return None else: return "The required directory '%s' does not exist" % self._ROOT_DIR @classmethod def PowercycleNode(cls, hvparams=None): """Chroot powercycle, just a wrapper over Linux powercycle. @type hvparams: dict of strings @param hvparams: hypervisor params to be used on this node """ cls.LinuxPowercycle() def MigrateInstance(self, cluster_name, instance, target, live): """Migrate an instance. @type cluster_name: string @param cluster_name: name of the cluster @type instance: L{objects.Instance} @param instance: the instance to be migrated @type target: string @param target: hostname (usually ip) of the target node @type live: boolean @param live: whether to do a live or non-live migration """ raise HypervisorError("Migration not supported by the chroot hypervisor") def GetMigrationStatus(self, instance): """Get the migration status @type instance: L{objects.Instance} @param instance: the instance that is being migrated @rtype: L{objects.MigrationStatus} @return: the status of the current migration (one of L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional progress info that can be retrieved from the hypervisor """ raise HypervisorError("Migration not supported by the chroot hypervisor") ganeti-3.1.0~rc2/lib/hypervisor/hv_fake.py000064400000000000000000000264521476477700300205520ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Fake hypervisor """ import os import os.path import logging from ganeti import utils from ganeti import constants from ganeti import errors from ganeti import objects from ganeti import pathutils from ganeti.hypervisor import hv_base class FakeHypervisor(hv_base.BaseHypervisor): """Fake hypervisor interface. This can be used for testing the ganeti code without having to have a real virtualisation software installed. """ PARAMETERS = { constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK, } CAN_MIGRATE = True _ROOT_DIR = pathutils.RUN_DIR + "/fake-hypervisor" def __init__(self): hv_base.BaseHypervisor.__init__(self) utils.EnsureDirs([(self._ROOT_DIR, constants.RUN_DIRS_MODE)]) def ListInstances(self, hvparams=None): """Get the list of running instances. """ return os.listdir(self._ROOT_DIR) def GetInstanceInfo(self, instance_name, hvparams=None): """Get instance properties. @type instance_name: string @param instance_name: the instance name @type hvparams: dict of strings @param hvparams: hvparams to be used with this instance @return: tuple of (name, id, memory, vcpus, stat, times) """ file_name = self._InstanceFile(instance_name) if not os.path.exists(file_name): return None try: fh = open(file_name, "r") try: inst_id = fh.readline().strip() memory = utils.TryConvert(int, fh.readline().strip()) vcpus = utils.TryConvert(int, fh.readline().strip()) stat = hv_base.HvInstanceState.RUNNING times = 0 return (instance_name, inst_id, memory, vcpus, stat, times) finally: fh.close() except IOError as err: raise errors.HypervisorError("Failed to list instance %s: %s" % (instance_name, err)) def GetAllInstancesInfo(self, hvparams=None): """Get properties of all instances. @type hvparams: dict of strings @param hvparams: hypervisor parameter @return: list of tuples (name, id, memory, vcpus, stat, times) """ data = [] for file_name in os.listdir(self._ROOT_DIR): try: fh = open(utils.PathJoin(self._ROOT_DIR, file_name), "r") inst_id = "-1" memory = 0 vcpus = 1 stat = hv_base.HvInstanceState.SHUTDOWN times = -1 try: inst_id = fh.readline().strip() memory = utils.TryConvert(int, fh.readline().strip()) vcpus = utils.TryConvert(int, fh.readline().strip()) stat = hv_base.HvInstanceState.RUNNING times = 0 finally: fh.close() data.append((file_name, inst_id, memory, vcpus, stat, times)) except IOError as err: raise errors.HypervisorError("Failed to list instances: %s" % err) return data @classmethod def _InstanceFile(cls, instance_name): """Compute the instance file for an instance name. """ return utils.PathJoin(cls._ROOT_DIR, instance_name) def _IsAlive(self, instance_name): """Checks if an instance is alive. """ file_name = self._InstanceFile(instance_name) return os.path.exists(file_name) def _MarkUp(self, instance, memory): """Mark the instance as running. This does no checks, which should be done by its callers. """ file_name = self._InstanceFile(instance.name) fh = open(file_name, "w") try: fh.write("0\n%d\n%d\n" % (memory, instance.beparams[constants.BE_VCPUS])) finally: fh.close() def _MarkDown(self, instance_name): """Mark the instance as running. This does no checks, which should be done by its callers. """ file_name = self._InstanceFile(instance_name) utils.RemoveFile(file_name) def StartInstance(self, instance, block_devices, startup_paused): """Start an instance. For the fake hypervisor, it just creates a file in the base dir, creating an exception if it already exists. We don't actually handle race conditions properly, since these are *FAKE* instances. """ if self._IsAlive(instance.name): raise errors.HypervisorError("Failed to start instance %s: %s" % (instance.name, "already running")) try: self._MarkUp(instance, self._InstanceStartupMemory(instance)) except IOError as err: raise errors.HypervisorError("Failed to start instance %s: %s" % (instance.name, err)) def StopInstance(self, instance, force=False, retry=False, name=None, timeout=None): """Stop an instance. For the fake hypervisor, this just removes the file in the base dir, if it exist, otherwise we raise an exception. """ assert(timeout is None or force is not None) if name is None: name = instance.name if not self._IsAlive(name): raise errors.HypervisorError("Failed to stop instance %s: %s" % (name, "not running")) self._MarkDown(name) def RebootInstance(self, instance): """Reboot an instance. For the fake hypervisor, this does nothing. """ return def BalloonInstanceMemory(self, instance, mem): """Balloon an instance memory to a certain value. @type instance: L{objects.Instance} @param instance: instance to be accepted @type mem: int @param mem: actual memory size to use for instance runtime """ if not self._IsAlive(instance.name): raise errors.HypervisorError("Failed to balloon memory for %s: %s" % (instance.name, "not running")) try: self._MarkUp(instance, mem) except EnvironmentError as err: raise errors.HypervisorError("Failed to balloon memory for %s: %s" % (instance.name, utils.ErrnoOrStr(err))) def GetNodeInfo(self, hvparams=None): """Return information about the node. See L{BaseHypervisor.GetLinuxNodeInfo}. """ result = self.GetLinuxNodeInfo() # substract running instances all_instances = self.GetAllInstancesInfo() result["memory_free"] -= min(result["memory_free"], sum([row[2] for row in all_instances])) return result @classmethod def GetInstanceConsole(cls, instance, primary_node, node_group, hvparams, beparams): """Return information for connecting to the console of an instance. """ return objects.InstanceConsole(instance=instance.name, kind=constants.CONS_MESSAGE, message=("Console not available for fake" " hypervisor")) def Verify(self, hvparams=None): """Verify the hypervisor. For the fake hypervisor, it just checks the existence of the base dir. @type hvparams: dict of strings @param hvparams: hypervisor parameters to be verified against; not used for fake hypervisors @return: Problem description if something is wrong, C{None} otherwise """ if os.path.exists(self._ROOT_DIR): return None else: return "The required directory '%s' does not exist" % self._ROOT_DIR @classmethod def PowercycleNode(cls, hvparams=None): """Fake hypervisor powercycle, just a wrapper over Linux powercycle. @type hvparams: dict of strings @param hvparams: hypervisor params to be used on this node """ cls.LinuxPowercycle() def AcceptInstance(self, instance, info, target): """Prepare to accept an instance. @type instance: L{objects.Instance} @param instance: instance to be accepted @type info: string @param info: instance info, not used @type target: string @param target: target host (usually ip), on this node """ if self._IsAlive(instance.name): raise errors.HypervisorError("Can't accept instance, already running") def MigrateInstance(self, cluster_name, instance, target, live): """Migrate an instance. @type cluster_name: string @param cluster_name: name of the cluster @type instance: L{objects.Instance} @param instance: the instance to be migrated @type target: string @param target: hostname (usually ip) of the target node @type live: boolean @param live: whether to do a live or non-live migration """ logging.debug("Fake hypervisor migrating %s to %s (live=%s)", instance, target, live) def FinalizeMigrationDst(self, instance, info, success): """Finalize the instance migration on the target node. For the fake hv, this just marks the instance up. @type instance: L{objects.Instance} @param instance: instance whose migration is being finalized @type info: string/data (opaque) @param info: migration information, from the source node @type success: boolean @param success: whether the migration was a success or a failure """ if success: self._MarkUp(instance, self._InstanceStartupMemory(instance)) else: # ensure it's down self._MarkDown(instance.name) def FinalizeMigrationSource(self, instance, success, live): """Finalize the instance migration on the source node. @type instance: L{objects.Instance} @param instance: the instance that was migrated @type success: bool @param success: whether the migration succeeded or not @type live: bool @param live: whether the user requested a live migration or not """ # pylint: disable=W0613 if success: self._MarkDown(instance.name) def GetMigrationStatus(self, instance): """Get the migration status The fake hypervisor migration always succeeds. @type instance: L{objects.Instance} @param instance: the instance that is being migrated @rtype: L{objects.MigrationStatus} @return: the status of the current migration (one of L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional progress info that can be retrieved from the hypervisor """ return objects.MigrationStatus(status=constants.HV_MIGRATION_COMPLETED) ganeti-3.1.0~rc2/lib/hypervisor/hv_kvm/000075500000000000000000000000001476477700300200565ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/hypervisor/hv_kvm/__init__.py000064400000000000000000003136221476477700300221760ustar00rootroot00000000000000# # # Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """KVM hypervisor """ import errno import os import os.path import re import tempfile import time import logging import pwd import shlex import shutil import urllib.request, urllib.error, urllib.parse from bitarray import bitarray try: import psutil # pylint: disable=F0401 if psutil.version_info < (2, 0, 0): # The psutil version seems too old, we ignore it psutil_err = \ "too old (2.x.x or newer needed, %s found)" % psutil.__version__ psutil = None else: psutil_err = "" except ImportError: psutil_err = "not found" psutil = None from ganeti import utils from ganeti import constants from ganeti import errors from ganeti import objects from ganeti import uidpool from ganeti import ssconf from ganeti import netutils from ganeti import pathutils from ganeti.hypervisor import hv_base from ganeti.utils import wrapper as utils_wrapper from ganeti.hypervisor.hv_kvm.monitor import QmpConnection, QmpMessage from ganeti.hypervisor.hv_kvm.netdev import OpenTap from ganeti.hypervisor.hv_kvm.kvm_runtime import KVMRuntime from ganeti.hypervisor.hv_kvm.validation import check_boot_parameters, \ check_console_parameters, \ check_disk_cache_parameters, \ check_security_model,\ check_spice_parameters, \ check_vnc_parameters, \ validate_machine_version, \ validate_security_model, \ validate_spice_parameters, \ validate_vnc_parameters, \ validate_disk_parameters import ganeti.hypervisor.hv_kvm.kvm_utils as kvm_utils _KVM_NETWORK_SCRIPT = pathutils.CONF_DIR + "/kvm-vif-bridge" _KVM_START_PAUSED_FLAG = "-S" # below constants show the format of runtime file # the nics are in second possition, while the disks in 4th (last) # moreover disk entries are stored as a list of in tuples # (L{objects.Disk}, link_name, uri) _KVM_NICS_RUNTIME_INDEX = 1 _KVM_DISKS_RUNTIME_INDEX = 3 _DEVICE_RUNTIME_INDEX = { constants.HOTPLUG_TARGET_DISK: _KVM_DISKS_RUNTIME_INDEX, constants.HOTPLUG_TARGET_NIC: _KVM_NICS_RUNTIME_INDEX } _FIND_RUNTIME_ENTRY = { constants.HOTPLUG_TARGET_NIC: lambda nic, kvm_nics: [n for n in kvm_nics if n.uuid == nic.uuid], constants.HOTPLUG_TARGET_DISK: lambda disk, kvm_disks: [(d, l, u) for (d, l, u) in kvm_disks if d.uuid == disk.uuid] } _RUNTIME_DEVICE = { constants.HOTPLUG_TARGET_NIC: lambda d: d, constants.HOTPLUG_TARGET_DISK: lambda d_e_x: d_e_x[0] } _RUNTIME_ENTRY = { constants.HOTPLUG_TARGET_NIC: lambda d, e: d, constants.HOTPLUG_TARGET_DISK: lambda d, e: (d, e[0], e[1]) } _DEVICE_TYPE = { constants.HOTPLUG_TARGET_NIC: lambda hvp: hvp[constants.HV_NIC_TYPE], constants.HOTPLUG_TARGET_DISK: lambda hvp: hvp[constants.HV_DISK_TYPE], } _DEVICE_DRIVER = { constants.HOTPLUG_TARGET_NIC: lambda ht: "virtio-net-pci" if ht == constants.HT_NIC_PARAVIRTUAL else ht, constants.HOTPLUG_TARGET_DISK: lambda ht: "virtio-blk-pci" if ht == constants.HT_DISK_PARAVIRTUAL else ht, } # NICs and paravirtual disks # show up as devices on the PCI bus (one slot per device). # SCSI disks will be placed on the SCSI bus. _DEVICE_BUS = { constants.HOTPLUG_TARGET_NIC: lambda _: _PCI_BUS, constants.HOTPLUG_TARGET_DISK: lambda ht: _SCSI_BUS if ht in constants.HT_SCSI_DEVICE_TYPES else _PCI_BUS } _HOTPLUGGABLE_DEVICE_TYPES = { # All available NIC types except for ne2k_isa constants.HOTPLUG_TARGET_NIC: [ constants.HT_NIC_E1000, constants.HT_NIC_I82551, constants.HT_NIC_I8259ER, constants.HT_NIC_I85557B, constants.HT_NIC_NE2K_PCI, constants.HT_NIC_PARAVIRTUAL, constants.HT_NIC_PCNET, constants.HT_NIC_RTL8139, ], constants.HOTPLUG_TARGET_DISK: [ constants.HT_DISK_PARAVIRTUAL, constants.HT_DISK_SCSI_BLOCK, constants.HT_DISK_SCSI_GENERIC, constants.HT_DISK_SCSI_HD, constants.HT_DISK_SCSI_CD, ] } _PCI_BUS = "pci.0" _SCSI_BUS = "scsi.0" _MIGRATION_CAPS_DELIM = ":" # in future make dirty_sync_count configurable _POSTCOPY_SYNC_COUNT_THRESHOLD = 2 # Precopy passes before enabling postcopy def _with_qmp(fn): """Wrapper used on hotplug related methods""" def wrapper(self, *args, **kwargs): """Create a QmpConnection and run the wrapped method""" if not getattr(self, "qmp", None): for arg in args: if isinstance(arg, objects.Instance): instance = arg break else: raise(RuntimeError("QMP decorator could not find" " a valid ganeti instance object")) filename = self._InstanceQmpMonitor(instance.name)# pylint: disable=W0212 self.qmp = QmpConnection(filename) return fn(self, *args, **kwargs) return wrapper def _GetDriveURI(disk, link, uri): """Helper function to get the drive uri to be used in -blockdev kvm option Invoked during startup and disk hot-add. In latter case and if no userspace access mode is used it will be overriden with /dev/fdset/ (see HotAddDisk() and AddFd() of QmpConnection). @type disk: L{objects.Disk} @param disk: A disk configuration object @type link: string @param link: The device link as returned by _SymlinkBlockDev() @type uri: string @param uri: The drive uri as returned by _CalculateDeviceURI() @return: The drive uri to use in kvm option """ access_mode = disk.params.get(constants.LDP_ACCESS, constants.DISK_KERNELSPACE) # If uri is available, use it during startup/hot-add if uri and access_mode == constants.DISK_USERSPACE: drive_uri = uri # Otherwise use the link previously created else: drive_uri = link return drive_uri def _GenerateDeviceKVMId(dev_type, dev): """Helper function to generate a unique device name used by KVM QEMU monitor commands use names to identify devices. Since the UUID is too long for a device ID (36 chars vs. 30), we choose to use only the part until the third '-' with a disk/nic prefix. For example if a disk has UUID '932df160-7a22-4067-a566-7e0ca8386133' the resulting device ID would be 'disk-932df160-7a22-4067'. @type dev_type: string @param dev_type: device type of param dev (HOTPLUG_TARGET_DISK|NIC) @type dev: L{objects.Disk} or L{objects.NIC} @param dev: the device object for which we generate a kvm name """ return "%s-%s" % (dev_type.lower(), dev.uuid.rsplit("-", 2)[0]) def _GenerateDeviceHVInfoStr(hvinfo): """Construct the -device option string for hvinfo dict PV disk: virtio-blk-pci,id=disk-1234,bus=pci.0,addr=0x9 PV NIC: virtio-net-pci,id=nic-1234,bus=pci.0,addr=0x9 SG disk: scsi-generic,id=disk-1234,bus=scsi.0,channel=0,scsi-id=1,lun=0 @type hvinfo: dict @param hvinfo: dictionary created by _GenerateDeviceHVInfo() @rtype: string @return: The constructed string to be passed along with a -device option """ # work on a copy d = dict(hvinfo) hvinfo_str = d.pop("driver") for k, v in d.items(): hvinfo_str += ",%s=%s" % (k, v) return hvinfo_str def _GenerateDeviceHVInfo(dev_type, kvm_devid, hv_dev_type, bus_slots): """Helper function to generate hvinfo of a device (disk, NIC) hvinfo will hold all necessary info for generating the -device QEMU option. We have two main buses: a PCI bus and a SCSI bus (created by a SCSI controller on the PCI bus). In case of PCI devices we add them on a free PCI slot (addr) on the first PCI bus (pci.0), and in case of SCSI devices we decide to put each disk on a different SCSI target (scsi-id) on the first SCSI bus (scsi.0). @type dev_type: string @param dev_type: either HOTPLUG_TARGET_DISK or HOTPLUG_TARGET_NIC @type kvm_devid: string @param kvm_devid: the id of the device @type hv_dev_type: string @param hv_dev_type: either disk_type or nic_type hvparam @type bus_slots: dict @param bus_slots: the current slots of the first PCI and SCSI buses @rtype: dict @return: dict including all necessary info (driver, id, bus and bus location) for generating a -device QEMU option for either a disk or a NIC """ driver = _DEVICE_DRIVER[dev_type](hv_dev_type) bus = _DEVICE_BUS[dev_type](hv_dev_type) slots = bus_slots[bus] slot = utils.GetFreeSlot(slots, reserve=True) hvinfo = { "driver": driver, "id": kvm_devid, "bus": bus, } if bus == _PCI_BUS: hvinfo.update({ "addr": hex(slot), }) elif bus == _SCSI_BUS: hvinfo.update({ "channel": 0, "scsi-id": slot, "lun": 0, }) return hvinfo def _GetExistingDeviceInfo(dev_type, device, runtime): """Helper function to get an existing device inside the runtime file Used when an instance is running. Load kvm runtime file and search for a device based on its type and uuid. @type dev_type: sting @param dev_type: device type of param dev @type device: L{objects.Disk} or L{objects.NIC} @param device: the device object for which we generate a kvm name @type runtime: tuple (cmd, nics, hvparams, disks) @param runtime: the runtime data to search for the device @raise errors.HotplugError: in case the requested device does not exist (e.g. device has been added without --hotplug option) """ index = _DEVICE_RUNTIME_INDEX[dev_type] found = _FIND_RUNTIME_ENTRY[dev_type](device, runtime[index]) if not found: raise errors.HotplugError("Cannot find runtime info for %s with UUID %s" % (dev_type, device.uuid)) return found[0] class HeadRequest(urllib.request.Request): def get_method(self): return "HEAD" def _CheckUrl(url): """Check if a given URL exists on the server """ try: urllib.request.urlopen(HeadRequest(url)) return True except urllib.error.URLError: return False class KVMHypervisor(hv_base.BaseHypervisor): """KVM hypervisor interface """ CAN_MIGRATE = True _ROOT_DIR = pathutils.RUN_DIR + "/kvm-hypervisor" _PIDS_DIR = _ROOT_DIR + "/pid" # contains live instances pids _UIDS_DIR = _ROOT_DIR + "/uid" # contains instances reserved uids _CTRL_DIR = _ROOT_DIR + "/ctrl" # contains instances control sockets _CONF_DIR = _ROOT_DIR + "/conf" # contains instances startup data _NICS_DIR = _ROOT_DIR + "/nic" # contains instances nic <-> tap associations # KVM instances with chroot enabled are started in empty chroot directories. _CHROOT_DIR = _ROOT_DIR + "/chroot" # for empty chroot directories # After an instance is stopped, its chroot directory is removed. # If the chroot directory is not empty, it can't be removed. # A non-empty chroot directory indicates a possible security incident. # To support forensics, the non-empty chroot directory is quarantined in # a separate directory, called 'chroot-quarantine'. _CHROOT_QUARANTINE_DIR = _ROOT_DIR + "/chroot-quarantine" _DIRS = [_ROOT_DIR, _PIDS_DIR, _UIDS_DIR, _CTRL_DIR, _CONF_DIR, _NICS_DIR, _CHROOT_DIR, _CHROOT_QUARANTINE_DIR] PARAMETERS = { constants.HV_KVM_PATH: hv_base.REQ_FILE_CHECK, constants.HV_KERNEL_PATH: hv_base.OPT_FILE_CHECK, constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK, constants.HV_ROOT_PATH: hv_base.NO_CHECK, constants.HV_KERNEL_ARGS: hv_base.NO_CHECK, constants.HV_ACPI: hv_base.NO_CHECK, constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK, constants.HV_SERIAL_SPEED: hv_base.NO_CHECK, constants.HV_VNC_BIND_ADDRESS: hv_base.NO_CHECK, # will be checked later constants.HV_VNC_TLS: hv_base.NO_CHECK, constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK, constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK, constants.HV_VNC_PASSWORD_FILE: hv_base.OPT_FILE_CHECK, constants.HV_KVM_SPICE_BIND: hv_base.NO_CHECK, # will be checked later constants.HV_KVM_SPICE_IP_VERSION: (False, lambda x: (x == constants.IFACE_NO_IP_VERSION_SPECIFIED or x in constants.VALID_IP_VERSIONS), "The SPICE IP version should be 4 or 6", None, None), constants.HV_KVM_SPICE_PASSWORD_FILE: hv_base.OPT_FILE_CHECK, constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR: hv_base.ParamInSet( False, constants.HT_KVM_SPICE_VALID_LOSSLESS_IMG_COMPR_OPTIONS), constants.HV_KVM_SPICE_JPEG_IMG_COMPR: hv_base.ParamInSet( False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS), constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR: hv_base.ParamInSet( False, constants.HT_KVM_SPICE_VALID_LOSSY_IMG_COMPR_OPTIONS), constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION: hv_base.ParamInSet( False, constants.HT_KVM_SPICE_VALID_VIDEO_STREAM_DETECTION_OPTIONS), constants.HV_KVM_SPICE_AUDIO_COMPR: hv_base.NO_CHECK, constants.HV_KVM_SPICE_USE_TLS: hv_base.NO_CHECK, constants.HV_KVM_SPICE_TLS_CIPHERS: hv_base.NO_CHECK, constants.HV_KVM_SPICE_USE_VDAGENT: hv_base.NO_CHECK, constants.HV_KVM_FLOPPY_IMAGE_PATH: hv_base.OPT_FILE_CHECK, constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_OR_URL_CHECK, constants.HV_KVM_CDROM2_IMAGE_PATH: hv_base.OPT_FILE_OR_URL_CHECK, constants.HV_BOOT_ORDER: hv_base.ParamInSet(True, constants.HT_KVM_VALID_BO_TYPES), constants.HV_NIC_TYPE: hv_base.ParamInSet(True, constants.HT_KVM_VALID_NIC_TYPES), constants.HV_DISK_TYPE: hv_base.ParamInSet(True, constants.HT_KVM_VALID_DISK_TYPES), constants.HV_KVM_SCSI_CONTROLLER_TYPE: hv_base.ParamInSet(True, constants.HT_KVM_VALID_SCSI_CONTROLLER_TYPES), constants.HV_DISK_DISCARD: hv_base.ParamInSet(False, constants.HT_VALID_DISCARD_TYPES), constants.HV_KVM_CDROM_DISK_TYPE: hv_base.ParamInSet(False, constants.HT_KVM_VALID_DISK_TYPES), constants.HV_USB_MOUSE: hv_base.ParamInSet(False, constants.HT_KVM_VALID_MOUSE_TYPES), constants.HV_KEYMAP: hv_base.NO_CHECK, constants.HV_MIGRATION_PORT: hv_base.REQ_NET_PORT_CHECK, constants.HV_MIGRATION_BANDWIDTH: hv_base.REQ_NONNEGATIVE_INT_CHECK, constants.HV_MIGRATION_DOWNTIME: hv_base.REQ_NONNEGATIVE_INT_CHECK, constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK, constants.HV_USE_GUEST_AGENT: hv_base.NO_CHECK, constants.HV_USE_LOCALTIME: hv_base.NO_CHECK, constants.HV_DISK_CACHE: hv_base.ParamInSet(True, constants.HT_VALID_CACHE_TYPES), constants.HV_KVM_DISK_AIO: hv_base.ParamInSet(False, constants.HT_KVM_VALID_AIO_TYPES), constants.HV_SECURITY_MODEL: hv_base.ParamInSet(True, constants.HT_KVM_VALID_SM_TYPES), constants.HV_SECURITY_DOMAIN: hv_base.NO_CHECK, constants.HV_KVM_FLAG: hv_base.ParamInSet(False, constants.HT_KVM_FLAG_VALUES), constants.HV_VHOST_NET: hv_base.NO_CHECK, constants.HV_VIRTIO_NET_QUEUES: hv_base.OPT_VIRTIO_NET_QUEUES_CHECK, constants.HV_KVM_USE_CHROOT: hv_base.NO_CHECK, constants.HV_KVM_USER_SHUTDOWN: hv_base.NO_CHECK, constants.HV_MEM_PATH: hv_base.OPT_DIR_CHECK, constants.HV_REBOOT_BEHAVIOR: hv_base.ParamInSet(True, constants.REBOOT_BEHAVIORS), constants.HV_CPU_MASK: hv_base.OPT_MULTI_CPU_MASK_CHECK, constants.HV_CPU_TYPE: hv_base.NO_CHECK, constants.HV_CPU_CORES: hv_base.OPT_NONNEGATIVE_INT_CHECK, constants.HV_CPU_THREADS: hv_base.OPT_NONNEGATIVE_INT_CHECK, constants.HV_CPU_SOCKETS: hv_base.OPT_NONNEGATIVE_INT_CHECK, constants.HV_SOUNDHW: hv_base.NO_CHECK, constants.HV_USB_DEVICES: hv_base.NO_CHECK, constants.HV_VGA: hv_base.NO_CHECK, constants.HV_KVM_EXTRA: hv_base.NO_CHECK, constants.HV_KVM_MACHINE_VERSION: hv_base.NO_CHECK, constants.HV_KVM_MIGRATION_CAPS: hv_base.NO_CHECK, constants.HV_KVM_PCI_RESERVATIONS: (False, lambda x: (x >= 0 and x <= constants.QEMU_PCI_SLOTS), "The number of PCI slots managed by QEMU (max: %s)" % constants.QEMU_PCI_SLOTS, None, None), constants.HV_VNET_HDR: hv_base.NO_CHECK, } _VIRTIO = "virtio" _VIRTIO_NET_PCI = "virtio-net-pci" _VIRTIO_BLK_PCI = "virtio-blk-pci" _MIGRATION_INFO_MAX_BAD_ANSWERS = 5 _MIGRATION_INFO_RETRY_DELAY = 2 _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)(\.(\d+))?\b") _DEFAULT_MACHINE_VERSION_RE = re.compile(r"^(\S+).*\(default\)", re.M) _QMP_RE = re.compile(r"^-qmp\s", re.M) _VHOST_RE = re.compile(r"^-netdev\stap.*,vhost=on\|off", re.M | re.S) _VIRTIO_NET_QUEUES_RE = re.compile(r"^-netdev\stap.*,fds=x:y:...:z", re.M) _ENABLE_KVM_RE = re.compile(r"^-enable-kvm\s", re.M) _DISABLE_KVM_RE = re.compile(r"^-disable-kvm\s", re.M) _NETDEV_RE = re.compile(r"^-netdev\s", re.M) _DISPLAY_RE = re.compile(r"^-display\s", re.M) _MACHINE_RE = re.compile(r"^-machine\s", re.M) _DEVICE_DRIVER_SUPPORTED = \ staticmethod(lambda drv, devlist: re.compile(r"^name \"%s\"" % drv, re.M).search(devlist)) # match -drive.*boot=on|off on different lines, but in between accept only # dashes not preceeded by a new line (which would mean another option # different than -drive is starting) _BOOT_RE = re.compile(r"^-(drive|blockdev)\s([^-]|(? KVMRuntime: """Generate KVM information to start an instance. @type kvmhelp: string @param kvmhelp: output of kvm --help @attention: this function must not have any side-effects; for example, it must not write to the filesystem, or read values from the current system the are expected to differ between nodes, since it is only run once at instance startup; actions/kvm arguments that can vary between systems should be done in L{_ExecuteKVMRuntime} """ # pylint: disable=R0912,R0914,R0915 hvp = instance.hvparams self.ValidateParameters(hvp) pidfile = self._InstancePidFile(instance.name) kvm = hvp[constants.HV_KVM_PATH] kvm_cmd = [kvm] # used just by the vnc server, if enabled kvm_cmd.extend(["-name", instance.name]) kvm_cmd.extend(["-m", instance.beparams[constants.BE_MAXMEM]]) smp_list = ["%s" % instance.beparams[constants.BE_VCPUS]] if hvp[constants.HV_CPU_CORES]: smp_list.append("cores=%s" % hvp[constants.HV_CPU_CORES]) if hvp[constants.HV_CPU_THREADS]: smp_list.append("threads=%s" % hvp[constants.HV_CPU_THREADS]) if hvp[constants.HV_CPU_SOCKETS]: smp_list.append("sockets=%s" % hvp[constants.HV_CPU_SOCKETS]) kvm_cmd.extend(["-smp", ",".join(smp_list)]) kvm_cmd.extend(["-pidfile", pidfile]) bus_slots = self._GetBusSlots(hvp) if hvp[constants.HV_DISK_TYPE] in constants.HT_SCSI_DEVICE_TYPES \ or hvp[constants.HV_KVM_CDROM_DISK_TYPE]\ in constants.HT_SCSI_DEVICE_TYPES: # In case a SCSI disk is given, QEMU adds a SCSI contorller # (LSI Logic / Symbios Logic 53c895a) implicitly. # Here, we add the controller explicitly with the default id. kvm_cmd.extend([ "-device", "%s,id=scsi" % hvp[constants.HV_KVM_SCSI_CONTROLLER_TYPE] ]) kvm_cmd.extend(["-device", "virtio-balloon"]) kvm_cmd.extend(["-daemonize"]) # logfile for qemu qemu_logfile = utils.PathJoin(pathutils.LOG_KVM_DIR, "%s.log" % instance.name) kvm_cmd.extend(["-D", qemu_logfile]) if instance.hvparams[constants.HV_REBOOT_BEHAVIOR] == \ constants.INSTANCE_REBOOT_EXIT: kvm_cmd.extend(["-no-reboot"]) machine_params = [] mversion = hvp[constants.HV_KVM_MACHINE_VERSION] if not mversion: mversion = self._GetDefaultMachineVersion(kvm) machine_params.append(mversion) if not instance.hvparams[constants.HV_ACPI]: if self._ACPI_RE.search(kvmhelp): # this parameter has been replaced starting with Qemu 9.0 kvm_cmd.extend(["-no-acpi"]) else: machine_params.append("acpi=off") if hvp[constants.HV_KVM_FLAG] == constants.HT_KVM_ENABLED: machine_params.append("accel=kvm") kvm_cmd.extend(["-machine", ",".join(machine_params)]) kernel_path = hvp[constants.HV_KERNEL_PATH] if kernel_path: boot_cdrom = boot_floppy = boot_network = False else: boot_cdrom = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_CDROM boot_floppy = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_FLOPPY boot_network = hvp[constants.HV_BOOT_ORDER] == constants.HT_BO_NETWORK if startup_paused: kvm_cmd.extend([_KVM_START_PAUSED_FLAG]) if boot_network: kvm_cmd.extend(["-boot", "order=n"]) disk_type = hvp[constants.HV_DISK_TYPE] # Now we can specify a different device type for CDROM devices. cdrom_disk_type = hvp[constants.HV_KVM_CDROM_DISK_TYPE] if not cdrom_disk_type: cdrom_disk_type = disk_type cdrom_image1 = hvp[constants.HV_CDROM_IMAGE_PATH] if cdrom_image1: self._CdromOption(kvm_cmd, cdrom_disk_type, cdrom_image1, boot_cdrom, "cdrom1") cdrom_image2 = hvp[constants.HV_KVM_CDROM2_IMAGE_PATH] if cdrom_image2: self._CdromOption(kvm_cmd, cdrom_disk_type, cdrom_image2, False, "cdrom2") floppy_image = hvp[constants.HV_KVM_FLOPPY_IMAGE_PATH] if floppy_image: self._FloppyOption(kvm_cmd, floppy_image, boot_floppy) if kernel_path: kvm_cmd.extend(["-kernel", kernel_path]) initrd_path = hvp[constants.HV_INITRD_PATH] if initrd_path: kvm_cmd.extend(["-initrd", initrd_path]) root_append = ["root=%s" % hvp[constants.HV_ROOT_PATH], hvp[constants.HV_KERNEL_ARGS]] if hvp[constants.HV_SERIAL_CONSOLE]: serial_speed = hvp[constants.HV_SERIAL_SPEED] root_append.append("console=ttyS0,%s" % serial_speed) kvm_cmd.extend(["-append", " ".join(root_append)]) mem_path = hvp[constants.HV_MEM_PATH] if mem_path: kvm_cmd.extend(["-mem-path", mem_path, "-mem-prealloc"]) monitor_dev = ("unix:%s,server,nowait" % self._InstanceMonitor(instance.name)) kvm_cmd.extend(["-monitor", monitor_dev]) if hvp[constants.HV_SERIAL_CONSOLE]: serial_dev = ("unix:%s,server,nowait" % self._InstanceSerial(instance.name)) kvm_cmd.extend(["-serial", serial_dev]) else: kvm_cmd.extend(["-serial", "none"]) mouse_type = hvp[constants.HV_USB_MOUSE] vnc_bind_address = hvp[constants.HV_VNC_BIND_ADDRESS] spice_bind = hvp[constants.HV_KVM_SPICE_BIND] spice_ip_version = None kvm_cmd.extend(["-usb"]) if mouse_type: kvm_cmd.extend(["-usbdevice", mouse_type]) elif vnc_bind_address: kvm_cmd.extend(["-usbdevice", constants.HT_MOUSE_TABLET]) if vnc_bind_address: if netutils.IsValidInterface(vnc_bind_address): if_addresses = netutils.GetInterfaceIpAddresses(vnc_bind_address) if_ip4_addresses = if_addresses[constants.IP4_VERSION] if len(if_ip4_addresses) < 1: logging.error("Could not determine IPv4 address of interface %s", vnc_bind_address) else: vnc_bind_address = if_ip4_addresses[0] if (netutils.IP4Address.IsValid(vnc_bind_address) or netutils.IP6Address.IsValid(vnc_bind_address)): if instance.network_port > constants.VNC_BASE_PORT: display = instance.network_port - constants.VNC_BASE_PORT if vnc_bind_address == constants.IP4_ADDRESS_ANY: vnc_arg = ":%d" % (display) elif netutils.IP6Address.IsValid(vnc_bind_address): vnc_arg = "[%s]:%d" % (vnc_bind_address, display) else: vnc_arg = "%s:%d" % (vnc_bind_address, display) else: logging.error("Network port is not a valid VNC display (%d < %d)," " not starting VNC", instance.network_port, constants.VNC_BASE_PORT) vnc_arg = "none" # Only allow tls and other option when not binding to a file, for now. # kvm/qemu gets confused otherwise about the filename to use. vnc_append = "" if hvp[constants.HV_VNC_TLS]: vnc_append = "%s,tls-creds=vnctls0" % vnc_append tls_obj = "tls-creds-anon" tls_obj_options = ["id=vnctls0", "endpoint=server"] if hvp[constants.HV_VNC_X509_VERIFY]: tls_obj = "tls-creds-x509" tls_obj_options.extend(["dir=%s" % hvp[constants.HV_VNC_X509], "verify-peer=yes"]) elif hvp[constants.HV_VNC_X509]: tls_obj = "tls-creds-x509" tls_obj_options.extend(["dir=%s" % hvp[constants.HV_VNC_X509], "verify-peer=no"]) kvm_cmd.extend(["-object", "%s,%s" % (tls_obj, ",".join(tls_obj_options))]) if hvp[constants.HV_VNC_PASSWORD_FILE]: vnc_append = "%s,password" % vnc_append vnc_arg = "%s%s" % (vnc_arg, vnc_append) else: vnc_arg = "unix:%s/%s.vnc" % (vnc_bind_address, instance.name) kvm_cmd.extend(["-vnc", vnc_arg]) elif spice_bind: # FIXME: this is wrong here; the iface ip address differs # between systems, so it should be done in _ExecuteKVMRuntime if netutils.IsValidInterface(spice_bind): # The user specified a network interface, we have to figure out the IP # address. addresses = netutils.GetInterfaceIpAddresses(spice_bind) spice_ip_version = hvp[constants.HV_KVM_SPICE_IP_VERSION] # if the user specified an IP version and the interface does not # have that kind of IP addresses, throw an exception if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED: if not addresses[spice_ip_version]: raise errors.HypervisorError("SPICE: Unable to get an IPv%s address" " for %s" % (spice_ip_version, spice_bind)) # the user did not specify an IP version, we have to figure it out elif (addresses[constants.IP4_VERSION] and addresses[constants.IP6_VERSION]): # we have both ipv4 and ipv6, let's use the cluster default IP # version cluster_family = ssconf.SimpleStore().GetPrimaryIPFamily() spice_ip_version = \ netutils.IPAddress.GetVersionFromAddressFamily(cluster_family) elif addresses[constants.IP4_VERSION]: spice_ip_version = constants.IP4_VERSION elif addresses[constants.IP6_VERSION]: spice_ip_version = constants.IP6_VERSION else: raise errors.HypervisorError("SPICE: Unable to get an IP address" " for %s" % (spice_bind)) spice_address = addresses[spice_ip_version][0] else: # spice_bind is known to be a valid IP address, because # ValidateParameters checked it. spice_address = spice_bind spice_arg = "addr=%s" % spice_address if hvp[constants.HV_KVM_SPICE_USE_TLS]: spice_arg = ("%s,tls-port=%s,x509-cacert-file=%s" % (spice_arg, instance.network_port, pathutils.SPICE_CACERT_FILE)) spice_arg = ("%s,x509-key-file=%s,x509-cert-file=%s" % (spice_arg, pathutils.SPICE_CERT_FILE, pathutils.SPICE_CERT_FILE)) tls_ciphers = hvp[constants.HV_KVM_SPICE_TLS_CIPHERS] if tls_ciphers: spice_arg = "%s,tls-ciphers=%s" % (spice_arg, tls_ciphers) else: spice_arg = "%s,port=%s" % (spice_arg, instance.network_port) if not hvp[constants.HV_KVM_SPICE_PASSWORD_FILE]: spice_arg = "%s,disable-ticketing" % spice_arg if spice_ip_version: spice_arg = "%s,ipv%s" % (spice_arg, spice_ip_version) # Image compression options img_lossless = hvp[constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR] img_jpeg = hvp[constants.HV_KVM_SPICE_JPEG_IMG_COMPR] img_zlib_glz = hvp[constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR] if img_lossless: spice_arg = "%s,image-compression=%s" % (spice_arg, img_lossless) if img_jpeg: spice_arg = "%s,jpeg-wan-compression=%s" % (spice_arg, img_jpeg) if img_zlib_glz: spice_arg = "%s,zlib-glz-wan-compression=%s" % (spice_arg, img_zlib_glz) # Video stream detection video_streaming = hvp[constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION] if video_streaming: spice_arg = "%s,streaming-video=%s" % (spice_arg, video_streaming) # Audio compression, by default in qemu-kvm it is on if not hvp[constants.HV_KVM_SPICE_AUDIO_COMPR]: spice_arg = "%s,playback-compression=off" % spice_arg if not hvp[constants.HV_KVM_SPICE_USE_VDAGENT]: spice_arg = "%s,agent-mouse=off" % spice_arg else: # Enable the spice agent communication channel between the host and the # agent. kvm_cmd.extend(["-device", "virtio-serial-pci,id=spice"]) kvm_cmd.extend([ "-device", "virtserialport,chardev=spicechannel0,name=com.redhat.spice.0", ]) kvm_cmd.extend(["-chardev", "spicevmc,id=spicechannel0,name=vdagent"]) logging.info("KVM: SPICE will listen on port %s", instance.network_port) kvm_cmd.extend(["-spice", spice_arg]) else: # From qemu 1.4 -nographic is incompatible with -daemonize. The new way # also works in earlier versions though (tested with 1.1 and 1.3) if self._DISPLAY_RE.search(kvmhelp): kvm_cmd.extend(["-display", "none"]) else: kvm_cmd.extend(["-nographic"]) # As requested by music lovers if hvp[constants.HV_SOUNDHW]: soundhw = hvp[constants.HV_SOUNDHW] if self._SOUND_RE.search(kvmhelp): kvm_cmd.extend(["-soundhw", soundhw]) else: # Qemu versions >= 7.1 do not support -soundhw anymore # also, we need to pick a host backend/driver. we'll use # spice if that is configured, otherwise there will be # no sound output if spice_bind: driver = "spice" else: driver = "none" kvm_cmd.extend(["-audio", "driver={},model={},id=soundhw" .format(driver, soundhw)]) if hvp[constants.HV_USE_LOCALTIME]: kvm_cmd.extend(["-rtc", "base=localtime"]) # Add qemu-KVM -cpu param if hvp[constants.HV_CPU_TYPE]: kvm_cmd.extend(["-cpu", hvp[constants.HV_CPU_TYPE]]) # Pass a -vga option if requested, or if spice is used, for backwards # compatibility. if hvp[constants.HV_VGA]: kvm_cmd.extend(["-vga", hvp[constants.HV_VGA]]) elif spice_bind: kvm_cmd.extend(["-vga", "qxl"]) # Various types of usb devices, comma separated if hvp[constants.HV_USB_DEVICES]: for dev in hvp[constants.HV_USB_DEVICES].split(","): kvm_cmd.extend(["-usbdevice", dev]) # Set system UUID to instance UUID if self._UUID_RE.search(kvmhelp): kvm_cmd.extend(["-uuid", instance.uuid]) # Add guest agent socket if hvp[constants.HV_USE_GUEST_AGENT]: qga_path = self._InstanceQemuGuestAgentMonitor(instance.name) logging.info("KVM: Guest Agent available at %s", qga_path) # The 'qga0' identified can change, but the 'org.qemu.guest_agent.0' # string is the default expected by the Guest Agent. kvm_cmd.extend([ "-chardev", "socket,path=%s,server,nowait,id=qga0" % qga_path, "-device", "virtio-serial,id=qga0", "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0", ]) if hvp[constants.HV_KVM_EXTRA]: kvm_cmd.extend( shlex.split(hvp[constants.HV_KVM_EXTRA]) ) def _generate_kvm_device(dev_type, dev): """Helper for generating a kvm device out of a Ganeti device.""" kvm_devid = _GenerateDeviceKVMId(dev_type, dev) hv_dev_type = _DEVICE_TYPE[dev_type](hvp) dev.hvinfo = _GenerateDeviceHVInfo(dev_type, kvm_devid, hv_dev_type, bus_slots) kvm_disks = [] for disk, link_name, uri in block_devices: _generate_kvm_device(constants.HOTPLUG_TARGET_DISK, disk) kvm_disks.append((disk, link_name, uri)) kvm_nics = [] for nic in instance.nics: _generate_kvm_device(constants.HOTPLUG_TARGET_NIC, nic) kvm_nics.append(nic) hvparams = hvp return KVMRuntime([kvm_cmd, kvm_nics, hvparams, kvm_disks]) def _WriteKVMRuntime(self, instance_name, data): """Write an instance's KVM runtime """ try: utils.WriteFile(self._InstanceKVMRuntime(instance_name), data=data) except EnvironmentError as err: raise errors.HypervisorError("Failed to save KVM runtime file: %s" % err) def _ReadKVMRuntime(self, instance_name): """Read an instance's KVM runtime """ try: file_content = utils.ReadFile(self._InstanceKVMRuntime(instance_name)) except EnvironmentError as err: raise errors.HypervisorError("Failed to load KVM runtime file: %s" % err) return file_content def _SaveKVMRuntime(self, instance, kvm_runtime: KVMRuntime): """Save an instance's KVM runtime """ self._WriteKVMRuntime(instance.name, kvm_runtime.serialize()) def _LoadKVMRuntime(self, instance, serialized_runtime=None) -> KVMRuntime: """Load an instance's KVM runtime """ if not serialized_runtime: serialized_runtime = self._ReadKVMRuntime(instance.name) return KVMRuntime.from_serialized(serialized_runtime) def _RunKVMCmd(self, name, kvm_cmd, tap_fds=None): """Run the KVM cmd and check for errors @type name: string @param name: instance name @type kvm_cmd: list of strings @param kvm_cmd: runcmd input for kvm @type tap_fds: list of int @param tap_fds: fds of tap devices opened by Ganeti """ try: result = utils.RunCmd(kvm_cmd, noclose_fds=tap_fds) finally: for fd in tap_fds: utils_wrapper.CloseFdNoError(fd) if result.failed: raise errors.HypervisorError("Failed to start instance %s: %s (%s)" % (name, result.fail_reason, result.output)) if not self._InstancePidAlive(name)[2]: raise errors.HypervisorError("Failed to start instance %s" % name) @staticmethod def _GenerateKvmTapName(nic): """Generate a TAP network interface name for a NIC. See L{hv_base.GenerateTapName}. For the case of the empty string, see L{OpenTap} @type nic: ganeti.objects.NIC @param nic: NIC object for the name should be generated @rtype: string @return: TAP network interface name, or the empty string if the NIC is not used in instance communication """ if nic.name is None or not \ nic.name.startswith(constants.INSTANCE_COMMUNICATION_NIC_PREFIX): return "" return hv_base.GenerateTapName() def _GetNetworkDeviceFeatures(self, up_hvp, devlist, kvmhelp): """Get network device options to properly enable supported features. Return a dict of supported and enabled tap features with nic_model along with the extra strings to be appended to the --netdev and --device options. This function is called before opening a new tap device. Currently the features_dict includes the following attributes: - vhost (boolean) - vnet_hdr (boolean) - mq (boolean, int) @rtype: (dict, str, str) tuple @return: The supported features, the string to be appended to the --netdev option, the string to be appended to the --device option """ nic_type = up_hvp[constants.HV_NIC_TYPE] nic_extra_str = "" tap_extra_str = "" features = { "vhost": False, "vnet_hdr": False, "mq": (False, 1) } update_features = {} if nic_type == constants.HT_NIC_PARAVIRTUAL: if self._DEVICE_DRIVER_SUPPORTED(self._VIRTIO_NET_PCI, devlist): nic_model = self._VIRTIO_NET_PCI update_features["vnet_hdr"] = up_hvp[constants.HV_VNET_HDR] else: # Older versions of kvm don't support DEVICE_LIST, but they don't # have new virtio syntax either. nic_model = self._VIRTIO if up_hvp[constants.HV_VHOST_NET]: # Check for vhost_net support. if self._VHOST_RE.search(kvmhelp): update_features["vhost"] = True tap_extra_str = ",vhost=on" else: raise errors.HypervisorError("vhost_net is configured" " but it is not available") virtio_net_queues = up_hvp.get(constants.HV_VIRTIO_NET_QUEUES, 1) if virtio_net_queues > 1: # Check for multiqueue virtio-net support. if self._VIRTIO_NET_QUEUES_RE.search(kvmhelp): # As advised at http://www.linux-kvm.org/page/Multiqueue formula # for calculating vector size is: vectors=2*N+2 where N is the # number of queues (HV_VIRTIO_NET_QUEUES). nic_extra_str = ",mq=on,vectors=%d" % (2 * virtio_net_queues + 2) update_features["mq"] = (True, virtio_net_queues) else: raise errors.HypervisorError("virtio_net_queues is configured" " but it is not available") else: nic_model = nic_type update_features["driver"] = nic_model features.update(update_features) return features, tap_extra_str, nic_extra_str def _GenerateRunwith(self, username=None, chroot_dir=None, kvmhelp=None): args = [] if self._RUNWITH_RE.search(kvmhelp): if username: args.append("user=%s" % username) if chroot_dir: args.append("chroot=%s" % chroot_dir) return(["-run-with", ",".join(args)]) else: if username: args.extend(["-runas", username]) if chroot_dir: args.extend(["-chroot", chroot_dir]) return(args) # nothing to do return([]) # too many local variables # pylint: disable=R0914 @_with_qmp def _ExecuteKVMRuntime(self, instance, kvm_runtime: KVMRuntime, kvmhelp, incoming=None): """Execute a KVM cmd, after completing it with some last minute data. @type instance: L{objects.Instance} object @param instance: the VM this command acts upon @type kvm_runtime: tuple of (list of str, list of L{objects.NIC} objects, dict of hypervisor options, list of tuples (L{objects.Disk}, str, str) @param kvm_runtime: (kvm command, NICs of the instance, options at startup of the instance, [(disk, link_name, uri)..]) @type incoming: tuple of strings @param incoming: (target_host_ip, port) for migration. @type kvmhelp: string @param kvmhelp: output of kvm --help """ # Small _ExecuteKVMRuntime hv parameters programming howto: # - conf_hvp contains the parameters as configured on ganeti. they might # have changed since the instance started; only use them if the change # won't affect the inside of the instance (which hasn't been rebooted). # - up_hvp contains the parameters as they were when the instance was # started, plus any new parameter which has been added between ganeti # versions: it is paramount that those default to a value which won't # affect the inside of the instance as well. conf_hvp = instance.hvparams name = instance.name self._CheckDown(name) self._ClearUserShutdown(instance.name) self._StartKvmd(instance.hvparams) temp_files = [] kvm_cmd = kvm_runtime.kvm_cmd kvm_nics = kvm_runtime.kvm_nics up_hvp = kvm_runtime.up_hvp kvm_disks = kvm_runtime.kvm_disks # the first element of kvm_cmd is always the path to the kvm binary kvm_path = kvm_cmd[0] up_hvp = objects.FillDict(conf_hvp, up_hvp) # the VNC keymap keymap = conf_hvp[constants.HV_KEYMAP] if keymap: kvm_cmd.extend(["-k", keymap]) # We have reasons to believe changing something like the nic driver/type # upon migration won't exactly fly with the instance kernel, so for nic # related parameters we'll use up_hvp tapfds = [] taps = [] devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST) if not kvm_nics: kvm_cmd.extend(["-net", "none"]) else: features, tap_extra, nic_extra = \ self._GetNetworkDeviceFeatures(up_hvp, devlist, kvmhelp) nic_model = features["driver"] kvm_supports_netdev = self._NETDEV_RE.search(kvmhelp) for nic_seq, nic in enumerate(kvm_nics): tapname, nic_tapfds, nic_vhostfds = \ OpenTap(features=features, name=self._GenerateKvmTapName(nic)) tapfds.extend(nic_tapfds) tapfds.extend(nic_vhostfds) taps.append(tapname) tapfd = "%s%s" % ("fds=" if len(nic_tapfds) > 1 else "fd=", ":".join(str(fd) for fd in nic_tapfds)) if nic_vhostfds: vhostfd = "%s%s" % (",vhostfds=" if len(nic_vhostfds) > 1 else ",vhostfd=", ":".join(str(fd) for fd in nic_vhostfds)) else: vhostfd = "" if kvm_supports_netdev: # Non paravirtual NICs hvinfo is empty if "id" in nic.hvinfo: nic_val = _GenerateDeviceHVInfoStr(nic.hvinfo) netdev = nic.hvinfo["id"] else: nic_val = "%s" % nic_model netdev = "netdev%d" % nic_seq nic_val += (",netdev=%s,mac=%s%s" % (netdev, nic.mac, nic_extra)) tap_val = ("type=tap,id=%s,%s%s%s" % (netdev, tapfd, vhostfd, tap_extra)) kvm_cmd.extend(["-netdev", tap_val, "-device", nic_val]) else: nic_val = "nic,vlan=%s,macaddr=%s,model=%s" % (nic_seq, nic.mac, nic_model) tap_val = "tap,vlan=%s,%s" % (nic_seq, tapfd) kvm_cmd.extend(["-net", tap_val, "-net", nic_val]) if incoming: target, port = incoming kvm_cmd.extend(["-incoming", "tcp:%s:%s" % (target, port)]) # Changing the vnc password doesn't bother the guest that much. At most it # will surprise people who connect to it. Whether positively or negatively # it's debatable. vnc_pwd_file = conf_hvp[constants.HV_VNC_PASSWORD_FILE] vnc_pwd = None if vnc_pwd_file: try: vnc_pwd = utils.ReadOneLineFile(vnc_pwd_file, strict=True) except EnvironmentError as err: raise errors.HypervisorError("Failed to open VNC password file %s: %s" % (vnc_pwd_file, err)) if conf_hvp[constants.HV_KVM_USE_CHROOT]: utils.EnsureDirs([(self._InstanceChrootDir(name), constants.SECURE_DIR_MODE)]) # Automatically enable QMP if version is >= 0.14 if self._QMP_RE.search(kvmhelp): logging.debug("Enabling QMP") kvm_cmd.extend(["-qmp", "unix:%s,server,nowait" % self._InstanceQmpMonitor(instance.name)]) # Add a second monitor for kvmd kvm_cmd.extend(["-qmp", "unix:%s,server,nowait" % self._InstanceKvmdMonitor(instance.name)]) # Configure the network now for starting instances and bridged/OVS # interfaces, during FinalizeMigration for incoming instances' routed # interfaces. for nic_seq, nic in enumerate(kvm_nics): if (incoming and nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_ROUTED): continue self._ConfigureNIC(instance, nic_seq, nic, taps[nic_seq]) bdev_opts = self._GenerateKVMBlockDevicesOptions(up_hvp, kvm_disks, kvmhelp, devlist) kvm_cmd.extend(bdev_opts) # CPU affinity requires kvm to start paused, so we set this flag if the # instance is not already paused and if we are not going to accept a # migrating instance. In the latter case, pausing is not needed. start_kvm_paused = not (_KVM_START_PAUSED_FLAG in kvm_cmd) and not incoming if start_kvm_paused: kvm_cmd.extend([_KVM_START_PAUSED_FLAG]) # Note: CPU pinning is using up_hvp since changes take effect # during instance startup anyway, and to avoid problems when soft # rebooting the instance. cpu_pinning = False if up_hvp.get(constants.HV_CPU_MASK, None) \ and up_hvp[constants.HV_CPU_MASK] != constants.CPU_PINNING_ALL: cpu_pinning = True # chroot and user are combind into -runwith since qemu-9.0 chroot_dir = None security_model = conf_hvp[constants.HV_SECURITY_MODEL] if conf_hvp[constants.HV_KVM_USE_CHROOT]: chroot_dir = self._InstanceChrootDir(instance.name) # only chroot is set, no run user if security_model == constants.HT_SM_NONE: kvm_cmd.extend( self._GenerateRunwith(chroot_dir=chroot_dir, kvmhelp=kvmhelp) ) if security_model == constants.HT_SM_USER: username = conf_hvp[constants.HV_SECURITY_DOMAIN] kvm_cmd.extend( self._GenerateRunwith(chroot_dir=chroot_dir, username=username, kvmhelp=kvmhelp) ) if security_model == constants.HT_SM_POOL: ss = ssconf.SimpleStore() uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\n") all_uids = set(uidpool.ExpandUidPool(uid_pool)) uid = uidpool.RequestUnusedUid(all_uids) try: username = pwd.getpwuid(uid.GetUid()).pw_name kvm_cmd.extend( self._GenerateRunwith(chroot_dir=chroot_dir, username=username, kvmhelp=kvmhelp) ) self._RunKVMCmd(name, kvm_cmd, tapfds) except: uidpool.ReleaseUid(uid) raise else: uid.Unlock() utils.WriteFile(self._InstanceUidFile(name), data=uid.AsStr()) else: self._RunKVMCmd(name, kvm_cmd, tapfds) utils.EnsureDirs([(self._InstanceNICDir(instance.name), constants.RUN_DIRS_MODE)]) for nic_seq, tap in enumerate(taps): utils.WriteFile(self._InstanceNICFile(instance.name, nic_seq), data=tap) if vnc_pwd: self.qmp.SetVNCPassword(vnc_pwd) # Setting SPICE password. We are not vulnerable to malicious passwordless # connection attempts because SPICE by default does not allow connections # if neither a password nor the "disable_ticketing" options are specified. # As soon as we send the password via QMP, that password is a valid ticket # for connection. spice_password_file = conf_hvp[constants.HV_KVM_SPICE_PASSWORD_FILE] if spice_password_file: spice_pwd = "" try: spice_pwd = utils.ReadOneLineFile(spice_password_file, strict=True) except EnvironmentError as err: raise errors.HypervisorError("Failed to open SPICE password file %s: %s" % (spice_password_file, err)) self.qmp.SetSpicePassword(spice_pwd) for filename in temp_files: utils.RemoveFile(filename) # If requested, set CPU affinity and resume instance execution if cpu_pinning: self._ExecuteCpuAffinity(instance, up_hvp[constants.HV_CPU_MASK]) start_memory = self._InstanceStartupMemory(instance) if start_memory < instance.beparams[constants.BE_MAXMEM]: self.BalloonInstanceMemory(instance, start_memory) if start_kvm_paused: # To control CPU pinning, ballooning, and vnc/spice passwords # the VM was started in a frozen state. If freezing was not # explicitly requested resume the vm status. self.qmp.ContinueGuestEmulation() @staticmethod def _StartKvmd(hvparams): """Ensure that the Kvm daemon is running. @type hvparams: dict of strings @param hvparams: hypervisor parameters """ if hvparams is None \ or not hvparams[constants.HV_KVM_USER_SHUTDOWN] \ or utils.IsDaemonAlive(constants.KVMD): return result = utils.RunCmd([pathutils.DAEMON_UTIL, "start", constants.KVMD]) if result.failed: raise errors.HypervisorError("Failed to start KVM daemon") def StartInstance(self, instance, block_devices, startup_paused): """Start an instance. """ self._CheckDown(instance.name) kvmpath = instance.hvparams[constants.HV_KVM_PATH] kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP) kvm_runtime = self._GenerateKVMRuntime(instance, block_devices, startup_paused, kvmhelp) self._SaveKVMRuntime(instance, kvm_runtime) self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp) @_with_qmp def VerifyHotplugSupport(self, instance, action, dev_type): """Verifies that hotplug is supported. Hotplug is not supported if: - the instance is not running - the device type is not hotplug-able - the QMP version does not support the corresponding commands @raise errors.HypervisorError: if one of the above applies """ runtime = self._LoadKVMRuntime(instance) device_type = _DEVICE_TYPE[dev_type](runtime[2]) if device_type not in _HOTPLUGGABLE_DEVICE_TYPES[dev_type]: msg = "Hotplug is not supported for device type %s" % device_type raise errors.HypervisorError(msg) if dev_type == constants.HOTPLUG_TARGET_DISK: if action == constants.HOTPLUG_ACTION_ADD: self.qmp.CheckDiskHotAddSupport() if dev_type == constants.HOTPLUG_TARGET_NIC: if action == constants.HOTPLUG_ACTION_ADD: self.qmp.CheckNicHotAddSupport() @_with_qmp def ResizeDisk(self, instance, disk, new_size): """ Notify the HV about a disk change. """ disk_id = _GenerateDeviceKVMId('disk', disk) self.qmp.ResizeBlockDevice(disk_id, new_size) @_with_qmp def HotplugSupported(self, instance): """Checks if hotplug is generally supported. Hotplug is *not* supported in case of: - qemu versions < 1.7 (where all qmp related commands are supported) - for stopped instances @raise errors.HypervisorError: in one of the previous cases """ try: version = self.qmp.GetVersion() except errors.HypervisorError: raise errors.HotplugError("Instance is probably down") #TODO: delegate more fine-grained checks to VerifyHotplugSupport if version < (1, 7, 0): raise errors.HotplugError("Hotplug not supported for qemu versions < 1.7") def _GetBusSlots(self, hvp=None, runtime: KVMRuntime=None): """Helper function to get the slots of PCI and SCSI QEMU buses. This will return the status of the first PCI and SCSI buses. By default QEMU boots with one PCI bus (pci.0) and occupies the first 3 PCI slots. If a SCSI disk is found then a SCSI controller is added on the PCI bus and a SCSI bus (scsi.0) is created. During hotplug we could query QEMU via info qtree HMP command but parsing the result is too complicated. Instead we use the info stored in runtime files. We parse NIC and disk entries and based on their hvinfo we reserve the corresponding slots. The runtime argument is a tuple as returned by _LoadKVMRuntime(). Obtain disks and NICs from it. In case a runtime file is not available (see _GenerateKVMRuntime()) we return the bus slots that QEMU boots with by default. """ # This is by default and returned during _GenerateKVMRuntime() bus_slots = { _PCI_BUS: bitarray(self._DEFAULT_PCI_RESERVATIONS), _SCSI_BUS: bitarray(self._DEFAULT_SCSI_RESERVATIONS), } # Adjust the empty slots depending of the corresponding hvparam if hvp and constants.HV_KVM_PCI_RESERVATIONS in hvp: res = hvp[constants.HV_KVM_PCI_RESERVATIONS] pci = bitarray(constants.QEMU_PCI_SLOTS) pci.setall(False) # pylint: disable=E1101 pci[0:res:1] = True bus_slots[_PCI_BUS] = pci # This is during hot-add if runtime: nics = runtime.kvm_nics disks = [d for d, _, _ in runtime.kvm_disks] for d in disks + nics: if not d.hvinfo or "bus" not in d.hvinfo: continue bus = d.hvinfo["bus"] slots = bus_slots[bus] if bus == _PCI_BUS: slot = d.hvinfo["addr"] slots[int(slot, 16)] = True elif bus == _SCSI_BUS: slot = d.hvinfo["scsi-id"] slots[slot] = True return bus_slots @_with_qmp def _VerifyHotplugCommand(self, _instance, kvm_devid, should_exist): """Checks if a previous hotplug command has succeeded. Depending on the should_exist value, verifies that an entry identified by device ID is present or not. @raise errors.HypervisorError: if result is not the expected one """ for i in range(5): found = self.qmp.HasDevice(kvm_devid) logging.info("Verifying hotplug command (retry %s): %s", i, found) if found and should_exist: break if not found and not should_exist: break time.sleep(1) if found and not should_exist: msg = "Device %s should have been removed but is still there" % kvm_devid raise errors.HypervisorError(msg) if not found and should_exist: msg = "Device %s should have been added but is missing" % kvm_devid raise errors.HypervisorError(msg) logging.info("Device %s has been correctly hot-plugged", kvm_devid) @_with_qmp def HotAddDevice(self, instance, dev_type, device, extra, seq): """ Helper method to hot-add a new device It generates the device ID and hvinfo, and invokes the device-specific method. """ kvm_devid = _GenerateDeviceKVMId(dev_type, device) runtime = self._LoadKVMRuntime(instance) up_hvp = runtime[2] device_type = _DEVICE_TYPE[dev_type](up_hvp) bus_state = self._GetBusSlots(up_hvp, runtime) # in case of hot-mod this is given if not device.hvinfo: device.hvinfo = _GenerateDeviceHVInfo(dev_type, kvm_devid, device_type, bus_state) new_runtime_entry = _RUNTIME_ENTRY[dev_type](device, extra) if dev_type == constants.HOTPLUG_TARGET_DISK: disk_info = new_runtime_entry[0] access_mode = disk_info.params.get(constants.LDP_ACCESS, constants.DISK_KERNELSPACE) writeback, direct, no_flush = kvm_utils.GetCacheSettings( up_hvp[constants.HV_DISK_CACHE], disk_info.dev_type) target = _GetDriveURI(device, extra[0], extra[1]) blockdevice = self._GenerateKVMBlockDevice(target, disk_info, up_hvp, kvm_devid) self.qmp.HotAddDisk(device, access_mode, writeback, direct, blockdevice) elif dev_type == constants.HOTPLUG_TARGET_NIC: kvmpath = instance.hvparams[constants.HV_KVM_PATH] is_chrooted = instance.hvparams[constants.HV_KVM_USE_CHROOT] kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP) devlist = self._GetKVMOutput(kvmpath, self._KVMOPT_DEVICELIST) features, _, _ = self._GetNetworkDeviceFeatures(up_hvp, devlist, kvmhelp) (tap, tapfds, vhostfds) = OpenTap(features=features) self._ConfigureNIC(instance, seq, device, tap) self.qmp.HotAddNic(device, kvm_devid, tapfds, vhostfds, features, is_chrooted) utils.WriteFile(self._InstanceNICFile(instance.name, seq), data=tap) self._VerifyHotplugCommand(instance, kvm_devid, True) # update relevant entries in runtime file index = _DEVICE_RUNTIME_INDEX[dev_type] runtime[index].append(new_runtime_entry) self._SaveKVMRuntime(instance, runtime) @_with_qmp def HotDelDevice(self, instance, dev_type, device, _, seq): """ Helper method for hot-del device It gets device info from runtime file, generates the device name and invokes the device-specific method. """ runtime = self._LoadKVMRuntime(instance) entry = _GetExistingDeviceInfo(dev_type, device, runtime) kvm_device = _RUNTIME_DEVICE[dev_type](entry) kvm_devid = _GenerateDeviceKVMId(dev_type, kvm_device) if dev_type == constants.HOTPLUG_TARGET_DISK: self.qmp.HotDelDisk(kvm_devid) elif dev_type == constants.HOTPLUG_TARGET_NIC: self.qmp.HotDelNic(kvm_devid) utils.RemoveFile(self._InstanceNICFile(instance.name, seq)) self._VerifyHotplugCommand(instance, kvm_devid, False) index = _DEVICE_RUNTIME_INDEX[dev_type] runtime[index].remove(entry) self._SaveKVMRuntime(instance, runtime) return kvm_device.hvinfo def HotModDevice(self, instance, dev_type, device, _, seq): """ Helper method for hot-mod device It gets device info from runtime file, generates the device name and invokes the device-specific method. Currently only NICs support hot-mod """ if dev_type == constants.HOTPLUG_TARGET_NIC: # putting it back in the same bus and slot device.hvinfo = self.HotDelDevice(instance, dev_type, device, _, seq) self.HotAddDevice(instance, dev_type, device, _, seq) @classmethod def _ParseKVMVersion(cls, text): """Parse the KVM version from the --help output. @type text: string @param text: output of kvm --help @return: (version, v_maj, v_min, v_rev) @raise errors.HypervisorError: when the KVM version cannot be retrieved """ match = cls._VERSION_RE.search(text.splitlines()[0]) if not match: raise errors.HypervisorError("Unable to get KVM version") v_all = match.group(0) v_maj = int(match.group(1)) v_min = int(match.group(2)) if match.group(4): v_rev = int(match.group(4)) else: v_rev = 0 return (v_all, v_maj, v_min, v_rev) @classmethod def _GetKVMOutput(cls, kvm_path, option): """Return the output of a kvm invocation @type kvm_path: string @param kvm_path: path to the kvm executable @type option: a key of _KVMOPTS_CMDS @param option: kvm option to fetch the output from @return: output a supported kvm invocation @raise errors.HypervisorError: when the KVM help output cannot be retrieved """ assert option in cls._KVMOPTS_CMDS, "Invalid output option" optlist, can_fail = cls._KVMOPTS_CMDS[option] result = utils.RunCmd([kvm_path] + optlist) if result.failed and not can_fail: raise errors.HypervisorError("Unable to get KVM %s output" % " ".join(optlist)) return result.output @classmethod def _GetKVMVersion(cls, kvm_path): """Return the installed KVM version. @return: (version, v_maj, v_min, v_rev) @raise errors.HypervisorError: when the KVM version cannot be retrieved """ return cls._ParseKVMVersion(cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP)) @classmethod def _GetDefaultMachineVersion(cls, kvm_path): """Return the default hardware revision (e.g. pc-1.1) """ output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST) match = cls._DEFAULT_MACHINE_VERSION_RE.search(output) if match: return match.group(1) else: return "pc" @_with_qmp def _StopInstance(self, instance, force=False, name=None, timeout=None): """Stop an instance. """ assert(timeout is None or force is not None) if name is not None and not force: raise errors.HypervisorError("Cannot shutdown cleanly by name only") if name is None: name = instance.name acpi = instance.hvparams[constants.HV_ACPI] else: acpi = False _, pid, alive = self._InstancePidAlive(name) if pid > 0 and alive: if force or not acpi: utils.KillProcess(pid) else: self.qmp.Powerdown() self._ClearUserShutdown(instance.name) def StopInstance(self, instance, force=False, retry=False, name=None, timeout=None): """Stop an instance. """ self._StopInstance(instance, force, name=name, timeout=timeout) def CleanupInstance(self, instance_name): """Cleanup after a stopped instance """ pidfile, pid, alive = self._InstancePidAlive(instance_name) if pid > 0 and alive: raise errors.HypervisorError("Cannot cleanup a live instance") self._RemoveInstanceRuntimeFiles(pidfile, instance_name) self._ClearUserShutdown(instance_name) def RebootInstance(self, instance): """Reboot an instance. """ # For some reason if we do a 'send-key ctrl-alt-delete' to the control # socket the instance will stop, but now power up again. So we'll resort # to shutdown and restart. _, _, alive = self._InstancePidAlive(instance.name) if not alive: raise errors.HypervisorError("Failed to reboot instance %s:" " not running" % instance.name) # StopInstance will delete the saved KVM runtime so: # ...first load it... kvm_runtime = self._LoadKVMRuntime(instance) # ...now we can safely call StopInstance... if not self.StopInstance(instance): self.StopInstance(instance, force=True) # ...and finally we can save it again, and execute it... self._SaveKVMRuntime(instance, kvm_runtime) kvmpath = instance.hvparams[constants.HV_KVM_PATH] kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP) self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp) def MigrationInfo(self, instance): """Get instance information to perform a migration. @type instance: L{objects.Instance} @param instance: instance to be migrated @rtype: string @return: content of the KVM runtime file """ return self._ReadKVMRuntime(instance.name) def AcceptInstance(self, instance, info, target): """Prepare to accept an instance. @type instance: L{objects.Instance} @param instance: instance to be accepted @type info: string @param info: content of the KVM runtime file on the source node @type target: string @param target: target host (usually ip), on this node """ kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info) incoming_address = (target, instance.hvparams[constants.HV_MIGRATION_PORT]) kvmpath = instance.hvparams[constants.HV_KVM_PATH] kvmhelp = self._GetKVMOutput(kvmpath, self._KVMOPT_HELP) self._ExecuteKVMRuntime(instance, kvm_runtime, kvmhelp, incoming=incoming_address) self._SetInstanceMigrationCapabilities(instance) def _ConfigureRoutedNICs(self, instance, info): """Configures all NICs in routed mode @type instance: L{objects.Instance} @param instance: the instance to be configured @type info: string @param info: serialized KVM runtime info """ kvm_runtime = self._LoadKVMRuntime(instance, serialized_runtime=info) kvm_nics = kvm_runtime[1] for nic_seq, nic in enumerate(kvm_nics): if nic.nicparams[constants.NIC_MODE] != constants.NIC_MODE_ROUTED: # Bridged/OVS interfaces have already been configured continue try: tap = utils.ReadFile(self._InstanceNICFile(instance.name, nic_seq)) except EnvironmentError as err: logging.warning("Failed to find host interface for %s NIC #%d: %s", instance.name, nic_seq, str(err)) continue try: self._ConfigureNIC(instance, nic_seq, nic, tap) except errors.HypervisorError as err: logging.warning(str(err)) def FinalizeMigrationDst(self, instance, info, success): """Finalize the instance migration on the target node. Stop the incoming mode KVM. @type instance: L{objects.Instance} @param instance: instance whose migration is being finalized """ if success: self._ConfigureRoutedNICs(instance, info) self._WriteKVMRuntime(instance.name, info) self._ClearInstanceMigrationCapabilities(instance) else: self.StopInstance(instance, force=True) @_with_qmp def MigrateInstance(self, cluster_name, instance, target, live_migration): """Migrate an instance to a target node. The migration will not be attempted if the instance is not currently running. @type cluster_name: string @param cluster_name: name of the cluster @type instance: L{objects.Instance} @param instance: the instance to be migrated @type target: string @param target: ip address of the target node @type live_migration: boolean @param live_migration: perform a live migration """ instance_name = instance.name port = instance.hvparams[constants.HV_MIGRATION_PORT] _, _, alive = self._InstancePidAlive(instance_name) if not alive: raise errors.HypervisorError("Instance not running, cannot migrate") if not live_migration: self.qmp.StopGuestEmulation() max_bandwidth_in_bytes = \ instance.hvparams[constants.HV_MIGRATION_BANDWIDTH] * 1024 * 1024 self.qmp.SetMigrationParameters(max_bandwidth_in_bytes, instance.hvparams[constants.HV_MIGRATION_DOWNTIME]) self._SetInstanceMigrationCapabilities(instance) self.qmp.StartMigration(target, port) @_with_qmp def FinalizeMigrationSource(self, instance, success, _): """Finalize the instance migration on the source node. @type instance: L{objects.Instance} @param instance: the instance that was migrated @type success: bool @param success: whether the migration succeeded or not """ if success: pidfile, pid, _ = self._InstancePidAlive(instance.name) utils.KillProcess(pid) self._RemoveInstanceRuntimeFiles(pidfile, instance.name) self._ClearUserShutdown(instance.name) else: # Detect if PID is alive rather than deciding if we were to perform a live # migration. _, _, alive = self._InstancePidAlive(instance.name) if alive: self.qmp.ContinueGuestEmulation() self._ClearInstanceMigrationCapabilities(instance) else: self.CleanupInstance(instance.name) @_with_qmp def GetMigrationStatus(self, instance): """Get the migration status @type instance: L{objects.Instance} @param instance: the instance that is being migrated @rtype: L{objects.MigrationStatus} @return: the status of the current migration (one of L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional progress info that can be retrieved from the hypervisor """ for _ in range(self._MIGRATION_INFO_MAX_BAD_ANSWERS): query_migrate = self.qmp.GetMigrationStatus() if "status" in query_migrate: if query_migrate["status"] in constants.HV_KVM_MIGRATION_VALID_STATUSES: migration_status = objects.MigrationStatus(status= query_migrate["status"]) if "ram" in query_migrate: migration_status.transferred_ram = \ query_migrate["ram"]["transferred"] migration_status.total_ram = query_migrate["ram"]["total"] migration_status.postcopy_status = None migration_caps = instance.hvparams[constants.HV_KVM_MIGRATION_CAPS] # migration_caps is a ':' delimited string, so checking # if 'postcopy-ram' is a substring also covers using # x-postcopy-ram for QEMU 2.5 if migration_caps and "postcopy-ram" in migration_caps: dirty_sync_count = query_migrate["ram"]["dirty-sync-count"] if (migration_status.status == constants.HV_MIGRATION_ACTIVE and dirty_sync_count >= _POSTCOPY_SYNC_COUNT_THRESHOLD): self.qmp.StartPostcopyMigration() logging.info("switched live migration for instance %s to" " postcopy mode", instance.name) # qmp.StartPostcopyMigration() seems asynchronous. Doing # qmp.GetMigrationStatus() right after, the ["status"] won't be # HV_KVM_MIGRATION_POSTCOPY_ACTIVE. This is why an extra field # ["postcopy_status"] is introduced migration_status.postcopy_status = \ constants.HV_KVM_MIGRATION_POSTCOPY_ACTIVE return migration_status else: logging.warning("KVM: unknown migration status '%s'", query_migrate["status"]) else: logging.warning("KVM: unknown 'query-migrate' result: %s", query_migrate) time.sleep(self._MIGRATION_INFO_RETRY_DELAY) return objects.MigrationStatus(status=constants.HV_MIGRATION_FAILED) @_with_qmp def BalloonInstanceMemory(self, instance, mem): """Balloon an instance memory to a certain value. @type instance: L{objects.Instance} @param instance: instance to be accepted @type mem: int @param mem: actual memory size to use for instance runtime """ self.qmp.SetBalloonMemory(mem) def GetNodeInfo(self, hvparams=None): """Return information about the node. @type hvparams: dict of strings @param hvparams: hypervisor parameters, not used in this class @return: a dict as returned by L{BaseHypervisor.GetLinuxNodeInfo} plus the following keys: - hv_version: the hypervisor version in the form (major, minor, revision) """ result = self.GetLinuxNodeInfo() kvmpath = constants.KVM_PATH if hvparams is not None: kvmpath = hvparams.get(constants.HV_KVM_PATH, constants.KVM_PATH) _, v_major, v_min, v_rev = self._GetKVMVersion(kvmpath) result[constants.HV_NODEINFO_KEY_VERSION] = (v_major, v_min, v_rev) return result @classmethod def GetInstanceConsole(cls, instance, primary_node, node_group, hvparams, beparams): """Return a command for connecting to the console of an instance. """ if hvparams[constants.HV_SERIAL_CONSOLE]: cmd = [pathutils.KVM_CONSOLE_WRAPPER, constants.SOCAT_PATH, utils.ShellQuote(instance.name), utils.ShellQuote(cls._InstanceMonitor(instance.name)), "STDIO,%s" % cls._SocatUnixConsoleParams(), "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)] ndparams = node_group.FillND(primary_node) return objects.InstanceConsole(instance=instance.name, kind=constants.CONS_SSH, host=primary_node.name, port=ndparams.get(constants.ND_SSH_PORT), user=constants.SSH_CONSOLE_USER, command=cmd) vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS] if vnc_bind_address and instance.network_port > constants.VNC_BASE_PORT: display = instance.network_port - constants.VNC_BASE_PORT return objects.InstanceConsole(instance=instance.name, kind=constants.CONS_VNC, host=vnc_bind_address, port=instance.network_port, display=display) spice_bind = hvparams[constants.HV_KVM_SPICE_BIND] if spice_bind: return objects.InstanceConsole(instance=instance.name, kind=constants.CONS_SPICE, host=spice_bind, port=instance.network_port) return objects.InstanceConsole(instance=instance.name, kind=constants.CONS_MESSAGE, message=("No serial shell for instance %s" % instance.name)) def Verify(self, hvparams=None): """Verify the hypervisor. Check that the required binaries exist. @type hvparams: dict of strings @param hvparams: hypervisor parameters to be verified against, not used here @return: Problem description if something is wrong, C{None} otherwise """ msgs = [] kvmpath = constants.KVM_PATH if hvparams is not None: kvmpath = hvparams.get(constants.HV_KVM_PATH, constants.KVM_PATH) if not os.path.exists(kvmpath): msgs.append("The KVM binary ('%s') does not exist" % kvmpath) if not os.path.exists(constants.SOCAT_PATH): msgs.append("The socat binary ('%s') does not exist" % constants.SOCAT_PATH) return self._FormatVerifyResults(msgs) @classmethod def CheckParameterSyntax(cls, hvparams): """Check the given parameters for validity. @type hvparams: dict of strings @param hvparams: hypervisor parameters @raise errors.HypervisorError: when a parameter is not valid """ super(KVMHypervisor, cls).CheckParameterSyntax(hvparams) check_boot_parameters(hvparams) check_security_model(hvparams) check_console_parameters(hvparams) check_vnc_parameters(hvparams) check_spice_parameters(hvparams) check_disk_cache_parameters(hvparams) @classmethod def ValidateParameters(cls, hvparams): """Check the given parameters for validity. @type hvparams: dict of strings @param hvparams: hypervisor parameters @raise errors.HypervisorError: when a parameter is not valid """ super(KVMHypervisor, cls).ValidateParameters(hvparams) validate_security_model(hvparams) validate_vnc_parameters(hvparams) kvm_path = hvparams[constants.HV_KVM_PATH] kvm_output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_HELP) validate_spice_parameters(hvparams, kvm_output) kvmpath = constants.KVM_PATH kvm_version = cls._GetKVMVersion(kvmpath) validate_disk_parameters(hvparams, kvm_version) kvm_output = cls._GetKVMOutput(kvm_path, cls._KVMOPT_MLIST) validate_machine_version(hvparams, kvm_output) @classmethod def AssessParameters(cls, hvparams): """Check the given parameters for uncommon/suboptimal values This should check the passed set of parameters for suboptimal values. @type hvparams: dict @param hvparams: dictionary with parameter names/value """ warnings = [] cpu_type = hvparams[constants.HV_CPU_TYPE] if not cpu_type: warnings.append("cpu_type is currently unset and defaults to 'qemu64'" ", please read the gnt-instance man page on the security" " and performance implications of this parameter.") elif cpu_type in ("qemu32", "qemu64", "kvm32", "kvm64"): warnings.append("cpu_type is currently set to '%s'" ", please read the gnt-instance man page on the security" " and performance implications of this parameter." % cpu_type) elif cpu_type == "host": warnings.append("cpu_type is currently set to 'host', please make" " sure all your cluster nodes have the exact same" " CPU type to allow live migrations.") vhost_net = hvparams[constants.HV_VHOST_NET] if not vhost_net: warnings.append("vhost_net should be enabled for paravirtual NICs to" " improve network latency and throughput.") migration_bandwidth = hvparams[constants.HV_MIGRATION_BANDWIDTH] default_migration_bw = constants.HVC_DEFAULTS[constants.HT_KVM]\ [constants.HV_MIGRATION_BANDWIDTH] if migration_bandwidth == default_migration_bw: warnings.append("migration_bandwidth is still set to its default value" "(%s MiB/s), please consider adjusting it to a higher " "value that leverages todays network speeds." % default_migration_bw) disk_type = hvparams[constants.HV_DISK_TYPE] if disk_type != "paravirtual": warnings.append("disk_type is set to '%s' instead of 'paravirtual'" ", please consider virtio-based disks for the best" " performance." % disk_type) nic_type = hvparams[constants.HV_NIC_TYPE] if nic_type != "paravirtual": warnings.append("nic_type is set to '%s' instead of 'paravirtual'" ", please consider virtio-based networking for the" " best performance." % nic_type) vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS] vnc_tls_enabled = hvparams[constants.HV_VNC_TLS] if vnc_bind_address and vnc_bind_address != "127.0.0.1" \ and not vnc_tls_enabled: warnings.append("VNC is configured but without TLS, please" " consider setting vnc_tls to 'true'" " for additional security.") spice_bind_address = hvparams[constants.HV_KVM_SPICE_BIND] spice_tls_enabled = hvparams[constants.HV_KVM_SPICE_USE_TLS] if spice_bind_address and spice_bind_address != "127.0.0.1" \ and not spice_tls_enabled: warnings.append("Spice is configured but without TLS, please" " consider setting spice_use_tls to 'true'" " for additional security.") machine_version = hvparams[constants.HV_KVM_MACHINE_VERSION] if not machine_version: warnings.append("machine_version is not explicitly configured so " "Ganeti will autodetect the default value from " "'kvm -M ?', this might produce unexpected results.") if machine_version == "pc": warnings.append("machine_version is set to the 'pc' type which " "could cause problems during live migration") if machine_version.startswith("q35"): warnings.append("machine_version is set to the 'q35' type which " "is currently not properly supported.") if machine_version in ["none", "microvm", "isapc", "x-remote"]: warnings.append("machine_version is set to %s which is unsupported " "in Ganeti." % machine_version) return warnings @classmethod def PowercycleNode(cls, hvparams=None): """KVM powercycle, just a wrapper over Linux powercycle. @type hvparams: dict of strings @param hvparams: hypervisor parameters to be used on this node """ cls.LinuxPowercycle() ganeti-3.1.0~rc2/lib/hypervisor/hv_kvm/kvm_runtime.py000064400000000000000000000170221476477700300227720ustar00rootroot00000000000000# # # Copyright (C) 2022 the Ganeti project # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from typing import List, Any, Dict from ganeti import utils from ganeti import serializer from ganeti import constants from ganeti import objects _KVM_CMD_RUNTIME_INDEX = 0 _KVM_NICS_RUNTIME_INDEX = 1 _KVM_HV_RUNTIME_INDEX = 2 _KVM_DISKS_RUNTIME_INDEX = 3 _PCI_BUS = "pci.0" _DEVICE_TYPE = { constants.HOTPLUG_TARGET_NIC: lambda hvp: hvp[constants.HV_NIC_TYPE], constants.HOTPLUG_TARGET_DISK: lambda hvp: hvp[constants.HV_DISK_TYPE], } _DEVICE_DRIVER = { constants.HOTPLUG_TARGET_NIC: lambda ht: "virtio-net-pci" if ht == constants.HT_NIC_PARAVIRTUAL else ht, constants.HOTPLUG_TARGET_DISK: lambda ht: "virtio-blk-pci" if ht == constants.HT_DISK_PARAVIRTUAL else ht, } class KVMRuntime: def __init__(self, data: List): assert len(data) == 4 self.data = data def __getitem__(self, idx): return self.data[idx] @property def kvm_cmd(self) -> List[str]: return self.data[_KVM_CMD_RUNTIME_INDEX] @property def kvm_nics(self) -> List[Any]: return self.data[_KVM_NICS_RUNTIME_INDEX] @property def up_hvp(self) -> Dict: return self.data[_KVM_HV_RUNTIME_INDEX] @property def kvm_disks(self) -> List[objects.Disk]: return self.data[_KVM_DISKS_RUNTIME_INDEX] def serialize(self) -> str: serialized_nics = [nic.ToDict() for nic in self.kvm_nics] serialized_disks = [(blk.ToDict(), link, uri) for blk, link, uri in self.kvm_disks] serialized = serializer.Dump((self.kvm_cmd, serialized_nics, self.up_hvp, serialized_disks)) return serialized @staticmethod def from_serialized(serialized: str, upgrade: bool = True) -> 'KVMRuntime': loaded_runtime = serializer.Load(serialized) if upgrade: _upgrade_serialized_runtime(loaded_runtime) kvm_cmd = loaded_runtime[_KVM_CMD_RUNTIME_INDEX] up_hvp = loaded_runtime[_KVM_HV_RUNTIME_INDEX] nics = [objects.NIC.FromDict(nic) for nic in loaded_runtime[_KVM_NICS_RUNTIME_INDEX]] disks = [(objects.Disk.FromDict(sdisk), link, uri) for sdisk, link, uri in loaded_runtime[_KVM_DISKS_RUNTIME_INDEX]] return KVMRuntime([kvm_cmd, nics, up_hvp, disks]) def _upgrade_serialized_runtime(loaded_runtime: List) -> List: """Upgrade runtime data Remove any deprecated fields or change the format of the data. The runtime files are not upgraded when Ganeti is upgraded, so the required modification have to be performed here. @type loaded_runtime: List @param loaded_runtime: List of unserialized items (dict or list) @return: List[cmd, nic dicts, hvparams, bdev dicts] @rtype: List """ kvm_cmd, serialized_nics, hvparams = loaded_runtime[:3] if len(loaded_runtime) >= 4: serialized_disks = loaded_runtime[3] else: serialized_disks = [] def update_hvinfo(dev, dev_type): """ Remove deprecated pci slot and substitute it with hvinfo """ if "hvinfo" not in dev: dev["hvinfo"] = {} uuid = dev["uuid"] # Ganeti used to save the PCI slot of paravirtual devices # (virtio-blk-pci, virtio-net-pci) in runtime files during # _GenerateKVMRuntime() and HotAddDevice(). # In this case we had a -device QEMU option in the command line with id, # drive|netdev, bus, and addr params. All other devices did not have an # id nor placed explicitly on a bus. # hot- prefix is removed in 2.16. Here we add it explicitly to # handle old instances in the cluster properly. if "pci" in dev: # This is practically the old _GenerateDeviceKVMId() hv_dev_type = _DEVICE_TYPE[dev_type](hvparams) dev["hvinfo"]["driver"] = _DEVICE_DRIVER[dev_type](hv_dev_type) dev["hvinfo"]["id"] = "hot%s-%s-%s-%s" % (dev_type.lower(), uuid.split("-")[0], "pci", dev["pci"]) dev["hvinfo"]["addr"] = hex(dev["pci"]) dev["hvinfo"]["bus"] = _PCI_BUS del dev["pci"] for nic in serialized_nics: # Add a dummy uuid slot if an pre-2.8 NIC is found if "uuid" not in nic: nic["uuid"] = utils.NewUUID() update_hvinfo(nic, constants.HOTPLUG_TARGET_NIC) for disk_entry in serialized_disks: # We have a (Disk, link, uri) tuple update_hvinfo(disk_entry[0], constants.HOTPLUG_TARGET_DISK) # Handle KVM command line argument changes try: idx = kvm_cmd.index("-localtime") except ValueError: pass else: kvm_cmd[idx:idx+1] = ["-rtc", "base=localtime"] try: idx = kvm_cmd.index("-balloon") except ValueError: pass else: balloon_args = kvm_cmd[idx+1].split(",")[1:] balloon_str = "virtio-balloon" if balloon_args: balloon_str += ",%s" % ",".join(balloon_args) kvm_cmd[idx:idx+2] = ["-device", balloon_str] try: idx = kvm_cmd.index("-vnc") except ValueError: pass else: # Check to see if TLS is enabled orig_vnc_args = kvm_cmd[idx+1].split(",") vnc_args = [] tls_obj = None tls_obj_args = ["id=vnctls0", "endpoint=server"] for arg in orig_vnc_args: if arg == "tls": tls_obj = "tls-creds-anon" vnc_args.append("tls-creds=vnctls0") continue elif arg.startswith("x509verify=") or arg.startswith("x509="): pki_path = arg.split("=", 1)[-1] tls_obj = "tls-creds-x509" tls_obj_args.append("dir=%s" % pki_path) if arg.startswith("x509verify="): tls_obj_args.append("verify-peer=yes") else: tls_obj_args.append("verify-peer=no") continue vnc_args.append(arg) if tls_obj is not None: vnc_cmd = ["-vnc", ",".join(vnc_args)] tls_obj_cmd = ["-object", "%s,%s" % (tls_obj, ",".join(tls_obj_args))] # Replace the original vnc argument with the new ones kvm_cmd[idx:idx+2] = tls_obj_cmd + vnc_cmd # with 3.1 the 'default' value for disk_discard has been dropped # and replaced by 'ignore' if constants.HV_DISK_DISCARD in hvparams \ and hvparams[constants.HV_DISK_DISCARD] not in \ constants.HT_VALID_DISCARD_TYPES: hvparams[constants.HV_DISK_DISCARD] = constants.HT_DISCARD_IGNORE return [kvm_cmd, serialized_nics, hvparams, serialized_disks] ganeti-3.1.0~rc2/lib/hypervisor/hv_kvm/kvm_utils.py000064400000000000000000000110251476477700300224440ustar00rootroot00000000000000# # # Copyright (C) 2022 the Ganeti project # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import re import logging from ganeti import constants from ganeti import errors _BLOCKDEV_URI_REGEX_GLUSTER = ( r"^gluster:\/\/(?P[a-z0-9-.]+):" r"(?P\d+)/(?P[^/]+)/(?P.+)$" ) _BLOCKDEV_URI_REGEX_RBD = r"^rbd:(?P\w+)/(?P[a-z0-9-\.]+)$" def TranslateBoolToOnOff(value): """Converts a given boolean to 'on'|'off' for use in QEMUs cmdline @param value: bool @return: 'on' or 'off' @rtype: string """ if value: return 'on' else: return 'off' def ParseStorageUriToBlockdevParam(uri): """Parse a storage uri into qemu blockdev params @type uri: string @param uri: storage-describing URI @return: dict """ if (match := re.match(_BLOCKDEV_URI_REGEX_GLUSTER, uri)) is not None: return { "driver": "gluster", "server": [ { 'type': 'inet', 'host': match.group("host"), 'port': match.group("port"), } ], "volume": match.group("volume"), "path": match.group("path") } elif (match := re.match(_BLOCKDEV_URI_REGEX_RBD, uri)) is not None: return { "driver": "rbd", "pool": match.group("pool"), "image": match.group("image") } raise errors.HypervisorError("Unsupported storage URI scheme: %s" % (uri)) def FlattenDict(d, parent_key='', sep='.'): """Helper method to convert nested dicts to flat string representation """ items = [] for k, v in d.items(): if isinstance(v, bool): v = TranslateBoolToOnOff(v) new_key = f"{parent_key}{sep}{k}" if parent_key else k if isinstance(v, dict): items.extend(FlattenDict(v, new_key, sep=sep).items()) else: items.append((new_key, v)) return dict(items) def DictToQemuStringNotation(data): """Convert dictionary to flat string representation This function is used to transform a blockdev QEMU parameter set for use as command line parameters (to QEMUs -blockdev parameter) @type data: dict @param data: data to convert @return: string """ logging.debug("Converting the following data structure " "to flat string: %s" % (data)) flat_str = ','.join(["%s=%s" % (key, value) for key, value in FlattenDict(data).items()]) logging.debug("Result: %s" % flat_str) return flat_str def GetCacheSettings(cache_type, dev_type): """Return cache settings suitable for use with -blockdev @param cache_type: string (one of L{constants.HT_VALID_CACHE_TYPES}) @param dev_type: string (one of L{constants.DISK_TEMPLATES} @return: (writeback, direct, no_flush) @rtype: tuple """ if dev_type in constants.DTS_EXT_MIRROR and dev_type != constants.DT_RBD: logging.warning("KVM: overriding disk_cache setting '%s' with 'none'" " to prevent shared storage corruption on migration", cache_type) cache_type = constants.HT_CACHE_NONE if cache_type == constants.HT_CACHE_NONE: return True, True, False elif cache_type in [constants.HT_CACHE_WBACK, constants.HT_CACHE_DEFAULT]: return True, False, False elif cache_type == constants.HT_CACHE_WTHROUGH: return False, False, False else: raise errors.HypervisorError("Invalid KVM cache setting '%s'" % cache_type)ganeti-3.1.0~rc2/lib/hypervisor/hv_kvm/monitor.py000064400000000000000000000634241476477700300221300ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Qemu monitor control classes """ import os import socket import io import logging import time from typing import Dict, Optional, Union, List from collections import namedtuple from bitarray import bitarray from ganeti import errors from ganeti import utils from ganeti import constants from ganeti import serializer import ganeti.hypervisor.hv_kvm.kvm_utils as kvm_utils class QmpCommandNotSupported(errors.HypervisorError): """QMP command not supported by the monitor. This is raised in case a QmpMonitor instance is asked to execute a command not supported by the instance. This is a KVM-specific exception, intended to assist in falling back to using the human monitor for operations QMP does not support. """ pass class QmpTimeoutError(errors.HypervisorError): """QMP socket timeout error """ class QmpMessage: """QEMU Messaging Protocol (QMP) message. """ def __init__(self, data: Dict): self.data = data def __getitem__(self, field_name): """Get the value of the required field if present, or None. Overrides the [] operator to provide access to the message data, returning None if the required item is not in the message @return: the value of the field_name field, or None if field_name is not contained in the message """ return self.data.get(field_name, None) def __setitem__(self, field_name, field_value): self.data[field_name] = field_value def __eq__(self, other: Union['QmpMessage', Dict]): if isinstance(other, QmpMessage): return self.data == other.data elif isinstance(other, dict): return self.data == other return False def __len__(self): """Return the number of fields stored in this QmpMessage. """ return len(self.data) def __delitem__(self, key): del self.data[key] @staticmethod def build_from_json_string(json_string: str) -> 'QmpMessage': """Build a QmpMessage from a JSON encoded string. @param json_string: JSON string representing the message @return: a L{QmpMessage} built from json_string """ # Parse the string data = serializer.LoadJson(json_string) return QmpMessage(data) def to_bytes(self) -> bytes: # The protocol expects the JSON object to be sent as a single line. return serializer.DumpJson(self.data) # debug only def to_json_string(self) -> str: return self.to_bytes().decode("utf-8") def __eq__(self, other: 'QmpMessage') -> bool: # compare only the data dict return self.data == other.data # define QMP timestamp as namedtuple QmpTimestamp = namedtuple('QMPTimestamp', 'seconds microseconds') class QmpEvent: """QEMU event message from a qmp socket. """ def __init__(self, timestamp: QmpTimestamp, event_type: str, data: Dict): self._timestamp = timestamp self._event_type = event_type self._data = data def __getitem__(self, field_name: str) -> any: return self._data.get(field_name, None) @property def timestamp(self) -> QmpTimestamp: return self._timestamp @property def event_type(self)-> str: return self._event_type @staticmethod def build_from_data(data: Dict) -> 'QmpEvent': """Build a QmpEvent from a data dict.""" timestamp = QmpTimestamp( seconds=data['timestamp']['seconds'], microseconds=data['timestamp']['microseconds'] ) return QmpEvent( timestamp=timestamp, event_type=data['event'], data=data['data'] ) def _ensure_connection(fn): """Decorator that wraps MonitorSocket external methods""" def wrapper(*args, **kwargs): """Ensure proper connect/close and exception propagation""" mon = args[0] already_connected = mon.is_connected() mon.connect() try: ret = fn(*args, **kwargs) finally: # In general this decorator wraps external methods. # Here we close the connection only if we initiated it before, # to protect us from using the socket after closing it # in case we invoke a decorated method internally by accident. if not already_connected: mon.close() return ret return wrapper class UnixFileSocketConnection: def __init__(self, socket_path: str, timeout: Union[int, float]): self.socket_path = socket_path self.timeout = timeout self.sock = None self._connected = False def __enter__(self): self.connect() return self def __exit__(self, exc_type, exc_value, exc_traceback): self.close() def connect(self): if not self.is_connected(): self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.sock.settimeout(self.timeout) self.sock.connect(self.socket_path) logging.debug("Create Socket Connection to %s.", {self.socket_path}) def close(self): if self.is_connected(): self.sock.close() self._connected = False logging.debug("Socket Connection to %s closed.", {self.socket_path}) def is_connected(self) -> bool: return self._connected def send(self, data: bytes): self.sock.sendall(data) def recv(self, bufsize: int) -> bytes: return self.sock.recv(bufsize) def reset_timeout(self) -> None: """Reset the timeout to self.timeout""" self.sock.settimeout(self.timeout) class QemuMonitorSocket(UnixFileSocketConnection): _ERROR_CLASS_KEY = "class" _ERROR_DESC_KEY = "desc" _EXECUTE_KEY = "execute" _ARGUMENTS_KEY = "arguments" _EVENT_KEY = "event" _ERROR_KEY = "error" _RETURN_KEY = "return" _MESSAGE_END_TOKEN = b"\r\n" _SEND_END_TOKEN = b"\n" def __init__(self, socket_path: str, timeout): super().__init__(socket_path, timeout) self._buffer = b"" def execute_qmp(self, command: str, arguments: Dict = None) -> QmpMessage: message = QmpMessage({self._EXECUTE_KEY: command}) if arguments: message[self._ARGUMENTS_KEY] = arguments self.send_qmp(message) return self.get_qmp_response(command) def send_qmp(self, message: QmpMessage): self.send(message.to_bytes() + b'\n') def get_qmp_response(self, command) -> Dict: while True: response = self.recv_qmp() err = response[self._ERROR_KEY] if err: raise errors.HypervisorError("kvm: error executing the {}" " command: {} ({}):".format( command, err[self._ERROR_DESC_KEY], err[self._ERROR_CLASS_KEY])) elif response[self._EVENT_KEY]: continue return response[self._RETURN_KEY] def wait_for_qmp_event(self, event_types: List[str], timeout: Union[int, float]) -> Optional[QmpEvent]: """Waits for one of the specified events and returns it. If the timeout is reached, returns None. """ self.sock.settimeout(timeout) try: while True: response = self.recv_qmp() if response[self._EVENT_KEY]: event = QmpEvent.build_from_data(response.data) if event.event_type in event_types: self.reset_timeout() return event else: continue except QmpTimeoutError: self.reset_timeout() return None def recv_qmp(self) -> QmpMessage: message = self._read_buffer() # check if the message is already in the buffer if message: return message recv_buffer = io.BytesIO(self._buffer) recv_buffer.seek(len(self._buffer)) try: while True: data = self.recv(4096) if not data: break recv_buffer.write(data) self._buffer = recv_buffer.getvalue() message = self._read_buffer() if message: return message except socket.timeout as err: raise QmpTimeoutError("Timeout while receiving a QMP message: " f"{err}") from err except socket.error as err: raise errors.HypervisorError("Unable to receive data from KVM using the" f" QMP protocol: {err}") def _read_buffer(self) -> QmpMessage: message = None # Check if we got the message end token (CRLF, as per the QEMU Protocol # Specification 0.1 - Section 2.1.1) pos = self._buffer.find(self._MESSAGE_END_TOKEN) if pos >= 0: try: message = QmpMessage.build_from_json_string(self._buffer[:pos + 1]) except Exception as err: raise errors.ProgrammerError(f"QMP data serialization error: {err}") self._buffer = self._buffer[pos + 1:] return message class QmpConnection(QemuMonitorSocket): """Connection to the QEMU Monitor using the QEMU Monitor Protocol (QMP). """ _QMP_TIMEOUT = 5 _FIRST_MESSAGE_KEY = "QMP" _RETURN_KEY = "return" _ACTUAL_KEY = ACTUAL_KEY = "actual" _VERSION_KEY = "version" _PACKAGE_KEY = "package" _QEMU_KEY = "qemu" _CAPABILITIES_COMMAND = "qmp_capabilities" _QUERY_COMMANDS = "query-commands" _MESSAGE_END_TOKEN = b"\r\n" # List of valid attributes for the device_add QMP command. # Extra attributes found in device's hvinfo will be ignored. _DEVICE_ATTRIBUTES = [ "driver", "id", "bus", "addr", "channel", "scsi-id", "lun" ] def __init__(self, socket_path: str): super().__init__(socket_path, self._QMP_TIMEOUT) self.version = None self.package = None self.supported_commands = None def __enter__(self): self.connect() return self def __exit__(self, exc_type, exc_value, tb): self.close() def execute_qmp(self, command: str, arguments: Dict = None) -> QmpMessage: # During the first calls of Execute, the list of supported commands has not # yet been populated, so we can't use it. if (self.supported_commands is not None and command not in self.supported_commands): raise QmpCommandNotSupported(f"Instance does not support the '{command}'" " QMP command.") message = QmpMessage({self._EXECUTE_KEY: command}) if arguments: message[self._ARGUMENTS_KEY] = arguments logging.debug("QMP JSON Command: %s", message.to_json_string()) self.send_qmp(message) ret = self.get_qmp_response(command) if command not in [self._QUERY_COMMANDS, self._CAPABILITIES_COMMAND]: logging.debug("QMP Response: %s %s: %s", command, arguments, ret) return ret def connect(self): """Connects to the QMP monitor. Connects to the UNIX socket and makes sure that we can actually send and receive data to the kvm instance via QMP. @raise errors.HypervisorError: when there are communication errors @raise errors.ProgrammerError: when there are data serialization errors """ super(QmpConnection, self).connect() # sometimes we receive asynchronous events instead of the intended greeting # message - we ignore these for now. However, only 5 times to not get stuck # in an endless connect() loop. for x in range(0, 4): # Check if we receive a correct greeting message from the server # (As per the QEMU Protocol Specification 0.1 - section 2.2) greeting = self.recv_qmp() if greeting[self._EVENT_KEY]: continue if not greeting[self._FIRST_MESSAGE_KEY]: self._connected = False raise errors.HypervisorError("kvm: QMP communication error (wrong" " server greeting)") else: break # Extract the version info from the greeting and make it available to users # of the monitor. version_info = greeting[self._FIRST_MESSAGE_KEY][self._VERSION_KEY] self.version = (version_info[self._QEMU_KEY]["major"], version_info[self._QEMU_KEY]["minor"], version_info[self._QEMU_KEY]["micro"]) self.package = version_info[self._PACKAGE_KEY].strip() # This is needed because QMP can return more than one greetings # see https://groups.google.com/d/msg/ganeti-devel/gZYcvHKDooU/SnukC8dgS5AJ self._buffer = b"" # Let's put the monitor in command mode using the qmp_capabilities # command, or else no command will be executable. # (As per the QEMU Protocol Specification 0.1 - section 4) self.execute_qmp(self._CAPABILITIES_COMMAND) self.supported_commands = self._GetSupportedCommands() def _GetSupportedCommands(self): """Update the list of supported commands. """ result = self.execute_qmp(self._QUERY_COMMANDS) return frozenset(com["name"] for com in result) def _filter_hvinfo(self, hvinfo): """Filter non valid keys of the device's hvinfo (if any).""" ret = {} for k in self._DEVICE_ATTRIBUTES: if k in hvinfo: ret[k] = hvinfo[k] return ret @_ensure_connection def HotAddNic(self, nic, devid, tapfds=None, vhostfds=None, features=None, is_chrooted=False): """Hot-add a NIC First pass the tapfds, then netdev_add and then device_add """ if tapfds is None: tapfds = [] if vhostfds is None: vhostfds = [] if features is None: features = {} enable_vhost = features.get("vhost", False) enable_mq, virtio_net_queues = features.get("mq", (False, 1)) fdnames = [] for i, fd in enumerate(tapfds): fdname = "%s-%d" % (devid, i) self._GetFd(fd, fdname) fdnames.append(fdname) arguments = { "type": "tap", "id": devid, "fds": ":".join(fdnames), } if enable_vhost: fdnames = [] for i, fd in enumerate(vhostfds): fdname = "%s-vhost-%d" % (devid, i) self._GetFd(fd, fdname) fdnames.append(fdname) arguments.update({ "vhost": True, "vhostfds": ":".join(fdnames), }) self.execute_qmp("netdev_add", arguments) arguments = { "netdev": devid, "mac": nic.mac, } if is_chrooted: # do not try to load a rom file when we are running qemu chrooted arguments.update({ "romfile": "", }) # Note that hvinfo that _GenerateDeviceHVInfo() creates # should include *only* the driver, id, bus, and addr keys arguments.update(self._filter_hvinfo(nic.hvinfo)) if enable_mq: arguments.update({ "mq": "on", "vectors": (2 * virtio_net_queues + 2), }) self.execute_qmp("device_add", arguments) @_ensure_connection def HotDelNic(self, devid): """Hot-del a NIC """ self.execute_qmp("device_del", {"id": devid}) self.execute_qmp("netdev_del", {"id": devid}) @_ensure_connection def HotAddDisk(self, disk, access_mode, cache_writeback, direct, blockdevice): """Hot-add a disk """ if access_mode == constants.DISK_KERNELSPACE: qemu_version = self.GetVersion() if qemu_version >= (9, 0, 0) and direct: # starting with qemu v9 it is required to # use O_DIRECT for opening the blockdevice # when `direct` is set flags = os.O_RDWR|os.O_DIRECT else: flags = os.O_RDWR fd = os.open(blockdevice["file"]["filename"], flags) fdset = self._AddFd(fd) os.close(fd) blockdevice["file"]["filename"] = "/dev/fdset/%s" % fdset else: fdset = None dev_arguments = { "drive": blockdevice["node-name"], "write-cache": kvm_utils.TranslateBoolToOnOff(cache_writeback) } # Note that hvinfo that _GenerateDeviceHVInfo() creates # should include *only* the driver, id, bus, and # addr or channel, scsi-id, and lun keys dev_arguments.update(self._filter_hvinfo(disk.hvinfo)) self.execute_qmp("blockdev-add", blockdevice) self.execute_qmp("device_add", dev_arguments) if fdset is not None: self._RemoveFdset(fdset) @_ensure_connection def HotDelDisk(self, devid): """Hot-del a Disk """ self.execute_qmp("device_del", {"id": devid}) # wait for the DEVICE_DELETED event with five seconds timeout event = self.wait_for_qmp_event(["DEVICE_DELETED", "DEVICE_UNPLUG_GUEST_ERROR"], 5) if event is None: raise errors.HypervisorError("DEVICE_DELETED event has not arrived") elif event.event_type == "DEVICE_UNPLUG_GUEST_ERROR": raise errors.HypervisorError("DEVICE_UNPLUG_GUEST_ERROR event has " "occurred") self.execute_qmp("blockdev-del", {"node-name": devid}) def _GetPCIDevices(self): """Get the devices of the first PCI bus of a running instance. """ pci = self.execute_qmp("query-pci") bus = pci[0] devices = bus["devices"] return devices @_ensure_connection def ResizeBlockDevice(self, disk_id: str, new_size: int): """ Notify the guest about a disk change. """ arguments = { "node-name": disk_id, "size": new_size } self.execute_qmp("block_resize", arguments) def _HasPCIDevice(self, devid): """Check if a specific device ID exists on the PCI bus. """ for d in self._GetPCIDevices(): if d["qdev_id"] == devid: return True return False def _GetBlockDevices(self): """Get the block devices of a running instance. The query-block QMP command returns a list of dictionaries including information for each virtual disk. For example: [{"device": "disk-049f140d", "inserted": {"file": ..., "image": ...}}] @rtype: list of dicts @return: Info about the virtual disks of the instance. """ devices = self.execute_qmp("query-block") return devices def _HasBlockDevice(self, devid): """Check if a specific device ID exists among block devices. """ for d in self._GetBlockDevices(): if d["device"] == devid: return True return False @_ensure_connection def HasDevice(self, devid): """Check if a specific device exists or not on a running instance. It first checks the PCI devices and then the block devices. """ if (self._HasPCIDevice(devid) or self._HasBlockDevice(devid)): return True return False @_ensure_connection def GetFreePCISlot(self): """Get the first available PCI slot of a running instance. """ slots = bitarray(constants.QEMU_PCI_SLOTS) slots.setall(False) # pylint: disable=E1101 for d in self._GetPCIDevices(): slot = d["slot"] slots[slot] = True return utils.GetFreeSlot(slots) @_ensure_connection def CheckDiskHotAddSupport(self): """Check if disk hotplug is possible Hotplug is *not* supported in case the add-fd and blockdev-add qmp commands are not supported """ def _raise(reason): raise errors.HotplugError("Cannot hot-add disk: %s." % reason) if "add-fd" not in self.supported_commands: _raise("add-fd qmp command is not supported") if "blockdev-add" not in self.supported_commands: _raise("blockdev-add qmp command is not supported") @_ensure_connection def CheckNicHotAddSupport(self): """Check if NIC hotplug is possible Hotplug is *not* supported in case the getfd and netdev_add qmp commands are not supported """ def _raise(reason): raise errors.HotplugError("Cannot hot-add NIC: %s." % reason) if "getfd" not in self.supported_commands: _raise("getfd qmp command is not supported") if "netdev_add" not in self.supported_commands: _raise("netdev_add qmp command is not supported") @_ensure_connection def GetVersion(self): """Return the QMP/qemu version field Accessing the version attribute directly might result in an error since the socket might not be yet connected. This getter method uses the @_ensure_connection decorator to work around this problem. """ return self.version @_ensure_connection def HasDynamicAutoReadOnly(self): """Check if QEMU uses dynamic auto-read-only for block devices Use QMP schema introspection (QEMU 2.5+) to check for the dynamic-auto-read-only feature. """ schema = self.execute_qmp("query-qmp-schema") # QEMU 4.0 did not have a feature flag, but has dynamic auto-read-only # support. if self.version[:2] == (4, 0): return True return any([x for x in schema if "dynamic-auto-read-only" in x.get("features",[])]) @_ensure_connection def SetMigrationParameters(self, max_bandwidth, downtime_limit): """Configute live migration parameters """ arguments = { "max-bandwidth": max_bandwidth, "downtime-limit": downtime_limit, } if self.version >= (3, 0, 0): arguments["max-postcopy-bandwidth"] = max_bandwidth self.execute_qmp("migrate-set-parameters", arguments) @_ensure_connection def SetMigrationCapabilities(self, capabilities, state): """Configure live migration capabilities """ arguments = { "capabilities": [] } for capability in capabilities: arguments["capabilities"].append({ "capability": capability, "state": state, }) self.execute_qmp("migrate-set-capabilities", arguments) @_ensure_connection def StopGuestEmulation(self): """Pause the running guest """ self.execute_qmp("stop") @_ensure_connection def ContinueGuestEmulation(self): """Continue the previously paused guest """ self.execute_qmp("cont") @_ensure_connection def StartMigration(self, target, port): """Start migration of an instance """ arguments = { "uri": "tcp:%s:%s" % (target, port) } self.execute_qmp("migrate", arguments) @_ensure_connection def StartPostcopyMigration(self): """ Start postcopy-ram migration """ self.execute_qmp("migrate-start-postcopy") @_ensure_connection def GetCpuInformation(self): """ Retrieve CPU/thread information uses the query-cpus-fast which does not interrupt the guest """ return self.execute_qmp("query-cpus-fast") @_ensure_connection def GetMigrationStatus(self): """Retrieve the current migration status """ return self.execute_qmp("query-migrate") @_ensure_connection def SetSpicePassword(self, spice_pwd): """Set Spice password of an instance """ arguments = { "protocol": "spice", "password": spice_pwd, } self.execute_qmp("set_password", arguments) @_ensure_connection def SetVNCPassword(self, vnc_pwd): """Set VNC password of an instance """ arguments = { "protocol": "vnc", "password": vnc_pwd, } self.execute_qmp("set_password", arguments) @_ensure_connection def SetBalloonMemory(self, memory): self.execute_qmp("balloon", {"value": memory * 1048576}) @_ensure_connection def Powerdown(self): self.execute_qmp("system_powerdown") def _GetFd(self, fd, fdname): """Wrapper around the getfd qmp command Send an fd to a running process via SCM_RIGHTS and then use the getfd qmp command to name it properly so that it can be used later by NIC hotplugging. @type fd: int @param fd: The file descriptor to pass @raise errors.HypervisorError: If getfd fails for some reason """ try: utils.SendFds(self.sock, b" ", [fd]) arguments = { "fdname": fdname, } self.execute_qmp("getfd", arguments) except errors.HypervisorError as err: logging.info("Passing fd %s via SCM_RIGHTS failed: %s", fd, err) raise def _AddFd(self, fd): """Wrapper around add-fd qmp command Send fd to a running process via SCM_RIGHTS and then add-fd qmp command to add it to an fdset so that it can be used later by disk hotplugging. @type fd: int @param fd: The file descriptor to pass @return: The fdset ID that the fd has been added to @raise errors.HypervisorError: If add-fd fails for some reason """ try: utils.SendFds(self.sock, b" ", [fd]) # Omit fdset-id and let qemu create a new one (see qmp-commands.hx) response = self.execute_qmp("add-fd") fdset = response["fdset-id"] except errors.HypervisorError as err: logging.info("Passing fd %s via SCM_RIGHTS failed: %s", fd, err) raise return fdset def _RemoveFdset(self, fdset): """Wrapper around remove-fd qmp command Remove the file descriptor previously passed. After qemu has dup'd the fd (e.g. during disk hotplug), it can be safely removed. """ # Omit the fd to cleanup all fds in the fdset (see qemu/qmp-commands.hx) try: self.execute_qmp("remove-fd", {"fdset-id": fdset}) except errors.HypervisorError as err: # There is no big deal if we cannot remove an fdset. This cleanup here is # done on a best effort basis. Upon next hot-add a new fdset will be # created. If we raise an exception here, that is after drive_add has # succeeded, the whole hot-add action will fail and the runtime file will # not be updated which will make the instance non migrate-able logging.info("Removing fdset with id %s failed: %s", fdset, err) ganeti-3.1.0~rc2/lib/hypervisor/hv_kvm/netdev.py000064400000000000000000000143201476477700300217150ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """KVM hypervisor tap device helpers """ import os import logging import struct import fcntl from ganeti import errors # TUN/TAP driver constants, taken from # They are architecture-independent and already hardcoded in qemu-kvm source, # so we can safely include them here. TUNSETIFF = 0x400454ca TUNGETIFF = 0x800454d2 TUNGETFEATURES = 0x800454cf IFF_TAP = 0x0002 IFF_NO_PI = 0x1000 IFF_ONE_QUEUE = 0x2000 IFF_VNET_HDR = 0x4000 IFF_MULTI_QUEUE = 0x0100 def _GetTunFeatures(fd, _ioctl=fcntl.ioctl): """Retrieves supported TUN features from file descriptor. @see: L{_ProbeTapVnetHdr} """ req = struct.pack("I", 0) try: buf = _ioctl(fd, TUNGETFEATURES, req) except EnvironmentError as err: logging.warning("ioctl(TUNGETFEATURES) failed: %s", err) return None else: (flags, ) = struct.unpack("I", buf) return flags def _ProbeTapVnetHdr(fd, _features_fn=_GetTunFeatures): """Check whether to enable the IFF_VNET_HDR flag. To do this, _all_ of the following conditions must be met: 1. TUNGETFEATURES ioctl() *must* be implemented 2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag 3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in drivers/net/tun.c there is no way to test this until after the tap device has been created using TUNSETIFF, and there is no way to change the IFF_VNET_HDR flag after creating the interface, catch-22! However both TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27, thus we can expect TUNGETIFF to be present if TUNGETFEATURES is. @type fd: int @param fd: the file descriptor of /dev/net/tun """ flags = _features_fn(fd) if flags is None: # Not supported return False result = bool(flags & IFF_VNET_HDR) if not result: logging.warning("Kernel does not support IFF_VNET_HDR, not enabling") return result def _ProbeTapMqVirtioNet(fd, _features_fn=_GetTunFeatures): """Check whether to enable the IFF_MULTI_QUEUE flag. This flag was introduced in Linux kernel 3.8. @type fd: int @param fd: the file descriptor of /dev/net/tun """ flags = _features_fn(fd) if flags is None: # Not supported return False result = bool(flags & IFF_MULTI_QUEUE) if not result: logging.warning("Kernel does not support IFF_MULTI_QUEUE, not enabling") return result def OpenTap(name="", features=None): """Open a new tap device and return its file descriptor. This is intended to be used by a qemu-type hypervisor together with the -net tap,fd= or -net tap,fds=x:y:...:z command line parameter. @type name: string @param name: name for the TAP interface being created; if an empty string is passed, the OS will generate a unique name @type features: dict @param features: A dict denoting whether vhost, vnet_hdr, mq netdev features are enabled or not. @return: (ifname, [tapfds], [vhostfds]) @rtype: tuple """ tapfds = [] vhostfds = [] if features is None: features = {} vhost = features.get("vhost", False) vnet_hdr = features.get("vnet_hdr", True) _, virtio_net_queues = features.get("mq", (False, 1)) ifreq_name = name.encode('ascii') for _ in range(virtio_net_queues): try: tapfd = os.open("/dev/net/tun", os.O_RDWR) except EnvironmentError: raise errors.HypervisorError("Failed to open /dev/net/tun") flags = IFF_TAP | IFF_NO_PI if vnet_hdr and _ProbeTapVnetHdr(tapfd): flags |= IFF_VNET_HDR # Check if it's ok to enable IFF_MULTI_QUEUE if virtio_net_queues > 1 and _ProbeTapMqVirtioNet(tapfd): flags |= IFF_MULTI_QUEUE else: flags |= IFF_ONE_QUEUE # The struct ifreq ioctl request (see netdevice(7)) ifr = struct.pack("16sh", ifreq_name, flags) try: res = fcntl.ioctl(tapfd, TUNSETIFF, ifr) except EnvironmentError as err: raise errors.HypervisorError("Failed to allocate a new TAP device: %s" % err) if vhost: # This is done regularly by the qemu process if vhost=on was passed with # --netdev option. Still, in case of hotplug and if the process does not # run with root privileges, we have to get the fds and pass them via # SCM_RIGHTS prior to qemu using them. try: vhostfd = os.open("/dev/vhost-net", os.O_RDWR) vhostfds.append(vhostfd) except EnvironmentError: raise errors.HypervisorError("Failed to open /dev/vhost-net") tapfds.append(tapfd) if not ifreq_name: # Set the tap device name after the first iteration of the loop going over # the number of net queues, if it is not already set. If we don't do this, # a new tap device will be created for each queue. ifreq_name = struct.unpack("16sh", res)[0].strip(b"\x00") # Get the interface name from the ioctl ifname = struct.unpack("16sh", res)[0].strip(b"\x00") return (ifname, tapfds, vhostfds) ganeti-3.1.0~rc2/lib/hypervisor/hv_kvm/validation.py000064400000000000000000000220211476477700300225570ustar00rootroot00000000000000# # # Copyright (C) 2022 the Ganeti project # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """KVM hypervisor parameter/syntax validation helpers """ import re import pwd from ganeti import constants from ganeti import netutils from ganeti import errors from ganeti import utils #: SPICE parameters which depend on L{constants.HV_KVM_SPICE_BIND} _SPICE_ADDITIONAL_PARAMS = frozenset([ constants.HV_KVM_SPICE_IP_VERSION, constants.HV_KVM_SPICE_PASSWORD_FILE, constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR, constants.HV_KVM_SPICE_JPEG_IMG_COMPR, constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR, constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION, constants.HV_KVM_SPICE_USE_TLS, ]) _SPICE_RE = re.compile(r"^-spice\s", re.M) _CHECK_MACHINE_VERSION_RE = [lambda x: re.compile(r"^(%s)[ ]+.*PC" % x, re.M)] _VERSION_RE = re.compile(r"\b(\d+)\.(\d+)(\.(\d+))?\b") def check_spice_parameters(hvparams): spice_bind = hvparams[constants.HV_KVM_SPICE_BIND] spice_ip_version = hvparams[constants.HV_KVM_SPICE_IP_VERSION] if spice_bind: if spice_ip_version != constants.IFACE_NO_IP_VERSION_SPECIFIED: # if an IP version is specified, the spice_bind parameter must be an # IP of that family if (netutils.IP4Address.IsValid(spice_bind) and spice_ip_version != constants.IP4_VERSION): raise errors.HypervisorError("SPICE: Got an IPv4 address (%s), but" " the specified IP version is %s" % (spice_bind, spice_ip_version)) if (netutils.IP6Address.IsValid(spice_bind) and spice_ip_version != constants.IP6_VERSION): raise errors.HypervisorError("SPICE: Got an IPv6 address (%s), but" " the specified IP version is %s" % (spice_bind, spice_ip_version)) else: # All the other SPICE parameters depend on spice_bind being set. Raise an # error if any of them is set without it. for param in _SPICE_ADDITIONAL_PARAMS: if hvparams[param]: raise errors.HypervisorError("SPICE: %s requires %s to be set" % (param, constants.HV_KVM_SPICE_BIND)) return True def validate_spice_parameters(hvparams, kvm_help_output): spice_bind = hvparams[constants.HV_KVM_SPICE_BIND] if spice_bind: # only one of VNC and SPICE can be used currently. if hvparams[constants.HV_VNC_BIND_ADDRESS]: raise errors.HypervisorError("Both SPICE and VNC are configured, but" " only one of them can be used at a" " given time") # check that KVM supports SPICE if not _SPICE_RE.search(kvm_help_output): raise errors.HypervisorError("SPICE is configured, but it is not" " supported according to 'kvm --help'") # if spice_bind is not an IP address, it must be a valid interface bound_to_addr = (netutils.IP4Address.IsValid(spice_bind) or netutils.IP6Address.IsValid(spice_bind)) if not bound_to_addr and not netutils.IsValidInterface(spice_bind): raise errors.HypervisorError("SPICE: The %s parameter must be either" " a valid IP address or interface name" % constants.HV_KVM_SPICE_BIND) return True def check_vnc_parameters(hvparams): if (hvparams[constants.HV_VNC_X509_VERIFY] and not hvparams[constants.HV_VNC_X509]): raise errors.HypervisorError("%s must be defined, if %s is" % (constants.HV_VNC_X509, constants.HV_VNC_X509_VERIFY)) return True def validate_vnc_parameters(hvparams): vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS] if vnc_bind_address: bound_to_addr = (netutils.IP4Address.IsValid(vnc_bind_address) or netutils.IP6Address.IsValid(vnc_bind_address)) is_interface = netutils.IsValidInterface(vnc_bind_address) is_path = utils.IsNormAbsPath(vnc_bind_address) if not bound_to_addr and not is_interface and not is_path: raise errors.HypervisorError("VNC: The %s parameter must be either" " a valid IP address, an interface name," " or an absolute path" % constants.HV_VNC_BIND_ADDRESS) return True def check_security_model(hvparams): security_model = hvparams[constants.HV_SECURITY_MODEL] if security_model == constants.HT_SM_USER: if not hvparams[constants.HV_SECURITY_DOMAIN]: raise errors.HypervisorError( "A security domain (user to run kvm as)" " must be specified") elif (security_model == constants.HT_SM_NONE or security_model == constants.HT_SM_POOL): if hvparams[constants.HV_SECURITY_DOMAIN]: raise errors.HypervisorError( "Cannot have a security domain when the" " security model is 'none' or 'pool'") return True def validate_security_model(hvparams): security_model = hvparams[constants.HV_SECURITY_MODEL] if security_model == constants.HT_SM_USER: username = hvparams[constants.HV_SECURITY_DOMAIN] try: pwd.getpwnam(username) except KeyError: raise errors.HypervisorError("Unknown security domain user %s" % username) return True def check_boot_parameters(hvparams): boot_order = hvparams[constants.HV_BOOT_ORDER] if (boot_order == constants.HT_BO_CDROM and not hvparams[constants.HV_CDROM_IMAGE_PATH]): raise errors.HypervisorError("Cannot boot from cdrom without an" " ISO path") kernel_path = hvparams[constants.HV_KERNEL_PATH] if kernel_path: if not hvparams[constants.HV_ROOT_PATH]: raise errors.HypervisorError("Need a root partition for the instance," " if a kernel is defined") return True def check_console_parameters(hvparams): if hvparams[constants.HV_SERIAL_CONSOLE]: serial_speed = hvparams[constants.HV_SERIAL_SPEED] valid_speeds = constants.VALID_SERIAL_SPEEDS if not serial_speed or serial_speed not in valid_speeds: raise errors.HypervisorError("Invalid serial console speed, must be" " one of: %s" % utils.CommaJoin(valid_speeds)) return True def validate_machine_version(hvparams, kvm_machine_output): machine_version = hvparams[constants.HV_KVM_MACHINE_VERSION] if machine_version: for test in _CHECK_MACHINE_VERSION_RE: if not test(machine_version).search(kvm_machine_output): raise errors.HypervisorError("Unsupported machine version: %s" % machine_version) return True def check_disk_cache_parameters(hvparams): disk_aio = hvparams[constants.HV_KVM_DISK_AIO] disk_cache = hvparams[constants.HV_DISK_CACHE] if disk_aio == constants.HT_KVM_AIO_NATIVE and \ disk_cache != constants.HT_CACHE_NONE: raise errors.HypervisorError("When 'disk_aio' is set to 'native', the " "only supported value for 'disk_cache' is " "'none'.") return True def validate_disk_parameters(hvparams, kvm_version): v_all, v_maj, v_min, v_rev = kvm_version disk_aio = hvparams[constants.HV_KVM_DISK_AIO] if disk_aio == constants.HT_KVM_AIO_IO_URING: if v_maj < 5: raise errors.HypervisorError("At least QEMU 5.0 required to use" "'disk_aio=io_uring'.") ganeti-3.1.0~rc2/lib/hypervisor/hv_lxc.py000064400000000000000000001104651476477700300204300ustar00rootroot00000000000000# # # Copyright (C) 2010, 2013, 2014, 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """LXC hypervisor """ import errno import os import os.path import logging import sys import re from ganeti import constants from ganeti import errors # pylint: disable=W0611 from ganeti import utils from ganeti import objects from ganeti import pathutils from ganeti import serializer from ganeti.hypervisor import hv_base from ganeti.errors import HypervisorError def _CreateBlankFile(path, mode): """Create blank file. Create a blank file for the path with specified mode. An existing file will be overwritten. """ try: utils.WriteFile(path, data="", mode=mode) except EnvironmentError as err: raise HypervisorError("Failed to create file %s: %s" % (path, err)) class LXCVersion(tuple): """LXC version class. """ # Let beta version following micro version, but don't care about it _VERSION_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)") @classmethod def _Parse(cls, version_string): """Parse a passed string as an LXC version string. @param version_string: a valid LXC version string @type version_string: string @raise ValueError: if version_string is an invalid LXC version string @rtype tuple(int, int, int) @return (major_num, minor_num, micro_num) """ match = cls._VERSION_RE.match(version_string) if match: return tuple(map(int, match.groups())) else: raise ValueError("'%s' is not a valid LXC version string" % version_string) def __new__(cls, version_string): version = super(LXCVersion, cls).__new__(cls, cls._Parse(version_string)) version.original_string = version_string return version def __str__(self): # pylint: disable=E1101 return self.original_string class LXCHypervisor(hv_base.BaseHypervisor): """LXC-based virtualization. """ _ROOT_DIR = pathutils.RUN_DIR + "/lxc" _LOG_DIR = pathutils.LOG_DIR + "/lxc" # The instance directory has to be structured in a way that would allow it to # be passed as an argument of the --lxcpath option in lxc- commands. # This means that: # Each LXC instance should have a directory carrying their name under this # directory. # Each instance directory should contain the "config" file that contains the # LXC container configuration of an instance. # # Therefore the structure of the directory tree should be: # # _INSTANCE_DIR # \_ instance1 # \_ config # \_ instance2 # \_ config # # Other instance specific files can also be placed under an instance # directory. _INSTANCE_DIR = _ROOT_DIR + "/instance" _CGROUP_ROOT_DIR = _ROOT_DIR + "/cgroup" _PROC_CGROUPS_FILE = "/proc/cgroups" _PROC_SELF_CGROUP_FILE = "/proc/self/cgroup" _LXC_MIN_VERSION_REQUIRED = LXCVersion("1.0.0") _LXC_COMMANDS_REQUIRED = [ "lxc-console", "lxc-ls", "lxc-start", "lxc-stop", "lxc-wait", ] _DIR_MODE = 0o755 _STASH_KEY_ALLOCATED_LOOP_DEV = "allocated_loopdev" _MEMORY_PARAMETER = "memory.limit_in_bytes" _MEMORY_SWAP_PARAMETER = "memory.memsw.limit_in_bytes" PARAMETERS = { constants.HV_CPU_MASK: hv_base.OPT_CPU_MASK_CHECK, constants.HV_LXC_DEVICES: hv_base.NO_CHECK, constants.HV_LXC_DROP_CAPABILITIES: hv_base.NO_CHECK, constants.HV_LXC_EXTRA_CGROUPS: hv_base.NO_CHECK, constants.HV_LXC_EXTRA_CONFIG: hv_base.NO_CHECK, constants.HV_LXC_NUM_TTYS: hv_base.REQ_NONNEGATIVE_INT_CHECK, constants.HV_LXC_STARTUP_TIMEOUT: hv_base.OPT_NONNEGATIVE_INT_CHECK, } _REBOOT_TIMEOUT = 120 # secs _REQUIRED_CGROUP_SUBSYSTEMS = [ "cpuset", "memory", "devices", "cpuacct", ] def __init__(self): hv_base.BaseHypervisor.__init__(self) self._EnsureDirectoryExistence() @classmethod def _InstanceDir(cls, instance_name): """Return the root directory for an instance. """ return utils.PathJoin(cls._INSTANCE_DIR, instance_name) @classmethod def _InstanceConfFilePath(cls, instance_name): """Return the configuration file for an instance. """ return utils.PathJoin(cls._InstanceDir(instance_name), "config") @classmethod def _InstanceLogFilePath(cls, instance): """Return the log file for an instance. @type instance: L{objects.Instance} """ filename = "%s.%s.log" % (instance.name, instance.uuid) return utils.PathJoin(cls._LOG_DIR, filename) @classmethod def _InstanceConsoleLogFilePath(cls, instance_name): """Return the console log file path for an instance. """ return utils.PathJoin(cls._InstanceDir(instance_name), "console.log") @classmethod def _InstanceStashFilePath(cls, instance_name): """Return the stash file path for an instance. The stash file is used to keep information needed to clean up after the destruction of the instance. """ return utils.PathJoin(cls._InstanceDir(instance_name), "stash") def _EnsureDirectoryExistence(self): """Ensures all the directories needed for LXC use exist. """ utils.EnsureDirs([ (self._ROOT_DIR, self._DIR_MODE), (self._LOG_DIR, 0o750), (self._INSTANCE_DIR, 0o750), ]) def _SaveInstanceStash(self, instance_name, data): """Save data to the instance stash file in serialized format. """ stash_file = self._InstanceStashFilePath(instance_name) serialized = serializer.Dump(data) try: utils.WriteFile(stash_file, data=serialized, mode=constants.SECURE_FILE_MODE) except EnvironmentError as err: raise HypervisorError("Failed to save instance stash file %s : %s" % (stash_file, err)) def _LoadInstanceStash(self, instance_name): """Load information stashed in file which was created by L{_SaveInstanceStash}. """ stash_file = self._InstanceStashFilePath(instance_name) try: return serializer.Load(utils.ReadFile(stash_file)) except (EnvironmentError, ValueError) as err: raise HypervisorError("Failed to load instance stash file %s : %s" % (stash_file, err)) @classmethod def _MountCgroupSubsystem(cls, subsystem): """Mount the cgroup subsystem fs under the cgroup root dir. @type subsystem: string @param subsystem: cgroup subsystem name to mount @rtype string @return path of subsystem mount point """ subsys_dir = utils.PathJoin(cls._GetCgroupMountPoint(), subsystem) if not os.path.isdir(subsys_dir): try: os.makedirs(subsys_dir) except EnvironmentError as err: raise HypervisorError("Failed to create directory %s: %s" % (subsys_dir, err)) mount_cmd = ["mount", "-t", "cgroup", "-o", subsystem, subsystem, subsys_dir] result = utils.RunCmd(mount_cmd) if result.failed: raise HypervisorError("Failed to mount cgroup subsystem '%s': %s" % (subsystem, result.output)) return subsys_dir def _CleanupInstance(self, instance_name, stash): """Actual implementation of the instance cleanup procedure. @type instance_name: string @param instance_name: instance name @type stash: dict(string:any) @param stash: dict that contains desired information for instance cleanup """ try: if self._STASH_KEY_ALLOCATED_LOOP_DEV in stash: loop_dev_path = stash[self._STASH_KEY_ALLOCATED_LOOP_DEV] utils.ReleaseBdevPartitionMapping(loop_dev_path) except errors.CommandError as err: raise HypervisorError("Failed to cleanup partition mapping : %s" % err) utils.RemoveFile(self._InstanceStashFilePath(instance_name)) def CleanupInstance(self, instance_name): """Cleanup after a stopped instance. """ stash = self._LoadInstanceStash(instance_name) self._CleanupInstance(instance_name, stash) @classmethod def _GetCgroupMountPoint(cls): """Return the directory that should be the base of cgroup fs. """ return cls._CGROUP_ROOT_DIR @classmethod def _GetOrPrepareCgroupSubsysMountPoint(cls, subsystem): """Prepare cgroup subsystem mount point. @type subsystem: string @param subsystem: cgroup subsystem name to mount @rtype string @return path of subsystem mount point """ for _, mpoint, fstype, options in utils.GetMounts(): if fstype == "cgroup" and subsystem in options.split(","): return mpoint return cls._MountCgroupSubsystem(subsystem) @classmethod def _GetCurrentCgroupSubsysGroups(cls): """Return the dict of cgroup subsystem hierarchies this process belongs to. The dictionary has the cgroup subsystem as a key and its hierarchy as a value. Information is read from /proc/self/cgroup. """ try: cgroup_list = utils.ReadFile(cls._PROC_SELF_CGROUP_FILE) except EnvironmentError as err: raise HypervisorError("Failed to read %s : %s" % (cls._PROC_SELF_CGROUP_FILE, err)) cgroups = {} for line in filter(None, cgroup_list.split("\n")): _, subsystems, hierarchy = line.split(":") for subsys in subsystems.split(","): cgroups[subsys] = hierarchy[1:] # discard first '/' return cgroups @classmethod def _GetCgroupSubsysDir(cls, subsystem): """Return the directory of the cgroup subsystem we use. @type subsystem: string @param subsystem: cgroup subsystem name @rtype: string @return: path of the hierarchy directory for the subsystem """ subsys_dir = cls._GetOrPrepareCgroupSubsysMountPoint(subsystem) base_group = cls._GetCurrentCgroupSubsysGroups().get(subsystem, "") return utils.PathJoin(subsys_dir, base_group, "lxc") @classmethod def _GetCgroupParamPath(cls, param_name, instance_name=None): """Return the path of the specified cgroup parameter file. @type param_name: string @param param_name: cgroup subsystem parameter name @rtype: string @return: path of the cgroup subsystem parameter file """ subsystem = param_name.split(".", 1)[0] subsys_dir = cls._GetCgroupSubsysDir(subsystem) if instance_name is not None: return utils.PathJoin(subsys_dir, instance_name, param_name) else: return utils.PathJoin(subsys_dir, param_name) @classmethod def _GetCgroupInstanceValue(cls, instance_name, param_name): """Return the value of the specified cgroup parameter. @type instance_name: string @param instance_name: instance name @type param_name: string @param param_name: cgroup subsystem parameter name @rtype string @return value read from cgroup subsystem fs """ param_path = cls._GetCgroupParamPath(param_name, instance_name=instance_name) return utils.ReadFile(param_path).rstrip("\n") @classmethod def _SetCgroupInstanceValue(cls, instance_name, param_name, param_value): """Set the value to the specified instance cgroup parameter. @type instance_name: string @param instance_name: instance name @type param_name: string @param param_name: cgroup subsystem parameter name @type param_value: string @param param_value: cgroup subsystem parameter value to be set """ param_path = cls._GetCgroupParamPath(param_name, instance_name=instance_name) # When interacting with cgroup fs, errno is quite important information # to see what happened when setting a cgroup parameter, so just throw # an error to the upper level. # e.g., we could know that the container can't reclaim its memory by # checking if the errno is EBUSY when setting the # memory.memsw.limit_in_bytes. fd = -1 try: fd = os.open(param_path, os.O_WRONLY) os.write(fd, param_value.encode("ascii")) finally: if fd != -1: os.close(fd) @classmethod def _IsCgroupParameterPresent(cls, parameter, hvparams=None): """Return whether a cgroup parameter can be used. This is checked by seeing whether there is a file representation of the parameter in the location where the cgroup is mounted. @type parameter: string @param parameter: The name of the parameter. @param hvparams: dict @param hvparams: The hypervisor parameters, optional. @rtype: boolean """ cls._EnsureCgroupMounts(hvparams) param_path = cls._GetCgroupParamPath(parameter) return os.path.exists(param_path) @classmethod def _GetCgroupCpuList(cls, instance_name): """Return the list of CPU ids for an instance. """ try: cpumask = cls._GetCgroupInstanceValue(instance_name, "cpuset.cpus") except EnvironmentError as err: raise errors.HypervisorError("Getting CPU list for instance" " %s failed: %s" % (instance_name, err)) return utils.ParseCpuMask(cpumask) @classmethod def _GetCgroupCpuUsage(cls, instance_name): """Return the CPU usage of an instance. """ try: cputime_ns = cls._GetCgroupInstanceValue(instance_name, "cpuacct.usage") except EnvironmentError as err: raise HypervisorError("Failed to get the cpu usage of %s: %s" % (instance_name, err)) return float(cputime_ns) / 10 ** 9 # nano secs to float secs @classmethod def _GetCgroupMemoryLimit(cls, instance_name): """Return the memory limit for an instance """ try: mem_limit = cls._GetCgroupInstanceValue(instance_name, "memory.limit_in_bytes") return int(mem_limit) except EnvironmentError as err: raise HypervisorError("Can't get instance memory limit of %s: %s" % (instance_name, err)) def ListInstances(self, hvparams=None): """Get the list of running instances. """ return self._ListAliveInstances() @classmethod def _IsInstanceAlive(cls, instance_name): """Return True if instance is alive. """ result = utils.RunCmd(["lxc-ls", "--running", re.escape(instance_name)]) if result.failed: raise HypervisorError("Failed to get running LXC containers list: %s" % result.output) return instance_name in result.stdout.split() @classmethod def _ListAliveInstances(cls): """Return list of alive instances. """ result = utils.RunCmd(["lxc-ls", "--running"]) if result.failed: raise HypervisorError("Failed to get running LXC containers list: %s" % result.output) return result.stdout.split() def GetInstanceInfo(self, instance_name, hvparams=None): """Get instance properties. @type instance_name: string @param instance_name: the instance name @type hvparams: dict of strings @param hvparams: hvparams to be used with this instance @rtype: tuple of strings @return: (name, id, memory, vcpus, stat, times) """ if not self._IsInstanceAlive(instance_name): return None return self._GetInstanceInfoInner(instance_name) def _GetInstanceInfoInner(self, instance_name): """Get instance properties. @type instance_name: string @param instance_name: the instance name @rtype: tuple of strings @return: (name, id, memory, vcpus, stat, times) """ cpu_list = self._GetCgroupCpuList(instance_name) memory = self._GetCgroupMemoryLimit(instance_name) // (1024 ** 2) cputime = self._GetCgroupCpuUsage(instance_name) return (instance_name, 0, memory, len(cpu_list), hv_base.HvInstanceState.RUNNING, cputime) def GetAllInstancesInfo(self, hvparams=None): """Get properties of all instances. @type hvparams: dict of strings @param hvparams: hypervisor parameter @return: [(name, id, memory, vcpus, stat, times),...] """ data = [] running_instances = self._ListAliveInstances() filter_fn = lambda x: os.path.isdir(utils.PathJoin(self._INSTANCE_DIR, x)) for dirname in filter(filter_fn, os.listdir(self._INSTANCE_DIR)): if dirname not in running_instances: continue try: info = self._GetInstanceInfoInner(dirname) except errors.HypervisorError: continue if info: data.append(info) return data @classmethod def _GetInstanceDropCapabilities(cls, hvparams): """Get and parse the drop capabilities list from the instance hvparams. @type hvparams: dict of strings @param hvparams: instance hvparams @rtype list(string) @return list of drop capabilities """ drop_caps = hvparams[constants.HV_LXC_DROP_CAPABILITIES] return drop_caps.split(",") def _CreateConfigFile(self, instance, sda_dev_path): """Create an lxc.conf file for an instance. """ out = [] # hostname out.append("lxc.utsname = %s" % instance.name) # separate pseudo-TTY instances out.append("lxc.pts = 255") # standard TTYs num_ttys = instance.hvparams[constants.HV_LXC_NUM_TTYS] if num_ttys: # if it is the number greater than 0 out.append("lxc.tty = %s" % num_ttys) # console log file # After the following patch was applied, we lost the console log file output # until the lxc.console.logfile parameter was introduced in 1.0.6. # https:// # lists.linuxcontainers.org/pipermail/lxc-devel/2014-March/008470.html lxc_version = self._GetLXCVersionFromCmd("lxc-start") if lxc_version >= LXCVersion("1.0.6"): console_log_path = self._InstanceConsoleLogFilePath(instance.name) _CreateBlankFile(console_log_path, constants.SECURE_FILE_MODE) out.append("lxc.console.logfile = %s" % console_log_path) else: logging.warn("Console log file is not supported in LXC version %s," " disabling.", lxc_version) # root FS out.append("lxc.rootfs = %s" % sda_dev_path) # Necessary file systems out.append("lxc.mount.entry = proc proc proc nodev,noexec,nosuid 0 0") out.append("lxc.mount.entry = sysfs sys sysfs defaults 0 0") # CPUs if instance.hvparams[constants.HV_CPU_MASK]: cpu_list = utils.ParseCpuMask(instance.hvparams[constants.HV_CPU_MASK]) cpus_in_mask = len(cpu_list) if cpus_in_mask != instance.beparams["vcpus"]: raise errors.HypervisorError("Number of VCPUs (%d) doesn't match" " the number of CPUs in the" " cpu_mask (%d)" % (instance.beparams["vcpus"], cpus_in_mask)) out.append("lxc.cgroup.cpuset.cpus = %s" % instance.hvparams[constants.HV_CPU_MASK]) # Memory out.append("lxc.cgroup.memory.limit_in_bytes = %dM" % instance.beparams[constants.BE_MAXMEM]) if LXCHypervisor._IsCgroupParameterPresent(self._MEMORY_SWAP_PARAMETER, instance.hvparams): out.append("lxc.cgroup.memory.memsw.limit_in_bytes = %dM" % instance.beparams[constants.BE_MAXMEM]) # Device control # deny direct device access out.append("lxc.cgroup.devices.deny = a") dev_specs = instance.hvparams[constants.HV_LXC_DEVICES] for dev_spec in dev_specs.split(","): out.append("lxc.cgroup.devices.allow = %s" % dev_spec) # Networking for idx, nic in enumerate(instance.nics): out.append("# NIC %d" % idx) mode = nic.nicparams[constants.NIC_MODE] link = nic.nicparams[constants.NIC_LINK] if mode == constants.NIC_MODE_BRIDGED: out.append("lxc.network.type = veth") out.append("lxc.network.link = %s" % link) else: raise errors.HypervisorError("LXC hypervisor only supports" " bridged mode (NIC %d has mode %s)" % (idx, mode)) out.append("lxc.network.hwaddr = %s" % nic.mac) out.append("lxc.network.flags = up") # Capabilities for cap in self._GetInstanceDropCapabilities(instance.hvparams): out.append("lxc.cap.drop = %s" % cap) # Extra config # TODO: Currently a configuration parameter that includes comma # in its value can't be added via this parameter. # Make this parameter able to read from a file once the # "parameter from a file" feature added. extra_configs = instance.hvparams[constants.HV_LXC_EXTRA_CONFIG] if extra_configs: out.append("# User defined configs") out.extend(extra_configs.split(",")) return "\n".join(out) + "\n" @classmethod def _GetCgroupEnabledKernelSubsystems(cls): """Return cgroup subsystems list that are enabled in current kernel. """ try: subsys_table = utils.ReadFile(cls._PROC_CGROUPS_FILE) except EnvironmentError as err: raise HypervisorError("Failed to read cgroup info from %s: %s" % (cls._PROC_CGROUPS_FILE, err)) return [x.split(None, 1)[0] for x in subsys_table.split("\n") if x and not x.startswith("#")] @classmethod def _EnsureCgroupMounts(cls, hvparams=None): """Ensures all cgroup subsystems required to run LXC container are mounted. """ # Check cgroup subsystems required by the Ganeti LXC hypervisor for subsystem in cls._REQUIRED_CGROUP_SUBSYSTEMS: cls._GetOrPrepareCgroupSubsysMountPoint(subsystem) # Check cgroup subsystems required by the LXC if hvparams is None or not hvparams[constants.HV_LXC_EXTRA_CGROUPS]: enable_subsystems = cls._GetCgroupEnabledKernelSubsystems() else: enable_subsystems = hvparams[constants.HV_LXC_EXTRA_CGROUPS].split(",") for subsystem in enable_subsystems: cls._GetOrPrepareCgroupSubsysMountPoint(subsystem) @classmethod def _PrepareInstanceRootFsBdev(cls, storage_path, stash): """Return mountable path for storage_path. This function creates a partition mapping for storage_path and returns the first partition device path as a rootfs partition, and stashes the loopback device path. If storage_path is not a multi-partition block device, just return storage_path. """ try: ret = utils.CreateBdevPartitionMapping(storage_path) except errors.CommandError as err: raise HypervisorError("Failed to create partition mapping for %s" ": %s" % (storage_path, err)) if ret is None: return storage_path else: loop_dev_path, dm_dev_paths = ret stash[cls._STASH_KEY_ALLOCATED_LOOP_DEV] = loop_dev_path return dm_dev_paths[0] @classmethod def _WaitForInstanceState(cls, instance_name, state, timeout): """Wait for an instance state transition within timeout Return True if an instance state changed to the desired state within timeout secs. """ result = utils.RunCmd(["lxc-wait", "-n", instance_name, "-s", state], timeout=timeout) if result.failed_by_timeout: return False elif result.failed: raise HypervisorError("Failure while waiting for instance state" " transition: %s" % result.output) else: return True def _SpawnLXC(self, instance, log_file, conf_file): """Execute lxc-start and wait until container health is confirmed. """ lxc_start_cmd = [ "lxc-start", "-n", instance.name, "-o", log_file, "-l", "DEBUG", "-f", conf_file, "-d" ] result = utils.RunCmd(lxc_start_cmd) if result.failed: raise HypervisorError("Failed to start instance %s : %s" % (instance.name, result.output)) lxc_startup_timeout = instance.hvparams[constants.HV_LXC_STARTUP_TIMEOUT] if not self._WaitForInstanceState(instance.name, constants.LXC_STATE_RUNNING, lxc_startup_timeout): raise HypervisorError("Instance %s state didn't change to RUNNING within" " %s secs" % (instance.name, lxc_startup_timeout)) # Ensure that the instance is running correctly after being daemonized if not self._IsInstanceAlive(instance.name): raise HypervisorError("Failed to start instance %s :" " lxc process exited after being daemonized" % instance.name) @classmethod def _VerifyDiskRequirements(cls, block_devices): """Insures that the disks provided work with the current implementation. """ if len(block_devices) == 0: raise HypervisorError("LXC cannot have diskless instances.") if len(block_devices) > 1: raise HypervisorError("At the moment, LXC cannot support more than one" " disk attached to it. Please create this" " instance anew with fewer disks.") def StartInstance(self, instance, block_devices, startup_paused): """Start an instance. For LXC, we try to mount the block device and execute 'lxc-start'. We use volatile containers. """ LXCHypervisor._VerifyDiskRequirements(block_devices) stash = {} # Since LXC version >= 1.0.0, the LXC strictly requires all cgroup # subsystems mounted before starting a container. # Try to mount all cgroup subsystems needed to start a LXC container. self._EnsureCgroupMounts(instance.hvparams) root_dir = self._InstanceDir(instance.name) try: utils.EnsureDirs([(root_dir, self._DIR_MODE)]) except errors.GenericError as err: raise HypervisorError("Creating instance directory failed: %s", str(err)) log_file = self._InstanceLogFilePath(instance) if not os.path.exists(log_file): _CreateBlankFile(log_file, constants.SECURE_FILE_MODE) try: sda_dev_path = block_devices[0][1] # LXC needs to use partition mapping devices to access each partition # of the storage sda_dev_path = self._PrepareInstanceRootFsBdev(sda_dev_path, stash) conf_file = self._InstanceConfFilePath(instance.name) conf = self._CreateConfigFile(instance, sda_dev_path) utils.WriteFile(conf_file, data=conf) logging.info("Starting LXC container") try: self._SpawnLXC(instance, log_file, conf_file) except: logging.error("Failed to start instance %s. Please take a look at %s to" " see LXC errors.", instance.name, log_file) raise except: # Save the original error exc_info = sys.exc_info() try: self._CleanupInstance(instance.name, stash) except HypervisorError as err: logging.warn("Cleanup for instance %s incomplete: %s", instance.name, err) raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) self._SaveInstanceStash(instance.name, stash) def StopInstance(self, instance, force=False, retry=False, name=None, timeout=None): """Stop an instance. """ assert(timeout is None or force is not None) if name is None: name = instance.name if self._IsInstanceAlive(instance.name): lxc_stop_cmd = ["lxc-stop", "-n", name] if force: lxc_stop_cmd.append("--kill") result = utils.RunCmd(lxc_stop_cmd, timeout=timeout) if result.failed: raise HypervisorError("Failed to kill instance %s: %s" % (name, result.output)) else: # The --timeout=-1 option is needed to prevent lxc-stop performs # hard-stop(kill) for the container after the default timing out. lxc_stop_cmd.extend(["--nokill", "--timeout", "-1"]) result = utils.RunCmd(lxc_stop_cmd, timeout=timeout) if result.failed: logging.error("Failed to stop instance %s: %s", name, result.output) def RebootInstance(self, instance): """Reboot an instance. """ if "sys_boot" in self._GetInstanceDropCapabilities(instance.hvparams): raise HypervisorError("The LXC container can't perform a reboot with the" " SYS_BOOT capability dropped.") # We can't use the --timeout=-1 approach as same as the StopInstance due to # the following patch was applied in lxc-1.0.5 and we are supporting # LXC >= 1.0.0. # http://lists.linuxcontainers.org/pipermail/lxc-devel/2014-July/009742.html result = utils.RunCmd(["lxc-stop", "-n", instance.name, "--reboot", "--timeout", str(self._REBOOT_TIMEOUT)]) if result.failed: raise HypervisorError("Failed to reboot instance %s: %s" % (instance.name, result.output)) def BalloonInstanceMemory(self, instance, mem): """Balloon an instance memory to a certain value. @type instance: L{objects.Instance} @param instance: instance to be accepted @type mem: int @param mem: actual memory size to use for instance runtime """ mem_in_bytes = mem * 1024 ** 2 current_mem_usage = self._GetCgroupMemoryLimit(instance.name) shrinking = mem_in_bytes <= current_mem_usage # The memsw.limit_in_bytes parameter might be present depending on kernel # parameters. # If present, it has to be modified at the same time as limit_in_bytes. if LXCHypervisor._IsCgroupParameterPresent(self._MEMORY_SWAP_PARAMETER, instance.hvparams): # memory.memsw.limit_in_bytes is the superlimit of memory.limit_in_bytes # so the order of setting these parameters is quite important. cgparams = [self._MEMORY_SWAP_PARAMETER, self._MEMORY_PARAMETER] else: cgparams = [self._MEMORY_PARAMETER] if shrinking: cgparams.reverse() for i, cgparam in enumerate(cgparams): try: self._SetCgroupInstanceValue(instance.name, cgparam, str(mem_in_bytes)) except EnvironmentError as err: if shrinking and err.errno == errno.EBUSY: logging.warn("Unable to reclaim memory or swap usage from instance" " %s", instance.name) # Restore changed parameters for an atomicity for restore_param in cgparams[0:i]: try: self._SetCgroupInstanceValue(instance.name, restore_param, str(current_mem_usage)) except EnvironmentError as restore_err: logging.warn("Can't restore the cgroup parameter %s of %s: %s", restore_param, instance.name, restore_err) raise HypervisorError("Failed to balloon the memory of %s, can't set" " cgroup parameter %s: %s" % (instance.name, cgparam, err)) def GetNodeInfo(self, hvparams=None): """Return information about the node. See L{BaseHypervisor.GetLinuxNodeInfo}. """ return self.GetLinuxNodeInfo() @classmethod def GetInstanceConsole(cls, instance, primary_node, node_group, hvparams, beparams): """Return a command for connecting to the console of an instance. """ ndparams = node_group.FillND(primary_node) return objects.InstanceConsole(instance=instance.name, kind=constants.CONS_SSH, host=primary_node.name, port=ndparams.get(constants.ND_SSH_PORT), user=constants.SSH_CONSOLE_USER, command=["lxc-console", "-n", instance.name]) @classmethod def _GetLXCVersionFromCmd(cls, from_cmd): """Return the LXC version currently used in the system. Version information will be retrieved by command specified by from_cmd. @param from_cmd: the lxc command used to retrieve version information @type from_cmd: string @rtype: L{LXCVersion} @return: a version object which represents the version retrieved from the command """ result = utils.RunCmd([from_cmd, "--version"]) if result.failed: raise HypervisorError("Failed to get version info from command %s: %s" % (from_cmd, result.output)) try: return LXCVersion(result.stdout.strip()) except ValueError as err: raise HypervisorError("Can't parse LXC version from %s: %s" % (from_cmd, err)) @classmethod def _VerifyLXCCommands(cls): """Verify the validity of lxc command line tools. @rtype: list(str) @return: list of problem descriptions. the blank list will be returned if there is no problem. """ msgs = [] for cmd in cls._LXC_COMMANDS_REQUIRED: try: # lxc-ls needs special checking procedure. # there are two different version of lxc-ls, one is written in python # and the other is written in shell script. # we have to ensure the python version of lxc-ls is installed. if cmd == "lxc-ls": help_string = utils.RunCmd(["lxc-ls", "--help"]).output if "--running" not in help_string: # shell script version has no --running switch msgs.append("The python version of 'lxc-ls' is required." " Maybe lxc was installed without --enable-python") else: try: version = cls._GetLXCVersionFromCmd(cmd) except HypervisorError as err: msgs.append(str(err)) continue if version < cls._LXC_MIN_VERSION_REQUIRED: msgs.append("LXC version >= %s is required but command %s has" " version %s" % (cls._LXC_MIN_VERSION_REQUIRED, cmd, version)) except errors.OpExecError: msgs.append("Required command %s not found" % cmd) return msgs def Verify(self, hvparams=None): """Verify the hypervisor. For the LXC manager, it just checks the existence of the base dir. @type hvparams: dict of strings @param hvparams: hypervisor parameters to be verified against; not used here @return: Problem description if something is wrong, C{None} otherwise """ msgs = [] if not os.path.exists(self._ROOT_DIR): msgs.append("The required directory '%s' does not exist" % self._ROOT_DIR) try: self._EnsureCgroupMounts(hvparams) except errors.HypervisorError as err: msgs.append(str(err)) msgs.extend(self._VerifyLXCCommands()) return self._FormatVerifyResults(msgs) @classmethod def PowercycleNode(cls, hvparams=None): """LXC powercycle, just a wrapper over Linux powercycle. @type hvparams: dict of strings @param hvparams: hypervisor params to be used on this node """ cls.LinuxPowercycle() def MigrateInstance(self, cluster_name, instance, target, live): """Migrate an instance. @type cluster_name: string @param cluster_name: name of the cluster @type instance: L{objects.Instance} @param instance: the instance to be migrated @type target: string @param target: hostname (usually ip) of the target node @type live: boolean @param live: whether to do a live or non-live migration """ raise HypervisorError("Migration is not supported by the LXC hypervisor") def GetMigrationStatus(self, instance): """Get the migration status @type instance: L{objects.Instance} @param instance: the instance that is being migrated @rtype: L{objects.MigrationStatus} @return: the status of the current migration (one of L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional progress info that can be retrieved from the hypervisor """ raise HypervisorError("Migration is not supported by the LXC hypervisor") ganeti-3.1.0~rc2/lib/hypervisor/hv_xen.py000064400000000000000000001555401476477700300204370ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Xen hypervisors """ import logging import errno import os import string # pylint: disable=W0402 import shutil import time from io import StringIO from ganeti import constants from ganeti import errors from ganeti import utils from ganeti.hypervisor import hv_base from ganeti import netutils from ganeti import objects from ganeti import pathutils XL_CONFIG_FILE = utils.PathJoin(pathutils.XEN_CONFIG_DIR, "xen/xl.conf") VIF_BRIDGE_SCRIPT = utils.PathJoin(pathutils.XEN_CONFIG_DIR, "scripts/vif-bridge") _DOM0_NAME = "Domain-0" _DISK_LETTERS = string.ascii_lowercase XEN_COMMAND = "xl" _FILE_DRIVER_MAP = { constants.FD_LOOP: "file", constants.FD_BLKTAP: "tap:aio", constants.FD_BLKTAP2: "tap2:tapdisk:aio", } def _CreateConfigCpus(cpu_mask): """Create a CPU config string for Xen's config file. """ # Convert the string CPU mask to a list of list of int's cpu_list = utils.ParseMultiCpuMask(cpu_mask) if len(cpu_list) == 1: all_cpu_mapping = cpu_list[0] if all_cpu_mapping == constants.CPU_PINNING_OFF: # If CPU pinning has 1 entry that's "all", then remove the # parameter from the config file return None else: # If CPU pinning has one non-all entry, mapping all vCPUS (the entire # VM) to one physical CPU, using format 'cpu = "C"' return "cpu = \"%s\"" % ",".join(map(str, all_cpu_mapping)) else: def _GetCPUMap(vcpu): if vcpu[0] == constants.CPU_PINNING_ALL_VAL: cpu_map = constants.CPU_PINNING_ALL_XEN else: cpu_map = ",".join(map(str, vcpu)) return "\"%s\"" % cpu_map # build the result string in format 'cpus = [ "c", "c", "c" ]', # where each c is a physical CPU number, a range, a list, or any # combination return "cpus = [ %s ]" % ", ".join(map(_GetCPUMap, cpu_list)) def _RunInstanceList(fn, instance_list_errors): """Helper function for L{_GetAllInstanceList} to retrieve the list of instances from xen. @type fn: callable @param fn: Function to query xen for the list of instances @type instance_list_errors: list @param instance_list_errors: Error list @rtype: list """ result = fn() if result.failed: logging.error("Retrieving the instance list from xen failed (%s): %s", result.fail_reason, result.output) instance_list_errors.append(result) raise utils.RetryAgain() # skip over the heading return result.stdout.splitlines() class _InstanceCrashed(errors.GenericError): """Instance has reached a violent ending. This is raised within the Xen hypervisor only, and should not be seen or used outside. """ def _ParseInstanceList(lines, include_node): """Parses the output of listing instances by xen. @type lines: list @param lines: Result of retrieving the instance list from xen @type include_node: boolean @param include_node: If True, return information for Dom0 @return: list of tuple containing (name, id, memory, vcpus, state, time spent) """ result = [] # Iterate through all lines while ignoring header for line in lines[1:]: # The format of lines is: # Name ID Mem(MiB) VCPUs State Time(s) # Domain-0 0 3418 4 r----- 266.2 data = line.split() if len(data) != 6: raise errors.HypervisorError("Can't parse instance list," " line: %s" % line) try: # TODO: Cleanup this mess - introduce a namedtuple/dict/class data[1] = int(data[1]) data[2] = int(data[2]) data[3] = int(data[3]) data[4] = _XenToHypervisorInstanceState(data[4]) data[5] = float(data[5]) except (TypeError, ValueError) as err: raise errors.HypervisorError("Can't parse instance list," " line: %s, error: %s" % (line, err)) except _InstanceCrashed: # The crashed instance can be interpreted as being down, so we omit it # from the instance list. continue # skip the Domain-0 (optional) if include_node or data[0] != _DOM0_NAME: result.append(data) return result def _InstanceDomID(info): """Get instance domain ID from instance info tuple. @type info: tuple @param info: instance info as parsed by _ParseInstanceList() @return: int, instance domain ID """ return info[1] def _InstanceRunning(info): """Get instance runtime from instance info tuple. @type info: tuple @param info: instance info as parsed by _ParseInstanceList() @return: bool """ return info[4] == hv_base.HvInstanceState.RUNNING def _InstanceRuntime(info): """Get instance runtime from instance info tuple. @type info: tuple @param info: instance info as parsed by _ParseInstanceList() @return: float value of instance runtime """ return info[5] def _GetAllInstanceList(fn, include_node, delays, timeout): """Return the list of instances including running and shutdown. See L{_RunInstanceList} and L{_ParseInstanceList} for parameter details. """ instance_list_errors = [] try: lines = utils.Retry(_RunInstanceList, delays, timeout, args=(fn, instance_list_errors)) except utils.RetryTimeout: if instance_list_errors: instance_list_result = instance_list_errors.pop() errmsg = ("listing instances failed, timeout exceeded (%s): %s" % (instance_list_result.fail_reason, instance_list_result.output)) else: errmsg = "listing instances failed" raise errors.HypervisorError(errmsg) return _ParseInstanceList(lines, include_node) def _IsInstanceRunning(instance_info): """Determine whether an instance is running. An instance is running if it is in the following Xen states: running, blocked, paused, or dying (about to be destroyed / shutdown). For some strange reason, Xen once printed 'rb----' which does not make any sense because an instance cannot be both running and blocked. Fortunately, for Ganeti 'running' or 'blocked' is the same as 'running'. A state of nothing '------' means that the domain is runnable but it is not currently running. That means it is in the queue behind other domains waiting to be scheduled to run. http://old-list-archives.xenproject.org/xen-users/2007-06/msg00849.html A dying instance is about to be removed, but it is still consuming resources, and counts as running. @type instance_info: string @param instance_info: Information about instance, as supplied by Xen. @rtype: bool @return: Whether an instance is running. """ allowable_running_prefixes = [ "r--", "rb-", "-b-", "---", ] def _RunningWithSuffix(suffix): return [x + suffix for x in allowable_running_prefixes] # The shutdown suspend ("ss") state is encountered during migration, where # the instance is still considered to be running. # The shutdown restart ("sr") is probably encountered during restarts - still # running. # See Xen commit e1475a6693aac8cddc4bdd456548aa05a625556b return instance_info in _RunningWithSuffix("---") \ or instance_info in _RunningWithSuffix("ss-") \ or instance_info in _RunningWithSuffix("sr-") \ or instance_info == "-----d" def _IsInstanceShutdown(instance_info): """Determine whether the instance is shutdown. An instance is shutdown when a user shuts it down from within, and we do not remove domains to be able to detect that. The dying state has been added as a precaution, as Xen's status reporting is weird. """ return instance_info == "---s--" \ or instance_info == "---s-d" def _IgnorePaused(instance_info): """Removes information about whether a Xen state is paused from the state. As it turns out, an instance can be reported as paused in almost any condition. Paused instances can be paused, running instances can be paused for scheduling, and any other condition can appear to be paused as a result of races or improbable conditions in Xen's status reporting. As we do not use Xen's pause commands in any way at the time, we can simply ignore the paused field and save ourselves a lot of trouble. Should we ever use the pause commands, several samples would be needed before we could confirm the domain as paused. """ return instance_info.replace('p', '-') def _IsCrashed(instance_info): """Returns whether an instance is in the crashed Xen state. When a horrible misconfiguration happens to a Xen domain, it can crash, meaning that it encounters a violent ending. While this state usually flashes only temporarily before the domain is restarted, being able to check for it allows Ganeti not to act confused and do something about it. """ return instance_info.count('c') > 0 def _XenToHypervisorInstanceState(instance_info): """Maps Xen states to hypervisor states. @type instance_info: string @param instance_info: Information about instance, as supplied by Xen. @rtype: L{hv_base.HvInstanceState} """ instance_info = _IgnorePaused(instance_info) if _IsCrashed(instance_info): raise _InstanceCrashed("Instance detected as crashed, should be omitted") if _IsInstanceRunning(instance_info): return hv_base.HvInstanceState.RUNNING elif _IsInstanceShutdown(instance_info): return hv_base.HvInstanceState.SHUTDOWN else: raise errors.HypervisorError("hv_xen._XenToHypervisorInstanceState:" " unhandled Xen instance state '%s'" % instance_info) def _GetRunningInstanceList(fn, include_node, delays, timeout): """Return the list of running instances. See L{_GetAllInstanceList} for parameter details. """ instances = _GetAllInstanceList(fn, include_node, delays, timeout) return [i for i in instances if hv_base.HvInstanceState.IsRunning(i[4])] def _GetShutdownInstanceList(fn, include_node, delays, timeout): """Return the list of shutdown instances. See L{_GetAllInstanceList} for parameter details. """ instances = _GetAllInstanceList(fn, include_node, delays, timeout) return [i for i in instances if hv_base.HvInstanceState.IsShutdown(i[4])] def _ParseNodeInfo(info): """Return information about the node. @return: a dict with the following keys (memory values in MiB): - memory_total: the total memory size on the node - memory_free: the available memory on the node for instances - nr_cpus: total number of CPUs - nr_nodes: in a NUMA system, the number of domains - nr_sockets: the number of physical CPU sockets in the node - hv_version: the hypervisor version in the form (major, minor) """ result = {} cores_per_socket = threads_per_core = nr_cpus = None xen_major, xen_minor = None, None memory_total = None memory_free = None for line in info.splitlines(): fields = line.split(":", 1) if len(fields) < 2: continue (key, val) = [s.strip() for s in fields] # Note: in Xen 3, memory has changed to total_memory if key in ("memory", "total_memory"): memory_total = int(val) elif key == "free_memory": memory_free = int(val) elif key == "nr_cpus": nr_cpus = result["cpu_total"] = int(val) elif key == "nr_nodes": result["cpu_nodes"] = int(val) elif key == "cores_per_socket": cores_per_socket = int(val) elif key == "threads_per_core": threads_per_core = int(val) elif key == "xen_major": xen_major = int(val) elif key == "xen_minor": xen_minor = int(val) if None not in [cores_per_socket, threads_per_core, nr_cpus]: result["cpu_sockets"] = nr_cpus // (cores_per_socket * threads_per_core) if memory_free is not None: result["memory_free"] = memory_free if memory_total is not None: result["memory_total"] = memory_total if not (xen_major is None or xen_minor is None): result[constants.HV_NODEINFO_KEY_VERSION] = (xen_major, xen_minor) return result def _MergeInstanceInfo(info, instance_list): """Updates node information from L{_ParseNodeInfo} with instance info. @type info: dict @param info: Result from L{_ParseNodeInfo} @type instance_list: list of tuples @param instance_list: list of instance information; one tuple per instance @rtype: dict """ total_instmem = 0 for (name, _, mem, vcpus, _, _) in instance_list: if name == _DOM0_NAME: info["memory_dom0"] = mem info["cpu_dom0"] = vcpus # Include Dom0 in total memory usage total_instmem += mem memory_free = info.get("memory_free") memory_total = info.get("memory_total") # Calculate memory used by hypervisor if None not in [memory_total, memory_free, total_instmem]: info["memory_hv"] = memory_total - memory_free - total_instmem return info def _GetNodeInfo(info, instance_list): """Combines L{_MergeInstanceInfo} and L{_ParseNodeInfo}. @type instance_list: list of tuples @param instance_list: list of instance information; one tuple per instance """ return _MergeInstanceInfo(_ParseNodeInfo(info), instance_list) def _GetConfigFileDiskData(block_devices, blockdev_prefix, _letters=_DISK_LETTERS): """Get disk directives for Xen config file. This method builds the xen config disk directive according to the given disk_template and block_devices. @param block_devices: list of tuples (cfdev, rldev): - cfdev: dict containing ganeti config disk part - rldev: ganeti.block.bdev.BlockDev object @param blockdev_prefix: a string containing blockdevice prefix, e.g. "sd" for /dev/sda @return: string containing disk directive for xen instance config file """ if len(block_devices) > len(_letters): raise errors.HypervisorError("Too many disks") disk_data = [] for sd_suffix, (cfdev, dev_path, _) in zip(_letters, block_devices): sd_name = blockdev_prefix + sd_suffix if cfdev.mode == constants.DISK_RDWR: mode = "w" else: mode = "r" if cfdev.dev_type in constants.DTS_FILEBASED: driver = _FILE_DRIVER_MAP[cfdev.logical_id[0]] else: driver = "phy" disk_data.append("'%s:%s,%s,%s'" % (driver, dev_path, sd_name, mode)) return disk_data def _QuoteCpuidField(data): """Add quotes around the CPUID field only if necessary. Xen CPUID fields come in two shapes: LIBXL strings, which need quotes around them, and lists of XEND strings, which don't. @param data: Either type of parameter. @return: The quoted version thereof. """ return "'%s'" % data if data.startswith("host") else data def _ConfigureNIC(instance, seq, nic, tap): """Run the network configuration script for a specified NIC See L{hv_base.ConfigureNIC}. @type instance: instance object @param instance: instance we're acting on @type seq: int @param seq: nic sequence number @type nic: nic object @param nic: nic we're acting on @type tap: str @param tap: the host's tap interface this NIC corresponds to """ hv_base.ConfigureNIC(pathutils.XEN_IFUP_OS, instance, seq, nic, tap) class XenHypervisor(hv_base.BaseHypervisor): """Xen generic hypervisor interface This is the Xen base class used for both Xen PVM and HVM. It contains all the functionality that is identical for both. """ CAN_MIGRATE = True REBOOT_RETRY_COUNT = 60 REBOOT_RETRY_INTERVAL = 10 _ROOT_DIR = pathutils.RUN_DIR + "/xen-hypervisor" # contains NICs' info _NICS_DIR = _ROOT_DIR + "/nic" # contains the pidfiles of socat processes used to migrate instaces under xl _MIGRATION_DIR = _ROOT_DIR + "/migration" _DIRS = [_ROOT_DIR, _NICS_DIR, _MIGRATION_DIR] _INSTANCE_LIST_DELAYS = (0.3, 1.5, 1.0) _INSTANCE_LIST_TIMEOUT = 5 ANCILLARY_FILES = [ XL_CONFIG_FILE, VIF_BRIDGE_SCRIPT, ] ANCILLARY_FILES_OPT = [ XL_CONFIG_FILE, ] def __init__(self, _cfgdir=None, _run_cmd_fn=None, _cmd=None): hv_base.BaseHypervisor.__init__(self) if _cfgdir is None: self._cfgdir = pathutils.XEN_CONFIG_DIR else: self._cfgdir = _cfgdir if _run_cmd_fn is None: self._run_cmd_fn = utils.RunCmd else: self._run_cmd_fn = _run_cmd_fn self._cmd = _cmd def _RunXen(self, args, timeout=None): """Wrapper around L{utils.process.RunCmd} to run Xen command. @type timeout: int or None @param timeout: if a timeout (in seconds) is specified, the command will be terminated after that number of seconds. @see: L{utils.process.RunCmd} """ cmd = [] if timeout is not None: cmd.extend(["timeout", str(timeout)]) cmd.extend([XEN_COMMAND]) cmd.extend(args) return self._run_cmd_fn(cmd) def _ConfigFileName(self, instance_name): """Get the config file name for an instance. @param instance_name: instance name @type instance_name: str @return: fully qualified path to instance config file @rtype: str """ return utils.PathJoin(self._cfgdir, instance_name) @classmethod def _EnsureDirs(cls, extra_dirs=None): """Makes sure that the directories needed by the hypervisor exist. @type extra_dirs: list of string or None @param extra_dirs: Additional directories which ought to exist. """ if extra_dirs is None: extra_dirs = [] dirs = [(dname, constants.RUN_DIRS_MODE) for dname in (cls._DIRS + extra_dirs)] utils.EnsureDirs(dirs) @classmethod def _WriteNICInfoFile(cls, instance, idx, nic): """Write the Xen config file for the instance. This version of the function just writes the config file from static data. """ instance_name = instance.name cls._EnsureDirs(extra_dirs=[cls._InstanceNICDir(instance_name)]) cfg_file = cls._InstanceNICFile(instance_name, idx) data = StringIO() data.write("TAGS=\"%s\"\n" % r"\ ".join(instance.GetTags())) if nic.netinfo: netinfo = objects.Network.FromDict(nic.netinfo) for k, v in netinfo.HooksDict().items(): data.write("%s=\"%s\"\n" % (k, v)) data.write("MAC=%s\n" % nic.mac) if nic.ip: data.write("IP=%s\n" % nic.ip) data.write("INTERFACE_INDEX=%s\n" % str(idx)) if nic.name: data.write("INTERFACE_NAME=%s\n" % nic.name) data.write("INTERFACE_UUID=%s\n" % nic.uuid) data.write("MODE=%s\n" % nic.nicparams[constants.NIC_MODE]) data.write("LINK=%s\n" % nic.nicparams[constants.NIC_LINK]) data.write("VLAN=%s\n" % nic.nicparams[constants.NIC_VLAN]) try: utils.WriteFile(cfg_file, data=data.getvalue()) except EnvironmentError as err: raise errors.HypervisorError("Cannot write Xen instance configuration" " file %s: %s" % (cfg_file, err)) @staticmethod def VersionsSafeForMigration(src, target): """Decide if migration is likely to suceed for hypervisor versions. Given two versions of a hypervisor, give a guess whether live migration from the one version to the other version is likely to succeed. For Xen, the heuristics is, that an increase by one on the second digit is OK. This fits with the current numbering scheme. @type src: list or tuple @type target: list or tuple @rtype: bool """ if src == target: return True if len(src) < 2 or len(target) < 2: return False return src[0] == target[0] and target[1] in [src[1], src[1] + 1] @classmethod def _InstanceNICDir(cls, instance_name): """Returns the directory holding the tap device files for a given instance. """ return utils.PathJoin(cls._NICS_DIR, instance_name) @classmethod def _InstanceNICFile(cls, instance_name, seq): """Returns the name of the file containing the tap device for a given NIC """ return utils.PathJoin(cls._InstanceNICDir(instance_name), str(seq)) @classmethod def _InstanceMigrationPidfile(cls, _instance_name): """Returns the name of the pid file for a socat process used to migrate. """ #TODO(riba): At the moment, we are using a single pidfile because we # use a single port for migrations at the moment. This is because we do not # allow more migrations, so dynamic port selection and the needed port # modifications are not needed. # The _instance_name parameter has been left here for future use. return utils.PathJoin(cls._MIGRATION_DIR, constants.XL_MIGRATION_PIDFILE) @classmethod def _GetConfig(cls, instance, startup_memory, block_devices): """Build Xen configuration for an instance. """ raise NotImplementedError def _WriteNicConfig(self, config, instance, hvp): vif_data = [] # only XenHvmHypervisor has these hvparams nic_type = hvp.get(constants.HV_NIC_TYPE, None) vif_type = hvp.get(constants.HV_VIF_TYPE, None) nic_type_str = "" if nic_type or vif_type: if nic_type is None: if vif_type: nic_type_str = ", type=%s" % vif_type elif nic_type == constants.HT_NIC_PARAVIRTUAL: nic_type_str = ", type=%s" % constants.HT_HVM_VIF_VIF else: # parameter 'model' is only valid with type 'ioemu' nic_type_str = ", model=%s, type=%s" % \ (nic_type, constants.HT_HVM_VIF_IOEMU) for idx, nic in enumerate(instance.nics): nic_args = {} nic_args["mac"] = "%s%s" % (nic.mac, nic_type_str) if nic.name and \ nic.name.startswith(constants.INSTANCE_COMMUNICATION_NIC_PREFIX): tap = hv_base.GenerateTapName() nic_args["vifname"] = tap nic_args["script"] = pathutils.XEN_VIF_METAD_SETUP nic.name = tap else: ip = getattr(nic, "ip", None) if ip is not None: nic_args["ip"] = ip if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: nic_args["bridge"] = nic.nicparams[constants.NIC_LINK] elif nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_OVS: nic_args["bridge"] = nic.nicparams[constants.NIC_LINK] if nic.nicparams[constants.NIC_VLAN]: nic_args["bridge"] += nic.nicparams[constants.NIC_VLAN] if hvp[constants.HV_VIF_SCRIPT]: nic_args["script"] = hvp[constants.HV_VIF_SCRIPT] nic_str = ", ".join(["%s=%s" % p for p in nic_args.items()]) vif_data.append("'%s'" % (nic_str, )) self._WriteNICInfoFile(instance, idx, nic) config.write("vif = [%s]\n" % ",".join(vif_data)) def _WriteConfigFile(self, instance_name, data): """Write the Xen config file for the instance. This version of the function just writes the config file from static data. """ cfg_file = self._ConfigFileName(instance_name) # just in case it exists utils.RemoveFile(utils.PathJoin(self._cfgdir, "auto", instance_name)) try: utils.WriteFile(cfg_file, data=data) except EnvironmentError as err: raise errors.HypervisorError("Cannot write Xen instance configuration" " file %s: %s" % (cfg_file, err)) def _ReadConfigFile(self, instance_name): """Returns the contents of the instance config file. """ filename = self._ConfigFileName(instance_name) try: file_content = utils.ReadFile(filename) except EnvironmentError as err: raise errors.HypervisorError("Failed to load Xen config file: %s" % err) return file_content def _RemoveConfigFile(self, instance_name): """Remove the xen configuration file. """ utils.RemoveFile(self._ConfigFileName(instance_name)) try: shutil.rmtree(self._InstanceNICDir(instance_name)) except OSError as err: if err.errno != errno.ENOENT: raise def _StashConfigFile(self, instance_name): """Move the Xen config file to the log directory and return its new path. """ old_filename = self._ConfigFileName(instance_name) base = ("%s-%s" % (instance_name, utils.TimestampForFilename())) new_filename = utils.PathJoin(pathutils.LOG_XEN_DIR, base) utils.RenameFile(old_filename, new_filename) return new_filename def _GetInstanceList(self, include_node): """Wrapper around module level L{_GetAllInstanceList}. """ return _GetAllInstanceList(lambda: self._RunXen(["list"]), include_node, delays=self._INSTANCE_LIST_DELAYS, timeout=self._INSTANCE_LIST_TIMEOUT) def ListInstances(self, hvparams=None): """Get the list of running instances. @type hvparams: dict of strings @param hvparams: the instance's hypervisor params @rtype: list of strings @return: names of running instances """ instance_list = _GetRunningInstanceList( lambda: self._RunXen(["list"]), False, delays=self._INSTANCE_LIST_DELAYS, timeout=self._INSTANCE_LIST_TIMEOUT) return [info[0] for info in instance_list] def GetInstanceInfo(self, instance_name, hvparams=None): """Get instance properties. @type instance_name: string @param instance_name: the instance name @type hvparams: dict of strings @param hvparams: the instance's hypervisor params @return: tuple (name, id, memory, vcpus, stat, times) """ instance_list = self._GetInstanceList(instance_name == _DOM0_NAME) for data in instance_list: if data[0] == instance_name: return data return None def GetAllInstancesInfo(self, hvparams=None): """Get properties of all instances. @type hvparams: dict of strings @param hvparams: hypervisor parameters @rtype: (string, string, int, int, HypervisorInstanceState, int) @return: list of tuples (name, id, memory, vcpus, state, times) """ return self._GetInstanceList(False) def _MakeConfigFile(self, instance, startup_memory, block_devices): """Gather configuration details and write to disk. See L{_GetConfig} for arguments. """ buf = StringIO() buf.write("# Automatically generated by Ganeti. Do not edit!\n") buf.write("\n") buf.write(self._GetConfig(instance, startup_memory, block_devices)) buf.write("\n") self._WriteConfigFile(instance.name, buf.getvalue()) def VerifyInstance(self, instance): """Verify if running instance (configuration) is in correct state. @type instance: L{objects.Instance} @param instance: instance to verify @return: bool, if instance in correct state """ config_file = utils.PathJoin(self._cfgdir, "auto", instance.name) return os.path.exists(config_file) def RestoreInstance(self, instance, block_devices): """Fixup running instance's state. @type instance: L{objects.Instance} @param instance: instance to restore @type block_devices: list of tuples (disk_object, link_name, drive_uri) @param block_devices: blockdevices assigned to this instance """ startup_memory = self._InstanceStartupMemory(instance) self._MakeConfigFile(instance, startup_memory, block_devices) def StartInstance(self, instance, block_devices, startup_paused): """Start an instance. @type instance: L{objects.Instance} @param instance: instance to start @type block_devices: list of tuples (cfdev, rldev) - cfdev: dict containing ganeti config disk part - rldev: ganeti.block.bdev.BlockDev object @param block_devices: blockdevices assigned to this instance @type startup_paused: bool @param startup_paused: if instance should be paused at startup """ startup_memory = self._InstanceStartupMemory(instance) self._MakeConfigFile(instance, startup_memory, block_devices) cmd = ["create"] if startup_paused: cmd.append("-p") cmd.append(self._ConfigFileName(instance.name)) result = self._RunXen(cmd) if result.failed: # Move the Xen configuration file to the log directory to avoid # leaving a stale config file behind. stashed_config = self._StashConfigFile(instance.name) raise errors.HypervisorError("Failed to start instance %s: %s (%s). Moved" " config file to %s" % (instance.name, result.fail_reason, result.output, stashed_config)) for nic_seq, nic in enumerate(instance.nics): if nic.name and nic.name.startswith("gnt.com."): _ConfigureNIC(instance, nic_seq, nic, nic.name) def StopInstance(self, instance, force=False, retry=False, name=None, timeout=None): """Stop an instance. A soft shutdown can be interrupted. A hard shutdown tries forever. """ assert(timeout is None or force is not None) if name is None: name = instance.name return self._StopInstance(name, force, instance.hvparams, timeout) def _ShutdownInstance(self, name, hvparams, timeout): """Shutdown an instance if the instance is running. The '-w' flag waits for shutdown to complete which avoids the need to poll in the case where we want to destroy the domain immediately after shutdown. @type name: string @param name: name of the instance to stop @type hvparams: dict of string @param hvparams: hypervisor parameters of the instance @type timeout: int or None @param timeout: a timeout after which the shutdown command should be killed, or None for no timeout """ info = self.GetInstanceInfo(name, hvparams=hvparams) if info is None or hv_base.HvInstanceState.IsShutdown(info[4]): logging.info("Failed to shutdown instance %s, not running", name) return None return self._RunXen(["shutdown", "-w", name], timeout) def _DestroyInstance(self, name, hvparams): """Destroy an instance if the instance exists. @type name: string @param name: name of the instance to destroy @type hvparams: dict of string @param hvparams: hypervisor parameters of the instance """ instance_info = self.GetInstanceInfo(name, hvparams=hvparams) if instance_info is None: logging.info("Failed to destroy instance %s, does not exist", name) return None return self._RunXen(["destroy", name]) # Destroy a domain only if necessary # # This method checks if the domain has already been destroyed before # issuing the 'destroy' command. This step is necessary to handle # domains created by other versions of Ganeti. For example, an # instance created with 2.10 will be destroy by the # '_ShutdownInstance', thus not requiring an additional destroy, # which would cause an error if issued. See issue 619. def _DestroyInstanceIfAlive(self, name, hvparams): instance_info = self.GetInstanceInfo(name, hvparams=hvparams) if instance_info is None: raise errors.HypervisorError("Failed to destroy instance %s, already" " destroyed" % name) else: self._DestroyInstance(name, hvparams) def _RenameInstance(self, old_name, new_name): """Rename an instance (domain). @type old_name: string @param old_name: current name of the instance @type new_name: string @param new_name: future (requested) name of the instace """ return self._RunXen(["rename", old_name, new_name]) def _StopInstance(self, name, force, hvparams, timeout): """Stop an instance. @type name: string @param name: name of the instance to destroy @type force: boolean @param force: whether to do a "hard" stop (destroy) @type hvparams: dict of string @param hvparams: hypervisor parameters of the instance @type timeout: int or None @param timeout: a timeout after which the shutdown command should be killed, or None for no timeout """ instance_info = self.GetInstanceInfo(name, hvparams=hvparams) if instance_info is None: raise errors.HypervisorError("Failed to shutdown instance %s," " not running" % name) if not force: self._ShutdownInstance(name, hvparams, timeout) # TODO: Xen always destroys the instance after trying a graceful shutdown. # That means doing another attempt with force=True will not make any # difference. This differs in behaviour from other hypervisors and should # be cleaned up. result = self._DestroyInstanceIfAlive(name, hvparams) if result is not None and result.failed and \ self.GetInstanceInfo(name, hvparams=hvparams) is not None: raise errors.HypervisorError("Failed to stop instance %s: %s, %s" % (name, result.fail_reason, result.output)) # Remove configuration file if stopping/starting instance was successful self._RemoveConfigFile(name) def RebootInstance(self, instance): """Reboot an instance. """ ini_info = self.GetInstanceInfo(instance.name, hvparams=instance.hvparams) if ini_info is None: raise errors.HypervisorError("Failed to reboot instance %s," " not running" % instance.name) result = self._RunXen(["reboot", instance.name]) if result.failed: raise errors.HypervisorError("Failed to reboot instance %s: %s, %s" % (instance.name, result.fail_reason, result.output)) def _CheckInstance(): new_info = self.GetInstanceInfo(instance.name, hvparams=instance.hvparams) # check if the domain ID has changed or the run time has decreased if (new_info is not None and (_InstanceDomID(new_info) != _InstanceDomID(ini_info) or ( _InstanceRuntime(new_info) < _InstanceRuntime(ini_info)))): return raise utils.RetryAgain() try: utils.Retry(_CheckInstance, self.REBOOT_RETRY_INTERVAL, self.REBOOT_RETRY_INTERVAL * self.REBOOT_RETRY_COUNT) except utils.RetryTimeout: raise errors.HypervisorError("Failed to reboot instance %s: instance" " did not reboot in the expected interval" % (instance.name, )) def BalloonInstanceMemory(self, instance, mem): """Balloon an instance memory to a certain value. @type instance: L{objects.Instance} @param instance: instance to be accepted @type mem: int @param mem: actual memory size to use for instance runtime """ result = self._RunXen(["mem-set", instance.name, mem]) if result.failed: raise errors.HypervisorError("Failed to balloon instance %s: %s (%s)" % (instance.name, result.fail_reason, result.output)) # Update configuration file cmd = ["sed", "-ie", "s/^memory.*$/memory = %s/" % mem] cmd.append(self._ConfigFileName(instance.name)) result = utils.RunCmd(cmd) if result.failed: raise errors.HypervisorError("Failed to update memory for %s: %s (%s)" % (instance.name, result.fail_reason, result.output)) def GetNodeInfo(self, hvparams=None): """Return information about the node. @see: L{_GetNodeInfo} and L{_ParseNodeInfo} """ result = self._RunXen(["info"]) if result.failed: logging.error("Can't retrieve xen hypervisor information (%s): %s", result.fail_reason, result.output) return None instance_list = self._GetInstanceList(True) return _GetNodeInfo(result.stdout, instance_list) @classmethod def GetInstanceConsole(cls, instance, primary_node, node_group, hvparams, beparams): """Return a command for connecting to the console of an instance. """ ndparams = node_group.FillND(primary_node) return objects.InstanceConsole(instance=instance.name, kind=constants.CONS_SSH, host=primary_node.name, port=ndparams.get(constants.ND_SSH_PORT), user=constants.SSH_CONSOLE_USER, command=[pathutils.XEN_CONSOLE_WRAPPER, XEN_COMMAND, instance.name]) def Verify(self, hvparams=None): """Verify the hypervisor. For Xen, this verifies that the XL toolstack is present and functional @type hvparams: dict of strings @param hvparams: hypervisor parameters to be verified against @return: Problem description if something is wrong, C{None} otherwise """ if hvparams is None: return "Could not verify the hypervisor, because no hvparams were" \ " provided." try: self._CheckToolstackXlConfigured() except errors.HypervisorError: return "The xen toolstack 'xl' is not available on this node." result = self._RunXen(["info"]) if result.failed: return "Retrieving information from xen failed: %s, %s" % \ (result.fail_reason, result.output) return None def MigrationInfo(self, instance): """Get instance information to perform a migration. @type instance: L{objects.Instance} @param instance: instance to be migrated @rtype: string @return: content of the xen config file """ return self._ReadConfigFile(instance.name) @classmethod def _KillMigrationDaemon(cls, instance): """Kills the migration daemon if present. """ pidfile = cls._InstanceMigrationPidfile(instance.name) read_pid = utils.ReadPidFile(pidfile) # There is no pidfile, hence nothing for us to do if read_pid == 0: return if utils.IsProcessAlive(read_pid): # If the process is alive, let's make sure we are killing the right one cmdline = ' '.join(utils.GetProcCmdline(read_pid)) if cmdline.count("xl migrate-receive") > 0: utils.KillProcess(read_pid) # By this point the process is not running, whether killed or initially # nonexistent, so it is safe to remove the pidfile. utils.RemoveFile(pidfile) def AcceptInstance(self, instance, info, target): """Prepare to accept an instance. @type instance: L{objects.Instance} @param instance: instance to be accepted @type info: string @param info: content of the xen config file on the source node @type target: string @param target: target host (usually ip), on this node """ port = instance.hvparams[constants.HV_MIGRATION_PORT] # Make sure there is somewhere to put the pidfile. XenHypervisor._EnsureDirs() pidfile = XenHypervisor._InstanceMigrationPidfile(instance.name) # And try and kill a previous daemon XenHypervisor._KillMigrationDaemon(instance) listening_arg = "TCP-LISTEN:%d,bind=%s,reuseaddr" % (port, target) socat_pid = utils.StartDaemon(["socat", "-b524288", listening_arg, "SYSTEM:'xl migrate-receive'"], pidfile=pidfile) # Wait for a while to make sure the socat process has successfully started # listening time.sleep(1) if not utils.IsProcessAlive(socat_pid): raise errors.HypervisorError("Could not start receiving socat process" " on port %d: check if port is available" % port) def FinalizeMigrationDst(self, instance, config, success): """Finalize an instance migration. Write a config file if the instance is running on the destination node regardles if we think the migration succeeded or not. This will cover cases, when the migration succeeded but due to a timeout on the source node we think it failed. If we think the migration failed and there is an unstarted domain, clean it up. @type instance: L{objects.Instance} @param instance: instance whose migration is being finalized @type config: string @param config: content of the xen config file from the source node @type success: boolean @param success: whether the master node thinks the migration succeeded """ # We should recreate the config file if the domain is present and running, # regardless if we think the migration succeeded or not. info = self.GetInstanceInfo(instance.name, hvparams=instance.hvparams) if info and _InstanceRunning(info): self._WriteConfigFile(instance.name, config) if not success: XenHypervisor._KillMigrationDaemon(instance) # Fix the common failure when the domain was created but never started: # this happens if the memory transfer didn't complete and the instance # is running on the source node. if info and _InstanceRuntime(info) == 0: self._DestroyInstance(instance.name, instance.hvparams) def MigrateInstance(self, _cluster_name, instance, target, live): """Migrate an instance to a target node. The migration will not be attempted if the instance is not currently running. @type instance: L{objects.Instance} @param instance: the instance to be migrated @type target: string @param target: ip address of the target node @type live: boolean @param live: perform a live migration """ port = instance.hvparams[constants.HV_MIGRATION_PORT] return self._MigrateInstance(instance.name, target, port, instance.hvparams) def _MigrateInstance(self, instance_name, target, port, hvparams, _ping_fn=netutils.TcpPing): """Migrate an instance to a target node. @see: L{MigrateInstance} for details """ if hvparams is None: raise errors.HypervisorError("No hvparams provided.") if self.GetInstanceInfo(instance_name, hvparams=hvparams) is None: raise errors.HypervisorError("Instance not running, cannot migrate") args = ["migrate"] # Rather than using SSH, use socat as Ganeti cannot guarantee the presence # of usable SSH keys as of 2.13 args.extend([ "-s", constants.XL_SOCAT_CMD % (target, port), "-C", self._ConfigFileName(instance_name), ]) args.extend([instance_name, target]) result = self._RunXen(args) if result.failed: raise errors.HypervisorError("Failed to migrate instance %s: %s" % (instance_name, result.output)) def FinalizeMigrationSource(self, instance, success, _): """Finalize the instance migration on the source node. @type instance: L{objects.Instance} @param instance: the instance that was migrated @type success: bool @param success: whether the master thinks the migration succeeded """ # pylint: disable=W0613 if success: # Remove old xen file after migration succeeded # Note that _RemoveConfigFile silently succeeds if the file is already # deleted, that makes this function idempotent try: self._RemoveConfigFile(instance.name) except EnvironmentError: logging.exception("Failure while removing instance config file") else: # Cleanup the most common failure case when the source instance fails # to freeze and remains running renamed: '${oldname}--migratedaway' temp_name = instance.name + '--migratedaway' info = self.GetInstanceInfo(temp_name, hvparams=instance.hvparams) if info: self._RenameInstance(temp_name, instance.name) def GetMigrationStatus(self, instance): """Get the migration status As MigrateInstance for Xen is still blocking, if this method is called it means that MigrateInstance has completed successfully. So we can safely assume that the migration was successful and notify this fact to the client. @type instance: L{objects.Instance} @param instance: the instance that is being migrated @rtype: L{objects.MigrationStatus} @return: the status of the current migration (one of L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional progress info that can be retrieved from the hypervisor """ return objects.MigrationStatus(status=constants.HV_MIGRATION_COMPLETED) def PowercycleNode(self, hvparams=None): """Xen-specific powercycle. This first does a Linux reboot (which triggers automatically a Xen reboot), and if that fails it tries to do a Xen reboot. The reason we don't try a Xen reboot first is that the xen reboot launches an external command which connects to the Xen hypervisor, and that won't work in case the root filesystem is broken and/or the xend daemon is not working. @type hvparams: dict of strings @param hvparams: hypervisor params to be used on this node """ try: self.LinuxPowercycle() finally: utils.RunCmd([XEN_COMMAND, "debug", "R"]) def _CheckToolstackXlConfigured(self): """Checks whether xl is enabled on an xl-capable node. @rtype: bool @returns: C{True} if 'xl' is enabled, C{False} otherwise """ result = self._run_cmd_fn([XEN_COMMAND, "help"]) if not result.failed: return True elif result.failed: if "toolstack" in result.stderr: return False # xl fails for some other reason than the toolstack else: raise errors.HypervisorError("Cannot run xen ('%s'). Error: %s." % (XEN_COMMAND, result.stderr)) def WriteXenConfigEvents(config, hvp): config.write("on_poweroff = 'preserve'\n") if hvp[constants.HV_REBOOT_BEHAVIOR] == constants.INSTANCE_REBOOT_ALLOWED: config.write("on_reboot = 'restart'\n") else: config.write("on_reboot = 'destroy'\n") config.write("on_crash = 'restart'\n") class XenPvmHypervisor(XenHypervisor): """Xen PVM hypervisor interface""" PARAMETERS = { constants.HV_USE_BOOTLOADER: hv_base.NO_CHECK, constants.HV_BOOTLOADER_PATH: hv_base.OPT_FILE_CHECK, constants.HV_BOOTLOADER_ARGS: hv_base.NO_CHECK, constants.HV_KERNEL_PATH: hv_base.REQ_FILE_CHECK, constants.HV_INITRD_PATH: hv_base.OPT_FILE_CHECK, constants.HV_ROOT_PATH: hv_base.NO_CHECK, constants.HV_KERNEL_ARGS: hv_base.NO_CHECK, constants.HV_MIGRATION_PORT: hv_base.REQ_NET_PORT_CHECK, constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK, # TODO: Add a check for the blockdev prefix (matching [a-z:] or similar). constants.HV_BLOCKDEV_PREFIX: hv_base.NO_CHECK, constants.HV_REBOOT_BEHAVIOR: hv_base.ParamInSet(True, constants.REBOOT_BEHAVIORS), constants.HV_CPU_MASK: hv_base.OPT_MULTI_CPU_MASK_CHECK, constants.HV_CPU_CAP: hv_base.OPT_NONNEGATIVE_INT_CHECK, constants.HV_CPU_WEIGHT: (False, lambda x: 0 < x < 65536, "invalid weight", None, None), constants.HV_VIF_SCRIPT: hv_base.OPT_FILE_CHECK, constants.HV_XEN_CPUID: hv_base.NO_CHECK, constants.HV_SOUNDHW: hv_base.NO_CHECK, } def _GetConfig(self, instance, startup_memory, block_devices): """Write the Xen config file for the instance. """ hvp = instance.hvparams config = StringIO() config.write("# this is autogenerated by Ganeti, please do not edit\n#\n") # if bootloader is True, use bootloader instead of kernel and ramdisk # parameters. if hvp[constants.HV_USE_BOOTLOADER]: # bootloader handling bootloader_path = hvp[constants.HV_BOOTLOADER_PATH] if bootloader_path: config.write("bootloader = '%s'\n" % bootloader_path) else: raise errors.HypervisorError("Bootloader enabled, but missing" " bootloader path") bootloader_args = hvp[constants.HV_BOOTLOADER_ARGS] if bootloader_args: config.write("bootargs = '%s'\n" % bootloader_args) else: # kernel handling kpath = hvp[constants.HV_KERNEL_PATH] config.write("kernel = '%s'\n" % kpath) # initrd handling initrd_path = hvp[constants.HV_INITRD_PATH] if initrd_path: config.write("ramdisk = '%s'\n" % initrd_path) # rest of the settings config.write("memory = %d\n" % startup_memory) config.write("maxmem = %d\n" % instance.beparams[constants.BE_MAXMEM]) config.write("vcpus = %d\n" % instance.beparams[constants.BE_VCPUS]) cpu_pinning = _CreateConfigCpus(hvp[constants.HV_CPU_MASK]) if cpu_pinning: config.write("%s\n" % cpu_pinning) cpu_cap = hvp[constants.HV_CPU_CAP] if cpu_cap: config.write("cpu_cap=%d\n" % cpu_cap) cpu_weight = hvp[constants.HV_CPU_WEIGHT] if cpu_weight: config.write("cpu_weight=%d\n" % cpu_weight) config.write("name = '%s'\n" % instance.name) self._WriteNicConfig(config, instance, hvp) disk_data = \ _GetConfigFileDiskData(block_devices, hvp[constants.HV_BLOCKDEV_PREFIX]) config.write("disk = [%s]\n" % ",".join(disk_data)) if hvp[constants.HV_ROOT_PATH]: config.write("root = '%s'\n" % hvp[constants.HV_ROOT_PATH]) WriteXenConfigEvents(config, hvp) config.write("extra = '%s'\n" % hvp[constants.HV_KERNEL_ARGS]) cpuid = hvp[constants.HV_XEN_CPUID] if cpuid: config.write("cpuid = %s\n" % _QuoteCpuidField(cpuid)) if hvp[constants.HV_SOUNDHW]: config.write("soundhw = '%s'\n" % hvp[constants.HV_SOUNDHW]) return config.getvalue() class XenHvmHypervisor(XenHypervisor): """Xen HVM hypervisor interface""" ANCILLARY_FILES = XenHypervisor.ANCILLARY_FILES + [ pathutils.VNC_PASSWORD_FILE, ] ANCILLARY_FILES_OPT = XenHypervisor.ANCILLARY_FILES_OPT + [ pathutils.VNC_PASSWORD_FILE, ] PARAMETERS = { constants.HV_ACPI: hv_base.NO_CHECK, constants.HV_BOOT_ORDER: (True, ) + (lambda x: x and len(x.strip("acdn")) == 0, "Invalid boot order specified, must be one or more of [acdn]", None, None), constants.HV_CDROM_IMAGE_PATH: hv_base.OPT_FILE_CHECK, constants.HV_DISK_TYPE: hv_base.ParamInSet(True, constants.HT_HVM_VALID_DISK_TYPES), constants.HV_NIC_TYPE: hv_base.ParamInSet(True, constants.HT_HVM_VALID_NIC_TYPES), constants.HV_PAE: hv_base.NO_CHECK, constants.HV_VNC_BIND_ADDRESS: (False, netutils.IP4Address.IsValid, "VNC bind address is not a valid IP address", None, None), constants.HV_KERNEL_PATH: hv_base.REQ_FILE_CHECK, constants.HV_DEVICE_MODEL: hv_base.REQ_FILE_CHECK, constants.HV_VNC_PASSWORD_FILE: hv_base.REQ_FILE_CHECK, constants.HV_MIGRATION_PORT: hv_base.REQ_NET_PORT_CHECK, constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK, constants.HV_USE_LOCALTIME: hv_base.NO_CHECK, # TODO: Add a check for the blockdev prefix (matching [a-z:] or similar). constants.HV_BLOCKDEV_PREFIX: hv_base.NO_CHECK, # Add PCI passthrough constants.HV_PASSTHROUGH: hv_base.NO_CHECK, constants.HV_REBOOT_BEHAVIOR: hv_base.ParamInSet(True, constants.REBOOT_BEHAVIORS), constants.HV_CPU_MASK: hv_base.OPT_MULTI_CPU_MASK_CHECK, constants.HV_CPU_CAP: hv_base.NO_CHECK, constants.HV_CPU_WEIGHT: (False, lambda x: 0 < x < 65535, "invalid weight", None, None), constants.HV_VIF_TYPE: hv_base.ParamInSet(False, constants.HT_HVM_VALID_VIF_TYPES), constants.HV_VIF_SCRIPT: hv_base.OPT_FILE_CHECK, constants.HV_VIRIDIAN: hv_base.NO_CHECK, constants.HV_XEN_CPUID: hv_base.NO_CHECK, constants.HV_SOUNDHW: hv_base.NO_CHECK, } def _GetConfig(self, instance, startup_memory, block_devices): """Create a Xen 3.1 HVM config file. """ hvp = instance.hvparams config = StringIO() # kernel handling kpath = hvp[constants.HV_KERNEL_PATH] config.write("kernel = '%s'\n" % kpath) config.write("builder = 'hvm'\n") config.write("memory = %d\n" % startup_memory) config.write("maxmem = %d\n" % instance.beparams[constants.BE_MAXMEM]) config.write("vcpus = %d\n" % instance.beparams[constants.BE_VCPUS]) cpu_pinning = _CreateConfigCpus(hvp[constants.HV_CPU_MASK]) if cpu_pinning: config.write("%s\n" % cpu_pinning) cpu_cap = hvp[constants.HV_CPU_CAP] if cpu_cap: config.write("cpu_cap=%d\n" % cpu_cap) cpu_weight = hvp[constants.HV_CPU_WEIGHT] if cpu_weight: config.write("cpu_weight=%d\n" % cpu_weight) config.write("name = '%s'\n" % instance.name) if hvp[constants.HV_PAE]: config.write("pae = 1\n") else: config.write("pae = 0\n") if hvp[constants.HV_ACPI]: config.write("acpi = 1\n") else: config.write("acpi = 0\n") if hvp[constants.HV_VIRIDIAN]: config.write("viridian = 1\n") else: config.write("viridian = 0\n") config.write("apic = 1\n") config.write("device_model = '%s'\n" % hvp[constants.HV_DEVICE_MODEL]) config.write("boot = '%s'\n" % hvp[constants.HV_BOOT_ORDER]) config.write("sdl = 0\n") config.write("usb = 1\n") config.write("usbdevice = 'tablet'\n") config.write("vnc = 1\n") if hvp[constants.HV_VNC_BIND_ADDRESS] is None: config.write("vnclisten = '%s'\n" % constants.VNC_DEFAULT_BIND_ADDRESS) else: config.write("vnclisten = '%s'\n" % hvp[constants.HV_VNC_BIND_ADDRESS]) if (instance.network_port is not None and instance.network_port > constants.VNC_BASE_PORT): display = instance.network_port - constants.VNC_BASE_PORT config.write("vncdisplay = %s\n" % display) config.write("vncunused = 0\n") else: config.write("# vncdisplay = 1\n") config.write("vncunused = 1\n") vnc_pwd_file = hvp[constants.HV_VNC_PASSWORD_FILE] try: password = utils.ReadFile(vnc_pwd_file) except EnvironmentError as err: raise errors.HypervisorError("Failed to open VNC password file %s: %s" % (vnc_pwd_file, err)) config.write("vncpasswd = '%s'\n" % password.rstrip()) config.write("serial = 'pty'\n") if hvp[constants.HV_USE_LOCALTIME]: config.write("localtime = 1\n") self._WriteNicConfig(config, instance, hvp) disk_data = \ _GetConfigFileDiskData(block_devices, hvp[constants.HV_BLOCKDEV_PREFIX]) iso_path = hvp[constants.HV_CDROM_IMAGE_PATH] if iso_path: iso = "'file:%s,hdc:cdrom,r'" % iso_path disk_data.append(iso) config.write("disk = [%s]\n" % (",".join(disk_data))) # Add PCI passthrough pci_pass_arr = [] pci_pass = hvp[constants.HV_PASSTHROUGH] if pci_pass: pci_pass_arr = pci_pass.split(";") config.write("pci = %s\n" % pci_pass_arr) WriteXenConfigEvents(config, hvp) cpuid = hvp[constants.HV_XEN_CPUID] if cpuid: config.write("cpuid = %s\n" % _QuoteCpuidField(cpuid)) if hvp[constants.HV_SOUNDHW]: config.write("soundhw = '%s'\n" % hvp[constants.HV_SOUNDHW]) return config.getvalue() ganeti-3.1.0~rc2/lib/impexpd/000075500000000000000000000000001476477700300160205ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/impexpd/__init__.py000064400000000000000000000465321476477700300201430ustar00rootroot00000000000000# # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Classes and functions for import/export daemon. """ import os import re import socket import logging import signal import errno import time from io import StringIO from ganeti import constants from ganeti import errors from ganeti import utils from ganeti import netutils from ganeti import compat #: Used to recognize point at which socat(1) starts to listen on its socket. #: The local address is required for the remote peer to connect (in particular #: the port number). LISTENING_RE = re.compile(r"^listening on\s+" r"AF=(?P\d+)\s+" r"(?P

.+):(?P\d+)$", re.I) #: Used to recognize point at which socat(1) is sending data over the wire TRANSFER_LOOP_RE = re.compile(r"^starting data transfer loop with FDs\s+.*$", re.I) SOCAT_LOG_DEBUG = "D" SOCAT_LOG_INFO = "I" SOCAT_LOG_NOTICE = "N" SOCAT_LOG_WARNING = "W" SOCAT_LOG_ERROR = "E" SOCAT_LOG_FATAL = "F" SOCAT_LOG_IGNORE = compat.UniqueFrozenset([ SOCAT_LOG_DEBUG, SOCAT_LOG_INFO, SOCAT_LOG_NOTICE, ]) #: Used to parse GNU dd(1) statistics DD_INFO_RE = re.compile(r"^(?P\d+)\s*byte(?:|s)\s.*\scopied,\s*" r"(?P[\d.]+)\s*s(?:|econds),.*$", re.I) #: Used to ignore "N+N records in/out" on dd(1)'s stderr DD_STDERR_IGNORE = re.compile(r"^\d+\+\d+\s*records\s+(?:in|out)$", re.I) #: Signal upon which dd(1) will print statistics (on some platforms, SIGINFO is #: unavailable and SIGUSR1 is used instead) DD_INFO_SIGNAL = getattr(signal, "SIGINFO", signal.SIGUSR1) #: Buffer size: at most this many bytes are transferred at once BUFSIZE = 1024 * 1024 # Common options for socat SOCAT_TCP_OPTS = ["keepalive", "keepidle=60", "keepintvl=10", "keepcnt=5"] SOCAT_OPENSSL_OPTS = ["verify=1", "cipher=%s" % constants.OPENSSL_CIPHERS] if constants.SOCAT_USE_COMPRESS: # Disables all compression in by OpenSSL. Only supported in patched versions # of socat (as of November 2010). See INSTALL for more information. SOCAT_OPENSSL_OPTS.append("compress=none") SOCAT_OPTION_MAXLEN = 400 (PROG_OTHER, PROG_SOCAT, PROG_DD, PROG_DD_PID, PROG_EXP_SIZE) = range(1, 6) PROG_ALL = compat.UniqueFrozenset([ PROG_OTHER, PROG_SOCAT, PROG_DD, PROG_DD_PID, PROG_EXP_SIZE, ]) class CommandBuilder(object): _SOCAT_VERSION = (0,) def __init__(self, mode, opts, socat_stderr_fd, dd_stderr_fd, dd_pid_fd): """Initializes this class. @param mode: Daemon mode (import or export) @param opts: Options object @type socat_stderr_fd: int @param socat_stderr_fd: File descriptor socat should write its stderr to @type dd_stderr_fd: int @param dd_stderr_fd: File descriptor dd should write its stderr to @type dd_pid_fd: int @param dd_pid_fd: File descriptor the child should write dd's PID to """ self._opts = opts self._mode = mode self._socat_stderr_fd = socat_stderr_fd self._dd_stderr_fd = dd_stderr_fd self._dd_pid_fd = dd_pid_fd assert (self._opts.magic is None or constants.IE_MAGIC_RE.match(self._opts.magic)) @staticmethod def GetBashCommand(cmd): """Prepares a command to be run in Bash. """ return ["bash", "-o", "errexit", "-o", "pipefail", "-c", cmd] @classmethod def _GetSocatVersion(cls): """Returns the socat version, as a tuple of ints. The version is memoized in a class variable for future use. """ if cls._SOCAT_VERSION > (0,): return cls._SOCAT_VERSION socat = utils.RunCmd([constants.SOCAT_PATH, "-V"]) # No need to check for errors here. If -V is not there, socat is really # old. Any other failure will be handled when running the actual socat # command. for line in socat.output.splitlines(): match = re.match(r"socat version ((\d+\.)*(\d+))", line) if match: try: cls._SOCAT_VERSION = tuple(int(x) for x in match.group(1).split('.')) except TypeError: pass break return cls._SOCAT_VERSION def _GetSocatCommand(self): """Returns the socat command. """ common_addr_opts = SOCAT_TCP_OPTS + SOCAT_OPENSSL_OPTS + [ "key=%s" % self._opts.key, "cert=%s" % self._opts.cert, "cafile=%s" % self._opts.ca, ] if self._opts.bind is not None: common_addr_opts.append("bind=%s" % self._opts.bind) assert not (self._opts.ipv4 and self._opts.ipv6) if self._opts.ipv4: common_addr_opts.append("pf=ipv4") elif self._opts.ipv6: common_addr_opts.append("pf=ipv6") if self._mode == constants.IEM_IMPORT: if self._opts.port is None: port = 0 else: port = self._opts.port addr1 = [ "OPENSSL-LISTEN:%s" % port, "reuseaddr", # Retry to listen if connection wasn't established successfully, up to # 100 times a second. Note that this still leaves room for DoS attacks. "forever", "intervall=0.01", ] + common_addr_opts addr2 = ["stdout"] elif self._mode == constants.IEM_EXPORT: if self._opts.host and netutils.IP6Address.IsValid(self._opts.host): host = "[%s]" % self._opts.host else: host = self._opts.host addr1 = ["stdin"] addr2 = [ "OPENSSL:%s:%s" % (host, self._opts.port), # How long to wait per connection attempt "connect-timeout=%s" % self._opts.connect_timeout, # Retry a few times before giving up to connect (once per second) "retry=%s" % self._opts.connect_retries, "intervall=1", ] + common_addr_opts # For socat versions >= 1.7.3, we need to also specify # openssl-commonname, otherwise server certificate verification will # fail. x509_cert_cn = host # we were previously hardcoding constants.X509_CERT_CN here, but # that's always ganeti.example.com while the other end generates # an actual cert that matches the hostname. unfortunately here # we are typically given an IP address, so let's try to find a # real hostname for the certificate check if host and netutils.IPAddress.IsValid(host): # this looks like an IP address, override based on the reverse # DNS try: x509_cert_cn, _, _ = socket.gethostbyaddr(host) logging.warning("overriden IP address %s to reverse hostname %s", host, x509_cert_cn) except OSError as e: logging.error( "failed to resolve IP address %s, reverting to default %s: %s", host, constants.X509_CERT_CN, e, ) x509_cert_cn = constants.X509_CERT_CN if self._GetSocatVersion() >= (1, 7, 3): addr2 += ["openssl-commonname=%s" % x509_cert_cn] else: raise errors.GenericError("Invalid mode '%s'" % self._mode) for i in [addr1, addr2]: for value in i: if len(value) > SOCAT_OPTION_MAXLEN: raise errors.GenericError("Socat option longer than %s" " characters: %r" % (SOCAT_OPTION_MAXLEN, value)) if "," in value: raise errors.GenericError("Comma not allowed in socat option" " value: %r" % value) return [ constants.SOCAT_PATH, # Log to stderr "-ls", # Log level "-d", "-d", # Buffer size "-b%s" % BUFSIZE, # Unidirectional mode, the first address is only used for reading, and the # second address is only used for writing "-u", ",".join(addr1), ",".join(addr2), ] def _GetMagicCommand(self): """Returns the command to read/write the magic value. """ if not self._opts.magic: return None # Prefix to ensure magic isn't interpreted as option to "echo" magic = "M=%s" % self._opts.magic cmd = StringIO() if self._mode == constants.IEM_IMPORT: cmd.write("{ ") cmd.write(utils.ShellQuoteArgs(["read", "-n", str(len(magic)), "magic"])) cmd.write(" && ") cmd.write("if test \"$magic\" != %s; then" % utils.ShellQuote(magic)) cmd.write(" echo %s >&2;" % utils.ShellQuote("Magic value mismatch")) cmd.write(" exit 1;") cmd.write("fi;") cmd.write(" }") elif self._mode == constants.IEM_EXPORT: cmd.write(utils.ShellQuoteArgs(["echo", "-E", "-n", magic])) else: raise errors.GenericError("Invalid mode '%s'" % self._mode) return cmd.getvalue() def _GetDdCommand(self): """Returns the command for measuring throughput. """ dd_cmd = StringIO() magic_cmd = self._GetMagicCommand() if magic_cmd: dd_cmd.write("{ ") dd_cmd.write(magic_cmd) dd_cmd.write(" && ") dd_cmd.write("{ ") # Setting LC_ALL since we want to parse the output and explicitly # redirecting stdin, as the background process (dd) would have # /dev/null as stdin otherwise dd_cmd.write("LC_ALL=C dd bs=%s <&0 2>&%d & pid=${!};" % (BUFSIZE, self._dd_stderr_fd)) # Send PID to daemon dd_cmd.write(" echo $pid >&%d;" % self._dd_pid_fd) # And wait for dd dd_cmd.write(" wait $pid;") dd_cmd.write(" }") if magic_cmd: dd_cmd.write(" }") return dd_cmd.getvalue() def _GetTransportCommand(self): """Returns the command for the transport part of the daemon. """ socat_cmd = ("%s 2>&%d" % (utils.ShellQuoteArgs(self._GetSocatCommand()), self._socat_stderr_fd)) dd_cmd = self._GetDdCommand() compr = self._opts.compress parts = [] if self._mode == constants.IEM_IMPORT: parts.append(socat_cmd) if compr in [constants.IEC_GZIP, constants.IEC_GZIP_FAST, constants.IEC_GZIP_SLOW, constants.IEC_LZOP]: utility_name = constants.IEC_COMPRESSION_UTILITIES.get(compr, compr) parts.append("%s -d -c" % utility_name) elif compr != constants.IEC_NONE: parts.append("%s -d" % compr) else: # No compression pass parts.append(dd_cmd) elif self._mode == constants.IEM_EXPORT: parts.append(dd_cmd) if compr in [constants.IEC_GZIP_SLOW, constants.IEC_LZOP]: utility_name = constants.IEC_COMPRESSION_UTILITIES.get(compr, compr) parts.append("%s -c" % utility_name) elif compr in [constants.IEC_GZIP_FAST, constants.IEC_GZIP]: parts.append("gzip -1 -c") elif compr != constants.IEC_NONE: parts.append(compr) else: # No compression pass parts.append(socat_cmd) else: raise errors.GenericError("Invalid mode '%s'" % self._mode) # TODO: Run transport as separate user # The transport uses its own shell to simplify running it as a separate user # in the future. return self.GetBashCommand(" | ".join(parts)) def GetCommand(self): """Returns the complete child process command. """ transport_cmd = self._GetTransportCommand() buf = StringIO() if self._opts.cmd_prefix: buf.write(self._opts.cmd_prefix) buf.write(" ") buf.write(utils.ShellQuoteArgs(transport_cmd)) if self._opts.cmd_suffix: buf.write(" ") buf.write(self._opts.cmd_suffix) return self.GetBashCommand(buf.getvalue()) def _VerifyListening(family, address, port): """Verify address given as listening address by socat. """ if family not in (socket.AF_INET, socket.AF_INET6): raise errors.GenericError("Address family %r not supported" % family) if (family == socket.AF_INET6 and address.startswith("[") and address.endswith("]")): address = address.lstrip("[").rstrip("]") try: packed_address = socket.inet_pton(family, address) except socket.error: raise errors.GenericError("Invalid address %r for family %s" % (address, family)) return (socket.inet_ntop(family, packed_address), port) class ChildIOProcessor(object): def __init__(self, debug, status_file, logger, throughput_samples, exp_size): """Initializes this class. """ self._debug = debug self._status_file = status_file self._logger = logger self._splitter = dict([(prog, utils.LineSplitter(self._ProcessOutput, prog)) for prog in PROG_ALL]) self._dd_pid = None self._dd_ready = False self._dd_tp_samples = throughput_samples self._dd_progress = [] # Expected size of transferred data self._exp_size = exp_size def GetLineSplitter(self, prog): """Returns the line splitter for a program. """ return self._splitter[prog] def FlushAll(self): """Flushes all line splitters. """ for ls in self._splitter.values(): ls.flush() def CloseAll(self): """Closes all line splitters. """ for ls in self._splitter.values(): ls.close() self._splitter.clear() def NotifyDd(self): """Tells dd(1) to write statistics. """ if self._dd_pid is None: # Can't notify return False if not self._dd_ready: # There's a race condition between starting the program and sending # signals. The signal handler is only registered after some time, so we # have to check whether the program is ready. If it isn't, sending a # signal will invoke the default handler (and usually abort the program). if not utils.IsProcessHandlingSignal(self._dd_pid, DD_INFO_SIGNAL): logging.debug("dd is not yet ready for signal %s", DD_INFO_SIGNAL) return False logging.debug("dd is now handling signal %s", DD_INFO_SIGNAL) self._dd_ready = True logging.debug("Sending signal %s to PID %s", DD_INFO_SIGNAL, self._dd_pid) try: os.kill(self._dd_pid, DD_INFO_SIGNAL) except EnvironmentError as err: if err.errno != errno.ESRCH: raise # Process no longer exists logging.debug("dd exited") self._dd_pid = None return True def _ProcessOutput(self, line, prog): """Takes care of child process output. @type line: string @param line: Child output line @type prog: number @param prog: Program from which the line originates """ force_update = False forward_line = line if prog == PROG_SOCAT: level = None parts = line.split(None, 4) if len(parts) == 5: (_, _, _, level, msg) = parts force_update = self._ProcessSocatOutput(self._status_file, level, msg) if self._debug or (level and level not in SOCAT_LOG_IGNORE): forward_line = "socat: %s %s" % (level, msg) else: forward_line = None else: forward_line = "socat: %s" % line elif prog == PROG_DD: (should_forward, force_update) = self._ProcessDdOutput(line) if should_forward or self._debug: forward_line = "dd: %s" % line else: forward_line = None elif prog == PROG_DD_PID: if self._dd_pid: raise RuntimeError("dd PID reported more than once") logging.debug("Received dd PID %r", line) self._dd_pid = int(line) forward_line = None elif prog == PROG_EXP_SIZE: logging.debug("Received predicted size %r", line) forward_line = None if line: try: exp_size = utils.BytesToMebibyte(int(line)) except (ValueError, TypeError) as err: logging.error("Failed to convert predicted size %r to number: %s", line, err) exp_size = None else: exp_size = None self._exp_size = exp_size if forward_line: self._logger.info(forward_line) self._status_file.AddRecentOutput(forward_line) self._status_file.Update(force_update) @staticmethod def _ProcessSocatOutput(status_file, level, msg): """Interprets socat log output. """ if level == SOCAT_LOG_NOTICE: if status_file.GetListenPort() is None: # TODO: Maybe implement timeout to not listen forever m = LISTENING_RE.match(msg) if m: (_, port) = _VerifyListening(int(m.group("family")), m.group("address"), int(m.group("port"))) status_file.SetListenPort(port) return True if not status_file.GetConnected(): m = TRANSFER_LOOP_RE.match(msg) if m: logging.debug("Connection established") status_file.SetConnected() return True return False def _ProcessDdOutput(self, line): """Interprets a line of dd(1)'s output. """ m = DD_INFO_RE.match(line) if m: seconds = float(m.group("seconds")) mbytes = utils.BytesToMebibyte(int(m.group("bytes"))) self._UpdateDdProgress(seconds, mbytes) return (False, True) m = DD_STDERR_IGNORE.match(line) if m: # Ignore return (False, False) # Forward line return (True, False) def _UpdateDdProgress(self, seconds, mbytes): """Updates the internal status variables for dd(1) progress. @type seconds: float @param seconds: Timestamp of this update @type mbytes: float @param mbytes: Total number of MiB transferred so far """ # Add latest sample self._dd_progress.append((seconds, mbytes)) # Remove old samples del self._dd_progress[:-self._dd_tp_samples] # Calculate throughput throughput = _CalcThroughput(self._dd_progress) # Calculate percent and ETA percent = None eta = None if self._exp_size is not None: if self._exp_size != 0: percent = max(0, min(100, (100.0 * mbytes) / self._exp_size)) if throughput: eta = max(0, float(self._exp_size - mbytes) / throughput) self._status_file.SetProgress(mbytes, throughput, percent, eta) def _CalcThroughput(samples): """Calculates the throughput in MiB/second. @type samples: sequence @param samples: List of samples, each consisting of a (timestamp, mbytes) tuple @rtype: float or None @return: Throughput in MiB/second """ if len(samples) < 2: # Can't calculate throughput return None (start_time, start_mbytes) = samples[0] (end_time, end_mbytes) = samples[-1] return (float(end_mbytes) - start_mbytes) / (float(end_time) - start_time) ganeti-3.1.0~rc2/lib/jqueue/000075500000000000000000000000001476477700300156505ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/jqueue/__init__.py000064400000000000000000001454361476477700300177760ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing the job queue handling. """ import logging import errno import time import weakref import threading import itertools import operator import os try: # pylint: disable=E0611 from pyinotify import pyinotify except ImportError: import pyinotify from ganeti import asyncnotifier from ganeti import constants from ganeti import serializer from ganeti import locking from ganeti import luxi from ganeti import opcodes from ganeti import opcodes_base from ganeti import errors from ganeti import mcpu from ganeti import utils from ganeti import jstore import ganeti.rpc.node as rpc from ganeti import runtime from ganeti import netutils from ganeti import compat from ganeti import ht from ganeti import query from ganeti import qlang from ganeti import pathutils from ganeti import vcluster from ganeti.cmdlib import cluster #: Retrieves "id" attribute _GetIdAttr = operator.attrgetter("id") class CancelJob(Exception): """Special exception to cancel a job. """ def TimeStampNow(): """Returns the current timestamp. @rtype: tuple @return: the current time in the (seconds, microseconds) format """ return utils.SplitTime(time.time()) def _CallJqUpdate(runner, names, file_name, content): """Updates job queue file after virtualizing filename. """ virt_file_name = vcluster.MakeVirtualPath(file_name) return runner.call_jobqueue_update(names, virt_file_name, content) class _QueuedOpCode(object): """Encapsulates an opcode object. @ivar log: holds the execution log and consists of tuples of the form C{(log_serial, timestamp, level, message)} @ivar input: the OpCode we encapsulate @ivar status: the current status @ivar result: the result of the LU execution @ivar start_timestamp: timestamp for the start of the execution @ivar exec_timestamp: timestamp for the actual LU Exec() function invocation @ivar stop_timestamp: timestamp for the end of the execution """ __slots__ = ["input", "status", "result", "log", "priority", "start_timestamp", "exec_timestamp", "end_timestamp", "__weakref__"] def __init__(self, op): """Initializes instances of this class. @type op: L{opcodes.OpCode} @param op: the opcode we encapsulate """ self.input = op self.status = constants.OP_STATUS_QUEUED self.result = None self.log = [] self.start_timestamp = None self.exec_timestamp = None self.end_timestamp = None # Get initial priority (it might change during the lifetime of this opcode) self.priority = getattr(op, "priority", constants.OP_PRIO_DEFAULT) @classmethod def Restore(cls, state): """Restore the _QueuedOpCode from the serialized form. @type state: dict @param state: the serialized state @rtype: _QueuedOpCode @return: a new _QueuedOpCode instance """ obj = _QueuedOpCode.__new__(cls) obj.input = opcodes.OpCode.LoadOpCode(state["input"]) obj.status = state["status"] obj.result = state["result"] obj.log = state["log"] obj.start_timestamp = state.get("start_timestamp", None) obj.exec_timestamp = state.get("exec_timestamp", None) obj.end_timestamp = state.get("end_timestamp", None) obj.priority = state.get("priority", constants.OP_PRIO_DEFAULT) return obj def Serialize(self): """Serializes this _QueuedOpCode. @rtype: dict @return: the dictionary holding the serialized state """ return { "input": self.input.__getstate__(), "status": self.status, "result": self.result, "log": self.log, "start_timestamp": self.start_timestamp, "exec_timestamp": self.exec_timestamp, "end_timestamp": self.end_timestamp, "priority": self.priority, } class _QueuedJob(object): """In-memory job representation. This is what we use to track the user-submitted jobs. Locking must be taken care of by users of this class. @type queue: L{JobQueue} @ivar queue: the parent queue @ivar id: the job ID @type ops: list @ivar ops: the list of _QueuedOpCode that constitute the job @type log_serial: int @ivar log_serial: holds the index for the next log entry @ivar received_timestamp: the timestamp for when the job was received @ivar start_timestmap: the timestamp for start of execution @ivar end_timestamp: the timestamp for end of execution @ivar writable: Whether the job is allowed to be modified """ # pylint: disable=W0212 __slots__ = ["queue", "id", "ops", "log_serial", "ops_iter", "cur_opctx", "received_timestamp", "start_timestamp", "end_timestamp", "writable", "archived", "livelock", "process_id", "__weakref__"] def AddReasons(self, pickup=False): """Extend the reason trail Add the reason for all the opcodes of this job to be executed. """ count = 0 for queued_op in self.ops: op = queued_op.input if pickup: reason_src_prefix = constants.OPCODE_REASON_SRC_PICKUP else: reason_src_prefix = constants.OPCODE_REASON_SRC_OPCODE reason_src = opcodes_base.NameToReasonSrc(op.__class__.__name__, reason_src_prefix) reason_text = "job=%d;index=%d" % (self.id, count) reason = getattr(op, "reason", []) reason.append((reason_src, reason_text, utils.EpochNano())) op.reason = reason count = count + 1 def __init__(self, queue, job_id, ops, writable): """Constructor for the _QueuedJob. @type queue: L{JobQueue} @param queue: our parent queue @type job_id: job_id @param job_id: our job id @type ops: list @param ops: the list of opcodes we hold, which will be encapsulated in _QueuedOpCodes @type writable: bool @param writable: Whether job can be modified """ if not ops: raise errors.GenericError("A job needs at least one opcode") self.queue = queue self.id = int(job_id) self.ops = [_QueuedOpCode(op) for op in ops] self.AddReasons() self.log_serial = 0 self.received_timestamp = TimeStampNow() self.start_timestamp = None self.end_timestamp = None self.archived = False self.livelock = None self.process_id = None self._InitInMemory(self, writable) assert not self.archived, "New jobs can not be marked as archived" @staticmethod def _InitInMemory(obj, writable): """Initializes in-memory variables. """ obj.writable = writable obj.ops_iter = None obj.cur_opctx = None def __repr__(self): status = ["%s.%s" % (self.__class__.__module__, self.__class__.__name__), "id=%s" % self.id, "ops=%s" % ",".join([op.input.Summary() for op in self.ops])] return "<%s at %#x>" % (" ".join(status), id(self)) @classmethod def Restore(cls, queue, state, writable, archived): """Restore a _QueuedJob from serialized state: @type queue: L{JobQueue} @param queue: to which queue the restored job belongs @type state: dict @param state: the serialized state @type writable: bool @param writable: Whether job can be modified @type archived: bool @param archived: Whether job was already archived @rtype: _JobQueue @return: the restored _JobQueue instance """ obj = _QueuedJob.__new__(cls) obj.queue = queue obj.id = int(state["id"]) obj.received_timestamp = state.get("received_timestamp", None) obj.start_timestamp = state.get("start_timestamp", None) obj.end_timestamp = state.get("end_timestamp", None) obj.archived = archived obj.livelock = state.get("livelock", None) obj.process_id = state.get("process_id", None) if obj.process_id is not None: obj.process_id = int(obj.process_id) obj.ops = [] obj.log_serial = 0 for op_state in state["ops"]: op = _QueuedOpCode.Restore(op_state) for log_entry in op.log: obj.log_serial = max(obj.log_serial, log_entry[0]) obj.ops.append(op) cls._InitInMemory(obj, writable) return obj def Serialize(self): """Serialize the _JobQueue instance. @rtype: dict @return: the serialized state """ return { "id": self.id, "ops": [op.Serialize() for op in self.ops], "start_timestamp": self.start_timestamp, "end_timestamp": self.end_timestamp, "received_timestamp": self.received_timestamp, "livelock": self.livelock, "process_id": self.process_id, } def CalcStatus(self): """Compute the status of this job. This function iterates over all the _QueuedOpCodes in the job and based on their status, computes the job status. The algorithm is: - if we find a cancelled, or finished with error, the job status will be the same - otherwise, the last opcode with the status one of: - waitlock - canceling - running will determine the job status - otherwise, it means either all opcodes are queued, or success, and the job status will be the same @return: the job status """ status = constants.JOB_STATUS_QUEUED all_success = True for op in self.ops: if op.status == constants.OP_STATUS_SUCCESS: continue all_success = False if op.status == constants.OP_STATUS_QUEUED: pass elif op.status == constants.OP_STATUS_WAITING: status = constants.JOB_STATUS_WAITING elif op.status == constants.OP_STATUS_RUNNING: status = constants.JOB_STATUS_RUNNING elif op.status == constants.OP_STATUS_CANCELING: status = constants.JOB_STATUS_CANCELING break elif op.status == constants.OP_STATUS_ERROR: status = constants.JOB_STATUS_ERROR # The whole job fails if one opcode failed break elif op.status == constants.OP_STATUS_CANCELED: status = constants.OP_STATUS_CANCELED break if all_success: status = constants.JOB_STATUS_SUCCESS return status def CalcPriority(self): """Gets the current priority for this job. Only unfinished opcodes are considered. When all are done, the default priority is used. @rtype: int """ priorities = [op.priority for op in self.ops if op.status not in constants.OPS_FINALIZED] if not priorities: # All opcodes are done, assume default priority return constants.OP_PRIO_DEFAULT return min(priorities) def GetLogEntries(self, newer_than): """Selectively returns the log entries. @type newer_than: None or int @param newer_than: if this is None, return all log entries, otherwise return only the log entries with serial higher than this value @rtype: list @return: the list of the log entries selected """ if newer_than is None: serial = -1 else: serial = newer_than entries = [] for op in self.ops: entries.extend([entry for entry in op.log if entry[0] > serial]) return entries def MarkUnfinishedOps(self, status, result): """Mark unfinished opcodes with a given status and result. This is an utility function for marking all running or waiting to be run opcodes with a given status. Opcodes which are already finalised are not changed. @param status: a given opcode status @param result: the opcode result """ not_marked = True for op in self.ops: if op.status in constants.OPS_FINALIZED: assert not_marked, "Finalized opcodes found after non-finalized ones" continue op.status = status op.result = result not_marked = False def Finalize(self): """Marks the job as finalized. """ self.end_timestamp = TimeStampNow() def Cancel(self): """Marks job as canceled/-ing if possible. @rtype: tuple; (bool, string) @return: Boolean describing whether job was successfully canceled or marked as canceling and a text message """ status = self.CalcStatus() if status == constants.JOB_STATUS_QUEUED: self.MarkUnfinishedOps(constants.OP_STATUS_CANCELED, "Job canceled by request") self.Finalize() return (True, "Job %s canceled" % self.id) elif status == constants.JOB_STATUS_WAITING: # The worker will notice the new status and cancel the job self.MarkUnfinishedOps(constants.OP_STATUS_CANCELING, None) return (True, "Job %s will be canceled" % self.id) else: logging.debug("Job %s is no longer waiting in the queue", self.id) return (False, "Job %s is no longer waiting in the queue" % self.id) def ChangePriority(self, priority): """Changes the job priority. @type priority: int @param priority: New priority @rtype: tuple; (bool, string) @return: Boolean describing whether job's priority was successfully changed and a text message """ status = self.CalcStatus() if status in constants.JOBS_FINALIZED: return (False, "Job %s is finished" % self.id) elif status == constants.JOB_STATUS_CANCELING: return (False, "Job %s is cancelling" % self.id) else: assert status in (constants.JOB_STATUS_QUEUED, constants.JOB_STATUS_WAITING, constants.JOB_STATUS_RUNNING) changed = False for op in self.ops: if (op.status == constants.OP_STATUS_RUNNING or op.status in constants.OPS_FINALIZED): assert not changed, \ ("Found opcode for which priority should not be changed after" " priority has been changed for previous opcodes") continue assert op.status in (constants.OP_STATUS_QUEUED, constants.OP_STATUS_WAITING) changed = True # Set new priority (doesn't modify opcode input) op.priority = priority if changed: return (True, ("Priorities of pending opcodes for job %s have been" " changed to %s" % (self.id, priority))) else: return (False, "Job %s had no pending opcodes" % self.id) def SetPid(self, pid): """Sets the job's process ID @type pid: int @param pid: the process ID """ status = self.CalcStatus() if status in (constants.JOB_STATUS_QUEUED, constants.JOB_STATUS_WAITING): if self.process_id is not None: logging.warning("Replacing the process id %s of job %s with %s", self.process_id, self.id, pid) self.process_id = pid else: logging.warning("Can set pid only for queued/waiting jobs") class _OpExecCallbacks(mcpu.OpExecCbBase): def __init__(self, queue, job, op): """Initializes this class. @type queue: L{JobQueue} @param queue: Job queue @type job: L{_QueuedJob} @param job: Job object @type op: L{_QueuedOpCode} @param op: OpCode """ super(_OpExecCallbacks, self).__init__() assert queue, "Queue is missing" assert job, "Job is missing" assert op, "Opcode is missing" self._queue = queue self._job = job self._op = op def _CheckCancel(self): """Raises an exception to cancel the job if asked to. """ # Cancel here if we were asked to if self._op.status == constants.OP_STATUS_CANCELING: logging.debug("Canceling opcode") raise CancelJob() def NotifyStart(self): """Mark the opcode as running, not lock-waiting. This is called from the mcpu code as a notifier function, when the LU is finally about to start the Exec() method. Of course, to have end-user visible results, the opcode must be initially (before calling into Processor.ExecOpCode) set to OP_STATUS_WAITING. """ assert self._op in self._job.ops assert self._op.status in (constants.OP_STATUS_WAITING, constants.OP_STATUS_CANCELING) # Cancel here if we were asked to self._CheckCancel() logging.debug("Opcode is now running") self._op.status = constants.OP_STATUS_RUNNING self._op.exec_timestamp = TimeStampNow() # And finally replicate the job status self._queue.UpdateJobUnlocked(self._job) def NotifyRetry(self): """Mark opcode again as lock-waiting. This is called from the mcpu code just after calling PrepareRetry. The opcode will now again acquire locks (more, hopefully). """ self._op.status = constants.OP_STATUS_WAITING logging.debug("Opcode will be retried. Back to waiting.") def _AppendFeedback(self, timestamp, log_type, log_msgs): """Internal feedback append function, with locks @type timestamp: tuple (int, int) @param timestamp: timestamp of the log message @type log_type: string @param log_type: log type (one of Types.ELogType) @type log_msgs: any @param log_msgs: log data to append """ # This should be removed once Feedback() has a clean interface. # Feedback can be called with anything, we interpret ELogMessageList as # messages that have to be individually added to the log list, but pushed # in a single update. Other msgtypes are only transparently passed forward. if log_type == constants.ELOG_MESSAGE_LIST: log_type = constants.ELOG_MESSAGE else: log_msgs = [log_msgs] for msg in log_msgs: self._job.log_serial += 1 self._op.log.append((self._job.log_serial, timestamp, log_type, msg)) self._queue.UpdateJobUnlocked(self._job, replicate=False) # TODO: Cleanup calling conventions, make them explicit def Feedback(self, *args): """Append a log entry. Calling conventions: arg[0]: (optional) string, message type (Types.ELogType) arg[1]: data to be interpreted as a message """ assert len(args) < 3 # TODO: Use separate keyword arguments for a single string vs. a list. if len(args) == 1: log_type = constants.ELOG_MESSAGE log_msg = args[0] else: (log_type, log_msg) = args # The time is split to make serialization easier and not lose # precision. timestamp = utils.SplitTime(time.time()) self._AppendFeedback(timestamp, log_type, log_msg) def CurrentPriority(self): """Returns current priority for opcode. """ assert self._op.status in (constants.OP_STATUS_WAITING, constants.OP_STATUS_CANCELING) # Cancel here if we were asked to self._CheckCancel() return self._op.priority def SubmitManyJobs(self, jobs): """Submits jobs for processing. See L{JobQueue.SubmitManyJobs}. """ # Locking is done in job queue return self._queue.SubmitManyJobs(jobs) def _EncodeOpError(err): """Encodes an error which occurred while processing an opcode. """ if isinstance(err, errors.GenericError): to_encode = err else: to_encode = errors.OpExecError(str(err)) return errors.EncodeException(to_encode) class _TimeoutStrategyWrapper(object): def __init__(self, fn): """Initializes this class. """ self._fn = fn self._next = None def _Advance(self): """Gets the next timeout if necessary. """ if self._next is None: self._next = self._fn() def Peek(self): """Returns the next timeout. """ self._Advance() return self._next def Next(self): """Returns the current timeout and advances the internal state. """ self._Advance() result = self._next self._next = None return result class _OpExecContext(object): def __init__(self, op, index, log_prefix, timeout_strategy_factory): """Initializes this class. """ self.op = op self.index = index self.log_prefix = log_prefix self.summary = op.input.Summary() # Create local copy to modify if getattr(op.input, opcodes_base.DEPEND_ATTR, None): self.jobdeps = op.input.depends[:] else: self.jobdeps = None self._timeout_strategy_factory = timeout_strategy_factory self._ResetTimeoutStrategy() def _ResetTimeoutStrategy(self): """Creates a new timeout strategy. """ self._timeout_strategy = \ _TimeoutStrategyWrapper(self._timeout_strategy_factory().NextAttempt) def CheckPriorityIncrease(self): """Checks whether priority can and should be increased. Called when locks couldn't be acquired. """ op = self.op # Exhausted all retries and next round should not use blocking acquire # for locks? if (self._timeout_strategy.Peek() is None and op.priority > constants.OP_PRIO_HIGHEST): logging.debug("Increasing priority") op.priority -= 1 self._ResetTimeoutStrategy() return True return False def GetNextLockTimeout(self): """Returns the next lock acquire timeout. """ return self._timeout_strategy.Next() class _JobProcessor(object): (DEFER, WAITDEP, FINISHED) = range(1, 4) def __init__(self, queue, opexec_fn, job, _timeout_strategy_factory=mcpu.LockAttemptTimeoutStrategy): """Initializes this class. """ self.queue = queue self.opexec_fn = opexec_fn self.job = job self._timeout_strategy_factory = _timeout_strategy_factory @staticmethod def _FindNextOpcode(job, timeout_strategy_factory): """Locates the next opcode to run. @type job: L{_QueuedJob} @param job: Job object @param timeout_strategy_factory: Callable to create new timeout strategy """ # Create some sort of a cache to speed up locating next opcode for future # lookups # TODO: Consider splitting _QueuedJob.ops into two separate lists, one for # pending and one for processed ops. if job.ops_iter is None: job.ops_iter = enumerate(job.ops) # Find next opcode to run while True: try: (idx, op) = next(job.ops_iter) except StopIteration: raise errors.ProgrammerError("Called for a finished job") if op.status == constants.OP_STATUS_RUNNING: # Found an opcode already marked as running raise errors.ProgrammerError("Called for job marked as running") opctx = _OpExecContext(op, idx, "Op %s/%s" % (idx + 1, len(job.ops)), timeout_strategy_factory) if op.status not in constants.OPS_FINALIZED: return opctx # This is a job that was partially completed before master daemon # shutdown, so it can be expected that some opcodes are already # completed successfully (if any did error out, then the whole job # should have been aborted and not resubmitted for processing). logging.info("%s: opcode %s already processed, skipping", opctx.log_prefix, opctx.summary) @staticmethod def _MarkWaitlock(job, op): """Marks an opcode as waiting for locks. The job's start timestamp is also set if necessary. @type job: L{_QueuedJob} @param job: Job object @type op: L{_QueuedOpCode} @param op: Opcode object """ assert op in job.ops assert op.status in (constants.OP_STATUS_QUEUED, constants.OP_STATUS_WAITING) update = False op.result = None if op.status == constants.OP_STATUS_QUEUED: op.status = constants.OP_STATUS_WAITING update = True if op.start_timestamp is None: op.start_timestamp = TimeStampNow() update = True if job.start_timestamp is None: job.start_timestamp = op.start_timestamp update = True assert op.status == constants.OP_STATUS_WAITING return update @staticmethod def _CheckDependencies(queue, job, opctx): """Checks if an opcode has dependencies and if so, processes them. @type queue: L{JobQueue} @param queue: Queue object @type job: L{_QueuedJob} @param job: Job object @type opctx: L{_OpExecContext} @param opctx: Opcode execution context @rtype: bool @return: Whether opcode will be re-scheduled by dependency tracker """ op = opctx.op result = False while opctx.jobdeps: (dep_job_id, dep_status) = opctx.jobdeps[0] (depresult, depmsg) = queue.depmgr.CheckAndRegister(job, dep_job_id, dep_status) assert ht.TNonEmptyString(depmsg), "No dependency message" logging.info("%s: %s", opctx.log_prefix, depmsg) if depresult == _JobDependencyManager.CONTINUE: # Remove dependency and continue opctx.jobdeps.pop(0) elif depresult == _JobDependencyManager.WAIT: # Need to wait for notification, dependency tracker will re-add job # to workerpool result = True break elif depresult == _JobDependencyManager.CANCEL: # Job was cancelled, cancel this job as well job.Cancel() assert op.status == constants.OP_STATUS_CANCELING break elif depresult in (_JobDependencyManager.WRONGSTATUS, _JobDependencyManager.ERROR): # Job failed or there was an error, this job must fail op.status = constants.OP_STATUS_ERROR op.result = _EncodeOpError(errors.OpExecError(depmsg)) break else: raise errors.ProgrammerError("Unknown dependency result '%s'" % depresult) return result def _ExecOpCodeUnlocked(self, opctx): """Processes one opcode and returns the result. """ op = opctx.op assert op.status in (constants.OP_STATUS_WAITING, constants.OP_STATUS_CANCELING) # The very last check if the job was cancelled before trying to execute if op.status == constants.OP_STATUS_CANCELING: return (constants.OP_STATUS_CANCELING, None) timeout = opctx.GetNextLockTimeout() try: # Make sure not to hold queue lock while calling ExecOpCode result = self.opexec_fn(op.input, _OpExecCallbacks(self.queue, self.job, op), timeout=timeout) except mcpu.LockAcquireTimeout: assert timeout is not None, "Received timeout for blocking acquire" logging.debug("Couldn't acquire locks in %0.6fs", timeout) assert op.status in (constants.OP_STATUS_WAITING, constants.OP_STATUS_CANCELING) # Was job cancelled while we were waiting for the lock? if op.status == constants.OP_STATUS_CANCELING: return (constants.OP_STATUS_CANCELING, None) # Stay in waitlock while trying to re-acquire lock return (constants.OP_STATUS_WAITING, None) except CancelJob: logging.exception("%s: Canceling job", opctx.log_prefix) assert op.status == constants.OP_STATUS_CANCELING return (constants.OP_STATUS_CANCELING, None) except Exception as err: # pylint: disable=W0703 logging.exception("%s: Caught exception in %s", opctx.log_prefix, opctx.summary) return (constants.OP_STATUS_ERROR, _EncodeOpError(err)) else: logging.debug("%s: %s successful", opctx.log_prefix, opctx.summary) return (constants.OP_STATUS_SUCCESS, result) def __call__(self, _nextop_fn=None): """Continues execution of a job. @param _nextop_fn: Callback function for tests @return: C{FINISHED} if job is fully processed, C{DEFER} if the job should be deferred and C{WAITDEP} if the dependency manager (L{_JobDependencyManager}) will re-schedule the job when appropriate """ queue = self.queue job = self.job logging.debug("Processing job %s", job.id) try: opcount = len(job.ops) assert job.writable, "Expected writable job" # Don't do anything for finalized jobs if job.CalcStatus() in constants.JOBS_FINALIZED: return self.FINISHED # Is a previous opcode still pending? if job.cur_opctx: opctx = job.cur_opctx job.cur_opctx = None else: if __debug__ and _nextop_fn: _nextop_fn() opctx = self._FindNextOpcode(job, self._timeout_strategy_factory) op = opctx.op # Consistency check assert compat.all(i.status in (constants.OP_STATUS_QUEUED, constants.OP_STATUS_CANCELING) for i in job.ops[opctx.index + 1:]) assert op.status in (constants.OP_STATUS_QUEUED, constants.OP_STATUS_WAITING, constants.OP_STATUS_CANCELING) assert (op.priority <= constants.OP_PRIO_LOWEST and op.priority >= constants.OP_PRIO_HIGHEST) waitjob = None if op.status != constants.OP_STATUS_CANCELING: assert op.status in (constants.OP_STATUS_QUEUED, constants.OP_STATUS_WAITING) # Prepare to start opcode if self._MarkWaitlock(job, op): # Write to disk queue.UpdateJobUnlocked(job) assert op.status == constants.OP_STATUS_WAITING assert job.CalcStatus() == constants.JOB_STATUS_WAITING assert job.start_timestamp and op.start_timestamp assert waitjob is None # Check if waiting for a job is necessary waitjob = self._CheckDependencies(queue, job, opctx) assert op.status in (constants.OP_STATUS_WAITING, constants.OP_STATUS_CANCELING, constants.OP_STATUS_ERROR) if not (waitjob or op.status in (constants.OP_STATUS_CANCELING, constants.OP_STATUS_ERROR)): logging.info("%s: opcode %s waiting for locks", opctx.log_prefix, opctx.summary) assert not opctx.jobdeps, "Not all dependencies were removed" (op_status, op_result) = self._ExecOpCodeUnlocked(opctx) op.status = op_status op.result = op_result assert not waitjob if op.status in (constants.OP_STATUS_WAITING, constants.OP_STATUS_QUEUED): # waiting: Couldn't get locks in time # queued: Queue is shutting down assert not op.end_timestamp else: # Finalize opcode op.end_timestamp = TimeStampNow() if op.status == constants.OP_STATUS_CANCELING: assert not compat.any(i.status != constants.OP_STATUS_CANCELING for i in job.ops[opctx.index:]) else: assert op.status in constants.OPS_FINALIZED if op.status == constants.OP_STATUS_QUEUED: # Queue is shutting down assert not waitjob finalize = False # Reset context job.cur_opctx = None # In no case must the status be finalized here assert job.CalcStatus() == constants.JOB_STATUS_QUEUED elif op.status == constants.OP_STATUS_WAITING or waitjob: finalize = False if not waitjob and opctx.CheckPriorityIncrease(): # Priority was changed, need to update on-disk file queue.UpdateJobUnlocked(job) # Keep around for another round job.cur_opctx = opctx assert (op.priority <= constants.OP_PRIO_LOWEST and op.priority >= constants.OP_PRIO_HIGHEST) # In no case must the status be finalized here assert job.CalcStatus() == constants.JOB_STATUS_WAITING else: # Ensure all opcodes so far have been successful assert (opctx.index == 0 or compat.all(i.status == constants.OP_STATUS_SUCCESS for i in job.ops[:opctx.index])) # Reset context job.cur_opctx = None if op.status == constants.OP_STATUS_SUCCESS: finalize = False elif op.status == constants.OP_STATUS_ERROR: # If we get here, we cannot afford to check for any consistency # any more, we just want to clean up. # TODO: Actually, it wouldn't be a bad idea to start a timer # here to kill the whole process. to_encode = errors.OpExecError("Preceding opcode failed") job.MarkUnfinishedOps(constants.OP_STATUS_ERROR, _EncodeOpError(to_encode)) finalize = True elif op.status == constants.OP_STATUS_CANCELING: job.MarkUnfinishedOps(constants.OP_STATUS_CANCELED, "Job canceled by request") finalize = True else: raise errors.ProgrammerError("Unknown status '%s'" % op.status) if opctx.index == (opcount - 1): # Finalize on last opcode finalize = True if finalize: # All opcodes have been run, finalize job job.Finalize() # Write to disk. If the job status is final, this is the final write # allowed. Once the file has been written, it can be archived anytime. queue.UpdateJobUnlocked(job) assert not waitjob if finalize: logging.info("Finished job %s, status = %s", job.id, job.CalcStatus()) return self.FINISHED assert not waitjob or queue.depmgr.JobWaiting(job) if waitjob: return self.WAITDEP else: return self.DEFER finally: assert job.writable, "Job became read-only while being processed" class _JobDependencyManager(object): """Keeps track of job dependencies. """ (WAIT, ERROR, CANCEL, CONTINUE, WRONGSTATUS) = range(1, 6) def __init__(self, getstatus_fn): """Initializes this class. """ self._getstatus_fn = getstatus_fn self._waiters = {} def JobWaiting(self, job): """Checks if a job is waiting. """ return compat.any(job in jobs for jobs in self._waiters.values()) def CheckAndRegister(self, job, dep_job_id, dep_status): """Checks if a dependency job has the requested status. If the other job is not yet in a finalized status, the calling job will be notified (re-added to the workerpool) at a later point. @type job: L{_QueuedJob} @param job: Job object @type dep_job_id: int @param dep_job_id: ID of dependency job @type dep_status: list @param dep_status: Required status """ assert ht.TJobId(job.id) assert ht.TJobId(dep_job_id) assert ht.TListOf(ht.TElemOf(constants.JOBS_FINALIZED))(dep_status) if job.id == dep_job_id: return (self.ERROR, "Job can't depend on itself") # Get status of dependency job try: status = self._getstatus_fn(dep_job_id) except errors.JobLost as err: return (self.ERROR, "Dependency error: %s" % err) assert status in constants.JOB_STATUS_ALL job_id_waiters = self._waiters.setdefault(dep_job_id, set()) if status not in constants.JOBS_FINALIZED: # Register for notification and wait for job to finish job_id_waiters.add(job) return (self.WAIT, "Need to wait for job %s, wanted status '%s'" % (dep_job_id, dep_status)) # Remove from waiters list if job in job_id_waiters: job_id_waiters.remove(job) if (status == constants.JOB_STATUS_CANCELED and constants.JOB_STATUS_CANCELED not in dep_status): return (self.CANCEL, "Dependency job %s was cancelled" % dep_job_id) elif not dep_status or status in dep_status: return (self.CONTINUE, "Dependency job %s finished with status '%s'" % (dep_job_id, status)) else: return (self.WRONGSTATUS, "Dependency job %s finished with status '%s'," " not one of '%s' as required" % (dep_job_id, status, utils.CommaJoin(dep_status))) def _RemoveEmptyWaitersUnlocked(self): """Remove all jobs without actual waiters. """ for job_id in [job_id for (job_id, waiters) in self._waiters.items() if not waiters]: del self._waiters[job_id] class JobQueue(object): """Queue used to manage the jobs. """ def __init__(self, context, cfg): """Constructor for JobQueue. The constructor will initialize the job queue object and then start loading the current jobs from disk, either for starting them (if they were queue) or for aborting them (if they were already running). @type context: GanetiContext @param context: the context object for access to the configuration data and other ganeti objects """ self.context = context self._memcache = weakref.WeakValueDictionary() self._my_hostname = netutils.Hostname.GetSysName() # Get initial list of nodes self._nodes = dict((n.name, n.primary_ip) for n in cfg.GetAllNodesInfo().values() if n.master_candidate) # Remove master node self._nodes.pop(self._my_hostname, None) # Job dependencies self.depmgr = _JobDependencyManager(self._GetJobStatusForDependencies) def _GetRpc(self, address_list): """Gets RPC runner with context. """ return rpc.JobQueueRunner(self.context, address_list) @staticmethod def _CheckRpcResult(result, nodes, failmsg): """Verifies the status of an RPC call. Since we aim to keep consistency should this node (the current master) fail, we will log errors if our rpc fail, and especially log the case when more than half of the nodes fails. @param result: the data as returned from the rpc call @type nodes: list @param nodes: the list of nodes we made the call to @type failmsg: str @param failmsg: the identifier to be used for logging """ failed = [] success = [] for node in nodes: msg = result[node].fail_msg if msg: failed.append(node) logging.error("RPC call %s (%s) failed on node %s: %s", result[node].call, failmsg, node, msg) else: success.append(node) # +1 for the master node if (len(success) + 1) < len(failed): # TODO: Handle failing nodes logging.error("More than half of the nodes failed") def _GetNodeIp(self): """Helper for returning the node name/ip list. @rtype: (list, list) @return: a tuple of two lists, the first one with the node names and the second one with the node addresses """ # TODO: Change to "tuple(map(list, zip(*self._nodes.items())))"? name_list = list(self._nodes) addr_list = [self._nodes[name] for name in name_list] return name_list, addr_list def _UpdateJobQueueFile(self, file_name, data, replicate): """Writes a file locally and then replicates it to all nodes. This function will replace the contents of a file on the local node and then replicate it to all the other nodes we have. @type file_name: str @param file_name: the path of the file to be replicated @type data: str @param data: the new contents of the file @type replicate: boolean @param replicate: whether to spread the changes to the remote nodes """ getents = runtime.GetEnts() utils.WriteFile(file_name, data=data, uid=getents.masterd_uid, gid=getents.daemons_gid, mode=constants.JOB_QUEUE_FILES_PERMS) if replicate: names, addrs = self._GetNodeIp() result = _CallJqUpdate(self._GetRpc(addrs), names, file_name, data) self._CheckRpcResult(result, self._nodes, "Updating %s" % file_name) def _RenameFilesUnlocked(self, rename): """Renames a file locally and then replicate the change. This function will rename a file in the local queue directory and then replicate this rename to all the other nodes we have. @type rename: list of (old, new) @param rename: List containing tuples mapping old to new names """ # Rename them locally for old, new in rename: utils.RenameFile(old, new, mkdir=True) # ... and on all nodes names, addrs = self._GetNodeIp() result = self._GetRpc(addrs).call_jobqueue_rename(names, rename) self._CheckRpcResult(result, self._nodes, "Renaming files (%r)" % rename) @staticmethod def _GetJobPath(job_id): """Returns the job file for a given job id. @type job_id: str @param job_id: the job identifier @rtype: str @return: the path to the job file """ return utils.PathJoin(pathutils.QUEUE_DIR, "job-%s" % job_id) @staticmethod def _GetArchivedJobPath(job_id): """Returns the archived job file for a give job id. @type job_id: str @param job_id: the job identifier @rtype: str @return: the path to the archived job file """ return utils.PathJoin(pathutils.JOB_QUEUE_ARCHIVE_DIR, jstore.GetArchiveDirectory(job_id), "job-%s" % job_id) @staticmethod def _DetermineJobDirectories(archived): """Build list of directories containing job files. @type archived: bool @param archived: Whether to include directories for archived jobs @rtype: list """ result = [pathutils.QUEUE_DIR] if archived: archive_path = pathutils.JOB_QUEUE_ARCHIVE_DIR result.extend(utils.PathJoin(archive_path, job_file) for job_file in utils.ListVisibleFiles(archive_path)) return result @classmethod def _GetJobIDsUnlocked(cls, sort=True, archived=False): """Return all known job IDs. The method only looks at disk because it's a requirement that all jobs are present on disk (so in the _memcache we don't have any extra IDs). @type sort: boolean @param sort: perform sorting on the returned job ids @rtype: list @return: the list of job IDs """ jlist = [] for path in cls._DetermineJobDirectories(archived): for filename in utils.ListVisibleFiles(path): m = constants.JOB_FILE_RE.match(filename) if m: jlist.append(int(m.group(1))) if sort: jlist.sort() return jlist def _LoadJobUnlocked(self, job_id): """Loads a job from the disk or memory. Given a job id, this will return the cached job object if existing, or try to load the job from the disk. If loading from disk, it will also add the job to the cache. @type job_id: int @param job_id: the job id @rtype: L{_QueuedJob} or None @return: either None or the job object """ assert isinstance(job_id, int), "Job queue: Supplied job id is not an int!" job = self._memcache.get(job_id, None) if job: logging.debug("Found job %s in memcache", job_id) assert job.writable, "Found read-only job in memcache" return job try: job = self._LoadJobFromDisk(job_id, False) if job is None: return job except errors.JobFileCorrupted: old_path = self._GetJobPath(job_id) new_path = self._GetArchivedJobPath(job_id) if old_path == new_path: # job already archived (future case) logging.exception("Can't parse job %s", job_id) else: # non-archived case logging.exception("Can't parse job %s, will archive.", job_id) self._RenameFilesUnlocked([(old_path, new_path)]) return None assert job.writable, "Job just loaded is not writable" self._memcache[job_id] = job logging.debug("Added job %s to the cache", job_id) return job def _LoadJobFromDisk(self, job_id, try_archived, writable=None): """Load the given job file from disk. Given a job file, read, load and restore it in a _QueuedJob format. @type job_id: int @param job_id: job identifier @type try_archived: bool @param try_archived: Whether to try loading an archived job @rtype: L{_QueuedJob} or None @return: either None or the job object """ path_functions = [(self._GetJobPath, False)] if try_archived: path_functions.append((self._GetArchivedJobPath, True)) raw_data = None archived = None for (fn, archived) in path_functions: filepath = fn(job_id) logging.debug("Loading job from %s", filepath) try: raw_data = utils.ReadFile(filepath) except EnvironmentError as err: if err.errno != errno.ENOENT: raise else: break if not raw_data: logging.debug("No data available for job %s", job_id) return None if writable is None: writable = not archived try: data = serializer.LoadJson(raw_data) job = _QueuedJob.Restore(self, data, writable, archived) except Exception as err: # pylint: disable=W0703 raise errors.JobFileCorrupted(err) return job def SafeLoadJobFromDisk(self, job_id, try_archived, writable=None): """Load the given job file from disk. Given a job file, read, load and restore it in a _QueuedJob format. In case of error reading the job, it gets returned as None, and the exception is logged. @type job_id: int @param job_id: job identifier @type try_archived: bool @param try_archived: Whether to try loading an archived job @rtype: L{_QueuedJob} or None @return: either None or the job object """ try: return self._LoadJobFromDisk(job_id, try_archived, writable=writable) except (errors.JobFileCorrupted, EnvironmentError): logging.exception("Can't load/parse job %s", job_id) return None @classmethod def SubmitManyJobs(cls, jobs): """Create and store multiple jobs. """ return luxi.Client(address=pathutils.QUERY_SOCKET).SubmitManyJobs(jobs) @staticmethod def _ResolveJobDependencies(resolve_fn, deps): """Resolves relative job IDs in dependencies. @type resolve_fn: callable @param resolve_fn: Function to resolve a relative job ID @type deps: list @param deps: Dependencies @rtype: tuple; (boolean, string or list) @return: If successful (first tuple item), the returned list contains resolved job IDs along with the requested status; if not successful, the second element is an error message """ result = [] for (dep_job_id, dep_status) in deps: if ht.TRelativeJobId(dep_job_id): assert ht.TInt(dep_job_id) and dep_job_id < 0 try: job_id = resolve_fn(dep_job_id) except IndexError: # Abort return (False, "Unable to resolve relative job ID %s" % dep_job_id) else: job_id = dep_job_id result.append((job_id, dep_status)) return (True, result) def _GetJobStatusForDependencies(self, job_id): """Gets the status of a job for dependencies. @type job_id: int @param job_id: Job ID @raise errors.JobLost: If job can't be found """ # Not using in-memory cache as doing so would require an exclusive lock # Try to load from disk job = self.SafeLoadJobFromDisk(job_id, True, writable=False) if job: assert not job.writable, "Got writable job" # pylint: disable=E1101 if job: return job.CalcStatus() raise errors.JobLost("Job %s not found" % job_id) def UpdateJobUnlocked(self, job, replicate=True): """Update a job's on disk storage. After a job has been modified, this function needs to be called in order to write the changes to disk and replicate them to the other nodes. @type job: L{_QueuedJob} @param job: the changed job @type replicate: boolean @param replicate: whether to replicate the change to remote nodes """ if __debug__: finalized = job.CalcStatus() in constants.JOBS_FINALIZED assert (finalized ^ (job.end_timestamp is None)) assert job.writable, "Can't update read-only job" assert not job.archived, "Can't update archived job" filename = self._GetJobPath(job.id) data = serializer.DumpJson(job.Serialize()) logging.debug("Writing job %s to %s", job.id, filename) self._UpdateJobQueueFile(filename, data, replicate) def HasJobBeenFinalized(self, job_id): """Checks if a job has been finalized. @type job_id: int @param job_id: Job identifier @rtype: boolean @return: True if the job has been finalized, False if the timeout has been reached, None if the job doesn't exist """ job = self.SafeLoadJobFromDisk(job_id, True, writable=False) if job is not None: return job.CalcStatus() in constants.JOBS_FINALIZED elif cluster.LUClusterDestroy.clusterHasBeenDestroyed: # FIXME: The above variable is a temporary workaround until the Python job # queue is completely removed. When removing the job queue, also remove # the variable from LUClusterDestroy. return True else: return None def CancelJob(self, job_id): """Cancels a job. This will only succeed if the job has not started yet. @type job_id: int @param job_id: job ID of job to be cancelled. """ logging.info("Cancelling job %s", job_id) return self._ModifyJobUnlocked(job_id, lambda job: job.Cancel()) def ChangeJobPriority(self, job_id, priority): """Changes a job's priority. @type job_id: int @param job_id: ID of the job whose priority should be changed @type priority: int @param priority: New priority """ logging.info("Changing priority of job %s to %s", job_id, priority) if priority not in constants.OP_PRIO_SUBMIT_VALID: allowed = utils.CommaJoin(constants.OP_PRIO_SUBMIT_VALID) raise errors.GenericError("Invalid priority %s, allowed are %s" % (priority, allowed)) def fn(job): (success, msg) = job.ChangePriority(priority) return (success, msg) return self._ModifyJobUnlocked(job_id, fn) def _ModifyJobUnlocked(self, job_id, mod_fn): """Modifies a job. @type job_id: int @param job_id: Job ID @type mod_fn: callable @param mod_fn: Modifying function, receiving job object as parameter, returning tuple of (status boolean, message string) """ job = self._LoadJobUnlocked(job_id) if not job: logging.debug("Job %s not found", job_id) return (False, "Job %s not found" % job_id) assert job.writable, "Can't modify read-only job" assert not job.archived, "Can't modify archived job" (success, msg) = mod_fn(job) if success: # If the job was finalized (e.g. cancelled), this is the final write # allowed. The job can be archived anytime. self.UpdateJobUnlocked(job) return (success, msg) ganeti-3.1.0~rc2/lib/jqueue/exec.py000064400000000000000000000150111476477700300171440ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing executing of a job as a separate process The complete protocol of initializing a job is described in the haskell module Ganeti.Query.Exec """ import contextlib import logging import os import signal import sys import time from ganeti import mcpu from ganeti.server import masterd from ganeti.rpc import transport from ganeti import serializer from ganeti import utils from ganeti import pathutils from ganeti.utils import livelock from ganeti.jqueue import _JobProcessor def _SetupJob(): """Setup the process to execute the job Retrieve the job id from argv, create a livelock and pass it to the master process and finally obtain any secret parameters. This also closes standard input/output. @rtype: (int, string, json encoding of a list of dicts) """ job_id = int(sys.argv[1]) ts = time.time() llock = livelock.LiveLock("job_%06d" % job_id) logging.debug("Opening transport over stdin/out") with contextlib.closing(transport.FdTransport((0, 1))) as trans: logging.debug("Sending livelock name to master") trans.Call(llock.GetPath()) # pylint: disable=E1101 logging.debug("Reading secret parameters from the master process") secret_params = trans.Call("") # pylint: disable=E1101 logging.debug("Got secret parameters.") return (job_id, llock, secret_params) def RestorePrivateValueWrapping(json): """Wrap private values in JSON decoded structure. @param json: the json-decoded value to protect. """ result = [] for secrets_dict in json: if secrets_dict is None: data = serializer.PrivateDict() else: data = serializer.PrivateDict(secrets_dict) result.append(data) return result def main(): debug = int(os.environ["GNT_DEBUG"]) logname = pathutils.GetLogFilename("jobs") utils.SetupLogging(logname, "job-startup", debug=debug) (job_id, llock, secret_params_serialized) = _SetupJob() secret_params = "" if secret_params_serialized: secret_params_json = serializer.LoadJson(secret_params_serialized) secret_params = RestorePrivateValueWrapping(secret_params_json) utils.SetupLogging(logname, "job-%s" % (job_id,), debug=debug) try: logging.debug("Preparing the context and the configuration") context = masterd.GanetiContext(llock) logging.debug("Registering signal handlers") cancel = [False] prio_change = [False] def _TermHandler(signum, _frame): logging.info("Killed by signal %d", signum) cancel[0] = True signal.signal(signal.SIGTERM, _TermHandler) def _HupHandler(signum, _frame): logging.debug("Received signal %d, old flag was %s, will set to True", signum, mcpu.sighupReceived) mcpu.sighupReceived[0] = True signal.signal(signal.SIGHUP, _HupHandler) def _User1Handler(signum, _frame): logging.info("Received signal %d, indicating priority change", signum) prio_change[0] = True signal.signal(signal.SIGUSR1, _User1Handler) job = context.jobqueue.SafeLoadJobFromDisk(job_id, False) job.SetPid(os.getpid()) if secret_params: for i in range(0, len(secret_params)): if hasattr(job.ops[i].input, "osparams_secret"): job.ops[i].input.osparams_secret = secret_params[i] execfun = mcpu.Processor(context, job_id, job_id).ExecOpCode proc = _JobProcessor(context.jobqueue, execfun, job) result = _JobProcessor.DEFER while result != _JobProcessor.FINISHED: result = proc() if result == _JobProcessor.WAITDEP and not cancel[0]: # Normally, the scheduler should avoid starting a job where the # dependencies are not yet finalised. So warn, but wait an continue. logging.warning("Got started despite a dependency not yet finished") time.sleep(5) if cancel[0]: logging.debug("Got cancel request, cancelling job %d", job_id) r = context.jobqueue.CancelJob(job_id) job = context.jobqueue.SafeLoadJobFromDisk(job_id, False) proc = _JobProcessor(context.jobqueue, execfun, job) logging.debug("CancelJob result for job %d: %s", job_id, r) cancel[0] = False if prio_change[0]: logging.debug("Received priority-change request") try: fname = os.path.join(pathutils.LUXID_MESSAGE_DIR, "%d.prio" % job_id) new_prio = int(utils.ReadFile(fname)) utils.RemoveFile(fname) logging.debug("Changing priority of job %d to %d", job_id, new_prio) r = context.jobqueue.ChangeJobPriority(job_id, new_prio) job = context.jobqueue.SafeLoadJobFromDisk(job_id, False) proc = _JobProcessor(context.jobqueue, execfun, job) logging.debug("Result of changing priority of %d to %d: %s", job_id, new_prio, r) except Exception: # pylint: disable=W0703 logging.warning("Informed of priority change, but could not" " read new priority") prio_change[0] = False except Exception: # pylint: disable=W0703 logging.exception("Exception when trying to run job %d", job_id) finally: logging.debug("Job %d finalized", job_id) logging.debug("Removing livelock file %s", llock.GetPath()) os.remove(llock.GetPath()) sys.exit(0) if __name__ == '__main__': main() ganeti-3.1.0~rc2/lib/jstore.py000064400000000000000000000152361476477700300162410ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing the job queue handling.""" import errno import os from ganeti import constants from ganeti import errors from ganeti import runtime from ganeti import utils from ganeti import pathutils JOBS_PER_ARCHIVE_DIRECTORY = constants.JSTORE_JOBS_PER_ARCHIVE_DIRECTORY def _ReadNumericFile(file_name): """Reads a file containing a number. @rtype: None or int @return: None if file is not found, otherwise number """ try: contents = utils.ReadFile(file_name) except EnvironmentError as err: if err.errno in (errno.ENOENT, ): return None raise try: return int(contents) except (ValueError, TypeError) as err: # Couldn't convert to int raise errors.JobQueueError("Content of file '%s' is not numeric: %s" % (file_name, err)) def ReadSerial(): """Read the serial file. The queue should be locked while this function is called. """ return _ReadNumericFile(pathutils.JOB_QUEUE_SERIAL_FILE) def ReadVersion(): """Read the queue version. The queue should be locked while this function is called. """ return _ReadNumericFile(pathutils.JOB_QUEUE_VERSION_FILE) def InitAndVerifyQueue(must_lock): """Open and lock job queue. If necessary, the queue is automatically initialized. @type must_lock: bool @param must_lock: Whether an exclusive lock must be held. @rtype: utils.FileLock @return: Lock object for the queue. This can be used to change the locking mode. """ getents = runtime.GetEnts() # Lock queue queue_lock = utils.FileLock.Open(pathutils.JOB_QUEUE_LOCK_FILE) try: # The queue needs to be locked in exclusive mode to write to the serial and # version files. if must_lock: queue_lock.Exclusive(blocking=True) holding_lock = True else: try: queue_lock.Exclusive(blocking=False) holding_lock = True except errors.LockError: # Ignore errors and assume the process keeping the lock checked # everything. holding_lock = False if holding_lock: # Verify version version = ReadVersion() if version is None: # Write new version file utils.WriteFile(pathutils.JOB_QUEUE_VERSION_FILE, uid=getents.masterd_uid, gid=getents.daemons_gid, mode=constants.JOB_QUEUE_FILES_PERMS, data="%s\n" % constants.JOB_QUEUE_VERSION) # Read again version = ReadVersion() if version != constants.JOB_QUEUE_VERSION: raise errors.JobQueueError("Found job queue version %s, expected %s", version, constants.JOB_QUEUE_VERSION) serial = ReadSerial() if serial is None: # Write new serial file utils.WriteFile(pathutils.JOB_QUEUE_SERIAL_FILE, uid=getents.masterd_uid, gid=getents.daemons_gid, mode=constants.JOB_QUEUE_FILES_PERMS, data="%s\n" % 0) # Read again serial = ReadSerial() if serial is None: # There must be a serious problem raise errors.JobQueueError("Can't read/parse the job queue" " serial file") if not must_lock: # There's no need for more error handling. Closing the lock # file below in case of an error will unlock it anyway. queue_lock.Unlock() except: queue_lock.Close() raise return queue_lock def CheckDrainFlag(): """Check if the queue is marked to be drained. This currently uses the queue drain file, which makes it a per-node flag. In the future this can be moved to the config file. @rtype: boolean @return: True if the job queue is marked drained """ return os.path.exists(pathutils.JOB_QUEUE_DRAIN_FILE) def SetDrainFlag(drain_flag): """Sets the drain flag for the queue. @type drain_flag: boolean @param drain_flag: Whether to set or unset the drain flag @attention: This function should only called the current holder of the queue lock """ getents = runtime.GetEnts() if drain_flag: utils.WriteFile(pathutils.JOB_QUEUE_DRAIN_FILE, data="", uid=getents.masterd_uid, gid=getents.daemons_gid, mode=constants.JOB_QUEUE_FILES_PERMS) else: utils.RemoveFile(pathutils.JOB_QUEUE_DRAIN_FILE) assert (not drain_flag) ^ CheckDrainFlag() def FormatJobID(job_id): """Convert a job ID to int format. Currently this just is a no-op that performs some checks, but if we want to change the job id format this will abstract this change. @type job_id: int or long @param job_id: the numeric job id @rtype: int @return: the formatted job id """ if not isinstance(job_id, int): raise errors.ProgrammerError("Job ID '%s' not numeric" % job_id) if job_id < 0: raise errors.ProgrammerError("Job ID %s is negative" % job_id) return job_id def GetArchiveDirectory(job_id): """Returns the archive directory for a job. @type job_id: str @param job_id: Job identifier @rtype: str @return: Directory name """ return str(ParseJobId(job_id) // JOBS_PER_ARCHIVE_DIRECTORY) def ParseJobId(job_id): """Parses a job ID and converts it to integer. """ try: return int(job_id) except (ValueError, TypeError): raise errors.ParameterError("Invalid job ID '%s'" % job_id) ganeti-3.1.0~rc2/lib/locking.py000064400000000000000000000640101476477700300163530ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing the Ganeti locking code.""" # pylint: disable=W0212 # W0212 since e.g. LockSet methods use (a lot) the internals of # SharedLock import os import select import threading import errno import logging import heapq import time from ganeti import errors from ganeti import utils from ganeti import compat from ganeti import query _EXCLUSIVE_TEXT = "exclusive" _SHARED_TEXT = "shared" _DELETED_TEXT = "deleted" _DEFAULT_PRIORITY = 0 #: Minimum timeout required to consider scheduling a pending acquisition #: (seconds) _LOCK_ACQUIRE_MIN_TIMEOUT = (1.0 / 1000) def ssynchronized(mylock, shared=0): """Shared Synchronization decorator. Calls the function holding the given lock, either in exclusive or shared mode. It requires the passed lock to be a SharedLock (or support its semantics). @type mylock: lockable object or string @param mylock: lock to acquire or class member name of the lock to acquire """ def wrap(fn): def sync_function(*args, **kwargs): if isinstance(mylock, str): assert args, "cannot ssynchronize on non-class method: self not found" # args[0] is "self" lock = getattr(args[0], mylock) else: lock = mylock lock.acquire(shared=shared) try: return fn(*args, **kwargs) finally: lock.release() return sync_function return wrap class _SingleNotifyPipeConditionWaiter(object): """Helper class for SingleNotifyPipeCondition """ __slots__ = [ "_fd", ] def __init__(self, fd): """Constructor for _SingleNotifyPipeConditionWaiter @type fd: int @param fd: File descriptor to wait for """ object.__init__(self) self._fd = fd def __call__(self, timeout): """Wait for something to happen on the pipe. @type timeout: float or None @param timeout: Timeout for waiting (can be None) """ running_timeout = utils.RunningTimeout(timeout, True) poller = select.poll() poller.register(self._fd, select.POLLHUP) while True: remaining_time = running_timeout.Remaining() if remaining_time is not None: if remaining_time < 0.0: break # Our calculation uses seconds, poll() wants milliseconds remaining_time *= 1000 try: result = poller.poll(remaining_time) except EnvironmentError as err: if err.errno != errno.EINTR: raise result = None # Check whether we were notified if result and result[0][0] == self._fd: break class _BaseCondition(object): """Base class containing common code for conditions. Some of this code is taken from python's threading module. """ __slots__ = [ "_lock", "acquire", "release", "_is_owned", "_acquire_restore", "_release_save", ] def __init__(self, lock): """Constructor for _BaseCondition. @type lock: threading.Lock @param lock: condition base lock """ object.__init__(self) try: self._release_save = lock._release_save except AttributeError: self._release_save = self._base_release_save try: self._acquire_restore = lock._acquire_restore except AttributeError: self._acquire_restore = self._base_acquire_restore try: self._is_owned = lock.is_owned except AttributeError: self._is_owned = self._base_is_owned self._lock = lock # Export the lock's acquire() and release() methods self.acquire = lock.acquire self.release = lock.release def _base_is_owned(self): """Check whether lock is owned by current thread. """ if self._lock.acquire(0): self._lock.release() return False return True def _base_release_save(self): self._lock.release() def _base_acquire_restore(self, _): self._lock.acquire() def _check_owned(self): """Raise an exception if the current thread doesn't own the lock. """ if not self._is_owned(): raise RuntimeError("cannot work with un-aquired lock") class SingleNotifyPipeCondition(_BaseCondition): """Condition which can only be notified once. This condition class uses pipes and poll, internally, to be able to wait for notification with a timeout, without resorting to polling. It is almost compatible with Python's threading.Condition, with the following differences: - notify_all can only be called once, and no wait can happen after that - notify is not supported, only notify_all """ __slots__ = [ "_read_fd", "_write_fd", "_nwaiters", "_notified", ] _waiter_class = _SingleNotifyPipeConditionWaiter def __init__(self, lock): """Constructor for SingleNotifyPipeCondition """ _BaseCondition.__init__(self, lock) self._nwaiters = 0 self._notified = False self._read_fd = None self._write_fd = None def _check_unnotified(self): """Throws an exception if already notified. """ if self._notified: raise RuntimeError("cannot use already notified condition") def _Cleanup(self): """Cleanup open file descriptors, if any. """ if self._read_fd is not None: os.close(self._read_fd) self._read_fd = None if self._write_fd is not None: os.close(self._write_fd) self._write_fd = None def wait(self, timeout): """Wait for a notification. @type timeout: float or None @param timeout: Waiting timeout (can be None) """ self._check_owned() self._check_unnotified() self._nwaiters += 1 try: if self._read_fd is None: (self._read_fd, self._write_fd) = os.pipe() wait_fn = self._waiter_class(self._read_fd) state = self._release_save() try: # Wait for notification wait_fn(timeout) finally: # Re-acquire lock self._acquire_restore(state) finally: self._nwaiters -= 1 if self._nwaiters == 0: self._Cleanup() def notify_all(self): """Close the writing side of the pipe to notify all waiters. """ self._check_owned() self._check_unnotified() self._notified = True if self._write_fd is not None: os.close(self._write_fd) self._write_fd = None class PipeCondition(_BaseCondition): """Group-only non-polling condition with counters. This condition class uses pipes and poll, internally, to be able to wait for notification with a timeout, without resorting to polling. It is almost compatible with Python's threading.Condition, but only supports notify_all and non-recursive locks. As an additional features it's able to report whether there are any waiting threads. """ __slots__ = [ "_waiters", "_single_condition", ] _single_condition_class = SingleNotifyPipeCondition def __init__(self, lock): """Initializes this class. """ _BaseCondition.__init__(self, lock) self._waiters = set() self._single_condition = self._single_condition_class(self._lock) def wait(self, timeout): """Wait for a notification. @type timeout: float or None @param timeout: Waiting timeout (can be None) """ self._check_owned() # Keep local reference to the pipe. It could be replaced by another thread # notifying while we're waiting. cond = self._single_condition self._waiters.add(threading.current_thread()) try: cond.wait(timeout) finally: self._check_owned() self._waiters.remove(threading.current_thread()) def notify_all(self): """Notify all currently waiting threads. """ self._check_owned() self._single_condition.notify_all() self._single_condition = self._single_condition_class(self._lock) def get_waiting(self): """Returns a list of all waiting threads. """ self._check_owned() return self._waiters def has_waiting(self): """Returns whether there are active waiters. """ self._check_owned() return bool(self._waiters) def __repr__(self): return ("<%s.%s waiters=%s at %#x>" % (self.__class__.__module__, self.__class__.__name__, self._waiters, id(self))) class _PipeConditionWithMode(PipeCondition): __slots__ = [ "shared", ] def __init__(self, lock, shared): """Initializes this class. """ self.shared = shared PipeCondition.__init__(self, lock) class SharedLock(object): """Implements a shared lock. Multiple threads can acquire the lock in a shared way by calling C{acquire(shared=1)}. In order to acquire the lock in an exclusive way threads can call C{acquire(shared=0)}. Notes on data structures: C{__pending} contains a priority queue (heapq) of all pending acquires: C{[(priority1: prioqueue1), (priority2: prioqueue2), ...]}. Each per-priority queue contains a normal in-order list of conditions to be notified when the lock can be acquired. Shared locks are grouped together by priority and the condition for them is stored in C{__pending_shared} if it already exists. C{__pending_by_prio} keeps references for the per-priority queues indexed by priority for faster access. @type name: string @ivar name: the name of the lock """ __slots__ = [ "__weakref__", "__deleted", "__exc", "__lock", "__pending", "__pending_by_prio", "__pending_shared", "__shr", "__time_fn", "name", ] __condition_class = _PipeConditionWithMode def __init__(self, name, monitor=None, _time_fn=time.time): """Construct a new SharedLock. @param name: the name of the lock @param monitor: Lock monitor with which to register """ object.__init__(self) self.name = name # Used for unittesting self.__time_fn = _time_fn # Internal lock self.__lock = threading.Lock() # Queue containing waiting acquires self.__pending = [] self.__pending_by_prio = {} self.__pending_shared = {} # Current lock holders self.__shr = set() self.__exc = None # is this lock in the deleted state? self.__deleted = False # Register with lock monitor if monitor: logging.debug("Adding lock %s to monitor", name) monitor.RegisterLock(self) def __repr__(self): return ("<%s.%s name=%s at %#x>" % (self.__class__.__module__, self.__class__.__name__, self.name, id(self))) def GetLockInfo(self, requested): """Retrieves information for querying locks. @type requested: set @param requested: Requested information, see C{query.LQ_*} """ self.__lock.acquire() try: # Note: to avoid unintentional race conditions, no references to # modifiable objects should be returned unless they were created in this # function. mode = None owner_names = None if query.LQ_MODE in requested: if self.__deleted: mode = _DELETED_TEXT assert not (self.__exc or self.__shr) elif self.__exc: mode = _EXCLUSIVE_TEXT elif self.__shr: mode = _SHARED_TEXT # Current owner(s) are wanted if query.LQ_OWNER in requested: if self.__exc: owner = [self.__exc] else: owner = self.__shr if owner: assert not self.__deleted owner_names = [i.name for i in owner] # Pending acquires are wanted if query.LQ_PENDING in requested: pending = [] # Sorting instead of copying and using heaq functions for simplicity for (_, prioqueue) in sorted(self.__pending): for cond in prioqueue: if cond.shared: pendmode = _SHARED_TEXT else: pendmode = _EXCLUSIVE_TEXT # List of names will be sorted in L{query._GetLockPending} pending.append((pendmode, [i.name for i in cond.get_waiting()])) else: pending = None return [(self.name, mode, owner_names, pending)] finally: self.__lock.release() def __check_deleted(self): """Raises an exception if the lock has been deleted. """ if self.__deleted: raise errors.LockError("Deleted lock %s" % self.name) def __is_sharer(self): """Is the current thread sharing the lock at this time? """ return threading.current_thread() in self.__shr def __is_exclusive(self): """Is the current thread holding the lock exclusively at this time? """ return threading.current_thread() == self.__exc def __is_owned(self, shared=-1): """Is the current thread somehow owning the lock at this time? This is a private version of the function, which presumes you're holding the internal lock. """ if shared < 0: return self.__is_sharer() or self.__is_exclusive() elif shared: return self.__is_sharer() else: return self.__is_exclusive() def is_owned(self, shared=-1): """Is the current thread somehow owning the lock at this time? @param shared: - < 0: check for any type of ownership (default) - 0: check for exclusive ownership - > 0: check for shared ownership """ self.__lock.acquire() try: return self.__is_owned(shared=shared) finally: self.__lock.release() #: Necessary to remain compatible with threading.Condition, which tries to #: retrieve a locks' "_is_owned" attribute _is_owned = is_owned def _count_pending(self): """Returns the number of pending acquires. @rtype: int """ self.__lock.acquire() try: return sum(len(prioqueue) for (_, prioqueue) in self.__pending) finally: self.__lock.release() def _check_empty(self): """Checks whether there are any pending acquires. @rtype: bool """ self.__lock.acquire() try: # Order is important: __find_first_pending_queue modifies __pending (_, prioqueue) = self.__find_first_pending_queue() return not (prioqueue or self.__pending or self.__pending_by_prio or self.__pending_shared) finally: self.__lock.release() def __do_acquire(self, shared): """Actually acquire the lock. """ if shared: self.__shr.add(threading.current_thread()) else: self.__exc = threading.current_thread() def __can_acquire(self, shared): """Determine whether lock can be acquired. """ if shared: return self.__exc is None else: return len(self.__shr) == 0 and self.__exc is None def __find_first_pending_queue(self): """Tries to find the topmost queued entry with pending acquires. Removes empty entries while going through the list. """ while self.__pending: (priority, prioqueue) = self.__pending[0] if prioqueue: return (priority, prioqueue) # Remove empty queue heapq.heappop(self.__pending) del self.__pending_by_prio[priority] assert priority not in self.__pending_shared return (None, None) def __is_on_top(self, cond): """Checks whether the passed condition is on top of the queue. The caller must make sure the queue isn't empty. """ (_, prioqueue) = self.__find_first_pending_queue() return cond == prioqueue[0] def __acquire_unlocked(self, shared, timeout, priority): """Acquire a shared lock. @param shared: whether to acquire in shared mode; by default an exclusive lock will be acquired @param timeout: maximum waiting time before giving up @type priority: integer @param priority: Priority for acquiring lock """ self.__check_deleted() # We cannot acquire the lock if we already have it assert not self.__is_owned(), ("double acquire() on a non-recursive lock" " %s" % self.name) # Remove empty entries from queue self.__find_first_pending_queue() # Check whether someone else holds the lock or there are pending acquires. if not self.__pending and self.__can_acquire(shared): # Apparently not, can acquire lock directly. self.__do_acquire(shared) return True # The lock couldn't be acquired right away, so if a timeout is given and is # considered too short, return right away as scheduling a pending # acquisition is quite expensive if timeout is not None and timeout < _LOCK_ACQUIRE_MIN_TIMEOUT: return False prioqueue = self.__pending_by_prio.get(priority, None) if shared: # Try to re-use condition for shared acquire wait_condition = self.__pending_shared.get(priority, None) assert (wait_condition is None or (wait_condition.shared and wait_condition in prioqueue)) else: wait_condition = None if wait_condition is None: if prioqueue is None: assert priority not in self.__pending_by_prio prioqueue = [] heapq.heappush(self.__pending, (priority, prioqueue)) self.__pending_by_prio[priority] = prioqueue wait_condition = self.__condition_class(self.__lock, shared) prioqueue.append(wait_condition) if shared: # Keep reference for further shared acquires on same priority. This is # better than trying to find it in the list of pending acquires. assert priority not in self.__pending_shared self.__pending_shared[priority] = wait_condition wait_start = self.__time_fn() acquired = False try: # Wait until we become the topmost acquire in the queue or the timeout # expires. while True: if self.__is_on_top(wait_condition) and self.__can_acquire(shared): self.__do_acquire(shared) acquired = True break # A lot of code assumes blocking acquires always succeed, therefore we # can never return False for a blocking acquire if (timeout is not None and utils.TimeoutExpired(wait_start, timeout, _time_fn=self.__time_fn)): break # Wait for notification wait_condition.wait(timeout) self.__check_deleted() finally: # Remove condition from queue if there are no more waiters if not wait_condition.has_waiting(): prioqueue.remove(wait_condition) if wait_condition.shared: # Remove from list of shared acquires if it wasn't while releasing # (e.g. on lock deletion) self.__pending_shared.pop(priority, None) return acquired def acquire(self, shared=0, timeout=None, priority=None, test_notify=None): """Acquire a shared lock. @type shared: integer (0/1) used as a boolean @param shared: whether to acquire in shared mode; by default an exclusive lock will be acquired @type timeout: float @param timeout: maximum waiting time before giving up @type priority: integer @param priority: Priority for acquiring lock @type test_notify: callable or None @param test_notify: Special callback function for unittesting """ if priority is None: priority = _DEFAULT_PRIORITY self.__lock.acquire() try: # We already got the lock, notify now if __debug__ and callable(test_notify): test_notify() return self.__acquire_unlocked(shared, timeout, priority) finally: self.__lock.release() def downgrade(self): """Changes the lock mode from exclusive to shared. Pending acquires in shared mode on the same priority will go ahead. """ self.__lock.acquire() try: assert self.__is_owned(), "Lock must be owned" if self.__is_exclusive(): # Do nothing if the lock is already acquired in shared mode self.__exc = None self.__do_acquire(1) # Important: pending shared acquires should only jump ahead if there # was a transition from exclusive to shared, otherwise an owner of a # shared lock can keep calling this function to push incoming shared # acquires (priority, prioqueue) = self.__find_first_pending_queue() if prioqueue: # Is there a pending shared acquire on this priority? cond = self.__pending_shared.pop(priority, None) if cond: assert cond.shared assert cond in prioqueue # Ensure shared acquire is on top of queue if len(prioqueue) > 1: prioqueue.remove(cond) prioqueue.insert(0, cond) # Notify cond.notify_all() assert not self.__is_exclusive() assert self.__is_sharer() return True finally: self.__lock.release() def release(self): """Release a Shared Lock. You must have acquired the lock, either in shared or in exclusive mode, before calling this function. """ self.__lock.acquire() try: assert self.__is_exclusive() or self.__is_sharer(), \ "Cannot release non-owned lock" # Autodetect release type if self.__is_exclusive(): self.__exc = None notify = True else: self.__shr.remove(threading.current_thread()) notify = not self.__shr # Notify topmost condition in queue if there are no owners left (for # shared locks) if notify: self.__notify_topmost() finally: self.__lock.release() def __notify_topmost(self): """Notifies topmost condition in queue of pending acquires. """ (priority, prioqueue) = self.__find_first_pending_queue() if prioqueue: cond = prioqueue[0] cond.notify_all() if cond.shared: # Prevent further shared acquires from sneaking in while waiters are # notified self.__pending_shared.pop(priority, None) def _notify_topmost(self): """Exported version of L{__notify_topmost}. """ self.__lock.acquire() try: return self.__notify_topmost() finally: self.__lock.release() def delete(self, timeout=None, priority=None): """Delete a Shared Lock. This operation will declare the lock for removal. First the lock will be acquired in exclusive mode if you don't already own it, then the lock will be put in a state where any future and pending acquire() fail. @type timeout: float @param timeout: maximum waiting time before giving up @type priority: integer @param priority: Priority for acquiring lock """ if priority is None: priority = _DEFAULT_PRIORITY self.__lock.acquire() try: assert not self.__is_sharer(), "Cannot delete() a lock while sharing it" self.__check_deleted() # The caller is allowed to hold the lock exclusively already. acquired = self.__is_exclusive() if not acquired: acquired = self.__acquire_unlocked(0, timeout, priority) if acquired: assert self.__is_exclusive() and not self.__is_sharer(), \ "Lock wasn't acquired in exclusive mode" self.__deleted = True self.__exc = None assert not (self.__exc or self.__shr), "Found owner during deletion" # Notify all acquires. They'll throw an error. for (_, prioqueue) in self.__pending: for cond in prioqueue: cond.notify_all() assert self.__deleted return acquired finally: self.__lock.release() def _release_save(self): shared = self.__is_sharer() self.release() return shared def _acquire_restore(self, shared): self.acquire(shared=shared) # Whenever we want to acquire a full LockSet we pass None as the value # to acquire. Hide this behind this nicely named constant. ALL_SET = None LOCKSET_NAME = "[lockset]" def _TimeoutZero(): """Returns the number zero. """ return 0 class _AcquireTimeout(Exception): """Internal exception to abort an acquire on a timeout. """ # Locking levels, must be acquired in increasing order. Current rules are: # - At level LEVEL_CLUSTER resides the Big Ganeti Lock (BGL) which must be # acquired before performing any operation, either in shared or exclusive # mode. Acquiring the BGL in exclusive mode is discouraged and should be # avoided.. # - At levels LEVEL_NODE and LEVEL_INSTANCE reside node and instance locks. If # you need more than one node, or more than one instance, acquire them at the # same time. # - LEVEL_NODE_RES is for node resources and should be used by operations with # possibly high impact on the node's disks. (LEVEL_CLUSTER, LEVEL_INSTANCE, LEVEL_NODEGROUP, LEVEL_NODE, LEVEL_NODE_RES, LEVEL_NETWORK) = range(0, 6) LEVELS = [ LEVEL_CLUSTER, LEVEL_INSTANCE, LEVEL_NODEGROUP, LEVEL_NODE, LEVEL_NODE_RES, LEVEL_NETWORK, ] # Lock levels which are modifiable LEVELS_MOD = compat.UniqueFrozenset([ LEVEL_NODE_RES, LEVEL_NODE, LEVEL_NODEGROUP, LEVEL_INSTANCE, LEVEL_NETWORK, ]) #: Lock level names (make sure to use singular form) LEVEL_NAMES = { LEVEL_CLUSTER: "cluster", LEVEL_INSTANCE: "instance", LEVEL_NODEGROUP: "nodegroup", LEVEL_NODE: "node", LEVEL_NODE_RES: "node-res", LEVEL_NETWORK: "network", } # Constant for the big ganeti lock BGL = "BGL" ganeti-3.1.0~rc2/lib/luxi.py000064400000000000000000000223521476477700300157110ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module for the LUXI protocol This module implements the local unix socket protocol. You only need this module and the opcodes module in the client program in order to communicate with the master. The module is also used by the master daemon. """ from ganeti import constants from ganeti import pathutils from ganeti import objects import ganeti.rpc.client as cl from ganeti.rpc.errors import RequestError from ganeti.rpc.transport import Transport __all__ = [ # classes: "Client" ] REQ_SUBMIT_JOB = constants.LUXI_REQ_SUBMIT_JOB REQ_SUBMIT_JOB_TO_DRAINED_QUEUE = constants.LUXI_REQ_SUBMIT_JOB_TO_DRAINED_QUEUE REQ_SUBMIT_MANY_JOBS = constants.LUXI_REQ_SUBMIT_MANY_JOBS REQ_PICKUP_JOB = constants.LUXI_REQ_PICKUP_JOB REQ_WAIT_FOR_JOB_CHANGE = constants.LUXI_REQ_WAIT_FOR_JOB_CHANGE REQ_CANCEL_JOB = constants.LUXI_REQ_CANCEL_JOB REQ_ARCHIVE_JOB = constants.LUXI_REQ_ARCHIVE_JOB REQ_CHANGE_JOB_PRIORITY = constants.LUXI_REQ_CHANGE_JOB_PRIORITY REQ_AUTO_ARCHIVE_JOBS = constants.LUXI_REQ_AUTO_ARCHIVE_JOBS REQ_QUERY = constants.LUXI_REQ_QUERY REQ_QUERY_FIELDS = constants.LUXI_REQ_QUERY_FIELDS REQ_QUERY_JOBS = constants.LUXI_REQ_QUERY_JOBS REQ_QUERY_FILTERS = constants.LUXI_REQ_QUERY_FILTERS REQ_REPLACE_FILTER = constants.LUXI_REQ_REPLACE_FILTER REQ_DELETE_FILTER = constants.LUXI_REQ_DELETE_FILTER REQ_QUERY_INSTANCES = constants.LUXI_REQ_QUERY_INSTANCES REQ_QUERY_NODES = constants.LUXI_REQ_QUERY_NODES REQ_QUERY_GROUPS = constants.LUXI_REQ_QUERY_GROUPS REQ_QUERY_NETWORKS = constants.LUXI_REQ_QUERY_NETWORKS REQ_QUERY_EXPORTS = constants.LUXI_REQ_QUERY_EXPORTS REQ_QUERY_CONFIG_VALUES = constants.LUXI_REQ_QUERY_CONFIG_VALUES REQ_QUERY_CLUSTER_INFO = constants.LUXI_REQ_QUERY_CLUSTER_INFO REQ_QUERY_TAGS = constants.LUXI_REQ_QUERY_TAGS REQ_SET_DRAIN_FLAG = constants.LUXI_REQ_SET_DRAIN_FLAG REQ_SET_WATCHER_PAUSE = constants.LUXI_REQ_SET_WATCHER_PAUSE REQ_ALL = constants.LUXI_REQ_ALL DEF_RWTO = constants.LUXI_DEF_RWTO WFJC_TIMEOUT = constants.LUXI_WFJC_TIMEOUT class Client(cl.AbstractClient): """High-level client implementation. This uses a backing Transport-like class on top of which it implements data serialization/deserialization. """ def __init__(self, address=None, timeouts=None, transport=Transport): """Constructor for the Client class. Arguments are the same as for L{AbstractClient}. """ super(Client, self).__init__(timeouts, transport) # Override the version of the protocol: self.version = constants.LUXI_VERSION # Store the socket address if address is None: address = pathutils.QUERY_SOCKET self.address = address self._InitTransport() def _GetAddress(self): return self.address def SetQueueDrainFlag(self, drain_flag): return self.CallMethod(REQ_SET_DRAIN_FLAG, (drain_flag, )) def SetWatcherPause(self, until): return self.CallMethod(REQ_SET_WATCHER_PAUSE, (until, )) def PickupJob(self, job): return self.CallMethod(REQ_PICKUP_JOB, (job,)) def SubmitJob(self, ops): ops_state = [op.__getstate__() if not isinstance(op, objects.ConfigObject) else op.ToDict(_with_private=True) for op in ops] return self.CallMethod(REQ_SUBMIT_JOB, (ops_state, )) def SubmitJobToDrainedQueue(self, ops): ops_state = [op.__getstate__() for op in ops] return self.CallMethod(REQ_SUBMIT_JOB_TO_DRAINED_QUEUE, (ops_state, )) def SubmitManyJobs(self, jobs): jobs_state = [] for ops in jobs: jobs_state.append([op.__getstate__() for op in ops]) return self.CallMethod(REQ_SUBMIT_MANY_JOBS, (jobs_state, )) @staticmethod def _PrepareJobId(request_name, job_id): try: return int(job_id) except ValueError: raise RequestError("Invalid parameter passed to %s as job id: " " expected integer, got value %s" % (request_name, job_id)) def CancelJob(self, job_id, kill=False): job_id = Client._PrepareJobId(REQ_CANCEL_JOB, job_id) return self.CallMethod(REQ_CANCEL_JOB, (job_id, kill)) def ArchiveJob(self, job_id): job_id = Client._PrepareJobId(REQ_ARCHIVE_JOB, job_id) return self.CallMethod(REQ_ARCHIVE_JOB, (job_id, )) def ChangeJobPriority(self, job_id, priority): job_id = Client._PrepareJobId(REQ_CHANGE_JOB_PRIORITY, job_id) return self.CallMethod(REQ_CHANGE_JOB_PRIORITY, (job_id, priority)) def AutoArchiveJobs(self, age): timeout = (DEF_RWTO - 1) // 2 return self.CallMethod(REQ_AUTO_ARCHIVE_JOBS, (age, timeout)) def WaitForJobChangeOnce(self, job_id, fields, prev_job_info, prev_log_serial, timeout=WFJC_TIMEOUT): """Waits for changes on a job. @param job_id: Job ID @type fields: list @param fields: List of field names to be observed @type prev_job_info: None or list @param prev_job_info: Previously received job information @type prev_log_serial: None or int/long @param prev_log_serial: Highest log serial number previously received @type timeout: int/float @param timeout: Timeout in seconds (values larger than L{WFJC_TIMEOUT} will be capped to that value) """ assert timeout >= 0, "Timeout can not be negative" return self.CallMethod(REQ_WAIT_FOR_JOB_CHANGE, (job_id, fields, prev_job_info, prev_log_serial, min(WFJC_TIMEOUT, timeout))) def WaitForJobChange(self, job_id, fields, prev_job_info, prev_log_serial): job_id = Client._PrepareJobId(REQ_WAIT_FOR_JOB_CHANGE, job_id) while True: result = self.WaitForJobChangeOnce(job_id, fields, prev_job_info, prev_log_serial) if result != constants.JOB_NOTCHANGED: break return result def Query(self, what, fields, qfilter): """Query for resources/items. @param what: One of L{constants.QR_VIA_LUXI} @type fields: List of strings @param fields: List of requested fields @type qfilter: None or list @param qfilter: Query filter @rtype: L{objects.QueryResponse} """ result = self.CallMethod(REQ_QUERY, (what, fields, qfilter)) return objects.QueryResponse.FromDict(result) def QueryFields(self, what, fields): """Query for available fields. @param what: One of L{constants.QR_VIA_LUXI} @type fields: None or list of strings @param fields: List of requested fields @rtype: L{objects.QueryFieldsResponse} """ result = self.CallMethod(REQ_QUERY_FIELDS, (what, fields)) return objects.QueryFieldsResponse.FromDict(result) def QueryJobs(self, job_ids, fields): return self.CallMethod(REQ_QUERY_JOBS, (job_ids, fields)) def QueryFilters(self, uuids, fields): return self.CallMethod(REQ_QUERY_FILTERS, (uuids, fields)) def ReplaceFilter(self, uuid, priority, predicates, action, reason): return self.CallMethod(REQ_REPLACE_FILTER, (uuid, priority, predicates, action, reason)) def DeleteFilter(self, uuid): return self.CallMethod(REQ_DELETE_FILTER, (uuid, )) def QueryInstances(self, names, fields, use_locking): return self.CallMethod(REQ_QUERY_INSTANCES, (names, fields, use_locking)) def QueryNodes(self, names, fields, use_locking): return self.CallMethod(REQ_QUERY_NODES, (names, fields, use_locking)) def QueryGroups(self, names, fields, use_locking): return self.CallMethod(REQ_QUERY_GROUPS, (names, fields, use_locking)) def QueryNetworks(self, names, fields, use_locking): return self.CallMethod(REQ_QUERY_NETWORKS, (names, fields, use_locking)) def QueryExports(self, nodes, use_locking): return self.CallMethod(REQ_QUERY_EXPORTS, (nodes, use_locking)) def QueryClusterInfo(self): return self.CallMethod(REQ_QUERY_CLUSTER_INFO, ()) def QueryConfigValues(self, fields): return self.CallMethod(REQ_QUERY_CONFIG_VALUES, (fields, )) def QueryTags(self, kind, name): return self.CallMethod(REQ_QUERY_TAGS, (kind, name)) ganeti-3.1.0~rc2/lib/masterd/000075500000000000000000000000001476477700300160115ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/masterd/__init__.py000064400000000000000000000026221476477700300201240ustar00rootroot00000000000000# # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # empty file for package definition """Masterd-related classes and functions. """ ganeti-3.1.0~rc2/lib/masterd/iallocator.py000064400000000000000000000724231476477700300205240ustar00rootroot00000000000000# # # Copyright (C) 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing the iallocator code.""" import logging from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import ht from ganeti import outils from ganeti import opcodes from ganeti import serializer from ganeti import utils import ganeti.rpc.node as rpc import ganeti.masterd.instance as gmi _STRING_LIST = ht.TListOf(ht.TString) _JOB_LIST = ht.TListOf(ht.TListOf(ht.TStrictDict(True, False, { # pylint: disable=E1101 # Class '...' has no 'OP_ID' member "OP_ID": ht.TElemOf([opcodes.OpInstanceFailover.OP_ID, opcodes.OpInstanceMigrate.OP_ID, opcodes.OpInstanceReplaceDisks.OP_ID]), }))) _NEVAC_MOVED = \ ht.TListOf(ht.TAnd(ht.TIsLength(3), ht.TItems([ht.TNonEmptyString, ht.TNonEmptyString, ht.TListOf(ht.TNonEmptyString), ]))) _NEVAC_FAILED = \ ht.TListOf(ht.TAnd(ht.TIsLength(2), ht.TItems([ht.TNonEmptyString, ht.TMaybeString, ]))) _NEVAC_RESULT = ht.TAnd(ht.TIsLength(3), ht.TItems([_NEVAC_MOVED, _NEVAC_FAILED, _JOB_LIST])) _INST_NAME = ("name", ht.TNonEmptyString) _INST_UUID = ("inst_uuid", ht.TNonEmptyString) class _AutoReqParam(outils.AutoSlots): """Meta class for request definitions. """ @classmethod def _GetSlots(mcs, attrs): """Extract the slots out of REQ_PARAMS. """ params = attrs.setdefault("REQ_PARAMS", []) return [slot for (slot, _) in params] class IARequestBase(outils.ValidatedSlots, metaclass=_AutoReqParam): """A generic IAllocator request object. """ MODE = NotImplemented REQ_PARAMS = [] REQ_RESULT = NotImplemented def __init__(self, **kwargs): """Constructor for IARequestBase. The constructor takes only keyword arguments and will set attributes on this object based on the passed arguments. As such, it means that you should not pass arguments which are not in the REQ_PARAMS attribute for this class. """ outils.ValidatedSlots.__init__(self, **kwargs) self.Validate() def Validate(self): """Validates all parameters of the request. This method returns L{None} if the validation succeeds, or raises an exception otherwise. @rtype: NoneType @return: L{None}, if the validation succeeds @raise Exception: validation fails """ assert self.MODE in constants.VALID_IALLOCATOR_MODES for (param, validator) in self.REQ_PARAMS: if not hasattr(self, param): raise errors.OpPrereqError("Request is missing '%s' parameter" % param, errors.ECODE_INVAL) value = getattr(self, param) if not validator(value): raise errors.OpPrereqError(("Request parameter '%s' has invalid" " type %s/value %s") % (param, type(value), value), errors.ECODE_INVAL) def GetRequest(self, cfg): """Gets the request data dict. @param cfg: The configuration instance """ raise NotImplementedError def GetExtraParams(self): # pylint: disable=R0201 """Gets extra parameters to the IAllocator call. """ return {} def ValidateResult(self, ia, result): """Validates the result of an request. @param ia: The IAllocator instance @param result: The IAllocator run result @raises ResultValidationError: If validation fails """ if ia.success and not self.REQ_RESULT(result): # pylint: disable=E1102 raise errors.ResultValidationError("iallocator returned invalid result," " expected %s, got %s" % (self.REQ_RESULT, result)) class IAReqInstanceAlloc(IARequestBase): """An instance allocation request. """ # pylint: disable=E1101 MODE = constants.IALLOCATOR_MODE_ALLOC REQ_PARAMS = [ _INST_NAME, ("memory", ht.TNonNegativeInt), ("spindle_use", ht.TNonNegativeInt), ("disks", ht.TListOf(ht.TDict)), ("disk_template", ht.TString), ("group_name", ht.TMaybe(ht.TNonEmptyString)), ("os", ht.TString), ("tags", _STRING_LIST), ("nics", ht.TListOf(ht.TDict)), ("vcpus", ht.TInt), ("hypervisor", ht.TString), ("node_whitelist", ht.TMaybeListOf(ht.TNonEmptyString)), ] REQ_RESULT = ht.TList def RequiredNodes(self): """Calculates the required nodes based on the disk_template. """ if self.disk_template in constants.DTS_INT_MIRROR: return 2 else: return 1 def GetRequest(self, cfg): """Requests a new instance. The checks for the completeness of the opcode must have already been done. """ for d in self.disks: d[constants.IDISK_TYPE] = self.disk_template disk_space = gmi.ComputeDiskSize(self.disks) return { "name": self.name, "disk_template": self.disk_template, "group_name": self.group_name, "tags": self.tags, "os": self.os, "vcpus": self.vcpus, "memory": self.memory, "spindle_use": self.spindle_use, "disks": self.disks, "disk_space_total": disk_space, "nics": self.nics, "required_nodes": self.RequiredNodes(), "hypervisor": self.hypervisor, } def ValidateResult(self, ia, result): """Validates an single instance allocation request. """ IARequestBase.ValidateResult(self, ia, result) if ia.success and len(result) != self.RequiredNodes(): raise errors.ResultValidationError("iallocator returned invalid number" " of nodes (%s), required %s" % (len(result), self.RequiredNodes())) class IAReqInstanceAllocateSecondary(IARequestBase): """Request to find a secondary node for plain to DRBD conversion. """ # pylint: disable=E1101 MODE = constants.IALLOCATOR_MODE_ALLOCATE_SECONDARY REQ_PARAMS = [ _INST_NAME, ] REQ_RESULT = ht.TString def GetRequest(self, cfg): return { "name": self.name } class IAReqMultiInstanceAlloc(IARequestBase): """An multi instance allocation request. """ # pylint: disable=E1101 MODE = constants.IALLOCATOR_MODE_MULTI_ALLOC REQ_PARAMS = [ ("instances", ht.TListOf(ht.TInstanceOf(IAReqInstanceAlloc))), ] _MASUCCESS = \ ht.TListOf(ht.TAnd(ht.TIsLength(2), ht.TItems([ht.TNonEmptyString, ht.TListOf(ht.TNonEmptyString), ]))) _MAFAILED = ht.TListOf(ht.TNonEmptyString) REQ_RESULT = ht.TAnd(ht.TList, ht.TIsLength(2), ht.TItems([_MASUCCESS, _MAFAILED])) def GetRequest(self, cfg): return { "instances": [iareq.GetRequest(cfg) for iareq in self.instances], } class IAReqRelocate(IARequestBase): """A relocation request. """ # pylint: disable=E1101 MODE = constants.IALLOCATOR_MODE_RELOC REQ_PARAMS = [ _INST_UUID, ("relocate_from_node_uuids", _STRING_LIST), ] REQ_RESULT = ht.TList def GetRequest(self, cfg): """Request an relocation of an instance The checks for the completeness of the opcode must have already been done. """ instance = cfg.GetInstanceInfo(self.inst_uuid) disks = cfg.GetInstanceDisks(self.inst_uuid) if instance is None: raise errors.ProgrammerError("Unknown instance '%s' passed to" " IAllocator" % self.inst_uuid) if not utils.AllDiskOfType(disks, constants.DTS_MIRRORED): raise errors.OpPrereqError("Can't relocate non-mirrored instances", errors.ECODE_INVAL) secondary_nodes = cfg.GetInstanceSecondaryNodes(instance.uuid) if (utils.AnyDiskOfType(disks, constants.DTS_INT_MIRROR) and len(secondary_nodes) != 1): raise errors.OpPrereqError("Instance has not exactly one secondary node", errors.ECODE_STATE) disk_sizes = [{constants.IDISK_SIZE: disk.size, constants.IDISK_TYPE: disk.dev_type} for disk in disks] disk_space = gmi.ComputeDiskSize(disk_sizes) return { "name": instance.name, "disk_space_total": disk_space, "required_nodes": 1, "relocate_from": cfg.GetNodeNames(self.relocate_from_node_uuids), } def ValidateResult(self, ia, result): """Validates the result of an relocation request. """ IARequestBase.ValidateResult(self, ia, result) node2group = dict((name, ndata["group"]) for (name, ndata) in ia.in_data["nodes"].items()) fn = compat.partial(self._NodesToGroups, node2group, ia.in_data["nodegroups"]) instance = ia.cfg.GetInstanceInfo(self.inst_uuid) request_groups = fn(ia.cfg.GetNodeNames(self.relocate_from_node_uuids) + ia.cfg.GetNodeNames([instance.primary_node])) result_groups = fn(result + ia.cfg.GetNodeNames([instance.primary_node])) if ia.success and not set(result_groups).issubset(request_groups): raise errors.ResultValidationError("Groups of nodes returned by" " iallocator (%s) differ from original" " groups (%s)" % (utils.CommaJoin(result_groups), utils.CommaJoin(request_groups))) @staticmethod def _NodesToGroups(node2group, groups, nodes): """Returns a list of unique group names for a list of nodes. @type node2group: dict @param node2group: Map from node name to group UUID @type groups: dict @param groups: Group information @type nodes: list @param nodes: Node names """ result = set() for node in nodes: try: group_uuid = node2group[node] except KeyError: # Ignore unknown node pass else: try: group = groups[group_uuid] except KeyError: # Can't find group, let's use UUID group_name = group_uuid else: group_name = group["name"] result.add(group_name) return sorted(result) class IAReqNodeEvac(IARequestBase): """A node evacuation request. """ # pylint: disable=E1101 MODE = constants.IALLOCATOR_MODE_NODE_EVAC REQ_PARAMS = [ ("instances", _STRING_LIST), ("evac_mode", ht.TEvacMode), ("ignore_soft_errors", ht.TMaybe(ht.TBool)), ] REQ_RESULT = _NEVAC_RESULT def GetRequest(self, cfg): """Get data for node-evacuate requests. """ return { "instances": self.instances, "evac_mode": self.evac_mode, } def GetExtraParams(self): """Get extra iallocator command line options for node-evacuate requests. """ if self.ignore_soft_errors: return {"ignore-soft-errors": None} else: return {} class IAReqGroupChange(IARequestBase): """A group change request. """ # pylint: disable=E1101 MODE = constants.IALLOCATOR_MODE_CHG_GROUP REQ_PARAMS = [ ("instances", _STRING_LIST), ("target_groups", _STRING_LIST), ] REQ_RESULT = _NEVAC_RESULT def GetRequest(self, cfg): """Get data for node-evacuate requests. """ return { "instances": self.instances, "target_groups": self.target_groups, } class IAllocator(object): """IAllocator framework. An IAllocator instance has three sets of attributes: - cfg that is needed to query the cluster - input data (all members of the _KEYS class attribute are required) - four buffer attributes (in|out_data|text), that represent the input (to the external script) in text and data structure format, and the output from it, again in two formats - the result variables from the script (success, info, nodes) for easy usage """ # pylint: disable=R0902 # lots of instance attributes def __init__(self, cfg, rpc_runner, req): self.cfg = cfg self.rpc = rpc_runner self.req = req # init buffer variables self.in_text = self.out_text = self.in_data = self.out_data = None # init result fields self.success = self.info = self.result = None self._BuildInputData(req) def _ComputeClusterDataNodeInfo(self, disk_templates, node_list, cluster_info, hypervisor_name): """Prepare and execute node info call. @type disk_templates: list of string @param disk_templates: the disk templates of the instances to be allocated @type node_list: list of strings @param node_list: list of nodes' UUIDs @type cluster_info: L{objects.Cluster} @param cluster_info: the cluster's information from the config @type hypervisor_name: string @param hypervisor_name: the hypervisor name @rtype: same as the result of the node info RPC call @return: the result of the node info RPC call """ storage_units_raw = utils.storage.GetStorageUnits(self.cfg, disk_templates) storage_units = rpc.PrepareStorageUnitsForNodes(self.cfg, storage_units_raw, node_list) hvspecs = [(hypervisor_name, cluster_info.hvparams[hypervisor_name])] return self.rpc.call_node_info(node_list, storage_units, hvspecs) def _ComputeClusterData(self, disk_template=None): """Compute the generic allocator input data. @type disk_template: list of string @param disk_template: the disk templates of the instances to be allocated """ cfg = self.cfg.GetDetachedConfig() cluster_info = cfg.GetClusterInfo() # cluster data data = { "version": constants.IALLOCATOR_VERSION, "cluster_name": cluster_info.cluster_name, "cluster_tags": list(cluster_info.GetTags()), "enabled_hypervisors": list(cluster_info.enabled_hypervisors), "ipolicy": cluster_info.ipolicy, } ginfo = cfg.GetAllNodeGroupsInfo() ninfo = cfg.GetAllNodesInfo() iinfo = cfg.GetAllInstancesInfo() i_list = [(inst, cluster_info.FillBE(inst)) for inst in iinfo.values()] # node data node_list = [n.uuid for n in ninfo.values() if n.vm_capable] if isinstance(self.req, IAReqInstanceAlloc): hypervisor_name = self.req.hypervisor elif isinstance(self.req, IAReqRelocate): hypervisor_name = iinfo[self.req.inst_uuid].hypervisor else: hypervisor_name = cluster_info.primary_hypervisor if not disk_template: disk_template = cluster_info.enabled_disk_templates[0] node_data = self._ComputeClusterDataNodeInfo([disk_template], node_list, cluster_info, hypervisor_name) node_iinfo = \ self.rpc.call_all_instances_info(node_list, cluster_info.enabled_hypervisors, cluster_info.hvparams) data["nodegroups"] = self._ComputeNodeGroupData(cluster_info, ginfo) config_ndata = self._ComputeBasicNodeData(cfg, ninfo) data["nodes"] = self._ComputeDynamicNodeData( ninfo, node_data, node_iinfo, i_list, config_ndata, disk_template) assert len(data["nodes"]) == len(ninfo), \ "Incomplete node data computed" data["instances"] = self._ComputeInstanceData(cfg, cluster_info, i_list) self.in_data = data @staticmethod def _ComputeNodeGroupData(cluster, ginfo): """Compute node groups data. """ ng = dict((guuid, { "name": gdata.name, "alloc_policy": gdata.alloc_policy, "networks": list(gdata.networks), "ipolicy": gmi.CalculateGroupIPolicy(cluster, gdata), "tags": list(gdata.GetTags()), }) for guuid, gdata in ginfo.items()) return ng @staticmethod def _ComputeBasicNodeData(cfg, node_cfg): """Compute global node data. @rtype: dict @returns: a dict of name: (node dict, node config) """ # fill in static (config-based) values node_results = dict((ninfo.name, { "tags": list(ninfo.GetTags()), "primary_ip": ninfo.primary_ip, "secondary_ip": ninfo.secondary_ip, "offline": ninfo.offline, "drained": ninfo.drained, "master_candidate": ninfo.master_candidate, "group": ninfo.group, "master_capable": ninfo.master_capable, "vm_capable": ninfo.vm_capable, "ndparams": cfg.GetNdParams(ninfo), }) for ninfo in node_cfg.values()) return node_results @staticmethod def _GetAttributeFromHypervisorNodeData(hv_info, node_name, attr): """Extract an attribute from the hypervisor's node information. This is a helper function to extract data from the hypervisor's information about the node, as part of the result of a node_info query. @type hv_info: dict of strings @param hv_info: dictionary of node information from the hypervisor @type node_name: string @param node_name: name of the node @type attr: string @param attr: key of the attribute in the hv_info dictionary @rtype: integer @return: the value of the attribute @raises errors.OpExecError: if key not in dictionary or value not integer """ if attr not in hv_info: raise errors.OpExecError("Node '%s' didn't return attribute" " '%s'" % (node_name, attr)) value = hv_info[attr] if not isinstance(value, int): raise errors.OpExecError("Node '%s' returned invalid value" " for '%s': %s" % (node_name, attr, value)) return value @staticmethod def _ComputeStorageDataFromSpaceInfoByTemplate( space_info, node_name, disk_template): """Extract storage data from node info. @type space_info: see result of the RPC call node info @param space_info: the storage reporting part of the result of the RPC call node info @type node_name: string @param node_name: the node's name @type disk_template: string @param disk_template: the disk template to report space for @rtype: 4-tuple of integers @return: tuple of storage info (total_disk, free_disk, total_spindles, free_spindles) """ storage_type = constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[disk_template] if storage_type not in constants.STS_REPORT: total_disk = total_spindles = 0 free_disk = free_spindles = 0 else: template_space_info = utils.storage.LookupSpaceInfoByDiskTemplate( space_info, disk_template) if not template_space_info: raise errors.OpExecError("Node '%s' didn't return space info for disk" "template '%s'" % (node_name, disk_template)) total_disk = template_space_info["storage_size"] free_disk = template_space_info["storage_free"] total_spindles = 0 free_spindles = 0 if disk_template in constants.DTS_LVM: lvm_pv_info = utils.storage.LookupSpaceInfoByStorageType( space_info, constants.ST_LVM_PV) if lvm_pv_info: total_spindles = lvm_pv_info["storage_size"] free_spindles = lvm_pv_info["storage_free"] return (total_disk, free_disk, total_spindles, free_spindles) @staticmethod def _ComputeStorageDataFromSpaceInfo(space_info, node_name, has_lvm): """Extract storage data from node info. @type space_info: see result of the RPC call node info @param space_info: the storage reporting part of the result of the RPC call node info @type node_name: string @param node_name: the node's name @type has_lvm: boolean @param has_lvm: whether or not LVM storage information is requested @rtype: 4-tuple of integers @return: tuple of storage info (total_disk, free_disk, total_spindles, free_spindles) """ # TODO: replace this with proper storage reporting if has_lvm: lvm_vg_info = utils.storage.LookupSpaceInfoByStorageType( space_info, constants.ST_LVM_VG) if not lvm_vg_info: raise errors.OpExecError("Node '%s' didn't return LVM vg space info." % (node_name)) total_disk = lvm_vg_info["storage_size"] free_disk = lvm_vg_info["storage_free"] lvm_pv_info = utils.storage.LookupSpaceInfoByStorageType( space_info, constants.ST_LVM_PV) if not lvm_pv_info: raise errors.OpExecError("Node '%s' didn't return LVM pv space info." % (node_name)) total_spindles = lvm_pv_info["storage_size"] free_spindles = lvm_pv_info["storage_free"] else: # we didn't even ask the node for VG status, so use zeros total_disk = free_disk = 0 total_spindles = free_spindles = 0 return (total_disk, free_disk, total_spindles, free_spindles) @staticmethod def _ComputeInstanceMemory(instance_list, node_instances_info, node_uuid, input_mem_free): """Compute memory used by primary instances. @rtype: tuple (int, int, int) @returns: A tuple of three integers: 1. the sum of memory used by primary instances on the node (including the ones that are currently down), 2. the sum of memory used by primary instances of the node that are up, 3. the amount of memory that is free on the node considering the current usage of the instances. """ i_p_mem = i_p_up_mem = 0 mem_free = input_mem_free for iinfo, beinfo in instance_list: if iinfo.primary_node == node_uuid: i_p_mem += beinfo[constants.BE_MAXMEM] if iinfo.name not in node_instances_info[node_uuid].payload: i_used_mem = 0 else: i_used_mem = int(node_instances_info[node_uuid] .payload[iinfo.name]["memory"]) i_mem_diff = beinfo[constants.BE_MAXMEM] - i_used_mem if iinfo.admin_state == constants.ADMINST_UP \ and not iinfo.forthcoming: mem_free -= max(0, i_mem_diff) i_p_up_mem += beinfo[constants.BE_MAXMEM] return (i_p_mem, i_p_up_mem, mem_free) def _ComputeDynamicNodeData(self, node_cfg, node_data, node_iinfo, i_list, node_results, disk_template): """Compute global node data. @param node_results: the basic node structures as filled from the config """ #TODO(dynmem): compute the right data on MAX and MIN memory # make a copy of the current dict node_results = dict(node_results) for nuuid, nresult in node_data.items(): ninfo = node_cfg[nuuid] assert ninfo.name in node_results, "Missing basic data for node %s" % \ ninfo.name if not ninfo.offline: nresult.Raise("Can't get data for node %s" % ninfo.name) node_iinfo[nuuid].Raise("Can't get node instance info from node %s" % ninfo.name) (_, space_info, (hv_info, )) = nresult.payload mem_free = self._GetAttributeFromHypervisorNodeData(hv_info, ninfo.name, "memory_free") (i_p_mem, i_p_up_mem, mem_free) = self._ComputeInstanceMemory( i_list, node_iinfo, nuuid, mem_free) (total_disk, free_disk, total_spindles, free_spindles) = \ self._ComputeStorageDataFromSpaceInfoByTemplate( space_info, ninfo.name, disk_template) # compute memory used by instances pnr_dyn = { "total_memory": self._GetAttributeFromHypervisorNodeData( hv_info, ninfo.name, "memory_total"), "reserved_memory": self._GetAttributeFromHypervisorNodeData( hv_info, ninfo.name, "memory_dom0"), "free_memory": mem_free, "total_disk": total_disk, "free_disk": free_disk, "total_spindles": total_spindles, "free_spindles": free_spindles, "total_cpus": self._GetAttributeFromHypervisorNodeData( hv_info, ninfo.name, "cpu_total"), "reserved_cpus": self._GetAttributeFromHypervisorNodeData( hv_info, ninfo.name, "cpu_dom0"), "i_pri_memory": i_p_mem, "i_pri_up_memory": i_p_up_mem, } pnr_dyn.update(node_results[ninfo.name]) node_results[ninfo.name] = pnr_dyn return node_results @staticmethod def _ComputeInstanceData(cfg, cluster_info, i_list): """Compute global instance data. """ instance_data = {} for iinfo, beinfo in i_list: nic_data = [] for nic in iinfo.nics: filled_params = cluster_info.SimpleFillNIC(nic.nicparams) nic_dict = { "mac": nic.mac, "ip": nic.ip, "mode": filled_params[constants.NIC_MODE], "link": filled_params[constants.NIC_LINK], } if filled_params[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: nic_dict["bridge"] = filled_params[constants.NIC_LINK] nic_data.append(nic_dict) inst_disks = cfg.GetInstanceDisks(iinfo.uuid) inst_disktemplate = cfg.GetInstanceDiskTemplate(iinfo.uuid) pir = { "tags": list(iinfo.GetTags()), "admin_state": iinfo.admin_state, "vcpus": beinfo[constants.BE_VCPUS], "memory": beinfo[constants.BE_MAXMEM], "spindle_use": beinfo[constants.BE_SPINDLE_USE], "os": iinfo.os, "nodes": [cfg.GetNodeName(iinfo.primary_node)] + cfg.GetNodeNames( cfg.GetInstanceSecondaryNodes(iinfo.uuid)), "nics": nic_data, "disks": [{constants.IDISK_TYPE: dsk.dev_type, constants.IDISK_SIZE: dsk.size, constants.IDISK_MODE: dsk.mode, constants.IDISK_SPINDLES: dsk.spindles} for dsk in inst_disks], "disk_template": inst_disktemplate, "disks_active": iinfo.disks_active, "hypervisor": iinfo.hypervisor, } pir["disk_space_total"] = gmi.ComputeDiskSize(pir["disks"]) instance_data[iinfo.name] = pir return instance_data def _BuildInputData(self, req): """Build input data structures. """ request = req.GetRequest(self.cfg) disk_template = None if request.get("disk_template") is not None: disk_template = request["disk_template"] elif isinstance(req, IAReqRelocate): disk_template = self.cfg.GetInstanceDiskTemplate(self.req.inst_uuid) self._ComputeClusterData(disk_template=disk_template) request["type"] = req.MODE if isinstance(self.req, IAReqInstanceAlloc): node_whitelist = self.req.node_whitelist else: node_whitelist = None if node_whitelist is not None: request["restrict-to-nodes"] = node_whitelist self.in_data["request"] = request self.in_text = serializer.Dump(self.in_data) logging.debug("IAllocator request: %s", self.in_text) def Run(self, name, validate=True, call_fn=None): """Run an instance allocator and return the results. """ if call_fn is None: call_fn = self.rpc.call_iallocator_runner ial_params = self.cfg.GetDefaultIAllocatorParameters() for ial_param in self.req.GetExtraParams().items(): ial_params[ial_param[0]] = ial_param[1] result = call_fn(self.cfg.GetMasterNode(), name, self.in_text, ial_params) result.Raise("Failure while running the iallocator script") self.out_text = result.payload if validate: self._ValidateResult() def _ValidateResult(self): """Process the allocator results. This will process and if successful save the result in self.out_data and the other parameters. """ try: rdict = serializer.Load(self.out_text) except Exception as err: raise errors.OpExecError("Can't parse iallocator results: %s" % str(err)) if not isinstance(rdict, dict): raise errors.OpExecError("Can't parse iallocator results: not a dict") # TODO: remove backwards compatiblity in later versions if "nodes" in rdict and "result" not in rdict: rdict["result"] = rdict["nodes"] del rdict["nodes"] for key in "success", "info", "result": if key not in rdict: raise errors.OpExecError("Can't parse iallocator results:" " missing key '%s'" % key) setattr(self, key, rdict[key]) self.req.ValidateResult(self, self.result) self.out_data = rdict ganeti-3.1.0~rc2/lib/masterd/instance.py000064400000000000000000001447151476477700300202030ustar00rootroot00000000000000# # # Copyright (C) 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Instance-related functions and classes for masterd. """ import logging import time import OpenSSL from hashlib import sha1 from ganeti import constants from ganeti import errors from ganeti import compat from ganeti import utils from ganeti import objects from ganeti import netutils from ganeti import pathutils class _ImportExportError(Exception): """Local exception to report import/export errors. """ class ImportExportTimeouts(object): #: Time until daemon starts writing status file DEFAULT_READY_TIMEOUT = 10 #: Length of time until errors cause hard failure DEFAULT_ERROR_TIMEOUT = 10 #: Time after which daemon must be listening DEFAULT_LISTEN_TIMEOUT = 10 #: Progress update interval DEFAULT_PROGRESS_INTERVAL = 60 __slots__ = [ "error", "ready", "listen", "connect", "progress", ] def __init__(self, connect, listen=DEFAULT_LISTEN_TIMEOUT, error=DEFAULT_ERROR_TIMEOUT, ready=DEFAULT_READY_TIMEOUT, progress=DEFAULT_PROGRESS_INTERVAL): """Initializes this class. @type connect: number @param connect: Timeout for establishing connection @type listen: number @param listen: Timeout for starting to listen for connections @type error: number @param error: Length of time until errors cause hard failure @type ready: number @param ready: Timeout for daemon to become ready @type progress: number @param progress: Progress update interval """ self.error = error self.ready = ready self.listen = listen self.connect = connect self.progress = progress class ImportExportCbBase(object): """Callbacks for disk import/export. """ def ReportListening(self, ie, private, component): """Called when daemon started listening. @type ie: Subclass of L{_DiskImportExportBase} @param ie: Import/export object @param private: Private data passed to import/export object @param component: transfer component name """ def ReportConnected(self, ie, private): """Called when a connection has been established. @type ie: Subclass of L{_DiskImportExportBase} @param ie: Import/export object @param private: Private data passed to import/export object """ def ReportProgress(self, ie, private): """Called when new progress information should be reported. @type ie: Subclass of L{_DiskImportExportBase} @param ie: Import/export object @param private: Private data passed to import/export object """ def ReportFinished(self, ie, private): """Called when a transfer has finished. @type ie: Subclass of L{_DiskImportExportBase} @param ie: Import/export object @param private: Private data passed to import/export object """ class _DiskImportExportBase(object): MODE_TEXT = None def __init__(self, lu, node_uuid, opts, instance, component, timeouts, cbs, private=None): """Initializes this class. @param lu: Logical unit instance @type node_uuid: string @param node_uuid: Node UUID for import @type opts: L{objects.ImportExportOptions} @param opts: Import/export daemon options @type instance: L{objects.Instance} @param instance: Instance object @type component: string @param component: which part of the instance is being imported @type timeouts: L{ImportExportTimeouts} @param timeouts: Timeouts for this import @type cbs: L{ImportExportCbBase} @param cbs: Callbacks @param private: Private data for callback functions """ assert self.MODE_TEXT self._lu = lu self.node_uuid = node_uuid self.node_name = lu.cfg.GetNodeName(node_uuid) self._opts = opts.Copy() self._instance = instance self._component = component self._timeouts = timeouts self._cbs = cbs self._private = private # Set master daemon's timeout in options for import/export daemon assert self._opts.connect_timeout is None self._opts.connect_timeout = timeouts.connect # Parent loop self._loop = None # Timestamps self._ts_begin = None self._ts_connected = None self._ts_finished = None self._ts_cleanup = None self._ts_last_progress = None self._ts_last_error = None # Transfer status self.success = None self.final_message = None # Daemon status self._daemon_name = None self._daemon = None @property def recent_output(self): """Returns the most recent output from the daemon. """ if self._daemon: return "\n".join(self._daemon.recent_output) return None @property def progress(self): """Returns transfer progress information. """ if not self._daemon: return None return (self._daemon.progress_mbytes, self._daemon.progress_throughput, self._daemon.progress_percent, self._daemon.progress_eta) @property def magic(self): """Returns the magic value for this import/export. """ return self._opts.magic @property def active(self): """Determines whether this transport is still active. """ return self.success is None @property def loop(self): """Returns parent loop. @rtype: L{ImportExportLoop} """ return self._loop def SetLoop(self, loop): """Sets the parent loop. @type loop: L{ImportExportLoop} """ if self._loop: raise errors.ProgrammerError("Loop can only be set once") self._loop = loop def _StartDaemon(self): """Starts the import/export daemon. """ raise NotImplementedError() def CheckDaemon(self): """Checks whether daemon has been started and if not, starts it. @rtype: string @return: Daemon name """ assert self._ts_cleanup is None if self._daemon_name is None: assert self._ts_begin is None result = self._StartDaemon() if result.fail_msg: raise _ImportExportError("Failed to start %s on %s: %s" % (self.MODE_TEXT, self.node_name, result.fail_msg)) daemon_name = result.payload logging.info("Started %s '%s' on %s", self.MODE_TEXT, daemon_name, self.node_name) self._ts_begin = time.time() self._daemon_name = daemon_name return self._daemon_name def GetDaemonName(self): """Returns the daemon name. """ assert self._daemon_name, "Daemon has not been started" assert self._ts_cleanup is None return self._daemon_name def Abort(self): """Sends SIGTERM to import/export daemon (if still active). """ if self._daemon_name: self._lu.LogWarning("Aborting %s '%s' on %s", self.MODE_TEXT, self._daemon_name, self.node_uuid) result = self._lu.rpc.call_impexp_abort(self.node_uuid, self._daemon_name) if result.fail_msg: self._lu.LogWarning("Failed to abort %s '%s' on %s: %s", self.MODE_TEXT, self._daemon_name, self.node_uuid, result.fail_msg) return False return True def _SetDaemonData(self, data): """Internal function for updating status daemon data. @type data: L{objects.ImportExportStatus} @param data: Daemon status data """ assert self._ts_begin is not None if not data: if utils.TimeoutExpired(self._ts_begin, self._timeouts.ready): raise _ImportExportError("Didn't become ready after %s seconds" % self._timeouts.ready) return False self._daemon = data return True def SetDaemonData(self, success, data): """Updates daemon status data. @type success: bool @param success: Whether fetching data was successful or not @type data: L{objects.ImportExportStatus} @param data: Daemon status data """ if not success: if self._ts_last_error is None: self._ts_last_error = time.time() elif utils.TimeoutExpired(self._ts_last_error, self._timeouts.error): raise _ImportExportError("Too many errors while updating data") return False self._ts_last_error = None return self._SetDaemonData(data) def CheckListening(self): """Checks whether the daemon is listening. """ raise NotImplementedError() def _GetConnectedCheckEpoch(self): """Returns timeout to calculate connect timeout. """ raise NotImplementedError() def CheckConnected(self): """Checks whether the daemon is connected. @rtype: bool @return: Whether the daemon is connected """ assert self._daemon, "Daemon status missing" if self._ts_connected is not None: return True if self._daemon.connected: self._ts_connected = time.time() # TODO: Log remote peer logging.debug("%s '%s' on %s is now connected", self.MODE_TEXT, self._daemon_name, self.node_uuid) self._cbs.ReportConnected(self, self._private) return True if utils.TimeoutExpired(self._GetConnectedCheckEpoch(), self._timeouts.connect): raise _ImportExportError("Not connected after %s seconds" % self._timeouts.connect) return False def _CheckProgress(self): """Checks whether a progress update should be reported. """ if ((self._ts_last_progress is None or utils.TimeoutExpired(self._ts_last_progress, self._timeouts.progress)) and self._daemon and self._daemon.progress_mbytes is not None and self._daemon.progress_throughput is not None): self._cbs.ReportProgress(self, self._private) self._ts_last_progress = time.time() def CheckFinished(self): """Checks whether the daemon exited. @rtype: bool @return: Whether the transfer is finished """ assert self._daemon, "Daemon status missing" if self._ts_finished: return True if self._daemon.exit_status is None: # TODO: Adjust delay for ETA expiring soon self._CheckProgress() return False self._ts_finished = time.time() self._ReportFinished(self._daemon.exit_status == 0, self._daemon.error_message) return True def _ReportFinished(self, success, message): """Transfer is finished or daemon exited. @type success: bool @param success: Whether the transfer was successful @type message: string @param message: Error message """ assert self.success is None self.success = success self.final_message = message if success: logging.info("%s '%s' on %s succeeded", self.MODE_TEXT, self._daemon_name, self.node_uuid) elif self._daemon_name: self._lu.LogWarning("%s '%s' on %s failed: %s", self.MODE_TEXT, self._daemon_name, self._lu.cfg.GetNodeName(self.node_uuid), message) else: self._lu.LogWarning("%s on %s failed: %s", self.MODE_TEXT, self._lu.cfg.GetNodeName(self.node_uuid), message) self._cbs.ReportFinished(self, self._private) def _Finalize(self): """Makes the RPC call to finalize this import/export. """ return self._lu.rpc.call_impexp_cleanup(self.node_uuid, self._daemon_name) def Finalize(self, error=None): """Finalizes this import/export. """ if self._daemon_name: logging.info("Finalizing %s '%s' on %s", self.MODE_TEXT, self._daemon_name, self.node_uuid) result = self._Finalize() if result.fail_msg: self._lu.LogWarning("Failed to finalize %s '%s' on %s: %s", self.MODE_TEXT, self._daemon_name, self.node_uuid, result.fail_msg) return False # Daemon is no longer running self._daemon_name = None self._ts_cleanup = time.time() if error: self._ReportFinished(False, error) return True class DiskImport(_DiskImportExportBase): MODE_TEXT = "import" def __init__(self, lu, node_uuid, opts, instance, component, dest, dest_args, timeouts, cbs, private=None): """Initializes this class. @param lu: Logical unit instance @type node_uuid: string @param node_uuid: Node name for import @type opts: L{objects.ImportExportOptions} @param opts: Import/export daemon options @type instance: L{objects.Instance} @param instance: Instance object @type component: string @param component: which part of the instance is being imported @param dest: I/O destination @param dest_args: I/O arguments @type timeouts: L{ImportExportTimeouts} @param timeouts: Timeouts for this import @type cbs: L{ImportExportCbBase} @param cbs: Callbacks @param private: Private data for callback functions """ _DiskImportExportBase.__init__(self, lu, node_uuid, opts, instance, component, timeouts, cbs, private) self._dest = dest self._dest_args = dest_args # Timestamps self._ts_listening = None @property def listen_port(self): """Returns the port the daemon is listening on. """ if self._daemon: return self._daemon.listen_port return None def _StartDaemon(self): """Starts the import daemon. """ return self._lu.rpc.call_import_start(self.node_uuid, self._opts, self._instance, self._component, (self._dest, self._dest_args)) def CheckListening(self): """Checks whether the daemon is listening. @rtype: bool @return: Whether the daemon is listening """ assert self._daemon, "Daemon status missing" if self._ts_listening is not None: return True port = self._daemon.listen_port if port is not None: self._ts_listening = time.time() logging.debug("Import '%s' on %s is now listening on port %s", self._daemon_name, self.node_uuid, port) self._cbs.ReportListening(self, self._private, self._component) return True if utils.TimeoutExpired(self._ts_begin, self._timeouts.listen): raise _ImportExportError("Not listening after %s seconds" % self._timeouts.listen) return False def _GetConnectedCheckEpoch(self): """Returns the time since we started listening. """ assert self._ts_listening is not None, \ ("Checking whether an import is connected is only useful" " once it's been listening") return self._ts_listening class DiskExport(_DiskImportExportBase): MODE_TEXT = "export" def __init__(self, lu, node_uuid, opts, dest_host, dest_port, instance, component, source, source_args, timeouts, cbs, private=None): """Initializes this class. @param lu: Logical unit instance @type node_uuid: string @param node_uuid: Node UUID for import @type opts: L{objects.ImportExportOptions} @param opts: Import/export daemon options @type dest_host: string @param dest_host: Destination host name or IP address @type dest_port: number @param dest_port: Destination port number @type instance: L{objects.Instance} @param instance: Instance object @type component: string @param component: which part of the instance is being imported @param source: I/O source @param source_args: I/O source @type timeouts: L{ImportExportTimeouts} @param timeouts: Timeouts for this import @type cbs: L{ImportExportCbBase} @param cbs: Callbacks @param private: Private data for callback functions """ _DiskImportExportBase.__init__(self, lu, node_uuid, opts, instance, component, timeouts, cbs, private) self._dest_host = dest_host self._dest_port = dest_port self._source = source self._source_args = source_args def _StartDaemon(self): """Starts the export daemon. """ return self._lu.rpc.call_export_start(self.node_uuid, self._opts, self._dest_host, self._dest_port, self._instance, self._component, (self._source, self._source_args)) def CheckListening(self): """Checks whether the daemon is listening. """ # Only an import can be listening return True def _GetConnectedCheckEpoch(self): """Returns the time since the daemon started. """ assert self._ts_begin is not None return self._ts_begin def FormatProgress(progress): """Formats progress information for user consumption """ (mbytes, throughput, percent, eta) = progress parts = [ utils.FormatUnit(mbytes, "h"), # Not using FormatUnit as it doesn't support kilobytes "%0.1f MiB/s" % throughput, ] if percent is not None: parts.append("%d%%" % percent) if eta is not None: parts.append("ETA %s" % utils.FormatSeconds(eta)) return utils.CommaJoin(parts) class ImportExportLoop(object): MIN_DELAY = 1.0 MAX_DELAY = 20.0 def __init__(self, lu): """Initializes this class. """ self._lu = lu self._queue = [] self._pending_add = [] def Add(self, diskie): """Adds an import/export object to the loop. @type diskie: Subclass of L{_DiskImportExportBase} @param diskie: Import/export object """ assert diskie not in self._pending_add assert diskie.loop is None diskie.SetLoop(self) # Adding new objects to a staging list is necessary, otherwise the main # loop gets confused if callbacks modify the queue while the main loop is # iterating over it. self._pending_add.append(diskie) @staticmethod def _CollectDaemonStatus(lu, daemons): """Collects the status for all import/export daemons. """ daemon_status = {} for node_name, names in daemons.items(): result = lu.rpc.call_impexp_status(node_name, names) if result.fail_msg: lu.LogWarning("Failed to get daemon status on %s: %s", node_name, result.fail_msg) continue assert len(names) == len(result.payload) daemon_status[node_name] = dict(zip(names, result.payload)) return daemon_status @staticmethod def _GetActiveDaemonNames(queue): """Gets the names of all active daemons. """ result = {} for diskie in queue: if not diskie.active: continue try: # Start daemon if necessary daemon_name = diskie.CheckDaemon() except _ImportExportError as err: logging.exception("%s failed", diskie.MODE_TEXT) diskie.Finalize(error=str(err)) continue result.setdefault(diskie.node_name, []).append(daemon_name) assert len(queue) >= len(result) assert len(queue) >= sum([len(names) for names in result.values()]) logging.debug("daemons=%r", result) return result def _AddPendingToQueue(self): """Adds all pending import/export objects to the internal queue. """ assert compat.all(diskie not in self._queue and diskie.loop == self for diskie in self._pending_add) self._queue.extend(self._pending_add) del self._pending_add[:] def Run(self): """Utility main loop. """ while True: self._AddPendingToQueue() # Collect all active daemon names daemons = self._GetActiveDaemonNames(self._queue) if not daemons: break # Collection daemon status data data = self._CollectDaemonStatus(self._lu, daemons) # Use data delay = self.MAX_DELAY for diskie in self._queue: if not diskie.active: continue try: try: all_daemon_data = data[diskie.node_name] except KeyError: result = diskie.SetDaemonData(False, None) else: result = \ diskie.SetDaemonData(True, all_daemon_data[diskie.GetDaemonName()]) if not result: # Daemon not yet ready, retry soon delay = min(3.0, delay) continue if diskie.CheckFinished(): # Transfer finished diskie.Finalize() continue # Normal case: check again in 5 seconds delay = min(5.0, delay) if not diskie.CheckListening(): # Not yet listening, retry soon delay = min(1.0, delay) continue if not diskie.CheckConnected(): # Not yet connected, retry soon delay = min(1.0, delay) continue except _ImportExportError as err: logging.exception("%s failed", diskie.MODE_TEXT) diskie.Finalize(error=str(err)) if not compat.any(diskie.active for diskie in self._queue): break # Wait a bit delay = min(self.MAX_DELAY, max(self.MIN_DELAY, delay)) logging.debug("Waiting for %ss", delay) time.sleep(delay) def FinalizeAll(self): """Finalizes all pending transfers. """ success = True for diskie in self._queue: success = diskie.Finalize() and success return success class _TransferInstCbBase(ImportExportCbBase): def __init__(self, lu, feedback_fn, instance, timeouts, src_node_uuid, src_cbs, dest_node_uuid, dest_ip): """Initializes this class. """ ImportExportCbBase.__init__(self) self.lu = lu self.feedback_fn = feedback_fn self.instance = instance self.timeouts = timeouts self.src_node_uuid = src_node_uuid self.src_cbs = src_cbs self.dest_node_uuid = dest_node_uuid self.dest_ip = dest_ip class _TransferInstSourceCb(_TransferInstCbBase): def ReportConnected(self, ie, dtp): """Called when a connection has been established. """ assert self.src_cbs is None assert dtp.src_export == ie assert dtp.dest_import self.feedback_fn("%s is sending data on %s" % (dtp.data.name, ie.node_name)) def ReportProgress(self, ie, dtp): """Called when new progress information should be reported. """ progress = ie.progress if not progress: return self.feedback_fn("%s sent %s" % (dtp.data.name, FormatProgress(progress))) def ReportFinished(self, ie, dtp): """Called when a transfer has finished. """ assert self.src_cbs is None assert dtp.src_export == ie assert dtp.dest_import if ie.success: self.feedback_fn("%s finished sending data" % dtp.data.name) else: self.feedback_fn("%s failed to send data: %s (recent output: %s)" % (dtp.data.name, ie.final_message, ie.recent_output)) dtp.RecordResult(ie.success) cb = dtp.data.finished_fn if cb: cb() # TODO: Check whether sending SIGTERM right away is okay, maybe we should # give the daemon a moment to sort things out if dtp.dest_import and not ie.success: dtp.dest_import.Abort() class _TransferInstDestCb(_TransferInstCbBase): def ReportListening(self, ie, dtp, component): """Called when daemon started listening. """ assert self.src_cbs assert dtp.src_export is None assert dtp.dest_import assert dtp.export_opts self.feedback_fn("%s is now listening, starting export" % dtp.data.name) # Start export on source node de = DiskExport(self.lu, self.src_node_uuid, dtp.export_opts, self.dest_ip, ie.listen_port, self.instance, component, dtp.data.src_io, dtp.data.src_ioargs, self.timeouts, self.src_cbs, private=dtp) ie.loop.Add(de) dtp.src_export = de def ReportConnected(self, ie, dtp): """Called when a connection has been established. """ self.feedback_fn("%s is receiving data on %s" % (dtp.data.name, self.lu.cfg.GetNodeName(self.dest_node_uuid))) def ReportFinished(self, ie, dtp): """Called when a transfer has finished. """ if ie.success: self.feedback_fn("%s finished receiving data" % dtp.data.name) else: self.feedback_fn("%s failed to receive data: %s (recent output: %s)" % (dtp.data.name, ie.final_message, ie.recent_output)) dtp.RecordResult(ie.success) # TODO: Check whether sending SIGTERM right away is okay, maybe we should # give the daemon a moment to sort things out if dtp.src_export and not ie.success: dtp.src_export.Abort() class DiskTransfer(object): def __init__(self, name, src_io, src_ioargs, dest_io, dest_ioargs, finished_fn): """Initializes this class. @type name: string @param name: User-visible name for this transfer (e.g. "disk/0") @param src_io: Source I/O type @param src_ioargs: Source I/O arguments @param dest_io: Destination I/O type @param dest_ioargs: Destination I/O arguments @type finished_fn: callable @param finished_fn: Function called once transfer has finished """ self.name = name self.src_io = src_io self.src_ioargs = src_ioargs self.dest_io = dest_io self.dest_ioargs = dest_ioargs self.finished_fn = finished_fn class _DiskTransferPrivate(object): def __init__(self, data, success, export_opts): """Initializes this class. @type data: L{DiskTransfer} @type success: bool """ self.data = data self.success = success self.export_opts = export_opts self.src_export = None self.dest_import = None def RecordResult(self, success): """Updates the status. One failed part will cause the whole transfer to fail. """ self.success = self.success and success def _GetInstDiskMagic(base, instance_name, index): """Computes the magic value for a disk export or import. @type base: string @param base: Random seed value (can be the same for all disks of a transfer) @type instance_name: string @param instance_name: Name of instance @type index: number @param index: Disk index """ h = sha1() for value in (str(constants.RIE_VERSION), base, instance_name, str(index)): h.update(value.encode("utf-8")) return h.hexdigest() def TransferInstanceData(lu, feedback_fn, src_node_uuid, dest_node_uuid, dest_ip, compress, instance, all_transfers): """Transfers an instance's data from one node to another. @param lu: Logical unit instance @param feedback_fn: Feedback function @type src_node_uuid: string @param src_node_uuid: Source node UUID @type dest_node_uuid: string @param dest_node_uuid: Destination node UUID @type dest_ip: string @param dest_ip: IP address of destination node @type compress: string @param compress: Compression tool to use @type instance: L{objects.Instance} @param instance: Instance object @type all_transfers: list of L{DiskTransfer} instances @param all_transfers: List of all disk transfers to be made @rtype: list @return: List with a boolean (True=successful, False=failed) for success for each transfer """ src_node_name = lu.cfg.GetNodeName(src_node_uuid) dest_node_name = lu.cfg.GetNodeName(dest_node_uuid) logging.debug("Source node %s, destination node %s, compression '%s'", src_node_name, dest_node_name, compress) timeouts = ImportExportTimeouts(constants.DISK_TRANSFER_CONNECT_TIMEOUT) src_cbs = _TransferInstSourceCb(lu, feedback_fn, instance, timeouts, src_node_uuid, None, dest_node_uuid, dest_ip) dest_cbs = _TransferInstDestCb(lu, feedback_fn, instance, timeouts, src_node_uuid, src_cbs, dest_node_uuid, dest_ip) all_dtp = [] base_magic = utils.GenerateSecret(6) ieloop = ImportExportLoop(lu) try: for idx, transfer in enumerate(all_transfers): if transfer: feedback_fn("Exporting %s from %s to %s" % (transfer.name, src_node_name, dest_node_name)) magic = _GetInstDiskMagic(base_magic, instance.name, idx) opts = objects.ImportExportOptions(key_name=None, ca_pem=None, compress=compress, magic=magic) dtp = _DiskTransferPrivate(transfer, True, opts) di = DiskImport(lu, dest_node_uuid, opts, instance, "disk%d" % idx, transfer.dest_io, transfer.dest_ioargs, timeouts, dest_cbs, private=dtp) ieloop.Add(di) dtp.dest_import = di else: dtp = _DiskTransferPrivate(None, False, None) all_dtp.append(dtp) ieloop.Run() finally: ieloop.FinalizeAll() assert len(all_dtp) == len(all_transfers) assert compat.all((dtp.src_export is None or dtp.src_export.success is not None) and (dtp.dest_import is None or dtp.dest_import.success is not None) for dtp in all_dtp), \ "Not all imports/exports are finalized" return [bool(dtp.success) for dtp in all_dtp] class _RemoteExportCb(ImportExportCbBase): def __init__(self, feedback_fn, disk_count): """Initializes this class. """ ImportExportCbBase.__init__(self) self._feedback_fn = feedback_fn self._dresults = [None] * disk_count @property def disk_results(self): """Returns per-disk results. """ return self._dresults def ReportConnected(self, ie, private): """Called when a connection has been established. """ (idx, _) = private self._feedback_fn("Disk %s is now sending data" % idx) def ReportProgress(self, ie, private): """Called when new progress information should be reported. """ (idx, _) = private progress = ie.progress if not progress: return self._feedback_fn("Disk %s sent %s" % (idx, FormatProgress(progress))) def ReportFinished(self, ie, private): """Called when a transfer has finished. """ (idx, finished_fn) = private if ie.success: self._feedback_fn("Disk %s finished sending data" % idx) else: self._feedback_fn("Disk %s failed to send data: %s (recent output: %s)" % (idx, ie.final_message, ie.recent_output)) self._dresults[idx] = bool(ie.success) if finished_fn: finished_fn() class ExportInstanceHelper(object): def __init__(self, lu, feedback_fn, instance): """Initializes this class. @param lu: Logical unit instance @param feedback_fn: Feedback function @type instance: L{objects.Instance} @param instance: Instance object """ self._lu = lu self._feedback_fn = feedback_fn self._instance = instance self._snapshots = [None] * len(instance.disks) self._snapshots_removed = [False] * len(instance.disks) def _SnapshotsReady(self): """Returns true if snapshots are ready to be used in exports. """ return all(self._snapshots) def CreateSnapshots(self): """Attempts to create a snapshot for every disk of the instance. Currently support drbd, plain and ext disk templates. @rtype: bool @return: Whether following transfers can use snapshots """ if any(self._snapshots): raise errors.ProgrammerError("Snapshot creation was invoked more than " "once") instance = self._instance inst_disks = self._lu.cfg.GetInstanceDisks(instance.uuid) # A quick check whether we can support snapshots at all if not all([d.SupportsSnapshots() for d in inst_disks]): return False src_node = instance.primary_node src_node_name = self._lu.cfg.GetNodeName(src_node) for idx, disk in enumerate(inst_disks): self._feedback_fn("Creating a snapshot of disk/%s on node %s" % (idx, src_node_name)) # result.payload will be a snapshot of an lvm leaf of the one we # passed result = self._lu.rpc.call_blockdev_snapshot(src_node, (disk, instance), None, None) msg = result.fail_msg if msg: self._lu.LogWarning("Could not snapshot disk/%s on node %s: %s", idx, src_node_name, msg) elif (not isinstance(result.payload, (tuple, list)) or len(result.payload) != 2): self._lu.LogWarning("Could not snapshot disk/%s on node %s: invalid" " result '%s'", idx, src_node_name, result.payload) else: disk_id = tuple(result.payload) # Snapshot is currently supported for ExtStorage and LogicalVolume. # In case disk is of type drbd the snapshot will be of type plain. if disk.dev_type == constants.DT_EXT: dev_type = constants.DT_EXT else: dev_type = constants.DT_PLAIN disk_params = constants.DISK_LD_DEFAULTS[dev_type].copy() disk_params.update(disk.params) new_dev = objects.Disk(dev_type=dev_type, size=disk.size, logical_id=disk_id, iv_name=disk.iv_name, params=disk_params) new_dev.uuid = self._lu.cfg.GenerateUniqueID(self._lu.proc.GetECId()) self._snapshots[idx] = new_dev self._snapshots_removed[idx] = False # One final check to see if we have managed to snapshot everything if self._SnapshotsReady(): return True else: # If we failed to do so, the existing snapshots are of little value to us # so we can remove them straight away. self.Cleanup() return False def _RemoveSnapshot(self, disk_index): """Removes an LVM snapshot. @type disk_index: number @param disk_index: Index of the snapshot to be removed """ snapshot = self._snapshots[disk_index] if snapshot is not None and not self._snapshots_removed[disk_index]: src_node_uuid = self._instance.primary_node src_node_name = self._lu.cfg.GetNodeName(src_node_uuid) self._feedback_fn("Removing snapshot of disk/%s on node %s" % (disk_index, src_node_name)) result = self._lu.rpc.call_blockdev_remove(src_node_uuid, (snapshot, self._instance)) if result.fail_msg: self._lu.LogWarning("Could not remove snapshot for disk/%d from node" " %s: %s", disk_index, src_node_name, result.fail_msg) else: self._snapshots_removed[disk_index] = True def _GetDisksToTransfer(self): """Returns disks to be transferred, whether snapshots or instance disks. @rtype: list of L{objects.Disk} @return: The disks to transfer """ if self._SnapshotsReady(): return self._snapshots else: return self._lu.cfg.GetInstanceDisks(self._instance.uuid) def _GetDiskLabel(self, idx): """Returns a label which should be used to represent a disk to transfer. @type idx: int @param idx: The disk index @rtype: string """ if self._SnapshotsReady(): return "snapshot/%d" % idx else: return "disk/%d" % idx def LocalExport(self, dest_node, compress): """Intra-cluster instance export. @type dest_node: L{objects.Node} @param dest_node: Destination node @type compress: string @param compress: Compression tool to use """ disks_to_transfer = self._GetDisksToTransfer() instance = self._instance src_node_uuid = instance.primary_node transfers = [] for idx, dev in enumerate(disks_to_transfer): path = utils.PathJoin(pathutils.EXPORT_DIR, "%s.new" % instance.name, dev.uuid) finished_fn = compat.partial(self._TransferFinished, idx) if instance.os: src_io = constants.IEIO_SCRIPT src_ioargs = ((dev, instance), idx) else: src_io = constants.IEIO_RAW_DISK src_ioargs = (dev, instance) # FIXME: pass debug option from opcode to backend dt = DiskTransfer(self._GetDiskLabel(idx), src_io, src_ioargs, constants.IEIO_FILE, (path, ), finished_fn) transfers.append(dt) # Actually export data dresults = TransferInstanceData(self._lu, self._feedback_fn, src_node_uuid, dest_node.uuid, dest_node.secondary_ip, compress, instance, transfers) assert len(dresults) == len(instance.disks) # Finalize only if all the disks have been exported successfully if all(dresults): self._feedback_fn("Finalizing export on %s" % dest_node.name) result = self._lu.rpc.call_finalize_export(dest_node.uuid, instance, disks_to_transfer) msg = result.fail_msg fin_resu = not msg if msg: self._lu.LogWarning("Could not finalize export for instance %s" " on node %s: %s", instance.name, dest_node.name, msg) else: fin_resu = False self._lu.LogWarning("Some disk exports have failed; there may be " "leftover data for instance %s on node %s", instance.name, dest_node.name) return (fin_resu, dresults) def RemoteExport(self, disk_info, key_name, dest_ca_pem, compress, timeouts): """Inter-cluster instance export. @type disk_info: list @param disk_info: Per-disk destination information @type key_name: string @param key_name: Name of X509 key to use @type dest_ca_pem: string @param dest_ca_pem: Destination X509 CA in PEM format @type compress: string @param compress: Compression tool to use @type timeouts: L{ImportExportTimeouts} @param timeouts: Timeouts for this import """ instance = self._instance disks_to_transfer = self._GetDisksToTransfer() assert len(disk_info) == len(disks_to_transfer) cbs = _RemoteExportCb(self._feedback_fn, len(disks_to_transfer)) ieloop = ImportExportLoop(self._lu) try: for idx, (dev, (host, port, magic)) in enumerate(zip(disks_to_transfer, disk_info)): # Decide whether to use IPv6 ipv6 = netutils.IP6Address.IsValid(host) opts = objects.ImportExportOptions(key_name=key_name, ca_pem=dest_ca_pem, magic=magic, compress=compress, ipv6=ipv6) if instance.os: src_io = constants.IEIO_SCRIPT src_ioargs = ((dev, instance), idx) else: src_io = constants.IEIO_RAW_DISK src_ioargs = (dev, instance) self._feedback_fn("Sending disk %s to %s:%s" % (idx, host, port)) finished_fn = compat.partial(self._TransferFinished, idx) ieloop.Add(DiskExport(self._lu, instance.primary_node, opts, host, port, instance, "disk%d" % idx, src_io, src_ioargs, timeouts, cbs, private=(idx, finished_fn))) ieloop.Run() finally: ieloop.FinalizeAll() return (True, cbs.disk_results) def _TransferFinished(self, idx): """Called once a transfer has finished. @type idx: number @param idx: Disk index """ logging.debug("Transfer %s finished", idx) self._RemoveSnapshot(idx) def Cleanup(self): """Remove all snapshots. """ for idx in range(len(self._snapshots)): self._RemoveSnapshot(idx) class _RemoteImportCb(ImportExportCbBase): def __init__(self, feedback_fn, cds, x509_cert_pem, disk_count, external_address): """Initializes this class. @type cds: string @param cds: Cluster domain secret @type x509_cert_pem: string @param x509_cert_pem: CA used for signing import key @type disk_count: number @param disk_count: Number of disks @type external_address: string @param external_address: External address of destination node """ ImportExportCbBase.__init__(self) self._feedback_fn = feedback_fn self._cds = cds self._x509_cert_pem = x509_cert_pem self._disk_count = disk_count self._external_address = external_address self._dresults = [None] * disk_count self._daemon_port = [None] * disk_count self._salt = utils.GenerateSecret(8) @property def disk_results(self): """Returns per-disk results. """ return self._dresults def _CheckAllListening(self): """Checks whether all daemons are listening. If all daemons are listening, the information is sent to the client. """ if not compat.all(dp is not None for dp in self._daemon_port): return host = self._external_address disks = [] for idx, (port, magic) in enumerate(self._daemon_port): disks.append(ComputeRemoteImportDiskInfo(self._cds, self._salt, idx, host, port, magic)) assert len(disks) == self._disk_count self._feedback_fn(constants.ELOG_REMOTE_IMPORT, { "disks": disks, "x509_ca": self._x509_cert_pem, }) def ReportListening(self, ie, private, _): """Called when daemon started listening. """ (idx, ) = private self._feedback_fn("Disk %s is now listening" % idx) assert self._daemon_port[idx] is None self._daemon_port[idx] = (ie.listen_port, ie.magic) self._CheckAllListening() def ReportConnected(self, ie, private): """Called when a connection has been established. """ (idx, ) = private self._feedback_fn("Disk %s is now receiving data" % idx) def ReportFinished(self, ie, private): """Called when a transfer has finished. """ (idx, ) = private # Daemon is certainly no longer listening self._daemon_port[idx] = None if ie.success: self._feedback_fn("Disk %s finished receiving data" % idx) else: self._feedback_fn(("Disk %s failed to receive data: %s" " (recent output: %s)") % (idx, ie.final_message, ie.recent_output)) self._dresults[idx] = bool(ie.success) def RemoteImport(lu, feedback_fn, instance, pnode, source_x509_ca, cds, compress, timeouts): """Imports an instance from another cluster. @param lu: Logical unit instance @param feedback_fn: Feedback function @type instance: L{objects.Instance} @param instance: Instance object @type pnode: L{objects.Node} @param pnode: Primary node of instance as an object @type source_x509_ca: OpenSSL.crypto.X509 @param source_x509_ca: Import source's X509 CA @type cds: string @param cds: Cluster domain secret @type compress: string @param compress: Compression tool to use @type timeouts: L{ImportExportTimeouts} @param timeouts: Timeouts for this import """ source_ca_pem = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, source_x509_ca) magic_base = utils.GenerateSecret(6) # Decide whether to use IPv6 ipv6 = netutils.IP6Address.IsValid(pnode.primary_ip) # Create crypto key result = lu.rpc.call_x509_cert_create(instance.primary_node, constants.RIE_CERT_VALIDITY) result.Raise("Can't create X509 key and certificate on %s" % result.node) (x509_key_name, x509_cert_pem) = result.payload try: # Load certificate x509_cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, x509_cert_pem) # Sign certificate signed_x509_cert_pem = \ utils.SignX509Certificate(x509_cert, cds, utils.GenerateSecret(8)) cbs = _RemoteImportCb(feedback_fn, cds, signed_x509_cert_pem, len(instance.disks), pnode.primary_ip) ieloop = ImportExportLoop(lu) inst_disks = lu.cfg.GetInstanceDisks(instance.uuid) try: for idx, dev in enumerate(inst_disks): magic = _GetInstDiskMagic(magic_base, instance.name, idx) # Import daemon options opts = objects.ImportExportOptions(key_name=x509_key_name, ca_pem=source_ca_pem, magic=magic, compress=compress, ipv6=ipv6) if instance.os: src_io = constants.IEIO_SCRIPT src_ioargs = ((dev, instance), idx) else: src_io = constants.IEIO_RAW_DISK src_ioargs = (dev, instance) ieloop.Add(DiskImport(lu, instance.primary_node, opts, instance, "disk%d" % idx, src_io, src_ioargs, timeouts, cbs, private=(idx, ))) ieloop.Run() finally: ieloop.FinalizeAll() finally: # Remove crypto key and certificate result = lu.rpc.call_x509_cert_remove(instance.primary_node, x509_key_name) result.Raise("Can't remove X509 key and certificate on %s" % result.node) return cbs.disk_results def _GetImportExportHandshakeMessage(version): """Returns the handshake message for a RIE protocol version. @type version: number """ return "%s:%s" % (version, constants.RIE_HANDSHAKE) def ComputeRemoteExportHandshake(cds): """Computes the remote import/export handshake. @type cds: string @param cds: Cluster domain secret """ salt = utils.GenerateSecret(8) msg = _GetImportExportHandshakeMessage(constants.RIE_VERSION) return (constants.RIE_VERSION, utils.Sha1Hmac(cds, msg, salt=salt), salt) def CheckRemoteExportHandshake(cds, handshake): """Checks the handshake of a remote import/export. @type cds: string @param cds: Cluster domain secret @type handshake: sequence @param handshake: Handshake sent by remote peer """ try: (version, hmac_digest, hmac_salt) = handshake except (TypeError, ValueError) as err: return "Invalid data: %s" % err if not utils.VerifySha1Hmac(cds, _GetImportExportHandshakeMessage(version), hmac_digest, salt=hmac_salt): return "Hash didn't match, clusters don't share the same domain secret" if version != constants.RIE_VERSION: return ("Clusters don't have the same remote import/export protocol" " (local=%s, remote=%s)" % (constants.RIE_VERSION, version)) return None def _GetRieDiskInfoMessage(disk_index, host, port, magic): """Returns the hashed text for import/export disk information. @type disk_index: number @param disk_index: Index of disk (included in hash) @type host: string @param host: Hostname @type port: number @param port: Daemon port @type magic: string @param magic: Magic value """ return "%s:%s:%s:%s" % (disk_index, host, port, magic) def CheckRemoteExportDiskInfo(cds, disk_index, disk_info): """Verifies received disk information for an export. @type cds: string @param cds: Cluster domain secret @type disk_index: number @param disk_index: Index of disk (included in hash) @type disk_info: sequence @param disk_info: Disk information sent by remote peer """ try: (host, port, magic, hmac_digest, hmac_salt) = disk_info except (TypeError, ValueError) as err: raise errors.GenericError("Invalid data: %s" % err) if not (host and port and magic): raise errors.GenericError("Missing destination host, port or magic") msg = _GetRieDiskInfoMessage(disk_index, host, port, magic) if not utils.VerifySha1Hmac(cds, msg, hmac_digest, salt=hmac_salt): raise errors.GenericError("HMAC is wrong") if netutils.IP6Address.IsValid(host) or netutils.IP4Address.IsValid(host): destination = host else: destination = netutils.Hostname.GetNormalizedName(host) return (destination, utils.ValidateServiceName(port), magic) def ComputeRemoteImportDiskInfo(cds, salt, disk_index, host, port, magic): """Computes the signed disk information for a remote import. @type cds: string @param cds: Cluster domain secret @type salt: string @param salt: HMAC salt @type disk_index: number @param disk_index: Index of disk (included in hash) @type host: string @param host: Hostname @type port: number @param port: Daemon port @type magic: string @param magic: Magic value """ msg = _GetRieDiskInfoMessage(disk_index, host, port, magic) hmac_digest = utils.Sha1Hmac(cds, msg, salt=salt) return (host, port, magic, hmac_digest, salt) def CalculateGroupIPolicy(cluster, group): """Calculate instance policy for group. """ return cluster.SimpleFillIPolicy(group.ipolicy) def ComputeDiskSize(disks): """Compute disk size requirements according to disk template """ # Required free disk space as a function of disk and swap space def size_f(d): dev_type = d[constants.IDISK_TYPE] req_size_dict = { constants.DT_DISKLESS: 0, constants.DT_PLAIN: d[constants.IDISK_SIZE], # Extra space for drbd metadata is added to each disk constants.DT_DRBD8: d[constants.IDISK_SIZE] + constants.DRBD_META_SIZE, constants.DT_FILE: d[constants.IDISK_SIZE], constants.DT_SHARED_FILE: d[constants.IDISK_SIZE], constants.DT_GLUSTER: d[constants.IDISK_SIZE], constants.DT_BLOCK: 0, constants.DT_RBD: d[constants.IDISK_SIZE], constants.DT_EXT: d[constants.IDISK_SIZE], } if dev_type not in req_size_dict: raise errors.ProgrammerError("Disk template '%s' size requirement" " is unknown" % dev_type) return req_size_dict[dev_type] return sum(map(size_f, disks)) ganeti-3.1.0~rc2/lib/mcpu.py000064400000000000000000000607671476477700300157100ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing the logic behind the cluster operations This module implements the logic for doing operations in the cluster. There are two kinds of classes defined: - logical units, which know how to deal with their specific opcode only - the processor, which dispatches the opcodes to their logical units """ import sys import logging import random import time import itertools import traceback from ganeti import opcodes from ganeti import opcodes_base from ganeti import constants from ganeti import errors from ganeti import hooksmaster from ganeti import cmdlib from ganeti import locking from ganeti import utils from ganeti import wconfd sighupReceived = [False] lusExecuting = [0] _OP_PREFIX = "Op" _LU_PREFIX = "LU" class LockAcquireTimeout(Exception): """Exception to report timeouts on acquiring locks. """ def _CalculateLockAttemptTimeouts(): """Calculate timeouts for lock attempts. """ result = [constants.LOCK_ATTEMPTS_MINWAIT] running_sum = result[0] # Wait for a total of at least LOCK_ATTEMPTS_TIMEOUT before doing a # blocking acquire while running_sum < constants.LOCK_ATTEMPTS_TIMEOUT: timeout = (result[-1] * 1.05) ** 1.25 # Cap max timeout. This gives other jobs a chance to run even if # we're still trying to get our locks, before finally moving to a # blocking acquire. timeout = min(timeout, constants.LOCK_ATTEMPTS_MAXWAIT) # And also cap the lower boundary for safety timeout = max(timeout, constants.LOCK_ATTEMPTS_MINWAIT) result.append(timeout) running_sum += timeout return result class LockAttemptTimeoutStrategy(object): """Class with lock acquire timeout strategy. """ __slots__ = [ "_timeouts", "_random_fn", "_time_fn", ] _TIMEOUT_PER_ATTEMPT = _CalculateLockAttemptTimeouts() def __init__(self, _time_fn=time.time, _random_fn=random.random): """Initializes this class. @param _time_fn: Time function for unittests @param _random_fn: Random number generator for unittests """ object.__init__(self) self._timeouts = iter(self._TIMEOUT_PER_ATTEMPT) self._time_fn = _time_fn self._random_fn = _random_fn def NextAttempt(self): """Returns the timeout for the next attempt. """ try: timeout = next(self._timeouts) except StopIteration: # No more timeouts, do blocking acquire timeout = None if timeout is not None: # Add a small variation (-/+ 5%) to timeout. This helps in situations # where two or more jobs are fighting for the same lock(s). variation_range = timeout * 0.1 timeout += ((self._random_fn() * variation_range) - (variation_range * 0.5)) return timeout class OpExecCbBase(object): # pylint: disable=W0232 """Base class for OpCode execution callbacks. """ def NotifyStart(self): """Called when we are about to execute the LU. This function is called when we're about to start the lu's Exec() method, that is, after we have acquired all locks. """ def NotifyRetry(self): """Called when we are about to reset an LU to retry again. This function is called after PrepareRetry successfully completed. """ # TODO: Cleanup calling conventions, make them explicit. def Feedback(self, *args): """Sends feedback from the LU code to the end-user. """ def CurrentPriority(self): # pylint: disable=R0201 """Returns current priority or C{None}. """ return None def SubmitManyJobs(self, jobs): """Submits jobs for processing. See L{jqueue.JobQueue.SubmitManyJobs}. """ raise NotImplementedError def _LUNameForOpName(opname): """Computes the LU name for a given OpCode name. """ assert opname.startswith(_OP_PREFIX), \ "Invalid OpCode name, doesn't start with %s: %s" % (_OP_PREFIX, opname) return _LU_PREFIX + opname[len(_OP_PREFIX):] def _ComputeDispatchTable(): """Computes the opcode-to-lu dispatch table. """ return dict((op, getattr(cmdlib, _LUNameForOpName(op.__name__))) for op in opcodes.OP_MAPPING.values() if op.WITH_LU) def _SetBaseOpParams(src, defcomment, dst): """Copies basic opcode parameters. @type src: L{opcodes.OpCode} @param src: Source opcode @type defcomment: string @param defcomment: Comment to specify if not already given @type dst: L{opcodes.OpCode} @param dst: Destination opcode """ if hasattr(src, "debug_level"): dst.debug_level = src.debug_level if (getattr(dst, "priority", None) is None and hasattr(src, "priority")): dst.priority = src.priority if not getattr(dst, opcodes_base.COMMENT_ATTR, None): dst.comment = defcomment if hasattr(src, constants.OPCODE_REASON): dst.reason = list(getattr(dst, constants.OPCODE_REASON, [])) dst.reason.extend(getattr(src, constants.OPCODE_REASON, [])) def _ProcessResult(submit_fn, op, result): """Examines opcode result. If necessary, additional processing on the result is done. """ if isinstance(result, cmdlib.ResultWithJobs): # Copy basic parameters (e.g. priority) for op2 in itertools.chain(*result.jobs): _SetBaseOpParams(op, "Submitted by %s" % op.OP_ID, op2) # Submit jobs job_submission = submit_fn(result.jobs) # Build dictionary result = result.other assert constants.JOB_IDS_KEY not in result, \ "Key '%s' found in additional return values" % constants.JOB_IDS_KEY result[constants.JOB_IDS_KEY] = job_submission return result def _FailingSubmitManyJobs(_): """Implementation of L{OpExecCbBase.SubmitManyJobs} to raise an exception. """ raise errors.ProgrammerError("Opcodes processed without callbacks (e.g." " queries) can not submit jobs") def _LockList(names): """If 'names' is a string, make it a single-element list. @type names: list or string or NoneType @param names: Lock names @rtype: a list of strings @return: if 'names' argument is an iterable, a list of it; if it's a string, make it a one-element list; if L{locking.ALL_SET}, L{locking.ALL_SET} """ if names == locking.ALL_SET: return names elif isinstance(names, str): return [names] else: return list(names) def _CheckSecretParameters(op): """Check if secret parameters are expected, but missing. """ if hasattr(op, "osparams_secret") and op.osparams_secret: for secret_param in op.osparams_secret: if op.osparams_secret[secret_param].Get() == constants.REDACTED: raise errors.OpPrereqError("Please re-submit secret parameters to job.", errors.ECODE_INVAL) class Processor(object): """Object which runs OpCodes""" DISPATCH_TABLE = _ComputeDispatchTable() def __init__(self, context, ec_id, enable_locks=True): """Constructor for Processor @type context: GanetiContext @param context: global Ganeti context @type ec_id: string @param ec_id: execution context identifier """ self._ec_id = ec_id self._cbs = None self.cfg = context.GetConfig(ec_id) self.rpc = context.GetRpc(self.cfg) self.hmclass = hooksmaster.HooksMaster self._enable_locks = enable_locks self.wconfd = wconfd # Indirection to allow testing self._wconfdcontext = context.GetWConfdContext(ec_id) def _CheckLocksEnabled(self): """Checks if locking is enabled. @raise errors.ProgrammerError: In case locking is not enabled """ if not self._enable_locks: raise errors.ProgrammerError("Attempted to use disabled locks") def _RequestAndWait(self, request, timeout): """Request locks from WConfD and wait for them to be granted. @type request: list @param request: the lock request to be sent to WConfD @type timeout: float @param timeout: the time to wait for the request to be granted @raise LockAcquireTimeout: In case locks couldn't be acquired in specified amount of time; in this case, locks still might be acquired or a request pending. """ logging.debug("Trying %ss to request %s for %s", timeout, request, self._wconfdcontext) if self._cbs: priority = self._cbs.CurrentPriority() # pylint: disable=W0612 else: priority = None if priority is None: priority = constants.OP_PRIO_DEFAULT ## Expect a signal if sighupReceived[0]: logging.warning("Ignoring unexpected SIGHUP") sighupReceived[0] = False # Request locks self.wconfd.Client().UpdateLocksWaiting(self._wconfdcontext, priority, request) pending = self.wconfd.Client().HasPendingRequest(self._wconfdcontext) if pending: def _HasPending(): if sighupReceived[0]: return self.wconfd.Client().HasPendingRequest(self._wconfdcontext) else: return True pending = utils.SimpleRetry(False, _HasPending, 0.05, timeout) signal = sighupReceived[0] if pending: pending = self.wconfd.Client().HasPendingRequest(self._wconfdcontext) if pending and signal: logging.warning("Ignoring unexpected SIGHUP") sighupReceived[0] = False logging.debug("Finished trying. Pending: %s", pending) if pending: raise LockAcquireTimeout() def _AcquireLocks(self, level, names, shared, opportunistic, timeout, opportunistic_count=1, request_only=False): """Acquires locks via the Ganeti lock manager. @type level: int @param level: Lock level @type names: list or string @param names: Lock names @type shared: bool @param shared: Whether the locks should be acquired in shared mode @type opportunistic: bool @param opportunistic: Whether to acquire opportunistically @type timeout: None or float @param timeout: Timeout for acquiring the locks @type request_only: bool @param request_only: do not acquire the locks, just return the request @raise LockAcquireTimeout: In case locks couldn't be acquired in specified amount of time; in this case, locks still might be acquired or a request pending. """ self._CheckLocksEnabled() if self._cbs: priority = self._cbs.CurrentPriority() # pylint: disable=W0612 else: priority = None if priority is None: priority = constants.OP_PRIO_DEFAULT if names == locking.ALL_SET: if opportunistic: expand_fns = { locking.LEVEL_CLUSTER: (lambda: [locking.BGL]), locking.LEVEL_INSTANCE: self.cfg.GetInstanceList, locking.LEVEL_NODEGROUP: self.cfg.GetNodeGroupList, locking.LEVEL_NODE: self.cfg.GetNodeList, locking.LEVEL_NODE_RES: self.cfg.GetNodeList, locking.LEVEL_NETWORK: self.cfg.GetNetworkList, } names = expand_fns[level]() else: names = locking.LOCKSET_NAME names = _LockList(names) # For locks of the same level, the lock order is lexicographic names.sort() levelname = locking.LEVEL_NAMES[level] locks = ["%s/%s" % (levelname, lock) for lock in list(names)] if not names: logging.debug("Acquiring no locks for (%s) at level %s", self._wconfdcontext, levelname) return [] if shared: request = [[lock, "shared"] for lock in locks] else: request = [[lock, "exclusive"] for lock in locks] if request_only: logging.debug("Lock request for level %s is %s", level, request) return request self.cfg.OutDate() if timeout is None: ## Note: once we are so desperate for locks to request them ## unconditionally, we no longer care about an original plan ## to acquire locks opportunistically. logging.info("Definitely requesting %s for %s", request, self._wconfdcontext) ## The only way to be sure of not getting starved is to sequentially ## acquire the locks one by one (in lock order). for r in request: logging.debug("Definite request %s for %s", r, self._wconfdcontext) self.wconfd.Client().UpdateLocksWaiting(self._wconfdcontext, priority, [r]) while True: pending = self.wconfd.Client().HasPendingRequest(self._wconfdcontext) if not pending: break time.sleep(10.0 * random.random()) elif opportunistic: logging.debug("For %ss trying to opportunistically acquire" " at least %d of %s for %s.", timeout, opportunistic_count, locks, self._wconfdcontext) locks = utils.SimpleRetry( lambda l: l != [], self.wconfd.Client().GuardedOpportunisticLockUnion, 2.0, timeout, args=[opportunistic_count, self._wconfdcontext, request]) logging.debug("Managed to get the following locks: %s", locks) if locks == []: raise LockAcquireTimeout() else: self._RequestAndWait(request, timeout) return locks def _ExecLU(self, lu): """Logical Unit execution sequence. """ write_count = self.cfg.write_count lu.cfg.OutDate() lu.CheckPrereq() hm = self.BuildHooksManager(lu) h_results = hm.RunPhase(constants.HOOKS_PHASE_PRE) lu.HooksCallBack(constants.HOOKS_PHASE_PRE, h_results, self.Log, None) if getattr(lu.op, "dry_run", False): # in this mode, no post-hooks are run, and the config is not # written (as it might have been modified by another LU, and we # shouldn't do writeout on behalf of other threads self.LogInfo("dry-run mode requested, not actually executing" " the operation") return lu.dry_run_result if self._cbs: submit_mj_fn = self._cbs.SubmitManyJobs else: submit_mj_fn = _FailingSubmitManyJobs lusExecuting[0] += 1 try: result = _ProcessResult(submit_mj_fn, lu.op, lu.Exec(self.Log)) h_results = hm.RunPhase(constants.HOOKS_PHASE_POST) result = lu.HooksCallBack(constants.HOOKS_PHASE_POST, h_results, self.Log, result) finally: # FIXME: This needs locks if not lu_class.REQ_BGL lusExecuting[0] -= 1 if write_count != self.cfg.write_count: hm.RunConfigUpdate() return result def BuildHooksManager(self, lu): return self.hmclass.BuildFromLu(lu.rpc.call_hooks_runner, lu) def _LockAndExecLU(self, lu, level, calc_timeout, pending=None): """Execute a Logical Unit, with the needed locks. This is a recursive function that starts locking the given level, and proceeds up, till there are no more locks to acquire. Then it executes the given LU and its opcodes. """ pending = pending or [] logging.debug("Looking at locks of level %s, still need to obtain %s", level, pending) adding_locks = level in lu.add_locks acquiring_locks = level in lu.needed_locks if level not in locking.LEVELS: if pending: self._RequestAndWait(pending, calc_timeout()) lu.cfg.OutDate() lu.wconfdlocks = self.wconfd.Client().ListLocks(self._wconfdcontext) pending = [] logging.debug("Finished acquiring locks") if self._cbs: self._cbs.NotifyStart() try: result = self._ExecLU(lu) except errors.OpPrereqError as err: if len(err.args) < 2 or err.args[1] != errors.ECODE_TEMP_NORES: raise logging.debug("Temporarily out of resources; will retry internally") try: lu.PrepareRetry(self.Log) if self._cbs: self._cbs.NotifyRetry() except errors.OpRetryNotSupportedError: logging.debug("LU does not know how to retry.") raise err raise LockAcquireTimeout() except AssertionError as err: # this is a bit ugly, as we don't know from which phase # (prereq, exec) this comes; but it's better than an exception # with no information (_, _, tb) = sys.exc_info() err_info = traceback.format_tb(tb) del tb logging.exception("Detected AssertionError") raise errors.OpExecError("Internal assertion error: please report" " this as a bug.\nError message: '%s';" " location:\n%s" % (str(err), err_info[-1])) return result # Determine if the acquiring is opportunistic up front opportunistic = lu.opportunistic_locks[level] dont_collate = lu.dont_collate_locks[level] if dont_collate and pending: self._RequestAndWait(pending, calc_timeout()) lu.cfg.OutDate() lu.wconfdlocks = self.wconfd.Client().ListLocks(self._wconfdcontext) pending = [] if adding_locks and opportunistic: # We could simultaneously acquire locks opportunistically and add new # ones, but that would require altering the API, and no use cases are # present in the system at the moment. raise NotImplementedError("Can't opportunistically acquire locks when" " adding new ones") if adding_locks and acquiring_locks and \ lu.needed_locks[level] == locking.ALL_SET: # It would also probably be possible to acquire all locks of a certain # type while adding new locks, but there is no use case at the moment. raise NotImplementedError("Can't request all locks of a certain level" " and add new locks") if adding_locks or acquiring_locks: self._CheckLocksEnabled() lu.DeclareLocks(level) share = lu.share_locks[level] opportunistic_count = lu.opportunistic_locks_count[level] try: if acquiring_locks: needed_locks = _LockList(lu.needed_locks[level]) else: needed_locks = [] if adding_locks: needed_locks.extend(_LockList(lu.add_locks[level])) timeout = calc_timeout() if timeout is not None and not opportunistic: pending = pending + self._AcquireLocks(level, needed_locks, share, opportunistic, timeout, request_only=True) else: if pending: self._RequestAndWait(pending, calc_timeout()) lu.cfg.OutDate() lu.wconfdlocks = self.wconfd.Client().ListLocks(self._wconfdcontext) pending = [] self._AcquireLocks(level, needed_locks, share, opportunistic, timeout, opportunistic_count=opportunistic_count) lu.wconfdlocks = self.wconfd.Client().ListLocks(self._wconfdcontext) result = self._LockAndExecLU(lu, level + 1, calc_timeout, pending=pending) finally: levelname = locking.LEVEL_NAMES[level] logging.debug("Freeing locks at level %s for %s", levelname, self._wconfdcontext) self.wconfd.Client().FreeLocksLevel(self._wconfdcontext, levelname) else: result = self._LockAndExecLU(lu, level + 1, calc_timeout, pending=pending) return result # pylint: disable=R0201 def _CheckLUResult(self, op, result): """Check the LU result against the contract in the opcode. """ resultcheck_fn = op.OP_RESULT if not (resultcheck_fn is None or resultcheck_fn(result)): logging.error("Expected opcode result matching %s, got %s", resultcheck_fn, result) if not getattr(op, "dry_run", False): # FIXME: LUs should still behave in dry_run mode, or # alternately we should have OP_DRYRUN_RESULT; in the # meantime, we simply skip the OP_RESULT check in dry-run mode raise errors.OpResultError("Opcode result does not match %s: %s" % (resultcheck_fn, utils.Truncate(result, 80))) def ExecOpCode(self, op, cbs, timeout=None): """Execute an opcode. @type op: an OpCode instance @param op: the opcode to be executed @type cbs: L{OpExecCbBase} @param cbs: Runtime callbacks @type timeout: float or None @param timeout: Maximum time to acquire all locks, None for no timeout @raise LockAcquireTimeout: In case locks couldn't be acquired in specified amount of time """ if not isinstance(op, opcodes.OpCode): raise errors.ProgrammerError("Non-opcode instance passed" " to ExecOpcode (%s)" % type(op)) lu_class = self.DISPATCH_TABLE.get(op.__class__, None) if lu_class is None: raise errors.OpCodeUnknown("Unknown opcode") if timeout is None: calc_timeout = lambda: None else: calc_timeout = utils.RunningTimeout(timeout, False).Remaining self._cbs = cbs try: if self._enable_locks: # Acquire the Big Ganeti Lock exclusively if this LU requires it, # and in a shared fashion otherwise (to prevent concurrent run with # an exclusive LU. self._AcquireLocks(locking.LEVEL_CLUSTER, locking.BGL, not lu_class.REQ_BGL, False, calc_timeout()) elif lu_class.REQ_BGL: raise errors.ProgrammerError("Opcode '%s' requires BGL, but locks are" " disabled" % op.OP_ID) lu = lu_class(self, op, self.cfg, self.rpc, self._wconfdcontext, self.wconfd) lu.wconfdlocks = self.wconfd.Client().ListLocks(self._wconfdcontext) _CheckSecretParameters(op) lu.ExpandNames() assert lu.needed_locks is not None, "needed_locks not set by LU" try: result = self._LockAndExecLU(lu, locking.LEVEL_CLUSTER + 1, calc_timeout) finally: if self._ec_id: self.cfg.DropECReservations(self._ec_id) finally: self.wconfd.Client().FreeLocksLevel( self._wconfdcontext, locking.LEVEL_NAMES[locking.LEVEL_CLUSTER]) self._cbs = None self._CheckLUResult(op, result) return result def Log(self, *args): """Forward call to feedback callback function. """ if self._cbs: self._cbs.Feedback(*args) def LogStep(self, current, total, message): """Log a change in LU execution progress. """ logging.debug("Step %d/%d %s", current, total, message) self.Log("STEP %d/%d %s" % (current, total, message)) def LogWarning(self, message, *args, **kwargs): """Log a warning to the logs and the user. The optional keyword argument is 'hint' and can be used to show a hint to the user (presumably related to the warning). If the message is empty, it will not be printed at all, allowing one to show only a hint. """ assert not kwargs or (len(kwargs) == 1 and "hint" in kwargs), \ "Invalid keyword arguments for LogWarning (%s)" % str(kwargs) if args: message = message % tuple(args) if message: logging.warning(message) self.Log(" - WARNING: %s" % message) if "hint" in kwargs: self.Log(" Hint: %s" % kwargs["hint"]) def LogInfo(self, message, *args): """Log an informational message to the logs and the user. """ if args: message = message % tuple(args) logging.info(message) self.Log(" - INFO: %s" % message) def GetECId(self): """Returns the current execution context ID. """ if not self._ec_id: raise errors.ProgrammerError("Tried to use execution context id when" " not set") return self._ec_id ganeti-3.1.0~rc2/lib/metad.py000064400000000000000000000064371476477700300160300ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module for the Metad protocol """ import logging import random import time from ganeti import constants from ganeti import errors import ganeti.rpc.client as cl from ganeti.rpc.transport import Transport from ganeti.rpc.errors import TimeoutError # If the metadata daemon is disabled, there is no stub generated for it. # So import the module and define the client class only if enabled, otherwise # just generate an empty placeholder class. if constants.ENABLE_METAD: import ganeti.rpc.stub.metad as stub class Client(cl.AbstractStubClient, stub.ClientRpcStub): """High-level Metad client implementation. This uses a backing Transport-like class on top of which it implements data serialization/deserialization. """ def __init__(self, timeouts=None, transport=Transport): """Constructor for the Client class. Arguments are the same as for L{AbstractClient}. """ cl.AbstractStubClient.__init__(self, timeouts, transport) stub.ClientRpcStub.__init__(self) retries = 12 for try_no in range(0, retries): try: self._InitTransport() return except TimeoutError: logging.debug("Timout trying to connect to MetaD") if try_no == retries - 1: raise logging.debug("Will retry") time.sleep(try_no * 10 + 10 * random.random()) def _InitTransport(self): """(Re)initialize the transport if needed. """ if self.transport is None: self.transport = self.transport_class(self._GetAddress(), timeouts=self.timeouts, allow_non_master=True) else: class Client(object): """An empty client representation that just throws an exception. """ def __init__(self, _timeouts=None, _transport=None): raise errors.ProgrammerError("The metadata deamon is disabled, yet" " the client has been called") ganeti-3.1.0~rc2/lib/netutils.py000064400000000000000000000502451476477700300166010ustar00rootroot00000000000000# # # Copyright (C) 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Ganeti network utility module. This module holds functions that can be used in both daemons (all) and the command line scripts. """ import errno import os import re import socket import struct import logging from ganeti import constants from ganeti import errors from ganeti import utils from ganeti import vcluster # Structure definition for getsockopt(SOL_SOCKET, SO_PEERCRED, ...): # struct ucred { pid_t pid; uid_t uid; gid_t gid; }; # # The GNU C Library defines gid_t and uid_t to be "unsigned int" and # pid_t to "int". # # IEEE Std 1003.1-2008: # "nlink_t, uid_t, gid_t, and id_t shall be integer types" # "blksize_t, pid_t, and ssize_t shall be signed integer types" _STRUCT_UCRED = "iII" _STRUCT_UCRED_SIZE = struct.calcsize(_STRUCT_UCRED) # Regexes used to find IP addresses in the output of ip. _IP_RE_TEXT = r"[.:a-z0-9]+" # separate for testing purposes _IP_FAMILY_RE = re.compile(r"(?Pinet6?)\s+(?P%s)/" % _IP_RE_TEXT, re.IGNORECASE) # Dict used to convert from a string representing an IP family to an IP # version _NAME_TO_IP_VER = { "inet": constants.IP4_VERSION, "inet6": constants.IP6_VERSION, } def _GetIpAddressesFromIpOutput(ip_output): """Parses the output of the ip command and retrieves the IP addresses and version. @param ip_output: string containing the output of the ip command; @rtype: dict; (int, list) @return: a dict having as keys the IP versions and as values the corresponding list of addresses found in the IP output. """ addr = dict((i, []) for i in _NAME_TO_IP_VER.values()) for row in ip_output.splitlines(): match = _IP_FAMILY_RE.search(row) if match and IPAddress.IsValid(match.group("ip")): addr[_NAME_TO_IP_VER[match.group("family")]].append(match.group("ip")) return addr def GetSocketCredentials(sock): """Returns the credentials of the foreign process connected to a socket. @param sock: Unix socket @rtype: tuple; (number, number, number) @return: The PID, UID and GID of the connected foreign process. """ peercred = sock.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, _STRUCT_UCRED_SIZE) return struct.unpack(_STRUCT_UCRED, peercred) def IsValidInterface(ifname): """Validate an interface name. @type ifname: string @param ifname: Name of the network interface @return: boolean indicating whether the interface name is valid or not. """ return os.path.exists(utils.PathJoin("/sys/class/net", ifname)) def GetInterfaceIpAddresses(ifname): """Returns the IP addresses associated to the interface. @type ifname: string @param ifname: Name of the network interface @return: A dict having for keys the IP version (either L{constants.IP4_VERSION} or L{constants.IP6_VERSION}) and for values the lists of IP addresses of the respective version associated to the interface """ result = utils.RunCmd([constants.IP_COMMAND_PATH, "-o", "addr", "show", "dev", ifname]) if result.failed: logging.error("Error running the ip command while getting the IP" " addresses of %s", ifname) return None return _GetIpAddressesFromIpOutput(result.output) def GetHostname(name=None, family=None): """Returns a Hostname object. @type name: str @param name: hostname or None @type family: int @param family: AF_INET | AF_INET6 | None @rtype: L{Hostname} @return: Hostname object @raise errors.OpPrereqError: in case of errors in resolving """ try: return Hostname(name=name, family=family) except errors.ResolverError as err: raise errors.OpPrereqError("The given name (%s) does not resolve: %s" % (err.args[0], err.args[2]), errors.ECODE_RESOLVER) class Hostname(object): """Class implementing resolver and hostname functionality. """ _VALID_NAME_RE = re.compile("^[a-z0-9._-]{1,255}$") def __init__(self, name=None, family=None): """Initialize the host name object. If the name argument is None, it will use this system's name. @type family: int @param family: AF_INET | AF_INET6 | None @type name: str @param name: hostname or None """ self.name = self.GetFqdn(name) self.ip = self.GetIP(self.name, family=family) @classmethod def GetSysName(cls): """Legacy method the get the current system's name. """ return cls.GetFqdn() @classmethod def GetFqdn(cls, hostname=None): """Return fqdn. If hostname is None the system's fqdn is returned. @type hostname: str @param hostname: name to be fqdn'ed @rtype: str @return: fqdn of given name, if it exists, unmodified name otherwise """ if hostname is None: virtfqdn = vcluster.GetVirtualHostname() if virtfqdn: result = virtfqdn else: result = socket.getfqdn() else: result = socket.getfqdn(hostname) return cls.GetNormalizedName(result) @staticmethod def GetIP(hostname, family=None): """Return IP address of given hostname. Supports both IPv4 and IPv6. @type hostname: str @param hostname: hostname to look up @type family: int @param family: AF_INET | AF_INET6 | None @rtype: str @return: IP address @raise errors.ResolverError: in case of errors in resolving """ try: if family in (socket.AF_INET, socket.AF_INET6): result = socket.getaddrinfo(hostname, None, family) else: result = socket.getaddrinfo(hostname, None) except (socket.gaierror, socket.herror, socket.error) as err: # hostname not found in DNS, or other socket exception in the # (code, description format) raise errors.ResolverError(hostname, err.args[0], err.args[1]) # getaddrinfo() returns a list of 5-tupes (family, socktype, proto, # canonname, sockaddr). We return the first tuple's first address in # sockaddr try: return result[0][4][0] except IndexError as err: # we don't have here an actual error code, it's just that the # data type returned by getaddrinfo is not what we expected; # let's keep the same format in the exception arguments with a # dummy error code raise errors.ResolverError(hostname, 0, "Unknown error in getaddrinfo(): %s" % err) @classmethod def GetNormalizedName(cls, hostname): """Validate and normalize the given hostname. @attention: the validation is a bit more relaxed than the standards require; most importantly, we allow underscores in names @raise errors.OpPrereqError: when the name is not valid """ hostname = hostname.lower() if (not cls._VALID_NAME_RE.match(hostname) or # double-dots, meaning empty label ".." in hostname or # empty initial label hostname.startswith(".")): raise errors.OpPrereqError("Invalid hostname '%s'" % hostname, errors.ECODE_INVAL) if hostname.endswith("."): hostname = hostname.rstrip(".") return hostname def ValidatePortNumber(port): """Returns the validated integer port number if it is valid. @param port: the port number to be validated @raise ValueError: if the port is not valid @rtype: int @return: the validated value. """ try: port = int(port) except TypeError: raise errors.ProgrammerError("ValidatePortNumber called with non-numeric" " type %s." % port.__class__.__name__) except ValueError: raise ValueError("Invalid port value: '%s'" % port) if not 0 < port < 2 ** 16: raise ValueError("Invalid port value: '%d'" % port) return port def TcpPing(target, port, timeout=10, live_port_needed=False, source=None): """Simple ping implementation using TCP connect(2). Check if the given IP is reachable by doing attempting a TCP connect to it. @type target: str @param target: the IP to ping @type port: int @param port: the port to connect to @type timeout: int @param timeout: the timeout on the connection attempt @type live_port_needed: boolean @param live_port_needed: whether a closed port will cause the function to return failure, as if there was a timeout @type source: str or None @param source: if specified, will cause the connect to be made from this specific source address; failures to bind other than C{EADDRNOTAVAIL} will be ignored """ logging.debug("Attempting to reach TCP port %s on target %s with a timeout" " of %s seconds", port, target, timeout) try: family = IPAddress.GetAddressFamily(target) except errors.IPAddressError as err: raise errors.ProgrammerError("Family of IP address given in parameter" " 'target' can't be determined: %s" % err) sock = socket.socket(family, socket.SOCK_STREAM) success = False if source is not None: try: sock.bind((source, 0)) except socket.error as err: if err.errno == errno.EADDRNOTAVAIL: success = False sock.settimeout(timeout) try: sock.connect((target, port)) success = True except socket.timeout: success = False except socket.error as err: success = (not live_port_needed) and (err.errno == errno.ECONNREFUSED) finally: sock.close() return success def GetDaemonPort(daemon_name): """Get the daemon port for this cluster. Note that this routine does not read a ganeti-specific file, but instead uses C{socket.getservbyname} to allow pre-customization of this parameter outside of Ganeti. @type daemon_name: string @param daemon_name: daemon name (in constants.DAEMONS_PORTS) @rtype: int """ if daemon_name not in constants.DAEMONS_PORTS: raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name) (proto, default_port) = constants.DAEMONS_PORTS[daemon_name] try: port = socket.getservbyname(daemon_name, proto) except socket.error: port = default_port return port class IPAddress(object): """Class that represents an IP address. """ iplen = 0 family = None loopback_cidr = None @staticmethod def _GetIPIntFromString(address): """Abstract method to please pylint. """ raise NotImplementedError @classmethod def IsValid(cls, address): """Validate a IP address. @type address: str @param address: IP address to be checked @rtype: bool @return: True if valid, False otherwise """ if cls.family is None: try: family = cls.GetAddressFamily(address) except errors.IPAddressError: return False else: family = cls.family try: socket.inet_pton(family, address) return True except socket.error: return False @classmethod def ValidateNetmask(cls, netmask): """Validate a netmask suffix in CIDR notation. @type netmask: int @param netmask: netmask suffix to validate @rtype: bool @return: True if valid, False otherwise """ assert isinstance(netmask, int) return 0 < netmask <= cls.iplen @classmethod def Own(cls, address): """Check if the current host has the the given IP address. This is done by trying to bind the given address. We return True if we succeed or false if a socket.error is raised. @type address: str @param address: IP address to be checked @rtype: bool @return: True if we own the address, False otherwise """ if cls.family is None: try: family = cls.GetAddressFamily(address) except errors.IPAddressError: return False else: family = cls.family s = socket.socket(family, socket.SOCK_DGRAM) success = False try: try: s.bind((address, 0)) success = True except socket.error: success = False finally: s.close() return success @classmethod def InNetwork(cls, cidr, address): """Determine whether an address is within a network. @type cidr: string @param cidr: Network in CIDR notation, e.g. '192.0.2.0/24', '2001:db8::/64' @type address: str @param address: IP address @rtype: bool @return: True if address is in cidr, False otherwise """ address_int = cls._GetIPIntFromString(address) subnet = cidr.split("/") assert len(subnet) == 2 try: prefix = int(subnet[1]) except ValueError: return False assert 0 <= prefix <= cls.iplen target_int = cls._GetIPIntFromString(subnet[0]) # Convert prefix netmask to integer value of netmask netmask_int = (2 ** cls.iplen) - 1 ^ ((2 ** cls.iplen) - 1 >> prefix) # Calculate hostmask hostmask_int = netmask_int ^ (2 ** cls.iplen) - 1 # Calculate network address by and'ing netmask network_int = target_int & netmask_int # Calculate broadcast address by or'ing hostmask broadcast_int = target_int | hostmask_int return network_int <= address_int <= broadcast_int @staticmethod def GetAddressFamily(address): """Get the address family of the given address. @type address: str @param address: ip address whose family will be returned @rtype: int @return: C{socket.AF_INET} or C{socket.AF_INET6} @raise errors.GenericError: for invalid addresses """ try: return IP4Address(address).family except errors.IPAddressError: pass try: return IP6Address(address).family except errors.IPAddressError: pass raise errors.IPAddressError("Invalid address '%s'" % address) @staticmethod def GetVersionFromAddressFamily(family): """Convert an IP address family to the corresponding IP version. @type family: int @param family: IP address family, one of socket.AF_INET or socket.AF_INET6 @return: an int containing the IP version, one of L{constants.IP4_VERSION} or L{constants.IP6_VERSION} @raise errors.ProgrammerError: for unknown families """ if family == socket.AF_INET: return constants.IP4_VERSION elif family == socket.AF_INET6: return constants.IP6_VERSION raise errors.ProgrammerError("%s is not a valid IP address family" % family) @staticmethod def GetAddressFamilyFromVersion(version): """Convert an IP version to the corresponding IP address family. @type version: int @param version: IP version, one of L{constants.IP4_VERSION} or L{constants.IP6_VERSION} @return: an int containing the IP address family, one of C{socket.AF_INET} or C{socket.AF_INET6} @raise errors.ProgrammerError: for unknown IP versions """ if version == constants.IP4_VERSION: return socket.AF_INET elif version == constants.IP6_VERSION: return socket.AF_INET6 raise errors.ProgrammerError("%s is not a valid IP version" % version) @staticmethod def GetClassFromIpVersion(version): """Return the IPAddress subclass for the given IP version. @type version: int @param version: IP version, one of L{constants.IP4_VERSION} or L{constants.IP6_VERSION} @return: a subclass of L{netutils.IPAddress} @raise errors.ProgrammerError: for unknowo IP versions """ if version == constants.IP4_VERSION: return IP4Address elif version == constants.IP6_VERSION: return IP6Address raise errors.ProgrammerError("%s is not a valid IP version" % version) @staticmethod def GetClassFromIpFamily(family): """Return the IPAddress subclass for the given IP family. @param family: IP family (one of C{socket.AF_INET} or C{socket.AF_INET6} @return: a subclass of L{netutils.IPAddress} @raise errors.ProgrammerError: for unknowo IP versions """ return IPAddress.GetClassFromIpVersion( IPAddress.GetVersionFromAddressFamily(family)) @classmethod def IsLoopback(cls, address): """Determine whether it is a loopback address. @type address: str @param address: IP address to be checked @rtype: bool @return: True if loopback, False otherwise """ try: return cls.InNetwork(cls.loopback_cidr, address) except errors.IPAddressError: return False class IP4Address(IPAddress): """IPv4 address class. """ iplen = 32 family = socket.AF_INET loopback_cidr = "127.0.0.0/8" def __init__(self, address): """Constructor for IPv4 address. @type address: str @param address: IP address @raises errors.IPAddressError: if address invalid """ IPAddress.__init__(self) if not self.IsValid(address): raise errors.IPAddressError("IPv4 Address %s invalid" % address) self.address = address @staticmethod def _GetIPIntFromString(address): """Get integer value of IPv4 address. @type address: str @param address: IPv6 address @rtype: int @return: integer value of given IP address """ address_int = 0 parts = address.split(".") assert len(parts) == 4 for part in parts: address_int = (address_int << 8) | int(part) return address_int class IP6Address(IPAddress): """IPv6 address class. """ iplen = 128 family = socket.AF_INET6 loopback_cidr = "::1/128" def __init__(self, address): """Constructor for IPv6 address. @type address: str @param address: IP address @raises errors.IPAddressError: if address invalid """ IPAddress.__init__(self) if not self.IsValid(address): raise errors.IPAddressError("IPv6 Address [%s] invalid" % address) self.address = address @staticmethod def _GetIPIntFromString(address): """Get integer value of IPv6 address. @type address: str @param address: IPv6 address @rtype: int @return: integer value of given IP address """ doublecolons = address.count("::") assert not doublecolons > 1 if doublecolons == 1: # We have a shorthand address, expand it parts = [] twoparts = address.split("::") sep = len(twoparts[0].split(":")) + len(twoparts[1].split(":")) parts = twoparts[0].split(":") parts.extend(["0"] * (8 - sep)) parts += twoparts[1].split(":") else: parts = address.split(":") address_int = 0 for part in parts: address_int = (address_int << 16) + int(part or "0", 16) return address_int def FormatAddress(address, family=None): """Format a socket address @type address: family specific (usually tuple) @param address: address, as reported by this class @type family: integer @param family: socket family (one of socket.AF_*) or None """ if family is None: try: family = IPAddress.GetAddressFamily(address[0]) except errors.IPAddressError: raise errors.ParameterError(address) if family == socket.AF_UNIX and len(address) == 3: return "pid=%s, uid=%s, gid=%s" % address if family in (socket.AF_INET, socket.AF_INET6) and len(address) == 2: host, port = address if family == socket.AF_INET6: res = "[%s]" % host else: res = host if port is not None: res += ":%s" % port return res raise errors.ParameterError(family, address) ganeti-3.1.0~rc2/lib/network.py000064400000000000000000000205611476477700300164210ustar00rootroot00000000000000# # # Copyright (C) 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """IP address pool management functions. """ import ipaddress from bitarray import bitarray from ganeti import errors def _ComputeIpv4NumHosts(network_size): """Derives the number of hosts in an IPv4 network from the size. """ return 2 ** (32 - network_size) IPV4_NETWORK_MIN_SIZE = 30 # FIXME: This limit is for performance reasons. Remove when refactoring # for performance tuning was successful. IPV4_NETWORK_MAX_SIZE = 16 IPV4_NETWORK_MIN_NUM_HOSTS = _ComputeIpv4NumHosts(IPV4_NETWORK_MIN_SIZE) IPV4_NETWORK_MAX_NUM_HOSTS = _ComputeIpv4NumHosts(IPV4_NETWORK_MAX_SIZE) class AddressPool(object): """Address pool class, wrapping an C{objects.Network} object. This class provides methods to manipulate address pools, backed by L{objects.Network} objects. """ FREE = bitarray("0") RESERVED = bitarray("1") def __init__(self, network): """Initialize a new IPv4 address pool from an L{objects.Network} object. @type network: L{objects.Network} @param network: the network object from which the pool will be generated """ self.network = None self.gateway = None self.network6 = None self.gateway6 = None self.net = network self.network = ipaddress.ip_network(self.net.network) if self.network.num_addresses > IPV4_NETWORK_MAX_NUM_HOSTS: raise errors.AddressPoolError("A big network with %s host(s) is currently" " not supported. please specify at most a" " /%s network" % (str(self.network.num_addresses), IPV4_NETWORK_MAX_SIZE)) if self.network.num_addresses < IPV4_NETWORK_MIN_NUM_HOSTS: raise errors.AddressPoolError("A network with only %s host(s) is too" " small, please specify at least a /%s" " network" % (str(self.network.num_addresses), IPV4_NETWORK_MIN_SIZE)) if self.net.gateway: self.gateway = ipaddress.ip_address(self.net.gateway) if self.net.network6: self.network6 = ipaddress.IPv6Network(self.net.network6) if self.net.gateway6: self.gateway6 = ipaddress.IPv6Address(self.net.gateway6) if self.net.reservations: self.reservations = bitarray(self.net.reservations) else: self.reservations = bitarray(self.network.num_addresses) # pylint: disable=E1103 self.reservations.setall(False) if self.net.ext_reservations: self.ext_reservations = bitarray(self.net.ext_reservations) else: self.ext_reservations = bitarray(self.network.num_addresses) # pylint: disable=E1103 self.ext_reservations.setall(False) assert len(self.reservations) == self.network.num_addresses assert len(self.ext_reservations) == self.network.num_addresses def Contains(self, address): if address is None: return False addr = ipaddress.ip_address(address) return addr in self.network def _GetAddrIndex(self, address): addr = ipaddress.ip_address(address) if not addr in self.network: raise errors.AddressPoolError("%s does not contain %s" % (self.network, addr)) return int(addr) - int(self.network.network_address) def Update(self): """Write address pools back to the network object. """ # pylint: disable=E1103 self.net.ext_reservations = self.ext_reservations.to01() self.net.reservations = self.reservations.to01() def _Mark(self, address, value=True, external=False): idx = self._GetAddrIndex(address) if external: self.ext_reservations[idx] = value else: self.reservations[idx] = value self.Update() def _GetSize(self): return 2 ** (32 - self.network.prefixlen) @property def all_reservations(self): """Return a combined map of internal and external reservations. """ return (self.reservations | self.ext_reservations) def Validate(self): assert len(self.reservations) == self._GetSize() assert len(self.ext_reservations) == self._GetSize() if self.gateway is not None: assert self.gateway in self.network if self.network6 and self.gateway6: assert self.gateway6 in self.network6 or self.gateway6.is_link_local def IsFull(self): """Check whether the network is full. """ return self.all_reservations.all() def GetReservedCount(self): """Get the count of reserved addresses. """ return self.all_reservations.count(True) def GetFreeCount(self): """Get the count of unused addresses. """ return self.all_reservations.count(False) def GetMap(self): """Return a textual representation of the network's occupation status. """ return self.all_reservations.to01().replace("1", "X").replace("0", ".") def IsReserved(self, address, external=False): """Checks if the given IP is reserved. """ idx = self._GetAddrIndex(address) if external: return self.ext_reservations[idx] else: return self.reservations[idx] def Reserve(self, address, external=False): """Mark an address as used. """ if self.IsReserved(address, external): if external: msg = "IP %s is already externally reserved" % address else: msg = "IP %s is already used by an instance" % address raise errors.AddressPoolError(msg) self._Mark(address, external=external) def Release(self, address, external=False): """Release a given address reservation. """ if not self.IsReserved(address, external): if external: msg = "IP %s is not externally reserved" % address else: msg = "IP %s is not used by an instance" % address raise errors.AddressPoolError(msg) self._Mark(address, value=False, external=external) def GetFreeAddress(self): """Returns the first available address. """ if self.IsFull(): raise errors.AddressPoolError("%s is full" % self.network) idx = self.all_reservations.index(False) address = str(self.network[idx]) self.Reserve(address) return address def GenerateFree(self): """Returns the first free address of the network. @raise errors.AddressPoolError: Pool is full """ idx = self.all_reservations.search(self.FREE, 1) if idx: return str(self.network[idx[0]]) else: raise errors.AddressPoolError("%s is full" % self.network) def GetExternalReservations(self): """Returns a list of all externally reserved addresses. """ # pylint: disable=E1103 idxs = self.ext_reservations.search(self.RESERVED) return [str(self.network[idx]) for idx in idxs] @classmethod def InitializeNetwork(cls, net): """Initialize an L{objects.Network} object. Reserve the network, broadcast and gateway IP addresses. """ obj = cls(net) obj.Update() for ip in [obj.network[0], obj.network[-1]]: obj.Reserve(ip, external=True) if obj.net.gateway is not None: obj.Reserve(obj.net.gateway, external=True) obj.Validate() return obj ganeti-3.1.0~rc2/lib/objects.py000064400000000000000000002226631476477700300163700ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Transportable objects for Ganeti. This module provides small, mostly data-only objects which are safe to pass to and from external parties. """ # pylint: disable=E0203,E0237,W0201,R0902 # E0203: Access to member %r before its definition, since we use # objects.py which doesn't explicitly initialise its members # E0237: Assigning to attribute not defined in class slots. pylint doesn't # appear to notice many of the slots defined in __slots__ for several objects. # W0201: Attribute '%s' defined outside __init__ # R0902: Allow instances of these objects to have more than 20 attributes import configparser import re import copy import logging import time from io import StringIO from socket import AF_INET from ganeti import errors from ganeti import constants from ganeti import netutils from ganeti import outils from ganeti import utils from ganeti import serializer __all__ = ["ConfigObject", "ConfigData", "NIC", "Disk", "Instance", "OS", "Node", "NodeGroup", "Cluster", "FillDict", "Network", "Filter"] _TIMESTAMPS = ["ctime", "mtime"] _UUID = ["uuid"] def FillDict(defaults_dict, custom_dict, skip_keys=None): """Basic function to apply settings on top a default dict. @type defaults_dict: dict @param defaults_dict: dictionary holding the default values @type custom_dict: dict @param custom_dict: dictionary holding customized value @type skip_keys: list @param skip_keys: which keys not to fill @rtype: dict @return: dict with the 'full' values """ ret_dict = copy.deepcopy(defaults_dict) ret_dict.update(custom_dict) if skip_keys: for k in skip_keys: if k in ret_dict: del ret_dict[k] return ret_dict def FillIPolicy(default_ipolicy, custom_ipolicy): """Fills an instance policy with defaults. """ assert frozenset(default_ipolicy) == constants.IPOLICY_ALL_KEYS ret_dict = copy.deepcopy(custom_ipolicy) for key in default_ipolicy: if key not in ret_dict: ret_dict[key] = copy.deepcopy(default_ipolicy[key]) elif key == constants.ISPECS_STD: ret_dict[key] = FillDict(default_ipolicy[key], ret_dict[key]) return ret_dict def FillDiskParams(default_dparams, custom_dparams, skip_keys=None): """Fills the disk parameter defaults. @see: L{FillDict} for parameters and return value """ return dict((dt, FillDict(default_dparams.get(dt, {}), custom_dparams.get(dt, {}), skip_keys=skip_keys)) for dt in constants.DISK_TEMPLATES) def UpgradeGroupedParams(target, defaults): """Update all groups for the target parameter. @type target: dict of dicts @param target: {group: {parameter: value}} @type defaults: dict @param defaults: default parameter values """ if target is None: target = {constants.PP_DEFAULT: defaults} else: for group in target: target[group] = FillDict(defaults, target[group]) return target def UpgradeBeParams(target): """Update the be parameters dict to the new format. @type target: dict @param target: "be" parameters dict """ if constants.BE_MEMORY in target: memory = target[constants.BE_MEMORY] target[constants.BE_MAXMEM] = memory target[constants.BE_MINMEM] = memory del target[constants.BE_MEMORY] def UpgradeDiskParams(diskparams): """Upgrade the disk parameters. @type diskparams: dict @param diskparams: disk parameters to upgrade @rtype: dict @return: the upgraded disk parameters dict """ if not diskparams: result = {} else: result = FillDiskParams(constants.DISK_DT_DEFAULTS, diskparams) return result def UpgradeNDParams(ndparams): """Upgrade ndparams structure. @type ndparams: dict @param ndparams: disk parameters to upgrade @rtype: dict @return: the upgraded node parameters dict """ if ndparams is None: ndparams = {} if (constants.ND_OOB_PROGRAM in ndparams and ndparams[constants.ND_OOB_PROGRAM] is None): # will be reset by the line below del ndparams[constants.ND_OOB_PROGRAM] return FillDict(constants.NDC_DEFAULTS, ndparams) def MakeEmptyIPolicy(): """Create empty IPolicy dictionary. """ return {} class ConfigObject(outils.ValidatedSlots): """A generic config object. It has the following properties: - provides somewhat safe recursive unpickling and pickling for its classes - unset attributes which are defined in slots are always returned as None instead of raising an error Classes derived from this must always declare __slots__ (we use many config objects and the memory reduction is useful) """ __slots__ = [] def __getattr__(self, name): if name not in self.GetAllSlots(): raise AttributeError("Invalid object attribute %s.%s" % (type(self).__name__, name)) return None def __setstate__(self, state): slots = self.GetAllSlots() for name in state: if name in slots: setattr(self, name, state[name]) def Validate(self): """Validates the slots. This method returns L{None} if the validation succeeds, or raises an exception otherwise. This method must be implemented by the child classes. @rtype: NoneType @return: L{None}, if the validation succeeds @raise Exception: validation fails """ def ToDict(self, _with_private=False): """Convert to a dict holding only standard python types. The generic routine just dumps all of this object's attributes in a dict. It does not work if the class has children who are ConfigObjects themselves (e.g. the nics list in an Instance), in which case the object should subclass the function in order to make sure all objects returned are only standard python types. Private fields can be included or not with the _with_private switch. The actual implementation of this switch is left for those subclassses with private fields to implement. @type _with_private: bool @param _with_private: if True, the object will leak its private fields in the dictionary representation. If False, the values will be replaced with None. """ result = {} for name in self.GetAllSlots(): value = getattr(self, name, None) if value is not None: result[name] = value return result __getstate__ = ToDict @classmethod def FromDict(cls, val): """Create an object from a dictionary. This generic routine takes a dict, instantiates a new instance of the given class, and sets attributes based on the dict content. As for `ToDict`, this does not work if the class has children who are ConfigObjects themselves (e.g. the nics list in an Instance), in which case the object should subclass the function and alter the objects. """ if not isinstance(val, dict): raise errors.ConfigurationError("Invalid object passed to FromDict:" " expected dict, got %s" % type(val)) val_str = dict([(str(k), v) for k, v in val.items()]) obj = cls(**val_str) return obj def Copy(self): """Makes a deep copy of the current object and its children. """ dict_form = self.ToDict() clone_obj = self.__class__.FromDict(dict_form) return clone_obj def __repr__(self): """Implement __repr__ for ConfigObjects.""" return repr(self.ToDict()) def UpgradeConfig(self): """Fill defaults for missing configuration values. This method will be called at configuration load time, and its implementation will be object dependent. """ pass class TaggableObject(ConfigObject): """An generic class supporting tags. """ __slots__ = ["tags"] VALID_TAG_RE = re.compile(r"^[\w.+*/:@-]+$") @classmethod def ValidateTag(cls, tag): """Check if a tag is valid. If the tag is invalid, an errors.TagError will be raised. The function has no return value. """ if not isinstance(tag, str): raise errors.TagError("Invalid tag type (not a string)") if len(tag) > constants.MAX_TAG_LEN: raise errors.TagError("Tag too long (>%d characters)" % constants.MAX_TAG_LEN) if not tag: raise errors.TagError("Tags cannot be empty") if not cls.VALID_TAG_RE.match(tag): raise errors.TagError("Tag contains invalid characters") def GetTags(self): """Return the tags list. """ tags = getattr(self, "tags", None) if tags is None: tags = self.tags = set() return tags def AddTag(self, tag): """Add a new tag. """ self.ValidateTag(tag) tags = self.GetTags() if len(tags) >= constants.MAX_TAGS_PER_OBJ: raise errors.TagError("Too many tags") self.GetTags().add(tag) def RemoveTag(self, tag): """Remove a tag. """ self.ValidateTag(tag) tags = self.GetTags() try: tags.remove(tag) except KeyError: raise errors.TagError("Tag not found") def ToDict(self, _with_private=False): """Taggable-object-specific conversion to standard python types. This replaces the tags set with a list. """ bo = super(TaggableObject, self).ToDict(_with_private=_with_private) tags = bo.get("tags", None) if isinstance(tags, set): bo["tags"] = list(tags) return bo @classmethod def FromDict(cls, val): """Custom function for instances. """ obj = super(TaggableObject, cls).FromDict(val) if hasattr(obj, "tags") and isinstance(obj.tags, list): obj.tags = set(obj.tags) return obj class MasterNetworkParameters(ConfigObject): """Network configuration parameters for the master @ivar uuid: master nodes UUID @ivar ip: master IP @ivar netmask: master netmask @ivar netdev: master network device @ivar ip_family: master IP family """ __slots__ = [ "uuid", "ip", "netmask", "netdev", "ip_family", ] class ConfigData(ConfigObject): """Top-level config object.""" __slots__ = [ "version", "cluster", "nodes", "nodegroups", "instances", "networks", "disks", "filters", "serial_no", ] + _TIMESTAMPS def ToDict(self, _with_private=False): """Custom function for top-level config data. This just replaces the list of nodes, instances, nodegroups, networks, disks and the cluster with standard python types. """ mydict = super(ConfigData, self).ToDict(_with_private=_with_private) mydict["cluster"] = mydict["cluster"].ToDict() for key in ("nodes", "instances", "nodegroups", "networks", "disks", "filters"): mydict[key] = outils.ContainerToDicts(mydict[key]) return mydict @classmethod def FromDict(cls, val): """Custom function for top-level config data """ obj = super(ConfigData, cls).FromDict(val) obj.cluster = Cluster.FromDict(obj.cluster) obj.nodes = outils.ContainerFromDicts(obj.nodes, dict, Node) obj.instances = \ outils.ContainerFromDicts(obj.instances, dict, Instance) obj.nodegroups = \ outils.ContainerFromDicts(obj.nodegroups, dict, NodeGroup) obj.networks = outils.ContainerFromDicts(obj.networks, dict, Network) obj.disks = outils.ContainerFromDicts(obj.disks, dict, Disk) obj.filters = outils.ContainerFromDicts(obj.filters, dict, Filter) return obj def DisksOfType(self, dev_type): """Check if in there is at disk of the given type in the configuration. @type dev_type: L{constants.DTS_BLOCK} @param dev_type: the type to look for @rtype: list of disks @return: all disks of the dev_type """ return [disk for disk in self.disks.values() if disk.IsBasedOnDiskType(dev_type)] def UpgradeConfig(self): """Fill defaults for missing configuration values. """ self.cluster.UpgradeConfig() for node in self.nodes.values(): node.UpgradeConfig() for instance in self.instances.values(): instance.UpgradeConfig() self._UpgradeEnabledDiskTemplates() if self.nodegroups is None: self.nodegroups = {} for nodegroup in self.nodegroups.values(): nodegroup.UpgradeConfig() InstancePolicy.UpgradeDiskTemplates( nodegroup.ipolicy, self.cluster.enabled_disk_templates) if self.cluster.drbd_usermode_helper is None: if self.cluster.IsDiskTemplateEnabled(constants.DT_DRBD8): self.cluster.drbd_usermode_helper = constants.DEFAULT_DRBD_HELPER if self.networks is None: self.networks = {} for network in self.networks.values(): network.UpgradeConfig() for disk in self.disks.values(): disk.UpgradeConfig() if self.filters is None: self.filters = {} def _UpgradeEnabledDiskTemplates(self): """Upgrade the cluster's enabled disk templates by inspecting the currently enabled and/or used disk templates. """ if not self.cluster.enabled_disk_templates: template_set = \ set([d.dev_type for d in self.disks.values()]) if any(not inst.disks for inst in self.instances.values()): template_set.add(constants.DT_DISKLESS) # Add drbd and plain, if lvm is enabled (by specifying a volume group) if self.cluster.volume_group_name: template_set.add(constants.DT_DRBD8) template_set.add(constants.DT_PLAIN) # Set enabled_disk_templates to the inferred disk templates. Order them # according to a preference list that is based on Ganeti's history of # supported disk templates. self.cluster.enabled_disk_templates = [] for preferred_template in constants.DISK_TEMPLATE_PREFERENCE: if preferred_template in template_set: self.cluster.enabled_disk_templates.append(preferred_template) template_set.remove(preferred_template) self.cluster.enabled_disk_templates.extend(list(template_set)) InstancePolicy.UpgradeDiskTemplates( self.cluster.ipolicy, self.cluster.enabled_disk_templates) class NIC(ConfigObject): """Config object representing a network card.""" __slots__ = ["name", "mac", "ip", "network", "nicparams", "netinfo", "pci", "hvinfo"] + _UUID @classmethod def CheckParameterSyntax(cls, nicparams): """Check the given parameters for validity. @type nicparams: dict @param nicparams: dictionary with parameter names/value @raise errors.ConfigurationError: when a parameter is not valid """ mode = nicparams[constants.NIC_MODE] if (mode not in constants.NIC_VALID_MODES and mode != constants.VALUE_AUTO): raise errors.ConfigurationError("Invalid NIC mode '%s'" % mode) if (mode == constants.NIC_MODE_BRIDGED and not nicparams[constants.NIC_LINK]): raise errors.ConfigurationError("Missing bridged NIC link") class Filter(ConfigObject): """Config object representing a filter rule.""" __slots__ = ["watermark", "priority", "predicates", "action", "reason_trail"] + _UUID class Disk(ConfigObject): """Config object representing a block device.""" __slots__ = [ "forthcoming", "name", "dev_type", "logical_id", "children", "nodes", "iv_name", "size", "mode", "params", "spindles", "pci", "hvinfo", "serial_no", # dynamic_params is special. It depends on the node this instance # is sent to, and should not be persisted. "dynamic_params" ] + _UUID + _TIMESTAMPS def _ComputeAllNodes(self): """Compute the list of all nodes covered by a device and its children.""" def _Helper(nodes, device): """Recursively compute nodes given a top device.""" if device.dev_type in constants.DTS_DRBD: nodes.extend(device.logical_id[:2]) if device.children: for child in device.children: _Helper(nodes, child) all_nodes = list() _Helper(all_nodes, self) return tuple(set(all_nodes)) all_nodes = property(_ComputeAllNodes, None, None, "List of names of all the nodes of a disk") def CreateOnSecondary(self): """Test if this device needs to be created on a secondary node.""" return self.dev_type in (constants.DT_DRBD8, constants.DT_PLAIN) def AssembleOnSecondary(self): """Test if this device needs to be assembled on a secondary node.""" return self.dev_type in (constants.DT_DRBD8, constants.DT_PLAIN) def OpenOnSecondary(self): """Test if this device needs to be opened on a secondary node.""" return self.dev_type in (constants.DT_PLAIN,) def SupportsSnapshots(self): """Test if this device supports snapshots.""" return self.dev_type in constants.DTS_SNAPSHOT_CAPABLE def StaticDevPath(self): """Return the device path if this device type has a static one. Some devices (LVM for example) live always at the same /dev/ path, irrespective of their status. For such devices, we return this path, for others we return None. @warning: The path returned is not a normalized pathname; callers should check that it is a valid path. """ if self.dev_type == constants.DT_PLAIN: return "/dev/%s/%s" % (self.logical_id[0], self.logical_id[1]) elif self.dev_type == constants.DT_BLOCK: return self.logical_id[1] elif self.dev_type == constants.DT_RBD: return "/dev/%s/%s" % (self.logical_id[0], self.logical_id[1]) return None def ChildrenNeeded(self): """Compute the needed number of children for activation. This method will return either -1 (all children) or a positive number denoting the minimum number of children needed for activation (only mirrored devices will usually return >=0). Currently, only DRBD8 supports diskless activation (therefore we return 0), for all other we keep the previous semantics and return -1. """ if self.dev_type == constants.DT_DRBD8: return 0 return -1 def IsBasedOnDiskType(self, dev_type): """Check if the disk or its children are based on the given type. @type dev_type: L{constants.DTS_BLOCK} @param dev_type: the type to look for @rtype: boolean @return: boolean indicating if a device of the given type was found or not """ if self.children: for child in self.children: if child.IsBasedOnDiskType(dev_type): return True return self.dev_type == dev_type def GetNodes(self, node_uuid): """This function returns the nodes this device lives on. Given the node on which the parent of the device lives on (or, in case of a top-level device, the primary node of the devices' instance), this function will return a list of nodes on which this devices needs to (or can) be assembled. """ if self.dev_type in [constants.DT_PLAIN, constants.DT_FILE, constants.DT_BLOCK, constants.DT_RBD, constants.DT_EXT, constants.DT_SHARED_FILE, constants.DT_GLUSTER]: result = [node_uuid] elif self.dev_type in constants.DTS_DRBD: result = [self.logical_id[0], self.logical_id[1]] if node_uuid not in result: raise errors.ConfigurationError("DRBD device passed unknown node") else: raise errors.ProgrammerError("Unhandled device type %s" % self.dev_type) return result def GetPrimaryNode(self, node_uuid): """This function returns the primary node of the device. If the device is not a DRBD device, we still return the node the device lives on. """ if self.dev_type in constants.DTS_DRBD: return self.logical_id[0] return node_uuid def ComputeNodeTree(self, parent_node_uuid): """Compute the node/disk tree for this disk and its children. This method, given the node on which the parent disk lives, will return the list of all (node UUID, disk) pairs which describe the disk tree in the most compact way. For example, a drbd/lvm stack will be returned as (primary_node, drbd) and (secondary_node, drbd) which represents all the top-level devices on the nodes. """ my_nodes = self.GetNodes(parent_node_uuid) result = [(node, self) for node in my_nodes] if not self.children: # leaf device return result for node in my_nodes: for child in self.children: child_result = child.ComputeNodeTree(node) if len(child_result) == 1: # child (and all its descendants) is simple, doesn't split # over multiple hosts, so we don't need to describe it, our # own entry for this node describes it completely continue else: # check if child nodes differ from my nodes; note that # subdisk can differ from the child itself, and be instead # one of its descendants for subnode, subdisk in child_result: if subnode not in my_nodes: result.append((subnode, subdisk)) # otherwise child is under our own node, so we ignore this # entry (but probably the other results in the list will # be different) return result def ComputeGrowth(self, amount): """Compute the per-VG growth requirements. This only works for VG-based disks. @type amount: integer @param amount: the desired increase in (user-visible) disk space @rtype: dict @return: a dictionary of volume-groups and the required size """ if self.dev_type == constants.DT_PLAIN: return {self.logical_id[0]: amount} elif self.dev_type == constants.DT_DRBD8: if self.children: return self.children[0].ComputeGrowth(amount) else: return {} else: # Other disk types do not require VG space return {} def RecordGrow(self, amount): """Update the size of this disk after growth. This method recurses over the disks's children and updates their size correspondigly. The method needs to be kept in sync with the actual algorithms from bdev. """ if self.dev_type in (constants.DT_PLAIN, constants.DT_FILE, constants.DT_RBD, constants.DT_EXT, constants.DT_SHARED_FILE, constants.DT_GLUSTER): self.size += amount elif self.dev_type == constants.DT_DRBD8: if self.children: self.children[0].RecordGrow(amount) self.size += amount else: raise errors.ProgrammerError("Disk.RecordGrow called for unsupported" " disk type %s" % self.dev_type) def Update(self, size=None, mode=None, spindles=None): """Apply changes to size, spindles and mode. """ if self.dev_type == constants.DT_DRBD8: if self.children: self.children[0].Update(size=size, mode=mode) else: assert not self.children if size is not None: self.size = size if mode is not None: self.mode = mode if spindles is not None: self.spindles = spindles def UnsetSize(self): """Sets recursively the size to zero for the disk and its children. """ if self.children: for child in self.children: child.UnsetSize() self.size = 0 def UpdateDynamicDiskParams(self, target_node_uuid, nodes_ip): """Updates the dynamic disk params for the given node. This is mainly used for drbd, which needs ip/port configuration. Arguments: - target_node_uuid: the node UUID we wish to configure for - nodes_ip: a mapping of node name to ip The target_node must exist in nodes_ip, and should be one of the nodes in the logical ID if this device is a DRBD device. """ if self.children: for child in self.children: child.UpdateDynamicDiskParams(target_node_uuid, nodes_ip) dyn_disk_params = {} if self.logical_id is not None and self.dev_type in constants.DTS_DRBD: pnode_uuid, snode_uuid, _, pminor, sminor, _ = self.logical_id if target_node_uuid not in (pnode_uuid, snode_uuid): # disk object is being sent to neither the primary nor the secondary # node. reset the dynamic parameters, the target node is not # supposed to use them. self.dynamic_params = dyn_disk_params return pnode_ip = nodes_ip.get(pnode_uuid, None) snode_ip = nodes_ip.get(snode_uuid, None) if pnode_ip is None or snode_ip is None: raise errors.ConfigurationError("Can't find primary or secondary node" " for %s" % str(self)) if pnode_uuid == target_node_uuid: dyn_disk_params[constants.DDP_LOCAL_IP] = pnode_ip dyn_disk_params[constants.DDP_REMOTE_IP] = snode_ip dyn_disk_params[constants.DDP_LOCAL_MINOR] = pminor dyn_disk_params[constants.DDP_REMOTE_MINOR] = sminor else: # it must be secondary, we tested above dyn_disk_params[constants.DDP_LOCAL_IP] = snode_ip dyn_disk_params[constants.DDP_REMOTE_IP] = pnode_ip dyn_disk_params[constants.DDP_LOCAL_MINOR] = sminor dyn_disk_params[constants.DDP_REMOTE_MINOR] = pminor self.dynamic_params = dyn_disk_params # pylint: disable=W0221 def ToDict(self, include_dynamic_params=False, _with_private=False): """Disk-specific conversion to standard python types. This replaces the children lists of objects with lists of standard python types. """ bo = super(Disk, self).ToDict(_with_private=_with_private) if not include_dynamic_params and "dynamic_params" in bo: del bo["dynamic_params"] if _with_private and "logical_id" in bo: mutable_id = list(bo["logical_id"]) mutable_id[5] = mutable_id[5].Get() bo["logical_id"] = tuple(mutable_id) for attr in ("children",): alist = bo.get(attr, None) if alist: bo[attr] = outils.ContainerToDicts(alist) return bo @classmethod def FromDict(cls, val): """Custom function for Disks """ obj = super(Disk, cls).FromDict(val) if obj.children: obj.children = outils.ContainerFromDicts(obj.children, list, Disk) if obj.logical_id and isinstance(obj.logical_id, list): obj.logical_id = tuple(obj.logical_id) if obj.dev_type in constants.DTS_DRBD: # we need a tuple of length six here if len(obj.logical_id) < 6: obj.logical_id += (None,) * (6 - len(obj.logical_id)) # If we do have a tuple of length 6, make the last entry (secret key) # private elif (len(obj.logical_id) == 6 and not isinstance(obj.logical_id[-1], serializer.Private)): obj.logical_id = obj.logical_id[:-1] + \ (serializer.Private(obj.logical_id[-1]),) return obj def __str__(self): """Custom str() formatter for disks. """ if self.dev_type == constants.DT_PLAIN: val = " parameters @rtype: list(dict) @return: a list of dicts, one for each node of the disk hierarchy. Each dict contains the LD parameters of the node. The tree is flattened in-order. """ if disk_template not in constants.DISK_TEMPLATES: raise errors.ProgrammerError("Unknown disk template %s" % disk_template) assert disk_template in disk_params result = list() dt_params = disk_params[disk_template] if disk_template == constants.DT_DRBD8: result.append(FillDict(constants.DISK_LD_DEFAULTS[constants.DT_DRBD8], { constants.LDP_RESYNC_RATE: dt_params[constants.DRBD_RESYNC_RATE], constants.LDP_BARRIERS: dt_params[constants.DRBD_DISK_BARRIERS], constants.LDP_NO_META_FLUSH: dt_params[constants.DRBD_META_BARRIERS], constants.LDP_DEFAULT_METAVG: dt_params[constants.DRBD_DEFAULT_METAVG], constants.LDP_DISK_CUSTOM: dt_params[constants.DRBD_DISK_CUSTOM], constants.LDP_NET_CUSTOM: dt_params[constants.DRBD_NET_CUSTOM], constants.LDP_PROTOCOL: dt_params[constants.DRBD_PROTOCOL], constants.LDP_DYNAMIC_RESYNC: dt_params[constants.DRBD_DYNAMIC_RESYNC], constants.LDP_PLAN_AHEAD: dt_params[constants.DRBD_PLAN_AHEAD], constants.LDP_FILL_TARGET: dt_params[constants.DRBD_FILL_TARGET], constants.LDP_DELAY_TARGET: dt_params[constants.DRBD_DELAY_TARGET], constants.LDP_MAX_RATE: dt_params[constants.DRBD_MAX_RATE], constants.LDP_MIN_RATE: dt_params[constants.DRBD_MIN_RATE], })) # data LV result.append(FillDict(constants.DISK_LD_DEFAULTS[constants.DT_PLAIN], { constants.LDP_STRIPES: dt_params[constants.DRBD_DATA_STRIPES], })) # metadata LV result.append(FillDict(constants.DISK_LD_DEFAULTS[constants.DT_PLAIN], { constants.LDP_STRIPES: dt_params[constants.DRBD_META_STRIPES], })) else: defaults = constants.DISK_LD_DEFAULTS[disk_template] values = {} for field in defaults: values[field] = dt_params[field] result.append(FillDict(defaults, values)) return result class InstancePolicy(ConfigObject): """Config object representing instance policy limits dictionary. Note that this object is not actually used in the config, it's just used as a placeholder for a few functions. """ @classmethod def UpgradeDiskTemplates(cls, ipolicy, enabled_disk_templates): """Upgrades the ipolicy configuration.""" if constants.IPOLICY_DTS in ipolicy: if not set(ipolicy[constants.IPOLICY_DTS]).issubset( set(enabled_disk_templates)): ipolicy[constants.IPOLICY_DTS] = list( set(ipolicy[constants.IPOLICY_DTS]) & set(enabled_disk_templates)) @classmethod def CheckParameterSyntax(cls, ipolicy, check_std): """ Check the instance policy for validity. @type ipolicy: dict @param ipolicy: dictionary with min/max/std specs and policies @type check_std: bool @param check_std: Whether to check std value or just assume compliance @raise errors.ConfigurationError: when the policy is not legal """ InstancePolicy.CheckISpecSyntax(ipolicy, check_std) if constants.IPOLICY_DTS in ipolicy: InstancePolicy.CheckDiskTemplates(ipolicy[constants.IPOLICY_DTS]) for key in constants.IPOLICY_PARAMETERS: if key in ipolicy: InstancePolicy.CheckParameter(key, ipolicy[key]) wrong_keys = frozenset(ipolicy) - constants.IPOLICY_ALL_KEYS if wrong_keys: raise errors.ConfigurationError("Invalid keys in ipolicy: %s" % utils.CommaJoin(wrong_keys)) @classmethod def _CheckIncompleteSpec(cls, spec, keyname): missing_params = constants.ISPECS_PARAMETERS - frozenset(spec) if missing_params: msg = ("Missing instance specs parameters for %s: %s" % (keyname, utils.CommaJoin(missing_params))) raise errors.ConfigurationError(msg) @classmethod def CheckISpecSyntax(cls, ipolicy, check_std): """Check the instance policy specs for validity. @type ipolicy: dict @param ipolicy: dictionary with min/max/std specs @type check_std: bool @param check_std: Whether to check std value or just assume compliance @raise errors.ConfigurationError: when specs are not valid """ if constants.ISPECS_MINMAX not in ipolicy: # Nothing to check return if check_std and constants.ISPECS_STD not in ipolicy: msg = "Missing key in ipolicy: %s" % constants.ISPECS_STD raise errors.ConfigurationError(msg) stdspec = ipolicy.get(constants.ISPECS_STD) if check_std: InstancePolicy._CheckIncompleteSpec(stdspec, constants.ISPECS_STD) if not ipolicy[constants.ISPECS_MINMAX]: raise errors.ConfigurationError("Empty minmax specifications") std_is_good = False for minmaxspecs in ipolicy[constants.ISPECS_MINMAX]: missing = constants.ISPECS_MINMAX_KEYS - frozenset(minmaxspecs) if missing: msg = "Missing instance specification: %s" % utils.CommaJoin(missing) raise errors.ConfigurationError(msg) for (key, spec) in minmaxspecs.items(): InstancePolicy._CheckIncompleteSpec(spec, key) spec_std_ok = True for param in constants.ISPECS_PARAMETERS: par_std_ok = InstancePolicy._CheckISpecParamSyntax(minmaxspecs, stdspec, param, check_std) spec_std_ok = spec_std_ok and par_std_ok std_is_good = std_is_good or spec_std_ok if not std_is_good: raise errors.ConfigurationError("Invalid std specifications") @classmethod def _CheckISpecParamSyntax(cls, minmaxspecs, stdspec, name, check_std): """Check the instance policy specs for validity on a given key. We check if the instance specs makes sense for a given key, that is if minmaxspecs[min][name] <= stdspec[name] <= minmaxspec[max][name]. @type minmaxspecs: dict @param minmaxspecs: dictionary with min and max instance spec @type stdspec: dict @param stdspec: dictionary with standard instance spec @type name: string @param name: what are the limits for @type check_std: bool @param check_std: Whether to check std value or just assume compliance @rtype: bool @return: C{True} when specs are valid, C{False} when standard spec for the given name is not valid @raise errors.ConfigurationError: when min/max specs for the given name are not valid """ minspec = minmaxspecs[constants.ISPECS_MIN] maxspec = minmaxspecs[constants.ISPECS_MAX] min_v = minspec[name] max_v = maxspec[name] if min_v > max_v: err = ("Invalid specification of min/max values for %s: %s/%s" % (name, min_v, max_v)) raise errors.ConfigurationError(err) elif check_std: std_v = stdspec.get(name, min_v) return std_v >= min_v and std_v <= max_v else: return True @classmethod def CheckDiskTemplates(cls, disk_templates): """Checks the disk templates for validity. """ if not disk_templates: raise errors.ConfigurationError("Instance policy must contain" + " at least one disk template") wrong = frozenset(disk_templates).difference(constants.DISK_TEMPLATES) if wrong: raise errors.ConfigurationError("Invalid disk template(s) %s" % utils.CommaJoin(wrong)) @classmethod def CheckParameter(cls, key, value): """Checks a parameter. Currently we expect all parameters to be float values. """ try: float(value) except (TypeError, ValueError) as err: raise errors.ConfigurationError("Invalid value for key" " '%s':" " '%s', error: %s" % (key, value, err)) def GetOSImage(osparams): """Gets the OS image value from the OS parameters. @type osparams: L{dict} or NoneType @param osparams: OS parameters or None @rtype: string or NoneType @return: value of OS image contained in OS parameters, or None if the OS parameters are None or the OS parameters do not contain an OS image """ if osparams is None: return None else: return osparams.get("os-image", None) def PutOSImage(osparams, os_image): """Update OS image value in the OS parameters @type osparams: L{dict} @param osparams: OS parameters @type os_image: string @param os_image: OS image @rtype: NoneType @return: None """ osparams["os-image"] = os_image class Instance(TaggableObject): """Config object representing an instance.""" __slots__ = [ "forthcoming", "name", "primary_node", "secondary_nodes", "os", "hypervisor", "hvparams", "beparams", "osparams", "osparams_private", "admin_state", "admin_state_source", "nics", "disks", "disks_info", "disk_template", "disks_active", "network_port", "serial_no", ] + _TIMESTAMPS + _UUID def FindDisk(self, idx): """Find a disk given having a specified index. This is just a wrapper that does validation of the index. @type idx: int @param idx: the disk index @rtype: string @return: the corresponding disk's uuid @raise errors.OpPrereqError: when the given index is not valid """ try: idx = int(idx) return self.disks[idx] except (TypeError, ValueError) as err: raise errors.OpPrereqError("Invalid disk index: '%s'" % str(err), errors.ECODE_INVAL) except IndexError: raise errors.OpPrereqError("Invalid disk index: %d (instace has disks" " 0 to %d" % (idx, len(self.disks) - 1), errors.ECODE_INVAL) def ToDict(self, _with_private=False): """Instance-specific conversion to standard python types. This replaces the children lists of objects with lists of standard python types. """ bo = super(Instance, self).ToDict(_with_private=_with_private) if _with_private: bo["osparams_private"] = self.osparams_private.Unprivate() for attr in ("nics",): alist = bo.get(attr, None) if alist: nlist = outils.ContainerToDicts(alist) else: nlist = [] bo[attr] = nlist if 'disk_template' in bo: del bo['disk_template'] return bo @classmethod def FromDict(cls, val): """Custom function for instances. """ if "admin_state" not in val: if val.get("admin_up", False): val["admin_state"] = constants.ADMINST_UP else: val["admin_state"] = constants.ADMINST_DOWN if "admin_up" in val: del val["admin_up"] obj = super(Instance, cls).FromDict(val) obj.nics = outils.ContainerFromDicts(obj.nics, list, NIC) # attribute 'disks_info' is only present when deserializing from a RPC # call in the backend disks_info = getattr(obj, "disks_info", None) if disks_info: obj.disks_info = outils.ContainerFromDicts(disks_info, list, Disk) return obj def UpgradeConfig(self): """Fill defaults for missing configuration values. """ if self.admin_state_source is None: self.admin_state_source = constants.ADMIN_SOURCE for nic in self.nics: nic.UpgradeConfig() if self.disks is None: self.disks = [] if self.hvparams: for key in constants.HVC_GLOBALS: try: del self.hvparams[key] except KeyError: pass if self.osparams is None: self.osparams = {} if self.osparams_private is None: self.osparams_private = serializer.PrivateDict() UpgradeBeParams(self.beparams) if self.disks_active is None: self.disks_active = self.admin_state == constants.ADMINST_UP class OS(ConfigObject): """Config object representing an operating system. @type supported_parameters: list @ivar supported_parameters: a list of tuples, name and description, containing the supported parameters by this OS @type VARIANT_DELIM: string @cvar VARIANT_DELIM: the variant delimiter """ __slots__ = [ "name", "path", "api_versions", "create_script", "create_script_untrusted", "export_script", "import_script", "rename_script", "verify_script", "supported_variants", "supported_parameters", ] VARIANT_DELIM = "+" @classmethod def SplitNameVariant(cls, name): """Splits the name into the proper name and variant. @param name: the OS (unprocessed) name @rtype: list @return: a list of two elements; if the original name didn't contain a variant, it's returned as an empty string """ nv = name.split(cls.VARIANT_DELIM, 1) if len(nv) == 1: nv.append("") return nv @classmethod def GetName(cls, name): """Returns the proper name of the os (without the variant). @param name: the OS (unprocessed) name """ return cls.SplitNameVariant(name)[0] @classmethod def GetVariant(cls, name): """Returns the variant the os (without the base name). @param name: the OS (unprocessed) name """ return cls.SplitNameVariant(name)[1] def IsTrusted(self): """Returns whether this OS is trusted. @rtype: bool @return: L{True} if this OS is trusted, L{False} otherwise """ return not self.create_script_untrusted class ExtStorage(ConfigObject): """Config object representing an External Storage Provider. """ __slots__ = [ "name", "path", "create_script", "remove_script", "grow_script", "attach_script", "detach_script", "setinfo_script", "verify_script", "snapshot_script", "open_script", "close_script", "supported_parameters", ] class NodeHvState(ConfigObject): """Hypvervisor state on a node. @ivar mem_total: Total amount of memory @ivar mem_node: Memory used by, or reserved for, the node itself (not always available) @ivar mem_hv: Memory used by hypervisor or lost due to instance allocation rounding @ivar mem_inst: Memory used by instances living on node @ivar cpu_total: Total node CPU core count @ivar cpu_node: Number of CPU cores reserved for the node itself """ __slots__ = [ "mem_total", "mem_node", "mem_hv", "mem_inst", "cpu_total", "cpu_node", ] + _TIMESTAMPS class NodeDiskState(ConfigObject): """Disk state on a node. """ __slots__ = [ "total", "reserved", "overhead", ] + _TIMESTAMPS class Node(TaggableObject): """Config object representing a node. @ivar hv_state: Hypervisor state (e.g. number of CPUs) @ivar hv_state_static: Hypervisor state overriden by user @ivar disk_state: Disk state (e.g. free space) @ivar disk_state_static: Disk state overriden by user """ __slots__ = [ "name", "primary_ip", "secondary_ip", "serial_no", "master_candidate", "offline", "drained", "group", "master_capable", "vm_capable", "ndparams", "powered", "hv_state", "hv_state_static", "disk_state", "disk_state_static", ] + _TIMESTAMPS + _UUID def UpgradeConfig(self): """Fill defaults for missing configuration values. """ # pylint: disable=E0203 # because these are "defined" via slots, not manually if self.master_capable is None: self.master_capable = True if self.vm_capable is None: self.vm_capable = True if self.ndparams is None: self.ndparams = {} # And remove any global parameter for key in constants.NDC_GLOBALS: if key in self.ndparams: logging.warning("Ignoring %s node parameter for node %s", key, self.name) del self.ndparams[key] if self.powered is None: self.powered = True def ToDict(self, _with_private=False): """Custom function for serializing. """ data = super(Node, self).ToDict(_with_private=_with_private) hv_state = data.get("hv_state", None) if hv_state is not None: data["hv_state"] = outils.ContainerToDicts(hv_state) disk_state = data.get("disk_state", None) if disk_state is not None: data["disk_state"] = \ dict((key, outils.ContainerToDicts(value)) for (key, value) in disk_state.items()) return data @classmethod def FromDict(cls, val): """Custom function for deserializing. """ obj = super(Node, cls).FromDict(val) if obj.hv_state is not None: obj.hv_state = \ outils.ContainerFromDicts(obj.hv_state, dict, NodeHvState) if obj.disk_state is not None: obj.disk_state = \ dict((key, outils.ContainerFromDicts(value, dict, NodeDiskState)) for (key, value) in obj.disk_state.items()) return obj class NodeGroup(TaggableObject): """Config object representing a node group.""" __slots__ = [ "name", "members", "ndparams", "diskparams", "ipolicy", "serial_no", "hv_state_static", "disk_state_static", "alloc_policy", "networks", ] + _TIMESTAMPS + _UUID def ToDict(self, _with_private=False): """Custom function for nodegroup. This discards the members object, which gets recalculated and is only kept in memory. """ mydict = super(NodeGroup, self).ToDict(_with_private=_with_private) del mydict["members"] return mydict @classmethod def FromDict(cls, val): """Custom function for nodegroup. The members slot is initialized to an empty list, upon deserialization. """ obj = super(NodeGroup, cls).FromDict(val) obj.members = [] return obj def UpgradeConfig(self): """Fill defaults for missing configuration values. """ if self.ndparams is None: self.ndparams = {} if self.serial_no is None: self.serial_no = 1 if self.alloc_policy is None: self.alloc_policy = constants.ALLOC_POLICY_PREFERRED # We only update mtime, and not ctime, since we would not be able # to provide a correct value for creation time. if self.mtime is None: self.mtime = time.time() if self.diskparams is None: self.diskparams = {} if self.ipolicy is None: self.ipolicy = MakeEmptyIPolicy() if self.networks is None: self.networks = {} for network, netparams in self.networks.items(): self.networks[network] = FillDict(constants.NICC_DEFAULTS, netparams) def FillND(self, node): """Return filled out ndparams for L{objects.Node} @type node: L{objects.Node} @param node: A Node object to fill @return a copy of the node's ndparams with defaults filled """ return self.SimpleFillND(node.ndparams) def SimpleFillND(self, ndparams): """Fill a given ndparams dict with defaults. @type ndparams: dict @param ndparams: the dict to fill @rtype: dict @return: a copy of the passed in ndparams with missing keys filled from the node group defaults """ return FillDict(self.ndparams, ndparams) class Cluster(TaggableObject): """Config object representing the cluster.""" __slots__ = [ "serial_no", "rsahostkeypub", "dsahostkeypub", "highest_used_port", "tcpudp_port_pool", "mac_prefix", "volume_group_name", "reserved_lvs", "drbd_usermode_helper", "default_bridge", "default_hypervisor", "master_node", "master_ip", "master_netdev", "master_netmask", "use_external_mip_script", "cluster_name", "file_storage_dir", "shared_file_storage_dir", "gluster_storage_dir", "enabled_hypervisors", "hvparams", "ipolicy", "os_hvp", "beparams", "osparams", "osparams_private_cluster", "nicparams", "ndparams", "diskparams", "candidate_pool_size", "modify_etc_hosts", "modify_ssh_setup", "maintain_node_health", "uid_pool", "default_iallocator", "default_iallocator_params", "hidden_os", "blacklisted_os", "primary_ip_family", "prealloc_wipe_disks", "hv_state_static", "disk_state_static", "enabled_disk_templates", "candidate_certs", "max_running_jobs", "max_tracked_jobs", "install_image", "instance_communication_network", "zeroing_image", "compression_tools", "enabled_user_shutdown", "data_collectors", "ssh_key_type", "ssh_key_bits", ] + _TIMESTAMPS + _UUID def UpgradeConfig(self): """Fill defaults for missing configuration values. """ # pylint: disable=E0203 # because these are "defined" via slots, not manually if self.hvparams is None: self.hvparams = constants.HVC_DEFAULTS else: for hypervisor in constants.HYPER_TYPES: try: existing_params = self.hvparams[hypervisor] except KeyError: existing_params = {} self.hvparams[hypervisor] = FillDict( constants.HVC_DEFAULTS[hypervisor], existing_params) if self.os_hvp is None: self.os_hvp = {} if self.osparams is None: self.osparams = {} # osparams_private_cluster added in 2.12 if self.osparams_private_cluster is None: self.osparams_private_cluster = {} self.ndparams = UpgradeNDParams(self.ndparams) self.beparams = UpgradeGroupedParams(self.beparams, constants.BEC_DEFAULTS) for beparams_group in self.beparams: UpgradeBeParams(self.beparams[beparams_group]) migrate_default_bridge = not self.nicparams self.nicparams = UpgradeGroupedParams(self.nicparams, constants.NICC_DEFAULTS) if migrate_default_bridge: self.nicparams[constants.PP_DEFAULT][constants.NIC_LINK] = \ self.default_bridge if self.modify_etc_hosts is None: self.modify_etc_hosts = True if self.modify_ssh_setup is None: self.modify_ssh_setup = True # default_bridge is no longer used in 2.1. The slot is left there to # support auto-upgrading. It can be removed once we decide to deprecate # upgrading straight from 2.0. if self.default_bridge is not None: self.default_bridge = None # default_hypervisor is just the first enabled one in 2.1. This slot and # code can be removed once upgrading straight from 2.0 is deprecated. if self.default_hypervisor is not None: self.enabled_hypervisors = ([self.default_hypervisor] + [hvname for hvname in self.enabled_hypervisors if hvname != self.default_hypervisor]) self.default_hypervisor = None # maintain_node_health added after 2.1.1 if self.maintain_node_health is None: self.maintain_node_health = False if self.uid_pool is None: self.uid_pool = [] if self.default_iallocator is None: self.default_iallocator = "" if self.default_iallocator_params is None: self.default_iallocator_params = {} # reserved_lvs added before 2.2 if self.reserved_lvs is None: self.reserved_lvs = [] # hidden and blacklisted operating systems added before 2.2.1 if self.hidden_os is None: self.hidden_os = [] if self.blacklisted_os is None: self.blacklisted_os = [] # primary_ip_family added before 2.3 if self.primary_ip_family is None: self.primary_ip_family = AF_INET if self.master_netmask is None: ipcls = netutils.IPAddress.GetClassFromIpFamily(self.primary_ip_family) self.master_netmask = ipcls.iplen if self.prealloc_wipe_disks is None: self.prealloc_wipe_disks = False # shared_file_storage_dir added before 2.5 if self.shared_file_storage_dir is None: self.shared_file_storage_dir = "" # gluster_storage_dir added in 2.11 if self.gluster_storage_dir is None: self.gluster_storage_dir = "" if self.use_external_mip_script is None: self.use_external_mip_script = False if self.diskparams: self.diskparams = UpgradeDiskParams(self.diskparams) else: self.diskparams = constants.DISK_DT_DEFAULTS.copy() # instance policy added before 2.6 if self.ipolicy is None: self.ipolicy = FillIPolicy(constants.IPOLICY_DEFAULTS, {}) else: # we can either make sure to upgrade the ipolicy always, or only # do it in some corner cases (e.g. missing keys); note that this # will break any removal of keys from the ipolicy dict wrongkeys = frozenset(self.ipolicy) - constants.IPOLICY_ALL_KEYS if wrongkeys: # These keys would be silently removed by FillIPolicy() msg = ("Cluster instance policy contains spurious keys: %s" % utils.CommaJoin(wrongkeys)) raise errors.ConfigurationError(msg) self.ipolicy = FillIPolicy(constants.IPOLICY_DEFAULTS, self.ipolicy) # hv_state_static added in 2.7 if self.hv_state_static is None: self.hv_state_static = {} if self.disk_state_static is None: self.disk_state_static = {} if self.candidate_certs is None: self.candidate_certs = {} if self.max_running_jobs is None: self.max_running_jobs = constants.LUXID_MAXIMAL_RUNNING_JOBS_DEFAULT if self.max_tracked_jobs is None: self.max_tracked_jobs = constants.LUXID_MAXIMAL_TRACKED_JOBS_DEFAULT if self.instance_communication_network is None: self.instance_communication_network = "" if self.install_image is None: self.install_image = "" if self.compression_tools is None: self.compression_tools = constants.IEC_DEFAULT_TOOLS if self.enabled_user_shutdown is None: self.enabled_user_shutdown = False if self.ssh_key_type is None: self.ssh_key_type = constants.SSH_DEFAULT_KEY_TYPE if self.ssh_key_bits is None: self.ssh_key_bits = constants.SSH_DEFAULT_KEY_BITS @property def primary_hypervisor(self): """The first hypervisor is the primary. Useful, for example, for L{Node}'s hv/disk state. """ return self.enabled_hypervisors[0] def ToDict(self, _with_private=False): """Custom function for cluster. """ mydict = super(Cluster, self).ToDict(_with_private=_with_private) # Explicitly save private parameters. if _with_private: for os in mydict["osparams_private_cluster"]: mydict["osparams_private_cluster"][os] = \ self.osparams_private_cluster[os].Unprivate() if self.tcpudp_port_pool is None: tcpudp_port_pool = [] else: tcpudp_port_pool = list(self.tcpudp_port_pool) mydict["tcpudp_port_pool"] = tcpudp_port_pool return mydict @classmethod def FromDict(cls, val): """Custom function for cluster. """ obj = super(Cluster, cls).FromDict(val) if obj.tcpudp_port_pool is None: obj.tcpudp_port_pool = set() elif not isinstance(obj.tcpudp_port_pool, set): obj.tcpudp_port_pool = set(obj.tcpudp_port_pool) return obj def SimpleFillDP(self, diskparams): """Fill a given diskparams dict with cluster defaults. @param diskparams: The diskparams @return: The defaults dict """ return FillDiskParams(self.diskparams, diskparams) def GetHVDefaults(self, hypervisor, os_name=None, skip_keys=None): """Get the default hypervisor parameters for the cluster. @param hypervisor: the hypervisor name @param os_name: if specified, we'll also update the defaults for this OS @param skip_keys: if passed, list of keys not to use @return: the defaults dict """ if skip_keys is None: skip_keys = [] fill_stack = [self.hvparams.get(hypervisor, {})] if os_name is not None: os_hvp = self.os_hvp.get(os_name, {}).get(hypervisor, {}) fill_stack.append(os_hvp) ret_dict = {} for o_dict in fill_stack: ret_dict = FillDict(ret_dict, o_dict, skip_keys=skip_keys) return ret_dict def SimpleFillHV(self, hv_name, os_name, hvparams, skip_globals=False): """Fill a given hvparams dict with cluster defaults. @type hv_name: string @param hv_name: the hypervisor to use @type os_name: string @param os_name: the OS to use for overriding the hypervisor defaults @type skip_globals: boolean @param skip_globals: if True, the global hypervisor parameters will not be filled @rtype: dict @return: a copy of the given hvparams with missing keys filled from the cluster defaults """ if skip_globals: skip_keys = constants.HVC_GLOBALS else: skip_keys = [] def_dict = self.GetHVDefaults(hv_name, os_name, skip_keys=skip_keys) return FillDict(def_dict, hvparams, skip_keys=skip_keys) def FillHV(self, instance, skip_globals=False): """Fill an instance's hvparams dict with cluster defaults. @type instance: L{objects.Instance} @param instance: the instance parameter to fill @type skip_globals: boolean @param skip_globals: if True, the global hypervisor parameters will not be filled @rtype: dict @return: a copy of the instance's hvparams with missing keys filled from the cluster defaults """ return self.SimpleFillHV(instance.hypervisor, instance.os, instance.hvparams, skip_globals) def SimpleFillBE(self, beparams): """Fill a given beparams dict with cluster defaults. @type beparams: dict @param beparams: the dict to fill @rtype: dict @return: a copy of the passed in beparams with missing keys filled from the cluster defaults """ return FillDict(self.beparams.get(constants.PP_DEFAULT, {}), beparams) def FillBE(self, instance): """Fill an instance's beparams dict with cluster defaults. @type instance: L{objects.Instance} @param instance: the instance parameter to fill @rtype: dict @return: a copy of the instance's beparams with missing keys filled from the cluster defaults """ return self.SimpleFillBE(instance.beparams) def SimpleFillNIC(self, nicparams): """Fill a given nicparams dict with cluster defaults. @type nicparams: dict @param nicparams: the dict to fill @rtype: dict @return: a copy of the passed in nicparams with missing keys filled from the cluster defaults """ return FillDict(self.nicparams.get(constants.PP_DEFAULT, {}), nicparams) def SimpleFillOS(self, os_name, os_params_public, os_params_private=None, os_params_secret=None): """Fill an instance's osparams dict with cluster defaults. @type os_name: string @param os_name: the OS name to use @type os_params_public: dict @param os_params_public: the dict to fill with default values @type os_params_private: dict @param os_params_private: the dict with private fields to fill with default values. Not passing this field results in no private fields being added to the return value. Private fields will be wrapped in L{Private} objects. @type os_params_secret: dict @param os_params_secret: the dict with secret fields to fill with default values. Not passing this field results in no secret fields being added to the return value. Private fields will be wrapped in L{Private} objects. @rtype: dict @return: a copy of the instance's osparams with missing keys filled from the cluster defaults. Private and secret parameters are not included unless the respective optional parameters are supplied. """ if os_name is None: name_only = None else: name_only = OS.GetName(os_name) defaults_base_public = self.osparams.get(name_only, {}) defaults_public = FillDict(defaults_base_public, self.osparams.get(os_name, {})) params_public = FillDict(defaults_public, os_params_public) if os_params_private is not None: defaults_base_private = self.osparams_private_cluster.get(name_only, {}) defaults_private = FillDict(defaults_base_private, self.osparams_private_cluster.get(os_name, {})) params_private = FillDict(defaults_private, os_params_private) else: params_private = {} if os_params_secret is not None: # There can't be default secret settings, so there's nothing to be done. params_secret = os_params_secret else: params_secret = {} # Enforce that the set of keys be distinct: duplicate_keys = utils.GetRepeatedKeys(params_public, params_private, params_secret) if not duplicate_keys: # Actually update them: params_public.update(params_private) params_public.update(params_secret) return params_public else: def formatter(keys): return utils.CommaJoin(sorted(map(repr, keys))) if keys else "(none)" #Lose the values. params_public = set(params_public) params_private = set(params_private) params_secret = set(params_secret) msg = """Cannot assign multiple values to OS parameters. Conflicting OS parameters that would have been set by this operation: - at public visibility: {public} - at private visibility: {private} - at secret visibility: {secret} """.format(public=formatter(params_public & duplicate_keys), private=formatter(params_private & duplicate_keys), secret=formatter(params_secret & duplicate_keys)) raise errors.OpPrereqError(msg) @staticmethod def SimpleFillHvState(hv_state): """Fill an hv_state sub dict with cluster defaults. """ return FillDict(constants.HVST_DEFAULTS, hv_state) @staticmethod def SimpleFillDiskState(disk_state): """Fill an disk_state sub dict with cluster defaults. """ return FillDict(constants.DS_DEFAULTS, disk_state) def FillND(self, node, nodegroup): """Return filled out ndparams for L{objects.NodeGroup} and L{objects.Node} @type node: L{objects.Node} @param node: A Node object to fill @type nodegroup: L{objects.NodeGroup} @param nodegroup: A Node object to fill @return a copy of the node's ndparams with defaults filled """ return self.SimpleFillND(nodegroup.FillND(node)) def FillNDGroup(self, nodegroup): """Return filled out ndparams for just L{objects.NodeGroup} @type nodegroup: L{objects.NodeGroup} @param nodegroup: A Node object to fill @return a copy of the node group's ndparams with defaults filled """ return self.SimpleFillND(nodegroup.SimpleFillND({})) def SimpleFillND(self, ndparams): """Fill a given ndparams dict with defaults. @type ndparams: dict @param ndparams: the dict to fill @rtype: dict @return: a copy of the passed in ndparams with missing keys filled from the cluster defaults """ return FillDict(self.ndparams, ndparams) def SimpleFillIPolicy(self, ipolicy): """ Fill instance policy dict with defaults. @type ipolicy: dict @param ipolicy: the dict to fill @rtype: dict @return: a copy of passed ipolicy with missing keys filled from the cluster defaults """ return FillIPolicy(self.ipolicy, ipolicy) def IsDiskTemplateEnabled(self, disk_template): """Checks if a particular disk template is enabled. """ return utils.storage.IsDiskTemplateEnabled( disk_template, self.enabled_disk_templates) def IsFileStorageEnabled(self): """Checks if file storage is enabled. """ return utils.storage.IsFileStorageEnabled(self.enabled_disk_templates) def IsSharedFileStorageEnabled(self): """Checks if shared file storage is enabled. """ return utils.storage.IsSharedFileStorageEnabled( self.enabled_disk_templates) class BlockDevStatus(ConfigObject): """Config object representing the status of a block device.""" __slots__ = [ "dev_path", "major", "minor", "sync_percent", "estimated_time", "is_degraded", "ldisk_status", ] class ImportExportStatus(ConfigObject): """Config object representing the status of an import or export.""" __slots__ = [ "recent_output", "listen_port", "connected", "progress_mbytes", "progress_throughput", "progress_eta", "progress_percent", "exit_status", "error_message", ] + _TIMESTAMPS class ImportExportOptions(ConfigObject): """Options for import/export daemon @ivar key_name: X509 key name (None for cluster certificate) @ivar ca_pem: Remote peer CA in PEM format (None for cluster certificate) @ivar compress: Compression tool to use @ivar magic: Used to ensure the connection goes to the right disk @ivar ipv6: Whether to use IPv6 @ivar connect_timeout: Number of seconds for establishing connection """ __slots__ = [ "key_name", "ca_pem", "compress", "magic", "ipv6", "connect_timeout", ] class ConfdRequest(ConfigObject): """Object holding a confd request. @ivar protocol: confd protocol version @ivar type: confd query type @ivar query: query request @ivar rsalt: requested reply salt """ __slots__ = [ "protocol", "type", "query", "rsalt", ] class ConfdReply(ConfigObject): """Object holding a confd reply. @ivar protocol: confd protocol version @ivar status: reply status code (ok, error) @ivar answer: confd query reply @ivar serial: configuration serial number """ __slots__ = [ "protocol", "status", "answer", "serial", ] class QueryFieldDefinition(ConfigObject): """Object holding a query field definition. @ivar name: Field name @ivar title: Human-readable title @ivar kind: Field type @ivar doc: Human-readable description """ __slots__ = [ "name", "title", "kind", "doc", ] class _QueryResponseBase(ConfigObject): __slots__ = [ "fields", ] def ToDict(self, _with_private=False): """Custom function for serializing. """ mydict = super(_QueryResponseBase, self).ToDict() mydict["fields"] = outils.ContainerToDicts(mydict["fields"]) return mydict @classmethod def FromDict(cls, val): """Custom function for de-serializing. """ obj = super(_QueryResponseBase, cls).FromDict(val) obj.fields = \ outils.ContainerFromDicts(obj.fields, list, QueryFieldDefinition) return obj class QueryResponse(_QueryResponseBase): """Object holding the response to a query. @ivar fields: List of L{QueryFieldDefinition} objects @ivar data: Requested data """ __slots__ = [ "data", ] class QueryFieldsRequest(ConfigObject): """Object holding a request for querying available fields. """ __slots__ = [ "what", "fields", ] class QueryFieldsResponse(_QueryResponseBase): """Object holding the response to a query for fields. @ivar fields: List of L{QueryFieldDefinition} objects """ __slots__ = [] class MigrationStatus(ConfigObject): """Object holding the status of a migration. """ __slots__ = [ "status", "transferred_ram", "total_ram", # to signal, if migration has switched to postcopy "postcopy_status", ] class InstanceConsole(ConfigObject): """Object describing how to access the console of an instance. """ __slots__ = [ "instance", "kind", "message", "host", "port", "user", "command", "display", ] def Validate(self): """Validates contents of this object. """ assert self.kind in constants.CONS_ALL, "Unknown console type" assert self.instance, "Missing instance name" assert self.message or self.kind in [constants.CONS_SSH, constants.CONS_SPICE, constants.CONS_VNC] assert self.host or self.kind == constants.CONS_MESSAGE assert self.port or self.kind in [constants.CONS_MESSAGE, constants.CONS_SSH] assert self.user or self.kind in [constants.CONS_MESSAGE, constants.CONS_SPICE, constants.CONS_VNC] assert self.command or self.kind in [constants.CONS_MESSAGE, constants.CONS_SPICE, constants.CONS_VNC] assert self.display or self.kind in [constants.CONS_MESSAGE, constants.CONS_SPICE, constants.CONS_SSH] class Network(TaggableObject): """Object representing a network definition for ganeti. """ __slots__ = [ "name", "serial_no", "mac_prefix", "network", "network6", "gateway", "gateway6", "reservations", "ext_reservations", ] + _TIMESTAMPS + _UUID def HooksDict(self, prefix=""): """Export a dictionary used by hooks with a network's information. @type prefix: String @param prefix: Prefix to prepend to the dict entries """ result = { "%sNETWORK_NAME" % prefix: self.name, "%sNETWORK_UUID" % prefix: self.uuid, "%sNETWORK_TAGS" % prefix: " ".join(self.GetTags()), } if self.network: result["%sNETWORK_SUBNET" % prefix] = self.network if self.gateway: result["%sNETWORK_GATEWAY" % prefix] = self.gateway if self.network6: result["%sNETWORK_SUBNET6" % prefix] = self.network6 if self.gateway6: result["%sNETWORK_GATEWAY6" % prefix] = self.gateway6 if self.mac_prefix: result["%sNETWORK_MAC_PREFIX" % prefix] = self.mac_prefix return result @classmethod def FromDict(cls, val): """Custom function for networks. Remove deprecated network_type and family. """ if "network_type" in val: del val["network_type"] if "family" in val: del val["family"] obj = super(Network, cls).FromDict(val) return obj # need to inherit object in order to use super() class SerializableConfigParser(configparser.ConfigParser, object): """Simple wrapper over ConfigParse that allows serialization. This class is basically configparser.ConfigParser with two additional methods that allow it to serialize/unserialize to/from a buffer. """ def Dumps(self): """Dump this instance and return the string representation.""" buf = StringIO() self.write(buf) return buf.getvalue() @classmethod def Loads(cls, data): """Load data from a string.""" buf = StringIO(data) cfp = cls() cfp.read_file(buf) return cfp def get(self, section, option, **kwargs): value = None try: value = super(SerializableConfigParser, self).get(section, option, **kwargs) if value.lower() == constants.VALUE_NONE: value = None except configparser.NoOptionError: r = re.compile(r"(disk|nic)\d+_name|nic\d+_(network|vlan)") match = r.match(option) if match: pass else: raise return value class LvmPvInfo(ConfigObject): """Information about an LVM physical volume (PV). @type name: string @ivar name: name of the PV @type vg_name: string @ivar vg_name: name of the volume group containing the PV @type size: float @ivar size: size of the PV in MiB @type free: float @ivar free: free space in the PV, in MiB @type attributes: string @ivar attributes: PV attributes @type lv_list: list of strings @ivar lv_list: names of the LVs hosted on the PV """ __slots__ = [ "name", "vg_name", "size", "free", "attributes", "lv_list" ] def IsEmpty(self): """Is this PV empty? """ return self.size <= (self.free + 1) def IsAllocatable(self): """Is this PV allocatable? """ return ("a" in self.attributes) ganeti-3.1.0~rc2/lib/opcodes.py.in_after000064400000000000000000000006071476477700300201510ustar00rootroot00000000000000 def _GetOpList(): """Returns list of all defined opcodes. Does not eliminate duplicates by C{OP_ID}. """ return [v for v in globals().values() if (isinstance(v, type) and issubclass(v, OpCode) and hasattr(v, "OP_ID") and v is not OpCode and v.OP_ID != 'OP_INSTANCE_MULTI_ALLOC_BASE')] OP_MAPPING = dict((v.OP_ID, v) for v in _GetOpList()) ganeti-3.1.0~rc2/lib/opcodes.py.in_before000064400000000000000000000177521476477700300203230ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """OpCodes module Note that this file is autogenerated using @src/hs2py@ with a header from @lib/opcodes.py.in_before@ and a footer from @lib/opcodes.py.in_after@. This module implements part of the data structures which define the cluster operations - the so-called opcodes. Every operation which modifies the cluster state is expressed via opcodes. """ # this are practically structures, so disable the message about too # few public methods: # pylint: disable=R0903 # pylint: disable=C0301 from ganeti import constants from ganeti import ht from ganeti import opcodes_base def FormatDuration(value): """Custom formatter for duration. """ try: return "%d ns" % round(value * 1e9) except TypeError: return str(value) class OpCode(opcodes_base.BaseOpCode): """Abstract OpCode. This is the root of the actual OpCode hierarchy. All clases derived from this class should override OP_ID. @cvar OP_ID: The ID of this opcode. This should be unique amongst all children of this class. @cvar OP_DSC_FIELD: The name of a field whose value will be included in the string returned by Summary(); see the docstring of that method for details). @cvar OP_DSC_FORMATTER: A callable that should format the OP_DSC_FIELD; if not present, then the field will be simply converted to string @cvar OP_PARAMS: List of opcode attributes, the default values they should get if not already defined, and types they must match. @cvar OP_RESULT: Callable to verify opcode result @cvar WITH_LU: Boolean that specifies whether this should be included in mcpu's dispatch table @ivar dry_run: Whether the LU should be run in dry-run mode, i.e. just the check steps @ivar priority: Opcode priority for queue """ # pylint: disable=E1101 # as OP_ID is dynamically defined WITH_LU = True OP_PARAMS = [ ("dry_run", None, ht.TMaybe(ht.TBool), "Run checks only, don't execute"), ("debug_level", None, ht.TMaybe(ht.TNonNegative(ht.TInt)), "Debug level"), ("priority", constants.OP_PRIO_DEFAULT, ht.TElemOf(constants.OP_PRIO_SUBMIT_VALID), "Opcode priority"), (opcodes_base.DEPEND_ATTR, None, opcodes_base.BuildJobDepCheck(True), "Job dependencies; if used through ``SubmitManyJobs`` relative (negative)" " job IDs can be used; see :doc:`design document `" " for details"), (opcodes_base.COMMENT_ATTR, None, ht.TMaybe(ht.TString), "Comment describing the purpose of the opcode"), (constants.OPCODE_REASON, [], ht.TMaybe(ht.TListOf(ht.TAny)), "The reason trail, describing why the OpCode is executed"), ] OP_RESULT = None def __getstate__(self): """Specialized getstate for opcodes. This method adds to the state dictionary the OP_ID of the class, so that on unload we can identify the correct class for instantiating the opcode. @rtype: C{dict} @return: the state as a dictionary """ data = opcodes_base.BaseOpCode.__getstate__(self) data["OP_ID"] = self.OP_ID return data @classmethod def LoadOpCode(cls, data): """Generic load opcode method. The method identifies the correct opcode class from the dict-form by looking for a OP_ID key, if this is not found, or its value is not available in this module as a child of this class, we fail. @type data: C{dict} @param data: the serialized opcode """ if not isinstance(data, dict): raise ValueError("Invalid data to LoadOpCode (%s)" % type(data)) if "OP_ID" not in data: raise ValueError("Invalid data to LoadOpcode, missing OP_ID") op_id = data["OP_ID"] op_class = None if op_id in OP_MAPPING: op_class = OP_MAPPING[op_id] else: raise ValueError("Invalid data to LoadOpCode: OP_ID %s unsupported" % op_id) op = op_class() new_data = data.copy() del new_data["OP_ID"] op.__setstate__(new_data) return op def Summary(self): """Generates a summary description of this opcode. The summary is the value of the OP_ID attribute (without the "OP_" prefix), plus the value of the OP_DSC_FIELD attribute, if one was defined; this field should allow to easily identify the operation (for an instance creation job, e.g., it would be the instance name). """ assert self.OP_ID is not None and len(self.OP_ID) > 3 # all OP_ID start with OP_, we remove that txt = self.OP_ID[3:] field_name = getattr(self, "OP_DSC_FIELD", None) if field_name: field_value = getattr(self, field_name, None) field_formatter = getattr(self, "OP_DSC_FORMATTER", None) if callable(field_formatter): field_value = field_formatter(field_value) # pylint: disable=E1102 elif isinstance(field_value, (list, tuple)): field_value = ",".join(str(i) for i in field_value) txt = "%s(%s)" % (txt, field_value) return txt def TinySummary(self): """Generates a compact summary description of the opcode. """ assert self.OP_ID.startswith("OP_") text = self.OP_ID[3:] for (prefix, supplement) in opcodes_base.SUMMARY_PREFIX.items(): if text.startswith(prefix): return supplement + text[len(prefix):] return text class OpInstanceMultiAllocBase(OpCode): """Allocates multiple instances. """ def __getstate__(self): """Generic serializer. """ state = OpCode.__getstate__(self) if hasattr(self, "instances"): # pylint: disable=E1101 state["instances"] = [inst.__getstate__() for inst in self.instances] return state def __setstate__(self, state): """Generic unserializer. This method just restores from the serialized state the attributes of the current instance. @param state: the serialized opcode data @type state: C{dict} """ if not isinstance(state, dict): raise ValueError("Invalid data to __setstate__: expected dict, got %s" % type(state)) if "instances" in state: state["instances"] = [OpCode.LoadOpCode(x) for x in state["instances"]] return OpCode.__setstate__(self, state) def Validate(self, set_defaults): """Validates this opcode. We do this recursively. @type set_defaults: bool @param set_defaults: whether to set default values @rtype: NoneType @return: L{None}, if the validation succeeds @raise errors.OpPrereqError: when a parameter value doesn't match requirements """ OpCode.Validate(self, set_defaults) for inst in self.instances: # pylint: disable=E1101 inst.Validate(set_defaults) ganeti-3.1.0~rc2/lib/opcodes_base.py000064400000000000000000000206451476477700300173610ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """OpCodes base module This module implements part of the data structures which define the cluster operations - the so-called opcodes. Every operation which modifies the cluster state is expressed via opcodes. """ # this are practically structures, so disable the message about too # few public methods: # pylint: disable=R0903 import copy import logging import re from ganeti import constants from ganeti import errors from ganeti import ht from ganeti import outils #: OP_ID conversion regular expression _OPID_RE = re.compile("([a-z])([A-Z])") SUMMARY_PREFIX = { "CLUSTER_": "C_", "GROUP_": "G_", "NODE_": "N_", "INSTANCE_": "I_", } #: Attribute name for dependencies DEPEND_ATTR = "depends" #: Attribute name for comment COMMENT_ATTR = "comment" def _NameComponents(name): """Split an opcode class name into its components @type name: string @param name: the class name, as OpXxxYyy @rtype: array of strings @return: the components of the name """ assert name.startswith("Op") # Note: (?<=[a-z])(?=[A-Z]) would be ideal, since it wouldn't # consume any input, and hence we would just have all the elements # in the list, one by one; but it seems that split doesn't work on # non-consuming input, hence we have to process the input string a # bit name = _OPID_RE.sub(r"\1,\2", name) elems = name.split(",") return elems def _NameToId(name): """Convert an opcode class name to an OP_ID. @type name: string @param name: the class name, as OpXxxYyy @rtype: string @return: the name in the OP_XXXX_YYYY format """ if not name.startswith("Op"): return None return "_".join(n.upper() for n in _NameComponents(name)) def NameToReasonSrc(name, prefix): """Convert an opcode class name to a source string for the reason trail @type name: string @param name: the class name, as OpXxxYyy @type prefix: string @param prefix: the prefix that will be prepended to the opcode name @rtype: string @return: the name in the OP_XXXX_YYYY format """ if not name.startswith("Op"): return None return "%s:%s" % (prefix, "_".join(n.lower() for n in _NameComponents(name))) class _AutoOpParamSlots(outils.AutoSlots): """Meta class for opcode definitions. """ def __new__(mcs, name, bases, attrs): """Called when a class should be created. @param mcs: The meta class @param name: Name of created class @param bases: Base classes @type attrs: dict @param attrs: Class attributes """ assert "OP_ID" not in attrs, "Class '%s' defining OP_ID" % name slots = mcs._GetSlots(attrs) assert "OP_DSC_FIELD" not in attrs or attrs["OP_DSC_FIELD"] in slots, \ "Class '%s' uses unknown field in OP_DSC_FIELD" % name assert ("OP_DSC_FORMATTER" not in attrs or callable(attrs["OP_DSC_FORMATTER"])), \ ("Class '%s' uses non-callable in OP_DSC_FORMATTER (%s)" % (name, type(attrs["OP_DSC_FORMATTER"]))) attrs["OP_ID"] = _NameToId(name) # pylint: disable=E1121 return outils.AutoSlots.__new__(mcs, name, bases, attrs) @classmethod def _GetSlots(mcs, attrs): """Build the slots out of OP_PARAMS. """ # Always set OP_PARAMS to avoid duplicates in BaseOpCode.GetAllParams params = attrs.setdefault("OP_PARAMS", []) # Use parameter names as slots return [pname for (pname, _, _, _) in params] class BaseOpCode(outils.ValidatedSlots, metaclass=_AutoOpParamSlots): """A simple serializable object. This object serves as a parent class for OpCode without any custom field handling. """ def __init__(self, **kwargs): outils.ValidatedSlots.__init__(self, **kwargs) for key, default, _, _ in self.__class__.GetAllParams(): if not hasattr(self, key): setattr(self, key, default) def __getstate__(self): """Generic serializer. This method just returns the contents of the instance as a dictionary. @rtype: C{dict} @return: the instance attributes and their values """ state = {} for name in self.GetAllSlots(): if hasattr(self, name): state[name] = getattr(self, name) return state def __setstate__(self, state): """Generic unserializer. This method just restores from the serialized state the attributes of the current instance. @param state: the serialized opcode data @type state: C{dict} """ if not isinstance(state, dict): raise ValueError("Invalid data to __setstate__: expected dict, got %s" % type(state)) for name in self.GetAllSlots(): if name not in state and hasattr(self, name): delattr(self, name) for name in state: setattr(self, name, state[name]) @classmethod def GetAllParams(cls): """Compute list of all parameters for an opcode. """ slots = [] for parent in cls.__mro__: slots.extend(getattr(parent, "OP_PARAMS", [])) return slots def Validate(self, set_defaults): # pylint: disable=W0221 """Validate opcode parameters, optionally setting default values. @type set_defaults: bool @param set_defaults: whether to set default values @rtype: NoneType @return: L{None}, if the validation succeeds @raise errors.OpPrereqError: when a parameter value doesn't match requirements """ for (attr_name, default, test, _) in self.GetAllParams(): assert callable(test) if hasattr(self, attr_name): attr_val = getattr(self, attr_name) else: attr_val = copy.deepcopy(default) if test(attr_val): if set_defaults: setattr(self, attr_name, attr_val) elif ht.TInt(attr_val) and test(float(attr_val)): if set_defaults: setattr(self, attr_name, float(attr_val)) else: op_id = self.OP_ID # pylint: disable=E1101 logging.error("OpCode %s, parameter %s, has invalid type %s/value" " '%s' expecting type %s", op_id, attr_name, type(attr_val), attr_val, test) if attr_val is None: logging.error("OpCode %s, parameter %s, has default value None which" " is does not check against the parameter's type: this" " means this parameter is required but no value was" " given", op_id, attr_name) raise errors.OpPrereqError("Parameter '%s.%s' fails validation" % (op_id, attr_name), errors.ECODE_INVAL) def BuildJobDepCheck(relative): """Builds check for job dependencies (L{DEPEND_ATTR}). @type relative: bool @param relative: Whether to accept relative job IDs (negative) @rtype: callable """ if relative: job_id = ht.TOr(ht.TJobId, ht.TRelativeJobId) else: job_id = ht.TJobId job_dep = \ ht.TAnd(ht.TOr(ht.TListOf(ht.TAny), ht.TTuple), ht.TIsLength(2), ht.TItems([job_id, ht.TListOf(ht.TElemOf(constants.JOBS_FINALIZED))])) return ht.TMaybe(ht.TListOf(job_dep)) TNoRelativeJobDependencies = BuildJobDepCheck(False) ganeti-3.1.0~rc2/lib/outils.py000064400000000000000000000121551476477700300162470ustar00rootroot00000000000000# # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module for object related utils.""" #: Supported container types for serialization/de-serialization (must be a #: tuple as it's used as a parameter for C{isinstance}) _SEQUENCE_TYPES = (list, tuple, set, frozenset) class AutoSlots(type): """Meta base class for __slots__ definitions. """ def __new__(mcs, name, bases, attrs): """Called when a class should be created. @param mcs: The meta class @param name: Name of created class @param bases: Base classes @type attrs: dict @param attrs: Class attributes """ assert "__slots__" not in attrs, \ "Class '%s' defines __slots__ when it should not" % name attrs["__slots__"] = mcs._GetSlots(attrs) return type.__new__(mcs, name, bases, attrs) @classmethod def _GetSlots(mcs, attrs): """Used to get the list of defined slots. @param attrs: The attributes of the class """ raise NotImplementedError class ValidatedSlots(object): """Sets and validates slots. """ __slots__ = [] def __init__(self, **kwargs): """Constructor for BaseOpCode. The constructor takes only keyword arguments and will set attributes on this object based on the passed arguments. As such, it means that you should not pass arguments which are not in the __slots__ attribute for this class. """ slots = self.GetAllSlots() for (key, value) in kwargs.items(): if key not in slots: raise TypeError("Object %s doesn't support the parameter '%s'" % (self.__class__.__name__, key)) setattr(self, key, value) @classmethod def GetAllSlots(cls): """Compute the list of all declared slots for a class. """ slots = [] for parent in cls.__mro__: slots.extend(getattr(parent, "__slots__", [])) return slots def Validate(self): """Validates the slots. This method returns L{None} if the validation succeeds, or raises an exception otherwise. @rtype: NoneType @return: L{None}, if the validation succeeds @raise Exception: validation fails This method must be implemented by the child classes. """ raise NotImplementedError def ContainerToDicts(container): """Convert the elements of a container to standard Python types. This method converts a container with elements to standard Python types. If the input container is of the type C{dict}, only its values are touched. Those values, as well as all elements of input sequences, must support a C{ToDict} method returning a serialized version. @type container: dict or sequence (see L{_SEQUENCE_TYPES}) """ if isinstance(container, dict): ret = dict([(k, v.ToDict()) for k, v in container.items()]) elif isinstance(container, _SEQUENCE_TYPES): ret = [elem.ToDict() for elem in container] else: raise TypeError("Unknown container type '%s'" % type(container)) return ret def ContainerFromDicts(source, c_type, e_type): """Convert a container from standard python types. This method converts a container with standard Python types to objects. If the container is a dict, we don't touch the keys, only the values. @type source: None, dict or sequence (see L{_SEQUENCE_TYPES}) @param source: Input data @type c_type: type class @param c_type: Desired type for returned container @type e_type: element type class @param e_type: Item type for elements in returned container (must have a C{FromDict} class method) """ if not isinstance(c_type, type): raise TypeError("Container type '%s' is not a type" % type(c_type)) if source is None: source = c_type() if c_type is dict: ret = dict([(k, e_type.FromDict(v)) for k, v in source.items()]) elif c_type in _SEQUENCE_TYPES: ret = c_type(map(e_type.FromDict, source)) else: raise TypeError("Unknown container type '%s'" % c_type) return ret ganeti-3.1.0~rc2/lib/ovf.py000064400000000000000000002051651476477700300155270ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Converter tools between ovf and ganeti config file """ # pylint: disable=F0401, E1101, C0413 # F0401 because ElementTree is not default for python 2.4 # E1101 makes no sense - pylint assumes that ElementTree object is a tuple # C0413 Wrong import position import configparser import errno import logging import os import os.path import re import shutil import tarfile import tempfile import xml.dom.minidom import xml.parsers.expat try: import xml.etree.ElementTree as ET except ImportError: import elementtree.ElementTree as ET try: ParseError = ET.ParseError # pylint: disable=E1103 except AttributeError: ParseError = None from ganeti import constants from ganeti import errors from ganeti import utils from ganeti import pathutils # Schemas used in OVF format GANETI_SCHEMA = "http://ganeti" OVF_SCHEMA = "http://schemas.dmtf.org/ovf/envelope/1" RASD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/" "CIM_ResourceAllocationSettingData") VSSD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/" "CIM_VirtualSystemSettingData") XML_SCHEMA = "http://www.w3.org/2001/XMLSchema-instance" # File extensions in OVF package OVA_EXT = ".ova" OVF_EXT = ".ovf" MF_EXT = ".mf" CERT_EXT = ".cert" COMPRESSION_EXT = ".gz" FILE_EXTENSIONS = [ OVF_EXT, MF_EXT, CERT_EXT, ] COMPRESSION_TYPE = "gzip" NO_COMPRESSION = [None, "identity"] COMPRESS = "compression" DECOMPRESS = "decompression" ALLOWED_ACTIONS = [COMPRESS, DECOMPRESS] VMDK = "vmdk" RAW = "raw" COW = "cow" ALLOWED_FORMATS = [RAW, COW, VMDK] # ResourceType values RASD_TYPE = { "vcpus": "3", "memory": "4", "scsi-controller": "6", "ethernet-adapter": "10", "disk": "17", } SCSI_SUBTYPE = "lsilogic" VS_TYPE = { "ganeti": "ganeti-ovf", "external": "vmx-04", } # AllocationUnits values and conversion ALLOCATION_UNITS = { "b": ["bytes", "b"], "kb": ["kilobytes", "kb", "byte * 2^10", "kibibytes", "kib"], "mb": ["megabytes", "mb", "byte * 2^20", "mebibytes", "mib"], "gb": ["gigabytes", "gb", "byte * 2^30", "gibibytes", "gib"], } CONVERT_UNITS_TO_MB = { "b": lambda x: x // (1024 * 1024), "kb": lambda x: x // 1024, "mb": lambda x: x, "gb": lambda x: x * 1024, } # Names of the config fields NAME = "name" OS = "os" HYPERV = "hypervisor" VCPUS = "vcpus" MEMORY = "memory" AUTO_BALANCE = "auto_balance" DISK_TEMPLATE = "disk_template" TAGS = "tags" VERSION = "version" # Instance IDs of System and SCSI controller INSTANCE_ID = { "system": 0, "vcpus": 1, "memory": 2, "scsi": 3, } # Disk format descriptions DISK_FORMAT = { RAW: "http://en.wikipedia.org/wiki/Byte", VMDK: "http://www.vmware.com/interfaces/specifications/vmdk.html" "#monolithicSparse", COW: "http://www.gnome.org/~markmc/qcow-image-format.html", } def CheckQemuImg(): """ Make sure that qemu-img is present before performing operations. @raise errors.OpPrereqError: when qemu-img was not found in the system """ if not constants.QEMUIMG_PATH: raise errors.OpPrereqError("qemu-img not found at build time, unable" " to continue", errors.ECODE_STATE) def LinkFile(old_path, prefix=None, suffix=None, directory=None): """Create link with a given prefix and suffix. This is a wrapper over os.link. It tries to create a hard link for given file, but instead of rising error when file exists, the function changes the name a little bit. @type old_path:string @param old_path: path to the file that is to be linked @type prefix: string @param prefix: prefix of filename for the link @type suffix: string @param suffix: suffix of the filename for the link @type directory: string @param directory: directory of the link @raise errors.OpPrereqError: when error on linking is different than "File exists" """ assert(prefix is not None or suffix is not None) if directory is None: directory = os.getcwd() new_path = utils.PathJoin(directory, "%s%s" % (prefix, suffix)) counter = 1 while True: try: os.link(old_path, new_path) break except OSError as err: if err.errno == errno.EEXIST: new_path = utils.PathJoin(directory, "%s_%s%s" % (prefix, counter, suffix)) counter += 1 else: raise errors.OpPrereqError("Error moving the file %s to %s location:" " %s" % (old_path, new_path, err), errors.ECODE_ENVIRON) return new_path class OVFReader(object): """Reader class for OVF files. @type files_list: list @ivar files_list: list of files in the OVF package @type tree: ET.ElementTree @ivar tree: XML tree of the .ovf file @type schema_name: string @ivar schema_name: name of the .ovf file @type input_dir: string @ivar input_dir: directory in which the .ovf file resides """ def __init__(self, input_path): """Initialiaze the reader - load the .ovf file to XML parser. It is assumed that names of manifesto (.mf), certificate (.cert) and ovf files are the same. In order to account any other files as part of the ovf package, they have to be explicitly mentioned in the Resources section of the .ovf file. @type input_path: string @param input_path: absolute path to the .ovf file @raise errors.OpPrereqError: when .ovf file is not a proper XML file or some of the files mentioned in Resources section do not exist """ self.tree = ET.ElementTree() try: self.tree.parse(input_path) except (ParseError, xml.parsers.expat.ExpatError) as err: raise errors.OpPrereqError("Error while reading %s file: %s" % (OVF_EXT, err), errors.ECODE_ENVIRON) # Create a list of all files in the OVF package (input_dir, input_file) = os.path.split(input_path) (input_name, _) = os.path.splitext(input_file) files_directory = utils.ListVisibleFiles(input_dir) files_list = [] for file_name in files_directory: (name, extension) = os.path.splitext(file_name) if extension in FILE_EXTENSIONS and name == input_name: files_list.append(file_name) files_list += self._GetAttributes("{%s}References/{%s}File" % (OVF_SCHEMA, OVF_SCHEMA), "{%s}href" % OVF_SCHEMA) for file_name in files_list: file_path = utils.PathJoin(input_dir, file_name) if not os.path.exists(file_path): raise errors.OpPrereqError("File does not exist: %s" % file_path, errors.ECODE_ENVIRON) logging.info("Files in the OVF package: %s", " ".join(files_list)) self.files_list = files_list self.input_dir = input_dir self.schema_name = input_name def _GetAttributes(self, path, attribute): """Get specified attribute from all nodes accessible using given path. Function follows the path from root node to the desired tags using path, then reads the apropriate attribute values. @type path: string @param path: path of nodes to visit @type attribute: string @param attribute: attribute for which we gather the information @rtype: list @return: for each accessible tag with the attribute value set, value of the attribute """ current_list = self.tree.findall(path) results = [x.get(attribute) for x in current_list] return [r for r in results if r] def _GetElementMatchingAttr(self, path, match_attr): """Searches for element on a path that matches certain attribute value. Function follows the path from root node to the desired tags using path, then searches for the first one matching the attribute value. @type path: string @param path: path of nodes to visit @type match_attr: tuple @param match_attr: pair (attribute, value) for which we search @rtype: ET.ElementTree or None @return: first element matching match_attr or None if nothing matches """ potential_elements = self.tree.findall(path) (attr, val) = match_attr for elem in potential_elements: if elem.get(attr) == val: return elem return None def _GetElementMatchingText(self, path, match_text): """Searches for element on a path that matches certain text value. Function follows the path from root node to the desired tags using path, then searches for the first one matching the text value. @type path: string @param path: path of nodes to visit @type match_text: tuple @param match_text: pair (node, text) for which we search @rtype: ET.ElementTree or None @return: first element matching match_text or None if nothing matches """ potential_elements = self.tree.findall(path) (node, text) = match_text for elem in potential_elements: if elem.findtext(node) == text: return elem return None @staticmethod def _GetDictParameters(root, schema): """Reads text in all children and creates the dictionary from the contents. @type root: ET.ElementTree or None @param root: father of the nodes we want to collect data about @type schema: string @param schema: schema name to be removed from the tag @rtype: dict @return: dictionary containing tags and their text contents, tags have their schema fragment removed or empty dictionary, when root is None """ if root is None: return {} results = {} for element in list(root): pref_len = len("{%s}" % schema) assert(schema in element.tag) tag = element.tag[pref_len:] results[tag] = element.text return results def VerifyManifest(self): """Verifies manifest for the OVF package, if one is given. @raise errors.OpPrereqError: if SHA1 checksums do not match """ if "%s%s" % (self.schema_name, MF_EXT) in self.files_list: logging.warning("Verifying SHA1 checksums, this may take a while") manifest_filename = "%s%s" % (self.schema_name, MF_EXT) manifest_path = utils.PathJoin(self.input_dir, manifest_filename) manifest_content = utils.ReadFile(manifest_path).splitlines() manifest_files = {} regexp = r"SHA1\((\S+)\)= (\S+)" for line in manifest_content: match = re.match(regexp, line) if match: file_name = match.group(1) sha1_sum = match.group(2) manifest_files[file_name] = sha1_sum files_with_paths = [utils.PathJoin(self.input_dir, file_name) for file_name in self.files_list] sha1_sums = utils.FingerprintFiles(files_with_paths) for file_name, value in manifest_files.items(): if sha1_sums.get(utils.PathJoin(self.input_dir, file_name)) != value: raise errors.OpPrereqError("SHA1 checksum of %s does not match the" " value in manifest file" % file_name, errors.ECODE_ENVIRON) logging.info("SHA1 checksums verified") def GetInstanceName(self): """Provides information about instance name. @rtype: string @return: instance name string """ find_name = "{%s}VirtualSystem/{%s}Name" % (OVF_SCHEMA, OVF_SCHEMA) return self.tree.findtext(find_name) def GetDiskTemplate(self): """Returns disk template from .ovf file @rtype: string or None @return: name of the template """ find_template = ("{%s}GanetiSection/{%s}DiskTemplate" % (GANETI_SCHEMA, GANETI_SCHEMA)) return self.tree.findtext(find_template) def GetHypervisorData(self): """Provides hypervisor information - hypervisor name and options. @rtype: dict @return: dictionary containing name of the used hypervisor and all the specified options """ hypervisor_search = ("{%s}GanetiSection/{%s}Hypervisor" % (GANETI_SCHEMA, GANETI_SCHEMA)) hypervisor_data = self.tree.find(hypervisor_search) if hypervisor_data is None: return {"hypervisor_name": constants.VALUE_AUTO} results = { "hypervisor_name": hypervisor_data.findtext("{%s}Name" % GANETI_SCHEMA, default=constants.VALUE_AUTO), } parameters = hypervisor_data.find("{%s}Parameters" % GANETI_SCHEMA) results.update(self._GetDictParameters(parameters, GANETI_SCHEMA)) return results def GetOSData(self): """ Provides operating system information - os name and options. @rtype: dict @return: dictionary containing name and options for the chosen OS """ results = {} os_search = ("{%s}GanetiSection/{%s}OperatingSystem" % (GANETI_SCHEMA, GANETI_SCHEMA)) os_data = self.tree.find(os_search) if os_data is not None: results["os_name"] = os_data.findtext("{%s}Name" % GANETI_SCHEMA) parameters = os_data.find("{%s}Parameters" % GANETI_SCHEMA) results.update(self._GetDictParameters(parameters, GANETI_SCHEMA)) return results def GetBackendData(self): """ Provides backend information - vcpus, memory, auto balancing options. @rtype: dict @return: dictionary containing options for vcpus, memory and auto balance settings """ results = {} find_vcpus = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item" % (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA)) match_vcpus = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["vcpus"]) vcpus = self._GetElementMatchingText(find_vcpus, match_vcpus) if vcpus is not None: vcpus_count = vcpus.findtext("{%s}VirtualQuantity" % RASD_SCHEMA, default=constants.VALUE_AUTO) else: vcpus_count = constants.VALUE_AUTO results["vcpus"] = str(vcpus_count) find_memory = find_vcpus match_memory = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["memory"]) memory = self._GetElementMatchingText(find_memory, match_memory) memory_raw = None if memory is not None: alloc_units = memory.findtext("{%s}AllocationUnits" % RASD_SCHEMA) matching_units = [units for units, variants in ALLOCATION_UNITS.items() if alloc_units.lower() in variants] if matching_units == []: raise errors.OpPrereqError("Unit %s for RAM memory unknown" % alloc_units, errors.ECODE_INVAL) units = matching_units[0] memory_raw = int(memory.findtext("{%s}VirtualQuantity" % RASD_SCHEMA, default=constants.VALUE_AUTO)) memory_count = CONVERT_UNITS_TO_MB[units](memory_raw) else: memory_count = constants.VALUE_AUTO results["memory"] = str(memory_count) find_balance = ("{%s}GanetiSection/{%s}AutoBalance" % (GANETI_SCHEMA, GANETI_SCHEMA)) balance = self.tree.findtext(find_balance, default=constants.VALUE_AUTO) results["auto_balance"] = balance return results def GetTagsData(self): """Provides tags information for instance. @rtype: string or None @return: string of comma-separated tags for the instance """ find_tags = "{%s}GanetiSection/{%s}Tags" % (GANETI_SCHEMA, GANETI_SCHEMA) results = self.tree.findtext(find_tags) if results: return results else: return None def GetVersionData(self): """Provides version number read from .ovf file @rtype: string @return: string containing the version number """ find_version = ("{%s}GanetiSection/{%s}Version" % (GANETI_SCHEMA, GANETI_SCHEMA)) return self.tree.findtext(find_version) def GetNetworkData(self): """Provides data about the network in the OVF instance. The method gathers the data about networks used by OVF instance. It assumes that 'name' tag means something - in essence, if it contains one of the words 'bridged' or 'routed' then that will be the mode of this network in Ganeti. The information about the network can be either in GanetiSection or VirtualHardwareSection. @rtype: dict @return: dictionary containing all the network information """ results = {} networks_search = ("{%s}NetworkSection/{%s}Network" % (OVF_SCHEMA, OVF_SCHEMA)) network_names = self._GetAttributes(networks_search, "{%s}name" % OVF_SCHEMA) required = ["ip", "mac", "link", "mode", "network"] for (counter, network_name) in enumerate(network_names): network_search = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item" % (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA)) ganeti_search = ("{%s}GanetiSection/{%s}Network/{%s}Nic" % (GANETI_SCHEMA, GANETI_SCHEMA, GANETI_SCHEMA)) network_match = ("{%s}Connection" % RASD_SCHEMA, network_name) ganeti_match = ("{%s}name" % OVF_SCHEMA, network_name) network_data = self._GetElementMatchingText(network_search, network_match) network_ganeti_data = self._GetElementMatchingAttr(ganeti_search, ganeti_match) ganeti_data = {} if network_ganeti_data is not None: ganeti_data["mode"] = network_ganeti_data.findtext("{%s}Mode" % GANETI_SCHEMA) ganeti_data["mac"] = network_ganeti_data.findtext("{%s}MACAddress" % GANETI_SCHEMA) ganeti_data["ip"] = network_ganeti_data.findtext("{%s}IPAddress" % GANETI_SCHEMA) ganeti_data["link"] = network_ganeti_data.findtext("{%s}Link" % GANETI_SCHEMA) ganeti_data["network"] = network_ganeti_data.findtext("{%s}Net" % GANETI_SCHEMA) mac_data = None if network_data is not None: mac_data = network_data.findtext("{%s}Address" % RASD_SCHEMA) network_name = network_name.lower() # First, some not Ganeti-specific information is collected if constants.NIC_MODE_BRIDGED in network_name: results["nic%s_mode" % counter] = "bridged" elif constants.NIC_MODE_ROUTED in network_name: results["nic%s_mode" % counter] = "routed" results["nic%s_mac" % counter] = mac_data # GanetiSection data overrides 'manually' collected data for name, value in ganeti_data.items(): results["nic%s_%s" % (counter, name)] = value # Bridged network has no IP - unless specifically stated otherwise if (results.get("nic%s_mode" % counter) == "bridged" and not results.get("nic%s_ip" % counter)): results["nic%s_ip" % counter] = constants.VALUE_NONE for option in required: if not results.get("nic%s_%s" % (counter, option)): results["nic%s_%s" % (counter, option)] = constants.VALUE_AUTO if network_names: results["nic_count"] = str(len(network_names)) return results def GetDisksNames(self): """Provides list of file names for the disks used by the instance. @rtype: list @return: list of file names, as referenced in .ovf file """ results = [] disks_search = "{%s}DiskSection/{%s}Disk" % (OVF_SCHEMA, OVF_SCHEMA) disk_ids = self._GetAttributes(disks_search, "{%s}fileRef" % OVF_SCHEMA) for disk in disk_ids: disk_search = "{%s}References/{%s}File" % (OVF_SCHEMA, OVF_SCHEMA) disk_match = ("{%s}id" % OVF_SCHEMA, disk) disk_elem = self._GetElementMatchingAttr(disk_search, disk_match) if disk_elem is None: raise errors.OpPrereqError("%s file corrupted - disk %s not found in" " references" % (OVF_EXT, disk), errors.ECODE_ENVIRON) disk_name = disk_elem.get("{%s}href" % OVF_SCHEMA) disk_compression = disk_elem.get("{%s}compression" % OVF_SCHEMA) results.append((disk_name, disk_compression)) return results def SubElementText(parent, tag, text, attrib={}, **extra): # pylint: disable=W0102 """This is just a wrapper on ET.SubElement that always has text content. """ if text is None: return None elem = ET.SubElement(parent, tag, attrib=attrib, **extra) elem.text = str(text) return elem class OVFWriter(object): """Writer class for OVF files. @type tree: ET.ElementTree @ivar tree: XML tree that we are constructing @type virtual_system_type: string @ivar virtual_system_type: value of vssd:VirtualSystemType, for external usage in VMWare this requires to be vmx @type hardware_list: list @ivar hardware_list: list of items prepared for VirtualHardwareSection @type next_instance_id: int @ivar next_instance_id: next instance id to be used when creating elements on hardware_list """ def __init__(self, has_gnt_section): """Initialize the writer - set the top element. @type has_gnt_section: bool @param has_gnt_section: if the Ganeti schema should be added - i.e. this means that Ganeti section will be present """ env_attribs = { "xmlns:xsi": XML_SCHEMA, "xmlns:vssd": VSSD_SCHEMA, "xmlns:rasd": RASD_SCHEMA, "xmlns:ovf": OVF_SCHEMA, "xmlns": OVF_SCHEMA, "xml:lang": "en-US", } if has_gnt_section: env_attribs["xmlns:gnt"] = GANETI_SCHEMA self.virtual_system_type = VS_TYPE["ganeti"] else: self.virtual_system_type = VS_TYPE["external"] self.tree = ET.Element("Envelope", attrib=env_attribs) self.hardware_list = [] # INSTANCE_ID contains statically assigned IDs, starting from 0 self.next_instance_id = len(INSTANCE_ID) # FIXME: hackish def SaveDisksData(self, disks): """Convert disk information to certain OVF sections. @type disks: list @param disks: list of dictionaries of disk options from config.ini """ references = ET.SubElement(self.tree, "References") disk_section = ET.SubElement(self.tree, "DiskSection") SubElementText(disk_section, "Info", "Virtual disk information") for counter, disk in enumerate(disks): file_id = "file%s" % counter disk_id = "disk%s" % counter file_attribs = { "ovf:href": disk["path"], "ovf:size": str(disk["real-size"]), "ovf:id": file_id, } disk_attribs = { "ovf:capacity": str(disk["virt-size"]), "ovf:diskId": disk_id, "ovf:fileRef": file_id, "ovf:format": DISK_FORMAT.get(disk["format"], disk["format"]), } if "compression" in disk: file_attribs["ovf:compression"] = disk["compression"] ET.SubElement(references, "File", attrib=file_attribs) ET.SubElement(disk_section, "Disk", attrib=disk_attribs) # Item in VirtualHardwareSection creation disk_item = ET.Element("Item") SubElementText(disk_item, "rasd:ElementName", disk_id) SubElementText(disk_item, "rasd:HostResource", "ovf:/disk/%s" % disk_id) SubElementText(disk_item, "rasd:InstanceID", self.next_instance_id) SubElementText(disk_item, "rasd:Parent", INSTANCE_ID["scsi"]) SubElementText(disk_item, "rasd:ResourceType", RASD_TYPE["disk"]) self.hardware_list.append(disk_item) self.next_instance_id += 1 def SaveNetworksData(self, networks): """Convert network information to NetworkSection. @type networks: list @param networks: list of dictionaries of network options form config.ini """ network_section = ET.SubElement(self.tree, "NetworkSection") SubElementText(network_section, "Info", "List of logical networks") for counter, network in enumerate(networks): network_name = "%s%s" % (network["mode"], counter) network_attrib = {"ovf:name": network_name} ET.SubElement(network_section, "Network", attrib=network_attrib) # Item in VirtualHardwareSection creation network_item = ET.Element("Item") SubElementText(network_item, "rasd:Address", network["mac"]) SubElementText(network_item, "rasd:Connection", network_name) SubElementText(network_item, "rasd:ElementName", network_name) SubElementText(network_item, "rasd:InstanceID", self.next_instance_id) SubElementText(network_item, "rasd:ResourceType", RASD_TYPE["ethernet-adapter"]) self.hardware_list.append(network_item) self.next_instance_id += 1 @staticmethod def _SaveNameAndParams(root, data): """Save name and parameters information under root using data. @type root: ET.Element @param root: root element for the Name and Parameters @type data: dict @param data: data from which we gather the values """ assert(data.get("name")) name = SubElementText(root, "gnt:Name", data["name"]) params = ET.SubElement(root, "gnt:Parameters") for name, value in data.items(): if name != "name": SubElementText(params, "gnt:%s" % name, value) def SaveGanetiData(self, ganeti, networks): """Convert Ganeti-specific information to GanetiSection. @type ganeti: dict @param ganeti: dictionary of Ganeti-specific options from config.ini @type networks: list @param networks: list of dictionaries of network options form config.ini """ ganeti_section = ET.SubElement(self.tree, "gnt:GanetiSection") SubElementText(ganeti_section, "gnt:Version", ganeti.get("version")) SubElementText(ganeti_section, "gnt:DiskTemplate", ganeti.get("disk_template")) SubElementText(ganeti_section, "gnt:AutoBalance", ganeti.get("auto_balance")) SubElementText(ganeti_section, "gnt:Tags", ganeti.get("tags")) osys = ET.SubElement(ganeti_section, "gnt:OperatingSystem") self._SaveNameAndParams(osys, ganeti["os"]) hypervisor = ET.SubElement(ganeti_section, "gnt:Hypervisor") self._SaveNameAndParams(hypervisor, ganeti["hypervisor"]) network_section = ET.SubElement(ganeti_section, "gnt:Network") for counter, network in enumerate(networks): network_name = "%s%s" % (network["mode"], counter) nic_attrib = {"ovf:name": network_name} nic = ET.SubElement(network_section, "gnt:Nic", attrib=nic_attrib) SubElementText(nic, "gnt:Mode", network["mode"]) SubElementText(nic, "gnt:MACAddress", network["mac"]) SubElementText(nic, "gnt:IPAddress", network["ip"]) SubElementText(nic, "gnt:Link", network["link"]) SubElementText(nic, "gnt:Net", network["network"]) def SaveVirtualSystemData(self, name, vcpus, memory): """Convert virtual system information to OVF sections. @type name: string @param name: name of the instance @type vcpus: int @param vcpus: number of VCPUs @type memory: int @param memory: RAM memory in MB """ assert(vcpus > 0) assert(memory > 0) vs_attrib = {"ovf:id": name} virtual_system = ET.SubElement(self.tree, "VirtualSystem", attrib=vs_attrib) SubElementText(virtual_system, "Info", "A virtual machine") name_section = ET.SubElement(virtual_system, "Name") name_section.text = name os_attrib = {"ovf:id": "0"} os_section = ET.SubElement(virtual_system, "OperatingSystemSection", attrib=os_attrib) SubElementText(os_section, "Info", "Installed guest operating system") hardware_section = ET.SubElement(virtual_system, "VirtualHardwareSection") SubElementText(hardware_section, "Info", "Virtual hardware requirements") # System description system = ET.SubElement(hardware_section, "System") SubElementText(system, "vssd:ElementName", "Virtual Hardware Family") SubElementText(system, "vssd:InstanceID", INSTANCE_ID["system"]) SubElementText(system, "vssd:VirtualSystemIdentifier", name) SubElementText(system, "vssd:VirtualSystemType", self.virtual_system_type) # Item for vcpus vcpus_item = ET.SubElement(hardware_section, "Item") SubElementText(vcpus_item, "rasd:ElementName", "%s virtual CPU(s)" % vcpus) SubElementText(vcpus_item, "rasd:InstanceID", INSTANCE_ID["vcpus"]) SubElementText(vcpus_item, "rasd:ResourceType", RASD_TYPE["vcpus"]) SubElementText(vcpus_item, "rasd:VirtualQuantity", vcpus) # Item for memory memory_item = ET.SubElement(hardware_section, "Item") SubElementText(memory_item, "rasd:AllocationUnits", "byte * 2^20") SubElementText(memory_item, "rasd:ElementName", "%sMB of memory" % memory) SubElementText(memory_item, "rasd:InstanceID", INSTANCE_ID["memory"]) SubElementText(memory_item, "rasd:ResourceType", RASD_TYPE["memory"]) SubElementText(memory_item, "rasd:VirtualQuantity", memory) # Item for scsi controller scsi_item = ET.SubElement(hardware_section, "Item") SubElementText(scsi_item, "rasd:Address", INSTANCE_ID["system"]) SubElementText(scsi_item, "rasd:ElementName", "scsi_controller0") SubElementText(scsi_item, "rasd:InstanceID", INSTANCE_ID["scsi"]) SubElementText(scsi_item, "rasd:ResourceSubType", SCSI_SUBTYPE) SubElementText(scsi_item, "rasd:ResourceType", RASD_TYPE["scsi-controller"]) # Other items - from self.hardware_list for item in self.hardware_list: hardware_section.append(item) def PrettyXmlDump(self): """Formatter of the XML file. @rtype: string @return: XML tree in the form of nicely-formatted string """ raw_string = ET.tostring(self.tree, encoding="unicode") parsed_xml = xml.dom.minidom.parseString(raw_string) xml_string = parsed_xml.toprettyxml(indent=" ") text_re = re.compile(r">\n\s+([^<>\s].*?)\n\s+\g<1>": OP_GT, ">=": OP_GE, } binary_cond = (field_name + pyp.oneOf(list(binopstbl)) + rval) binary_cond.setParseAction(lambda lhs_op_rhs: [[binopstbl[lhs_op_rhs[1]], lhs_op_rhs[0], lhs_op_rhs[2]]]) # "in" condition in_cond = (rval + pyp.Suppress("in") + field_name) in_cond.setParseAction(lambda value_field: [[OP_CONTAINS, value_field[1], value_field[0]]]) # "not in" condition not_in_cond = (rval + pyp.Suppress("not") + pyp.Suppress("in") + field_name) not_in_cond.setParseAction(lambda value_field: [[OP_NOT, [OP_CONTAINS, value_field[1], value_field[0]]]]) # Regular expression, e.g. m/foobar/i regexp_val = pyp.Group(pyp.Optional("m").suppress() + pyp.MatchFirst([pyp.QuotedString(i, escChar="\\") for i in _KNOWN_REGEXP_DELIM]) + pyp.Optional(pyp.Word(pyp.alphas), default="")) regexp_val.setParseAction(_ConvertRegexpValue) regexp_cond = (field_name + pyp.Suppress("=~") + regexp_val) regexp_cond.setParseAction(lambda field_value: [[OP_REGEXP, field_value[0], field_value[1]]]) not_regexp_cond = (field_name + pyp.Suppress("!~") + regexp_val) not_regexp_cond.setParseAction(lambda field_value: [[OP_NOT, [OP_REGEXP, field_value[0], field_value[1]]]]) # Globbing, e.g. name =* "*.site" glob_cond = (field_name + pyp.Suppress("=*") + quoted_string) glob_cond.setParseAction(lambda field_value: [[OP_REGEXP, field_value[0], utils.DnsNameGlobPattern(field_value[1])]]) not_glob_cond = (field_name + pyp.Suppress("!*") + quoted_string) not_glob_cond.setParseAction(lambda field_value: [[OP_NOT, [OP_REGEXP, field_value[0], utils.DnsNameGlobPattern(field_value[1])]]]) # All possible conditions condition = (binary_cond ^ bool_cond ^ in_cond ^ not_in_cond ^ regexp_cond ^ not_regexp_cond ^ glob_cond ^ not_glob_cond) # Associativity operators filter_expr = pyp.infixNotation(condition, [ (pyp.Keyword("not").suppress(), 1, pyp.opAssoc.RIGHT, lambda toks: [[OP_NOT, toks[0][0]]]), (pyp.Keyword("and").suppress(), 2, pyp.opAssoc.LEFT, _ConvertLogicOp(OP_AND)), (pyp.Keyword("or").suppress(), 2, pyp.opAssoc.LEFT, _ConvertLogicOp(OP_OR)), ]) parser = pyp.StringStart() + filter_expr + pyp.StringEnd() parser.parseWithTabs() # Originally C{parser.validate} was called here, but there seems to be some # issue causing it to fail whenever the "not" operator is included above. return parser def ParseFilter(text, parser=None): """Parses a query filter. @type text: string @param text: Query filter @type parser: pyparsing.ParserElement @param parser: Pyparsing object @rtype: list """ logging.debug("Parsing as query filter: %s", text) if parser is None: parser = BuildFilterParser() try: return parser.parseString(text)[0] except pyp.ParseBaseException as err: raise errors.QueryFilterParseError("Failed to parse query filter" " '%s': %s" % (text, err), err) def _CheckFilter(text): """CHecks if a string could be a filter. @rtype: bool """ return bool(frozenset(text) & FILTER_DETECTION_CHARS) def _CheckGlobbing(text): """Checks if a string could be a globbing pattern. @rtype: bool """ return bool(frozenset(text) & GLOB_DETECTION_CHARS) def _MakeFilterPart(namefield, text, isnumeric=False): """Generates filter for one argument. """ if isnumeric: try: number = int(text) except (TypeError, ValueError) as err: raise errors.OpPrereqError("Invalid job ID passed: %s" % str(err), errors.ECODE_INVAL) return [OP_EQUAL, namefield, number] elif _CheckGlobbing(text): return [OP_REGEXP, namefield, utils.DnsNameGlobPattern(text)] else: return [OP_EQUAL, namefield, text] def MakeFilter(args, force_filter, namefield=None, isnumeric=False): """Try to make a filter from arguments to a command. If the name could be a filter it is parsed as such. If it's just a globbing pattern, e.g. "*.site", such a filter is constructed. As a last resort the names are treated just as a plain name filter. @type args: list of string @param args: Arguments to command @type force_filter: bool @param force_filter: Whether to force treatment as a full-fledged filter @type namefield: string @param namefield: Name of field to use for simple filters (use L{None} for a default of "name") @type isnumeric: bool @param isnumeric: Whether the namefield type is numeric, as opposed to the default string type; this influences how the filter is built @rtype: list @return: Query filter """ if namefield is None: namefield = "name" if (force_filter or (args and len(args) == 1 and _CheckFilter(args[0]))): try: (filter_text, ) = args except (TypeError, ValueError): raise errors.OpPrereqError("Exactly one argument must be given as a" " filter", errors.ECODE_INVAL) result = ParseFilter(filter_text) elif args: result = [OP_OR] + [ _MakeFilterPart(namefield, arg, isnumeric=isnumeric) for arg in args ] else: result = None return result ganeti-3.1.0~rc2/lib/query.py000064400000000000000000002625711476477700300161060ustar00rootroot00000000000000# # # Copyright (C) 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module for query operations How it works: - Add field definitions - See how L{NODE_FIELDS} is built - Each field gets: - Query field definition (L{objects.QueryFieldDefinition}, use L{_MakeField} for creating), containing: - Name, must be lowercase and match L{FIELD_NAME_RE} - Title for tables, must not contain whitespace and match L{TITLE_RE} - Value data type, e.g. L{constants.QFT_NUMBER} - Human-readable description, must not end with punctuation or contain newlines - Data request type, see e.g. C{NQ_*} - OR-ed flags, see C{QFF_*} - A retrieval function, see L{Query.__init__} for description - Pass list of fields through L{_PrepareFieldList} for preparation and checks - Instantiate L{Query} with prepared field list definition and selected fields - Call L{Query.RequestedData} to determine what data to collect/compute - Call L{Query.Query} or L{Query.OldStyleQuery} with collected data and use result - Data container must support iteration using C{__iter__} - Items are passed to retrieval functions and can have any format - Call L{Query.GetFields} to get list of definitions for selected fields @attention: Retrieval functions must be idempotent. They can be called multiple times, in any order and any number of times. """ import logging import operator import re from ganeti import constants from ganeti import errors from ganeti import utils from ganeti import compat from ganeti import objects from ganeti import ht from ganeti import runtime from ganeti import qlang from ganeti import jstore from ganeti.hypervisor import hv_base from ganeti.constants import (QFT_UNKNOWN, QFT_TEXT, QFT_BOOL, QFT_NUMBER, QFT_NUMBER_FLOAT, QFT_UNIT, QFT_TIMESTAMP, QFT_OTHER, RS_NORMAL, RS_UNKNOWN, RS_NODATA, RS_UNAVAIL, RS_OFFLINE) (NETQ_CONFIG, NETQ_GROUP, NETQ_STATS, NETQ_INST) = range(300, 304) # Constants for requesting data from the caller/data provider. Each property # collected/computed separately by the data provider should have its own to # only collect the requested data and not more. (NQ_CONFIG, NQ_INST, NQ_LIVE, NQ_GROUP, NQ_OOB) = range(1, 6) (IQ_CONFIG, IQ_LIVE, IQ_DISKUSAGE, IQ_CONSOLE, IQ_NODES, IQ_NETWORKS) = range(100, 106) (LQ_MODE, LQ_OWNER, LQ_PENDING) = range(10, 13) (GQ_CONFIG, GQ_NODE, GQ_INST, GQ_DISKPARAMS) = range(200, 204) (CQ_CONFIG, CQ_QUEUE_DRAINED, CQ_WATCHER_PAUSE) = range(300, 303) (JQ_ARCHIVED, ) = range(400, 401) # Query field flags QFF_HOSTNAME = 0x01 QFF_IP_ADDRESS = 0x02 QFF_JOB_ID = 0x04 QFF_SPLIT_TIMESTAMP = 0x08 # Next values: 0x10, 0x20, 0x40, 0x80, 0x100, 0x200 QFF_ALL = (QFF_HOSTNAME | QFF_IP_ADDRESS | QFF_JOB_ID | QFF_SPLIT_TIMESTAMP) FIELD_NAME_RE = re.compile(r"^[a-z0-9/._]+$") TITLE_RE = re.compile(r"^[^\s]+$") DOC_RE = re.compile(r"^[A-Z].*[^.,?!]$") #: Verification function for each field type _VERIFY_FN = { QFT_UNKNOWN: ht.TNone, QFT_TEXT: ht.TString, QFT_BOOL: ht.TBool, QFT_NUMBER: ht.TInt, QFT_NUMBER_FLOAT: ht.TFloat, QFT_UNIT: ht.TInt, QFT_TIMESTAMP: ht.TNumber, QFT_OTHER: lambda _: True, } # Unique objects for special field statuses _FS_UNKNOWN = object() _FS_NODATA = object() _FS_UNAVAIL = object() _FS_OFFLINE = object() #: List of all special status _FS_ALL = compat.UniqueFrozenset([ _FS_UNKNOWN, _FS_NODATA, _FS_UNAVAIL, _FS_OFFLINE, ]) #: VType to QFT mapping _VTToQFT = { # TODO: fix validation of empty strings constants.VTYPE_STRING: QFT_OTHER, # since VTYPE_STRINGs can be empty constants.VTYPE_MAYBE_STRING: QFT_OTHER, constants.VTYPE_BOOL: QFT_BOOL, constants.VTYPE_SIZE: QFT_UNIT, constants.VTYPE_INT: QFT_NUMBER, constants.VTYPE_FLOAT: QFT_NUMBER_FLOAT, } _SERIAL_NO_DOC = "%s object serial number, incremented on each modification" def _GetUnknownField(ctx, item): # pylint: disable=W0613 """Gets the contents of an unknown field. """ return _FS_UNKNOWN def _GetQueryFields(fielddefs, selected): """Calculates the internal list of selected fields. Unknown fields are returned as L{constants.QFT_UNKNOWN}. @type fielddefs: dict @param fielddefs: Field definitions @type selected: list of strings @param selected: List of selected fields """ result = [] for name in selected: try: fdef = fielddefs[name] except KeyError: fdef = (_MakeField(name, name, QFT_UNKNOWN, "Unknown field '%s'" % name), None, 0, _GetUnknownField) assert len(fdef) == 4 result.append(fdef) return result def GetAllFields(fielddefs): """Extract L{objects.QueryFieldDefinition} from field definitions. @rtype: list of L{objects.QueryFieldDefinition} """ return [fdef for (fdef, _, _, _) in fielddefs] class _FilterHints(object): """Class for filter analytics. When filters are used, the user of the L{Query} class usually doesn't know exactly which items will be necessary for building the result. It therefore has to prepare and compute the input data for potentially returning everything. There are two ways to optimize this. The first, and simpler, is to assign each field a group of data, so that the caller can determine which computations are necessary depending on the data groups requested. The list of referenced groups must also be computed for fields referenced in the filter. The second is restricting the items based on a primary key. The primary key is usually a unique name (e.g. a node name). This class extracts all referenced names from a filter. If it encounters any filter condition which disallows such a list to be determined (e.g. a non-equality filter), all names will be requested. The end-effect is that any operation other than L{qlang.OP_OR} and L{qlang.OP_EQUAL} will make the query more expensive. """ def __init__(self, namefield): """Initializes this class. @type namefield: string @param namefield: Field caller is interested in """ self._namefield = namefield #: Whether all names need to be requested (e.g. if a non-equality operator #: has been used) self._allnames = False #: Which names to request self._names = None #: Data kinds referenced by the filter (used by L{Query.RequestedData}) self._datakinds = set() def RequestedNames(self): """Returns all requested values. Returns C{None} if list of values can't be determined (e.g. encountered non-equality operators). @rtype: list """ if self._allnames or self._names is None: return None return utils.UniqueSequence(self._names) def ReferencedData(self): """Returns all kinds of data referenced by the filter. """ return frozenset(self._datakinds) def _NeedAllNames(self): """Changes internal state to request all names. """ self._allnames = True self._names = None def NoteLogicOp(self, op): """Called when handling a logic operation. @type op: string @param op: Operator """ if op != qlang.OP_OR: self._NeedAllNames() def NoteUnaryOp(self, op, datakind): # pylint: disable=W0613 """Called when handling an unary operation. @type op: string @param op: Operator """ if datakind is not None: self._datakinds.add(datakind) self._NeedAllNames() def NoteBinaryOp(self, op, datakind, name, value): """Called when handling a binary operation. @type op: string @param op: Operator @type name: string @param name: Left-hand side of operator (field name) @param value: Right-hand side of operator """ if datakind is not None: self._datakinds.add(datakind) if self._allnames: return # If any operator other than equality was used, all names need to be # retrieved EQ_OPS = [qlang.OP_EQUAL, qlang.OP_EQUAL_LEGACY] if op in EQ_OPS and name == self._namefield: if self._names is None: self._names = [] self._names.append(value) else: self._NeedAllNames() def _WrapLogicOp(op_fn, sentences, ctx, item): """Wrapper for logic operator functions. """ return op_fn(fn(ctx, item) for fn in sentences) def _WrapUnaryOp(op_fn, inner, ctx, item): """Wrapper for unary operator functions. """ return op_fn(inner(ctx, item)) def _WrapBinaryOp(op_fn, retrieval_fn, value, ctx, item): """Wrapper for binary operator functions. """ return op_fn(retrieval_fn(ctx, item), value) def _WrapNot(fn, lhs, rhs): """Negates the result of a wrapped function. """ return not fn(lhs, rhs) def _PrepareRegex(pattern): """Compiles a regular expression. """ try: return re.compile(pattern) except re.error as err: raise errors.ParameterError("Invalid regex pattern (%s)" % err) def _PrepareSplitTimestamp(value): """Prepares a value for comparison by L{_MakeSplitTimestampComparison}. """ if ht.TNumber(value): return value else: return utils.MergeTime(value) def _MakeSplitTimestampComparison(fn): """Compares split timestamp values after converting to float. """ return lambda lhs, rhs: fn(utils.MergeTime(lhs), rhs) def _MakeComparisonChecks(fn): """Prepares flag-specific comparisons using a comparison function. """ return [ (QFF_SPLIT_TIMESTAMP, _MakeSplitTimestampComparison(fn), _PrepareSplitTimestamp), (QFF_JOB_ID, lambda lhs, rhs: fn(jstore.ParseJobId(lhs), rhs), jstore.ParseJobId), (None, fn, None), ] class _FilterCompilerHelper(object): """Converts a query filter to a callable usable for filtering. """ # String statement has no effect, pylint: disable=W0105 #: How deep filters can be nested _LEVELS_MAX = 10 # Unique identifiers for operator groups (_OPTYPE_LOGIC, _OPTYPE_UNARY, _OPTYPE_BINARY) = range(1, 4) """Functions for equality checks depending on field flags. List of tuples containing flags and a callable receiving the left- and right-hand side of the operator. The flags are an OR-ed value of C{QFF_*} (e.g. L{QFF_HOSTNAME} or L{QFF_SPLIT_TIMESTAMP}). Order matters. The first item with flags will be used. Flags are checked using binary AND. """ _EQUALITY_CHECKS = [ (QFF_HOSTNAME, lambda lhs, rhs: utils.MatchNameComponent(rhs, [lhs], case_sensitive=False), None), (QFF_SPLIT_TIMESTAMP, _MakeSplitTimestampComparison(operator.eq), _PrepareSplitTimestamp), (None, operator.eq, None), ] """Known operators Operator as key (C{qlang.OP_*}), value a tuple of operator group (C{_OPTYPE_*}) and a group-specific value: - C{_OPTYPE_LOGIC}: Callable taking any number of arguments; used by L{_HandleLogicOp} - C{_OPTYPE_UNARY}: Always C{None}; details handled by L{_HandleUnaryOp} - C{_OPTYPE_BINARY}: Callable taking exactly two parameters, the left- and right-hand side of the operator, used by L{_HandleBinaryOp} """ _OPS = { # Logic operators qlang.OP_OR: (_OPTYPE_LOGIC, compat.any), qlang.OP_AND: (_OPTYPE_LOGIC, compat.all), # Unary operators qlang.OP_NOT: (_OPTYPE_UNARY, None), qlang.OP_TRUE: (_OPTYPE_UNARY, None), # Binary operators qlang.OP_EQUAL: (_OPTYPE_BINARY, _EQUALITY_CHECKS), qlang.OP_EQUAL_LEGACY: (_OPTYPE_BINARY, _EQUALITY_CHECKS), qlang.OP_NOT_EQUAL: (_OPTYPE_BINARY, [(flags, compat.partial(_WrapNot, fn), valprepfn) for (flags, fn, valprepfn) in _EQUALITY_CHECKS]), qlang.OP_LT: (_OPTYPE_BINARY, _MakeComparisonChecks(operator.lt)), qlang.OP_LE: (_OPTYPE_BINARY, _MakeComparisonChecks(operator.le)), qlang.OP_GT: (_OPTYPE_BINARY, _MakeComparisonChecks(operator.gt)), qlang.OP_GE: (_OPTYPE_BINARY, _MakeComparisonChecks(operator.ge)), qlang.OP_REGEXP: (_OPTYPE_BINARY, [ (None, lambda lhs, rhs: rhs.search(lhs), _PrepareRegex), ]), qlang.OP_CONTAINS: (_OPTYPE_BINARY, [ (None, operator.contains, None), ]), } def __init__(self, fields): """Initializes this class. @param fields: Field definitions (return value of L{_PrepareFieldList}) """ self._fields = fields self._hints = None self._op_handler = None def __call__(self, hints, qfilter): """Converts a query filter into a callable function. @type hints: L{_FilterHints} or None @param hints: Callbacks doing analysis on filter @type qfilter: list @param qfilter: Filter structure @rtype: callable @return: Function receiving context and item as parameters, returning boolean as to whether item matches filter """ self._op_handler = { self._OPTYPE_LOGIC: (self._HandleLogicOp, getattr(hints, "NoteLogicOp", None)), self._OPTYPE_UNARY: (self._HandleUnaryOp, getattr(hints, "NoteUnaryOp", None)), self._OPTYPE_BINARY: (self._HandleBinaryOp, getattr(hints, "NoteBinaryOp", None)), } try: filter_fn = self._Compile(qfilter, 0) finally: self._op_handler = None return filter_fn def _Compile(self, qfilter, level): """Inner function for converting filters. Calls the correct handler functions for the top-level operator. This function is called recursively (e.g. for logic operators). """ if not (isinstance(qfilter, (list, tuple)) and qfilter): raise errors.ParameterError("Invalid filter on level %s" % level) # Limit recursion if level >= self._LEVELS_MAX: raise errors.ParameterError("Only up to %s levels are allowed (filter" " nested too deep)" % self._LEVELS_MAX) # Create copy to be modified operands = qfilter[:] op = operands.pop(0) try: (kind, op_data) = self._OPS[op] except KeyError: raise errors.ParameterError("Unknown operator '%s'" % op) (handler, hints_cb) = self._op_handler[kind] return handler(hints_cb, level, op, op_data, operands) def _LookupField(self, name): """Returns a field definition by name. """ try: return self._fields[name] except KeyError: raise errors.ParameterError("Unknown field '%s'" % name) def _HandleLogicOp(self, hints_fn, level, op, op_fn, operands): """Handles logic operators. @type hints_fn: callable @param hints_fn: Callback doing some analysis on the filter @type level: integer @param level: Current depth @type op: string @param op: Operator @type op_fn: callable @param op_fn: Function implementing operator @type operands: list @param operands: List of operands """ if hints_fn: hints_fn(op) return compat.partial(_WrapLogicOp, op_fn, [self._Compile(op, level + 1) for op in operands]) def _HandleUnaryOp(self, hints_fn, level, op, op_fn, operands): """Handles unary operators. @type hints_fn: callable @param hints_fn: Callback doing some analysis on the filter @type level: integer @param level: Current depth @type op: string @param op: Operator @type op_fn: callable @param op_fn: Function implementing operator @type operands: list @param operands: List of operands """ assert op_fn is None if len(operands) != 1: raise errors.ParameterError("Unary operator '%s' expects exactly one" " operand" % op) if op == qlang.OP_TRUE: (_, datakind, _, retrieval_fn) = self._LookupField(operands[0]) if hints_fn: hints_fn(op, datakind) op_fn = operator.truth arg = retrieval_fn elif op == qlang.OP_NOT: if hints_fn: hints_fn(op, None) op_fn = operator.not_ arg = self._Compile(operands[0], level + 1) else: raise errors.ProgrammerError("Can't handle operator '%s'" % op) return compat.partial(_WrapUnaryOp, op_fn, arg) def _HandleBinaryOp(self, hints_fn, level, op, op_data, operands): """Handles binary operators. @type hints_fn: callable @param hints_fn: Callback doing some analysis on the filter @type level: integer @param level: Current depth @type op: string @param op: Operator @param op_data: Functions implementing operators @type operands: list @param operands: List of operands """ # Unused arguments, pylint: disable=W0613 try: (name, value) = operands except (ValueError, TypeError): raise errors.ParameterError("Invalid binary operator, expected exactly" " two operands") (fdef, datakind, field_flags, retrieval_fn) = self._LookupField(name) assert fdef.kind != QFT_UNKNOWN # TODO: Type conversions? verify_fn = _VERIFY_FN[fdef.kind] if not verify_fn(value): raise errors.ParameterError("Unable to compare field '%s' (type '%s')" " with '%s', expected %s" % (name, fdef.kind, value.__class__.__name__, verify_fn)) if hints_fn: hints_fn(op, datakind, name, value) for (fn_flags, fn, valprepfn) in op_data: if fn_flags is None or fn_flags & field_flags: # Prepare value if necessary (e.g. compile regular expression) if valprepfn: value = valprepfn(value) return compat.partial(_WrapBinaryOp, fn, retrieval_fn, value) raise errors.ProgrammerError("Unable to find operator implementation" " (op '%s', flags %s)" % (op, field_flags)) def _CompileFilter(fields, hints, qfilter): """Converts a query filter into a callable function. See L{_FilterCompilerHelper} for details. @rtype: callable """ return _FilterCompilerHelper(fields)(hints, qfilter) class Query(object): def __init__(self, fieldlist, selected, qfilter=None, namefield=None): """Initializes this class. The field definition is a dictionary with the field's name as a key and a tuple containing, in order, the field definition object (L{objects.QueryFieldDefinition}, the data kind to help calling code collect data and a retrieval function. The retrieval function is called with two parameters, in order, the data container and the item in container (see L{Query.Query}). Users of this class can call L{RequestedData} before preparing the data container to determine what data is needed. @type fieldlist: dictionary @param fieldlist: Field definitions @type selected: list of strings @param selected: List of selected fields """ assert namefield is None or namefield in fieldlist self._fields = _GetQueryFields(fieldlist, selected) self._filter_fn = None self._requested_names = None self._filter_datakinds = frozenset() if qfilter is not None: # Collect requested names if wanted if namefield: hints = _FilterHints(namefield) else: hints = None # Build filter function self._filter_fn = _CompileFilter(fieldlist, hints, qfilter) if hints: self._requested_names = hints.RequestedNames() self._filter_datakinds = hints.ReferencedData() if namefield is None: self._name_fn = None else: (_, _, _, self._name_fn) = fieldlist[namefield] def RequestedNames(self): """Returns all names referenced in the filter. If there is no filter or operators are preventing determining the exact names, C{None} is returned. """ return self._requested_names def RequestedData(self): """Gets requested kinds of data. @rtype: frozenset """ return (self._filter_datakinds | frozenset(datakind for (_, datakind, _, _) in self._fields if datakind is not None)) def GetFields(self): """Returns the list of fields for this query. Includes unknown fields. @rtype: List of L{objects.QueryFieldDefinition} """ return GetAllFields(self._fields) def Query(self, ctx, sort_by_name=True): """Execute a query. @param ctx: Data container passed to field retrieval functions, must support iteration using C{__iter__} @type sort_by_name: boolean @param sort_by_name: Whether to sort by name or keep the input data's ordering """ sort = (self._name_fn and sort_by_name) result = [] for idx, item in enumerate(ctx): if not (self._filter_fn is None or self._filter_fn(ctx, item)): continue row = [_ProcessResult(fn(ctx, item)) for (_, _, _, fn) in self._fields] # Verify result if __debug__: _VerifyResultRow(self._fields, row) if sort: (status, name) = _ProcessResult(self._name_fn(ctx, item)) assert status == constants.RS_NORMAL # TODO: Are there cases where we wouldn't want to use NiceSort? # Answer: if the name field is non-string... result.append((utils.NiceSortKey(name), idx, row)) else: result.append(row) if not sort: return result # TODO: Would "heapq" be more efficient than sorting? # Sorting in-place instead of using "sorted()" result.sort() assert not result or (len(result[0]) == 3 and len(result[-1]) == 3) return [r[2] for r in result] def OldStyleQuery(self, ctx, sort_by_name=True): """Query with "old" query result format. See L{Query.Query} for arguments. """ unknown = set(fdef.name for (fdef, _, _, _) in self._fields if fdef.kind == QFT_UNKNOWN) if unknown: raise errors.OpPrereqError("Unknown output fields selected: %s" % (utils.CommaJoin(unknown), ), errors.ECODE_INVAL) return [[value for (_, value) in row] for row in self.Query(ctx, sort_by_name=sort_by_name)] def _ProcessResult(value): """Converts result values into externally-visible ones. """ if value is _FS_UNKNOWN: return (RS_UNKNOWN, None) elif value is _FS_NODATA: return (RS_NODATA, None) elif value is _FS_UNAVAIL: return (RS_UNAVAIL, None) elif value is _FS_OFFLINE: return (RS_OFFLINE, None) else: return (RS_NORMAL, value) def _VerifyResultRow(fields, row): """Verifies the contents of a query result row. @type fields: list @param fields: Field definitions for result @type row: list of tuples @param row: Row data """ assert len(row) == len(fields) errs = [] for ((status, value), (fdef, _, _, _)) in zip(row, fields): if status == RS_NORMAL: if not _VERIFY_FN[fdef.kind](value): errs.append("normal field %s fails validation (value is %s)" % (fdef.name, value)) elif value is not None: errs.append("abnormal field %s has a non-None value" % fdef.name) assert not errs, ("Failed validation: %s in row %s" % (utils.CommaJoin(errs), row)) def _FieldDictKey(field): """Generates key for field dictionary. """ (fdef, _, flags, fn) = field assert fdef.name and fdef.title, "Name and title are required" assert FIELD_NAME_RE.match(fdef.name) assert TITLE_RE.match(fdef.title) assert (DOC_RE.match(fdef.doc) and len(fdef.doc.splitlines()) == 1 and fdef.doc.strip() == fdef.doc), \ "Invalid description for field '%s'" % fdef.name assert callable(fn) assert (flags & ~QFF_ALL) == 0, "Unknown flags for field '%s'" % fdef.name return fdef.name def _PrepareFieldList(fields, aliases): """Prepares field list for use by L{Query}. Converts the list to a dictionary and does some verification. @type fields: list of tuples; (L{objects.QueryFieldDefinition}, data kind, retrieval function) @param fields: List of fields, see L{Query.__init__} for a better description @type aliases: list of tuples; (alias, target) @param aliases: list of tuples containing aliases; for each alias/target pair, a duplicate will be created in the field list @rtype: dict @return: Field dictionary for L{Query} """ if __debug__: duplicates = utils.FindDuplicates(fdef.title.lower() for (fdef, _, _, _) in fields) assert not duplicates, "Duplicate title(s) found: %r" % duplicates result = utils.SequenceToDict(fields, key=_FieldDictKey) for alias, target in aliases: assert alias not in result, "Alias %s overrides an existing field" % alias assert target in result, "Missing target %s for alias %s" % (target, alias) (fdef, k, flags, fn) = result[target] fdef = fdef.Copy() fdef.name = alias result[alias] = (fdef, k, flags, fn) assert len(result) == len(fields) + len(aliases) assert compat.all(name == fdef.name for (name, (fdef, _, _, _)) in result.items()) return result def GetQueryResponse(query, ctx, sort_by_name=True): """Prepares the response for a query. @type query: L{Query} @param ctx: Data container, see L{Query.Query} @type sort_by_name: boolean @param sort_by_name: Whether to sort by name or keep the input data's ordering """ return objects.QueryResponse(data=query.Query(ctx, sort_by_name=sort_by_name), fields=query.GetFields()).ToDict() def QueryFields(fielddefs, selected): """Returns list of available fields. @type fielddefs: dict @param fielddefs: Field definitions @type selected: list of strings @param selected: List of selected fields @return: List of L{objects.QueryFieldDefinition} """ if selected is None: # Client requests all fields, sort by name fdefs = utils.NiceSort(GetAllFields(fielddefs.values()), key=operator.attrgetter("name")) else: # Keep order as requested by client fdefs = Query(fielddefs, selected).GetFields() return objects.QueryFieldsResponse(fields=fdefs).ToDict() def _MakeField(name, title, kind, doc): """Wrapper for creating L{objects.QueryFieldDefinition} instances. @param name: Field name as a regular expression @param title: Human-readable title @param kind: Field type @param doc: Human-readable description """ return objects.QueryFieldDefinition(name=name, title=title, kind=kind, doc=doc) def _StaticValueInner(value, ctx, _): # pylint: disable=W0613 """Returns a static value. """ return value def _StaticValue(value): """Prepares a function to return a static value. """ return compat.partial(_StaticValueInner, value) def _GetNodeRole(node, master_uuid): """Determine node role. @type node: L{objects.Node} @param node: Node object @type master_uuid: string @param master_uuid: Master node UUID """ if node.uuid == master_uuid: return constants.NR_MASTER elif node.master_candidate: return constants.NR_MCANDIDATE elif node.drained: return constants.NR_DRAINED elif node.offline: return constants.NR_OFFLINE else: return constants.NR_REGULAR def _GetItemAttr(attr): """Returns a field function to return an attribute of the item. @param attr: Attribute name """ getter = operator.attrgetter(attr) return lambda _, item: getter(item) def _GetItemMaybeAttr(attr): """Returns a field function to return a not-None attribute of the item. If the value is None, then C{_FS_UNAVAIL} will be returned instead. @param attr: Attribute name """ def _helper(_, obj): val = getattr(obj, attr) if val is None: return _FS_UNAVAIL else: return val return _helper def _GetNDParam(name): """Return a field function to return an ND parameter out of the context. """ def _helper(ctx, _): if ctx.ndparams is None: return _FS_UNAVAIL else: return ctx.ndparams.get(name, None) return _helper def _BuildNDFields(is_group): """Builds all the ndparam fields. @param is_group: whether this is called at group or node level """ if is_group: field_kind = GQ_CONFIG else: field_kind = NQ_GROUP return [(_MakeField("ndp/%s" % name, constants.NDS_PARAMETER_TITLES.get(name, "ndp/%s" % name), _VTToQFT[kind], "The \"%s\" node parameter" % name), field_kind, 0, _GetNDParam(name)) for name, kind in constants.NDS_PARAMETER_TYPES.items()] def _ConvWrapInner(convert, fn, ctx, item): """Wrapper for converting values. @param convert: Conversion function receiving value as single parameter @param fn: Retrieval function """ value = fn(ctx, item) # Is the value an abnormal status? if compat.any(value is fs for fs in _FS_ALL): # Return right away return value # TODO: Should conversion function also receive context, item or both? return convert(value) def _ConvWrap(convert, fn): """Convenience wrapper for L{_ConvWrapInner}. @param convert: Conversion function receiving value as single parameter @param fn: Retrieval function """ return compat.partial(_ConvWrapInner, convert, fn) def _GetItemTimestamp(getter): """Returns function for getting timestamp of item. @type getter: callable @param getter: Function to retrieve timestamp attribute """ def fn(_, item): """Returns a timestamp of item. """ timestamp = getter(item) if timestamp is None: # Old configs might not have all timestamps return _FS_UNAVAIL else: return timestamp return fn def _GetItemTimestampFields(datatype): """Returns common timestamp fields. @param datatype: Field data type for use by L{Query.RequestedData} """ return [ (_MakeField("ctime", "CTime", QFT_TIMESTAMP, "Creation timestamp"), datatype, 0, _GetItemTimestamp(operator.attrgetter("ctime"))), (_MakeField("mtime", "MTime", QFT_TIMESTAMP, "Modification timestamp"), datatype, 0, _GetItemTimestamp(operator.attrgetter("mtime"))), ] class NodeQueryData(object): """Data container for node data queries. """ def __init__(self, nodes, live_data, master_uuid, node_to_primary, node_to_secondary, inst_uuid_to_inst_name, groups, oob_support, cluster): """Initializes this class. """ self.nodes = nodes self.live_data = live_data self.master_uuid = master_uuid self.node_to_primary = node_to_primary self.node_to_secondary = node_to_secondary self.inst_uuid_to_inst_name = inst_uuid_to_inst_name self.groups = groups self.oob_support = oob_support self.cluster = cluster # Used for individual rows self.curlive_data = None self.ndparams = None def __iter__(self): """Iterate over all nodes. This function has side-effects and only one instance of the resulting generator should be used at a time. """ for node in self.nodes: group = self.groups.get(node.group, None) if group is None: self.ndparams = None else: self.ndparams = self.cluster.FillND(node, group) if self.live_data: self.curlive_data = self.live_data.get(node.uuid, None) else: self.curlive_data = None yield node #: Fields that are direct attributes of an L{objects.Node} object _NODE_SIMPLE_FIELDS = { "drained": ("Drained", QFT_BOOL, 0, "Whether node is drained"), "master_candidate": ("MasterC", QFT_BOOL, 0, "Whether node is a master candidate"), "master_capable": ("MasterCapable", QFT_BOOL, 0, "Whether node can become a master candidate"), "name": ("Node", QFT_TEXT, QFF_HOSTNAME, "Node name"), "offline": ("Offline", QFT_BOOL, 0, "Whether node is marked offline"), "serial_no": ("SerialNo", QFT_NUMBER, 0, _SERIAL_NO_DOC % "Node"), "uuid": ("UUID", QFT_TEXT, 0, "Node UUID"), "vm_capable": ("VMCapable", QFT_BOOL, 0, "Whether node can host instances"), } #: Fields requiring talking to the node # Note that none of these are available for non-vm_capable nodes _NODE_LIVE_FIELDS = { "bootid": ("BootID", QFT_TEXT, "bootid", "Random UUID renewed for each system reboot, can be used" " for detecting reboots by tracking changes"), "cnodes": ("CNodes", QFT_NUMBER, "cpu_nodes", "Number of NUMA domains on node (if exported by hypervisor)"), "cnos": ("CNOs", QFT_NUMBER, "cpu_dom0", "Number of logical processors used by the node OS (dom0 for Xen)"), "csockets": ("CSockets", QFT_NUMBER, "cpu_sockets", "Number of physical CPU sockets (if exported by hypervisor)"), "ctotal": ("CTotal", QFT_NUMBER, "cpu_total", "Number of logical processors"), "dfree": ("DFree", QFT_UNIT, "storage_free", "Available storage space in storage unit"), "dtotal": ("DTotal", QFT_UNIT, "storage_size", "Total storage space in storage unit used for instance disk" " allocation"), "spfree": ("SpFree", QFT_NUMBER, "spindles_free", "Available spindles in volume group (exclusive storage only)"), "sptotal": ("SpTotal", QFT_NUMBER, "spindles_total", "Total spindles in volume group (exclusive storage only)"), "mfree": ("MFree", QFT_UNIT, "memory_free", "Memory available for instance allocations"), "mnode": ("MNode", QFT_UNIT, "memory_dom0", "Amount of memory used by node (dom0 for Xen)"), "mtotal": ("MTotal", QFT_UNIT, "memory_total", "Total amount of memory of physical machine"), } def _GetGroup(cb): """Build function for calling another function with an node group. @param cb: The callback to be called with the nodegroup """ def fn(ctx, node): """Get group data for a node. @type ctx: L{NodeQueryData} @type inst: L{objects.Node} @param inst: Node object """ ng = ctx.groups.get(node.group, None) if ng is None: # Nodes always have a group, or the configuration is corrupt return _FS_UNAVAIL return cb(ctx, node, ng) return fn def _GetNodeGroup(ctx, node, ng): # pylint: disable=W0613 """Returns the name of a node's group. @type ctx: L{NodeQueryData} @type node: L{objects.Node} @param node: Node object @type ng: L{objects.NodeGroup} @param ng: The node group this node belongs to """ return ng.name def _GetNodePower(ctx, node): """Returns the node powered state @type ctx: L{NodeQueryData} @type node: L{objects.Node} @param node: Node object """ if ctx.oob_support[node.uuid]: return node.powered return _FS_UNAVAIL def _GetNdParams(ctx, node, ng): """Returns the ndparams for this node. @type ctx: L{NodeQueryData} @type node: L{objects.Node} @param node: Node object @type ng: L{objects.NodeGroup} @param ng: The node group this node belongs to """ return ctx.cluster.SimpleFillND(ng.FillND(node)) def _GetLiveNodeField(field, kind, ctx, node): """Gets the value of a "live" field from L{NodeQueryData}. @param field: Live field name @param kind: Data kind, one of L{constants.QFT_ALL} @type ctx: L{NodeQueryData} @type node: L{objects.Node} @param node: Node object """ if node.offline: return _FS_OFFLINE if not node.vm_capable: return _FS_UNAVAIL if not ctx.curlive_data: return _FS_NODATA return _GetStatsField(field, kind, ctx.curlive_data) def _GetStatsField(field, kind, data): """Gets a value from live statistics. If the value is not found, L{_FS_UNAVAIL} is returned. If the field kind is numeric a conversion to integer is attempted. If that fails, L{_FS_UNAVAIL} is returned. @param field: Live field name @param kind: Data kind, one of L{constants.QFT_ALL} @type data: dict @param data: Statistics """ try: value = data[field] except KeyError: return _FS_UNAVAIL if kind == QFT_TEXT: return value assert kind in (QFT_NUMBER, QFT_UNIT) # Try to convert into number try: return int(value) except (ValueError, TypeError): logging.exception("Failed to convert node field '%s' (value %r) to int", field, value) return _FS_UNAVAIL def _GetNodeHvState(_, node): """Converts node's hypervisor state for query result. """ hv_state = node.hv_state if hv_state is None: return _FS_UNAVAIL return dict((name, value.ToDict()) for (name, value) in hv_state.items()) def _GetNodeDiskState(_, node): """Converts node's disk state for query result. """ disk_state = node.disk_state if disk_state is None: return _FS_UNAVAIL return dict((disk_kind, dict((name, value.ToDict()) for (name, value) in kind_state.items())) for (disk_kind, kind_state) in disk_state.items()) def _BuildNodeFields(): """Builds list of fields for node queries. """ fields = [ (_MakeField("pip", "PrimaryIP", QFT_TEXT, "Primary IP address"), NQ_CONFIG, 0, _GetItemAttr("primary_ip")), (_MakeField("sip", "SecondaryIP", QFT_TEXT, "Secondary IP address"), NQ_CONFIG, 0, _GetItemAttr("secondary_ip")), (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), NQ_CONFIG, 0, lambda ctx, node: list(node.GetTags())), (_MakeField("master", "IsMaster", QFT_BOOL, "Whether node is master"), NQ_CONFIG, 0, lambda ctx, node: node.uuid == ctx.master_uuid), (_MakeField("group", "Group", QFT_TEXT, "Node group"), NQ_GROUP, 0, _GetGroup(_GetNodeGroup)), (_MakeField("group.uuid", "GroupUUID", QFT_TEXT, "UUID of node group"), NQ_CONFIG, 0, _GetItemAttr("group")), (_MakeField("powered", "Powered", QFT_BOOL, "Whether node is thought to be powered on"), NQ_OOB, 0, _GetNodePower), (_MakeField("ndparams", "NodeParameters", QFT_OTHER, "Merged node parameters"), NQ_GROUP, 0, _GetGroup(_GetNdParams)), (_MakeField("custom_ndparams", "CustomNodeParameters", QFT_OTHER, "Custom node parameters"), NQ_GROUP, 0, _GetItemAttr("ndparams")), (_MakeField("hv_state", "HypervisorState", QFT_OTHER, "Hypervisor state"), NQ_CONFIG, 0, _GetNodeHvState), (_MakeField("disk_state", "DiskState", QFT_OTHER, "Disk state"), NQ_CONFIG, 0, _GetNodeDiskState), ] fields.extend(_BuildNDFields(False)) # Node role role_values = (constants.NR_MASTER, constants.NR_MCANDIDATE, constants.NR_REGULAR, constants.NR_DRAINED, constants.NR_OFFLINE) role_doc = ("Node role; \"%s\" for master, \"%s\" for master candidate," " \"%s\" for regular, \"%s\" for drained, \"%s\" for offline" % role_values) fields.append((_MakeField("role", "Role", QFT_TEXT, role_doc), NQ_CONFIG, 0, lambda ctx, node: _GetNodeRole(node, ctx.master_uuid))) assert set(role_values) == constants.NR_ALL def _GetLength(getter): return lambda ctx, node: len(getter(ctx)[node.uuid]) def _GetList(getter): return lambda ctx, node: utils.NiceSort( [ctx.inst_uuid_to_inst_name[uuid] for uuid in getter(ctx)[node.uuid]]) # Add fields operating on instance lists for prefix, titleprefix, docword, getter in \ [("p", "Pri", "primary", operator.attrgetter("node_to_primary")), ("s", "Sec", "secondary", operator.attrgetter("node_to_secondary"))]: # TODO: Allow filterting by hostname in list fields.extend([ (_MakeField("%sinst_cnt" % prefix, "%sinst" % prefix.upper(), QFT_NUMBER, "Number of instances with this node as %s" % docword), NQ_INST, 0, _GetLength(getter)), (_MakeField("%sinst_list" % prefix, "%sInstances" % titleprefix, QFT_OTHER, "List of instances with this node as %s" % docword), NQ_INST, 0, _GetList(getter)), ]) # Add simple fields fields.extend([ (_MakeField(name, title, kind, doc), NQ_CONFIG, flags, _GetItemAttr(name)) for (name, (title, kind, flags, doc)) in _NODE_SIMPLE_FIELDS.items()]) # Add fields requiring live data fields.extend([ (_MakeField(name, title, kind, doc), NQ_LIVE, 0, compat.partial(_GetLiveNodeField, nfield, kind)) for (name, (title, kind, nfield, doc)) in _NODE_LIVE_FIELDS.items()]) # Add timestamps fields.extend(_GetItemTimestampFields(NQ_CONFIG)) return _PrepareFieldList(fields, []) class InstanceQueryData(object): """Data container for instance data queries. """ def __init__(self, instances, cluster, disk_usage, offline_node_uuids, bad_node_uuids, live_data, wrongnode_inst, console, nodes, groups, networks): """Initializes this class. @param instances: List of instance objects @param cluster: Cluster object @type disk_usage: dict; instance UUID as key @param disk_usage: Per-instance disk usage @type offline_node_uuids: list of strings @param offline_node_uuids: List of offline nodes @type bad_node_uuids: list of strings @param bad_node_uuids: List of faulty nodes @type live_data: dict; instance UUID as key @param live_data: Per-instance live data @type wrongnode_inst: set @param wrongnode_inst: Set of instances running on wrong node(s) @type console: dict; instance UUID as key @param console: Per-instance console information @type nodes: dict; node UUID as key @param nodes: Node objects @type networks: dict; net_uuid as key @param networks: Network objects """ assert len(set(bad_node_uuids) & set(offline_node_uuids)) == \ len(offline_node_uuids), \ "Offline nodes not included in bad nodes" assert not (set(live_data.keys()) & set(bad_node_uuids)), \ "Found live data for bad or offline nodes" self.instances = instances self.cluster = cluster self.disk_usage = disk_usage self.offline_nodes = offline_node_uuids self.bad_nodes = bad_node_uuids self.live_data = live_data self.wrongnode_inst = wrongnode_inst self.console = console self.nodes = nodes self.groups = groups self.networks = networks # Used for individual rows self.inst_hvparams = None self.inst_beparams = None self.inst_osparams = None self.inst_nicparams = None def __iter__(self): """Iterate over all instances. This function has side-effects and only one instance of the resulting generator should be used at a time. """ for inst in self.instances: self.inst_hvparams = self.cluster.FillHV(inst, skip_globals=True) self.inst_beparams = self.cluster.FillBE(inst) self.inst_osparams = self.cluster.SimpleFillOS(inst.os, inst.osparams) self.inst_nicparams = [self.cluster.SimpleFillNIC(nic.nicparams) for nic in inst.nics] yield inst def _GetInstOperState(ctx, inst): """Get instance's operational status. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ # Can't use RS_OFFLINE here as it would describe the instance to # be offline when we actually don't know due to missing data if inst.primary_node in ctx.bad_nodes: return _FS_NODATA else: return bool(ctx.live_data.get(inst.uuid)) def _GetInstLiveData(name): """Build function for retrieving live data. @type name: string @param name: Live data field name """ def fn(ctx, inst): """Get live data for an instance. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ if (inst.primary_node in ctx.bad_nodes or inst.primary_node in ctx.offline_nodes): # Can't use RS_OFFLINE here as it would describe the instance to be # offline when we actually don't know due to missing data return _FS_NODATA if inst.uuid in ctx.live_data: data = ctx.live_data[inst.uuid] if name in data: return data[name] return _FS_UNAVAIL return fn def _GetLiveInstStatus(ctx, instance, instance_state): hvparams = ctx.cluster.FillHV(instance, skip_globals=True) allow_userdown = \ ctx.cluster.enabled_user_shutdown and \ (instance.hypervisor != constants.HT_KVM or hvparams[constants.HV_KVM_USER_SHUTDOWN]) if instance.uuid in ctx.wrongnode_inst: return constants.INSTST_WRONGNODE else: if hv_base.HvInstanceState.IsShutdown(instance_state): if instance.admin_state == constants.ADMINST_UP and allow_userdown: return constants.INSTST_USERDOWN elif instance.admin_state == constants.ADMINST_UP: return constants.INSTST_ERRORDOWN else: return constants.INSTST_ADMINDOWN else: if instance.admin_state == constants.ADMINST_UP: return constants.INSTST_RUNNING else: return constants.INSTST_ERRORUP def _GetDeadInstStatus(inst): if inst.admin_state == constants.ADMINST_UP: return constants.INSTST_ERRORDOWN elif inst.admin_state == constants.ADMINST_DOWN: if inst.admin_state_source == constants.USER_SOURCE: return constants.INSTST_USERDOWN else: return constants.INSTST_ADMINDOWN else: return constants.INSTST_ADMINOFFLINE def _GetInstStatus(ctx, inst): """Get instance status. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ if inst.primary_node in ctx.offline_nodes: return constants.INSTST_NODEOFFLINE if inst.primary_node in ctx.bad_nodes: return constants.INSTST_NODEDOWN instance_live_data = ctx.live_data.get(inst.uuid) if bool(instance_live_data): return _GetLiveInstStatus(ctx, inst, instance_live_data["state"]) else: return _GetDeadInstStatus(inst) def _GetInstDisk(index, cb): """Build function for calling another function with an instance Disk. @type index: int @param index: Disk index @type cb: callable @param cb: Callback """ def fn(ctx, inst): """Call helper function with instance Disk. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ try: nic = inst.disks[index] except IndexError: return _FS_UNAVAIL return cb(ctx, index, nic) return fn def _GetInstDiskSize(ctx, _, disk): # pylint: disable=W0613 """Get a Disk's size. @type ctx: L{InstanceQueryData} @type disk: L{objects.Disk} @param disk: The Disk object """ if disk.size is None: return _FS_UNAVAIL else: return disk.size def _GetInstDiskSpindles(ctx, _, disk): # pylint: disable=W0613 """Get a Disk's spindles. @type disk: L{objects.Disk} @param disk: The Disk object """ if disk.spindles is None: return _FS_UNAVAIL else: return disk.spindles def _GetInstDeviceName(ctx, _, device): # pylint: disable=W0613 """Get a Device's Name. @type ctx: L{InstanceQueryData} @type device: L{objects.NIC} or L{objects.Disk} @param device: The NIC or Disk object """ if device.name is None: return _FS_UNAVAIL else: return device.name def _GetInstDeviceUUID(ctx, _, device): # pylint: disable=W0613 """Get a Device's UUID. @type ctx: L{InstanceQueryData} @type device: L{objects.NIC} or L{objects.Disk} @param device: The NIC or Disk object """ if device.uuid is None: return _FS_UNAVAIL else: return device.uuid def _GetInstNic(index, cb): """Build function for calling another function with an instance NIC. @type index: int @param index: NIC index @type cb: callable @param cb: Callback """ def fn(ctx, inst): """Call helper function with instance NIC. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ try: nic = inst.nics[index] except IndexError: return _FS_UNAVAIL return cb(ctx, index, nic) return fn def _GetInstNicNetworkName(ctx, _, nic): # pylint: disable=W0613 """Get a NIC's Network. @type ctx: L{InstanceQueryData} @type nic: L{objects.NIC} @param nic: NIC object """ if nic.network is None: return _FS_UNAVAIL else: return ctx.networks[nic.network].name def _GetInstNicNetwork(ctx, _, nic): # pylint: disable=W0613 """Get a NIC's Network. @type ctx: L{InstanceQueryData} @type nic: L{objects.NIC} @param nic: NIC object """ if nic.network is None: return _FS_UNAVAIL else: return nic.network def _GetInstNicIp(ctx, _, nic): # pylint: disable=W0613 """Get a NIC's IP address. @type ctx: L{InstanceQueryData} @type nic: L{objects.NIC} @param nic: NIC object """ if nic.ip is None: return _FS_UNAVAIL else: return nic.ip def _GetInstNicBridge(ctx, index, _): """Get a NIC's bridge. @type ctx: L{InstanceQueryData} @type index: int @param index: NIC index """ assert len(ctx.inst_nicparams) >= index nicparams = ctx.inst_nicparams[index] if nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: return nicparams[constants.NIC_LINK] else: return _FS_UNAVAIL def _GetInstNicVLan(ctx, index, _): """Get a NIC's VLAN. @type ctx: L{InstanceQueryData} @type index: int @param index: NIC index """ assert len(ctx.inst_nicparams) >= index nicparams = ctx.inst_nicparams[index] if nicparams[constants.NIC_MODE] == constants.NIC_MODE_OVS: return nicparams[constants.NIC_VLAN] else: return _FS_UNAVAIL def _GetInstAllNicNetworkNames(ctx, inst): """Get all network names for an instance. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ result = [] for nic in inst.nics: name = None if nic.network: name = ctx.networks[nic.network].name result.append(name) assert len(result) == len(inst.nics) return result def _GetInstAllNicVlans(ctx, inst): """Get all network VLANs for an instance. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ assert len(ctx.inst_nicparams) == len(inst.nics) result = [] for nicp in ctx.inst_nicparams: if nicp[constants.NIC_MODE] in \ [constants.NIC_MODE_BRIDGED, constants.NIC_MODE_OVS]: result.append(nicp[constants.NIC_VLAN]) else: result.append(None) assert len(result) == len(inst.nics) return result def _GetInstAllNicBridges(ctx, inst): """Get all network bridges for an instance. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ assert len(ctx.inst_nicparams) == len(inst.nics) result = [] for nicp in ctx.inst_nicparams: if nicp[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: result.append(nicp[constants.NIC_LINK]) else: result.append(None) assert len(result) == len(inst.nics) return result def _GetInstNicParam(name): """Build function for retrieving a NIC parameter. @type name: string @param name: Parameter name """ def fn(ctx, index, _): """Get a NIC's bridge. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object @type nic: L{objects.NIC} @param nic: NIC object """ assert len(ctx.inst_nicparams) >= index return ctx.inst_nicparams[index][name] return fn def _GetInstanceNetworkFields(): """Get instance fields involving network interfaces. @return: Tuple containing list of field definitions used as input for L{_PrepareFieldList} and a list of aliases """ nic_mac_fn = lambda ctx, _, nic: nic.mac nic_mode_fn = _GetInstNicParam(constants.NIC_MODE) nic_link_fn = _GetInstNicParam(constants.NIC_LINK) fields = [ # All NICs (_MakeField("nic.count", "NICs", QFT_NUMBER, "Number of network interfaces"), IQ_CONFIG, 0, lambda ctx, inst: len(inst.nics)), (_MakeField("nic.macs", "NIC_MACs", QFT_OTHER, "List containing each network interface's MAC address"), IQ_CONFIG, 0, lambda ctx, inst: [nic.mac for nic in inst.nics]), (_MakeField("nic.ips", "NIC_IPs", QFT_OTHER, "List containing each network interface's IP address"), IQ_CONFIG, 0, lambda ctx, inst: [nic.ip for nic in inst.nics]), (_MakeField("nic.names", "NIC_Names", QFT_OTHER, "List containing each network interface's name"), IQ_CONFIG, 0, lambda ctx, inst: [nic.name for nic in inst.nics]), (_MakeField("nic.uuids", "NIC_UUIDs", QFT_OTHER, "List containing each network interface's UUID"), IQ_CONFIG, 0, lambda ctx, inst: [nic.uuid for nic in inst.nics]), (_MakeField("nic.modes", "NIC_modes", QFT_OTHER, "List containing each network interface's mode"), IQ_CONFIG, 0, lambda ctx, inst: [nicp[constants.NIC_MODE] for nicp in ctx.inst_nicparams]), (_MakeField("nic.links", "NIC_links", QFT_OTHER, "List containing each network interface's link"), IQ_CONFIG, 0, lambda ctx, inst: [nicp[constants.NIC_LINK] for nicp in ctx.inst_nicparams]), (_MakeField("nic.vlans", "NIC_VLANs", QFT_OTHER, "List containing each network interface's VLAN"), IQ_CONFIG, 0, _GetInstAllNicVlans), (_MakeField("nic.bridges", "NIC_bridges", QFT_OTHER, "List containing each network interface's bridge"), IQ_CONFIG, 0, _GetInstAllNicBridges), (_MakeField("nic.networks", "NIC_networks", QFT_OTHER, "List containing each interface's network"), IQ_CONFIG, 0, lambda ctx, inst: [nic.network for nic in inst.nics]), (_MakeField("nic.networks.names", "NIC_networks_names", QFT_OTHER, "List containing each interface's network"), IQ_NETWORKS, 0, _GetInstAllNicNetworkNames) ] # NICs by number for i in range(constants.MAX_NICS): numtext = utils.FormatOrdinal(i + 1) fields.extend([ (_MakeField("nic.ip/%s" % i, "NicIP/%s" % i, QFT_TEXT, "IP address of %s network interface" % numtext), IQ_CONFIG, 0, _GetInstNic(i, _GetInstNicIp)), (_MakeField("nic.mac/%s" % i, "NicMAC/%s" % i, QFT_TEXT, "MAC address of %s network interface" % numtext), IQ_CONFIG, 0, _GetInstNic(i, nic_mac_fn)), (_MakeField("nic.name/%s" % i, "NicName/%s" % i, QFT_TEXT, "Name address of %s network interface" % numtext), IQ_CONFIG, 0, _GetInstNic(i, _GetInstDeviceName)), (_MakeField("nic.uuid/%s" % i, "NicUUID/%s" % i, QFT_TEXT, "UUID address of %s network interface" % numtext), IQ_CONFIG, 0, _GetInstNic(i, _GetInstDeviceUUID)), (_MakeField("nic.mode/%s" % i, "NicMode/%s" % i, QFT_TEXT, "Mode of %s network interface" % numtext), IQ_CONFIG, 0, _GetInstNic(i, nic_mode_fn)), (_MakeField("nic.link/%s" % i, "NicLink/%s" % i, QFT_TEXT, "Link of %s network interface" % numtext), IQ_CONFIG, 0, _GetInstNic(i, nic_link_fn)), (_MakeField("nic.bridge/%s" % i, "NicBridge/%s" % i, QFT_TEXT, "Bridge of %s network interface" % numtext), IQ_CONFIG, 0, _GetInstNic(i, _GetInstNicBridge)), (_MakeField("nic.vlan/%s" % i, "NicVLAN/%s" % i, QFT_TEXT, "VLAN of %s network interface" % numtext), IQ_CONFIG, 0, _GetInstNic(i, _GetInstNicVLan)), (_MakeField("nic.network/%s" % i, "NicNetwork/%s" % i, QFT_TEXT, "Network of %s network interface" % numtext), IQ_CONFIG, 0, _GetInstNic(i, _GetInstNicNetwork)), (_MakeField("nic.network.name/%s" % i, "NicNetworkName/%s" % i, QFT_TEXT, "Network name of %s network interface" % numtext), IQ_NETWORKS, 0, _GetInstNic(i, _GetInstNicNetworkName)), ]) aliases = [ # Legacy fields for first NIC ("ip", "nic.ip/0"), ("mac", "nic.mac/0"), ("bridge", "nic.bridge/0"), ("nic_mode", "nic.mode/0"), ("nic_link", "nic.link/0"), ("nic_network", "nic.network/0"), ] return (fields, aliases) def _GetInstDiskUsage(ctx, inst): """Get disk usage for an instance. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ usage = ctx.disk_usage[inst.uuid] if usage is None: usage = 0 return usage def _GetInstanceConsole(ctx, inst): """Get console information for instance. @type ctx: L{InstanceQueryData} @type inst: L{objects.Instance} @param inst: Instance object """ consinfo = ctx.console[inst.uuid] if consinfo is None: return _FS_UNAVAIL return consinfo def _GetInstanceDiskFields(): """Get instance fields involving disks. @return: List of field definitions used as input for L{_PrepareFieldList} """ fields = [ (_MakeField("disk_usage", "DiskUsage", QFT_UNIT, "Total disk space used by instance on each of its nodes;" " this is not the disk size visible to the instance, but" " the usage on the node"), IQ_DISKUSAGE, 0, _GetInstDiskUsage), (_MakeField("disk.count", "Disks", QFT_NUMBER, "Number of disks"), IQ_CONFIG, 0, lambda ctx, inst: len(inst.disks)), (_MakeField("disk.sizes", "Disk_sizes", QFT_OTHER, "List of disk sizes"), IQ_CONFIG, 0, lambda ctx, inst: [disk.size for disk in inst.disks]), (_MakeField("disk.spindles", "Disk_spindles", QFT_OTHER, "List of disk spindles"), IQ_CONFIG, 0, lambda ctx, inst: [disk.spindles for disk in inst.disks]), (_MakeField("disk.names", "Disk_names", QFT_OTHER, "List of disk names"), IQ_CONFIG, 0, lambda ctx, inst: [disk.name for disk in inst.disks]), (_MakeField("disk.uuids", "Disk_UUIDs", QFT_OTHER, "List of disk UUIDs"), IQ_CONFIG, 0, lambda ctx, inst: [disk.uuid for disk in inst.disks]), ] # Disks by number for i in range(constants.MAX_DISKS): numtext = utils.FormatOrdinal(i + 1) fields.extend([ (_MakeField("disk.size/%s" % i, "Disk/%s" % i, QFT_UNIT, "Disk size of %s disk" % numtext), IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDiskSize)), (_MakeField("disk.spindles/%s" % i, "DiskSpindles/%s" % i, QFT_NUMBER, "Spindles of %s disk" % numtext), IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDiskSpindles)), (_MakeField("disk.name/%s" % i, "DiskName/%s" % i, QFT_TEXT, "Name of %s disk" % numtext), IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDeviceName)), (_MakeField("disk.uuid/%s" % i, "DiskUUID/%s" % i, QFT_TEXT, "UUID of %s disk" % numtext), IQ_CONFIG, 0, _GetInstDisk(i, _GetInstDeviceUUID))]) return fields def _GetInstanceParameterFields(): """Get instance fields involving parameters. @return: List of field definitions used as input for L{_PrepareFieldList} """ fields = [ # Filled parameters (_MakeField("hvparams", "HypervisorParameters", QFT_OTHER, "Hypervisor parameters (merged)"), IQ_CONFIG, 0, lambda ctx, _: ctx.inst_hvparams), (_MakeField("beparams", "BackendParameters", QFT_OTHER, "Backend parameters (merged)"), IQ_CONFIG, 0, lambda ctx, _: ctx.inst_beparams), (_MakeField("osparams", "OpSysParameters", QFT_OTHER, "Operating system parameters (merged)"), IQ_CONFIG, 0, lambda ctx, _: ctx.inst_osparams), # Unfilled parameters (_MakeField("custom_hvparams", "CustomHypervisorParameters", QFT_OTHER, "Custom hypervisor parameters"), IQ_CONFIG, 0, _GetItemAttr("hvparams")), (_MakeField("custom_beparams", "CustomBackendParameters", QFT_OTHER, "Custom backend parameters",), IQ_CONFIG, 0, _GetItemAttr("beparams")), (_MakeField("custom_osparams", "CustomOpSysParameters", QFT_OTHER, "Custom operating system parameters",), IQ_CONFIG, 0, _GetItemAttr("osparams")), (_MakeField("custom_nicparams", "CustomNicParameters", QFT_OTHER, "Custom network interface parameters"), IQ_CONFIG, 0, lambda ctx, inst: [nic.nicparams for nic in inst.nics]), ] # HV params def _GetInstHvParam(name): return lambda ctx, _: ctx.inst_hvparams.get(name, _FS_UNAVAIL) fields.extend([ (_MakeField("hv/%s" % name, constants.HVS_PARAMETER_TITLES.get(name, "hv/%s" % name), _VTToQFT[kind], "The \"%s\" hypervisor parameter" % name), IQ_CONFIG, 0, _GetInstHvParam(name)) for name, kind in constants.HVS_PARAMETER_TYPES.items() if name not in constants.HVC_GLOBALS]) # BE params def _GetInstBeParam(name): return lambda ctx, _: ctx.inst_beparams.get(name, None) fields.extend([ (_MakeField("be/%s" % name, constants.BES_PARAMETER_TITLES.get(name, "be/%s" % name), _VTToQFT[kind], "The \"%s\" backend parameter" % name), IQ_CONFIG, 0, _GetInstBeParam(name)) for name, kind in constants.BES_PARAMETER_TYPES.items()]) return fields _INST_SIMPLE_FIELDS = { "disk_template": ("Disk_template", QFT_TEXT, 0, "Instance disk template"), "hypervisor": ("Hypervisor", QFT_TEXT, 0, "Hypervisor name"), "name": ("Instance", QFT_TEXT, QFF_HOSTNAME, "Instance name"), # Depending on the hypervisor, the port can be None "network_port": ("Network_port", QFT_OTHER, 0, "Instance network port if available (e.g. for VNC console)"), "os": ("OS", QFT_TEXT, 0, "Operating system"), "serial_no": ("SerialNo", QFT_NUMBER, 0, _SERIAL_NO_DOC % "Instance"), "uuid": ("UUID", QFT_TEXT, 0, "Instance UUID"), } def _GetNodeName(ctx, default, node_uuid): """Gets node name of a node. @type ctx: L{InstanceQueryData} @param default: Default value @type node_uuid: string @param node_uuid: Node UUID """ try: node = ctx.nodes[node_uuid] except KeyError: return default else: return node.name def _GetInstNodeGroup(ctx, default, node_uuid): """Gets group UUID of an instance node. @type ctx: L{InstanceQueryData} @param default: Default value @type node_uuid: string @param node_uuid: Node UUID """ try: node = ctx.nodes[node_uuid] except KeyError: return default else: return node.group def _GetInstNodeGroupName(ctx, default, node_uuid): """Gets group name of an instance node. @type ctx: L{InstanceQueryData} @param default: Default value @type node_uuid: string @param node_uuid: Node UUID """ try: node = ctx.nodes[node_uuid] except KeyError: return default try: group = ctx.groups[node.group] except KeyError: return default return group.name def _BuildInstanceFields(): """Builds list of fields for instance queries. """ fields = [ (_MakeField("pnode", "Primary_node", QFT_TEXT, "Primary node"), IQ_NODES, QFF_HOSTNAME, lambda ctx, inst: _GetNodeName(ctx, None, inst.primary_node)), (_MakeField("pnode.group", "PrimaryNodeGroup", QFT_TEXT, "Primary node's group"), IQ_NODES, 0, lambda ctx, inst: _GetInstNodeGroupName(ctx, _FS_UNAVAIL, inst.primary_node)), (_MakeField("pnode.group.uuid", "PrimaryNodeGroupUUID", QFT_TEXT, "Primary node's group UUID"), IQ_NODES, 0, lambda ctx, inst: _GetInstNodeGroup(ctx, _FS_UNAVAIL, inst.primary_node)), # TODO: Allow filtering by secondary node as hostname (_MakeField("snodes", "Secondary_Nodes", QFT_OTHER, "Secondary nodes; usually this will just be one node"), IQ_NODES, 0, lambda ctx, inst: [ _GetNodeName(ctx, None, uuid) for uuid in inst.secondary_nodes ]), (_MakeField("snodes.group", "SecondaryNodesGroups", QFT_OTHER, "Node groups of secondary nodes"), IQ_NODES, 0, lambda ctx, inst: [ _GetInstNodeGroupName(ctx, None, uuid) for uuid in inst.secondary_nodes ]), (_MakeField("snodes.group.uuid", "SecondaryNodesGroupsUUID", QFT_OTHER, "Node group UUIDs of secondary nodes"), IQ_NODES, 0, lambda ctx, inst: [ _GetInstNodeGroup(ctx, None, uuid) for uuid in inst.secondary_nodes ]), (_MakeField("admin_state", "InstanceState", QFT_TEXT, "Desired state of the instance"), IQ_CONFIG, 0, _GetItemAttr("admin_state")), (_MakeField("admin_up", "Autostart", QFT_BOOL, "Desired state of the instance"), IQ_CONFIG, 0, lambda ctx, inst: inst.admin_state == constants.ADMINST_UP), (_MakeField("admin_state_source", "InstanceStateSource", QFT_TEXT, "Who last changed the desired state of the instance"), IQ_CONFIG, 0, _GetItemAttr("admin_state_source")), (_MakeField("disks_active", "DisksActive", QFT_BOOL, "Desired state of the instance disks"), IQ_CONFIG, 0, _GetItemAttr("disks_active")), (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), IQ_CONFIG, 0, lambda ctx, inst: list(inst.GetTags())), (_MakeField("console", "Console", QFT_OTHER, "Instance console information"), IQ_CONSOLE, 0, _GetInstanceConsole), (_MakeField("forthcoming", "Forthcoming", QFT_BOOL, "Whether the Instance is forthcoming"), IQ_CONFIG, 0, lambda _, inst: bool(inst.forthcoming)), ] # Add simple fields fields.extend([ (_MakeField(name, title, kind, doc), IQ_CONFIG, flags, _GetItemAttr(name)) for (name, (title, kind, flags, doc)) in _INST_SIMPLE_FIELDS.items()]) # Fields requiring talking to the node fields.extend([ (_MakeField("oper_state", "Running", QFT_BOOL, "Actual state of instance"), IQ_LIVE, 0, _GetInstOperState), (_MakeField("oper_ram", "Memory", QFT_UNIT, "Actual memory usage as seen by hypervisor"), IQ_LIVE, 0, _GetInstLiveData("memory")), (_MakeField("oper_vcpus", "VCPUs", QFT_NUMBER, "Actual number of VCPUs as seen by hypervisor"), IQ_LIVE, 0, _GetInstLiveData("vcpus")), ]) # Status field status_values = (constants.INSTST_RUNNING, constants.INSTST_ADMINDOWN, constants.INSTST_WRONGNODE, constants.INSTST_ERRORUP, constants.INSTST_ERRORDOWN, constants.INSTST_NODEDOWN, constants.INSTST_NODEOFFLINE, constants.INSTST_ADMINOFFLINE, constants.INSTST_USERDOWN) status_doc = ("Instance status; \"%s\" if instance is set to be running" " and actually is, \"%s\" if instance is stopped and" " is not running, \"%s\" if instance running, but not on its" " designated primary node, \"%s\" if instance should be" " stopped, but is actually running, \"%s\" if instance should" " run, but doesn't, \"%s\" if instance's primary node is down," " \"%s\" if instance's primary node is marked offline," " \"%s\" if instance is offline and does not use dynamic," " \"%s\" if the user shutdown the instance" " resources" % status_values) fields.append((_MakeField("status", "Status", QFT_TEXT, status_doc), IQ_LIVE, 0, _GetInstStatus)) assert set(status_values) == constants.INSTST_ALL, \ "Status documentation mismatch" (network_fields, network_aliases) = _GetInstanceNetworkFields() fields.extend(network_fields) fields.extend(_GetInstanceParameterFields()) fields.extend(_GetInstanceDiskFields()) fields.extend(_GetItemTimestampFields(IQ_CONFIG)) aliases = [ ("vcpus", "be/vcpus"), ("be/memory", "be/maxmem"), ("sda_size", "disk.size/0"), ("sdb_size", "disk.size/1"), ] + network_aliases return _PrepareFieldList(fields, aliases) class LockQueryData(object): """Data container for lock data queries. """ def __init__(self, lockdata): """Initializes this class. """ self.lockdata = lockdata def __iter__(self): """Iterate over all locks. """ return iter(self.lockdata) def _GetLockOwners(_, data): """Returns a sorted list of a lock's current owners. """ (_, _, owners, _) = data if owners: owners = utils.NiceSort(owners) return owners def _GetLockPending(_, data): """Returns a sorted list of a lock's pending acquires. """ (_, _, _, pending) = data if pending: pending = [(mode, utils.NiceSort(names)) for (mode, names) in pending] return pending def _BuildLockFields(): """Builds list of fields for lock queries. """ return _PrepareFieldList([ # TODO: Lock names are not always hostnames. Should QFF_HOSTNAME be used? (_MakeField("name", "Name", QFT_TEXT, "Lock name"), None, 0, lambda ctx, lock_info: lock_info[0]), (_MakeField("mode", "Mode", QFT_OTHER, "Mode in which the lock is currently acquired" " (exclusive or shared)"), LQ_MODE, 0, lambda ctx, lock_info: lock_info[1]), (_MakeField("owner", "Owner", QFT_OTHER, "Current lock owner(s)"), LQ_OWNER, 0, _GetLockOwners), (_MakeField("pending", "Pending", QFT_OTHER, "Threads waiting for the lock"), LQ_PENDING, 0, _GetLockPending), ], []) class GroupQueryData(object): """Data container for node group data queries. """ def __init__(self, cluster, groups, group_to_nodes, group_to_instances, want_diskparams): """Initializes this class. @param cluster: Cluster object @param groups: List of node group objects @type group_to_nodes: dict; group UUID as key @param group_to_nodes: Per-group list of nodes @type group_to_instances: dict; group UUID as key @param group_to_instances: Per-group list of (primary) instances @type want_diskparams: bool @param want_diskparams: Whether diskparamters should be calculated """ self.groups = groups self.group_to_nodes = group_to_nodes self.group_to_instances = group_to_instances self.cluster = cluster self.want_diskparams = want_diskparams # Used for individual rows self.group_ipolicy = None self.ndparams = None self.group_dp = None def __iter__(self): """Iterate over all node groups. This function has side-effects and only one instance of the resulting generator should be used at a time. """ for group in self.groups: self.group_ipolicy = self.cluster.SimpleFillIPolicy(group.ipolicy) self.ndparams = self.cluster.SimpleFillND(group.ndparams) if self.want_diskparams: self.group_dp = self.cluster.SimpleFillDP(group.diskparams) else: self.group_dp = None yield group _GROUP_SIMPLE_FIELDS = { "alloc_policy": ("AllocPolicy", QFT_TEXT, "Allocation policy for group"), "name": ("Group", QFT_TEXT, "Group name"), "serial_no": ("SerialNo", QFT_NUMBER, _SERIAL_NO_DOC % "Group"), "uuid": ("UUID", QFT_TEXT, "Group UUID"), } def _BuildGroupFields(): """Builds list of fields for node group queries. """ # Add simple fields fields = [(_MakeField(name, title, kind, doc), GQ_CONFIG, 0, _GetItemAttr(name)) for (name, (title, kind, doc)) in _GROUP_SIMPLE_FIELDS.items()] def _GetLength(getter): return lambda ctx, group: len(getter(ctx)[group.uuid]) def _GetSortedList(getter): return lambda ctx, group: utils.NiceSort(getter(ctx)[group.uuid]) group_to_nodes = operator.attrgetter("group_to_nodes") group_to_instances = operator.attrgetter("group_to_instances") # Add fields for nodes fields.extend([ (_MakeField("node_cnt", "Nodes", QFT_NUMBER, "Number of nodes"), GQ_NODE, 0, _GetLength(group_to_nodes)), (_MakeField("node_list", "NodeList", QFT_OTHER, "List of nodes"), GQ_NODE, 0, _GetSortedList(group_to_nodes)), ]) # Add fields for instances fields.extend([ (_MakeField("pinst_cnt", "Instances", QFT_NUMBER, "Number of primary instances"), GQ_INST, 0, _GetLength(group_to_instances)), (_MakeField("pinst_list", "InstanceList", QFT_OTHER, "List of primary instances"), GQ_INST, 0, _GetSortedList(group_to_instances)), ]) # Other fields fields.extend([ (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), GQ_CONFIG, 0, lambda ctx, group: list(group.GetTags())), (_MakeField("ipolicy", "InstancePolicy", QFT_OTHER, "Instance policy limitations (merged)"), GQ_CONFIG, 0, lambda ctx, _: ctx.group_ipolicy), (_MakeField("custom_ipolicy", "CustomInstancePolicy", QFT_OTHER, "Custom instance policy limitations"), GQ_CONFIG, 0, _GetItemAttr("ipolicy")), (_MakeField("custom_ndparams", "CustomNDParams", QFT_OTHER, "Custom node parameters"), GQ_CONFIG, 0, _GetItemAttr("ndparams")), (_MakeField("ndparams", "NDParams", QFT_OTHER, "Node parameters"), GQ_CONFIG, 0, lambda ctx, _: ctx.ndparams), (_MakeField("diskparams", "DiskParameters", QFT_OTHER, "Disk parameters (merged)"), GQ_DISKPARAMS, 0, lambda ctx, _: ctx.group_dp), (_MakeField("custom_diskparams", "CustomDiskParameters", QFT_OTHER, "Custom disk parameters"), GQ_CONFIG, 0, _GetItemAttr("diskparams")), ]) # ND parameters fields.extend(_BuildNDFields(True)) fields.extend(_GetItemTimestampFields(GQ_CONFIG)) return _PrepareFieldList(fields, []) class OsInfo(objects.ConfigObject): __slots__ = [ "name", "valid", "hidden", "blacklisted", "variants", "api_versions", "parameters", "node_status", "os_hvp", "osparams", "trusted" ] def _BuildOsFields(): """Builds list of fields for operating system queries. """ fields = [ (_MakeField("name", "Name", QFT_TEXT, "Operating system name"), None, 0, _GetItemAttr("name")), (_MakeField("valid", "Valid", QFT_BOOL, "Whether operating system definition is valid"), None, 0, _GetItemAttr("valid")), (_MakeField("hidden", "Hidden", QFT_BOOL, "Whether operating system is hidden"), None, 0, _GetItemAttr("hidden")), (_MakeField("blacklisted", "Blacklisted", QFT_BOOL, "Whether operating system is blacklisted"), None, 0, _GetItemAttr("blacklisted")), (_MakeField("variants", "Variants", QFT_OTHER, "Operating system variants"), None, 0, _ConvWrap(utils.NiceSort, _GetItemAttr("variants"))), (_MakeField("api_versions", "ApiVersions", QFT_OTHER, "Operating system API versions"), None, 0, _ConvWrap(sorted, _GetItemAttr("api_versions"))), (_MakeField("parameters", "Parameters", QFT_OTHER, "Operating system parameters"), None, 0, _ConvWrap(compat.partial(utils.NiceSort, key=compat.fst), _GetItemAttr("parameters"))), (_MakeField("node_status", "NodeStatus", QFT_OTHER, "Status from node"), None, 0, _GetItemAttr("node_status")), (_MakeField("os_hvp", "OsHypervisorParams", QFT_OTHER, "Operating system specific hypervisor parameters"), None, 0, _GetItemAttr("os_hvp")), (_MakeField("osparams", "OsParameters", QFT_OTHER, "Operating system specific parameters"), None, 0, _GetItemAttr("osparams")), (_MakeField("trusted", "Trusted", QFT_BOOL, "Whether this OS is trusted"), None, 0, _GetItemAttr("trusted")), ] return _PrepareFieldList(fields, []) class ExtStorageInfo(objects.ConfigObject): __slots__ = [ "name", "node_status", "nodegroup_status", "parameters", ] def _BuildExtStorageFields(): """Builds list of fields for extstorage provider queries. """ fields = [ (_MakeField("name", "Name", QFT_TEXT, "ExtStorage provider name"), None, 0, _GetItemAttr("name")), (_MakeField("node_status", "NodeStatus", QFT_OTHER, "Status from node"), None, 0, _GetItemAttr("node_status")), (_MakeField("nodegroup_status", "NodegroupStatus", QFT_OTHER, "Overall Nodegroup status"), None, 0, _GetItemAttr("nodegroup_status")), (_MakeField("parameters", "Parameters", QFT_OTHER, "ExtStorage provider parameters"), None, 0, _GetItemAttr("parameters")), ] return _PrepareFieldList(fields, []) def _JobUnavailInner(fn, ctx, jid_job): # pylint: disable=W0613 """Return L{_FS_UNAVAIL} if job is None. When listing specifc jobs (e.g. "gnt-job list 1 2 3"), a job may not be found, in which case this function converts it to L{_FS_UNAVAIL}. """ job = jid_job[1] if job is None: return _FS_UNAVAIL else: return fn(job) def _JobUnavail(inner): """Wrapper for L{_JobUnavailInner}. """ return compat.partial(_JobUnavailInner, inner) def _PerJobOpInner(fn, job): """Executes a function per opcode in a job. """ return [fn(op) for op in job.ops] def _PerJobOp(fn): """Wrapper for L{_PerJobOpInner}. """ return _JobUnavail(compat.partial(_PerJobOpInner, fn)) def _JobTimestampInner(fn, job): """Converts unavailable timestamp to L{_FS_UNAVAIL}. """ timestamp = fn(job) if timestamp is None: return _FS_UNAVAIL else: return timestamp def _JobTimestamp(fn): """Wrapper for L{_JobTimestampInner}. """ return _JobUnavail(compat.partial(_JobTimestampInner, fn)) def _BuildJobFields(): """Builds list of fields for job queries. """ fields = [ (_MakeField("id", "ID", QFT_NUMBER, "Job ID"), None, QFF_JOB_ID, lambda _, jid_job: jid_job[0]), (_MakeField("status", "Status", QFT_TEXT, "Job status"), None, 0, _JobUnavail(lambda job: job.CalcStatus())), (_MakeField("priority", "Priority", QFT_NUMBER, ("Current job priority (%s to %s)" % (constants.OP_PRIO_LOWEST, constants.OP_PRIO_HIGHEST))), None, 0, _JobUnavail(lambda job: job.CalcPriority())), (_MakeField("archived", "Archived", QFT_BOOL, "Whether job is archived"), JQ_ARCHIVED, 0, lambda _, jid_job: jid_job[1].archived), (_MakeField("ops", "OpCodes", QFT_OTHER, "List of all opcodes"), None, 0, _PerJobOp(lambda op: op.input.__getstate__())), (_MakeField("opresult", "OpCode_result", QFT_OTHER, "List of opcodes results"), None, 0, _PerJobOp(operator.attrgetter("result"))), (_MakeField("opstatus", "OpCode_status", QFT_OTHER, "List of opcodes status"), None, 0, _PerJobOp(operator.attrgetter("status"))), (_MakeField("oplog", "OpCode_log", QFT_OTHER, "List of opcode output logs"), None, 0, _PerJobOp(operator.attrgetter("log"))), (_MakeField("opstart", "OpCode_start", QFT_OTHER, "List of opcode start timestamps (before acquiring locks)"), None, 0, _PerJobOp(operator.attrgetter("start_timestamp"))), (_MakeField("opexec", "OpCode_exec", QFT_OTHER, "List of opcode execution start timestamps (after acquiring" " locks)"), None, 0, _PerJobOp(operator.attrgetter("exec_timestamp"))), (_MakeField("opend", "OpCode_end", QFT_OTHER, "List of opcode execution end timestamps"), None, 0, _PerJobOp(operator.attrgetter("end_timestamp"))), (_MakeField("oppriority", "OpCode_prio", QFT_OTHER, "List of opcode priorities"), None, 0, _PerJobOp(operator.attrgetter("priority"))), (_MakeField("summary", "Summary", QFT_OTHER, "List of per-opcode summaries"), None, 0, _PerJobOp(lambda op: op.input.Summary())), ] # Timestamp fields for (name, attr, title, desc) in [ ("received_ts", "received_timestamp", "Received", "Timestamp of when job was received"), ("start_ts", "start_timestamp", "Start", "Timestamp of job start"), ("end_ts", "end_timestamp", "End", "Timestamp of job end"), ]: getter = operator.attrgetter(attr) fields.extend([ (_MakeField(name, title, QFT_OTHER, "%s (tuple containing seconds and microseconds)" % desc), None, QFF_SPLIT_TIMESTAMP, _JobTimestamp(getter)), ]) return _PrepareFieldList(fields, []) def _GetExportName(_, node_export_name): # pylint: disable=W0613 """Returns an export name if available. """ expname = node_export_name[1] if expname is None: return _FS_NODATA else: return expname def _BuildExportFields(): """Builds list of fields for exports. """ fields = [ (_MakeField("node", "Node", QFT_TEXT, "Node name"), None, QFF_HOSTNAME, lambda _, node_expname: node_expname[0]), (_MakeField("export", "Export", QFT_TEXT, "Export name"), None, 0, _GetExportName), ] return _PrepareFieldList(fields, []) _CLUSTER_VERSION_FIELDS = { "software_version": ("SoftwareVersion", QFT_TEXT, constants.RELEASE_VERSION, "Software version"), "protocol_version": ("ProtocolVersion", QFT_NUMBER, constants.PROTOCOL_VERSION, "RPC protocol version"), "config_version": ("ConfigVersion", QFT_NUMBER, constants.CONFIG_VERSION, "Configuration format version"), "os_api_version": ("OsApiVersion", QFT_NUMBER, max(constants.OS_API_VERSIONS), "API version for OS template scripts"), "export_version": ("ExportVersion", QFT_NUMBER, constants.EXPORT_VERSION, "Import/export file format version"), "vcs_version": ("VCSVersion", QFT_TEXT, constants.VCS_VERSION, "VCS version"), } _CLUSTER_SIMPLE_FIELDS = { "cluster_name": ("Name", QFT_TEXT, QFF_HOSTNAME, "Cluster name"), "volume_group_name": ("VgName", QFT_TEXT, 0, "LVM volume group name"), } class ClusterQueryData(object): def __init__(self, cluster, nodes, drain_flag, watcher_pause): """Initializes this class. @type cluster: L{objects.Cluster} @param cluster: Instance of cluster object @type nodes: dict; node UUID as key @param nodes: Node objects @type drain_flag: bool @param drain_flag: Whether job queue is drained @type watcher_pause: number @param watcher_pause: Until when watcher is paused (Unix timestamp) """ self._cluster = cluster self.nodes = nodes self.drain_flag = drain_flag self.watcher_pause = watcher_pause def __iter__(self): return iter([self._cluster]) def _ClusterWatcherPause(ctx, _): """Returns until when watcher is paused (if available). """ if ctx.watcher_pause is None: return _FS_UNAVAIL else: return ctx.watcher_pause def _BuildClusterFields(): """Builds list of fields for cluster information. """ fields = [ (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), CQ_CONFIG, 0, lambda ctx, cluster: list(cluster.GetTags())), (_MakeField("architecture", "ArchInfo", QFT_OTHER, "Architecture information"), None, 0, lambda ctx, _: runtime.GetArchInfo()), (_MakeField("drain_flag", "QueueDrained", QFT_BOOL, "Flag whether job queue is drained"), CQ_QUEUE_DRAINED, 0, lambda ctx, _: ctx.drain_flag), (_MakeField("watcher_pause", "WatcherPause", QFT_TIMESTAMP, "Until when watcher is paused"), CQ_WATCHER_PAUSE, 0, _ClusterWatcherPause), (_MakeField("master_node", "Master", QFT_TEXT, "Master node name"), CQ_CONFIG, QFF_HOSTNAME, lambda ctx, cluster: _GetNodeName(ctx, None, cluster.master_node)), ] # Simple fields fields.extend([ (_MakeField(name, title, kind, doc), CQ_CONFIG, flags, _GetItemAttr(name)) for (name, (title, kind, flags, doc)) in _CLUSTER_SIMPLE_FIELDS.items() ],) # Version fields fields.extend([ (_MakeField(name, title, kind, doc), None, 0, _StaticValue(value)) for (name, (title, kind, value, doc)) in _CLUSTER_VERSION_FIELDS.items()]) # Add timestamps fields.extend(_GetItemTimestampFields(CQ_CONFIG)) return _PrepareFieldList(fields, [ ("name", "cluster_name")]) class NetworkQueryData(object): """Data container for network data queries. """ def __init__(self, networks, network_to_groups, network_to_instances, stats): """Initializes this class. @param networks: List of network objects @type network_to_groups: dict; network UUID as key @param network_to_groups: Per-network list of groups @type network_to_instances: dict; network UUID as key @param network_to_instances: Per-network list of instances @type stats: dict; network UUID as key @param stats: Per-network usage statistics """ self.networks = networks self.network_to_groups = network_to_groups self.network_to_instances = network_to_instances self.stats = stats def __iter__(self): """Iterate over all networks. """ for net in self.networks: if self.stats: self.curstats = self.stats.get(net.uuid, None) else: self.curstats = None yield net _NETWORK_SIMPLE_FIELDS = { "name": ("Network", QFT_TEXT, 0, "Name"), "network": ("Subnet", QFT_TEXT, 0, "IPv4 subnet"), "gateway": ("Gateway", QFT_OTHER, 0, "IPv4 gateway"), "network6": ("IPv6Subnet", QFT_OTHER, 0, "IPv6 subnet"), "gateway6": ("IPv6Gateway", QFT_OTHER, 0, "IPv6 gateway"), "mac_prefix": ("MacPrefix", QFT_OTHER, 0, "MAC address prefix"), "serial_no": ("SerialNo", QFT_NUMBER, 0, _SERIAL_NO_DOC % "Network"), "uuid": ("UUID", QFT_TEXT, 0, "Network UUID"), } _NETWORK_STATS_FIELDS = { "free_count": ("FreeCount", QFT_NUMBER, 0, "Number of available addresses"), "reserved_count": ("ReservedCount", QFT_NUMBER, 0, "Number of reserved addresses"), "map": ("Map", QFT_TEXT, 0, "Actual mapping"), "external_reservations": ("ExternalReservations", QFT_TEXT, 0, "External reservations"), } def _GetNetworkStatsField(field, kind, ctx, _): """Gets the value of a "stats" field from L{NetworkQueryData}. @param field: Field name @param kind: Data kind, one of L{constants.QFT_ALL} @type ctx: L{NetworkQueryData} """ return _GetStatsField(field, kind, ctx.curstats) def _BuildNetworkFields(): """Builds list of fields for network queries. """ fields = [ (_MakeField("tags", "Tags", QFT_OTHER, "Tags"), IQ_CONFIG, 0, lambda ctx, inst: list(inst.GetTags())), ] # Add simple fields fields.extend([ (_MakeField(name, title, kind, doc), NETQ_CONFIG, 0, _GetItemMaybeAttr(name)) for (name, (title, kind, _, doc)) in _NETWORK_SIMPLE_FIELDS.items()]) def _GetLength(getter): return lambda ctx, network: len(getter(ctx)[network.uuid]) def _GetSortedList(getter): return lambda ctx, network: utils.NiceSort(getter(ctx)[network.uuid]) network_to_groups = operator.attrgetter("network_to_groups") network_to_instances = operator.attrgetter("network_to_instances") # Add fields for node groups fields.extend([ (_MakeField("group_cnt", "NodeGroups", QFT_NUMBER, "Number of nodegroups"), NETQ_GROUP, 0, _GetLength(network_to_groups)), (_MakeField("group_list", "GroupList", QFT_OTHER, "List of nodegroups (group name, NIC mode, NIC link)"), NETQ_GROUP, 0, lambda ctx, network: network_to_groups(ctx)[network.uuid]), ]) # Add fields for instances fields.extend([ (_MakeField("inst_cnt", "Instances", QFT_NUMBER, "Number of instances"), NETQ_INST, 0, _GetLength(network_to_instances)), (_MakeField("inst_list", "InstanceList", QFT_OTHER, "List of instances"), NETQ_INST, 0, _GetSortedList(network_to_instances)), ]) # Add fields for usage statistics fields.extend([ (_MakeField(name, title, kind, doc), NETQ_STATS, 0, compat.partial(_GetNetworkStatsField, name, kind)) for (name, (title, kind, _, doc)) in _NETWORK_STATS_FIELDS.items()]) # Add timestamps fields.extend(_GetItemTimestampFields(IQ_NETWORKS)) return _PrepareFieldList(fields, []) _FILTER_SIMPLE_FIELDS = { "watermark": ("Watermark", QFT_NUMBER, 0, "Watermark"), "priority": ("Priority", QFT_NUMBER, 0, "Priority"), "predicates": ("Predicates", QFT_OTHER, 0, "Predicates"), "action": ("Action", QFT_OTHER, 0, "Action"), "reason_trail": ("ReasonTrail", QFT_OTHER, 0, "Reason trail"), "uuid": ("UUID", QFT_TEXT, 0, "Network UUID"), } def _BuildFilterFields(): """Builds list of fields for job filter queries. """ fields = [ (_MakeField(name, title, kind, doc), None, 0, _GetItemMaybeAttr(name)) for (name, (title, kind, _, doc)) in _FILTER_SIMPLE_FIELDS.items() ] return _PrepareFieldList(fields, []) #: Fields for cluster information CLUSTER_FIELDS = _BuildClusterFields() #: Fields available for node queries NODE_FIELDS = _BuildNodeFields() #: Fields available for instance queries INSTANCE_FIELDS = _BuildInstanceFields() #: Fields available for lock queries LOCK_FIELDS = _BuildLockFields() #: Fields available for node group queries GROUP_FIELDS = _BuildGroupFields() #: Fields available for operating system queries OS_FIELDS = _BuildOsFields() #: Fields available for extstorage provider queries EXTSTORAGE_FIELDS = _BuildExtStorageFields() #: Fields available for job queries JOB_FIELDS = _BuildJobFields() #: Fields available for exports EXPORT_FIELDS = _BuildExportFields() #: Fields available for network queries NETWORK_FIELDS = _BuildNetworkFields() #: Fields available for job filter queries FILTER_FIELDS = _BuildFilterFields() #: All available resources ALL_FIELDS = { constants.QR_CLUSTER: CLUSTER_FIELDS, constants.QR_INSTANCE: INSTANCE_FIELDS, constants.QR_NODE: NODE_FIELDS, constants.QR_LOCK: LOCK_FIELDS, constants.QR_GROUP: GROUP_FIELDS, constants.QR_OS: OS_FIELDS, constants.QR_EXTSTORAGE: EXTSTORAGE_FIELDS, constants.QR_JOB: JOB_FIELDS, constants.QR_EXPORT: EXPORT_FIELDS, constants.QR_NETWORK: NETWORK_FIELDS, constants.QR_FILTER: FILTER_FIELDS, } #: All available field lists ALL_FIELD_LISTS = list(ALL_FIELDS.values()) ganeti-3.1.0~rc2/lib/rapi/000075500000000000000000000000001476477700300153055ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/rapi/__init__.py000064400000000000000000000030171476477700300174170ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Ganeti RAPI module""" from ganeti import compat RAPI_ACCESS_WRITE = "write" RAPI_ACCESS_READ = "read" RAPI_ACCESS_ALL = compat.UniqueFrozenset([ RAPI_ACCESS_WRITE, RAPI_ACCESS_READ, ]) ganeti-3.1.0~rc2/lib/rapi/baserlib.py000064400000000000000000000551571476477700300174570ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Remote API base resources library. """ # pylint: disable=C0103 # C0103: Invalid name, since the R_* names are not conforming import logging from ganeti import luxi import ganeti.rpc.errors as rpcerr from ganeti import rapi from ganeti import http from ganeti import errors from ganeti import compat from ganeti import constants from ganeti import utils # Dummy value to detect unchanged parameters _DEFAULT = object() #: Supported HTTP methods _SUPPORTED_METHODS = compat.UniqueFrozenset([ http.HTTP_DELETE, http.HTTP_GET, http.HTTP_POST, http.HTTP_PUT, ]) class OpcodeAttributes(object): """Acts as a structure containing the per-method attribute names. """ __slots__ = [ "method", "opcode", "rename", "aliases", "forbidden", "get_input", ] def __init__(self, method_name): """Initializes the opcode attributes for the given method name. """ self.method = method_name self.opcode = "%s_OPCODE" % method_name self.rename = "%s_RENAME" % method_name self.aliases = "%s_ALIASES" % method_name self.forbidden = "%s_FORBIDDEN" % method_name self.get_input = "Get%sOpInput" % method_name.capitalize() def GetModifiers(self): """Returns the names of all the attributes that replace or modify a method. """ return [self.opcode, self.rename, self.aliases, self.forbidden, self.get_input] def GetAll(self): return [self.method] + self.GetModifiers() def _BuildOpcodeAttributes(): """Builds list of attributes used for per-handler opcodes. """ return [OpcodeAttributes(method) for method in _SUPPORTED_METHODS] OPCODE_ATTRS = _BuildOpcodeAttributes() def BuildUriList(ids, uri_format, uri_fields=("name", "uri")): """Builds a URI list as used by index resources. @param ids: list of ids as strings @param uri_format: format to be applied for URI @param uri_fields: optional parameter for field IDs """ (field_id, field_uri) = uri_fields def _MapId(m_id): return { field_id: m_id, field_uri: uri_format % m_id, } # Make sure the result is sorted, makes it nicer to look at and simplifies # unittests. ids.sort() return [_MapId(id) for id in ids] def MapFields(names, data): """Maps two lists into one dictionary. Example:: >>> MapFields(["a", "b"], ["foo", 123]) {'a': 'foo', 'b': 123} @param names: field names (list of strings) @param data: field data (list) """ if len(names) != len(data): raise AttributeError("Names and data must have the same length") return dict(zip(names, data)) def MapBulkFields(itemslist, fields): """Map value to field name in to one dictionary. @param itemslist: a list of items values @param fields: a list of items names @return: a list of mapped dictionaries """ items_details = [] for item in itemslist: mapped = MapFields(fields, item) items_details.append(mapped) return items_details def FillOpcode(opcls, body, static, rename=None): """Fills an opcode with body parameters. Parameter types are checked. @type opcls: L{opcodes.OpCode} @param opcls: Opcode class @type body: dict @param body: Body parameters as received from client @type static: dict @param static: Static parameters which can't be modified by client @type rename: dict @param rename: Renamed parameters, key as old name, value as new name @return: Opcode object """ if body is None: params = {} else: CheckType(body, dict, "Body contents") # Make copy to be modified params = body.copy() if rename: for old, new in rename.items(): if new in params and old in params: raise http.HttpBadRequest("Parameter '%s' was renamed to '%s', but" " both are specified" % (old, new)) if old in params: assert new not in params params[new] = params.pop(old) if static: overwritten = set(params.keys()) & set(static.keys()) if overwritten: raise http.HttpBadRequest("Can't overwrite static parameters %r" % overwritten) params.update(static) # Convert keys to strings (simplejson decodes them as unicode) params = dict((str(key), value) for (key, value) in params.items()) try: op = opcls(**params) op.Validate(False) except (errors.OpPrereqError, TypeError) as err: raise http.HttpBadRequest("Invalid body parameters: %s" % err) return op def HandleItemQueryErrors(fn, *args, **kwargs): """Converts errors when querying a single item. """ try: result = fn(*args, **kwargs) except errors.OpPrereqError as err: if len(err.args) == 2 and err.args[1] == errors.ECODE_NOENT: raise http.HttpNotFound() raise # In case split query mechanism is used if not result: raise http.HttpNotFound() return result def FeedbackFn(msg): """Feedback logging function for jobs. We don't have a stdout for printing log messages, so log them to the http log at least. @param msg: the message """ (_, log_type, log_msg) = msg logging.info("%s: %s", log_type, log_msg) def CheckType(value, exptype, descr): """Abort request if value type doesn't match expected type. @param value: Value @type exptype: type @param exptype: Expected type @type descr: string @param descr: Description of value @return: Value (allows inline usage) """ if not isinstance(value, exptype): raise http.HttpBadRequest("%s: Type is '%s', but '%s' is expected" % (descr, type(value).__name__, exptype.__name__)) return value def CheckParameter(data, name, default=_DEFAULT, exptype=_DEFAULT): """Check and return the value for a given parameter. If no default value was given and the parameter doesn't exist in the input data, an error is raise. @type data: dict @param data: Dictionary containing input data @type name: string @param name: Parameter name @param default: Default value (can be None) @param exptype: Expected type (can be None) """ try: value = data[name] except KeyError: if default is not _DEFAULT: return default raise http.HttpBadRequest("Required parameter '%s' is missing" % name) if exptype is _DEFAULT: return value return CheckType(value, exptype, "'%s' parameter" % name) class ResourceBase(object): """Generic class for resources. """ # Default permission requirements GET_ACCESS = [] PUT_ACCESS = [rapi.RAPI_ACCESS_WRITE] POST_ACCESS = [rapi.RAPI_ACCESS_WRITE] DELETE_ACCESS = [rapi.RAPI_ACCESS_WRITE] def __init__(self, items, queryargs, req, _client_cls=None): """Generic resource constructor. @param items: a list with variables encoded in the URL @param queryargs: a dictionary with additional options from URL @param req: Request context @param _client_cls: L{luxi} client class (unittests only) """ assert isinstance(queryargs, dict) self.items = items self.queryargs = queryargs self._req = req if _client_cls is None: _client_cls = luxi.Client self._client_cls = _client_cls def _GetRequestBody(self): """Returns the body data. """ return self._req.private.body_data request_body = property(fget=_GetRequestBody) def _checkIntVariable(self, name, default=0): """Return the parsed value of an int argument. """ val = self.queryargs.get(name, default) if isinstance(val, list): if val: val = val[0] else: val = default try: val = int(val) except (ValueError, TypeError): raise http.HttpBadRequest("Invalid value for the" " '%s' parameter" % (name,)) return val def _checkStringVariable(self, name, default=None): """Return the parsed value of a string argument. """ val = self.queryargs.get(name, default) if isinstance(val, list): if val: val = val[0] else: val = default return val def getBodyParameter(self, name, *args): """Check and return the value for a given parameter. If a second parameter is not given, an error will be returned, otherwise this parameter specifies the default value. @param name: the required parameter """ if args: return CheckParameter(self.request_body, name, default=args[0]) return CheckParameter(self.request_body, name) def useLocking(self): """Check if the request specifies locking. """ return bool(self._checkIntVariable("lock")) def useBulk(self): """Check if the request specifies bulk querying. """ return bool(self._checkIntVariable("bulk")) def useForce(self): """Check if the request specifies a forced operation. """ return bool(self._checkIntVariable("force")) def dryRun(self): """Check if the request specifies dry-run mode. """ return bool(self._checkIntVariable("dry-run")) def GetClient(self): """Wrapper for L{luxi.Client} with HTTP-specific error handling. """ # Could be a function, pylint: disable=R0201 try: return self._client_cls() except rpcerr.NoMasterError as err: raise http.HttpBadGateway("Can't connect to master daemon: %s" % err) except rpcerr.PermissionError: raise http.HttpInternalServerError("Internal error: no permission to" " connect to the master daemon") def SubmitJob(self, op, cl=None): """Generic wrapper for submit job, for better http compatibility. @type op: list @param op: the list of opcodes for the job @type cl: None or luxi.Client @param cl: optional luxi client to use @rtype: string @return: the job ID """ if cl is None: cl = self.GetClient() try: return cl.SubmitJob(op) except errors.JobQueueFull: raise http.HttpServiceUnavailable("Job queue is full, needs archiving") except errors.JobQueueDrainError: raise http.HttpServiceUnavailable("Job queue is drained, cannot submit") except rpcerr.NoMasterError as err: raise http.HttpBadGateway("Master seems to be unreachable: %s" % err) except rpcerr.PermissionError: raise http.HttpInternalServerError("Internal error: no permission to" " connect to the master daemon") except rpcerr.TimeoutError as err: raise http.HttpGatewayTimeout("Timeout while talking to the master" " daemon: %s" % err) def GetResourceOpcodes(cls): """Returns all opcodes used by a resource. """ return frozenset(opcode for opcode in (getattr(cls, method_attrs.opcode, None) for method_attrs in OPCODE_ATTRS) if opcode) def GetHandlerAccess(handler, method): """Returns the access rights for a method on a handler. @type handler: L{ResourceBase} @type method: string @rtype: string or None """ return getattr(handler, "%s_ACCESS" % method, None) def GetHandler(get_fn, aliases): result = get_fn() if not isinstance(result, dict) or aliases is None: return result for (param, alias) in aliases.items(): if param in result: if alias in result: raise http.HttpBadRequest("Parameter '%s' has an alias of '%s', but" " both values are present in response" % (param, alias)) result[alias] = result[param] return result # Constant used to denote that a parameter cannot be set ALL_VALUES_FORBIDDEN = "all_values_forbidden" def ProduceForbiddenParamDict(class_name, method_name, param_list): """Turns a list of parameter names and possibly values into a dictionary. @type class_name: string @param class_name: The name of the handler class @type method_name: string @param method_name: The name of the HTTP method @type param_list: list of string or tuple of (string, list of any) @param param_list: A list of forbidden parameters, specified in the RAPI handler class @return: The dictionary of forbidden param names to values or ALL_VALUES_FORBIDDEN """ # A simple error-raising function def _RaiseError(message): raise errors.ProgrammerError( "While examining the %s_FORBIDDEN field of class %s: %s" % (method_name, class_name, message) ) param_dict = {} for value in param_list: if isinstance(value, str): param_dict[value] = ALL_VALUES_FORBIDDEN elif isinstance(value, tuple): if len(value) != 2: _RaiseError("Tuples of only length 2 allowed") param_name, forbidden_values = value param_dict[param_name] = forbidden_values else: _RaiseError("Only strings or tuples allowed, found %s" % value) return param_dict def InspectParams(params_dict, forbidden_params, rename_dict): """Inspects a dictionary of params, looking for forbidden values. @type params_dict: dict of string to anything @param params_dict: A dictionary of supplied parameters @type forbidden_params: dict of string to string or list of any @param forbidden_params: The forbidden parameters, with a list of forbidden values or the constant ALL_VALUES_FORBIDDEN signifying that all values are forbidden @type rename_dict: None or dict of string to string @param rename_dict: The list of parameter renamings used by the method @raise http.HttpForbidden: If a forbidden param has been set """ for param in params_dict: # Check for possible renames to ensure nothing slips through if rename_dict is not None and param in rename_dict: param = rename_dict[param] # Now see if there are restrictions on this parameter if param in forbidden_params: forbidden_values = forbidden_params[param] if forbidden_values == ALL_VALUES_FORBIDDEN: raise http.HttpForbidden("The parameter %s cannot be set via RAPI" % param) param_value = params_dict[param] if param_value in forbidden_values: raise http.HttpForbidden("The parameter %s cannot be set to the value" " %s via RAPI" % (param, param_value)) class _MetaOpcodeResource(type): """Meta class for RAPI resources. """ def __call__(mcs, *args, **kwargs): """Instantiates class and patches it for use by the RAPI daemon. """ # Access to private attributes of a client class, pylint: disable=W0212 obj = type.__call__(mcs, *args, **kwargs) for m_attrs in OPCODE_ATTRS: method, op_attr, rename_attr, aliases_attr, _, fn_attr = m_attrs.GetAll() if hasattr(obj, method): # If the method handler is already defined, "*_RENAME" or # "Get*OpInput" shouldn't be (they're only used by the automatically # generated handler) assert not hasattr(obj, rename_attr) assert not hasattr(obj, fn_attr) # The aliases are allowed only on GET calls assert not hasattr(obj, aliases_attr) or method == http.HTTP_GET # GET methods can add aliases of values they return under a different # name if method == http.HTTP_GET and hasattr(obj, aliases_attr): setattr(obj, method, compat.partial(GetHandler, getattr(obj, method), getattr(obj, aliases_attr))) else: # Try to generate handler method on handler instance try: opcode = getattr(obj, op_attr) except AttributeError: pass else: setattr(obj, method, compat.partial(obj._GenericHandler, opcode, getattr(obj, rename_attr, None), getattr(obj, fn_attr, obj._GetDefaultData))) # Finally, the method (generated or not) should be wrapped to handle # forbidden values if hasattr(obj, m_attrs.forbidden): forbidden_dict = ProduceForbiddenParamDict( obj.__class__.__name__, method, getattr(obj, m_attrs.forbidden) ) setattr( obj, method, compat.partial(obj._ForbiddenHandler, getattr(obj, method), forbidden_dict, getattr(obj, m_attrs.rename, None)) ) return obj class OpcodeResource(ResourceBase, metaclass=_MetaOpcodeResource): """Base class for opcode-based RAPI resources. Instances of this class automatically gain handler functions through L{_MetaOpcodeResource} for any method for which a C{$METHOD$_OPCODE} variable is defined at class level. Subclasses can define a C{Get$Method$OpInput} method to do their own opcode input processing (e.g. for static values). The C{$METHOD$_RENAME} variable defines which values are renamed (see L{baserlib.FillOpcode}). Still default behavior cannot be totally overriden. There are opcode params that are available to all opcodes, e.g. "depends". In case those params (currently only "depends") are found in the original request's body, they are added to the dictionary of parsed parameters and eventually passed to the opcode. If the parsed body is not represented as a dictionary object, the values are not added. @cvar GET_OPCODE: Set this to a class derived from L{opcodes.OpCode} to automatically generate a GET handler submitting the opcode @cvar GET_RENAME: Set this to rename parameters in the GET handler (see L{baserlib.FillOpcode}) @cvar GET_FORBIDDEN: Set this to disable listed parameters and optionally specific values from being set through the GET handler (see L{baserlib.InspectParams}) @cvar GET_ALIASES: Set this to duplicate return values in GET results (see L{baserlib.GetHandler}) @ivar GetGetOpInput: Define this to override the default method for getting opcode parameters (see L{baserlib.OpcodeResource._GetDefaultData}) @cvar PUT_OPCODE: Set this to a class derived from L{opcodes.OpCode} to automatically generate a PUT handler submitting the opcode @cvar PUT_RENAME: Set this to rename parameters in the PUT handler (see L{baserlib.FillOpcode}) @cvar PUT_FORBIDDEN: Set this to disable listed parameters and optionally specific values from being set through the PUT handler (see L{baserlib.InspectParams}) @ivar GetPutOpInput: Define this to override the default method for getting opcode parameters (see L{baserlib.OpcodeResource._GetDefaultData}) @cvar POST_OPCODE: Set this to a class derived from L{opcodes.OpCode} to automatically generate a POST handler submitting the opcode @cvar POST_RENAME: Set this to rename parameters in the POST handler (see L{baserlib.FillOpcode}) @cvar POST_FORBIDDEN: Set this to disable listed parameters and optionally specific values from being set through the POST handler (see L{baserlib.InspectParams}) @ivar GetPostOpInput: Define this to override the default method for getting opcode parameters (see L{baserlib.OpcodeResource._GetDefaultData}) @cvar DELETE_OPCODE: Set this to a class derived from L{opcodes.OpCode} to automatically generate a DELETE handler submitting the opcode @cvar DELETE_RENAME: Set this to rename parameters in the DELETE handler (see L{baserlib.FillOpcode}) @cvar DELETE_FORBIDDEN: Set this to disable listed parameters and optionally specific values from being set through the DELETE handler (see L{baserlib.InspectParams}) @ivar GetDeleteOpInput: Define this to override the default method for getting opcode parameters (see L{baserlib.OpcodeResource._GetDefaultData}) """ def _ForbiddenHandler(self, method_fn, forbidden_params, rename_dict): """Examines provided parameters for forbidden values. """ InspectParams(self.queryargs, forbidden_params, rename_dict) InspectParams(self.request_body, forbidden_params, rename_dict) return method_fn() def _GetDefaultData(self): return (self.request_body, None) def _GetRapiOpName(self): """Extracts the name of the RAPI operation from the class name """ if self.__class__.__name__.startswith("R_2_"): return self.__class__.__name__[4:] return self.__class__.__name__ def _GetCommonStatic(self): """Return the static parameters common to all the RAPI calls The reason is a parameter present in all the RAPI calls, and the reason trail has to be build for all of them, so the parameter is read here and used to build the reason trail, that is the actual parameter passed forward. """ trail = [] usr_reason = self._checkStringVariable("reason", default=None) if usr_reason: trail.append((constants.OPCODE_REASON_SRC_USER, usr_reason, utils.EpochNano())) reason_src = "%s:%s" % (constants.OPCODE_REASON_SRC_RLIB2, self._GetRapiOpName()) trail.append((reason_src, "", utils.EpochNano())) common_static = { "reason": trail, } return common_static def _GetDepends(self): ret = {} if isinstance(self.request_body, dict): depends = self.getBodyParameter("depends", None) if depends: ret.update({"depends": depends}) return ret def _GenericHandler(self, opcode, rename, fn): (body, specific_static) = fn() if isinstance(body, dict): body.update(self._GetDepends()) static = self._GetCommonStatic() if specific_static: static.update(specific_static) op = FillOpcode(opcode, body, static, rename=rename) return self.SubmitJob([op]) ganeti-3.1.0~rc2/lib/rapi/client.py000064400000000000000000002345351476477700300171510ustar00rootroot00000000000000# # # Copyright (C) 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Ganeti RAPI client. @attention: To use the RAPI client, the application B{must} call C{pycurl.global_init} during initialization and C{pycurl.global_cleanup} before exiting the process. This is very important in multi-threaded programs. See curl_global_init(3) and curl_global_cleanup(3) for details. The decorator L{UsesRapiClient} can be used. """ # No Ganeti-specific modules should be imported. The RAPI client is supposed to # be standalone. import json import logging import socket import threading import time try: from urllib import urlencode except ImportError: from urllib.parse import urlencode import pycurl from io import StringIO, BytesIO GANETI_RAPI_PORT = 5080 GANETI_RAPI_VERSION = 2 HTTP_DELETE = "DELETE" HTTP_GET = "GET" HTTP_PUT = "PUT" HTTP_POST = "POST" HTTP_OK = 200 HTTP_NOT_FOUND = 404 HTTP_APP_JSON = "application/json" REPLACE_DISK_PRI = "replace_on_primary" REPLACE_DISK_SECONDARY = "replace_on_secondary" REPLACE_DISK_CHG = "replace_new_secondary" REPLACE_DISK_AUTO = "replace_auto" NODE_EVAC_PRI = "primary-only" NODE_EVAC_SEC = "secondary-only" NODE_EVAC_ALL = "all" NODE_ROLE_DRAINED = "drained" NODE_ROLE_MASTER_CANDIDATE = "master-candidate" NODE_ROLE_MASTER = "master" NODE_ROLE_OFFLINE = "offline" NODE_ROLE_REGULAR = "regular" JOB_STATUS_QUEUED = "queued" JOB_STATUS_WAITING = "waiting" JOB_STATUS_CANCELING = "canceling" JOB_STATUS_RUNNING = "running" JOB_STATUS_CANCELED = "canceled" JOB_STATUS_SUCCESS = "success" JOB_STATUS_ERROR = "error" JOB_STATUS_PENDING = frozenset([ JOB_STATUS_QUEUED, JOB_STATUS_WAITING, JOB_STATUS_CANCELING, ]) JOB_STATUS_FINALIZED = frozenset([ JOB_STATUS_CANCELED, JOB_STATUS_SUCCESS, JOB_STATUS_ERROR, ]) JOB_STATUS_ALL = frozenset([ JOB_STATUS_RUNNING, ]) | JOB_STATUS_PENDING | JOB_STATUS_FINALIZED # Legacy name JOB_STATUS_WAITLOCK = JOB_STATUS_WAITING # Internal constants _REQ_DATA_VERSION_FIELD = "__version__" _QPARAM_DRY_RUN = "dry-run" _QPARAM_FORCE = "force" # Feature strings INST_CREATE_REQV1 = "instance-create-reqv1" INST_REINSTALL_REQV1 = "instance-reinstall-reqv1" NODE_MIGRATE_REQV1 = "node-migrate-reqv1" NODE_EVAC_RES1 = "node-evac-res1" # Old feature constant names in case they're references by users of this module _INST_CREATE_REQV1 = INST_CREATE_REQV1 _INST_REINSTALL_REQV1 = INST_REINSTALL_REQV1 _NODE_MIGRATE_REQV1 = NODE_MIGRATE_REQV1 _NODE_EVAC_RES1 = NODE_EVAC_RES1 #: Resolver errors ECODE_RESOLVER = "resolver_error" #: Not enough resources (iallocator failure, disk space, memory, etc.) ECODE_NORES = "insufficient_resources" #: Temporarily out of resources; operation can be tried again ECODE_TEMP_NORES = "temp_insufficient_resources" #: Wrong arguments (at syntax level) ECODE_INVAL = "wrong_input" #: Wrong entity state ECODE_STATE = "wrong_state" #: Entity not found ECODE_NOENT = "unknown_entity" #: Entity already exists ECODE_EXISTS = "already_exists" #: Resource not unique (e.g. MAC or IP duplication) ECODE_NOTUNIQUE = "resource_not_unique" #: Internal cluster error ECODE_FAULT = "internal_error" #: Environment error (e.g. node disk error) ECODE_ENVIRON = "environment_error" #: List of all failure types ECODE_ALL = frozenset([ ECODE_RESOLVER, ECODE_NORES, ECODE_TEMP_NORES, ECODE_INVAL, ECODE_STATE, ECODE_NOENT, ECODE_EXISTS, ECODE_NOTUNIQUE, ECODE_FAULT, ECODE_ENVIRON, ]) # Older pycURL versions don't have all error constants try: _CURLE_SSL_CACERT = pycurl.E_SSL_CACERT _CURLE_SSL_CACERT_BADFILE = pycurl.E_SSL_CACERT_BADFILE except AttributeError: _CURLE_SSL_CACERT = 60 _CURLE_SSL_CACERT_BADFILE = 77 _CURL_SSL_CERT_ERRORS = frozenset([ _CURLE_SSL_CACERT, _CURLE_SSL_CACERT_BADFILE, ]) class Error(Exception): """Base error class for this module. """ pass class GanetiApiError(Error): """Generic error raised from Ganeti API. """ def __init__(self, msg, code=None): Error.__init__(self, msg) self.code = code class CertificateError(GanetiApiError): """Raised when a problem is found with the SSL certificate. """ pass def EpochNano(): """Return the current timestamp expressed as number of nanoseconds since the unix epoch @return: nanoseconds since the Unix epoch """ return int(time.time() * 1000000000) def _AppendIf(container, condition, value): """Appends to a list if a condition evaluates to truth. """ if condition: container.append(value) return condition def _AppendDryRunIf(container, condition): """Appends a "dry-run" parameter if a condition evaluates to truth. """ return _AppendIf(container, condition, (_QPARAM_DRY_RUN, 1)) def _AppendForceIf(container, condition): """Appends a "force" parameter if a condition evaluates to truth. """ return _AppendIf(container, condition, (_QPARAM_FORCE, 1)) def _AppendReason(container, reason): """Appends an element to the reason trail. If the user provided a reason, it is added to the reason trail. """ return _AppendIf(container, reason, ("reason", reason)) def _SetItemIf(container, condition, item, value): """Sets an item if a condition evaluates to truth. """ if condition: container[item] = value return condition def UsesRapiClient(fn): """Decorator for code using RAPI client to initialize pycURL. """ def wrapper(*args, **kwargs): # curl_global_init(3) and curl_global_cleanup(3) must be called with only # one thread running. This check is just a safety measure -- it doesn't # cover all cases. assert threading.activeCount() == 1, \ "Found active threads when initializing pycURL" pycurl.global_init(pycurl.GLOBAL_ALL) try: return fn(*args, **kwargs) finally: pycurl.global_cleanup() return wrapper def GenericCurlConfig(verbose=False, use_signal=False, use_curl_cabundle=False, cafile=None, capath=None, proxy=None, verify_hostname=False, connect_timeout=None, timeout=None, _pycurl_version_fn=pycurl.version_info): """Curl configuration function generator. @type verbose: bool @param verbose: Whether to set cURL to verbose mode @type use_signal: bool @param use_signal: Whether to allow cURL to use signals @type use_curl_cabundle: bool @param use_curl_cabundle: Whether to use cURL's default CA bundle @type cafile: string @param cafile: In which file we can find the certificates @type capath: string @param capath: In which directory we can find the certificates @type proxy: string @param proxy: Proxy to use, None for default behaviour and empty string for disabling proxies (see curl_easy_setopt(3)) @type verify_hostname: bool @param verify_hostname: Whether to verify the remote peer certificate's commonName @type connect_timeout: number @param connect_timeout: Timeout for establishing connection in seconds @type timeout: number @param timeout: Timeout for complete transfer in seconds (see curl_easy_setopt(3)). """ if use_curl_cabundle and (cafile or capath): raise Error("Can not use default CA bundle when CA file or path is set") def _ConfigCurl(curl, logger): """Configures a cURL object @type curl: pycurl.Curl @param curl: cURL object """ logger.debug("Using cURL version %s", pycurl.version) # pycurl.version_info returns a tuple with information about the used # version of libcurl. Item 5 is the SSL library linked to it. # e.g.: (3, '7.18.0', 463360, 'x86_64-pc-linux-gnu', 1581, 'GnuTLS/2.0.4', # 0, '1.2.3.3', ...) sslver = _pycurl_version_fn()[5] if not sslver: raise Error("No SSL support in cURL") lcsslver = sslver.lower() if lcsslver.startswith("openssl/"): pass elif lcsslver.startswith("nss/"): # TODO: investigate compatibility beyond a simple test pass elif lcsslver.startswith("gnutls/"): if capath: raise Error("cURL linked against GnuTLS has no support for a" " CA path (%s)" % (pycurl.version, )) elif lcsslver.startswith("boringssl"): pass else: raise NotImplementedError("cURL uses unsupported SSL version '%s'" % sslver) curl.setopt(pycurl.VERBOSE, verbose) curl.setopt(pycurl.NOSIGNAL, not use_signal) # Whether to verify remote peer's CN if verify_hostname: # curl_easy_setopt(3): "When CURLOPT_SSL_VERIFYHOST is 2, that # certificate must indicate that the server is the server to which you # meant to connect, or the connection fails. [...] When the value is 1, # the certificate must contain a Common Name field, but it doesn't matter # what name it says. [...]" curl.setopt(pycurl.SSL_VERIFYHOST, 2) else: curl.setopt(pycurl.SSL_VERIFYHOST, 0) if cafile or capath or use_curl_cabundle: # Require certificates to be checked curl.setopt(pycurl.SSL_VERIFYPEER, True) if cafile: curl.setopt(pycurl.CAINFO, str(cafile)) if capath: curl.setopt(pycurl.CAPATH, str(capath)) # Not changing anything for using default CA bundle else: # Disable SSL certificate verification curl.setopt(pycurl.SSL_VERIFYPEER, False) if proxy is not None: curl.setopt(pycurl.PROXY, str(proxy)) # Timeouts if connect_timeout is not None: curl.setopt(pycurl.CONNECTTIMEOUT, connect_timeout) if timeout is not None: curl.setopt(pycurl.TIMEOUT, timeout) return _ConfigCurl class _CompatIO(object): """ Stream that lazy-allocates its buffer based on the first write's type This is a wrapper around BytesIO/StringIO that will allocate the respective internal buffer based on whether the first write is bytes or not. It is intended to be used with PycURL, which returns the response as bytes in Python 3 and string in Python 2. """ def __init__(self): self.buffer = None def write(self, data, *args, **kwargs): if self.buffer is None: self.buffer = BytesIO() if isinstance(data, bytes) else StringIO() return self.buffer.write(data, *args, **kwargs) def read(self, *args, **kwargs): return self.buffer.read(*args, **kwargs) def tell(self): if self.buffer is None: # We were never written to return 0 return self.buffer.tell() def seek(self, *args, **kwargs): return self.buffer.seek(*args, **kwargs) class GanetiRapiClient(object): # pylint: disable=R0904 """Ganeti RAPI client. """ USER_AGENT = "Ganeti RAPI Client" _json_encoder = json.JSONEncoder(sort_keys=True) def __init__(self, host, port=GANETI_RAPI_PORT, username=None, password=None, logger=logging, curl_config_fn=None, curl_factory=None): """Initializes this class. @type host: string @param host: the ganeti cluster master to interact with @type port: int @param port: the port on which the RAPI is running (default is 5080) @type username: string @param username: the username to connect with @type password: string @param password: the password to connect with @type curl_config_fn: callable @param curl_config_fn: Function to configure C{pycurl.Curl} object @param logger: Logging object """ self._username = username self._password = password self._logger = logger self._curl_config_fn = curl_config_fn self._curl_factory = curl_factory try: socket.inet_pton(socket.AF_INET6, host) address = "[%s]:%s" % (host, port) except socket.error: address = "%s:%s" % (host, port) self._base_url = "https://%s" % address if username is not None: if password is None: raise Error("Password not specified") elif password: raise Error("Specified password without username") def _CreateCurl(self): """Creates a cURL object. """ # Create pycURL object if no factory is provided if self._curl_factory: curl = self._curl_factory() else: curl = pycurl.Curl() # Default cURL settings curl.setopt(pycurl.VERBOSE, False) curl.setopt(pycurl.FOLLOWLOCATION, False) curl.setopt(pycurl.MAXREDIRS, 5) curl.setopt(pycurl.NOSIGNAL, True) curl.setopt(pycurl.USERAGENT, self.USER_AGENT) curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.setopt(pycurl.SSL_VERIFYPEER, False) curl.setopt(pycurl.HTTPHEADER, [ "Accept: %s" % HTTP_APP_JSON, "Content-type: %s" % HTTP_APP_JSON, ]) assert ((self._username is None and self._password is None) ^ (self._username is not None and self._password is not None)) if self._username: # Setup authentication curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) curl.setopt(pycurl.USERPWD, str("%s:%s" % (self._username, self._password))) # Call external configuration function if self._curl_config_fn: self._curl_config_fn(curl, self._logger) return curl @staticmethod def _EncodeQuery(query): """Encode query values for RAPI URL. @type query: list of two-tuples @param query: Query arguments @rtype: list @return: Query list with encoded values """ result = [] for name, value in query: if value is None: result.append((name, "")) elif isinstance(value, bool): # Boolean values must be encoded as 0 or 1 result.append((name, int(value))) elif isinstance(value, (list, tuple, dict)): raise ValueError("Invalid query data type %r" % type(value).__name__) else: result.append((name, value)) return result def _SendRequest(self, method, path, query, content): """Sends an HTTP request. This constructs a full URL, encodes and decodes HTTP bodies, and handles invalid responses in a pythonic way. @type method: string @param method: HTTP method to use @type path: string @param path: HTTP URL path @type query: list of two-tuples @param query: query arguments to pass to urlencode @type content: str or None @param content: HTTP body content @rtype: str @return: JSON-Decoded response @raises CertificateError: If an invalid SSL certificate is found @raises GanetiApiError: If an invalid response is returned """ assert path.startswith("/") curl = self._CreateCurl() if content is not None: encoded_content = self._json_encoder.encode(content) else: encoded_content = "" # Build URL urlparts = [self._base_url, path] if query: urlparts.append("?") urlparts.append(urlencode(self._EncodeQuery(query))) url = "".join(urlparts) self._logger.debug("Sending request %s %s (content=%r)", method, url, encoded_content) # Buffer for response encoded_resp_body = _CompatIO() # Configure cURL curl.setopt(pycurl.CUSTOMREQUEST, str(method)) curl.setopt(pycurl.URL, str(url)) curl.setopt(pycurl.POSTFIELDS, str(encoded_content)) curl.setopt(pycurl.WRITEFUNCTION, encoded_resp_body.write) try: # Send request and wait for response try: curl.perform() except pycurl.error as err: if err.args[0] in _CURL_SSL_CERT_ERRORS: raise CertificateError("SSL certificate error %s" % err, code=err.args[0]) raise GanetiApiError(str(err), code=err.args[0]) finally: # Reset settings to not keep references to large objects in memory # between requests curl.setopt(pycurl.POSTFIELDS, "") curl.setopt(pycurl.WRITEFUNCTION, lambda _: None) # Get HTTP response code http_code = curl.getinfo(pycurl.RESPONSE_CODE) # Was anything written to the response buffer? if encoded_resp_body.tell(): encoded_resp_body.seek(0) response_content = json.load(encoded_resp_body) else: response_content = None if http_code != HTTP_OK: if isinstance(response_content, dict): msg = ("%s %s: %s" % (response_content["code"], response_content["message"], response_content["explain"])) else: msg = str(response_content) raise GanetiApiError(msg, code=http_code) return response_content def GetVersion(self): """Gets the Remote API version running on the cluster. @rtype: int @return: Ganeti Remote API version """ return self._SendRequest(HTTP_GET, "/version", None, None) def GetFeatures(self): """Gets the list of optional features supported by RAPI server. @rtype: list @return: List of optional features """ try: return self._SendRequest(HTTP_GET, "/%s/features" % GANETI_RAPI_VERSION, None, None) except GanetiApiError as err: # Older RAPI servers don't support this resource if err.code == HTTP_NOT_FOUND: return [] raise def GetOperatingSystems(self, reason=None): """Gets the Operating Systems running in the Ganeti cluster. @rtype: list of str @return: operating systems @type reason: string @param reason: the reason for executing this operation """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, "/%s/os" % GANETI_RAPI_VERSION, query, None) def GetInfo(self, reason=None): """Gets info about the cluster. @type reason: string @param reason: the reason for executing this operation @rtype: dict @return: information about the cluster """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, "/%s/info" % GANETI_RAPI_VERSION, query, None) def RedistributeConfig(self, reason=None): """Tells the cluster to redistribute its configuration files. @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, "/%s/redistribute-config" % GANETI_RAPI_VERSION, query, None) def ModifyCluster(self, reason=None, **kwargs): """Modifies cluster parameters. More details for parameters can be found in the RAPI documentation. @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendReason(query, reason) body = kwargs return self._SendRequest(HTTP_PUT, "/%s/modify" % GANETI_RAPI_VERSION, query, body) def GetClusterTags(self, reason=None): """Gets the cluster tags. @type reason: string @param reason: the reason for executing this operation @rtype: list of str @return: cluster tags """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, "/%s/tags" % GANETI_RAPI_VERSION, query, None) def AddClusterTags(self, tags, dry_run=False, reason=None): """Adds tags to the cluster. @type tags: list of str @param tags: tags to add to the cluster @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, "/%s/tags" % GANETI_RAPI_VERSION, query, None) def DeleteClusterTags(self, tags, dry_run=False, reason=None): """Deletes tags from the cluster. @type tags: list of str @param tags: tags to delete @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_DELETE, "/%s/tags" % GANETI_RAPI_VERSION, query, None) def GetInstances(self, bulk=False, reason=None): """Gets information about instances on the cluster. @type bulk: bool @param bulk: whether to return all information about all instances @type reason: string @param reason: the reason for executing this operation @rtype: list of dict or list of str @return: if bulk is True, info about the instances, else a list of instances """ query = [] _AppendIf(query, bulk, ("bulk", 1)) _AppendReason(query, reason) instances = self._SendRequest(HTTP_GET, "/%s/instances" % GANETI_RAPI_VERSION, query, None) if bulk: return instances else: return [i["id"] for i in instances] def GetInstance(self, instance, reason=None): """Gets information about an instance. @type instance: str @param instance: instance whose info to return @type reason: string @param reason: the reason for executing this operation @rtype: dict @return: info about the instance """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, ("/%s/instances/%s" % (GANETI_RAPI_VERSION, instance)), query, None) def GetInstanceInfo(self, instance, static=None, reason=None): """Gets information about an instance. @type instance: string @param instance: Instance name @type reason: string @param reason: the reason for executing this operation @rtype: string @return: Job ID """ query = [] if static is not None: query.append(("static", static)) _AppendReason(query, reason) return self._SendRequest(HTTP_GET, ("/%s/instances/%s/info" % (GANETI_RAPI_VERSION, instance)), query, None) @staticmethod def _UpdateWithKwargs(base, **kwargs): """Updates the base with params from kwargs. @param base: The base dict, filled with required fields @note: This is an inplace update of base """ conflicts = set(kwargs.keys()) & set(base.keys()) if conflicts: raise GanetiApiError("Required fields can not be specified as" " keywords: %s" % ", ".join(conflicts)) base.update((key, value) for key, value in kwargs.items() if key != "dry_run") def InstanceAllocation(self, mode, name, disk_template, disks, nics, **kwargs): """Generates an instance allocation as used by multiallocate. More details for parameters can be found in the RAPI documentation. It is the same as used by CreateInstance. @type mode: string @param mode: Instance creation mode @type name: string @param name: Hostname of the instance to create @type disk_template: string @param disk_template: Disk template for instance (e.g. plain, diskless, file, or drbd) @type disks: list of dicts @param disks: List of disk definitions @type nics: list of dicts @param nics: List of NIC definitions @return: A dict with the generated entry """ # All required fields for request data version 1 alloc = { "mode": mode, "name": name, "disk_template": disk_template, "disks": disks, "nics": nics, } self._UpdateWithKwargs(alloc, **kwargs) return alloc def InstancesMultiAlloc(self, instances, reason=None, **kwargs): """Tries to allocate multiple instances. More details for parameters can be found in the RAPI documentation. @param instances: A list of L{InstanceAllocation} results """ query = [] body = { "instances": instances, } self._UpdateWithKwargs(body, **kwargs) _AppendDryRunIf(query, kwargs.get("dry_run")) _AppendReason(query, reason) return self._SendRequest(HTTP_POST, "/%s/instances-multi-alloc" % GANETI_RAPI_VERSION, query, body) def CreateInstance(self, mode, name, disk_template, disks, nics, reason=None, **kwargs): """Creates a new instance. More details for parameters can be found in the RAPI documentation. @type mode: string @param mode: Instance creation mode @type name: string @param name: Hostname of the instance to create @type disk_template: string @param disk_template: Disk template for instance (e.g. plain, diskless, file, or drbd) @type disks: list of dicts @param disks: List of disk definitions @type nics: list of dicts @param nics: List of NIC definitions @type dry_run: bool @keyword dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendDryRunIf(query, kwargs.get("dry_run")) _AppendReason(query, reason) if _INST_CREATE_REQV1 in self.GetFeatures(): body = self.InstanceAllocation(mode, name, disk_template, disks, nics, **kwargs) body[_REQ_DATA_VERSION_FIELD] = 1 else: raise GanetiApiError("Server does not support new-style (version 1)" " instance creation requests") return self._SendRequest(HTTP_POST, "/%s/instances" % GANETI_RAPI_VERSION, query, body) def DeleteInstance(self, instance, dry_run=False, reason=None, **kwargs): """Deletes an instance. @type instance: str @param instance: the instance to delete @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] body = kwargs _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_DELETE, ("/%s/instances/%s" % (GANETI_RAPI_VERSION, instance)), query, body) def ModifyInstance(self, instance, reason=None, **kwargs): """Modifies an instance. More details for parameters can be found in the RAPI documentation. @type instance: string @param instance: Instance name @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ body = kwargs query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/modify" % (GANETI_RAPI_VERSION, instance)), query, body) def ActivateInstanceDisks(self, instance, ignore_size=None, reason=None): """Activates an instance's disks. @type instance: string @param instance: Instance name @type ignore_size: bool @param ignore_size: Whether to ignore recorded size @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendIf(query, ignore_size, ("ignore_size", 1)) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/activate-disks" % (GANETI_RAPI_VERSION, instance)), query, None) def DeactivateInstanceDisks(self, instance, reason=None, force=False): """Deactivates an instance's disks. @type instance: string @param instance: Instance name @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendForceIf(query, force) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/deactivate-disks" % (GANETI_RAPI_VERSION, instance)), query, None) def RecreateInstanceDisks(self, instance, disks=None, nodes=None, reason=None, iallocator=None): """Recreate an instance's disks. @type instance: string @param instance: Instance name @type disks: list of int @param disks: List of disk indexes @type nodes: list of string @param nodes: New instance nodes, if relocation is desired @type reason: string @param reason: the reason for executing this operation @type iallocator: str or None @param iallocator: instance allocator plugin to use @rtype: string @return: job id """ body = {} _SetItemIf(body, disks is not None, "disks", disks) _SetItemIf(body, nodes is not None, "nodes", nodes) _SetItemIf(body, iallocator is not None, "iallocator", iallocator) query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_POST, ("/%s/instances/%s/recreate-disks" % (GANETI_RAPI_VERSION, instance)), query, body) def GrowInstanceDisk(self, instance, disk, amount, wait_for_sync=None, reason=None): """Grows a disk of an instance. More details for parameters can be found in the RAPI documentation. @type instance: string @param instance: Instance name @type disk: integer @param disk: Disk index @type amount: integer @param amount: Grow disk by this amount (MiB) @type wait_for_sync: bool @param wait_for_sync: Wait for disk to synchronize @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ body = { "amount": amount, } _SetItemIf(body, wait_for_sync is not None, "wait_for_sync", wait_for_sync) query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_POST, ("/%s/instances/%s/disk/%s/grow" % (GANETI_RAPI_VERSION, instance, disk)), query, body) def GetInstanceTags(self, instance, reason=None): """Gets tags for an instance. @type instance: str @param instance: instance whose tags to return @type reason: string @param reason: the reason for executing this operation @rtype: list of str @return: tags for the instance """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, ("/%s/instances/%s/tags" % (GANETI_RAPI_VERSION, instance)), query, None) def AddInstanceTags(self, instance, tags, dry_run=False, reason=None): """Adds tags to an instance. @type instance: str @param instance: instance to add tags to @type tags: list of str @param tags: tags to add to the instance @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/tags" % (GANETI_RAPI_VERSION, instance)), query, None) def DeleteInstanceTags(self, instance, tags, dry_run=False, reason=None): """Deletes tags from an instance. @type instance: str @param instance: instance to delete tags from @type tags: list of str @param tags: tags to delete @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_DELETE, ("/%s/instances/%s/tags" % (GANETI_RAPI_VERSION, instance)), query, None) def RebootInstance(self, instance, reboot_type=None, ignore_secondaries=None, dry_run=False, reason=None, **kwargs): """Reboots an instance. @type instance: str @param instance: instance to reboot @type reboot_type: str @param reboot_type: one of: hard, soft, full @type ignore_secondaries: bool @param ignore_secondaries: if True, ignores errors for the secondary node while re-assembling disks (in hard-reboot mode only) @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for the reboot @rtype: string @return: job id """ query = [] body = kwargs _AppendDryRunIf(query, dry_run) _AppendIf(query, reboot_type, ("type", reboot_type)) _AppendIf(query, ignore_secondaries is not None, ("ignore_secondaries", ignore_secondaries)) _AppendReason(query, reason) return self._SendRequest(HTTP_POST, ("/%s/instances/%s/reboot" % (GANETI_RAPI_VERSION, instance)), query, body) def ShutdownInstance(self, instance, dry_run=False, no_remember=False, reason=None, **kwargs): """Shuts down an instance. @type instance: str @param instance: the instance to shut down @type dry_run: bool @param dry_run: whether to perform a dry run @type no_remember: bool @param no_remember: if true, will not record the state change @type reason: string @param reason: the reason for the shutdown @rtype: string @return: job id """ query = [] body = kwargs _AppendDryRunIf(query, dry_run) _AppendIf(query, no_remember, ("no_remember", 1)) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/shutdown" % (GANETI_RAPI_VERSION, instance)), query, body) def StartupInstance(self, instance, dry_run=False, no_remember=False, reason=None): """Starts up an instance. @type instance: str @param instance: the instance to start up @type dry_run: bool @param dry_run: whether to perform a dry run @type no_remember: bool @param no_remember: if true, will not record the state change @type reason: string @param reason: the reason for the startup @rtype: string @return: job id """ query = [] _AppendDryRunIf(query, dry_run) _AppendIf(query, no_remember, ("no_remember", 1)) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/startup" % (GANETI_RAPI_VERSION, instance)), query, None) def ReinstallInstance(self, instance, os=None, no_startup=False, osparams=None, reason=None): """Reinstalls an instance. @type instance: str @param instance: The instance to reinstall @type os: str or None @param os: The operating system to reinstall. If None, the instance's current operating system will be installed again @type no_startup: bool @param no_startup: Whether to start the instance automatically @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendReason(query, reason) if _INST_REINSTALL_REQV1 in self.GetFeatures(): body = { "start": not no_startup, } _SetItemIf(body, os is not None, "os", os) _SetItemIf(body, osparams is not None, "osparams", osparams) return self._SendRequest(HTTP_POST, ("/%s/instances/%s/reinstall" % (GANETI_RAPI_VERSION, instance)), query, body) # Use old request format if osparams: raise GanetiApiError("Server does not support specifying OS parameters" " for instance reinstallation") query = [] _AppendIf(query, os, ("os", os)) _AppendIf(query, no_startup, ("nostartup", 1)) return self._SendRequest(HTTP_POST, ("/%s/instances/%s/reinstall" % (GANETI_RAPI_VERSION, instance)), query, None) def ReplaceInstanceDisks(self, instance, disks=None, mode=REPLACE_DISK_AUTO, remote_node=None, iallocator=None, reason=None, early_release=None): """Replaces disks on an instance. @type instance: str @param instance: instance whose disks to replace @type disks: list of ints @param disks: Indexes of disks to replace @type mode: str @param mode: replacement mode to use (defaults to replace_auto) @type remote_node: str or None @param remote_node: new secondary node to use (for use with replace_new_secondary mode) @type iallocator: str or None @param iallocator: instance allocator plugin to use (for use with replace_auto mode) @type reason: string @param reason: the reason for executing this operation @type early_release: bool @param early_release: whether to release locks as soon as possible @rtype: string @return: job id """ query = [ ("mode", mode), ] # TODO: Convert to body parameters if disks is not None: _AppendIf(query, True, ("disks", ",".join(str(idx) for idx in disks))) _AppendIf(query, remote_node is not None, ("remote_node", remote_node)) _AppendIf(query, iallocator is not None, ("iallocator", iallocator)) _AppendReason(query, reason) _AppendIf(query, early_release is not None, ("early_release", early_release)) return self._SendRequest(HTTP_POST, ("/%s/instances/%s/replace-disks" % (GANETI_RAPI_VERSION, instance)), query, None) def PrepareExport(self, instance, mode, reason=None): """Prepares an instance for an export. @type instance: string @param instance: Instance name @type mode: string @param mode: Export mode @type reason: string @param reason: the reason for executing this operation @rtype: string @return: Job ID """ query = [("mode", mode)] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/prepare-export" % (GANETI_RAPI_VERSION, instance)), query, None) def ExportInstance(self, instance, mode, destination, shutdown=None, remove_instance=None, x509_key_name=None, destination_x509_ca=None, compress=None, reason=None): """Exports an instance. @type instance: string @param instance: Instance name @type mode: string @param mode: Export mode @type reason: string @param reason: the reason for executing this operation @rtype: string @return: Job ID """ body = { "destination": destination, "mode": mode, } _SetItemIf(body, shutdown is not None, "shutdown", shutdown) _SetItemIf(body, remove_instance is not None, "remove_instance", remove_instance) _SetItemIf(body, x509_key_name is not None, "x509_key_name", x509_key_name) _SetItemIf(body, destination_x509_ca is not None, "destination_x509_ca", destination_x509_ca) _SetItemIf(body, compress is not None, "compress", compress) query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/export" % (GANETI_RAPI_VERSION, instance)), query, body) def MigrateInstance(self, instance, mode=None, cleanup=None, target_node=None, reason=None): """Migrates an instance. @type instance: string @param instance: Instance name @type mode: string @param mode: Migration mode @type cleanup: bool @param cleanup: Whether to clean up a previously failed migration @type target_node: string @param target_node: Target Node for externally mirrored instances @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ body = {} _SetItemIf(body, mode is not None, "mode", mode) _SetItemIf(body, cleanup is not None, "cleanup", cleanup) _SetItemIf(body, target_node is not None, "target_node", target_node) query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/migrate" % (GANETI_RAPI_VERSION, instance)), query, body) def FailoverInstance(self, instance, iallocator=None, ignore_consistency=None, target_node=None, reason=None): """Does a failover of an instance. @type instance: string @param instance: Instance name @type iallocator: string @param iallocator: Iallocator for deciding the target node for shared-storage instances @type ignore_consistency: bool @param ignore_consistency: Whether to ignore disk consistency @type target_node: string @param target_node: Target node for shared-storage instances @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ body = {} _SetItemIf(body, iallocator is not None, "iallocator", iallocator) _SetItemIf(body, ignore_consistency is not None, "ignore_consistency", ignore_consistency) _SetItemIf(body, target_node is not None, "target_node", target_node) query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/failover" % (GANETI_RAPI_VERSION, instance)), query, body) def RenameInstance(self, instance, new_name, ip_check=None, name_check=None, reason=None): """Changes the name of an instance. @type instance: string @param instance: Instance name @type new_name: string @param new_name: New instance name @type ip_check: bool @param ip_check: Whether to ensure instance's IP address is inactive @type name_check: bool @param name_check: Whether to ensure instance's name is resolvable @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ body = { "new_name": new_name, } _SetItemIf(body, ip_check is not None, "ip_check", ip_check) _SetItemIf(body, name_check is not None, "name_check", name_check) query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/instances/%s/rename" % (GANETI_RAPI_VERSION, instance)), query, body) def GetInstanceConsole(self, instance, reason=None): """Request information for connecting to instance's console. @type instance: string @param instance: Instance name @type reason: string @param reason: the reason for executing this operation @rtype: dict @return: dictionary containing information about instance's console """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, ("/%s/instances/%s/console" % (GANETI_RAPI_VERSION, instance)), query, None) def GetJobs(self, bulk=False): """Gets all jobs for the cluster. @type bulk: bool @param bulk: Whether to return detailed information about jobs. @rtype: list of int @return: List of job ids for the cluster or list of dicts with detailed information about the jobs if bulk parameter was true. """ query = [] _AppendIf(query, bulk, ("bulk", 1)) if bulk: return self._SendRequest(HTTP_GET, "/%s/jobs" % GANETI_RAPI_VERSION, query, None) else: return [int(j["id"]) for j in self._SendRequest(HTTP_GET, "/%s/jobs" % GANETI_RAPI_VERSION, None, None)] def GetJobStatus(self, job_id): """Gets the status of a job. @type job_id: string @param job_id: job id whose status to query @rtype: dict @return: job status """ return self._SendRequest(HTTP_GET, "/%s/jobs/%s" % (GANETI_RAPI_VERSION, job_id), None, None) def WaitForJobCompletion(self, job_id, period=5, retries=-1): """Polls cluster for job status until completion. Completion is defined as any of the following states listed in L{JOB_STATUS_FINALIZED}. @type job_id: string @param job_id: job id to watch @type period: int @param period: how often to poll for status (optional, default 5s) @type retries: int @param retries: how many time to poll before giving up (optional, default -1 means unlimited) @rtype: bool @return: C{True} if job succeeded or C{False} if failed/status timeout @deprecated: It is recommended to use L{WaitForJobChange} wherever possible; L{WaitForJobChange} returns immediately after a job changed and does not use polling """ while retries != 0: job_result = self.GetJobStatus(job_id) if job_result and job_result["status"] == JOB_STATUS_SUCCESS: return True elif not job_result or job_result["status"] in JOB_STATUS_FINALIZED: return False if period: time.sleep(period) if retries > 0: retries -= 1 return False def WaitForJobChange(self, job_id, fields, prev_job_info, prev_log_serial): """Waits for job changes. @type job_id: string @param job_id: Job ID for which to wait @return: C{None} if no changes have been detected and a dict with two keys, C{job_info} and C{log_entries} otherwise. @rtype: dict """ body = { "fields": fields, "previous_job_info": prev_job_info, "previous_log_serial": prev_log_serial, } return self._SendRequest(HTTP_GET, "/%s/jobs/%s/wait" % (GANETI_RAPI_VERSION, job_id), None, body) def CancelJob(self, job_id, dry_run=False): """Cancels a job. @type job_id: string @param job_id: id of the job to delete @type dry_run: bool @param dry_run: whether to perform a dry run @rtype: tuple @return: tuple containing the result, and a message (bool, string) """ query = [] _AppendDryRunIf(query, dry_run) return self._SendRequest(HTTP_DELETE, "/%s/jobs/%s" % (GANETI_RAPI_VERSION, job_id), query, None) def GetNodes(self, bulk=False, reason=None): """Gets all nodes in the cluster. @type bulk: bool @param bulk: whether to return all information about all instances @type reason: string @param reason: the reason for executing this operation @rtype: list of dict or str @return: if bulk is true, info about nodes in the cluster, else list of nodes in the cluster """ query = [] _AppendIf(query, bulk, ("bulk", 1)) _AppendReason(query, reason) nodes = self._SendRequest(HTTP_GET, "/%s/nodes" % GANETI_RAPI_VERSION, query, None) if bulk: return nodes else: return [n["id"] for n in nodes] def GetNode(self, node, reason=None): """Gets information about a node. @type node: str @param node: node whose info to return @type reason: string @param reason: the reason for executing this operation @rtype: dict @return: info about the node """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, "/%s/nodes/%s" % (GANETI_RAPI_VERSION, node), query, None) def EvacuateNode(self, node, iallocator=None, remote_node=None, dry_run=False, early_release=None, mode=None, accept_old=False, reason=None): """Evacuates instances from a Ganeti node. @type node: str @param node: node to evacuate @type iallocator: str or None @param iallocator: instance allocator to use @type remote_node: str @param remote_node: node to evaucate to @type dry_run: bool @param dry_run: whether to perform a dry run @type early_release: bool @param early_release: whether to enable parallelization @type mode: string @param mode: Node evacuation mode @type accept_old: bool @param accept_old: Whether caller is ready to accept old-style (pre-2.5) results @type reason: string @param reason: the reason for executing this operation @rtype: string, or a list for pre-2.5 results @return: Job ID or, if C{accept_old} is set and server is pre-2.5, list of (job ID, instance name, new secondary node); if dry_run was specified, then the actual move jobs were not submitted and the job IDs will be C{None} @raises GanetiApiError: if an iallocator and remote_node are both specified """ if iallocator and remote_node: raise GanetiApiError("Only one of iallocator or remote_node can be used") query = [] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) if _NODE_EVAC_RES1 in self.GetFeatures(): # Server supports body parameters body = {} _SetItemIf(body, iallocator is not None, "iallocator", iallocator) _SetItemIf(body, remote_node is not None, "remote_node", remote_node) _SetItemIf(body, early_release is not None, "early_release", early_release) _SetItemIf(body, mode is not None, "mode", mode) else: # Pre-2.5 request format body = None if not accept_old: raise GanetiApiError("Server is version 2.4 or earlier and caller does" " not accept old-style results (parameter" " accept_old)") # Pre-2.5 servers can only evacuate secondaries if mode is not None and mode != NODE_EVAC_SEC: raise GanetiApiError("Server can only evacuate secondary instances") _AppendIf(query, iallocator, ("iallocator", iallocator)) _AppendIf(query, remote_node, ("remote_node", remote_node)) _AppendIf(query, early_release, ("early_release", 1)) return self._SendRequest(HTTP_POST, ("/%s/nodes/%s/evacuate" % (GANETI_RAPI_VERSION, node)), query, body) def MigrateNode(self, node, mode=None, dry_run=False, iallocator=None, target_node=None, reason=None): """Migrates all primary instances from a node. @type node: str @param node: node to migrate @type mode: string @param mode: if passed, it will overwrite the live migration type, otherwise the hypervisor default will be used @type dry_run: bool @param dry_run: whether to perform a dry run @type iallocator: string @param iallocator: instance allocator to use @type target_node: string @param target_node: Target node for shared-storage instances @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) if _NODE_MIGRATE_REQV1 in self.GetFeatures(): body = {} _SetItemIf(body, mode is not None, "mode", mode) _SetItemIf(body, iallocator is not None, "iallocator", iallocator) _SetItemIf(body, target_node is not None, "target_node", target_node) assert len(query) <= 1 return self._SendRequest(HTTP_POST, ("/%s/nodes/%s/migrate" % (GANETI_RAPI_VERSION, node)), query, body) else: # Use old request format if target_node is not None: raise GanetiApiError("Server does not support specifying target node" " for node migration") _AppendIf(query, mode is not None, ("mode", mode)) return self._SendRequest(HTTP_POST, ("/%s/nodes/%s/migrate" % (GANETI_RAPI_VERSION, node)), query, None) def GetNodeRole(self, node, reason=None): """Gets the current role for a node. @type node: str @param node: node whose role to return @type reason: string @param reason: the reason for executing this operation @rtype: str @return: the current role for a node """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, ("/%s/nodes/%s/role" % (GANETI_RAPI_VERSION, node)), query, None) def SetNodeRole(self, node, role, force=False, auto_promote=None, reason=None): """Sets the role for a node. @type node: str @param node: the node whose role to set @type role: str @param role: the role to set for the node @type force: bool @param force: whether to force the role change @type auto_promote: bool @param auto_promote: Whether node(s) should be promoted to master candidate if necessary @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendForceIf(query, force) _AppendIf(query, auto_promote is not None, ("auto-promote", auto_promote)) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/nodes/%s/role" % (GANETI_RAPI_VERSION, node)), query, role) def PowercycleNode(self, node, force=False, reason=None): """Powercycles a node. @type node: string @param node: Node name @type force: bool @param force: Whether to force the operation @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendForceIf(query, force) _AppendReason(query, reason) return self._SendRequest(HTTP_POST, ("/%s/nodes/%s/powercycle" % (GANETI_RAPI_VERSION, node)), query, None) def ModifyNode(self, node, reason=None, **kwargs): """Modifies a node. More details for parameters can be found in the RAPI documentation. @type node: string @param node: Node name @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_POST, ("/%s/nodes/%s/modify" % (GANETI_RAPI_VERSION, node)), query, kwargs) def GetNodeStorageUnits(self, node, storage_type, output_fields, reason=None): """Gets the storage units for a node. @type node: str @param node: the node whose storage units to return @type storage_type: str @param storage_type: storage type whose units to return @type output_fields: str @param output_fields: storage type fields to return @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id where results can be retrieved """ query = [ ("storage_type", storage_type), ("output_fields", output_fields), ] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, ("/%s/nodes/%s/storage" % (GANETI_RAPI_VERSION, node)), query, None) def ModifyNodeStorageUnits(self, node, storage_type, name, allocatable=None, reason=None): """Modifies parameters of storage units on the node. @type node: str @param node: node whose storage units to modify @type storage_type: str @param storage_type: storage type whose units to modify @type name: str @param name: name of the storage unit @type allocatable: bool or None @param allocatable: Whether to set the "allocatable" flag on the storage unit (None=no modification, True=set, False=unset) @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [ ("storage_type", storage_type), ("name", name), ] _AppendIf(query, allocatable is not None, ("allocatable", allocatable)) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/nodes/%s/storage/modify" % (GANETI_RAPI_VERSION, node)), query, None) def RepairNodeStorageUnits(self, node, storage_type, name, reason=None): """Repairs a storage unit on the node. @type node: str @param node: node whose storage units to repair @type storage_type: str @param storage_type: storage type to repair @type name: str @param name: name of the storage unit to repair @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [ ("storage_type", storage_type), ("name", name), ] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/nodes/%s/storage/repair" % (GANETI_RAPI_VERSION, node)), query, None) def GetNodeTags(self, node, reason=None): """Gets the tags for a node. @type node: str @param node: node whose tags to return @type reason: string @param reason: the reason for executing this operation @rtype: list of str @return: tags for the node """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, ("/%s/nodes/%s/tags" % (GANETI_RAPI_VERSION, node)), query, None) def AddNodeTags(self, node, tags, dry_run=False, reason=None): """Adds tags to a node. @type node: str @param node: node to add tags to @type tags: list of str @param tags: tags to add to the node @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/nodes/%s/tags" % (GANETI_RAPI_VERSION, node)), query, tags) def DeleteNodeTags(self, node, tags, dry_run=False, reason=None): """Delete tags from a node. @type node: str @param node: node to remove tags from @type tags: list of str @param tags: tags to remove from the node @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_DELETE, ("/%s/nodes/%s/tags" % (GANETI_RAPI_VERSION, node)), query, None) def GetNetworks(self, bulk=False, reason=None): """Gets all networks in the cluster. @type bulk: bool @param bulk: whether to return all information about the networks @rtype: list of dict or str @return: if bulk is true, a list of dictionaries with info about all networks in the cluster, else a list of names of those networks """ query = [] _AppendIf(query, bulk, ("bulk", 1)) _AppendReason(query, reason) networks = self._SendRequest(HTTP_GET, "/%s/networks" % GANETI_RAPI_VERSION, query, None) if bulk: return networks else: return [n["name"] for n in networks] def GetNetwork(self, network, reason=None): """Gets information about a network. @type network: str @param network: name of the network whose info to return @type reason: string @param reason: the reason for executing this operation @rtype: dict @return: info about the network """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, "/%s/networks/%s" % (GANETI_RAPI_VERSION, network), query, None) def CreateNetwork(self, network_name, network, gateway=None, network6=None, gateway6=None, mac_prefix=None, add_reserved_ips=None, tags=None, dry_run=False, reason=None): """Creates a new network. @type network_name: str @param network_name: the name of network to create @type dry_run: bool @param dry_run: whether to peform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) if add_reserved_ips: add_reserved_ips = add_reserved_ips.split(",") if tags: tags = tags.split(",") body = { "network_name": network_name, "gateway": gateway, "network": network, "gateway6": gateway6, "network6": network6, "mac_prefix": mac_prefix, "add_reserved_ips": add_reserved_ips, "tags": tags, } return self._SendRequest(HTTP_POST, "/%s/networks" % GANETI_RAPI_VERSION, query, body) def ConnectNetwork(self, network_name, group_name, mode, link, vlan="", dry_run=False, reason=None): """Connects a Network to a NodeGroup with the given netparams """ body = { "group_name": group_name, "network_mode": mode, "network_link": link, "network_vlan": vlan, } query = [] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/networks/%s/connect" % (GANETI_RAPI_VERSION, network_name)), query, body) def DisconnectNetwork(self, network_name, group_name, dry_run=False, reason=None): """Connects a Network to a NodeGroup with the given netparams """ body = { "group_name": group_name, } query = [] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/networks/%s/disconnect" % (GANETI_RAPI_VERSION, network_name)), query, body) def ModifyNetwork(self, network, reason=None, **kwargs): """Modifies a network. More details for parameters can be found in the RAPI documentation. @type network: string @param network: Network name @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/networks/%s/modify" % (GANETI_RAPI_VERSION, network)), None, kwargs) def DeleteNetwork(self, network, dry_run=False, reason=None): """Deletes a network. @type network: str @param network: the network to delete @type dry_run: bool @param dry_run: whether to peform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_DELETE, ("/%s/networks/%s" % (GANETI_RAPI_VERSION, network)), query, None) def RenameNetwork(self, network, new_name, reason=None): """Changes the name of a network. @type network: string @param network: Network name @type new_name: string @param new_name: New network name @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ body = { "new_name": new_name, } query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/networks/%s/rename" % (GANETI_RAPI_VERSION, network)), query, body) def GetNetworkTags(self, network, reason=None): """Gets tags for a network. @type network: string @param network: Node group whose tags to return @type reason: string @param reason: the reason for executing this operation @rtype: list of strings @return: tags for the network """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, ("/%s/networks/%s/tags" % (GANETI_RAPI_VERSION, network)), query, None) def AddNetworkTags(self, network, tags, dry_run=False, reason=None): """Adds tags to a network. @type network: str @param network: network to add tags to @type tags: list of string @param tags: tags to add to the network @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/networks/%s/tags" % (GANETI_RAPI_VERSION, network)), query, None) def DeleteNetworkTags(self, network, tags, dry_run=False, reason=None): """Deletes tags from a network. @type network: str @param network: network to delete tags from @type tags: list of string @param tags: tags to delete @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_DELETE, ("/%s/networks/%s/tags" % (GANETI_RAPI_VERSION, network)), query, None) def GetGroups(self, bulk=False, reason=None): """Gets all node groups in the cluster. @type bulk: bool @param bulk: whether to return all information about the groups @type reason: string @param reason: the reason for executing this operation @rtype: list of dict or str @return: if bulk is true, a list of dictionaries with info about all node groups in the cluster, else a list of names of those node groups """ query = [] _AppendIf(query, bulk, ("bulk", 1)) _AppendReason(query, reason) groups = self._SendRequest(HTTP_GET, "/%s/groups" % GANETI_RAPI_VERSION, query, None) if bulk: return groups else: return [g["name"] for g in groups] def GetGroup(self, group, reason=None): """Gets information about a node group. @type group: str @param group: name of the node group whose info to return @type reason: string @param reason: the reason for executing this operation @rtype: dict @return: info about the node group """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, "/%s/groups/%s" % (GANETI_RAPI_VERSION, group), query, None) def CreateGroup(self, name, alloc_policy=None, dry_run=False, reason=None): """Creates a new node group. @type name: str @param name: the name of node group to create @type alloc_policy: str @param alloc_policy: the desired allocation policy for the group, if any @type dry_run: bool @param dry_run: whether to peform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) body = { "name": name, "alloc_policy": alloc_policy, } return self._SendRequest(HTTP_POST, "/%s/groups" % GANETI_RAPI_VERSION, query, body) def ModifyGroup(self, group, reason=None, **kwargs): """Modifies a node group. More details for parameters can be found in the RAPI documentation. @type group: string @param group: Node group name @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/groups/%s/modify" % (GANETI_RAPI_VERSION, group)), query, kwargs) def DeleteGroup(self, group, dry_run=False, reason=None): """Deletes a node group. @type group: str @param group: the node group to delete @type dry_run: bool @param dry_run: whether to peform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_DELETE, ("/%s/groups/%s" % (GANETI_RAPI_VERSION, group)), query, None) def RenameGroup(self, group, new_name, reason=None): """Changes the name of a node group. @type group: string @param group: Node group name @type new_name: string @param new_name: New node group name @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ body = { "new_name": new_name, } query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/groups/%s/rename" % (GANETI_RAPI_VERSION, group)), query, body) def AssignGroupNodes(self, group, nodes, force=False, dry_run=False, reason=None): """Assigns nodes to a group. @type group: string @param group: Node group name @type nodes: list of strings @param nodes: List of nodes to assign to the group @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendForceIf(query, force) _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) body = { "nodes": nodes, } return self._SendRequest(HTTP_PUT, ("/%s/groups/%s/assign-nodes" % (GANETI_RAPI_VERSION, group)), query, body) def GetGroupTags(self, group, reason=None): """Gets tags for a node group. @type group: string @param group: Node group whose tags to return @type reason: string @param reason: the reason for executing this operation @rtype: list of strings @return: tags for the group """ query = [] _AppendReason(query, reason) return self._SendRequest(HTTP_GET, ("/%s/groups/%s/tags" % (GANETI_RAPI_VERSION, group)), query, None) def AddGroupTags(self, group, tags, dry_run=False, reason=None): """Adds tags to a node group. @type group: str @param group: group to add tags to @type tags: list of string @param tags: tags to add to the group @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_PUT, ("/%s/groups/%s/tags" % (GANETI_RAPI_VERSION, group)), query, None) def DeleteGroupTags(self, group, tags, dry_run=False, reason=None): """Deletes tags from a node group. @type group: str @param group: group to delete tags from @type tags: list of string @param tags: tags to delete @type dry_run: bool @param dry_run: whether to perform a dry run @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [("tag", t) for t in tags] _AppendDryRunIf(query, dry_run) _AppendReason(query, reason) return self._SendRequest(HTTP_DELETE, ("/%s/groups/%s/tags" % (GANETI_RAPI_VERSION, group)), query, None) def Query(self, what, fields, qfilter=None, reason=None): """Retrieves information about resources. @type what: string @param what: Resource name, one of L{constants.QR_VIA_RAPI} @type fields: list of string @param fields: Requested fields @type qfilter: None or list @param qfilter: Query filter @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendReason(query, reason) body = { "fields": fields, } _SetItemIf(body, qfilter is not None, "qfilter", qfilter) # TODO: remove "filter" after 2.7 _SetItemIf(body, qfilter is not None, "filter", qfilter) return self._SendRequest(HTTP_PUT, ("/%s/query/%s" % (GANETI_RAPI_VERSION, what)), query, body) def QueryFields(self, what, fields=None, reason=None): """Retrieves available fields for a resource. @type what: string @param what: Resource name, one of L{constants.QR_VIA_RAPI} @type fields: list of string @param fields: Requested fields @type reason: string @param reason: the reason for executing this operation @rtype: string @return: job id """ query = [] _AppendReason(query, reason) if fields is not None: _AppendIf(query, True, ("fields", ",".join(fields))) return self._SendRequest(HTTP_GET, ("/%s/query/%s/fields" % (GANETI_RAPI_VERSION, what)), query, None) def GetFilters(self, bulk=False): """Gets all filter rules in the cluster. @type bulk: bool @param bulk: whether to return all information about the networks @rtype: list of dict or str @return: if bulk is true, a list of dictionaries with info about all filter rules in the cluster, else a list of UUIDs of those filters """ query = [] _AppendIf(query, bulk, ("bulk", 1)) filters = self._SendRequest(HTTP_GET, "/%s/filters" % GANETI_RAPI_VERSION, query, None) if bulk: return filters else: return [f["uuid"] for f in filters] def GetFilter(self, filter_uuid): """Gets information about a filter rule. @type filter_uuid: str @param filter_uuid: UUID of the filter whose info to return @rtype: dict @return: info about the filter """ query = [] return self._SendRequest(HTTP_GET, "/%s/filters/%s" % (GANETI_RAPI_VERSION, filter_uuid), query, None) def AddFilter(self, priority, predicates, action, reason_trail=None): """Adds a filter rule @type reason_trail: list of (str, str, int) triples @param reason_trail: the reason trail for executing this operation, or None @rtype: string @return: filter UUID of the added filter """ if reason_trail is None: reason_trail = [] assert isinstance(reason_trail, list) reason_trail.append(("gnt:client", "", EpochNano(),)) # add client reason body = { "priority": priority, "predicates": predicates, "action": action, "reason": reason_trail, } query = [] return self._SendRequest(HTTP_POST, ("/%s/filters" % (GANETI_RAPI_VERSION)), query, body) def ReplaceFilter(self, uuid, priority, predicates, action, reason_trail=None): """Replaces a filter rule, or creates one if it doesn't already exist @type reason_trail: list of (str, str, int) triples @param reason_trail: the reason trail for executing this operation, or None @rtype: string @return: filter UUID of the replaced/added filter """ if reason_trail is None: reason_trail = [] assert isinstance(reason_trail, list) reason_trail.append(("gnt:client", "", EpochNano(),)) # add client reason body = { "priority": priority, "predicates": predicates, "action": action, "reason": reason_trail, } query = [] return self._SendRequest(HTTP_PUT, ("/%s/filters/%s" % (GANETI_RAPI_VERSION, uuid)), query, body) def DeleteFilter(self, uuid): """Deletes a filter rule @return: None """ body = {} query = [] return self._SendRequest(HTTP_DELETE, ("/%s/filters/%s" % (GANETI_RAPI_VERSION, uuid)), query, body) ganeti-3.1.0~rc2/lib/rapi/client_utils.py000064400000000000000000000066661476477700300203730ustar00rootroot00000000000000# # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """RAPI client utilities. """ from ganeti import constants from ganeti import cli from ganeti.rapi import client # Local constant to avoid importing ganeti.http HTTP_NOT_FOUND = 404 class RapiJobPollCb(cli.JobPollCbBase): def __init__(self, cl): """Initializes this class. @param cl: RAPI client instance """ cli.JobPollCbBase.__init__(self) self.cl = cl def WaitForJobChangeOnce(self, job_id, fields, prev_job_info, prev_log_serial, timeout=constants.DEFAULT_WFJC_TIMEOUT): """Waits for changes on a job. """ try: result = self.cl.WaitForJobChange(job_id, fields, prev_job_info, prev_log_serial) except client.GanetiApiError as err: if err.code == HTTP_NOT_FOUND: return None raise if result is None: return constants.JOB_NOTCHANGED return (result["job_info"], result["log_entries"]) def QueryJobs(self, job_ids, fields): """Returns the given fields for the selected job IDs. @type job_ids: list of numbers @param job_ids: Job IDs @type fields: list of strings @param fields: Fields """ if len(job_ids) != 1: raise NotImplementedError("Only one job supported at this time") try: result = self.cl.GetJobStatus(job_ids[0]) except client.GanetiApiError as err: if err.code == HTTP_NOT_FOUND: return [None] raise return [[result[name] for name in fields], ] def CancelJob(self, job_id): """Cancels a currently running job. """ return self.cl.CancelJob(job_id) def PollJob(rapi_client, job_id, reporter): """Function to poll for the result of a job. @param rapi_client: RAPI client instance @type job_id: number @param job_id: Job ID @type reporter: L{cli.JobPollReportCbBase} @param reporter: PollJob reporter instance @return: The opresult of the job @raise errors.JobLost: If job can't be found @raise errors.OpExecError: if job didn't succeed @see: L{ganeti.cli.GenericPollJob} """ return cli.GenericPollJob(job_id, RapiJobPollCb(rapi_client), reporter) ganeti-3.1.0~rc2/lib/rapi/connector.py000064400000000000000000000240241476477700300176530ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Remote API connection map. """ # pylint: disable=C0103 # C0103: Invalid name, since the R_* names are not conforming import re import urllib.parse from ganeti import constants from ganeti import http from ganeti import utils from ganeti.rapi import rlib2 _NAME_PATTERN = r"[\w\._-]+" _DISK_PATTERN = r"\d+" # the connection map is created at the end of this file CONNECTOR = {} class Mapper(object): """Map resource to method. """ def __init__(self, connector=None): """Resource mapper constructor. @param connector: a dictionary, mapping method name with URL path regexp """ if connector is None: connector = CONNECTOR self._connector = connector def getController(self, uri): """Find method for a given URI. @param uri: string with URI @return: None if no method is found or a tuple containing the following fields: - method: name of method mapped to URI - items: a list of variable intems in the path - args: a dictionary with additional parameters from URL """ if "?" in uri: (path, query) = uri.split("?", 1) args = urllib.parse.parse_qs(query) else: path = uri query = None args = {} # Try to find handler for request path result = utils.FindMatch(self._connector, path) if result is None: raise http.HttpNotFound() (handler, groups) = result return (handler, groups, args) def _ConvertPattern(value): """Converts URI pattern into a regular expression group. Used by L{_CompileHandlerPath}. """ if isinstance(value, UriPattern): return "(%s)" % value.content else: return value def _CompileHandlerPath(*args): """Compiles path for RAPI resource into regular expression. @return: Compiled regular expression object """ return re.compile("^%s$" % "".join(map(_ConvertPattern, args))) class UriPattern(object): __slots__ = [ "content", ] def __init__(self, content): self.content = content def GetHandlers(node_name_pattern, instance_name_pattern, group_name_pattern, network_name_pattern, job_id_pattern, disk_pattern, filter_pattern, query_res_pattern, translate=None): """Returns all supported resources and their handlers. C{node_name_pattern} and the other C{*_pattern} parameters are wrapped in L{UriPattern} and, if used in a URI, passed to the function specified using C{translate}. C{translate} receives 1..N parameters which are either plain strings or instances of L{UriPattern} and returns a dictionary key suitable for the caller of C{GetHandlers}. The default implementation in L{_CompileHandlerPath} returns a compiled regular expression in which each pattern is a group. @rtype: dict """ if translate is None: translate_fn = _CompileHandlerPath else: translate_fn = translate node_name = UriPattern(node_name_pattern) instance_name = UriPattern(instance_name_pattern) group_name = UriPattern(group_name_pattern) network_name = UriPattern(network_name_pattern) job_id = UriPattern(job_id_pattern) disk = UriPattern(disk_pattern) filter_uuid = UriPattern(filter_pattern) query_res = UriPattern(query_res_pattern) # Important note: New resources should always be added under /2. During a # discussion in July 2010 it was decided that having per-resource versions # is more flexible and future-compatible than versioning the whole remote # API. # TODO: Consider a different data structure where all keys are of the same # type. Strings are faster to look up in a dictionary than iterating and # matching regular expressions, therefore maybe two separate dictionaries # should be used. return { "/": rlib2.R_root, "/2": rlib2.R_2, "/version": rlib2.R_version, "/2/nodes": rlib2.R_2_nodes, translate_fn("/2/nodes/", node_name): rlib2.R_2_nodes_name, translate_fn("/2/nodes/", node_name, "/powercycle"): rlib2.R_2_nodes_name_powercycle, translate_fn("/2/nodes/", node_name, "/tags"): rlib2.R_2_nodes_name_tags, translate_fn("/2/nodes/", node_name, "/role"): rlib2.R_2_nodes_name_role, translate_fn("/2/nodes/", node_name, "/evacuate"): rlib2.R_2_nodes_name_evacuate, translate_fn("/2/nodes/", node_name, "/migrate"): rlib2.R_2_nodes_name_migrate, translate_fn("/2/nodes/", node_name, "/modify"): rlib2.R_2_nodes_name_modify, translate_fn("/2/nodes/", node_name, "/storage"): rlib2.R_2_nodes_name_storage, translate_fn("/2/nodes/", node_name, "/storage/modify"): rlib2.R_2_nodes_name_storage_modify, translate_fn("/2/nodes/", node_name, "/storage/repair"): rlib2.R_2_nodes_name_storage_repair, "/2/instances": rlib2.R_2_instances, translate_fn("/2/instances/", instance_name): rlib2.R_2_instances_name, translate_fn("/2/instances/", instance_name, "/info"): rlib2.R_2_instances_name_info, translate_fn("/2/instances/", instance_name, "/tags"): rlib2.R_2_instances_name_tags, translate_fn("/2/instances/", instance_name, "/reboot"): rlib2.R_2_instances_name_reboot, translate_fn("/2/instances/", instance_name, "/reinstall"): rlib2.R_2_instances_name_reinstall, translate_fn("/2/instances/", instance_name, "/replace-disks"): rlib2.R_2_instances_name_replace_disks, translate_fn("/2/instances/", instance_name, "/shutdown"): rlib2.R_2_instances_name_shutdown, translate_fn("/2/instances/", instance_name, "/startup"): rlib2.R_2_instances_name_startup, translate_fn("/2/instances/", instance_name, "/activate-disks"): rlib2.R_2_instances_name_activate_disks, translate_fn("/2/instances/", instance_name, "/deactivate-disks"): rlib2.R_2_instances_name_deactivate_disks, translate_fn("/2/instances/", instance_name, "/recreate-disks"): rlib2.R_2_instances_name_recreate_disks, translate_fn("/2/instances/", instance_name, "/prepare-export"): rlib2.R_2_instances_name_prepare_export, translate_fn("/2/instances/", instance_name, "/export"): rlib2.R_2_instances_name_export, translate_fn("/2/instances/", instance_name, "/migrate"): rlib2.R_2_instances_name_migrate, translate_fn("/2/instances/", instance_name, "/failover"): rlib2.R_2_instances_name_failover, translate_fn("/2/instances/", instance_name, "/rename"): rlib2.R_2_instances_name_rename, translate_fn("/2/instances/", instance_name, "/modify"): rlib2.R_2_instances_name_modify, translate_fn("/2/instances/", instance_name, "/disk/", disk, "/grow"): rlib2.R_2_instances_name_disk_grow, translate_fn("/2/instances/", instance_name, "/console"): rlib2.R_2_instances_name_console, "/2/networks": rlib2.R_2_networks, translate_fn("/2/networks/", network_name): rlib2.R_2_networks_name, translate_fn("/2/networks/", network_name, "/connect"): rlib2.R_2_networks_name_connect, translate_fn("/2/networks/", network_name, "/disconnect"): rlib2.R_2_networks_name_disconnect, translate_fn("/2/networks/", network_name, "/rename"): rlib2.R_2_networks_name_rename, translate_fn("/2/networks/", network_name, "/modify"): rlib2.R_2_networks_name_modify, translate_fn("/2/networks/", network_name, "/tags"): rlib2.R_2_networks_name_tags, "/2/groups": rlib2.R_2_groups, translate_fn("/2/groups/", group_name): rlib2.R_2_groups_name, translate_fn("/2/groups/", group_name, "/modify"): rlib2.R_2_groups_name_modify, translate_fn("/2/groups/", group_name, "/rename"): rlib2.R_2_groups_name_rename, translate_fn("/2/groups/", group_name, "/assign-nodes"): rlib2.R_2_groups_name_assign_nodes, translate_fn("/2/groups/", group_name, "/tags"): rlib2.R_2_groups_name_tags, "/2/jobs": rlib2.R_2_jobs, translate_fn("/2/jobs/", job_id): rlib2.R_2_jobs_id, translate_fn("/2/jobs/", job_id, "/wait"): rlib2.R_2_jobs_id_wait, "/2/instances-multi-alloc": rlib2.R_2_instances_multi_alloc, "/2/tags": rlib2.R_2_tags, "/2/info": rlib2.R_2_info, "/2/os": rlib2.R_2_os, "/2/redistribute-config": rlib2.R_2_redist_config, "/2/features": rlib2.R_2_features, "/2/modify": rlib2.R_2_cluster_modify, translate_fn("/2/query/", query_res): rlib2.R_2_query, translate_fn("/2/query/", query_res, "/fields"): rlib2.R_2_query_fields, "/2/filters": rlib2.R_2_filters, translate_fn("/2/filters/", filter_uuid): rlib2.R_2_filters_uuid, } CONNECTOR.update(GetHandlers(_NAME_PATTERN, _NAME_PATTERN, _NAME_PATTERN, _NAME_PATTERN, constants.JOB_ID_TEMPLATE, _DISK_PATTERN, _NAME_PATTERN, _NAME_PATTERN)) ganeti-3.1.0~rc2/lib/rapi/rlib2.py000064400000000000000000001340251476477700300166760ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Remote API resource implementations. PUT or POST? ============ According to RFC2616 the main difference between PUT and POST is that POST can create new resources but PUT can only create the resource the URI was pointing to on the PUT request. In the context of this module POST on ``/2/instances`` to change an existing entity is legitimate, while PUT would not be. PUT creates a new entity (e.g. a new instance) with a name specified in the request. Quoting from RFC2616, section 9.6:: The fundamental difference between the POST and PUT requests is reflected in the different meaning of the Request-URI. The URI in a POST request identifies the resource that will handle the enclosed entity. That resource might be a data-accepting process, a gateway to some other protocol, or a separate entity that accepts annotations. In contrast, the URI in a PUT request identifies the entity enclosed with the request -- the user agent knows what URI is intended and the server MUST NOT attempt to apply the request to some other resource. If the server desires that the request be applied to a different URI, it MUST send a 301 (Moved Permanently) response; the user agent MAY then make its own decision regarding whether or not to redirect the request. So when adding new methods, if they are operating on the URI entity itself, PUT should be prefered over POST. """ # pylint: disable=C0103 # C0103: Invalid name, since the R_* names are not conforming import errno import OpenSSL import socket from ganeti import opcodes from ganeti import objects from ganeti import http from ganeti import constants from ganeti import cli from ganeti import rapi from ganeti import ht from ganeti import compat from ganeti.rapi import baserlib _COMMON_FIELDS = ["ctime", "mtime", "uuid", "serial_no", "tags"] I_FIELDS = ["name", "admin_state", "os", "pnode", "snodes", "disk_template", "nic.ips", "nic.macs", "nic.modes", "nic.uuids", "nic.names", "nic.links", "nic.networks", "nic.networks.names", "nic.bridges", "network_port", "disk.sizes", "disk.spindles", "disk_usage", "disk.uuids", "disk.names", "beparams", "hvparams", "oper_state", "oper_ram", "oper_vcpus", "status", "custom_hvparams", "custom_beparams", "custom_nicparams", "custom_osparams", ] + _COMMON_FIELDS N_FIELDS = ["name", "offline", "master_candidate", "drained", "dtotal", "dfree", "sptotal", "spfree", "mtotal", "mnode", "mfree", "pinst_cnt", "sinst_cnt", "ctotal", "cnos", "cnodes", "csockets", "pip", "sip", "role", "pinst_list", "sinst_list", "master_capable", "vm_capable", "ndparams", "group.uuid", ] + _COMMON_FIELDS NET_FIELDS = ["name", "network", "gateway", "network6", "gateway6", "mac_prefix", "free_count", "reserved_count", "map", "group_list", "inst_list", "external_reservations", ] + _COMMON_FIELDS G_FIELDS = [ "alloc_policy", "name", "node_cnt", "node_list", "ipolicy", "custom_ipolicy", "diskparams", "custom_diskparams", "ndparams", "custom_ndparams", ] + _COMMON_FIELDS FILTER_RULE_FIELDS = [ "watermark", "priority", "predicates", "action", "reason_trail", "uuid", ] J_FIELDS_BULK = [ "id", "ops", "status", "summary", "opstatus", "received_ts", "start_ts", "end_ts", ] J_FIELDS = J_FIELDS_BULK + [ "oplog", "opresult", ] _NR_DRAINED = "drained" _NR_MASTER_CANDIDATE = "master-candidate" _NR_MASTER = "master" _NR_OFFLINE = "offline" _NR_REGULAR = "regular" _NR_MAP = { constants.NR_MASTER: _NR_MASTER, constants.NR_MCANDIDATE: _NR_MASTER_CANDIDATE, constants.NR_DRAINED: _NR_DRAINED, constants.NR_OFFLINE: _NR_OFFLINE, constants.NR_REGULAR: _NR_REGULAR, } assert frozenset(_NR_MAP) == constants.NR_ALL # Request data version field _REQ_DATA_VERSION = "__version__" # Feature string for instance creation request data version 1 _INST_CREATE_REQV1 = "instance-create-reqv1" # Feature string for instance reinstall request version 1 _INST_REINSTALL_REQV1 = "instance-reinstall-reqv1" # Feature string for node migration version 1 _NODE_MIGRATE_REQV1 = "node-migrate-reqv1" # Feature string for node evacuation with LU-generated jobs _NODE_EVAC_RES1 = "node-evac-res1" ALL_FEATURES = compat.UniqueFrozenset([ _INST_CREATE_REQV1, _INST_REINSTALL_REQV1, _NODE_MIGRATE_REQV1, _NODE_EVAC_RES1, ]) # Timeout for /2/jobs/[job_id]/wait. Gives job up to 10 seconds to change. _WFJC_TIMEOUT = 10 # FIXME: For compatibility we update the beparams/memory field. Needs to be # removed in Ganeti 2.8 def _UpdateBeparams(inst): """Updates the beparams dict of inst to support the memory field. @param inst: Inst dict @return: Updated inst dict """ beparams = inst["beparams"] beparams[constants.BE_MEMORY] = beparams[constants.BE_MAXMEM] return inst def _CheckIfConnectionDropped(sock): """Utility function to monitor the state of an open connection. @param sock: Connection's open socket @return: True if the connection was remotely closed, otherwise False """ try: result = sock.recv(0) if result == "": return True # The connection is still open except OpenSSL.SSL.WantReadError: return False # The connection has been terminated gracefully except OpenSSL.SSL.ZeroReturnError: return True # The connection was terminated except OpenSSL.SSL.SysCallError: return True # The usual EAGAIN is raised when the read would block, but only if SSL is # disabled (The SSL case is covered by WantReadError above). except socket.error as err: if getattr(err, 'errno') == errno.EAGAIN: return False else: raise return False class R_root(baserlib.ResourceBase): """/ resource. """ @staticmethod def GET(): """Supported for legacy reasons. """ return None class R_2(R_root): """/2 resource. """ class R_version(baserlib.ResourceBase): """/version resource. This resource should be used to determine the remote API version and to adapt clients accordingly. """ @staticmethod def GET(): """Returns the remote API version. """ return constants.RAPI_VERSION class R_2_info(baserlib.OpcodeResource): """/2/info resource. """ GET_OPCODE = opcodes.OpClusterQuery GET_ALIASES = { "volume_group_name": "vg_name", "drbd_usermode_helper": "drbd_helper", } def GET(self): """Returns cluster information. """ client = self.GetClient() return client.QueryClusterInfo() class R_2_features(baserlib.ResourceBase): """/2/features resource. """ @staticmethod def GET(): """Returns list of optional RAPI features implemented. """ return list(ALL_FEATURES) class R_2_os(baserlib.OpcodeResource): """/2/os resource. """ GET_OPCODE = opcodes.OpOsDiagnose def GET(self): """Return a list of all OSes. Can return error 500 in case of a problem. Example: ["debian-etch"] """ cl = self.GetClient() op = opcodes.OpOsDiagnose(output_fields=["name", "variants"], names=[]) cancel_fn = (lambda: _CheckIfConnectionDropped(self._req.request_sock)) job_id = self.SubmitJob([op], cl=cl) # we use custom feedback function, instead of print we log the status result = cli.PollJob(job_id, cl, feedback_fn=baserlib.FeedbackFn, cancel_fn=cancel_fn) diagnose_data = result[0] if not isinstance(diagnose_data, list): raise http.HttpBadGateway(message="Can't get OS list") os_names = [] for (name, variants) in diagnose_data: os_names.extend(cli.CalculateOSNames(name, variants)) return os_names class R_2_redist_config(baserlib.OpcodeResource): """/2/redistribute-config resource. """ PUT_OPCODE = opcodes.OpClusterRedistConf class R_2_cluster_modify(baserlib.OpcodeResource): """/2/modify resource. """ PUT_OPCODE = opcodes.OpClusterSetParams PUT_FORBIDDEN = [ "compression_tools", ] def checkFilterParameters(data): """Checks and extracts filter rule parameters from a request body. @return: the checked parameters: (priority, predicates, action). """ if not isinstance(data, dict): raise http.HttpBadRequest("Invalid body contents, not a dictionary") # Forbid unknown parameters allowed_params = set(["priority", "predicates", "action", "reason"]) for param in data: if param not in allowed_params: raise http.HttpBadRequest("Invalid body parameters: filter rule doesn't" " support the parameter '%s'" % param) priority = baserlib.CheckParameter( data, "priority", exptype=int, default=0) # We leave the deeper check into the predicates list to the server. predicates = baserlib.CheckParameter( data, "predicates", exptype=list, default=[]) # The action can be a string or a list; we leave the check to the server. action = baserlib.CheckParameter(data, "action", default="CONTINUE") reason = baserlib.CheckParameter(data, "reason", exptype=list, default=[]) return (priority, predicates, action, reason) class R_2_filters(baserlib.ResourceBase): """/2/filters resource. """ def GET(self): """Returns a list of all filter rules. @return: a dictionary with filter rule UUID and uri. """ client = self.GetClient() if self.useBulk(): bulkdata = client.QueryFilters(None, FILTER_RULE_FIELDS) return baserlib.MapBulkFields(bulkdata, FILTER_RULE_FIELDS) else: jobdata = [f[0] for f in client.QueryFilters(None, ["uuid"])] return baserlib.BuildUriList(jobdata, "/2/filters/%s", uri_fields=("uuid", "uri")) def POST(self): """Adds a filter rule. @return: the UUID of the newly created filter rule. """ priority, predicates, action, reason = \ checkFilterParameters(self.request_body) # ReplaceFilter(None, ...) inserts a new filter. return self.GetClient().ReplaceFilter(None, priority, predicates, action, reason) class R_2_filters_uuid(baserlib.ResourceBase): """/2/filters/[filter_uuid] resource. """ def GET(self): """Returns a filter rule. @return: a dictionary with job parameters. The result includes: - uuid: unique filter ID string - watermark: highest job ID ever used as a number - priority: filter priority as a non-negative number - predicates: filter predicates, each one being a list with the first element being the name of the predicate and the rest being parameters suitable for that predicate - action: effect of the filter as a string - reason_trail: reasons for the addition of this filter as a list of lists """ uuid = self.items[0] result = baserlib.HandleItemQueryErrors(self.GetClient().QueryFilters, uuids=[uuid], fields=FILTER_RULE_FIELDS) return baserlib.MapFields(FILTER_RULE_FIELDS, result[0]) def PUT(self): """Replaces an existing filter rule, or creates one if it doesn't exist already. @return: the UUID of the changed or created filter rule. """ uuid = self.items[0] priority, predicates, action, reason = \ checkFilterParameters(self.request_body) return self.GetClient().ReplaceFilter(uuid, priority, predicates, action, reason) def DELETE(self): """Deletes a filter rule. """ uuid = self.items[0] return self.GetClient().DeleteFilter(uuid) class R_2_jobs(baserlib.ResourceBase): """/2/jobs resource. """ def GET(self): """Returns a dictionary of jobs. @return: a dictionary with jobs id and uri. """ client = self.GetClient() if self.useBulk(): bulkdata = client.QueryJobs(None, J_FIELDS_BULK) return baserlib.MapBulkFields(bulkdata, J_FIELDS_BULK) else: jobdata = [j[0] for j in client.QueryJobs(None, ["id"])] return baserlib.BuildUriList(jobdata, "/2/jobs/%s", uri_fields=("id", "uri")) class R_2_jobs_id(baserlib.ResourceBase): """/2/jobs/[job_id] resource. """ def GET(self): """Returns a job status. @return: a dictionary with job parameters. The result includes: - id: job ID as a number - status: current job status as a string - ops: involved OpCodes as a list of dictionaries for each opcodes in the job - opstatus: OpCodes status as a list - opresult: OpCodes results as a list of lists """ job_id = self.items[0] result = self.GetClient().QueryJobs([job_id, ], J_FIELDS)[0] if result is None: raise http.HttpNotFound() return baserlib.MapFields(J_FIELDS, result) def DELETE(self): """Cancel not-yet-started job. """ job_id = self.items[0] result = self.GetClient().CancelJob(job_id) return result class R_2_jobs_id_wait(baserlib.ResourceBase): """/2/jobs/[job_id]/wait resource. """ # WaitForJobChange provides access to sensitive information and blocks # machine resources (it's a blocking RAPI call), hence restricting access. GET_ACCESS = [rapi.RAPI_ACCESS_WRITE] def GET(self): """Waits for job changes. """ job_id = self.items[0] fields = self.getBodyParameter("fields") prev_job_info = self.getBodyParameter("previous_job_info", None) prev_log_serial = self.getBodyParameter("previous_log_serial", None) if not isinstance(fields, list): raise http.HttpBadRequest("The 'fields' parameter should be a list") if not (prev_job_info is None or isinstance(prev_job_info, list)): raise http.HttpBadRequest("The 'previous_job_info' parameter should" " be a list") if not (prev_log_serial is None or isinstance(prev_log_serial, int)): raise http.HttpBadRequest("The 'previous_log_serial' parameter should" " be a number") client = self.GetClient() result = client.WaitForJobChangeOnce(job_id, fields, prev_job_info, prev_log_serial, timeout=_WFJC_TIMEOUT) if not result: raise http.HttpNotFound() if result == constants.JOB_NOTCHANGED: # No changes return None (job_info, log_entries) = result return { "job_info": job_info, "log_entries": log_entries, } class R_2_nodes(baserlib.OpcodeResource): """/2/nodes resource. """ def GET(self): """Returns a list of all nodes. """ client = self.GetClient() if self.useBulk(): bulkdata = client.QueryNodes([], N_FIELDS, False) return baserlib.MapBulkFields(bulkdata, N_FIELDS) else: nodesdata = client.QueryNodes([], ["name"], False) nodeslist = [row[0] for row in nodesdata] return baserlib.BuildUriList(nodeslist, "/2/nodes/%s", uri_fields=("id", "uri")) class R_2_nodes_name(baserlib.OpcodeResource): """/2/nodes/[node_name] resource. """ GET_ALIASES = { "sip": "secondary_ip", } def GET(self): """Send information about a node. """ node_name = self.items[0] client = self.GetClient() result = baserlib.HandleItemQueryErrors(client.QueryNodes, names=[node_name], fields=N_FIELDS, use_locking=self.useLocking()) return baserlib.MapFields(N_FIELDS, result[0]) class R_2_nodes_name_powercycle(baserlib.OpcodeResource): """/2/nodes/[node_name]/powercycle resource. """ POST_OPCODE = opcodes.OpNodePowercycle def GetPostOpInput(self): """Tries to powercycle a node. """ return (self.request_body, { "node_name": self.items[0], "force": self.useForce(), }) class R_2_nodes_name_role(baserlib.OpcodeResource): """/2/nodes/[node_name]/role resource. """ PUT_OPCODE = opcodes.OpNodeSetParams def GET(self): """Returns the current node role. @return: Node role """ node_name = self.items[0] client = self.GetClient() result = client.QueryNodes(names=[node_name], fields=["role"], use_locking=self.useLocking()) return _NR_MAP[result[0][0]] def GetPutOpInput(self): """Sets the node role. """ baserlib.CheckType(self.request_body, str, "Body contents") role = self.request_body if role == _NR_REGULAR: candidate = False offline = False drained = False elif role == _NR_MASTER_CANDIDATE: candidate = True offline = drained = None elif role == _NR_DRAINED: drained = True candidate = offline = None elif role == _NR_OFFLINE: offline = True candidate = drained = None else: raise http.HttpBadRequest("Can't set '%s' role" % role) assert len(self.items) == 1 return ({}, { "node_name": self.items[0], "master_candidate": candidate, "offline": offline, "drained": drained, "force": self.useForce(), "auto_promote": bool(self._checkIntVariable("auto-promote", default=0)), }) class R_2_nodes_name_evacuate(baserlib.OpcodeResource): """/2/nodes/[node_name]/evacuate resource. """ POST_OPCODE = opcodes.OpNodeEvacuate def GetPostOpInput(self): """Evacuate all instances off a node. """ return (self.request_body, { "node_name": self.items[0], "dry_run": self.dryRun(), }) class R_2_nodes_name_migrate(baserlib.OpcodeResource): """/2/nodes/[node_name]/migrate resource. """ POST_OPCODE = opcodes.OpNodeMigrate def GetPostOpInput(self): """Migrate all primary instances from a node. """ if self.queryargs: # Support old-style requests if "live" in self.queryargs and "mode" in self.queryargs: raise http.HttpBadRequest("Only one of 'live' and 'mode' should" " be passed") if "live" in self.queryargs: if self._checkIntVariable("live", default=1): mode = constants.HT_MIGRATION_LIVE else: mode = constants.HT_MIGRATION_NONLIVE else: mode = self._checkStringVariable("mode", default=None) data = { "mode": mode, } else: data = self.request_body return (data, { "node_name": self.items[0], }) class R_2_nodes_name_modify(baserlib.OpcodeResource): """/2/nodes/[node_name]/modify resource. """ POST_OPCODE = opcodes.OpNodeSetParams def GetPostOpInput(self): """Changes parameters of a node. """ assert len(self.items) == 1 return (self.request_body, { "node_name": self.items[0], }) class R_2_nodes_name_storage(baserlib.OpcodeResource): """/2/nodes/[node_name]/storage resource. """ # LUNodeQueryStorage acquires locks, hence restricting access to GET GET_ACCESS = [rapi.RAPI_ACCESS_WRITE] GET_OPCODE = opcodes.OpNodeQueryStorage def GetGetOpInput(self): """List storage available on a node. """ storage_type = self._checkStringVariable("storage_type", None) output_fields = self._checkStringVariable("output_fields", None) if not output_fields: raise http.HttpBadRequest("Missing the required 'output_fields'" " parameter") return ({}, { "nodes": [self.items[0]], "storage_type": storage_type, "output_fields": output_fields.split(","), }) class R_2_nodes_name_storage_modify(baserlib.OpcodeResource): """/2/nodes/[node_name]/storage/modify resource. """ PUT_OPCODE = opcodes.OpNodeModifyStorage def GetPutOpInput(self): """Modifies a storage volume on a node. """ storage_type = self._checkStringVariable("storage_type", None) name = self._checkStringVariable("name", None) if not name: raise http.HttpBadRequest("Missing the required 'name'" " parameter") changes = {} if "allocatable" in self.queryargs: changes[constants.SF_ALLOCATABLE] = \ bool(self._checkIntVariable("allocatable", default=1)) return ({}, { "node_name": self.items[0], "storage_type": storage_type, "name": name, "changes": changes, }) class R_2_nodes_name_storage_repair(baserlib.OpcodeResource): """/2/nodes/[node_name]/storage/repair resource. """ PUT_OPCODE = opcodes.OpRepairNodeStorage def GetPutOpInput(self): """Repairs a storage volume on a node. """ storage_type = self._checkStringVariable("storage_type", None) name = self._checkStringVariable("name", None) if not name: raise http.HttpBadRequest("Missing the required 'name'" " parameter") return ({}, { "node_name": self.items[0], "storage_type": storage_type, "name": name, }) class R_2_networks(baserlib.OpcodeResource): """/2/networks resource. """ POST_OPCODE = opcodes.OpNetworkAdd POST_RENAME = { "name": "network_name", } def GetPostOpInput(self): """Create a network. """ assert not self.items return (self.request_body, { "dry_run": self.dryRun(), }) def GET(self): """Returns a list of all networks. """ client = self.GetClient() if self.useBulk(): bulkdata = client.QueryNetworks([], NET_FIELDS, False) return baserlib.MapBulkFields(bulkdata, NET_FIELDS) else: data = client.QueryNetworks([], ["name"], False) networknames = [row[0] for row in data] return baserlib.BuildUriList(networknames, "/2/networks/%s", uri_fields=("name", "uri")) class R_2_networks_name(baserlib.OpcodeResource): """/2/networks/[network_name] resource. """ DELETE_OPCODE = opcodes.OpNetworkRemove def GET(self): """Send information about a network. """ network_name = self.items[0] client = self.GetClient() result = baserlib.HandleItemQueryErrors(client.QueryNetworks, names=[network_name], fields=NET_FIELDS, use_locking=self.useLocking()) return baserlib.MapFields(NET_FIELDS, result[0]) def GetDeleteOpInput(self): """Delete a network. """ assert len(self.items) == 1 return (self.request_body, { "network_name": self.items[0], "dry_run": self.dryRun(), }) class R_2_networks_name_connect(baserlib.OpcodeResource): """/2/networks/[network_name]/connect resource. """ PUT_OPCODE = opcodes.OpNetworkConnect def GetPutOpInput(self): """Changes some parameters of node group. """ assert self.items return (self.request_body, { "network_name": self.items[0], "dry_run": self.dryRun(), }) class R_2_networks_name_disconnect(baserlib.OpcodeResource): """/2/networks/[network_name]/disconnect resource. """ PUT_OPCODE = opcodes.OpNetworkDisconnect def GetPutOpInput(self): """Changes some parameters of node group. """ assert self.items return (self.request_body, { "network_name": self.items[0], "dry_run": self.dryRun(), }) class R_2_networks_name_modify(baserlib.OpcodeResource): """/2/networks/[network_name]/modify resource. """ PUT_OPCODE = opcodes.OpNetworkSetParams def GetPutOpInput(self): """Changes some parameters of network. """ assert self.items return (self.request_body, { "network_name": self.items[0], }) class R_2_networks_name_rename(baserlib.OpcodeResource): """/2/networks/[network_name]/rename resource. """ PUT_OPCODE = opcodes.OpNetworkRename def GetPutOpInput(self): """Changes the name of a network. """ assert len(self.items) == 1 return (self.request_body, { "network_name": self.items[0], "dry_run": self.dryRun(), }) class R_2_groups(baserlib.OpcodeResource): """/2/groups resource. """ POST_OPCODE = opcodes.OpGroupAdd POST_RENAME = { "name": "group_name", } def GetPostOpInput(self): """Create a node group. """ assert not self.items return (self.request_body, { "dry_run": self.dryRun(), }) def GET(self): """Returns a list of all node groups. """ client = self.GetClient() if self.useBulk(): bulkdata = client.QueryGroups([], G_FIELDS, False) return baserlib.MapBulkFields(bulkdata, G_FIELDS) else: data = client.QueryGroups([], ["name"], False) groupnames = [row[0] for row in data] return baserlib.BuildUriList(groupnames, "/2/groups/%s", uri_fields=("name", "uri")) class R_2_groups_name(baserlib.OpcodeResource): """/2/groups/[group_name] resource. """ DELETE_OPCODE = opcodes.OpGroupRemove def GET(self): """Send information about a node group. """ group_name = self.items[0] client = self.GetClient() result = baserlib.HandleItemQueryErrors(client.QueryGroups, names=[group_name], fields=G_FIELDS, use_locking=self.useLocking()) return baserlib.MapFields(G_FIELDS, result[0]) def GetDeleteOpInput(self): """Delete a node group. """ assert len(self.items) == 1 return ({}, { "group_name": self.items[0], "dry_run": self.dryRun(), }) class R_2_groups_name_modify(baserlib.OpcodeResource): """/2/groups/[group_name]/modify resource. """ PUT_OPCODE = opcodes.OpGroupSetParams PUT_RENAME = { "custom_ndparams": "ndparams", "custom_ipolicy": "ipolicy", "custom_diskparams": "diskparams", } def GetPutOpInput(self): """Changes some parameters of node group. """ assert self.items return (self.request_body, { "group_name": self.items[0], }) class R_2_groups_name_rename(baserlib.OpcodeResource): """/2/groups/[group_name]/rename resource. """ PUT_OPCODE = opcodes.OpGroupRename def GetPutOpInput(self): """Changes the name of a node group. """ assert len(self.items) == 1 return (self.request_body, { "group_name": self.items[0], "dry_run": self.dryRun(), }) class R_2_groups_name_assign_nodes(baserlib.OpcodeResource): """/2/groups/[group_name]/assign-nodes resource. """ PUT_OPCODE = opcodes.OpGroupAssignNodes def GetPutOpInput(self): """Assigns nodes to a group. """ assert len(self.items) == 1 return (self.request_body, { "group_name": self.items[0], "dry_run": self.dryRun(), "force": self.useForce(), }) def _ConvertUsbDevices(data): """Convert in place the usb_devices string to the proper format. In Ganeti 2.8.4 the separator for the usb_devices hvparam was changed from comma to space because commas cannot be accepted on the command line (they already act as the separator between different hvparams). RAPI should be able to accept commas for backwards compatibility, but we want it to also accept the new space separator. Therefore, we convert spaces into commas here and keep the old parsing logic elsewhere. """ try: hvparams = data["hvparams"] usb_devices = hvparams[constants.HV_USB_DEVICES] hvparams[constants.HV_USB_DEVICES] = usb_devices.replace(" ", ",") data["hvparams"] = hvparams except KeyError: #No usb_devices, no modification required pass class R_2_instances(baserlib.OpcodeResource): """/2/instances resource. """ POST_OPCODE = opcodes.OpInstanceCreate POST_RENAME = { "os": "os_type", "name": "instance_name", } def GET(self): """Returns a list of all available instances. """ client = self.GetClient() use_locking = self.useLocking() if self.useBulk(): bulkdata = client.QueryInstances([], I_FIELDS, use_locking) return [_UpdateBeparams(f) for f in baserlib.MapBulkFields(bulkdata, I_FIELDS)] else: instancesdata = client.QueryInstances([], ["name"], use_locking) instanceslist = [row[0] for row in instancesdata] return baserlib.BuildUriList(instanceslist, "/2/instances/%s", uri_fields=("id", "uri")) def GetPostOpInput(self): """Create an instance. @return: a job id """ baserlib.CheckType(self.request_body, dict, "Body contents") # Default to request data version 0 data_version = self.getBodyParameter(_REQ_DATA_VERSION, 0) if data_version == 0: raise http.HttpBadRequest("Instance creation request version 0 is no" " longer supported") elif data_version != 1: raise http.HttpBadRequest("Unsupported request data version %s" % data_version) data = self.request_body.copy() # Remove "__version__" data.pop(_REQ_DATA_VERSION, None) _ConvertUsbDevices(data) return (data, { "dry_run": self.dryRun(), }) class R_2_instances_multi_alloc(baserlib.OpcodeResource): """/2/instances-multi-alloc resource. """ POST_OPCODE = opcodes.OpInstanceMultiAlloc def GetPostOpInput(self): """Try to allocate multiple instances. @return: A dict with submitted jobs, allocatable instances and failed allocations """ if "instances" not in self.request_body: raise http.HttpBadRequest("Request is missing required 'instances' field" " in body") # Unlike most other RAPI calls, this one is composed of individual opcodes, # and we have to do the filling ourselves OPCODE_RENAME = { "os": "os_type", "name": "instance_name", } body = objects.FillDict(self.request_body, { "instances": [ baserlib.FillOpcode(opcodes.OpInstanceCreate, inst, {}, rename=OPCODE_RENAME) for inst in self.request_body["instances"] ], }) return (body, { "dry_run": self.dryRun(), }) class R_2_instances_name(baserlib.OpcodeResource): """/2/instances/[instance_name] resource. """ DELETE_OPCODE = opcodes.OpInstanceRemove def GET(self): """Send information about an instance. """ client = self.GetClient() instance_name = self.items[0] result = baserlib.HandleItemQueryErrors(client.QueryInstances, names=[instance_name], fields=I_FIELDS, use_locking=self.useLocking()) return _UpdateBeparams(baserlib.MapFields(I_FIELDS, result[0])) def GetDeleteOpInput(self): """Delete an instance. """ assert len(self.items) == 1 return (self.request_body, { "instance_name": self.items[0], "ignore_failures": False, "dry_run": self.dryRun(), }) class R_2_instances_name_info(baserlib.OpcodeResource): """/2/instances/[instance_name]/info resource. """ GET_OPCODE = opcodes.OpInstanceQueryData def GetGetOpInput(self): """Request detailed instance information. """ assert len(self.items) == 1 return ({}, { "instances": [self.items[0]], "static": bool(self._checkIntVariable("static", default=0)), }) class R_2_instances_name_reboot(baserlib.OpcodeResource): """/2/instances/[instance_name]/reboot resource. Implements an instance reboot. """ POST_OPCODE = opcodes.OpInstanceReboot def GetPostOpInput(self): """Reboot an instance. The URI takes type=[hard|soft|full] and ignore_secondaries=[False|True] parameters. """ return (self.request_body, { "instance_name": self.items[0], "reboot_type": self.queryargs.get("type", [constants.INSTANCE_REBOOT_HARD])[0], "ignore_secondaries": bool(self._checkIntVariable("ignore_secondaries")), "dry_run": self.dryRun(), }) class R_2_instances_name_startup(baserlib.OpcodeResource): """/2/instances/[instance_name]/startup resource. Implements an instance startup. """ PUT_OPCODE = opcodes.OpInstanceStartup def GetPutOpInput(self): """Startup an instance. The URI takes force=[False|True] parameter to start the instance if even if secondary disks are failing. """ return ({}, { "instance_name": self.items[0], "force": self.useForce(), "dry_run": self.dryRun(), "no_remember": bool(self._checkIntVariable("no_remember")), }) class R_2_instances_name_shutdown(baserlib.OpcodeResource): """/2/instances/[instance_name]/shutdown resource. Implements an instance shutdown. """ PUT_OPCODE = opcodes.OpInstanceShutdown def GetPutOpInput(self): """Shutdown an instance. """ return (self.request_body, { "instance_name": self.items[0], "no_remember": bool(self._checkIntVariable("no_remember")), "dry_run": self.dryRun(), }) def _ParseInstanceReinstallRequest(name, data): """Parses a request for reinstalling an instance. """ if not isinstance(data, dict): raise http.HttpBadRequest("Invalid body contents, not a dictionary") ostype = baserlib.CheckParameter(data, "os", default=None) start = baserlib.CheckParameter(data, "start", exptype=bool, default=True) osparams = baserlib.CheckParameter(data, "osparams", default=None) ops = [ opcodes.OpInstanceShutdown(instance_name=name), opcodes.OpInstanceReinstall(instance_name=name, os_type=ostype, osparams=osparams), ] if start: ops.append(opcodes.OpInstanceStartup(instance_name=name, force=False)) return ops class R_2_instances_name_reinstall(baserlib.OpcodeResource): """/2/instances/[instance_name]/reinstall resource. Implements an instance reinstall. """ POST_OPCODE = opcodes.OpInstanceReinstall def POST(self): """Reinstall an instance. The URI takes os=name and nostartup=[0|1] optional parameters. By default, the instance will be started automatically. """ if self.request_body: if self.queryargs: raise http.HttpBadRequest("Can't combine query and body parameters") body = self.request_body elif self.queryargs: # Legacy interface, do not modify/extend body = { "os": self._checkStringVariable("os"), "start": not self._checkIntVariable("nostartup"), } else: body = {} ops = _ParseInstanceReinstallRequest(self.items[0], body) return self.SubmitJob(ops) class R_2_instances_name_replace_disks(baserlib.OpcodeResource): """/2/instances/[instance_name]/replace-disks resource. """ POST_OPCODE = opcodes.OpInstanceReplaceDisks def GetPostOpInput(self): """Replaces disks on an instance. """ static = { "instance_name": self.items[0], } if self.request_body: data = self.request_body elif self.queryargs: # Legacy interface, do not modify/extend data = { "remote_node": self._checkStringVariable("remote_node", default=None), "mode": self._checkStringVariable("mode", default=None), "disks": self._checkStringVariable("disks", default=None), "iallocator": self._checkStringVariable("iallocator", default=None), } else: data = {} # Parse disks try: raw_disks = data.pop("disks") except KeyError: pass else: if raw_disks: if ht.TListOf(ht.TInt)(raw_disks): # pylint: disable=E1102 data["disks"] = raw_disks else: # Backwards compatibility for strings of the format "1, 2, 3" try: data["disks"] = [int(part) for part in raw_disks.split(",")] except (TypeError, ValueError) as err: raise http.HttpBadRequest("Invalid disk index passed: %s" % err) return (data, static) class R_2_instances_name_activate_disks(baserlib.OpcodeResource): """/2/instances/[instance_name]/activate-disks resource. """ PUT_OPCODE = opcodes.OpInstanceActivateDisks def GetPutOpInput(self): """Activate disks for an instance. The URI might contain ignore_size to ignore current recorded size. """ return ({}, { "instance_name": self.items[0], "ignore_size": bool(self._checkIntVariable("ignore_size")), }) class R_2_instances_name_deactivate_disks(baserlib.OpcodeResource): """/2/instances/[instance_name]/deactivate-disks resource. """ PUT_OPCODE = opcodes.OpInstanceDeactivateDisks def GetPutOpInput(self): """Deactivate disks for an instance. """ return ({}, { "instance_name": self.items[0], "force": self.useForce(), }) class R_2_instances_name_recreate_disks(baserlib.OpcodeResource): """/2/instances/[instance_name]/recreate-disks resource. """ POST_OPCODE = opcodes.OpInstanceRecreateDisks def GetPostOpInput(self): """Recreate disks for an instance. """ return (self.request_body, { "instance_name": self.items[0], }) class R_2_instances_name_prepare_export(baserlib.OpcodeResource): """/2/instances/[instance_name]/prepare-export resource. """ PUT_OPCODE = opcodes.OpBackupPrepare def GetPutOpInput(self): """Prepares an export for an instance. """ return ({}, { "instance_name": self.items[0], "mode": self._checkStringVariable("mode"), }) class R_2_instances_name_export(baserlib.OpcodeResource): """/2/instances/[instance_name]/export resource. """ PUT_OPCODE = opcodes.OpBackupExport PUT_RENAME = { "destination": "target_node", } def GetPutOpInput(self): """Exports an instance. """ return (self.request_body, { "instance_name": self.items[0], }) class R_2_instances_name_migrate(baserlib.OpcodeResource): """/2/instances/[instance_name]/migrate resource. """ PUT_OPCODE = opcodes.OpInstanceMigrate def GetPutOpInput(self): """Migrates an instance. """ return (self.request_body, { "instance_name": self.items[0], }) class R_2_instances_name_failover(baserlib.OpcodeResource): """/2/instances/[instance_name]/failover resource. """ PUT_OPCODE = opcodes.OpInstanceFailover def GetPutOpInput(self): """Does a failover of an instance. """ return (self.request_body, { "instance_name": self.items[0], }) class R_2_instances_name_rename(baserlib.OpcodeResource): """/2/instances/[instance_name]/rename resource. """ PUT_OPCODE = opcodes.OpInstanceRename def GetPutOpInput(self): """Changes the name of an instance. """ return (self.request_body, { "instance_name": self.items[0], }) class R_2_instances_name_modify(baserlib.OpcodeResource): """/2/instances/[instance_name]/modify resource. """ PUT_OPCODE = opcodes.OpInstanceSetParams PUT_RENAME = { "custom_beparams": "beparams", "custom_hvparams": "hvparams", } def GetPutOpInput(self): """Changes parameters of an instance. """ data = self.request_body.copy() _ConvertUsbDevices(data) return (data, { "instance_name": self.items[0], }) class R_2_instances_name_disk_grow(baserlib.OpcodeResource): """/2/instances/[instance_name]/disk/[disk_index]/grow resource. """ POST_OPCODE = opcodes.OpInstanceGrowDisk def GetPostOpInput(self): """Increases the size of an instance disk. """ return (self.request_body, { "instance_name": self.items[0], "disk": int(self.items[1]), }) class R_2_instances_name_console(baserlib.ResourceBase): """/2/instances/[instance_name]/console resource. """ GET_ACCESS = [rapi.RAPI_ACCESS_WRITE, rapi.RAPI_ACCESS_READ] GET_OPCODE = opcodes.OpInstanceConsole def GET(self): """Request information for connecting to instance's console. @return: Serialized instance console description, see L{objects.InstanceConsole} """ instance_name = self.items[0] client = self.GetClient() (console, oper_state) = \ client.QueryInstances([instance_name], ["console", "oper_state"], False)[0] if not oper_state: raise http.HttpServiceUnavailable("Instance console unavailable") assert isinstance(console, dict) return console def _GetQueryFields(args): """Tries to extract C{fields} query parameter. @type args: dictionary @rtype: list of string @raise http.HttpBadRequest: When parameter can't be found """ try: fields = args["fields"] except KeyError: raise http.HttpBadRequest("Missing 'fields' query argument") return _SplitQueryFields(fields[0]) def _SplitQueryFields(fields): """Splits fields as given for a query request. @type fields: string @rtype: list of string """ return [i.strip() for i in fields.split(",")] class R_2_query(baserlib.ResourceBase): """/2/query/[resource] resource. """ # Results might contain sensitive information GET_ACCESS = [rapi.RAPI_ACCESS_WRITE, rapi.RAPI_ACCESS_READ] PUT_ACCESS = GET_ACCESS GET_OPCODE = opcodes.OpQuery PUT_OPCODE = opcodes.OpQuery def _Query(self, fields, qfilter): client = self.GetClient() return client.Query(self.items[0], fields, qfilter).ToDict() def GET(self): """Returns resource information. @return: Query result, see L{objects.QueryResponse} """ return self._Query(_GetQueryFields(self.queryargs), None) def PUT(self): """Submits job querying for resources. @return: Query result, see L{objects.QueryResponse} """ body = self.request_body baserlib.CheckType(body, dict, "Body contents") try: fields = body["fields"] except KeyError: fields = _GetQueryFields(self.queryargs) qfilter = body.get("qfilter", None) # TODO: remove this after 2.7 if qfilter is None: qfilter = body.get("filter", None) return self._Query(fields, qfilter) class R_2_query_fields(baserlib.ResourceBase): """/2/query/[resource]/fields resource. """ GET_OPCODE = opcodes.OpQueryFields def GET(self): """Retrieves list of available fields for a resource. @return: List of serialized L{objects.QueryFieldDefinition} """ try: raw_fields = self.queryargs["fields"] except KeyError: fields = None else: fields = _SplitQueryFields(raw_fields[0]) return self.GetClient().QueryFields(self.items[0], fields).ToDict() class _R_Tags(baserlib.OpcodeResource): """Quasiclass for tagging resources. Manages tags. When inheriting this class you must define the TAG_LEVEL for it. """ TAG_LEVEL = None GET_OPCODE = opcodes.OpTagsGet PUT_OPCODE = opcodes.OpTagsSet DELETE_OPCODE = opcodes.OpTagsDel def __init__(self, items, queryargs, req, **kwargs): """A tag resource constructor. We have to override the default to sort out cluster naming case. """ baserlib.OpcodeResource.__init__(self, items, queryargs, req, **kwargs) if self.TAG_LEVEL == constants.TAG_CLUSTER: self.name = None else: self.name = items[0] def GET(self): """Returns a list of tags. Example: ["tag1", "tag2", "tag3"] """ kind = self.TAG_LEVEL if kind in constants.VALID_TAG_TYPES: cl = self.GetClient() if kind == constants.TAG_CLUSTER: if self.name: raise http.HttpBadRequest("Can't specify a name" " for cluster tag request") tags = list(cl.QueryTags(kind, "")) else: if not self.name: raise http.HttpBadRequest("Missing name on tag request") tags = list(cl.QueryTags(kind, self.name)) else: raise http.HttpBadRequest("Unhandled tag type!") return list(tags) def GetPutOpInput(self): """Add a set of tags. The request as a list of strings should be PUT to this URI. And you'll have back a job id. """ return ({}, { "kind": self.TAG_LEVEL, "name": self.name, "tags": self.queryargs.get("tag", []), "dry_run": self.dryRun(), }) def GetDeleteOpInput(self): """Delete a tag. In order to delete a set of tags, the DELETE request should be addressed to URI like: /tags?tag=[tag]&tag=[tag] """ # Re-use code return self.GetPutOpInput() class R_2_instances_name_tags(_R_Tags): """ /2/instances/[instance_name]/tags resource. Manages per-instance tags. """ TAG_LEVEL = constants.TAG_INSTANCE class R_2_nodes_name_tags(_R_Tags): """ /2/nodes/[node_name]/tags resource. Manages per-node tags. """ TAG_LEVEL = constants.TAG_NODE class R_2_groups_name_tags(_R_Tags): """ /2/groups/[group_name]/tags resource. Manages per-nodegroup tags. """ TAG_LEVEL = constants.TAG_NODEGROUP class R_2_networks_name_tags(_R_Tags): """ /2/networks/[network_name]/tags resource. Manages per-network tags. """ TAG_LEVEL = constants.TAG_NETWORK class R_2_tags(_R_Tags): """ /2/tags resource. Manages cluster tags. """ TAG_LEVEL = constants.TAG_CLUSTER ganeti-3.1.0~rc2/lib/rapi/testutils.py000064400000000000000000000255361476477700300177320ustar00rootroot00000000000000# # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Remote API test utilities. """ import base64 import logging import re from io import StringIO import pycurl from ganeti import errors from ganeti import opcodes from ganeti import http from ganeti import server from ganeti import utils from ganeti import compat from ganeti import luxi import ganeti.rpc.client as rpccl from ganeti import rapi import ganeti.http.server # pylint: disable=W0611 import ganeti.server.rapi # pylint: disable=W0611 import ganeti.rapi.client # pylint: disable=W0611 _URI_RE = re.compile(r"https://(?P.*):(?P\d+)(?P/.*)") class VerificationError(Exception): """Dedicated error class for test utilities. This class is used to hide all of Ganeti's internal exception, so that external users of these utilities don't have to integrate Ganeti's exception hierarchy. """ def _GetOpById(op_id): """Tries to get an opcode class based on its C{OP_ID}. """ try: return opcodes.OP_MAPPING[op_id] except KeyError: raise VerificationError("Unknown opcode ID '%s'" % op_id) def _HideInternalErrors(fn): """Hides Ganeti-internal exceptions, see L{VerificationError}. """ def wrapper(*args, **kwargs): try: return fn(*args, **kwargs) except (errors.GenericError, rapi.client.GanetiApiError) as err: raise VerificationError("Unhandled Ganeti error: %s" % err) return wrapper @_HideInternalErrors def VerifyOpInput(op_id, data): """Verifies opcode parameters according to their definition. @type op_id: string @param op_id: Opcode ID (C{OP_ID} attribute), e.g. C{OP_CLUSTER_VERIFY} @type data: dict @param data: Opcode parameter values @raise VerificationError: Parameter verification failed """ op_cls = _GetOpById(op_id) try: op = op_cls(**data) except TypeError as err: raise VerificationError("Unable to create opcode instance: %s" % err) try: op.Validate(False) except errors.OpPrereqError as err: raise VerificationError("Parameter validation for opcode '%s' failed: %s" % (op_id, err)) @_HideInternalErrors def VerifyOpResult(op_id, result): """Verifies opcode results used in tests (e.g. in a mock). @type op_id: string @param op_id: Opcode ID (C{OP_ID} attribute), e.g. C{OP_CLUSTER_VERIFY} @param result: Mocked opcode result @raise VerificationError: Return value verification failed """ resultcheck_fn = _GetOpById(op_id).OP_RESULT if not resultcheck_fn: logging.warning("Opcode '%s' has no result type definition", op_id) elif not resultcheck_fn(result): raise VerificationError("Given result does not match result description" " for opcode '%s': %s" % (op_id, resultcheck_fn)) def _GetPathFromUri(uri): """Gets the path and query from a URI. """ match = _URI_RE.match(uri) if match: return match.groupdict()["path"] else: return None def _FormatHeaders(headers): """Formats HTTP headers. @type headers: sequence of strings @rtype: string """ assert compat.all(": " in header for header in headers) return "\n".join(headers) class FakeCurl(object): """Fake cURL object. """ def __init__(self, handler): """Initialize this class @param handler: Request handler instance """ self._handler = handler self._opts = {} self._info = {} def setopt(self, opt, value): self._opts[opt] = value def getopt(self, opt): return self._opts.get(opt) def unsetopt(self, opt): self._opts.pop(opt, None) def getinfo(self, info): return self._info[info] def perform(self): method = self._opts[pycurl.CUSTOMREQUEST] url = self._opts[pycurl.URL] request_body = self._opts[pycurl.POSTFIELDS] writefn = self._opts[pycurl.WRITEFUNCTION] if pycurl.HTTPHEADER in self._opts: baseheaders = _FormatHeaders(self._opts[pycurl.HTTPHEADER]) else: baseheaders = "" headers = http.ParseHeaders(StringIO(baseheaders)) if request_body: headers[http.HTTP_CONTENT_LENGTH] = str(len(request_body)) if self._opts.get(pycurl.HTTPAUTH, 0) & pycurl.HTTPAUTH_BASIC: try: userpwd = self._opts[pycurl.USERPWD].encode("utf-8") except KeyError: raise errors.ProgrammerError("Basic authentication requires username" " and password") headers[http.HTTP_AUTHORIZATION] = \ "%s %s" % (http.auth.HTTP_BASIC_AUTH, base64.b64encode(userpwd).decode("ascii")) path = _GetPathFromUri(url) (code, _, resp_body) = \ self._handler.FetchResponse(path, method, headers, request_body) self._info[pycurl.RESPONSE_CODE] = code if isinstance(resp_body, bytes): resp_body = resp_body.decode("utf-8") if resp_body is not None: writefn(resp_body) class _RapiMock(object): """Mocking out the RAPI server parts. """ def __init__(self, user_fn, luxi_client, reqauth=False): """Initialize this class. @type user_fn: callable @param user_fn: Function to authentication username @param luxi_client: A LUXI client implementation """ self.handler = \ server.rapi.RemoteApiHandler(user_fn, reqauth, _client_cls=luxi_client) def FetchResponse(self, path, method, headers, request_body): """This is a callback method used to fetch a response. This method is called by the FakeCurl.perform method @type path: string @param path: Requested path @type method: string @param method: HTTP method @type request_body: string @param request_body: Request body @type headers: email.message.Message @param headers: Request headers @return: Tuple containing status code, response headers and response body """ req_msg = http.HttpMessage() req_msg.start_line = \ http.HttpClientToServerStartLine(method, path, http.HTTP_1_0) req_msg.headers = headers req_msg.body = request_body req_reader = type('TestReader', (object, ), {'sock': None})() (_, _, _, resp_msg) = \ http.server.HttpResponder(self.handler)(lambda: (req_msg, req_reader)) return (resp_msg.start_line.code, resp_msg.headers, resp_msg.body) class _TestLuxiTransport(object): """Mocked LUXI transport. Raises L{errors.RapiTestResult} for all method calls, no matter the arguments. """ def __init__(self, record_fn, address, timeouts=None, # pylint: disable=W0613 allow_non_master=None): # pylint: disable=W0613 """Initializes this class. """ self._record_fn = record_fn def Close(self): pass def Call(self, data): """Calls LUXI method. In this test class the method is not actually called, but added to a list of called methods and then an exception (L{errors.RapiTestResult}) is raised. There is no return value. """ (method, _, _) = rpccl.ParseRequest(data) # Take a note of called method self._record_fn(method) # Everything went fine until here, so let's abort the test raise errors.RapiTestResult class _LuxiCallRecorder(object): """Records all called LUXI client methods. """ def __init__(self): """Initializes this class. """ self._called = set() def Record(self, name): """Records a called function name. """ self._called.add(name) def CalledNames(self): """Returns a list of called LUXI methods. """ return self._called def __call__(self, address=None): """Creates an instrumented LUXI client. The LUXI client will record all method calls (use L{CalledNames} to retrieve them). """ return luxi.Client(transport=compat.partial(_TestLuxiTransport, self.Record), address=address) def _TestWrapper(fn, *args, **kwargs): """Wrapper for ignoring L{errors.RapiTestResult}. """ try: return fn(*args, **kwargs) except errors.RapiTestResult: # Everything was fine up to the point of sending a LUXI request return NotImplemented class InputTestClient(object): """Test version of RAPI client. Instances of this class can be used to test input arguments for RAPI client calls. See L{rapi.client.GanetiRapiClient} for available methods and their arguments. Functions can return C{NotImplemented} if all arguments are acceptable, but a LUXI request would be necessary to provide an actual return value. In case of an error, L{VerificationError} is raised. @see: An example on how to use this class can be found in C{doc/examples/rapi_testutils.py} """ def __init__(self): """Initializes this class. """ username = utils.GenerateSecret() password = utils.GenerateSecret() def user_fn(wanted): """Called to verify user credentials given in HTTP request. """ assert username == wanted return http.auth.PasswordFileUser(username, password, [rapi.RAPI_ACCESS_WRITE]) self._lcr = _LuxiCallRecorder() # Create a mock RAPI server handler = _RapiMock(user_fn, self._lcr) self._client = \ rapi.client.GanetiRapiClient("master.example.com", username=username, password=password, curl_factory=lambda: FakeCurl(handler)) def _GetLuxiCalls(self): """Returns the names of all called LUXI client functions. """ return self._lcr.CalledNames() def __getattr__(self, name): """Finds method by name. The method is wrapped using L{_TestWrapper} to produce the actual test result. """ return _HideInternalErrors(compat.partial(_TestWrapper, getattr(self._client, name))) ganeti-3.1.0~rc2/lib/rpc/000075500000000000000000000000001476477700300151365ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/rpc/__init__.py000064400000000000000000000025511476477700300172520ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Empty file for package definition. """ ganeti-3.1.0~rc2/lib/rpc/client.py000064400000000000000000000210351476477700300167670ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module for generic RPC clients. """ import logging import time import ganeti.rpc.transport as t from ganeti import constants from ganeti import errors from ganeti.rpc.errors import (ProtocolError, RequestError, LuxiError) from ganeti import serializer KEY_METHOD = constants.LUXI_KEY_METHOD KEY_ARGS = constants.LUXI_KEY_ARGS KEY_SUCCESS = constants.LUXI_KEY_SUCCESS KEY_RESULT = constants.LUXI_KEY_RESULT KEY_VERSION = constants.LUXI_KEY_VERSION def ParseRequest(msg): """Parses a request message. """ try: request = serializer.LoadJson(msg) except ValueError as err: raise ProtocolError("Invalid RPC request (parsing error): %s" % err) logging.debug("RPC request: %s", request) if not isinstance(request, dict): logging.error("RPC request not a dict: %r", msg) raise ProtocolError("Invalid RPC request (not a dict)") method = request.get(KEY_METHOD, None) # pylint: disable=E1103 args = request.get(KEY_ARGS, None) # pylint: disable=E1103 version = request.get(KEY_VERSION, None) # pylint: disable=E1103 if method is None or args is None: logging.error("RPC request missing method or arguments: %r", msg) raise ProtocolError(("Invalid RPC request (no method or arguments" " in request): %r") % msg) return (method, args, version) def ParseResponse(msg): """Parses a response message. """ # Parse the result try: data = serializer.LoadJson(msg) except KeyboardInterrupt: raise except Exception as err: raise ProtocolError("Error while deserializing response: %s" % str(err)) # Validate response if not (isinstance(data, dict) and KEY_SUCCESS in data and KEY_RESULT in data): raise ProtocolError("Invalid response from server: %r" % data) return (data[KEY_SUCCESS], data[KEY_RESULT], data.get(KEY_VERSION, None)) # pylint: disable=E1103 def FormatResponse(success, result, version=None): """Formats a response message. """ response = { KEY_SUCCESS: success, KEY_RESULT: result, } if version is not None: response[KEY_VERSION] = version logging.debug("RPC response: %s", response) return serializer.DumpJson(response) def FormatRequest(method, args, version=None): """Formats a request message. """ # Build request request = { KEY_METHOD: method, KEY_ARGS: args, } if version is not None: request[KEY_VERSION] = version # Serialize the request return serializer.DumpJson(request, private_encoder=serializer.EncodeWithPrivateFields) def CallRPCMethod(transport_cb, method, args, version=None): """Send a RPC request via a transport and return the response. """ assert callable(transport_cb) t1 = time.time() * 1000 request_msg = FormatRequest(method, args, version=version) t2 = time.time() * 1000 # Send request and wait for response response_msg = transport_cb(request_msg) t3 = time.time() * 1000 (success, result, resp_version) = ParseResponse(response_msg) t4 = time.time() * 1000 logging.debug("CallRPCMethod %s: format: %dms, sock: %dms, parse: %dms", method, int(t2 - t1), int(t3 - t2), int(t4 - t3)) # Verify version if there was one in the response if resp_version is not None and resp_version != version: raise LuxiError("RPC version mismatch, client %s, response %s" % (version, resp_version)) if success: return result errors.MaybeRaise(result) raise RequestError(result) class AbstractClient(object): """High-level client abstraction. This uses a backing Transport-like class on top of which it implements data serialization/deserialization. """ def __init__(self, timeouts=None, transport=t.Transport, allow_non_master=False): """Constructor for the Client class. If timeout is not passed, the default timeouts of the transport class are used. @type timeouts: list of ints @param timeouts: timeouts to be used on connect and read/write @type transport: L{Transport} or another compatible class @param transport: the underlying transport to use for the RPC calls @type allow_non_master: bool @param allow_non_master: skip checks for the master node on errors """ self.timeouts = timeouts self.transport_class = transport self.allow_non_master = allow_non_master self.transport = None # The version used in RPC communication, by default unused: self.version = None def _GetAddress(self): """Returns the socket address """ raise NotImplementedError def _InitTransport(self): """(Re)initialize the transport if needed. """ if self.transport is None: self.transport = \ self.transport_class(self._GetAddress(), timeouts=self.timeouts, allow_non_master=self.allow_non_master) def _CloseTransport(self): """Close the transport, ignoring errors. """ if self.transport is None: return try: old_transp = self.transport self.transport = None old_transp.Close() except Exception: # pylint: disable=W0703 pass def _SendMethodCall(self, data): # Send request and wait for response def send(try_no): if try_no: logging.debug("RPC peer disconnected, retrying") self._InitTransport() return self.transport.Call(data) return t.Transport.RetryOnNetworkError(send, lambda _: self._CloseTransport()) def Close(self): """Close the underlying connection. """ self._CloseTransport() def close(self): """Same as L{Close}, to be used with contextlib.closing(...). """ self.Close() def CallMethod(self, method, args): """Send a generic request and return the response. """ if not isinstance(args, (list, tuple)): raise errors.ProgrammerError("Invalid parameter passed to CallMethod:" " expected list, got %s" % type(args)) return CallRPCMethod(self._SendMethodCall, method, args, version=self.version) class AbstractStubClient(AbstractClient): """An abstract Client that connects a generated stub client to a L{Transport}. Subclasses should inherit from this class (first) as well and a designated stub (second). """ def __init__(self, timeouts=None, transport=t.Transport, allow_non_master=None): """Constructor for the class. Arguments are the same as for L{AbstractClient}. Checks that SOCKET_PATH attribute is defined (in the stub class). @type timeouts: list of ints @param timeouts: timeouts to be used on connect and read/write @type transport: L{Transport} or another compatible class @param transport: the underlying transport to use for the RPC calls @type allow_non_master: bool @param allow_non_master: skip checks for the master node on errors """ super(AbstractStubClient, self).__init__(timeouts=timeouts, transport=transport, allow_non_master=allow_non_master) def _GenericInvoke(self, method, *args): return self.CallMethod(method, args) def _GetAddress(self): return self._GetSocketPath() # pylint: disable=E1101 ganeti-3.1.0~rc2/lib/rpc/errors.py000064400000000000000000000045771476477700300170410ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module that defines a transport for RPC connections. A transport can send to and receive messages from some endpoint. """ from ganeti.errors import LuxiError class ProtocolError(LuxiError): """Denotes an error in the LUXI protocol.""" class ConnectionClosedError(ProtocolError): """Connection closed error.""" class TimeoutError(ProtocolError): """Operation timeout error.""" class RequestError(ProtocolError): """Error on request. This signifies an error in the request format or request handling, but not (e.g.) an error in starting up an instance. Some common conditions that can trigger this exception: - job submission failed because the job data was wrong - query failed because required fields were missing """ class NoMasterError(ProtocolError): """The master cannot be reached. This means that the master daemon is not running or the socket has been removed. """ class PermissionError(ProtocolError): """Permission denied while connecting to the master socket. This means the user doesn't have the proper rights. """ ganeti-3.1.0~rc2/lib/rpc/node.py000064400000000000000000001016111476477700300164350ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Inter-node RPC library. """ # pylint: disable=C0103,R0201,R0904 # C0103: Invalid name, since call_ are not valid # R0201: Method could be a function, we keep all rpcs instance methods # as not to change them back and forth between static/instance methods # if they need to start using instance attributes # R0904: Too many public methods import base64 import copy import logging import os import threading import zlib import pycurl from ganeti import utils from ganeti import objects from ganeti import http from ganeti import serializer from ganeti import constants from ganeti import errors from ganeti import netutils from ganeti import ssconf from ganeti import runtime from ganeti import compat from ganeti import rpc_defs from ganeti import pathutils from ganeti import vcluster # Special module generated at build time from ganeti import _generated_rpc # pylint has a bug here, doesn't see this import import ganeti.http.client # pylint: disable=W0611 _RPC_CLIENT_HEADERS = [ "Content-type: %s" % http.HTTP_APP_JSON, "Expect:", ] #: Special value to describe an offline host _OFFLINE = object() def Init(): """Initializes the module-global HTTP client manager. Must be called before using any RPC function and while exactly one thread is running. """ # curl_global_init(3) and curl_global_cleanup(3) must be called with only # one thread running. This check is just a safety measure -- it doesn't # cover all cases. assert threading.activeCount() == 1, \ "Found more than one active thread when initializing pycURL" logging.info("Using PycURL %s", pycurl.version) pycurl.global_init(pycurl.GLOBAL_ALL) def Shutdown(): """Stops the module-global HTTP client manager. Must be called before quitting the program and while exactly one thread is running. """ pycurl.global_cleanup() def _ConfigRpcCurl(curl): noded_cert = pathutils.NODED_CERT_FILE noded_client_cert = pathutils.NODED_CLIENT_CERT_FILE # This fallback is required for backwards compatibility with 2.10. Ganeti # 2.11 introduced per-node client certificates, but when we restart after # an upgrade from 2.10, the client certs are not in place yet, and we need # to fall back to using the cluster-wide server cert. if not os.path.exists(noded_client_cert): logging.warn("Using server certificate as client certificate for RPC" "call.") noded_client_cert = noded_cert curl.setopt(pycurl.FOLLOWLOCATION, False) curl.setopt(pycurl.CAINFO, noded_cert) curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.setopt(pycurl.SSL_VERIFYPEER, True) curl.setopt(pycurl.SSLCERTTYPE, "PEM") curl.setopt(pycurl.SSLCERT, noded_client_cert) curl.setopt(pycurl.SSLKEYTYPE, "PEM") curl.setopt(pycurl.SSLKEY, noded_client_cert) curl.setopt(pycurl.CONNECTTIMEOUT, constants.RPC_CONNECT_TIMEOUT) def RunWithRPC(fn): """RPC-wrapper decorator. When applied to a function, it runs it with the RPC system initialized, and it shutsdown the system afterwards. This means the function must be called without RPC being initialized. """ def wrapper(*args, **kwargs): Init() try: return fn(*args, **kwargs) finally: Shutdown() return wrapper def _Compress(_, data): """Compresses a string for transport over RPC. Small amounts of data are not compressed. @type data: str @param data: Data @rtype: tuple @return: Encoded data to send """ # Small amounts of data are not compressed if len(data) < 512: return (constants.RPC_ENCODING_NONE, data) # Compress with zlib and encode in base64 return (constants.RPC_ENCODING_ZLIB_BASE64, base64.b64encode(zlib.compress(data, 3))) class RpcResult(object): """RPC Result class. This class holds an RPC result. It is needed since in multi-node calls we can't raise an exception just because one out of many failed, and therefore we use this class to encapsulate the result. @ivar data: the data payload, for successful results, or None @ivar call: the name of the RPC call @ivar node: the name of the node to which we made the call @ivar offline: whether the operation failed because the node was offline, as opposed to actual failure; offline=True will always imply failed=True, in order to allow simpler checking if the user doesn't care about the exact failure mode @ivar fail_msg: the error message if the call failed """ def __init__(self, data=None, failed=False, offline=False, call=None, node=None): self.offline = offline self.call = call self.node = node if offline: self.fail_msg = "Node is marked offline" self.data = self.payload = None elif failed: self.fail_msg = self._EnsureErr(data) self.data = self.payload = None else: self.data = data if not isinstance(self.data, (tuple, list)): self.fail_msg = ("RPC layer error: invalid result type (%s)" % type(self.data)) self.payload = None elif len(data) != 2: self.fail_msg = ("RPC layer error: invalid result length (%d), " "expected 2" % len(self.data)) self.payload = None elif not self.data[0]: self.fail_msg = self._EnsureErr(self.data[1]) self.payload = None else: # finally success self.fail_msg = None self.payload = data[1] for attr_name in ["call", "data", "fail_msg", "node", "offline", "payload"]: assert hasattr(self, attr_name), "Missing attribute %s" % attr_name def __repr__(self): return ("RpcResult(data=%s, call=%s, node=%s, offline=%s, fail_msg=%s)" % (self.offline, self.call, self.node, self.offline, self.fail_msg)) @staticmethod def _EnsureErr(val): """Helper to ensure we return a 'True' value for error.""" if val: return val else: return "No error information" def Raise(self, msg, prereq=False, ecode=None): """If the result has failed, raise an OpExecError. This is used so that LU code doesn't have to check for each result, but instead can call this function. """ if not self.fail_msg: return if not msg: # one could pass None for default message msg = ("Call '%s' to node '%s' has failed: %s" % (self.call, self.node, self.fail_msg)) else: msg = "%s: %s" % (msg, self.fail_msg) if prereq: ec = errors.OpPrereqError else: ec = errors.OpExecError if ecode is not None: args = (msg, ecode) else: args = (msg, ) raise ec(*args) def Warn(self, msg, feedback_fn): """If the result has failed, call the feedback_fn. This is used to in cases were LU wants to warn the user about a failure, but continue anyway. """ if not self.fail_msg: return msg = "%s: %s" % (msg, self.fail_msg) feedback_fn(msg) def _SsconfResolver(ssconf_ips, node_list, _, ssc=ssconf.SimpleStore, nslookup_fn=netutils.Hostname.GetIP): """Return addresses for given node names. @type ssconf_ips: bool @param ssconf_ips: Use the ssconf IPs @type node_list: list @param node_list: List of node names @type ssc: class @param ssc: SimpleStore class that is used to obtain node->ip mappings @type nslookup_fn: callable @param nslookup_fn: function use to do NS lookup @rtype: list of tuple; (string, string) @return: List of tuples containing node name and IP address """ ss = ssc() family = ss.GetPrimaryIPFamily() if ssconf_ips: iplist = ss.GetNodePrimaryIPList() ipmap = dict(entry.split() for entry in iplist) else: ipmap = {} result = [] for node in node_list: ip = ipmap.get(node) if ip is None: ip = nslookup_fn(node, family=family) result.append((node, ip, node)) return result class _StaticResolver(object): def __init__(self, addresses): """Initializes this class. """ self._addresses = addresses def __call__(self, hosts, _): """Returns static addresses for hosts. """ assert len(hosts) == len(self._addresses) return list(zip(hosts, self._addresses, hosts)) def _CheckConfigNode(node_uuid_or_name, node, accept_offline_node): """Checks if a node is online. @type node_uuid_or_name: string @param node_uuid_or_name: Node UUID @type node: L{objects.Node} or None @param node: Node object """ if node is None: # Assume that the passed parameter was actually a node name, so depend on # DNS for name resolution return (node_uuid_or_name, node_uuid_or_name, node_uuid_or_name) else: if node.offline and not accept_offline_node: ip = _OFFLINE else: ip = node.primary_ip return (node.name, ip, node_uuid_or_name) def _NodeConfigResolver(single_node_fn, all_nodes_fn, node_uuids, opts): """Calculate node addresses using configuration. Note that strings in node_uuids are treated as node names if the UUID is not found in the configuration. """ accept_offline_node = (opts is rpc_defs.ACCEPT_OFFLINE_NODE) assert accept_offline_node or opts is None, "Unknown option" # Special case for single-host lookups if len(node_uuids) == 1: (uuid, ) = node_uuids return [_CheckConfigNode(uuid, single_node_fn(uuid), accept_offline_node)] else: all_nodes = all_nodes_fn() return [_CheckConfigNode(uuid, all_nodes.get(uuid, None), accept_offline_node) for uuid in node_uuids] class _RpcProcessor(object): def __init__(self, resolver, port, lock_monitor_cb=None): """Initializes this class. @param resolver: callable accepting a list of node UUIDs or hostnames, returning a list of tuples containing name, IP address and original name of the resolved node. IP address can be the name or the special value L{_OFFLINE} to mark offline machines. @type port: int @param port: TCP port @param lock_monitor_cb: Callable for registering with lock monitor """ self._resolver = resolver self._port = port self._lock_monitor_cb = lock_monitor_cb @staticmethod def _PrepareRequests(hosts, port, procedure, body, read_timeout): """Prepares requests by sorting offline hosts into separate list. @type body: dict @param body: a dictionary with per-host body data """ results = {} requests = {} assert isinstance(body, dict) assert len(body) == len(hosts) assert compat.all(isinstance(v, (str, bytes)) for v in body.values()) assert frozenset(h[2] for h in hosts) == frozenset(body), \ "%s != %s" % (hosts, list(body)) for (name, ip, original_name) in hosts: if ip is _OFFLINE: # Node is marked as offline results[original_name] = RpcResult(node=name, offline=True, call=procedure) else: requests[original_name] = \ http.client.HttpClientRequest(str(ip), port, http.HTTP_POST, str("/%s" % procedure), headers=_RPC_CLIENT_HEADERS, post_data=body[original_name], read_timeout=read_timeout, nicename="%s/%s" % (name, procedure), curl_config_fn=_ConfigRpcCurl) return (results, requests) @staticmethod def _CombineResults(results, requests, procedure): """Combines pre-computed results for offline hosts with actual call results. """ for name, req in requests.items(): if req.success and req.resp_status_code == http.HTTP_OK: host_result = RpcResult(data=serializer.LoadJson(req.resp_body), node=name, call=procedure) else: # TODO: Better error reporting if req.error: msg = req.error else: msg = req.resp_body logging.error("RPC error in %s on node %s: %s", procedure, name, msg) host_result = RpcResult(data=msg, failed=True, node=name, call=procedure) results[name] = host_result return results def __call__(self, nodes, procedure, body, read_timeout, resolver_opts, _req_process_fn=None): """Makes an RPC request to a number of nodes. @type nodes: sequence @param nodes: node UUIDs or Hostnames @type procedure: string @param procedure: Request path @type body: dictionary @param body: dictionary with request bodies per host @type read_timeout: int or None @param read_timeout: Read timeout for request @rtype: dictionary @return: a dictionary mapping host names to rpc.RpcResult objects """ assert read_timeout is not None, \ "Missing RPC read timeout for procedure '%s'" % procedure if _req_process_fn is None: _req_process_fn = http.client.ProcessRequests (results, requests) = \ self._PrepareRequests(self._resolver(nodes, resolver_opts), self._port, procedure, body, read_timeout) _req_process_fn(list(requests.values()), lock_monitor_cb=self._lock_monitor_cb) assert not frozenset(results).intersection(requests) return self._CombineResults(results, requests, procedure) class _RpcClientBase(object): def __init__(self, resolver, encoder_fn, lock_monitor_cb=None, _req_process_fn=None): """Initializes this class. """ proc = _RpcProcessor(resolver, netutils.GetDaemonPort(constants.NODED), lock_monitor_cb=lock_monitor_cb) self._proc = compat.partial(proc, _req_process_fn=_req_process_fn) self._encoder = compat.partial(self._EncodeArg, encoder_fn) @staticmethod def _EncodeArg(encoder_fn, node, arg): """Encode argument. """ (argkind, value) = arg if argkind is None: return value else: return encoder_fn(argkind)(node, value) def _Call(self, cdef, node_list, args): """Entry point for automatically generated RPC wrappers. """ (procedure, _, resolver_opts, timeout, argdefs, prep_fn, postproc_fn, _) = cdef if callable(timeout): read_timeout = timeout(args) else: read_timeout = timeout if callable(resolver_opts): req_resolver_opts = resolver_opts(args) else: req_resolver_opts = resolver_opts if len(args) != len(argdefs): raise errors.ProgrammerError("Number of passed arguments doesn't match") if prep_fn is None: prep_fn = lambda _, args: args assert callable(prep_fn) # encode the arguments for each node individually, pass them and the node # name to the prep_fn, and serialise its return value encode_args_fn = lambda node: [self._encoder(node, (argdef[1], val)) for (argdef, val) in zip(argdefs, args)] pnbody = dict( (n, serializer.DumpJson(prep_fn(n, encode_args_fn(n)), private_encoder=serializer.EncodeWithPrivateFields)) for n in node_list ) result = self._proc(node_list, procedure, pnbody, read_timeout, req_resolver_opts) if postproc_fn: return dict((k, postproc_fn(v)) for (k, v) in result.items()) else: return result def _ObjectToDict(_, value): """Converts an object to a dictionary. @note: See L{objects}. """ return value.ToDict() def _ObjectListToDict(node, value): """Converts a list of L{objects} to dictionaries. """ return [_ObjectToDict(node, v) for v in value] def _PrepareFileUpload(getents_fn, node, filename): """Loads a file and prepares it for an upload to nodes. """ statcb = utils.FileStatHelper() data = _Compress(node, utils.ReadBinaryFile(filename, preread=statcb)) st = statcb.st if getents_fn is None: getents_fn = runtime.GetEnts getents = getents_fn() virt_filename = vcluster.MakeVirtualPath(filename) return [virt_filename, data, st.st_mode, getents.LookupUid(st.st_uid), getents.LookupGid(st.st_gid), st.st_atime, st.st_mtime] def _PrepareFinalizeExportDisks(_, snap_disks): """Encodes disks for finalizing export. """ flat_disks = [] for disk in snap_disks: if isinstance(disk, bool): flat_disks.append(disk) else: flat_disks.append(disk.ToDict()) return flat_disks def _EncodeBlockdevRename(_, value): """Encodes information for renaming block devices. """ return [(d.ToDict(), uid) for d, uid in value] def _AddSpindlesToLegacyNodeInfo(result, space_info): """Extracts the spindle information from the space info and adds it to the result dictionary. @type result: dict of strings @param result: dictionary holding the result of the legacy node info @type space_info: list of dicts of strings @param space_info: list, each row holding space information of one storage unit @rtype: None @return: does not return anything, manipulates the C{result} variable """ lvm_pv_info = utils.storage.LookupSpaceInfoByStorageType( space_info, constants.ST_LVM_PV) if lvm_pv_info: result["spindles_free"] = lvm_pv_info["storage_free"] result["spindles_total"] = lvm_pv_info["storage_size"] else: result["spindles_free"] = 0 result["spindles_total"] = 0 def _AddStorageInfoToLegacyNodeInfoByTemplate( result, space_info, disk_template): """Extracts the storage space information of the disk template from the space info and adds it to the result dictionary. @see: C{_AddSpindlesToLegacyNodeInfo} for parameter information. """ if utils.storage.DiskTemplateSupportsSpaceReporting(disk_template): disk_info = utils.storage.LookupSpaceInfoByDiskTemplate( space_info, disk_template) result["name"] = disk_info["name"] result["storage_free"] = disk_info["storage_free"] result["storage_size"] = disk_info["storage_size"] else: # FIXME: consider displaying '-' in this case result["storage_free"] = 0 result["storage_size"] = 0 def MakeLegacyNodeInfo(data, disk_template): """Formats the data returned by call_node_info. Converts the data into a single dictionary. This is fine for most use cases, but some require information from more than one volume group or hypervisor. """ (bootid, space_info, (hv_info, )) = data ret = utils.JoinDisjointDicts(hv_info, {"bootid": bootid}) _AddSpindlesToLegacyNodeInfo(ret, space_info) _AddStorageInfoToLegacyNodeInfoByTemplate(ret, space_info, disk_template) return ret def _AnnotateDParamsDRBD(disk, params): """Annotates just DRBD disks layouts. """ (drbd_params, data_params, meta_params) = params assert disk.dev_type == constants.DT_DRBD8 disk.params = objects.FillDict(drbd_params, disk.params) (dev_data, dev_meta) = disk.children dev_data.params = objects.FillDict(data_params, dev_data.params) dev_meta.params = objects.FillDict(meta_params, dev_meta.params) return disk def _AnnotateDParamsGeneric(disk, params): """Generic disk parameter annotation routine. """ assert disk.dev_type != constants.DT_DRBD8 disk.params = objects.FillDict(params[0], disk.params) return disk def AnnotateDiskParams(disks, disk_params): """Annotates the disk objects with the disk parameters. @param disks: The list of disks objects to annotate @param disk_params: The disk parameters for annotation @returns: A list of disk objects annotated """ def AnnotateDisk(disk): if disk.dev_type == constants.DT_DISKLESS: return disk ld_params = objects.Disk.ComputeLDParams(disk.dev_type, disk_params) if disk.dev_type == constants.DT_DRBD8: return _AnnotateDParamsDRBD(disk, ld_params) else: return _AnnotateDParamsGeneric(disk, ld_params) return [AnnotateDisk(disk.Copy()) for disk in disks] def _GetExclusiveStorageFlag(cfg, node_uuid): ni = cfg.GetNodeInfo(node_uuid) if ni is None: raise errors.OpPrereqError("Invalid node name %s" % node_uuid, errors.ECODE_NOENT) return cfg.GetNdParams(ni)[constants.ND_EXCLUSIVE_STORAGE] def _AddExclusiveStorageFlagToLvmStorageUnits(storage_units, es_flag): """Adds the exclusive storage flag to lvm units. This function creates a copy of the storage_units lists, with the es_flag being added to all lvm storage units. @type storage_units: list of pairs (string, string) @param storage_units: list of 'raw' storage units, consisting only of (storage_type, storage_key) @type es_flag: boolean @param es_flag: exclusive storage flag @rtype: list of tuples (string, string, list) @return: list of storage units (storage_type, storage_key, params) with the params containing the es_flag for lvm-vg storage units """ result = [] for (storage_type, storage_key) in storage_units: if storage_type in [constants.ST_LVM_VG]: result.append((storage_type, storage_key, [es_flag])) if es_flag: result.append((constants.ST_LVM_PV, storage_key, [es_flag])) else: result.append((storage_type, storage_key, [])) return result def GetExclusiveStorageForNodes(cfg, node_uuids): """Return the exclusive storage flag for all the given nodes. @type cfg: L{config.ConfigWriter} @param cfg: cluster configuration @type node_uuids: list or tuple @param node_uuids: node UUIDs for which to read the flag @rtype: dict @return: mapping from node uuids to exclusive storage flags @raise errors.OpPrereqError: if any given node name has no corresponding node """ getflag = lambda n: _GetExclusiveStorageFlag(cfg, n) flags = map(getflag, node_uuids) return dict(zip(node_uuids, flags)) def PrepareStorageUnitsForNodes(cfg, storage_units, node_uuids): """Return the lvm storage unit for all the given nodes. Main purpose of this function is to map the exclusive storage flag, which can be different for each node, to the default LVM storage unit. @type cfg: L{config.ConfigWriter} @param cfg: cluster configuration @type storage_units: list of pairs (string, string) @param storage_units: list of 'raw' storage units, e.g. pairs of (storage_type, storage_key) @type node_uuids: list or tuple @param node_uuids: node UUIDs for which to read the flag @rtype: dict @return: mapping from node uuids to a list of storage units which include the exclusive storage flag for lvm storage @raise errors.OpPrereqError: if any given node name has no corresponding node """ getunit = lambda n: _AddExclusiveStorageFlagToLvmStorageUnits( storage_units, _GetExclusiveStorageFlag(cfg, n)) flags = map(getunit, node_uuids) return dict(zip(node_uuids, flags)) #: Generic encoders _ENCODERS = { rpc_defs.ED_OBJECT_DICT: _ObjectToDict, rpc_defs.ED_OBJECT_DICT_LIST: _ObjectListToDict, rpc_defs.ED_COMPRESS: _Compress, rpc_defs.ED_FINALIZE_EXPORT_DISKS: _PrepareFinalizeExportDisks, rpc_defs.ED_BLOCKDEV_RENAME: _EncodeBlockdevRename, } class RpcRunner(_RpcClientBase, _generated_rpc.RpcClientDefault, _generated_rpc.RpcClientBootstrap, _generated_rpc.RpcClientDnsOnly, _generated_rpc.RpcClientConfig): """RPC runner class. """ def __init__(self, cfg, lock_monitor_cb, _req_process_fn=None, _getents=None): """Initialized the RPC runner. @type cfg: L{config.ConfigWriter} @param cfg: Configuration @type lock_monitor_cb: callable @param lock_monitor_cb: Lock monitor callback """ self._cfg = cfg encoders = _ENCODERS.copy() encoders.update({ # Encoders requiring configuration object rpc_defs.ED_INST_DICT: self._InstDict, rpc_defs.ED_INST_DICT_HVP_BEP_DP: self._InstDictHvpBepDp, rpc_defs.ED_INST_DICT_OSP_DP: self._InstDictOspDp, rpc_defs.ED_NIC_DICT: self._NicDict, rpc_defs.ED_DEVICE_DICT: self._DeviceDict, # Encoders annotating disk parameters rpc_defs.ED_DISKS_DICT_DP: self._DisksDictDP, rpc_defs.ED_MULTI_DISKS_DICT_DP: self._MultiDiskDictDP, rpc_defs.ED_SINGLE_DISK_DICT_DP: self._SingleDiskDictDP, rpc_defs.ED_NODE_TO_DISK_DICT_DP: self._EncodeNodeToDiskDictDP, # Encoders with special requirements rpc_defs.ED_FILE_DETAILS: compat.partial(_PrepareFileUpload, _getents), rpc_defs.ED_IMPEXP_IO: self._EncodeImportExportIO, }) # Resolver using configuration resolver = compat.partial(_NodeConfigResolver, cfg.GetNodeInfo, cfg.GetAllNodesInfo) # Pylint doesn't recognize multiple inheritance properly, see # and # # pylint: disable=W0233 _RpcClientBase.__init__(self, resolver, encoders.get, lock_monitor_cb=lock_monitor_cb, _req_process_fn=_req_process_fn) _generated_rpc.RpcClientConfig.__init__(self) _generated_rpc.RpcClientBootstrap.__init__(self) _generated_rpc.RpcClientDnsOnly.__init__(self) _generated_rpc.RpcClientDefault.__init__(self) def _NicDict(self, _, nic): """Convert the given nic to a dict and encapsulate netinfo """ n = copy.deepcopy(nic) if n.network: net_uuid = self._cfg.LookupNetwork(n.network) if net_uuid: nobj = self._cfg.GetNetwork(net_uuid) n.netinfo = objects.Network.ToDict(nobj) return n.ToDict() def _DeviceDict(self, _, devinstance): (device, instance) = devinstance if isinstance(device, objects.NIC): return self._NicDict(None, device) elif isinstance(device, objects.Disk): return self._SingleDiskDictDP(None, (device, instance)) def _InstDict(self, node, instance, hvp=None, bep=None, osp=None): """Convert the given instance to a dict. This is done via the instance's ToDict() method and additionally we fill the hvparams with the cluster defaults. @type instance: L{objects.Instance} @param instance: an Instance object @type hvp: dict or None @param hvp: a dictionary with overridden hypervisor parameters @type bep: dict or None @param bep: a dictionary with overridden backend parameters @type osp: dict or None @param osp: a dictionary with overridden os parameters @rtype: dict @return: the instance dict, with the hvparams filled with the cluster defaults """ idict = instance.ToDict() cluster = self._cfg.GetClusterInfo() idict["hvparams"] = cluster.FillHV(instance) idict["secondary_nodes"] = \ self._cfg.GetInstanceSecondaryNodes(instance.uuid) if hvp is not None: idict["hvparams"].update(hvp) idict["beparams"] = cluster.FillBE(instance) if bep is not None: idict["beparams"].update(bep) idict["osparams"] = cluster.SimpleFillOS(instance.os, instance.osparams) if osp is not None: idict["osparams"].update(osp) disks = self._cfg.GetInstanceDisks(instance.uuid) idict["disks_info"] = self._DisksDictDP(node, (disks, instance)) for nic in idict["nics"]: nic["nicparams"] = objects.FillDict( cluster.nicparams[constants.PP_DEFAULT], nic["nicparams"]) network = nic.get("network", None) if network: net_uuid = self._cfg.LookupNetwork(network) if net_uuid: nobj = self._cfg.GetNetwork(net_uuid) nic["netinfo"] = objects.Network.ToDict(nobj) return idict def _InstDictHvpBepDp(self, node, instance_params): """Wrapper for L{_InstDict}. """ (instance, hvp, bep) = instance_params return self._InstDict(node, instance, hvp=hvp, bep=bep) def _InstDictOspDp(self, node, instance_osparams): """Wrapper for L{_InstDict}. """ (instance, osparams) = instance_osparams return self._InstDict(node, instance, osp=osparams) def _DisksDictDP(self, node, instance_disks): """Wrapper for L{AnnotateDiskParams}. """ (disks, instance) = instance_disks diskparams = self._cfg.GetInstanceDiskParams(instance) ret = [] for disk in AnnotateDiskParams(disks, diskparams): disk_node_uuids = disk.GetNodes(instance.primary_node) node_ips = dict((uuid, node.secondary_ip) for (uuid, node) in self._cfg.GetMultiNodeInfo(disk_node_uuids)) disk.UpdateDynamicDiskParams(node, node_ips) ret.append(disk.ToDict(include_dynamic_params=True)) return ret def _MultiDiskDictDP(self, node, disks_insts): """Wrapper for L{AnnotateDiskParams}. Supports a list of (disk, instance) tuples. """ return [disk for disk_inst in disks_insts for disk in self._DisksDictDP(node, disk_inst)] def _SingleDiskDictDP(self, node, instance_disk): """Wrapper for L{AnnotateDiskParams}. """ (disk, instance) = instance_disk anno_disk = self._DisksDictDP(node, ([disk], instance))[0] return anno_disk def _EncodeNodeToDiskDictDP(self, node, value): """Encode dict of node name -> list of (disk, instance) tuples as values. """ return dict((name, [self._SingleDiskDictDP(node, disk) for disk in disks]) for name, disks in value.items()) def _EncodeImportExportIO(self, node, ieinfo): """Encodes import/export I/O information. """ (ieio, ieioargs) = ieinfo if ieio == constants.IEIO_RAW_DISK: assert len(ieioargs) == 2 return (ieio, (self._SingleDiskDictDP(node, ieioargs), )) if ieio == constants.IEIO_SCRIPT: assert len(ieioargs) == 2 return (ieio, (self._SingleDiskDictDP(node, ieioargs[0]), ieioargs[1])) return (ieio, ieioargs) class JobQueueRunner(_RpcClientBase, _generated_rpc.RpcClientJobQueue): """RPC wrappers for job queue. """ def __init__(self, _context, address_list): """Initializes this class. """ if address_list is None: resolver = compat.partial(_SsconfResolver, True) else: # Caller provided an address list resolver = _StaticResolver(address_list) _RpcClientBase.__init__(self, resolver, _ENCODERS.get, lock_monitor_cb=lambda _: None) _generated_rpc.RpcClientJobQueue.__init__(self) class BootstrapRunner(_RpcClientBase, _generated_rpc.RpcClientBootstrap, _generated_rpc.RpcClientDnsOnly): """RPC wrappers for bootstrapping. """ def __init__(self): """Initializes this class. """ # Pylint doesn't recognize multiple inheritance properly, see # and # # pylint: disable=W0233 _RpcClientBase.__init__(self, compat.partial(_SsconfResolver, True), _ENCODERS.get) _generated_rpc.RpcClientBootstrap.__init__(self) _generated_rpc.RpcClientDnsOnly.__init__(self) class DnsOnlyRunner(_RpcClientBase, _generated_rpc.RpcClientDnsOnly): """RPC wrappers for calls using only DNS. """ def __init__(self): """Initialize this class. """ _RpcClientBase.__init__(self, compat.partial(_SsconfResolver, False), _ENCODERS.get) _generated_rpc.RpcClientDnsOnly.__init__(self) class ConfigRunner(_RpcClientBase, _generated_rpc.RpcClientConfig): """RPC wrappers for L{config}. """ def __init__(self, _context, address_list, _req_process_fn=None, _getents=None): """Initializes this class. """ lock_monitor_cb = None if address_list is None: resolver = compat.partial(_SsconfResolver, True) else: # Caller provided an address list resolver = _StaticResolver(address_list) encoders = _ENCODERS.copy() encoders.update({ rpc_defs.ED_FILE_DETAILS: compat.partial(_PrepareFileUpload, _getents), }) _RpcClientBase.__init__(self, resolver, encoders.get, lock_monitor_cb=lock_monitor_cb, _req_process_fn=_req_process_fn) _generated_rpc.RpcClientConfig.__init__(self) ganeti-3.1.0~rc2/lib/rpc/stub/000075500000000000000000000000001476477700300161135ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/rpc/stub/__init__.py000064400000000000000000000026731476477700300202340ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Empty file for package definition. Files in this module are generated automatically from Haskell during compilation. """ ganeti-3.1.0~rc2/lib/rpc/transport.py000064400000000000000000000241121476477700300175440ustar00rootroot00000000000000# # # Copyright (C) 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module that defines a transport for RPC connections. A transport can send to and receive messages from some endpoint. """ import collections import errno import io import logging import socket import time from ganeti import constants import ganeti.errors from ganeti import ssconf from ganeti import utils from ganeti.rpc import errors DEF_CTMO = constants.LUXI_DEF_CTMO DEF_RWTO = constants.LUXI_DEF_RWTO class Transport(object): """Low-level transport class. This is used on the client side. This could be replaced by any other class that provides the same semantics to the Client. This means: - can send messages and receive messages - safe for multithreading """ def __init__(self, address, timeouts=None, allow_non_master=None): """Constructor for the Client class. There are two timeouts used since we might want to wait for a long time for a response, but the connect timeout should be lower. If not passed, we use the default luxi timeouts from the global constants file. Note that on reading data, since the timeout applies to an invidual receive, it might be that the total duration is longer than timeout value passed (we make a hard limit at twice the read timeout). @type address: socket address @param address: address the transport connects to @type timeouts: list of ints @param timeouts: timeouts to be used on connect and read/write @type allow_non_master: bool @param allow_non_master: skip checks for the master node on errors """ self.address = address if timeouts is None: self._ctimeout, self._rwtimeout = DEF_CTMO, DEF_RWTO else: self._ctimeout, self._rwtimeout = timeouts self.socket = None self._buffer = b"" self._msgs = collections.deque() try: self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # Try to connect try: utils.Retry(self._Connect, 1.0, self._ctimeout, args=(self.socket, address, self._ctimeout, allow_non_master)) except utils.RetryTimeout: raise errors.TimeoutError("Connect timed out") self.socket.settimeout(self._rwtimeout) except (socket.error, errors.NoMasterError): if self.socket is not None: self.socket.close() self.socket = None raise @staticmethod def _Connect(sock, address, timeout, allow_non_master): sock.settimeout(timeout) try: sock.connect(address) except socket.timeout as err: raise errors.TimeoutError("Connect timed out: %s" % str(err)) except socket.error as err: error_code = err.args[0] if error_code in (errno.ENOENT, errno.ECONNREFUSED): if not allow_non_master: # Verify if we're actually on the master node before trying # again. ss = ssconf.SimpleStore() try: master, myself = ssconf.GetMasterAndMyself(ss=ss) except ganeti.errors.ConfigurationError: raise errors.NoMasterError(address) if master != myself: raise errors.NoMasterError(address) raise utils.RetryAgain() elif error_code in (errno.EPERM, errno.EACCES): raise errors.PermissionError(address) elif error_code == errno.EAGAIN: # Server's socket backlog is full at the moment raise utils.RetryAgain() raise def _CheckSocket(self): """Make sure we are connected. """ if self.socket is None: raise errors.ProtocolError("Connection is closed") def Send(self, msg): """Send a message. This just sends a message and doesn't wait for the response. """ if isinstance(msg, str): msg = msg.encode("utf-8") if constants.LUXI_EOM in msg: raise errors.ProtocolError("Message terminator found in payload") self._CheckSocket() try: # TODO: sendall is not guaranteed to send everything self.socket.sendall(msg + constants.LUXI_EOM) except socket.timeout as err: raise errors.TimeoutError("Sending timeout: %s" % str(err)) def Recv(self): """Try to receive a message from the socket. In case we already have messages queued, we just return from the queue. Otherwise, we try to read data with a _rwtimeout network timeout, and making sure we don't go over 2x_rwtimeout as a global limit. """ self._CheckSocket() etime = time.time() + self._rwtimeout while not self._msgs: if time.time() > etime: raise errors.TimeoutError("Extended receive timeout") while True: try: data = self.socket.recv(4096) except socket.timeout as err: raise errors.TimeoutError("Receive timeout: %s" % str(err)) except socket.error as err: if err.args and err.args[0] == errno.EAGAIN: continue raise break if not data: raise errors.ConnectionClosedError("Connection closed while reading") new_msgs = (self._buffer + data).split(constants.LUXI_EOM) self._buffer = new_msgs.pop() self._msgs.extend(new_msgs) return self._msgs.popleft().decode("utf-8") def Call(self, msg): """Send a message and wait for the response. This is just a wrapper over Send and Recv. """ self.Send(msg) return self.Recv() @staticmethod def RetryOnNetworkError(fn, on_error, retries=15, wait_on_error=5): """Calls a given function, retrying if it fails on a network IO exception. This allows to re-establish a broken connection and retry an IO operation. The function receives one an integer argument stating the current retry number, 0 being the first call, 1 being the first retry, 2 the second, and so on. If any exception occurs, on_error is invoked first with the exception given as an argument. Then, if the exception is a network exception, the function call is retried once more. """ for try_no in range(0, retries): try: return fn(try_no) except (socket.error, errors.ConnectionClosedError) as ex: on_error(ex) # we retry on a network error, unless it's the last try if try_no == retries - 1: raise logging.error("Network error: %s, retrying (retry attempt number %d)", ex, try_no + 1) time.sleep(wait_on_error * try_no) except Exception as ex: on_error(ex) raise assert False # we should never get here def Close(self): """Close the socket""" if self.socket is not None: self.socket.close() self.socket = None class FdTransport(object): """Low-level transport class that works on arbitrary file descriptors. Unlike L{Transport}, this doesn't use timeouts. """ def __init__(self, fds, timeouts=None, allow_non_master=None): # pylint: disable=W0613 """Constructor for the Client class. @type fds: pair of file descriptors @param fds: the file descriptor for reading (the first in the pair) and the file descriptor for writing (the second) @type timeouts: int @param timeouts: unused @type allow_non_master: bool @param allow_non_master: unused """ self._rstream = io.open(fds[0], 'rb', 0) self._wstream = io.open(fds[1], 'wb', 0) self._buffer = b"" self._msgs = collections.deque() def _CheckSocket(self): """Make sure we are connected. """ if self._rstream is None or self._wstream is None: raise errors.ProtocolError("Connection is closed") def Send(self, msg): """Send a message. This just sends a message and doesn't wait for the response. """ if isinstance(msg, str): msg = msg.encode("utf-8") if constants.LUXI_EOM in msg: raise errors.ProtocolError("Message terminator found in payload") self._CheckSocket() self._wstream.write(msg + constants.LUXI_EOM) self._wstream.flush() def Recv(self): """Try to receive a message from the read part of the socket. In case we already have messages queued, we just return from the queue. """ self._CheckSocket() while not self._msgs: data = self._rstream.read(4096) if not data: raise errors.ConnectionClosedError("Connection closed while reading") new_msgs = (self._buffer + data).split(constants.LUXI_EOM) self._buffer = new_msgs.pop() self._msgs.extend(new_msgs) return self._msgs.popleft().decode("utf-8") def Call(self, msg): """Send a message and wait for the response. This is just a wrapper over Send and Recv. """ self.Send(msg) return self.Recv() def Close(self): """Close the socket""" if self._rstream is not None: self._rstream.close() self._rstream = None if self._wstream is not None: self._wstream.close() self._wstream = None def close(self): self.Close() ganeti-3.1.0~rc2/lib/rpc_defs.py000064400000000000000000000751711476477700300165240ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """RPC definitions for communication between master and node daemons. RPC definition fields: - Name as string - L{SINGLE} for single-node calls, L{MULTI} for multi-node - Name resolver option(s), can be callable receiving all arguments in a tuple - Timeout (e.g. L{constants.RPC_TMO_NORMAL}), or callback receiving all arguments in a tuple to calculate timeout - List of arguments as tuples - Name as string - Argument kind used for encoding/decoding - Description for docstring (can be C{None}) - Custom body encoder (e.g. for preparing per-node bodies) - Return value wrapper (e.g. for deserializing into L{objects}-based objects) - Short call description for docstring """ from ganeti import constants from ganeti import utils from ganeti import objects # Guidelines for choosing timeouts: # - call used during watcher: timeout of 1min, constants.RPC_TMO_URGENT # - trivial (but be sure it is trivial) # (e.g. reading a file): 5min, constants.RPC_TMO_FAST # - other calls: 15 min, constants.RPC_TMO_NORMAL # - special calls (instance add, etc.): # either constants.RPC_TMO_SLOW (1h) or huge timeouts SINGLE = "single-node" MULTI = "multi-node" ACCEPT_OFFLINE_NODE = object() # Constants for encoding/decoding (ED_OBJECT_DICT, ED_OBJECT_DICT_LIST, ED_INST_DICT, ED_INST_DICT_HVP_BEP_DP, ED_NODE_TO_DISK_DICT_DP, ED_INST_DICT_OSP_DP, ED_IMPEXP_IO, ED_FILE_DETAILS, ED_FINALIZE_EXPORT_DISKS, ED_COMPRESS, ED_BLOCKDEV_RENAME, ED_DISKS_DICT_DP, ED_MULTI_DISKS_DICT_DP, ED_SINGLE_DISK_DICT_DP, ED_NIC_DICT, ED_DEVICE_DICT) = range(1, 17) def _Prepare(calls): """Converts list of calls to dictionary. """ return utils.SequenceToDict(calls) def _MigrationStatusPostProc(result): """Post-processor for L{rpc.node.RpcRunner.call_instance_get_migration_status} """ if not result.fail_msg and result.payload is not None: result.payload = objects.MigrationStatus.FromDict(result.payload) return result def _BlockdevFindPostProc(result): """Post-processor for L{rpc.node.RpcRunner.call_blockdev_find}. """ if not result.fail_msg and result.payload is not None: result.payload = objects.BlockDevStatus.FromDict(result.payload) return result def _BlockdevGetMirrorStatusPostProc(result): """Post-processor for call_blockdev_getmirrorstatus. """ if not result.fail_msg: result.payload = [objects.BlockDevStatus.FromDict(d) for d in result.payload] return result def _BlockdevGetMirrorStatusMultiPreProc(node, args): """Prepares the appropriate node values for blockdev_getmirrorstatus_multi. """ # there should be only one argument to this RPC, already holding a # node->disks dictionary, we just need to extract the value for the # current node assert len(args) == 1 return [args[0][node]] def _BlockdevGetMirrorStatusMultiPostProc(result): """Post-processor for call_blockdev_getmirrorstatus_multi. """ if not result.fail_msg: for idx, (success, status) in enumerate(result.payload): if success: result.payload[idx] = (success, objects.BlockDevStatus.FromDict(status)) return result def _NodeInfoPreProc(node, args): """Prepare the storage_units argument for node_info calls.""" assert len(args) == 2 # The storage_units argument is either a dictionary with one value for each # node, or a fixed value to be used for all the nodes if isinstance(args[0], dict): return [args[0][node], args[1]] else: return args def _ImpExpStatusPostProc(result): """Post-processor for import/export status. @rtype: Payload containing list of L{objects.ImportExportStatus} instances @return: Returns a list of the state of each named import/export or None if a status couldn't be retrieved """ if not result.fail_msg: decoded = [] for i in result.payload: if i is None: decoded.append(None) continue decoded.append(objects.ImportExportStatus.FromDict(i)) result.payload = decoded return result def _TestDelayTimeout(duration): """Calculate timeout for "test_delay" RPC. """ _duration = duration[0] return int(_duration + 5) _FILE_STORAGE_CALLS = [ ("file_storage_dir_create", SINGLE, None, constants.RPC_TMO_FAST, [ ("file_storage_dir", None, "File storage directory"), ], None, None, "Create the given file storage directory"), ("file_storage_dir_remove", SINGLE, None, constants.RPC_TMO_FAST, [ ("file_storage_dir", None, "File storage directory"), ], None, None, "Remove the given file storage directory"), ("file_storage_dir_rename", SINGLE, None, constants.RPC_TMO_FAST, [ ("old_file_storage_dir", None, "Old name"), ("new_file_storage_dir", None, "New name"), ], None, None, "Rename file storage directory"), ] _STORAGE_CALLS = [ ("storage_list", MULTI, None, constants.RPC_TMO_NORMAL, [ ("su_name", None, None), ("su_args", None, None), ("name", None, None), ("fields", None, None), ], None, None, "Get list of storage units"), ("storage_modify", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("su_name", None, None), ("su_args", None, None), ("name", None, None), ("changes", None, None), ], None, None, "Modify a storage unit"), ("storage_execute", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("su_name", None, None), ("su_args", None, None), ("name", None, None), ("op", None, None), ], None, None, "Executes an operation on a storage unit"), ] _INSTANCE_CALLS = [ ("instance_info", SINGLE, None, constants.RPC_TMO_URGENT, [ ("instance", None, "Instance name"), ("hname", None, "Hypervisor type"), ("hvparams", None, "Hypervisor parameters"), ], None, None, "Returns information about a single instance"), ("all_instances_info", MULTI, None, constants.RPC_TMO_URGENT, [ ("hypervisor_list", None, "Hypervisors to query for instances"), ("all_hvparams", None, "Dictionary mapping hypervisor names to hvparams"), ], None, None, "Returns information about all instances on the given nodes"), ("instance_list", MULTI, None, constants.RPC_TMO_URGENT, [ ("hypervisor_list", None, "Hypervisors to query for instances"), ("hvparams", None, "Hvparams of all hypervisors"), ], None, None, "Returns the list of running instances on the given nodes"), ("instance_reboot", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("inst", ED_INST_DICT, "Instance object"), ("reboot_type", None, None), ("shutdown_timeout", None, None), ("reason", None, "The reason for the reboot"), ], None, None, "Returns the list of running instances on the given nodes"), ("instance_shutdown", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, "Instance object"), ("timeout", None, None), ("reason", None, "The reason for the shutdown"), ], None, None, "Stops an instance"), ("instance_balloon_memory", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, "Instance object"), ("memory", None, None), ], None, None, "Modify the amount of an instance's runtime memory"), ("instance_run_rename", SINGLE, None, constants.RPC_TMO_SLOW, [ ("instance", ED_INST_DICT, "Instance object"), ("old_name", None, None), ("debug", None, None), ], None, None, "Run the OS rename script for an instance"), ("instance_migratable", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, "Instance object"), ], None, None, "Checks whether the given instance can be migrated"), ("migration_info", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, "Instance object"), ], None, None, "Gather the information necessary to prepare an instance migration"), ("accept_instance", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, "Instance object"), ("info", None, "Result for the call_migration_info call"), ("target", None, "Target hostname (usually an IP address)"), ], None, None, "Prepare a node to accept an instance"), ("instance_finalize_migration_dst", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, "Instance object"), ("info", None, "Result for the call_migration_info call"), ("success", None, "Whether the migration was a success or failure"), ], None, None, "Finalize any target-node migration specific operation"), ("instance_migrate", SINGLE, None, constants.RPC_TMO_SLOW, [ ("cluster_name", None, "Cluster name"), ("instance", ED_INST_DICT, "Instance object"), ("target", None, "Target node name"), ("live", None, "Whether the migration should be done live or not"), ], None, None, "Migrate an instance"), ("instance_finalize_migration_src", SINGLE, None, constants.RPC_TMO_SLOW, [ ("instance", ED_INST_DICT, "Instance object"), ("success", None, "Whether the migration succeeded or not"), ("live", None, "Whether the user requested a live migration or not"), ], None, None, "Finalize the instance migration on the source node"), ("instance_get_migration_status", SINGLE, None, constants.RPC_TMO_SLOW, [ ("instance", ED_INST_DICT, "Instance object"), ], None, _MigrationStatusPostProc, "Report migration status"), ("instance_start", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance_hvp_bep", ED_INST_DICT_HVP_BEP_DP, None), ("startup_paused", None, None), ("reason", None, "The reason for the startup"), ], None, None, "Starts an instance"), ("instance_os_add", SINGLE, None, constants.RPC_TMO_1DAY, [ ("instance_osp", ED_INST_DICT_OSP_DP, "Tuple: (target instance," " temporary OS parameters" " overriding configuration)"), ("reinstall", None, "Whether the instance is being reinstalled"), ("debug", None, "Debug level for the OS install script to use"), ], None, None, "Installs an operative system onto an instance"), ("hotplug_device", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, "Instance object"), ("action", None, "Hotplug Action"), ("dev_type", None, "Device type"), ("device", ED_DEVICE_DICT, "Device dict"), ("extra", None, "Extra info for device (dev_path for disk)"), ("seq", None, "Device seq"), ], None, None, "Hotplug a device to a running instance"), ("resize_disk", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, "Instance object"), ("disk", None, "The disk to be resized"), ("new_size", None, "The new disk size in bytes"), ], None, None, "Notify the HV about a disk resize"), ("hotplug_supported", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, "Instance object"), ], None, None, "Check if hotplug is supported"), ("instance_metadata_modify", SINGLE, None, constants.RPC_TMO_URGENT, [ ("instance", None, "Instance object"), ], None, None, "Modify instance metadata"), ] _IMPEXP_CALLS = [ ("import_start", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("opts", ED_OBJECT_DICT, None), ("instance", ED_INST_DICT, None), ("component", None, None), ("dest", ED_IMPEXP_IO, "Import destination"), ], None, None, "Starts an import daemon"), ("export_start", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("opts", ED_OBJECT_DICT, None), ("host", None, None), ("port", None, None), ("instance", ED_INST_DICT, None), ("component", None, None), ("source", ED_IMPEXP_IO, "Export source"), ], None, None, "Starts an export daemon"), ("impexp_status", SINGLE, None, constants.RPC_TMO_FAST, [ ("names", None, "Import/export names"), ], None, _ImpExpStatusPostProc, "Gets the status of an import or export"), ("impexp_abort", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("name", None, "Import/export name"), ], None, None, "Aborts an import or export"), ("impexp_cleanup", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("name", None, "Import/export name"), ], None, None, "Cleans up after an import or export"), ("export_info", SINGLE, None, constants.RPC_TMO_FAST, [ ("path", None, None), ], None, None, "Queries the export information in a given path"), ("finalize_export", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance", ED_INST_DICT, None), ("snap_disks", ED_FINALIZE_EXPORT_DISKS, None), ], None, None, "Request the completion of an export operation"), ("export_list", MULTI, None, constants.RPC_TMO_FAST, [], None, None, "Gets the stored exports list"), ("export_remove", SINGLE, None, constants.RPC_TMO_FAST, [ ("export", None, None), ], None, None, "Requests removal of a given export"), ] _X509_CALLS = [ ("x509_cert_create", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("validity", None, "Validity in seconds"), ], None, None, "Creates a new X509 certificate for SSL/TLS"), ("x509_cert_remove", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("name", None, "Certificate name"), ], None, None, "Removes a X509 certificate"), ] _BLOCKDEV_CALLS = [ ("bdev_sizes", MULTI, None, constants.RPC_TMO_URGENT, [ ("devices", None, None), ], None, None, "Gets the sizes of requested block devices present on a node"), ("blockdev_create", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("bdev", ED_SINGLE_DISK_DICT_DP, None), ("size", None, None), ("owner", None, None), ("on_primary", None, None), ("info", None, None), ("exclusive_storage", None, None), ], None, None, "Request creation of a given block device"), ("blockdev_convert", SINGLE, None, constants.RPC_TMO_SLOW, [ ("bdev_src", ED_SINGLE_DISK_DICT_DP, None), ("bdev_dest", ED_SINGLE_DISK_DICT_DP, None), ], None, None, "Request the copy of the source block device to the destination one"), ("blockdev_image", SINGLE, None, constants.RPC_TMO_SLOW, [ ("bdev", ED_SINGLE_DISK_DICT_DP, None), ("image", None, None), ("size", None, None), ], None, None, "Request to dump an image with given size onto a block device"), ("blockdev_wipe", SINGLE, None, constants.RPC_TMO_SLOW, [ ("bdev", ED_SINGLE_DISK_DICT_DP, None), ("offset", None, None), ("size", None, None), ], None, None, "Request wipe at given offset with given size of a block device"), ("blockdev_remove", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("bdev", ED_SINGLE_DISK_DICT_DP, None), ], None, None, "Request removal of a given block device"), ("blockdev_pause_resume_sync", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("disks", ED_DISKS_DICT_DP, None), ("pause", None, None), ], None, None, "Request a pause/resume of given block device"), ("blockdev_assemble", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("disk", ED_SINGLE_DISK_DICT_DP, None), ("instance", ED_INST_DICT, None), ("on_primary", None, None), ("idx", None, None), ], None, None, "Request assembling of a given block device"), ("blockdev_shutdown", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("disk", ED_SINGLE_DISK_DICT_DP, None), ], None, None, "Request shutdown of a given block device"), ("blockdev_addchildren", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("bdev", ED_SINGLE_DISK_DICT_DP, None), ("ndevs", ED_DISKS_DICT_DP, None), ], None, None, "Request adding a list of children to a (mirroring) device"), ("blockdev_removechildren", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("bdev", ED_SINGLE_DISK_DICT_DP, None), ("ndevs", ED_DISKS_DICT_DP, None), ], None, None, "Request removing a list of children from a (mirroring) device"), ("blockdev_close", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance_name", None, None), ("disks", ED_DISKS_DICT_DP, None), ], None, None, "Closes the given block devices"), ("blockdev_open", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("instance_name", None, None), ("disks", ED_DISKS_DICT_DP, None), ("exclusive", None, None), ], None, None, "Opens the given block devices in required mode"), ("blockdev_getdimensions", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("disks", ED_MULTI_DISKS_DICT_DP, None), ], None, None, "Returns size and spindles of the given disks"), ("drbd_disconnect_net", MULTI, None, constants.RPC_TMO_NORMAL, [ ("disks", ED_DISKS_DICT_DP, None), ], None, None, "Disconnects the network of the given drbd devices"), ("drbd_attach_net", MULTI, None, constants.RPC_TMO_NORMAL, [ ("disks", ED_DISKS_DICT_DP, None), ("multimaster", None, None), ], None, None, "Connects the given DRBD devices"), ("drbd_wait_sync", MULTI, None, constants.RPC_TMO_SLOW, [ ("disks", ED_DISKS_DICT_DP, None), ], None, None, "Waits for the synchronization of drbd devices is complete"), ("drbd_needs_activation", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("disks", ED_MULTI_DISKS_DICT_DP, None), ], None, None, "Returns the drbd disks which need activation"), ("blockdev_grow", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("cf_bdev", ED_SINGLE_DISK_DICT_DP, None), ("amount", None, None), ("dryrun", None, None), ("backingstore", None, None), ("es_flag", None, None), ], None, None, "Request growing of the given block device by a" " given amount"), ("blockdev_snapshot", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("cf_bdev", ED_SINGLE_DISK_DICT_DP, None), ("snap_name", None, None), ("snap_size", None, None), ], None, None, "Export a given disk to another node"), ("blockdev_rename", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("devlist", ED_BLOCKDEV_RENAME, None), ], None, None, "Request rename of the given block devices"), ("blockdev_find", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("disk", ED_SINGLE_DISK_DICT_DP, None), ], None, _BlockdevFindPostProc, "Request identification of a given block device"), ("blockdev_getmirrorstatus", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("disks", ED_DISKS_DICT_DP, None), ], None, _BlockdevGetMirrorStatusPostProc, "Request status of a (mirroring) device"), ("blockdev_getmirrorstatus_multi", MULTI, None, constants.RPC_TMO_NORMAL, [ ("node_disks", ED_NODE_TO_DISK_DICT_DP, None), ], _BlockdevGetMirrorStatusMultiPreProc, _BlockdevGetMirrorStatusMultiPostProc, "Request status of (mirroring) devices from multiple nodes"), ("blockdev_setinfo", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("disk", ED_SINGLE_DISK_DICT_DP, None), ("info", None, None), ], None, None, "Sets metadata information on a given block device"), ] _OS_CALLS = [ ("os_diagnose", MULTI, None, constants.RPC_TMO_FAST, [], None, None, "Request a diagnose of OS definitions"), ("os_validate", MULTI, None, constants.RPC_TMO_FAST, [ ("required", None, None), ("name", None, None), ("checks", None, None), ("params", None, None), ("force_variant", None, None), ], None, None, "Run a validation routine for a given OS"), ("os_export", SINGLE, None, constants.RPC_TMO_FAST, [ ("instance", ED_INST_DICT, None), ("override_env", None, None), ], None, None, "Export an OS for a given instance"), ] _EXTSTORAGE_CALLS = [ ("extstorage_diagnose", MULTI, None, constants.RPC_TMO_FAST, [], None, None, "Request a diagnose of ExtStorage Providers"), ] _NODE_CALLS = [ ("node_has_ip_address", SINGLE, None, constants.RPC_TMO_FAST, [ ("address", None, "IP address"), ], None, None, "Checks if a node has the given IP address"), ("node_info", MULTI, None, constants.RPC_TMO_URGENT, [ ("storage_units", None, "List of tuples ',,[]' to ask for disk space" " information; the parameter list varies depending on the storage_type"), ("hv_specs", None, "List of hypervisor specification (name, hvparams) to ask for node " "information"), ], _NodeInfoPreProc, None, "Return node information"), ("node_verify", MULTI, None, constants.RPC_TMO_NORMAL, [ ("checkdict", None, "What to verify"), ("cluster_name", None, "Cluster name"), ("all_hvparams", None, "Dictionary mapping hypervisor names to hvparams"), ], None, None, "Request verification of given parameters"), ("node_volumes", MULTI, None, constants.RPC_TMO_FAST, [], None, None, "Gets all volumes on node(s)"), ("node_demote_from_mc", SINGLE, None, constants.RPC_TMO_FAST, [], None, None, "Demote a node from the master candidate role"), ("node_powercycle", SINGLE, ACCEPT_OFFLINE_NODE, constants.RPC_TMO_NORMAL, [ ("hypervisor", None, "Hypervisor type"), ("hvparams", None, "Hypervisor parameters"), ], None, None, "Tries to powercycle a node"), ("node_configure_ovs", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("ovs_name", None, "Name of the OpenvSwitch to create"), ("ovs_link", None, "Link of the OpenvSwitch to the outside"), ], None, None, "This will create and setup the OpenvSwitch"), ("node_crypto_tokens", SINGLE, None, constants.RPC_TMO_SLOW, [ ("token_request", None, "List of tuples of requested crypto token types, actions"), ], None, None, "Handle crypto tokens of the node."), ("node_ensure_daemon", MULTI, None, constants.RPC_TMO_URGENT, [ ("daemon", None, "Daemon name"), ("run", None, "Whether the daemon should be running or stopped"), ], None, None, "Ensure daemon is running on the node."), ("node_ssh_key_add", MULTI, None, constants.RPC_TMO_FAST, [ ("node_uuid", None, "UUID of the node whose key is distributed"), ("node_name", None, "Name of the node whose key is distributed"), ("potential_master_candidates", None, "Potential master candidates"), ("to_authorized_keys", None, "Whether the node's key should be added" " to all nodes' 'authorized_keys' file"), ("to_public_keys", None, "Whether the node's key should be added" " to all nodes' public key file"), ("get_public_keys", None, "Whether the node should get the other nodes'" " public keys")], None, None, "Distribute a new node's public SSH key on the cluster."), ("node_ssh_key_remove", MULTI, None, constants.RPC_TMO_FAST, [ ("node_uuid", None, "UUID of the node whose key is removed"), ("node_name", None, "Name of the node whose key is removed"), ("master_candidate_uuids", None, "List of UUIDs of master candidates."), ("potential_master_candidates", None, "Potential master candidates"), ("from_authorized_keys", None, "If the key should be removed from the 'authorized_keys' file."), ("from_public_keys", None, "If the key should be removed from the public key file."), ("clear_authorized_keys", None, "If the 'authorized_keys' file of the node should be cleared."), ("clear_public_keys", None, "If the 'ganeti_pub_keys' file of the node should be cleared."), ("readd", None, "Whether this is a readd operation.")], None, None, "Remove a node's SSH key from the other nodes' key files."), ("node_ssh_keys_renew", MULTI, None, constants.RPC_TMO_4HRS, [ ("node_uuids", None, "UUIDs of the nodes whose key is renewed"), ("node_names", None, "Names of the nodes whose key is renewed"), ("master_candidate_uuids", None, "List of UUIDs of master candidates."), ("potential_master_candidates", None, "Potential master candidates"), ("old_key_type", None, "The type of key previously used"), ("new_key_type", None, "The type of key to generate"), ("new_key_bits", None, "The length of the key to generate")], None, None, "Renew all SSH key pairs of all nodes nodes."), ] _MISC_CALLS = [ ("lv_list", MULTI, None, constants.RPC_TMO_URGENT, [ ("vg_name", None, None), ], None, None, "Gets the logical volumes present in a given volume group"), ("vg_list", MULTI, None, constants.RPC_TMO_URGENT, [], None, None, "Gets the volume group list"), ("bridges_exist", SINGLE, None, constants.RPC_TMO_URGENT, [ ("bridges_list", None, "Bridges which must be present on remote node"), ], None, None, "Checks if a node has all the bridges given"), ("etc_hosts_modify", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("mode", None, "Mode to operate; currently L{constants.ETC_HOSTS_ADD} or" " L{constants.ETC_HOSTS_REMOVE}"), ("name", None, "Hostname to be modified"), ("ip", None, "IP address (L{constants.ETC_HOSTS_ADD} only)"), ], None, None, "Modify hosts file with name"), ("drbd_helper", MULTI, None, constants.RPC_TMO_URGENT, [], None, None, "Gets DRBD helper"), ("restricted_command", MULTI, None, constants.RPC_TMO_SLOW, [ ("cmd", None, "Command name"), ], None, None, "Runs restricted command"), ("run_oob", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("oob_program", None, None), ("command", None, None), ("remote_node", None, None), ("timeout", None, None), ], None, None, "Runs out-of-band command"), ("hooks_runner", MULTI, None, constants.RPC_TMO_NORMAL, [ ("hpath", None, None), ("phase", None, None), ("env", None, None), ], None, None, "Call the hooks runner"), ("iallocator_runner", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("name", None, "Iallocator name"), ("idata", None, "JSON-encoded input string"), ("default_iallocator_params", None, "Additional iallocator parameters"), ], None, None, "Call an iallocator on a remote node"), ("test_delay", MULTI, None, _TestDelayTimeout, [ ("duration", None, None), ], None, None, "Sleep for a fixed time on given node(s)"), ("hypervisor_validate_params", MULTI, None, constants.RPC_TMO_NORMAL, [ ("hvname", None, "Hypervisor name"), ("hvfull", None, "Parameters to be validated"), ], None, None, "Validate hypervisor params"), ("get_watcher_pause", SINGLE, None, constants.RPC_TMO_URGENT, [], None, None, "Get watcher pause end"), ("set_watcher_pause", MULTI, None, constants.RPC_TMO_URGENT, [ ("until", None, None), ], None, None, "Set watcher pause end"), ("get_file_info", SINGLE, None, constants.RPC_TMO_FAST, [ ("file_path", None, None), ], None, None, "Checks if a file exists and reports on it"), ] CALLS = { "RpcClientDefault": _Prepare(_IMPEXP_CALLS + _X509_CALLS + _OS_CALLS + _NODE_CALLS + _FILE_STORAGE_CALLS + _MISC_CALLS + _INSTANCE_CALLS + _BLOCKDEV_CALLS + _STORAGE_CALLS + _EXTSTORAGE_CALLS), "RpcClientJobQueue": _Prepare([ ("jobqueue_update", MULTI, None, constants.RPC_TMO_URGENT, [ ("file_name", None, None), ("content", ED_COMPRESS, None), ], None, None, "Update job queue file"), ("jobqueue_purge", SINGLE, None, constants.RPC_TMO_NORMAL, [], None, None, "Purge job queue"), ("jobqueue_rename", MULTI, None, constants.RPC_TMO_URGENT, [ ("rename", None, None), ], None, None, "Rename job queue file"), ("jobqueue_set_drain_flag", MULTI, None, constants.RPC_TMO_URGENT, [ ("flag", None, None), ], None, None, "Set job queue drain flag"), ]), "RpcClientBootstrap": _Prepare([ ("node_start_master_daemons", SINGLE, None, constants.RPC_TMO_FAST, [ ("no_voting", None, None), ], None, None, "Starts master daemons on a node"), ("node_activate_master_ip", SINGLE, None, constants.RPC_TMO_FAST, [ ("master_params", ED_OBJECT_DICT, "Network parameters of the master"), ("use_external_mip_script", None, "Whether to use the user-provided master IP address setup script"), ], None, None, "Activates master IP on a node"), ("node_stop_master", SINGLE, None, constants.RPC_TMO_FAST, [], None, None, "Deactivates master IP and stops master daemons on a node"), ("node_deactivate_master_ip", SINGLE, None, constants.RPC_TMO_FAST, [ ("master_params", ED_OBJECT_DICT, "Network parameters of the master"), ("use_external_mip_script", None, "Whether to use the user-provided master IP address setup script"), ], None, None, "Deactivates master IP on a node"), ("node_change_master_netmask", SINGLE, None, constants.RPC_TMO_FAST, [ ("old_netmask", None, "The old value of the netmask"), ("netmask", None, "The new value of the netmask"), ("master_ip", None, "The master IP"), ("master_netdev", None, "The master network device"), ], None, None, "Change master IP netmask"), ("node_leave_cluster", SINGLE, None, constants.RPC_TMO_NORMAL, [ ("modify_ssh_setup", None, None), ], None, None, "Requests a node to clean the cluster information it has"), ("master_node_name", MULTI, None, constants.RPC_TMO_URGENT, [], None, None, "Returns the master node name"), ]), "RpcClientDnsOnly": _Prepare([ ("version", MULTI, ACCEPT_OFFLINE_NODE, constants.RPC_TMO_URGENT, [], None, None, "Query node version"), ("node_verify_light", MULTI, None, constants.RPC_TMO_NORMAL, [ ("checkdict", None, "What to verify"), ("cluster_name", None, "Cluster name"), ("hvparams", None, "Dictionary mapping hypervisor names to hvparams"), ], None, None, "Request verification of given parameters"), ]), "RpcClientConfig": _Prepare([ ("upload_file", MULTI, None, constants.RPC_TMO_NORMAL, [ ("file_name", ED_FILE_DETAILS, None), ], None, None, "Upload files"), ("upload_file_single", MULTI, None, constants.RPC_TMO_NORMAL, [ ("file_name", None, "The name of the file"), ("content", ED_COMPRESS, "The data to be uploaded"), ("mode", None, "The mode of the file or None"), ("uid", None, "The owner of the file"), ("gid", None, "The group of the file"), ("atime", None, "The file's last access time"), ("mtime", None, "The file's last modification time"), ], None, None, "Upload files"), ("write_ssconf_files", MULTI, None, constants.RPC_TMO_NORMAL, [ ("values", None, None), ], None, None, "Write ssconf files"), ]), } ganeti-3.1.0~rc2/lib/runtime.py000064400000000000000000000221071476477700300164110ustar00rootroot00000000000000# # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing configuration details at runtime. """ import grp import pwd import threading import platform from ganeti import constants from ganeti import errors from ganeti import luxi from ganeti.rpc.errors import NoMasterError from ganeti import pathutils from ganeti import ssconf from ganeti import utils _priv = None _priv_lock = threading.Lock() #: Architecture information _arch = None def GetUid(user, _getpwnam): """Retrieve the uid from the database. @type user: string @param user: The username to retrieve @return: The resolved uid """ try: return _getpwnam(user).pw_uid except KeyError as err: raise errors.ConfigurationError("User '%s' not found (%s)" % (user, err)) def GetGid(group, _getgrnam): """Retrieve the gid from the database. @type group: string @param group: The group name to retrieve @return: The resolved gid """ try: return _getgrnam(group).gr_gid except KeyError as err: raise errors.ConfigurationError("Group '%s' not found (%s)" % (group, err)) class GetentResolver(object): """Resolves Ganeti uids and gids by name. @ivar masterd_uid: The resolved uid of the masterd user @ivar masterd_gid: The resolved gid of the masterd group @ivar confd_uid: The resolved uid of the confd user @ivar confd_gid: The resolved gid of the confd group @ivar wconfd_uid: The resolved uid of the wconfd user @ivar wconfd_gid: The resolved gid of the wconfd group @ivar luxid_uid: The resolved uid of the luxid user @ivar luxid_gid: The resolved gid of the luxid group @ivar rapi_uid: The resolved uid of the rapi user @ivar rapi_gid: The resolved gid of the rapi group @ivar noded_uid: The resolved uid of the noded user @ivar noded_gid: The resolved uid of the noded group @ivar mond_uid: The resolved uid of the mond user @ivar mond_gid: The resolved gid of the mond group @ivar metad_uid: The resolved uid of the metad user @ivar metad_gid: The resolved gid of the metad group @ivar daemons_gid: The resolved gid of the daemons group @ivar admin_gid: The resolved gid of the admin group """ def __init__(self, _getpwnam=pwd.getpwnam, _getgrnam=grp.getgrnam): """Initialize the resolver. """ # Daemon pairs self.masterd_uid = GetUid(constants.MASTERD_USER, _getpwnam) self.masterd_gid = GetGid(constants.MASTERD_GROUP, _getgrnam) self.confd_uid = GetUid(constants.CONFD_USER, _getpwnam) self.confd_gid = GetGid(constants.CONFD_GROUP, _getgrnam) self.wconfd_uid = GetUid(constants.WCONFD_USER, _getpwnam) self.wconfd_gid = GetGid(constants.WCONFD_GROUP, _getgrnam) self.luxid_uid = GetUid(constants.LUXID_USER, _getpwnam) self.luxid_gid = GetGid(constants.LUXID_GROUP, _getgrnam) self.rapi_uid = GetUid(constants.RAPI_USER, _getpwnam) self.rapi_gid = GetGid(constants.RAPI_GROUP, _getgrnam) self.noded_uid = GetUid(constants.NODED_USER, _getpwnam) self.noded_gid = GetGid(constants.NODED_GROUP, _getgrnam) self.mond_uid = GetUid(constants.MOND_USER, _getpwnam) self.mond_gid = GetGid(constants.MOND_GROUP, _getgrnam) self.metad_uid = GetUid(constants.METAD_USER, _getpwnam) self.metad_gid = GetGid(constants.METAD_GROUP, _getgrnam) # Misc Ganeti groups self.daemons_gid = GetGid(constants.DAEMONS_GROUP, _getgrnam) self.admin_gid = GetGid(constants.ADMIN_GROUP, _getgrnam) self._uid2user = { self.masterd_uid: constants.MASTERD_USER, self.confd_uid: constants.CONFD_USER, self.wconfd_uid: constants.WCONFD_USER, self.luxid_uid: constants.LUXID_USER, self.rapi_uid: constants.RAPI_USER, self.noded_uid: constants.NODED_USER, self.metad_uid: constants.METAD_USER, self.mond_uid: constants.MOND_USER, } self._gid2group = { self.masterd_gid: constants.MASTERD_GROUP, self.confd_gid: constants.CONFD_GROUP, self.wconfd_gid: constants.WCONFD_GROUP, self.luxid_gid: constants.LUXID_GROUP, self.rapi_gid: constants.RAPI_GROUP, self.noded_gid: constants.NODED_GROUP, self.mond_gid: constants.MOND_GROUP, self.metad_gid: constants.METAD_GROUP, self.daemons_gid: constants.DAEMONS_GROUP, self.admin_gid: constants.ADMIN_GROUP, } self._user2uid = utils.InvertDict(self._uid2user) self._group2gid = utils.InvertDict(self._gid2group) def LookupUid(self, uid): """Looks which Ganeti user belongs to this uid. @param uid: The uid to lookup @returns The user name associated with that uid """ try: return self._uid2user[uid] except KeyError: raise errors.ConfigurationError("Unknown Ganeti uid '%d'" % uid) def LookupGid(self, gid): """Looks which Ganeti group belongs to this gid. @param gid: The gid to lookup @returns The group name associated with that gid """ try: return self._gid2group[gid] except KeyError: raise errors.ConfigurationError("Unknown Ganeti gid '%d'" % gid) def LookupUser(self, name): """Looks which uid belongs to this name. @param name: The name to lookup @returns The uid associated with that user name """ try: return self._user2uid[name] except KeyError: raise errors.ConfigurationError("Unknown Ganeti user '%s'" % name) def LookupGroup(self, name): """Looks which gid belongs to this name. @param name: The name to lookup @returns The gid associated with that group name """ try: return self._group2gid[name] except KeyError: raise errors.ConfigurationError("Unknown Ganeti group '%s'" % name) def GetEnts(resolver=GetentResolver): """Singleton wrapper around resolver instance. As this method is accessed by multiple threads at the same time we need to take thread-safety carefully. """ # We need to use the global keyword here global _priv # pylint: disable=W0603 if not _priv: _priv_lock.acquire() try: if not _priv: # W0621: Redefine '_priv' from outer scope (used for singleton) _priv = resolver() # pylint: disable=W0621 finally: _priv_lock.release() return _priv def InitArchInfo(): """Initialize architecture information. We can assume this information never changes during the lifetime of a process, therefore the information can easily be cached. @note: This function uses C{platform.architecture} to retrieve the Python binary architecture and does so by forking to run C{file} (see Python documentation for more information). Therefore it must not be used in a multi-threaded environment. """ global _arch # pylint: disable=W0603 if _arch is not None: raise errors.ProgrammerError("Architecture information can only be" " initialized once") _arch = (platform.architecture()[0], platform.machine()) def GetArchInfo(): """Returns previsouly initialized architecture information. """ if _arch is None: raise errors.ProgrammerError("Architecture information hasn't been" " initialized") return _arch def GetClient(): """Connects to the a luxi socket and returns a client. """ try: client = luxi.Client(address=pathutils.QUERY_SOCKET) except NoMasterError: ss = ssconf.SimpleStore() # Try to read ssconf file try: ss.GetMasterNode() except errors.ConfigurationError: raise errors.OpPrereqError("Cluster not initialized or this machine is" " not part of a cluster", errors.ECODE_INVAL) master, myself = ssconf.GetMasterAndMyself(ss=ss) if master != myself: raise errors.OpPrereqError("This is not the master node, please connect" " to node '%s' and rerun the command" % master, errors.ECODE_INVAL) raise return client ganeti-3.1.0~rc2/lib/serializer.py000064400000000000000000000272761476477700300171130ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Serializer abstraction module This module introduces a simple abstraction over the serialization backend (currently json). """ # pylint: disable=C0103 # C0103: Invalid name, since pylint doesn't see that Dump points to a # function and not a constant import re import json from copy import deepcopy from ganeti import errors from ganeti import utils from ganeti import constants _RE_EOLSP = re.compile("[ \t]+$", re.MULTILINE) def DumpJson(data, private_encoder=None): """Serialize a given object. @param data: the data to serialize @return: the bytes representation of data @param private_encoder: specify L{serializer.EncodeWithPrivateFields} if you require the produced JSON to also contain private parameters. Otherwise, they will encode to null. """ if private_encoder is None: # Do not leak private fields by default. private_encoder = EncodeWithoutPrivateFields encoded = json.dumps(data, cls=private_encoder) txt = _RE_EOLSP.sub("", encoded) if not txt.endswith("\n"): txt += "\n" return txt.encode("utf-8") def LoadJson(data): """Unserialize data from bytes. @param data: the json-encoded form @type data: str or bytes @return: the original data @raise JSONDecodeError: if L{txt} is not a valid JSON document """ # convert data to string if data is a byte array if isinstance(data, bytes): data = data.decode('utf-8') values = json.loads(data) # Hunt and seek for Private fields and wrap them. WrapPrivateValues(values) return values def WrapPrivateValues(json_data): """Crawl a JSON decoded structure for private values and wrap them. @param json_data: the json-decoded value to protect. """ # This function used to be recursive. I use this list to avoid actual # recursion, however, since this is a very high-traffic area. todo = [json_data] while todo: data = todo.pop() if isinstance(data, list): # Array for item in data: todo.append(item) elif isinstance(data, dict): # Object # This is kind of a kludge, but the only place where we know what should # be protected is in ganeti.opcodes, and not in a way that is helpful to # us, especially in such a high traffic method; on the other hand, the # Haskell `py_compat_fields` test should complain whenever this check # does not protect fields properly. for field in data: value = data[field] if field in constants.PRIVATE_PARAMETERS_BLACKLIST: if not field.endswith("_cluster"): data[field] = PrivateDict(value) elif data[field] is not None: for os in data[field]: value[os] = PrivateDict(value[os]) else: todo.append(value) else: # Values pass def DumpSignedJson(data, key, salt=None, key_selector=None, private_encoder=None): """Serialize a given object and authenticate it. @param data: the data to serialize @param key: shared hmac key @param key_selector: name/id that identifies the key (in case there are multiple keys in use, e.g. in a multi-cluster environment) @param private_encoder: see L{DumpJson} @return: the string representation of data signed by the hmac key """ txt = DumpJson(data, private_encoder=private_encoder) if salt is None: salt = "" signed_dict = { "msg": txt, "salt": salt, } if key_selector: signed_dict["key_selector"] = key_selector else: key_selector = "" signed_dict["hmac"] = utils.Sha1Hmac(key, txt, salt=salt + key_selector) return DumpJson(signed_dict) def LoadSignedJson(txt, key): """Verify that a given message was signed with the given key, and load it. @param txt: json-encoded hmac-signed message @param key: the shared hmac key or a callable taking one argument (the key selector), which returns the hmac key belonging to the key selector. Typical usage is to pass a reference to the get method of a dict. @rtype: tuple of original data, string @return: original data, salt @raises errors.SignatureError: if the message signature doesn't verify """ signed_dict = LoadJson(txt) WrapPrivateValues(signed_dict) if not isinstance(signed_dict, dict): raise errors.SignatureError("Invalid external message") try: msg = signed_dict["msg"] salt = signed_dict["salt"] hmac_sign = signed_dict["hmac"] except KeyError: raise errors.SignatureError("Invalid external message") if callable(key): # pylint: disable=E1103 key_selector = signed_dict.get("key_selector", None) hmac_key = key(key_selector) if not hmac_key: raise errors.SignatureError("No key with key selector '%s' found" % key_selector) else: key_selector = "" hmac_key = key if not utils.VerifySha1Hmac(hmac_key, msg, hmac_sign, salt=salt + key_selector): raise errors.SignatureError("Invalid Signature") return LoadJson(msg), salt def LoadAndVerifyJson(raw, verify_fn): """Parses and verifies JSON data. @type raw: string @param raw: Input data in JSON format @type verify_fn: callable @param verify_fn: Verification function, usually from L{ht} @return: De-serialized data """ try: data = LoadJson(raw) except Exception as err: raise errors.ParseError("Can't parse input data: %s" % err) if not verify_fn(data): raise errors.ParseError("Data does not match expected format: %s" % verify_fn) return data Dump = DumpJson Load = LoadJson DumpSigned = DumpSignedJson LoadSigned = LoadSignedJson class Private(object): """Wrap a value so it is hard to leak it accidentally. >>> x = Private("foo") >>> print("Value: %s" % x) Value: >>> print("Value: {0}".format(x)) Value: >>> x.upper() == "FOO" True """ def __init__(self, item, descr="redacted"): if isinstance(item, Private): raise ValueError("Attempted to nest Private values.") self._item = item self._descr = descr def Get(self): "Return the wrapped value." return self._item def __str__(self): return "<%s>" % (self._descr, ) def __repr__(self): return "Private(?, descr=%r)" % (self._descr, ) # pylint: disable=W0212 # If it doesn't access _item directly, the call will go through __getattr__ # because this class defines __slots__ and "item" is not in it. # OTOH, if we do add it there, we'd risk shadowing an "item" attribute. def __eq__(self, other): if isinstance(other, Private): return self._item == other._item else: return self._item == other def __hash__(self): return hash(self._item) def __format__(self, *_1, **_2): return self.__str__() def __copy__(self): return Private(self._item, self._descr) def __deepcopy__(self, memo): new_item = deepcopy(self._item, memo) return Private(new_item, self._descr) def __getattr__(self, attr): return Private(getattr(self._item, attr), descr="%s.%s" % (self._descr, attr)) def __call__(self, *args, **kwargs): return Private(self._item(*args, **kwargs), descr="%s()" % (self._descr, )) # pylint: disable=R0201 # While this could get away with being a function, it needs to be a method. # Required by the copy.deepcopy function used by FillDict. def __getnewargs__(self): return tuple() def __bool__(self): return bool(self._item) # Get in the way of Pickle by implementing __slots__ but not __getstate__ # ...and get a performance boost, too. __slots__ = ["_item", "_descr"] class PrivateDict(dict): """A dictionary that turns its values to private fields. >>> PrivateDict() {} >>> supersekkrit = PrivateDict({"password": "foobar"}) >>> print(supersekkrit["password"]) >>> supersekkrit["password"].Get() 'foobar' >>> supersekkrit.GetPrivate("password") 'foobar' >>> supersekkrit["user"] = "eggspam" >>> supersekkrit.Unprivate() {'password': 'foobar', 'user': 'eggspam'} """ def __init__(self, data=None): dict.__init__(self) self.update(data) def __setitem__(self, item, value): if not isinstance(value, Private): if not isinstance(item, dict): value = Private(value, descr=item) else: value = PrivateDict(value) dict.__setitem__(self, item, value) # The actual conversion to Private containers is done by __setitem__ # copied straight from cpython/Lib/UserDict.py # Copyright (c) 2001-2014 Python Software Foundation; All Rights Reserved def update(self, other=None, **kwargs): # Make progressively weaker assumptions about "other" if other is None: pass elif hasattr(other, 'iteritems'): # iteritems saves memory and lookups for k, v in other.items(): self[k] = v elif hasattr(other, 'keys'): for k in other.keys(): self[k] = other[k] else: for k, v in other: self[k] = v if kwargs: self.update(kwargs) def GetPrivate(self, *args): """Like dict.get, but extracting the value in the process. Arguments are semantically equivalent to ``dict.get`` >>> PrivateDict({"foo": "bar"}).GetPrivate("foo") 'bar' >>> PrivateDict({"foo": "bar"}).GetPrivate("baz", "spam") 'spam' """ if len(args) == 1: key, = args return self[key].Get() elif len(args) == 2: key, default = args if key not in self: return default else: return self[key].Get() else: raise TypeError("GetPrivate() takes 2 arguments (%d given)" % len(args)) def Unprivate(self): """Turn this dict of Private() values to a dict of values. >>> PrivateDict({"foo": "bar"}).Unprivate() {'foo': 'bar'} @rtype: dict """ returndict = {} for key in self: returndict[key] = self[key].Get() return returndict # This class extends the default JsonEncoder to serialize byte arrays. # Unlike simplejson, python build-in json cannot encode byte arrays. class ByteEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, bytes): return str(o, encoding="ascii") return super().default(o) class EncodeWithoutPrivateFields(ByteEncoder): def default(self, o): if isinstance(o, Private): return None return super().default(o) class EncodeWithPrivateFields(ByteEncoder): def default(self, o): if isinstance(o, Private): return o.Get() return super().default(o) ganeti-3.1.0~rc2/lib/server/000075500000000000000000000000001476477700300156605ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/server/__init__.py000064400000000000000000000025511476477700300177740ustar00rootroot00000000000000# # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Empty file for package definition. """ ganeti-3.1.0~rc2/lib/server/masterd.py000064400000000000000000000071631476477700300177000ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Master daemon program. Some classes deviates from the standard style guide since the inheritance from parent classes requires it. """ # pylint: disable=C0103 # C0103: Invalid name ganeti-masterd import logging from ganeti import config from ganeti import constants from ganeti import jqueue from ganeti import utils import ganeti.rpc.node as rpc CLIENT_REQUEST_WORKERS = 16 EXIT_NOTMASTER = constants.EXIT_NOTMASTER EXIT_NODESETUP_ERROR = constants.EXIT_NODESETUP_ERROR class GanetiContext(object): """Context common to all ganeti threads. This class creates and holds common objects shared by all threads. """ # pylint: disable=W0212 # we do want to ensure a singleton here _instance = None def __init__(self, livelock=None): """Constructs a new GanetiContext object. There should be only a GanetiContext object at any time, so this function raises an error if this is not the case. """ assert self.__class__._instance is None, "double GanetiContext instance" # Create a livelock file if livelock is None: self.livelock = utils.livelock.LiveLock("masterd") else: self.livelock = livelock # Job queue cfg = self.GetConfig(None) logging.debug("Creating the job queue") self.jobqueue = jqueue.JobQueue(self, cfg) # setting this also locks the class against attribute modifications self.__class__._instance = self def __setattr__(self, name, value): """Setting GanetiContext attributes is forbidden after initialization. """ assert self.__class__._instance is None, "Attempt to modify Ganeti Context" object.__setattr__(self, name, value) def GetWConfdContext(self, ec_id): return config.GetWConfdContext(ec_id, self.livelock) def GetConfig(self, ec_id): return config.GetConfig(ec_id, self.livelock) # pylint: disable=R0201 # method could be a function, but keep interface backwards compatible def GetRpc(self, cfg): return rpc.RpcRunner(cfg, lambda _: None) def AddNode(self, cfg, node, ec_id): """Adds a node to the configuration. """ # Add it to the configuration cfg.AddNode(node, ec_id) def RemoveNode(self, cfg, node): """Removes a node from the configuration and lock manager. """ # Remove node from configuration cfg.RemoveNode(node.uuid) ganeti-3.1.0~rc2/lib/server/noded.py000064400000000000000000001231751476477700300173340ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Ganeti node daemon""" # pylint: disable=C0103 # C0103: Functions in this module need to have a given name structure, # and the name of the daemon doesn't match from __future__ import print_function import os import sys import logging import signal from optparse import OptionParser from ganeti import backend from ganeti import constants from ganeti import objects from ganeti import errors from ganeti import jstore from ganeti import daemon from ganeti import http from ganeti import utils from ganeti.storage import container from ganeti import serializer from ganeti import netutils from ganeti import pathutils from ganeti import ssconf import ganeti.http.server # pylint: disable=W0611 queue_lock = None def _extendReasonTrail(trail, source, reason=""): """Extend the reason trail with noded information The trail is extended by appending the name of the noded functionality """ assert trail is not None trail_source = "%s:%s" % (constants.OPCODE_REASON_SRC_NODED, source) trail.append((trail_source, reason, utils.EpochNano())) def _PrepareQueueLock(): """Try to prepare the queue lock. @return: None for success, otherwise an exception object """ global queue_lock # pylint: disable=W0603 if queue_lock is not None: return None # Prepare job queue try: queue_lock = jstore.InitAndVerifyQueue(must_lock=False) return None except EnvironmentError as err: return err def _RequireJobQueueLock(fn): """Decorator for job queue manipulating functions. """ QUEUE_LOCK_TIMEOUT = 10 def wrapper(*args, **kwargs): # Locking in exclusive, blocking mode because there could be several # children running at the same time. Waiting up to 10 seconds. if _PrepareQueueLock() is not None: raise errors.JobQueueError("Job queue failed initialization," " cannot update jobs") queue_lock.Exclusive(blocking=True, timeout=QUEUE_LOCK_TIMEOUT) try: return fn(*args, **kwargs) finally: queue_lock.Unlock() return wrapper def _DecodeImportExportIO(ieio, ieioargs): """Decodes import/export I/O information. """ if ieio == constants.IEIO_RAW_DISK: assert len(ieioargs) == 1 return (objects.Disk.FromDict(ieioargs[0]), ) if ieio == constants.IEIO_SCRIPT: assert len(ieioargs) == 2 return (objects.Disk.FromDict(ieioargs[0]), ieioargs[1]) return ieioargs def _DefaultAlternative(value, default): """Returns value or, if evaluating to False, a default value. Returns the given value, unless it evaluates to False. In the latter case the default value is returned. @param value: Value to return if it doesn't evaluate to False @param default: Default value @return: Given value or the default """ if value: return value return default class MlockallRequestExecutor(http.server.HttpServerRequestExecutor): """Subclass ensuring request handlers are locked in RAM. """ def __init__(self, *args, **kwargs): utils.Mlockall() http.server.HttpServerRequestExecutor.__init__(self, *args, **kwargs) class NodeRequestHandler(http.server.HttpServerHandler): """The server implementation. This class holds all methods exposed over the RPC interface. """ # too many public methods, and unused args - all methods get params # due to the API # pylint: disable=R0904,W0613 def __init__(self): http.server.HttpServerHandler.__init__(self) self.noded_pid = os.getpid() def HandleRequest(self, req): """Handle a request. """ if req.request_method.upper() != http.HTTP_POST: raise http.HttpBadRequest("Only the POST method is supported") path = req.request_path if path.startswith("/"): path = path[1:] method = getattr(self, "perspective_%s" % path, None) if method is None: raise http.HttpNotFound() try: result = (True, method(serializer.LoadJson(req.request_body))) except backend.RPCFail as err: # our custom failure exception; str(err) works fine if the # exception was constructed with a single argument, and in # this case, err.message == err.args[0] == str(err) result = (False, str(err)) except errors.QuitGanetiException as err: # Tell parent to quit logging.info("Shutting down the node daemon, arguments: %s", str(err.args)) os.kill(self.noded_pid, signal.SIGTERM) # And return the error's arguments, which must be already in # correct tuple format result = err.args except Exception as err: # pylint: disable=W0703 logging.exception("Error in RPC call") result = (False, "Error while executing backend function: %s" % str(err)) return serializer.DumpJson(result) # the new block devices -------------------------- @staticmethod def perspective_blockdev_create(params): """Create a block device. """ (bdev_s, size, owner, on_primary, info, excl_stor) = params bdev = objects.Disk.FromDict(bdev_s) if bdev is None: raise ValueError("can't unserialize data!") return backend.BlockdevCreate(bdev, size, owner, on_primary, info, excl_stor) @staticmethod def perspective_blockdev_convert(params): """Copy data from source block device to target. """ disk_src, disk_dest = params bdev_src = objects.Disk.FromDict(disk_src) bdev_dest = objects.Disk.FromDict(disk_dest) return backend.BlockdevConvert(bdev_src, bdev_dest) @staticmethod def perspective_blockdev_pause_resume_sync(params): """Pause/resume sync of a block device. """ disks_s, pause = params disks = [objects.Disk.FromDict(bdev_s) for bdev_s in disks_s] return backend.BlockdevPauseResumeSync(disks, pause) @staticmethod def perspective_blockdev_image(params): """Image a block device. """ bdev_s, image, size = params bdev = objects.Disk.FromDict(bdev_s) return backend.BlockdevImage(bdev, image, size) @staticmethod def perspective_blockdev_wipe(params): """Wipe a block device. """ bdev_s, offset, size = params bdev = objects.Disk.FromDict(bdev_s) return backend.BlockdevWipe(bdev, offset, size) @staticmethod def perspective_blockdev_remove(params): """Remove a block device. """ bdev_s = params[0] bdev = objects.Disk.FromDict(bdev_s) return backend.BlockdevRemove(bdev) @staticmethod def perspective_blockdev_rename(params): """Remove a block device. """ devlist = [(objects.Disk.FromDict(ds), uid) for ds, uid in params[0]] return backend.BlockdevRename(devlist) @staticmethod def perspective_blockdev_assemble(params): """Assemble a block device. """ bdev_s, idict, on_primary, idx = params bdev = objects.Disk.FromDict(bdev_s) instance = objects.Instance.FromDict(idict) if bdev is None: raise ValueError("can't unserialize data!") return backend.BlockdevAssemble(bdev, instance, on_primary, idx) @staticmethod def perspective_blockdev_shutdown(params): """Shutdown a block device. """ bdev_s = params[0] bdev = objects.Disk.FromDict(bdev_s) if bdev is None: raise ValueError("can't unserialize data!") return backend.BlockdevShutdown(bdev) @staticmethod def perspective_blockdev_addchildren(params): """Add a child to a mirror device. Note: this is only valid for mirror devices. It's the caller's duty to send a correct disk, otherwise we raise an error. """ bdev_s, ndev_s = params bdev = objects.Disk.FromDict(bdev_s) ndevs = [objects.Disk.FromDict(disk_s) for disk_s in ndev_s] if bdev is None or ndevs.count(None) > 0: raise ValueError("can't unserialize data!") return backend.BlockdevAddchildren(bdev, ndevs) @staticmethod def perspective_blockdev_removechildren(params): """Remove a child from a mirror device. This is only valid for mirror devices, of course. It's the callers duty to send a correct disk, otherwise we raise an error. """ bdev_s, ndev_s = params bdev = objects.Disk.FromDict(bdev_s) ndevs = [objects.Disk.FromDict(disk_s) for disk_s in ndev_s] if bdev is None or ndevs.count(None) > 0: raise ValueError("can't unserialize data!") return backend.BlockdevRemovechildren(bdev, ndevs) @staticmethod def perspective_blockdev_getmirrorstatus(params): """Return the mirror status for a list of disks. """ disks = [objects.Disk.FromDict(dsk_s) for dsk_s in params[0]] return [status.ToDict() for status in backend.BlockdevGetmirrorstatus(disks)] @staticmethod def perspective_blockdev_getmirrorstatus_multi(params): """Return the mirror status for a list of disks. """ (node_disks, ) = params disks = [objects.Disk.FromDict(dsk_s) for dsk_s in node_disks] result = [] for (success, status) in backend.BlockdevGetmirrorstatusMulti(disks): if success: result.append((success, status.ToDict())) else: result.append((success, status)) return result @staticmethod def perspective_blockdev_find(params): """Expose the FindBlockDevice functionality for a disk. This will try to find but not activate a disk. """ disk = objects.Disk.FromDict(params[0]) result = backend.BlockdevFind(disk) if result is None: return None return result.ToDict() @staticmethod def perspective_blockdev_snapshot(params): """Create a snapshot device. Note that this is only valid for LVM and ExtStorage disks, if we get passed something else we raise an exception. The snapshot device can be remove by calling the generic block device remove call. """ (disk, snap_name, snap_size) = params cfbd = objects.Disk.FromDict(disk) return backend.BlockdevSnapshot(cfbd, snap_name, snap_size) @staticmethod def perspective_blockdev_grow(params): """Grow a stack of devices. """ if len(params) < 5: raise ValueError("Received only %s parameters in blockdev_grow," " old master?" % len(params)) cfbd = objects.Disk.FromDict(params[0]) amount = params[1] dryrun = params[2] backingstore = params[3] excl_stor = params[4] return backend.BlockdevGrow(cfbd, amount, dryrun, backingstore, excl_stor) @staticmethod def perspective_blockdev_close(params): """Closes the given block devices. """ disks = [objects.Disk.FromDict(cf) for cf in params[1]] return backend.BlockdevClose(params[0], disks) @staticmethod def perspective_blockdev_open(params): """Opens the given block devices. """ disks = [objects.Disk.FromDict(cf) for cf in params[1]] exclusive = params[2] return backend.BlockdevOpen(params[0], disks, exclusive) @staticmethod def perspective_blockdev_getdimensions(params): """Compute the sizes of the given block devices. """ disks = [objects.Disk.FromDict(cf) for cf in params[0]] return backend.BlockdevGetdimensions(disks) @staticmethod def perspective_blockdev_setinfo(params): """Sets metadata information on the given block device. """ (disk, info) = params disk = objects.Disk.FromDict(disk) return backend.BlockdevSetInfo(disk, info) # blockdev/drbd specific methods ---------- @staticmethod def perspective_drbd_disconnect_net(params): """Disconnects the network connection of drbd disks. Note that this is only valid for drbd disks, so the members of the disk list must all be drbd devices. """ (disks,) = params disks = [objects.Disk.FromDict(disk) for disk in disks] return backend.DrbdDisconnectNet(disks) @staticmethod def perspective_drbd_attach_net(params): """Attaches the network connection of drbd disks. Note that this is only valid for drbd disks, so the members of the disk list must all be drbd devices. """ disks, multimaster = params disks = [objects.Disk.FromDict(disk) for disk in disks] return backend.DrbdAttachNet(disks, multimaster) @staticmethod def perspective_drbd_wait_sync(params): """Wait until DRBD disks are synched. Note that this is only valid for drbd disks, so the members of the disk list must all be drbd devices. """ (disks,) = params disks = [objects.Disk.FromDict(disk) for disk in disks] return backend.DrbdWaitSync(disks) @staticmethod def perspective_drbd_needs_activation(params): """Checks if the drbd devices need activation Note that this is only valid for drbd disks, so the members of the disk list must all be drbd devices. """ (disks,) = params disks = [objects.Disk.FromDict(disk) for disk in disks] return backend.DrbdNeedsActivation(disks) @staticmethod def perspective_drbd_helper(_): """Query drbd helper. """ return backend.GetDrbdUsermodeHelper() # export/import -------------------------- @staticmethod def perspective_finalize_export(params): """Expose the finalize export functionality. """ instance = objects.Instance.FromDict(params[0]) snap_disks = [] for disk in params[1]: if isinstance(disk, bool): snap_disks.append(disk) else: snap_disks.append(objects.Disk.FromDict(disk)) return backend.FinalizeExport(instance, snap_disks) @staticmethod def perspective_export_info(params): """Query information about an existing export on this node. The given path may not contain an export, in which case we return None. """ path = params[0] return backend.ExportInfo(path) @staticmethod def perspective_export_list(params): """List the available exports on this node. Note that as opposed to export_info, which may query data about an export in any path, this only queries the standard Ganeti path (pathutils.EXPORT_DIR). """ return backend.ListExports() @staticmethod def perspective_export_remove(params): """Remove an export. """ export = params[0] return backend.RemoveExport(export) # block device --------------------- @staticmethod def perspective_bdev_sizes(params): """Query the list of block devices """ devices = params[0] return backend.GetBlockDevSizes(devices) # volume -------------------------- @staticmethod def perspective_lv_list(params): """Query the list of logical volumes in a given volume group. """ vgname = params[0] return backend.GetVolumeList(vgname) @staticmethod def perspective_vg_list(params): """Query the list of volume groups. """ return backend.ListVolumeGroups() # Storage -------------------------- @staticmethod def perspective_storage_list(params): """Get list of storage units. """ (su_name, su_args, name, fields) = params return container.GetStorage(su_name, *su_args).List(name, fields) @staticmethod def perspective_storage_modify(params): """Modify a storage unit. """ (su_name, su_args, name, changes) = params return container.GetStorage(su_name, *su_args).Modify(name, changes) @staticmethod def perspective_storage_execute(params): """Execute an operation on a storage unit. """ (su_name, su_args, name, op) = params return container.GetStorage(su_name, *su_args).Execute(name, op) # bridge -------------------------- @staticmethod def perspective_bridges_exist(params): """Check if all bridges given exist on this node. """ bridges_list = params[0] return backend.BridgesExist(bridges_list) # instance -------------------------- @staticmethod def perspective_instance_os_add(params): """Install an OS on a given instance. """ inst_s = params[0] inst = objects.Instance.FromDict(inst_s) reinstall = params[1] debug = params[2] return backend.InstanceOsAdd(inst, reinstall, debug) @staticmethod def perspective_instance_run_rename(params): """Runs the OS rename script for an instance. """ inst_s, old_name, debug = params inst = objects.Instance.FromDict(inst_s) return backend.RunRenameInstance(inst, old_name, debug) @staticmethod def perspective_instance_shutdown(params): """Shutdown an instance. """ instance = objects.Instance.FromDict(params[0]) timeout = params[1] trail = params[2] _extendReasonTrail(trail, "shutdown") return backend.InstanceShutdown(instance, timeout, trail) @staticmethod def perspective_instance_start(params): """Start an instance. """ (instance_name, startup_paused, trail) = params instance = objects.Instance.FromDict(instance_name) _extendReasonTrail(trail, "start") return backend.StartInstance(instance, startup_paused, trail) @staticmethod def perspective_resize_disk(params): """Notify the hypervisor about a disk change. """ (idict, disk, new_size) = params instance = objects.Instance.FromDict(idict) disk = objects.Disk.FromDict(disk) return backend.ResizeDisk(instance, disk, new_size) @staticmethod def perspective_hotplug_device(params): """Hotplugs device to a running instance. """ (idict, action, dev_type, ddict, extra, seq) = params instance = objects.Instance.FromDict(idict) if dev_type == constants.HOTPLUG_TARGET_DISK: device = objects.Disk.FromDict(ddict) elif dev_type == constants.HOTPLUG_TARGET_NIC: device = objects.NIC.FromDict(ddict) else: assert dev_type in constants.HOTPLUG_ALL_TARGETS return backend.HotplugDevice(instance, action, dev_type, device, extra, seq) @staticmethod def perspective_hotplug_supported(params): """Checks if hotplug is supported. """ instance = objects.Instance.FromDict(params[0]) return backend.HotplugSupported(instance) @staticmethod def perspective_instance_metadata_modify(params): """Modify instance metadata. """ instance = params[0] return backend.ModifyInstanceMetadata(instance) @staticmethod def perspective_migration_info(params): """Gather information about an instance to be migrated. """ instance = objects.Instance.FromDict(params[0]) return backend.MigrationInfo(instance) @staticmethod def perspective_accept_instance(params): """Prepare the node to accept an instance. """ instance, info, target = params instance = objects.Instance.FromDict(instance) return backend.AcceptInstance(instance, info, target) @staticmethod def perspective_instance_finalize_migration_dst(params): """Finalize the instance migration on the destination node. """ instance, info, success = params instance = objects.Instance.FromDict(instance) return backend.FinalizeMigrationDst(instance, info, success) @staticmethod def perspective_instance_migrate(params): """Migrates an instance. """ cluster_name, instance, target, live = params instance = objects.Instance.FromDict(instance) return backend.MigrateInstance(cluster_name, instance, target, live) @staticmethod def perspective_instance_finalize_migration_src(params): """Finalize the instance migration on the source node. """ instance, success, live = params instance = objects.Instance.FromDict(instance) return backend.FinalizeMigrationSource(instance, success, live) @staticmethod def perspective_instance_get_migration_status(params): """Reports migration status. """ instance = objects.Instance.FromDict(params[0]) return backend.GetMigrationStatus(instance).ToDict() @staticmethod def perspective_instance_reboot(params): """Reboot an instance. """ instance = objects.Instance.FromDict(params[0]) reboot_type = params[1] shutdown_timeout = params[2] trail = params[3] _extendReasonTrail(trail, "reboot") return backend.InstanceReboot(instance, reboot_type, shutdown_timeout, trail) @staticmethod def perspective_instance_balloon_memory(params): """Modify instance runtime memory. """ instance_dict, memory = params instance = objects.Instance.FromDict(instance_dict) return backend.InstanceBalloonMemory(instance, memory) @staticmethod def perspective_instance_info(params): """Query instance information. """ (instance_name, hypervisor_name, hvparams) = params return backend.GetInstanceInfo(instance_name, hypervisor_name, hvparams) @staticmethod def perspective_instance_migratable(params): """Query whether the specified instance can be migrated. """ instance = objects.Instance.FromDict(params[0]) return backend.GetInstanceMigratable(instance) @staticmethod def perspective_all_instances_info(params): """Query information about all instances. """ (hypervisor_list, all_hvparams) = params return backend.GetAllInstancesInfo(hypervisor_list, all_hvparams) @staticmethod def perspective_instance_console_info(params): """Query information on how to get console access to instances """ return backend.GetInstanceConsoleInfo(params) @staticmethod def perspective_instance_list(params): """Query the list of running instances. """ (hypervisor_list, hvparams) = params return backend.GetInstanceList(hypervisor_list, hvparams) # node -------------------------- @staticmethod def perspective_node_has_ip_address(params): """Checks if a node has the given ip address. """ return netutils.IPAddress.Own(params[0]) @staticmethod def perspective_node_info(params): """Query node information. """ (storage_units, hv_specs) = params return backend.GetNodeInfo(storage_units, hv_specs) @staticmethod def perspective_etc_hosts_modify(params): """Modify a node entry in /etc/hosts. """ backend.EtcHostsModify(params[0], params[1], params[2]) return True @staticmethod def perspective_node_verify(params): """Run a verify sequence on this node. """ (what, cluster_name, hvparams) = params return backend.VerifyNode(what, cluster_name, hvparams) @classmethod def perspective_node_verify_light(cls, params): """Run a light verify sequence on this node. This call is meant to perform a less strict verification of the node in certain situations. Right now, it is invoked only when a node is just about to be added to a cluster, and even then, it performs the same checks as L{perspective_node_verify}. """ return cls.perspective_node_verify(params) @staticmethod def perspective_node_start_master_daemons(params): """Start the master daemons on this node. """ return backend.StartMasterDaemons(params[0]) @staticmethod def perspective_node_activate_master_ip(params): """Activate the master IP on this node. """ master_params = objects.MasterNetworkParameters.FromDict(params[0]) return backend.ActivateMasterIp(master_params, params[1]) @staticmethod def perspective_node_deactivate_master_ip(params): """Deactivate the master IP on this node. """ master_params = objects.MasterNetworkParameters.FromDict(params[0]) return backend.DeactivateMasterIp(master_params, params[1]) @staticmethod def perspective_node_stop_master(params): """Stops master daemons on this node. """ return backend.StopMasterDaemons() @staticmethod def perspective_node_change_master_netmask(params): """Change the master IP netmask. """ return backend.ChangeMasterNetmask(params[0], params[1], params[2], params[3]) @staticmethod def perspective_node_leave_cluster(params): """Cleanup after leaving a cluster. """ return backend.LeaveCluster(params[0]) @staticmethod def perspective_node_volumes(params): """Query the list of all logical volume groups. """ return backend.NodeVolumes() @staticmethod def perspective_node_demote_from_mc(params): """Demote a node from the master candidate role. """ return backend.DemoteFromMC() @staticmethod def perspective_node_powercycle(params): """Tries to powercycle the node. """ (hypervisor_type, hvparams) = params return backend.PowercycleNode(hypervisor_type, hvparams) @staticmethod def perspective_node_configure_ovs(params): """Sets up OpenvSwitch on the node. """ (ovs_name, ovs_link) = params return backend.ConfigureOVS(ovs_name, ovs_link) @staticmethod def perspective_node_crypto_tokens(params): """Gets the node's public crypto tokens. """ token_requests = params[0] return backend.GetCryptoTokens(token_requests) @staticmethod def perspective_node_ensure_daemon(params): """Ensure daemon is running. """ (daemon_name, run) = params return backend.EnsureDaemon(daemon_name, run) @staticmethod def perspective_node_ssh_key_add(params): """Distributes a new node's SSH key if authorized. """ (node_uuid, node_name, potential_master_candidates, to_authorized_keys, to_public_keys, get_public_keys) = params return backend.AddNodeSshKey(node_uuid, node_name, potential_master_candidates, to_authorized_keys=to_authorized_keys, to_public_keys=to_public_keys, get_public_keys=get_public_keys) @staticmethod def perspective_node_ssh_keys_renew(params): """Generates a new root SSH key pair on the node. """ (node_uuids, node_names, master_candidate_uuids, potential_master_candidates, old_key_type, new_key_type, new_key_bits) = params return backend.RenewSshKeys(node_uuids, node_names, master_candidate_uuids, potential_master_candidates, old_key_type, new_key_type, new_key_bits) @staticmethod def perspective_node_ssh_key_remove(params): """Removes a node's SSH key from the other nodes' SSH files. """ (node_uuid, node_name, master_candidate_uuids, potential_master_candidates, from_authorized_keys, from_public_keys, clear_authorized_keys, clear_public_keys, readd) = params return backend.RemoveNodeSshKey(node_uuid, node_name, master_candidate_uuids, potential_master_candidates, from_authorized_keys=from_authorized_keys, from_public_keys=from_public_keys, clear_authorized_keys=clear_authorized_keys, clear_public_keys=clear_public_keys, readd=readd) # cluster -------------------------- @staticmethod def perspective_version(params): """Query version information. """ return constants.PROTOCOL_VERSION @staticmethod def perspective_upload_file(params): """Upload a file. Note that the backend implementation imposes strict rules on which files are accepted. """ return backend.UploadFile(*(params[0])) @staticmethod def perspective_upload_file_single(params): """Upload a file. Note that the backend implementation imposes strict rules on which files are accepted. """ return backend.UploadFile(*params) @staticmethod def perspective_master_node_name(params): """Returns the master node name. """ return backend.GetMasterNodeName() @staticmethod def perspective_run_oob(params): """Runs oob on node. """ output = backend.RunOob(params[0], params[1], params[2], params[3]) if output: result = serializer.LoadJson(output) else: result = None return result @staticmethod def perspective_restricted_command(params): """Runs a restricted command. """ (cmd, ) = params return backend.RunRestrictedCmd(cmd) @staticmethod def perspective_write_ssconf_files(params): """Write ssconf files. """ (values,) = params return ssconf.WriteSsconfFiles(values) @staticmethod def perspective_get_watcher_pause(params): """Get watcher pause end. """ return utils.ReadWatcherPauseFile(pathutils.WATCHER_PAUSEFILE) @staticmethod def perspective_set_watcher_pause(params): """Set watcher pause. """ (until, ) = params return backend.SetWatcherPause(until) @staticmethod def perspective_get_file_info(params): """Get info on whether a file exists and its properties. """ (path, ) = params return backend.GetFileInfo(path) # os ----------------------- @staticmethod def perspective_os_diagnose(params): """Query detailed information about existing OSes. """ return backend.DiagnoseOS() @staticmethod def perspective_os_validate(params): """Run a given OS' validation routine. """ required, name, checks, params, force_variant = params return backend.ValidateOS(required, name, checks, params, force_variant) @staticmethod def perspective_os_export(params): """Export an OS definition into an instance specific package. """ instance = objects.Instance.FromDict(params[0]) override_env = params[1] return backend.ExportOS(instance, override_env) # extstorage ----------------------- @staticmethod def perspective_extstorage_diagnose(params): """Query detailed information about existing extstorage providers. """ return backend.DiagnoseExtStorage() # hooks ----------------------- @staticmethod def perspective_hooks_runner(params): """Run hook scripts. """ hpath, phase, env = params hr = backend.HooksRunner() return hr.RunHooks(hpath, phase, env) # iallocator ----------------- @staticmethod def perspective_iallocator_runner(params): """Run an iallocator script. """ name, idata, ial_params_dict = params ial_params = [] for ial_param in ial_params_dict.items(): if ial_param[1] is not None: ial_params.append("--" + ial_param[0] + "=" + ial_param[1]) else: ial_params.append("--" + ial_param[0]) iar = backend.IAllocatorRunner() return iar.Run(name, idata, ial_params) # test ----------------------- @staticmethod def perspective_test_delay(params): """Run test delay. """ duration = params[0] status, rval = utils.TestDelay(duration) if not status: raise backend.RPCFail(rval) return rval # file storage --------------- @staticmethod def perspective_file_storage_dir_create(params): """Create the file storage directory. """ file_storage_dir = params[0] return backend.CreateFileStorageDir(file_storage_dir) @staticmethod def perspective_file_storage_dir_remove(params): """Remove the file storage directory. """ file_storage_dir = params[0] return backend.RemoveFileStorageDir(file_storage_dir) @staticmethod def perspective_file_storage_dir_rename(params): """Rename the file storage directory. """ old_file_storage_dir = params[0] new_file_storage_dir = params[1] return backend.RenameFileStorageDir(old_file_storage_dir, new_file_storage_dir) # jobs ------------------------ @staticmethod @_RequireJobQueueLock def perspective_jobqueue_update(params): """Update job queue. """ (file_name, content) = params return backend.JobQueueUpdate(file_name, content) @staticmethod @_RequireJobQueueLock def perspective_jobqueue_purge(params): """Purge job queue. """ return backend.JobQueuePurge() @staticmethod @_RequireJobQueueLock def perspective_jobqueue_rename(params): """Rename a job queue file. """ # TODO: What if a file fails to rename? return [backend.JobQueueRename(old, new) for old, new in params[0]] @staticmethod @_RequireJobQueueLock def perspective_jobqueue_set_drain_flag(params): """Set job queue's drain flag. """ (flag, ) = params return jstore.SetDrainFlag(flag) # hypervisor --------------- @staticmethod def perspective_hypervisor_validate_params(params): """Validate the hypervisor parameters. """ (hvname, hvparams) = params return backend.ValidateHVParams(hvname, hvparams) # Crypto @staticmethod def perspective_x509_cert_create(params): """Creates a new X509 certificate for SSL/TLS. """ (validity, ) = params return backend.CreateX509Certificate(validity) @staticmethod def perspective_x509_cert_remove(params): """Removes a X509 certificate. """ (name, ) = params return backend.RemoveX509Certificate(name) # Import and export @staticmethod def perspective_import_start(params): """Starts an import daemon. """ (opts_s, instance, component, (dest, dest_args)) = params opts = objects.ImportExportOptions.FromDict(opts_s) return backend.StartImportExportDaemon(constants.IEM_IMPORT, opts, None, None, objects.Instance.FromDict(instance), component, dest, _DecodeImportExportIO(dest, dest_args)) @staticmethod def perspective_export_start(params): """Starts an export daemon. """ (opts_s, host, port, instance, component, (source, source_args)) = params opts = objects.ImportExportOptions.FromDict(opts_s) return backend.StartImportExportDaemon(constants.IEM_EXPORT, opts, host, port, objects.Instance.FromDict(instance), component, source, _DecodeImportExportIO(source, source_args)) @staticmethod def perspective_impexp_status(params): """Retrieves the status of an import or export daemon. """ return backend.GetImportExportStatus(params[0]) @staticmethod def perspective_impexp_abort(params): """Aborts an import or export. """ return backend.AbortImportExport(params[0]) @staticmethod def perspective_impexp_cleanup(params): """Cleans up after an import or export. """ return backend.CleanupImportExport(params[0]) def CheckNoded(options, args): """Initial checks whether to run or exit with a failure. """ if args: # noded doesn't take any arguments print("Usage: %s [-f] [-d] [-p port] [-b ADDRESS]" % sys.argv[0], file=sys.stderr) sys.exit(constants.EXIT_FAILURE) if options.max_clients < 1: print("%s --max-clients argument must be >= 1" % sys.argv[0], file=sys.stderr) sys.exit(constants.EXIT_FAILURE) def SSLVerifyPeer(conn, cert, errnum, errdepth, ok): """Callback function to verify a peer against the candidate cert map. Note that we have a chicken-and-egg problem during cluster init and upgrade. This method checks whether the incoming connection comes from a master candidate by comparing it to the master certificate map in the cluster configuration. However, during cluster init and cluster upgrade there are various RPC calls done to the master node itself, before the candidate certificate list is established and the cluster configuration is written. In this case, we cannot check against the master candidate map. This problem is solved by checking whether the candidate map is empty. An initialized 2.11 or higher cluster has at least one entry for the master node in the candidate map. If the map is empty, we know that we are still in the bootstrap/upgrade phase. In this case, we read the server certificate digest and compare it to the incoming request. This means that after an upgrade of Ganeti, the system continues to operate like before, using server certificates only. After the client certificates are generated with ``gnt-cluster renew-crypto --new-node-certificates``, RPC communication is switched to using client certificates and the trick of using server certificates does not work anymore. @type conn: C{OpenSSL.SSL.Connection} @param conn: the OpenSSL connection object @type cert: C{OpenSSL.X509} @param cert: the peer's SSL certificate @type errdepth: integer @param errdepth: number of the step in the certificate chain starting at 0 for the actual client certificate. """ # some parameters are unused, but this is the API # pylint: disable=W0613 # If we receive a certificate from the certificate chain that is higher # than the lowest element of the chain, we have to check it against the # server certificate. cert_digest = cert.digest("sha1").decode("ascii") if errdepth > 0: server_digest = utils.GetCertificateDigest( cert_filename=pathutils.NODED_CERT_FILE) match = cert_digest == server_digest if not match: logging.debug("Received certificate from the certificate chain, which" " does not match the server certficate. Digest of the" " received certificate: %s. Digest of the server" " certificate: %s.", cert_digest, server_digest) return match elif errdepth == 0: sstore = ssconf.SimpleStore() try: candidate_certs = sstore.GetMasterCandidatesCertMap() except errors.ConfigurationError: logging.info("No candidate certificates found. Switching to " "bootstrap/update mode.") candidate_certs = None if not candidate_certs: candidate_certs = { constants.CRYPTO_BOOTSTRAP: utils.GetCertificateDigest( cert_filename=pathutils.NODED_CERT_FILE)} match = cert_digest in candidate_certs.values() if not match: logging.debug("Received certificate which is not a certificate of a" " master candidate. Certificate digest: %s. List of master" " candidate certificate digests: %s.", cert_digest, str(candidate_certs)) return match else: logging.error("Invalid errdepth value: %s.", errdepth) return False def PrepNoded(options, _): """Preparation node daemon function, executed with the PID file held. """ if options.mlock: request_executor_class = MlockallRequestExecutor try: utils.Mlockall() except errors.NoCtypesError: logging.warning("Cannot set memory lock, ctypes module not found") request_executor_class = http.server.HttpServerRequestExecutor else: request_executor_class = http.server.HttpServerRequestExecutor # Read SSL certificate if options.ssl: ssl_params = http.HttpSslParams(ssl_key_path=options.ssl_key, ssl_cert_path=options.ssl_cert) else: ssl_params = None err = _PrepareQueueLock() if err is not None: # this might be some kind of file-system/permission error; while # this breaks the job queue functionality, we shouldn't prevent # startup of the whole node daemon because of this logging.critical("Can't init/verify the queue, proceeding anyway: %s", err) handler = NodeRequestHandler() mainloop = daemon.Mainloop() server = http.server.HttpServer( mainloop, options.bind_address, options.port, options.max_clients, handler, ssl_params=ssl_params, ssl_verify_peer=True, request_executor_class=request_executor_class, ssl_verify_callback=SSLVerifyPeer) server.Start() return (mainloop, server) def ExecNoded(options, args, prep_data): # pylint: disable=W0613 """Main node daemon function, executed with the PID file held. """ (mainloop, server) = prep_data try: mainloop.Run() finally: server.Stop() def Main(): """Main function for the node daemon. """ parser = OptionParser(description="Ganeti node daemon", usage=("%prog [-f] [-d] [-p port] [-b ADDRESS]" " [-i INTERFACE]"), version="%%prog (ganeti) %s" % constants.RELEASE_VERSION) parser.add_option("--no-mlock", dest="mlock", help="Do not mlock the node memory in ram", default=True, action="store_false") parser.add_option("--max-clients", dest="max_clients", default=20, type="int", help="Number of simultaneous connections accepted" " by noded") daemon.GenericMain(constants.NODED, parser, CheckNoded, PrepNoded, ExecNoded, default_ssl_cert=pathutils.NODED_CERT_FILE, default_ssl_key=pathutils.NODED_CERT_FILE, console_logging=True, warn_breach=True) ganeti-3.1.0~rc2/lib/server/rapi.py000064400000000000000000000276561476477700300172050ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Ganeti Remote API master script. """ # pylint: disable=C0103 # C0103: Invalid name ganeti-watcher from __future__ import print_function import logging import optparse import sys import os import os.path import errno try: from pyinotify import pyinotify # pylint: disable=E0611 except ImportError: import pyinotify from ganeti import asyncnotifier from ganeti import constants from ganeti import http from ganeti import daemon from ganeti import ssconf import ganeti.rpc.errors as rpcerr from ganeti import serializer from ganeti import compat from ganeti import utils from ganeti import pathutils from ganeti.rapi import connector from ganeti.rapi import baserlib import ganeti.http.auth # pylint: disable=W0611 import ganeti.http.server # pylint: disable=W0611 class RemoteApiRequestContext(object): """Data structure for Remote API requests. """ def __init__(self): self.handler = None self.handler_fn = None self.handler_access = None self.body_data = None class RemoteApiHandler(http.auth.HttpServerRequestAuthentication, http.server.HttpServerHandler): """REST Request Handler Class. """ AUTH_REALM = "Ganeti Remote API" def __init__(self, user_fn, reqauth, _client_cls=None): """Initializes this class. @type user_fn: callable @param user_fn: Function receiving username as string and returning L{http.auth.PasswordFileUser} or C{None} if user is not found @type reqauth: bool @param reqauth: Whether to require authentication """ # pylint: disable=W0233 # it seems pylint doesn't see the second parent class there http.server.HttpServerHandler.__init__(self) http.auth.HttpServerRequestAuthentication.__init__(self) self._client_cls = _client_cls self._resmap = connector.Mapper() self._user_fn = user_fn self._reqauth = reqauth @staticmethod def FormatErrorMessage(values): """Formats the body of an error message. @type values: dict @param values: dictionary with keys C{code}, C{message} and C{explain}. @rtype: tuple; (string, string) @return: Content-type and response body """ return (http.HTTP_APP_JSON, serializer.DumpJson(values)) def _GetRequestContext(self, req): """Returns the context for a request. The context is cached in the req.private variable. """ if req.private is None: (HandlerClass, items, args) = \ self._resmap.getController(req.request_path) ctx = RemoteApiRequestContext() ctx.handler = HandlerClass(items, args, req, _client_cls=self._client_cls) method = req.request_method.upper() try: ctx.handler_fn = getattr(ctx.handler, method) except AttributeError: raise http.HttpNotImplemented("Method %s is unsupported for path %s" % (method, req.request_path)) ctx.handler_access = baserlib.GetHandlerAccess(ctx.handler, method) # Require permissions definition (usually in the base class) if ctx.handler_access is None: raise AssertionError("Permissions definition missing") # This is only made available in HandleRequest ctx.body_data = None req.private = ctx # Check for expected attributes assert req.private.handler assert req.private.handler_fn assert req.private.handler_access is not None return req.private def AuthenticationRequired(self, req): """Determine whether authentication is required. """ return self._reqauth or bool(self._GetRequestContext(req).handler_access) def Authenticate(self, req, username, password): """Checks whether a user can access a resource. """ ctx = self._GetRequestContext(req) user = self._user_fn(username) if not (user and self.VerifyBasicAuthPassword(req, username, password, user.password)): # Unknown user or password wrong return False if (not ctx.handler_access or set(user.options).intersection(ctx.handler_access)): # Allow access return True # Access forbidden raise http.HttpForbidden() def HandleRequest(self, req): """Handles a request. """ ctx = self._GetRequestContext(req) # Deserialize request parameters if req.request_body: # RFC2616, 7.2.1: Any HTTP/1.1 message containing an entity-body SHOULD # include a Content-Type header field defining the media type of that # body. [...] If the media type remains unknown, the recipient SHOULD # treat it as type "application/octet-stream". req_content_type = req.request_headers.get(http.HTTP_CONTENT_TYPE, http.HTTP_APP_OCTET_STREAM) if req_content_type.lower() != http.HTTP_APP_JSON.lower(): raise http.HttpUnsupportedMediaType() try: ctx.body_data = serializer.LoadJson(req.request_body) except Exception: raise http.HttpBadRequest(message="Unable to parse JSON data") else: ctx.body_data = None try: result = ctx.handler_fn() except rpcerr.TimeoutError: raise http.HttpGatewayTimeout() except rpcerr.ProtocolError as err: raise http.HttpBadGateway(str(err)) req.resp_headers[http.HTTP_CONTENT_TYPE] = http.HTTP_APP_JSON return serializer.DumpJson(result) class RapiUsers(object): def __init__(self): """Initializes this class. """ self._users = None def Get(self, username): """Checks whether a user exists. """ if self._users: return self._users.get(username, None) else: return None def Load(self, filename): """Loads a file containing users and passwords. @type filename: string @param filename: Path to file """ logging.info("Reading users file at %s", filename) try: try: contents = utils.ReadFile(filename) except EnvironmentError as err: self._users = None if err.errno == errno.ENOENT: logging.warning("No users file at %s", filename) else: logging.warning("Error while reading %s: %s", filename, err) return False users = http.auth.ParsePasswordFile(contents) except Exception as err: # pylint: disable=W0703 # We don't care about the type of exception logging.error("Error while parsing %s: %s", filename, err) return False self._users = users return True class FileEventHandler(asyncnotifier.FileEventHandlerBase): def __init__(self, wm, path, cb): """Initializes this class. @param wm: Inotify watch manager @type path: string @param path: File path @type cb: callable @param cb: Function called on file change """ asyncnotifier.FileEventHandlerBase.__init__(self, wm) self._cb = cb self._filename = os.path.basename(path) # Different Pyinotify versions have the flag constants at different places, # hence not accessing them directly mask = (pyinotify.EventsCodes.ALL_FLAGS["IN_CLOSE_WRITE"] | pyinotify.EventsCodes.ALL_FLAGS["IN_DELETE"] | pyinotify.EventsCodes.ALL_FLAGS["IN_MOVED_FROM"] | pyinotify.EventsCodes.ALL_FLAGS["IN_MOVED_TO"]) self._handle = self.AddWatch(os.path.dirname(path), mask) def process_default(self, event): """Called upon inotify event. """ if event.name == self._filename: logging.debug("Received inotify event %s", event) self._cb() def SetupFileWatcher(filename, cb): """Configures an inotify watcher for a file. @type filename: string @param filename: File to watch @type cb: callable @param cb: Function called on file change """ wm = pyinotify.WatchManager() handler = FileEventHandler(wm, filename, cb) asyncnotifier.AsyncNotifier(wm, default_proc_fun=handler) def CheckRapi(options, args): """Initial checks whether to run or exit with a failure. """ if args: # rapi doesn't take any arguments print("Usage: %s [-f] [-d] [-p port] [-b ADDRESS]" % sys.argv[0], file=sys.stderr) sys.exit(constants.EXIT_FAILURE) if options.max_clients < 1: print("%s --max-clients argument must be >= 1" % sys.argv[0], file=sys.stderr) sys.exit(constants.EXIT_FAILURE) ssconf.CheckMaster(options.debug) # Read SSL certificate (this is a little hackish to read the cert as root) if options.ssl: options.ssl_params = http.HttpSslParams(ssl_key_path=options.ssl_key, ssl_cert_path=options.ssl_cert, ssl_chain_path=options.ssl_chain) else: options.ssl_params = None def PrepRapi(options, _): """Prep remote API function, executed with the PID file held. """ mainloop = daemon.Mainloop() users = RapiUsers() handler = RemoteApiHandler(users.Get, options.reqauth) # Setup file watcher (it'll be driven by asyncore) SetupFileWatcher(pathutils.RAPI_USERS_FILE, compat.partial(users.Load, pathutils.RAPI_USERS_FILE)) users.Load(pathutils.RAPI_USERS_FILE) server = http.server.HttpServer( mainloop, options.bind_address, options.port, options.max_clients, handler, ssl_params=options.ssl_params, ssl_verify_peer=False) server.Start() return (mainloop, server) def ExecRapi(options, args, prep_data): # pylint: disable=W0613 """Main remote API function, executed with the PID file held. """ (mainloop, server) = prep_data try: mainloop.Run() finally: server.Stop() def Main(): """Main function. """ parser = optparse.OptionParser(description="Ganeti Remote API", usage=("%prog [-f] [-d] [-p port] [-b ADDRESS]" " [-i INTERFACE]"), version="%%prog (ganeti) %s" % constants.RELEASE_VERSION) parser.add_option("--require-authentication", dest="reqauth", default=False, action="store_true", help=("Disable anonymous HTTP requests and require" " authentication")) parser.add_option("--max-clients", dest="max_clients", default=20, type="int", help="Number of simultaneous connections accepted" " by ganeti-rapi") parser.add_option("--ssl-chain", dest="ssl_chain", help="SSL Certificate chain path", default=None, type="string") daemon.GenericMain(constants.RAPI, parser, CheckRapi, PrepRapi, ExecRapi, default_ssl_cert=pathutils.RAPI_CERT_FILE, default_ssl_key=pathutils.RAPI_CERT_FILE) ganeti-3.1.0~rc2/lib/ssconf.py000064400000000000000000000347771476477700300162410ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Global Configuration data for Ganeti. This module provides the interface to a special case of cluster configuration data, which is mostly static and available to all nodes. """ import sys import errno import logging from ganeti import errors from ganeti import constants from ganeti import utils from ganeti import netutils from ganeti import pathutils SSCONF_LOCK_TIMEOUT = 10 #: Valid ssconf keys _VALID_KEYS = constants.VALID_SS_KEYS #: Maximum size for ssconf files _MAX_SIZE = 128 * 1024 def ReadSsconfFile(filename): """Reads an ssconf file and verifies its size. @type filename: string @param filename: Path to file @rtype: string @return: File contents without newlines at the end @raise RuntimeError: When the file size exceeds L{_MAX_SIZE} """ statcb = utils.FileStatHelper() data = utils.ReadFile(filename, size=_MAX_SIZE, preread=statcb) if statcb.st.st_size > _MAX_SIZE: msg = ("File '%s' has a size of %s bytes (up to %s allowed)" % (filename, statcb.st.st_size, _MAX_SIZE)) raise RuntimeError(msg) return data.rstrip("\n") class SimpleStore(object): """Interface to static cluster data. This is different that the config.ConfigWriter and SimpleConfigReader classes in that it holds data that will always be present, even on nodes which don't have all the cluster data. Other particularities of the datastore: - keys are restricted to predefined values """ def __init__(self, cfg_location=None, _lockfile=pathutils.SSCONF_LOCK_FILE): if cfg_location is None: self._cfg_dir = pathutils.DATA_DIR else: self._cfg_dir = cfg_location self._lockfile = _lockfile def KeyToFilename(self, key): """Convert a given key into filename. """ if key not in _VALID_KEYS: raise errors.ProgrammerError("Invalid key requested from SSConf: '%s'" % str(key)) filename = self._cfg_dir + "/" + constants.SSCONF_FILEPREFIX + key return filename def _ReadFile(self, key, default=None): """Generic routine to read keys. This will read the file which holds the value requested. Errors will be changed into ConfigurationErrors. """ filename = self.KeyToFilename(key) try: return ReadSsconfFile(filename) except EnvironmentError as err: if err.errno == errno.ENOENT and default is not None: return default raise errors.ConfigurationError("Can't read ssconf file %s: %s" % (filename, str(err))) def ReadAll(self): """Reads all keys and returns their values. @rtype: dict @return: Dictionary, ssconf key as key, value as value """ result = [] for key in _VALID_KEYS: try: value = self._ReadFile(key) except errors.ConfigurationError: # Ignore non-existing files pass else: result.append((key, value)) return dict(result) def WriteFiles(self, values, dry_run=False): """Writes ssconf files used by external scripts. @type values: dict @param values: Dictionary of (name, value) @type dry_run boolean @param dry_run: Whether to perform a dry run """ ssconf_lock = utils.FileLock.Open(self._lockfile) # Get lock while writing files ssconf_lock.Exclusive(blocking=True, timeout=SSCONF_LOCK_TIMEOUT) try: for name, value in values.items(): if isinstance(value, (list, tuple)): value = "\n".join(value) if value and not value.endswith("\n"): value += "\n" if len(value) > _MAX_SIZE: msg = ("Value '%s' has a length of %s bytes, but only up to %s are" " allowed" % (name, len(value), _MAX_SIZE)) raise errors.ConfigurationError(msg) utils.WriteFile(self.KeyToFilename(name), data=value, mode=constants.SS_FILE_PERMS, dry_run=dry_run) finally: ssconf_lock.Unlock() def GetFileList(self): """Return the list of all config files. This is used for computing node replication data. """ return [self.KeyToFilename(key) for key in _VALID_KEYS] def GetClusterName(self): """Get the cluster name. """ return self._ReadFile(constants.SS_CLUSTER_NAME) def GetFileStorageDir(self): """Get the file storage dir. """ return self._ReadFile(constants.SS_FILE_STORAGE_DIR) def GetSharedFileStorageDir(self): """Get the shared file storage dir. """ return self._ReadFile(constants.SS_SHARED_FILE_STORAGE_DIR) def GetGlusterStorageDir(self): """Get the Gluster storage dir. """ return self._ReadFile(constants.SS_GLUSTER_STORAGE_DIR) def GetMasterCandidates(self): """Return the list of master candidates. """ data = self._ReadFile(constants.SS_MASTER_CANDIDATES) nl = data.splitlines(False) return nl def GetMasterCandidatesIPList(self): """Return the list of master candidates' primary IP. """ data = self._ReadFile(constants.SS_MASTER_CANDIDATES_IPS) nl = data.splitlines(False) return nl def _GetDictOfSsconfMap(self, ss_file_key): """Reads a file with lines like key=value and returns a dict. This utility function reads a file containing ssconf values of the form "key=value", splits the lines at "=" and returns a dictionary mapping the keys to the values. @type ss_file_key: string @param ss_file_key: the constant referring to an ssconf file @rtype: dict of string to string @return: a dictionary mapping the keys to the values """ data = self._ReadFile(ss_file_key) lines = data.splitlines(False) mapping = {} for line in lines: (key, value) = line.split("=") mapping[key] = value return mapping def GetMasterCandidatesCertMap(self): """Returns the map of master candidate UUIDs to ssl cert. @rtype: dict of string to string @return: dictionary mapping the master candidates' UUIDs to their SSL certificate digests """ return self._GetDictOfSsconfMap(constants.SS_MASTER_CANDIDATES_CERTS) def GetSshPortMap(self): """Returns the map of node names to SSH port. @rtype: dict of string to string @return: dictionary mapping the node names to their SSH port """ return dict([(node_name, int(ssh_port)) for node_name, ssh_port in self._GetDictOfSsconfMap(constants.SS_SSH_PORTS).items()]) def GetMasterIP(self): """Get the IP of the master node for this cluster. """ return self._ReadFile(constants.SS_MASTER_IP) def GetMasterNetdev(self): """Get the netdev to which we'll add the master ip. """ return self._ReadFile(constants.SS_MASTER_NETDEV) def GetMasterNetmask(self): """Get the master netmask. """ try: return self._ReadFile(constants.SS_MASTER_NETMASK) except errors.ConfigurationError: family = self.GetPrimaryIPFamily() ipcls = netutils.IPAddress.GetClassFromIpFamily(family) return ipcls.iplen def GetMasterNode(self): """Get the hostname of the master node for this cluster. """ return self._ReadFile(constants.SS_MASTER_NODE) def GetNodeList(self): """Return the list of cluster nodes. """ data = self._ReadFile(constants.SS_NODE_LIST) nl = data.splitlines(False) return nl def GetOnlineNodeList(self): """Return the list of online cluster nodes. """ data = self._ReadFile(constants.SS_ONLINE_NODES) nl = data.splitlines(False) return nl def GetNodePrimaryIPList(self): """Return the list of cluster nodes' primary IP. """ data = self._ReadFile(constants.SS_NODE_PRIMARY_IPS) nl = data.splitlines(False) return nl def GetNodeSecondaryIPList(self): """Return the list of cluster nodes' secondary IP. """ data = self._ReadFile(constants.SS_NODE_SECONDARY_IPS) nl = data.splitlines(False) return nl def GetNodesVmCapable(self): """Return the cluster nodes' vm capable value. @rtype: dict of string to bool @return: mapping of node names to vm capable values """ data = self._ReadFile(constants.SS_NODE_VM_CAPABLE) vm_capable = {} for line in data.splitlines(False): (node_uuid, node_vm_capable) = line.split("=") vm_capable[node_uuid] = node_vm_capable == "True" return vm_capable def GetNodegroupList(self): """Return the list of nodegroups. """ data = self._ReadFile(constants.SS_NODEGROUPS) nl = data.splitlines(False) return nl def GetNetworkList(self): """Return the list of networks. """ data = self._ReadFile(constants.SS_NETWORKS) nl = data.splitlines(False) return nl def GetClusterTags(self): """Return the cluster tags. """ data = self._ReadFile(constants.SS_CLUSTER_TAGS) nl = data.splitlines(False) return nl def GetHypervisorList(self): """Return the list of enabled hypervisors. """ data = self._ReadFile(constants.SS_HYPERVISOR_LIST) nl = data.splitlines(False) return nl def GetHvparamsForHypervisor(self, hvname): """Return the hypervisor parameters of the given hypervisor. @type hvname: string @param hvname: name of the hypervisor, must be in C{constants.HYPER_TYPES} @rtype: dict of strings @returns: dictionary with hypervisor parameters """ return self._GetDictOfSsconfMap(constants.SS_HVPARAMS_PREF + hvname) def GetHvparams(self): """Return the hypervisor parameters of all hypervisors. @rtype: dict of dict of strings @returns: dictionary mapping hypervisor names to hvparams """ all_hvparams = {} for hv in constants.HYPER_TYPES: all_hvparams[hv] = self.GetHvparamsForHypervisor(hv) return all_hvparams def GetMaintainNodeHealth(self): """Return the value of the maintain_node_health option. """ data = self._ReadFile(constants.SS_MAINTAIN_NODE_HEALTH) # we rely on the bool serialization here return data == "True" def GetUidPool(self): """Return the user-id pool definition string. The separator character is a newline. The return value can be parsed using uidpool.ParseUidPool():: ss = ssconf.SimpleStore() uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\\n") """ data = self._ReadFile(constants.SS_UID_POOL) return data def GetPrimaryIPFamily(self): """Return the cluster-wide primary address family. """ try: return int(self._ReadFile(constants.SS_PRIMARY_IP_FAMILY, default=netutils.IP4Address.family)) except (ValueError, TypeError) as err: raise errors.ConfigurationError("Error while trying to parse primary IP" " family: %s" % err) def GetEnabledUserShutdown(self): """Return whether user shutdown is enabled. @rtype: bool @return: 'True' if user shutdown is enabled, 'False' otherwise """ return self._ReadFile(constants.SS_ENABLED_USER_SHUTDOWN) == "True" def WriteSsconfFiles(values, dry_run=False): """Update all ssconf files. Wrapper around L{SimpleStore.WriteFiles}. """ SimpleStore().WriteFiles(values, dry_run=dry_run) def GetMasterAndMyself(ss=None): """Get the master node and my own hostname. This can be either used for a 'soft' check (compared to CheckMaster, which exits) or just for computing both at the same time. The function does not handle any errors, these should be handled in the caller (errors.ConfigurationError, errors.ResolverError). @param ss: either a sstore.SimpleConfigReader or a sstore.SimpleStore instance @rtype: tuple @return: a tuple (master node name, my own name) """ if ss is None: ss = SimpleStore() return ss.GetMasterNode(), netutils.Hostname.GetSysName() def CheckMaster(debug, ss=None): """Checks the node setup. If this is the master, the function will return. Otherwise it will exit with an exit code based on the node status. """ try: master_name, myself = GetMasterAndMyself(ss) except errors.ConfigurationError as err: print("Cluster configuration incomplete: '%s'" % str(err)) sys.exit(constants.EXIT_NODESETUP_ERROR) except errors.ResolverError as err: sys.stderr.write("Cannot resolve my own name (%s)\n" % err.args[0]) sys.exit(constants.EXIT_NODESETUP_ERROR) if myself != master_name: if debug: sys.stderr.write("Not master, exiting.\n") sys.exit(constants.EXIT_NOTMASTER) def VerifyClusterName(name, _cfg_location=None): """Verifies cluster name against a local cluster name. @type name: string @param name: Cluster name """ sstore = SimpleStore(cfg_location=_cfg_location) try: local_name = sstore.GetClusterName() except errors.ConfigurationError as err: logging.debug("Can't get local cluster name: %s", err) else: if name != local_name: raise errors.GenericError("Current cluster name is '%s'" % local_name) def VerifyKeys(keys): """Raises an exception if unknown ssconf keys are given. @type keys: sequence @param keys: Key names to verify @raise errors.GenericError: When invalid keys were found """ invalid = frozenset(keys) - _VALID_KEYS if invalid: raise errors.GenericError("Invalid ssconf keys: %s" % utils.CommaJoin(sorted(invalid))) ganeti-3.1.0~rc2/lib/ssh.py000064400000000000000000001122141476477700300155220ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module encapsulating ssh functionality. """ import logging import os import tempfile from collections import namedtuple from functools import partial from ganeti import utils from ganeti import errors from ganeti import constants from ganeti import netutils from ganeti import pathutils from ganeti import vcluster from ganeti import compat from ganeti import serializer from ganeti import ssconf def GetUserFiles(user, mkdir=False, dircheck=True, kind=constants.SSHK_DSA, _homedir_fn=None): """Return the paths of a user's SSH files. @type user: string @param user: Username @type mkdir: bool @param mkdir: Whether to create ".ssh" directory if it doesn't exist @type dircheck: bool @param dircheck: Whether to check if ".ssh" directory exists @type kind: string @param kind: One of L{constants.SSHK_ALL} @rtype: tuple; (string, string, string) @return: Tuple containing three file system paths; the private SSH key file, the public SSH key file and the user's C{authorized_keys} file @raise errors.OpExecError: When home directory of the user can not be determined @raise errors.OpExecError: Regardless of the C{mkdir} parameters, this exception is raised if C{~$user/.ssh} is not a directory and C{dircheck} is set to C{True} """ if _homedir_fn is None: _homedir_fn = utils.GetHomeDir user_dir = _homedir_fn(user) if not user_dir: raise errors.OpExecError("Cannot resolve home of user '%s'" % user) if kind == constants.SSHK_DSA: suffix = "dsa" elif kind == constants.SSHK_RSA: suffix = "rsa" elif kind == constants.SSHK_ECDSA: suffix = "ecdsa" else: raise errors.ProgrammerError("Unknown SSH key kind '%s'" % kind) ssh_dir = utils.PathJoin(user_dir, ".ssh") if mkdir: utils.EnsureDirs([(ssh_dir, constants.SECURE_DIR_MODE)]) elif dircheck and not os.path.isdir(ssh_dir): raise errors.OpExecError("Path %s is not a directory" % ssh_dir) return [utils.PathJoin(ssh_dir, base) for base in ["id_%s" % suffix, "id_%s.pub" % suffix, "authorized_keys"]] def GetAllUserFiles(user, mkdir=False, dircheck=True, _homedir_fn=None): """Wrapper over L{GetUserFiles} to retrieve files for all SSH key types. See L{GetUserFiles} for details. @rtype: tuple; (string, dict with string as key, tuple of (string, string) as value) """ helper = compat.partial(GetUserFiles, user, mkdir=mkdir, dircheck=dircheck, _homedir_fn=_homedir_fn) result = [(kind, helper(kind=kind)) for kind in constants.SSHK_ALL] authorized_keys = [i for (_, (_, _, i)) in result] assert len(frozenset(authorized_keys)) == 1, \ "Different paths for authorized_keys were returned" return (authorized_keys[0], dict((kind, (privkey, pubkey)) for (kind, (privkey, pubkey, _)) in result)) def _SplitSshKey(key): """Splits a line for SSH's C{authorized_keys} file. If the line has no options (e.g. no C{command="..."}), only the significant parts, the key type and its hash, are used. Otherwise the whole line is used (split at whitespace). @type key: string @param key: Key line @rtype: tuple """ parts = key.split() if parts and parts[0] in constants.SSHAK_ALL: # If the key has no options in front of it, we only want the significant # fields return (False, parts[:2]) else: # Can't properly split the line, so use everything return (True, parts) def AddAuthorizedKeys(file_obj, keys): """Adds a list of SSH public key to an authorized_keys file. @type file_obj: str or file handle @param file_obj: path to authorized_keys file @type keys: list of str @param keys: list of strings containing keys """ key_field_list = [(key, _SplitSshKey(key)) for key in keys] if isinstance(file_obj, str): f = open(file_obj, "a+") f.seek(0) else: f = file_obj try: nl = True for line in f: # Ignore whitespace changes line_key = _SplitSshKey(line) key_field_list[:] = [(key, split_key) for (key, split_key) in key_field_list if split_key != line_key] nl = line.endswith("\n") if not nl: f.write("\n") for (key, _) in key_field_list: f.write(key.rstrip("\r\n")) f.write("\n") f.flush() finally: f.close() def HasAuthorizedKey(file_obj, key): """Check if a particular key is in the 'authorized_keys' file. @type file_obj: str or file handle @param file_obj: path to authorized_keys file @type key: str @param key: string containing key """ key_fields = _SplitSshKey(key) if isinstance(file_obj, str): f = open(file_obj, "r") else: f = file_obj try: for line in f: # Ignore whitespace changes line_key = _SplitSshKey(line) if line_key == key_fields: return True finally: f.close() return False def CheckForMultipleKeys(file_obj, node_names): """Check if there is at most one key per host in 'authorized_keys' file. @type file_obj: str or file handle @param file_obj: path to authorized_keys file @type node_names: list of str @param node_names: list of names of nodes of the cluster @returns: a dictionary with hostnames which occur more than once """ if isinstance(file_obj, str): f = open(file_obj, "r") else: f = file_obj occurrences = {} try: index = 0 for line in f: index += 1 if line.startswith("#"): continue chunks = line.split() # find the chunk with user@hostname user_hostname = [chunk.strip() for chunk in chunks if "@" in chunk][0] if not user_hostname in occurrences: occurrences[user_hostname] = [] occurrences[user_hostname].append(index) finally: f.close() bad_occurrences = {} for user_hostname, occ in occurrences.items(): _, hostname = user_hostname.split("@") if hostname in node_names and len(occ) > 1: bad_occurrences[user_hostname] = occ return bad_occurrences def AddAuthorizedKey(file_obj, key): """Adds an SSH public key to an authorized_keys file. @type file_obj: str or file handle @param file_obj: path to authorized_keys file @type key: str @param key: string containing key """ AddAuthorizedKeys(file_obj, [key]) def RemoveAuthorizedKeys(file_name, keys): """Removes public SSH keys from an authorized_keys file. @type file_name: str @param file_name: path to authorized_keys file @type keys: list of str @param keys: list of strings containing keys """ key_field_list = [_SplitSshKey(key) for key in keys] fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name)) try: out = os.fdopen(fd, "w") try: f = open(file_name, "r") try: for line in f: # Ignore whitespace changes while comparing lines if _SplitSshKey(line) not in key_field_list: out.write(line) out.flush() os.rename(tmpname, file_name) finally: f.close() finally: out.close() except: utils.RemoveFile(tmpname) raise def RemoveAuthorizedKey(file_name, key): """Removes an SSH public key from an authorized_keys file. @type file_name: str @param file_name: path to authorized_keys file @type key: str @param key: string containing key """ RemoveAuthorizedKeys(file_name, [key]) def _AddPublicKeyProcessLine(new_uuid, new_key, line_uuid, line_key, found): """Processes one line of the public key file when adding a key. This is a sub function that can be called within the C{_ManipulatePublicKeyFile} function. It processes one line of the public key file, checks if this line contains the key to add already and if so, notes the occurrence in the return value. @type new_uuid: string @param new_uuid: the node UUID of the node whose key is added @type new_key: string @param new_key: the SSH key to be added @type line_uuid: the UUID of the node whose line in the public key file is processed in this function call @param line_key: the SSH key of the node whose line in the public key file is processed in this function call @type found: boolean @param found: whether or not the (UUID, key) pair of the node whose key is being added was found in the public key file already. @rtype: (boolean, string) @return: a possibly updated value of C{found} and the processed line """ if line_uuid == new_uuid and line_key == new_key: logging.debug("SSH key of node '%s' already in key file.", new_uuid) found = True return (found, "%s %s\n" % (line_uuid, line_key)) def _AddPublicKeyElse(new_uuid, new_key): """Adds a new SSH key to the key file if it did not exist already. This is an auxiliary function for C{_ManipulatePublicKeyFile} which is carried out when a new key is added to the public key file and after processing the whole file, we found out that the key does not exist in the file yet but needs to be appended at the end. @type new_uuid: string @param new_uuid: the UUID of the node whose key is added @type new_key: string @param new_key: the SSH key to be added @rtype: string @return: a new line to be added to the file """ return "%s %s\n" % (new_uuid, new_key) def _RemovePublicKeyProcessLine( target_uuid, _target_key, line_uuid, line_key, found): """Processes a line in the public key file when aiming for removing a key. This is an auxiliary function for C{_ManipulatePublicKeyFile} when we are removing a key from the public key file. This particular function only checks if the current line contains the UUID of the node in question and writes the line to the temporary file otherwise. @type target_uuid: string @param target_uuid: UUID of the node whose key is being removed @type _target_key: string @param _target_key: SSH key of the node (not used) @type line_uuid: string @param line_uuid: UUID of the node whose line is processed in this call @type line_key: string @param line_key: SSH key of the nodes whose line is processed in this call @type found: boolean @param found: whether or not the UUID was already found. @rtype: (boolean, string) @return: a tuple, indicating if the target line was found and the processed line; the line is 'None', if the original line is removed """ if line_uuid != target_uuid: return (found, "%s %s\n" % (line_uuid, line_key)) else: return (True, None) def _RemovePublicKeyElse( target_uuid, _target_key): """Logs when we tried to remove a key that does not exist. This is an auxiliary function for C{_ManipulatePublicKeyFile} which is run after we have processed the complete public key file and did not find the key to be removed. @type target_uuid: string @param target_uuid: the UUID of the node whose key was supposed to be removed @type _target_key: string @param _target_key: the key of the node which was supposed to be removed (not used) @rtype: string @return: in this case, always None """ logging.debug("Trying to remove key of node '%s' which is not in list" " of public keys.", target_uuid) return None def _ReplaceNameByUuidProcessLine( node_name, _key, line_identifier, line_key, found, node_uuid=None): """Replaces a node's name with its UUID on a matching line in the key file. This is an auxiliary function for C{_ManipulatePublicKeyFile} which processes a line of the ganeti public key file. If the line in question matches the node's name, the name will be replaced by the node's UUID. @type node_name: string @param node_name: name of the node to be replaced by the UUID @type _key: string @param _key: SSH key of the node (not used) @type line_identifier: string @param line_identifier: an identifier of a node in a line of the public key file. This can be either a node name or a node UUID, depending on if it got replaced already or not. @type line_key: string @param line_key: SSH key of the node whose line is processed @type found: boolean @param found: whether or not the line matches the node's name @type node_uuid: string @param node_uuid: the node's UUID which will replace the node name @rtype: (boolean, string) @return: a tuple indicating whether the target line was found and the processed line """ if node_name == line_identifier: return (True, "%s %s\n" % (node_uuid, line_key)) else: return (found, "%s %s\n" % (line_identifier, line_key)) def _ReplaceNameByUuidElse( node_uuid, node_name, _key): """Logs a debug message when we try to replace a key that is not there. This is an implementation of the auxiliary C{process_else_fn} function for the C{_ManipulatePubKeyFile} function when we use it to replace a line in the public key file that is indexed by the node's name instead of the node's UUID. @type node_uuid: string @param node_uuid: the node's UUID @type node_name: string @param node_name: the node's UUID @type _key: string (not used) @param _key: the node's SSH key (not used) @rtype: string @return: in this case, always None """ logging.debug("Trying to replace node name '%s' with UUID '%s', but" " no line with that name was found.", node_name, node_uuid) return None def _ParseKeyLine(line, error_fn): """Parses a line of the public key file. @type line: string @param line: line of the public key file @type error_fn: function @param error_fn: function to process error messages @rtype: tuple (string, string) @return: a tuple containing the UUID of the node and a string containing the SSH key and possible more parameters for the key """ if len(line.rstrip()) == 0: return (None, None) chunks = line.split(" ") if len(chunks) < 2: raise error_fn("Error parsing public SSH key file. Line: '%s'" % line) uuid = chunks[0] key = " ".join(chunks[1:]).rstrip() return (uuid, key) def _ManipulatePubKeyFile(target_identifier, target_key, key_file=pathutils.SSH_PUB_KEYS, error_fn=errors.ProgrammerError, process_line_fn=None, process_else_fn=None): """Manipulates the list of public SSH keys of the cluster. This is a general function to manipulate the public key file. It needs two auxiliary functions C{process_line_fn} and C{process_else_fn} to work. Generally, the public key file is processed as follows: 1) The function processes each line of the original ganeti public key file, applies the C{process_line_fn} function on it, which returns a possibly manipulated line and an indicator whether the line in question was found. If a line is returned, it is added to a list of lines for later writing to the file. 2) If all lines are processed and the 'found' variable is False, the seconds auxiliary function C{process_else_fn} is called to possibly add more lines to the list of lines. 3) Finally, the list of lines is assembled to a string and written atomically to the public key file, thereby overriding it. If the public key file does not exist, we create it. This is necessary for a smooth transition after an upgrade. @type target_identifier: str @param target_identifier: identifier of the node whose key is added; in most cases this is the node's UUID, but in some it is the node's host name @type target_key: str @param target_key: string containing a public SSH key (a complete line possibly including more parameters than just the key) @type key_file: str @param key_file: filename of the file of public node keys (optional parameter for testing) @type error_fn: function @param error_fn: Function that returns an exception, used to customize exception types depending on the calling context @type process_line_fn: function @param process_line_fn: function to process one line of the public key file @type process_else_fn: function @param process_else_fn: function to be called if no line of the key file matches the target uuid """ assert process_else_fn is not None assert process_line_fn is not None old_lines = [] f_orig = None if os.path.exists(key_file): try: f_orig = open(key_file, "r") old_lines = f_orig.readlines() finally: f_orig.close() else: try: f_orig = open(key_file, "w") f_orig.close() except IOError as e: raise errors.SshUpdateError("Cannot create public key file: %s" % e) found = False new_lines = [] for line in old_lines: (uuid, key) = _ParseKeyLine(line, error_fn) if not uuid: continue (new_found, new_line) = process_line_fn(target_identifier, target_key, uuid, key, found) if new_found: found = True if new_line is not None: new_lines.append(new_line) if not found: new_line = process_else_fn(target_identifier, target_key) if new_line is not None: new_lines.append(new_line) new_file_content = "".join(new_lines) utils.WriteFile(key_file, data=new_file_content) def AddPublicKey(new_uuid, new_key, key_file=pathutils.SSH_PUB_KEYS, error_fn=errors.ProgrammerError): """Adds a new key to the list of public keys. @see: _ManipulatePubKeyFile for parameter descriptions. """ _ManipulatePubKeyFile(new_uuid, new_key, key_file=key_file, process_line_fn=_AddPublicKeyProcessLine, process_else_fn=_AddPublicKeyElse, error_fn=error_fn) def RemovePublicKey(target_uuid, key_file=pathutils.SSH_PUB_KEYS, error_fn=errors.ProgrammerError): """Removes a key from the list of public keys. @see: _ManipulatePubKeyFile for parameter descriptions. """ _ManipulatePubKeyFile(target_uuid, None, key_file=key_file, process_line_fn=_RemovePublicKeyProcessLine, process_else_fn=_RemovePublicKeyElse, error_fn=error_fn) def ReplaceNameByUuid(node_uuid, node_name, key_file=pathutils.SSH_PUB_KEYS, error_fn=errors.ProgrammerError): """Replaces a host name with the node's corresponding UUID. When a node is added to the cluster, we don't know it's UUID yet. So first its SSH key gets added to the public key file and in a second step, the node's name gets replaced with the node's UUID as soon as we know the UUID. @type node_uuid: string @param node_uuid: the node's UUID to replace the node's name @type node_name: string @param node_name: the node's name to be replaced by the node's UUID @see: _ManipulatePubKeyFile for the other parameter descriptions. """ process_line_fn = partial(_ReplaceNameByUuidProcessLine, node_uuid=node_uuid) process_else_fn = partial(_ReplaceNameByUuidElse, node_uuid=node_uuid) _ManipulatePubKeyFile(node_name, None, key_file=key_file, process_line_fn=process_line_fn, process_else_fn=process_else_fn, error_fn=error_fn) def ClearPubKeyFile(key_file=pathutils.SSH_PUB_KEYS, mode=0o600): """Resets the content of the public key file. """ utils.WriteFile(key_file, data="", mode=mode) def OverridePubKeyFile(key_map, key_file=pathutils.SSH_PUB_KEYS): """Overrides the public key file with a list of given keys. @type key_map: dict from str to list of str @param key_map: dictionary mapping uuids to lists of SSH keys """ new_lines = [] for (uuid, keys) in key_map.items(): for key in keys: new_lines.append("%s %s\n" % (uuid, key)) new_file_content = "".join(new_lines) utils.WriteFile(key_file, data=new_file_content) def QueryPubKeyFile(target_uuids, key_file=pathutils.SSH_PUB_KEYS, error_fn=errors.ProgrammerError): """Retrieves a map of keys for the requested node UUIDs. @type target_uuids: str or list of str @param target_uuids: UUID of the node to retrieve the key for or a list of UUIDs of nodes to retrieve the keys for @type key_file: str @param key_file: filename of the file of public node keys (optional parameter for testing) @type error_fn: function @param error_fn: Function that returns an exception, used to customize exception types depending on the calling context @rtype: dict mapping strings to list of strings @return: dictionary mapping node uuids to their ssh keys """ all_keys = target_uuids is None if isinstance(target_uuids, str): target_uuids = [target_uuids] result = {} f = open(key_file, "r") try: for line in f: (uuid, key) = _ParseKeyLine(line, error_fn) if not uuid: continue if all_keys or (uuid in target_uuids): if uuid not in result: result[uuid] = [] result[uuid].append(key) finally: f.close() return result def InitSSHSetup(key_type, key_bits, error_fn=errors.OpPrereqError, _homedir_fn=None, _suffix=""): """Setup the SSH configuration for the node. This generates a dsa keypair for root, adds the pub key to the permitted hosts and adds the hostkey to its own known hosts. @param key_type: the type of SSH keypair to be generated @param key_bits: the key length, in bits, to be used """ priv_key, _, auth_keys = GetUserFiles(constants.SSH_LOGIN_USER, kind=key_type, mkdir=True, _homedir_fn=_homedir_fn) new_priv_key_name = priv_key + _suffix new_pub_key_name = priv_key + _suffix + ".pub" for name in new_priv_key_name, new_pub_key_name: if os.path.exists(name): utils.CreateBackup(name) utils.RemoveFile(name) result = utils.RunCmd(["ssh-keygen", "-b", str(key_bits), "-t", key_type, "-f", new_priv_key_name, "-q", "-N", ""]) if result.failed: raise error_fn("Could not generate ssh keypair, error %s" % result.output) AddAuthorizedKey(auth_keys, utils.ReadFile(new_pub_key_name)) def InitPubKeyFile(master_uuid, key_type, key_file=pathutils.SSH_PUB_KEYS): """Creates the public key file and adds the master node's SSH key. @type master_uuid: str @param master_uuid: the master node's UUID @type key_type: one of L{constants.SSHK_ALL} @param key_type: the type of ssh key to be used @type key_file: str @param key_file: name of the file containing the public keys """ _, pub_key, _ = GetUserFiles(constants.SSH_LOGIN_USER, kind=key_type) ClearPubKeyFile(key_file=key_file) key = utils.ReadFile(pub_key) AddPublicKey(master_uuid, key, key_file=key_file) class SshRunner(object): """Wrapper for SSH commands. """ def __init__(self, cluster_name): """Initializes this class. @type cluster_name: str @param cluster_name: name of the cluster """ self.cluster_name = cluster_name family = ssconf.SimpleStore().GetPrimaryIPFamily() self.ipv6 = (family == netutils.IP6Address.family) def _BuildSshOptions(self, batch, ask_key, use_cluster_key, strict_host_check, private_key=None, quiet=True, port=None): """Builds a list with needed SSH options. @param batch: same as ssh's batch option @param ask_key: allows ssh to ask for key confirmation; this parameter conflicts with the batch one @param use_cluster_key: if True, use the cluster name as the HostKeyAlias name @param strict_host_check: this makes the host key checking strict @param private_key: use this private key instead of the default @param quiet: whether to enable -q to ssh @param port: the SSH port to use, or None to use the default @rtype: list @return: the list of options ready to use in L{utils.process.RunCmd} """ options = [ "-oEscapeChar=none", "-oHashKnownHosts=no", "-oGlobalKnownHostsFile=%s" % pathutils.SSH_KNOWN_HOSTS_FILE, "-oUserKnownHostsFile=/dev/null", "-oCheckHostIp=no", "-oConnectTimeout=10", ] if use_cluster_key: options.append("-oHostKeyAlias=%s" % self.cluster_name) if quiet: options.append("-q") if private_key: options.append("-i%s" % private_key) if port: options.append("-oPort=%d" % port) # TODO: Too many boolean options, maybe convert them to more descriptive # constants. # Note: ask_key conflicts with batch mode if batch: if ask_key: raise errors.ProgrammerError("SSH call requested conflicting options") options.append("-oBatchMode=yes") if strict_host_check: options.append("-oStrictHostKeyChecking=yes") else: options.append("-oStrictHostKeyChecking=no") else: # non-batch mode if ask_key: options.append("-oStrictHostKeyChecking=ask") elif strict_host_check: options.append("-oStrictHostKeyChecking=yes") else: options.append("-oStrictHostKeyChecking=no") if self.ipv6: options.append("-6") else: options.append("-4") return options def BuildCmd(self, hostname, user, command, batch=True, ask_key=False, tty=False, use_cluster_key=True, strict_host_check=True, private_key=None, quiet=True, port=None): """Build an ssh command to execute a command on a remote node. @param hostname: the target host, string @param user: user to auth as @param command: the command @param batch: if true, ssh will run in batch mode with no prompting @param ask_key: if true, ssh will run with StrictHostKeyChecking=ask, so that we can connect to an unknown host (not valid in batch mode) @param use_cluster_key: whether to expect and use the cluster-global SSH key @param strict_host_check: whether to check the host's SSH key at all @param private_key: use this private key instead of the default @param quiet: whether to enable -q to ssh @param port: the SSH port on which the node's daemon is running @return: the ssh call to run 'command' on the remote host. """ argv = [constants.SSH] argv.extend(self._BuildSshOptions(batch, ask_key, use_cluster_key, strict_host_check, private_key, quiet=quiet, port=port)) if tty: argv.extend(["-t", "-t"]) argv.append("%s@%s" % (user, hostname)) # Insert variables for virtual nodes argv.extend("export %s=%s;" % (utils.ShellQuote(name), utils.ShellQuote(value)) for (name, value) in vcluster.EnvironmentForHost(hostname).items()) argv.append(command) return argv def Run(self, *args, **kwargs): """Runs a command on a remote node. This method has the same return value as `utils.RunCmd()`, which it uses to launch ssh. Args: see SshRunner.BuildCmd. @rtype: L{utils.process.RunResult} @return: the result as from L{utils.process.RunCmd()} """ return utils.RunCmd(self.BuildCmd(*args, **kwargs)) def CopyFileToNode(self, node, port, filename): """Copy a file to another node with scp. @param node: node in the cluster @param filename: absolute pathname of a local file @rtype: boolean @return: the success of the operation """ if not os.path.isabs(filename): logging.error("File %s must be an absolute path", filename) return False if not os.path.isfile(filename): logging.error("File %s does not exist", filename) return False command = [constants.SCP, "-p"] command.extend(self._BuildSshOptions(True, False, True, True, port=port)) command.append(filename) if netutils.IP6Address.IsValid(node): node = netutils.FormatAddress((node, None)) command.append("%s:%s" % (node, vcluster.ExchangeNodeRoot(node, filename))) result = utils.RunCmd(command) if result.failed: logging.error("Copy to node %s failed (%s) error '%s'," " command was '%s'", node, result.fail_reason, result.output, result.cmd) return not result.failed def VerifyNodeHostname(self, node, ssh_port): """Verify hostname consistency via SSH. This functions connects via ssh to a node and compares the hostname reported by the node to the name with have (the one that we connected to). This is used to detect problems in ssh known_hosts files (conflicting known hosts) and inconsistencies between dns/hosts entries and local machine names @param node: nodename of a host to check; can be short or full qualified hostname @param ssh_port: the port of a SSH daemon running on the node @return: (success, detail), where: - success: True/False - detail: string with details """ cmd = ("if test -z \"$GANETI_HOSTNAME\"; then" " hostname --fqdn;" "else" " echo \"$GANETI_HOSTNAME\";" "fi") retval = self.Run(node, constants.SSH_LOGIN_USER, cmd, quiet=False, port=ssh_port) if retval.failed: msg = "ssh problem" output = retval.output if output: msg += ": %s" % output else: msg += ": %s (no output)" % retval.fail_reason logging.error("Command %s failed: %s", retval.cmd, msg) return False, msg remotehostname = retval.stdout.strip() if not remotehostname or remotehostname != node: if node.startswith(remotehostname + "."): msg = "hostname not FQDN" else: msg = "hostname mismatch" return False, ("%s: expected %s but got %s" % (msg, node, remotehostname)) return True, "host matches" def WriteKnownHostsFile(cfg, file_name): """Writes the cluster-wide equally known_hosts file. """ data = "" if cfg.GetRsaHostKey(): data += "%s ssh-rsa %s\n" % (cfg.GetClusterName(), cfg.GetRsaHostKey()) if cfg.GetDsaHostKey(): data += "%s ssh-dss %s\n" % (cfg.GetClusterName(), cfg.GetDsaHostKey()) utils.WriteFile(file_name, mode=0o600, data=data) def _EnsureCorrectGanetiVersion(cmd): """Ensured the correct Ganeti version before running a command via SSH. Before a command is run on a node via SSH, it makes sense in some situations to ensure that this node is indeed running the correct version of Ganeti like the rest of the cluster. @type cmd: string @param cmd: string @rtype: list of strings @return: a list of commands with the newly added ones at the beginning """ logging.debug("Ensure correct Ganeti version: %s", cmd) version = constants.DIR_VERSION all_cmds = [["test", "-d", os.path.join(pathutils.PKGLIBDIR, version)]] if constants.HAS_GNU_LN: all_cmds.extend([["ln", "-s", "-f", "-T", os.path.join(pathutils.PKGLIBDIR, version), os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")], ["ln", "-s", "-f", "-T", os.path.join(pathutils.SHAREDIR, version), os.path.join(pathutils.SYSCONFDIR, "ganeti/share")]]) else: all_cmds.extend([["rm", "-f", os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")], ["ln", "-s", "-f", os.path.join(pathutils.PKGLIBDIR, version), os.path.join(pathutils.SYSCONFDIR, "ganeti/lib")], ["rm", "-f", os.path.join(pathutils.SYSCONFDIR, "ganeti/share")], ["ln", "-s", "-f", os.path.join(pathutils.SHAREDIR, version), os.path.join(pathutils.SYSCONFDIR, "ganeti/share")]]) all_cmds.append(cmd) return all_cmds def RunSshCmdWithStdin(cluster_name, node, basecmd, port, data, debug=False, verbose=False, use_cluster_key=False, ask_key=False, strict_host_check=False, ensure_version=False): """Runs a command on a remote machine via SSH and provides input in stdin. @type cluster_name: string @param cluster_name: Cluster name @type node: string @param node: Node name @type basecmd: string @param basecmd: Base command (path on the remote machine) @type port: int @param port: The SSH port of the remote machine or None for the default @param data: JSON-serializable input data for script (passed to stdin) @type debug: bool @param debug: Enable debug output @type verbose: bool @param verbose: Enable verbose output @type use_cluster_key: bool @param use_cluster_key: See L{ssh.SshRunner.BuildCmd} @type ask_key: bool @param ask_key: See L{ssh.SshRunner.BuildCmd} @type strict_host_check: bool @param strict_host_check: See L{ssh.SshRunner.BuildCmd} """ cmd = [basecmd] # Pass --debug/--verbose to the external script if set on our invocation if debug: cmd.append("--debug") if verbose: cmd.append("--verbose") if ensure_version: all_cmds = _EnsureCorrectGanetiVersion(cmd) else: all_cmds = [cmd] if port is None: port = netutils.GetDaemonPort(constants.SSH) srun = SshRunner(cluster_name) scmd = srun.BuildCmd(node, constants.SSH_LOGIN_USER, utils.ShellQuoteArgs( utils.ShellCombineCommands(all_cmds)), batch=False, ask_key=ask_key, quiet=False, strict_host_check=strict_host_check, use_cluster_key=use_cluster_key, port=port) tempfh = tempfile.TemporaryFile() try: tempfh.write(serializer.DumpJson(data)) tempfh.seek(0) result = utils.RunCmd(scmd, interactive=True, input_fd=tempfh) finally: tempfh.close() if result.failed: raise errors.OpExecError("Command '%s' failed: %s" % (result.cmd, result.fail_reason)) def ReadRemoteSshPubKeys(pub_key_file, node, cluster_name, port, ask_key, strict_host_check): """Fetches a public SSH key from a node via SSH. @type pub_key_file: string @param pub_key_file: a tuple consisting of the file name of the public DSA key """ ssh_runner = SshRunner(cluster_name) cmd = ["cat", pub_key_file] ssh_cmd = ssh_runner.BuildCmd(node, constants.SSH_LOGIN_USER, utils.ShellQuoteArgs(cmd), batch=False, ask_key=ask_key, quiet=False, strict_host_check=strict_host_check, use_cluster_key=False, port=port) result = utils.RunCmd(ssh_cmd) if result.failed: raise errors.OpPrereqError("Could not fetch a public SSH key (%s) from node" " '%s': ran command '%s', failure reason: '%s'." % (pub_key_file, node, cmd, result.fail_reason), errors.ECODE_INVAL) return result.stdout # Update gnt-cluster.rst when changing which combinations are valid. KeyBitInfo = namedtuple('KeyBitInfo', ['default', 'validation_fn']) SSH_KEY_VALID_BITS = { constants.SSHK_DSA: KeyBitInfo(1024, lambda b: b == 1024), constants.SSHK_RSA: KeyBitInfo(2048, lambda b: b >= 768), constants.SSHK_ECDSA: KeyBitInfo(384, lambda b: b in [256, 384, 521]), } def DetermineKeyBits(key_type, key_bits, old_key_type, old_key_bits): """Checks the key bits to be used for a given key type, or provides defaults. @type key_type: one of L{constants.SSHK_ALL} @param key_type: The key type to use. @type key_bits: positive int or None @param key_bits: The number of bits to use, if supplied by user. @type old_key_type: one of L{constants.SSHK_ALL} or None @param old_key_type: The previously used key type, if any. @type old_key_bits: positive int or None @param old_key_bits: The previously used number of bits, if any. @rtype: positive int @return: The number of bits to use. """ if key_bits is None: if old_key_type is not None and old_key_type == key_type: key_bits = old_key_bits else: key_bits = SSH_KEY_VALID_BITS[key_type].default if not SSH_KEY_VALID_BITS[key_type].validation_fn(key_bits): raise errors.OpPrereqError("Invalid key type and bit size combination:" " %s with %s bits" % (key_type, key_bits), errors.ECODE_INVAL) return key_bits ganeti-3.1.0~rc2/lib/storage/000075500000000000000000000000001476477700300160165ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/storage/__init__.py000064400000000000000000000025531476477700300201340ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Block device abstraction """ ganeti-3.1.0~rc2/lib/storage/base.py000064400000000000000000000406351476477700300173120ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Block device abstraction - base class and utility functions""" import logging from ganeti import objects from ganeti import constants from ganeti import utils from ganeti import errors class BlockDev(object): """Block device abstract class. A block device can be in the following states: - not existing on the system, and by `Create()` it goes into: - existing but not setup/not active, and by `Assemble()` goes into: - active read-write and by `Open()` it goes into - online (=used, or ready for use) A device can also be online but read-only, however we are not using the readonly state (LV has it, if needed in the future) and we are usually looking at this like at a stack, so it's easier to conceptualise the transition from not-existing to online and back like a linear one. The many different states of the device are due to the fact that we need to cover many device types: - logical volumes are created, lvchange -a y $lv, and used - drbd devices are attached to a local disk/remote peer and made primary A block device is identified by three items: - the /dev path of the device (dynamic) - a unique ID of the device (static) - it's major/minor pair (dynamic) Not all devices implement both the first two as distinct items. LVM logical volumes have their unique ID (the pair volume group, logical volume name) in a 1-to-1 relation to the dev path. For DRBD devices, the /dev path is again dynamic and the unique id is the pair (host1, dev1), (host2, dev2). You can get to a device in two ways: - creating the (real) device, which returns you an attached instance (lvcreate) - attaching of a python instance to an existing (real) device The second point, the attachment to a device, is different depending on whether the device is assembled or not. At init() time, we search for a device with the same unique_id as us. If found, good. It also means that the device is already assembled. If not, after assembly we'll have our correct major/minor. """ # pylint: disable=W0613 def __init__(self, unique_id, children, size, params, dyn_params, **kwargs): self._children = children self.dev_path = None self.unique_id = unique_id self.major = None self.minor = None self.attached = False self.size = size self.params = params self.dyn_params = dyn_params def __eq__(self, other): if not isinstance(self, type(other)): return False return (self._children == other._children and # pylint: disable=W0212 self.dev_path == other.dev_path and self.unique_id == other.unique_id and self.major == other.major and self.minor == other.minor and self.attached == other.attached and self.size == other.size and self.params == other.params and self.dyn_params == other.dyn_params) def Assemble(self): """Assemble the device from its components. Implementations of this method by child classes must ensure that: - after the device has been assembled, it knows its major/minor numbers; this allows other devices (usually parents) to probe correctly for their children - calling this method on an existing, in-use device is safe - if the device is already configured (and in an OK state), this method is idempotent """ pass def Attach(self, **kwargs): """Find a device which matches our config and attach to it. """ raise NotImplementedError def Close(self): """Notifies that the device will no longer be used for I/O. """ raise NotImplementedError @classmethod def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, **kwargs): """Create the device. If the device cannot be created, it will return None instead. Error messages go to the logging system. Note that for some devices, the unique_id is used, and for other, the children. The idea is that these two, taken together, are enough for both creation and assembly (later). @type unique_id: 2-element tuple or list @param unique_id: unique identifier; the details depend on the actual device type @type children: list of L{BlockDev} @param children: for hierarchical devices, the child devices @type size: float @param size: size in MiB @type spindles: int @param spindles: number of physical disk to dedicate to the device @type params: dict @param params: device-specific options/parameters @type excl_stor: bool @param excl_stor: whether exclusive_storage is active @type dyn_params: dict @param dyn_params: dynamic parameters of the disk only valid for this node. As set by L{objects.Disk.UpdateDynamicDiskParams}. @rtype: L{BlockDev} @return: the created device, or C{None} in case of an error """ raise NotImplementedError def Remove(self): """Remove this device. This makes sense only for some of the device types: LV and file storage. Also note that if the device can't attach, the removal can't be completed. """ raise NotImplementedError def Rename(self, new_id): """Rename this device. This may or may not make sense for a given device type. """ raise NotImplementedError def Open(self, force=False, exclusive=True): """Make the device ready for use. This makes the device ready for I/O. The force parameter signifies that if the device has any kind of --force thing, it should be used, we know what we are doing. The exclusive parameter denotes whether the device will be opened for exclusive access (True) or for concurrent shared access by multiple nodes (False) (e.g. during migration). @type force: boolean """ raise NotImplementedError def Shutdown(self): """Shut down the device, freeing its children. This undoes the `Assemble()` work, except for the child assembling; as such, the children on the device are still assembled after this call. """ raise NotImplementedError def Import(self): """Builds the shell command for importing data to device. This method returns the command that will be used by the caller to import data to the target device during the disk template conversion operation. Block devices that provide a more efficient way to transfer their data can override this method to use their specific utility. @rtype: list of strings @return: List containing the import command for device """ if not self.minor and not self.Attach(): ThrowError("Can't attach to target device during Import()") # we use the 'notrunc' argument to not attempt to truncate on the # given device return [constants.DD_CMD, "of=%s" % self.dev_path, "bs=%s" % constants.DD_BLOCK_SIZE, "oflag=direct", "conv=notrunc"] def Export(self): """Builds the shell command for exporting data from device. This method returns the command that will be used by the caller to export data from the source device during the disk template conversion operation. Block devices that provide a more efficient way to transfer their data can override this method to use their specific utility. @rtype: list of strings @return: List containing the export command for device """ if not self.minor and not self.Attach(): ThrowError("Can't attach to source device during Import()") return [constants.DD_CMD, "if=%s" % self.dev_path, "bs=%s" % constants.DD_BLOCK_SIZE, "count=%s" % self.size, "iflag=direct"] def Snapshot(self, snap_name, snap_size): """Creates a snapshot of the block device. Currently this is used only during LUInstanceExport. @type snap_name: string @param snap_name: The name of the snapshot. @type snap_size: int @param snap_size: The size of the snapshot. @rtype: tuple @return: The logical id of the newly created disk. """ ThrowError("Snapshot is not supported for disk %s of type %s.", self.unique_id, self.__class__.__name__) def SetSyncParams(self, params): """Adjust the synchronization parameters of the mirror. In case this is not a mirroring device, this is no-op. @param params: dictionary of LD level disk parameters related to the synchronization. @rtype: list @return: a list of error messages, emitted both by the current node and by children. An empty list means no errors. """ result = [] if self._children: for child in self._children: result.extend(child.SetSyncParams(params)) return result def PauseResumeSync(self, pause): """Pause/Resume the sync of the mirror. In case this is not a mirroring device, this is no-op. @type pause: boolean @param pause: Whether to pause or resume """ result = True if self._children: for child in self._children: result = result and child.PauseResumeSync(pause) return result def GetSyncStatus(self): """Returns the sync status of the device. If this device is a mirroring device, this function returns the status of the mirror. If sync_percent is None, it means the device is not syncing. If estimated_time is None, it means we can't estimate the time needed, otherwise it's the time left in seconds. If is_degraded is True, it means the device is missing redundancy. This is usually a sign that something went wrong in the device setup, if sync_percent is None. The ldisk parameter represents the degradation of the local data. This is only valid for some devices, the rest will always return False (not degraded). @rtype: objects.BlockDevStatus """ return objects.BlockDevStatus(dev_path=self.dev_path, major=self.major, minor=self.minor, sync_percent=None, estimated_time=None, is_degraded=False, ldisk_status=constants.LDS_OKAY) def CombinedSyncStatus(self): """Calculate the mirror status recursively for our children. The return value is the same as for `GetSyncStatus()` except the minimum percent and maximum time are calculated across our children. @rtype: objects.BlockDevStatus """ status = self.GetSyncStatus() min_percent = status.sync_percent max_time = status.estimated_time is_degraded = status.is_degraded ldisk_status = status.ldisk_status if self._children: for child in self._children: child_status = child.GetSyncStatus() if min_percent is None: min_percent = child_status.sync_percent elif child_status.sync_percent is not None: min_percent = min(min_percent, child_status.sync_percent) if max_time is None: max_time = child_status.estimated_time elif child_status.estimated_time is not None: max_time = max(max_time, child_status.estimated_time) is_degraded = is_degraded or child_status.is_degraded if ldisk_status is None: ldisk_status = child_status.ldisk_status elif child_status.ldisk_status is not None: ldisk_status = max(ldisk_status, child_status.ldisk_status) return objects.BlockDevStatus(dev_path=self.dev_path, major=self.major, minor=self.minor, sync_percent=min_percent, estimated_time=max_time, is_degraded=is_degraded, ldisk_status=ldisk_status) def SetInfo(self, text): """Update metadata with info text. Only supported for some device types. """ for child in self._children: child.SetInfo(text) def Grow(self, amount, dryrun, backingstore, excl_stor): """Grow the block device. @type amount: integer @param amount: the amount (in mebibytes) to grow with @type dryrun: boolean @param dryrun: whether to execute the operation in simulation mode only, without actually increasing the size @param backingstore: whether to execute the operation on backing storage only, or on "logical" storage only; e.g. DRBD is logical storage, whereas LVM, file, RBD are backing storage @type excl_stor: boolean @param excl_stor: Whether exclusive_storage is active """ raise NotImplementedError def GetActualSize(self): """Return the actual disk size. @note: the device needs to be active when this is called """ assert self.attached, "BlockDevice not attached in GetActualSize()" result = utils.RunCmd(["blockdev", "--getsize64", self.dev_path]) if result.failed: ThrowError("blockdev failed (%s): %s", result.fail_reason, result.output) try: sz = int(result.output.strip()) except (ValueError, TypeError) as err: ThrowError("Failed to parse blockdev output: %s", str(err)) return sz def GetActualSpindles(self): """Return the actual number of spindles used. This is not supported by all devices; if not supported, C{None} is returned. @note: the device needs to be active when this is called """ assert self.attached, "BlockDevice not attached in GetActualSpindles()" return None def GetActualDimensions(self): """Return the actual disk size and number of spindles used. @rtype: tuple @return: (size, spindles); spindles is C{None} when they are not supported @note: the device needs to be active when this is called """ return (self.GetActualSize(), self.GetActualSpindles()) def GetUserspaceAccessUri(self, hypervisor): """Return URIs hypervisors can use to access disks in userspace mode. @rtype: string @return: userspace device URI @raise errors.BlockDeviceError: if userspace access is not supported """ ThrowError("Userspace access with %s block device and %s hypervisor is not " "supported." % (self.__class__.__name__, hypervisor)) def __repr__(self): return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" % (self.__class__, self.unique_id, self._children, self.major, self.minor, self.dev_path)) def ThrowError(msg, *args): """Log an error to the node daemon and the raise an exception. @type msg: string @param msg: the text of the exception @raise errors.BlockDeviceError """ if args: msg = msg % args logging.error(msg) raise errors.BlockDeviceError(msg) def IgnoreError(fn, *args, **kwargs): """Executes the given function, ignoring BlockDeviceErrors. This is used in order to simplify the execution of cleanup or rollback functions. @rtype: boolean @return: True when fn didn't raise an exception, False otherwise """ try: fn(*args, **kwargs) return True except errors.BlockDeviceError as err: logging.warning("Caught BlockDeviceError but ignoring: %s", str(err)) return False ganeti-3.1.0~rc2/lib/storage/bdev.py000064400000000000000000001333701476477700300173170ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Block device abstraction. """ import re import stat import os import logging import math from ganeti import utils from ganeti import errors from ganeti import constants from ganeti import objects from ganeti import compat from ganeti import serializer from ganeti.storage import base from ganeti.storage import drbd from ganeti.storage.filestorage import FileStorage from ganeti.storage.gluster import GlusterStorage from ganeti.storage.extstorage import ExtStorageDevice class RbdShowmappedJsonError(Exception): """`rbd showmmapped' JSON formatting error Exception class. """ pass def _CheckResult(result): """Throws an error if the given result is a failed one. @param result: result from RunCmd """ if result.failed: base.ThrowError("Command: %s error: %s - %s", result.cmd, result.fail_reason, result.output) class LogicalVolume(base.BlockDev): """Logical Volume block device. """ _VALID_NAME_RE = re.compile("^[a-zA-Z0-9+_.-]*$") _PARSE_PV_DEV_RE = re.compile(r"^([^ ()]+)\([0-9]+\)$") _INVALID_NAMES = compat.UniqueFrozenset([".", "..", "snapshot", "pvmove"]) _INVALID_SUBSTRINGS = compat.UniqueFrozenset(["_mlog", "_mimage"]) def __init__(self, unique_id, children, size, params, dyn_params, **kwargs): """Attaches to a LV device. The unique_id is a tuple (vg_name, lv_name) """ super(LogicalVolume, self).__init__(unique_id, children, size, params, dyn_params, **kwargs) if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise ValueError("Invalid configuration data %s" % str(unique_id)) self._vg_name, self._lv_name = unique_id self._ValidateName(self._vg_name) self._ValidateName(self._lv_name) self.dev_path = utils.PathJoin("/dev", self._vg_name, self._lv_name) self._degraded = True self.major = self.minor = self.pe_size = self.stripe_count = None self.pv_names = None lvs_cache = kwargs.get("lvs_cache") if lvs_cache: lv_info = lvs_cache.get(self.dev_path) self.Attach(lv_info=lv_info) else: self.Attach() @staticmethod def _GetStdPvSize(pvs_info): """Return the the standard PV size (used with exclusive storage). @param pvs_info: list of objects.LvmPvInfo, cannot be empty @rtype: float @return: size in MiB """ assert len(pvs_info) > 0 smallest = min([pv.size for pv in pvs_info]) return smallest // (1 + constants.PART_MARGIN + constants.PART_RESERVED) @staticmethod def _ComputeNumPvs(size, pvs_info): """Compute the number of PVs needed for an LV (with exclusive storage). @type size: float @param size: LV size in MiB @param pvs_info: list of objects.LvmPvInfo, cannot be empty @rtype: integer @return: number of PVs needed """ assert len(pvs_info) > 0 pv_size = float(LogicalVolume._GetStdPvSize(pvs_info)) return int(math.ceil(float(size) / pv_size)) @staticmethod def _GetEmptyPvNames(pvs_info, max_pvs=None): """Return a list of empty PVs, by name. """ empty_pvs = [pv for pv in pvs_info if objects.LvmPvInfo.IsEmpty(pv)] if max_pvs is not None: empty_pvs = empty_pvs[:max_pvs] return [pv.name for pv in empty_pvs] @classmethod def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, **kwargs): """Create a new logical volume. """ if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise errors.ProgrammerError("Invalid configuration data %s" % str(unique_id)) vg_name, lv_name = unique_id cls._ValidateName(vg_name) cls._ValidateName(lv_name) pvs_info = cls.GetPVInfo([vg_name]) if not pvs_info: if excl_stor: msg = "No (empty) PVs found" else: msg = "Can't compute PV info for vg %s" % vg_name base.ThrowError(msg) pvs_info.sort(key=(lambda pv: pv.free), reverse=True) pvlist = [pv.name for pv in pvs_info] if compat.any(":" in v for v in pvlist): base.ThrowError("Some of your PVs have the invalid character ':' in their" " name, this is not supported - please filter them out" " in lvm.conf using either 'filter' or 'preferred_names'") current_pvs = len(pvlist) desired_stripes = params[constants.LDP_STRIPES] stripes = min(current_pvs, desired_stripes) if excl_stor: if spindles is None: base.ThrowError("Unspecified number of spindles: this is required" "when exclusive storage is enabled, try running" " gnt-cluster repair-disk-sizes") (err_msgs, _) = utils.LvmExclusiveCheckNodePvs(pvs_info) if err_msgs: for m in err_msgs: logging.warning(m) req_pvs = cls._ComputeNumPvs(size, pvs_info) if spindles < req_pvs: base.ThrowError("Requested number of spindles (%s) is not enough for" " a disk of %d MB (at least %d spindles needed)", spindles, size, req_pvs) else: req_pvs = spindles pvlist = cls._GetEmptyPvNames(pvs_info, req_pvs) current_pvs = len(pvlist) if current_pvs < req_pvs: base.ThrowError("Not enough empty PVs (spindles) to create a disk of %d" " MB: %d available, %d needed", size, current_pvs, req_pvs) assert current_pvs == len(pvlist) # We must update stripes to be sure to use all the desired spindles stripes = current_pvs if stripes > desired_stripes: # Don't warn when lowering stripes, as it's no surprise logging.warning("Using %s stripes instead of %s, to be able to use" " %s spindles", stripes, desired_stripes, current_pvs) else: if stripes < desired_stripes: logging.warning("Could not use %d stripes for VG %s, as only %d PVs are" " available.", desired_stripes, vg_name, current_pvs) free_size = sum([pv.free for pv in pvs_info]) # The size constraint should have been checked from the master before # calling the create function. if free_size < size: base.ThrowError("Not enough free space: required %s," " available %s", size, free_size) # If the free space is not well distributed, we won't be able to # create an optimally-striped volume; in that case, we want to try # with N, N-1, ..., 2, and finally 1 (non-stripped) number of # stripes # When run non-interactively, newer LVM versions will fail (unless # `--yes` is specified) when an existing filesystem signature is # encountered while creating a new LV. Using `-Wn` disables this check. cmd = ["lvcreate", "-Wn", "-L%dm" % size, "-n%s" % lv_name] for stripes_arg in range(stripes, 0, -1): result = utils.RunCmd(cmd + ["-i%d" % stripes_arg] + [vg_name] + pvlist) if not result.failed: break if result.failed: base.ThrowError("LV create failed (%s): %s", result.fail_reason, result.output) return LogicalVolume(unique_id, children, size, params, dyn_params, **kwargs) @staticmethod def _GetVolumeInfo(lvm_cmd, fields): """Returns LVM Volume infos using lvm_cmd @param lvm_cmd: Should be one of "pvs", "vgs" or "lvs" @param fields: Fields to return @return: A list of dicts each with the parsed fields """ if not fields: raise errors.ProgrammerError("No fields specified") sep = "|" cmd = [lvm_cmd, "--noheadings", "--nosuffix", "--units=m", "--unbuffered", "--separator=%s" % sep, "-o%s" % ",".join(fields)] result = utils.RunCmd(cmd) if result.failed: raise errors.CommandError("Can't get the volume information: %s - %s" % (result.fail_reason, result.output)) data = [] for line in result.stdout.splitlines(): splitted_fields = line.strip().split(sep) if len(fields) != len(splitted_fields): raise errors.CommandError("Can't parse %s output: line '%s'" % (lvm_cmd, line)) data.append(splitted_fields) return data @classmethod def GetPVInfo(cls, vg_names, filter_allocatable=True, include_lvs=False): """Get the free space info for PVs in a volume group. @param vg_names: list of volume group names, if empty all will be returned @param filter_allocatable: whether to skip over unallocatable PVs @param include_lvs: whether to include a list of LVs hosted on each PV @rtype: list @return: list of objects.LvmPvInfo objects """ # We request "lv_name" field only if we care about LVs, so we don't get # a long list of entries with many duplicates unless we really have to. # The duplicate "pv_name" field will be ignored. if include_lvs: lvfield = "lv_name" else: lvfield = "pv_name" try: info = cls._GetVolumeInfo("pvs", ["pv_name", "vg_name", "pv_free", "pv_attr", "pv_size", lvfield]) except errors.GenericError as err: logging.error("Can't get PV information: %s", err) return None # When asked for LVs, "pvs" may return multiple entries for the same PV-LV # pair. We sort entries by PV name and then LV name, so it's easy to weed # out duplicates. if include_lvs: info.sort(key=(lambda i: (i[0], i[5]))) data = [] lastpvi = None for (pv_name, vg_name, pv_free, pv_attr, pv_size, lv_name) in info: # (possibly) skip over pvs which are not allocatable if filter_allocatable and pv_attr[0] != "a": continue # (possibly) skip over pvs which are not in the right volume group(s) if vg_names and vg_name not in vg_names: continue # Beware of duplicates (check before inserting) if lastpvi and lastpvi.name == pv_name: if include_lvs and lv_name: if not lastpvi.lv_list or lastpvi.lv_list[-1] != lv_name: lastpvi.lv_list.append(lv_name) else: if include_lvs and lv_name: lvl = [lv_name] else: lvl = [] lastpvi = objects.LvmPvInfo(name=pv_name, vg_name=vg_name, size=float(pv_size), free=float(pv_free), attributes=pv_attr, lv_list=lvl) data.append(lastpvi) return data @classmethod def _GetRawFreePvInfo(cls, vg_name): """Return info (size/free) about PVs. @type vg_name: string @param vg_name: VG name @rtype: tuple @return: (standard_pv_size_in_MiB, number_of_free_pvs, total_number_of_pvs) """ pvs_info = cls.GetPVInfo([vg_name]) if not pvs_info: pv_size = 0.0 free_pvs = 0 num_pvs = 0 else: pv_size = cls._GetStdPvSize(pvs_info) free_pvs = len(cls._GetEmptyPvNames(pvs_info)) num_pvs = len(pvs_info) return (pv_size, free_pvs, num_pvs) @classmethod def _GetExclusiveStorageVgFree(cls, vg_name): """Return the free disk space in the given VG, in exclusive storage mode. @type vg_name: string @param vg_name: VG name @rtype: float @return: free space in MiB """ (pv_size, free_pvs, _) = cls._GetRawFreePvInfo(vg_name) return pv_size * free_pvs @classmethod def GetVgSpindlesInfo(cls, vg_name): """Get the free space info for specific VGs. @param vg_name: volume group name @rtype: tuple @return: (free_spindles, total_spindles) """ (_, free_pvs, num_pvs) = cls._GetRawFreePvInfo(vg_name) return (free_pvs, num_pvs) @classmethod def GetVGInfo(cls, vg_names, excl_stor, filter_readonly=True): """Get the free space info for specific VGs. @param vg_names: list of volume group names, if empty all will be returned @param excl_stor: whether exclusive_storage is enabled @param filter_readonly: whether to skip over readonly VGs @rtype: list @return: list of tuples (free_space, total_size, name) with free_space in MiB """ try: info = cls._GetVolumeInfo("vgs", ["vg_name", "vg_free", "vg_attr", "vg_size"]) except errors.GenericError as err: logging.error("Can't get VG information: %s", err) return None data = [] for vg_name, vg_free, vg_attr, vg_size in info: # (possibly) skip over vgs which are not writable if filter_readonly and vg_attr[0] == "r": continue # (possibly) skip over vgs which are not in the right volume group(s) if vg_names and vg_name not in vg_names: continue # Exclusive storage needs a different concept of free space if excl_stor: es_free = cls._GetExclusiveStorageVgFree(vg_name) assert es_free <= vg_free vg_free = es_free data.append((float(vg_free), float(vg_size), vg_name)) return data @classmethod def _ValidateName(cls, name): """Validates that a given name is valid as VG or LV name. The list of valid characters and restricted names is taken out of the lvm(8) manpage, with the simplification that we enforce both VG and LV restrictions on the names. """ if (not cls._VALID_NAME_RE.match(name) or name in cls._INVALID_NAMES or compat.any(substring in name for substring in cls._INVALID_SUBSTRINGS)): base.ThrowError("Invalid LVM name '%s'", name) def Remove(self): """Remove this logical volume. """ if not self.minor and not self.Attach(): # the LV does not exist return result = utils.RunCmd(["lvremove", "-f", "%s/%s" % (self._vg_name, self._lv_name)]) if result.failed: base.ThrowError("Can't lvremove: %s - %s", result.fail_reason, result.output) def Rename(self, new_id): """Rename this logical volume. """ if not isinstance(new_id, (tuple, list)) or len(new_id) != 2: raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id) new_vg, new_name = new_id if new_vg != self._vg_name: raise errors.ProgrammerError("Can't move a logical volume across" " volume groups (from %s to to %s)" % (self._vg_name, new_vg)) result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name]) if result.failed: base.ThrowError("Failed to rename the logical volume: %s", result.output) self._lv_name = new_name self.dev_path = utils.PathJoin("/dev", self._vg_name, self._lv_name) @staticmethod def _ParseLvInfoLine(line, sep): """Parse one line of the lvs output used in L{GetLvGlobalInfo}. """ elems = line.strip().split(sep) # The previous iteration of code here assumed that LVM might put another # separator to the right of the output. The PV info might be empty for # thin volumes, so stripping off the separators might cut off the last # empty element - do this instead. if len(elems) == 9 and elems[-1] == "": elems.pop() if len(elems) != 8: base.ThrowError("Can't parse LVS output, len(%s) != 8", str(elems)) (vg_name, lv_name, status, major, minor, pe_size, stripes, pvs) = elems path = os.path.join(os.environ.get('DM_DEV_DIR', '/dev'), vg_name, lv_name) if len(status) < 6: base.ThrowError("lvs lv_attr is not at least 6 characters (%s)", status) try: major = int(major) minor = int(minor) except (TypeError, ValueError) as err: base.ThrowError("lvs major/minor cannot be parsed: %s", str(err)) try: pe_size = int(float(pe_size)) except (TypeError, ValueError) as err: base.ThrowError("Can't parse vg extent size: %s", err) try: stripes = int(stripes) except (TypeError, ValueError) as err: base.ThrowError("Can't parse the number of stripes: %s", err) pv_names = [] if pvs != "": for pv in pvs.split(","): m = re.match(LogicalVolume._PARSE_PV_DEV_RE, pv) if not m: base.ThrowError("Can't parse this device list: %s", pvs) pv_names.append(m.group(1)) return (path, (status, major, minor, pe_size, stripes, pv_names)) @staticmethod def GetLvGlobalInfo(_run_cmd=utils.RunCmd): """Obtain the current state of the existing LV disks. @return: a dict containing the state of each disk with the disk path as key """ sep = "|" result = _run_cmd(["lvs", "--noheadings", "--separator=%s" % sep, "--units=k", "--nosuffix", "-ovg_name,lv_name,lv_attr,lv_kernel_major," "lv_kernel_minor,vg_extent_size,stripes,devices"]) if result.failed: logging.warning("lvs command failed, the LV cache will be empty!") logging.info("lvs failure: %r", result.stderr) return {} out = result.stdout.splitlines() if not out: logging.warning("lvs command returned an empty output, the LV cache will" "be empty!") return {} return dict([LogicalVolume._ParseLvInfoLine(line, sep) for line in out]) def Attach(self, lv_info=None, **kwargs): """Attach to an existing LV. This method will try to see if an existing and active LV exists which matches our name. If so, its major/minor will be recorded. """ self.attached = False if not lv_info: lv_info = LogicalVolume.GetLvGlobalInfo().get(self.dev_path) if not lv_info: return False (status, major, minor, pe_size, stripes, pv_names) = lv_info self.major = major self.minor = minor self.pe_size = pe_size self.stripe_count = stripes self._degraded = status[0] == "v" # virtual volume, i.e. doesn't backing # storage self.pv_names = pv_names self.attached = True return True def Assemble(self): """Assemble the device. We always run `lvchange -ay` on the LV to ensure it's active before use, as there were cases when xenvg was not active after boot (also possibly after disk issues). """ result = utils.RunCmd(["lvchange", "-ay", self.dev_path]) if result.failed: base.ThrowError("Can't activate lv %s: %s", self.dev_path, result.output) def Shutdown(self): """Shutdown the device. This is a no-op for the LV device type, as we don't deactivate the volumes on shutdown. """ pass def GetSyncStatus(self): """Returns the sync status of the device. If this device is a mirroring device, this function returns the status of the mirror. For logical volumes, sync_percent and estimated_time are always None (no recovery in progress, as we don't handle the mirrored LV case). The is_degraded parameter is the inverse of the ldisk parameter. For the ldisk parameter, we check if the logical volume has the 'virtual' type, which means it's not backed by existing storage anymore (read from it return I/O error). This happens after a physical disk failure and subsequent 'vgreduce --removemissing' on the volume group. The status was already read in Attach, so we just return it. @rtype: objects.BlockDevStatus """ if self._degraded: ldisk_status = constants.LDS_FAULTY else: ldisk_status = constants.LDS_OKAY return objects.BlockDevStatus(dev_path=self.dev_path, major=self.major, minor=self.minor, sync_percent=None, estimated_time=None, is_degraded=self._degraded, ldisk_status=ldisk_status) def Open(self, force=False, exclusive=True): """Make the device ready for I/O. This is a no-op for the LV device type. """ pass def Close(self): """Notifies that the device will no longer be used for I/O. This is a no-op for the LV device type. """ pass def Snapshot(self, snap_name=None, snap_size=None): """Create a snapshot copy of an lvm block device. @returns: tuple (vg, lv) """ if not snap_name: snap_name = self._lv_name + ".snap" if not snap_size: # FIXME: choose a saner value for the snapshot size # let's stay on the safe side and ask for the full size, for now snap_size = self.size # remove existing snapshot if found snap = LogicalVolume((self._vg_name, snap_name), None, snap_size, self.params, self.dyn_params) base.IgnoreError(snap.Remove) vg_info = self.GetVGInfo([self._vg_name], False) if not vg_info: base.ThrowError("Can't compute VG info for vg %s", self._vg_name) free_size, _, _ = vg_info[0] if free_size < snap_size: base.ThrowError("Not enough free space: required %s," " available %s", snap_size, free_size) _CheckResult(utils.RunCmd(["lvcreate", "-L%dm" % snap_size, "-s", "-n%s" % snap_name, self.dev_path])) return (self._vg_name, snap_name) def _RemoveOldInfo(self): """Try to remove old tags from the lv. """ result = utils.RunCmd(["lvs", "-o", "tags", "--noheadings", "--nosuffix", self.dev_path]) _CheckResult(result) raw_tags = result.stdout.strip() if raw_tags: for tag in raw_tags.split(","): _CheckResult(utils.RunCmd(["lvchange", "--deltag", tag.strip(), self.dev_path])) def SetInfo(self, text): """Update metadata with info text. """ base.BlockDev.SetInfo(self, text) self._RemoveOldInfo() # Replace invalid characters text = re.sub("^[^A-Za-z0-9_+.]", "_", text) text = re.sub("[^-A-Za-z0-9_+.]", "_", text) # Only up to 128 characters are allowed text = text[:128] _CheckResult(utils.RunCmd(["lvchange", "--addtag", text, self.dev_path])) def _GetGrowthAvaliabilityExclStor(self): """Return how much the disk can grow with exclusive storage. @rtype: float @return: available space in Mib """ pvs_info = self.GetPVInfo([self._vg_name]) if not pvs_info: base.ThrowError("Cannot get information about PVs for %s", self.dev_path) std_pv_size = self._GetStdPvSize(pvs_info) free_space = sum(pvi.free - (pvi.size - std_pv_size) for pvi in pvs_info if pvi.name in self.pv_names) return free_space def Grow(self, amount, dryrun, backingstore, excl_stor): """Grow the logical volume. """ if not backingstore: return if self.pe_size is None or self.stripe_count is None: if not self.Attach(): base.ThrowError("Can't attach to LV during Grow()") full_stripe_size = self.pe_size * self.stripe_count # pe_size is in KB amount *= 1024 rest = amount % full_stripe_size if rest != 0: amount += full_stripe_size - rest cmd = ["lvextend", "-L", "+%dk" % amount] if dryrun: cmd.append("--test") if excl_stor: free_space = self._GetGrowthAvaliabilityExclStor() # amount is in KiB, free_space in MiB if amount > free_space * 1024: base.ThrowError("Not enough free space to grow %s: %d MiB required," " %d available", self.dev_path, amount // 1024, free_space) # Disk growth doesn't grow the number of spindles, so we must stay within # our assigned volumes pvlist = list(self.pv_names) else: pvlist = [] # we try multiple algorithms since the 'best' ones might not have # space available in the right place, but later ones might (since # they have less constraints); also note that only recent LVM # supports 'cling' for alloc_policy in "contiguous", "cling", "normal": result = utils.RunCmd(cmd + ["--alloc", alloc_policy, self.dev_path] + pvlist) if not result.failed: return base.ThrowError("Can't grow LV %s: %s", self.dev_path, result.output) def GetActualSpindles(self): """Return the number of spindles used. """ assert self.attached, "BlockDevice not attached in GetActualSpindles()" return len(self.pv_names) class PersistentBlockDevice(base.BlockDev): """A block device with persistent node May be either directly attached, or exposed through DM (e.g. dm-multipath). udev helpers are probably required to give persistent, human-friendly names. For the time being, pathnames are required to lie under /dev. """ def __init__(self, unique_id, children, size, params, dyn_params, **kwargs): """Attaches to a static block device. The unique_id is a path under /dev. """ super(PersistentBlockDevice, self).__init__(unique_id, children, size, params, dyn_params, **kwargs) if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise ValueError("Invalid configuration data %s" % str(unique_id)) self.dev_path = unique_id[1] if not os.path.realpath(self.dev_path).startswith("/dev/"): raise ValueError("Full path '%s' lies outside /dev" % os.path.realpath(self.dev_path)) # TODO: this is just a safety guard checking that we only deal with devices # we know how to handle. In the future this will be integrated with # external storage backends and possible values will probably be collected # from the cluster configuration. if unique_id[0] != constants.BLOCKDEV_DRIVER_MANUAL: raise ValueError("Got persistent block device of invalid type: %s" % unique_id[0]) self.major = self.minor = None self.Attach() @classmethod def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, **kwargs): """Create a new device This is a noop, we only return a PersistentBlockDevice instance """ if excl_stor: raise errors.ProgrammerError("Persistent block device requested with" " exclusive_storage") return PersistentBlockDevice(unique_id, children, 0, params, dyn_params, **kwargs) def Remove(self): """Remove a device This is a noop """ pass def Rename(self, new_id): """Rename this device. """ base.ThrowError("Rename is not supported for PersistentBlockDev storage") def Attach(self): """Attach to an existing block device. """ self.attached = False try: st = os.stat(self.dev_path) except OSError as err: logging.error("Error stat()'ing %s: %s", self.dev_path, str(err)) return False if not stat.S_ISBLK(st.st_mode): logging.error("%s is not a block device", self.dev_path) return False self.major = os.major(st.st_rdev) self.minor = utils.osminor(st.st_rdev) self.attached = True return True def Assemble(self): """Assemble the device. """ pass def Shutdown(self): """Shutdown the device. """ pass def Open(self, force=False, exclusive=True): """Make the device ready for I/O. """ pass def Close(self): """Notifies that the device will no longer be used for I/O. """ pass def Grow(self, amount, dryrun, backingstore, excl_stor): """Grow the logical volume. """ base.ThrowError("Grow is not supported for PersistentBlockDev storage") def Import(self): """Builds the shell command for importing data to device. @see: L{BlockDev.Import} for details """ base.ThrowError("Importing data is not supported for the" " PersistentBlockDevice template") class RADOSBlockDevice(base.BlockDev): """A RADOS Block Device (rbd). This class implements the RADOS Block Device for the backend. You need the rbd kernel driver, the RADOS Tools and a working RADOS cluster for this to be functional. """ def __init__(self, unique_id, children, size, params, dyn_params, **kwargs): """Attaches to an rbd device. """ super(RADOSBlockDevice, self).__init__(unique_id, children, size, params, dyn_params, **kwargs) if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise ValueError("Invalid configuration data %s" % str(unique_id)) self.driver, self.rbd_name = unique_id self.rbd_pool = params[constants.LDP_POOL] self.major = self.minor = None self.Attach() @classmethod def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, **kwargs): """Create a new rbd device. Provision a new rbd volume inside a RADOS pool. """ if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise errors.ProgrammerError("Invalid configuration data %s" % str(unique_id)) if excl_stor: raise errors.ProgrammerError("RBD device requested with" " exclusive_storage") rbd_pool = params[constants.LDP_POOL] rbd_name = unique_id[1] # Provision a new rbd volume (Image) inside the RADOS cluster. cmd = [constants.RBD_CMD, "create", "-p", rbd_pool, rbd_name, "--size", "%s" % size] result = utils.RunCmd(cmd) if result.failed: base.ThrowError("rbd creation failed (%s): %s", result.fail_reason, result.output) return RADOSBlockDevice(unique_id, children, size, params, dyn_params, **kwargs) def Remove(self): """Remove the rbd device. """ rbd_pool = self.params[constants.LDP_POOL] rbd_name = self.unique_id[1] if not self.minor and not self.Attach(): # The rbd device doesn't exist. return # First shutdown the device (remove mappings). self.Shutdown() # Remove the actual Volume (Image) from the RADOS cluster. cmd = [constants.RBD_CMD, "rm", "-p", rbd_pool, rbd_name] result = utils.RunCmd(cmd) if result.failed: base.ThrowError("Can't remove Volume from cluster with rbd rm: %s - %s", result.fail_reason, result.output) def Rename(self, new_id): """Rename this device. """ pass def Attach(self): """Attach to an existing rbd device. This method maps the rbd volume that matches our name with an rbd device and then attaches to this device. """ self.attached = False # Map the rbd volume to a block device under /dev self.dev_path = self._MapVolumeToBlockdev(self.unique_id) try: st = os.stat(self.dev_path) except OSError as err: logging.error("Error stat()'ing %s: %s", self.dev_path, str(err)) return False if not stat.S_ISBLK(st.st_mode): logging.error("%s is not a block device", self.dev_path) return False self.major = os.major(st.st_rdev) self.minor = utils.osminor(st.st_rdev) self.attached = True return True def _MapVolumeToBlockdev(self, unique_id): """Maps existing rbd volumes to block devices. This method should be idempotent if the mapping already exists. @rtype: string @return: the block device path that corresponds to the volume """ pool = self.params[constants.LDP_POOL] name = unique_id[1] # Check if the mapping already exists. rbd_dev = self._VolumeToBlockdev(pool, name) if rbd_dev: # The mapping exists. Return it. return rbd_dev # The mapping doesn't exist. Create it. map_cmd = [constants.RBD_CMD, "map", "-p", pool, name] result = utils.RunCmd(map_cmd) if result.failed: base.ThrowError("rbd map failed (%s): %s", result.fail_reason, result.output) # Find the corresponding rbd device. rbd_dev = self._VolumeToBlockdev(pool, name) if not rbd_dev: base.ThrowError("rbd map succeeded, but could not find the rbd block" " device in output of showmapped, for volume: %s", name) # The device was successfully mapped. Return it. return rbd_dev @classmethod def _VolumeToBlockdev(cls, pool, volume_name): """Do the 'volume name'-to-'rbd block device' resolving. @type pool: string @param pool: RADOS pool to use @type volume_name: string @param volume_name: the name of the volume whose device we search for @rtype: string or None @return: block device path if the volume is mapped, else None """ try: # Newer versions of the rbd tool support json output formatting. Use it # if available. showmap_cmd = [ constants.RBD_CMD, "showmapped", "--format", "json" ] result = utils.RunCmd(showmap_cmd) if result.failed: logging.error("rbd JSON output formatting returned error (%s): %s," "falling back to plain output parsing", result.fail_reason, result.output) raise RbdShowmappedJsonError return cls._ParseRbdShowmappedJson(result.output, pool, volume_name) except RbdShowmappedJsonError: # For older versions of rbd, we have to parse the plain / text output # manually. showmap_cmd = [constants.RBD_CMD, "showmapped", "-p", pool] result = utils.RunCmd(showmap_cmd) if result.failed: base.ThrowError("rbd showmapped failed (%s): %s", result.fail_reason, result.output) return cls._ParseRbdShowmappedPlain(result.output, volume_name) @staticmethod def _ParseRbdShowmappedJson(output, volume_pool, volume_name): """Parse the json output of `rbd showmapped'. This method parses the json output of `rbd showmapped' and returns the rbd block device path (e.g. /dev/rbd0) that matches the given rbd volume. @type output: string @param output: the json output of `rbd showmapped' @type volume_pool: string @param volume_pool: name of the volume whose device we search for @type volume_name: string @param volume_name: name of the pool in which we search @rtype: string or None @return: block device path if the volume is mapped, else None """ try: devices = serializer.LoadJson(output) except ValueError as err: base.ThrowError("Unable to parse JSON data: %s" % err) # since ceph mimic the json output changed from dict to list if isinstance(devices, dict): devices = list(devices.values()) rbd_dev = None for d in devices: try: name = d["name"] except KeyError: base.ThrowError("'name' key missing from json object %s", devices) try: pool = d["pool"] except KeyError: base.ThrowError("'pool' key missing from json object %s", devices) if name == volume_name and pool == volume_pool: if rbd_dev is not None: base.ThrowError("rbd volume %s is mapped more than once", volume_name) rbd_dev = d["device"] return rbd_dev @staticmethod def _ParseRbdShowmappedPlain(output, volume_name): """Parse the (plain / text) output of `rbd showmapped'. This method parses the output of `rbd showmapped' and returns the rbd block device path (e.g. /dev/rbd0) that matches the given rbd volume. @type output: string @param output: the plain text output of `rbd showmapped' @type volume_name: string @param volume_name: the name of the volume whose device we search for @rtype: string or None @return: block device path if the volume is mapped, else None """ allfields = 5 volumefield = 2 devicefield = 4 lines = output.splitlines() # Try parsing the new output format (ceph >= 0.55). splitted_lines = [l.split() for l in lines] # Check for empty output. if not splitted_lines: return None # Check showmapped output, to determine number of fields. field_cnt = len(splitted_lines[0]) if field_cnt != allfields: # Parsing the new format failed. Fallback to parsing the old output # format (< 0.55). splitted_lines = [l.split("\t") for l in lines] if field_cnt != allfields: base.ThrowError("Cannot parse rbd showmapped output expected %s fields," " found %s", allfields, field_cnt) matched_lines = [l for l in splitted_lines if len(l) == allfields and l[volumefield] == volume_name] if len(matched_lines) > 1: base.ThrowError("rbd volume %s mapped more than once", volume_name) if matched_lines: # rbd block device found. Return it. rbd_dev = matched_lines[0][devicefield] return rbd_dev # The given volume is not mapped. return None def Assemble(self): """Assemble the device. """ pass def Shutdown(self): """Shutdown the device. """ if not self.minor and not self.Attach(): # The rbd device doesn't exist. return # Unmap the block device from the Volume. self._UnmapVolumeFromBlockdev(self.unique_id) self.minor = None self.dev_path = None def _UnmapVolumeFromBlockdev(self, unique_id): """Unmaps the rbd device from the Volume it is mapped. Unmaps the rbd device from the Volume it was previously mapped to. This method should be idempotent if the Volume isn't mapped. """ pool = self.params[constants.LDP_POOL] name = unique_id[1] # Check if the mapping already exists. rbd_dev = self._VolumeToBlockdev(pool, name) if rbd_dev: # The mapping exists. Unmap the rbd device. unmap_cmd = [constants.RBD_CMD, "unmap", "%s" % rbd_dev] result = utils.RunCmd(unmap_cmd) if result.failed: base.ThrowError("rbd unmap failed (%s): %s", result.fail_reason, result.output) def Open(self, force=False, exclusive=True): """Make the device ready for I/O. """ pass def Close(self): """Notifies that the device will no longer be used for I/O. """ pass def Grow(self, amount, dryrun, backingstore, excl_stor): """Grow the Volume. @type amount: integer @param amount: the amount (in mebibytes) to grow with @type dryrun: boolean @param dryrun: whether to execute the operation in simulation mode only, without actually increasing the size """ if not backingstore: return if not self.Attach(): base.ThrowError("Can't attach to rbd device during Grow()") if dryrun: # the rbd tool does not support dry runs of resize operations. # Since rbd volumes are thinly provisioned, we assume # there is always enough free space for the operation. return rbd_pool = self.params[constants.LDP_POOL] rbd_name = self.unique_id[1] new_size = self.size + amount # Resize the rbd volume (Image) inside the RADOS cluster. cmd = [constants.RBD_CMD, "resize", "-p", rbd_pool, rbd_name, "--size", "%s" % new_size] result = utils.RunCmd(cmd) if result.failed: base.ThrowError("rbd resize failed (%s): %s", result.fail_reason, result.output) def Import(self): """Builds the shell command for importing data to device. @see: L{BlockDev.Import} for details """ if not self.minor and not self.Attach(): # The rbd device doesn't exist. base.ThrowError("Can't attach to rbd device during Import()") rbd_pool = self.params[constants.LDP_POOL] rbd_name = self.unique_id[1] # Currently, the 'rbd import' command imports data only to non-existing # volumes. If the rbd volume exists the command will fail. # The disk conversion mechanism though, has already created the new rbd # volume at the time we perform the data copy, so we have to first remove # the volume before starting to import its data. The 'rbd import' will # re-create the rbd volume. We choose to remove manually the rbd device # instead of calling its 'Remove()' method to avoid affecting the 'self.' # parameters of the device. Also, this part of the removal code will go # away once 'rbd import' has support for importing into an existing volume. # TODO: update this method when the 'rbd import' command supports the # '--force' option, which will allow importing to an existing volume. # Unmap the block device from the Volume. self._UnmapVolumeFromBlockdev(self.unique_id) # Remove the actual Volume (Image) from the RADOS cluster. cmd = [constants.RBD_CMD, "rm", "-p", rbd_pool, rbd_name] result = utils.RunCmd(cmd) if result.failed: base.ThrowError("Can't remove Volume from cluster with rbd rm: %s - %s", result.fail_reason, result.output) # We use "-" for importing from stdin return [constants.RBD_CMD, "import", "-p", rbd_pool, "-", rbd_name] def Export(self): """Builds the shell command for exporting data from device. @see: L{BlockDev.Export} for details """ if not self.minor and not self.Attach(): # The rbd device doesn't exist. base.ThrowError("Can't attach to rbd device during Export()") rbd_pool = self.params[constants.LDP_POOL] rbd_name = self.unique_id[1] # We use "-" for exporting to stdout. return [constants.RBD_CMD, "export", "-p", rbd_pool, rbd_name, "-"] def GetUserspaceAccessUri(self, hypervisor): """Generate KVM userspace URIs to be used as `-drive file` settings. @see: L{BlockDev.GetUserspaceAccessUri} """ if hypervisor == constants.HT_KVM: return "rbd:" + self.rbd_pool + "/" + self.rbd_name else: base.ThrowError("Hypervisor %s doesn't support RBD userspace access" % hypervisor) def _VerifyDiskType(dev_type): if dev_type not in DEV_MAP: raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type) def _VerifyDiskParams(disk): """Verifies if all disk parameters are set. """ missing = set(constants.DISK_LD_DEFAULTS[disk.dev_type]) - set(disk.params) if missing: raise errors.ProgrammerError("Block device is missing disk parameters: %s" % missing) def FindDevice(disk, children, **kwargs): """Search for an existing, assembled device. This will succeed only if the device exists and is assembled, but it does not do any actions in order to activate the device. @type disk: L{objects.Disk} @param disk: the disk object to find @type children: list of L{bdev.BlockDev} @param children: the list of block devices that are children of the device represented by the disk parameter """ _VerifyDiskType(disk.dev_type) device = DEV_MAP[disk.dev_type](disk.logical_id, children, disk.size, disk.params, disk.dynamic_params, name=disk.name, uuid=disk.uuid, **kwargs) if not device.attached: return None return device def Assemble(disk, children): """Try to attach or assemble an existing device. This will attach to assemble the device, as needed, to bring it fully up. It must be safe to run on already-assembled devices. @type disk: L{objects.Disk} @param disk: the disk object to assemble @type children: list of L{bdev.BlockDev} @param children: the list of block devices that are children of the device represented by the disk parameter """ _VerifyDiskType(disk.dev_type) _VerifyDiskParams(disk) device = DEV_MAP[disk.dev_type](disk.logical_id, children, disk.size, disk.params, disk.dynamic_params, name=disk.name, uuid=disk.uuid) device.Assemble() return device def Create(disk, children, excl_stor): """Create a device. @type disk: L{objects.Disk} @param disk: the disk object to create @type children: list of L{bdev.BlockDev} @param children: the list of block devices that are children of the device represented by the disk parameter @type excl_stor: boolean @param excl_stor: Whether exclusive_storage is active @rtype: L{bdev.BlockDev} @return: the created device, or C{None} in case of an error """ _VerifyDiskType(disk.dev_type) _VerifyDiskParams(disk) device = DEV_MAP[disk.dev_type].Create(disk.logical_id, children, disk.size, disk.spindles, disk.params, excl_stor, disk.dynamic_params, name=disk.name, uuid=disk.uuid) return device # Please keep this at the bottom of the file for visibility. DEV_MAP = { constants.DT_PLAIN: LogicalVolume, constants.DT_DRBD8: drbd.DRBD8Dev, constants.DT_BLOCK: PersistentBlockDevice, constants.DT_RBD: RADOSBlockDevice, constants.DT_EXT: ExtStorageDevice, constants.DT_FILE: FileStorage, constants.DT_SHARED_FILE: FileStorage, constants.DT_GLUSTER: GlusterStorage, } """Map disk types to disk type classes. @see: L{Assemble}, L{FindDevice}, L{Create}.""" # pylint: disable=W0105 ganeti-3.1.0~rc2/lib/storage/container.py000064400000000000000000000336631476477700300203650ustar00rootroot00000000000000# # # Copyright (C) 2009, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Storage container abstraction. """ # pylint: disable=W0232,R0201 # W0232, since we use these as singletons rather than object holding # data # R0201, for the same reason # TODO: FileStorage initialised with paths whereas the others not import logging from ganeti import errors from ganeti import constants from ganeti import utils def _ParseSize(value): return int(round(float(value), 0)) class _Base(object): """Base class for storage abstraction. """ def List(self, name, fields): """Returns a list of all entities within the storage unit. @type name: string or None @param name: Entity name or None for all @type fields: list @param fields: List with all requested result fields (order is preserved) """ raise NotImplementedError() def Modify(self, name, changes): # pylint: disable=W0613 """Modifies an entity within the storage unit. @type name: string @param name: Entity name @type changes: dict @param changes: New field values """ # Don't raise an error if no changes are requested if changes: raise errors.ProgrammerError("Unable to modify the following" "fields: %r" % (list(changes), )) def Execute(self, name, op): """Executes an operation on an entity within the storage unit. @type name: string @param name: Entity name @type op: string @param op: Operation name """ raise NotImplementedError() class FileStorage(_Base): # pylint: disable=W0223 """File storage unit. """ def __init__(self, paths): """Initializes this class. @type paths: list @param paths: List of file storage paths """ super(FileStorage, self).__init__() self._paths = paths def List(self, name, fields): """Returns a list of all entities within the storage unit. See L{_Base.List}. """ rows = [] if name is None: paths = self._paths else: paths = [name] for path in paths: rows.append(self._ListInner(path, fields)) return rows @staticmethod def _ListInner(path, fields): """Gathers requested information from directory. @type path: string @param path: Path to directory @type fields: list @param fields: Requested fields """ values = [] # Pre-calculate information in case it's requested more than once if constants.SF_USED in fields: dirsize = utils.CalculateDirectorySize(path) else: dirsize = None if constants.SF_FREE in fields or constants.SF_SIZE in fields: fsstats = utils.GetFilesystemStats(path) else: fsstats = None # Make sure to update constants.VALID_STORAGE_FIELDS when changing fields. for field_name in fields: if field_name == constants.SF_NAME: values.append(path) elif field_name == constants.SF_USED: values.append(dirsize) elif field_name == constants.SF_FREE: values.append(fsstats[1]) elif field_name == constants.SF_SIZE: values.append(fsstats[0]) elif field_name == constants.SF_ALLOCATABLE: values.append(True) else: raise errors.StorageError("Unknown field: %r" % field_name) return values class _LvmBase(_Base): # pylint: disable=W0223 """Base class for LVM storage containers. @cvar LIST_FIELDS: list of tuples consisting of three elements: SF_* constants, lvm command output fields (list), and conversion function or static value (for static value, the lvm output field can be an empty list) """ LIST_SEP = "|" LIST_COMMAND = None LIST_FIELDS = None def List(self, name, wanted_field_names): """Returns a list of all entities within the storage unit. See L{_Base.List}. """ # Get needed LVM fields lvm_fields = self._GetLvmFields(self.LIST_FIELDS, wanted_field_names) # Build LVM command cmd_args = self._BuildListCommand(self.LIST_COMMAND, self.LIST_SEP, lvm_fields, name) # Run LVM command cmd_result = self._RunListCommand(cmd_args) # Split and rearrange LVM command output return self._BuildList(self._SplitList(cmd_result, self.LIST_SEP, len(lvm_fields)), self.LIST_FIELDS, wanted_field_names, lvm_fields) @staticmethod def _GetLvmFields(fields_def, wanted_field_names): """Returns unique list of fields wanted from LVM command. @type fields_def: list @param fields_def: Field definitions @type wanted_field_names: list @param wanted_field_names: List of requested fields """ field_to_idx = dict([(field_name, idx) for (idx, (field_name, _, _)) in enumerate(fields_def)]) lvm_fields = [] for field_name in wanted_field_names: try: idx = field_to_idx[field_name] except IndexError: raise errors.StorageError("Unknown field: %r" % field_name) (_, lvm_names, _) = fields_def[idx] lvm_fields.extend(lvm_names) return utils.UniqueSequence(lvm_fields) @classmethod def _BuildList(cls, cmd_result, fields_def, wanted_field_names, lvm_fields): """Builds the final result list. @type cmd_result: iterable @param cmd_result: Iterable of LVM command output (iterable of lists) @type fields_def: list @param fields_def: Field definitions @type wanted_field_names: list @param wanted_field_names: List of requested fields @type lvm_fields: list @param lvm_fields: LVM fields """ lvm_name_to_idx = dict([(lvm_name, idx) for (idx, lvm_name) in enumerate(lvm_fields)]) field_to_idx = dict([(field_name, idx) for (idx, (field_name, _, _)) in enumerate(fields_def)]) data = [] for raw_data in cmd_result: row = [] for field_name in wanted_field_names: (_, lvm_names, mapper) = fields_def[field_to_idx[field_name]] values = [raw_data[lvm_name_to_idx[i]] for i in lvm_names] if callable(mapper): # we got a function, call it with all the declared fields val = mapper(*values) elif len(values) == 1: assert mapper is None, ("Invalid mapper value (neither callable" " nor None) for one-element fields") # we don't have a function, but we had a single field # declared, pass it unchanged val = values[0] else: # let's make sure there are no fields declared (cannot map > # 1 field without a function) assert not values, "LVM storage has multi-fields without a function" val = mapper row.append(val) data.append(row) return data @staticmethod def _BuildListCommand(cmd, sep, options, name): """Builds LVM command line. @type cmd: string @param cmd: Command name @type sep: string @param sep: Field separator character @type options: list of strings @param options: Wanted LVM fields @type name: name or None @param name: Name of requested entity """ args = [cmd, "--noheadings", "--units=m", "--nosuffix", "--separator", sep, "--options", ",".join(options)] if name is not None: args.append(name) return args @staticmethod def _RunListCommand(args): """Run LVM command. """ result = utils.RunCmd(args) if result.failed: raise errors.StorageError("Failed to run %r, command output: %s" % (args[0], result.output)) return result.stdout @staticmethod def _SplitList(data, sep, fieldcount): """Splits LVM command output into rows and fields. @type data: string @param data: LVM command output @type sep: string @param sep: Field separator character @type fieldcount: int @param fieldcount: Expected number of fields """ for line in data.splitlines(): fields = line.strip().split(sep) if len(fields) != fieldcount: logging.warning("Invalid line returned from lvm command: %s", line) continue yield fields def _LvmPvGetAllocatable(attr): """Determines whether LVM PV is allocatable. @rtype: bool """ if attr: return (attr[0] == "a") else: logging.warning("Invalid PV attribute: %r", attr) return False class LvmPvStorage(_LvmBase): # pylint: disable=W0223 """LVM Physical Volume storage unit. """ LIST_COMMAND = "pvs" # Make sure to update constants.VALID_STORAGE_FIELDS when changing field # definitions. LIST_FIELDS = [ (constants.SF_NAME, ["pv_name"], None), (constants.SF_SIZE, ["pv_size"], _ParseSize), (constants.SF_USED, ["pv_used"], _ParseSize), (constants.SF_FREE, ["pv_free"], _ParseSize), (constants.SF_ALLOCATABLE, ["pv_attr"], _LvmPvGetAllocatable), ] def _SetAllocatable(self, name, allocatable): """Sets the "allocatable" flag on a physical volume. @type name: string @param name: Physical volume name @type allocatable: bool @param allocatable: Whether to set the "allocatable" flag """ args = ["pvchange", "--allocatable"] if allocatable: args.append("y") else: args.append("n") args.append(name) result = utils.RunCmd(args) if result.failed: raise errors.StorageError("Failed to modify physical volume," " pvchange output: %s" % result.output) def Modify(self, name, changes): """Modifies flags on a physical volume. See L{_Base.Modify}. """ if constants.SF_ALLOCATABLE in changes: self._SetAllocatable(name, changes[constants.SF_ALLOCATABLE]) del changes[constants.SF_ALLOCATABLE] # Other changes will be handled (and maybe refused) by the base class. return _LvmBase.Modify(self, name, changes) class LvmVgStorage(_LvmBase): """LVM Volume Group storage unit. """ LIST_COMMAND = "vgs" VGREDUCE_COMMAND = "vgreduce" # Make sure to update constants.VALID_STORAGE_FIELDS when changing field # definitions. LIST_FIELDS = [ (constants.SF_NAME, ["vg_name"], None), (constants.SF_SIZE, ["vg_size"], _ParseSize), (constants.SF_FREE, ["vg_free"], _ParseSize), (constants.SF_USED, ["vg_size", "vg_free"], lambda x, y: _ParseSize(x) - _ParseSize(y)), (constants.SF_ALLOCATABLE, [], True), ] def _RemoveMissing(self, name, _runcmd_fn=utils.RunCmd): """Runs "vgreduce --removemissing" on a volume group. @type name: string @param name: Volume group name """ # Ignoring vgreduce exit code. Older versions exit with an error even tough # the VG is already consistent. This was fixed in later versions, but we # cannot depend on it. result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing", name]) # Keep output in case something went wrong vgreduce_output = result.output # work around newer LVM version if ("Wrote out consistent volume group" not in vgreduce_output or "vgreduce --removemissing --force" in vgreduce_output): # we need to re-run with --force result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing", "--force", name]) vgreduce_output += "\n" + result.output result = _runcmd_fn([self.LIST_COMMAND, "--noheadings", "--nosuffix", name]) # we also need to check the output if result.failed or "Couldn't find device with uuid" in result.output: raise errors.StorageError(("Volume group '%s' still not consistent," " 'vgreduce' output: %r," " 'vgs' output: %r") % (name, vgreduce_output, result.output)) def Execute(self, name, op): """Executes an operation on a virtual volume. See L{_Base.Execute}. """ if op == constants.SO_FIX_CONSISTENCY: return self._RemoveMissing(name) return _LvmBase.Execute(self, name, op) # Lookup table for storage types _STORAGE_TYPES = { constants.ST_FILE: FileStorage, constants.ST_LVM_PV: LvmPvStorage, constants.ST_LVM_VG: LvmVgStorage, constants.ST_SHARED_FILE: FileStorage, constants.ST_GLUSTER: FileStorage, } def GetStorageClass(name): """Returns the class for a storage type. @type name: string @param name: Storage type """ try: return _STORAGE_TYPES[name] except KeyError: raise errors.StorageError("Unknown storage type: %r" % name) def GetStorage(name, *args): """Factory function for storage methods. @type name: string @param name: Storage type """ return GetStorageClass(name)(*args) ganeti-3.1.0~rc2/lib/storage/drbd.py000064400000000000000000001127671476477700300173210ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """DRBD block device related functionality""" import errno import logging import time from ganeti import constants from ganeti import utils from ganeti import errors from ganeti import netutils from ganeti import objects from ganeti.storage import base from ganeti.storage.drbd_info import DRBD8Info from ganeti.storage import drbd_info from ganeti.storage import drbd_cmdgen # Size of reads in _CanReadDevice _DEVICE_READ_SIZE = 128 * 1024 class DRBD8(object): """Various methods to deals with the DRBD system as a whole. This class provides a set of methods to deal with the DRBD installation on the node or with uninitialized devices as opposed to a DRBD device. """ _USERMODE_HELPER_FILE = "/sys/module/drbd/parameters/usermode_helper" _MAX_MINORS = 255 @staticmethod def GetUsermodeHelper(filename=_USERMODE_HELPER_FILE): """Returns DRBD usermode_helper currently set. @type filename: string @param filename: the filename to read the usermode helper from @rtype: string @return: the currently configured DRBD usermode helper """ try: helper = utils.ReadFile(filename).splitlines()[0] except EnvironmentError as err: if err.errno == errno.ENOENT: base.ThrowError("The file %s cannot be opened, check if the module" " is loaded (%s)", filename, str(err)) else: base.ThrowError("Can't read DRBD helper file %s: %s", filename, str(err)) if not helper: base.ThrowError("Can't read any data from %s", filename) return helper @staticmethod def GetProcInfo(): """Reads and parses information from /proc/drbd. @rtype: DRBD8Info @return: a L{DRBD8Info} instance containing the current /proc/drbd info """ return DRBD8Info.CreateFromFile() @staticmethod def GetUsedDevs(): """Compute the list of used DRBD minors. @rtype: list of ints """ info = DRBD8.GetProcInfo() return [m for m in info.GetMinors() if not info.GetMinorStatus(m).is_unconfigured] @staticmethod def FindUnusedMinor(): """Find an unused DRBD device. This is specific to 8.x as the minors are allocated dynamically, so non-existing numbers up to a max minor count are actually free. @rtype: int """ highest = -1 info = DRBD8.GetProcInfo() for minor in info.GetMinors(): status = info.GetMinorStatus(minor) if not status.is_in_use: return minor highest = max(highest, minor) if highest >= DRBD8._MAX_MINORS: logging.error("Error: no free drbd minors!") raise errors.BlockDeviceError("Can't find a free DRBD minor") return highest + 1 @staticmethod def GetCmdGenerator(info): """Creates a suitable L{BaseDRBDCmdGenerator} based on the given info. @type info: DRBD8Info @rtype: BaseDRBDCmdGenerator """ version = info.GetVersion() if version["k_minor"] <= 3: return drbd_cmdgen.DRBD83CmdGenerator(version) else: return drbd_cmdgen.DRBD84CmdGenerator(version) @staticmethod def ShutdownAll(minor): """Deactivate the device. This will, of course, fail if the device is in use. @type minor: int @param minor: the minor to shut down """ info = DRBD8.GetProcInfo() cmd_gen = DRBD8.GetCmdGenerator(info) cmd = cmd_gen.GenDownCmd(minor) result = utils.RunCmd(cmd) if result.failed: base.ThrowError("drbd%d: can't shutdown drbd device: %s", minor, result.output) class DRBD8Dev(base.BlockDev): """DRBD v8.x block device. This implements the local host part of the DRBD device, i.e. it doesn't do anything to the supposed peer. If you need a fully connected DRBD pair, you need to use this class on both hosts. The unique_id for the drbd device is a (pnode_uuid, snode_uuid, port, pnode_minor, lnode_minor, secret) tuple, and it must have two children: the data device and the meta_device. The meta device is checked for valid size and is zeroed on create. """ _DRBD_MAJOR = 147 # timeout constants _NET_RECONFIG_TIMEOUT = 60 def __init__(self, unique_id, children, size, params, dyn_params, **kwargs): if children and children.count(None) > 0: children = [] if len(children) not in (0, 2): raise ValueError("Invalid configuration data %s" % str(children)) if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 6: raise ValueError("Invalid configuration data %s" % str(unique_id)) if constants.DDP_LOCAL_IP not in dyn_params or \ constants.DDP_REMOTE_IP not in dyn_params or \ constants.DDP_LOCAL_MINOR not in dyn_params or \ constants.DDP_REMOTE_MINOR not in dyn_params: raise ValueError("Invalid dynamic parameters %s" % str(dyn_params)) self._lhost = dyn_params[constants.DDP_LOCAL_IP] self._lport = unique_id[2] self._rhost = dyn_params[constants.DDP_REMOTE_IP] self._rport = unique_id[2] self._aminor = dyn_params[constants.DDP_LOCAL_MINOR] # The secret is wrapped in the Private data type, and it has to be extracted # before use self._secret = unique_id[5].Get() if children: if not _CanReadDevice(children[1].dev_path): logging.info("drbd%s: Ignoring unreadable meta device", self._aminor) children = [] super(DRBD8Dev, self).__init__(unique_id, children, size, params, dyn_params, **kwargs) self.major = self._DRBD_MAJOR info = DRBD8.GetProcInfo() version = info.GetVersion() if version["k_major"] != 8: base.ThrowError("Mismatch in DRBD kernel version and requested ganeti" " usage: kernel is %s.%s, ganeti wants 8.x", version["k_major"], version["k_minor"]) if version["k_minor"] <= 3: self._show_info_cls = drbd_info.DRBD83ShowInfo else: self._show_info_cls = drbd_info.DRBD84ShowInfo self._cmd_gen = DRBD8.GetCmdGenerator(info) if (self._lhost is not None and self._lhost == self._rhost and self._lport == self._rport): raise ValueError("Invalid configuration data, same local/remote %s, %s" % (unique_id, dyn_params)) self.Attach() @staticmethod def _DevPath(minor): """Return the path to a drbd device for a given minor. @type minor: int @rtype: string """ return "/dev/drbd%d" % minor def _SetFromMinor(self, minor): """Set our parameters based on the given minor. This sets our minor variable and our dev_path. @type minor: int """ if minor is None: self.minor = self.dev_path = None self.attached = False else: self.minor = minor self.dev_path = self._DevPath(minor) self.attached = True @staticmethod def _CheckMetaSize(meta_device): """Check if the given meta device looks like a valid one. This currently only checks the size, which must be around 128MiB. @type meta_device: string @param meta_device: the path to the device to check """ result = utils.RunCmd(["blockdev", "--getsize", meta_device]) if result.failed: base.ThrowError("Failed to get device size: %s - %s", result.fail_reason, result.output) try: sectors = int(result.stdout) except (TypeError, ValueError): base.ThrowError("Invalid output from blockdev: '%s'", result.stdout) num_bytes = sectors * 512 if num_bytes < 128 * 1024 * 1024: # less than 128MiB base.ThrowError("Meta device too small (%.2fMib)", (num_bytes / 1024 / 1024)) # the maximum *valid* size of the meta device when living on top # of LVM is hard to compute: it depends on the number of stripes # and the PE size; e.g. a 2-stripe, 64MB PE will result in a 128MB # (normal size), but an eight-stripe 128MB PE will result in a 1GB # size meta device; as such, we restrict it to 1GB (a little bit # too generous, but making assumptions about PE size is hard) if num_bytes > 1024 * 1024 * 1024: base.ThrowError("Meta device too big (%.2fMiB)", (num_bytes / 1024 / 1024)) def _GetShowData(self, minor): """Return the `drbdsetup show` data. @type minor: int @param minor: the minor to collect show output for @rtype: string """ result = utils.RunCmd(self._cmd_gen.GenShowCmd(minor)) if result.failed: logging.error("Can't display the drbd config: %s - %s", result.fail_reason, result.output) return None return result.stdout def _GetShowInfo(self, minor): """Return parsed information from `drbdsetup show`. @type minor: int @param minor: the minor to return information for @rtype: dict as described in L{drbd_info.BaseShowInfo.GetDevInfo} """ return self._show_info_cls.GetDevInfo(self._GetShowData(minor)) @staticmethod def _NeedsLocalSyncerParams(): # For DRBD >= 8.4, syncer init must be done after local, not in net. info = DRBD8.GetProcInfo() version = info.GetVersion() return version["k_minor"] >= 4 def _MatchesLocal(self, info): """Test if our local config matches with an existing device. The parameter should be as returned from `_GetShowInfo()`. This method tests if our local backing device is the same as the one in the info parameter, in effect testing if we look like the given device. @type info: dict as described in L{drbd_info.BaseShowInfo.GetDevInfo} @rtype: boolean """ if self._children: backend, meta = self._children else: backend = meta = None if backend is not None: retval = ("local_dev" in info and info["local_dev"] == backend.dev_path) else: retval = ("local_dev" not in info) if meta is not None: retval = retval and ("meta_dev" in info and info["meta_dev"] == meta.dev_path) if "meta_index" in info: retval = retval and info["meta_index"] == 0 else: retval = retval and ("meta_dev" not in info and "meta_index" not in info) return retval def _MatchesNet(self, info): """Test if our network config matches with an existing device. The parameter should be as returned from `_GetShowInfo()`. This method tests if our network configuration is the same as the one in the info parameter, in effect testing if we look like the given device. @type info: dict as described in L{drbd_info.BaseShowInfo.GetDevInfo} @rtype: boolean """ if (((self._lhost is None and not ("local_addr" in info)) and (self._rhost is None and not ("remote_addr" in info)))): return True if self._lhost is None: return False if not ("local_addr" in info and "remote_addr" in info): return False retval = (info["local_addr"] == (self._lhost, self._lport)) retval = (retval and info["remote_addr"] == (self._rhost, self._rport)) return retval def _AssembleLocal(self, minor, backend, meta, size): """Configure the local part of a DRBD device. @type minor: int @param minor: the minor to assemble locally @type backend: string @param backend: path to the data device to use @type meta: string @param meta: path to the meta device to use @type size: int @param size: size in MiB """ cmds = self._cmd_gen.GenLocalInitCmds(minor, backend, meta, size, self.params) for cmd in cmds: result = utils.RunCmd(cmd) if result.failed: base.ThrowError("drbd%d: can't attach local disk: %s", minor, result.output) def _WaitForMinorSyncParams(): """Call _SetMinorSyncParams and raise RetryAgain on errors. """ if self._SetMinorSyncParams(minor, self.params): raise utils.RetryAgain() if self._NeedsLocalSyncerParams(): # Retry because disk config for DRBD resource may be still uninitialized. try: utils.Retry(_WaitForMinorSyncParams, 1.0, 5.0) except utils.RetryTimeout as e: base.ThrowError("drbd%d: can't set the synchronization parameters: %s" % (minor, utils.CommaJoin(e.args[0]))) def _AssembleNet(self, minor, net_info, dual_pri=False, hmac=None, secret=None): """Configure the network part of the device. @type minor: int @param minor: the minor to assemble the network for @type net_info: (string, int, string, int) @param net_info: tuple containing the local address, local port, remote address and remote port @type dual_pri: boolean @param dual_pri: whether two primaries should be allowed or not @type hmac: string @param hmac: the HMAC algorithm to use @type secret: string @param secret: the shared secret to use """ lhost, lport, rhost, rport = net_info if None in net_info: # we don't want network connection and actually want to make # sure its shutdown self._ShutdownNet(minor) return if dual_pri: protocol = constants.DRBD_MIGRATION_NET_PROTOCOL else: protocol = self.params[constants.LDP_PROTOCOL] # Workaround for a race condition. When DRBD is doing its dance to # establish a connection with its peer, it also sends the # synchronization speed over the wire. In some cases setting the # sync speed only after setting up both sides can race with DRBD # connecting, hence we set it here before telling DRBD anything # about its peer. if not self._NeedsLocalSyncerParams(): sync_errors = self._SetMinorSyncParams(minor, self.params) if sync_errors: base.ThrowError("drbd%d: can't set the synchronization parameters: %s" % (minor, utils.CommaJoin(sync_errors))) family = self._GetNetFamily(minor, lhost, rhost) cmds = self._cmd_gen.GenNetInitCmds(minor, family, lhost, lport, rhost, rport, protocol, dual_pri, hmac, secret, self.params) for cmd in cmds: result = utils.RunCmd(cmd) if result.failed: base.ThrowError("drbd%d: can't setup network: %s - %s", minor, result.fail_reason, result.output) def _CheckNetworkConfig(): info = self._GetShowInfo(minor) if not "local_addr" in info or not "remote_addr" in info: raise utils.RetryAgain() if (info["local_addr"] != (lhost, lport) or info["remote_addr"] != (rhost, rport)): raise utils.RetryAgain() try: utils.Retry(_CheckNetworkConfig, 1.0, 10.0) except utils.RetryTimeout: base.ThrowError("drbd%d: timeout while configuring network", minor) # Once the assembly is over, try to set the synchronization parameters if not self._NeedsLocalSyncerParams(): try: # The minor may not have been set yet, requiring us to set it at least # temporarily old_minor = self.minor self._SetFromMinor(minor) sync_errors = self.SetSyncParams(self.params) if sync_errors: base.ThrowError("drbd%d: can't set the synchronization parameters: " "%s" % (self.minor, utils.CommaJoin(sync_errors))) finally: # Undo the change, regardless of whether it will have to be done again # soon self._SetFromMinor(old_minor) @staticmethod def _GetNetFamily(minor, lhost, rhost): if netutils.IP6Address.IsValid(lhost): if not netutils.IP6Address.IsValid(rhost): base.ThrowError("drbd%d: can't connect ip %s to ip %s" % (minor, lhost, rhost)) return "ipv6" elif netutils.IP4Address.IsValid(lhost): if not netutils.IP4Address.IsValid(rhost): base.ThrowError("drbd%d: can't connect ip %s to ip %s" % (minor, lhost, rhost)) return "ipv4" else: base.ThrowError("drbd%d: Invalid ip %s" % (minor, lhost)) def AddChildren(self, devices): """Add a disk to the DRBD device. @type devices: list of L{BlockDev} @param devices: a list of exactly two L{BlockDev} objects; the first denotes the data device, the second the meta device for this DRBD device """ if self.minor is None: base.ThrowError("drbd%d: can't attach to dbrd8 during AddChildren", self._aminor) if len(devices) != 2: base.ThrowError("drbd%d: need two devices for AddChildren", self.minor) info = self._GetShowInfo(self.minor) if "local_dev" in info: base.ThrowError("drbd%d: already attached to a local disk", self.minor) backend, meta = devices if backend.dev_path is None or meta.dev_path is None: base.ThrowError("drbd%d: children not ready during AddChildren", self.minor) backend.Open() meta.Open() self._CheckMetaSize(meta.dev_path) self._InitMeta(DRBD8.FindUnusedMinor(), meta.dev_path) self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path, self.size) self._children = devices def RemoveChildren(self, devices): """Detach the drbd device from local storage. @type devices: list of L{BlockDev} @param devices: a list of exactly two L{BlockDev} objects; the first denotes the data device, the second the meta device for this DRBD device """ if self.minor is None: base.ThrowError("drbd%d: can't attach to drbd8 during RemoveChildren", self._aminor) # early return if we don't actually have backing storage info = self._GetShowInfo(self.minor) if "local_dev" not in info: return if len(self._children) != 2: base.ThrowError("drbd%d: we don't have two children: %s", self.minor, self._children) if self._children.count(None) == 2: # we don't actually have children :) logging.warning("drbd%d: requested detach while detached", self.minor) return if len(devices) != 2: base.ThrowError("drbd%d: we need two children in RemoveChildren", self.minor) for child, dev in zip(self._children, devices): if dev != child.dev_path: base.ThrowError("drbd%d: mismatch in local storage (%s != %s) in" " RemoveChildren", self.minor, dev, child.dev_path) self._ShutdownLocal(self.minor) self._children = [] def _SetMinorSyncParams(self, minor, params): """Set the parameters of the DRBD syncer. This is the low-level implementation. @type minor: int @param minor: the drbd minor whose settings we change @type params: dict @param params: LD level disk parameters related to the synchronization @rtype: list @return: a list of error messages """ cmd = self._cmd_gen.GenSyncParamsCmd(minor, params) result = utils.RunCmd(cmd) if result.failed: msg = ("Can't change syncer rate: %s - %s" % (result.fail_reason, result.output)) logging.error(msg) return [msg] return [] def SetSyncParams(self, params): """Set the synchronization parameters of the DRBD syncer. See L{BlockDev.SetSyncParams} for parameter description. """ if self.minor is None: err = "Not attached during SetSyncParams" logging.info(err) return [err] children_result = super(DRBD8Dev, self).SetSyncParams(params) children_result.extend(self._SetMinorSyncParams(self.minor, params)) return children_result def PauseResumeSync(self, pause): """Pauses or resumes the sync of a DRBD device. See L{BlockDev.PauseResumeSync} for parameter description. """ if self.minor is None: logging.info("Not attached during PauseSync") return False children_result = super(DRBD8Dev, self).PauseResumeSync(pause) if pause: cmd = self._cmd_gen.GenPauseSyncCmd(self.minor) else: cmd = self._cmd_gen.GenResumeSyncCmd(self.minor) result = utils.RunCmd(cmd) if result.failed: logging.error("Can't %s: %s - %s", cmd, result.fail_reason, result.output) return not result.failed and children_result def GetProcStatus(self): """Return the current status data from /proc/drbd for this device. @rtype: DRBD8Status """ if self.minor is None: base.ThrowError("drbd%d: GetStats() called while not attached", self._aminor) info = DRBD8.GetProcInfo() if not info.HasMinorStatus(self.minor): base.ThrowError("drbd%d: can't find myself in /proc", self.minor) return info.GetMinorStatus(self.minor) def GetSyncStatus(self): """Returns the sync status of the device. If sync_percent is None, it means all is ok If estimated_time is None, it means we can't estimate the time needed, otherwise it's the time left in seconds. We set the is_degraded parameter to True on two conditions: network not connected or local disk missing. We compute the ldisk parameter based on whether we have a local disk or not. @rtype: objects.BlockDevStatus """ if self.minor is None and not self.Attach(): base.ThrowError("drbd%d: can't Attach() in GetSyncStatus", self._aminor) stats = self.GetProcStatus() is_degraded = not stats.is_connected or not stats.is_disk_uptodate if stats.is_diskless or stats.is_standalone: ldisk_status = constants.LDS_FAULTY elif stats.is_disk_uptodate: ldisk_status = constants.LDS_OKAY elif stats.is_in_resync: ldisk_status = constants.LDS_SYNC else: ldisk_status = constants.LDS_UNKNOWN return objects.BlockDevStatus(dev_path=self.dev_path, major=self.major, minor=self.minor, sync_percent=stats.sync_percent, estimated_time=stats.est_time, is_degraded=is_degraded, ldisk_status=ldisk_status) def Open(self, force=False, exclusive=True): """Make the local state primary. If the 'force' parameter is given, DRBD is instructed to switch the device into primary mode. Since this is a potentially dangerous operation, the force flag should be only given after creation, when it actually is mandatory. """ if self.minor is None and not self.Attach(): logging.error("DRBD cannot attach to a device during open") return False cmd = self._cmd_gen.GenPrimaryCmd(self.minor, force) result = utils.RunCmd(cmd) if result.failed: base.ThrowError("drbd%d: can't make drbd device primary: %s", self.minor, result.output) def Close(self): """Make the local state secondary. This will, of course, fail if the device is in use. """ if self.minor is None and not self.Attach(): base.ThrowError("drbd%d: can't Attach() in Close()", self._aminor) cmd = self._cmd_gen.GenSecondaryCmd(self.minor) result = utils.RunCmd(cmd) if result.failed: base.ThrowError("drbd%d: can't switch drbd device to secondary: %s", self.minor, result.output) def DisconnectNet(self): """Removes network configuration. This method shutdowns the network side of the device. The method will wait up to a hardcoded timeout for the device to go into standalone after the 'disconnect' command before re-configuring it, as sometimes it takes a while for the disconnect to actually propagate and thus we might issue a 'net' command while the device is still connected. If the device will still be attached to the network and we time out, we raise an exception. """ if self.minor is None: base.ThrowError("drbd%d: disk not attached in re-attach net", self._aminor) if None in (self._lhost, self._lport, self._rhost, self._rport): base.ThrowError("drbd%d: DRBD disk missing network info in" " DisconnectNet()", self.minor) class _DisconnectStatus(object): def __init__(self, ever_disconnected): self.ever_disconnected = ever_disconnected dstatus = _DisconnectStatus(base.IgnoreError(self._ShutdownNet, self.minor)) def _WaitForDisconnect(): if self.GetProcStatus().is_standalone: return # retry the disconnect, it seems possible that due to a well-time # disconnect on the peer, my disconnect command might be ignored and # forgotten dstatus.ever_disconnected = \ base.IgnoreError(self._ShutdownNet, self.minor) or \ dstatus.ever_disconnected raise utils.RetryAgain() # Keep start time start_time = time.time() try: # Start delay at 100 milliseconds and grow up to 2 seconds utils.Retry(_WaitForDisconnect, (0.1, 1.5, 2.0), self._NET_RECONFIG_TIMEOUT) except utils.RetryTimeout: if dstatus.ever_disconnected: msg = ("drbd%d: device did not react to the" " 'disconnect' command in a timely manner") else: msg = "drbd%d: can't shutdown network, even after multiple retries" base.ThrowError(msg, self.minor) reconfig_time = time.time() - start_time if reconfig_time > (self._NET_RECONFIG_TIMEOUT * 0.25): logging.info("drbd%d: DisconnectNet: detach took %.3f seconds", self.minor, reconfig_time) def AttachNet(self, multimaster): """Reconnects the network. This method connects the network side of the device with a specified multi-master flag. The device needs to be 'Standalone' but have valid network configuration data. @type multimaster: boolean @param multimaster: init the network in dual-primary mode """ if self.minor is None: base.ThrowError("drbd%d: device not attached in AttachNet", self._aminor) if None in (self._lhost, self._lport, self._rhost, self._rport): base.ThrowError("drbd%d: missing network info in AttachNet()", self.minor) status = self.GetProcStatus() if not status.is_standalone: base.ThrowError("drbd%d: device is not standalone in AttachNet", self.minor) self._AssembleNet(self.minor, (self._lhost, self._lport, self._rhost, self._rport), dual_pri=multimaster, hmac=constants.DRBD_HMAC_ALG, secret=self._secret) def Attach(self, **kwargs): """Check if our minor is configured. This doesn't do any device configurations - it only checks if the minor is in a state different from Unconfigured. Note that this function will not change the state of the system in any way (except in case of side-effects caused by reading from /proc). """ used_devs = DRBD8.GetUsedDevs() if self._aminor in used_devs: minor = self._aminor else: minor = None self._SetFromMinor(minor) return minor is not None def Assemble(self): """Assemble the drbd. Method: - if we have a configured device, we try to ensure that it matches our config - if not, we create it from zero - anyway, set the device parameters """ super(DRBD8Dev, self).Assemble() self.Attach() if self.minor is None: # local device completely unconfigured self._FastAssemble() else: # we have to recheck the local and network status and try to fix # the device self._SlowAssemble() def _SlowAssemble(self): """Assembles the DRBD device from a (partially) configured device. In case of partially attached (local device matches but no network setup), we perform the network attach. If successful, we re-test the attach if can return success. """ # TODO: Rewrite to not use a for loop just because there is 'break' # pylint: disable=W0631 net_data = (self._lhost, self._lport, self._rhost, self._rport) for minor in (self._aminor,): info = self._GetShowInfo(minor) match_l = self._MatchesLocal(info) match_r = self._MatchesNet(info) if match_l and match_r: # everything matches break if match_l and not match_r and "local_addr" not in info: # disk matches, but not attached to network, attach and recheck self._AssembleNet(minor, net_data, hmac=constants.DRBD_HMAC_ALG, secret=self._secret) if self._MatchesNet(self._GetShowInfo(minor)): break else: base.ThrowError("drbd%d: network attach successful, but 'drbdsetup" " show' disagrees", minor) if match_r and "local_dev" not in info: # no local disk, but network attached and it matches self._AssembleLocal(minor, self._children[0].dev_path, self._children[1].dev_path, self.size) if self._MatchesLocal(self._GetShowInfo(minor)): break else: base.ThrowError("drbd%d: disk attach successful, but 'drbdsetup" " show' disagrees", minor) # this case must be considered only if we actually have local # storage, i.e. not in diskless mode, because all diskless # devices are equal from the point of view of local # configuration if (match_l and "local_dev" in info and not match_r and "local_addr" in info): # strange case - the device network part points to somewhere # else, even though its local storage is ours; as we own the # drbd space, we try to disconnect from the remote peer and # reconnect to our correct one try: self._ShutdownNet(minor) except errors.BlockDeviceError as err: base.ThrowError("drbd%d: device has correct local storage, wrong" " remote peer and is unable to disconnect in order" " to attach to the correct peer: %s", minor, str(err)) # note: _AssembleNet also handles the case when we don't want # local storage (i.e. one or more of the _[lr](host|port) is # None) self._AssembleNet(minor, net_data, hmac=constants.DRBD_HMAC_ALG, secret=self._secret) if self._MatchesNet(self._GetShowInfo(minor)): break else: base.ThrowError("drbd%d: network attach successful, but 'drbdsetup" " show' disagrees", minor) else: minor = None self._SetFromMinor(minor) if minor is None: base.ThrowError("drbd%d: cannot activate, unknown or unhandled reason", self._aminor) def _FastAssemble(self): """Assemble the drbd device from zero. This is run when in Assemble we detect our minor is unused. """ minor = self._aminor if self._children and self._children[0] and self._children[1]: self._AssembleLocal(minor, self._children[0].dev_path, self._children[1].dev_path, self.size) if self._lhost and self._lport and self._rhost and self._rport: self._AssembleNet(minor, (self._lhost, self._lport, self._rhost, self._rport), hmac=constants.DRBD_HMAC_ALG, secret=self._secret) self._SetFromMinor(minor) def _ShutdownLocal(self, minor): """Detach from the local device. I/Os will continue to be served from the remote device. If we don't have a remote device, this operation will fail. @type minor: int @param minor: the device to detach from the local device """ cmd = self._cmd_gen.GenDetachCmd(minor) result = utils.RunCmd(cmd) if result.failed: base.ThrowError("drbd%d: can't detach local disk: %s", minor, result.output) def _ShutdownNet(self, minor): """Disconnect from the remote peer. This fails if we don't have a local device. @type minor: boolean @param minor: the device to disconnect from the remote peer """ family = self._GetNetFamily(minor, self._lhost, self._rhost) cmd = self._cmd_gen.GenDisconnectCmd(minor, family, self._lhost, self._lport, self._rhost, self._rport) result = utils.RunCmd(cmd) if result.failed: base.ThrowError("drbd%d: can't shutdown network: %s", minor, result.output) def Shutdown(self): """Shutdown the DRBD device. """ if self.minor is None and not self.Attach(): logging.info("drbd%d: not attached during Shutdown()", self._aminor) return try: DRBD8.ShutdownAll(self.minor) finally: self.minor = None self.dev_path = None def Remove(self): """Stub remove for DRBD devices. """ self.Shutdown() def Rename(self, new_id): """Rename a device. This is not supported for drbd devices. """ raise errors.ProgrammerError("Can't rename a drbd device") def Grow(self, amount, dryrun, backingstore, excl_stor): """Resize the DRBD device and its backing storage. See L{BlockDev.Grow} for parameter description. """ if self.minor is None: base.ThrowError("drbd%d: Grow called while not attached", self._aminor) if len(self._children) != 2 or None in self._children: base.ThrowError("drbd%d: cannot grow diskless device", self.minor) self._children[0].Grow(amount, dryrun, backingstore, excl_stor) if dryrun or backingstore: # DRBD does not support dry-run mode and is not backing storage, # so we'll return here return cmd = self._cmd_gen.GenResizeCmd(self.minor, self.size + amount) result = utils.RunCmd(cmd) if result.failed: base.ThrowError("drbd%d: resize failed: %s", self.minor, result.output) @classmethod def _InitMeta(cls, minor, dev_path): """Initialize a meta device. This will not work if the given minor is in use. @type minor: int @param minor: the DRBD minor whose (future) meta device should be initialized @type dev_path: string @param dev_path: path to the meta device to initialize """ # Zero the metadata first, in order to make sure drbdmeta doesn't # try to auto-detect existing filesystems or similar (see # https://github.com/ganeti/ganeti/issues/238); we only # care about the first 128MB of data in the device, even though it # can be bigger result = utils.RunCmd([constants.DD_CMD, "if=/dev/zero", "of=%s" % dev_path, "bs=%s" % constants.DD_BLOCK_SIZE, "count=128", "oflag=direct"]) if result.failed: base.ThrowError("Can't wipe the meta device: %s", result.output) info = DRBD8.GetProcInfo() cmd_gen = DRBD8.GetCmdGenerator(info) cmd = cmd_gen.GenInitMetaCmd(minor, dev_path) result = utils.RunCmd(cmd) if result.failed: base.ThrowError("Can't initialize meta device: %s", result.output) @classmethod def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, **kwargs): """Create a new DRBD8 device. Since DRBD devices are not created per se, just assembled, this function only initializes the metadata. """ if len(children) != 2: raise errors.ProgrammerError("Invalid setup for the drbd device") if excl_stor: raise errors.ProgrammerError("DRBD device requested with" " exclusive_storage") if constants.DDP_LOCAL_MINOR not in dyn_params: raise errors.ProgrammerError("Invalid dynamic params for drbd device %s" % dyn_params) # check that the minor is unused aminor = dyn_params[constants.DDP_LOCAL_MINOR] info = DRBD8.GetProcInfo() if info.HasMinorStatus(aminor): status = info.GetMinorStatus(aminor) in_use = status.is_in_use else: in_use = False if in_use: base.ThrowError("drbd%d: minor is already in use at Create() time", aminor) meta = children[1] meta.Assemble() if not meta.Attach(): base.ThrowError("drbd%d: can't attach to meta device '%s'", aminor, meta) cls._CheckMetaSize(meta.dev_path) cls._InitMeta(aminor, meta.dev_path) return cls(unique_id, children, size, params, dyn_params) def _CanReadDevice(path): """Check if we can read from the given device. This tries to read the first 128k of the device. @type path: string """ try: utils.ReadBinaryFile(path, size=_DEVICE_READ_SIZE) return True except EnvironmentError: logging.warning("Can't read from device %s", path, exc_info=True) return False ganeti-3.1.0~rc2/lib/storage/drbd_cmdgen.py000064400000000000000000000364661476477700300206370ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """DRBD command generating classes""" import logging import shlex from ganeti import constants from ganeti import errors class BaseDRBDCmdGenerator(object): """Base class for DRBD command generators. This class defines the interface for the command generators and holds shared code. """ def __init__(self, version): self._version = version def GenShowCmd(self, minor): raise NotImplementedError def GenInitMetaCmd(self, minor, meta_dev): raise NotImplementedError def GenLocalInitCmds(self, minor, data_dev, meta_dev, size_mb, params): raise NotImplementedError def GenNetInitCmds(self, minor, family, lhost, lport, rhost, rport, protocol, dual_pri, hmac, secret, params): raise NotImplementedError def GenSyncParamsCmd(self, minor, params): raise NotImplementedError def GenPauseSyncCmd(self, minor): raise NotImplementedError def GenResumeSyncCmd(self, minor): raise NotImplementedError def GenPrimaryCmd(self, minor, force): raise NotImplementedError def GenSecondaryCmd(self, minor): raise NotImplementedError def GenDetachCmd(self, minor): raise NotImplementedError def GenDisconnectCmd(self, minor, family, lhost, lport, rhost, rport): raise NotImplementedError def GenDownCmd(self, minor): raise NotImplementedError def GenResizeCmd(self, minor, size_mb): raise NotImplementedError @staticmethod def _DevPath(minor): """Return the path to a drbd device for a given minor. """ return "/dev/drbd%d" % minor class DRBD83CmdGenerator(BaseDRBDCmdGenerator): """Generates drbdsetup commands suited for the DRBD <= 8.3 syntax. """ # command line options for barriers _DISABLE_DISK_OPTION = "--no-disk-barrier" # -a _DISABLE_DRAIN_OPTION = "--no-disk-drain" # -D _DISABLE_FLUSH_OPTION = "--no-disk-flushes" # -i _DISABLE_META_FLUSH_OPTION = "--no-md-flushes" # -m def __init__(self, version): super(DRBD83CmdGenerator, self).__init__(version) def GenShowCmd(self, minor): return ["drbdsetup", self._DevPath(minor), "show"] def GenInitMetaCmd(self, minor, meta_dev): return ["drbdmeta", "--force", self._DevPath(minor), "v08", meta_dev, "0", "create-md"] def GenLocalInitCmds(self, minor, data_dev, meta_dev, size_mb, params): args = ["drbdsetup", self._DevPath(minor), "disk", data_dev, meta_dev, "0", "-e", "detach", "--create-device"] if size_mb: args.extend(["-d", "%sm" % size_mb]) vmaj = self._version["k_major"] vmin = self._version["k_minor"] vrel = self._version["k_point"] barrier_args = \ self._ComputeDiskBarrierArgs(vmaj, vmin, vrel, params[constants.LDP_BARRIERS], params[constants.LDP_NO_META_FLUSH]) args.extend(barrier_args) if params[constants.LDP_DISK_CUSTOM]: args.extend(shlex.split(params[constants.LDP_DISK_CUSTOM])) return [args] def GenNetInitCmds(self, minor, family, lhost, lport, rhost, rport, protocol, dual_pri, hmac, secret, params): args = ["drbdsetup", self._DevPath(minor), "net", "%s:%s:%s" % (family, lhost, lport), "%s:%s:%s" % (family, rhost, rport), protocol, "-A", "discard-zero-changes", "-B", "consensus", "--create-device", ] if dual_pri: args.append("-m") if hmac and secret: args.extend(["-a", hmac, "-x", secret]) if params[constants.LDP_NET_CUSTOM]: args.extend(shlex.split(params[constants.LDP_NET_CUSTOM])) return [args] def GenSyncParamsCmd(self, minor, params): args = ["drbdsetup", self._DevPath(minor), "syncer"] if params[constants.LDP_DYNAMIC_RESYNC]: vmin = self._version["k_minor"] vrel = self._version["k_point"] # By definition we are using 8.x, so just check the rest of the version # number if vmin != 3 or vrel < 9: msg = ("The current DRBD version (8.%d.%d) does not support the " "dynamic resync speed controller" % (vmin, vrel)) logging.error(msg) return [msg] if params[constants.LDP_PLAN_AHEAD] == 0: msg = ("A value of 0 for c-plan-ahead disables the dynamic sync speed" " controller at DRBD level. If you want to disable it, please" " set the dynamic-resync disk parameter to False.") logging.error(msg) return [msg] # add the c-* parameters to args args.extend(["--c-plan-ahead", params[constants.LDP_PLAN_AHEAD], "--c-fill-target", params[constants.LDP_FILL_TARGET], "--c-delay-target", params[constants.LDP_DELAY_TARGET], "--c-max-rate", params[constants.LDP_MAX_RATE], "--c-min-rate", params[constants.LDP_MIN_RATE], ]) else: args.extend(["-r", "%d" % params[constants.LDP_RESYNC_RATE]]) args.append("--create-device") return args def GenPauseSyncCmd(self, minor): return ["drbdsetup", self._DevPath(minor), "pause-sync"] def GenResumeSyncCmd(self, minor): return ["drbdsetup", self._DevPath(minor), "resume-sync"] def GenPrimaryCmd(self, minor, force): cmd = ["drbdsetup", self._DevPath(minor), "primary"] if force: cmd.append("-o") return cmd def GenSecondaryCmd(self, minor): return ["drbdsetup", self._DevPath(minor), "secondary"] def GenDetachCmd(self, minor): return ["drbdsetup", self._DevPath(minor), "detach"] def GenDisconnectCmd(self, minor, family, lhost, lport, rhost, rport): return ["drbdsetup", self._DevPath(minor), "disconnect"] def GenDownCmd(self, minor): return ["drbdsetup", self._DevPath(minor), "down"] def GenResizeCmd(self, minor, size_mb): return ["drbdsetup", self._DevPath(minor), "resize", "-s", "%dm" % size_mb] @classmethod def _ComputeDiskBarrierArgs(cls, vmaj, vmin, vrel, disabled_barriers, disable_meta_flush): """Compute the DRBD command line parameters for disk barriers Returns a list of the disk barrier parameters as requested via the disabled_barriers and disable_meta_flush arguments, and according to the supported ones in the DRBD version vmaj.vmin.vrel If the desired option is unsupported, raises errors.BlockDeviceError. """ disabled_barriers_set = frozenset(disabled_barriers) if not disabled_barriers_set in constants.DRBD_VALID_BARRIER_OPT: raise errors.BlockDeviceError("%s is not a valid option set for DRBD" " barriers" % disabled_barriers) args = [] # The following code assumes DRBD 8.x, with x < 4 and x != 1 (DRBD 8.1.x # does not exist) if not vmaj == 8 and vmin in (0, 2, 3): raise errors.BlockDeviceError("Unsupported DRBD version: %d.%d.%d" % (vmaj, vmin, vrel)) def _AppendOrRaise(option, min_version): """Helper for DRBD options""" if min_version is not None and vrel >= min_version: args.append(option) else: raise errors.BlockDeviceError("Could not use the option %s as the" " DRBD version %d.%d.%d does not support" " it." % (option, vmaj, vmin, vrel)) # the minimum version for each feature is encoded via pairs of (minor # version -> x) where x is version in which support for the option was # introduced. meta_flush_supported = disk_flush_supported = { 0: 12, 2: 7, 3: 0, } disk_drain_supported = { 2: 7, 3: 0, } disk_barriers_supported = { 3: 0, } # meta flushes if disable_meta_flush: _AppendOrRaise(cls._DISABLE_META_FLUSH_OPTION, meta_flush_supported.get(vmin, None)) # disk flushes if constants.DRBD_B_DISK_FLUSH in disabled_barriers_set: _AppendOrRaise(cls._DISABLE_FLUSH_OPTION, disk_flush_supported.get(vmin, None)) # disk drain if constants.DRBD_B_DISK_DRAIN in disabled_barriers_set: _AppendOrRaise(cls._DISABLE_DRAIN_OPTION, disk_drain_supported.get(vmin, None)) # disk barriers if constants.DRBD_B_DISK_BARRIERS in disabled_barriers_set: _AppendOrRaise(cls._DISABLE_DISK_OPTION, disk_barriers_supported.get(vmin, None)) return args class DRBD84CmdGenerator(BaseDRBDCmdGenerator): """Generates drbdsetup commands suited for the DRBD >= 8.4 syntax. """ # command line options for barriers _DISABLE_DISK_OPTION = "--disk-barrier=no" _DISABLE_DRAIN_OPTION = "--disk-drain=no" _DISABLE_FLUSH_OPTION = "--disk-flushes=no" _DISABLE_META_FLUSH_OPTION = "--md-flushes=no" def __init__(self, version): super(DRBD84CmdGenerator, self).__init__(version) def GenShowCmd(self, minor): return ["drbdsetup", "show", minor] def GenInitMetaCmd(self, minor, meta_dev): return ["drbdmeta", "--force", self._DevPath(minor), "v08", meta_dev, "flex-external", "create-md"] def GenLocalInitCmds(self, minor, data_dev, meta_dev, size_mb, params): cmds = [] cmds.append(["drbdsetup", "new-resource", self._GetResource(minor)]) cmds.append(["drbdsetup", "new-minor", self._GetResource(minor), str(minor), "0"]) # We need to apply the activity log before attaching the disk else drbdsetup # will fail. cmds.append(["drbdmeta", self._DevPath(minor), "v08", meta_dev, "flex-external", "apply-al"]) attach_cmd = ["drbdsetup", "attach", minor, data_dev, meta_dev, "flexible", "--on-io-error=detach"] if size_mb: attach_cmd.extend(["--size", "%sm" % size_mb]) barrier_args = \ self._ComputeDiskBarrierArgs(params[constants.LDP_BARRIERS], params[constants.LDP_NO_META_FLUSH]) attach_cmd.extend(barrier_args) if params[constants.LDP_DISK_CUSTOM]: attach_cmd.extend(shlex.split(params[constants.LDP_DISK_CUSTOM])) cmds.append(attach_cmd) return cmds def GenNetInitCmds(self, minor, family, lhost, lport, rhost, rport, protocol, dual_pri, hmac, secret, params): cmds = [] cmds.append(["drbdsetup", "new-resource", self._GetResource(minor)]) cmds.append(["drbdsetup", "new-minor", self._GetResource(minor), str(minor), "0"]) args = ["drbdsetup", "connect", self._GetResource(minor), "%s:%s:%s" % (family, lhost, lport), "%s:%s:%s" % (family, rhost, rport), "--protocol", protocol, "--after-sb-0pri", "discard-zero-changes", "--after-sb-1pri", "consensus" ] if dual_pri: args.append("--allow-two-primaries") if hmac and secret: args.extend(["--cram-hmac-alg", hmac, "--shared-secret", secret]) if params[constants.LDP_NET_CUSTOM]: args.extend(shlex.split(params[constants.LDP_NET_CUSTOM])) cmds.append(args) return cmds def GenSyncParamsCmd(self, minor, params): args = ["drbdsetup", "disk-options", minor] if params[constants.LDP_DYNAMIC_RESYNC]: if params[constants.LDP_PLAN_AHEAD] == 0: msg = ("A value of 0 for c-plan-ahead disables the dynamic sync speed" " controller at DRBD level. If you want to disable it, please" " set the dynamic-resync disk parameter to False.") logging.error(msg) return [msg] # add the c-* parameters to args args.extend(["--c-plan-ahead", params[constants.LDP_PLAN_AHEAD], "--c-fill-target", params[constants.LDP_FILL_TARGET], "--c-delay-target", params[constants.LDP_DELAY_TARGET], "--c-max-rate", params[constants.LDP_MAX_RATE], "--c-min-rate", params[constants.LDP_MIN_RATE], ]) else: args.extend(["--resync-rate", "%d" % params[constants.LDP_RESYNC_RATE]]) return args def GenPauseSyncCmd(self, minor): return ["drbdsetup", "pause-sync", minor] def GenResumeSyncCmd(self, minor): return ["drbdsetup", "resume-sync", minor] def GenPrimaryCmd(self, minor, force): cmd = ["drbdsetup", "primary", minor] if force: cmd.append("--force") return cmd def GenSecondaryCmd(self, minor): return ["drbdsetup", "secondary", minor] def GenDetachCmd(self, minor): return ["drbdsetup", "detach", minor] def GenDisconnectCmd(self, minor, family, lhost, lport, rhost, rport): return ["drbdsetup", "disconnect", "%s:%s:%s" % (family, lhost, lport), "%s:%s:%s" % (family, rhost, rport)] def GenDownCmd(self, minor): return ["drbdsetup", "down", self._GetResource(minor)] def GenResizeCmd(self, minor, size_mb): return ["drbdsetup", "resize", minor, "--size", "%dm" % size_mb] @staticmethod def _GetResource(minor): """Return the resource name for a given minor. Currently we don't support DRBD volumes which share a resource, so we generate the resource name based on the minor the resulting volumes is assigned to. """ return "resource%d" % minor @classmethod def _ComputeDiskBarrierArgs(cls, disabled_barriers, disable_meta_flush): """Compute the DRBD command line parameters for disk barriers """ disabled_barriers_set = frozenset(disabled_barriers) if not disabled_barriers_set in constants.DRBD_VALID_BARRIER_OPT: raise errors.BlockDeviceError("%s is not a valid option set for DRBD" " barriers" % disabled_barriers) args = [] # meta flushes if disable_meta_flush: args.append(cls._DISABLE_META_FLUSH_OPTION) # disk flushes if constants.DRBD_B_DISK_FLUSH in disabled_barriers_set: args.append(cls._DISABLE_FLUSH_OPTION) # disk drain if constants.DRBD_B_DISK_DRAIN in disabled_barriers_set: args.append(cls._DISABLE_DRAIN_OPTION) # disk barriers if constants.DRBD_B_DISK_BARRIERS in disabled_barriers_set: args.append(cls._DISABLE_DISK_OPTION) return args ganeti-3.1.0~rc2/lib/storage/drbd_info.py000064400000000000000000000363771476477700300203360ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """DRBD information parsing utilities""" import errno import re import pyparsing as pyp from ganeti import constants from ganeti import utils from ganeti import errors from ganeti import compat from ganeti.storage import base class DRBD8Status(object): # pylint: disable=R0902 """A DRBD status representation class. Note that this class is meant to be used to parse one of the entries returned from L{DRBD8Info._JoinLinesPerMinor}. """ UNCONF_RE = re.compile(r"\s*[0-9]+:\s*cs:Unconfigured$") LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+(?:st|ro):([^/]+)/(\S+)" r"\s+ds:([^/]+)/(\S+)\s+.*$") SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*" # Due to a bug in drbd in the kernel, introduced in # commit 4b0715f096 (still unfixed as of 2011-08-22) r"(?:\s|M)" r"finish: ([0-9]+):([0-9]+):([0-9]+)\s.*$") CS_UNCONFIGURED = "Unconfigured" CS_STANDALONE = "StandAlone" CS_WFCONNECTION = "WFConnection" CS_WFREPORTPARAMS = "WFReportParams" CS_CONNECTED = "Connected" CS_STARTINGSYNCS = "StartingSyncS" CS_STARTINGSYNCT = "StartingSyncT" CS_WFBITMAPS = "WFBitMapS" CS_WFBITMAPT = "WFBitMapT" CS_WFSYNCUUID = "WFSyncUUID" CS_SYNCSOURCE = "SyncSource" CS_SYNCTARGET = "SyncTarget" CS_PAUSEDSYNCS = "PausedSyncS" CS_PAUSEDSYNCT = "PausedSyncT" CSET_SYNC = compat.UniqueFrozenset([ CS_WFREPORTPARAMS, CS_STARTINGSYNCS, CS_STARTINGSYNCT, CS_WFBITMAPS, CS_WFBITMAPT, CS_WFSYNCUUID, CS_SYNCSOURCE, CS_SYNCTARGET, CS_PAUSEDSYNCS, CS_PAUSEDSYNCT, ]) DS_DISKLESS = "Diskless" DS_ATTACHING = "Attaching" # transient state DS_FAILED = "Failed" # transient state, next: diskless DS_NEGOTIATING = "Negotiating" # transient state DS_INCONSISTENT = "Inconsistent" # while syncing or after creation DS_OUTDATED = "Outdated" DS_DUNKNOWN = "DUnknown" # shown for peer disk when not connected DS_CONSISTENT = "Consistent" DS_UPTODATE = "UpToDate" # normal state RO_PRIMARY = "Primary" RO_SECONDARY = "Secondary" RO_UNKNOWN = "Unknown" def __init__(self, procline): u = self.UNCONF_RE.match(procline) if u: self.cstatus = self.CS_UNCONFIGURED self.lrole = self.rrole = self.ldisk = self.rdisk = None else: m = self.LINE_RE.match(procline) if not m: raise errors.BlockDeviceError("Can't parse input data '%s'" % procline) self.cstatus = m.group(1) self.lrole = m.group(2) self.rrole = m.group(3) self.ldisk = m.group(4) self.rdisk = m.group(5) # end reading of data from the LINE_RE or UNCONF_RE self.is_standalone = self.cstatus == self.CS_STANDALONE self.is_wfconn = self.cstatus == self.CS_WFCONNECTION self.is_connected = self.cstatus == self.CS_CONNECTED self.is_unconfigured = self.cstatus == self.CS_UNCONFIGURED self.is_primary = self.lrole == self.RO_PRIMARY self.is_secondary = self.lrole == self.RO_SECONDARY self.peer_primary = self.rrole == self.RO_PRIMARY self.peer_secondary = self.rrole == self.RO_SECONDARY self.both_primary = self.is_primary and self.peer_primary self.both_secondary = self.is_secondary and self.peer_secondary self.is_diskless = self.ldisk == self.DS_DISKLESS self.is_disk_uptodate = self.ldisk == self.DS_UPTODATE self.peer_disk_uptodate = self.rdisk == self.DS_UPTODATE self.is_in_resync = self.cstatus in self.CSET_SYNC self.is_in_use = self.cstatus != self.CS_UNCONFIGURED m = self.SYNC_RE.match(procline) if m: self.sync_percent = float(m.group(1)) hours = int(m.group(2)) minutes = int(m.group(3)) seconds = int(m.group(4)) self.est_time = hours * 3600 + minutes * 60 + seconds else: # we have (in this if branch) no percent information, but if # we're resyncing we need to 'fake' a sync percent information, # as this is how cmdlib determines if it makes sense to wait for # resyncing or not if self.is_in_resync: self.sync_percent = 0 else: self.sync_percent = None self.est_time = None def __repr__(self): return ("<%s: cstatus=%s, lrole=%s, rrole=%s, ldisk=%s, rdisk=%s>" % (self.__class__, self.cstatus, self.lrole, self.rrole, self.ldisk, self.rdisk)) class DRBD8Info(object): """Represents information DRBD exports (usually via /proc/drbd). An instance of this class is created by one of the CreateFrom... methods. """ _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)(?:([.-])(\d+))?" r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)") _VALID_LINE_RE = re.compile("^ *([0-9]+): cs:([^ ]+).*$") def __init__(self, lines): self._version = self._ParseVersion(lines) self._minors, self._line_per_minor = self._JoinLinesPerMinor(lines) def GetVersion(self): """Return the DRBD version. This will return a dict with keys: - k_major - k_minor - k_point - k_fix (only on some drbd versions) - k_fix_separator (only when k_fix is present) - api - proto - proto2 (only on drbd > 8.2.X) """ return self._version def GetVersionString(self): """Return the DRBD version as a single string. """ version = self.GetVersion() retval = "%d.%d.%d" % \ (version["k_major"], version["k_minor"], version["k_point"]) if ("k_fix_separator" in version) and ("k_fix" in version): retval += "%s%s" % (version["k_fix_separator"], version["k_fix"]) retval += " (api:%d/proto:%d" % (version["api"], version["proto"]) if "proto2" in version: retval += "-%s" % version["proto2"] retval += ")" return retval def GetMinors(self): """Return a list of minor for which information is available. This list is ordered in exactly the order which was found in the underlying data. """ return self._minors def HasMinorStatus(self, minor): return minor in self._line_per_minor def GetMinorStatus(self, minor): return DRBD8Status(self._line_per_minor[minor]) def _ParseVersion(self, lines): first_line = lines[0].strip() version = self._VERSION_RE.match(first_line) if not version: raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" % first_line) values = version.groups() retval = { "k_major": int(values[0]), "k_minor": int(values[1]), "k_point": int(values[2]), "api": int(values[5]), "proto": int(values[6]), } if (values[3] is not None) and (values[4] is not None): retval["k_fix_separator"] = values[3] retval["k_fix"] = values[4] if values[7] is not None: retval["proto2"] = values[7] return retval def _JoinLinesPerMinor(self, lines): """Transform the raw lines into a dictionary based on the minor. @return: a dictionary of minor: joined lines from /proc/drbd for that minor """ minors = [] results = {} old_minor = old_line = None for line in lines: if not line: # completely empty lines, as can be returned by drbd8.0+ continue lresult = self._VALID_LINE_RE.match(line) if lresult is not None: if old_minor is not None: minors.append(old_minor) results[old_minor] = old_line old_minor = int(lresult.group(1)) old_line = line else: if old_minor is not None: old_line += " " + line.strip() # add last line if old_minor is not None: minors.append(old_minor) results[old_minor] = old_line return minors, results @staticmethod def CreateFromLines(lines): return DRBD8Info(lines) @staticmethod def CreateFromFile(filename=constants.DRBD_STATUS_FILE): try: lines = utils.ReadFile(filename).splitlines() except EnvironmentError as err: if err.errno == errno.ENOENT: base.ThrowError("The file %s cannot be opened, check if the module" " is loaded (%s)", filename, str(err)) else: base.ThrowError("Can't read the DRBD proc file %s: %s", filename, str(err)) if not lines: base.ThrowError("Can't read any data from %s", filename) return DRBD8Info.CreateFromLines(lines) class BaseShowInfo(object): """Base class for parsing the `drbdsetup show` output. Holds various common pyparsing expressions which are used by subclasses. Also provides caching of the constructed parser. """ _PARSE_SHOW = None # pyparsing setup _lbrace = pyp.Literal("{").suppress() _rbrace = pyp.Literal("}").suppress() _lbracket = pyp.Literal("[").suppress() _rbracket = pyp.Literal("]").suppress() _semi = pyp.Literal(";").suppress() _colon = pyp.Literal(":").suppress() # this also converts the value to an int _number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0])) _comment = pyp.Literal("#") + pyp.Optional(pyp.restOfLine) _defa = pyp.Literal("_is_default").suppress() _dbl_quote = pyp.Literal('"').suppress() _keyword = pyp.Word(pyp.alphanums + "-") # value types _value = pyp.Word(pyp.alphanums + "_-/.:") _quoted = _dbl_quote + pyp.CharsNotIn('"') + _dbl_quote _ipv4_addr = (pyp.Optional(pyp.Literal("ipv4")).suppress() + pyp.Word(pyp.nums + ".") + _colon + _number) _ipv6_addr = (pyp.Optional(pyp.Literal("ipv6")).suppress() + pyp.Optional(_lbracket) + pyp.Word(pyp.hexnums + ":") + pyp.Optional(_rbracket) + _colon + _number) # meta device, extended syntax _meta_value = ((_value ^ _quoted) + _lbracket + _number + _rbracket) # device name, extended syntax _device_value = pyp.Literal("minor").suppress() + _number # a statement _stmt = (~_rbrace + _keyword + ~_lbrace + pyp.Optional(_ipv4_addr ^ _ipv6_addr ^ _value ^ _quoted ^ _meta_value ^ _device_value) + pyp.Optional(_defa) + _semi + pyp.Optional(pyp.restOfLine).suppress()) @classmethod def GetDevInfo(cls, show_data): """Parse details about a given DRBD minor. This returns, if available, the local backing device (as a path) and the local and remote (ip, port) information from a string containing the output of the `drbdsetup show` command as returned by DRBD8Dev._GetShowData. This will return a dict with keys: - local_dev - meta_dev - meta_index - local_addr - remote_addr """ if not show_data: return {} try: # run pyparse results = (cls._GetShowParser()).parseString(show_data) except pyp.ParseException as err: base.ThrowError("Can't parse drbdsetup show output: %s", str(err)) return cls._TransformParseResult(results) @classmethod def _TransformParseResult(cls, parse_result): raise NotImplementedError @classmethod def _GetShowParser(cls): """Return a parser for `drbd show` output. This will either create or return an already-created parser for the output of the command `drbd show`. """ if cls._PARSE_SHOW is None: cls._PARSE_SHOW = cls._ConstructShowParser() return cls._PARSE_SHOW @classmethod def _ConstructShowParser(cls): raise NotImplementedError class DRBD83ShowInfo(BaseShowInfo): @classmethod def _ConstructShowParser(cls): # an entire section section_name = pyp.Word(pyp.alphas + "_") section = section_name + \ cls._lbrace + \ pyp.ZeroOrMore(pyp.Group(cls._stmt)) + \ cls._rbrace bnf = pyp.ZeroOrMore(pyp.Group(section ^ cls._stmt)) bnf.ignore(cls._comment) return bnf @classmethod def _TransformParseResult(cls, parse_result): retval = {} for section in parse_result: sname = section[0] if sname == "_this_host": for lst in section[1:]: if lst[0] == "disk": retval["local_dev"] = lst[1] elif lst[0] == "meta-disk": retval["meta_dev"] = lst[1] retval["meta_index"] = lst[2] elif lst[0] == "address": retval["local_addr"] = tuple(lst[1:]) elif sname == "_remote_host": for lst in section[1:]: if lst[0] == "address": retval["remote_addr"] = tuple(lst[1:]) return retval class DRBD84ShowInfo(BaseShowInfo): @classmethod def _ConstructShowParser(cls): # an entire section (sections can be nested in DRBD 8.4, and there exist # sections like "volume 0") section_name = pyp.Word(pyp.alphas + "_") + \ pyp.Optional(pyp.Word(pyp.nums)).suppress() # skip volume idx section = pyp.Forward() # pylint: disable=W0106 section << (section_name + cls._lbrace + pyp.ZeroOrMore(pyp.Group(cls._stmt ^ section)) + cls._rbrace) resource_name = pyp.Word(pyp.alphanums + "_-.") resource = (pyp.Literal("resource") + resource_name).suppress() + \ cls._lbrace + \ pyp.ZeroOrMore(pyp.Group(section)) + \ cls._rbrace resource.ignore(cls._comment) return resource @classmethod def _TransformVolumeSection(cls, vol_content, retval): for entry in vol_content: if entry[0] == "disk" and len(entry) == 2 and \ isinstance(entry[1], str): retval["local_dev"] = entry[1] elif entry[0] == "meta-disk": if len(entry) > 1: retval["meta_dev"] = entry[1] if len(entry) > 2: retval["meta_index"] = entry[2] @classmethod def _TransformParseResult(cls, parse_result): retval = {} for section in parse_result: sname = section[0] if sname == "_this_host": for lst in section[1:]: if lst[0] == "address": retval["local_addr"] = tuple(lst[1:]) elif lst[0] == "volume": cls._TransformVolumeSection(lst[1:], retval) elif sname == "_remote_host": for lst in section[1:]: if lst[0] == "address": retval["remote_addr"] = tuple(lst[1:]) return retval ganeti-3.1.0~rc2/lib/storage/extstorage.py000064400000000000000000000477331476477700300205730ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ExtStorage Interface related functionality """ import re import stat import os import logging from ganeti import utils from ganeti import errors from ganeti import constants from ganeti import objects from ganeti import pathutils from ganeti.storage import base class ExtStorageDevice(base.BlockDev): """A block device provided by an ExtStorage Provider. This class implements the External Storage Interface, which means handling of the externally provided block devices. """ def __init__(self, unique_id, children, size, params, dyn_params, **kwargs): """Attaches to an extstorage block device. """ super(ExtStorageDevice, self).__init__(unique_id, children, size, params, dyn_params, **kwargs) self.name = kwargs["name"] self.uuid = kwargs["uuid"] if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise ValueError("Invalid configuration data %s" % str(unique_id)) self.driver, self.vol_name = unique_id self.ext_params = params self.major = self.minor = None self.uris = [] self.Attach() @classmethod def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, **kwargs): """Create a new extstorage device. Provision a new volume using an extstorage provider, which will then be mapped to a block device. """ if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise errors.ProgrammerError("Invalid configuration data %s" % str(unique_id)) if excl_stor: raise errors.ProgrammerError("extstorage device requested with" " exclusive_storage") # Call the External Storage's create script, # to provision a new Volume inside the External Storage _ExtStorageAction(constants.ES_ACTION_CREATE, unique_id, params, size=size, name=kwargs["name"], uuid=kwargs["uuid"]) return ExtStorageDevice(unique_id, children, size, params, dyn_params, **kwargs) def Remove(self): """Remove the extstorage device. """ if not self.minor and not self.Attach(): # The extstorage device doesn't exist. return # First shutdown the device (remove mappings). self.Shutdown() # Call the External Storage's remove script, # to remove the Volume from the External Storage _ExtStorageAction(constants.ES_ACTION_REMOVE, self.unique_id, self.ext_params, name=self.name, uuid=self.uuid) def Rename(self, new_id): """Rename this device. """ pass def Attach(self, **kwargs): """Attach to an existing extstorage device. This method maps the extstorage volume that matches our name with a corresponding block device and then attaches to this device. """ self.attached = False # Call the External Storage's attach script, # to attach an existing Volume to a block device under /dev result = _ExtStorageAction(constants.ES_ACTION_ATTACH, self.unique_id, self.ext_params, name=self.name, uuid=self.uuid) # Attach script returns the block device path and optionally # the URIs to be used for userspace access (one URI for # each hypervisor supported). # If the provider doesn't support userspace access, then # the 'uris' variable will be an empty list. result = result.split("\n") self.dev_path = result[0] self.uris = result[1:] if not self.dev_path: logging.info("A local block device is not available") self.dev_path = None if not self.uris: logging.error("Neither a block device nor a userspace URI is available") return False self.attached = True return True # Verify that dev_path exists and is a block device try: st = os.stat(self.dev_path) except OSError as err: logging.error("Error stat()'ing %s: %s", self.dev_path, str(err)) return False if not stat.S_ISBLK(st.st_mode): logging.error("%s is not a block device", self.dev_path) return False self.major = os.major(st.st_rdev) self.minor = utils.osminor(st.st_rdev) self.attached = True return True def Assemble(self): """Assemble the device. """ pass def Shutdown(self): """Shutdown the device. """ if not self.minor and not self.Attach(): # The extstorage device doesn't exist. return # Call the External Storage's detach script, # to detach an existing Volume from it's block device under /dev _ExtStorageAction(constants.ES_ACTION_DETACH, self.unique_id, self.ext_params, name=self.name, uuid=self.uuid) self.minor = None self.dev_path = None def Open(self, force=False, exclusive=True): """Make the device ready for I/O. """ _ExtStorageAction(constants.ES_ACTION_OPEN, self.unique_id, self.ext_params, name=self.name, uuid=self.uuid, exclusive=exclusive) def Close(self): """Notifies that the device will no longer be used for I/O. """ _ExtStorageAction(constants.ES_ACTION_CLOSE, self.unique_id, self.ext_params, name=self.name, uuid=self.uuid) def Grow(self, amount, dryrun, backingstore, excl_stor): """Grow the Volume. @type amount: integer @param amount: the amount (in mebibytes) to grow with @type dryrun: boolean @param dryrun: whether to execute the operation in simulation mode only, without actually increasing the size """ if not backingstore: return if not self.Attach(): base.ThrowError("Can't attach to extstorage device during Grow()") if dryrun: # we do not support dry runs of resize operations for now. return new_size = self.size + amount # Call the External Storage's grow script, # to grow an existing Volume inside the External Storage _ExtStorageAction(constants.ES_ACTION_GROW, self.unique_id, self.ext_params, size=self.size, grow=new_size, name=self.name, uuid=self.uuid) def SetInfo(self, text): """Update metadata with info text. """ # Replace invalid characters text = re.sub("^[^A-Za-z0-9_+.]", "_", text) text = re.sub("[^-A-Za-z0-9_+.]", "_", text) # Only up to 128 characters are allowed text = text[:128] # Call the External Storage's setinfo script, # to set metadata for an existing Volume inside the External Storage _ExtStorageAction(constants.ES_ACTION_SETINFO, self.unique_id, self.ext_params, metadata=text, name=self.name, uuid=self.uuid) def GetUserspaceAccessUri(self, hypervisor): """Generate KVM userspace URIs to be used as `-drive file` settings. @see: L{base.BlockDev.GetUserspaceAccessUri} """ if not self.Attach(): base.ThrowError("Can't attach to ExtStorage device") # If the provider supports userspace access, the attach script has # returned a list of URIs prefixed with the corresponding hypervisor. prefix = hypervisor.lower() + ":" for uri in self.uris: if uri[:len(prefix)].lower() == prefix: return uri[len(prefix):] base.ThrowError("Userspace access is not supported by the '%s'" " ExtStorage provider for the '%s' hypervisor" % (self.driver, hypervisor)) def Snapshot(self, snap_name=None, snap_size=None): """Take a snapshot of the block device. """ provider, vol_name = self.unique_id if not snap_name: snap_name = vol_name + ".snap" if not snap_size: snap_size = self.size _ExtStorageAction(constants.ES_ACTION_SNAPSHOT, self.unique_id, self.ext_params, snap_name=snap_name, snap_size=snap_size) return (provider, snap_name) def _ExtStorageAction(action, unique_id, ext_params, size=None, grow=None, metadata=None, name=None, uuid=None, snap_name=None, snap_size=None, exclusive=None): """Take an External Storage action. Take an External Storage action concerning or affecting a specific Volume inside the External Storage. @type action: string @param action: which action to perform. One of: create / remove / grow / attach / detach / snapshot @type unique_id: tuple (driver, vol_name) @param unique_id: a tuple containing the type of ExtStorage (driver) and the Volume name @type ext_params: dict @param ext_params: ExtStorage parameters @type size: integer @param size: the size of the Volume in mebibytes @type grow: integer @param grow: the new size in mebibytes (after grow) @type metadata: string @param metadata: metadata info of the Volume, for use by the provider @type name: string @param name: name of the Volume (objects.Disk.name) @type uuid: string @type snap_size: integer @param snap_size: the size of the snapshot @type snap_name: string @param snap_name: the name of the snapshot @type exclusive: boolean @param exclusive: Whether the Volume will be opened exclusively or not @param uuid: uuid of the Volume (objects.Disk.uuid) @rtype: None or a block device path (during attach) """ driver, vol_name = unique_id # Create an External Storage instance of type `driver' status, inst_es = ExtStorageFromDisk(driver) if not status: base.ThrowError("%s" % inst_es) # Create the basic environment for the driver's scripts create_env = _ExtStorageEnvironment(unique_id, ext_params, size, grow, metadata, name, uuid, snap_name, snap_size, exclusive) # Do not use log file for action `attach' as we need # to get the output from RunResult # TODO: find a way to have a log file for attach too logfile = None if action is not constants.ES_ACTION_ATTACH: logfile = _VolumeLogName(action, driver, vol_name) # Make sure the given action results in a valid script if action not in constants.ES_SCRIPTS: base.ThrowError("Action '%s' doesn't result in a valid ExtStorage script" % action) # Find out which external script to run according the given action script_name = action + "_script" script = getattr(inst_es, script_name) # Here script is either a valid file path or None if the script is optional if not script: logging.info("Optional action '%s' is not supported by provider '%s'," " skipping", action, driver) return # Run the external script # pylint: disable=E1103 result = utils.RunCmd([script], env=create_env, cwd=inst_es.path, output=logfile,) if result.failed: logging.error("External storage's %s command '%s' returned" " error: %s, logfile: %s, output: %s", action, result.cmd, result.fail_reason, logfile, result.output) # If logfile is 'None' (during attach), it breaks TailFile # TODO: have a log file for attach too if action is not constants.ES_ACTION_ATTACH: lines = [utils.SafeEncode(val) for val in utils.TailFile(logfile, lines=20)] else: lines = result.output.splitlines()[-20:] base.ThrowError("External storage's %s script failed (%s), last" " lines of output:\n%s", action, result.fail_reason, "\n".join(lines)) if action == constants.ES_ACTION_ATTACH: return result.stdout def _CheckExtStorageFile(base_dir, filename, required): """Check prereqs for an ExtStorage file. Check if file exists, if it is a regular file and in case it is one of extstorage scripts if it is executable. @type base_dir: string @param base_dir: Base directory containing ExtStorage installations. @type filename: string @param filename: The basename of the ExtStorage file. @type required: bool @param required: Whether the file is required or not. @rtype: String @return: The file path if the file is found and is valid, None if the file is not found and not required. @raises BlockDeviceError: In case prereqs are not met (found and not valid/executable, not found and required) """ file_path = utils.PathJoin(base_dir, filename) try: st = os.stat(file_path) except EnvironmentError as err: if not required: logging.info("Optional file '%s' under path '%s' is missing", filename, base_dir) return None base.ThrowError("File '%s' under path '%s' is missing (%s)" % (filename, base_dir, utils.ErrnoOrStr(err))) if not stat.S_ISREG(stat.S_IFMT(st.st_mode)): base.ThrowError("File '%s' under path '%s' is not a regular file" % (filename, base_dir)) if filename in constants.ES_SCRIPTS: if stat.S_IMODE(st.st_mode) & stat.S_IXUSR != stat.S_IXUSR: base.ThrowError("File '%s' under path '%s' is not executable" % (filename, base_dir)) return file_path def ExtStorageFromDisk(name, base_dir=None): """Create an ExtStorage instance from disk. This function will return an ExtStorage instance if the given name is a valid ExtStorage name. @type base_dir: string @keyword base_dir: Base directory containing ExtStorage installations. Defaults to a search in all the ES_SEARCH_PATH dirs. @rtype: tuple @return: True and the ExtStorage instance if we find a valid one, or False and the diagnose message on error """ if base_dir is None: es_base_dir = pathutils.ES_SEARCH_PATH else: es_base_dir = [base_dir] es_dir = utils.FindFile(name, es_base_dir, os.path.isdir) if es_dir is None: return False, ("Directory for External Storage Provider %s not" " found in search path" % name) # ES Files dictionary: this will be populated later with the absolute path # names for each script; currently we denote for each script if it is # required (True) or optional (False) es_files = dict.fromkeys(constants.ES_SCRIPTS, True) # Let the snapshot, open, and close scripts be optional # for backwards compatibility es_files[constants.ES_SCRIPT_SNAPSHOT] = False es_files[constants.ES_SCRIPT_OPEN] = False es_files[constants.ES_SCRIPT_CLOSE] = False es_files[constants.ES_PARAMETERS_FILE] = True for (filename, required) in es_files.items(): try: # Here we actually fill the dict with the ablsolute path name for each # script or None, depending on the corresponding checks. See the # function's docstrings for more on these checks. es_files[filename] = _CheckExtStorageFile(es_dir, filename, required) except errors.BlockDeviceError as err: return False, str(err) parameters = [] if constants.ES_PARAMETERS_FILE in es_files: parameters_file = es_files[constants.ES_PARAMETERS_FILE] try: parameters = utils.ReadFile(parameters_file).splitlines() except EnvironmentError as err: return False, ("Error while reading the EXT parameters file at %s: %s" % (parameters_file, utils.ErrnoOrStr(err))) parameters = [v.split(None, 1) for v in parameters] es_obj = \ objects.ExtStorage(name=name, path=es_dir, create_script=es_files[constants.ES_SCRIPT_CREATE], remove_script=es_files[constants.ES_SCRIPT_REMOVE], grow_script=es_files[constants.ES_SCRIPT_GROW], attach_script=es_files[constants.ES_SCRIPT_ATTACH], detach_script=es_files[constants.ES_SCRIPT_DETACH], setinfo_script=es_files[constants.ES_SCRIPT_SETINFO], verify_script=es_files[constants.ES_SCRIPT_VERIFY], snapshot_script=es_files[constants.ES_SCRIPT_SNAPSHOT], open_script=es_files[constants.ES_SCRIPT_OPEN], close_script=es_files[constants.ES_SCRIPT_CLOSE], supported_parameters=parameters) return True, es_obj def _ExtStorageEnvironment(unique_id, ext_params, size=None, grow=None, metadata=None, name=None, uuid=None, snap_name=None, snap_size=None, exclusive=None): """Calculate the environment for an External Storage script. @type unique_id: tuple (driver, vol_name) @param unique_id: ExtStorage pool and name of the Volume @type ext_params: dict @param ext_params: the EXT parameters @type size: integer @param size: size of the Volume (in mebibytes) @type grow: integer @param grow: new size of Volume after grow (in mebibytes) @type metadata: string @param metadata: metadata info of the Volume @type name: string @param name: name of the Volume (objects.Disk.name) @type uuid: string @param uuid: uuid of the Volume (objects.Disk.uuid) @type snap_size: integer @param snap_size: the size of the snapshot @type snap_name: string @param snap_name: the name of the snapshot @type exclusive: boolean @param exclusive: Whether the Volume will be opened exclusively or not @rtype: dict @return: dict of environment variables """ vol_name = unique_id[1] result = {} result["VOL_NAME"] = vol_name # EXT params for pname, pvalue in ext_params.items(): result["EXTP_%s" % pname.upper()] = str(pvalue) if size is not None: result["VOL_SIZE"] = str(size) if grow is not None: result["VOL_NEW_SIZE"] = str(grow) if metadata is not None: result["VOL_METADATA"] = metadata if name is not None: result["VOL_CNAME"] = name if uuid is not None: result["VOL_UUID"] = uuid if snap_name is not None: result["VOL_SNAPSHOT_NAME"] = snap_name if snap_size is not None: result["VOL_SNAPSHOT_SIZE"] = str(snap_size) if exclusive is not None: result["VOL_OPEN_EXCLUSIVE"] = str(exclusive) return result def _VolumeLogName(kind, es_name, volume): """Compute the ExtStorage log filename for a given Volume and operation. @type kind: string @param kind: the operation type (e.g. create, remove etc.) @type es_name: string @param es_name: the ExtStorage name @type volume: string @param volume: the name of the Volume inside the External Storage """ # Check if the extstorage log dir is a valid dir if not os.path.isdir(pathutils.LOG_ES_DIR): base.ThrowError("Cannot find log directory: %s", pathutils.LOG_ES_DIR) # TODO: Use tempfile.mkstemp to create unique filename basename = ("%s-%s-%s-%s.log" % (kind, es_name, volume, utils.TimestampForFilename())) return utils.PathJoin(pathutils.LOG_ES_DIR, basename) ganeti-3.1.0~rc2/lib/storage/filestorage.py000064400000000000000000000336731476477700300207100ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Filesystem-based access functions and disk templates. """ import logging import errno import os from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import pathutils from ganeti import utils from ganeti.utils import io from ganeti.storage import base class FileDeviceHelper(object): @classmethod def CreateFile(cls, path, size, create_folders=False, _file_path_acceptance_fn=None): """Create a new file and its file device helper. @param size: the size in MiBs the file should be truncated to. @param create_folders: create the directories for the path if necessary (using L{ganeti.utils.io.Makedirs}) @rtype: FileDeviceHelper @return: The FileDeviceHelper object representing the object. @raise errors.FileStoragePathError: if the file path is disallowed by policy """ if not _file_path_acceptance_fn: _file_path_acceptance_fn = CheckFileStoragePathAcceptance _file_path_acceptance_fn(path) if create_folders: folder = os.path.dirname(path) io.Makedirs(folder) try: fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL) f = os.fdopen(fd, "w") f.truncate(size * 1024 * 1024) f.close() except EnvironmentError as err: base.ThrowError("%s: can't create: %s", path, str(err)) return FileDeviceHelper(path, _file_path_acceptance_fn=_file_path_acceptance_fn) def __init__(self, path, _file_path_acceptance_fn=None): """Create a new file device helper. @raise errors.FileStoragePathError: if the file path is disallowed by policy """ if not _file_path_acceptance_fn: _file_path_acceptance_fn = CheckFileStoragePathAcceptance _file_path_acceptance_fn(path) self.file_path_acceptance_fn = _file_path_acceptance_fn self.path = path def Exists(self, assert_exists=None): """Check for the existence of the given file. @param assert_exists: creates an assertion on the result value: - if true, raise errors.BlockDeviceError if the file doesn't exist - if false, raise errors.BlockDeviceError if the file does exist @rtype: boolean @return: True if the file exists """ exists = os.path.isfile(self.path) if not exists and assert_exists is True: raise base.ThrowError("%s: No such file", self.path) if exists and assert_exists is False: raise base.ThrowError("%s: File exists", self.path) return exists def Remove(self): """Remove the file backing the block device. @rtype: boolean @return: True if the removal was successful """ try: os.remove(self.path) return True except OSError as err: if err.errno != errno.ENOENT: base.ThrowError("%s: can't remove: %s", self.path, err) return False def Size(self): """Return the actual disk size in bytes. @rtype: int @return: The file size in bytes. """ self.Exists(assert_exists=True) try: return os.stat(self.path).st_size except OSError as err: base.ThrowError("%s: can't stat: %s", self.path, err) def Grow(self, amount, dryrun, backingstore, _excl_stor): """Grow the file @param amount: the amount (in mebibytes) to grow by. """ # Check that the file exists self.Exists(assert_exists=True) if amount < 0: base.ThrowError("%s: can't grow by negative amount", self.path) if dryrun: return if not backingstore: return current_size = self.Size() new_size = current_size + amount * 1024 * 1024 try: f = open(self.path, "a+") f.truncate(new_size) f.close() except EnvironmentError as err: base.ThrowError("%s: can't grow: ", self.path, str(err)) def Move(self, new_path): """Move file to a location inside the file storage dir. """ # Check that the file exists self.Exists(assert_exists=True) self.file_path_acceptance_fn(new_path) try: os.rename(self.path, new_path) self.path = new_path except OSError as err: base.ThrowError("%s: can't rename to %s: ", str(err), new_path) class FileStorage(base.BlockDev): """File device. This class represents a file storage backend device. The unique_id for the file device is a (file_driver, file_path) tuple. """ def __init__(self, unique_id, children, size, params, dyn_params, **kwargs): """Initalizes a file device backend. """ if children: raise errors.BlockDeviceError("Invalid setup for file device") super(FileStorage, self).__init__(unique_id, children, size, params, dyn_params, **kwargs) if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise ValueError("Invalid configuration data %s" % str(unique_id)) self.driver = unique_id[0] self.dev_path = unique_id[1] self.file = FileDeviceHelper(self.dev_path) self.Attach() def Assemble(self): """Assemble the device. Checks whether the file device exists, raises BlockDeviceError otherwise. """ self.file.Exists(assert_exists=True) def Shutdown(self): """Shutdown the device. This is a no-op for the file type, as we don't deactivate the file on shutdown. """ pass def Open(self, force=False, exclusive=True): """Make the device ready for I/O. This is a no-op for the file type. """ pass def Close(self): """Notifies that the device will no longer be used for I/O. This is a no-op for the file type. """ pass def Remove(self): """Remove the file backing the block device. @rtype: boolean @return: True if the removal was successful """ return self.file.Remove() def Rename(self, new_id): """Renames the file. """ return self.file.Move(new_id[1]) def Grow(self, amount, dryrun, backingstore, excl_stor): """Grow the file @param amount: the amount (in mebibytes) to grow with """ if not backingstore: return if dryrun: return self.file.Grow(amount, dryrun, backingstore, excl_stor) def Attach(self, **kwargs): """Attach to an existing file. Check if this file already exists. @rtype: boolean @return: True if file exists """ self.attached = self.file.Exists() return self.attached def GetActualSize(self): """Return the actual disk size. @note: the device needs to be active when this is called """ return self.file.Size() @classmethod def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, **kwargs): """Create a new file. @type size: int @param size: the size of file in MiB @rtype: L{bdev.FileStorage} @return: an instance of FileStorage """ if excl_stor: raise errors.ProgrammerError("FileStorage device requested with" " exclusive_storage") if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise ValueError("Invalid configuration data %s" % str(unique_id)) dev_path = unique_id[1] FileDeviceHelper.CreateFile(dev_path, size) return FileStorage(unique_id, children, size, params, dyn_params, **kwargs) def GetFileStorageSpaceInfo(path): """Retrieves the free and total space of the device where the file is located. @type path: string @param path: Path of the file whose embracing device's capacity is reported. @return: a dictionary containing 'vg_size' and 'vg_free' given in MebiBytes """ try: result = os.statvfs(path) free = (result.f_frsize * result.f_bavail) // (1024 * 1024) size = (result.f_frsize * result.f_blocks) // (1024 * 1024) return {"type": constants.ST_FILE, "name": path, "storage_size": size, "storage_free": free} except OSError as e: raise errors.CommandError("Failed to retrieve file system information about" " path: %s - %s" % (path, e.strerror)) def _GetForbiddenFileStoragePaths(): """Builds a list of path prefixes which shouldn't be used for file storage. @rtype: frozenset """ paths = set([ "/boot", "/dev", "/etc", "/home", "/proc", "/root", "/sys", ]) for prefix in ["", "/usr", "/usr/local"]: paths.update( "%s/%s" % (prefix, s) for s in ["bin", "lib", "lib32", "lib64", "sbin"]) return compat.UniqueFrozenset(map(os.path.normpath, paths)) def _ComputeWrongFileStoragePaths(paths, _forbidden=_GetForbiddenFileStoragePaths()): """Cross-checks a list of paths for prefixes considered bad. Some paths, e.g. "/bin", should not be used for file storage. @type paths: list @param paths: List of paths to be checked @rtype: list @return: Sorted list of paths for which the user should be warned """ def _Check(path): return (not os.path.isabs(path) or path in _forbidden or [p for p in _forbidden if utils.IsBelowDir(p, path)]) return utils.NiceSort(filter(_Check, map(os.path.normpath, paths))) def ComputeWrongFileStoragePaths(_filename=pathutils.FILE_STORAGE_PATHS_FILE): """Returns a list of file storage paths whose prefix is considered bad. See L{_ComputeWrongFileStoragePaths}. """ return _ComputeWrongFileStoragePaths(_LoadAllowedFileStoragePaths(_filename)) def _CheckFileStoragePath(path, allowed, exact_match_ok=False): """Checks if a path is in a list of allowed paths for file storage. @type path: string @param path: Path to check @type allowed: list @param allowed: List of allowed paths @type exact_match_ok: bool @param exact_match_ok: whether or not it is okay when the path is exactly equal to an allowed path and not a subdir of it @raise errors.FileStoragePathError: If the path is not allowed """ if not os.path.isabs(path): raise errors.FileStoragePathError("File storage path must be absolute," " got '%s'" % path) for i in allowed: if not os.path.isabs(i): logging.info("Ignoring relative path '%s' for file storage", i) continue if exact_match_ok: if os.path.normpath(i) == os.path.normpath(path): break if utils.IsBelowDir(i, path): break else: raise errors.FileStoragePathError("Path '%s' is not acceptable for file" " storage" % path) def _LoadAllowedFileStoragePaths(filename): """Loads file containing allowed file storage paths. @rtype: list @return: List of allowed paths (can be an empty list) """ try: contents = utils.ReadFile(filename) except EnvironmentError: return [] else: return utils.FilterEmptyLinesAndComments(contents) def CheckFileStoragePathAcceptance( path, _filename=pathutils.FILE_STORAGE_PATHS_FILE, exact_match_ok=False): """Checks if a path is allowed for file storage. @type path: string @param path: Path to check @raise errors.FileStoragePathError: If the path is not allowed """ allowed = _LoadAllowedFileStoragePaths(_filename) if not allowed: raise errors.FileStoragePathError("No paths are valid or path file '%s'" " was not accessible." % _filename) if _ComputeWrongFileStoragePaths([path]): raise errors.FileStoragePathError("Path '%s' uses a forbidden prefix" % path) _CheckFileStoragePath(path, allowed, exact_match_ok=exact_match_ok) def _CheckFileStoragePathExistance(path): """Checks whether the given path is usable on the file system. This checks wether the path is existing, a directory and writable. @type path: string @param path: path to check """ if not os.path.isdir(path): raise errors.FileStoragePathError("Path '%s' does not exist or is not a" " directory." % path) if not os.access(path, os.W_OK): raise errors.FileStoragePathError("Path '%s' is not writable" % path) def CheckFileStoragePath( path, _allowed_paths_file=pathutils.FILE_STORAGE_PATHS_FILE): """Checks whether the path exists and is acceptable to use. Can be used for any file-based storage, for example shared-file storage. @type path: string @param path: path to check @rtype: string @returns: error message if the path is not ready to use """ try: CheckFileStoragePathAcceptance(path, _filename=_allowed_paths_file, exact_match_ok=True) except errors.FileStoragePathError as e: return str(e) if not os.path.isdir(path): return "Path '%s' is not existing or not a directory." % path if not os.access(path, os.W_OK): return "Path '%s' is not writable" % path ganeti-3.1.0~rc2/lib/storage/gluster.py000064400000000000000000000334451476477700300200660ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Gluster storage class. This class is very similar to FileStorage, given that Gluster when mounted behaves essentially like a regular file system. Unlike RBD, there are no special provisions for block device abstractions (yet). """ import logging import os import socket from ganeti import utils from ganeti import errors from ganeti import netutils from ganeti import constants from ganeti import ssconf from ganeti.utils import io from ganeti.storage import base from ganeti.storage.filestorage import FileDeviceHelper class GlusterVolume(object): """This class represents a Gluster volume. Volumes are uniquely identified by: - their IP address - their port - the volume name itself Two GlusterVolume objects x, y with same IP address, port and volume name are considered equal. """ def __init__(self, server_addr, port, volume, _run_cmd=utils.RunCmd, _mount_point=None): """Creates a Gluster volume object. @type server_addr: str @param server_addr: The address to connect to @type port: int @param port: The port to connect to (Gluster standard is 24007) @type volume: str @param volume: The gluster volume to use for storage. """ self.server_addr = server_addr server_ip = netutils.Hostname.GetIP(self.server_addr) self._server_ip = server_ip port = netutils.ValidatePortNumber(port) self._port = port self._volume = volume if _mount_point: # tests self.mount_point = _mount_point else: self.mount_point = ssconf.SimpleStore().GetGlusterStorageDir() self._run_cmd = _run_cmd @property def server_ip(self): return self._server_ip @property def port(self): return self._port @property def volume(self): return self._volume def __eq__(self, other): return (self.server_ip, self.port, self.volume) == \ (other.server_ip, other.port, other.volume) def __repr__(self): return """GlusterVolume("{ip}", {port}, "{volume}")""" \ .format(ip=self.server_ip, port=self.port, volume=self.volume) def __hash__(self): return (self.server_ip, self.port, self.volume).__hash__() def _IsMounted(self): """Checks if we are mounted or not. @rtype: bool @return: True if this volume is mounted. """ if not os.path.exists(self.mount_point): return False return os.path.ismount(self.mount_point) def _GuessMountFailReasons(self): """Try and give reasons why the mount might've failed. @rtype: str @return: A semicolon-separated list of problems found with the current setup suitable for display to the user. """ reasons = [] # Does the mount point exist? if not os.path.exists(self.mount_point): reasons.append("%r: does not exist" % self.mount_point) # Okay, it exists, but is it a directory? elif not os.path.isdir(self.mount_point): reasons.append("%r: not a directory" % self.mount_point) # If, for some unfortunate reason, this folder exists before mounting: # # /var/run/ganeti/gluster/gv0/10.0.0.1:30000:gv0/ # '--------- cwd ------------' # # and you _are_ trying to mount the gluster volume gv0 on 10.0.0.1:30000, # then the mount.glusterfs command parser gets confused and this command: # # mount -t glusterfs 10.0.0.1:30000:gv0 /var/run/ganeti/gluster/gv0 # '-- remote end --' '------ mountpoint -------' # # gets parsed instead like this: # # mount -t glusterfs 10.0.0.1:30000:gv0 /var/run/ganeti/gluster/gv0 # '-- mountpoint --' '----- syntax error ------' # # and if there _is_ a gluster server running locally at the default remote # end, localhost:24007, then this is not a network error and therefore... no # usage message gets printed out. All you get is a Byson parser error in the # gluster log files about an unexpected token in line 1, "". (That's stdin.) # # Not that we rely on that output in any way whatsoever... parser_confusing = io.PathJoin(self.mount_point, self._GetFUSEMountString()) if os.path.exists(parser_confusing): reasons.append("%r: please delete, rename or move." % parser_confusing) # Let's try something else: can we connect to the server? sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.connect((self.server_ip, self.port)) sock.close() except socket.error as err: reasons.append("%s:%d: %s" % (self.server_ip, self.port, err.strerror)) reasons.append("try running 'gluster volume info %s' on %s to ensure" " it exists, it is started and it is using the tcp" " transport" % (self.volume, self.server_ip)) return "; ".join(reasons) def _GetFUSEMountString(self): """Return the string FUSE needs to mount this volume. @rtype: str """ return "-o server-port={port} {ip}:/{volume}" \ .format(port=self.port, ip=self.server_ip, volume=self.volume) def GetKVMMountString(self, path): """Return the string KVM needs to use this volume. @rtype: str """ ip = self.server_ip if netutils.IPAddress.GetAddressFamily(ip) == socket.AF_INET6: ip = "[%s]" % ip return "gluster://{ip}:{port}/{volume}/{path}" \ .format(ip=ip, port=self.port, volume=self.volume, path=path) def Mount(self): """Try and mount the volume. No-op if the volume is already mounted. @raises BlockDeviceError: if the mount was unsuccessful @rtype: context manager @return: A simple context manager that lets you use this volume for short lived operations like so:: with volume.mount(): # Do operations on volume # Volume is now unmounted """ class _GlusterVolumeContextManager(object): def __init__(self, volume): self.volume = volume def __enter__(self): # We're already mounted. return self def __exit__(self, *exception_information): self.volume.Unmount() return False # do not swallow exceptions. if self._IsMounted(): return _GlusterVolumeContextManager(self) command = ["mount", "-t", "glusterfs", self._GetFUSEMountString(), self.mount_point] io.Makedirs(self.mount_point) self._run_cmd(" ".join(command), # Why set cwd? Because it's an area we control. If, # for some unfortunate reason, this folder exists: # "/%s/" % _GetFUSEMountString() # ...then the gluster parser gets confused and treats # _GetFUSEMountString() as your mount point and # self.mount_point becomes a syntax error. cwd=self.mount_point) # mount.glusterfs exits with code 0 even after failure. # https://bugzilla.redhat.com/show_bug.cgi?id=1031973 if not self._IsMounted(): reasons = self._GuessMountFailReasons() if not reasons: reasons = "%r failed." % (" ".join(command)) base.ThrowError("%r: mount failure: %s", self.mount_point, reasons) return _GlusterVolumeContextManager(self) def Unmount(self): """Try and unmount the volume. Failures are logged but otherwise ignored. @raises BlockDeviceError: if the volume was not mounted to begin with. """ if not self._IsMounted(): base.ThrowError("%r: should be mounted but isn't.", self.mount_point) result = self._run_cmd(["umount", self.mount_point]) if result.failed: logging.warning("Failed to unmount %r from %r: %s", self, self.mount_point, result.fail_reason) class GlusterStorage(base.BlockDev): """File device using the Gluster backend. This class represents a file storage backend device stored on Gluster. Ganeti mounts and unmounts the Gluster devices automatically. The unique_id for the file device is a (file_driver, file_path) tuple. """ def __init__(self, unique_id, children, size, params, dyn_params, **kwargs): """Initalizes a file device backend. """ if children: base.ThrowError("Invalid setup for file device") try: self.driver, self.path = unique_id except ValueError: # wrong number of arguments raise ValueError("Invalid configuration data %s" % repr(unique_id)) server_addr = params[constants.GLUSTER_HOST] port = params[constants.GLUSTER_PORT] volume = params[constants.GLUSTER_VOLUME] self.volume = GlusterVolume(server_addr, port, volume) self.full_path = io.PathJoin(self.volume.mount_point, self.path) self.file = None super(GlusterStorage, self).__init__(unique_id, children, size, params, dyn_params, **kwargs) self.Attach() def Assemble(self): """Assemble the device. Checks whether the file device exists, raises BlockDeviceError otherwise. """ assert self.attached, "Gluster file assembled without being attached" self.file.Exists(assert_exists=True) def Shutdown(self): """Shutdown the device. """ self.file = None self.dev_path = None self.attached = False def Open(self, force=False, exclusive=True): """Make the device ready for I/O. This is a no-op for the file type. """ assert self.attached, "Gluster file opened without being attached" def Close(self): """Notifies that the device will no longer be used for I/O. This is a no-op for the file type. """ pass def Remove(self): """Remove the file backing the block device. @rtype: boolean @return: True if the removal was successful """ with self.volume.Mount(): self.file = FileDeviceHelper(self.full_path) if self.file.Remove(): self.file = None return True else: return False def Rename(self, new_id): """Renames the file. """ # TODO: implement rename for file-based storage base.ThrowError("Rename is not supported for Gluster storage") def Grow(self, amount, dryrun, backingstore, excl_stor): """Grow the file @param amount: the amount (in mebibytes) to grow with """ self.file.Grow(amount, dryrun, backingstore, excl_stor) def Attach(self, **kwargs): """Attach to an existing file. Check if this file already exists. @rtype: boolean @return: True if file exists """ try: self.volume.Mount() self.file = FileDeviceHelper(self.full_path) self.dev_path = self.full_path except Exception as err: self.volume.Unmount() raise err self.attached = self.file.Exists() return self.attached def GetActualSize(self): """Return the actual disk size. @note: the device needs to be active when this is called """ return self.file.Size() def GetUserspaceAccessUri(self, hypervisor): """Generate KVM userspace URIs to be used as `-drive file` settings. @see: L{BlockDev.GetUserspaceAccessUri} @see: https://github.com/qemu/qemu/commit/8d6d89cb63c57569864ecdeb84d3a1c2eb """ if hypervisor == constants.HT_KVM: return self.volume.GetKVMMountString(self.path) else: base.ThrowError("Hypervisor %s doesn't support Gluster userspace access" % hypervisor) @classmethod def Create(cls, unique_id, children, size, spindles, params, excl_stor, dyn_params, **kwargs): """Create a new file. @param size: the size of file in MiB @rtype: L{bdev.FileStorage} @return: an instance of FileStorage """ if excl_stor: raise errors.ProgrammerError("FileStorage device requested with" " exclusive_storage") if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2: raise ValueError("Invalid configuration data %s" % str(unique_id)) full_path = unique_id[1] server_addr = params[constants.GLUSTER_HOST] port = params[constants.GLUSTER_PORT] volume = params[constants.GLUSTER_VOLUME] volume_obj = GlusterVolume(server_addr, port, volume) full_path = io.PathJoin(volume_obj.mount_point, full_path) # Possible optimization: defer actual creation to first Attach, rather # than mounting and unmounting here, then remounting immediately after. with volume_obj.Mount(): FileDeviceHelper.CreateFile(full_path, size, create_folders=True) return GlusterStorage(unique_id, children, size, params, dyn_params, **kwargs) ganeti-3.1.0~rc2/lib/tools/000075500000000000000000000000001476477700300155125ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/tools/__init__.py000064400000000000000000000025331476477700300176260ustar00rootroot00000000000000# # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Common tools modules. """ ganeti-3.1.0~rc2/lib/tools/burnin.py000075500000000000000000001337461476477700300174020ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Burnin program """ from __future__ import print_function import sys import optparse import time import socket import urllib.request, urllib.parse, urllib.error import random import string # pylint: disable=W0402 from functools import reduce from itertools import islice, cycle from io import StringIO from operator import or_ from ganeti import opcodes from ganeti import constants from ganeti import cli from ganeti import errors from ganeti import utils from ganeti import hypervisor from ganeti import compat from ganeti import pathutils from ganeti.confd import client as confd_client from ganeti.runtime import (GetClient) USAGE = ("\tburnin -o OS_NAME [options...] instance_name ...") MAX_RETRIES = 3 LOG_HEADERS = { 0: "- ", 1: "* ", 2: "", } #: Disk templates supporting a single node _SINGLE_NODE_DISK_TEMPLATES = compat.UniqueFrozenset([ constants.DT_DISKLESS, constants.DT_PLAIN, constants.DT_FILE, constants.DT_SHARED_FILE, constants.DT_EXT, constants.DT_RBD, constants.DT_GLUSTER ]) _SUPPORTED_DISK_TEMPLATES = compat.UniqueFrozenset([ constants.DT_DISKLESS, constants.DT_DRBD8, constants.DT_EXT, constants.DT_FILE, constants.DT_PLAIN, constants.DT_RBD, constants.DT_SHARED_FILE, constants.DT_GLUSTER ]) #: Disk templates for which import/export is tested _IMPEXP_DISK_TEMPLATES = (_SUPPORTED_DISK_TEMPLATES - frozenset([ constants.DT_DISKLESS, constants.DT_FILE, constants.DT_SHARED_FILE, constants.DT_GLUSTER ])) class InstanceDown(Exception): """The checked instance was not up""" class BurninFailure(Exception): """Failure detected during burning""" def Usage(): """Shows program usage information and exits the program.""" print("Usage:", file=sys.stderr) print(USAGE, file=sys.stderr) sys.exit(2) def Log(msg, *args, **kwargs): """Simple function that prints out its argument. """ if args: msg = msg % args indent = kwargs.get("indent", 0) sys.stdout.write("%*s%s%s\n" % (2 * indent, "", LOG_HEADERS.get(indent, " "), msg)) sys.stdout.flush() def Err(msg, exit_code=1): """Simple error logging that prints to stderr. """ sys.stderr.write(msg + "\n") sys.stderr.flush() sys.exit(exit_code) def RandomString(size=8, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for x in range(size)) class SimpleOpener(urllib.request.FancyURLopener): """A simple url opener""" # pylint: disable=W0221 def prompt_user_passwd(self, host, realm, clear_cache=0): """No-interaction version of prompt_user_passwd.""" # we follow parent class' API # pylint: disable=W0613 return None, None def http_error_default(self, url, fp, errcode, errmsg, headers): """Custom error handling""" # make sure sockets are not left in CLOSE_WAIT, this is similar # but with a different exception to the BasicURLOpener class _ = fp.read() # throw away data fp.close() raise InstanceDown("HTTP error returned: code %s, msg %s" % (errcode, errmsg)) OPTIONS = [ cli.cli_option("-o", "--os", dest="os", default=None, help="OS to use during burnin", metavar="", completion_suggest=cli.OPT_COMPL_ONE_OS), cli.HYPERVISOR_OPT, cli.OSPARAMS_OPT, cli.cli_option("--disk-size", dest="disk_size", help="Disk size (determines disk count)", default="1G", type="string", metavar="", completion_suggest=("512M 1G 4G 1G,256M" " 4G,1G,1G 10G").split()), cli.cli_option("--disk-growth", dest="disk_growth", help="Disk growth", default="128m", type="string", metavar=""), cli.cli_option("--mem-size", dest="mem_size", help="Memory size", default=None, type="unit", metavar="", completion_suggest=("128M 256M 512M 1G 4G 8G" " 12G 16G").split()), cli.cli_option("--maxmem-size", dest="maxmem_size", help="Max Memory size", default=256, type="unit", metavar="", completion_suggest=("128M 256M 512M 1G 4G 8G" " 12G 16G").split()), cli.cli_option("--minmem-size", dest="minmem_size", help="Min Memory size", default=128, type="unit", metavar="", completion_suggest=("128M 256M 512M 1G 4G 8G" " 12G 16G").split()), cli.cli_option("--vcpu-count", dest="vcpu_count", help="VCPU count", default=3, type="unit", metavar="", completion_suggest=("1 2 3 4").split()), cli.DEBUG_OPT, cli.VERBOSE_OPT, cli.IPCHECK_OPT, cli.NAMECHECK_OPT, cli.EARLY_RELEASE_OPT, cli.cli_option("--no-replace1", dest="do_replace1", help="Skip disk replacement with the same secondary", action="store_false", default=True), cli.cli_option("--no-replace2", dest="do_replace2", help="Skip disk replacement with a different secondary", action="store_false", default=True), cli.cli_option("--no-failover", dest="do_failover", help="Skip instance failovers", action="store_false", default=True), cli.cli_option("--no-migrate", dest="do_migrate", help="Skip instance live migration", action="store_false", default=True), cli.cli_option("--no-move", dest="do_move", help="Skip instance moves", action="store_false", default=True), cli.cli_option("--no-importexport", dest="do_importexport", help="Skip instance export/import", action="store_false", default=True), cli.cli_option("--no-startstop", dest="do_startstop", help="Skip instance stop/start", action="store_false", default=True), cli.cli_option("--no-reinstall", dest="do_reinstall", help="Skip instance reinstall", action="store_false", default=True), cli.cli_option("--no-reboot", dest="do_reboot", help="Skip instance reboot", action="store_false", default=True), cli.cli_option("--no-renamesame", dest="do_renamesame", help="Skip instance rename to same name", action="store_false", default=True), cli.cli_option("--reboot-types", dest="reboot_types", help="Specify the reboot types", default=None), cli.cli_option("--no-activate-disks", dest="do_activate_disks", help="Skip disk activation/deactivation", action="store_false", default=True), cli.cli_option("--no-add-disks", dest="do_addremove_disks", help="Skip disk addition/removal", action="store_false", default=True), cli.cli_option("--no-add-nics", dest="do_addremove_nics", help="Skip NIC addition/removal", action="store_false", default=True), cli.cli_option("--no-nics", dest="nics", help="No network interfaces", action="store_const", const=[], default=[{}]), cli.cli_option("--no-confd", dest="do_confd_tests", help="Skip confd queries", action="store_false", default=True), cli.cli_option("--rename", dest="rename", default=None, help=("Give one unused instance name which is taken" " to start the renaming sequence"), metavar=""), cli.cli_option("-t", "--disk-template", dest="disk_template", choices=list(_SUPPORTED_DISK_TEMPLATES), default=constants.DT_DRBD8, help=("Disk template (default %s, otherwise one of %s)" % (constants.DT_DRBD8, utils.CommaJoin(_SUPPORTED_DISK_TEMPLATES)))), cli.cli_option("-n", "--nodes", dest="nodes", default="", help=("Comma separated list of nodes to perform" " the burnin on (defaults to all nodes)"), completion_suggest=cli.OPT_COMPL_MANY_NODES), cli.cli_option("-I", "--iallocator", dest="iallocator", default=None, type="string", help=("Perform the allocation using an iallocator" " instead of fixed node spread (node restrictions no" " longer apply, therefore -n/--nodes must not be" " used"), completion_suggest=cli.OPT_COMPL_ONE_IALLOCATOR), cli.cli_option("-p", "--parallel", default=False, action="store_true", dest="parallel", help=("Enable parallelization of some operations in" " order to speed burnin or to test granular locking")), cli.cli_option("--net-timeout", default=15, type="int", dest="net_timeout", help=("The instance check network timeout in seconds" " (defaults to 15 seconds)"), completion_suggest="15 60 300 900".split()), cli.cli_option("-C", "--http-check", default=False, action="store_true", dest="http_check", help=("Enable checking of instance status via http," " looking for /hostname.txt that should contain the" " name of the instance")), cli.cli_option("-K", "--keep-instances", default=False, action="store_true", dest="keep_instances", help=("Leave instances on the cluster after burnin," " for investigation in case of errors or simply" " to use them")), cli.REASON_OPT, ] # Mainly used for bash completion ARGUMENTS = [cli.ArgInstance(min=1)] def _DoCheckInstances(fn): """Decorator for checking instances. """ def wrapper(self, *args, **kwargs): val = fn(self, *args, **kwargs) for instance in self.instances: self._CheckInstanceAlive(instance) # pylint: disable=W0212 return val return wrapper def _DoBatch(retry): """Decorator for possible batch operations. Must come after the _DoCheckInstances decorator (if any). @param retry: whether this is a retryable batch, will be passed to StartBatch """ def wrap(fn): def batched(self, *args, **kwargs): self.StartBatch(retry) val = fn(self, *args, **kwargs) self.CommitQueue() return val return batched return wrap class FeedbackAccumulator(object): """Feedback accumulator class.""" _feed_buf = StringIO() opts = None def ClearFeedbackBuf(self): """Clear the feedback buffer.""" self._feed_buf.truncate(0) def GetFeedbackBuf(self): """Return the contents of the buffer.""" return self._feed_buf.getvalue() def Feedback(self, msg): """Acumulate feedback in our buffer.""" formatted_msg = "%s %s" % (time.ctime(utils.MergeTime(msg[0])), msg[2]) self._feed_buf.write(formatted_msg + "\n") if self.opts.verbose: Log(formatted_msg, indent=3) class JobHandler(FeedbackAccumulator): """Class for handling Ganeti jobs.""" queued_ops = [] queue_retry = False def __init__(self): self.cl = cli.GetClient() def MaybeRetry(self, retry_count, msg, fn, *args): """Possibly retry a given function execution. @type retry_count: int @param retry_count: retry counter: - 0: non-retryable action - 1: last retry for a retryable action - MAX_RETRIES: original try for a retryable action @type msg: str @param msg: the kind of the operation @type fn: callable @param fn: the function to be called """ try: val = fn(*args) if retry_count > 0 and retry_count < MAX_RETRIES: Log("Idempotent %s succeeded after %d retries", msg, MAX_RETRIES - retry_count) return val except Exception as err: # pylint: disable=W0703 if retry_count == 0: Log("Non-idempotent %s failed, aborting", msg) raise elif retry_count == 1: Log("Idempotent %s repeated failure, aborting", msg) raise else: Log("Idempotent %s failed, retry #%d/%d: %s", msg, MAX_RETRIES - retry_count + 1, MAX_RETRIES, err) self.MaybeRetry(retry_count - 1, msg, fn, *args) def _ExecOp(self, *ops): """Execute one or more opcodes and manage the exec buffer. @return: if only opcode has been passed, we return its result; otherwise we return the list of results """ job_id = cli.SendJob(ops, cl=self.cl) results = cli.PollJob(job_id, cl=self.cl, feedback_fn=self.Feedback) if len(ops) == 1: return results[0] else: return results def ExecOp(self, retry, *ops): """Execute one or more opcodes and manage the exec buffer. @return: if only opcode has been passed, we return its result; otherwise we return the list of results """ if retry: rval = MAX_RETRIES else: rval = 0 cli.SetGenericOpcodeOpts(ops, self.opts) return self.MaybeRetry(rval, "opcode", self._ExecOp, *ops) def ExecOrQueue(self, name, ops, post_process=None): """Execute an opcode and manage the exec buffer.""" if self.opts.parallel: cli.SetGenericOpcodeOpts(ops, self.opts) self.queued_ops.append((ops, name, post_process)) else: val = self.ExecOp(self.queue_retry, *ops) if post_process is not None: post_process() return val def StartBatch(self, retry): """Start a new batch of jobs. @param retry: whether this is a retryable batch """ self.queued_ops = [] self.queue_retry = retry def CommitQueue(self): """Execute all submitted opcodes in case of parallel burnin""" if not self.opts.parallel or not self.queued_ops: return if self.queue_retry: rval = MAX_RETRIES else: rval = 0 try: results = self.MaybeRetry(rval, "jobset", self.ExecJobSet, self.queued_ops) finally: self.queued_ops = [] return results def ExecJobSet(self, jobs): """Execute a set of jobs and return once all are done. The method will return the list of results, if all jobs are successful. Otherwise, OpExecError will be raised from within cli.py. """ self.ClearFeedbackBuf() jex = cli.JobExecutor(cl=self.cl, feedback_fn=self.Feedback) for ops, name, _ in jobs: jex.QueueJob(name, *ops) try: results = jex.GetResults() except Exception as err: # pylint: disable=W0703 Log("Jobs failed: %s", err) raise BurninFailure() fail = False val = [] for (_, name, post_process), (success, result) in zip(jobs, results): if success: if post_process: try: post_process() except Exception as err: # pylint: disable=W0703 Log("Post process call for job %s failed: %s", name, err) fail = True val.append(result) else: fail = True if fail: raise BurninFailure() return val class Burner(JobHandler): """Burner class.""" def __init__(self): """Constructor.""" super(Burner, self).__init__() self.url_opener = SimpleOpener() self.nodes = [] self.instances = [] self.to_rem = [] self.disk_count = self.disk_growth = self.disk_size = None self.hvp = self.bep = None self.ParseOptions() self.disk_nodes = {} self.instance_nodes = {} self.GetState() self.confd_reply = None def ParseOptions(self): """Parses the command line options. In case of command line errors, it will show the usage and exit the program. """ parser = optparse.OptionParser(usage="\n%s" % USAGE, version=("%%prog (ganeti) %s" % constants.RELEASE_VERSION), option_list=OPTIONS) options, args = parser.parse_args() if len(args) < 1 or options.os is None: Usage() if options.mem_size: options.maxmem_size = options.mem_size options.minmem_size = options.mem_size elif options.minmem_size > options.maxmem_size: Err("Maximum memory lower than minimum memory") if options.disk_template not in _SUPPORTED_DISK_TEMPLATES: Err("Unknown or unsupported disk template '%s'" % options.disk_template) if options.disk_template == constants.DT_DISKLESS: disk_size = disk_growth = [] options.do_addremove_disks = False else: disk_size = [utils.ParseUnit(v) for v in options.disk_size.split(",")] disk_growth = [utils.ParseUnit(v) for v in options.disk_growth.split(",")] if len(disk_growth) != len(disk_size): Err("Wrong disk sizes/growth combination") if ((disk_size and options.disk_template == constants.DT_DISKLESS) or (not disk_size and options.disk_template != constants.DT_DISKLESS)): Err("Wrong disk count/disk template combination") self.disk_size = disk_size self.disk_growth = disk_growth self.disk_count = len(disk_size) if options.nodes and options.iallocator: Err("Give either the nodes option or the iallocator option, not both") if options.http_check and not options.name_check: Err("Can't enable HTTP checks without name checks") self.opts = options self.instances = args self.bep = { constants.BE_MINMEM: options.minmem_size, constants.BE_MAXMEM: options.maxmem_size, constants.BE_VCPUS: options.vcpu_count, } self.hypervisor = None self.hvp = {} if options.hypervisor: self.hypervisor, self.hvp = options.hypervisor if options.reboot_types is None: options.reboot_types = constants.REBOOT_TYPES else: options.reboot_types = options.reboot_types.split(",") rt_diff = set(options.reboot_types).difference(constants.REBOOT_TYPES) if rt_diff: Err("Invalid reboot types specified: %s" % utils.CommaJoin(rt_diff)) socket.setdefaulttimeout(options.net_timeout) def GetState(self): """Read the cluster state from the master daemon.""" if self.opts.nodes: names = self.opts.nodes.split(",") else: names = [] try: qcl = GetClient() result = qcl.QueryNodes(names, ["name", "offline", "drained"], False) except errors.GenericError as err: err_code, msg = cli.FormatError(err) Err(msg, exit_code=err_code) finally: qcl.Close() self.nodes = [data[0] for data in result if not (data[1] or data[2])] op_diagnose = opcodes.OpOsDiagnose(output_fields=["name", "variants", "hidden"], names=[]) result = self.ExecOp(True, op_diagnose) if not result: Err("Can't get the OS list") found = False for (name, variants, _) in result: if self.opts.os in cli.CalculateOSNames(name, variants): found = True break if not found: Err("OS '%s' not found" % self.opts.os) cluster_info = self.cl.QueryClusterInfo() self.cluster_info = cluster_info if not self.cluster_info: Err("Can't get cluster info") default_nic_params = self.cluster_info["nicparams"][constants.PP_DEFAULT] self.cluster_default_nicparams = default_nic_params if self.hypervisor is None: self.hypervisor = self.cluster_info["default_hypervisor"] self.hv_can_migrate = \ hypervisor.GetHypervisorClass(self.hypervisor).CAN_MIGRATE def FindMatchingDisk(self, instance): """Find a disk whose nodes match the instance's disk nodes.""" instance_nodes = self.instance_nodes[instance] for disk, disk_nodes in self.disk_nodes.items(): if instance_nodes == disk_nodes: # Erase that disk from the dictionary so that we don't pick it again. del self.disk_nodes[disk] return disk Err("Couldn't find matching detached disk for instance %s" % instance) @_DoCheckInstances @_DoBatch(False) def BurnCreateInstances(self): """Create the given instances. """ self.to_rem = [] mytor = zip(cycle(self.nodes), islice(cycle(self.nodes), 1, None), self.instances) Log("Creating instances") for pnode, snode, instance in mytor: Log("instance %s", instance, indent=1) if self.opts.iallocator: pnode = snode = None msg = "with iallocator %s" % self.opts.iallocator elif self.opts.disk_template not in constants.DTS_INT_MIRROR: snode = None msg = "on %s" % pnode else: msg = "on %s, %s" % (pnode, snode) Log(msg, indent=2) op = opcodes.OpInstanceCreate(instance_name=instance, disks=[{"size": size} for size in self.disk_size], disk_template=self.opts.disk_template, nics=self.opts.nics, mode=constants.INSTANCE_CREATE, os_type=self.opts.os, pnode=pnode, snode=snode, start=True, ip_check=self.opts.ip_check, name_check=self.opts.name_check, wait_for_sync=True, file_driver="loop", file_storage_dir=None, iallocator=self.opts.iallocator, beparams=self.bep, hvparams=self.hvp, hypervisor=self.hypervisor, osparams=self.opts.osparams, ) # NB the i=instance default param is needed here so the lambda captures # the variable. See https://docs.python.org/2/faq/programming.html#id11 rm_inst = lambda i=instance: self.to_rem.append(i) # pylint: disable=C0322 self.ExecOrQueue(instance, [op], post_process=rm_inst) @_DoBatch(False) def BurnModifyRuntimeMemory(self): """Alter the runtime memory.""" Log("Setting instance runtime memory") for instance in self.instances: Log("instance %s", instance, indent=1) tgt_mem = self.bep[constants.BE_MINMEM] op = opcodes.OpInstanceSetParams(instance_name=instance, runtime_mem=tgt_mem) Log("Set memory to %s MB", tgt_mem, indent=2) self.ExecOrQueue(instance, [op]) @_DoBatch(False) def BurnGrowDisks(self): """Grow both the os and the swap disks by the requested amount, if any.""" Log("Growing disks") for instance in self.instances: Log("instance %s", instance, indent=1) for idx, growth in enumerate(self.disk_growth): if growth > 0: op = opcodes.OpInstanceGrowDisk(instance_name=instance, disk=idx, amount=growth, wait_for_sync=True, ignore_ipolicy=True) Log("increase disk/%s by %s MB", idx, growth, indent=2) self.ExecOrQueue(instance, [op]) @_DoBatch(True) def BurnReplaceDisks1D8(self): """Replace disks on primary and secondary for drbd8.""" Log("Replacing disks on the same nodes") early_release = self.opts.early_release for instance in self.instances: Log("instance %s", instance, indent=1) ops = [] for mode in constants.REPLACE_DISK_SEC, constants.REPLACE_DISK_PRI: op = opcodes.OpInstanceReplaceDisks(instance_name=instance, mode=mode, disks=list(range(self.disk_count)), early_release=early_release) Log("run %s", mode, indent=2) ops.append(op) self.ExecOrQueue(instance, ops) @_DoBatch(True) def BurnReplaceDisks2(self): """Replace secondary node.""" Log("Changing the secondary node") mode = constants.REPLACE_DISK_CHG mytor = zip(islice(cycle(self.nodes), 2, None), self.instances) for tnode, instance in mytor: Log("instance %s", instance, indent=1) if self.opts.iallocator: tnode = None msg = "with iallocator %s" % self.opts.iallocator else: msg = tnode op = opcodes.OpInstanceReplaceDisks(instance_name=instance, mode=mode, remote_node=tnode, iallocator=self.opts.iallocator, disks=[], early_release=self.opts.early_release) Log("run %s %s", mode, msg, indent=2) self.ExecOrQueue(instance, [op]) @_DoCheckInstances @_DoBatch(False) def BurnFailover(self): """Failover the instances.""" Log("Failing over instances") for instance in self.instances: Log("instance %s", instance, indent=1) op = opcodes.OpInstanceFailover(instance_name=instance, ignore_consistency=False) self.ExecOrQueue(instance, [op]) @_DoCheckInstances @_DoBatch(False) def BurnMove(self): """Move the instances.""" Log("Moving instances") mytor = zip(islice(cycle(self.nodes), 1, None), self.instances) for tnode, instance in mytor: Log("instance %s", instance, indent=1) op = opcodes.OpInstanceMove(instance_name=instance, target_node=tnode) self.ExecOrQueue(instance, [op]) @_DoBatch(False) def BurnMigrate(self): """Migrate the instances.""" Log("Migrating instances") for instance in self.instances: Log("instance %s", instance, indent=1) op1 = opcodes.OpInstanceMigrate(instance_name=instance, mode=None, cleanup=False) op2 = opcodes.OpInstanceMigrate(instance_name=instance, mode=None, cleanup=True) Log("migration and migration cleanup", indent=2) self.ExecOrQueue(instance, [op1, op2]) @_DoCheckInstances @_DoBatch(False) def BurnImportExport(self): """Export the instance, delete it, and import it back. """ Log("Exporting and re-importing instances") mytor = zip(cycle(self.nodes), islice(cycle(self.nodes), 1, None), islice(cycle(self.nodes), 2, None), self.instances) qcl = GetClient() for pnode, snode, enode, instance in mytor: Log("instance %s", instance, indent=1) # read the full name of the instance (full_name, ) = qcl.QueryInstances([instance], ["name"], False)[0] if self.opts.iallocator: pnode = snode = None import_log_msg = ("import from %s" " with iallocator %s" % (enode, self.opts.iallocator)) elif self.opts.disk_template not in constants.DTS_INT_MIRROR: snode = None import_log_msg = ("import from %s to %s" % (enode, pnode)) else: import_log_msg = ("import from %s to %s, %s" % (enode, pnode, snode)) exp_op = opcodes.OpBackupExport(instance_name=instance, target_node=enode, mode=constants.EXPORT_MODE_LOCAL, shutdown=True) rem_op = opcodes.OpInstanceRemove(instance_name=instance, ignore_failures=True) imp_dir = utils.PathJoin(pathutils.EXPORT_DIR, full_name) imp_op = opcodes.OpInstanceCreate(instance_name=instance, disks=[{"size": size} for size in self.disk_size], disk_template=self.opts.disk_template, nics=self.opts.nics, mode=constants.INSTANCE_IMPORT, src_node=enode, src_path=imp_dir, pnode=pnode, snode=snode, start=True, ip_check=self.opts.ip_check, name_check=self.opts.name_check, wait_for_sync=True, file_storage_dir=None, file_driver="loop", iallocator=self.opts.iallocator, beparams=self.bep, hvparams=self.hvp, osparams=self.opts.osparams, ) erem_op = opcodes.OpBackupRemove(instance_name=instance) Log("export to node %s", enode, indent=2) Log("remove instance", indent=2) Log(import_log_msg, indent=2) Log("remove export", indent=2) self.ExecOrQueue(instance, [exp_op, rem_op, imp_op, erem_op]) qcl.Close() @staticmethod def StopInstanceOp(instance): """Stop given instance.""" return opcodes.OpInstanceShutdown(instance_name=instance) @staticmethod def StartInstanceOp(instance): """Start given instance.""" return opcodes.OpInstanceStartup(instance_name=instance, force=False) @staticmethod def RenameInstanceOp(instance, instance_new, name_check, ip_check): """Rename instance.""" return opcodes.OpInstanceRename(instance_name=instance, new_name=instance_new, name_check=name_check, ip_check=ip_check) @_DoCheckInstances @_DoBatch(True) def BurnStopStart(self): """Stop/start the instances.""" Log("Stopping and starting instances") for instance in self.instances: Log("instance %s", instance, indent=1) op1 = self.StopInstanceOp(instance) op2 = self.StartInstanceOp(instance) self.ExecOrQueue(instance, [op1, op2]) @_DoBatch(False) def BurnRemove(self): """Remove the instances.""" Log("Removing instances") for instance in self.to_rem: Log("instance %s", instance, indent=1) op = opcodes.OpInstanceRemove(instance_name=instance, ignore_failures=True) self.ExecOrQueue(instance, [op]) def BurnRename(self, name_check, ip_check): """Rename the instances. Note that this function will not execute in parallel, since we only have one target for rename. """ Log("Renaming instances") rename = self.opts.rename for instance in self.instances: Log("instance %s", instance, indent=1) op_stop1 = self.StopInstanceOp(instance) op_stop2 = self.StopInstanceOp(rename) op_rename1 = self.RenameInstanceOp(instance, rename, name_check, ip_check) op_rename2 = self.RenameInstanceOp(rename, instance, name_check, ip_check) op_start1 = self.StartInstanceOp(rename) op_start2 = self.StartInstanceOp(instance) self.ExecOp(False, op_stop1, op_rename1, op_start1) self._CheckInstanceAlive(rename) self.ExecOp(False, op_stop2, op_rename2, op_start2) self._CheckInstanceAlive(instance) @_DoCheckInstances @_DoBatch(True) def BurnReinstall(self): """Reinstall the instances.""" Log("Reinstalling instances") for instance in self.instances: Log("instance %s", instance, indent=1) op1 = self.StopInstanceOp(instance) op2 = opcodes.OpInstanceReinstall(instance_name=instance) Log("reinstall without passing the OS", indent=2) op3 = opcodes.OpInstanceReinstall(instance_name=instance, os_type=self.opts.os) Log("reinstall specifying the OS", indent=2) op4 = self.StartInstanceOp(instance) self.ExecOrQueue(instance, [op1, op2, op3, op4]) @_DoCheckInstances @_DoBatch(True) def BurnReboot(self): """Reboot the instances.""" Log("Rebooting instances") for instance in self.instances: Log("instance %s", instance, indent=1) ops = [] for reboot_type in self.opts.reboot_types: op = opcodes.OpInstanceReboot(instance_name=instance, reboot_type=reboot_type, ignore_secondaries=False) Log("reboot with type '%s'", reboot_type, indent=2) ops.append(op) self.ExecOrQueue(instance, ops) @_DoCheckInstances @_DoBatch(True) def BurnRenameSame(self, name_check, ip_check): """Rename the instances to their own name.""" Log("Renaming the instances to their own name") for instance in self.instances: Log("instance %s", instance, indent=1) op1 = self.StopInstanceOp(instance) op2 = self.RenameInstanceOp(instance, instance, name_check, ip_check) Log("rename to the same name", indent=2) op4 = self.StartInstanceOp(instance) self.ExecOrQueue(instance, [op1, op2, op4]) @_DoCheckInstances @_DoBatch(True) def BurnActivateDisks(self): """Activate and deactivate disks of the instances.""" Log("Activating/deactivating disks") for instance in self.instances: Log("instance %s", instance, indent=1) op_start = self.StartInstanceOp(instance) op_act = opcodes.OpInstanceActivateDisks(instance_name=instance) op_deact = opcodes.OpInstanceDeactivateDisks(instance_name=instance) op_stop = self.StopInstanceOp(instance) Log("activate disks when online", indent=2) Log("activate disks when offline", indent=2) Log("deactivate disks (when offline)", indent=2) self.ExecOrQueue(instance, [op_act, op_stop, op_act, op_deact, op_start]) @_DoBatch(False) def BurnAddRemoveNICs(self): """Add, change and remove an extra NIC for the instances.""" Log("Adding and removing NICs") for instance in self.instances: Log("instance %s", instance, indent=1) op_add = opcodes.OpInstanceSetParams( instance_name=instance, nics=[(constants.DDM_ADD, {})]) op_chg = opcodes.OpInstanceSetParams( instance_name=instance, nics=[(constants.DDM_MODIFY, -1, {"mac": constants.VALUE_GENERATE})]) op_rem = opcodes.OpInstanceSetParams( instance_name=instance, nics=[(constants.DDM_REMOVE, {})]) Log("adding a NIC", indent=2) Log("changing a NIC", indent=2) Log("removing last NIC", indent=2) self.ExecOrQueue(instance, [op_add, op_chg, op_rem]) def ConfdCallback(self, reply): """Callback for confd queries""" if reply.type == confd_client.UPCALL_REPLY: if reply.server_reply.status != constants.CONFD_REPL_STATUS_OK: Err("Query %s gave non-ok status %s: %s" % (reply.orig_request, reply.server_reply.status, reply.server_reply)) if reply.orig_request.type == constants.CONFD_REQ_PING: Log("Ping: OK", indent=1) elif reply.orig_request.type == constants.CONFD_REQ_CLUSTER_MASTER: if reply.server_reply.answer == self.cluster_info["master"]: Log("Master: OK", indent=1) else: Err("Master: wrong: %s" % reply.server_reply.answer) elif reply.orig_request.type == constants.CONFD_REQ_NODE_ROLE_BYNAME: if reply.server_reply.answer == constants.CONFD_NODE_ROLE_MASTER: Log("Node role for master: OK", indent=1) else: Err("Node role for master: wrong: %s" % reply.server_reply.answer) elif reply.orig_request.type == constants.CONFD_REQ_INSTANCE_DISKS: self.confd_reply = reply.server_reply.answer def DoConfdRequestReply(self, req): self.confd_counting_callback.RegisterQuery(req.rsalt) self.confd_client.SendRequest(req, async_=False) while not self.confd_counting_callback.AllAnswered(): if not self.confd_client.ReceiveReply(): Err("Did not receive all expected confd replies") break def BurnConfd(self): """Run confd queries for our instances. The following confd queries are tested: - CONFD_REQ_PING: simple ping - CONFD_REQ_CLUSTER_MASTER: cluster master - CONFD_REQ_NODE_ROLE_BYNAME: node role, for the master """ Log("Checking confd results") filter_callback = confd_client.ConfdFilterCallback(self.ConfdCallback) counting_callback = confd_client.ConfdCountingCallback(filter_callback) self.confd_counting_callback = counting_callback self.confd_client = confd_client.GetConfdClient(counting_callback) req = confd_client.ConfdClientRequest(type=constants.CONFD_REQ_PING) self.DoConfdRequestReply(req) req = confd_client.ConfdClientRequest( type=constants.CONFD_REQ_CLUSTER_MASTER) self.DoConfdRequestReply(req) req = confd_client.ConfdClientRequest( type=constants.CONFD_REQ_NODE_ROLE_BYNAME, query=self.cluster_info["master"]) self.DoConfdRequestReply(req) @_DoCheckInstances @_DoBatch(False) def BurnAddDisks(self): """Add an extra disk to every instance and then detach it.""" Log("Adding and detaching disks") # Instantiate a Confd client filter_callback = confd_client.ConfdFilterCallback(self.ConfdCallback) counting_callback = confd_client.ConfdCountingCallback(filter_callback) self.confd_counting_callback = counting_callback self.confd_client = confd_client.GetConfdClient(counting_callback) # Iterate all instances, start them, add a disk with a unique name and # detach it. Do all disk operations with hotplugging (if possible). for instance in self.instances: Log("instance %s", instance, indent=1) # Fetch disk info for an instance from the confd. The result of the query # will be stored in the confd_reply attribute of Burner. req = (confd_client.ConfdClientRequest( type=constants.CONFD_REQ_INSTANCE_DISKS, query=instance)) self.DoConfdRequestReply(req) disk_name = RandomString() nodes = [set(disk["nodes"]) for disk in self.confd_reply] nodes = reduce(or_, nodes) self.instance_nodes[instance] = nodes self.disk_nodes[disk_name] = nodes op_stop = self.StopInstanceOp(instance) op_add = opcodes.OpInstanceSetParams( instance_name=instance, disks=[(constants.DDM_ADD, {"size": self.disk_size[0], "name": disk_name})]) op_detach = opcodes.OpInstanceSetParams( instance_name=instance, disks=[(constants.DDM_DETACH, {})]) op_start = self.StartInstanceOp(instance) Log("adding a disk with name %s" % disk_name, indent=2) Log("detaching last disk", indent=2) self.ExecOrQueue(instance, [op_start, op_add, op_detach, op_stop, op_start]) @_DoCheckInstances @_DoBatch(False) def BurnRemoveDisks(self): """Attach a previously detached disk to an instance and then remove it.""" Log("Attaching and removing disks") # Iterate all instances in random order, attach the detached disks, remove # them and then restart the instances. Do all disk operation with # hotplugging (if possible). instances_copy = list(self.instances) random.shuffle(instances_copy) for instance in instances_copy: Log("instance %s", instance, indent=1) disk_name = self.FindMatchingDisk(instance) op_attach = opcodes.OpInstanceSetParams( instance_name=instance, disks=[(constants.DDM_ATTACH, {"name": disk_name})]) op_rem = opcodes.OpInstanceSetParams( instance_name=instance, disks=[(constants.DDM_REMOVE, {})]) op_stop = self.StopInstanceOp(instance) op_start = self.StartInstanceOp(instance) Log("attaching a disk with name %s" % disk_name, indent=2) Log("removing last disk", indent=2) self.ExecOrQueue(instance, [op_attach, op_rem, op_stop, op_start]) # Disk nodes are useful only for this test. del self.disk_nodes del self.instance_nodes def _CheckInstanceAlive(self, instance): """Check if an instance is alive by doing http checks. This will try to retrieve the url on the instance /hostname.txt and check that it contains the hostname of the instance. In case we get ECONNREFUSED, we retry up to the net timeout seconds, for any other error we abort. """ if not self.opts.http_check: return end_time = time.time() + self.opts.net_timeout url = None while time.time() < end_time and url is None: try: url = self.url_opener.open("http://%s/hostname.txt" % instance) except IOError: # here we can have connection refused, no route to host, etc. time.sleep(1) if url is None: raise InstanceDown(instance, "Cannot contact instance") hostname = url.read().strip() url.close() if hostname != instance: raise InstanceDown(instance, ("Hostname mismatch, expected %s, got %s" % (instance, hostname))) def BurninCluster(self): """Test a cluster intensively. This will create instances and then start/stop/failover them. It is safe for existing instances but could impact performance. """ Log("Testing global parameters") if (len(self.nodes) == 1 and self.opts.disk_template not in _SINGLE_NODE_DISK_TEMPLATES): Err("When one node is available/selected the disk template must" " be one of %s" % utils.CommaJoin(_SINGLE_NODE_DISK_TEMPLATES)) has_err = True try: self.BurnCreateInstances() if self.opts.do_startstop: self.BurnStopStart() if self.bep[constants.BE_MINMEM] < self.bep[constants.BE_MAXMEM]: self.BurnModifyRuntimeMemory() if self.opts.do_replace1 and \ self.opts.disk_template in constants.DTS_INT_MIRROR: self.BurnReplaceDisks1D8() if (self.opts.do_replace2 and len(self.nodes) > 2 and self.opts.disk_template in constants.DTS_INT_MIRROR): self.BurnReplaceDisks2() if (self.opts.disk_template in constants.DTS_GROWABLE and compat.any(n > 0 for n in self.disk_growth)): self.BurnGrowDisks() if self.opts.do_failover and \ self.opts.disk_template in constants.DTS_MIRRORED: self.BurnFailover() if self.opts.do_migrate: if self.opts.disk_template not in constants.DTS_MIRRORED: Log("Skipping migration (disk template %s does not support it)", self.opts.disk_template) elif not self.hv_can_migrate: Log("Skipping migration (hypervisor %s does not support it)", self.hypervisor) else: self.BurnMigrate() if (self.opts.do_move and len(self.nodes) > 1 and self.opts.disk_template in [constants.DT_PLAIN, constants.DT_FILE]): self.BurnMove() if (self.opts.do_importexport and self.opts.disk_template in _IMPEXP_DISK_TEMPLATES): self.BurnImportExport() if self.opts.do_reinstall: self.BurnReinstall() if self.opts.do_reboot: self.BurnReboot() if self.opts.do_renamesame: self.BurnRenameSame(self.opts.name_check, self.opts.ip_check) if self.opts.do_confd_tests: self.BurnConfd() default_nic_mode = self.cluster_default_nicparams[constants.NIC_MODE] # Don't add/remove nics in routed mode, as we would need an ip to add # them with if self.opts.do_addremove_nics: if default_nic_mode == constants.NIC_MODE_BRIDGED: self.BurnAddRemoveNICs() else: Log("Skipping nic add/remove as the cluster is not in bridged mode") if self.opts.do_activate_disks: self.BurnActivateDisks() if self.opts.do_addremove_disks: self.BurnAddDisks() self.BurnRemoveDisks() if self.opts.rename: self.BurnRename(self.opts.name_check, self.opts.ip_check) has_err = False finally: if has_err: Log("Error detected: opcode buffer follows:\n\n") Log(self.GetFeedbackBuf()) Log("\n\n") if not self.opts.keep_instances: try: self.BurnRemove() except Exception as err: # pylint: disable=W0703 if has_err: # already detected errors, so errors in removal # are quite expected Log("Note: error detected during instance remove: %s", err) else: # non-expected error raise return constants.EXIT_SUCCESS def Main(): """Main function. """ utils.SetupLogging(pathutils.LOG_BURNIN, sys.argv[0], debug=False, stderr_logging=True) return Burner().BurninCluster() ganeti-3.1.0~rc2/lib/tools/cfgupgrade.py000064400000000000000000001002111476477700300201660ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Library of the tools/cfgupgrade utility. This code handles only the types supported by simplejson. As an example, 'set' is a 'list'. """ import copy import os import os.path import sys import logging import optparse import time import functools from io import StringIO from ganeti import cli from ganeti import constants from ganeti import serializer from ganeti import utils from ganeti import bootstrap from ganeti import config from ganeti import pathutils from ganeti import netutils from ganeti.utils import version #: Target major version we will upgrade to TARGET_MAJOR = constants.CONFIG_MAJOR #: Target minor version we will upgrade to TARGET_MINOR = constants.CONFIG_MINOR #: Last supported v2.x minor LAST_V2_MINOR = 16 #: Target major version for downgrade DOWNGRADE_MAJOR = TARGET_MAJOR #: Target minor version for downgrade DOWNGRADE_MINOR = TARGET_MINOR - 1 # map of legacy device types # (mapping differing old LD_* constants to new DT_* constants) DEV_TYPE_OLD_NEW = {"lvm": constants.DT_PLAIN, "drbd8": constants.DT_DRBD8} # (mapping differing new DT_* constants to old LD_* constants) DEV_TYPE_NEW_OLD = dict((v, k) for k, v in DEV_TYPE_OLD_NEW.items()) class Error(Exception): """Generic exception""" pass def ParseOptions(args=None): parser = optparse.OptionParser(usage="%prog [--debug|--verbose] [--force]") parser.add_option("--dry-run", dest="dry_run", action="store_true", help="Try to do the conversion, but don't write" " output file") parser.add_option(cli.FORCE_OPT) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option("--ignore-hostname", dest="ignore_hostname", action="store_true", default=False, help="Don't abort if hostname doesn't match") parser.add_option("--path", help="Convert configuration in this" " directory instead of '%s'" % pathutils.DATA_DIR, default=pathutils.DATA_DIR, dest="data_dir") parser.add_option("--confdir", help=("Use this directory instead of '%s'" % pathutils.CONF_DIR), default=pathutils.CONF_DIR, dest="conf_dir") parser.add_option("--no-verify", help="Do not verify configuration after upgrade", action="store_true", dest="no_verify", default=False) parser.add_option("--downgrade", help="Downgrade to the previous stable version", action="store_true", dest="downgrade", default=False) return parser.parse_args(args=args) def OrFail(description=None): """Make failure non-fatal and improve reporting.""" def wrapper(f): @functools.wraps(f) def wrapped(self): safety = copy.deepcopy(self.config_data) try: f(self) except BaseException as e: msg = "%s failed:\n%s" % (description or f.func_name, e) logging.exception(msg) self.config_data = safety self.errors.append(msg) return wrapped return wrapper class CfgUpgrade(object): def __init__(self, opts, args): self.opts = opts self.args = args self.errors = [] def Run(self): """Main program. """ self._ComposePaths() self.SetupLogging() # Option checking if self.args: raise Error("No arguments expected") if self.opts.downgrade and not self.opts.no_verify: self.opts.no_verify = True # Check master name if not (self.CheckHostname(self.opts.SSCONF_MASTER_NODE) or self.opts.ignore_hostname): logging.error("Aborting due to hostname mismatch") sys.exit(constants.EXIT_FAILURE) self._AskUser() # Check whether it's a Ganeti configuration directory if not (os.path.isfile(self.opts.CONFIG_DATA_PATH) and os.path.isfile(self.opts.SERVER_PEM_PATH) and os.path.isfile(self.opts.KNOWN_HOSTS_PATH)): raise Error(("%s does not seem to be a Ganeti configuration" " directory") % self.opts.data_dir) if not os.path.isdir(self.opts.conf_dir): raise Error("Not a directory: %s" % self.opts.conf_dir) self.config_data = serializer.LoadJson(utils.ReadFile( self.opts.CONFIG_DATA_PATH)) try: config_version = self.config_data["version"] except KeyError: raise Error("Unable to determine configuration version") (config_major, config_minor, config_revision) = \ version.SplitVersion(config_version) logging.info("Found configuration version %s (%d.%d.%d)", config_version, config_major, config_minor, config_revision) if "config_version" in self.config_data["cluster"]: raise Error("Inconsistent configuration: found config_version in" " configuration file") # Downgrade to the previous stable version if self.opts.downgrade: self._Downgrade(config_major, config_minor, config_version, config_revision) # Upgrade from 2.0-2.16 and 3.0 to 3.1 elif ((config_major == TARGET_MAJOR and config_minor in range(TARGET_MINOR)) or (config_major == 2 and config_minor in range(LAST_V2_MINOR + 1))): if config_revision != 0: logging.warning("Config revision is %s, not 0", config_revision) if not self.UpgradeAll(): raise Error("Upgrade failed:\n%s" % '\n'.join(self.errors)) elif config_major == TARGET_MAJOR and config_minor == TARGET_MINOR: logging.info("No changes necessary") else: raise Error("Configuration version %d.%d.%d not supported by this tool" % (config_major, config_minor, config_revision)) try: logging.info("Writing configuration file to %s", self.opts.CONFIG_DATA_PATH) utils.WriteFile(file_name=self.opts.CONFIG_DATA_PATH, data=serializer.DumpJson(self.config_data), mode=0o600, dry_run=self.opts.dry_run, backup=True) if not self.opts.dry_run: # This creates the cluster certificate if it does not exist yet. # In this case, we do not automatically create a client certificate # as well, because if the cluster certificate did not exist before, # no client certificate will exist on any node yet. In this case # all client certificate should be renewed by 'gnt-cluster # renew-crypto --new-node-certificates'. This will be enforced # by a nagging warning in 'gnt-cluster verify'. bootstrap.GenerateClusterCrypto( False, False, False, False, False, False, None, nodecert_file=self.opts.SERVER_PEM_PATH, rapicert_file=self.opts.RAPI_CERT_FILE, spicecert_file=self.opts.SPICE_CERT_FILE, spicecacert_file=self.opts.SPICE_CACERT_FILE, hmackey_file=self.opts.CONFD_HMAC_KEY, cds_file=self.opts.CDS_FILE) except Exception: logging.critical("Writing configuration failed. It is probably in an" " inconsistent state and needs manual intervention.") raise self._TestLoadingConfigFile() def SetupLogging(self): """Configures the logging module. """ formatter = logging.Formatter("%(asctime)s: %(message)s") stderr_handler = logging.StreamHandler() stderr_handler.setFormatter(formatter) if self.opts.debug: stderr_handler.setLevel(logging.NOTSET) elif self.opts.verbose: stderr_handler.setLevel(logging.INFO) else: stderr_handler.setLevel(logging.WARNING) root_logger = logging.getLogger("") root_logger.setLevel(logging.NOTSET) root_logger.addHandler(stderr_handler) @staticmethod def CheckHostname(path): """Ensures hostname matches ssconf value. @param path: Path to ssconf file """ ssconf_master_node = utils.ReadOneLineFile(path) hostname = netutils.GetHostname().name if ssconf_master_node == hostname: return True logging.warning("Warning: ssconf says master node is '%s', but this" " machine's name is '%s'; this tool must be run on" " the master node", ssconf_master_node, hostname) return False @staticmethod def _FillIPolicySpecs(default_ipolicy, ipolicy): if "minmax" in ipolicy: for (key, spec) in ipolicy["minmax"][0].items(): for (par, val) in default_ipolicy["minmax"][0][key].items(): if par not in spec: spec[par] = val def UpgradeIPolicy(self, ipolicy, default_ipolicy, isgroup): minmax_keys = ["min", "max"] if any((k in ipolicy) for k in minmax_keys): minmax = {} for key in minmax_keys: if key in ipolicy: if ipolicy[key]: minmax[key] = ipolicy[key] del ipolicy[key] if minmax: ipolicy["minmax"] = [minmax] if isgroup and "std" in ipolicy: del ipolicy["std"] self._FillIPolicySpecs(default_ipolicy, ipolicy) @OrFail("Setting networks") def UpgradeNetworks(self): assert isinstance(self.config_data, dict) # pylint can't infer config_data type # pylint: disable=E1103 networks = self.config_data.get("networks", None) if not networks: self.config_data["networks"] = {} @OrFail("Upgrading cluster") def UpgradeCluster(self): assert isinstance(self.config_data, dict) # pylint can't infer config_data type # pylint: disable=E1103 cluster = self.config_data.get("cluster", None) if cluster is None: raise Error("Cannot find cluster") ipolicy = cluster.setdefault("ipolicy", None) if ipolicy: self.UpgradeIPolicy(ipolicy, constants.IPOLICY_DEFAULTS, False) ial_params = cluster.get("default_iallocator_params", None) if not ial_params: cluster["default_iallocator_params"] = {} if not "candidate_certs" in cluster: cluster["candidate_certs"] = {} cluster["instance_communication_network"] = \ cluster.get("instance_communication_network", "") cluster["install_image"] = \ cluster.get("install_image", "") cluster["zeroing_image"] = \ cluster.get("zeroing_image", "") cluster["compression_tools"] = \ cluster.get("compression_tools", constants.IEC_DEFAULT_TOOLS) if "enabled_user_shutdown" not in cluster: cluster["enabled_user_shutdown"] = False cluster["data_collectors"] = cluster.get("data_collectors", {}) for name in constants.DATA_COLLECTOR_NAMES: cluster["data_collectors"][name] = \ cluster["data_collectors"].get( name, dict(active=True, interval=constants.MOND_TIME_INTERVAL * 1e6)) # These parameters are set to pre-2.16 default values, which # differ from post-2.16 default values if "ssh_key_type" not in cluster: cluster["ssh_key_type"] = constants.SSHK_DSA if "ssh_key_bits" not in cluster: cluster["ssh_key_bits"] = 1024 if "hvparams" in cluster: variants = [constants.HT_XEN_PVM, constants.HT_XEN_HVM] for variant in variants: if variant in cluster["hvparams"]: cluster["hvparams"][variant].pop("xen_cmd", None) # HT_DISCARD_DEFAULT removed with 3.1.0 if constants.HV_DISK_DISCARD in cluster["hvparams"][constants.HT_KVM] \ and cluster["hvparams"][constants.HT_KVM][constants.HV_DISK_DISCARD] \ == "default": cluster["hvparams"][constants.HT_KVM][constants.HV_DISK_DISCARD] = \ constants.HT_DISCARD_IGNORE @OrFail("Upgrading groups") def UpgradeGroups(self): cl_ipolicy = self.config_data["cluster"].get("ipolicy") for group in self.config_data["nodegroups"].values(): networks = group.get("networks", None) if not networks: group["networks"] = {} ipolicy = group.get("ipolicy", None) if ipolicy: if cl_ipolicy is None: raise Error("A group defines an instance policy but there is no" " instance policy at cluster level") self.UpgradeIPolicy(ipolicy, cl_ipolicy, True) def GetExclusiveStorageValue(self): """Return a conservative value of the exclusive_storage flag. Return C{True} if the cluster or at least a nodegroup have the flag set. """ ret = False cluster = self.config_data["cluster"] ndparams = cluster.get("ndparams") if ndparams is not None and ndparams.get("exclusive_storage"): ret = True for group in self.config_data["nodegroups"].values(): ndparams = group.get("ndparams") if ndparams is not None and ndparams.get("exclusive_storage"): ret = True return ret def RemovePhysicalId(self, disk): if "children" in disk: for d in disk["children"]: self.RemovePhysicalId(d) if "physical_id" in disk: del disk["physical_id"] def ChangeDiskDevType(self, disk, dev_type_map): """Replaces disk's dev_type attributes according to the given map. This can be used for both, up or downgrading the disks. """ if disk["dev_type"] in dev_type_map: disk["dev_type"] = dev_type_map[disk["dev_type"]] if "children" in disk: for child in disk["children"]: self.ChangeDiskDevType(child, dev_type_map) def UpgradeDiskDevType(self, disk): """Upgrades the disks' device type.""" self.ChangeDiskDevType(disk, DEV_TYPE_OLD_NEW) @staticmethod def _ConvertNicNameToUuid(iobj, network2uuid): for nic in iobj["nics"]: name = nic.get("network", None) if name: uuid = network2uuid.get(name, None) if uuid: print("NIC with network name %s found." " Substituting with uuid %s." % (name, uuid)) nic["network"] = uuid @classmethod def AssignUuid(cls, disk): if not "uuid" in disk: disk["uuid"] = utils.io.NewUUID() if "children" in disk: for d in disk["children"]: cls.AssignUuid(d) def _ConvertDiskAndCheckMissingSpindles(self, iobj, instance): missing_spindles = False if "disks" not in iobj: raise Error("Instance '%s' doesn't have a disks entry?!" % instance) disks = iobj["disks"] if not all(isinstance(d, str) for d in disks): # Disks are not top level citizens for idx, dobj in enumerate(disks): self.RemovePhysicalId(dobj) expected = "disk/%s" % idx current = dobj.get("iv_name", "") if current != expected: logging.warning("Updating iv_name for instance %s/disk %s" " from '%s' to '%s'", instance, idx, current, expected) dobj["iv_name"] = expected if "dev_type" in dobj: self.UpgradeDiskDevType(dobj) if not "spindles" in dobj: missing_spindles = True self.AssignUuid(dobj) return missing_spindles @OrFail("Upgrading instance with spindles") def UpgradeInstances(self): """Upgrades the instances' configuration.""" network2uuid = dict((n["name"], n["uuid"]) for n in self.config_data["networks"].values()) if "instances" not in self.config_data: raise Error("Can't find the 'instances' key in the configuration!") missing_spindles = False for instance, iobj in self.config_data["instances"].items(): self._ConvertNicNameToUuid(iobj, network2uuid) if self._ConvertDiskAndCheckMissingSpindles(iobj, instance): missing_spindles = True if "admin_state_source" not in iobj: iobj["admin_state_source"] = constants.ADMIN_SOURCE if "hvparams" in iobj and constants.HV_DISK_DISCARD in iobj["hvparams"]: if iobj["hvparams"][constants.HV_DISK_DISCARD] == "default": iobj["hvparams"][constants.HV_DISK_DISCARD] = \ constants.HT_DISCARD_IGNORE logging.info("disk_discard was explicitly set to 'default' on " "instance '%s': migrated to 'ignore'" % iobj["name"]) if self.GetExclusiveStorageValue() and missing_spindles: # We cannot be sure that the instances that are missing spindles have # exclusive storage enabled (the check would be more complicated), so we # give a noncommittal message logging.warning("Some instance disks could be needing to update the" " spindles parameter; you can check by running" " 'gnt-cluster verify', and fix any problem with" " 'gnt-cluster repair-disk-sizes'") def UpgradeRapiUsers(self): if (os.path.isfile(self.opts.RAPI_USERS_FILE_PRE24) and not os.path.islink(self.opts.RAPI_USERS_FILE_PRE24)): if os.path.exists(self.opts.RAPI_USERS_FILE): raise Error("Found pre-2.4 RAPI users file at %s, but another file" " already exists at %s" % (self.opts.RAPI_USERS_FILE_PRE24, self.opts.RAPI_USERS_FILE)) logging.info("Found pre-2.4 RAPI users file at %s, renaming to %s", self.opts.RAPI_USERS_FILE_PRE24, self.opts.RAPI_USERS_FILE) if not self.opts.dry_run: utils.RenameFile(self.opts.RAPI_USERS_FILE_PRE24, self.opts.RAPI_USERS_FILE, mkdir=True, mkdir_mode=0o750) # Create a symlink for RAPI users file if (not (os.path.islink(self.opts.RAPI_USERS_FILE_PRE24) or os.path.isfile(self.opts.RAPI_USERS_FILE_PRE24)) and os.path.isfile(self.opts.RAPI_USERS_FILE)): logging.info("Creating symlink from %s to %s", self.opts.RAPI_USERS_FILE_PRE24, self.opts.RAPI_USERS_FILE) if not self.opts.dry_run: os.symlink(self.opts.RAPI_USERS_FILE, self.opts.RAPI_USERS_FILE_PRE24) def UpgradeWatcher(self): # Remove old watcher state file if it exists if os.path.exists(self.opts.WATCHER_STATEFILE): logging.info("Removing watcher state file %s", self.opts.WATCHER_STATEFILE) if not self.opts.dry_run: utils.RemoveFile(self.opts.WATCHER_STATEFILE) @OrFail("Upgrading file storage paths") def UpgradeFileStoragePaths(self): # Write file storage paths if not os.path.exists(self.opts.FILE_STORAGE_PATHS_FILE): cluster = self.config_data["cluster"] file_storage_dir = cluster.get("file_storage_dir") shared_file_storage_dir = cluster.get("shared_file_storage_dir") del cluster logging.info("Ganeti 2.7 and later only allow whitelisted directories" " for file storage; writing existing configuration values" " into '%s'", self.opts.FILE_STORAGE_PATHS_FILE) if file_storage_dir: logging.info("File storage directory: %s", file_storage_dir) if shared_file_storage_dir: logging.info("Shared file storage directory: %s", shared_file_storage_dir) buf = StringIO() buf.write("# List automatically generated from configuration by\n") buf.write("# cfgupgrade at %s\n" % time.asctime()) if file_storage_dir: buf.write("%s\n" % file_storage_dir) if shared_file_storage_dir: buf.write("%s\n" % shared_file_storage_dir) utils.WriteFile(file_name=self.opts.FILE_STORAGE_PATHS_FILE, data=buf.getvalue(), mode=0o600, dry_run=self.opts.dry_run, backup=True) @staticmethod def GetNewNodeIndex(nodes_by_old_key, old_key, new_key_field): if old_key not in nodes_by_old_key: logging.warning("Can't find node '%s' in configuration, " "assuming that it's already up-to-date", old_key) return old_key return nodes_by_old_key[old_key][new_key_field] def ChangeNodeIndices(self, config_data, old_key_field, new_key_field): def ChangeDiskNodeIndices(disk): # Note: 'drbd8' is a legacy device type from pre 2.9 and needs to be # considered when up/downgrading from/to any versions touching 2.9 on the # way. drbd_disk_types = set(["drbd8"]) | constants.DTS_DRBD if disk["dev_type"] in drbd_disk_types: for i in range(0, 2): disk["logical_id"][i] = self.GetNewNodeIndex(nodes_by_old_key, disk["logical_id"][i], new_key_field) if "children" in disk: for child in disk["children"]: ChangeDiskNodeIndices(child) nodes_by_old_key = {} nodes_by_new_key = {} for (_, node) in config_data["nodes"].items(): nodes_by_old_key[node[old_key_field]] = node nodes_by_new_key[node[new_key_field]] = node config_data["nodes"] = nodes_by_new_key cluster = config_data["cluster"] cluster["master_node"] = self.GetNewNodeIndex(nodes_by_old_key, cluster["master_node"], new_key_field) for inst in config_data["instances"].values(): inst["primary_node"] = self.GetNewNodeIndex(nodes_by_old_key, inst["primary_node"], new_key_field) for disk in config_data["disks"].values(): ChangeDiskNodeIndices(disk) @staticmethod def ChangeInstanceIndices(config_data, old_key_field, new_key_field): insts_by_old_key = {} insts_by_new_key = {} for (_, inst) in config_data["instances"].items(): insts_by_old_key[inst[old_key_field]] = inst insts_by_new_key[inst[new_key_field]] = inst config_data["instances"] = insts_by_new_key @OrFail("Changing node indices") def UpgradeNodeIndices(self): self.ChangeNodeIndices(self.config_data, "name", "uuid") @OrFail("Changing instance indices") def UpgradeInstanceIndices(self): self.ChangeInstanceIndices(self.config_data, "name", "uuid") @OrFail("Adding filters") def UpgradeFilters(self): # pylint can't infer config_data type # pylint: disable=E1103 filters = self.config_data.get("filters", None) if not filters: self.config_data["filters"] = {} @OrFail("Set top level disks") def UpgradeTopLevelDisks(self): """Upgrades the disks as config top level citizens.""" if "instances" not in self.config_data: raise Error("Can't find the 'instances' key in the configuration!") if "disks" in self.config_data: # Disks are already top level citizens return self.config_data["disks"] = dict() for iobj in self.config_data["instances"].values(): disk_uuids = [] for disk in iobj["disks"]: duuid = disk["uuid"] disk["serial_no"] = 1 # Instances may not have the ctime value, and the Haskell serialization # will have set it to zero. disk["ctime"] = disk["mtime"] = iobj.get("ctime", 0) self.config_data["disks"][duuid] = disk disk_uuids.append(duuid) iobj["disks"] = disk_uuids @OrFail("Removing disk template") def UpgradeDiskTemplate(self): if "instances" not in self.config_data: raise Error("Can't find the 'instances' dictionary in the configuration.") instances = self.config_data["instances"] for inst in instances.values(): if "disk_template" in inst: del inst["disk_template"] # The following function is based on a method of class Disk with the same # name, but adjusted to work with dicts and sets. def _ComputeAllNodes(self, disk): """Recursively compute nodes given a top device.""" nodes = set() if disk["dev_type"] in constants.DTS_DRBD: nodes = set(disk["logical_id"][:2]) for child in disk.get("children", []): nodes |= self._ComputeAllNodes(child) return nodes def _RecursiveUpdateNodes(self, disk, nodes): disk["nodes"] = nodes for child in disk.get("children", []): self._RecursiveUpdateNodes(child, nodes) @OrFail("Upgrading disk nodes") def UpgradeDiskNodes(self): """Specify the nodes from which a disk is accessible in its definition. For every disk that is attached to an instance, get the UUIDs of the nodes that it's accessible from. There are three main cases: 1) Internally mirrored disks (DRBD): These disks are accessible from two nodes, so the nodes list will include these. Their children (data, meta) are also accessible from two nodes, therefore they will inherit the nodes of the parent. 2) Externally mirrored disks (Blockdev, Ext, Gluster, RBD, Shared File): These disks should be accessible from any node in the cluster, therefore the nodes list will be empty. 3) Single-node disks (Plain, File): These disks are accessible from one node only, therefore the nodes list will consist only of the primary instance node. """ disks = self.config_data["disks"] for instance in self.config_data["instances"].values(): # Get all disk nodes for an instance instance_node = set([instance["primary_node"]]) disk_nodes = set() for disk_uuid in instance["disks"]: disk_nodes |= self._ComputeAllNodes(disks[disk_uuid]) all_nodes = list(instance_node | disk_nodes) # Populate the `nodes` list field of each disk. for disk_uuid in instance["disks"]: disk = disks[disk_uuid] if "nodes" in disk: # The "nodes" field has already been added for this disk. continue if disk["dev_type"] in constants.DTS_INT_MIRROR: self._RecursiveUpdateNodes(disk, all_nodes) elif disk["dev_type"] in (constants.DT_PLAIN, constants.DT_FILE): disk["nodes"] = all_nodes else: disk["nodes"] = [] def UpgradeAll(self): self.config_data["version"] = version.BuildVersion(TARGET_MAJOR, TARGET_MINOR, 0) self.UpgradeRapiUsers() self.UpgradeWatcher() steps = [self.UpgradeFileStoragePaths, self.UpgradeNetworks, self.UpgradeCluster, self.UpgradeGroups, self.UpgradeInstances, self.UpgradeTopLevelDisks, self.UpgradeNodeIndices, self.UpgradeInstanceIndices, self.UpgradeFilters, self.UpgradeDiskNodes, self.UpgradeDiskTemplate] for s in steps: s() return not self.errors # DOWNGRADE ------------------------------------------------------------ @OrFail("Adding xen_cmd parameter") def DowngradeXenSettings(self): """Re-adds the xen_cmd setting to the configuration. """ # pylint: disable=E1103 # Because config_data is a dictionary which has the get method. cluster = self.config_data.get("cluster", None) if cluster is None: raise Error("Can't find the cluster entry in the configuration") hvparams = cluster.get("hvparams", None) if hvparams is None: return variants = [constants.HT_XEN_PVM, constants.HT_XEN_HVM] for variant in variants: if variant in hvparams: hvparams[variant]["xen_cmd"] = "xl" def DowngradeAll(self): self.config_data["version"] = version.BuildVersion(DOWNGRADE_MAJOR, DOWNGRADE_MINOR, 0) self.DowngradeXenSettings() return not self.errors def _ComposePaths(self): # We need to keep filenames locally because they might be renamed between # versions. self.opts.data_dir = os.path.abspath(self.opts.data_dir) self.opts.CONFIG_DATA_PATH = self.opts.data_dir + "/config.data" self.opts.SERVER_PEM_PATH = self.opts.data_dir + "/server.pem" self.opts.CLIENT_PEM_PATH = self.opts.data_dir + "/client.pem" self.opts.KNOWN_HOSTS_PATH = self.opts.data_dir + "/known_hosts" self.opts.RAPI_CERT_FILE = self.opts.data_dir + "/rapi.pem" self.opts.SPICE_CERT_FILE = self.opts.data_dir + "/spice.pem" self.opts.SPICE_CACERT_FILE = self.opts.data_dir + "/spice-ca.pem" self.opts.RAPI_USERS_FILE = self.opts.data_dir + "/rapi/users" self.opts.RAPI_USERS_FILE_PRE24 = self.opts.data_dir + "/rapi_users" self.opts.CONFD_HMAC_KEY = self.opts.data_dir + "/hmac.key" self.opts.CDS_FILE = self.opts.data_dir + "/cluster-domain-secret" self.opts.SSCONF_MASTER_NODE = self.opts.data_dir + "/ssconf_master_node" self.opts.WATCHER_STATEFILE = self.opts.data_dir + "/watcher.data" self.opts.FILE_STORAGE_PATHS_FILE = (self.opts.conf_dir + "/file-storage-paths") def _AskUser(self): if not self.opts.force: if self.opts.downgrade: usertext = ("The configuration is going to be DOWNGRADED " "to version %s.%s. Some configuration data might be " " removed if they don't fit" " in the old format. Please make sure you have read the" " upgrade notes (available in the UPGRADE file and included" " in other documentation formats) to understand what they" " are. Continue with *DOWNGRADING* the configuration?" % (DOWNGRADE_MAJOR, DOWNGRADE_MINOR)) else: usertext = ("Please make sure you have read the upgrade notes for" " Ganeti %s (available in the UPGRADE file and included" " in other documentation formats). Continue with upgrading" " configuration?" % constants.RELEASE_VERSION) if not cli.AskUser(usertext): sys.exit(constants.EXIT_FAILURE) def _Downgrade(self, config_major, config_minor, config_version, config_revision): if not ((config_major == TARGET_MAJOR and config_minor == TARGET_MINOR) or (config_major == DOWNGRADE_MAJOR and config_minor == DOWNGRADE_MINOR)): raise Error("Downgrade supported only from the latest version (%s.%s)," " found %s (%s.%s.%s) instead" % (TARGET_MAJOR, TARGET_MINOR, config_version, config_major, config_minor, config_revision)) if not self.DowngradeAll(): raise Error("Downgrade failed:\n%s" % "\n".join(self.errors)) def _TestLoadingConfigFile(self): # test loading the config file all_ok = True if not (self.opts.dry_run or self.opts.no_verify): logging.info("Testing the new config file...") cfg = config.ConfigWriter(cfg_file=self.opts.CONFIG_DATA_PATH, accept_foreign=self.opts.ignore_hostname, offline=True) # if we reached this, it's all fine vrfy = cfg.VerifyConfig() if vrfy: logging.error("Errors after conversion:") for item in vrfy: logging.error(" - %s", item) all_ok = False else: logging.info("File loaded successfully after upgrading") del cfg if self.opts.downgrade: action = "downgraded" out_ver = "%s.%s" % (DOWNGRADE_MAJOR, DOWNGRADE_MINOR) else: action = "upgraded" out_ver = constants.RELEASE_VERSION if all_ok: cli.ToStderr("Configuration successfully %s to version %s.", action, out_ver) else: cli.ToStderr("Configuration %s to version %s, but there are errors." "\nPlease review the file.", action, out_ver) ganeti-3.1.0~rc2/lib/tools/common.py000064400000000000000000000166751476477700300173730ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Common functions for tool scripts. """ import logging import os import time from io import StringIO import OpenSSL from ganeti import constants from ganeti import errors from ganeti import pathutils from ganeti import utils from ganeti import serializer from ganeti import ssconf from ganeti import ssh def VerifyOptions(parser, opts, args): """Verifies options and arguments for correctness. """ if args: parser.error("No arguments are expected") return opts def _VerifyCertificateStrong(cert_pem, error_fn, _check_fn=utils.CheckNodeCertificate): """Verifies a certificate against the local node daemon certificate. Includes elaborate tests of encodings etc., and returns formatted certificate. @type cert_pem: string @param cert_pem: Certificate and key in PEM format @type error_fn: callable @param error_fn: function to call in case of an error @rtype: string @return: Formatted key and certificate """ try: cert = \ OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_pem) except Exception as err: raise error_fn("(stdin) Unable to load certificate: %s" % err) try: key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, cert_pem) except OpenSSL.crypto.Error as err: raise error_fn("(stdin) Unable to load private key: %s" % err) # Check certificate with given key; this detects cases where the key given on # stdin doesn't match the certificate also given on stdin try: utils.X509CertKeyCheck(cert, key) except OpenSSL.SSL.Error: raise error_fn("(stdin) Certificate is not signed with given key") # Standard checks, including check against an existing local certificate # (no-op if that doesn't exist) _check_fn(cert) key_encoded = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key) cert_encoded = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert) complete_cert_encoded = key_encoded + cert_encoded if not cert_pem == complete_cert_encoded.decode('ascii'): logging.error("The certificate differs after being reencoded. Please" " renew the certificates cluster-wide to prevent future" " inconsistencies.") # Format for storing on disk buf = StringIO() buf.write(cert_pem) return buf.getvalue() def _VerifyCertificateSoft(cert_pem, error_fn, _check_fn=utils.CheckNodeCertificate): """Verifies a certificate against the local node daemon certificate. @type cert_pem: string @param cert_pem: Certificate in PEM format (no key) """ try: OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, cert_pem) except OpenSSL.crypto.Error as err: pass else: raise error_fn("No private key may be given") try: cert = \ OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_pem) except Exception as err: raise errors.X509CertError("(stdin)", "Unable to load certificate: %s" % err) _check_fn(cert) def VerifyCertificateSoft(data, error_fn, _verify_fn=_VerifyCertificateSoft): """Verifies cluster certificate if existing. @type data: dict @type error_fn: callable @param error_fn: function to call in case of an error @rtype: string @return: Formatted key and certificate """ cert = data.get(constants.SSHS_NODE_DAEMON_CERTIFICATE) if cert: _verify_fn(cert, error_fn) def VerifyCertificateStrong(data, error_fn, _verify_fn=_VerifyCertificateStrong): """Verifies cluster certificate. Throws error when not existing. @type data: dict @type error_fn: callable @param error_fn: function to call in case of an error @rtype: string @return: Formatted key and certificate """ cert = data.get(constants.NDS_NODE_DAEMON_CERTIFICATE) if not cert: raise error_fn("Node daemon certificate must be specified") return _verify_fn(cert, error_fn) def VerifyClusterName(data, error_fn, cluster_name_constant, _verify_fn=ssconf.VerifyClusterName): """Verifies cluster name. @type data: dict """ name = data.get(cluster_name_constant) if name: _verify_fn(name) else: raise error_fn("Cluster name must be specified") return name def VerifyHmac(data, error_fn): """Verifies the presence of the hmac secret. @type data: dict """ hmac = data.get(constants.NDS_HMAC) if not hmac: raise error_fn("Hmac key must be provided") return hmac def LoadData(raw, data_check): """Parses and verifies input data. @rtype: dict """ result = None try: result = serializer.LoadAndVerifyJson(raw, data_check) logging.debug("Received data: %s", serializer.DumpJson(result)) except Exception as e: logging.warn("Received data is not valid json: %s.", str(raw)) raise e return result def GenerateRootSshKeys(key_type, key_bits, error_fn, _suffix="", _homedir_fn=None): """Generates root's SSH keys for this node. """ ssh.InitSSHSetup(key_type, key_bits, error_fn=error_fn, _homedir_fn=_homedir_fn, _suffix=_suffix) def GenerateClientCertificate( data, error_fn, client_cert=pathutils.NODED_CLIENT_CERT_FILE, signing_cert=pathutils.NODED_CERT_FILE): """Regenerates the client certificate of the node. @type data: string @param data: the JSON-formated input data """ if not os.path.exists(signing_cert): raise error_fn("The signing certificate '%s' cannot be found." % signing_cert) # TODO: This sets the serial number to the number of seconds # since epoch. This is technically not a correct serial number # (in the way SSL is supposed to be used), but it serves us well # enough for now, as we don't have any infrastructure for keeping # track of the number of signed certificates yet. serial_no = int(time.time()) # The hostname of the node is provided with the input data. hostname = data.get(constants.NDS_NODE_NAME) if not hostname: raise error_fn("No hostname found.") utils.GenerateSignedSslCert(client_cert, serial_no, signing_cert, common_name=hostname) ganeti-3.1.0~rc2/lib/tools/ensure_dirs.py000064400000000000000000000261641476477700300204170ustar00rootroot00000000000000# # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to ensure permissions on files/dirs are accurate. """ import os import os.path import optparse import sys import logging from ganeti import constants from ganeti import errors from ganeti import runtime from ganeti import ssconf from ganeti import utils from ganeti import cli from ganeti import pathutils from ganeti import compat (DIR, FILE, QUEUE_DIR) = range(1, 4) ALL_TYPES = compat.UniqueFrozenset([ DIR, FILE, QUEUE_DIR, ]) def RecursiveEnsure(path, uid, gid, dir_perm, file_perm): """Ensures permissions recursively down a directory. This functions walks the path and sets permissions accordingly. @param path: The absolute path to walk @param uid: The uid used as owner @param gid: The gid used as group @param dir_perm: The permission bits set for directories @param file_perm: The permission bits set for files """ assert os.path.isabs(path), "Path %s is not absolute" % path assert os.path.isdir(path), "Path %s is not a dir" % path logging.debug("Recursively processing %s", path) for root, dirs, files in os.walk(path): for subdir in dirs: utils.EnforcePermission(os.path.join(root, subdir), dir_perm, uid=uid, gid=gid) for filename in files: utils.EnforcePermission(os.path.join(root, filename), file_perm, uid=uid, gid=gid) def EnsureQueueDir(path, mode, uid, gid): """Sets the correct permissions on all job files in the queue. @param path: Directory path @param mode: Wanted file mode @param uid: Wanted user ID @param gid: Wanted group ID """ for filename in utils.ListVisibleFiles(path): if constants.JOB_FILE_RE.match(filename): utils.EnforcePermission(utils.PathJoin(path, filename), mode, uid=uid, gid=gid) def ProcessPath(path): """Processes a path component. @param path: A tuple of the path component to process """ (pathname, pathtype, mode, uid, gid) = path[0:5] assert pathtype in ALL_TYPES if pathtype in (DIR, QUEUE_DIR): # No additional parameters assert len(path) == 5 if pathtype == DIR: utils.MakeDirWithPerm(pathname, mode, uid, gid) elif pathtype == QUEUE_DIR: EnsureQueueDir(pathname, mode, uid, gid) elif pathtype == FILE: (must_exist, ) = path[5:] utils.EnforcePermission(pathname, mode, uid=uid, gid=gid, must_exist=must_exist) def GetPaths(): """Returns a tuple of path objects to process. """ getent = runtime.GetEnts() masterd_log = constants.DAEMONS_LOGFILES[constants.MASTERD] noded_log = constants.DAEMONS_LOGFILES[constants.NODED] confd_log = constants.DAEMONS_LOGFILES[constants.CONFD] wconfd_log = constants.DAEMONS_LOGFILES[constants.WCONFD] luxid_log = constants.DAEMONS_LOGFILES[constants.LUXID] rapi_log = constants.DAEMONS_LOGFILES[constants.RAPI] mond_log = constants.DAEMONS_LOGFILES[constants.MOND] metad_log = constants.DAEMONS_LOGFILES[constants.METAD] mond_extra_log = constants.DAEMONS_EXTRA_LOGFILES[constants.MOND] metad_extra_log = constants.DAEMONS_EXTRA_LOGFILES[constants.METAD] jobs_log = pathutils.GetLogFilename("jobs") rapi_dir = os.path.join(pathutils.DATA_DIR, "rapi") cleaner_log_dir = os.path.join(pathutils.LOG_DIR, "cleaner") master_cleaner_log_dir = os.path.join(pathutils.LOG_DIR, "master-cleaner") # A note on the ordering: The parent directory (type C{DIR}) must always be # listed before files (type C{FILE}) in that directory. Once the directory is # set, only files directly in that directory can be listed. paths = [ (pathutils.DATA_DIR, DIR, 0o755, getent.masterd_uid, getent.masterd_gid), (pathutils.CLUSTER_DOMAIN_SECRET_FILE, FILE, 0o640, getent.masterd_uid, getent.masterd_gid, False), (pathutils.CLUSTER_CONF_FILE, FILE, 0o640, getent.masterd_uid, getent.confd_gid, False), (pathutils.LOCK_STATUS_FILE, FILE, 0o640, getent.masterd_uid, getent.confd_gid, False), (pathutils.TEMP_RES_STATUS_FILE, FILE, 0o640, getent.masterd_uid, getent.confd_gid, False), (pathutils.CONFD_HMAC_KEY, FILE, 0o440, getent.confd_uid, getent.masterd_gid, False), (pathutils.SSH_KNOWN_HOSTS_FILE, FILE, 0o644, getent.masterd_uid, getent.masterd_gid, False), (pathutils.RAPI_CERT_FILE, FILE, 0o440, getent.rapi_uid, getent.masterd_gid, False), (pathutils.SPICE_CERT_FILE, FILE, 0o440, getent.noded_uid, getent.masterd_gid, False), (pathutils.SPICE_CACERT_FILE, FILE, 0o440, getent.noded_uid, getent.masterd_gid, False), (pathutils.NODED_CERT_FILE, FILE, pathutils.NODED_CERT_MODE, getent.masterd_uid, getent.masterd_gid, False), (pathutils.NODED_CLIENT_CERT_FILE, FILE, pathutils.NODED_CERT_MODE, getent.masterd_uid, getent.masterd_gid, False), (pathutils.WATCHER_PAUSEFILE, FILE, 0o644, getent.masterd_uid, getent.masterd_gid, False), ] ss = ssconf.SimpleStore() for ss_path in ss.GetFileList(): paths.append((ss_path, FILE, constants.SS_FILE_PERMS, getent.noded_uid, getent.noded_gid, False)) paths.extend([ (pathutils.QUEUE_DIR, DIR, 0o750, getent.masterd_uid, getent.daemons_gid), (pathutils.QUEUE_DIR, QUEUE_DIR, constants.JOB_QUEUE_FILES_PERMS, getent.masterd_uid, getent.daemons_gid), (pathutils.JOB_QUEUE_DRAIN_FILE, FILE, 0o644, getent.masterd_uid, getent.daemons_gid, False), (pathutils.JOB_QUEUE_LOCK_FILE, FILE, constants.JOB_QUEUE_FILES_PERMS, getent.masterd_uid, getent.daemons_gid, False), (pathutils.JOB_QUEUE_SERIAL_FILE, FILE, constants.JOB_QUEUE_FILES_PERMS, getent.masterd_uid, getent.daemons_gid, False), (pathutils.JOB_QUEUE_VERSION_FILE, FILE, constants.JOB_QUEUE_FILES_PERMS, getent.masterd_uid, getent.daemons_gid, False), (pathutils.JOB_QUEUE_ARCHIVE_DIR, DIR, 0o750, getent.masterd_uid, getent.daemons_gid), (rapi_dir, DIR, 0o750, getent.rapi_uid, getent.masterd_gid), (pathutils.RAPI_USERS_FILE, FILE, 0o640, getent.rapi_uid, getent.masterd_gid, False), (pathutils.RUN_DIR, DIR, 0o775, getent.masterd_uid, getent.daemons_gid), (pathutils.SOCKET_DIR, DIR, 0o770, getent.masterd_uid, getent.daemons_gid), (pathutils.MASTER_SOCKET, FILE, 0o660, getent.masterd_uid, getent.daemons_gid, False), (pathutils.QUERY_SOCKET, FILE, 0o660, getent.luxid_uid, getent.daemons_gid, False), (pathutils.BDEV_CACHE_DIR, DIR, 0o755, getent.noded_uid, getent.masterd_gid), (pathutils.UIDPOOL_LOCKDIR, DIR, 0o750, getent.noded_uid, getent.masterd_gid), (pathutils.DISK_LINKS_DIR, DIR, 0o755, getent.noded_uid, getent.masterd_gid), (pathutils.CRYPTO_KEYS_DIR, DIR, 0o700, getent.noded_uid, getent.masterd_gid), (pathutils.IMPORT_EXPORT_DIR, DIR, 0o755, getent.noded_uid, getent.masterd_gid), (pathutils.LOG_DIR, DIR, 0o770, getent.masterd_uid, getent.daemons_gid), (masterd_log, FILE, 0o600, getent.masterd_uid, getent.masterd_gid, False), (confd_log, FILE, 0o600, getent.confd_uid, getent.masterd_gid, False), (wconfd_log, FILE, 0o600, getent.wconfd_uid, getent.masterd_gid, False), (luxid_log, FILE, 0o600, getent.luxid_uid, getent.masterd_gid, False), (noded_log, FILE, 0o600, getent.noded_uid, getent.masterd_gid, False), (rapi_log, FILE, 0o600, getent.rapi_uid, getent.masterd_gid, False), (mond_log, FILE, 0o600, getent.mond_uid, getent.masterd_gid, False), (mond_extra_log["access"], FILE, 0o600, getent.mond_uid, getent.masterd_gid, False), (mond_extra_log["error"], FILE, 0o600, getent.mond_uid, getent.masterd_gid, False), (metad_log, FILE, 0o600, getent.metad_uid, getent.metad_gid, False), (metad_extra_log["access"], FILE, 0o600, getent.metad_uid, getent.metad_gid, False), (metad_extra_log["error"], FILE, 0o600, getent.metad_uid, getent.metad_gid, False), (jobs_log, FILE, 0o600, getent.luxid_uid, getent.luxid_gid, False), (pathutils.LOG_OS_DIR, DIR, 0o750, getent.noded_uid, getent.daemons_gid), (pathutils.LOG_XEN_DIR, DIR, 0o750, getent.noded_uid, getent.daemons_gid), (pathutils.LOG_KVM_DIR, DIR, 0o750, getent.noded_uid, getent.daemons_gid), (cleaner_log_dir, DIR, 0o750, getent.noded_uid, getent.noded_gid), (master_cleaner_log_dir, DIR, 0o750, getent.masterd_uid, getent.masterd_gid), (pathutils.INSTANCE_REASON_DIR, DIR, 0o755, getent.noded_uid, getent.noded_gid), (pathutils.LIVELOCK_DIR, DIR, 0o750, getent.masterd_uid, getent.daemons_gid), (pathutils.LUXID_MESSAGE_DIR, DIR, 0o750, getent.masterd_uid, getent.daemons_gid), ]) return paths def ParseOptions(): """Parses the options passed to the program. @return: Options and arguments """ program = os.path.basename(sys.argv[0]) parser = optparse.OptionParser(usage="%prog [--full-run]", prog=program) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option("--full-run", "-f", dest="full_run", action="store_true", default=False, help=("Make a full run and set permissions" " on archived jobs (time consuming)")) return parser.parse_args() def Main(): """Main routine. """ (opts, args) = ParseOptions() utils.SetupToolLogging(opts.debug, opts.verbose) if args: logging.error("No arguments are expected") return constants.EXIT_FAILURE if opts.full_run: logging.info("Running in full mode") getent = runtime.GetEnts() try: for path in GetPaths(): ProcessPath(path) if opts.full_run: RecursiveEnsure(pathutils.JOB_QUEUE_ARCHIVE_DIR, getent.masterd_uid, getent.daemons_gid, 0o750, constants.JOB_QUEUE_FILES_PERMS) except errors.GenericError as err: logging.error("An error occurred while setting permissions: %s", err) return constants.EXIT_FAILURE return constants.EXIT_SUCCESS ganeti-3.1.0~rc2/lib/tools/node_cleanup.py000064400000000000000000000077471476477700300205370ustar00rootroot00000000000000# # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to configure the node daemon. """ import os import os.path import optparse import sys import logging from ganeti import cli from ganeti import constants from ganeti import pathutils from ganeti import ssconf from ganeti import utils def ParseOptions(): """Parses the options passed to the program. @return: Options and arguments """ parser = optparse.OptionParser(usage="%prog [--no-backup]", prog=os.path.basename(sys.argv[0])) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option(cli.YES_DOIT_OPT) parser.add_option("--no-backup", dest="backup", default=True, action="store_false", help="Whether to create backup copies of deleted files") (opts, args) = parser.parse_args() return VerifyOptions(parser, opts, args) def VerifyOptions(parser, opts, args): """Verifies options and arguments for correctness. """ if args: parser.error("No arguments are expected") return opts def Main(): """Main routine. """ opts = ParseOptions() utils.SetupToolLogging(opts.debug, opts.verbose) try: # List of files to delete. Contains tuples consisting of the absolute path # and a boolean denoting whether a backup copy should be created before # deleting. clean_files = [ (pathutils.CONFD_HMAC_KEY, True), (pathutils.CLUSTER_CONF_FILE, True), (pathutils.CLUSTER_DOMAIN_SECRET_FILE, True), ] clean_files.extend((f, True) for f in pathutils.ALL_CERT_FILES) clean_files.extend((f, False) for f in ssconf.SimpleStore().GetFileList()) if not opts.yes_do_it: cli.ToStderr("Cleaning a node is irreversible. If you really want to" " clean this node, supply the --yes-do-it option.") return constants.EXIT_FAILURE logging.info("Stopping daemons") result = utils.RunCmd([pathutils.DAEMON_UTIL, "stop-all"], interactive=True) if result.failed: raise Exception("Could not stop daemons, command '%s' failed: %s" % (result.cmd, result.fail_reason)) for (filename, backup) in clean_files: if os.path.exists(filename): if opts.backup and backup: logging.info("Backing up %s", filename) utils.CreateBackup(filename) logging.info("Removing %s", filename) utils.RemoveFile(filename) logging.info("Node successfully cleaned") except Exception as err: # pylint: disable=W0703 logging.debug("Caught unhandled exception", exc_info=True) (retcode, message) = cli.FormatError(err) logging.error(message) return retcode else: return constants.EXIT_SUCCESS ganeti-3.1.0~rc2/lib/tools/node_daemon_setup.py000064400000000000000000000125121476477700300215550ustar00rootroot00000000000000# # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to configure the node daemon. """ import os import os.path import optparse import sys import logging from ganeti import cli from ganeti import constants from ganeti import errors from ganeti import pathutils from ganeti import utils from ganeti import runtime from ganeti import ht from ganeti import ssconf from ganeti.tools import common _DATA_CHECK = ht.TStrictDict(False, True, { constants.NDS_CLUSTER_NAME: ht.TNonEmptyString, constants.NDS_NODE_DAEMON_CERTIFICATE: ht.TNonEmptyString, constants.NDS_HMAC: ht.TNonEmptyString, constants.NDS_SSCONF: ht.TDictOf(ht.TNonEmptyString, ht.TString), constants.NDS_START_NODE_DAEMON: ht.TBool, constants.NDS_NODE_NAME: ht.TString, }) class SetupError(errors.GenericError): """Local class for reporting errors. """ def ParseOptions(): """Parses the options passed to the program. @return: Options and arguments """ parser = optparse.OptionParser(usage="%prog [--dry-run]", prog=os.path.basename(sys.argv[0])) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option(cli.DRY_RUN_OPT) (opts, args) = parser.parse_args() return VerifyOptions(parser, opts, args) def VerifyOptions(parser, opts, args): """Verifies options and arguments for correctness. """ if args: parser.error("No arguments are expected") return opts def VerifySsconf(data, cluster_name, _verify_fn=ssconf.VerifyKeys): """Verifies ssconf names. @type data: dict """ items = data.get(constants.NDS_SSCONF) if not items: raise SetupError("Ssconf values must be specified") # TODO: Should all keys be required? Right now any subset of valid keys is # accepted. _verify_fn(list(items)) if items.get(constants.SS_CLUSTER_NAME) != cluster_name: raise SetupError("Cluster name in ssconf does not match") return items def Main(): """Main routine. """ opts = ParseOptions() utils.SetupToolLogging(opts.debug, opts.verbose) try: getent = runtime.GetEnts() data = common.LoadData(sys.stdin.read(), SetupError) cluster_name = common.VerifyClusterName(data, SetupError, constants.NDS_CLUSTER_NAME) cert_pem = common.VerifyCertificateStrong(data, SetupError) hmac_key = common.VerifyHmac(data, SetupError) ssdata = VerifySsconf(data, cluster_name) logging.info("Writing ssconf files ...") ssconf.WriteSsconfFiles(ssdata, dry_run=opts.dry_run) logging.info("Writing hmac.key ...") utils.WriteFile(pathutils.CONFD_HMAC_KEY, data=hmac_key, mode=pathutils.NODED_CERT_MODE, uid=getent.masterd_uid, gid=getent.masterd_gid, dry_run=opts.dry_run) logging.info("Writing node daemon certificate ...") utils.WriteFile(pathutils.NODED_CERT_FILE, data=cert_pem, mode=pathutils.NODED_CERT_MODE, uid=getent.masterd_uid, gid=getent.masterd_gid, dry_run=opts.dry_run) common.GenerateClientCertificate(data, SetupError) if (data.get(constants.NDS_START_NODE_DAEMON) and # pylint: disable=E1103 not opts.dry_run): logging.info("Restarting node daemon ...") stop_cmd = "%s stop-all" % pathutils.DAEMON_UTIL noded_cmd = "%s start %s" % (pathutils.DAEMON_UTIL, constants.NODED) mond_cmd = "" if constants.ENABLE_MOND: mond_cmd = "%s start %s" % (pathutils.DAEMON_UTIL, constants.MOND) cmd = "; ".join([stop_cmd, noded_cmd, mond_cmd]) result = utils.RunCmd(cmd, interactive=True) if result.failed: raise SetupError("Could not start the node daemons, command '%s'" " failed: %s" % (result.cmd, result.fail_reason)) logging.info("Node daemon successfully configured") except Exception as err: # pylint: disable=W0703 logging.debug("Caught unhandled exception", exc_info=True) (retcode, message) = cli.FormatError(err) logging.error(message) return retcode else: return constants.EXIT_SUCCESS ganeti-3.1.0~rc2/lib/tools/prepare_node_join.py000064400000000000000000000152261476477700300215540ustar00rootroot00000000000000# # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to prepare a node for joining a cluster. """ import os import os.path import optparse import sys import logging from ganeti import cli from ganeti import constants from ganeti import errors from ganeti import pathutils from ganeti import utils from ganeti import ht from ganeti import ssh from ganeti.tools import common _SSH_KEY_LIST_ITEM = \ ht.TAnd(ht.TIsLength(3), ht.TItems([ ht.TSshKeyType, ht.Comment("public")(ht.TNonEmptyString), ht.Comment("private")(ht.TNonEmptyString), ])) _SSH_KEY_LIST = ht.TListOf(_SSH_KEY_LIST_ITEM) _DATA_CHECK = ht.TStrictDict(False, True, { constants.SSHS_CLUSTER_NAME: ht.TNonEmptyString, constants.SSHS_NODE_DAEMON_CERTIFICATE: ht.TNonEmptyString, constants.SSHS_SSH_HOST_KEY: _SSH_KEY_LIST, constants.SSHS_SSH_ROOT_KEY: _SSH_KEY_LIST, constants.SSHS_SSH_AUTHORIZED_KEYS: ht.TDictOf(ht.TNonEmptyString, ht.TListOf(ht.TNonEmptyString)), constants.SSHS_SSH_KEY_TYPE: ht.TSshKeyType, constants.SSHS_SSH_KEY_BITS: ht.TPositive, }) class JoinError(errors.GenericError): """Local class for reporting errors. """ def ParseOptions(): """Parses the options passed to the program. @return: Options and arguments """ program = os.path.basename(sys.argv[0]) parser = optparse.OptionParser(usage="%prog [--dry-run]", prog=program) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option(cli.DRY_RUN_OPT) (opts, args) = parser.parse_args() return common.VerifyOptions(parser, opts, args) def _UpdateKeyFiles(keys, dry_run, keyfiles): """Updates SSH key files. @type keys: sequence of tuple; (string, string, string) @param keys: Keys to write, tuples consist of key type (L{constants.SSHK_ALL}), public and private key @type dry_run: boolean @param dry_run: Whether to perform a dry run @type keyfiles: dict; (string as key, tuple with (string, string) as values) @param keyfiles: Mapping from key types (L{constants.SSHK_ALL}) to file names; value tuples consist of public key filename and private key filename """ assert set(keyfiles) == constants.SSHK_ALL for (kind, private_key, public_key) in keys: (private_file, public_file) = keyfiles[kind] logging.debug("Writing %s ...", private_file) utils.WriteFile(private_file, data=private_key, mode=0o600, backup=True, dry_run=dry_run) logging.debug("Writing %s ...", public_file) utils.WriteFile(public_file, data=public_key, mode=0o644, backup=True, dry_run=dry_run) def UpdateSshDaemon(data, dry_run, _runcmd_fn=utils.RunCmd, _keyfiles=None): """Updates SSH daemon's keys. Unless C{dry_run} is set, the daemon is restarted at the end. @type data: dict @param data: Input data @type dry_run: boolean @param dry_run: Whether to perform a dry run """ keys = data.get(constants.SSHS_SSH_HOST_KEY) if not keys: return if _keyfiles is None: _keyfiles = constants.SSH_DAEMON_KEYFILES logging.info("Updating SSH daemon key files") _UpdateKeyFiles(keys, dry_run, _keyfiles) if dry_run: logging.info("This is a dry run, not restarting SSH daemon") else: result = _runcmd_fn([pathutils.DAEMON_UTIL, "reload-ssh-keys"], interactive=True) if result.failed: raise JoinError("Could not reload SSH keys, command '%s'" " had exitcode %s and error %s" % (result.cmd, result.exit_code, result.output)) def UpdateSshRoot(data, dry_run, _homedir_fn=None): """Updates root's SSH keys. Root's C{authorized_keys} file is also updated with new public keys. @type data: dict @param data: Input data @type dry_run: boolean @param dry_run: Whether to perform a dry run """ authorized_keys = data.get(constants.SSHS_SSH_AUTHORIZED_KEYS) (auth_keys_file, _) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=True, _homedir_fn=_homedir_fn) if dry_run: logging.info("This is a dry run, not replacing the SSH keys.") else: ssh_key_type = data.get(constants.SSHS_SSH_KEY_TYPE) ssh_key_bits = data.get(constants.SSHS_SSH_KEY_BITS) common.GenerateRootSshKeys(ssh_key_type, ssh_key_bits, error_fn=JoinError, _homedir_fn=_homedir_fn) if authorized_keys: if dry_run: logging.info("This is a dry run, not modifying %s", auth_keys_file) else: all_authorized_keys = [] for keys in authorized_keys.values(): all_authorized_keys += keys ssh.AddAuthorizedKeys(auth_keys_file, all_authorized_keys) def Main(): """Main routine. """ opts = ParseOptions() utils.SetupToolLogging(opts.debug, opts.verbose) try: data = common.LoadData(sys.stdin.read(), _DATA_CHECK) # Check if input data is correct common.VerifyClusterName(data, JoinError, constants.SSHS_CLUSTER_NAME) common.VerifyCertificateSoft(data, JoinError) # Update SSH files UpdateSshDaemon(data, opts.dry_run) UpdateSshRoot(data, opts.dry_run) logging.info("Setup finished successfully") except Exception as err: # pylint: disable=W0703 logging.debug("Caught unhandled exception", exc_info=True) (retcode, message) = cli.FormatError(err) logging.error(message) return retcode else: return constants.EXIT_SUCCESS ganeti-3.1.0~rc2/lib/tools/ssh_update.py000064400000000000000000000164621476477700300202340ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to update a node's SSH key files. This script is used to update the node's 'authorized_keys' and 'ganeti_pub_key' files. It will be called via SSH from the master node. """ import os import os.path import optparse import sys import logging from ganeti import cli from ganeti import constants from ganeti import errors from ganeti import utils from ganeti import ht from ganeti import ssh from ganeti import pathutils from ganeti.tools import common _DATA_CHECK = ht.TStrictDict(False, True, { constants.SSHS_CLUSTER_NAME: ht.TNonEmptyString, constants.SSHS_NODE_DAEMON_CERTIFICATE: ht.TNonEmptyString, constants.SSHS_SSH_PUBLIC_KEYS: ht.TItems( [ht.TElemOf(constants.SSHS_ACTIONS), ht.TDictOf(ht.TNonEmptyString, ht.TListOf(ht.TNonEmptyString))]), constants.SSHS_SSH_AUTHORIZED_KEYS: ht.TItems( [ht.TElemOf(constants.SSHS_ACTIONS), ht.TDictOf(ht.TNonEmptyString, ht.TListOf(ht.TNonEmptyString))]), constants.SSHS_GENERATE: ht.TItems( [ht.TSshKeyType, # The type of key to generate ht.TPositive, # The number of bits in the key ht.TString]), # The suffix constants.SSHS_SSH_KEY_TYPE: ht.TSshKeyType, constants.SSHS_SSH_KEY_BITS: ht.TPositive, }) class SshUpdateError(errors.GenericError): """Local class for reporting errors. """ def ParseOptions(): """Parses the options passed to the program. @return: Options and arguments """ program = os.path.basename(sys.argv[0]) parser = optparse.OptionParser( usage="%prog [--dry-run] [--verbose] [--debug]", prog=program) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option(cli.DRY_RUN_OPT) (opts, args) = parser.parse_args() return common.VerifyOptions(parser, opts, args) def UpdateAuthorizedKeys(data, dry_run, _homedir_fn=None): """Updates root's C{authorized_keys} file. @type data: dict @param data: Input data @type dry_run: boolean @param dry_run: Whether to perform a dry run """ instructions = data.get(constants.SSHS_SSH_AUTHORIZED_KEYS) if not instructions: logging.info("No change to the authorized_keys file requested.") return (action, authorized_keys) = instructions (auth_keys_file, _) = \ ssh.GetAllUserFiles(constants.SSH_LOGIN_USER, mkdir=True, _homedir_fn=_homedir_fn) key_values = [] for key_value in authorized_keys.values(): key_values += key_value if action == constants.SSHS_ADD: if dry_run: logging.info("This is a dry run, not adding keys to %s", auth_keys_file) else: if not os.path.exists(auth_keys_file): utils.WriteFile(auth_keys_file, mode=0o600, data="") ssh.AddAuthorizedKeys(auth_keys_file, key_values) elif action == constants.SSHS_REMOVE: if dry_run: logging.info("This is a dry run, not removing keys from %s", auth_keys_file) else: ssh.RemoveAuthorizedKeys(auth_keys_file, key_values) else: raise SshUpdateError("Action '%s' not implemented for authorized keys." % action) def UpdatePubKeyFile(data, dry_run, key_file=pathutils.SSH_PUB_KEYS): """Updates the file of public SSH keys. @type data: dict @param data: Input data @type dry_run: boolean @param dry_run: Whether to perform a dry run """ instructions = data.get(constants.SSHS_SSH_PUBLIC_KEYS) if not instructions: logging.info("No instructions to modify public keys received." " Not modifying the public key file at all.") return (action, public_keys) = instructions if action == constants.SSHS_OVERRIDE: if dry_run: logging.info("This is a dry run, not overriding %s", key_file) else: ssh.OverridePubKeyFile(public_keys, key_file=key_file) elif action in [constants.SSHS_ADD, constants.SSHS_REPLACE_OR_ADD]: if dry_run: logging.info("This is a dry run, not adding or replacing a key to %s", key_file) else: for uuid, keys in public_keys.items(): if action == constants.SSHS_REPLACE_OR_ADD: ssh.RemovePublicKey(uuid, key_file=key_file) for key in keys: ssh.AddPublicKey(uuid, key, key_file=key_file) elif action == constants.SSHS_REMOVE: if dry_run: logging.info("This is a dry run, not removing keys from %s", key_file) else: for uuid in public_keys.keys(): ssh.RemovePublicKey(uuid, key_file=key_file) elif action == constants.SSHS_CLEAR: if dry_run: logging.info("This is a dry run, not clearing file %s", key_file) else: ssh.ClearPubKeyFile(key_file=key_file) else: raise SshUpdateError("Action '%s' not implemented for public keys." % action) def GenerateRootSshKeys(data, dry_run): """(Re-)generates the root SSH keys. @type data: dict @param data: Input data @type dry_run: boolean @param dry_run: Whether to perform a dry run """ generate_info = data.get(constants.SSHS_GENERATE) if generate_info: key_type, key_bits, suffix = generate_info if dry_run: logging.info("This is a dry run, not generating any files.") else: common.GenerateRootSshKeys(key_type, key_bits, SshUpdateError, _suffix=suffix) def Main(): """Main routine. """ opts = ParseOptions() utils.SetupToolLogging(opts.debug, opts.verbose) try: data = common.LoadData(sys.stdin.read(), _DATA_CHECK) # Check if input data is correct common.VerifyClusterName(data, SshUpdateError, constants.SSHS_CLUSTER_NAME) common.VerifyCertificateSoft(data, SshUpdateError) # Update / Generate SSH files UpdateAuthorizedKeys(data, opts.dry_run) UpdatePubKeyFile(data, opts.dry_run) GenerateRootSshKeys(data, opts.dry_run) logging.info("Setup finished successfully") except Exception as err: # pylint: disable=W0703 logging.debug("Caught unhandled exception", exc_info=True) (retcode, message) = cli.FormatError(err) logging.error(message) return retcode else: return constants.EXIT_SUCCESS ganeti-3.1.0~rc2/lib/tools/ssl_update.py000064400000000000000000000110521476477700300202260ustar00rootroot00000000000000# # # Copyright (C) 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script to recreate and sign the client SSL certificates. """ import os import os.path import optparse import sys import logging from ganeti import cli from ganeti import constants from ganeti import errors from ganeti import utils from ganeti import ht from ganeti import pathutils from ganeti.tools import common _DATA_CHECK = ht.TStrictDict(False, True, { constants.NDS_CLUSTER_NAME: ht.TNonEmptyString, constants.NDS_NODE_DAEMON_CERTIFICATE: ht.TNonEmptyString, constants.NDS_NODE_NAME: ht.TNonEmptyString, constants.NDS_ACTION: ht.TNonEmptyString, }) class SslSetupError(errors.GenericError): """Local class for reporting errors. """ def ParseOptions(): """Parses the options passed to the program. @return: Options and arguments """ parser = optparse.OptionParser(usage="%prog [--dry-run]", prog=os.path.basename(sys.argv[0])) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option(cli.DRY_RUN_OPT) (opts, args) = parser.parse_args() return common.VerifyOptions(parser, opts, args) def DeleteClientCertificate(): """Deleting the client certificate. This is necessary for downgrades.""" if os.path.exists(pathutils.NODED_CLIENT_CERT_FILE): os.remove(pathutils.NODED_CLIENT_CERT_FILE) else: logging.debug("Trying to delete the client certificate '%s' which did not" " exist.", pathutils.NODED_CLIENT_CERT_FILE) def ClearMasterCandidateSsconfList(): """Clear the ssconf list of master candidate certs. This is necessary when deleting the client certificates for a downgrade, because otherwise the master cannot distribute the configuration to the nodes via RPC during a downgrade anymore. """ ssconf_file = os.path.join( pathutils.DATA_DIR, "%s%s" % (constants.SSCONF_FILEPREFIX, constants.SS_MASTER_CANDIDATES_CERTS)) if os.path.exists: os.remove(ssconf_file) else: logging.debug("Trying to delete the ssconf file '%s' which does not" " exist.", ssconf_file) # pylint: disable=E1103 # This pyling message complains about 'data' as 'bool' not having a get # member, but obviously the type is wrongly inferred. def Main(): """Main routine. """ opts = ParseOptions() utils.SetupToolLogging(opts.debug, opts.verbose) try: data = common.LoadData(sys.stdin.read(), _DATA_CHECK) common.VerifyClusterName(data, SslSetupError, constants.NDS_CLUSTER_NAME) # Verifies whether the server certificate of the caller # is the same as on this node. common.VerifyCertificateStrong(data, SslSetupError) action = data.get(constants.NDS_ACTION) if not action: raise SslSetupError("No Action specified.") if action == constants.CRYPTO_ACTION_CREATE: common.GenerateClientCertificate(data, SslSetupError) elif action == constants.CRYPTO_ACTION_DELETE: DeleteClientCertificate() ClearMasterCandidateSsconfList() else: raise SslSetupError("Unsupported action: %s." % action) except Exception as err: # pylint: disable=W0703 logging.debug("Caught unhandled exception", exc_info=True) (retcode, message) = cli.FormatError(err) logging.error(message) return retcode else: return constants.EXIT_SUCCESS ganeti-3.1.0~rc2/lib/uidpool.py000064400000000000000000000304121476477700300163770ustar00rootroot00000000000000# # # Copyright (C) 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """User-id pool related functions. The user-id pool is cluster-wide configuration option. It is stored as a list of user-id ranges. This module contains functions used for manipulating the user-id pool parameter and for requesting/returning user-ids from the pool. """ import errno import logging import os import random from ganeti import errors from ganeti import constants from ganeti import utils from ganeti import pathutils def ParseUidPool(value, separator=None): """Parse a user-id pool definition. @param value: string representation of the user-id pool. The accepted input format is a list of integer ranges. The boundaries are inclusive. Example: '1000-5000,8000,9000-9010'. @param separator: the separator character between the uids/uid-ranges. Defaults to a comma. @return: a list of integer pairs (lower, higher range boundaries) """ if separator is None: separator = "," ranges = [] for range_def in value.split(separator): if not range_def: # Skip empty strings continue boundaries = range_def.split("-") n_elements = len(boundaries) if n_elements > 2: raise errors.OpPrereqError( "Invalid user-id range definition. Only one hyphen allowed: %s" % boundaries, errors.ECODE_INVAL) try: lower = int(boundaries[0]) except (ValueError, TypeError) as err: raise errors.OpPrereqError("Invalid user-id value for lower boundary of" " user-id range: %s" % str(err), errors.ECODE_INVAL) try: higher = int(boundaries[n_elements - 1]) except (ValueError, TypeError) as err: raise errors.OpPrereqError("Invalid user-id value for higher boundary of" " user-id range: %s" % str(err), errors.ECODE_INVAL) ranges.append((lower, higher)) ranges.sort() return ranges def AddToUidPool(uid_pool, add_uids): """Add a list of user-ids/user-id ranges to a user-id pool. @param uid_pool: a user-id pool (list of integer tuples) @param add_uids: user-id ranges to be added to the pool (list of integer tuples) """ for uid_range in add_uids: if uid_range not in uid_pool: uid_pool.append(uid_range) uid_pool.sort() def RemoveFromUidPool(uid_pool, remove_uids): """Remove a list of user-ids/user-id ranges from a user-id pool. @param uid_pool: a user-id pool (list of integer tuples) @param remove_uids: user-id ranges to be removed from the pool (list of integer tuples) """ for uid_range in remove_uids: if uid_range not in uid_pool: raise errors.OpPrereqError( "User-id range to be removed is not found in the current" " user-id pool: %s" % str(uid_range), errors.ECODE_INVAL) uid_pool.remove(uid_range) def _FormatUidRange(lower, higher): """Convert a user-id range definition into a string. """ if lower == higher: return str(lower) return "%s-%s" % (lower, higher) def FormatUidPool(uid_pool, separator=None): """Convert the internal representation of the user-id pool into a string. The output format is also accepted by ParseUidPool() @param uid_pool: a list of integer pairs representing UID ranges @param separator: the separator character between the uids/uid-ranges. Defaults to ", ". @return: a string with the formatted results """ if separator is None: separator = ", " return separator.join([_FormatUidRange(lower, higher) for lower, higher in uid_pool]) def CheckUidPool(uid_pool): """Sanity check user-id pool range definition values. @param uid_pool: a list of integer pairs (lower, higher range boundaries) """ for lower, higher in uid_pool: if lower > higher: raise errors.OpPrereqError( "Lower user-id range boundary value (%s)" " is larger than higher boundary value (%s)" % (lower, higher), errors.ECODE_INVAL) if lower < constants.UIDPOOL_UID_MIN: raise errors.OpPrereqError( "Lower user-id range boundary value (%s)" " is smaller than UIDPOOL_UID_MIN (%s)." % (lower, constants.UIDPOOL_UID_MIN), errors.ECODE_INVAL) if higher > constants.UIDPOOL_UID_MAX: raise errors.OpPrereqError( "Higher user-id boundary value (%s)" " is larger than UIDPOOL_UID_MAX (%s)." % (higher, constants.UIDPOOL_UID_MAX), errors.ECODE_INVAL) def ExpandUidPool(uid_pool): """Expands a uid-pool definition to a list of uids. @param uid_pool: a list of integer pairs (lower, higher range boundaries) @return: a list of integers """ uids = set() for lower, higher in uid_pool: uids.update(range(lower, higher + 1)) return list(uids) def _IsUidUsed(uid): """Check if there is any process in the system running with the given user-id @type uid: integer @param uid: the user-id to be checked. """ pgrep_command = [constants.PGREP, "-u", uid] result = utils.RunCmd(pgrep_command) if result.exit_code == 0: return True elif result.exit_code == 1: return False else: raise errors.CommandError("Running pgrep failed. exit code: %s" % result.exit_code) class LockedUid(object): """Class representing a locked user-id in the uid-pool. This binds together a userid and a lock. """ def __init__(self, uid, lock): """Constructor @param uid: a user-id @param lock: a utils.FileLock object """ self._uid = uid self._lock = lock def Unlock(self): # Release the exclusive lock and close the filedescriptor self._lock.Close() def GetUid(self): return self._uid def AsStr(self): return "%s" % self._uid def RequestUnusedUid(all_uids): """Tries to find an unused uid from the uid-pool, locks it and returns it. Usage pattern ============= 1. When starting a process:: from ganeti import ssconf from ganeti import uidpool # Get list of all user-ids in the uid-pool from ssconf ss = ssconf.SimpleStore() uid_pool = uidpool.ParseUidPool(ss.GetUidPool(), separator="\\n") all_uids = set(uidpool.ExpandUidPool(uid_pool)) uid = uidpool.RequestUnusedUid(all_uids) try: # Once the process is started, we can release the file lock uid.Unlock() except ... as err: # Return the UID to the pool uidpool.ReleaseUid(uid) 2. Stopping a process:: from ganeti import uidpool uid = uidpool.ReleaseUid(uid) @type all_uids: set of integers @param all_uids: a set containing all the user-ids in the user-id pool @return: a LockedUid object representing the unused uid. It's the caller's responsibility to unlock the uid once an instance is started with this uid. """ # Create the lock dir if it's not yet present try: utils.EnsureDirs([(pathutils.UIDPOOL_LOCKDIR, 0o755)]) except errors.GenericError as err: raise errors.LockError("Failed to create user-id pool lock dir: %s" % err) # Get list of currently used uids from the filesystem try: taken_uids = set() for taken_uid in os.listdir(pathutils.UIDPOOL_LOCKDIR): try: taken_uid = int(taken_uid) except ValueError as err: # Skip directory entries that can't be converted into an integer continue taken_uids.add(taken_uid) except OSError as err: raise errors.LockError("Failed to get list of used user-ids: %s" % err) # Filter out spurious entries from the directory listing taken_uids = all_uids.intersection(taken_uids) # Remove the list of used uids from the list of all uids unused_uids = list(all_uids - taken_uids) if not unused_uids: logging.info("All user-ids in the uid-pool are marked 'taken'") # Randomize the order of the unused user-id list random.shuffle(unused_uids) # Randomize the order of the unused user-id list taken_uids = list(taken_uids) random.shuffle(taken_uids) for uid in unused_uids + taken_uids: try: # Create the lock file # Note: we don't care if it exists. Only the fact that we can # (or can't) lock it later is what matters. uid_path = utils.PathJoin(pathutils.UIDPOOL_LOCKDIR, str(uid)) lock = utils.FileLock.Open(uid_path) except OSError as err: raise errors.LockError("Failed to create lockfile for user-id %s: %s" % (uid, err)) try: # Try acquiring an exclusive lock on the lock file lock.Exclusive() # Check if there is any process running with this user-id if _IsUidUsed(uid): logging.debug("There is already a process running under" " user-id %s", uid) lock.Unlock() continue return LockedUid(uid, lock) except IOError as err: if err.errno == errno.EAGAIN: # The file is already locked, let's skip it and try another unused uid logging.debug("Lockfile for user-id is already locked %s: %s", uid, err) continue except errors.LockError as err: # There was an unexpected error while trying to lock the file logging.error("Failed to lock the lockfile for user-id %s: %s", uid, err) raise raise errors.LockError("Failed to find an unused user-id") def ReleaseUid(uid): """This should be called when the given user-id is no longer in use. @type uid: LockedUid or integer @param uid: the uid to release back to the pool """ if isinstance(uid, LockedUid): # Make sure we release the exclusive lock, if there is any uid.Unlock() uid_filename = uid.AsStr() else: uid_filename = str(uid) try: uid_path = utils.PathJoin(pathutils.UIDPOOL_LOCKDIR, uid_filename) os.remove(uid_path) except OSError as err: raise errors.LockError("Failed to remove user-id lockfile" " for user-id %s: %s" % (uid_filename, err)) def ExecWithUnusedUid(fn, all_uids, *args, **kwargs): """Execute a callable and provide an unused user-id in its kwargs. This wrapper function provides a simple way to handle the requesting, unlocking and releasing a user-id. "fn" is called by passing a "uid" keyword argument that contains an unused user-id (as an integer) selected from the set of user-ids passed in all_uids. If there is an error while executing "fn", the user-id is returned to the pool. @param fn: a callable that accepts a keyword argument called "uid" @type all_uids: a set of integers @param all_uids: a set containing all user-ids in the user-id pool """ uid = RequestUnusedUid(all_uids) kwargs["uid"] = uid.GetUid() try: return_value = fn(*args, **kwargs) except: # The failure of "callabe" means that starting a process with the uid # failed, so let's put the uid back into the pool. ReleaseUid(uid) raise uid.Unlock() return return_value ganeti-3.1.0~rc2/lib/utils/000075500000000000000000000000001476477700300155125ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/utils/__init__.py000064400000000000000000000700451476477700300176310ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Ganeti utility module. This module holds functions that can be used in both daemons (all) and the command line scripts. """ # Allow wildcard import in pylint: disable=W0401 import os import re import array import errno import pwd import time import itertools import select import socket import logging import signal from ganeti import errors from ganeti import constants from ganeti import compat from ganeti import pathutils from ganeti.utils.algo import * from ganeti.utils.filelock import * from ganeti.utils.hash import * from ganeti.utils.io import * from ganeti.utils.livelock import * from ganeti.utils.log import * from ganeti.utils.lvm import * from ganeti.utils.mlock import * from ganeti.utils.nodesetup import * from ganeti.utils.process import * from ganeti.utils.retry import * from ganeti.utils.security import * from ganeti.utils.storage import * from ganeti.utils.tags import * from ganeti.utils.text import * from ganeti.utils.wrapper import * from ganeti.utils.version import * from ganeti.utils.x509 import * from ganeti.utils.bitarrays import * _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$") UUID_RE = re.compile(constants.UUID_REGEX) def ForceDictType(target, key_types, allowed_values=None): """Force the values of a dict to have certain types. @type target: dict @param target: the dict to update @type key_types: dict @param key_types: dict mapping target dict keys to types in constants.ENFORCEABLE_TYPES @type allowed_values: list @keyword allowed_values: list of specially allowed values """ if allowed_values is None: allowed_values = [] if not isinstance(target, dict): msg = "Expected dictionary, got '%s'" % target raise errors.TypeEnforcementError(msg) for key in target: if key not in key_types: msg = "Unknown parameter '%s'" % key raise errors.TypeEnforcementError(msg) if target[key] in allowed_values: continue ktype = key_types[key] if ktype not in constants.ENFORCEABLE_TYPES: msg = "'%s' has non-enforceable type %s" % (key, ktype) raise errors.ProgrammerError(msg) if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING): if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING: pass elif not isinstance(target[key], str): if isinstance(target[key], bool) and not target[key]: target[key] = "" else: msg = "'%s' (value %s) is not a valid string" % (key, target[key]) raise errors.TypeEnforcementError(msg) elif ktype == constants.VTYPE_BOOL: if isinstance(target[key], str) and target[key]: if target[key].lower() == constants.VALUE_FALSE: target[key] = False elif target[key].lower() == constants.VALUE_TRUE: target[key] = True else: msg = "'%s' (value %s) is not a valid boolean" % (key, target[key]) raise errors.TypeEnforcementError(msg) elif target[key]: target[key] = True else: target[key] = False elif ktype == constants.VTYPE_SIZE: try: target[key] = ParseUnit(target[key]) except errors.UnitParseError as err: msg = "'%s' (value %s) is not a valid size. error: %s" % \ (key, target[key], err) raise errors.TypeEnforcementError(msg) elif ktype == constants.VTYPE_INT: try: target[key] = int(target[key]) except (ValueError, TypeError): msg = "'%s' (value %s) is not a valid integer" % (key, target[key]) raise errors.TypeEnforcementError(msg) elif ktype == constants.VTYPE_FLOAT: try: target[key] = float(target[key]) except (ValueError, TypeError): msg = "'%s' (value %s) is not a valid float" % (key, target[key]) raise errors.TypeEnforcementError(msg) def ValidateServiceName(name): """Validate the given service name. @type name: number or string @param name: Service name or port specification """ try: numport = int(name) except (ValueError, TypeError): # Non-numeric service name valid = _VALID_SERVICE_NAME_RE.match(name) else: # Numeric port (protocols other than TCP or UDP might need adjustments # here) valid = (numport >= 0 and numport < (1 << 16)) if not valid: raise errors.OpPrereqError("Invalid service name '%s'" % name, errors.ECODE_INVAL) return name def _ComputeMissingKeys(key_path, options, defaults): """Helper functions to compute which keys a invalid. @param key_path: The current key path (if any) @param options: The user provided options @param defaults: The default dictionary @return: A list of invalid keys """ defaults_keys = frozenset(defaults) invalid = [] for key, value in options.items(): if key_path: new_path = "%s/%s" % (key_path, key) else: new_path = key if key not in defaults_keys: invalid.append(new_path) elif isinstance(value, dict): invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key])) return invalid def VerifyDictOptions(options, defaults): """Verify a dict has only keys set which also are in the defaults dict. @param options: The user provided options @param defaults: The default dictionary @raise error.OpPrereqError: If one of the keys is not supported """ invalid = _ComputeMissingKeys("", options, defaults) if invalid: raise errors.OpPrereqError("Provided option keys not supported: %s" % CommaJoin(invalid), errors.ECODE_INVAL) def ListVolumeGroups(): """List volume groups and their size @rtype: dict @return: Dictionary with keys volume name and values the size of the volume """ command = "vgs --noheadings --units m --nosuffix -o name,size" result = RunCmd(command) retval = {} if result.failed: return retval for line in result.stdout.splitlines(): try: name, size = line.split() size = int(float(size)) except (IndexError, ValueError) as err: logging.error("Invalid output from vgs (%s): %s", err, line) continue retval[name] = size return retval def BridgeExists(bridge): """Check whether the given bridge exists in the system @type bridge: str @param bridge: the bridge name to check @rtype: boolean @return: True if it does """ return os.path.isdir("/sys/class/net/%s/bridge" % bridge) def TryConvert(fn, val): """Try to convert a value ignoring errors. This function tries to apply function I{fn} to I{val}. If no C{ValueError} or C{TypeError} exceptions are raised, it will return the result, else it will return the original value. Any other exceptions are propagated to the caller. @type fn: callable @param fn: function to apply to the value @param val: the value to be converted @return: The converted value if the conversion was successful, otherwise the original value. """ try: nv = fn(val) except (ValueError, TypeError): nv = val return nv def ParseCpuMask(cpu_mask): """Parse a CPU mask definition and return the list of CPU IDs. CPU mask format: comma-separated list of CPU IDs or dash-separated ID ranges Example: "0-2,5" -> "0,1,2,5" @type cpu_mask: str @param cpu_mask: CPU mask definition @rtype: list of int @return: list of CPU IDs """ if not cpu_mask: return [] cpu_list = [] for range_def in cpu_mask.split(","): boundaries = range_def.split("-") n_elements = len(boundaries) if n_elements > 2: raise errors.ParseError("Invalid CPU ID range definition" " (only one hyphen allowed): %s" % range_def) try: lower = int(boundaries[0]) except (ValueError, TypeError) as err: raise errors.ParseError("Invalid CPU ID value for lower boundary of" " CPU ID range: %s" % str(err)) try: higher = int(boundaries[-1]) except (ValueError, TypeError) as err: raise errors.ParseError("Invalid CPU ID value for higher boundary of" " CPU ID range: %s" % str(err)) if lower > higher: raise errors.ParseError("Invalid CPU ID range definition" " (%d > %d): %s" % (lower, higher, range_def)) cpu_list.extend(range(lower, higher + 1)) return cpu_list def ParseMultiCpuMask(cpu_mask): """Parse a multiple CPU mask definition and return the list of CPU IDs. CPU mask format: colon-separated list of comma-separated list of CPU IDs or dash-separated ID ranges, with optional "all" as CPU value Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ] @type cpu_mask: str @param cpu_mask: multiple CPU mask definition @rtype: list of lists of int @return: list of lists of CPU IDs """ if not cpu_mask: return [] cpu_list = [] for range_def in cpu_mask.split(constants.CPU_PINNING_SEP): if range_def == constants.CPU_PINNING_ALL: cpu_list.append([constants.CPU_PINNING_ALL_VAL, ]) else: # Uniquify and sort the list before adding cpu_list.append(sorted(set(ParseCpuMask(range_def)))) return cpu_list def GetHomeDir(user, default=None): """Try to get the homedir of the given user. The user can be passed either as a string (denoting the name) or as an integer (denoting the user id). If the user is not found, the C{default} argument is returned, which defaults to C{None}. """ try: if isinstance(user, str): result = pwd.getpwnam(user) elif isinstance(user, int): result = pwd.getpwuid(user) else: raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" % type(user)) except KeyError: return default return result.pw_dir def FirstFree(seq, base=0): """Returns the first non-existing integer from seq. The seq argument should be a sorted list of positive integers. The first time the index of an element is smaller than the element value, the index will be returned. The base argument is used to start at a different offset, i.e. C{[3, 4, 6]} with I{offset=3} will return 5. Example: C{[0, 1, 3]} will return I{2}. @type seq: sequence @param seq: the sequence to be analyzed. @type base: int @param base: use this value as the base index of the sequence @rtype: int @return: the first non-used index in the sequence """ for idx, elem in enumerate(seq): assert elem >= base, "Passed element is higher than base offset" if elem > idx + base: # idx is not used return idx + base return None def SingleWaitForFdCondition(fdobj, event, timeout): """Waits for a condition to occur on the socket. Immediately returns at the first interruption. @type fdobj: integer or object supporting a fileno() method @param fdobj: entity to wait for events on @type event: integer @param event: ORed condition (see select module) @type timeout: float or None @param timeout: Timeout in seconds @rtype: int or None @return: None for timeout, otherwise occured conditions """ check = (event | select.POLLPRI | select.POLLNVAL | select.POLLHUP | select.POLLERR) if timeout is not None: # Poller object expects milliseconds timeout *= 1000 poller = select.poll() poller.register(fdobj, event) try: # TODO: If the main thread receives a signal and we have no timeout, we # could wait forever. This should check a global "quit" flag or something # every so often. io_events = poller.poll(timeout) except select.error as err: if err.errno != errno.EINTR: raise io_events = [] if io_events and io_events[0][1] & check: return io_events[0][1] else: return None class FdConditionWaiterHelper(object): """Retry helper for WaitForFdCondition. This class contains the retried and wait functions that make sure WaitForFdCondition can continue waiting until the timeout is actually expired. """ def __init__(self, timeout): self.timeout = timeout def Poll(self, fdobj, event): result = SingleWaitForFdCondition(fdobj, event, self.timeout) if result is None: raise RetryAgain() else: return result def UpdateTimeout(self, timeout): self.timeout = timeout def WaitForFdCondition(fdobj, event, timeout): """Waits for a condition to occur on the socket. Retries until the timeout is expired, even if interrupted. @type fdobj: integer or object supporting a fileno() method @param fdobj: entity to wait for events on @type event: integer @param event: ORed condition (see select module) @type timeout: float or None @param timeout: Timeout in seconds @rtype: int or None @return: None for timeout, otherwise occured conditions """ if timeout is not None: retrywaiter = FdConditionWaiterHelper(timeout) try: result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout, args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout) except RetryTimeout: result = None else: result = None while result is None: result = SingleWaitForFdCondition(fdobj, event, timeout) return result def EnsureDaemon(name): """Check for and start daemon if not alive. @type name: string @param name: daemon name @rtype: bool @return: 'True' if daemon successfully started, 'False' otherwise """ result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name]) if result.failed: logging.error("Can't start daemon '%s', failure %s, output: %s", name, result.fail_reason, result.output) return False return True def StopDaemon(name): """Stop daemon @type name: string @param name: daemon name @rtype: bool @return: 'True' if daemon successfully stopped, 'False' otherwise """ result = RunCmd([pathutils.DAEMON_UTIL, "stop", name]) if result.failed: logging.error("Can't stop daemon '%s', failure %s, output: %s", name, result.fail_reason, result.output) return False return True def SplitTime(value): """Splits time as floating point number into a tuple. @param value: Time in seconds @type value: int or float @return: Tuple containing (seconds, microseconds) """ (seconds, microseconds) = divmod(int(value * 1000000), 1000000) # pylint: disable=C0122 assert 0 <= seconds, \ "Seconds must be larger than or equal to 0, but are %s" % seconds assert 0 <= microseconds <= 999999, \ "Microseconds must be 0-999999, but are %s" % microseconds return (int(seconds), int(microseconds)) def MergeTime(timetuple): """Merges a tuple into time as a floating point number. @param timetuple: Time as tuple, (seconds, microseconds) @type timetuple: tuple @return: Time as a floating point number expressed in seconds """ (seconds, microseconds) = timetuple # pylint: disable=C0122 assert 0 <= seconds, \ "Seconds must be larger than or equal to 0, but are %s" % seconds assert 0 <= microseconds <= 999999, \ "Microseconds must be 0-999999, but are %s" % microseconds return float(seconds) + (float(microseconds) * 0.000001) def EpochNano(): """Return the current timestamp expressed as number of nanoseconds since the unix epoch @return: nanoseconds since the Unix epoch """ return int(time.time() * 1000000000) def FindMatch(data, name): """Tries to find an item in a dictionary matching a name. Callers have to ensure the data names aren't contradictory (e.g. a regexp that matches a string). If the name isn't a direct key, all regular expression objects in the dictionary are matched against it. @type data: dict @param data: Dictionary containing data @type name: string @param name: Name to look for @rtype: tuple; (value in dictionary, matched groups as list) """ if name in data: return (data[name], []) for key, value in data.items(): # Regex objects if hasattr(key, "match"): m = key.match(name) if m: return (value, list(m.groups())) return None def GetMounts(filename=constants.PROC_MOUNTS): """Returns the list of mounted filesystems. This function is Linux-specific. @param filename: path of mounts file (/proc/mounts by default) @rtype: list of tuples @return: list of mount entries (device, mountpoint, fstype, options) """ # TODO(iustin): investigate non-Linux options (e.g. via mount output) data = [] mountlines = ReadFile(filename).splitlines() for line in mountlines: device, mountpoint, fstype, options, _ = line.split(None, 4) data.append((device, mountpoint, fstype, options)) return data def SignalHandled(signums): """Signal Handled decoration. This special decorator installs a signal handler and then calls the target function. The function must accept a 'signal_handlers' keyword argument, which will contain a dict indexed by signal number, with SignalHandler objects as values. The decorator can be safely stacked with iself, to handle multiple signals with different handlers. @type signums: list @param signums: signals to intercept """ def wrap(fn): def sig_function(*args, **kwargs): assert "signal_handlers" not in kwargs or \ kwargs["signal_handlers"] is None or \ isinstance(kwargs["signal_handlers"], dict), \ "Wrong signal_handlers parameter in original function call" if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None: signal_handlers = kwargs["signal_handlers"] else: signal_handlers = {} kwargs["signal_handlers"] = signal_handlers sighandler = SignalHandler(signums) try: for sig in signums: signal_handlers[sig] = sighandler return fn(*args, **kwargs) finally: sighandler.Reset() return sig_function return wrap def TimeoutExpired(epoch, timeout, _time_fn=time.time): """Checks whether a timeout has expired. """ return _time_fn() > (epoch + timeout) class SignalWakeupFd(object): def _SetWakeupFd(self, fd): return signal.set_wakeup_fd(fd) def __init__(self): """Initializes this class. """ (read_fd, write_fd) = os.pipe() # signal.set_wakeup_fd requires the FD to be non-blocking SetNonblockFlag(write_fd, True) # Once these succeeded, the file descriptors will be closed automatically. # Buffer size 0 is important, otherwise .read() with a specified length # might buffer data and the file descriptors won't be marked readable. self._read_fh = os.fdopen(read_fd, "rb", 0) self._write_fh = os.fdopen(write_fd, "wb", 0) self._previous = self._SetWakeupFd(self._write_fh.fileno()) # Utility functions self.fileno = self._read_fh.fileno self.read = self._read_fh.read def Reset(self): """Restores the previous wakeup file descriptor. """ if hasattr(self, "_previous") and self._previous is not None: self._SetWakeupFd(self._previous) self._previous = None def Notify(self): """Notifies the wakeup file descriptor. """ self._write_fh.write(b"\x00") def __del__(self): """Called before object deletion. """ self.Reset() class SignalHandler(object): """Generic signal handler class. It automatically restores the original handler when deconstructed or when L{Reset} is called. You can either pass your own handler function in or query the L{called} attribute to detect whether the signal was sent. @type signum: list @ivar signum: the signals we handle @type called: boolean @ivar called: tracks whether any of the signals have been raised """ def __init__(self, signum, handler_fn=None, wakeup=None): """Constructs a new SignalHandler instance. @type signum: int or list of ints @param signum: Single signal number or set of signal numbers @type handler_fn: callable @param handler_fn: Signal handling function """ assert handler_fn is None or callable(handler_fn) self.signum = set(signum) self.called = False self._handler_fn = handler_fn self._wakeup = wakeup self._previous = {} try: for signum in self.signum: # Setup handler prev_handler = signal.signal(signum, self._HandleSignal) try: self._previous[signum] = prev_handler except: # Restore previous handler signal.signal(signum, prev_handler) raise except: # Reset all handlers self.Reset() # Here we have a race condition: a handler may have already been called, # but there's not much we can do about it at this point. raise def __del__(self): self.Reset() def Reset(self): """Restore previous handler. This will reset all the signals to their previous handlers. """ for signum, prev_handler in list(self._previous.items()): signal.signal(signum, prev_handler) # If successful, remove from dict del self._previous[signum] def Clear(self): """Unsets the L{called} flag. This function can be used in case a signal may arrive several times. """ self.called = False def _HandleSignal(self, signum, frame): """Actual signal handling function. """ # This is not nice and not absolutely atomic, but it appears to be the only # solution in Python -- there are no atomic types. self.called = True if self._wakeup: # Notify whoever is interested in signals self._wakeup.Notify() if self._handler_fn: self._handler_fn(signum, frame) def SetHandlerFn(self, fn): """Set the signal handling function """ self._handler_fn = fn class FieldSet(object): """A simple field set. Among the features are: - checking if a string is among a list of static string or regex objects - checking if a whole list of string matches - returning the matching groups from a regex match Internally, all fields are held as regular expression objects. """ def __init__(self, *items): self.items = [re.compile("^%s$" % value) for value in items] def Extend(self, other_set): """Extend the field set with the items from another one""" self.items.extend(other_set.items) def Matches(self, field): """Checks if a field matches the current set @type field: str @param field: the string to match @return: either None or a regular expression match object """ for m in filter(None, (val.match(field) for val in self.items)): return m return None def NonMatching(self, items): """Returns the list of fields not matching the current set @type items: list @param items: the list of fields to check @rtype: list @return: list of non-matching fields """ return [val for val in items if not self.Matches(val)] def ValidateDeviceNames(kind, container): """Validate instance device names. Check that a device container contains only unique and valid names. @type kind: string @param kind: One-word item description @type container: list @param container: Container containing the devices """ valid = [] for device in container: if isinstance(device, dict): if kind == "NIC": name = device.get(constants.INIC_NAME, None) elif kind == "disk": name = device.get(constants.IDISK_NAME, None) else: raise errors.OpPrereqError("Invalid container kind '%s'" % kind, errors.ECODE_INVAL) else: name = device.name # Check that a device name is not the UUID of another device valid.append(device.uuid) try: int(name) except (ValueError, TypeError): pass else: raise errors.OpPrereqError("Invalid name '%s'. Purely numeric %s names" " are not allowed" % (name, kind), errors.ECODE_INVAL) if name is not None and name.lower() != constants.VALUE_NONE: if name in valid: raise errors.OpPrereqError("%s name '%s' already used" % (kind, name), errors.ECODE_NOTUNIQUE) else: valid.append(name) def AllDiskOfType(disks_info, dev_types): """Checks if the instance has only disks of any of the dev_types. @type disks_info: list of L{Disk} @param disks_info: all the disks of the instance. @type dev_types: list of disk templates @param dev_types: the disk type required. @rtype: bool @return: True iff the instance only has disks of type dev_type. """ assert not isinstance(dev_types, str) if not disks_info and constants.DT_DISKLESS not in dev_types: return False for disk in disks_info: if disk.dev_type not in dev_types: return False return True def AnyDiskOfType(disks_info, dev_types): """Checks if the instance has some disks of any types in dev_types. @type disks_info: list of L{Disk} @param disks_info: all the disks of the instance. @type dev_types: list of disk template @param dev_types: the disk type required. @rtype: bool @return: True if the instance has disks of type dev_types or the instance has no disks and the dev_types allow DT_DISKLESS. """ assert not isinstance(dev_types, str) if not disks_info and constants.DT_DISKLESS in dev_types: return True for disk in disks_info: if disk.dev_type in dev_types: return True return False def GetDiskTemplateString(disk_types): """Gives a summary disk template from disk devtypes. @type disk_types: list of string @param disk_types: all the dev_types of the instance. @rtype disk template @returns the summarized disk template of the disk types. """ disk_types = set(dev_type for dev_type in disk_types) if not disk_types: return constants.DT_DISKLESS elif len(disk_types) > 1: return constants.DT_MIXED else: return disk_types.pop() def GetDiskTemplate(disks_info): """Gives a summary disk template from disks. @type disks_info: list of L{Disk} @param disks_info: all the disks of the instance. @rtype disk template @returns the summarized disk template of the disk types. """ return GetDiskTemplateString(d.dev_type for d in disks_info) def SendFds(sock, data, fds): """Sends a set of file descriptors over a socket using sendmsg(2) @type sock: socket.socket @param sock: the socket over which the fds will be sent @type data: bytes @param data: actual data for the sendmsg(2) call @type fds: list of file descriptors or file-like objects with fileno() methods @param fds: the file descriptors to send """ if not isinstance(data, bytes): raise errors.TypeEnforcementError("expecting bytes for data") _fds = array.array("i") for fd in fds: if isinstance(fd, int): _fds.append(fd) continue try: _fds.append(fd.fileno()) continue except AttributeError: pass raise errors.TypeEnforcementError("expected int or file-like object" " got %s" % type(fd).__name__) return sock.sendmsg([data], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, _fds)]) ganeti-3.1.0~rc2/lib/utils/algo.py000064400000000000000000000236211476477700300170120ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions with algorithms. """ import re import time import numbers import itertools from ganeti import compat from ganeti.utils import text _SORTER_GROUPS = 8 _SORTER_RE = re.compile("^%s(.*)$" % (_SORTER_GROUPS * r"(\D+|\d+)?")) def UniqueSequence(seq): """Returns a list with unique elements. Element order is preserved. @type seq: sequence @param seq: the sequence with the source elements @rtype: list @return: list of unique elements from seq """ seen = set() return [i for i in seq if i not in seen and not seen.add(i)] def JoinDisjointDicts(dict_a, dict_b): """Joins dictionaries with no conflicting keys. Enforces the constraint that the two key sets must be disjoint, and then merges the two dictionaries in a new dictionary that is returned to the caller. @type dict_a: dict @param dict_a: the first dictionary @type dict_b: dict @param dict_b: the second dictionary @rtype: dict @return: a new dictionary containing all the key/value pairs contained in the two dictionaries. """ assert not (set(dict_a) & set(dict_b)), ("Duplicate keys found while joining" " %s and %s" % (dict_a, dict_b)) result = dict_a.copy() result.update(dict_b) return result def FindDuplicates(seq): """Identifies duplicates in a list. Does not preserve element order. @type seq: sequence @param seq: Sequence with source elements @rtype: list @return: List of duplicate elements from seq """ dup = set() seen = set() for item in seq: if item in seen: dup.add(item) else: seen.add(item) return list(dup) def GetRepeatedKeys(*dicts): """Return the set of keys defined multiple times in the given dicts. >>> GetRepeatedKeys({"foo": 1, "bar": 2}, ... {"foo": 5, "baz": 7} ... ) set("foo") @type dicts: dict @param dicts: The dictionaries to check for duplicate keys. @rtype: set @return: Keys used more than once across all dicts """ if len(dicts) < 2: return set() keys = [] for dictionary in dicts: keys.extend(dictionary) return set(FindDuplicates(keys)) class _NiceSortAtom: """Helper class providing rich comparison between different types Wrap an object to provide rich comparison against None, numbers and strings and allow sorting of heterogeneous lists. """ __slots__ = ["_obj"] def __init__(self, obj): if not isinstance(obj, (numbers.Real, str, type(None))): raise ValueError("Cannot wrap type %s" % type(obj)) self._obj = obj def __lt__(self, other): if not isinstance(other, _NiceSortAtom): raise TypeError("Can compare only with _NiceSortAtom") try: return self._obj < other._obj except TypeError: pass if self._obj is None: # None is smaller than anything else return True elif isinstance(self._obj, numbers.Real): if other._obj is None: return False elif isinstance(other._obj, str): return True elif isinstance(self._obj, str): return False raise TypeError("Cannot compare with %s" % type(other._obj)) def __eq__(self, other): if not isinstance(other, _NiceSortAtom): raise TypeError("Can compare only with _NiceSortAtom") return self._obj == other._obj # Mark the rest as NotImplemented and let the intepreter derive them using # __eq__ and __lt__. def __ne__(self, other): return NotImplemented def __gt__(self, other): return NotImplemented def __ge__(self, other): return NotImplemented def __le__(self, other): return NotImplemented def _NiceSortGetKey(val): """Get a suitable sort key. Attempt to convert a value to an integer and wrap it in a _NiceSortKey. """ if val and val.isdigit(): val = int(val) return _NiceSortAtom(val) def NiceSortKey(value): """Extract key for sorting. """ return [_NiceSortGetKey(grp) for grp in _SORTER_RE.match(str(value)).groups()] def NiceSort(values, key=None): """Sort a list of strings based on digit and non-digit groupings. Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function will sort the list in the logical order C{['a1', 'a2', 'a10', 'a11']}. The sort algorithm breaks each name in groups of either only-digits or no-digits. Only the first eight such groups are considered, and after that we just use what's left of the string. @type values: list @param values: the names to be sorted @type key: callable or None @param key: function of one argument to extract a comparison key from each list element, must return string @rtype: list @return: a copy of the name list sorted with our algorithm """ if key is None: keyfunc = NiceSortKey else: keyfunc = lambda value: NiceSortKey(key(value)) return sorted(values, key=keyfunc) def InvertDict(dict_in): """Inverts the key/value mapping of a dict. @param dict_in: The dict to invert @return: the inverted dict """ return dict(zip(dict_in.values(), dict_in.keys())) def InsertAtPos(src, pos, other): """Inserts C{other} at given C{pos} into C{src}. @note: This function does not modify C{src} in place but returns a new copy @type src: list @param src: The source list in which we want insert elements @type pos: int @param pos: The position where we want to start insert C{other} @type other: list @param other: The other list to insert into C{src} @return: A copy of C{src} with C{other} inserted at C{pos} """ new = src[:pos] new.extend(other) new.extend(src[pos:]) return new def SequenceToDict(seq, key=compat.fst): """Converts a sequence to a dictionary with duplicate detection. @type seq: sequen @param seq: Input sequence @type key: callable @param key: Function for retrieving dictionary key from sequence element @rtype: dict """ keys = [key(s) for s in seq] duplicates = FindDuplicates(keys) if duplicates: raise ValueError("Duplicate keys found: %s" % text.CommaJoin(duplicates)) assert len(keys) == len(seq) return dict(zip(keys, seq)) def _MakeFlatToDict(data): """Helper function for C{FlatToDict}. This function is recursively called @param data: The input data as described in C{FlatToDict}, already splitted @returns: The so far converted dict """ if not compat.fst(compat.fst(data)): assert len(data) == 1, \ "not bottom most element, found %d elements, expected 1" % len(data) return compat.snd(compat.fst(data)) keyfn = lambda e: compat.fst(e).pop(0) return dict([(k, _MakeFlatToDict(list(g))) for (k, g) in itertools.groupby(sorted(data), keyfn)]) def FlatToDict(data, field_sep="/"): """Converts a flat structure to a fully fledged dict. It accept a list of tuples in the form:: [ ("foo/bar", {"key1": "data1", "key2": "data2"}), ("foo/baz", {"key3" :"data3" }), ] where the first element is the key separated by C{field_sep}. This would then return:: { "foo": { "bar": {"key1": "data1", "key2": "data2"}, "baz": {"key3" :"data3" }, }, } @type data: list of tuple @param data: Input list to convert @type field_sep: str @param field_sep: The separator for the first field of the tuple @returns: A dict based on the input list """ return _MakeFlatToDict([(keys.split(field_sep), value) for (keys, value) in data]) class RunningTimeout(object): """Class to calculate remaining timeout when doing several operations. """ __slots__ = [ "_allow_negative", "_start_time", "_time_fn", "_timeout", ] def __init__(self, timeout, allow_negative, _time_fn=time.time): """Initializes this class. @type timeout: float @param timeout: Timeout duration @type allow_negative: bool @param allow_negative: Whether to return values below zero @param _time_fn: Time function for unittests """ object.__init__(self) if timeout is not None and timeout < 0.0: raise ValueError("Timeout must not be negative") self._timeout = timeout self._allow_negative = allow_negative self._time_fn = _time_fn self._start_time = None def Remaining(self): """Returns the remaining timeout. """ if self._timeout is None: return None # Get start time on first calculation if self._start_time is None: self._start_time = self._time_fn() # Calculate remaining time remaining_timeout = self._start_time + self._timeout - self._time_fn() if not self._allow_negative: # Ensure timeout is always >= 0 return max(0.0, remaining_timeout) return remaining_timeout ganeti-3.1.0~rc2/lib/utils/bitarrays.py000064400000000000000000000045611476477700300200720ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for managing bitarrays. """ from bitarray import bitarray from ganeti import errors # Constant bitarray that reflects to a free slot # Use it with bitarray.search() _AVAILABLE_SLOT = bitarray("0") def GetFreeSlot(slots, slot=None, reserve=False): """Helper method to get first available slot in a bitarray @type slots: bitarray @param slots: the bitarray to operate on @type slot: integer @param slot: if given we check whether the slot is free @type reserve: boolean @param reserve: whether to reserve the first available slot or not @return: the idx of the (first) available slot @raise errors.OpPrereqError: If all slots in a bitarray are occupied or the given slot is not free. """ if slot is not None: assert slot < len(slots) if slots[slot]: raise errors.GenericError("Slot %d occupied" % slot) else: avail = slots.search(_AVAILABLE_SLOT, 1) if not avail: raise errors.GenericError("All slots occupied") slot = int(avail[0]) if reserve: slots[slot] = True return slot ganeti-3.1.0~rc2/lib/utils/filelock.py000064400000000000000000000132141476477700300176550ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for file-based locks. """ import fcntl import errno import os import logging from ganeti import errors from ganeti.utils import retry def LockFile(fd): """Locks a file using POSIX locks. @type fd: int @param fd: the file descriptor we need to lock """ try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as err: if err.errno == errno.EAGAIN: raise errors.LockError("File already locked") raise class FileLock(object): """Utility class for file locks. """ def __init__(self, fd, filename): """Constructor for FileLock. @type fd: file @param fd: File object @type filename: str @param filename: Path of the file opened at I{fd} """ self.fd = fd self.filename = filename @classmethod def Open(cls, filename): """Creates and opens a file to be used as a file-based lock. @type filename: string @param filename: path to the file to be locked """ # Using "os.open" is necessary to allow both opening existing file # read/write and creating if not existing. Vanilla "open" will truncate an # existing file -or- allow creating if not existing. _flags = os.O_RDWR | os.O_CREAT return cls(os.fdopen(os.open(filename, _flags, 0o664), "w+"), filename) def __del__(self): self.Close() def Close(self): """Close the file and release the lock. """ if hasattr(self, "fd") and self.fd: self.fd.close() self.fd = None def _flock(self, flag, blocking, timeout, errmsg): """Wrapper for fcntl.flock. @type flag: int @param flag: operation flag @type blocking: bool @param blocking: whether the operation should be done in blocking mode. @type timeout: None or float @param timeout: for how long the operation should be retried (implies non-blocking mode). @type errmsg: string @param errmsg: error message in case operation fails. """ assert self.fd, "Lock was closed" assert timeout is None or timeout >= 0, \ "If specified, timeout must be positive" assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set" # When a timeout is used, LOCK_NB must always be set if not (timeout is None and blocking): flag |= fcntl.LOCK_NB if timeout is None: self._Lock(self.fd, flag, timeout) else: try: retry.Retry(self._Lock, (0.1, 1.2, 1.0), timeout, args=(self.fd, flag, timeout)) except retry.RetryTimeout: raise errors.LockError(errmsg) @staticmethod def _Lock(fd, flag, timeout): try: fcntl.flock(fd, flag) except IOError as err: if timeout is not None and err.errno == errno.EAGAIN: raise retry.RetryAgain() logging.exception("fcntl.flock failed") raise def Exclusive(self, blocking=False, timeout=None): """Locks the file in exclusive mode. @type blocking: boolean @param blocking: whether to block and wait until we can lock the file or return immediately @type timeout: int or None @param timeout: if not None, the duration to wait for the lock (in blocking mode) """ self._flock(fcntl.LOCK_EX, blocking, timeout, "Failed to lock %s in exclusive mode" % self.filename) def Shared(self, blocking=False, timeout=None): """Locks the file in shared mode. @type blocking: boolean @param blocking: whether to block and wait until we can lock the file or return immediately @type timeout: int or None @param timeout: if not None, the duration to wait for the lock (in blocking mode) """ self._flock(fcntl.LOCK_SH, blocking, timeout, "Failed to lock %s in shared mode" % self.filename) def Unlock(self, blocking=True, timeout=None): """Unlocks the file. According to C{flock(2)}, unlocking can also be a nonblocking operation:: To make a non-blocking request, include LOCK_NB with any of the above operations. @type blocking: boolean @param blocking: whether to block and wait until we can lock the file or return immediately @type timeout: int or None @param timeout: if not None, the duration to wait for the lock (in blocking mode) """ self._flock(fcntl.LOCK_UN, blocking, timeout, "Failed to unlock %s" % self.filename) ganeti-3.1.0~rc2/lib/utils/hash.py000064400000000000000000000062611476477700300170140ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for hashing. """ import os import hmac from hashlib import sha1 from ganeti import compat def Sha1Hmac(key, msg, salt=None): """Calculates the HMAC-SHA1 digest of a message. HMAC is defined in RFC2104. @type key: string @param key: Secret key @type msg: string or bytes """ key = key.encode("utf-8") if isinstance(msg, str): msg = msg.encode("utf-8") if salt: salt = salt.encode("utf-8") salted_msg = salt + msg else: salted_msg = msg return hmac.new(key, salted_msg, sha1).hexdigest() def VerifySha1Hmac(key, text, digest, salt=None): """Verifies the HMAC-SHA1 digest of a text. HMAC is defined in RFC2104. @type key: string @param key: Secret key @type text: string @type digest: string @param digest: Expected digest @rtype: bool @return: Whether HMAC-SHA1 digest matches """ return digest.lower() == Sha1Hmac(key, text, salt=salt).lower() def _FingerprintFile(filename): """Compute the fingerprint of a file. If the file does not exist, a None will be returned instead. @type filename: str @param filename: the filename to checksum @rtype: str @return: the hex digest of the sha checksum of the contents of the file """ if not (os.path.exists(filename) and os.path.isfile(filename)): return None fp = sha1() with open(filename, "rb") as f: while True: data = f.read(4096) if not data: break fp.update(data) return fp.hexdigest() def FingerprintFiles(files): """Compute fingerprints for a list of files. @type files: list @param files: the list of filename to fingerprint @rtype: dict @return: a dictionary filename: fingerprint, holding only existing files """ ret = {} for filename in files: cksum = _FingerprintFile(filename) if cksum: ret[filename] = cksum return ret ganeti-3.1.0~rc2/lib/utils/io.py000064400000000000000000000736641476477700300165130ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for I/O. """ import os import re import logging import shutil import tempfile import errno import time import stat import grp import pwd from io import FileIO, TextIOWrapper, BufferedRWPair from ganeti import errors from ganeti import constants from ganeti import pathutils from ganeti.utils import filelock #: Directory used by fsck(8) to store recovered data, usually at a file #: system's root directory _LOST_AND_FOUND = "lost+found" # Possible values for keep_perms in WriteFile() KP_NEVER = 0 KP_ALWAYS = 1 KP_IF_EXISTS = 2 KEEP_PERMS_VALUES = [ KP_NEVER, KP_ALWAYS, KP_IF_EXISTS, ] def ErrnoOrStr(err): """Format an EnvironmentError exception. If the L{err} argument has an errno attribute, it will be looked up and converted into a textual C{E...} description. Otherwise the string representation of the error will be returned. @type err: L{EnvironmentError} @param err: the exception to format """ if hasattr(err, "errno"): detail = errno.errorcode[err.errno] else: detail = str(err) return detail class FileStatHelper(object): """Helper to store file handle's C{fstat}. Useful in combination with L{ReadFile}'s C{preread} parameter. """ def __init__(self): """Initializes this class. """ self.st = None def __call__(self, fh): """Calls C{fstat} on file handle. """ self.st = os.fstat(fh.fileno()) def ReadBinaryFile(file_name, size=-1, preread=None): """Reads a binary file. @type size: int @param size: Read at most size bytes (if negative, entire file) @type preread: callable receiving file handle as single parameter @param preread: Function called before file is read @rtype: bytes @return: the (possibly partial) content of the file @raise IOError: if the file cannot be opened """ f = open(file_name, "rb") try: if preread: preread(f) return f.read(size) finally: f.close() def ReadFile(file_name, size=-1, preread=None): """Reads a text file. @type size: int @param size: Read at most size bytes (if negative, entire file) @type preread: callable receiving file handle as single parameter @param preread: Function called before file is read @rtype: str @return: the (possibly partial) content of the file @raise IOError: if the file cannot be opened """ f = open(file_name, "r", encoding="utf-8") try: if preread: preread(f) return f.read(size) finally: f.close() def WriteFile(file_name, fn=None, data=None, mode=None, uid=-1, gid=-1, atime=None, mtime=None, close=True, dry_run=False, backup=False, prewrite=None, postwrite=None, keep_perms=KP_NEVER): """(Over)write a file atomically. The file_name and either fn (a function taking one argument, the file descriptor, and which should write the data to it) or data (the contents of the file) must be passed. The other arguments are optional and allow setting the file mode, owner and group, and the mtime/atime of the file. If the function doesn't raise an exception, it has succeeded and the target file has the new contents. If the function has raised an exception, an existing target file should be unmodified and the temporary file should be removed. @type file_name: str @param file_name: the target filename @type fn: callable @param fn: content writing function, called with file descriptor as parameter @type data: str or bytes @param data: contents of the file @type mode: int @param mode: file mode @type uid: int @param uid: the owner of the file @type gid: int @param gid: the group of the file @type atime: int @param atime: a custom access time to be set on the file @type mtime: int @param mtime: a custom modification time to be set on the file @type close: boolean @param close: whether to close file after writing it @type prewrite: callable @param prewrite: function to be called before writing content @type postwrite: callable @param postwrite: function to be called after writing content @type keep_perms: members of L{KEEP_PERMS_VALUES} @param keep_perms: if L{KP_NEVER} (default), owner, group, and mode are taken from the other parameters; if L{KP_ALWAYS}, owner, group, and mode are copied from the existing file; if L{KP_IF_EXISTS}, owner, group, and mode are taken from the file, and if the file doesn't exist, they are taken from the other parameters. It is an error to pass L{KP_ALWAYS} when the file doesn't exist or when C{uid}, C{gid}, or C{mode} are set to non-default values. @rtype: None or int @return: None if the 'close' parameter evaluates to True, otherwise the file descriptor @raise errors.ProgrammerError: if any of the arguments are not valid """ if not os.path.isabs(file_name): raise errors.ProgrammerError("Path passed to WriteFile is not" " absolute: '%s'" % file_name) if [fn, data].count(None) != 1: raise errors.ProgrammerError("fn or data required") if [atime, mtime].count(None) == 1: raise errors.ProgrammerError("Both atime and mtime must be either" " set or None") if keep_perms not in KEEP_PERMS_VALUES: raise errors.ProgrammerError("Invalid value for keep_perms: %s" % keep_perms) if keep_perms == KP_ALWAYS and (uid != -1 or gid != -1 or mode is not None): raise errors.ProgrammerError("When keep_perms==KP_ALWAYS, 'uid', 'gid'," " and 'mode' cannot be set") if backup and not dry_run and os.path.isfile(file_name): CreateBackup(file_name) if keep_perms == KP_ALWAYS or keep_perms == KP_IF_EXISTS: # os.stat() raises an exception if the file doesn't exist try: file_stat = os.stat(file_name) mode = stat.S_IMODE(file_stat.st_mode) uid = file_stat.st_uid gid = file_stat.st_gid except OSError: if keep_perms == KP_ALWAYS: raise # else: if keeep_perms == KP_IF_EXISTS it's ok if the file doesn't exist # Whether temporary file needs to be removed (e.g. if any error occurs) do_remove = True # Function result result = None (dir_name, base_name) = os.path.split(file_name) (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name, dir=dir_name) try: try: if uid != -1 or gid != -1: os.chown(new_name, uid, gid) if mode: os.chmod(new_name, mode) if callable(prewrite): prewrite(fd) if data is not None: assert isinstance(data, (str, bytes)) if isinstance(data, str): open_mode = "w" encoding = "utf-8" else: open_mode = "wb" encoding = None with open(fd, open_mode, closefd=False, encoding=encoding, buffering=8192) as out: out.write(data) else: fn(fd) if callable(postwrite): postwrite(fd) os.fsync(fd) if atime is not None and mtime is not None: os.utime(new_name, (atime, mtime)) finally: # Close file unless the file descriptor should be returned if close: os.close(fd) else: result = fd # Rename file to destination name if not dry_run: os.rename(new_name, file_name) # Successful, no need to remove anymore do_remove = False finally: if do_remove: RemoveFile(new_name) return result def GetFileID(path=None, fd=None): """Returns the file 'id', i.e. the dev/inode and mtime information. Either the path to the file or the fd must be given. @param path: the file path @param fd: a file descriptor @return: a tuple of (device number, inode number, mtime) """ if [path, fd].count(None) != 1: raise errors.ProgrammerError("One and only one of fd/path must be given") if fd is None: st = os.stat(path) else: st = os.fstat(fd) return (st.st_dev, st.st_ino, st.st_mtime) def VerifyFileID(fi_disk, fi_ours): """Verifies that two file IDs are matching. Differences in the inode/device are not accepted, but and older timestamp for fi_disk is accepted. @param fi_disk: tuple (dev, inode, mtime) representing the actual file data @param fi_ours: tuple (dev, inode, mtime) representing the last written file data @rtype: boolean """ (d1, i1, m1) = fi_disk (d2, i2, m2) = fi_ours return (d1, i1) == (d2, i2) and m1 <= m2 def SafeWriteFile(file_name, file_id, **kwargs): """Wraper over L{WriteFile} that locks the target file. By keeping the target file locked during WriteFile, we ensure that cooperating writers will safely serialise access to the file. @type file_name: str @param file_name: the target filename @type file_id: tuple @param file_id: a result from L{GetFileID} """ fd = os.open(file_name, os.O_RDONLY | os.O_CREAT) try: filelock.LockFile(fd) if file_id is not None: disk_id = GetFileID(fd=fd) if not VerifyFileID(disk_id, file_id): raise errors.LockError("Cannot overwrite file %s, it has been modified" " since last written" % file_name) return WriteFile(file_name, **kwargs) finally: os.close(fd) def ReadOneLineFile(file_name, strict=False): """Return the first non-empty line from a file. @type strict: boolean @param strict: if True, abort if the file has more than one non-empty line """ file_lines = ReadFile(file_name).splitlines() full_lines = [l for l in file_lines if l] if not file_lines or not full_lines: raise errors.GenericError("No data in one-liner file %s" % file_name) elif strict and len(full_lines) > 1: raise errors.GenericError("Too many lines in one-liner file %s" % file_name) return full_lines[0] def RemoveFile(filename): """Remove a file ignoring some errors. Remove a file, ignoring non-existing ones or directories. Other errors are passed. @type filename: str @param filename: the file to be removed """ try: os.unlink(filename) except OSError as err: if err.errno not in (errno.ENOENT, errno.EISDIR): raise def RemoveDir(dirname): """Remove an empty directory. Remove a directory, ignoring non-existing ones. Other errors are passed. This includes the case, where the directory is not empty, so it can't be removed. @type dirname: str @param dirname: the empty directory to be removed """ try: os.rmdir(dirname) except OSError as err: if err.errno != errno.ENOENT: raise def RenameFile(old, new, mkdir=False, mkdir_mode=0o750, dir_uid=None, dir_gid=None): """Renames a file. This just creates the very least directory if it does not exist and C{mkdir} is set to true. @type old: string @param old: Original path @type new: string @param new: New path @type mkdir: bool @param mkdir: Whether to create target directory if it doesn't exist @type mkdir_mode: int @param mkdir_mode: Mode for newly created directories @type dir_uid: int @param dir_uid: The uid for the (if fresh created) dir @type dir_gid: int @param dir_gid: The gid for the (if fresh created) dir """ try: return os.rename(old, new) except OSError as err: # In at least one use case of this function, the job queue, directory # creation is very rare. Checking for the directory before renaming is not # as efficient. if mkdir and err.errno == errno.ENOENT: # Create directory and try again dir_path = os.path.dirname(new) MakeDirWithPerm(dir_path, mkdir_mode, dir_uid, dir_gid) return os.rename(old, new) raise def EnforcePermission(path, mode, uid=None, gid=None, must_exist=True, _chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat): """Enforces that given path has given permissions. @param path: The path to the file @param mode: The mode of the file @param uid: The uid of the owner of this file @param gid: The gid of the owner of this file @param must_exist: Specifies if non-existance of path will be an error @param _chmod_fn: chmod function to use (unittest only) @param _chown_fn: chown function to use (unittest only) """ logging.debug("Checking %s", path) # chown takes -1 if you want to keep one part of the ownership, however # None is Python standard for that. So we remap them here. if uid is None: uid = -1 if gid is None: gid = -1 try: st = _stat_fn(path) fmode = stat.S_IMODE(st[stat.ST_MODE]) if fmode != mode: logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode) _chmod_fn(path, mode) if max(uid, gid) > -1: fuid = st[stat.ST_UID] fgid = st[stat.ST_GID] if fuid != uid or fgid != gid: logging.debug("Changing owner of %s from UID %s/GID %s to" " UID %s/GID %s", path, fuid, fgid, uid, gid) _chown_fn(path, uid, gid) except EnvironmentError as err: if err.errno == errno.ENOENT: if must_exist: raise errors.GenericError("Path %s should exist, but does not" % path) else: raise errors.GenericError("Error while changing permissions on %s: %s" % (path, err)) def MakeDirWithPerm(path, mode, uid, gid, _lstat_fn=os.lstat, _mkdir_fn=os.mkdir, _perm_fn=EnforcePermission): """Enforces that given path is a dir and has given mode, uid and gid set. @param path: The path to the file @param mode: The mode of the file @param uid: The uid of the owner of this file @param gid: The gid of the owner of this file @param _lstat_fn: Stat function to use (unittest only) @param _mkdir_fn: mkdir function to use (unittest only) @param _perm_fn: permission setter function to use (unittest only) """ logging.debug("Checking directory %s", path) try: # We don't want to follow symlinks st = _lstat_fn(path) except EnvironmentError as err: if err.errno != errno.ENOENT: raise errors.GenericError("stat(2) on %s failed: %s" % (path, err)) _mkdir_fn(path) else: if not stat.S_ISDIR(st[stat.ST_MODE]): raise errors.GenericError(("Path %s is expected to be a directory, but " "isn't") % path) _perm_fn(path, mode, uid=uid, gid=gid) def Makedirs(path, mode=0o750): """Super-mkdir; create a leaf directory and all intermediate ones. This is a wrapper around C{os.makedirs} adding error handling not implemented before Python 2.5. """ try: os.makedirs(path, mode) except OSError as err: # Ignore EEXIST. This is only handled in os.makedirs as included in # Python 2.5 and above. if err.errno != errno.EEXIST or not os.path.exists(path): raise def TimestampForFilename(): """Returns the current time formatted for filenames. The format doesn't contain colons as some shells and applications treat them as separators. Uses the local timezone. """ return time.strftime("%Y-%m-%d_%H_%M_%S") def CreateBackup(file_name): """Creates a backup of a file. @type file_name: str @param file_name: file to be backed up @rtype: str @return: the path to the newly created backup @raise errors.ProgrammerError: for invalid file names """ if not os.path.isfile(file_name): raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" % file_name) prefix = ("%s.backup-%s." % (os.path.basename(file_name), TimestampForFilename())) dir_name = os.path.dirname(file_name) fsrc = open(file_name, "rb") try: (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name) fdst = os.fdopen(fd, "wb") try: logging.debug("Backing up %s at %s", file_name, backup_name) shutil.copyfileobj(fsrc, fdst) finally: fdst.close() finally: fsrc.close() return backup_name def ListVisibleFiles(path, _is_mountpoint=os.path.ismount): """Returns a list of visible files in a directory. @type path: str @param path: the directory to enumerate @rtype: list @return: the list of all files not starting with a dot @raise ProgrammerError: if L{path} is not an absolue and normalized path """ if not IsNormAbsPath(path): raise errors.ProgrammerError("Path passed to ListVisibleFiles is not" " absolute/normalized: '%s'" % path) mountpoint = _is_mountpoint(path) def fn(name): """File name filter. Ignores files starting with a dot (".") as by Unix convention they're considered hidden. The "lost+found" directory found at the root of some filesystems is also hidden. """ return not (name.startswith(".") or (mountpoint and name == _LOST_AND_FOUND and os.path.isdir(os.path.join(path, name)))) return [f for f in os.listdir(path) if fn(f)] def EnsureDirs(dirs): """Make required directories, if they don't exist. @param dirs: list of tuples (dir_name, dir_mode) @type dirs: list of (string, integer) """ for dir_name, dir_mode in dirs: try: os.mkdir(dir_name, dir_mode) except EnvironmentError as err: if err.errno != errno.EEXIST: raise errors.GenericError("Cannot create needed directory" " '%s': %s" % (dir_name, err)) try: os.chmod(dir_name, dir_mode) except EnvironmentError as err: raise errors.GenericError("Cannot change directory permissions on" " '%s' to 0%o: %s" % (dir_name, dir_mode, err)) if not os.path.isdir(dir_name): raise errors.GenericError("%s is not a directory" % dir_name) def FindFile(name, search_path, test=os.path.exists): """Look for a filesystem object in a given path. This is an abstract method to search for filesystem object (files, dirs) under a given search path. @type name: str @param name: the name to look for @type search_path: iterable of string @param search_path: locations to start at @type test: callable @param test: a function taking one argument that should return True if the a given object is valid; the default value is os.path.exists, causing only existing files to be returned @rtype: str or None @return: full path to the object if found, None otherwise """ # validate the filename mask if constants.EXT_PLUGIN_MASK.match(name) is None: logging.critical("Invalid value passed for external script name: '%s'", name) return None for dir_name in search_path: # FIXME: investigate switch to PathJoin item_name = os.path.sep.join([dir_name, name]) # check the user test and that we're indeed resolving to the given # basename if test(item_name) and os.path.basename(item_name) == name: return item_name return None def IsNormAbsPath(path): """Check whether a path is absolute and also normalized This avoids things like /dir/../../other/path to be valid. """ return os.path.normpath(path) == path and os.path.isabs(path) def IsBelowDir(root, other_path): """Check whether a path is below a root dir. This works around the nasty byte-byte comparison of commonprefix. """ if not (os.path.isabs(root) and os.path.isabs(other_path)): raise ValueError("Provided paths '%s' and '%s' are not absolute" % (root, other_path)) norm_other = os.path.normpath(other_path) if norm_other == os.sep: # The root directory can never be below another path return False norm_root = os.path.normpath(root) if norm_root == os.sep: # This is the root directory, no need to add another slash prepared_root = norm_root else: prepared_root = "%s%s" % (norm_root, os.sep) return os.path.commonprefix([prepared_root, norm_other]) == prepared_root URL_RE = re.compile(r'(https?|ftps?)://') def IsUrl(path): """Check whether a path is a HTTP URL. """ return URL_RE.match(path) def PathJoin(*args): """Safe-join a list of path components. Requirements: - the first argument must be an absolute path - no component in the path must have backtracking (e.g. /../), since we check for normalization at the end @param args: the path components to be joined @raise ValueError: for invalid paths """ # ensure we're having at least two paths passed in if len(args) <= 1: raise errors.ProgrammerError("PathJoin requires two arguments") # ensure the first component is an absolute and normalized path name root = args[0] if not IsNormAbsPath(root): raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0])) result = os.path.join(*args) # ensure that the whole path is normalized if not IsNormAbsPath(result): raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args)) # check that we're still under the original prefix if not IsBelowDir(root, result): raise ValueError("Error: path joining resulted in different prefix" " (%s != %s)" % (result, root)) return result def TailFile(fname, lines=20): """Return the last lines from a file. @note: this function will only read and parse the last 4KB of the file; if the lines are very long, it could be that less than the requested number of lines are returned @param fname: the file name @type lines: int @param lines: the (maximum) number of lines to return """ fd = open(fname, "r") try: fd.seek(0, 2) pos = fd.tell() pos = max(0, pos - 4096) fd.seek(pos, 0) raw_data = fd.read() finally: fd.close() rows = raw_data.splitlines() return rows[-lines:] def BytesToMebibyte(value): """Converts bytes to mebibytes. @type value: int @param value: Value in bytes @rtype: int @return: Value in mebibytes """ return int(round(value / (1024.0 * 1024.0), 0)) def CalculateDirectorySize(path): """Calculates the size of a directory recursively. @type path: string @param path: Path to directory @rtype: int @return: Size in mebibytes """ size = 0 for (curpath, _, files) in os.walk(path): for filename in files: st = os.lstat(PathJoin(curpath, filename)) size += st.st_size return BytesToMebibyte(size) def GetFilesystemStats(path): """Returns the total and free space on a filesystem. @type path: string @param path: Path on filesystem to be examined @rtype: int @return: tuple of (Total space, Free space) in mebibytes """ st = os.statvfs(path) fsize = BytesToMebibyte(st.f_bavail * st.f_frsize) tsize = BytesToMebibyte(st.f_blocks * st.f_frsize) return (tsize, fsize) def ReadPidFile(pidfile): """Read a pid from a file. @type pidfile: string @param pidfile: path to the file containing the pid @rtype: int @return: The process id, if the file exists and contains a valid PID, otherwise 0 """ try: raw_data = ReadOneLineFile(pidfile) except EnvironmentError as err: if err.errno != errno.ENOENT: logging.exception("Can't read pid file") return 0 return _ParsePidFileContents(raw_data) def _ParsePidFileContents(data): """Tries to extract a process ID from a PID file's content. @type data: string @rtype: int @return: Zero if nothing could be read, PID otherwise """ try: pid = int(data) except (TypeError, ValueError): logging.info("Can't parse pid file contents", exc_info=True) return 0 else: return pid def ReadLockedPidFile(path): """Reads a locked PID file. This can be used together with L{utils.process.StartDaemon}. @type path: string @param path: Path to PID file @return: PID as integer or, if file was unlocked or couldn't be opened, None """ try: fd = os.open(path, os.O_RDONLY) except EnvironmentError as err: if err.errno == errno.ENOENT: # PID file doesn't exist return None raise try: try: # Try to acquire lock filelock.LockFile(fd) except errors.LockError: # Couldn't lock, daemon is running return int(os.read(fd, 100)) finally: os.close(fd) return None def DaemonPidFileName(name): """Compute a ganeti pid file absolute path @type name: str @param name: the daemon name @rtype: str @return: the full path to the pidfile corresponding to the given daemon name """ return PathJoin(pathutils.RUN_DIR, "%s.pid" % name) def WritePidFile(pidfile): """Write the current process pidfile. @type pidfile: string @param pidfile: the path to the file to be written @raise errors.LockError: if the pid file already exists and points to a live process @rtype: int @return: the file descriptor of the lock file; do not close this unless you want to unlock the pid file """ # We don't rename nor truncate the file to not drop locks under # existing processes fd_pidfile = os.open(pidfile, os.O_RDWR | os.O_CREAT, 0o600) # Lock the PID file (and fail if not possible to do so). Any code # wanting to send a signal to the daemon should try to lock the PID # file before reading it. If acquiring the lock succeeds, the daemon is # no longer running and the signal should not be sent. try: filelock.LockFile(fd_pidfile) except errors.LockError: msg = ["PID file '%s' is already locked by another process" % pidfile] # Try to read PID file pid = _ParsePidFileContents(os.read(fd_pidfile, 100)) if pid > 0: msg.append(", PID read from file is %s" % pid) raise errors.PidFileLockError("".join(msg)) os.write(fd_pidfile, b"%d\n" % os.getpid()) return fd_pidfile def ReadWatcherPauseFile(filename, now=None, remove_after=3600): """Reads the watcher pause file. @type filename: string @param filename: Path to watcher pause file @type now: None, float or int @param now: Current time as Unix timestamp @type remove_after: int @param remove_after: Remove watcher pause file after specified amount of seconds past the pause end time """ if now is None: now = time.time() try: value = ReadFile(filename) except IOError as err: if err.errno != errno.ENOENT: raise value = None if value is not None: try: value = int(value) except ValueError: logging.warning(("Watcher pause file (%s) contains invalid value," " removing it"), filename) RemoveFile(filename) value = None if value is not None: # Remove file if it's outdated if now > (value + remove_after): RemoveFile(filename) value = None elif now > value: value = None return value def NewUUID(): """Returns a random UUID. @note: This is a Linux-specific method as it uses the /proc filesystem. @rtype: str """ return ReadFile(constants.RANDOM_UUID_FILE, size=128).rstrip("\n") class TemporaryFileManager(object): """Stores the list of files to be deleted and removes them on demand. """ def __init__(self): self._files = [] def __del__(self): self.Cleanup() def Add(self, filename): """Add file to list of files to be deleted. @type filename: string @param filename: path to filename to be added """ self._files.append(filename) def Remove(self, filename): """Remove file from list of files to be deleted. @type filename: string @param filename: path to filename to be deleted """ self._files.remove(filename) def Cleanup(self): """Delete all files marked for deletion """ while self._files: RemoveFile(self._files.pop()) def IsUserInGroup(uid, gid): """Returns True if the user belongs to the group. @type uid: int @param uid: the user id @type gid: int @param gid: the group id @rtype: bool """ user = pwd.getpwuid(uid) group = grp.getgrgid(gid) return user.pw_gid == gid or user.pw_name in group.gr_mem def CanRead(username, filename): """Returns True if the user can access (read) the file. @type username: string @param username: the name of the user @type filename: string @param filename: the name of the file @rtype: bool """ filestats = os.stat(filename) user = pwd.getpwnam(username) uid = user.pw_uid user_readable = filestats.st_mode & stat.S_IRUSR != 0 group_readable = filestats.st_mode & stat.S_IRGRP != 0 return ((filestats.st_uid == uid and user_readable) or (filestats.st_uid != uid and IsUserInGroup(uid, filestats.st_gid) and group_readable)) def OpenTTY(device="/dev/tty"): """Returns a text I/O object pointing a TTY (/dev/tty by default) As of Python 3.7, /dev/tty cannot be opened in buffered and/or text mode, see https://bugs.python.org/issue20074. Work around this by using TextIOWrapper over a BufferedRWPair to return a line-buffered TTY I/O object. @type device: string @param device: path to the TTY/PTY device to open """ _tty = FileIO(device, "r+") _buffered_tty = BufferedRWPair(_tty, _tty) return TextIOWrapper(_buffered_tty, line_buffering=True) ganeti-3.1.0~rc2/lib/utils/livelock.py000064400000000000000000000061051476477700300176760ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Lockfiles to prove liveliness When requesting resources, like locks, from wconfd, requesters have to provide the name of a file they own an exclusive lock on, to prove that they are still alive. Provide methods to obtain such a file. """ import fcntl import os import struct import time from ganeti.utils.algo import NiceSort from ganeti import pathutils class LiveLock(object): """Utility for a lockfile needed to request resources from WconfD. """ def __init__(self, name=None): if name is None: name = "pid%d_" % os.getpid() # to avoid reusing existing lock files, extend name # by the current time name = "%s_%d" % (name, int(time.time())) fname = os.path.join(pathutils.LIVELOCK_DIR, name) self.lockfile = open(fname, 'w') # with LFS enabled, off_t is 64 bits even on 32-bit platforms try: os.O_LARGEFILE struct_flock = 'hhqqhh' except AttributeError: struct_flock = 'hhllhh' fcntl.fcntl(self.lockfile, fcntl.F_SETLKW, struct.pack(struct_flock, fcntl.F_WRLCK, 0, 0, 0, 0, 0)) def GetPath(self): return self.lockfile.name def close(self): """Close the lockfile and clean it up. """ self.lockfile.close() os.remove(self.lockfile.name) def __str__(self): return "LiveLock(" + self.GetPath() + ")" def GuessLockfileFor(name): """For a given name, take the latest file matching. @return: the file with the latest name matching the given prefix in LIVELOCK_DIR, or the plain name, if none exists. """ lockfiles = [n for n in os.listdir(pathutils.LIVELOCK_DIR) if n.startswith(name)] if len(lockfiles) > 0: lockfile = NiceSort(lockfiles)[-1] else: lockfile = name return os.path.join(pathutils.LIVELOCK_DIR, lockfile) ganeti-3.1.0~rc2/lib/utils/log.py000064400000000000000000000241321476477700300166470ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for logging. """ import os.path import logging import logging.handlers from io import StringIO from ganeti import constants from ganeti import compat class _ReopenableLogHandler(logging.handlers.BaseRotatingHandler): """Log handler with ability to reopen log file on request. In combination with a SIGHUP handler this class can reopen the log file on user request. """ def __init__(self, filename): """Initializes this class. @type filename: string @param filename: Path to logfile """ logging.handlers.BaseRotatingHandler.__init__(self, filename, "a") assert self.encoding is None, "Encoding not supported for logging" assert not hasattr(self, "_reopen"), "Base class has '_reopen' attribute" self._reopen = False def shouldRollover(self, _): # pylint: disable=C0103 """Determine whether log file should be reopened. """ return self._reopen or not self.stream def doRollover(self): # pylint: disable=C0103 """Reopens the log file. """ if self.stream: self.stream.flush() self.stream.close() self.stream = None # Reopen file # TODO: Handle errors? self.stream = open(self.baseFilename, "a") # Don't reopen on the next message self._reopen = False def RequestReopen(self): """Register a request to reopen the file. The file will be reopened before writing the next log record. """ self._reopen = True def _LogErrorsToConsole(base): """Create wrapper class writing errors to console. This needs to be in a function for unittesting. """ class wrapped(base): # pylint: disable=C0103 """Log handler that doesn't fallback to stderr. When an error occurs while writing on the logfile, logging.FileHandler tries to log on stderr. This doesn't work in Ganeti since stderr is redirected to a logfile. This class avoids failures by reporting errors to /dev/console. """ def __init__(self, console, *args, **kwargs): """Initializes this class. @type console: file-like object or None @param console: Open file-like object for console """ base.__init__(self, *args, **kwargs) assert not hasattr(self, "_console") self._console = console def handleError(self, record): # pylint: disable=C0103 """Handle errors which occur during an emit() call. Try to handle errors with FileHandler method, if it fails write to /dev/console. """ try: base.handleError(record) except Exception: # pylint: disable=W0703 if self._console: try: # Ignore warning about "self.format", pylint: disable=E1101 self._console.write("Cannot log message:\n%s\n" % self.format(record)) except Exception: # pylint: disable=W0703 # Log handler tried everything it could, now just give up pass return wrapped #: Custom log handler for writing to console with a reopenable handler _LogHandler = _LogErrorsToConsole(_ReopenableLogHandler) def _GetLogFormatter(program, multithreaded, debug, syslog): """Build log formatter. @param program: Program name @param multithreaded: Whether to add thread name to log messages @param debug: Whether to enable debug messages @param syslog: Whether the formatter will be used for syslog """ parts = [] if syslog: parts.append(program + "[%(process)d]:") else: parts.append("%(asctime)s: " + program + " pid=%(process)d") if multithreaded: if syslog: parts.append(" (%(threadName)s)") else: parts.append("/%(threadName)s") # Add debug info for non-syslog loggers if debug and not syslog: parts.append(" %(module)s:%(lineno)s") # Ses, we do want the textual level, as remote syslog will probably lose the # error level, and it's easier to grep for it. parts.append(" %(levelname)s %(message)s") return logging.Formatter("".join(parts)) def _ReopenLogFiles(handlers): """Wrapper for reopening all log handler's files in a sequence. """ for handler in handlers: handler.RequestReopen() logging.info("Received request to reopen log files") def SetupLogging(logfile, program, debug=0, stderr_logging=False, multithreaded=False, syslog=constants.SYSLOG_USAGE, console_logging=False, root_logger=None): """Configures the logging module. @type logfile: str @param logfile: the filename to which we should log @type program: str @param program: the name under which we should log messages @type debug: integer @param debug: if greater than zero, enable debug messages, otherwise only those at C{INFO} and above level @type stderr_logging: boolean @param stderr_logging: whether we should also log to the standard error @type multithreaded: boolean @param multithreaded: if True, will add the thread name to the log file @type syslog: string @param syslog: one of 'no', 'yes', 'only': - if no, syslog is not used - if yes, syslog is used (in addition to file-logging) - if only, only syslog is used @type console_logging: boolean @param console_logging: if True, will use a FileHandler which falls back to the system console if logging fails @type root_logger: logging.Logger @param root_logger: Root logger to use (for unittests) @raise EnvironmentError: if we can't open the log file and syslog/stderr logging is disabled @rtype: callable @return: Function reopening all open log files when called """ progname = os.path.basename(program) formatter = _GetLogFormatter(progname, multithreaded, debug, False) syslog_fmt = _GetLogFormatter(progname, multithreaded, debug, True) reopen_handlers = [] if root_logger is None: root_logger = logging.getLogger("") root_logger.setLevel(logging.NOTSET) # Remove all previously setup handlers for handler in root_logger.handlers: handler.close() root_logger.removeHandler(handler) if stderr_logging: stderr_handler = logging.StreamHandler() stderr_handler.setFormatter(formatter) if debug: stderr_handler.setLevel(logging.NOTSET) else: stderr_handler.setLevel(logging.CRITICAL) root_logger.addHandler(stderr_handler) if syslog in (constants.SYSLOG_YES, constants.SYSLOG_ONLY): facility = logging.handlers.SysLogHandler.LOG_DAEMON syslog_handler = logging.handlers.SysLogHandler(constants.SYSLOG_SOCKET, facility) syslog_handler.setFormatter(syslog_fmt) # Never enable debug over syslog syslog_handler.setLevel(logging.INFO) root_logger.addHandler(syslog_handler) if syslog != constants.SYSLOG_ONLY: # this can fail, if the logging directories are not setup or we have # a permisssion problem; in this case, it's best to log but ignore # the error if stderr_logging is True, and if false we re-raise the # exception since otherwise we could run but without any logs at all try: if console_logging: logfile_handler = _LogHandler(open(constants.DEV_CONSOLE, "w"), logfile) else: logfile_handler = _ReopenableLogHandler(logfile) logfile_handler.setFormatter(formatter) if debug: logfile_handler.setLevel(logging.DEBUG) else: logfile_handler.setLevel(logging.INFO) root_logger.addHandler(logfile_handler) reopen_handlers.append(logfile_handler) except EnvironmentError: if stderr_logging or syslog == constants.SYSLOG_YES: logging.exception("Failed to enable logging to file '%s'", logfile) else: # we need to re-raise the exception raise return compat.partial(_ReopenLogFiles, reopen_handlers) def SetupToolLogging(debug, verbose, threadname=False, _root_logger=None, _stream=None): """Configures the logging module for tools. All log messages are sent to stderr. @type debug: boolean @param debug: Disable log message filtering @type verbose: boolean @param verbose: Enable verbose log messages @type threadname: boolean @param threadname: Whether to include thread name in output """ if _root_logger is None: root_logger = logging.getLogger("") else: root_logger = _root_logger fmt = StringIO() fmt.write("%(asctime)s:") if threadname: fmt.write(" %(threadName)s") if debug or verbose: fmt.write(" %(levelname)s") fmt.write(" %(message)s") formatter = logging.Formatter(fmt.getvalue()) stderr_handler = logging.StreamHandler(_stream) stderr_handler.setFormatter(formatter) if debug: stderr_handler.setLevel(logging.NOTSET) elif verbose: stderr_handler.setLevel(logging.INFO) else: stderr_handler.setLevel(logging.WARNING) root_logger.setLevel(logging.NOTSET) root_logger.addHandler(stderr_handler) ganeti-3.1.0~rc2/lib/utils/lvm.py000064400000000000000000000065771476477700300167010ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for LVM. """ from ganeti import constants def CheckVolumeGroupSize(vglist, vgname, minsize): """Checks if the volume group list is valid. The function will check if a given volume group is in the list of volume groups and has a minimum size. @type vglist: dict @param vglist: dictionary of volume group names and their size @type vgname: str @param vgname: the volume group we should check @type minsize: int @param minsize: the minimum size we accept @rtype: None or str @return: None for success, otherwise the error message """ vgsize = vglist.get(vgname, None) if vgsize is None: return "volume group '%s' missing" % vgname elif vgsize < minsize: return ("volume group '%s' too small (%s MiB required, %d MiB found)" % (vgname, minsize, vgsize)) return None def LvmExclusiveCheckNodePvs(pvs_info): """Check consistency of PV sizes in a node for exclusive storage. @type pvs_info: list @param pvs_info: list of L{LvmPvInfo} objects @rtype: tuple @return: A pair composed of: 1. a list of error strings describing the violations found, or an empty list if everything is ok; 2. a pair containing the sizes of the smallest and biggest PVs, in MiB. """ errmsgs = [] sizes = [pv.size for pv in pvs_info] # The sizes of PVs must be the same (tolerance is constants.PART_MARGIN) small = min(sizes) big = max(sizes) if LvmExclusiveTestBadPvSizes(small, big): m = ("Sizes of PVs are too different: min=%d max=%d" % (small, big)) errmsgs.append(m) return (errmsgs, (small, big)) def LvmExclusiveTestBadPvSizes(small, big): """Test if the given PV sizes are permitted with exclusive storage. @param small: size of the smallest PV @param big: size of the biggest PV @return: True when the given sizes are bad, False otherwise """ # Test whether no X exists such that: # small >= X * (1 - constants.PART_MARGIN) and # big <= X * (1 + constants.PART_MARGIN) return (small * (1 + constants.PART_MARGIN) < big * (1 - constants.PART_MARGIN)) ganeti-3.1.0~rc2/lib/utils/mlock.py000064400000000000000000000056031476477700300171750ustar00rootroot00000000000000# # # Copyright (C) 2009, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Wrapper around mlockall(2). """ import os import logging from ganeti import errors try: # pylint: disable=F0401 import ctypes except ImportError: ctypes = None # Flags for mlockall(2) (from bits/mman.h) _MCL_CURRENT = 1 _MCL_FUTURE = 2 def Mlockall(_ctypes=ctypes): """Lock current process' virtual address space into RAM. This is equivalent to the C call C{mlockall(MCL_CURRENT | MCL_FUTURE)}. See mlockall(2) for more details. This function requires the C{ctypes} module. @raises errors.NoCtypesError: If the C{ctypes} module is not found """ if _ctypes is None: raise errors.NoCtypesError() try: libc = _ctypes.cdll.LoadLibrary("libc.so.6") except EnvironmentError as err: logging.error("Failure trying to load libc: %s", err) libc = None if libc is None: logging.error("Cannot set memory lock, ctypes cannot load libc") return # The ctypes module before Python 2.6 does not have built-in functionality to # access the global errno global (which, depending on the libc and build # options, is per thread), where function error codes are stored. Use GNU # libc's way to retrieve errno(3) instead, which is to use the pointer named # "__errno_location" (see errno.h and bits/errno.h). # pylint: disable=W0212 libc.__errno_location.restype = _ctypes.POINTER(_ctypes.c_int) if libc.mlockall(_MCL_CURRENT | _MCL_FUTURE): # pylint: disable=W0212 logging.error("Cannot set memory lock: %s", os.strerror(libc.__errno_location().contents.value)) return logging.debug("Memory lock set") ganeti-3.1.0~rc2/lib/utils/nodesetup.py000064400000000000000000000077541476477700300201070ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for manipulating /etc/hosts. """ from io import StringIO from ganeti import pathutils from ganeti.utils import algo from ganeti.utils import io def SetEtcHostsEntry(file_name, ip, hostname, aliases): """Sets the name of an IP address and hostname in /etc/hosts. @type file_name: str @param file_name: path to the file to modify (usually C{/etc/hosts}) @type ip: str @param ip: the IP address @type hostname: str @param hostname: the hostname to be added @type aliases: list @param aliases: the list of aliases to add for the hostname """ # Ensure aliases are unique names = algo.UniqueSequence([hostname] + aliases) out = StringIO() def _write_entry(written): if not written: out.write("%s\t%s\n" % (ip, " ".join(names))) return True written = False for line in io.ReadFile(file_name).splitlines(True): fields = line.split() if fields and not fields[0].startswith("#") and ip == fields[0]: written = _write_entry(written) else: out.write(line) _write_entry(written) io.WriteFile(file_name, data=out.getvalue(), uid=0, gid=0, mode=0o644, keep_perms=io.KP_IF_EXISTS) def AddHostToEtcHosts(hostname, ip): """Wrapper around SetEtcHostsEntry. @type hostname: str @param hostname: a hostname that will be resolved and added to L{pathutils.ETC_HOSTS} @type ip: str @param ip: The ip address of the host """ SetEtcHostsEntry(pathutils.ETC_HOSTS, ip, hostname, [hostname.split(".")[0]]) def RemoveEtcHostsEntry(file_name, hostname): """Removes a hostname from /etc/hosts. IP addresses without names are removed from the file. @type file_name: str @param file_name: path to the file to modify (usually C{/etc/hosts}) @type hostname: str @param hostname: the hostname to be removed """ out = StringIO() for line in io.ReadFile(file_name).splitlines(True): fields = line.split() if len(fields) > 1 and not fields[0].startswith("#"): names = fields[1:] if hostname in names: while hostname in names: names.remove(hostname) if names: out.write("%s %s\n" % (fields[0], " ".join(names))) continue out.write(line) io.WriteFile(file_name, data=out.getvalue(), uid=0, gid=0, mode=0o644, keep_perms=io.KP_IF_EXISTS) def RemoveHostFromEtcHosts(hostname): """Wrapper around RemoveEtcHostsEntry. @type hostname: str @param hostname: hostname that will be resolved and its full and shot name will be removed from L{pathutils.ETC_HOSTS} """ RemoveEtcHostsEntry(pathutils.ETC_HOSTS, hostname) RemoveEtcHostsEntry(pathutils.ETC_HOSTS, hostname.split(".")[0]) ganeti-3.1.0~rc2/lib/utils/process.py000064400000000000000000000763231476477700300175550ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for processes. """ import os import sys import subprocess import errno import select import logging import signal import resource from io import StringIO from ganeti import errors from ganeti import constants from ganeti import compat from ganeti.utils import retry as utils_retry from ganeti.utils import wrapper as utils_wrapper from ganeti.utils import text as utils_text from ganeti.utils import io as utils_io from ganeti.utils import algo as utils_algo #: when set to True, L{RunCmd} is disabled _no_fork = False (_TIMEOUT_NONE, _TIMEOUT_TERM, _TIMEOUT_KILL) = range(3) def DisableFork(): """Disables the use of fork(2). """ global _no_fork # pylint: disable=W0603 _no_fork = True class RunResult(object): """Holds the result of running external programs. @type exit_code: int @ivar exit_code: the exit code of the program, or None (if the program didn't exit()) @type signal: int or None @ivar signal: the signal that caused the program to finish, or None (if the program wasn't terminated by a signal) @type stdout: str @ivar stdout: the standard output of the program @type stderr: str @ivar stderr: the standard error of the program @type failed: boolean @ivar failed: True in case the program was terminated by a signal or exited with a non-zero exit code @type failed_by_timeout: boolean @ivar failed_by_timeout: True in case the program was terminated by timeout @ivar fail_reason: a string detailing the termination reason """ __slots__ = ["exit_code", "signal", "stdout", "stderr", "failed", "failed_by_timeout", "fail_reason", "cmd"] def __init__(self, exit_code, signal_, stdout, stderr, cmd, timeout_action, timeout): self.cmd = cmd self.exit_code = exit_code self.signal = signal_ self.stdout = stdout self.stderr = stderr self.failed = (signal_ is not None or exit_code != 0) self.failed_by_timeout = timeout_action != _TIMEOUT_NONE fail_msgs = [] if self.signal is not None: fail_msgs.append("terminated by signal %s" % self.signal) elif self.exit_code is not None: fail_msgs.append("exited with exit code %s" % self.exit_code) else: fail_msgs.append("unable to determine termination reason") if timeout_action == _TIMEOUT_TERM: fail_msgs.append("terminated after timeout of %.2f seconds" % timeout) elif timeout_action == _TIMEOUT_KILL: fail_msgs.append(("force termination after timeout of %.2f seconds" " and linger for another %.2f seconds") % (timeout, constants.CHILD_LINGER_TIMEOUT)) if fail_msgs and self.failed: self.fail_reason = utils_text.CommaJoin(fail_msgs) else: self.fail_reason = None if self.failed: logging.debug("Command '%s' failed (%s); output: %s", self.cmd, self.fail_reason, self.output) def _GetOutput(self): """Returns the combined stdout and stderr for easier usage. """ return self.stdout + self.stderr output = property(_GetOutput, None, None, "Return full output") def _BuildCmdEnvironment(env, reset): """Builds the environment for an external program. """ if reset: cmd_env = {} else: cmd_env = os.environ.copy() cmd_env["LC_ALL"] = "C" if env is not None: cmd_env.update(env) return cmd_env def RunCmd(cmd, env=None, output=None, cwd="/", reset_env=False, interactive=False, timeout=None, noclose_fds=None, input_fd=None, postfork_fn=None): """Execute a (shell) command. The command should not read from its standard input, as it will be closed. @type cmd: string or list @param cmd: Command to run @type env: dict @param env: Additional environment variables @type output: str @param output: if desired, the output of the command can be saved in a file instead of the RunResult instance; this parameter denotes the file name (if not None) @type cwd: string @param cwd: if specified, will be used as the working directory for the command; the default will be / @type reset_env: boolean @param reset_env: whether to reset or keep the default os environment @type interactive: boolean @param interactive: whether we pipe stdin, stdout and stderr (default behaviour) or run the command interactive @type timeout: int @param timeout: If not None, timeout in seconds until child process gets killed @type noclose_fds: list @param noclose_fds: list of additional (fd >=3) file descriptors to leave open for the child process @type input_fd: C{file}-like object or numeric file descriptor @param input_fd: File descriptor for process' standard input @type postfork_fn: Callable receiving PID as parameter @param postfork_fn: Callback run after fork but before timeout @rtype: L{RunResult} @return: RunResult instance @raise errors.ProgrammerError: if we call this when forks are disabled """ if _no_fork: raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled") if output and interactive: raise errors.ProgrammerError("Parameters 'output' and 'interactive' can" " not be provided at the same time") if not (output is None or input_fd is None): # The current logic in "_RunCmdFile", which is used when output is defined, # does not support input files (not hard to implement, though) raise errors.ProgrammerError("Parameters 'output' and 'input_fd' can" " not be used at the same time") if isinstance(cmd, str): strcmd = cmd shell = True else: cmd = [str(val) for val in cmd] strcmd = utils_text.ShellQuoteArgs(cmd) shell = False if output: logging.info("RunCmd %s, output file '%s'", strcmd, output) else: logging.info("RunCmd %s", strcmd) cmd_env = _BuildCmdEnvironment(env, reset_env) try: if output is None: out, err, status, timeout_action = _RunCmdPipe(cmd, cmd_env, shell, cwd, interactive, timeout, noclose_fds, input_fd, postfork_fn=postfork_fn) else: if postfork_fn: raise errors.ProgrammerError("postfork_fn is not supported if output" " should be captured") assert input_fd is None timeout_action = _TIMEOUT_NONE status = _RunCmdFile(cmd, cmd_env, shell, output, cwd, noclose_fds) out = err = "" except OSError as err: if err.errno == errno.ENOENT: raise errors.OpExecError("Can't execute '%s': not found (%s)" % (strcmd, err)) else: raise if status >= 0: exitcode = status signal_ = None else: exitcode = None signal_ = -status return RunResult(exitcode, signal_, out, err, strcmd, timeout_action, timeout) def SetupDaemonEnv(cwd="/", umask=0o77): """Setup a daemon's environment. This should be called between the first and second fork, due to setsid usage. @param cwd: the directory to which to chdir @param umask: the umask to setup """ os.chdir(cwd) os.umask(umask) os.setsid() def SetupDaemonFDs(output_file, output_fd): """Setups up a daemon's file descriptors. @param output_file: if not None, the file to which to redirect stdout/stderr @param output_fd: if not None, the file descriptor for stdout/stderr """ # check that at most one is defined assert [output_file, output_fd].count(None) >= 1 # Open /dev/null (read-only, only for stdin) devnull_fd = os.open(os.devnull, os.O_RDONLY) output_close = True if output_fd is not None: output_close = False elif output_file is not None: # Open output file try: output_fd = os.open(output_file, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) except EnvironmentError as err: raise Exception("Opening output file failed: %s" % err) else: output_fd = os.open(os.devnull, os.O_WRONLY) # Redirect standard I/O os.dup2(devnull_fd, 0) os.dup2(output_fd, 1) os.dup2(output_fd, 2) if devnull_fd > 2: utils_wrapper.CloseFdNoError(devnull_fd) if output_close and output_fd > 2: utils_wrapper.CloseFdNoError(output_fd) def StartDaemon(cmd, env=None, cwd="/", output=None, output_fd=None, pidfile=None): """Start a daemon process after forking twice. @type cmd: string or list @param cmd: Command to run @type env: dict @param env: Additional environment variables @type cwd: string @param cwd: Working directory for the program @type output: string @param output: Path to file in which to save the output @type output_fd: int @param output_fd: File descriptor for output @type pidfile: string @param pidfile: Process ID file @rtype: int @return: Daemon process ID @raise errors.ProgrammerError: if we call this when forks are disabled """ if _no_fork: raise errors.ProgrammerError("utils.StartDaemon() called with fork()" " disabled") if output and not (bool(output) ^ (output_fd is not None)): raise errors.ProgrammerError("Only one of 'output' and 'output_fd' can be" " specified") if isinstance(cmd, str): cmd = ["/bin/sh", "-c", cmd] strcmd = utils_text.ShellQuoteArgs(cmd) if output: logging.debug("StartDaemon %s, output file '%s'", strcmd, output) else: logging.debug("StartDaemon %s", strcmd) cmd_env = _BuildCmdEnvironment(env, False) # Create pipe for sending PID back (pidpipe_read, pidpipe_write) = os.pipe() try: try: # Create pipe for sending error messages (errpipe_read, errpipe_write) = os.pipe() try: try: # First fork pid = os.fork() if pid == 0: # Try to start child process, will either execve or exit on failure. _StartDaemonChild(errpipe_read, errpipe_write, pidpipe_read, pidpipe_write, cmd, cmd_env, cwd, output, output_fd, pidfile) finally: utils_wrapper.CloseFdNoError(errpipe_write) # Wait for daemon to be started (or an error message to # arrive) and read up to 100 KB as an error message errormsg = utils_wrapper.RetryOnSignal(os.read, errpipe_read, 100 * 1024).decode("utf-8") finally: utils_wrapper.CloseFdNoError(errpipe_read) finally: utils_wrapper.CloseFdNoError(pidpipe_write) # Read up to 128 bytes for PID pidtext = utils_wrapper.RetryOnSignal(os.read, pidpipe_read, 128) finally: utils_wrapper.CloseFdNoError(pidpipe_read) # Try to avoid zombies by waiting for child process try: os.waitpid(pid, 0) except OSError: pass if errormsg: raise errors.OpExecError("Error when starting daemon process: %r" % errormsg) try: return int(pidtext) except (ValueError, TypeError) as err: raise errors.OpExecError("Error while trying to parse PID %r: %s" % (pidtext, err)) def _StartDaemonChild(errpipe_read, errpipe_write, pidpipe_read, pidpipe_write, args, env, cwd, output, fd_output, pidfile): """Child process for starting daemon. """ try: # Close parent's side utils_wrapper.CloseFdNoError(errpipe_read) utils_wrapper.CloseFdNoError(pidpipe_read) # First child process SetupDaemonEnv() # And fork for the second time pid = os.fork() if pid != 0: # Exit first child process os._exit(0) # pylint: disable=W0212 # Make sure pipe is closed on execv* (and thereby notifies # original process) utils_wrapper.SetCloseOnExecFlag(errpipe_write, True) # List of file descriptors to be left open noclose_fds = [errpipe_write] # Open PID file if pidfile: fd_pidfile = utils_io.WritePidFile(pidfile) # Keeping the file open to hold the lock noclose_fds.append(fd_pidfile) utils_wrapper.SetCloseOnExecFlag(fd_pidfile, False) else: fd_pidfile = None SetupDaemonFDs(output, fd_output) # Send daemon PID to parent utils_wrapper.RetryOnSignal(os.write, pidpipe_write, b"%d" % os.getpid()) # Close all file descriptors except stdio and error message pipe CloseFDs(noclose_fds=noclose_fds) # Change working directory os.chdir(cwd) if env is None: os.execvp(args[0], args) else: os.execvpe(args[0], args, env) except: # pylint: disable=W0702 try: # Report errors to original process WriteErrorToFD(errpipe_write, str(sys.exc_info()[1])) except: # pylint: disable=W0702 # Ignore errors in error handling pass os._exit(1) # pylint: disable=W0212 def WriteErrorToFD(fd, err): """Possibly write an error message to a fd. @type fd: None or int (file descriptor) @param fd: if not None, the error will be written to this fd @param err: string, the error message """ if fd is None: return if not err: err = "" utils_wrapper.RetryOnSignal(os.write, fd, err.encode("utf-8")) def _CheckIfAlive(child): """Raises L{utils_retry.RetryAgain} if child is still alive. @raises utils_retry.RetryAgain: If child is still alive """ if child.poll() is None: raise utils_retry.RetryAgain() def _WaitForProcess(child, timeout): """Waits for the child to terminate or until we reach timeout. """ try: utils_retry.Retry(_CheckIfAlive, (1.0, 1.2, 5.0), max(0, timeout), args=[child]) except utils_retry.RetryTimeout: pass def _RunCmdPipe(cmd, env, via_shell, cwd, interactive, timeout, noclose_fds, input_fd, postfork_fn=None, _linger_timeout=constants.CHILD_LINGER_TIMEOUT): """Run a command and return its output. @type cmd: string or list @param cmd: Command to run @type env: dict @param env: The environment to use @type via_shell: bool @param via_shell: if we should run via the shell @type cwd: string @param cwd: the working directory for the program @type interactive: boolean @param interactive: Run command interactive (without piping) @type timeout: int @param timeout: Timeout after the programm gets terminated @type noclose_fds: list @param noclose_fds: list of additional (fd >=3) file descriptors to leave open for the child process @type input_fd: C{file}-like object or numeric file descriptor @param input_fd: File descriptor for process' standard input @type postfork_fn: Callable receiving PID as parameter @param postfork_fn: Function run after fork but before timeout @rtype: tuple @return: (out, err, status) """ # pylint: disable=R0101 poller = select.poll() if interactive: stderr = None stdout = None else: stderr = subprocess.PIPE stdout = subprocess.PIPE if input_fd: stdin = input_fd elif interactive: stdin = None else: stdin = subprocess.PIPE if noclose_fds: pass_fds = noclose_fds else: pass_fds = [] child = subprocess.Popen(cmd, shell=via_shell, stderr=stderr, stdout=stdout, stdin=stdin, pass_fds=pass_fds, env=env, cwd=cwd, encoding="utf-8") if postfork_fn: postfork_fn(child.pid) out = StringIO() err = StringIO() linger_timeout = None if timeout is None: poll_timeout = None else: poll_timeout = utils_algo.RunningTimeout(timeout, True).Remaining msg_timeout = ("Command %s (%d) run into execution timeout, terminating" % (cmd, child.pid)) msg_linger = ("Command %s (%d) run into linger timeout, killing" % (cmd, child.pid)) timeout_action = _TIMEOUT_NONE # subprocess: "If the stdin argument is PIPE, this attribute is a file object # that provides input to the child process. Otherwise, it is None." assert (stdin == subprocess.PIPE) ^ (child.stdin is None), \ "subprocess' stdin did not behave as documented" if not interactive: if child.stdin is not None: child.stdin.close() poller.register(child.stdout, select.POLLIN) poller.register(child.stderr, select.POLLIN) fdmap = { child.stdout.fileno(): (out, child.stdout), child.stderr.fileno(): (err, child.stderr), } for fd in fdmap: utils_wrapper.SetNonblockFlag(fd, True) while fdmap: if poll_timeout: pt = poll_timeout() * 1000 if pt < 0: if linger_timeout is None: logging.warning(msg_timeout) if child.poll() is None: timeout_action = _TIMEOUT_TERM utils_wrapper.IgnoreProcessNotFound(os.kill, child.pid, signal.SIGTERM) linger_timeout = \ utils_algo.RunningTimeout(_linger_timeout, True).Remaining pt = linger_timeout() * 1000 if pt < 0: break else: pt = None pollresult = utils_wrapper.RetryOnSignal(poller.poll, pt) for fd, event in pollresult: if event & select.POLLIN or event & select.POLLPRI: data = fdmap[fd][1].read() # no data from read signifies EOF (the same as POLLHUP) if not data: poller.unregister(fd) fdmap[fd][1].close() del fdmap[fd] continue fdmap[fd][0].write(data) if (event & select.POLLNVAL or event & select.POLLHUP or event & select.POLLERR): poller.unregister(fd) fdmap[fd][1].close() del fdmap[fd] if fdmap: for (_, handle) in fdmap.values(): handle.close() if timeout is not None: assert callable(poll_timeout) # We have no I/O left but it might still run if child.poll() is None: _WaitForProcess(child, poll_timeout()) # Terminate if still alive after timeout if child.poll() is None: if linger_timeout is None: logging.warning(msg_timeout) timeout_action = _TIMEOUT_TERM utils_wrapper.IgnoreProcessNotFound(os.kill, child.pid, signal.SIGTERM) lt = _linger_timeout else: lt = linger_timeout() _WaitForProcess(child, lt) # Okay, still alive after timeout and linger timeout? Kill it! if child.poll() is None: timeout_action = _TIMEOUT_KILL logging.warning(msg_linger) utils_wrapper.IgnoreProcessNotFound(os.kill, child.pid, signal.SIGKILL) out = out.getvalue() err = err.getvalue() status = child.wait() return out, err, status, timeout_action def _RunCmdFile(cmd, env, via_shell, output, cwd, noclose_fds): """Run a command and save its output to a file. @type cmd: string or list @param cmd: Command to run @type env: dict @param env: The environment to use @type via_shell: bool @param via_shell: if we should run via the shell @type output: str @param output: the filename in which to save the output @type cwd: string @param cwd: the working directory for the program @type noclose_fds: list @param noclose_fds: list of additional (fd >=3) file descriptors to leave open for the child process @rtype: int @return: the exit status """ fh = open(output, "a") if noclose_fds: pass_fds = noclose_fds else: pass_fds = [] try: child = subprocess.Popen(cmd, shell=via_shell, stderr=subprocess.STDOUT, stdout=fh, stdin=subprocess.PIPE, pass_fds=pass_fds, env=env, cwd=cwd, encoding="utf-8") child.stdin.close() status = child.wait() finally: fh.close() return status def RunParts(dir_name, env=None, reset_env=False): """Run Scripts or programs in a directory @type dir_name: string @param dir_name: absolute path to a directory @type env: dict @param env: The environment to use @type reset_env: boolean @param reset_env: whether to reset or keep the default os environment @rtype: list of tuples @return: list of (name, (one of RUNDIR_STATUS), RunResult) """ rr = [] try: dir_contents = utils_io.ListVisibleFiles(dir_name) except OSError as err: logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err) return rr for relname in sorted(dir_contents): fname = utils_io.PathJoin(dir_name, relname) if not (constants.EXT_PLUGIN_MASK.match(relname) is not None and utils_wrapper.IsExecutable(fname)): rr.append((relname, constants.RUNPARTS_SKIP, None)) else: try: result = RunCmd([fname], env=env, reset_env=reset_env) except Exception as err: # pylint: disable=W0703 rr.append((relname, constants.RUNPARTS_ERR, str(err))) else: rr.append((relname, constants.RUNPARTS_RUN, result)) return rr def _GetProcStatusPath(pid): """Returns the path for a PID's proc status file. @type pid: int @param pid: Process ID @rtype: string """ return "/proc/%d/status" % pid def GetProcCmdline(pid): """Returns the command line of a pid as a list of arguments. @type pid: int @param pid: Process ID @rtype: list of string @raise EnvironmentError: If the process does not exist """ proc_path = "/proc/%d/cmdline" % pid with open(proc_path, 'r') as f: nulled_cmdline = f.read() # Individual arguments are separated by nul chars in the contents of the proc # file return nulled_cmdline.split('\x00') def IsProcessAlive(pid): """Check if a given pid exists on the system. @note: zombie status is not handled, so zombie processes will be returned as alive @type pid: int @param pid: the process ID to check @rtype: boolean @return: True if the process exists """ def _TryStat(name): try: os.stat(name) return True except EnvironmentError as err: if err.errno in (errno.ENOENT, errno.ENOTDIR): return False elif err.errno == errno.EINVAL: raise utils_retry.RetryAgain(err) raise assert isinstance(pid, int), "pid must be an integer" if pid <= 0: return False # /proc in a multiprocessor environment can have strange behaviors. # Retry the os.stat a few times until we get a good result. try: return utils_retry.Retry(_TryStat, (0.01, 1.5, 0.1), 0.5, args=[_GetProcStatusPath(pid)]) except utils_retry.RetryTimeout as err: err.RaiseInner() def IsDaemonAlive(name): """Determines whether a daemon is alive @type name: string @param name: daemon name @rtype: boolean @return: True if daemon is running, False otherwise """ return IsProcessAlive(utils_io.ReadPidFile(utils_io.DaemonPidFileName(name))) def _ParseSigsetT(sigset): """Parse a rendered sigset_t value. This is the opposite of the Linux kernel's fs/proc/array.c:render_sigset_t function. @type sigset: string @param sigset: Rendered signal set from /proc/$pid/status @rtype: set @return: Set of all enabled signal numbers """ result = set() signum = 0 for ch in reversed(sigset): chv = int(ch, 16) # The following could be done in a loop, but it's easier to read and # understand in the unrolled form if chv & 1: result.add(signum + 1) if chv & 2: result.add(signum + 2) if chv & 4: result.add(signum + 3) if chv & 8: result.add(signum + 4) signum += 4 return result def _GetProcStatusField(pstatus, field): """Retrieves a field from the contents of a proc status file. @type pstatus: string @param pstatus: Contents of /proc/$pid/status @type field: string @param field: Name of field whose value should be returned @rtype: string """ for line in pstatus.splitlines(): parts = line.split(":", 1) if len(parts) < 2 or parts[0] != field: continue return parts[1].strip() return None def IsProcessHandlingSignal(pid, signum, status_path=None): """Checks whether a process is handling a signal. @type pid: int @param pid: Process ID @type signum: int @param signum: Signal number @rtype: bool """ if status_path is None: status_path = _GetProcStatusPath(pid) try: proc_status = utils_io.ReadFile(status_path) except EnvironmentError as err: # In at least one case, reading /proc/$pid/status failed with ESRCH. if err.errno in (errno.ENOENT, errno.ENOTDIR, errno.EINVAL, errno.ESRCH): return False raise sigcgt = _GetProcStatusField(proc_status, "SigCgt") if sigcgt is None: raise RuntimeError("%s is missing 'SigCgt' field" % status_path) # Now check whether signal is handled return signum in _ParseSigsetT(sigcgt) def Daemonize(logfile): """Daemonize the current process. This detaches the current process from the controlling terminal and runs it in the background as a daemon. @type logfile: str @param logfile: the logfile to which we should redirect stdout/stderr @rtype: tuple; (int, callable) @return: File descriptor of pipe(2) which must be closed to notify parent process and a callable to reopen log files """ # pylint: disable=W0212 # yes, we really want os._exit # TODO: do another attempt to merge Daemonize and StartDaemon, or at # least abstract the pipe functionality between them # Create pipe for sending error messages (rpipe, wpipe) = os.pipe() # this might fail pid = os.fork() if pid == 0: # The first child. SetupDaemonEnv() # this might fail pid = os.fork() # Fork a second child. if pid == 0: # The second child. utils_wrapper.CloseFdNoError(rpipe) else: # exit() or _exit()? See below. os._exit(0) # Exit parent (the first child) of the second child. else: utils_wrapper.CloseFdNoError(wpipe) # Wait for daemon to be started (or an error message to # arrive) and read up to 100 KB as an error message errormsg = utils_wrapper.RetryOnSignal(os.read, rpipe, 100 * 1024) if errormsg: sys.stderr.write("Error when starting daemon process: %r\n" % errormsg) rcode = 1 else: rcode = 0 os._exit(rcode) # Exit parent of the first child. reopen_fn = compat.partial(SetupDaemonFDs, logfile, None) # Open logs for the first time reopen_fn() return (wpipe, reopen_fn) def KillProcess(pid, signal_=signal.SIGTERM, timeout=30, waitpid=False): """Kill a process given by its pid. @type pid: int @param pid: The PID to terminate. @type signal_: int @param signal_: The signal to send, by default SIGTERM @type timeout: int @param timeout: The timeout after which, if the process is still alive, a SIGKILL will be sent. If not positive, no such checking will be done @type waitpid: boolean @param waitpid: If true, we should waitpid on this process after sending signals, since it's our own child and otherwise it would remain as zombie """ def _helper(pid, signal_, wait): """Simple helper to encapsulate the kill/waitpid sequence""" if utils_wrapper.IgnoreProcessNotFound(os.kill, pid, signal_) and wait: try: os.waitpid(pid, os.WNOHANG) except OSError: pass if pid <= 0: # kill with pid=0 == suicide raise errors.ProgrammerError("Invalid pid given '%s'" % pid) if not IsProcessAlive(pid): return _helper(pid, signal_, waitpid) if timeout <= 0: return def _CheckProcess(): if not IsProcessAlive(pid): return try: (result_pid, _) = os.waitpid(pid, os.WNOHANG) except OSError: raise utils_retry.RetryAgain() if result_pid > 0: return raise utils_retry.RetryAgain() try: # Wait up to $timeout seconds utils_retry.Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout) except utils_retry.RetryTimeout: pass if IsProcessAlive(pid): # Kill process if it's still alive _helper(pid, signal.SIGKILL, waitpid) def RunInSeparateProcess(fn, *args): """Runs a function in a separate process. Note: Only boolean return values are supported. @type fn: callable @param fn: Function to be called @rtype: bool @return: Function's result """ pid = os.fork() if pid == 0: # Child process try: # In case the function uses temporary files utils_wrapper.ResetTempfileModule() # Call function result = int(bool(fn(*args))) assert result in (0, 1) except: # pylint: disable=W0702 logging.exception("Error while calling function in separate process") # 0 and 1 are reserved for the return value result = 33 os._exit(result) # pylint: disable=W0212 # Parent process # Avoid zombies and check exit code (_, status) = os.waitpid(pid, 0) if os.WIFSIGNALED(status): exitcode = None signum = os.WTERMSIG(status) else: exitcode = os.WEXITSTATUS(status) signum = None if not (exitcode in (0, 1) and signum is None): raise errors.GenericError("Child program failed (code=%s, signal=%s)" % (exitcode, signum)) return bool(exitcode) def CloseFDs(noclose_fds=None): """Close file descriptors. This closes all file descriptors above 2 (i.e. except stdin/out/err). @type noclose_fds: list or None @param noclose_fds: if given, it denotes a list of file descriptor that should not be closed """ # Default maximum for the number of available file descriptors. if 'SC_OPEN_MAX' in os.sysconf_names: try: MAXFD = os.sysconf('SC_OPEN_MAX') if MAXFD < 0: MAXFD = 1024 except OSError: MAXFD = 1024 else: MAXFD = 1024 # Iterate through and close all file descriptors (except the standard ones) for fd in range(3, MAXFD): if noclose_fds and fd in noclose_fds: continue utils_wrapper.CloseFdNoError(fd) ganeti-3.1.0~rc2/lib/utils/retry.py000064400000000000000000000201321476477700300172270ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for retrying function calls with a timeout. """ import logging import time from ganeti import errors #: Special delay to specify whole remaining timeout RETRY_REMAINING_TIME = object() class RetryTimeout(Exception): """Retry loop timed out. Any arguments which was passed by the retried function to RetryAgain will be preserved in RetryTimeout, if it is raised. If such argument was an exception the RaiseInner helper method will reraise it. """ def RaiseInner(self): if self.args and isinstance(self.args[0], Exception): raise self.args[0] else: raise RetryTimeout(*self.args) class RetryAgain(Exception): """Retry again. Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as arguments to RetryTimeout. If an exception is passed, the RaiseInner() method of the RetryTimeout() method can be used to reraise it. """ class _RetryDelayCalculator(object): """Calculator for increasing delays. """ __slots__ = [ "_factor", "_limit", "_next", "_start", ] def __init__(self, start, factor, limit): """Initializes this class. @type start: float @param start: Initial delay @type factor: float @param factor: Factor for delay increase @type limit: float or None @param limit: Upper limit for delay or None for no limit """ assert start > 0.0 assert factor >= 1.0 assert limit is None or limit >= 0.0 self._start = start self._factor = factor self._limit = limit self._next = start def __call__(self): """Returns current delay and calculates the next one. """ current = self._next # Update for next run if self._limit is None or self._next < self._limit: self._next = min(self._limit, self._next * self._factor) return current def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep, _time_fn=time.time): """Call a function repeatedly until it succeeds. The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain} anymore. Between calls a delay, specified by C{delay}, is inserted. After a total of C{timeout} seconds, this function throws L{RetryTimeout}. C{delay} can be one of the following: - callable returning the delay length as a float - Tuple of (start, factor, limit) - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is useful when overriding L{wait_fn} to wait for an external event) - A static delay as a number (int or float) @type fn: callable @param fn: Function to be called @param delay: Either a callable (returning the delay), a tuple of (start, factor, limit) (see L{_RetryDelayCalculator}), L{RETRY_REMAINING_TIME} or a number (int or float) @type timeout: float @param timeout: Total timeout @type wait_fn: callable @param wait_fn: Waiting function @return: Return value of function """ assert callable(fn) assert callable(wait_fn) assert callable(_time_fn) if args is None: args = [] end_time = _time_fn() + timeout if callable(delay): # External function to calculate delay calc_delay = delay elif isinstance(delay, (tuple, list)): # Increasing delay with optional upper boundary (start, factor, limit) = delay calc_delay = _RetryDelayCalculator(start, factor, limit) elif delay is RETRY_REMAINING_TIME: # Always use the remaining time calc_delay = None else: # Static delay calc_delay = lambda: delay assert calc_delay is None or callable(calc_delay) while True: retry_args = [] try: return fn(*args) except RetryAgain as err: retry_args = err.args except RetryTimeout: raise errors.ProgrammerError("Nested retry loop detected that didn't" " handle RetryTimeout") remaining_time = end_time - _time_fn() if remaining_time <= 0.0: raise RetryTimeout(*retry_args) assert remaining_time > 0.0 if calc_delay is None: wait_fn(remaining_time) else: current_delay = calc_delay() if current_delay > 0.0: wait_fn(current_delay) def SimpleRetry(expected, fn, delay, timeout, args=None, wait_fn=time.sleep, _time_fn=time.time): """A wrapper over L{Retry} implementing a simpler interface. All the parameters are the same as for L{Retry}, except it has one extra argument: expected, which can be either a value (will be compared with the result of the function, or a callable (which will get the result passed and has to return a boolean). If the test is false, we will retry until either the timeout has passed or the tests succeeds. In both cases, the last result from calling the function will be returned. Note that this function is not expected to raise any retry-related exceptions, always simply returning values. As such, the function is designed to allow easy wrapping of code that doesn't use retry at all (e.g. "if fn(args)" replaced with "if SimpleRetry(True, fn, ...)". @see: L{Retry} """ rdict = {} def helper(*innerargs): result = rdict["result"] = fn(*innerargs) if not ((callable(expected) and expected(result)) or result == expected): raise RetryAgain() return result try: result = Retry(helper, delay, timeout, args=args, wait_fn=wait_fn, _time_fn=_time_fn) except RetryTimeout: assert "result" in rdict result = rdict["result"] return result def CountRetry(expected, fn, count, args=None): """A wrapper over L{SimpleRetry} implementing a count down. Where L{Retry} fixes the time, after which the command is assumed to be failing, this function assumes the total number of tries. @see: L{Retry} """ rdict = {"tries": 0} get_tries = lambda: rdict["tries"] def inc_tries(t): rdict["tries"] += t return SimpleRetry(expected, fn, 1, count, args=args, wait_fn=inc_tries, _time_fn=get_tries) def RetryByNumberOfTimes(max_retries, exception_class, fn, *args, **kwargs): """Retries calling a function up to the specified number of times. @type max_retries: integer @param max_retries: Maximum number of retries. @type exception_class: class @param exception_class: Exception class which is used for throwing the final exception. @type fn: callable @param fn: Function to be called (up to the specified maximum number of retries. """ last_exception = None for i in range(max_retries): try: fn(*args, **kwargs) break except errors.OpExecError as e: logging.error("Error after retry no. %s: %s.", i, e) last_exception = e else: if last_exception: raise exception_class("Error after %s retries. Last exception: %s." % (max_retries, last_exception)) ganeti-3.1.0~rc2/lib/utils/security.py000064400000000000000000000133471476477700300177430ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for security features of Ganeti. """ import logging import os import time import uuid as uuid_module import OpenSSL from ganeti.utils import io from ganeti.utils import x509 from ganeti import constants from ganeti import errors from ganeti import pathutils def UuidToInt(uuid): uuid_obj = uuid_module.UUID(uuid) return uuid_obj.int # pylint: disable=E1101 def GetCertificateDigest(cert_filename=pathutils.NODED_CLIENT_CERT_FILE): """Reads the SSL certificate and returns the sha1 digest. """ cert_plain = io.ReadFile(cert_filename) cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_plain) return cert.digest("sha1").decode("ascii") def GenerateNewSslCert(new_cert, cert_filename, serial_no, log_msg, uid=-1, gid=-1): """Creates a new server SSL certificate and backups the old one. @type new_cert: boolean @param new_cert: whether a new certificate should be created @type cert_filename: string @param cert_filename: filename of the certificate file @type serial_no: int @param serial_no: serial number of the certificate @type log_msg: string @param log_msg: log message to be written on certificate creation @type uid: int @param uid: the user ID of the user who will be owner of the certificate file @type gid: int @param gid: the group ID of the group who will own the certificate file """ cert_exists = os.path.exists(cert_filename) if new_cert or not cert_exists: if cert_exists: io.CreateBackup(cert_filename) logging.debug(log_msg) x509.GenerateSelfSignedSslCert(cert_filename, serial_no, uid=uid, gid=gid) def GenerateNewClientSslCert(cert_filename, signing_cert_filename, hostname): """Creates a new server SSL certificate and backups the old one. @type cert_filename: string @param cert_filename: filename of the certificate file @type signing_cert_filename: string @param signing_cert_filename: name of the certificate to be used for signing @type hostname: string @param hostname: name of the machine whose cert is created """ serial_no = int(time.time()) x509.GenerateSignedSslCert(cert_filename, serial_no, signing_cert_filename, common_name=hostname) def VerifyCertificate(filename): """Verifies a SSL certificate. @type filename: string @param filename: Path to PEM file """ try: cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, io.ReadFile(filename)) except Exception as err: # pylint: disable=W0703 return (constants.CV_ERROR, "Failed to load X509 certificate %s: %s" % (filename, err)) (errcode, msg) = \ x509.VerifyX509Certificate(cert, constants.SSL_CERT_EXPIRATION_WARN, constants.SSL_CERT_EXPIRATION_ERROR) if msg: fnamemsg = "While verifying %s: %s" % (filename, msg) else: fnamemsg = None if errcode is None: return (None, fnamemsg) elif errcode == x509.CERT_WARNING: return (constants.CV_WARNING, fnamemsg) elif errcode == x509.CERT_ERROR: return (constants.CV_ERROR, fnamemsg) raise errors.ProgrammerError("Unhandled certificate error code %r" % errcode) def IsCertificateSelfSigned(cert_filename): """Checks whether the certificate issuer is the same as the owner. Note that this does not actually verify the signature, it simply compares the certificates common name and the issuer's common name. This is sufficient, because now that Ganeti started creating non-self-signed client-certificates, it uses their hostnames as common names and thus they are distinguishable by common name from the server certificates. @type cert_filename: string @param cert_filename: filename of the certificate to examine """ try: cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, io.ReadFile(cert_filename)) except Exception as err: # pylint: disable=W0703 return (constants.CV_ERROR, "Failed to load X509 certificate %s: %s" % (cert_filename, err)) if cert.get_subject().CN == cert.get_issuer().CN: msg = "The certificate '%s' is self-signed. Please run 'gnt-cluster" \ " renew-crypto --new-node-certificates' to get a properly signed" \ " certificate." % cert_filename return (constants.CV_WARNING, msg) return (None, None) ganeti-3.1.0~rc2/lib/utils/storage.py000064400000000000000000000251671476477700300175430ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for storage. """ import logging from ganeti import constants from ganeti import errors from ganeti.utils import io as utils_io from ganeti.utils import process as utils_process def GetDiskTemplatesOfStorageTypes(*storage_types): """Given the storage type, returns a list of disk templates based on that storage type.""" return [dt for dt in constants.DISK_TEMPLATES if constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[dt] in storage_types] def IsDiskTemplateEnabled(disk_template, enabled_disk_templates): """Checks if a particular disk template is enabled. """ return disk_template in enabled_disk_templates def IsFileStorageEnabled(enabled_disk_templates): """Checks if file storage is enabled. """ return IsDiskTemplateEnabled(constants.DT_FILE, enabled_disk_templates) def IsSharedFileStorageEnabled(enabled_disk_templates): """Checks if shared file storage is enabled. """ return IsDiskTemplateEnabled(constants.DT_SHARED_FILE, enabled_disk_templates) def IsLvmEnabled(enabled_disk_templates): """Check whether or not any lvm-based disk templates are enabled.""" return len(constants.DTS_LVM & set(enabled_disk_templates)) != 0 def LvmGetsEnabled(enabled_disk_templates, new_enabled_disk_templates): """Checks whether lvm was not enabled before, but will be enabled after the operation. """ if IsLvmEnabled(enabled_disk_templates): return False return len(constants.DTS_LVM & set(new_enabled_disk_templates)) != 0 def _GetDefaultStorageUnitForDiskTemplate(cfg, disk_template): """Retrieves the identifier of the default storage entity for the given storage type. @type cfg: C{objects.ConfigData} @param cfg: the configuration data @type disk_template: string @param disk_template: a disk template, for example 'drbd' @rtype: string @return: identifier for a storage unit, for example the vg_name for lvm storage """ storage_type = constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[disk_template] cluster = cfg.GetClusterInfo() if disk_template in constants.DTS_LVM: return (storage_type, cfg.GetVGName()) elif disk_template == constants.DT_FILE: return (storage_type, cluster.file_storage_dir) elif disk_template == constants.DT_SHARED_FILE: return (storage_type, cluster.shared_file_storage_dir) elif disk_template == constants.DT_GLUSTER: return (storage_type, cluster.gluster_storage_dir) else: return (storage_type, None) def DiskTemplateSupportsSpaceReporting(disk_template): """Check whether the disk template supports storage space reporting.""" return (constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[disk_template] in constants.STS_REPORT) def GetStorageUnits(cfg, disk_templates): """Get the cluster's storage units for the given disk templates. If any lvm-based disk template is requested, spindle information is added to the request. @type cfg: L{config.ConfigWriter} @param cfg: Cluster configuration @type disk_templates: list of string @param disk_templates: list of disk templates for which the storage units will be computed @rtype: list of tuples (string, string) @return: list of storage units, each storage unit being a tuple of (storage_type, storage_key); storage_type is in C{constants.STORAGE_TYPES} and the storage_key a string to identify an entity of that storage type, for example a volume group name for LVM storage or a file for file storage. """ storage_units = [] for disk_template in disk_templates: if DiskTemplateSupportsSpaceReporting(disk_template): storage_units.append( _GetDefaultStorageUnitForDiskTemplate(cfg, disk_template)) return storage_units def LookupSpaceInfoByDiskTemplate(storage_space_info, disk_template): """Looks up the storage space info for a given disk template. @type storage_space_info: list of dicts @param storage_space_info: result of C{GetNodeInfo} @type disk_template: string @param disk_template: disk template to get storage space info @rtype: tuple @return: returns the element of storage_space_info that matches the given disk template """ storage_type = constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[disk_template] return LookupSpaceInfoByStorageType(storage_space_info, storage_type) def LookupSpaceInfoByStorageType(storage_space_info, storage_type): """Looks up the storage space info for a given storage type. Note that this lookup can be ambiguous if storage space reporting for several units of the same storage type was requested. This function is only supposed to be used for legacy code in situations where it actually is unambiguous. @type storage_space_info: list of dicts @param storage_space_info: result of C{GetNodeInfo} @type storage_type: string @param storage_type: a storage type, which is included in the storage_units list @rtype: tuple @return: returns the element of storage_space_info that matches the given storage type """ result = None for unit_info in storage_space_info: if unit_info["type"] == storage_type: if result is None: result = unit_info else: # There is more than one storage type in the query, log a warning logging.warning("Storage space information requested for" " ambiguous storage type '%s'.", storage_type) return result def GetDiskLabels(prefix, num_disks, start=0): """Generate disk labels for a number of disks Note that disk labels are generated in the range [start..num_disks[ (e.g., as in range(start, num_disks)) @type prefix: string @param prefix: disk label prefix (e.g., "/dev/sd") @type num_disks: int @param num_disks: number of disks (i.e., disk labels) @type start: int @param start: optional start index @rtype: generator @return: generator for the disk labels """ def _GetDiskSuffix(i): n = ord('z') - ord('a') + 1 if i < n: return chr(ord('a') + i) else: mod = int(i % n) pref = _GetDiskSuffix((i - mod) // (n + 1)) suf = _GetDiskSuffix(mod) return pref + suf for i in range(start, num_disks): yield prefix + _GetDiskSuffix(i) def CreateBdevPartitionMapping(image_path): """Create dm device for each partition of disk image. This operation will allocate a loopback and a device-mapper device to map partitions. You must call L{ReleaseBdevPartitionMapping} to clean up resources allocated by this function call. @type image_path: string @param image_path: path of multi-partition disk image @rtype: tuple(string, list(string)) or NoneType @return: returns the tuple(loopback_device, list(device_mapper_files)) if image_path is a multi-partition disk image. otherwise, returns None. """ # Unfortunately, there are two different losetup commands in this world. # One has the '-s' switch and the other has the '--show' switch to provide the # same functionality. result = utils_process.RunCmd(["losetup", "-f", "-s", image_path]) if result.failed and "invalid option -- 's'" in result.stderr: result = utils_process.RunCmd(["losetup", "-f", "--show", image_path]) if result.failed: raise errors.CommandError("Failed to setup loop device for %s: %s" % (image_path, result.output)) loop_dev_path = result.stdout.strip() logging.debug("Loop dev %s allocated for %s", loop_dev_path, image_path) result = utils_process.RunCmd(["kpartx", "-a", "-v", loop_dev_path]) if result.failed: # Just try to cleanup allocated loop device utils_process.RunCmd(["losetup", "-d", loop_dev_path]) raise errors.CommandError("Failed to add partition mapping for %s: %s" % (image_path, result.output)) dm_devs = [x.split(" ") for x in result.stdout.split("\n") if x] if dm_devs: dm_dev_paths = [utils_io.PathJoin("/dev/mapper", x[2]) for x in dm_devs] return (loop_dev_path, dm_dev_paths) else: # image_path is not a multi partition disk image, no need to use # device-mapper. logging.debug("Release loop dev %s allocated for %s", loop_dev_path, image_path) ReleaseBdevPartitionMapping(loop_dev_path) return None def ReleaseBdevPartitionMapping(loop_dev_path): """Release allocated dm devices and loopback devices. @type loop_dev_path: string @param loop_dev_path: path of loopback device returned by L{CreateBdevPartitionMapping} """ result = utils_process.RunCmd(["kpartx", "-d", loop_dev_path]) if result.failed: raise errors.CommandError("Failed to release partition mapping of %s: %s" % (loop_dev_path, result.output)) # The invocation of udevadm settle was added here because users had issues # with the loopback device still being busy after kpartx / earlier commands # did their work. result = utils_process.RunCmd(["udevadm", "settle"]) if result.failed: raise errors.CommandError("Waiting on udev failed: %s" % result.output) result = utils_process.RunCmd(["losetup", "-d", loop_dev_path]) if result.failed: raise errors.CommandError("Failed to detach %s: %s" % (loop_dev_path, result.output)) def osminor(dev): """Return the device minor number from a raw device number. This is a replacement for os.minor working around the issue that Python's os.minor still has the old definition. See Ganeti issue 1058 for more details. """ return (dev & 0xff) | ((dev >> 12) & ~0xff) ganeti-3.1.0~rc2/lib/utils/tags.py000064400000000000000000000036651476477700300170340ustar00rootroot00000000000000# # # Copyright (C) 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for tag related operations """ from ganeti import constants def GetExclusionPrefixes(ctags): """Extract the exclusion tag prefixes from the cluster tags """ prefixes = set([]) for tag in ctags: if tag.startswith(constants.EX_TAGS_PREFIX): prefixes.add(tag[len(constants.EX_TAGS_PREFIX):]) return prefixes def IsGoodTag(prefixes, tag): """Decide if a string is a tag @param prefixes: set of prefixes that would indicate the tag being suitable @param tag: the tag in question """ for prefix in prefixes: if tag.startswith(prefix): return True return False ganeti-3.1.0~rc2/lib/utils/text.py000064400000000000000000000450011476477700300170500ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for manipulating or working with text. """ import re import os import time import numbers import collections from ganeti import errors from ganeti import compat #: Unit checker regexp _PARSEUNIT_REGEX = re.compile(r"^([.\d]+)\s*([a-zA-Z]+)?$") #: Characters which don't need to be quoted for shell commands _SHELL_UNQUOTED_RE = re.compile("^[-.,=:/_+@A-Za-z0-9]+$") #: Shell param checker regexp _SHELLPARAM_REGEX = re.compile(r"^[-a-zA-Z0-9._+/:%@]+$") #: ASCII equivalent of unicode character 'HORIZONTAL ELLIPSIS' (U+2026) _ASCII_ELLIPSIS = "..." #: MAC address octet _MAC_ADDR_OCTET_RE = r"[0-9a-f]{2}" def MatchNameComponent(key, name_list, case_sensitive=True): """Try to match a name against a list. This function will try to match a name like test1 against a list like C{['test1.example.com', 'test2.example.com', ...]}. Against this list, I{'test1'} as well as I{'test1.example'} will match, but not I{'test1.ex'}. A multiple match will be considered as no match at all (e.g. I{'test1'} against C{['test1.example.com', 'test1.example.org']}), except when the key fully matches an entry (e.g. I{'test1'} against C{['test1', 'test1.example.com']}). @type key: str @param key: the name to be searched @type name_list: list @param name_list: the list of strings against which to search the key @type case_sensitive: boolean @param case_sensitive: whether to provide a case-sensitive match @rtype: None or str @return: None if there is no match I{or} if there are multiple matches, otherwise the element from the list which matches """ if key in name_list: return key re_flags = 0 if not case_sensitive: re_flags |= re.IGNORECASE key = key.upper() name_re = re.compile(r"^%s(\..*)?$" % re.escape(key), re_flags) names_filtered = [] string_matches = [] for name in name_list: if name_re.match(name) is not None: names_filtered.append(name) if not case_sensitive and key == name.upper(): string_matches.append(name) if len(string_matches) == 1: return string_matches[0] if len(names_filtered) == 1: return names_filtered[0] return None def _DnsNameGlobHelper(match): """Helper function for L{DnsNameGlobPattern}. Returns regular expression pattern for parts of the pattern. """ text = match.group(0) if text == "*": return "[^.]*" elif text == "?": return "[^.]" else: return re.escape(text) def DnsNameGlobPattern(pattern): """Generates regular expression from DNS name globbing pattern. A DNS name globbing pattern (e.g. C{*.site}) is converted to a regular expression. Escape sequences or ranges (e.g. [a-z]) are not supported. Matching always starts at the leftmost part. An asterisk (*) matches all characters except the dot (.) separating DNS name parts. A question mark (?) matches a single character except the dot (.). @type pattern: string @param pattern: DNS name globbing pattern @rtype: string @return: Regular expression """ return r"^%s(\..*)?$" % re.sub(r"\*|\?|[^*?]*", _DnsNameGlobHelper, pattern) def FormatUnit(value, units, roman=False): """Formats an incoming number of MiB with the appropriate unit. @type value: int @param value: integer representing the value in MiB (1048576) @type units: char @param units: the type of formatting we should do: - 'h' for automatic scaling - 'm' for MiBs - 'g' for GiBs - 't' for TiBs @rtype: str @return: the formatted value (with suffix) """ if units not in ("m", "g", "t", "h"): raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units)) if not isinstance(value, numbers.Real): raise errors.ProgrammerError("Invalid value specified '%s (%s)'" % ( value, type(value))) suffix = "" if units == "m" or (units == "h" and value < 1024): if units == "h": suffix = "M" return "%s%s" % (compat.RomanOrRounded(value, 0, roman), suffix) elif units == "g" or (units == "h" and value < (1024 * 1024)): if units == "h": suffix = "G" return "%s%s" % (compat.RomanOrRounded(float(value) / 1024, 1, roman), suffix) else: if units == "h": suffix = "T" return "%s%s" % (compat.RomanOrRounded(float(value) / 1024 / 1024, 1, roman), suffix) def ParseUnit(input_string): """Tries to extract number and scale from the given string. Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE* [UNIT]}. If no unit is specified, it defaults to MiB. Return value is always an int in MiB. """ m = _PARSEUNIT_REGEX.match(str(input_string)) if not m: raise errors.UnitParseError("Invalid format") value = float(m.groups()[0]) unit = m.groups()[1] if unit: lcunit = unit.lower() else: lcunit = "m" if lcunit in ("m", "mb", "mib"): # Value already in MiB pass elif lcunit in ("g", "gb", "gib"): value *= 1024 elif lcunit in ("t", "tb", "tib"): value *= 1024 * 1024 else: raise errors.UnitParseError("Unknown unit: %s" % unit) # Make sure we round up if int(value) < value: value += 1 # Round up to the next multiple of 4 value = int(value) if value % 4: value += 4 - value % 4 return value def ShellQuote(value): """Quotes shell argument according to POSIX. @type value: str @param value: the argument to be quoted @rtype: str @return: the quoted value """ if _SHELL_UNQUOTED_RE.match(value): return value else: return "'%s'" % value.replace("'", "'\\''") def ShellQuoteArgs(args): """Quotes a list of shell arguments. @type args: list @param args: list of arguments to be quoted @rtype: str @return: the quoted arguments concatenated with spaces """ return " ".join([ShellQuote(i) for i in args]) def ShellCombineCommands(cmdlist): """Out of a list of shell comands construct a single one. """ return ["/bin/sh", "-c", " && ".join(ShellQuoteArgs(c) for c in cmdlist)] class ShellWriter(object): """Helper class to write scripts with indentation. """ INDENT_STR = " " def __init__(self, fh, indent=True): """Initializes this class. """ self._fh = fh self._indent_enabled = indent self._indent = 0 def IncIndent(self): """Increase indentation level by 1. """ self._indent += 1 def DecIndent(self): """Decrease indentation level by 1. """ assert self._indent > 0 self._indent -= 1 def Write(self, txt, *args): """Write line to output file. """ assert self._indent >= 0 if args: line = txt % args else: line = txt if line and self._indent_enabled: # Indent only if there's something on the line self._fh.write(self._indent * self.INDENT_STR) self._fh.write(line) self._fh.write("\n") def GenerateSecret(numbytes=20): """Generates a random secret. This will generate a pseudo-random secret returning an hex string (so that it can be used where an ASCII string is needed). @param numbytes: the number of bytes which will be represented by the returned string (defaulting to 20, the length of a SHA1 hash) @rtype: str @return: an hex representation of the pseudo-random sequence """ return os.urandom(numbytes).hex() def _MakeMacAddrRegexp(octets): """Builds a regular expression for verifying MAC addresses. @type octets: integer @param octets: How many octets to expect (1-6) @return: Compiled regular expression """ assert octets > 0 assert octets <= 6 return re.compile("^%s$" % ":".join([_MAC_ADDR_OCTET_RE] * octets), re.I) #: Regular expression for full MAC address _MAC_CHECK_RE = _MakeMacAddrRegexp(6) #: Regular expression for half a MAC address _MAC_PREFIX_CHECK_RE = _MakeMacAddrRegexp(3) def _MacAddressCheck(check_re, mac, msg): """Checks a MAC address using a regular expression. @param check_re: Compiled regular expression as returned by C{re.compile} @type mac: string @param mac: MAC address to be validated @type msg: string @param msg: Error message (%s will be replaced with MAC address) """ if check_re.match(mac): return mac.lower() raise errors.OpPrereqError(msg % mac, errors.ECODE_INVAL) def NormalizeAndValidateMac(mac): """Normalizes and check if a MAC address is valid and contains six octets. Checks whether the supplied MAC address is formally correct. Accepts colon-separated format only. Normalize it to all lower case. @type mac: string @param mac: MAC address to be validated @rtype: string @return: Normalized and validated MAC address @raise errors.OpPrereqError: If the MAC address isn't valid """ return _MacAddressCheck(_MAC_CHECK_RE, mac, "Invalid MAC address '%s'") def NormalizeAndValidateThreeOctetMacPrefix(mac): """Normalizes a potential MAC address prefix (three octets). Checks whether the supplied string is a valid MAC address prefix consisting of three colon-separated octets. The result is normalized to all lower case. @type mac: string @param mac: Prefix to be validated @rtype: string @return: Normalized and validated prefix @raise errors.OpPrereqError: If the MAC address prefix isn't valid """ return _MacAddressCheck(_MAC_PREFIX_CHECK_RE, mac, "Invalid MAC address prefix '%s'") def SafeEncode(text): """Return a 'safe' version of a source string. This function mangles the input string and returns a version that should be safe to display/encode as ASCII. To this end, we first convert it to ASCII using the 'backslashreplace' encoding which should get rid of any non-ASCII chars, and then we process it through a loop copied from the string repr sources in the python; we don't use string_escape anymore since that escape single quotes and backslashes too, and that is too much; and that escaping is not stable, i.e. string_escape(string_escape(x)) != string_escape(x). @type text: str or unicode @param text: input data @rtype: str @return: a safe version of text """ if not isinstance(text, str): raise TypeError("Only str can be SafeEncoded") text = text.encode("ascii", "backslashreplace").decode("ascii") resu = "" for char in text: c = ord(char) if char == "\t": resu += r"\t" elif char == "\n": resu += r"\n" elif char == "\r": resu += r'\'r' elif c < 32 or c >= 127: # non-printable resu += "\\x%02x" % (c & 0xff) else: resu += char return resu def UnescapeAndSplit(text, sep=","): r"""Split and unescape a string based on a given separator. This function splits a string based on a separator where the separator itself can be escape in order to be an element of the elements. The escaping rules are (assuming coma being the separator): - a plain , separates the elements - a sequence \\\\, (double backslash plus comma) is handled as a backslash plus a separator comma - a sequence \, (backslash plus comma) is handled as a non-separator comma @type text: string @param text: the string to split @type sep: string @param text: the separator @rtype: string @return: a list of strings """ # we split the list by sep (with no escaping at this stage) slist = text.split(sep) # next, we revisit the elements and if any of them ended with an odd # number of backslashes, then we join it with the next rlist = [] while slist: e1 = slist.pop(0) if e1.endswith("\\"): num_b = len(e1) - len(e1.rstrip("\\")) if num_b % 2 == 1 and slist: e2 = slist.pop(0) # Merge the two elements and push the result back to the source list for # revisiting. If e2 ended with backslashes, further merging may need to # be done. slist.insert(0, e1 + sep + e2) continue # here the backslashes remain (all), and will be reduced in the next step rlist.append(e1) # finally, replace backslash-something with something rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist] return rlist def EscapeAndJoin(slist, sep=","): """Encode a list in a way parsable by UnescapeAndSplit. @type slist: list of strings @param slist: the strings to be encoded @rtype: string @return: the encoding of the list oas a string """ return sep.join([re.sub("\\" + sep, "\\\\" + sep, re.sub(r"\\", r"\\\\", v)) for v in slist]) def CommaJoin(names): """Nicely join a set of identifiers. @param names: set, list or tuple @return: a string with the formatted results """ return ", ".join([str(val) for val in names]) def FormatTime(val, usecs=None): """Formats a time value. @type val: float or None @param val: Timestamp as returned by time.time() (seconds since Epoch, 1970-01-01 00:00:00 UTC) @return: a string value or N/A if we don't have a valid timestamp """ if val is None or not isinstance(val, (int, float)): return "N/A" # these two codes works on Linux, but they are not guaranteed on all # platforms result = time.strftime("%F %T", time.localtime(val)) if usecs is not None: result += ".%06d" % usecs return result def FormatSeconds(secs): """Formats seconds for easier reading. @type secs: number @param secs: Number of seconds @rtype: string @return: Formatted seconds (e.g. "2d 9h 19m 49s") """ parts = [] secs = round(secs, 0) if secs > 0: # Negative values would be a bit tricky for unit, one in [("d", 24 * 60 * 60), ("h", 60 * 60), ("m", 60)]: (complete, secs) = divmod(secs, one) if complete or parts: parts.append("%d%s" % (complete, unit)) parts.append("%ds" % secs) return " ".join(parts) class LineSplitter(object): """Splits byte data chunks into lines of text separated by newline. Instances provide a file-like interface. """ def __init__(self, line_fn, *args): """Initializes this class. @type line_fn: callable @param line_fn: Function called for each line, first parameter is line @param args: Extra arguments for L{line_fn} """ assert callable(line_fn) if args: # Python 2.4 doesn't have functools.partial yet self._line_fn = lambda line: line_fn(line, *args) else: self._line_fn = line_fn self._lines = collections.deque() self._buffer = b"" def write(self, data): parts = (self._buffer + data).split(b"\n") self._buffer = parts.pop() self._lines.extend(p.decode() for p in parts) def flush(self): while self._lines: self._line_fn(self._lines.popleft().rstrip("\r\n")) def close(self): self.flush() if self._buffer: self._line_fn(self._buffer.decode()) def IsValidShellParam(word): """Verifies is the given word is safe from the shell's p.o.v. This means that we can pass this to a command via the shell and be sure that it doesn't alter the command line and is passed as such to the actual command. Note that we are overly restrictive here, in order to be on the safe side. @type word: str @param word: the word to check @rtype: boolean @return: True if the word is 'safe' """ return bool(_SHELLPARAM_REGEX.match(word)) def BuildShellCmd(template, *args): """Build a safe shell command line from the given arguments. This function will check all arguments in the args list so that they are valid shell parameters (i.e. they don't contain shell metacharacters). If everything is ok, it will return the result of template % args. @type template: str @param template: the string holding the template for the string formatting @rtype: str @return: the expanded command line """ for word in args: if not IsValidShellParam(word): raise errors.ProgrammerError("Shell argument '%s' contains" " invalid characters" % word) return template % args def FormatOrdinal(value): """Formats a number as an ordinal in the English language. E.g. the number 1 becomes "1st", 22 becomes "22nd". @type value: integer @param value: Number @rtype: string """ tens = value % 10 if value > 10 and value < 20: suffix = "th" elif tens == 1: suffix = "st" elif tens == 2: suffix = "nd" elif tens == 3: suffix = "rd" else: suffix = "th" return "%s%s" % (value, suffix) def Truncate(text, length): """Truncate string and add ellipsis if needed. @type text: string @param text: Text @type length: integer @param length: Desired length @rtype: string @return: Truncated text """ assert length > len(_ASCII_ELLIPSIS) # Serialize if necessary if not isinstance(text, str): text = str(text) if len(text) <= length: return text else: return text[:length - len(_ASCII_ELLIPSIS)] + _ASCII_ELLIPSIS def FilterEmptyLinesAndComments(text): """Filters empty lines and comments from a line-based string. Whitespace is also removed from the beginning and end of all lines. @type text: string @param text: Input string @rtype: list """ return [line for line in [s.strip() for s in text.splitlines()] # Ignore empty lines and comments if line and not line.startswith("#")] def FormatKeyValue(data): """Formats a dictionary as "key=value" parameters. The keys are sorted to have a stable order. @type data: dict @rtype: list of string """ return ["%s=%s" % (key, value) for (key, value) in sorted(data.items())] ganeti-3.1.0~rc2/lib/utils/version.py000064400000000000000000000151271476477700300175570ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Version utilities.""" import re from ganeti import constants _FULL_VERSION_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)") _SHORT_VERSION_RE = re.compile(r"(\d+)\.(\d+)") # The first Ganeti version that supports automatic upgrades FIRST_UPGRADE_VERSION = (2, 10, 0) CURRENT_VERSION = (constants.VERSION_MAJOR, constants.VERSION_MINOR, constants.VERSION_REVISION) # Format for CONFIG_VERSION: # 01 03 0123 = 01030123 # ^^ ^^ ^^^^ # | | + Configuration version/revision # | + Minor version # + Major version # # It is stored as an integer. Make sure not to write an octal number. # BuildVersion and SplitVersion must be in here because we can't import other # modules. The cfgupgrade tool must be able to read and write version numbers # and thus requires these functions. To avoid code duplication, they're kept in # here. def BuildVersion(major, minor, revision): """Calculates int version number from major, minor and revision numbers. Returns: int representing version number """ assert isinstance(major, int) assert isinstance(minor, int) assert isinstance(revision, int) return (1000000 * major + 10000 * minor + 1 * revision) def SplitVersion(version): """Splits version number stored in an int. Returns: tuple; (major, minor, revision) """ assert isinstance(version, int) (major, remainder) = divmod(version, 1000000) (minor, revision) = divmod(remainder, 10000) return (major, minor, revision) def ParseVersion(versionstring): """Parses a version string. @param versionstring: the version string to parse @type versionstring: string @rtype: tuple or None @return: (major, minor, revision) if parsable, None otherwise. """ m = _FULL_VERSION_RE.match(versionstring) if m is not None: return (int(m.group(1)), int(m.group(2)), int(m.group(3))) m = _SHORT_VERSION_RE.match(versionstring) if m is not None: return (int(m.group(1)), int(m.group(2)), 0) return None def UpgradeRange(target, current=CURRENT_VERSION): """Verify whether a version is within the range of automatic upgrades. @param target: The version to upgrade to as (major, minor, revision) @type target: tuple @param current: The version to upgrade from as (major, minor, revision) @type current: tuple @rtype: string or None @return: None, if within the range, and a human-readable error message otherwise """ if target < FIRST_UPGRADE_VERSION or current < FIRST_UPGRADE_VERSION: return "automatic upgrades only supported from 2.10 onwards" if target[0] != current[0]: # allow major upgrade from 2.16 to 3.0 if current[0:2] == (2,16) and target[0:2] == (3,0): return None # allow major downgrade from 3.0 to 2.16 if current[0:2] == (3,0) and target[0:2] == (2,16): return None # forbid any other major version up-/downgrades return "major version up- or downgrades are only supported between " \ "2.16 and 3.0" if target[1] < current[1] - 1: return "can only downgrade one minor version at a time" return None def ShouldCfgdowngrade(version, current=CURRENT_VERSION): """Decide whether cfgupgrade --downgrade should be called. Given the current version and the version to change to, decide if in the transition process cfgupgrade --downgrade should be called @param version: The version to upgrade to as (major, minor, revision) @type version: tuple @param current: The version to upgrade from as (major, minor, revision) @type current: tuple @rtype: bool @return: True, if cfgupgrade --downgrade should be called. """ return version[0] == current[0] and version[1] == current[1] - 1 def IsCorrectConfigVersion(targetversion, configversion): """Decide whether configuration version is compatible with the target. @param targetversion: The version to upgrade to as (major, minor, revision) @type targetversion: tuple @param configversion: The version of the current configuration @type configversion: tuple @rtype: bool @return: True, if the configversion fits with the target version. """ return (configversion[0] == targetversion[0] and configversion[1] == targetversion[1]) def IsBefore(version, major, minor, revision): """Decide if a given version is strictly before a given version. @param version: (major, minor, revision) or None, with None being before all versions @type version: (int, int, int) or None @param major: major version @type major: int @param minor: minor version @type minor: int @param revision: revision @type revision: int """ if version is None: return True return version < (major, minor, revision) def IsEqual(version, major, minor, revision): """Decide if a given version matches the given version. If the revision is set to None, only major and minor are compared. @param version: (major, minor, revision) or None, with None being before all versions @type version: (int, int, int) or None @param major: major version @type major: int @param minor: minor version @type minor: int @param revision: revision @type revision: int """ if version is None: return False if revision is None: current_major, current_minor, _ = version return (current_major, current_minor) == (major, minor) return version == (major, minor, revision) ganeti-3.1.0~rc2/lib/utils/wrapper.py000064400000000000000000000136671476477700300175610ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions wrapping other functions. """ import sys import time import socket import errno import tempfile import fcntl import os import select import logging def TestDelay(duration): """Sleep for a fixed amount of time. @type duration: float @param duration: the sleep duration, in seconds @rtype: (boolean, str) @return: False for negative value, and an accompanying error message; True otherwise (and msg is None) """ if duration < 0: return False, "Invalid sleep duration" time.sleep(duration) return True, None def CloseFdNoError(fd, retries=5): """Close a file descriptor ignoring errors. @type fd: int @param fd: the file descriptor @type retries: int @param retries: how many retries to make, in case we get any other error than EBADF """ try: os.close(fd) except OSError as err: if err.errno != errno.EBADF: if retries > 0: CloseFdNoError(fd, retries - 1) # else either it's closed already or we're out of retries, so we # ignore this and go on def SetCloseOnExecFlag(fd, enable): """Sets or unsets the close-on-exec flag on a file descriptor. @type fd: int @param fd: File descriptor @type enable: bool @param enable: Whether to set or unset it. """ flags = fcntl.fcntl(fd, fcntl.F_GETFD) if enable: flags |= fcntl.FD_CLOEXEC else: flags &= ~fcntl.FD_CLOEXEC fcntl.fcntl(fd, fcntl.F_SETFD, flags) def SetNonblockFlag(fd, enable): """Sets or unsets the O_NONBLOCK flag on on a file descriptor. @type fd: int @param fd: File descriptor @type enable: bool @param enable: Whether to set or unset it """ flags = fcntl.fcntl(fd, fcntl.F_GETFL) if enable: flags |= os.O_NONBLOCK else: flags &= ~os.O_NONBLOCK fcntl.fcntl(fd, fcntl.F_SETFL, flags) def RetryOnSignal(fn, *args, **kwargs): """Calls a function again if it failed due to EINTR. """ while True: try: return fn(*args, **kwargs) except EnvironmentError as err: if err.errno != errno.EINTR: raise except (socket.error, select.error) as err: # In python 2.6 and above select.error is an IOError, so it's handled # above, in 2.5 and below it's not, and it's handled here. if not (err.args and err.args[0] == errno.EINTR): raise def IgnoreProcessNotFound(fn, *args, **kwargs): """Ignores ESRCH when calling a process-related function. ESRCH is raised when a process is not found. @rtype: bool @return: Whether process was found """ try: fn(*args, **kwargs) except EnvironmentError as err: # Ignore ESRCH if err.errno == errno.ESRCH: return False raise return True def IgnoreSignals(fn, *args, **kwargs): """Tries to call a function ignoring failures due to EINTR. """ try: return fn(*args, **kwargs) except EnvironmentError as err: if err.errno == errno.EINTR: return None else: raise except (select.error, socket.error) as err: # In python 2.6 and above select.error is an IOError, so it's handled # above, in 2.5 and below it's not, and it's handled here. if err.args and err.args[0] == errno.EINTR: return None else: raise def GetClosedTempfile(*args, **kwargs): """Creates a temporary file and returns its path. """ (fd, path) = tempfile.mkstemp(*args, **kwargs) CloseFdNoError(fd) return path def IsExecutable(filename): """Checks whether a file exists and is executable. @type filename: string @param filename: Filename @rtype: bool """ return os.path.isfile(filename) and os.access(filename, os.X_OK) def ResetTempfileModule(_time=time.time): """Resets the random name generator of the tempfile module. This function should be called after C{os.fork} in the child process to ensure it creates a newly seeded random generator. Otherwise it would generate the same random parts as the parent process. If several processes race for the creation of a temporary file, this could lead to one not getting a temporary name. """ # pylint: disable=W0212 if ((sys.hexversion >= 0x020703F0 and sys.hexversion < 0x03000000) or sys.hexversion >= 0x030203F0): # Python 2.7 automatically resets the RNG on pid changes (i.e. forking) return try: lock = tempfile._once_lock lock.acquire() try: # Re-seed random name generator if tempfile._name_sequence: tempfile._name_sequence.rng.seed(hash(_time()) ^ os.getpid()) finally: lock.release() except AttributeError: logging.critical("The tempfile module misses at least one of the" " '_once_lock' and '_name_sequence' attributes") ganeti-3.1.0~rc2/lib/utils/x509.py000064400000000000000000000362401476477700300165760ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions for X509. """ import calendar import datetime import errno import logging import re import time import OpenSSL from ganeti import errors from ganeti import constants from ganeti import pathutils from ganeti.utils import text as utils_text from ganeti.utils import io as utils_io from ganeti.utils import hash as utils_hash HEX_CHAR_RE = r"[a-zA-Z0-9]" VALID_X509_SIGNATURE_SALT = re.compile("^%s+$" % HEX_CHAR_RE, re.S) X509_SIGNATURE = re.compile(r"^%s:\s*(?P%s+)/(?P%s+)$" % (re.escape(constants.X509_CERT_SIGNATURE_HEADER), HEX_CHAR_RE, HEX_CHAR_RE), re.S | re.I) # Certificate verification results (CERT_WARNING, CERT_ERROR) = range(1, 3) #: ASN1 time regexp _ASN1_TIME_REGEX = re.compile(r"^(\d+)([-+]\d\d)(\d\d)$") def _ParseAsn1Generalizedtime(value): """Parses an ASN1 GENERALIZEDTIME timestamp as used by pyOpenSSL. @type value: string or bytes @param value: ASN1 GENERALIZEDTIME timestamp @return: Seconds since the Epoch (1970-01-01 00:00:00 UTC) """ if value is None: return None if isinstance(value, bytes): value = value.decode("ascii") m = _ASN1_TIME_REGEX.match(value) if m: # We have an offset asn1time = m.group(1) hours = int(m.group(2)) minutes = int(m.group(3)) utcoffset = (60 * hours) + minutes else: if not value.endswith("Z"): raise ValueError("Missing timezone") asn1time = value[:-1] utcoffset = 0 parsed = time.strptime(asn1time, "%Y%m%d%H%M%S") tt = datetime.datetime(*(parsed[:7])) - datetime.timedelta(minutes=utcoffset) return calendar.timegm(tt.utctimetuple()) def GetX509CertValidity(cert): """Returns the validity period of the certificate. @type cert: OpenSSL.crypto.X509 @param cert: X509 certificate object """ not_before = _ParseAsn1Generalizedtime(cert.get_notBefore()) not_after = _ParseAsn1Generalizedtime(cert.get_notAfter()) return (not_before, not_after) def _VerifyCertificateInner(expired, not_before, not_after, now, warn_days, error_days): """Verifies certificate validity. @type expired: bool @param expired: Whether pyOpenSSL considers the certificate as expired @type not_before: number or None @param not_before: Unix timestamp before which certificate is not valid @type not_after: number or None @param not_after: Unix timestamp after which certificate is invalid @type now: number @param now: Current time as Unix timestamp @type warn_days: number or None @param warn_days: How many days before expiration a warning should be reported @type error_days: number or None @param error_days: How many days before expiration an error should be reported """ if expired: msg = "Certificate is expired" if not_before is not None and not_after is not None: msg += (" (valid from %s to %s)" % (utils_text.FormatTime(not_before), utils_text.FormatTime(not_after))) elif not_before is not None: msg += " (valid from %s)" % utils_text.FormatTime(not_before) elif not_after is not None: msg += " (valid until %s)" % utils_text.FormatTime(not_after) return (CERT_ERROR, msg) elif not_before is not None and not_before > now: return (CERT_WARNING, "Certificate not yet valid (valid from %s)" % utils_text.FormatTime(not_before)) elif not_after is not None: remaining_days = int((not_after - now) / (24 * 3600)) msg = "Certificate expires in about %d days" % remaining_days if error_days is not None and remaining_days <= error_days: return (CERT_ERROR, msg) if warn_days is not None and remaining_days <= warn_days: return (CERT_WARNING, msg) return (None, None) def VerifyX509Certificate(cert, warn_days, error_days): """Verifies a certificate for LUClusterVerify. @type cert: OpenSSL.crypto.X509 @param cert: X509 certificate object @type warn_days: number or None @param warn_days: How many days before expiration a warning should be reported @type error_days: number or None @param error_days: How many days before expiration an error should be reported """ # Depending on the pyOpenSSL version, this can just return (None, None) (not_before, not_after) = GetX509CertValidity(cert) now = time.time() + constants.NODE_MAX_CLOCK_SKEW return _VerifyCertificateInner(cert.has_expired(), not_before, not_after, now, warn_days, error_days) def SignX509Certificate(cert, key, salt): """Sign a X509 certificate. An RFC822-like signature header is added in front of the certificate. @type cert: OpenSSL.crypto.X509 @param cert: X509 certificate object @type key: string @param key: Key for HMAC @type salt: string @param salt: Salt for HMAC @rtype: string @return: Serialized and signed certificate in PEM format """ if not VALID_X509_SIGNATURE_SALT.match(salt): raise errors.GenericError("Invalid salt: %r" % salt) # Dumping as PEM here ensures the certificate is in a sane format cert_pem = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert).decode("ascii") return ("%s: %s/%s\n\n%s" % (constants.X509_CERT_SIGNATURE_HEADER, salt, utils_hash.Sha1Hmac(key, cert_pem, salt=salt), cert_pem)) def _ExtractX509CertificateSignature(cert_pem): """Helper function to extract signature from X509 certificate. """ if isinstance(cert_pem, bytes): cert_pem = cert_pem.decode("ascii") # Extract signature from original PEM data for line in cert_pem.splitlines(): if line.startswith("---"): break m = X509_SIGNATURE.match(line.strip()) if m: return (m.group("salt"), m.group("sign")) raise errors.GenericError("X509 certificate signature is missing") def LoadSignedX509Certificate(cert_pem, key): """Verifies a signed X509 certificate. @type cert_pem: string @param cert_pem: Certificate in PEM format and with signature header @type key: string @param key: Key for HMAC @rtype: tuple; (OpenSSL.crypto.X509, string) @return: X509 certificate object and salt """ (salt, signature) = _ExtractX509CertificateSignature(cert_pem) # Load and dump certificate to ensure it's in a sane format (cert, sane_pem) = ExtractX509Certificate(cert_pem) if not utils_hash.VerifySha1Hmac(key, sane_pem, signature, salt=salt): raise errors.GenericError("X509 certificate signature is invalid") return (cert, salt) def GenerateSelfSignedX509Cert(common_name, validity, serial_no): """Generates a self-signed X509 certificate. @type common_name: string @param common_name: commonName value @type validity: int @param validity: Validity for certificate in seconds @return: a tuple of strings containing the PEM-encoded private key and certificate """ # Create private and public key key = OpenSSL.crypto.PKey() key.generate_key(OpenSSL.crypto.TYPE_RSA, constants.RSA_KEY_BITS) # Create self-signed certificate cert = OpenSSL.crypto.X509() if common_name: cert.get_subject().CN = common_name cert.set_serial_number(serial_no) cert.gmtime_adj_notBefore(0) cert.gmtime_adj_notAfter(validity) cert.set_issuer(cert.get_subject()) cert.set_pubkey(key) cert.sign(key, constants.X509_CERT_SIGN_DIGEST) key_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key) cert_pem = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert) return (key_pem, cert_pem) def GenerateSelfSignedSslCert(filename, serial_no, common_name=constants.X509_CERT_CN, validity=constants.X509_CERT_DEFAULT_VALIDITY, uid=-1, gid=-1): """Legacy function to generate self-signed X509 certificate. @type filename: str @param filename: path to write certificate to @type common_name: string @param common_name: commonName value @type validity: int @param validity: validity of certificate in number of days @type uid: int @param uid: the user ID of the user who will be owner of the certificate file @type gid: int @param gid: the group ID of the group who will own the certificate file @return: a tuple of strings containing the PEM-encoded private key and certificate """ # TODO: Investigate using the cluster name instead of X505_CERT_CN for # common_name, as cluster-renames are very seldom, and it'd be nice if RAPI # and node daemon certificates have the proper Subject/Issuer. (key_pem, cert_pem) = GenerateSelfSignedX509Cert( common_name, validity * 24 * 60 * 60, serial_no) utils_io.WriteFile(filename, mode=0o440, data=key_pem + cert_pem, uid=uid, gid=gid) return (key_pem, cert_pem) def GenerateSignedX509Cert(common_name, validity, serial_no, signing_cert_pem): """Generates a signed (but not self-signed) X509 certificate. @type common_name: string @param common_name: commonName value, should be hostname of the machine @type validity: int @param validity: Validity for certificate in seconds @type signing_cert_pem: X509 key @param signing_cert_pem: PEM-encoded private key of the signing certificate @return: a tuple of strings containing the PEM-encoded private key and certificate """ # Create key pair with private and public key. key_pair = OpenSSL.crypto.PKey() key_pair.generate_key(OpenSSL.crypto.TYPE_RSA, constants.RSA_KEY_BITS) # Create certificate sigining request. req = OpenSSL.crypto.X509Req() req.get_subject().CN = common_name req.set_pubkey(key_pair) req.sign(key_pair, constants.X509_CERT_SIGN_DIGEST) # Load the certificates used for signing. signing_key = OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, signing_cert_pem) signing_cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, signing_cert_pem) # Create a certificate and sign it. cert = OpenSSL.crypto.X509() cert.set_subject(req.get_subject()) cert.set_serial_number(serial_no) cert.gmtime_adj_notBefore(0) cert.gmtime_adj_notAfter(validity) cert.set_issuer(signing_cert.get_subject()) cert.set_pubkey(req.get_pubkey()) cert.sign(signing_key, constants.X509_CERT_SIGN_DIGEST) # Encode the key and certificate in PEM format. key_pem = OpenSSL.crypto.dump_privatekey( OpenSSL.crypto.FILETYPE_PEM, key_pair) cert_pem = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, cert) return (key_pem, cert_pem) def GenerateSignedSslCert(filename_cert, serial_no, filename_signing_cert, common_name=constants.X509_CERT_CN, validity=constants.X509_CERT_DEFAULT_VALIDITY, uid=-1, gid=-1): signing_cert_pem = utils_io.ReadFile(filename_signing_cert) (key_pem, cert_pem) = GenerateSignedX509Cert( common_name, validity * 24 * 60 * 60, serial_no, signing_cert_pem) utils_io.WriteFile(filename_cert, mode=0o440, data=key_pem + cert_pem, uid=uid, gid=gid, backup=True) return (key_pem, cert_pem) def ExtractX509Certificate(pem): """Extracts the certificate from a PEM-formatted string. @type pem: string @rtype: tuple; (OpenSSL.X509 object, string) @return: Certificate object and PEM-formatted certificate """ cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pem) return (cert, OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) def X509CertKeyCheck(cert, key): """Function for verifying certificate with a certain private key. @type key: OpenSSL.crypto.PKey @param key: Private key object @type cert: OpenSSL.crypto.X509 @param cert: X509 certificate object @rtype: callable @return: Callable doing the actual check; will raise C{OpenSSL.SSL.Error} if certificate is not signed by given private key """ ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD) ctx.use_certificate(cert) ctx.use_privatekey(key) ctx.check_privatekey() def CheckNodeCertificate(cert, _noded_cert_file=pathutils.NODED_CERT_FILE): """Checks the local node daemon certificate against given certificate. Both certificates must be signed with the same key (as stored in the local L{pathutils.NODED_CERT_FILE} file). No error is raised if no local certificate can be found. @type cert: OpenSSL.crypto.X509 @param cert: X509 certificate object @raise errors.X509CertError: When an error related to X509 occurred @raise errors.GenericError: When the verification failed """ try: noded_pem = utils_io.ReadFile(_noded_cert_file) except EnvironmentError as err: if err.errno != errno.ENOENT: raise logging.debug("Node certificate file '%s' was not found", _noded_cert_file) return try: noded_cert = \ OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, noded_pem) except Exception as err: raise errors.X509CertError(_noded_cert_file, "Unable to load certificate: %s" % err) try: noded_key = \ OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, noded_pem) except Exception as err: raise errors.X509CertError(_noded_cert_file, "Unable to load private key: %s" % err) # Check consistency of server.pem file try: X509CertKeyCheck(noded_cert, noded_key) except OpenSSL.SSL.Error: # This should never happen as it would mean the certificate in server.pem # is out of sync with the private key stored in the same file raise errors.X509CertError(_noded_cert_file, "Certificate does not match with private key") # Check with supplied certificate with local key try: X509CertKeyCheck(cert, noded_key) except OpenSSL.SSL.Error: raise errors.GenericError("Given cluster certificate does not match" " local key") ganeti-3.1.0~rc2/lib/vcluster.py000064400000000000000000000204731476477700300166010ustar00rootroot00000000000000# # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module containing utilities for virtual clusters. Most functions manipulate file system paths and are no-ops when the environment variables C{GANETI_ROOTDIR} and C{GANETI_HOSTNAME} are not set. See the functions' docstrings for details. """ import os from ganeti import compat from ganeti import _constants # cannot use constants, as this would cause a circular import ETC_HOSTS = _constants.V_CLUSTER_ETC_HOSTS _VIRT_PATH_PREFIX = _constants.V_CLUSTER_VIRT_PATH_PREFIX _ROOTDIR_ENVNAME = _constants.V_CLUSTER_ROOTDIR_ENVNAME _HOSTNAME_ENVNAME = _constants.V_CLUSTER_HOSTNAME_ENVNAME #: List of paths which shouldn't be virtualized _VPATH_WHITELIST = _constants.V_CLUSTER_VPATH_WHITELIST def _GetRootDirectory(envname): """Retrieves root directory from an environment variable. @type envname: string @param envname: Environment variable name @rtype: string @return: Root directory (can be empty) """ path = os.getenv(envname) if path: if not os.path.isabs(path): raise RuntimeError("Root directory in '%s' must be absolute: %s" % (envname, path)) return os.path.normpath(path) return "" def _GetHostname(envname): """Retrieves virtual hostname from an environment variable. @type envname: string @param envname: Environment variable name @rtype: string @return: Host name (can be empty) """ return os.getenv(envname, default="") def _CheckHostname(hostname): """Very basic check for hostnames. @type hostname: string @param hostname: Hostname """ if os.path.basename(hostname) != hostname: raise RuntimeError("Hostname '%s' can not be used for a file system" " path" % hostname) def _PreparePaths(rootdir, hostname): """Checks if the root directory and hostname are acceptable. The (node-specific) root directory must have the hostname as its last component. The parent directory then becomes the cluster-wide root directory. This is necessary as some components must be able to predict the root path on a remote node (e.g. copying files via scp). @type rootdir: string @param rootdir: Root directory (from environment) @type hostname: string @param hostname: Hostname (from environment) @rtype: tuple; (string, string, string or None) @return: Tuple containing cluster-global root directory, node root directory and virtual hostname """ if bool(rootdir) ^ bool(hostname): raise RuntimeError("Both root directory and hostname must be specified" " using the environment variables %s and %s" % (_ROOTDIR_ENVNAME, _HOSTNAME_ENVNAME)) if rootdir: assert rootdir == os.path.normpath(rootdir), "Not normalized: " + rootdir _CheckHostname(hostname) if os.path.basename(rootdir) != hostname: raise RuntimeError("Last component of root directory ('%s') must match" " hostname ('%s')" % (rootdir, hostname)) return (os.path.dirname(rootdir), rootdir, hostname) else: return ("", "", None) (_VIRT_BASEDIR, _VIRT_NODEROOT, _VIRT_HOSTNAME) = \ _PreparePaths(_GetRootDirectory(_ROOTDIR_ENVNAME), _GetHostname(_HOSTNAME_ENVNAME)) assert (compat.all([_VIRT_BASEDIR, _VIRT_NODEROOT, _VIRT_HOSTNAME]) or not compat.any([_VIRT_BASEDIR, _VIRT_NODEROOT, _VIRT_HOSTNAME])) def GetVirtualHostname(): """Returns the virtual hostname. @rtype: string or L{None} """ return _VIRT_HOSTNAME def MakeNodeRoot(base, node_name): """Appends a node name to the base directory. """ _CheckHostname(node_name) return os.path.normpath("%s/%s" % (base, node_name)) def ExchangeNodeRoot(node_name, filename, _basedir=_VIRT_BASEDIR, _noderoot=_VIRT_NODEROOT): """Replaces the node-specific root directory in a path. Replaces it with the root directory for another node. Assuming C{/tmp/vcluster/node1} is the root directory for C{node1}, the result will be C{/tmp/vcluster/node3} for C{node3} (as long as a root directory is specified in the environment). """ if _basedir: pure = _RemoveNodePrefix(filename, _noderoot=_noderoot) result = "%s/%s" % (MakeNodeRoot(_basedir, node_name), pure) else: result = filename return os.path.normpath(result) def EnvironmentForHost(hostname, _basedir=_VIRT_BASEDIR): """Returns the environment variables for a host. """ if _basedir: return { _ROOTDIR_ENVNAME: MakeNodeRoot(_basedir, hostname), _HOSTNAME_ENVNAME: hostname, } else: return {} def AddNodePrefix(path, _noderoot=_VIRT_NODEROOT): """Adds a node-specific prefix to a path in a virtual cluster. Returned path includes user-specified root directory if specified in environment. As an example, the path C{/var/lib/ganeti} becomes C{/tmp/vcluster/node1/var/lib/ganeti} if C{/tmp/vcluster/node1} is the root directory specified in the environment. """ assert os.path.isabs(path), "Path not absolute: " + path if _noderoot: result = "%s/%s" % (_noderoot, path) else: result = path assert os.path.isabs(result), "Path not absolute: " + path return os.path.normpath(result) def _RemoveNodePrefix(path, _noderoot=_VIRT_NODEROOT): """Removes the node-specific prefix from a path. This is the opposite of L{AddNodePrefix} and removes a node-local prefix path. """ assert os.path.isabs(path), "Path not absolute: " + path norm_path = os.path.normpath(path) if _noderoot: # Make sure path is actually below node root norm_root = os.path.normpath(_noderoot) root_with_sep = "%s%s" % (norm_root, os.sep) prefix = os.path.commonprefix([root_with_sep, norm_path]) if prefix == root_with_sep: result = norm_path[len(norm_root):] else: raise RuntimeError("Path '%s' is not below node root '%s'" % (path, _noderoot)) else: result = norm_path assert os.path.isabs(result), "Path not absolute: " + path return result def MakeVirtualPath(path, _noderoot=_VIRT_NODEROOT): """Virtualizes a path. A path is "virtualized" by stripping it of its node-specific directory and prepending a prefix (L{_VIRT_PATH_PREFIX}). Use L{LocalizeVirtualPath} to undo the process. Virtual paths are meant to be transported via RPC. """ assert os.path.isabs(path), "Path not absolute: " + path if _noderoot and path not in _VPATH_WHITELIST: return _VIRT_PATH_PREFIX + _RemoveNodePrefix(path, _noderoot=_noderoot) else: return path def LocalizeVirtualPath(path, _noderoot=_VIRT_NODEROOT): """Localizes a virtual path. A "virtualized" path consists of a prefix (L{LocalizeVirtualPath}) and a local path. This function adds the node-specific directory to the local path. Virtual paths are meant to be transported via RPC. """ assert os.path.isabs(path), "Path not absolute: " + path if _noderoot and path not in _VPATH_WHITELIST: if path.startswith(_VIRT_PATH_PREFIX): return AddNodePrefix(path[len(_VIRT_PATH_PREFIX):], _noderoot=_noderoot) else: raise RuntimeError("Path '%s' is not a virtual path" % path) else: return path ganeti-3.1.0~rc2/lib/watcher/000075500000000000000000000000001476477700300160075ustar00rootroot00000000000000ganeti-3.1.0~rc2/lib/watcher/__init__.py000064400000000000000000000755601476477700300201350ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tool to restart erroneously downed virtual machines. This program and set of classes implement a watchdog to restart virtual machines in a Ganeti cluster that have crashed or been killed by a node reboot. Run from cron or similar. """ import os import os.path import sys import signal import time import logging import errno from optparse import OptionParser from ganeti import utils from ganeti import wconfd from ganeti import constants from ganeti import compat from ganeti import errors from ganeti import opcodes from ganeti import cli import ganeti.rpc.errors as rpcerr from ganeti import rapi from ganeti import netutils from ganeti import qlang from ganeti import ssconf from ganeti import ht from ganeti import pathutils import ganeti.rapi.client # pylint: disable=W0611 from ganeti.rapi.client import UsesRapiClient from ganeti.watcher import nodemaint from ganeti.watcher import state MAXTRIES = 5 BAD_STATES = compat.UniqueFrozenset([ constants.INSTST_ERRORDOWN, ]) HELPLESS_STATES = compat.UniqueFrozenset([ constants.INSTST_NODEDOWN, constants.INSTST_NODEOFFLINE, ]) NOTICE = "NOTICE" ERROR = "ERROR" #: Number of seconds to wait between starting child processes for node groups CHILD_PROCESS_DELAY = 1.0 #: How many seconds to wait for instance status file lock INSTANCE_STATUS_LOCK_TIMEOUT = 10.0 class NotMasterError(errors.GenericError): """Exception raised when this host is not the master.""" def ShouldPause(): """Check whether we should pause. """ return bool(utils.ReadWatcherPauseFile(pathutils.WATCHER_PAUSEFILE)) def StartNodeDaemons(): """Start all the daemons that should be running on all nodes. """ # on master or not, try to start the node daemon utils.EnsureDaemon(constants.NODED) # start confd as well. On non candidates it will be in disabled mode. utils.EnsureDaemon(constants.CONFD) # start mond as well: all nodes need monitoring if constants.ENABLE_MOND: utils.EnsureDaemon(constants.MOND) # start kvmd, which will quit if not needed to run utils.EnsureDaemon(constants.KVMD) def RunWatcherHooks(): """Run the watcher hooks. """ hooks_dir = utils.PathJoin(pathutils.HOOKS_BASE_DIR, constants.HOOKS_NAME_WATCHER) if not os.path.isdir(hooks_dir): return try: results = utils.RunParts(hooks_dir) except Exception as err: # pylint: disable=W0703 logging.exception("RunParts %s failed: %s", hooks_dir, err) return for (relname, status, runresult) in results: if status == constants.RUNPARTS_SKIP: logging.debug("Watcher hook %s: skipped", relname) elif status == constants.RUNPARTS_ERR: logging.warning("Watcher hook %s: error (%s)", relname, runresult) elif status == constants.RUNPARTS_RUN: if runresult.failed: logging.warning("Watcher hook %s: failed (exit: %d) (output: %s)", relname, runresult.exit_code, runresult.output) else: logging.debug("Watcher hook %s: success (output: %s)", relname, runresult.output) else: raise errors.ProgrammerError("Unknown status %s returned by RunParts", status) class Instance(object): """Abstraction for a Virtual Machine instance. """ def __init__(self, name, status, config_state, config_state_source, disks_active, snodes, disk_template): self.name = name self.status = status self.config_state = config_state self.config_state_source = config_state_source self.disks_active = disks_active self.snodes = snodes self.disk_template = disk_template def Restart(self, cl): """Encapsulates the start of an instance. """ op = opcodes.OpInstanceStartup(instance_name=self.name, force=False) op.reason = [(constants.OPCODE_REASON_SRC_WATCHER, "Restarting instance %s" % self.name, utils.EpochNano())] cli.SubmitOpCode(op, cl=cl) def ActivateDisks(self, cl): """Encapsulates the activation of all disks of an instance. """ op = opcodes.OpInstanceActivateDisks(instance_name=self.name) op.reason = [(constants.OPCODE_REASON_SRC_WATCHER, "Activating disks for instance %s" % self.name, utils.EpochNano())] cli.SubmitOpCode(op, cl=cl) def NeedsCleanup(self): """Determines whether the instance needs cleanup. Determines whether the instance needs cleanup after having been shutdown by the user. @rtype: bool @return: True if the instance needs cleanup, False otherwise. """ return self.status == constants.INSTST_USERDOWN and \ self.config_state != constants.ADMINST_DOWN class Node(object): """Data container representing cluster node. """ def __init__(self, name, bootid, offline, secondaries): """Initializes this class. """ self.name = name self.bootid = bootid self.offline = offline self.secondaries = secondaries def _CleanupInstance(cl, notepad, inst, locks): n = notepad.NumberOfCleanupAttempts(inst.name) if inst.name in locks: logging.info("Not cleaning up instance '%s', instance is locked", inst.name) return if n > MAXTRIES: logging.warning("Not cleaning up instance '%s', retries exhausted", inst.name) return logging.info("Instance '%s' was shutdown by the user, cleaning up instance", inst.name) op = opcodes.OpInstanceShutdown(instance_name=inst.name, admin_state_source=constants.USER_SOURCE) op.reason = [(constants.OPCODE_REASON_SRC_WATCHER, "Cleaning up instance %s" % inst.name, utils.EpochNano())] try: cli.SubmitOpCode(op, cl=cl) if notepad.NumberOfCleanupAttempts(inst.name): notepad.RemoveInstance(inst.name) except Exception: # pylint: disable=W0703 logging.exception("Error while cleaning up instance '%s'", inst.name) notepad.RecordCleanupAttempt(inst.name) def _CheckInstances(cl, notepad, instances, locks): """Make a pass over the list of instances, restarting downed ones. """ notepad.MaintainInstanceList(list(instances)) started = set() for inst in instances.values(): if inst.NeedsCleanup(): _CleanupInstance(cl, notepad, inst, locks) elif inst.status in BAD_STATES: n = notepad.NumberOfRestartAttempts(inst.name) if n > MAXTRIES: logging.warning("Not restarting instance '%s', retries exhausted", inst.name) continue if n == MAXTRIES: notepad.RecordRestartAttempt(inst.name) logging.error("Could not restart instance '%s' after %s attempts," " giving up", inst.name, MAXTRIES) continue try: logging.info("Restarting instance '%s' (attempt #%s)", inst.name, n + 1) inst.Restart(cl) except Exception: # pylint: disable=W0703 logging.exception("Error while restarting instance '%s'", inst.name) else: started.add(inst.name) notepad.RecordRestartAttempt(inst.name) else: if notepad.NumberOfRestartAttempts(inst.name): notepad.RemoveInstance(inst.name) if inst.status not in HELPLESS_STATES: logging.info("Restart of instance '%s' succeeded", inst.name) return started def _CheckDisks(cl, notepad, nodes, instances, started): """Check all nodes for restarted ones. """ check_nodes = [] for node in nodes.values(): old = notepad.GetNodeBootID(node.name) if not node.bootid: # Bad node, not returning a boot id if not node.offline: logging.debug("Node '%s' missing boot ID, skipping secondary checks", node.name) continue if old != node.bootid: # Node's boot ID has changed, probably through a reboot check_nodes.append(node) if check_nodes: # Activate disks for all instances with any of the checked nodes as a # secondary node. for node in check_nodes: for instance_name in node.secondaries: try: inst = instances[instance_name] except KeyError: logging.info("Can't find instance '%s', maybe it was ignored", instance_name) continue if not inst.disks_active: logging.info("Skipping disk activation for instance with not" " activated disks '%s'", inst.name) continue if inst.name in started: # we already tried to start the instance, which should have # activated its drives (if they can be at all) logging.debug("Skipping disk activation for instance '%s' as" " it was already started", inst.name) continue try: logging.info("Activating disks for instance '%s'", inst.name) inst.ActivateDisks(cl) except Exception: # pylint: disable=W0703 logging.exception("Error while activating disks for instance '%s'", inst.name) # Keep changed boot IDs for node in check_nodes: notepad.SetNodeBootID(node.name, node.bootid) def _CheckForOfflineNodes(nodes, instance): """Checks if given instances has any secondary in offline status. @param instance: The instance object @return: True if any of the secondary is offline, False otherwise """ return compat.any(nodes[node_name].offline for node_name in instance.snodes) def _GetPendingVerifyDisks(cl, uuid): """Checks if there are any currently running or pending group verify jobs and if so, returns their id. """ qfilter = qlang.MakeSimpleFilter("status", frozenset([constants.JOB_STATUS_RUNNING, constants.JOB_STATUS_QUEUED, constants.JOB_STATUS_WAITING])) qresult = cl.Query(constants.QR_JOB, ["id", "summary"], qfilter) ids = [jobid for ((_, jobid), (_, (job, ))) in qresult.data if job == ("GROUP_VERIFY_DISKS(%s)" % uuid)] return ids def _VerifyDisks(cl, uuid, nodes, instances, is_strict): """Run a per-group "gnt-cluster verify-disks". """ existing_jobs = _GetPendingVerifyDisks(cl, uuid) if existing_jobs: logging.info("There are verify disks jobs already pending (%s), skipping " "VerifyDisks step for %s.", utils.CommaJoin(existing_jobs), uuid) return op = opcodes.OpGroupVerifyDisks( group_name=uuid, priority=constants.OP_PRIO_LOW, is_strict=is_strict) op.reason = [(constants.OPCODE_REASON_SRC_WATCHER, "Verifying disks of group %s" % uuid, utils.EpochNano())] job_id = cl.SubmitJob([op]) ((_, offline_disk_instances, _), ) = \ cli.PollJob(job_id, cl=cl, feedback_fn=logging.debug) try: cl.ArchiveJob(job_id) except Exception as err: logging.exception("Error while archiving job %d" % job_id) if not offline_disk_instances: # nothing to do logging.debug("Verify-disks reported no offline disks, nothing to do") return logging.debug("Will activate disks for instance(s) %s", utils.CommaJoin(offline_disk_instances)) # We submit only one job, and wait for it. Not optimal, but this puts less # load on the job queue. job = [] for name in offline_disk_instances: try: inst = instances[name] except KeyError: logging.info("Can't find instance '%s', maybe it was ignored", name) continue if inst.status in HELPLESS_STATES or _CheckForOfflineNodes(nodes, inst): logging.info("Skipping instance '%s' because it is in a helpless state" " or has offline secondaries", name) continue op = opcodes.OpInstanceActivateDisks(instance_name=name) op.reason = [(constants.OPCODE_REASON_SRC_WATCHER, "Activating disks for instance %s" % name, utils.EpochNano())] job.append(op) if job: job_id = cli.SendJob(job, cl=cl) try: cli.PollJob(job_id, cl=cl, feedback_fn=logging.debug) except Exception: # pylint: disable=W0703 logging.exception("Error while activating disks") def IsRapiResponding(hostname): """Connects to RAPI port and does a simple test. Connects to RAPI port of hostname and does a simple test. At this time, the test is GetVersion. If RAPI responds with error code "401 Unauthorized", the test is successful, because the aim of this function is to assess whether RAPI is responding, not if it is accessible. @type hostname: string @param hostname: hostname of the node to connect to. @rtype: bool @return: Whether RAPI is working properly """ curl_config = rapi.client.GenericCurlConfig() rapi_client = rapi.client.GanetiRapiClient(hostname, curl_config_fn=curl_config) try: master_version = rapi_client.GetVersion() except rapi.client.CertificateError as err: logging.warning("RAPI certificate error: %s", err) return False except rapi.client.GanetiApiError as err: if err.code == 401: # Unauthorized, but RAPI is alive and responding return True else: logging.warning("RAPI error: %s", err) return False else: logging.debug("Reported RAPI version %s", master_version) return master_version == constants.RAPI_VERSION def IsWconfdResponding(): """Probes an echo RPC to WConfD. """ probe_string = "ganeti watcher probe %d" % time.time() try: result = wconfd.Client().Echo(probe_string) except Exception as err: # pylint: disable=W0703 logging.warning("WConfd connection error: %s", err) return False if result != probe_string: logging.warning("WConfd echo('%s') returned '%s'", probe_string, result) return False return True def ParseOptions(): """Parse the command line options. @return: (options, args) as from OptionParser.parse_args() """ parser = OptionParser(description="Ganeti cluster watcher", usage="%prog [-d]", version="%%prog (ganeti) %s" % constants.RELEASE_VERSION) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.NODEGROUP_OPT) parser.add_option("-A", "--job-age", dest="job_age", default=6 * 3600, help="Autoarchive jobs older than this age (default" " 6 hours)") parser.add_option("--ignore-pause", dest="ignore_pause", default=False, action="store_true", help="Ignore cluster pause setting") parser.add_option("--wait-children", dest="wait_children", action="store_true", help="Wait for child processes") parser.add_option("--no-wait-children", dest="wait_children", action="store_false", help="Don't wait for child processes") parser.add_option("--no-verify-disks", dest="no_verify_disks", default=False, action="store_true", help="Do not verify disk status") parser.add_option("--no-strict", dest="no_strict", default=False, action="store_true", help="Do not run group verify in strict mode") parser.add_option("--rapi-ip", dest="rapi_ip", default=constants.IP4_ADDRESS_LOCALHOST, help="Use this IP to talk to RAPI.") # See optparse documentation for why default values are not set by options parser.set_defaults(wait_children=True) options, args = parser.parse_args() options.job_age = cli.ParseTimespec(options.job_age) if args: parser.error("No arguments expected") return (options, args) def _WriteInstanceStatus(filename, data): """Writes the per-group instance status file. The entries are sorted. @type filename: string @param filename: Path to instance status file @type data: list of tuple; (instance name as string, status as string) @param data: Instance name and status """ logging.debug("Updating instance status file '%s' with %s instances", filename, len(data)) utils.WriteFile(filename, data="\n".join("%s %s" % (n, s) for (n, s) in sorted(data))) def _UpdateInstanceStatus(filename, instances): """Writes an instance status file from L{Instance} objects. @type filename: string @param filename: Path to status file @type instances: list of L{Instance} """ _WriteInstanceStatus(filename, [(inst.name, inst.status) for inst in instances]) def _ReadInstanceStatus(filename): """Reads an instance status file. @type filename: string @param filename: Path to status file @rtype: tuple; (None or number, list of lists containing instance name and status) @return: File's mtime and instance status contained in the file; mtime is C{None} if file can't be read """ logging.debug("Reading per-group instance status from '%s'", filename) statcb = utils.FileStatHelper() try: content = utils.ReadFile(filename, preread=statcb) except EnvironmentError as err: if err.errno == errno.ENOENT: logging.error("Can't read '%s', does not exist (yet)", filename) else: logging.exception("Unable to read '%s', ignoring", filename) return (None, None) else: return (statcb.st.st_mtime, [line.split(None, 1) for line in content.splitlines()]) def _MergeInstanceStatus(filename, pergroup_filename, groups): """Merges all per-group instance status files into a global one. @type filename: string @param filename: Path to global instance status file @type pergroup_filename: string @param pergroup_filename: Path to per-group status files, must contain "%s" to be replaced with group UUID @type groups: sequence @param groups: UUIDs of known groups """ # Lock global status file in exclusive mode lock = utils.FileLock.Open(filename) try: lock.Exclusive(blocking=True, timeout=INSTANCE_STATUS_LOCK_TIMEOUT) except errors.LockError as err: # All per-group processes will lock and update the file. None of them # should take longer than 10 seconds (the value of # INSTANCE_STATUS_LOCK_TIMEOUT). logging.error("Can't acquire lock on instance status file '%s', not" " updating: %s", filename, err) return logging.debug("Acquired exclusive lock on '%s'", filename) data = {} # Load instance status from all groups for group_uuid in groups: (mtime, instdata) = _ReadInstanceStatus(pergroup_filename % group_uuid) if mtime is not None: for (instance_name, status) in instdata: data.setdefault(instance_name, []).append((mtime, status)) # Select last update based on file mtime inststatus = [(instance_name, sorted(status, reverse=True)[0][1]) for (instance_name, status) in data.items()] # Write the global status file. Don't touch file after it's been # updated--there is no lock anymore. _WriteInstanceStatus(filename, inststatus) def GetLuxiClient(try_restart): """Tries to connect to the luxi daemon. @type try_restart: bool @param try_restart: Whether to attempt to restart the master daemon """ try: return cli.GetClient() except errors.OpPrereqError as err: # this is, from cli.GetClient, a not-master case raise NotMasterError("Not on master node (%s)" % err) except (rpcerr.NoMasterError, rpcerr.TimeoutError) as err: if not try_restart: raise logging.warning("Luxi daemon seems to be down (%s), trying to restart", err) if not utils.EnsureDaemon(constants.LUXID): raise errors.GenericError("Can't start the master daemon") # Retry the connection return cli.GetClient() def _StartGroupChildren(cl, wait): """Starts a new instance of the watcher for every node group. """ assert not compat.any(arg.startswith(cli.NODEGROUP_OPT_NAME) for arg in sys.argv) result = cl.QueryGroups([], ["name", "uuid"], False) children = [] for (idx, (name, uuid)) in enumerate(result): if idx > 0: # Let's not kill the system time.sleep(CHILD_PROCESS_DELAY) logging.debug("Spawning child for group %r (%s).", name, uuid) signal.signal(signal.SIGCHLD, signal.SIG_IGN) try: pid = os.fork() except OSError: logging.exception("Failed to fork for group %r (%s)", name, uuid) if pid == 0: (options, _) = ParseOptions() options.nodegroup = uuid _GroupWatcher(options) return else: logging.debug("Started with PID %s", pid) children.append(pid) if wait: for child in children: logging.debug("Waiting for child PID %s", child) try: result = utils.RetryOnSignal(os.waitpid, child, 0) except EnvironmentError as err: result = str(err) logging.debug("Child PID %s exited with status %s", child, result) def _ArchiveJobs(cl, age): """Archives old jobs. """ (arch_count, left_count) = cl.AutoArchiveJobs(age) logging.debug("Archived %s jobs, left %s", arch_count, left_count) def _CheckMaster(cl): """Ensures current host is master node. """ (master, ) = cl.QueryConfigValues(["master_node"]) if master != netutils.Hostname.GetSysName(): raise NotMasterError("This is not the master node") @UsesRapiClient def _GlobalWatcher(opts): """Main function for global watcher. At the end child processes are spawned for every node group. """ StartNodeDaemons() RunWatcherHooks() # Run node maintenance in all cases, even if master, so that old masters can # be properly cleaned up if nodemaint.NodeMaintenance.ShouldRun(): # pylint: disable=E0602 nodemaint.NodeMaintenance().Exec() # pylint: disable=E0602 try: client = GetLuxiClient(True) except NotMasterError: # Don't proceed on non-master nodes return constants.EXIT_SUCCESS # we are on master now utils.EnsureDaemon(constants.RAPI) utils.EnsureDaemon(constants.WCONFD) # If RAPI isn't responding to queries, try one restart logging.debug("Attempting to talk to remote API on %s", opts.rapi_ip) if not IsRapiResponding(opts.rapi_ip): logging.warning("Couldn't get answer from remote API, restaring daemon") utils.StopDaemon(constants.RAPI) utils.EnsureDaemon(constants.RAPI) logging.debug("Second attempt to talk to remote API") if not IsRapiResponding(opts.rapi_ip): logging.fatal("RAPI is not responding") logging.debug("Successfully talked to remote API") # If WConfD isn't responding to queries, try one restart logging.debug("Attempting to talk to WConfD") if not IsWconfdResponding(): logging.warning("WConfD not responsive, restarting daemon") utils.StopDaemon(constants.WCONFD) utils.EnsureDaemon(constants.WCONFD) logging.debug("Second attempt to talk to WConfD") if not IsWconfdResponding(): logging.fatal("WConfD is not responding") _CheckMaster(client) _ArchiveJobs(client, opts.job_age) # Spawn child processes for all node groups _StartGroupChildren(client, opts.wait_children) return constants.EXIT_SUCCESS def _GetGroupData(qcl, uuid): """Retrieves instances and nodes per node group. """ locks = qcl.Query(constants.QR_LOCK, ["name", "mode"], None) prefix = "instance/" prefix_len = len(prefix) locked_instances = set() for [[_, name], [_, lock]] in locks.data: if name.startswith(prefix) and lock: locked_instances.add(name[prefix_len:]) queries = [ (constants.QR_INSTANCE, ["name", "status", "admin_state", "admin_state_source", "disks_active", "snodes", "pnode.group.uuid", "snodes.group.uuid", "disk_template"], [qlang.OP_EQUAL, "pnode.group.uuid", uuid]), (constants.QR_NODE, ["name", "bootid", "offline"], [qlang.OP_EQUAL, "group.uuid", uuid]), ] results_data = [ qcl.Query(what, field, qfilter).data for (what, field, qfilter) in queries ] # Ensure results are tuples with two values assert compat.all( ht.TListOf(ht.TListOf(ht.TIsLength(2)))(d) for d in results_data) # Extract values ignoring result status (raw_instances, raw_nodes) = [[[v[1] for v in values] for values in res] for res in results_data] secondaries = {} instances = [] # Load all instances for (name, status, config_state, config_state_source, disks_active, snodes, pnode_group_uuid, snodes_group_uuid, disk_template) in raw_instances: if snodes and set([pnode_group_uuid]) != set(snodes_group_uuid): logging.error("Ignoring split instance '%s', primary group %s, secondary" " groups %s", name, pnode_group_uuid, utils.CommaJoin(snodes_group_uuid)) else: instances.append(Instance(name, status, config_state, config_state_source, disks_active, snodes, disk_template)) for node in snodes: secondaries.setdefault(node, set()).add(name) # Load all nodes nodes = [Node(name, bootid, offline, secondaries.get(name, set())) for (name, bootid, offline) in raw_nodes] return (dict((node.name, node) for node in nodes), dict((inst.name, inst) for inst in instances), locked_instances) def _LoadKnownGroups(): """Returns a list of all node groups known by L{ssconf}. """ groups = ssconf.SimpleStore().GetNodegroupList() result = list(line.split(None, 1)[0] for line in groups if line.strip()) if not compat.all(utils.UUID_RE.match(r) for r in result): raise errors.GenericError("Ssconf contains invalid group UUID") return result def _GroupWatcher(opts): """Main function for per-group watcher process. """ group_uuid = opts.nodegroup.lower() if not utils.UUID_RE.match(group_uuid): raise errors.GenericError("Node group parameter (%s) must be given a UUID," " got '%s'" % (cli.NODEGROUP_OPT_NAME, group_uuid)) logging.info("Watcher for node group '%s'", group_uuid) known_groups = _LoadKnownGroups() # Check if node group is known if group_uuid not in known_groups: raise errors.GenericError("Node group '%s' is not known by ssconf" % group_uuid) # Group UUID has been verified and should not contain any dangerous # characters state_path = pathutils.WATCHER_GROUP_STATE_FILE % group_uuid inst_status_path = pathutils.WATCHER_GROUP_INSTANCE_STATUS_FILE % group_uuid logging.debug("Using state file %s", state_path) # Group watcher file lock statefile = state.OpenStateFile(state_path) # pylint: disable=E0602 if not statefile: return constants.EXIT_FAILURE notepad = state.WatcherState(statefile) # pylint: disable=E0602 try: # Connect to master daemon client = GetLuxiClient(False) _CheckMaster(client) (nodes, instances, locks) = _GetGroupData(client, group_uuid) # Update per-group instance status file _UpdateInstanceStatus(inst_status_path, list(instances.values())) _MergeInstanceStatus(pathutils.INSTANCE_STATUS_FILE, pathutils.WATCHER_GROUP_INSTANCE_STATUS_FILE, known_groups) started = _CheckInstances(client, notepad, instances, locks) _CheckDisks(client, notepad, nodes, instances, started) except Exception as err: logging.info("Not updating status file due to failure: %s", err) raise else: # Save changes for next run notepad.Save(state_path) notepad.Close() # Check if the nodegroup only has ext storage type only_ext = compat.all(i.disk_template == constants.DT_EXT for i in instances.values()) # We skip current NodeGroup verification if there are only external storage # devices. Currently we provide an interface for external storage provider # for disk verification implementations, however current ExtStorageDevice # does not provide an API for this yet. # # This check needs to be revisited if ES_ACTION_VERIFY on ExtStorageDevice # is implemented. if not opts.no_verify_disks and not only_ext: is_strict = not opts.no_strict _VerifyDisks(client, group_uuid, nodes, instances, is_strict=is_strict) return constants.EXIT_SUCCESS def Main(): """Main function. """ (options, _) = ParseOptions() utils.SetupLogging(pathutils.LOG_WATCHER, sys.argv[0], debug=options.debug, stderr_logging=options.debug) if ShouldPause() and not options.ignore_pause: logging.debug("Pause has been set, exiting") return constants.EXIT_SUCCESS # Try to acquire global watcher lock in shared mode. # In case we are in the global watcher process, this lock will be held by all # children processes (one for each nodegroup) and will only be released when # all of them have finished running. lock = utils.FileLock.Open(pathutils.WATCHER_LOCK_FILE) try: lock.Shared(blocking=False) except (EnvironmentError, errors.LockError) as err: logging.error("Can't acquire lock on %s: %s", pathutils.WATCHER_LOCK_FILE, err) return constants.EXIT_SUCCESS if options.nodegroup is None: fn = _GlobalWatcher else: # Per-nodegroup watcher fn = _GroupWatcher try: return fn(options) except (SystemExit, KeyboardInterrupt): raise except NotMasterError: logging.debug("Not master, exiting") return constants.EXIT_NOTMASTER except errors.ResolverError as err: logging.error("Cannot resolve hostname '%s', exiting", err.args[0]) return constants.EXIT_NODESETUP_ERROR except errors.JobQueueFull: logging.error("Job queue is full, can't query cluster state") except errors.JobQueueDrainError: logging.error("Job queue is drained, can't maintain cluster state") except Exception as err: # pylint: disable=W0703 logging.exception(str(err)) return constants.EXIT_FAILURE return constants.EXIT_SUCCESS ganeti-3.1.0~rc2/lib/watcher/nodemaint.py000064400000000000000000000122741476477700300203450ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module doing node maintenance for Ganeti watcher. """ import logging from ganeti import constants from ganeti import errors from ganeti import hypervisor from ganeti import netutils from ganeti import ssconf from ganeti import utils from ganeti import confd from ganeti.storage import drbd import ganeti.confd.client # pylint: disable=W0611 class NodeMaintenance(object): """Talks to confd daemons and possible shutdown instances/drbd devices. """ def __init__(self): self.store_cb = confd.client.StoreResultCallback() self.filter_cb = confd.client.ConfdFilterCallback(self.store_cb) self.confd_client = confd.client.GetConfdClient(self.filter_cb) @staticmethod def ShouldRun(): """Checks whether node maintenance should run. """ try: return ssconf.SimpleStore().GetMaintainNodeHealth() except errors.ConfigurationError as err: logging.error("Configuration error, not activating node maintenance: %s", err) return False @staticmethod def GetRunningInstances(): """Compute list of hypervisor/running instances. """ hyp_list = ssconf.SimpleStore().GetHypervisorList() hvparams = ssconf.SimpleStore().GetHvparams() results = [] for hv_name in hyp_list: try: hv = hypervisor.GetHypervisor(hv_name) ilist = hv.ListInstances(hvparams=hvparams) results.extend([(iname, hv_name) for iname in ilist]) except: # pylint: disable=W0702 logging.error("Error while listing instances for hypervisor %s", hv_name, exc_info=True) return results @staticmethod def GetUsedDRBDs(): """Get list of used DRBD minors. """ return drbd.DRBD8.GetUsedDevs() @classmethod def DoMaintenance(cls, role): """Maintain the instance list. """ if role == constants.CONFD_NODE_ROLE_OFFLINE: inst_running = cls.GetRunningInstances() cls.ShutdownInstances(inst_running) drbd_running = cls.GetUsedDRBDs() cls.ShutdownDRBD(drbd_running) else: logging.debug("Not doing anything for role %s", role) @staticmethod def ShutdownInstances(inst_running): """Shutdown running instances. """ names_running = set([i[0] for i in inst_running]) if names_running: logging.info("Following instances should not be running," " shutting them down: %s", utils.CommaJoin(names_running)) # this dictionary will collapse duplicate instance names (only # xen pvm/vhm) into a single key, which is fine i2h = dict(inst_running) for name in names_running: hv_name = i2h[name] hv = hypervisor.GetHypervisor(hv_name) hv.StopInstance(None, force=True, name=name) @staticmethod def ShutdownDRBD(drbd_running): """Shutdown active DRBD devices. """ if drbd_running: logging.info("Following DRBD minors should not be active," " shutting them down: %s", utils.CommaJoin(drbd_running)) for minor in drbd_running: drbd.DRBD8.ShutdownAll(minor) def Exec(self): """Check node status versus cluster desired state. """ my_name = netutils.Hostname.GetSysName() req = \ confd.client.ConfdClientRequest(type=constants.CONFD_REQ_NODE_ROLE_BYNAME, query=my_name) self.confd_client.SendRequest(req, async_=False, coverage=-1) timed_out, _, _ = self.confd_client.WaitForReply(req.rsalt) if not timed_out: # should have a valid response status, result = self.store_cb.GetResponse(req.rsalt) assert status, "Missing result but received replies" if not self.filter_cb.consistent[req.rsalt]: logging.warning("Inconsistent replies, not doing anything") return self.DoMaintenance(result.server_reply.answer) else: logging.warning("Confd query timed out, cannot do maintenance actions") ganeti-3.1.0~rc2/lib/watcher/state.py000064400000000000000000000176241476477700300175130ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module keeping state for Ganeti watcher. """ import os import time import logging from ganeti import utils from ganeti import serializer from ganeti import errors # Delete any record that is older than 8 hours; this value is based on # the fact that the current retry counter is 5, and watcher runs every # 5 minutes, so it takes around half an hour to exceed the retry # counter, so 8 hours (16*1/2h) seems like a reasonable reset time RETRY_EXPIRATION = 8 * 3600 KEY_CLEANUP_COUNT = "cleanup_count" KEY_CLEANUP_WHEN = "cleanup_when" KEY_RESTART_COUNT = "restart_count" KEY_RESTART_WHEN = "restart_when" KEY_BOOT_ID = "bootid" def OpenStateFile(path): """Opens the state file and acquires a lock on it. @type path: string @param path: Path to state file """ # The two-step dance below is necessary to allow both opening existing # file read/write and creating if not existing. Vanilla open will truncate # an existing file -or- allow creating if not existing. statefile_fd = os.open(path, os.O_RDWR | os.O_CREAT) # Try to acquire lock on state file. If this fails, another watcher instance # might already be running or another program is temporarily blocking the # watcher from running. try: utils.LockFile(statefile_fd) except errors.LockError as err: logging.error("Can't acquire lock on state file %s: %s", path, err) return None return os.fdopen(statefile_fd, "w+") class WatcherState(object): """Interface to a state file recording restart attempts. """ def __init__(self, statefile): """Open, lock, read and parse the file. @type statefile: file @param statefile: State file object """ self.statefile = statefile try: state_data = self.statefile.read() if not state_data: self._data = {} else: self._data = serializer.Load(state_data) except Exception as msg: # pylint: disable=W0703 # Ignore errors while loading the file and treat it as empty self._data = {} logging.warning(("Invalid state file. Using defaults." " Error message: %s"), msg) if "instance" not in self._data: self._data["instance"] = {} if "node" not in self._data: self._data["node"] = {} self._orig_data = serializer.Dump(self._data) def Save(self, filename): """Save state to file. """ assert self.statefile serialized_form = serializer.Dump(self._data) if self._orig_data == serialized_form: logging.debug("Data didn't change, just touching status file") os.utime(filename, None) return # We need to make sure the file is locked before renaming it, otherwise # starting ganeti-watcher again at the same time will create a conflict. fd = utils.WriteFile(filename, data=serialized_form, prewrite=utils.LockFile, close=False) self.statefile = os.fdopen(fd, "w+") def Close(self): """Unlock configuration file and close it. """ assert self.statefile # Files are automatically unlocked when closing them self.statefile.close() self.statefile = None def GetNodeBootID(self, name): """Returns the last boot ID of a node or None. """ ndata = self._data["node"] if name in ndata and KEY_BOOT_ID in ndata[name]: return ndata[name][KEY_BOOT_ID] return None def SetNodeBootID(self, name, bootid): """Sets the boot ID of a node. """ assert bootid ndata = self._data["node"] ndata.setdefault(name, {})[KEY_BOOT_ID] = bootid def NumberOfRestartAttempts(self, instance_name): """Returns number of previous restart attempts. @type instance_name: string @param instance_name: the name of the instance to look up """ idata = self._data["instance"] return idata.get(instance_name, {}).get(KEY_RESTART_COUNT, 0) def NumberOfCleanupAttempts(self, instance_name): """Returns number of previous cleanup attempts. @type instance_name: string @param instance_name: the name of the instance to look up """ idata = self._data["instance"] return idata.get(instance_name, {}).get(KEY_CLEANUP_COUNT, 0) def MaintainInstanceList(self, instances): """Perform maintenance on the recorded instances. @type instances: list of string @param instances: the list of currently existing instances """ idict = self._data["instance"] # First, delete obsolete instances obsolete_instances = set(idict).difference(instances) for inst in obsolete_instances: logging.debug("Forgetting obsolete instance %s", inst) idict.pop(inst, None) # Second, delete expired records earliest = time.time() - RETRY_EXPIRATION expired_instances = [i for i in idict if idict[i].get(KEY_RESTART_WHEN, 0) < earliest] for inst in expired_instances: logging.debug("Expiring record for instance %s", inst) idict.pop(inst, None) @staticmethod def _RecordAttempt(instances, instance_name, key_when, key_count): """Record an event. @type instances: dict @param instances: contains instance data indexed by instance_name @type instance_name: string @param instance_name: name of the instance involved in the event @type key_when: @param key_when: dict key for the information for when the event occurred @type key_count: int @param key_count: dict key for the information for how many times the event occurred """ instance = instances.setdefault(instance_name, {}) instance[key_when] = time.time() instance[key_count] = instance.get(key_count, 0) + 1 def RecordRestartAttempt(self, instance_name): """Record a restart attempt. @type instance_name: string @param instance_name: the name of the instance being restarted """ self._RecordAttempt(self._data["instance"], instance_name, KEY_RESTART_WHEN, KEY_RESTART_COUNT) def RecordCleanupAttempt(self, instance_name): """Record a cleanup attempt. @type instance_name: string @param instance_name: the name of the instance being cleaned up """ self._RecordAttempt(self._data["instance"], instance_name, KEY_CLEANUP_WHEN, KEY_CLEANUP_COUNT) def RemoveInstance(self, instance_name): """Update state to reflect that a machine is running. This method removes the record for a named instance (as we only track down instances). @type instance_name: string @param instance_name: the name of the instance to remove from books """ idata = self._data["instance"] idata.pop(instance_name, None) ganeti-3.1.0~rc2/lib/wconfd.py000064400000000000000000000051201476477700300162020ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module for the WConfd protocol """ import logging import random import time import ganeti.rpc.client as cl import ganeti.rpc.stub.wconfd as stub from ganeti.rpc.transport import Transport from ganeti.rpc import errors class Client(cl.AbstractStubClient, stub.ClientRpcStub): # R0904: Too many public methods # pylint: disable=R0904 """High-level WConfD client implementation. This uses a backing Transport-like class on top of which it implements data serialization/deserialization. """ def __init__(self, timeouts=None, transport=Transport, allow_non_master=None): """Constructor for the Client class. Arguments are the same as for L{AbstractClient}. """ cl.AbstractStubClient.__init__(self, timeouts=timeouts, transport=transport, allow_non_master=allow_non_master) stub.ClientRpcStub.__init__(self) retries = 12 for try_no in range(0, retries): try: self._InitTransport() return except errors.TimeoutError: logging.debug("Timout trying to connect to WConfD") if try_no == retries -1: raise logging.debug("Will retry") time.sleep(try_no * 10 + 10 * random.random()) ganeti-3.1.0~rc2/lib/workerpool.py000064400000000000000000000441221476477700300171320ustar00rootroot00000000000000# # # Copyright (C) 2008, 2009, 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Base classes for worker pools. """ import logging import threading import heapq import itertools from ganeti import compat from ganeti import errors _TERMINATE = object() _DEFAULT_PRIORITY = 0 class DeferTask(Exception): """Special exception class to defer a task. This class can be raised by L{BaseWorker.RunTask} to defer the execution of a task. Optionally, the priority of the task can be changed. """ def __init__(self, priority=None): """Initializes this class. @type priority: number @param priority: New task priority (None means no change) """ Exception.__init__(self) self.priority = priority class NoSuchTask(Exception): """Exception raised when a task can't be found. """ class BaseWorker(threading.Thread, object): """Base worker class for worker pools. Users of a worker pool must override RunTask in a subclass. """ # pylint: disable=W0212 def __init__(self, pool, worker_id): """Constructor for BaseWorker thread. @param pool: the parent worker pool @param worker_id: identifier for this worker """ super(BaseWorker, self).__init__(name=worker_id) self.pool = pool self._worker_id = worker_id self._current_task = None assert self.name == worker_id def ShouldTerminate(self): """Returns whether this worker should terminate. Should only be called from within L{RunTask}. """ self.pool._lock.acquire() try: assert self._HasRunningTaskUnlocked() return self.pool._ShouldWorkerTerminateUnlocked(self) finally: self.pool._lock.release() def GetCurrentPriority(self): """Returns the priority of the current task. Should only be called from within L{RunTask}. """ self.pool._lock.acquire() try: assert self._HasRunningTaskUnlocked() (priority, _, _, _) = self._current_task return priority finally: self.pool._lock.release() def SetTaskName(self, taskname): """Sets the name of the current task. Should only be called from within L{RunTask}. @type taskname: string @param taskname: Task's name """ if taskname: name = "%s/%s" % (self._worker_id, taskname) else: name = self._worker_id # Set thread name self.name = name def _HasRunningTaskUnlocked(self): """Returns whether this worker is currently running a task. """ return (self._current_task is not None) def _GetCurrentOrderAndTaskId(self): """Returns the order and task ID of the current task. Should only be called from within L{RunTask}. """ self.pool._lock.acquire() try: assert self._HasRunningTaskUnlocked() (_, order_id, task_id, _) = self._current_task return (order_id, task_id) finally: self.pool._lock.release() def run(self): """Main thread function. Waits for new tasks to show up in the queue. """ pool = self.pool while True: assert self._current_task is None defer = None try: # Wait on lock to be told either to terminate or to do a task pool._lock.acquire() try: task = pool._WaitForTaskUnlocked(self) if task is _TERMINATE: # Told to terminate break if task is None: # Spurious notification, ignore continue self._current_task = task # No longer needed, dispose of reference del task assert self._HasRunningTaskUnlocked() finally: pool._lock.release() (priority, _, _, args) = self._current_task try: # Run the actual task assert defer is None logging.debug("Starting task %r, priority %s", args, priority) assert self.name == self._worker_id try: self.RunTask(*args) finally: self.SetTaskName(None) logging.debug("Done with task %r, priority %s", args, priority) except DeferTask as err: defer = err if defer.priority is None: # Use same priority defer.priority = priority logging.debug("Deferring task %r, new priority %s", args, defer.priority) assert self._HasRunningTaskUnlocked() except: # pylint: disable=W0702 logging.exception("Caught unhandled exception") assert self._HasRunningTaskUnlocked() finally: # Notify pool pool._lock.acquire() try: if defer: assert self._current_task # Schedule again for later run (_, _, task_id, args) = self._current_task pool._AddTaskUnlocked(args, defer.priority, task_id) if self._current_task: self._current_task = None pool._worker_to_pool.notify_all() finally: pool._lock.release() assert not self._HasRunningTaskUnlocked() logging.debug("Terminates") def RunTask(self, *args): """Function called to start a task. This needs to be implemented by child classes. """ raise NotImplementedError() class WorkerPool(object): """Worker pool with a queue. This class is thread-safe. Tasks are guaranteed to be started in the order in which they're added to the pool. Due to the nature of threading, they're not guaranteed to finish in the same order. @type _tasks: list of tuples @ivar _tasks: Each tuple has the format (priority, order ID, task ID, arguments). Priority and order ID are numeric and essentially control the sort order. The order ID is an increasing number denoting the order in which tasks are added to the queue. The task ID is controlled by user of workerpool, see L{AddTask} for details. The task arguments are C{None} for abandoned tasks, otherwise a sequence of arguments to be passed to L{BaseWorker.RunTask}). The list must fulfill the heap property (for use by the C{heapq} module). @type _taskdata: dict; (task IDs as keys, tuples as values) @ivar _taskdata: Mapping from task IDs to entries in L{_tasks} """ def __init__(self, name, num_workers, worker_class): """Constructor for worker pool. @param num_workers: number of workers to be started (dynamic resizing is not yet implemented) @param worker_class: the class to be instantiated for workers; should derive from L{BaseWorker} """ # Some of these variables are accessed by BaseWorker self._lock = threading.Lock() self._pool_to_pool = threading.Condition(self._lock) self._pool_to_worker = threading.Condition(self._lock) self._worker_to_pool = threading.Condition(self._lock) self._worker_class = worker_class self._name = name self._last_worker_id = 0 self._workers = [] self._quiescing = False # Terminating workers self._termworkers = [] # Queued tasks self._counter = itertools.count() self._tasks = [] self._taskdata = {} # Start workers self.Resize(num_workers) # TODO: Implement dynamic resizing? def _WaitWhileQuiescingUnlocked(self): """Wait until the worker pool has finished quiescing. """ while self._quiescing: self._pool_to_pool.wait() def _AddTaskUnlocked(self, args, priority, task_id): """Adds a task to the internal queue. @type args: sequence @param args: Arguments passed to L{BaseWorker.RunTask} @type priority: number @param priority: Task priority @param task_id: Task ID """ assert isinstance(args, (tuple, list)), "Arguments must be a sequence" assert isinstance(priority, int), "Priority must be numeric" assert task_id is None or isinstance(task_id, int), \ "Task ID must be numeric or None" task = [priority, next(self._counter), task_id, args] if task_id is not None: assert task_id not in self._taskdata # Keep a reference to change priority later if necessary self._taskdata[task_id] = task # A counter is used to ensure elements are processed in their incoming # order. For processing they're sorted by priority and then counter. heapq.heappush(self._tasks, task) # Notify a waiting worker self._pool_to_worker.notify() def AddTask(self, args, priority=_DEFAULT_PRIORITY, task_id=None): """Adds a task to the queue. @type args: sequence @param args: arguments passed to L{BaseWorker.RunTask} @type priority: number @param priority: Task priority @param task_id: Task ID @note: The task ID can be essentially anything that can be used as a dictionary key. Callers, however, must ensure a task ID is unique while a task is in the pool or while it might return to the pool due to deferring using L{DeferTask}. """ self._lock.acquire() try: self._WaitWhileQuiescingUnlocked() self._AddTaskUnlocked(args, priority, task_id) finally: self._lock.release() def AddManyTasks(self, tasks, priority=_DEFAULT_PRIORITY, task_id=None): """Add a list of tasks to the queue. @type tasks: list of tuples @param tasks: list of args passed to L{BaseWorker.RunTask} @type priority: number or list of numbers @param priority: Priority for all added tasks or a list with the priority for each task @type task_id: list @param task_id: List with the ID for each task @note: See L{AddTask} for a note on task IDs. """ assert compat.all(isinstance(task, (tuple, list)) for task in tasks), \ "Each task must be a sequence" assert (isinstance(priority, int) or compat.all(isinstance(prio, int) for prio in priority)), \ "Priority must be numeric or be a list of numeric values" assert task_id is None or isinstance(task_id, (tuple, list)), \ "Task IDs must be in a sequence" if isinstance(priority, int): priority = [priority] * len(tasks) elif len(priority) != len(tasks): raise errors.ProgrammerError("Number of priorities (%s) doesn't match" " number of tasks (%s)" % (len(priority), len(tasks))) if task_id is None: task_id = [None] * len(tasks) elif len(task_id) != len(tasks): raise errors.ProgrammerError("Number of task IDs (%s) doesn't match" " number of tasks (%s)" % (len(task_id), len(tasks))) self._lock.acquire() try: self._WaitWhileQuiescingUnlocked() assert compat.all(isinstance(prio, int) for prio in priority) assert len(tasks) == len(priority) assert len(tasks) == len(task_id) for (args, prio, tid) in zip(tasks, priority, task_id): self._AddTaskUnlocked(args, prio, tid) finally: self._lock.release() def ChangeTaskPriority(self, task_id, priority): """Changes a task's priority. @param task_id: Task ID @type priority: number @param priority: New task priority @raise NoSuchTask: When the task referred by C{task_id} can not be found (it may never have existed, may have already been processed, or is currently running) """ assert isinstance(priority, int), "Priority must be numeric" self._lock.acquire() try: logging.debug("About to change priority of task %s to %s", task_id, priority) # Find old task oldtask = self._taskdata.get(task_id, None) if oldtask is None: msg = "Task '%s' was not found" % task_id logging.debug(msg) raise NoSuchTask(msg) # Prepare new task newtask = [priority] + oldtask[1:] # Mark old entry as abandoned (this doesn't change the sort order and # therefore doesn't invalidate the heap property of L{self._tasks}). # See also . oldtask[-1] = None # Change reference to new task entry and forget the old one assert task_id is not None self._taskdata[task_id] = newtask # Add a new task with the old number and arguments heapq.heappush(self._tasks, newtask) # Notify a waiting worker self._pool_to_worker.notify() finally: self._lock.release() def _WaitForTaskUnlocked(self, worker): """Waits for a task for a worker. @type worker: L{BaseWorker} @param worker: Worker thread """ while True: if self._ShouldWorkerTerminateUnlocked(worker): return _TERMINATE # If there's a pending task, return it immediately if self._tasks: # Get task from queue and tell pool about it try: task = heapq.heappop(self._tasks) finally: self._worker_to_pool.notify_all() (_, _, task_id, args) = task # If the priority was changed, "args" is None if args is None: # Try again logging.debug("Found abandoned task (%r)", task) continue # Delete reference if task_id is not None: del self._taskdata[task_id] return task logging.debug("Waiting for tasks") # wait() releases the lock and sleeps until notified self._pool_to_worker.wait() logging.debug("Notified while waiting") def _ShouldWorkerTerminateUnlocked(self, worker): """Returns whether a worker should terminate. """ return (worker in self._termworkers) def _HasRunningTasksUnlocked(self): """Checks whether there's a task running in a worker. """ for worker in self._workers + self._termworkers: if worker._HasRunningTaskUnlocked(): # pylint: disable=W0212 return True return False def HasRunningTasks(self): """Checks whether there's at least one task running. """ self._lock.acquire() try: return self._HasRunningTasksUnlocked() finally: self._lock.release() def Quiesce(self): """Waits until the task queue is empty. """ self._lock.acquire() try: self._quiescing = True # Wait while there are tasks pending or running while self._tasks or self._HasRunningTasksUnlocked(): self._worker_to_pool.wait() finally: self._quiescing = False # Make sure AddTasks continues in case it was waiting self._pool_to_pool.notify_all() self._lock.release() def _NewWorkerIdUnlocked(self): """Return an identifier for a new worker. """ self._last_worker_id += 1 return "%s%d" % (self._name, self._last_worker_id) def _ResizeUnlocked(self, num_workers): """Changes the number of workers. """ assert num_workers >= 0, "num_workers must be >= 0" logging.debug("Resizing to %s workers", num_workers) current_count = len(self._workers) if current_count == num_workers: # Nothing to do pass elif current_count > num_workers: if num_workers == 0: # Create copy of list to iterate over while lock isn't held. termworkers = self._workers[:] del self._workers[:] else: # TODO: Implement partial downsizing raise NotImplementedError() #termworkers = ... self._termworkers += termworkers # Notify workers that something has changed self._pool_to_worker.notify_all() # Join all terminating workers self._lock.release() try: for worker in termworkers: logging.debug("Waiting for thread %s", worker.name) worker.join() finally: self._lock.acquire() # Remove terminated threads. This could be done in a more efficient way # (del self._termworkers[:]), but checking worker.is_alive() makes sure we # don't leave zombie threads around. for worker in termworkers: assert worker in self._termworkers, ("Worker not in list of" " terminating workers") if not worker.is_alive(): self._termworkers.remove(worker) assert not self._termworkers, "Zombie worker detected" elif current_count < num_workers: # Create (num_workers - current_count) new workers for _ in range(num_workers - current_count): worker = self._worker_class(self, self._NewWorkerIdUnlocked()) self._workers.append(worker) worker.start() def Resize(self, num_workers): """Changes the number of workers in the pool. @param num_workers: the new number of workers """ self._lock.acquire() try: return self._ResizeUnlocked(num_workers) finally: self._lock.release() def TerminateWorkers(self): """Terminate all worker threads. Unstarted tasks will be ignored. """ logging.debug("Terminating all workers") self._lock.acquire() try: self._ResizeUnlocked(0) if self._tasks: logging.debug("There are %s tasks left", len(self._tasks)) finally: self._lock.release() logging.debug("All workers terminated") ganeti-3.1.0~rc2/man/000075500000000000000000000000001476477700300143575ustar00rootroot00000000000000ganeti-3.1.0~rc2/man/footer.rst000064400000000000000000000052241476477700300164120ustar00rootroot00000000000000REPORTING BUGS -------------- Report bugs to the `project's issue tracker `_ or contact the developers using the `Ganeti mailing list `_. SEE ALSO -------- Ganeti overview and specifications: **ganeti**\(7) (general overview), **ganeti-os-interface**\(7) (guest OS definitions), **ganeti-extstorage-interface**\(7) (external storage providers). Ganeti commands: **gnt-cluster**\(8) (cluster-wide commands), **gnt-job**\(8) (job-related commands), **gnt-node**\(8) (node-related commands), **gnt-instance**\(8) (instance commands), **gnt-os**\(8) (guest OS commands), **gnt-storage**\(8) (storage commands), **gnt-group**\(8) (node group commands), **gnt-backup**\(8) (instance import/export commands), **gnt-debug**\(8) (debug commands). Ganeti daemons: **ganeti-watcher**\(8) (automatic instance restarter), **ganeti-cleaner**\(8) (job queue cleaner), **ganeti-noded**\(8) (node daemon), **ganeti-rapi**\(8) (remote API daemon). Ganeti htools: **htools**\(1) (generic binary), **hbal**\(1) (cluster balancer), **hspace**\(1) (capacity calculation), **hail**\(1) (IAllocator plugin), **hscan**\(1) (data gatherer from remote clusters), **hinfo**\(1) (cluster information printer), **mon-collector**\(7) (data collectors interface). COPYRIGHT --------- Copyright (C) 2006-2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-cleaner.rst000064400000000000000000000014421476477700300177700ustar00rootroot00000000000000ganeti-cleaner(8) Ganeti | Version @GANETI_VERSION@ =================================================== Name ---- ganeti-cleaner - Ganeti job queue cleaner Synopsis -------- **ganeti-cleaner** node|master DESCRIPTION ----------- The **ganeti-cleaner** is a periodically run script to remove old files. It can clean either node-specific or master-specific files. When called with ``node`` as argument, it will cleanup expired X509 certificates and keys from ``@LOCALSTATEDIR@/run/ganeti/crypto``, as well as outdated **ganeti-watcher** information. When called with ``master`` as argument, it will instead automatically remove all files older than 21 days from ``@LOCALSTATEDIR@/lib/ganeti/queue/archive``. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-confd.rst000064400000000000000000000042301476477700300174460ustar00rootroot00000000000000ganeti-confd(8) Ganeti | Version @GANETI_VERSION@ ================================================= Name ---- ganeti-confd - Ganeti conf daemon Synopsis -------- **ganeti-confd** [-f] [-d] [--syslog] [-p *PORT*] [-b *ADDRESS*] [--no-user-checks] DESCRIPTION ----------- **ganeti-confd** is a daemon used to answer queries related to the configuration of a Ganeti cluster. For testing purposes, you can give the ``-f`` option and the program won't detach from the running terminal. Debug-level message can be activated by giving the ``-d`` option. Logging to syslog, rather than its own log file, can be enabled by passing in the ``--syslog`` option. The **ganeti-confd** daemon listens to port 1814 UDP, on all interfaces, by default. The port can be overridden by an entry the services database (usually ``/etc/services``) or by passing the ``-p`` option. The ``-b`` option can be used to specify the address to bind to (defaults to ``0.0.0.0``). The daemon will refuse to start if the user and group do not match the one defined at build time; this behaviour can be overridden by the ``--no-user-checks`` option. ROLE ~~~~ The role of the conf daemon is to make sure we have a highly available and very fast way to query cluster configuration values. This daemon is automatically active on all master candidates, and so has no single point of failure. It communicates via UDP so each query can easily be sent to multiple servers, and it answers queries from a cached copy of the config it keeps in memory, so no disk access is required to get an answer. The config is reloaded from disk automatically when it changes, with a rate limit of once per second. If the conf daemon is stopped on all nodes, its clients won't be able to get query answers. COMMUNICATION PROTOCOL ~~~~~~~~~~~~~~~~~~~~~~ The confd protocol is an HMAC authenticated json-encoded custom format, over UDP. A client library is provided to make it easy to write software to query confd. More information can be found in the Ganeti 2.1 design doc, and an example usage can be seen in the (external) NBMA daemon for Ganeti. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-extstorage-interface.rst000064400000000000000000000270651476477700300225130ustar00rootroot00000000000000ganeti-extstorage-interface(7) Ganeti | Version @GANETI_VERSION@ ================================================================ Name ---- ganeti-extstorage-interface - Specifications for ExtStorage providers DESCRIPTION ----------- The method for supporting external shared storage in Ganeti is to have an ExtStorage provider for each external shared storage hardware type. The ExtStorage provider is a set of files (executable scripts and text files), contained inside a directory which is named after the provider. This directory must be present across all nodes of a nodegroup (Ganeti doesn't replicate it), in order for the provider to be usable by Ganeti for this nodegroup (valid). The external shared storage hardware should also be accessible by all nodes of this nodegroup too. REFERENCE --------- There are eight required files: *create*, *attach*, *detach*, *remove*, *grow*, *setinfo*, *verify*, (executables) and *parameters.list* (text file). There are also three optional files: *open*, *close*, and *snapshot* (executables). Common environment ~~~~~~~~~~~~~~~~~~ All commands will get their input via environment variables. A common set of variables will be exported for all commands, and some commands might have extra variables. Note that all counts are zero-based. Since Ganeti version 2.5, the environment will be cleaned up before being passed to scripts, therefore they will not inherit the environment in with which the ganeti node daemon was started. If you depend on any environment variables (non-Ganeti), then you will need to define or source them appropriately. VOL_NAME The name of the volume. This is unique for Ganeti and it uses it to refer to a specific volume inside the external storage. Its format is ``UUID.ext.diskX`` where ``UUID`` is produced by Ganeti and is unique inside the Ganeti context. ``X`` is the number of the disk count. VOL_SIZE Available only to the **create** and **grow** scripts. The volume's size in mebibytes. VOL_NEW_SIZE Available only to the **grow** script. It declares the new size of the volume after grow (in mebibytes). To find the amount of grow, the script should calculate the number VOL_NEW_SIZE - VOL_SIZE. EXTP_*name* Each ExtStorage parameter (see below) will be exported in its own variable, prefixed with ``EXTP_``, and upper-cased. For example, a ``fromsnap`` parameter will be exported as ``EXTP_FROMSNAP``. VOL_METADATA Available only to the **setinfo** script. A string containing metadata to be associated with the volume. Currently, Ganeti sets this value to ``originstname+X`` where ``X`` is the instance's name. VOL_CNAME The human-readable name of the Disk config object (optional). VOL_UUID The uuid of the Disk config object. VOL_SNAPSHOT_NAME The name of the volume's snapshot. VOL_SNAPSHOT_SIZE The size of the volume's snapshot. VOL_OPEN_EXCLUSIVE Whether the volume will be opened for exclusive access or not. This will be False (denoting shared access) during migration. EXECUTABLE SCRIPTS ------------------ create ~~~~~~ The **create** command is used for creating a new volume inside the external storage. The ``VOL_NAME`` denotes the volume's name, which should be unique. After creation, Ganeti will refer to this volume by this name for all other actions. Ganeti produces this name dynamically and ensures its uniqueness inside the Ganeti context. Therefore, you should make sure not to provision manually additional volumes inside the external storage with this type of name, because this will lead to conflicts and possible loss of data. The ``VOL_SIZE`` variable denotes the size of the new volume to be created in mebibytes. If the script ends successfully, a new volume of size ``VOL_SIZE`` should exist inside the external storage. e.g:: a lun inside a NAS appliance. The script returns ``0`` on success. attach ~~~~~~ This command is used in order to make an already created volume visible to the physical node which will host the instance. This is done by mapping the already provisioned volume to a block device inside the host node. The ``VOL_NAME`` variable denotes the volume to be mapped. After successful attachment the script returns to its stdout a string, which is the full path of the block device to which the volume is mapped. e.g:: /dev/dummy1 When attach returns, this path should be a valid block device on the host node. The attach script should be idempotent if the volume is already mapped. If the requested volume is already mapped, then the script should just return to its stdout the path which is already mapped to. In case the storage technology supports userspace access to volumes as well, e.g. the QEMU Hypervisor can see an RBD volume using its embedded driver for the RBD protocol, then the provider can return extra lines denoting the available userspace access URIs per hypervisor. The URI should be in the following format: :. For example, a RADOS provider should return kvm:rbd:/ in the second line of stdout after the local block device path (e.g. /dev/rbd1). So, if the ``access`` disk parameter is ``userspace`` for the ext disk template, then the QEMU command will end up having file= in the ``-drive`` option. In case the storage technology supports *only* userspace access to volumes, then the first line of stdout should be an empty line, denoting that a local block device is not available. If neither a block device nor a URI is returned, then Ganeti will complain. detach ~~~~~~ This command is used in order to unmap an already mapped volume from the host node. Detach undoes everything attach did. This is done by unmapping the requested volume from the block device it is mapped to. The ``VOL_NAME`` variable denotes the volume to be unmapped. ``detach`` doesn't affect the volume itself. It just unmaps it from the host node. The volume continues to exist inside the external storage. It's just not accessible by the node anymore. This script doesn't return anything to its stdout. The detach script should be idempotent if the volume is already unmapped. If the volume is not mapped, the script doesn't perform any action at all. The script returns ``0`` on success. remove ~~~~~~ This command is used to remove an existing volume from the external storage. The volume is permanently removed from inside the external storage along with all its data. The ``VOL_NAME`` variable denotes the volume to be removed. The script returns ``0`` on success. grow ~~~~ This command is used to grow an existing volume of the external storage. The ``VOL_NAME`` variable denotes the volume to grow. The ``VOL_SIZE`` variable denotes the current volume's size (in mebibytes). The ``VOL_NEW_SIZE`` variable denotes the final size after the volume has been grown (in mebibytes). The amount of grow can be easily calculated by the script and is: grow_amount = VOL_NEW_SIZE - VOL_SIZE (in mebibytes) Ganeti ensures that: ``VOL_NEW_SIZE`` > ``VOL_SIZE`` If the script returns successfully, then the volume inside the external storage will have a new size of ``VOL_NEW_SIZE``. This isn't immediately reflected to the instance's disk. See ``gnt-instance grow`` for more details on when the running instance becomes aware of its grown disk. The script returns ``0`` on success. setinfo ~~~~~~~ This script is used to add metadata to an existing volume. It is helpful when we need to keep an external, Ganeti-independent mapping between instances and volumes; primarily for recovery reasons. This is provider specific and the author of the provider chooses whether/how to implement this. You can just exit with ``0``, if you do not want to implement this feature, without harming the overall functionality of the provider. The ``VOL_METADATA`` variable contains the metadata of the volume. Currently, Ganeti sets this value to ``originstname+X`` where ``X`` is the instance's name. The script returns ``0`` on success. verify ~~~~~~ The *verify* script is used to verify consistency of the external parameters (ext-params) (see below). The command should take one or more arguments denoting what checks should be performed, and return a proper exit code depending on whether the validation failed or succeeded. Currently, the script is not invoked by Ganeti, but should be present for future use and consistency with gnt-os-interface's verify script. The script should return ``0`` on success. snapshot ~~~~~~~~ The *snapshot* script is used to take a snapshot of the given volume. The ``VOL_SNAPSHOT_NAME`` and ``VOL_SNAPSHOT_SIZE`` variables contain the name and size of the snapshot we are about to create. Currently this operation is used only during gnt-backup export and Ganeti sets those values to ``VOL_NAME.snap`` and ``VOL_SIZE`` respectively (see above). The script returns ``0`` on success. Please note that this script is optional and not all providers should implement it. Of course if it is not present, instance backup export will not be supported for the given provider. open ~~~~ The *open* script is used to open the given volume. This makes the volume ready for I/O. The ``VOL_OPEN_EXCLUSIVE`` variable denotes whether the volume will be opened for exclusive access or not. It is True by default and False (denoting shared access) during migration. The script returns ``0`` on success. Please note that this script is optional and not all providers should implement it. close ~~~~~ The *close* script is used to close the given volume. This disables I/O on the volume. The script returns ``0`` on success. Please note that this script is optional and not all providers should implement it. TEXT FILES ---------- parameters.list ~~~~~~~~~~~~~~~ This file declares the parameters supported by the ExtStorage provider, one parameter per line, with name and description (space and/or tab separated). For example:: fromsnap Snapshot name to create the volume from nas_ip The IP of the NAS appliance The parameters can then be used during instance add as follows:: # gnt-instance add --disk=0:fromsnap="file_name",nas_ip="1.2.3.4" ... EXAMPLES -------- In the following examples we assume that you have already installed successfully two ExtStorage providers: ``pvdr1`` and ``pvdr2`` Add a new instance with a 10G first disk provided by ``pvdr1`` and a 20G second disk provided by ``pvdr2``:: # gnt-instance add -t ext --disk=0:size=10G,provider=pvdr1 --disk=1:size=20G,provider=pvdr2 Add a new instance with a 5G first disk provided by provider ``pvdr1`` and also pass the ``prm1``, ``prm2`` parameters to the provider, with the corresponding values ``val1``, ``val2``:: # gnt-instance add -t ext --disk=0:size=5G,provider=pvdr1,prm1=val1,prm2=val2 Modify an existing instance of disk type ``ext`` by adding a new 30G disk provided by provider ``pvdr2``:: # gnt-instance modify --disk 1:add,size=30G,provider=pvdr2 Modify an existing instance of disk type ``ext`` by adding 2 new disks, of different providers, passing one parameter for the first one:: # gnt-instance modify --disk 2:add,size=3G,provider=pvdr1,prm1=val1 --disk 3:add,size=5G,provider=pvdr2 NOTES ----- Backwards compatibility ~~~~~~~~~~~~~~~~~~~~~~~ The ExtStorage Interface was introduced in Ganeti 2.7. Ganeti 2.7 and up is compatible with the ExtStorage Interface. Common behaviour ~~~~~~~~~~~~~~~~ All the scripts should display an usage message when called with a wrong number of arguments or when the first argument is ``-h`` or ``--help``. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-kvmd.rst000064400000000000000000000022051476477700300173160ustar00rootroot00000000000000ganeti-kvmd(8) Ganeti | Version @GANETI_VERSION@ ================================================ Name ---- ganeti-kvmd - Ganeti KVM daemon Synopsis -------- **ganeti-kvmd** DESCRIPTION ----------- The KVM daemon is responsible for determining whether a given KVM instance was shutdown by an administrator or a user. The KVM daemon monitors, using ``inotify``, KVM instances through their QMP sockets, which are provided by KVM. Using the QMP sockets, the KVM daemon listens for particular shutdown, powerdown, and stop events which will determine if a given instance was shutdown by the user or Ganeti, and this result is communicated to Ganeti via a special file in the filesystem. FILES ----- The KVM daemon monitors Qmp sockets of KVM instances, which are created in the KVM control directory, located under ``@LOCALSTATEDIR@/run/ganeti/kvm-hypervisor/ctrl/``. The KVM daemon also creates shutdown files in this directory. Finally, the KVM daemon's log file is located under ``@LOCALSTATEDIR@/log/ganeti/ganeti-kvmd.log``. Removal of the KVM control directory, the shutdown files, or the log file, will lead to no errors on the KVM daemon. ganeti-3.1.0~rc2/man/ganeti-listrunner.rst000064400000000000000000000053661476477700300205750ustar00rootroot00000000000000ganeti-listrunner(8) Ganeti | Version @GANETI_VERSION@ ====================================================== NAME ---- ganeti-listrunner - Run commands in parallel over multiple machines SYNOPSIS -------- **ganeti-listrunner** ``-l`` *logdir* {``-x`` *executable* | ``-c`` *shell-cmd*} {``-f`` *hostfile* | ``-h`` *hostlist*} [``-a`` *aux-file*] [``-b`` *batch-size*] [``-u`` *username*] [``-A``] DESCRIPTION ----------- **ganeti-listrunner** is a tool to run commands in parallel over multiple machines. It differs from ``dsh`` or other tools in that it asks for the password once (if not using ``ssh-agent``) and then reuses the password to connect to all machines, thus being easily usable even when public key authentication or Kerberos authentication is not available. It can run either a command or a script (which gets uploaded first and deleted after execution) on a list of hosts provided either via a file (one host per line) or as a comma-separated list on the commandline. The output (stdout and stderr are merged) of the remote execution is written to a logfile. One logfile per host is written. OPTIONS ------- The options that can be passed to the program are as follows: ``-l`` *logdir* The directory under which the logfiles files should be written. ``-x`` *executable* The executable to copy and run on the target hosts. ``-c`` *shell-cmd* The shell command to run on the remote hosts. ``-f`` *hostfile* The file with the target hosts, one hostname per line. ``-h`` *hostlist* Comma-separated list of target hosts. ``-a`` *aux-file* A file to copy to the target hosts. Can be given multiple times, in which case all files will be copied to the temporary directory. The executable or the shell command will be run from the (temporary) directory where these files have been copied. ``-b`` *batch-size* The host list will be split into batches of batch-size which will be processed in parallel. The default if 15, and should be increased if faster processing is needed. ``-u`` *username* Username to connect as instead of the default root username. ``-A`` Use an existing ssh-agent instead of password authentication. ``--args`` Arguments to pass to executable (``-x``). EXIT STATUS ----------- The exist status of the command will be zero, unless it was aborted in some way (e.g. ^C). EXAMPLE ------- Run a command on a list of hosts: .. code-block:: bash listrunner -l logdir -c "uname -a" -h host1,host2,host3 Upload a script, some auxiliary files and run the script: .. code-block:: bash listrunner -l logdir -x runme.sh \ -a seed.dat -a golden.dat \ -h host1,host2,host3 SEE ALSO -------- **dsh**\(1), **cssh**\(1) .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-luxid.rst000064400000000000000000000035411476477700300175060ustar00rootroot00000000000000ganeti-luxid(8) Ganeti | Version @GANETI_VERSION@ ================================================= Name ---- ganeti-luxid - Ganeti query daemon Synopsis -------- **ganeti-luxid** [-f] [-d] [--syslog] [--no-user-checks] [--no-voting --yes-do-it] DESCRIPTION ----------- **ganeti-luxid** is a daemon used to answer queries related to the configuration and the current live state of a Ganeti cluster. Additionally, it is the authoritative daemon for the Ganeti job queue. Jobs can be submitted via this daemon and it schedules and starts them. For testing purposes, you can give the ``-f`` option and the program won't detach from the running terminal. Debug-level message can be activated by giving the ``-d`` option. Logging to syslog, rather than its own log file, can be enabled by passing in the ``--syslog`` option. The **ganeti-luxid** daemon listens on a Unix socket (``@LOCALSTATEDIR@/run/ganeti/socket/ganeti-query``) on which it exports a ``Luxi`` endpoint supporting the full set of commands. The daemon will refuse to start if the user and group do not match the one defined at build time; this behaviour can be overridden by the ``--no-user-checks`` option. The daemon will refuse to start if it cannot verify that the majority of cluster nodes believes that it is running on the master node. To allow failover in a two-node cluster, this can be overridden by the ``--no-voting`` option. As it this is dangerous, the ``--yes-do-it`` option has to be given as well. Only queries which don't require locks can be handled by the luxi daemon, which might lead to slightly outdated results in some cases. The config is reloaded from disk automatically when it changes, with a rate limit of once per second. COMMUNICATION PROTOCOL ~~~~~~~~~~~~~~~~~~~~~~ See **gnt-master**\(8). .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-mond.rst000064400000000000000000000047411476477700300173210ustar00rootroot00000000000000ganeti-mond(8) Ganeti | Version @GANETI_VERSION@ ================================================= Name ---- ganeti-mond - Ganeti monitoring daemon Synopsis -------- **ganeti-mond** [-f] [-d] [-p *PORT*] [-b *ADDRESS*] [--no-user-checks] DESCRIPTION ----------- **ganeti-mond** is the daemon providing the Ganeti monitoring functionality. It is responsible for running the data collectors and to provide the collected information through a HTTP interface. For testing purposes, you can give the ``-f`` option and the program won't detach from the running terminal. Debug-level message can be activated by giving the ``-d`` option. The **ganeti-mond** daemon listens to port 1815 TCP, on all interfaces, by default. The port can be overridden by an entry the services database by passing the ``-p`` option. The ``-b`` option can be used to specify the address to bind to (defaults to ``0.0.0.0``). The daemon will refuse to start if the user and group do not match the one defined at build time; this behaviour can be overridden by the ``--no-user-checks`` option. COMMUNICATION PROTOCOL ~~~~~~~~~~~~~~~~~~~~~~ The queries to the monitoring agent will be HTTP GET requests on port 1815. The answer will be encoded in JSON format and will depend on the specific accessed resource. If a request is sent to a non-existing resource, a 404 error will be returned by the HTTP server. ``/`` +++++ The root resource. It will return the list of the supported protocol version numbers. ``/1/list/collectors`` ++++++++++++++++++++++ Returns a list of tuples (kind, category, name) showing all the collectors available in the system. ``/1/report/all`` +++++++++++++++++ A list of the reports of all the data collectors. `Status reporting collectors` will provide their output in non-verbose format. The verbose format can be requested by adding the parameter ``verbose=1`` to the request. ``/1/report/[category]/[collector_name]`` +++++++++++++++++++++++++++++++++++++++++ Returns the report of the collector ``[collector_name]`` that belongs to the specified ``[category]``. If a collector does not belong to any category, ``collector`` will be used as the value for ``[category]``. `Status reporting collectors` will provide their output in non-verbose format. The verbose format can be requested by adding the parameter ``verbose=1`` to the request. Further information can be found in the Ganeti Monitoring Agent design document. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-noded.rst000064400000000000000000000051051476477700300174500ustar00rootroot00000000000000ganeti-noded(8) Ganeti | Version @GANETI_VERSION@ ================================================= Name ---- ganeti-noded - Ganeti node daemon Synopsis -------- | **ganeti-noded** [-f] [-d] [-p *PORT*] [-b *ADDRESS*] [-i *INTERFACE*] | [\--max-clients *CLIENTS*] [\--no-mlock] [\--syslog] [\--no-ssl] | [-K *SSL_KEY_FILE*] [-C *SSL_CERT_FILE*] DESCRIPTION ----------- The **ganeti-noded** is the daemon which is responsible for the node functions in the Ganeti system. By default, in order to be able to support features such as node powercycling even on systems with a very damaged root disk, **ganeti-noded** locks itself in RAM using **mlockall**\(2). You can disable this feature by passing in the ``--no-mlock`` to the daemon. For testing purposes, you can give the ``-f`` option and the program won't detach from the running terminal. Debug-level message can be activated by giving the ``-d`` option. Logging to syslog, rather than its own log file, can be enabled by passing in the ``--syslog`` option. The **ganeti-noded** daemon listens to port 1811 TCP, on all interfaces, by default. The port can be overridden by an entry in the services database (usually ``/etc/services``) or by passing the ``-p`` option. The ``-b`` option can be used to specify the address to bind to (defaults to ``0.0.0.0``); alternatively, the ``-i`` option can be used to specify the interface to bind do. The maximum number of simultaneous client connections may be configured with the ``--max-clients`` option. This defaults to 20. Connections above this count are accepted, but no responses are sent until enough connections are closed. Ganeti noded communication is protected via SSL, with a key generated at cluster init time. This can be disabled with the ``--no-ssl`` option, or a different SSL key and certificate can be specified using the ``-K`` and ``-C`` options. ROLE ~~~~ The role of the node daemon is to do almost all the actions that change the state of the node. Things like creating disks for instances, activating disks, starting/stopping instance and so on are done via the node daemon. Also, in some cases the startup/shutdown of the master daemon are done via the node daemon, and the cluster IP address is also added/removed to the master node via it. If the node daemon is stopped, the instances are not affected, but the master won't be able to talk to that node. COMMUNICATION PROTOCOL ~~~~~~~~~~~~~~~~~~~~~~ Currently the master-node RPC is done using a simple RPC protocol built using JSON over HTTP(S). .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-os-interface.rst000064400000000000000000000321341476477700300207400ustar00rootroot00000000000000ganeti-os-interface(7) Ganeti | Version @GANETI_VERSION@ ======================================================== Name ---- ganeti-os-interface - Specifications for guest OS types DESCRIPTION ----------- The method of supporting guest operating systems in Ganeti is to have, for each guest OS type, a directory containing a number of required files. This directory must be present across all nodes (Ganeti doesn't replicate it) in order for the OS to be usable by Ganeti. REFERENCE --------- There are eight required files: *create*, *import*, *export*, *rename*, *verify* (executables), *ganeti_api_version*, *variants.list* and *parameters.list* (text files). Common environment ~~~~~~~~~~~~~~~~~~ All commands will get their input via environment variables. A common set of variables will be exported for all commands, and some of them might have extra ones. Note that all counts are zero-based. Since Ganeti version 2.5, the environment will be cleaned up before being passed to scripts, therefore they will not inherit the environment in with which the ganeti node daemon was started. If you depend on any environment variables (non-Ganeti), then you will need to define or source them appropriately. OS_API_VERSION The OS API version that the rest of the environment conforms to. INSTANCE_NAME The instance name the script should operate on. INSTANCE_OS, OS_NAME Both names point to the name of the instance's OS as Ganeti knows it. This can simplify the OS scripts by providing the same scripts under multiple names, and then the scripts can use this name to alter their behaviour. With OS API 15 changing the script behavior based on this variable is deprecated: OS_VARIANT should be used instead (see below). OS_VARIANT The variant of the OS which should be installed. Each OS must support all variants listed under its variants.list file, and may support more. Any more supported variants should be properly documented in the per-OS documentation. HYPERVISOR The hypervisor of this instance. DISK_COUNT The number of disks the instance has. The actual disk definitions are in a set of additional variables. The instance's disk will be numbered from 0 to this value minus one. DISK\_%N_PATH The path to the storage for disk N of the instance. This might be either a block device or a regular file, in which case the OS scripts should use ``losetup`` (if they need to mount it). E.g. the first disk of the instance might be exported as ``DISK_0_PATH=/dev/drbd0``. If the disk is only accessible via a userspace URI, this not be set. DISK\_%N_SIZE The configured size of disk N in mebibytes (available since Ganeti 3.0). DISK\_%N_URI The userspace URI to the storage for disk N of the instance, if configured. DISK\_%N_ACCESS This is how the hypervisor will export the instance disks: either read-write (``rw``) or read-only (``ro``). DISK\_%N_UUID The uuid associated with the N-th disk of the instance. DISK\_%N_NAME (Optional) The name, if any, associated with the N-th disk of the instance. DISK\_%N_FRONTEND_TYPE (Optional) If applicable to the current hypervisor type: the type of the device exported by the hypervisor. For example, the Xen HVM hypervisor can export disks as either ``paravirtual`` or ``ioemu``. DISK\_%N_BACKEND_TYPE How files are visible on the node side. This can be either ``block`` (when using block devices) or ``file:type``, where ``type`` is either ``loop``, ``blktap`` or ``blktap2``, depending on how the hypervisor will be configured. Note that not all backend types apply to all hypervisors. NIC_COUNT Similar to the ``DISK_COUNT``, this represents the number of NICs of the instance. NIC\_%N_MAC The MAC address associated with this interface. NIC\_%N_UUID The uuid associated with the N-th NIC of the instance. NIC\_%N_NAME (Optional) The name, if any, associated with the N-th NIC of the instance. NIC\_%N_IP The IP address, if any, associated with the N-th NIC of the instance. NIC\_%N_MODE The NIC mode, routed, bridged or openvswitch NIC\_%N_BRIDGE The bridge to which this NIC will be attached. This variable is defined only when the NIC is in bridged mode. NIC\_%N_LINK In bridged or openvswitch mode, this is the interface to which the NIC will be attached (same as ``NIC_%N_BRIDGE`` for bridged). In routed mode it is the routing table which will be used by the hypervisor to insert the appropriate routes. NIC\_%N_FRONTEND_TYPE (Optional) If applicable, the type of the exported NIC to the instance, this can be one of: ``rtl8139``, ``ne2k_pci``, ``ne2k_isa``, ``paravirtual``. NIC\_%d_NETWORK_NAME (Optional) If a NIC network is specified, the network's name. NIC\_%d_NETWORK_UUID (Optional) If a NIC network is specified, the network's uuid. NIC\_%d_NETWORK_FAMILY (Optional) If a NIC network is specified, the network's family. NIC\_%d_NETWORK_SUBNET (Optional) If a NIC network is specified, the network's IPv4 subnet. NIC\_%d_NETWORK_GATEWAY (Optional) If a NIC network is specified, the network's IPv4 gateway. NIC\_%d_NETWORK_SUBNET6 (Optional) If a NIC network is specified, the network's IPv6 subnet. NIC\_%d_NETWORK_GATEWAY6 (Optional) If a NIC network is specified, the network's IPv6 gateway. NIC\_%d_NETWORK_MAC_PREFIX (Optional) If a NIC network is specified, the network's mac prefix. NIC\_%d_NETWORK_TAGS (Optional) If a NIC network is specified, the network's tags, space separated. OSP\_*name* Each OS parameter (see below) will be exported in its own variable, prefixed with ``OSP_``, and upper-cased. For example, a ``dhcp`` parameter will be exported as ``OSP_DHCP``. DEBUG_LEVEL If non-zero, this should cause the OS script to generate verbose logs of its execution, for troubleshooting purposes. Currently only ``0`` and ``1`` are valid values. EXECUTABLE SCRIPTS ------------------ create ~~~~~~ The **create** command is used for creating a new instance from scratch. It has no additional environment variables beside the common ones. The ``INSTANCE_NAME`` variable denotes the name of the instance, which is guaranteed to resolve to an IP address. The create script should configure the instance according to this name. It can configure the IP statically or not, depending on the deployment environment. The ``INSTANCE_REINSTALL`` variable is set to ``1`` when this create request is reinstalling an existing instance, rather than creating a new one. This can be used, for example, to preserve some data in the old instance in an OS-specific way. export ~~~~~~ This command is used in order to make a backup of a given disk of the instance. The command should write to stdout a dump of the given block device. The output of this program will be passed during restore to the **import** command. The specific disk to backup is denoted by four additional environment variables: EXPORT_INDEX The index in the instance disks structure (and could be used for example to skip the second disk if not needed for backup). EXPORT_DISK_PATH Alias for ``DISK_N_PATH``. It is duplicated here for easier usage by shell scripts (rather than parse the ``DISK_...`` variables). EXPORT_DISK_URI Alias for ``DISK_N_URI``, analagous to ``EXPORT_DISK_PATH``. EXPORT_DEVICE Historical alias for ``EXPORT_DISK_PATH``. To provide the user with an estimate on how long the export will take, a predicted size can be written to the file descriptor passed in the variable ``EXP_SIZE_FD``. The value is in bytes and must be terminated by a newline character (``\n``). Older versions of Ganeti don't support this feature, hence the variable should be checked before use. Example:: if test -n "$EXP_SIZE_FD"; then blockdev --getsize64 $blockdev >&$EXP_SIZE_FD fi import ~~~~~~ The **import** command is used for restoring an instance from a backup as done by **export**. The arguments are the similar to those passed to **export**, whose output will be provided on stdin. The difference in variables is that the current disk is denoted by ``IMPORT_DISK_PATH``, ``IMPORT_DISK_URI``, ``IMPORT_DEVICE`` and ``IMPORT_INDEX`` (instead of ``EXPORT_...``). rename ~~~~~~ This command is used in order to perform a rename at the instance OS level, after the instance has been renamed in Ganeti. The command should do whatever steps are required to ensure that the instance is updated to use the new name, if the operating system supports it. Note that it is acceptable for the rename script to do nothing at all, however be warned that in this case, there will be a desynchronization between what gnt-instance list shows you and the actual hostname of the instance. The script will be passed one additional environment variable called ``OLD_INSTANCE_NAME`` which holds the old instance name. The ``INSTANCE_NAME`` variable holds the new instance name. A very simple rename script should at least change the hostname and IP address of the instance, leaving the administrator to update the other services. verify ~~~~~~ The *verify* script is used to verify consistency of the OS parameters (see below). The command should take one or more arguments denoting what checks should be performed, and return a proper exit code depending on whether the validation failed or succeeded. Currently (API version 20), only one parameter is supported: ``parameters``. This should validate the ``OSP_`` variables from the environment, and output diagnostic messages in case the validation fails. For the ``dhcp`` parameter given as example above, a verification script could be: .. code-block:: bash #!/bin/sh case $OSP_DHCP in ""|yes|no) ;; *) echo "Invalid value '$OSP_DHCP' for the dhcp parameter" 1>&2 exit 1; ;; esac exit 0 TEXT FILES ---------- ganeti_api_version ~~~~~~~~~~~~~~~~~~ The ganeti_api_version file is a plain text file containing the version(s) of the guest OS API that this OS definition complies with, one per line. The version documented by this man page is 20, so this file must contain the number 20 followed by a newline if only this version is supported. A script compatible with more than one Ganeti version should contain the most recent version first (i.e. 20), followed by the old version(s) (in this case 15 and/or 10). variants.list ~~~~~~~~~~~~~ variants.list is a plain text file containing all the declared supported variants for this OS, one per line. If this file is missing or empty, then the OS won't be considered to support variants. Empty lines and lines starting with a hash (``#``) are ignored. parameters.list ~~~~~~~~~~~~~~~ This file declares the parameters supported by the OS, one parameter per line, with name and description (space and/or tab separated). For example:: dhcp Whether to enable (yes) or disable (no) dhcp root_size The size of the root partition, in GiB The parameters can then be used in instance add or modification, as follows:: # gnt-instance add -O dhcp=no,root_size=8 ... NOTES ----- Backwards compatibility ~~~~~~~~~~~~~~~~~~~~~~~ Ganeti 2.3 and up is compatible with API versions 10, 15 and 20. The OS parameters and related scripts (verify) are only supported in version 20. The variants functionality (variants.list, and OS_VARIANT env. var) are supported/present only in version 15 and up. Common behaviour ~~~~~~~~~~~~~~~~ All the scripts should display an usage message when called with a wrong number of arguments or when the first argument is ``-h`` or ``--help``. Upgrading from old versions ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Version 15 to 20 ^^^^^^^^^^^^^^^^ The ``parameters.list`` file and ``verify`` script have been added. For no parameters, an empty parameters file and an empty verify script which returns success can be used. Version 10 to 15 ^^^^^^^^^^^^^^^^ The ``variants.list`` file has been added, so OSes should support at least one variant, declaring it in that file and must be prepared to parse the OS_VARIANT environment variable. OSes are free to support more variants than just the declared ones. Note that this file is optional; without it, the variants functionality is disabled. Version 5 to 10 ^^^^^^^^^^^^^^^ The method for passing data has changed from command line options to environment variables, so scripts should be modified to use these. For an example of how this can be done in a way compatible with both versions, feel free to look at the debootstrap instance's common.sh auxiliary script. Also, instances can have now a variable number of disks, not only two, and a variable number of NICs (instead of fixed one), so the scripts should deal with this. The biggest change is in the import/export, which are called once per disk, instead of once per instance. Version 4 to 5 ^^^^^^^^^^^^^^ The rename script has been added. If you don't want to do any changes on the instances after a rename, you can migrate the OS definition to version 5 by creating the rename script simply as: .. code-block:: bash #!/bin/sh exit 0 Note that the script must be executable. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-rapi.rst000064400000000000000000000046611476477700300173200ustar00rootroot00000000000000ganeti-rapi(8) Ganeti | Version @GANETI_VERSION@ ================================================ Name ---- ganeti-rapi - Ganeti remote API daemon Synopsis -------- | **ganeti-rapi** [-d] [-f] [-p *PORT*] [-b *ADDRESS*] [-i *INTERFACE*] | [\--max-clients *CLIENTS*] [\--no-ssl] [-K *SSL_KEY_FILE*] | [-C *SSL_CERT_FILE*] | [\--require-authentication] [\--ssl-chain *SSL_CHAIN_FILE*] DESCRIPTION ----------- **ganeti-rapi** is the daemon providing a remote API for Ganeti clusters. It is automatically started on the master node, and by default it uses SSL encryption. This can be disabled by passing the ``--no-ssl`` option, or alternatively the certificate used can be changed via the ``-C`` option and the key via the ``-K`` option. If your certificate chain involves intermediate certificates that the server needs to present that chain file can be privided with ``--ssl-chain``. The daemon will listen to the "ganeti-rapi" TCP port, as listed in the system services database, or if not defined, to port 5080 by default. The port can be overridden by passing the ``-p`` option. The ``-b`` option can be used to specify the address to bind to (defaults to ``0.0.0.0``). Note that if you specify the address, the watcher needs to be informed about it using its option ``--rapi-ip``, otherwise it will not be able to reach the RAPI interface and will attempt to restart it all the time. Alternatively to setting the IP with ``--b``, the ``-i`` option can be used to specify the interface to bind do. The maximum number of simultaneous client connections may be configured with the ``--max-clients`` option. This defaults to 20. Connections above this count are accepted, but no responses are sent until enough connections are closed. See the *Ganeti remote API* documentation for further information. Requests are logged to ``@LOCALSTATEDIR@/log/ganeti/rapi-daemon.log``, in the same format as for the node and master daemon. ACCESS CONTROLS --------------- Most query operations are allowed without authentication. Only the modification operations require authentication, in the form of basic authentication. Specify the ``--require-authentication`` command line flag to always require authentication. The users and their rights are defined in the ``@LOCALSTATEDIR@/lib/ganeti/rapi/users`` file. The format of this file is described in the Ganeti documentation (``rapi.html``). .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-watcher.rst000064400000000000000000000073301476477700300200160ustar00rootroot00000000000000ganeti-watcher(8) Ganeti | Version @GANETI_VERSION@ =================================================== Name ---- ganeti-watcher - Ganeti cluster watcher Synopsis -------- **ganeti-watcher** [\--debug] [\--job-age=*age* ] [\--ignore-pause] [\--rapi-ip=*IP*] [\--no-verify-disks] [\--no-strict] DESCRIPTION ----------- The **ganeti-watcher** is a periodically run script which is responsible for keeping the instances in the correct status. It has two separate functions, one for the master node and another one that runs on every node. If the watcher is disabled at cluster level (via the **gnt-cluster watcher pause** command), it will exit without doing anything. The cluster-level pause can be overridden via the ``--ignore-pause`` option, for example if during a maintenance the watcher needs to be disabled in general, but the administrator wants to run it just once. The ``--debug`` option will increase the verbosity of the watcher and also activate logging to the standard error. The ``--no-strict`` option runs the group verify disks job in a non-strict mode. This only verifies those disks whose node locks could be acquired in a best-effort attempt and will skip nodes that are recognized as busy with other jobs. The ``--rapi-ip`` option needs to be set if the RAPI daemon was started with a particular IP (using the ``-b`` option). The two options need to be exactly the same to ensure that the watcher can reach the RAPI interface. Master operations ~~~~~~~~~~~~~~~~~ Its primary function is to try to keep running all instances which are marked as *up* in the configuration file, by trying to start them a limited number of times. Another function is to "repair" DRBD links by reactivating the block devices of instances which have secondaries on nodes that have been rebooted. Additionally, it will verify and repair degraded DRBD disks; this will not happen, if the ``--no-verify-disks`` option is given. The watcher will also archive old jobs (older than the age given via the ``--job-age`` option, which defaults to 6 hours), in order to keep the job queue manageable. Node operations ~~~~~~~~~~~~~~~ The watcher will restart any down daemons that are appropriate for the current node. In addition, it will execute any scripts which exist under the "watcher" directory in the Ganeti hooks directory (``@SYSCONFDIR@/ganeti/hooks``). This should be used for lightweight actions, like starting any extra daemons. If the cluster parameter ``maintain_node_health`` is enabled, then the watcher will also shutdown instances and DRBD devices if the node is declared as offline by known master candidates. The watcher does synchronous queries but will submit jobs for executing the changes. Due to locking, it could be that the jobs execute much later than the watcher submits them. FILES ----- The command has a set of state files (one per group) located at ``@LOCALSTATEDIR@/lib/ganeti/watcher.GROUP-UUID.data`` (only used on the master) and a log file at ``@LOCALSTATEDIR@/log/ganeti/watcher.log``. Removal of either file(s) will not affect correct operation; the removal of the state file will just cause the restart counters for the instances to reset to zero, and mark nodes as freshly rebooted (so for example DRBD minors will be re-activated). In some cases, it's even desirable to reset the watcher state, for example after maintenance actions, or when you want to simulate the reboot of all nodes, so in this case, you can remove all state files: .. code-block:: bash rm -f @LOCALSTATEDIR@/lib/ganeti/watcher.*.data rm -f @LOCALSTATEDIR@/lib/ganeti/watcher.*.instance-status rm -f @LOCALSTATEDIR@/lib/ganeti/instance-status And then re-run the watcher. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/ganeti-wconfd.rst000064400000000000000000000034151476477700300176410ustar00rootroot00000000000000ganeti-wconfd(8) Ganeti | Version @GANETI_VERSION@ ================================================== Name ---- ganeti-wconfd - Ganeti configuration writing daemon Synopsis -------- **ganeti-wcond** [-f] [-d] [--syslog] [--no-user-checks] [--no-voting --yes-do-it] [--force-node] DESCRIPTION ----------- **ganeti-wconfd** is the daemon that has authoritative knowledge about the configuration and is the only entity that can accept changes to it. All jobs that need to modify the configuration will do so by sending appropriate requests to this daemon. For testing purposes, you can give the ``-f`` option and the program won't detach from the running terminal. Debug-level message can be activated by giving the ``-d`` option. Logging to syslog, rather than its own log file, can be enabled by passing in the ``--syslog`` option. The **ganeti-wconfd** daemon listens on a Unix socket (``@LOCALSTATEDIR@/run/ganeti/socket/ganeti-query``) on which it accepts all requests in an internal protocol format, used by Ganeti jobs. The daemon will refuse to start if the user and group do not match the one defined at build time; this behaviour can be overridden by the ``--no-user-checks`` option. The daemon will refuse to start if it cannot verify that the majority of cluster nodes believes that it is running on the master node. To allow failover in a two-node cluster, this can be overridden by the ``--no-voting`` option. As it this is dangerous, the ``--yes-do-it`` option has to be given as well. Also, if the option ``--force-node`` is given, it will accept to run on a non-master node; it should not be necessary to give this option manually, but ``gnt-cluster masterfailover`` will use it internally to start the daemon in order to update the master-node information in the configuration. ganeti-3.1.0~rc2/man/ganeti.rst000064400000000000000000000410701476477700300163620ustar00rootroot00000000000000ganeti(7) Ganeti | Version @GANETI_VERSION@ =========================================== Name ---- ganeti - cluster-based virtualization management Synopsis -------- :: # gnt-cluster init cluster1.example.com # gnt-node add node2.example.com # gnt-instance add -n node2.example.com \ > -o debootstrap --disk 0:size=30g \ > -t plain instance1.example.com DESCRIPTION ----------- The Ganeti software manages physical nodes and virtual instances of a cluster based on a virtualization software. The current version (2.3) supports Xen 3.x and KVM (72 or above) as hypervisors, and LXC as an experimental hypervisor. Quick start ----------- First you must install the software on all the cluster nodes, either from sources or (if available) from a package. The next step is to create the initial cluster configuration, using **gnt-cluster init**. Then you can add other nodes, or start creating instances. Cluster architecture -------------------- In Ganeti 2.0, the architecture of the cluster is a little more complicated than in 1.2. The cluster is coordinated by a master daemon (**ganeti-masterd**\(8)), running on the master node. Each node runs (as before) a node daemon, and the master has the RAPI daemon running too. Node roles ~~~~~~~~~~ Each node can be in one of the following states: master Only one node per cluster can be in this role, and this node is the one holding the authoritative copy of the cluster configuration and the one that can actually execute commands on the cluster and modify the cluster state. See more details under *Cluster configuration*. master_candidate The node receives the full cluster configuration (configuration file and jobs) and can become a master via the **gnt-cluster master-failover** command. Nodes that are not in this state cannot transition into the master role due to missing state. regular This the normal state of a node. drained Nodes in this state are functioning normally but cannot receive new instances, because the intention is to set them to *offline* or remove them from the cluster. offline These nodes are still recorded in the Ganeti configuration, but except for the master daemon startup voting procedure, they are not actually contacted by the master. This state was added in order to allow broken machines (that are being repaired) to remain in the cluster but without creating problems. Node flags ~~~~~~~~~~ Nodes have two flags which govern which roles they can take: master_capable The node can become a master candidate, and furthermore the master node. When this flag is disabled, the node cannot become a candidate; this can be useful for special networking cases, or less reliable hardware. vm_capable The node can host instances. When enabled (the default state), the node will participate in instance allocation, capacity calculation, etc. When disabled, the node will be skipped in many cluster checks and operations. Node Parameters ~~~~~~~~~~~~~~~ The ``ndparams`` refer to node parameters. These can be set as defaults on cluster and node group levels, but they take effect for nodes only. Currently we support the following node parameters: oob_program Path to an executable used as the out-of-band helper. It needs to implement the corresponding interface; in particular, in needs to support the ``power-on``, ``power-off``, ``power-cycle``, ``power-status``, and ``health`` commands. The full specification can be found in the `Ganeti Node OOB Management Framework` design document (implemented in Ganeti 2.4). Design documents are also available online on ``http://docs.ganeti.org/``. spindle_count This should reflect the I/O performance of local attached storage (e.g. for "file", "plain" and "drbd" disk templates). It doesn't have to match the actual spindle count of (any eventual) mechanical hard-drives, its meaning is site-local and just the relative values matter. exclusive_storage When this Boolean flag is enabled, physical disks on the node are assigned to instance disks in an exclusive manner, so as to lower I/O interference between instances. This parameter cannot be set on individual nodes, as its value must be the same within each node group. The `Partitioned Ganeti` design document (implemented in Ganeti 2.9) contains more details. ovs When this Boolean flag is enabled, OpenvSwitch will be used as the network layer. This will cause the initialization of OpenvSwitch on the nodes when added to the cluster. Per default this is not enabled. ovs_name When ovs is enabled, this parameter will represent the name of the OpenvSwitch to generate and use. This will default to `switch1`. ovs_link When ovs is enabled, a OpenvSwitch will be initialized on new nodes and will have this as its connection to the outside. This parameter is not set per default, as it depends very much on the specific setup. ssh_port The port used for SSH connections to nodes belonging to a group. The user is responsible for properly configuring the ports of SSH daemons on machines prior to adding them as Ganeti nodes or when modifying the parameter value of an existing group. Note that using non-standard SSH ports and downgrading to an older Ganeti version that doesn't support ``ssh_port`` will break the cluster. Hypervisor State Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Using ``--hypervisor-state`` you can set hypervisor specific states. The format is: ``hypervisor:option=value``. Currently we support the following hypervisor state values: mem_total Total node memory, as discovered by this hypervisor mem_node Memory used by, or reserved for, the node itself; note that some hypervisors can report this in an authoritative way, other not mem_hv Memory used either by the hypervisor itself or lost due to instance allocation rounding; usually this cannot be precisely computed, but only roughly estimated cpu_total Total node cpu (core) count; usually this can be discovered automatically cpu_node Number of cores reserved for the node itself; this can either be discovered or set manually. Only used for estimating how many VCPUs are left for instances Note that currently this option is unused by Ganeti; values will be recorded but will not influence the Ganeti operation. Disk State Parameters ~~~~~~~~~~~~~~~~~~~~~ Using ``--disk-state`` you can set disk specific states. The format is: ``storage_type/identifier:option=value``. Where we currently just support ``lvm`` as storage type. The identifier in this case is the LVM volume group. By default this is ``xenvg``. Currently we support the following hypervisor state values: disk_total Total disk size (usually discovered automatically) disk_reserved Reserved disk size; this is a lower limit on the free space, if such a limit is desired disk_overhead Disk that is expected to be used by other volumes (set via ``reserved_lvs``); usually should be zero Note that currently this option is unused by Ganeti; values will be recorded but will not influence the Ganeti operation. Cluster configuration ~~~~~~~~~~~~~~~~~~~~~ The master node keeps and is responsible for the cluster configuration. On the filesystem, this is stored under the ``@LOCALSTATEDIR@/ganeti/lib`` directory, and if the master daemon is stopped it can be backed up normally. The master daemon will replicate the configuration database called ``config.data`` and the job files to all the nodes in the master candidate role. It will also distribute a copy of some configuration values via the *ssconf* files, which are stored in the same directory and start with a ``ssconf_`` prefix, to all nodes. Jobs ~~~~ All cluster modification are done via jobs. A job consists of one or more opcodes, and the list of opcodes is processed serially. If an opcode fails, the entire job is failed and later opcodes are no longer processed. A job can be in one of the following states: queued The job has been submitted but not yet processed by the master daemon. waiting The job is waiting for for locks before the first of its opcodes. canceling The job is waiting for locks, but is has been marked for cancellation. It will not transition to *running*, but to *canceled*. running The job is currently being executed. canceled The job has been canceled before starting execution. success The job has finished successfully. error The job has failed during runtime, or the master daemon has been stopped during the job execution. Common command line features ---------------------------- Options ~~~~~~~ Many Ganeti commands provide the following options. The availability for a certain command can be checked by calling the command using the ``--help`` option. | **gnt-...** *command* [\--dry-run] [\--priority {low | normal | high}] | [\--submit] [\--print-jobid] The ``--dry-run`` option can be used to check whether an operation would succeed. The option ``--priority`` sets the priority for opcodes submitted by the command. The ``--submit`` option is used to send the job to the master daemon but not wait for its completion. The job ID will be shown so that it can be examined using **gnt-job info**. The ``--reason`` option allows to specify a reason for the submitted job. It is inherited by all jobs created by this job and intended to make it easier to track the reason why any given job exists. Some reason strings have special meanings: rate-limit:*n*:*label* Assigns the job to a rate-limiting bucket identified by the combination of (``n``, ``label``); that is ``rate-limit:4:mylabel`` and ``rate-limit:5:mylabel`` are different buckets. ``n`` must be a positive integer; ``label`` is an arbitrary ASCII string. The job scheduler will ensure that, for each rate-limiting bucket, there are at most ``n`` jobs belonging to that bucket that are running in parallel. The special-cases for reason strings above must be given in exactly the specified format; if they are preceded by other characters (whitespace included), they become normal reasons and have no special effect. The ``--print-jobid`` option makes the command print the job id as first line on stdout, so that it is easy to parse by other programs. Defaults ~~~~~~~~ For certain commands you can use environment variables to provide default command line arguments. Just assign the arguments as a string to the corresponding environment variable. The format of that variable name is **binary**_*command*. **binary** is the name of the ``gnt-*`` script all upper case and dashes replaced by underscores, and *command* is the command invoked on that script. Currently supported commands are ``gnt-node list``, ``gnt-group list`` and ``gnt-instance list``. So you can configure default command line flags by setting ``GNT_NODE_LIST``, ``GNT_GROUP_LIST`` and ``GNT_INSTANCE_LIST``. Debug options ~~~~~~~~~~~~~ If the variable ``FORCE_LUXI_SOCKET`` is set, it will override the socket used for LUXI connections by command-line tools (``gnt-*``). This is useful mostly for debugging, and some operations won't work at all if, for example, you point this variable to the confd-supplied query socket and try to submit a job. If the variable is set to the value ``master``, it will connect to the correct path for the master daemon (even if, for example, split queries are enabled and this is a query operation). If set to ``query``, it will always (try to) connect to the query socket, even if split queries are disabled. Otherwise, the value is taken to represent a filesystem path to the socket to use. Field formatting ---------------- Multiple ganeti commands use the same framework for tabular listing of resources (e.g. **gnt-instance list**, **gnt-node list**, **gnt-group list**, **gnt-debug locks**, etc.). For these commands, special states are denoted via a special symbol (in terse mode) or a string (in verbose mode): \*, (offline) The node in question is marked offline, and thus it cannot be queried for data. This result is persistent until the node is de-offlined. ?, (nodata) Ganeti expected to receive an answer from this entity, but the cluster RPC call failed and/or we didn't receive a valid answer; usually more information is available in the node daemon log (if the node is alive) or the master daemon log. This result is transient, and re-running command might return a different result. -, (unavail) The respective field doesn't make sense for this entity; e.g. querying a down instance for its current memory 'live' usage, or querying a non-vm_capable node for disk/memory data. This result is persistent, and until the entity state is changed via ganeti commands, the result won't change. ??, (unknown) This field is not known (note that this is different from entity being unknown). Either you have mis-typed the field name, or you are using a field that the running Ganeti master daemon doesn't know. This result is persistent, re-running the command won't change it. Key-value parameters ~~~~~~~~~~~~~~~~~~~~ Multiple options take parameters that are of the form ``key=value,key=value,...`` or ``category:key=value,...``. Examples are the hypervisor parameters, backend parameters, etc. For these, it's possible to use values that contain commas by escaping with via a backslash (which needs two if not single-quoted, due to shell behaviour):: # gnt-instance modify -H kernel_path=an\\,example instance1 # gnt-instance modify -H kernel_path='an\,example' instance1 Additionally, the following non-string parameters can be passed. To pass the boolean value ``True``, only mention the key (leaving out the equality sign and any value). To pass the boolean value ``False``, again only mention the key, but prefix it with ``no_``. To pass the special ``None`` value, again only mention the key, but prefix it with a single ``-`` sign. Query filters ~~~~~~~~~~~~~ Most commands listing resources (e.g. instances or nodes) support filtering. The filter language is similar to Python expressions with some elements from Perl. The language is not generic. Each condition must consist of a field name and a value (except for boolean checks), a field can not be compared to another field. Keywords are case-sensitive. Examples (see below for syntax details): - List webservers:: gnt-instance list --filter 'name =* "web*.example.com"' - List instances with three or six virtual CPUs and whose primary nodes reside in groups starting with the string "rack":: gnt-instance list --filter '(be/vcpus == 3 or be/vcpus == 6) and pnode.group =~ m/^rack/' - Nodes hosting primary instances:: gnt-node list --filter 'pinst_cnt != 0' - Nodes which aren't master candidates:: gnt-node list --filter 'not master_candidate' - Short version for globbing patterns:: gnt-instance list '*.site1' '*.site2' Syntax in pseudo-BNF:: ::= /* String quoted with single or double quotes, backslash for escaping */ ::= /* Number in base-10 positional notation */ ::= /* Regular expression */ /* Modifier "i": Case-insensitive matching, see http://docs.python.org/library/re#re.IGNORECASE Modifier "s": Make the "." special character match any character, including newline, see http://docs.python.org/library/re#re.DOTALL */ ::= /* empty */ | i | s ::= | ::= { /* Value comparison */ { == | != | < | <= | >= | > } /* Collection membership */ | [ not ] in /* Regular expressions (recognized delimiters are "/", "#", "^", and "|"; backslash for escaping) */ | { =~ | !~ } m// /* Globbing */ | { =* | !* } /* Boolean */ | } ::= { [ not ] | ( ) } [ { and | or } ] Operators: *==* Equality *!=* Inequality *<* Less than *<=* Less than or equal *>* Greater than *>=* Greater than or equal *=~* Pattern match using regular expression *!~* Logically negated from *=~* *=\** Globbing, see **glob**\(7), though only * and ? are supported *!\** Logically negated from *=\** *in*, *not in* Collection membership and negation Common daemon functionality --------------------------- All Ganeti daemons re-open the log file(s) when sent a SIGHUP signal. **logrotate**\(8) can be used to rotate Ganeti's log files. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-backup.rst000064400000000000000000000245761476477700300171620ustar00rootroot00000000000000gnt-backup(8) Ganeti | Version @GANETI_VERSION@ =============================================== Name ---- gnt-backup - Ganeti instance import/export Synopsis -------- **gnt-backup** {command} [arguments...] DESCRIPTION ----------- The **gnt-backup** is used for importing and exporting instances and their configuration from a Ganeti system. It is useful for backing up instances and also to migrate them between clusters. COMMANDS -------- EXPORT ~~~~~~ | **export** {-n *node-name*} | [\--shutdown-timeout=*N*] [\--noshutdown] [\--remove-instance] | [\--ignore-remove-failures] [\--submit] [\--print-jobid] | [\--transport-compression=*compression-mode*] | [\--zero-free-space] [\--zeroing-timeout-fixed] | [\--zeroing-timeout-per-mib] [\--long-sleep] | {*instance-name*} Exports an instance to the target node. All the instance data and its configuration will be exported under the ``@CUSTOM_EXPORT_DIR@/$instance`` directory on the target node. The ``--transport-compression`` option is used to specify which compression mode is used to try and speed up moves during the export. Valid values are 'none', and any values defined in the 'compression_tools' cluster parameter. The ``--shutdown-timeout`` is used to specify how much time (in seconds) to wait before forcing the shutdown (xl destroy in xen, killing the kvm process, for kvm). By default two minutes are given to each instance to stop. The ``--noshutdown`` option will create a snapshot disk of the instance without shutting it down first. While this is faster and involves no downtime, it cannot be guaranteed that the instance data will be in a consistent state in the exported dump. The ``--remove`` option can be used to remove the instance after it was exported. This is useful to make one last backup before removing the instance. The ``--zero-free-space`` option can be used to zero the free space of the instance prior to exporting it, saving space if compression is used. The ``--zeroing-timeout-fixed`` and ``--zeroing-timeout-per-mib`` options control the timeout, the former determining the minimum time to wait, and the latter how much longer to wait per MiB of data the instance has. The ``--long-sleep`` option allows Ganeti to keep the instance shut down for the entire duration of the export if necessary. This is needed if snapshots are not supported by the underlying storage type, or if the creation of snapshots fails for some reason - e.g. lack of space. Should the snapshotting or transfer of any of the instance disks fail, the backup will not complete and any previous backups will be preserved. The exact details of the failures will be shown during the command execution (and will be stored in the job log). It is recommended that for any non-zero exit code, the backup is considered invalid, and retried. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-backup export -n node1.example.com instance3.example.com IMPORT ~~~~~~ | **import** | {-n *node[:secondary-node]* | \--iallocator *name*} | [\--compress=*compression-mode*] | [\--disk *N*:size=*VAL* [,vg=*VG*], [,mode=*ro|rw*]...] | [\--net *N* [:options...] | \--no-nics] | [-B *BEPARAMS*] | [-H *HYPERVISOR* [: option=*value*... ]] | [\--src-node=*source-node*] [\--src-dir=*source-dir*] | [-t [diskless | plain | drbd | file]] | [\--identify-defaults] | [\--ignore-ipolicy] | [\--submit] [\--print-jobid] | {*instance-name*} Imports a new instance from an export residing on *source-node* in *source-dir*. *instance-name* must be in DNS and resolve to a IP in the same network as the nodes in the cluster. If the source node and directory are not passed, the last backup in the cluster is used, as visible with the **list** command. The ``disk`` option specifies the parameters for the disks of the instance. The numbering of disks starts at zero. For each disk, at least the size needs to be given, and optionally the access mode (read-only or the default of read-write) and LVM volume group can also be specified. The size is interpreted (when no unit is given) in mebibytes. You can also use one of the suffixes m, g or t to specify the exact the units used; these suffixes map to mebibytes, gibibytes and tebibytes. Alternatively, a single-disk instance can be created via the ``-s`` option which takes a single argument, the size of the disk. This is similar to the Ganeti 1.2 version (but will only create one disk). If no disk information is passed, the disk configuration saved at export time will be used. The minimum disk specification is therefore empty (export information will be used), a single disk can be specified as ``--disk 0:size=20G`` (or ``-s 20G`` when using the ``-s`` option), and a three-disk instance can be specified as ``--disk 0:size=20G --disk 1:size=4G --disk 2:size=100G``. The NICs of the instances can be specified via the ``--net`` option. By default, the NIC configuration of the original (exported) instance will be reused. Each NIC can take up to three parameters (all optional): mac either a value or ``generate`` to generate a new unique MAC, or ``auto`` to reuse the old MAC ip specifies the IP address assigned to the instance from the Ganeti side (this is not necessarily what the instance will use, but what the node expects the instance to use) mode specifies the connection mode for this NIC: ``routed``, ``bridged`` or ``openvswitch`` link in bridged and openvswitch mode specifies the interface to attach this NIC to, in routed mode it's intended to differentiate between different routing tables/instance groups (but the meaning is dependent on the network script in use, see **gnt-cluster**\(8) for more details) Of these ``mode`` and ``link`` are NIC parameters, and inherit their default at cluster level. If no network is desired for the instance, you should create a single empty NIC and delete it afterwards via **gnt-instance modify \--net delete**. The ``-B`` option specifies the backend parameters for the instance. If no such parameters are specified, the values are inherited from the export. Possible parameters are: maxmem the maximum memory size of the instance; as usual, suffixes can be used to denote the unit, otherwise the value is taken in mebibytes minmem the minimum memory size of the instance; as usual, suffixes can be used to denote the unit, otherwise the value is taken in mebibytes vcpus the number of VCPUs to assign to the instance (if this value makes sense for the hypervisor) auto_balance whether the instance is considered in the N+1 cluster checks (enough redundancy in the cluster to survive a node failure) always\_failover ``True`` or ``False``, whether the instance must be failed over (shut down and rebooted) always or it may be migrated (briefly suspended) The ``-t`` options specifies the disk layout type for the instance. If not passed, the configuration of the original instance is used. The available choices are: diskless This creates an instance with no disks. Its useful for testing only (or other special cases). plain Disk devices will be logical volumes. drbd Disk devices will be drbd (version 8.x) on top of lvm volumes. file Disk devices will be backed up by files, under the cluster's default file storage directory. By default, each instance will get a directory (as its own name) under this path, and each disk is stored as individual files in this (instance-specific) directory. The ``--iallocator`` option specifies the instance allocator plugin to use. If you pass in this option the allocator will select nodes for this instance automatically, so you don't need to pass them with the ``-n`` option. For more information please refer to the instance allocator documentation. The optional second value of the ``--node`` is used for the drbd template and specifies the remote node. The ``--compress`` option is used to specify which compression mode is used for moves during the import. Valid values are 'none' (the default) and 'gzip'. The ``--src-dir`` option allows importing instances from a directory below ``@CUSTOM_EXPORT_DIR@``. If ``--ignore-ipolicy`` is given any instance policy violations occurring during this operation are ignored. Since many of the parameters are by default read from the exported instance information and used as such, the new instance will have all parameters explicitly specified, the opposite of a newly added instance which has most parameters specified via cluster defaults. To change the import behaviour to recognize parameters whose saved value matches the current cluster default and mark it as such (default value), pass the ``--identify-defaults`` option. This will affect the hypervisor, backend and NIC parameters, both read from the export file and passed in via the command line. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example for identical instance import:: # gnt-backup import -n node1.example.com instance3.example.com Explicit configuration example:: # gnt-backup import -t plain --disk 0:size=1G -B memory=512 \ > -n node1.example.com \ > instance3.example.com LIST ~~~~ | **list** [\--node=*NODE*] [\--no-headers] [\--separator=*SEPARATOR*] | [-o *[+]FIELD,...*] Lists the exports currently available in the default directory in all the nodes of the current cluster, or optionally only a subset of them specified using the ``--node`` option (which can be used multiple times) The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The ``-o`` option takes a comma-separated list of output fields. The available fields and their meaning are: @QUERY_FIELDS_EXPORT@ If the value of the option starts with the character ``+``, the new fields will be added to the default list. This allows one to quickly see the default list plus a few other fields, instead of retyping the entire list of fields. Example:: # gnt-backup list --node node1 --node node2 LIST-FIELDS ~~~~~~~~~~~ **list-fields** [field...] Lists available fields for exports. REMOVE ~~~~~~ **remove** {*instance-name*} Removes the backup for the given instance name, if any. If the backup was for a deleted instance, it is needed to pass the FQDN of the instance, and not only the short hostname. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-cluster.rst000064400000000000000000001273611476477700300173720ustar00rootroot00000000000000gnt-cluster(8) Ganeti | Version @GANETI_VERSION@ ================================================ Name ---- gnt-cluster - Ganeti administration, cluster-wide Synopsis -------- **gnt-cluster** {command} [arguments...] DESCRIPTION ----------- The **gnt-cluster** is used for cluster-wide administration in the Ganeti system. COMMANDS -------- ACTIVATE-MASTER-IP ~~~~~~~~~~~~~~~~~~ **activate-master-ip** Activates the master IP on the master node. COMMAND ~~~~~~~ **command** [-n *node-name*] [-g *group*] [-M] {*command*} Executes a command on all nodes. This command is designed for simple usage. For more complex use cases the commands **dsh**\(1) or **cssh**\(1) should be used instead. If the option ``-n`` is not given, the command will be executed on all nodes, otherwise it will be executed only on the node(s) specified. Use the option multiple times for running it on multiple nodes, like:: # gnt-cluster command -n node1.example.com -n node2.example.com date The ``-g`` option can be used to run a command only on a specific node group, e.g.:: # gnt-cluster command -g default date The ``-M`` option can be used to prepend the node name to all output lines. The ``--failure-only`` option hides successful commands, making it easier to see failures. The command is executed serially on the selected nodes. If the master node is present in the list, the command will be executed last on the master. Regarding the other nodes, the execution order is somewhat alphabetic, so that node2.example.com will be earlier than node10.example.com but after node1.example.com. So given the node names node1, node2, node3, node10, node11, with node3 being the master, the order will be: node1, node2, node10, node11, node3. The command is constructed by concatenating all other command line arguments. For example, to list the contents of the /etc directory on all nodes, run:: # gnt-cluster command ls -l /etc and the command which will be executed will be ``ls -l /etc``. COPYFILE ~~~~~~~~ | **copyfile** [\--use-replication-network] [-n *node-name*] [-g *group*] | {*file*} Copies a file to all or to some nodes. The argument specifies the source file (on the current system), the ``-n`` argument specifies the target node, or nodes if the option is given multiple times. If ``-n`` is not given at all, the file will be copied to all nodes. The ``-g`` option can be used to only select nodes in a specific node group. Passing the ``--use-replication-network`` option will cause the copy to be done over the replication network (only matters if the primary/secondary IPs are different). Example:: # gnt-cluster copyfile -n node1.example.com -n node2.example.com /tmp/test This will copy the file /tmp/test from the current node to the two named nodes. DEACTIVATE-MASTER-IP ~~~~~~~~~~~~~~~~~~~~ **deactivate-master-ip** [\--yes] Deactivates the master IP on the master node. This should be run only locally or on a connection to the node ip directly, as a connection to the master ip will be broken by this operation. Because of this risk it will require user confirmation unless the ``--yes`` option is passed. DESTROY ~~~~~~~ **destroy** {\--yes-do-it} Remove all configuration files related to the cluster, so that a **gnt-cluster init** can be done again afterwards. Since this is a dangerous command, you are required to pass the argument *\--yes-do-it.* EPO ~~~ **epo** [\--on] [\--groups|\--all] [\--power-delay] *arguments* Performs an emergency power-off on nodes given as arguments. If ``--groups`` is given, arguments are node groups. If ``--all`` is provided, the whole cluster will be shut down. The ``--on`` flag recovers the cluster after an emergency power-off. When powering on the cluster you can use ``--power-delay`` to define the time in seconds (fractions allowed) waited between powering on individual nodes. Please note that the master node will not be turned down or up automatically. It will just be left in a state, where you can manually perform the shutdown of that one node. If the master is in the list of affected nodes and this is not a complete cluster emergency power-off (e.g. using ``--all``), you're required to do a master failover to another node not affected. GETMASTER ~~~~~~~~~ **getmaster** Displays the current master node. INFO ~~~~ **info** [\--roman] Shows runtime cluster information: cluster name, architecture (32 or 64 bit), master node, node list and instance list. Passing the ``--roman`` option gnt-cluster info will try to print its integer fields in a latin friendly way. This allows further diffusion of Ganeti among ancient cultures. SHOW-ISPECS-CMD ~~~~~~~~~~~~~~~ **show-ispecs-cmd** Shows the command line that can be used to recreate the cluster with the same options relative to specs in the instance policies. INIT ~~~~ | **init** | [{-s|\--secondary-ip} *secondary\_ip*] | [\--vg-name *vg-name*] | [\--master-netdev *interface-name*] | [\--master-netmask *netmask*] | [\--use-external-mip-script {yes \| no}] | [{-m|\--mac-prefix} *mac-prefix*] | [\--no-etc-hosts] | [\--no-ssh-init] | [\--file-storage-dir *dir*] | [\--shared-file-storage-dir *dir*] | [\--gluster-storage-dir *dir*] | [\--enabled-hypervisors *hypervisors*] | [{-H|\--hypervisor-parameters} *hypervisor*:*hv-param*=*value*[,*hv-param*=*value*...]] | [{-B|\--backend-parameters} *be-param*=*value*[,*be-param*=*value*...]] | [{-N|\--nic-parameters} *nic-param*=*value*[,*nic-param*=*value*...]] | [{-D|\--disk-parameters} *disk-template*:*disk-param*=*value*[,*disk-param*=*value*...]] | [\--maintain-node-health {yes \| no}] | [\--uid-pool *user-id pool definition*] | [{-I|\--default-iallocator} *default instance allocator*] | [\--default-iallocator-params *ial-param*=*value*,*ial-param*=*value*] | [\--primary-ip-version *version*] | [\--prealloc-wipe-disks {yes \| no}] | [\--node-parameters *ndparams*] | [{-C|\--candidate-pool-size} *candidate\_pool\_size*] | [\--specs-cpu-count *spec-param*=*value* [,*spec-param*=*value*...]] | [\--specs-disk-count *spec-param*=*value* [,*spec-param*=*value*...]] | [\--specs-disk-size *spec-param*=*value* [,*spec-param*=*value*...]] | [\--specs-mem-size *spec-param*=*value* [,*spec-param*=*value*...]] | [\--specs-nic-count *spec-param*=*value* [,*spec-param*=*value*...]] | [\--ipolicy-std-specs *spec*=*value* [,*spec*=*value*...]] | [\--ipolicy-bounds-specs *bounds_ispecs*] | [\--ipolicy-disk-templates *template* [,*template*...]] | [\--ipolicy-spindle-ratio *ratio*] | [\--ipolicy-vcpu-ratio *ratio*] | [\--disk-state *diskstate*] | [\--hypervisor-state *hvstate*] | [\--drbd-usermode-helper *helper*] | [\--enabled-disk-templates *template* [,*template*...]] | [\--install-image *image*] | [\--zeroing-image *image*] | [\--compression-tools [*tool*, [*tool*]]] | [\--user-shutdown {yes \| no}] | [\--ssh-key-type *type*] | [\--ssh-key-bits *bits*] | {*cluster-name*} This commands is only run once initially on the first node of the cluster. It will initialize the cluster configuration, setup the ssh-keys, start the daemons on the master node, etc. in order to have a working one-node cluster. Note that the *cluster-name* is not any random name. It has to be resolvable to an IP address using DNS, and it is best if you give the fully-qualified domain name. This hostname must resolve to an IP address reserved exclusively for this purpose, i.e. not already in use. The cluster can run in two modes: single-home or dual-homed. In the first case, all traffic (both public traffic, inter-node traffic and data replication traffic) goes over the same interface. In the dual-homed case, the data replication traffic goes over the second network. The ``-s (--secondary-ip)`` option here marks the cluster as dual-homed and its parameter represents this node's address on the second network. If you initialise the cluster with ``-s``, all nodes added must have a secondary IP as well. Note that for Ganeti it doesn't matter if the secondary network is actually a separate physical network, or is done using tunnelling, etc. For performance reasons, it's recommended to use a separate network, of course. The ``--vg-name`` option will let you specify a volume group different than "xenvg" for Ganeti to use when creating instance disks. This volume group must have the same name on all nodes. Once the cluster is initialized this can be altered by using the **modify** command. Note that if the volume group name is modified after the cluster creation and DRBD support is enabled you might have to manually modify the metavg as well. If you don't want to use lvm storage at all use the ``--enabled-disk-templates`` option to restrict the set of enabled disk templates. Once the cluster is initialized you can change this setup with the **modify** command. The ``--master-netdev`` option is useful for specifying a different interface on which the master will activate its IP address. It's important that all nodes have this interface because you'll need it for a master failover. The ``--master-netmask`` option allows to specify a netmask for the master IP. The netmask must be specified as an integer, and will be interpreted as a CIDR netmask. The default value is 32 for an IPv4 address and 128 for an IPv6 address. The ``--use-external-mip-script`` option allows to specify whether to use an user-supplied master IP address setup script, whose location is ``@SYSCONFDIR@/ganeti/scripts/master-ip-setup``. If the option value is set to False, the default script (located at ``@PKGLIBDIR@/tools/master-ip-setup``) will be executed. The ``-m (--mac-prefix)`` option will let you specify a three byte prefix under which the virtual MAC addresses of your instances will be generated. The prefix must be specified in the format ``XX:XX:XX`` and the default is ``aa:00:00``. The ``--no-etc-hosts`` option allows you to initialize the cluster without modifying the /etc/hosts file. The ``--no-ssh-init`` option allows you to initialize the cluster without creating or distributing SSH key pairs. This also sets the cluster-wide configuration parameter ``modify ssh setup`` to False. When adding nodes, Ganeti will consider this parameter to determine whether to create and distribute SSH key pairs on new nodes as well. The ``--file-storage-dir``, ``--shared-file-storage-dir`` and ``--gluster-storage-dir`` options allow you set the directory to use for storing the instance disk files when using respectively the file storage backend, the shared file storage backend and the gluster storage backend. Note that these directories must be an allowed directory for file storage. Those directories are specified in the ``@SYSCONFDIR@/ganeti/file-storage-paths`` file. The file storage directory can also be a subdirectory of an allowed one. The file storage directory should be present on all nodes. The ``--prealloc-wipe-disks`` sets a cluster wide configuration value for wiping disks prior to allocation and size changes (``gnt-instance grow-disk``). This increases security on instance level as the instance can't access untouched data from its underlying storage. The ``--enabled-hypervisors`` option allows you to set the list of hypervisors that will be enabled for this cluster. Instance hypervisors can only be chosen from the list of enabled hypervisors, and the first entry of this list will be used by default. Currently, the following hypervisors are available: xen-pvm Xen PVM hypervisor xen-hvm Xen HVM hypervisor kvm Linux KVM hypervisor chroot a simple chroot manager that starts chroot based on a script at the root of the filesystem holding the chroot fake fake hypervisor for development/testing Either a single hypervisor name or a comma-separated list of hypervisor names can be specified. If this option is not specified, only the xen-pvm hypervisor is enabled by default. The ``--user-shutdown`` option enables or disables user shutdown detection at the cluster level. User shutdown detection allows users to initiate instance poweroff from inside the instance, and Ganeti will report the instance status as 'USER_down' (as opposed, to 'ERROR_down') and the watcher will not restart these instances, thus preserving their instance status. This option is disabled by default. For KVM, the hypervisor parameter ``user_shutdown`` must also be set, either at the cluster level or on a per-instance basis (see **gnt-instance**\(8)). The ``-H (--hypervisor-parameters)`` option allows you to set default hypervisor specific parameters for the cluster. The format of this option is the name of the hypervisor, followed by a colon and a comma-separated list of key=value pairs. The keys available for each hypervisors are detailed in the **gnt-instance**\(8) man page, in the **add** command plus the following parameters which are only configurable globally (at cluster level): migration\_port Valid for the Xen PVM and KVM hypervisors. migration\_bandwidth Valid for the KVM hypervisor. This option specifies the maximum bandwidth that KVM will use for instance live migrations. The value is in MiB/s. This option is only effective with kvm versions >= 78 and qemu-kvm versions >= 0.10.0. The ``-B (--backend-parameters)`` option allows you to set the default backend parameters for the cluster. The parameter format is a comma-separated list of key=value pairs with the following supported keys: vcpus Number of VCPUs to set for an instance by default, must be an integer, will be set to 1 if no specified. maxmem Maximum amount of memory to allocate for an instance by default, can be either an integer or an integer followed by a unit (M for mebibytes and G for gibibytes are supported), will be set to 128M if not specified. minmem Minimum amount of memory to allocate for an instance by default, can be either an integer or an integer followed by a unit (M for mebibytes and G for gibibytes are supported), will be set to 128M if not specified. auto\_balance Value of the auto\_balance flag for instances to use by default, will be set to true if not specified. always\_failover Default value for the ``always_failover`` flag for instances; if not set, ``False`` is used. The ``-N (--nic-parameters)`` option allows you to set the default network interface parameters for the cluster. The parameter format is a comma-separated list of key=value pairs with the following supported keys: mode The default NIC mode, one of ``routed``, ``bridged`` or ``openvswitch``. link In ``bridged`` or ``openvswitch`` mode the default interface where to attach NICs. In ``routed`` mode it represents an hypervisor-vif-script dependent value to allow different instance groups. For example under the KVM default network script it is interpreted as a routing table number or name. Openvswitch support is also hypervisor dependent and currently works for the default KVM network script. Under Xen a custom network script must be provided. The ``-D (--disk-parameters)`` option allows you to set the default disk template parameters at cluster level. The format used for this option is similar to the one use by the ``-H`` option: the disk template name must be specified first, followed by a colon and by a comma-separated list of key-value pairs. These parameters can only be specified at cluster and node group level; the cluster-level parameter are inherited by the node group at the moment of its creation, and can be further modified at node group level using the **gnt-group**\(8) command. The following is the list of disk parameters available for the **drbd** template, with measurement units specified in square brackets at the end of the description (when applicable): resync-rate Static re-synchronization rate. [KiB/s] data-stripes Number of stripes to use for data LVs. meta-stripes Number of stripes to use for meta LVs. disk-barriers What kind of barriers to **disable** for disks. It can either assume the value "n", meaning no barrier disabled, or a non-empty string containing a subset of the characters "bfd". "b" means disable disk barriers, "f" means disable disk flushes, "d" disables disk drains. meta-barriers Boolean value indicating whether the meta barriers should be disabled (True) or not (False). metavg String containing the name of the default LVM volume group for DRBD metadata. By default, it is set to ``xenvg``. It can be overridden during the instance creation process by using the ``metavg`` key of the ``--disk`` parameter. disk-custom String containing additional parameters to be appended to the arguments list of ``drbdsetup disk``. net-custom String containing additional parameters to be appended to the arguments list of ``drbdsetup net``. protocol Replication protocol for the DRBD device. Has to be either "A", "B" or "C". Refer to the DRBD documentation for further information about the differences between the protocols. dynamic-resync Boolean indicating whether to use the dynamic resync speed controller or not. If enabled, c-plan-ahead must be non-zero and all the c-* parameters will be used by DRBD. Otherwise, the value of resync-rate will be used as a static resync speed. c-plan-ahead Agility factor of the dynamic resync speed controller. (the higher, the slower the algorithm will adapt the resync speed). A value of 0 (that is the default) disables the controller. [ds] c-fill-target Maximum amount of in-flight resync data for the dynamic resync speed controller. [sectors] c-delay-target Maximum estimated peer response latency for the dynamic resync speed controller. [ds] c-min-rate Minimum resync speed for the dynamic resync speed controller. [KiB/s] c-max-rate Upper bound on resync speed for the dynamic resync speed controller. [KiB/s] List of parameters available for the **plain** template: stripes Number of stripes to use for new LVs. List of parameters available for the **rbd** template: pool The RADOS cluster pool, inside which all rbd volumes will reside. When a new RADOS cluster is deployed, the default pool to put rbd volumes (Images in RADOS terminology) is 'rbd'. access If 'userspace', instances will access their disks directly without going through a block device, avoiding expensive context switches with kernel space and the potential for deadlocks_ in low memory scenarios. The default value is 'kernelspace' and it disables this behaviour. This setting may only be changed to 'userspace' if all instance disks in the affected group or cluster can be accessed in userspace. Attempts to use this feature without rbd support compiled in KVM result in a "no such file or directory" error messages. .. _deadlocks: http://tracker.ceph.com/issues/3076 The option ``--maintain-node-health`` allows one to enable/disable automatic maintenance actions on nodes. Currently these include automatic shutdown of instances and deactivation of DRBD devices on offline nodes; in the future it might be extended to automatic removal of unknown LVM volumes, etc. Note that this option is only useful if the use of ``ganeti-confd`` was enabled at compilation. The ``--uid-pool`` option initializes the user-id pool. The *user-id pool definition* can contain a list of user-ids and/or a list of user-id ranges. The parameter format is a comma-separated list of numeric user-ids or user-id ranges. The ranges are defined by a lower and higher boundary, separated by a dash. The boundaries are inclusive. If the ``--uid-pool`` option is not supplied, the user-id pool is initialized to an empty list. An empty list means that the user-id pool feature is disabled. The ``-I (--default-iallocator)`` option specifies the default instance allocator. The instance allocator will be used for operations like instance creation, instance and node migration, etc. when no manual override is specified. If this option is not specified and htools was not enabled at build time, the default instance allocator will be blank, which means that relevant operations will require the administrator to manually specify either an instance allocator, or a set of nodes. If the option is not specified but htools was enabled, the default iallocator will be **hail**\(1) (assuming it can be found on disk). The default iallocator can be changed later using the **modify** command. The option ``--default-iallocator-params`` sets the cluster-wide iallocator parameters used by the default iallocator only on instance allocations. The ``--primary-ip-version`` option specifies the IP version used for the primary address. Possible values are 4 and 6 for IPv4 and IPv6, respectively. This option is used when resolving node names and the cluster name. The ``--node-parameters`` option allows you to set default node parameters for the cluster. Please see **ganeti**\(7) for more information about supported key=value pairs. The ``-C (--candidate-pool-size)`` option specifies the ``candidate_pool_size`` cluster parameter. This is the number of nodes that the master will try to keep as master\_candidates. For more details about this role and other node roles, see the **ganeti**\(7). The ``--specs-...`` and ``--ipolicy-...`` options specify the instance policy on the cluster. The ``--ipolicy-bounds-specs`` option sets the minimum and maximum specifications for instances. The format is: min:*param*=*value*,.../max:*param*=*value*,... and further specifications pairs can be added by using ``//`` as a separator. The ``--ipolicy-std-specs`` option takes a list of parameter/value pairs. For both options, *param* can be: - ``cpu-count``: number of VCPUs for an instance - ``disk-count``: number of disk for an instance - ``disk-size``: size of each disk - ``memory-size``: instance memory - ``nic-count``: number of network interface - ``spindle-use``: spindle usage for an instance For the ``--specs-...`` options, each option can have three values: ``min``, ``max`` and ``std``, which can also be modified on group level (except for ``std``, which is defined once for the entire cluster). Please note, that ``std`` values are not the same as defaults set by ``--beparams``, but they are used for the capacity calculations. - ``--specs-cpu-count`` limits the number of VCPUs that can be used by an instance. - ``--specs-disk-count`` limits the number of disks - ``--specs-disk-size`` limits the disk size for every disk used - ``--specs-mem-size`` limits the amount of memory available - ``--specs-nic-count`` sets limits on the number of NICs used The ``--ipolicy-spindle-ratio`` option takes a decimal number. The ``--ipolicy-disk-templates`` option takes a comma-separated list of disk templates. This list of disk templates must be a subset of the list of cluster-wide enabled disk templates (which can be set with ``--enabled-disk-templates``). - ``--ipolicy-spindle-ratio`` limits the instances-spindles ratio - ``--ipolicy-vcpu-ratio`` limits the vcpu-cpu ratio All the instance policy elements can be overridden at group level. Group level overrides can be removed by specifying ``default`` as the value of an item. The ``--drbd-usermode-helper`` option can be used to specify a usermode helper. Check that this string is the one used by the DRBD kernel. For details about how to use ``--hypervisor-state`` and ``--disk-state`` have a look at **ganeti**\(7). The ``--enabled-disk-templates`` option specifies a list of disk templates that can be used by instances of the cluster. For the possible values in this list, see **gnt-instance**\(8). Note that in contrast to the list of disk templates in the ipolicy, this list is a hard restriction. It is not possible to create instances with disk templates that are not enabled in the cluster. It is also not possible to disable a disk template when there are still instances using it. The first disk template in the list of enabled disk template is the default disk template. It will be used for instance creation, if no disk template is requested explicitly. The ``--install-image`` option specifies the location of the OS image to use to run the OS scripts inside a virtualized environment. This can be a file path or a URL. In the case that a file path is used, nodes are expected to have the install image located at the given path, although that is enforced during a instance create with unsafe OS scripts operation only. The ``--zeroing-image`` option specifies the location of the OS image to use to zero out the free space of an instance. This can be a file path or a URL. In the case that a file path is used, nodes are expected to have the zeroing image located at the given path, although that is enforced during a zeroing operation only. The ``--compression-tools`` option specifies the tools that can be used to compress the disk data of instances in transfer. The default tools are: 'gzip', 'gzip-slow', and 'gzip-fast'. For compatibility reasons, the 'gzip' tool cannot be excluded from the list of compression tools. Ganeti knows how to use certain tools, but does not provide them as a default as they are not commonly present: currently only 'lzop'. The user should indicate their presence by specifying them through this option. Any other custom tool specified must have a simple executable name ('[-_a-zA-Z0-9]+'), accept input on stdin, and produce output on stdout. The '-d' flag specifies that decompression rather than compression is taking place. The '-h' flag must be supported as a means of testing whether the executable exists. These requirements are compatible with the gzip command line options, allowing many tools to be easily wrapped and used. The ``--ssh-key-type`` and ``--ssh-key-bits`` options determine the properties of the SSH keys Ganeti generates and uses to execute commands on nodes. The supported types are currently 'dsa', 'rsa', and 'ecdsa'. The supported bit sizes vary across keys, reflecting the options **ssh-keygen**\(1) exposes. These are currently: - dsa: 1024 bits - rsa: >=768 bits - ecdsa: 256, 384, or 521 bits Ganeti defaults to using 2048-bit RSA keys. MASTER-FAILOVER ~~~~~~~~~~~~~~~ | **master-failover** [\--no-voting] [\--yes-do-it] | [\--ignore-offline-nodes] Failover the master role to the current node. The ``--no-voting`` option skips the remote node agreement checks. This is dangerous, but necessary in some cases (for example failing over the master role in a 2 node cluster with the original master down). If the original master then comes up, it won't be able to start its master daemon because it won't have enough votes, but so won't the new master, if the master daemon ever needs a restart. You can pass ``--no-voting`` to **ganeti-luxid** and **ganeti-wconfd** on the new master to solve this problem, and run **gnt-cluster redist-conf** to make sure the cluster is consistent again. The option ``--yes-do-it`` is used together with ``--no-voting``, for skipping the interactive checks. This is even more dangerous, and should only be used in conjunction with other means (e.g. a HA suite) to confirm that the operation is indeed safe. Note that in order for remote node agreement checks to work, a strict majority of nodes still needs to be functional. To avoid situations with daemons not starting up on the new master, master-failover without the ``--no-voting`` option verifies a healthy majority of nodes and refuses the operation otherwise. The ``--ignore-offline-nodes`` flag ignores offline nodes when the cluster is voting on the master. Any nodes that are offline are not counted towards the vote or towards the healthy nodes required for a majority, as they will be brought into sync with the rest of the cluster during a node readd operation. MASTER-PING ~~~~~~~~~~~ **master-ping** Checks if the master daemon is alive. If the master daemon is alive and can respond to a basic query (the equivalent of **gnt-cluster info**), then the exit code of the command will be 0. If the master daemon is not alive (either due to a crash or because this is not the master node), the exit code will be 1. MODIFY ~~~~~~ | **modify** [\--submit] [\--print-jobid] | [\--force] | [\--vg-name *vg-name*] | [\--enabled-hypervisors *hypervisors*] | [{-H|\--hypervisor-parameters} *hypervisor*:*hv-param*=*value*[,*hv-param*=*value*...]] | [{-B|\--backend-parameters} *be-param*=*value*[,*be-param*=*value*...]] | [{-N|\--nic-parameters} *nic-param*=*value*[,*nic-param*=*value*...]] | [{-D|\--disk-parameters} *disk-template*:*disk-param*=*value*[,*disk-param*=*value*...]] | [\--uid-pool *user-id pool definition*] | [\--add-uids *user-id pool definition*] | [\--remove-uids *user-id pool definition*] | [{-C|\--candidate-pool-size} *candidate\_pool\_size*] | [\--max-running-jobs *count* ] | [\--max-tracked-jobs *count* ] | [\--maintain-node-health {yes \| no}] | [\--prealloc-wipe-disks {yes \| no}] | [{-I|\--default-iallocator} *default instance allocator*] | [\--default-iallocator-params *ial-param*=*value*,*ial-param*=*value*] | [\--reserved-lvs=*NAMES*] | [\--node-parameters *ndparams*] | [{-m|\--mac-prefix} *mac-prefix*] | [\--master-netdev *interface-name*] | [\--master-netmask *netmask*] | [\--modify-etc-hosts {yes \| no}] | [\--use-external-mip-script {yes \| no}] | [\--hypervisor-state *hvstate*] | [\--disk-state *diskstate*] | [\--ipolicy-std-specs *spec*=*value* [,*spec*=*value*...]] | [\--ipolicy-bounds-specs *bounds_ispecs*] | [\--ipolicy-disk-templates *template* [,*template*...]] | [\--ipolicy-spindle-ratio *ratio*] | [\--ipolicy-vcpu-ratio *ratio*] | [\--enabled-disk-templates *template* [,*template*...]] | [\--drbd-usermode-helper *helper*] | [\--file-storage-dir *dir*] | [\--shared-file-storage-dir *dir*] | [\--compression-tools [*tool*, [*tool*]]] | [\--instance-communication-network *network*] | [\--install-image *image*] | [\--zeroing-image *image*] | [\--user-shutdown {yes \| no}] | [\--enabled-data-collectors *collectors*] | [\--data-collector-interval *intervals*] Modify the options for the cluster. The ``--vg-name``, ``--enabled-hypervisors``, ``-H (--hypervisor-parameters)``, ``-B (--backend-parameters)``, ``-D (--disk-parameters)``, ``--nic-parameters``, ``-C (--candidate-pool-size)``, ``--maintain-node-health``, ``--prealloc-wipe-disks``, ``--uid-pool``, ``--node-parameters``, ``--mac-prefix``, ``--master-netdev``, ``--master-netmask``, ``--use-external-mip-script``, ``--drbd-usermode-helper``, ``--file-storage-dir``, ``--shared-file-storage-dir``, ``--compression-tools``, and ``--enabled-disk-templates`` options are described in the **init** command. ``--master-netdev``, ``--master-netmask``, ``--use-external-mip-script``, ``--drbd-usermode-helper``, ``--file-storage-dir``, ``--shared-file-storage-dir``, ``--enabled-disk-templates``, and ``--user-shutdown`` options are described in the **init** command. The ``--modify-etc-hosts`` option is described by ``--no-etc-hosts`` in the **init** command. The ``--hypervisor-state`` and ``--disk-state`` options are described in detail in **ganeti**\(7). The ``--max-running-jobs`` options allows to set limit on the number of jobs in non-finished jobs that are not queued, i.e., the number of jobs that are in waiting or running state. The ``--max-tracked-jobs`` options allows to set the limit on the tracked jobs. Normally, Ganeti will watch waiting and running jobs by tracking their job file with inotify. If this limit is exceeded, however, Ganeti will back off and only periodically pull for updates. The ``--add-uids`` and ``--remove-uids`` options can be used to modify the user-id pool by adding/removing a list of user-ids or user-id ranges. The option ``--reserved-lvs`` specifies a list (comma-separated) of logical volume group names (regular expressions) that will be ignored by the cluster verify operation. This is useful if the volume group used for Ganeti is shared with the system for other uses. Note that it's not recommended to create and mark as ignored logical volume names which match Ganeti's own name format (starting with UUID and then .diskN), as this option only skips the verification, but not the actual use of the names given. To remove all reserved logical volumes, pass in an empty argument to the option, as in ``--reserved-lvs=`` or ``--reserved-lvs ''``. The ``-I (--default-iallocator)`` is described in the **init** command. To clear the default iallocator, just pass an empty string (''). The option ``--default-iallocator-params`` is described in the **init** command. To clear the default iallocator parameters, just pass an empty string (''). The ``--ipolicy-...`` options are described in the **init** command. The ``--instance-communication-network`` enables instance communication by specifying the name of the Ganeti network that should be used for instance communication. If the supplied network does not exist, Ganeti will create a new network with the supplied name with the default parameters for instance communication. If the supplied network exists, Ganeti will check its parameters and warn about unusual configurations, but it will still use that network for instance communication. The ``--enabled-data-collectors`` and ``--data-collector-interval`` options are to control the behavior of the **ganeti-mond**\(8). The first expects a list name=bool pairs to activate or deactivate the mentioned data collector. The second option expects similar pairs of collector name and number of seconds specifying the interval at which the collector shall be collected. See **gnt-cluster init** for a description of ``--install-image`` and ``--zeroing-image``. See **ganeti**\(7) for a description of ``--submit`` and other common options. QUEUE ~~~~~ **queue** {drain | undrain | info} Change job queue properties. The ``drain`` option sets the drain flag on the job queue. No new jobs will be accepted, but jobs already in the queue will be processed. The ``undrain`` will unset the drain flag on the job queue. New jobs will be accepted. The ``info`` option shows the properties of the job queue. WATCHER ~~~~~~~ **watcher** {pause *duration* | continue | info} Make the watcher pause or let it continue. The ``pause`` option causes the watcher to pause for *duration* seconds. The ``continue`` option will let the watcher continue. The ``info`` option shows whether the watcher is currently paused. REDIST-CONF ~~~~~~~~~~~ **redist-conf** [\--submit] [\--print-jobid] This command forces a full push of configuration files from the master node to the other nodes in the cluster. This is normally not needed, but can be run if the **verify** complains about configuration mismatches. See **ganeti**\(7) for a description of ``--submit`` and other common options. RENAME ~~~~~~ **rename** [-f] {*new-name*} Renames the cluster and in the process updates the master IP address to the one the new name resolves to. At least one of either the name or the IP address must be different, otherwise the operation will be aborted. Note that since this command can be dangerous (especially when run over SSH), the command will require confirmation unless run with the ``-f`` option. RENEW-CRYPTO ~~~~~~~~~~~~ | **renew-crypto** [-f] | [\--new-cluster-certificate] | [\--new-node-certificates] | [\--new-confd-hmac-key] | [\--new-rapi-certificate] [\--rapi-certificate *rapi-cert*] | [\--new-spice-certificate | \--spice-certificate *spice-cert* | \--spice-ca-certificate *spice-ca-cert*] | [\--new-ssh-keys] [\--no-ssh-key-check] | [\--new-cluster-domain-secret] [\--cluster-domain-secret *filename*] | [\--ssh-key-type *type*] | [\--ssh-key-bits *bits*] This command will stop all Ganeti daemons in the cluster and start them again once the new certificates and keys are replicated. The option ``--new-confd-hmac-key`` can be used to regenerate the HMAC key used by **ganeti-confd**\(8). The option ``--new-cluster-certificate`` will regenerate the cluster-internal server SSL certificate. The option ``--new-node-certificates`` will generate new node SSL certificates for all nodes. Note that for the regeneration of of the server SSL certificate will invoke a regeneration of the node certificates as well, because node certificates are signed by the server certificate and thus have to be recreated and signed by the new server certificate. Nodes which are offline during a renewal of the server or the node certificates are not accessible anymore once they are marked as online again. To fix this, please readd the node instead. To generate a new self-signed RAPI certificate (used by **ganeti-rapi**\(8)) specify ``--new-rapi-certificate``. If you want to use your own certificate, e.g. one signed by a certificate authority (CA), pass its filename to ``--rapi-certificate``. To generate a new self-signed SPICE certificate, used for SPICE connections to the KVM hypervisor, specify the ``--new-spice-certificate`` option. If you want to provide a certificate, pass its filename to ``--spice-certificate`` and pass the signing CA certificate to ``--spice-ca-certificate``. The option ``--new-ssh-keys`` renews all SSH keys of all nodes and updates the ``authorized_keys`` files of all nodes to contain only the (new) public keys of all master candidates. To avoid having to confirm the fingerprint of each node use the ``--no-ssh-key-check`` option. Be aware of that this includes a security risk as you omit verifying the machines' identities. Finally ``--new-cluster-domain-secret`` generates a new, random cluster domain secret, and ``--cluster-domain-secret`` reads the secret from a file. The cluster domain secret is used to sign information exchanged between separate clusters via a third party. The options ``--ssh-key-type`` and ``ssh-key-bits`` determine the properties of the disk types used. They are described in more detail in the ``init`` option description. REPAIR-DISK-SIZES ~~~~~~~~~~~~~~~~~ **repair-disk-sizes** [instance-name...] This command checks that the recorded size of the given instance's disks matches the actual size and updates any mismatches found. This is needed if the Ganeti configuration is no longer consistent with reality, as it will impact some disk operations. If no arguments are given, all instances will be checked. When exclusive storage is active, also spindles are updated. Note that only active disks can be checked by this command; in case a disk cannot be activated it's advised to use **gnt-instance activate-disks \--ignore-size ...** to force activation without regard to the current size. When all the disk sizes are consistent, the command will return no output. Otherwise it will log details about the inconsistencies in the configuration. UPGRADE ~~~~~~~ **upgrade** {--to *version* | --resume} This command safely switches all nodes of the cluster to a new Ganeti version. It is a prerequisite that the new version is already installed, albeit not activated, on all nodes; this requisite is checked before any actions are done. If called with the ``--resume`` option, any pending upgrade is continued, that was interrupted by a power failure or similar on master. It will do nothing, if not run on the master node, or if no upgrade was in progress. VERIFY ~~~~~~ | **verify** [\--no-nplus1-mem] [\--no-hv-param-assessment] | [\--node-group *nodegroup*] | [\--error-codes] [{-I|\--ignore-errors} *errorcode*] | [{-I|\--ignore-errors} *errorcode*...] | [--verify-ssh-clutter] Verify correctness of cluster configuration. This is safe with respect to running instances, and incurs no downtime of the instances. If the ``--no-nplus1-mem`` option is given, Ganeti won't check whether if it loses a node it can restart all the instances on their secondaries (and report an error otherwise). The ``--no-hv-param-assessment`` option disables the evaluation of hypervisor parameters set on the cluster-level. By default Ganeti will warn about suboptimal settings, if implemented by the enabled hypervisors. This will only generate warnings and never influence the return code of the verify run. With ``--node-group``, restrict the verification to those nodes and instances that live in the named group. This will not verify global settings, but will allow to perform verification of a group while other operations are ongoing in other groups. The ``--error-codes`` option outputs each error in the following parseable format: *ftype*:*ecode*:*edomain*:*name*:*msg*. These fields have the following meaning: ftype Failure type. Can be *WARNING* or *ERROR*. ecode Error code of the failure. See below for a list of error codes. edomain Can be *cluster*, *node* or *instance*. name Contains the name of the item that is affected from the failure. msg Contains a descriptive error message about the error ``gnt-cluster verify`` will have a non-zero exit code if at least one of the failures that are found are of type *ERROR*. The ``--ignore-errors`` option can be used to change this behaviour, because it demotes the error represented by the error code received as a parameter to a warning. The option must be repeated for each error that should be ignored (e.g.: ``-I ENODEVERSION -I ENODEORPHANLV``). The ``--error-codes`` option can be used to determine the error code of a given error. Note that the verification of the configuration file consistency across master candidates can fail if there are other concurrently running operations that modify the configuration. The ``--verify-ssh-clutter`` option checks if more than one SSH key for the same 'user@hostname' pair exists in the 'authorized_keys' file. This is only checked for hostnames of nodes which belong to the cluster. This check is optional, because there might be other systems manipulating the 'authorized_keys' files, which would cause too many false positives otherwise. List of error codes: @CONSTANTS_ECODES@ VERIFY-DISKS ~~~~~~~~~~~~ **verify-disks** [\--node-group *nodegroup*] [\--no-strict] The command checks which instances have degraded DRBD disks and activates the disks of those instances. With ``--node-group``, restrict the verification to those nodes and instances that live in the named group. The ``--no-strict`` option runs the group verify disks job in a non-strict mode. This only verifies those disks whose node locks could be acquired in a best-effort attempt and will skip nodes that are recognized as busy with other jobs. This command is run from the **ganeti-watcher** tool, which also has a different, complementary algorithm for doing this check. Together, these two should ensure that DRBD disks are kept consistent. VERSION ~~~~~~~ **version** Show the cluster version. Tags ~~~~ ADD-TAGS ^^^^^^^^ **add-tags** [\--from *file*] {*tag*...} Add tags to the cluster. If any of the tags contains invalid characters, the entire operation will abort. If the ``--from`` option is given, the list of tags will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, both sources will be used). A file name of - will be interpreted as stdin. LIST-TAGS ^^^^^^^^^ **list-tags** List the tags of the cluster. REMOVE-TAGS ^^^^^^^^^^^ **remove-tags** [\--from *file*] {*tag*...} Remove tags from the cluster. If any of the tags are not existing on the cluster, the entire operation will abort. If the ``--from`` option is given, the list of tags to be removed will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, tags from both sources will be removed). A file name of - will be interpreted as stdin. SEARCH-TAGS ^^^^^^^^^^^ **search-tags** {*pattern*} Searches the tags on all objects in the cluster (the cluster itself, the nodes and the instances) for a given pattern. The pattern is interpreted as a regular expression and a search will be done on it (i.e. the given pattern is not anchored to the beginning of the string; if you want that, prefix the pattern with ^). If no tags are matching the pattern, the exit code of the command will be one. If there is at least one match, the exit code will be zero. Each match is listed on one line, the object and the tag separated by a space. The cluster will be listed as /cluster, a node will be listed as /nodes/*name*, and an instance as /instances/*name*. Example: :: # gnt-cluster search-tags time /cluster ctime:2007-09-01 /nodes/node1.example.com mtime:2007-10-04 .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-debug.rst000064400000000000000000000126101476477700300167650ustar00rootroot00000000000000gnt-debug(8) Ganeti | Version @GANETI_VERSION@ ============================================== Name ---- gnt-debug - Debug commands Synopsis -------- **gnt-debug** {command} [arguments...] DESCRIPTION ----------- The **gnt-debug** is used for debugging the Ganeti system. COMMANDS -------- IALLOCATOR ~~~~~~~~~~ **iallocator** [\--debug] [\--dir *direction*] {\--algorithm *allocator* } [\--mode *mode*] [\--mem *memory*] [\--disks *diskS*] [\--disk-template *template*] [\--nics *nics*] [\--os-type *OS*] [\--vcpus *vcpus*] [\--tags *tags*] {*instance-name*} Executes a test run of the *iallocator* framework. The command will build input for a given iallocator script (named with the ``--algorithm`` option), and either show this input data (if *direction* is ``in``) or run the iallocator script and show its output (if *direction* is ``out``). If the *mode* is ``allocate``, then an instance definition is built from the other arguments and sent to the script, otherwise (*mode* is ``relocate``) an existing instance name must be passed as the first argument. This build of Ganeti will look for iallocator scripts in the following directories: @CUSTOM_IALLOCATOR_SEARCH_PATH@; for more details about this framework, see the HTML or PDF documentation. DELAY ~~~~~ **delay** [\--debug] [\--no-master] [\--interruptible] [-n *node-name*...] {*duration*} Run a test opcode (a sleep) on the master and on selected nodes (via an RPC call). This serves no other purpose but to execute a test operation. The ``-n`` option can be given multiple times to select the nodes for the RPC call. By default, the delay will also be executed on the master, unless the ``--no-master`` option is passed. The ``--interruptible`` option allows a running delay opcode to be interrupted by communicating with a special domain socket. If any data is sent to the socket, the delay opcode terminates. If this option is used, no RPCs are performed, but locks are still acquired. The *delay* argument will be interpreted as a floating point number. SUBMIT-JOB ~~~~~~~~~~ **submit-job** [\--verbose] [\--timing-stats] [\--job-repeat *n*] [\--op-repeat *n*] [\--each] {opcodes_file...} This command builds a list of opcodes from files in JSON format and submits a job per file to the master daemon. It can be used to test options that are not available via command line. The ``verbose`` option will additionally display the corresponding job IDs and the progress in waiting for the jobs; the ``timing-stats`` option will show some overall statistics including the number of total opcodes, jobs submitted and time spent in each stage (submit, exec, total). The ``job-repeat`` and ``op-repeat`` options allow to submit multiple copies of the passed arguments; job-repeat will cause N copies of each job (input file) to be submitted (equivalent to passing the arguments N times) while op-repeat will cause N copies of each of the opcodes in the file to be executed (equivalent to each file containing N copies of the opcodes). The ``each`` option allow to submit each job separately (using ``N`` SubmitJob LUXI requests instead of one SubmitManyJobs request). TEST-JOBQUEUE ~~~~~~~~~~~~~ **test-jobqueue** Executes a few tests on the job queue. This command might generate failed jobs deliberately. TEST_OSPARAMS ~~~~~~~~~~~~~ **test-osparams** {--os-parameters-secret *param*=*value*... } Tests secret os parameter transmission. LOCKS ~~~~~ | **locks** [\--no-headers] [\--separator=*separator*] [-v] | [-o *[+]field,...*] [\--interval=*seconds*] Shows a list of locks in the master daemon. The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The ``-v`` option activates verbose mode, which changes the display of special field states (see **ganeti**\(7)). The ``-o`` option takes a comma-separated list of output fields. The available fields and their meaning are: @QUERY_FIELDS_LOCK@ If the value of the option starts with the character ``+``, the new fields will be added to the default list. This allows one to quickly see the default list plus a few other fields, instead of retyping the entire list of fields. Use ``--interval`` to repeat the listing. A delay specified by the option value in seconds is inserted. METAD ~~~~~ | **metad** echo *text* Tests the WConf daemon by invoking its ``echo`` function. A given text is sent to Metad through RPC, echoed back by Metad and printed to the console. WCONFD ~~~~~~ | **wconfd** echo *text* Tests the WConf daemon by invoking its ``echo`` function. A given text is sent to WConfd through RPC, echoed back by WConfd and printed to the console. | **wconfd** cleanuplocks A request to clean up all stale locks is sent to WConfd. | **wconfd** listlocks *job-id* A request to list the locks owned by the given job id is sent to WConfd and the answer is displayed. | **wconfd** listalllocks A request to list all locks in use, directly or indirectly, is sent to WConfd and the answer is displayed. | **wconfd** listalllocks A request to list all locks in use, directly or indirectly, together with their respective direct owners is sent to WConfd and the answer is displayed. | **wconfd** flushconfig A request to ensure that the configuration is fully distributed to the master candidates. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-filter.rst000064400000000000000000000154151476477700300171720ustar00rootroot00000000000000gnt-filter(8) Ganeti | Version @GANETI_VERSION@ =============================================== Name ---- gnt-filter - Ganeti job filter rule administration Synopsis -------- **gnt-filter** {command} [options...] [arguments...] DESCRIPTION ----------- The **gnt-filter** command is used for managing job filter rules in the Ganeti system. Filter rules are used by the Ganeti job scheduler to determine which jobs should be accepted, rejected, paused or rate-limited. Filter rules consist of the following: - A ``UUID``, used to refer to existing filters. - A ``watermark``. This is the highest job id ever used, as valid in the moment when the filter was added or replaced. - A ``priority``. This is a non-negative integer. Filters are processed in order of increasing priority. While there is a well-defined order in which rules of the same priority are evaluated (increasing watermark, then the UUID, are taken as tie breakers), it is not recommended to have rules of the same priority that overlap and have different actions associated. - A list of ``predicates`` to be matched against the job. A predicate is a list, with the first element being the name of the predicate and the rest being parameters suitable for that predicate. Most predicates take a single parameter, which is a boolean expression formulated in the of the Ganeti query language. The currently supported predicate names are: - ``jobid``. Only parameter is a boolean expression. For this expression, there is only one field available, ``id``, which represents the id the job to be filtered. In all value positions, the string ``watermark`` is replaced by the value of the watermark of the filter rule. - ``opcode``. Only parameter is a boolean expression. For this expression, ``OP_ID`` and all other fields present in the opcode are available. This predicate will hold true, if the expression is true for at least one opcode in the job. - ``reason``. Only parameter is a boolean expression. For this expression, the three fields ``source``, ``reason``, ``timestamp`` of reason trail entries are available. This predicate is true, if one of the entries of one of the opcodes in this job satisfies the expression. - An ``action``. One of: - ACCEPT. The job will be accepted; no further filter rules are applied. - PAUSE. The job will be accepted to the queue and remain there; however, it is not executed. Has no effect if the job is already running. - REJECT. The job is rejected. If it is already in the queue, it will be cancelled. - CONTINUE. The filtering continues processing with the next rule. Such a rule will never have any direct or indirect effect, but it can serve as documentation for a "normally present, but currently disabled" rule. - RATE_LIMIT ``n``, where ``n`` is a positive integer. The job will be held in the queue while ``n`` or more jobs where this rule applies are running. Jobs already running when this rule is added are not changed. Logically, this rule is applied job by job sequentially, so that the number of jobs where this rule applies is limited to ``n`` once the jobs running at rule addition have finished. - A reason trail, in the same format as reason trails for job opcodes (see the ``--reason`` option in **ganeti**\(7)). This allows to find out which maintenance (or other reason) caused the addition of this filter rule. COMMANDS -------- ADD ~~~ | **add** | [\--priority=*priority*] | [\--predicates=*predicates*] | [\--action=*action*] Creates a new filter rule. A UUID is automatically assigned. The ``--priority`` option sets the priority of the filter. It is a non-negative integer. Default: 0 (the highest possible priority). The ``--predicates`` option sets the predicates of the filter. It is a list of predicates in the format described in the **DESCRIPTION** above. Default: [] (no predicate, filter always matches). The ``--action`` option sets the action of the filter. It is one of the strings ``ACCEPT``, ``PAUSE``, ``REJECT``, ``CONTINUE``, or ``RATE_LIMIT n`` (see the **DESCRIPTION** above). Default: ``CONTINUE``. See **ganeti**\(7) for a description of ``--reason`` and other common options. REPLACE ~~~~~~~ | **replace** | [\--priority=*priority*] | [\--predicates=*predicates*] | [\--action=*action*] | [\--reason=*reason*] | {*filter-uuid*} Replaces a filter rule, or creates one if it doesn't already exist. Accepts all options described above in ``ADD``. When being replaced, the filter will be assigned an updated watermark. See **ganeti**\(7) for a description of ``--reason`` and other common options. DELETE ~~~~~~ | **delete** {*filter-uuid*} Deletes the indicated filter rule. LIST ~~~~ | **list** [\--no-headers] [\--separator=*separator*] [-v] | [-o *[+]field,...*] [filter-uuid...] Lists all existing filters in the cluster. If no filter UUIDs are given, then all filters are included. Otherwise, only the given filters will be listed. The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The ``-v`` option activates verbose mode, which changes the display of special field states (see **ganeti**\(7)). The ``-o`` option takes a comma-separated list of output fields. If the value of the option starts with the character ``+``, the new fields will be added to the default list. This allows to quickly see the default list plus a few other fields, instead of retyping the entire list of fields. The available fields and their meaning are: @QUERY_FIELDS_FILTER@ LIST-FIELDS ~~~~~~~~~~~ **list-fields** [field...] List available fields for filters. INFO ~~~~ | **info** [filter-uuid...] Displays information about a given filter. EXAMPLES -------- Draining the queue. :: gnt-filter add '--predicates=[["jobid", [">", "id", "watermark"]]]' --action=REJECT Soft draining could be achieved by replacing ``REJECT`` by ``PAUSE`` in the above example. Pausing all new jobs not belonging to a specific maintenance. :: gnt-filter add --priority=0 '--predicates=[["reason", ["=~", "reason", "maintenance pink bunny"]]]' --action=ACCEPT gnt-filter add --priority=1 '--predicates=[["jobid", [">", "id", "watermark"]]]' --action=PAUSE Cancelling all queued instance creations and disallowing new such jobs. :: gnt-filter add '--predicates=[["opcode", ["=", "OP_ID", "OP_INSTANCE_CREATE"]]]' --action=REJECT Limiting the number of simultaneous instance disk replacements to 10 in order to throttle replication traffic. :: gnt-filter add '--predicates=[["opcode", ["=", "OP_ID", "OP_INSTANCE_REPLACE_DISKS"]]]' '--action=RATE_LIMIT 10' .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-group.rst000064400000000000000000000205441476477700300170400ustar00rootroot00000000000000gnt-group(8) Ganeti | Version @GANETI_VERSION@ ============================================== Name ---- gnt-group - Ganeti node-group administration Synopsis -------- **gnt-group** {command} [arguments...] DESCRIPTION ----------- The **gnt-group** command is used for node group administration in the Ganeti system. COMMANDS -------- ADD ~~~ | **add** [\--submit] [\--print-jobid] | [\--node-parameters=*NDPARAMS*] | [\--alloc-policy=*POLICY*] | [{-D|\--disk-parameters} *disk-template*:*disk-param*=*value*[,*disk-param*=*value*...]] | [\--ipolicy-bounds-specs *bound_ispecs*] | [\--ipolicy-disk-templates *template* [,*template*...]] | [\--ipolicy-spindle-ratio *ratio*] | [\--ipolicy-vcpu-ratio *ratio*] | [\--disk-state *diskstate*] | [\--hypervisor-state *hvstate*] | {*group-name*} Creates a new group with the given name. The node group will be initially empty; to add nodes to it, use ``gnt-group assign-nodes``. The ``--node-parameters`` option allows you to set default node parameters for nodes in the group. Please see **ganeti**\(7) for more information about supported key=value pairs and their corresponding options. The ``--alloc-policy`` option allows you to set an allocation policy for the group at creation time. Possible values are: unallocable nodes in the group should not be candidates for instance allocation, and the operation (e.g., instance creation) should fail if only groups in this state could be found to satisfy the requirements. last_resort nodes in the group should not be used for instance allocations, unless this would be the only way to have the operation succeed. preferred nodes in the group can be used freely for allocation of instances (this is the default). Note that prioritization among groups in this state will be deferred to the iallocator plugin that's being used. The ``-D (--disk-parameters)`` option allows you to set the disk parameters for the node group; please see the section about **gnt-cluster add** in **gnt-cluster**\(8) for more information about disk parameters The ``--ipolicy-...`` options specify instance policies on the node group, and are documented in the **gnt-cluster**\(8) man page. See **ganeti**\(7) for a description of ``--submit`` and other common options. ASSIGN-NODES ~~~~~~~~~~~~ | **assign-nodes** | [\--force] [\--submit] [\--print-jobid] | {*group-name*} {*node-name*...} Assigns one or more nodes to the specified group, moving them from their original group (or groups). By default, this command will refuse to proceed if the move would split between groups any instance that was not previously split (a split instance is an instance with a mirrored disk template, e.g. DRBD, that has the primary and secondary nodes in different node groups). You can force the operation with ``--force``. See **ganeti**\(7) for a description of ``--submit`` and other common options. MODIFY ~~~~~~ | **modify** [\--submit] [\--print-jobid] | [\--node-parameters=*NDPARAMS*] | [\--alloc-policy=*POLICY*] | [\--hypervisor-state *hvstate*] | [{-D|\--disk-parameters} *disk-template*:*disk-param*=*value*[,*disk-param*=*value*...]] | [\--disk-state *diskstate*] | [\--ipolicy-bounds-specs *bound_ispecs*] | [\--ipolicy-disk-templates *template* [,*template*...]] | [\--ipolicy-spindle-ratio *ratio*] | [\--ipolicy-vcpu-ratio *ratio*] | {*group*} Modifies some parameters from the node group. The ``--node-parameters`` and ``--alloc-policy`` options are documented in the **add** command above. ``--hypervisor-state`` as well as ``--disk-state`` are documented in detail in **ganeti**\(7). The ``--node-parameters``, ``--alloc-policy``, ``-D (--disk-parameters)`` options are documented in the **add** command above. The ``--ipolicy-...`` options specify instance policies on the node group, and are documented in the **gnt-cluster**\(8) man page. See **ganeti**\(7) for a description of ``--submit`` and other common options. REMOVE ~~~~~~ | **remove** [\--submit] [\--print-jobid] {*group*} Deletes the indicated node group, which must be empty. There must always be at least one group, so the last group cannot be removed. See **ganeti**\(7) for a description of ``--submit`` and other common options. LIST ~~~~ | **list** [\--no-headers] [\--separator=*SEPARATOR*] [-v] | [-o *[+]FIELD,...*] [\--filter] [*group-name*...] Lists all existing node groups in the cluster. The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The ``-v`` option activates verbose mode, which changes the display of special field states (see **ganeti**\(7)). The ``-o`` option takes a comma-separated list of output fields. If the value of the option starts with the character ``+``, the new fields will be added to the default list. This allows one to quickly see the default list plus a few other fields, instead of retyping the entire list of fields. The available fields and their meaning are: @QUERY_FIELDS_GROUP@ If exactly one argument is given and it appears to be a query filter (see **ganeti**\(7)), the query result is filtered accordingly. For ambiguous cases (e.g. a single field name as a filter) the ``--filter`` (``-F``) option forces the argument to be treated as a filter. If no group names are given, then all groups are included. Otherwise, only the named groups will be listed. LIST-FIELDS ~~~~~~~~~~~ **list-fields** [field...] List available fields for node groups. RENAME ~~~~~~ | **rename** [\--submit] [\--print-jobid] {*oldname*} {*newname*} Renames a given group from *oldname* to *newname*. See **ganeti**\(7) for a description of ``--submit`` and other common options. EVACUATE ~~~~~~~~ | **evacuate** [\--submit] [\--print-jobid] [\--sequential] [\--force-failover] | [\--iallocator *name*] [\--to *group*...] {*source-group*} This command will move all instances out of the given node group. Instances are placed in a new group by an iallocator, either given on the command line or as a cluster default. If no specific destination groups are specified using ``--to``, all groups except the evacuated group are considered. The moves of the individual instances are handled as separate jobs to allow for maximal parallelism. If the ``--sequential`` option is given, the moves of the individual instances will be executed sequentially. This can be useful if the link between the groups is vulnerable to congestion. If the ``--force-failover`` option is given, no migrations will be made. This might be necessary if the group being evacuated is too different from the other groups in the cluster. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-group evacuate -I hail --to rack4 rack1 Tags ~~~~ ADD-TAGS ^^^^^^^^ **add-tags** [\--from *file*] {*group*} {*tag*...} Add tags to the given node group. If any of the tags contains invalid characters, the entire operation will abort. If the ``--from`` option is given, the list of tags will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, both sources will be used). A file name of ``-`` will be interpreted as stdin. LIST-TAGS ^^^^^^^^^ **list-tags** {*group*} List the tags of the given node group. REMOVE-TAGS ^^^^^^^^^^^ **remove-tags** [\--from *file*] {*group*} {*tag*...} Remove tags from the given node group. If any of the tags are not existing on the node, the entire operation will abort. If the ``--from`` option is given, the list of tags to be removed will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, tags from both sources will be removed). A file name of ``-`` will be interpreted as stdin. INFO ~~~~ **info** [*group*...] Shows config information for all (or given) groups. SHOW-ISPECS-CMD ~~~~~~~~~~~~~~~ **show-ispecs-cmd** [\--include-defaults] [*group*...] Shows the command line that can be used to recreate the given groups (or all groups, if none is given) with the same options relative to specs in the instance policies. If ``--include-defaults`` is specified, include also the default values (i.e. the cluster-level settings), and not only the configuration items that a group overrides. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-instance.rst000064400000000000000000002575541476477700300175250ustar00rootroot00000000000000gnt-instance(8) Ganeti | Version @GANETI_VERSION@ ================================================= Name ---- gnt-instance - Ganeti instance administration Synopsis -------- **gnt-instance** {command} [arguments...] DESCRIPTION ----------- The **gnt-instance** command is used for instance administration in the Ganeti system. COMMANDS -------- Creation/removal/querying ~~~~~~~~~~~~~~~~~~~~~~~~~ ADD ^^^ | **add** | {-t|\--disk-template {diskless \| file \| plain \| drbd \| rbd}} | {\--disk=*N*: {size=*VAL*[,spindles=*VAL*] \| adopt=*LV*}[,options...] | \| {size=*VAL*,provider=*PROVIDER*}[,param=*value*... ][,options...] | \| {-s|\--os-size} *SIZE*} | [\--ip-check] [\--name-check] [\--no-conflicts-check] | [\--no-start] [\--no-install] [{\--forthcoming \| \--commit}] | [\--net=*N* [:options...] \| \--no-nics] | [{-B|\--backend-parameters} *BEPARAMS*] | [{-H|\--hypervisor-parameters} *HYPERVISOR* [: option=*value*... ]] | [{-O|\--os-parameters} *param*=*value*... ] | [--os-parameters-private *param*=*value*... ] | [--os-parameters-secret *param*=*value*... ] | [\--file-storage-dir *dir\_path*] [\--file-driver {loop \| blktap \| blktap2}] | {{-n|\--node} *node[:secondary-node]* \| {-I|\--iallocator} *name* \| {-g|--node-group} *nodegroup*} | {{-o|\--os-type} *os-type*} | [\--submit] [\--print-jobid] | [\--ignore-ipolicy] | [\--no-wait-for-sync] | [{-c|\--communication=yes|no}] | [{--tags [: tag:value... ] | {*instance-name*} Creates a new instance on the specified host. The *instance-name* argument must be in DNS, but depending on the bridge/routing setup, need not be in the same network as the nodes in the cluster. The ``disk`` option specifies the parameters for the disks of the instance. The numbering of disks starts at zero, and at least one disk needs to be passed. For each disk, either the size or the adoption source needs to be given. The size is interpreted (when no unit is given) in mebibytes. You can also use one of the suffixes *m*, *g* or *t* to specify the exact the units used; these suffixes map to mebibytes, gibibytes and tebibytes. Each disk can also take these parameters (all optional): spindles How many spindles (physical disks on the node) the disk should span. mode The access mode. Either ``ro`` (read-only) or the default ``rw`` (read-write). name This option specifies a name for the disk, which can be used as a disk identifier. An instance can not have two disks with the same name. vg The LVM volume group. This works only for LVM and DRBD devices. metavg This option specifies a different VG for the metadata device. This works only for DRBD devices. If not specified, the default metavg of the node-group (possibly inherited from the cluster-wide settings) will be used. access If 'userspace', instance will access this disk directly without going through a block device, avoiding expensive context switches with kernel space. This option works only for RBD, Gluster and ExtStorage devices. If not specified, the default access of the node-group (possibly inherited from the cluster-wide settings) will be used. When creating ExtStorage disks, also arbitrary parameters can be passed, to the ExtStorage provider. Those parameters are passed as additional comma separated options. Therefore, an ExtStorage disk provided by provider ``pvdr1`` with parameters ``param1``, ``param2`` would be passed as ``--disk 0:size=10G,provider=pvdr1,param1=val1,param2=val2``. When using the ``adopt`` key in the disk definition, Ganeti will reuse those volumes (instead of creating new ones) as the instance's disks. Ganeti will rename these volumes to the standard format, and (without installing the OS) will use them as-is for the instance. This allows migrating instances from non-managed mode (e.g. plain KVM with LVM) to being managed via Ganeti. Please note that this works only for the \`plain' disk template (see below for template details). Alternatively, a single-disk instance can be created via the ``-s`` option which takes a single argument, the size of the disk. This is similar to the Ganeti 1.2 version (but will only create one disk). The minimum disk specification is therefore ``--disk 0:size=20G`` (or ``-s 20G`` when using the ``-s`` option), and a three-disk instance can be specified as ``--disk 0:size=20G --disk 1:size=4G --disk 2:size=100G``. The minimum information needed to specify an ExtStorage disk are the ``size`` and the ``provider``. For example: ``--disk 0:size=20G,provider=pvdr1``. The ``--ip-check`` option checks that the instance's IP is not already alive (i.e. reachable from the master node). If you pass this option you must also pass the ``--name-check`` option, because the name check is used to compute the IP address via the resolver (e.g. in DNS or /etc/hosts, depending on your setup). The ``--name-check`` checks for the instance name via the resolver (e.g. in DNS or /etc/hosts, depending on your setup). The name check can be used to compute the IP address (when set to ``auto``). If you don't want the instance to automatically start after creation, this is possible via the ``--no-start`` option. This will leave the instance down until a subsequent **gnt-instance start** command. The NICs of the instances can be specified via the ``--net`` option. By default, one NIC is created for the instance, with a random MAC, and set up according to the cluster level NIC parameters. Each NIC can take these parameters (all optional): mac either a value or 'generate' to generate a new unique MAC ip specifies the IP address assigned to the instance from the Ganeti side (this is not necessarily what the instance will use, but what the node expects the instance to use). Note that if an IP in the range of a network configured with **gnt-network**\(8) is used, and the NIC is not already connected to it, this network has to be passed in the **network** parameter if this NIC is meant to be connected to the said network. ``--no-conflicts-check`` can be used to override this check. The special value **pool** causes Ganeti to select an IP from the network the NIC is or will be connected to. The special value **auto** must be combined with ``--name-check`` and will use the resolved IP. One can pick an externally reserved IP of a network along with ``--no-conflict-check``. Note that this IP cannot be assigned to any other instance until it gets released. mode specifies the connection mode for this NIC: routed, bridged or openvswitch. link in bridged or openvswitch mode specifies the interface to attach this NIC to, in routed mode it's intended to differentiate between different routing tables/instance groups (but the meaning is dependent on the network script, see **gnt-cluster**\(8) for more details). Note that openvswitch support is also hypervisor dependent. network derives the mode and the link from the settings of the network which is identified by its name. If the network option is chosen, link and mode must not be specified. Note that the mode and link depend on the network-to-nodegroup connection, thus allowing different nodegroups to be connected to the same network in different ways. name this option specifies a name for the NIC, which can be used as a NIC identifier. An instance can not have two NICs with the same name. vlan in bridged and openvswitch mode specifies the VLANs that the NIC will be connected to. To connect as an access port use ``n`` or ``.n`` with **n** being the VLAN ID. To connect as a trunk port use ``:n[:n]``. A hybrid port can be created with ``.n:n[:n]``. For bridged mode, the bridge needs VLAN filtering enabled. Assuming a bridge named **gnt-br**, this is accomplished by running ``ip link set dev gnt-br type bridge vlan_filtering 1``. Please make sure to only use VLAN IDs accepted by your network equipment (e.g. do not set the PVID) as this will break traffic flow otherwise. Of these "mode" and "link" are NIC parameters, and inherit their default at cluster level. Alternatively, if no network is desired for the instance, you can prevent the default of one NIC with the ``--no-nics`` option. The ``-o (--os-type)`` option specifies the operating system to be installed. The available operating systems can be listed with **gnt-os list**. Passing ``--no-install`` will however skip the OS installation, allowing a manual import if so desired. Note that the no-installation mode will automatically disable the start-up of the instance (without an OS, it most likely won't be able to start-up successfully). Passing the ``--forthcoming`` option, Ganeti will not at all try to create the instance or its disks. Instead the instance will only be added to the configuration, so that the resources are reserved. If the ``--commit`` option is passed, then it is a prerequisite that an instance with that name has already been added to the configuration as a forthcoming instance and the request is to replace this instance by the newly created real one. Note that if the reason for reserving an instance is that DNS names still need to be propagated, the reservation has to be done without ``--name-check`` and ``--ip-check``. The ``-B (--backend-parameters)`` option specifies the backend parameters for the instance. If no such parameters are specified, the values are inherited from the cluster. Possible parameters are: maxmem the maximum memory size of the instance; as usual, suffixes can be used to denote the unit, otherwise the value is taken in mebibytes minmem the minimum memory size of the instance; as usual, suffixes can be used to denote the unit, otherwise the value is taken in mebibytes vcpus the number of VCPUs to assign to the instance (if this value makes sense for the hypervisor) auto\_balance whether the instance is considered in the N+1 cluster checks (enough redundancy in the cluster to survive a node failure) always\_failover ``True`` or ``False``, whether the instance must be failed over (shut down and rebooted) always or it may be migrated (briefly suspended) Note that before 2.6 Ganeti had a ``memory`` parameter, which was the only value of memory an instance could have. With the ``maxmem``/``minmem`` change Ganeti guarantees that at least the minimum memory is always available for an instance, but allows more memory to be used (up to the maximum memory) should it be free. The ``-H (--hypervisor-parameters)`` option specified the hypervisor to use for the instance (must be one of the enabled hypervisors on the cluster) and optionally custom parameters for this instance. If not other options are used (i.e. the invocation is just -H *NAME*) the instance will inherit the cluster options. The defaults below show the cluster defaults at cluster creation time. The possible hypervisor options are as follows: boot\_order Valid for the Xen HVM and KVM hypervisors. A string value denoting the boot order. This has different meaning for the Xen HVM hypervisor and for the KVM one. For Xen HVM, The boot order is a string of letters listing the boot devices, with valid device letters being: a floppy drive c hard disk d CDROM drive n network boot (PXE) The default is not to set an HVM boot order, which is interpreted as 'dc'. For KVM the boot order is either "floppy", "cdrom", "disk" or "network". Please note that older versions of KVM couldn't netboot from virtio interfaces. This has been fixed in more recent versions and is confirmed to work at least with qemu-kvm 0.11.1. Also note that if you have set the ``kernel_path`` option, that will be used for booting, and this setting will be silently ignored. blockdev\_prefix Valid for the Xen HVM and PVM hypervisors. Relevant to non-pvops guest kernels, in which the disk device names are given by the host. Allows one to specify 'xvd', which helps run Red Hat based installers, driven by anaconda. floppy\_image\_path Valid for the KVM hypervisor. The path to a floppy disk image to attach to the instance. This is useful to install Windows operating systems on Virt/IO disks because you can specify here the floppy for the drivers at installation time. cdrom\_image\_path Valid for the Xen HVM and KVM hypervisors. The path to a CDROM image to attach to the instance. cdrom2\_image\_path Valid for the KVM hypervisor. The path to a second CDROM image to attach to the instance. **NOTE**: This image can't be used to boot the system. To do that you have to use the 'cdrom\_image\_path' option. nic\_type Valid for the Xen HVM and KVM hypervisors. This parameter determines the way the network cards are presented to the instance. The possible options are: - rtl8139 (default for Xen HVM) (HVM & KVM) - ne2k\_isa (HVM & KVM) - ne2k\_pci (HVM & KVM) - i82551 (KVM) - i82557b (KVM) - i82559er (KVM) - pcnet (KVM) - e1000 (KVM) - paravirtual (default for KVM) (HVM & KVM) vif\_type Valid for the Xen HVM hypervisor. This parameter specifies the vif type of the nic configuration of the instance. Unsetting the value leads to no type being specified in the configuration. Note that this parameter only takes effect when the 'nic_type' is not set. The possible options are: - ioemu - vif scsi\_controller\_type Valid for the KVM hypervisor. This parameter specifies which type of SCSI controller to use. The possible options are: - lsi [default] - megasas - virtio-scsi-pci kvm\_pci\_reservations Valid for the KVM hypervisor. The number of PCI slots that QEMU will manage implicitly. By default Ganeti will let QEMU use the first 12 slots (i.e. PCI slots 0-11) on its own and will start adding disks and NICs from the 13rd slot (i.e. PCI slot 12) onwards. So by default one can add 20 PCI devices (32 - 12). To support more than that, this hypervisor parameter should be set accordingly (e.g. to 8). disk\_type Valid for the Xen HVM and KVM hypervisors. This parameter determines the way the disks are presented to the instance. The possible options are: - ioemu [default] (HVM & KVM) - paravirtual (HVM & KVM) - ide (KVM) - scsi (KVM) - sd (KVM) - mtd (KVM) - pflash (KVM) cdrom\_disk\_type Valid for the KVM hypervisor. This parameter determines the way the cdroms disks are presented to the instance. The default behavior is to get the same value of the earlier parameter (disk_type). The possible options are: - paravirtual - ide - scsi - sd - mtd - pflash vnc\_bind\_address Valid for the Xen HVM and KVM hypervisors. Specifies the address that the VNC listener for this instance should bind to. Valid values are IPv4 addresses. Use the address 0.0.0.0 to bind to all available interfaces (this is the default) or specify the address of one of the interfaces on the node to restrict listening to that interface. vnc\_password\_file Valid for the Xen HVM and KVM hypervisors. Specifies the location of the file containing the password for connections using VNC. The default is a file named vnc-cluster-password which can be found in the configuration directory. vnc\_tls Valid for the KVM hypervisor. A boolean option that controls whether the VNC connection is secured with TLS. vnc\_x509\_path Valid for the KVM hypervisor. If ``vnc_tls`` is enabled, this options specifies the path to the x509 certificate to use. vnc\_x509\_verify Valid for the KVM hypervisor. spice\_bind Valid for the KVM hypervisor. Specifies the address or interface on which the SPICE server will listen. Valid values are: - IPv4 addresses, including 0.0.0.0 and 127.0.0.1 - IPv6 addresses, including :: and ::1 - names of network interfaces If a network interface is specified, the SPICE server will be bound to one of the addresses of that interface. spice\_ip\_version Valid for the KVM hypervisor. Specifies which version of the IP protocol should be used by the SPICE server. It is mainly intended to be used for specifying what kind of IP addresses should be used if a network interface with both IPv4 and IPv6 addresses is specified via the ``spice_bind`` parameter. In this case, if the ``spice_ip_version`` parameter is not used, the default IP version of the cluster will be used. spice\_password\_file Valid for the KVM hypervisor. Specifies a file containing the password that must be used when connecting via the SPICE protocol. If the option is not specified, passwordless connections are allowed. spice\_image\_compression Valid for the KVM hypervisor. Configures the SPICE lossless image compression. Valid values are: - auto_glz - auto_lz - quic - glz - lz - off spice\_jpeg\_wan\_compression Valid for the KVM hypervisor. Configures how SPICE should use the jpeg algorithm for lossy image compression on slow links. Valid values are: - auto - never - always spice\_zlib\_glz\_wan\_compression Valid for the KVM hypervisor. Configures how SPICE should use the zlib-glz algorithm for lossy image compression on slow links. Valid values are: - auto - never - always spice\_streaming\_video Valid for the KVM hypervisor. Configures how SPICE should detect video streams. Valid values are: - off - all - filter spice\_playback\_compression Valid for the KVM hypervisor. Configures whether SPICE should compress audio streams or not. spice\_use\_tls Valid for the KVM hypervisor. Specifies that the SPICE server must use TLS to encrypt all the traffic with the client. spice\_tls\_ciphers Valid for the KVM hypervisor. Specifies a list of comma-separated ciphers that SPICE should use for TLS connections. For the format, see man **cipher**\(1). spice\_use\_vdagent Valid for the KVM hypervisor. Enables or disables passing mouse events via SPICE vdagent. cpu\_type Valid for the KVM hypervisor. This parameter determines the emulated cpu for the instance. If this parameter is empty (which is the default configuration), it will not be passed to KVM which defaults to the emulated 'qemu64' type. Be aware of setting this parameter to ``"host"`` if you have nodes with mixed CPU models. Live migration may stop working or crash the instance in this situation. If you leave this parameter unset or use one of the generic types (e.g. ``"qemu32"``, ``"qemu64"``, ``"kvm32"`` or ``"kvm64"``) your instances will not be able to benefit from advanced CPU instructions such as AES-NI or RDRAND. Please be aware these generic CPU types also lack security features such as mitigations to the Meltdown and Spectre family of processor vulnerabilities. You can query for supported CPU types by running the QEMU/KVM binary with the following parameter: .. code-block:: bash qemu-system-x86_64 -cpu ? More information can be found in the Qemu / KVM CPU model configuration documentation. Please check there for the recommended settings. acpi Valid for the Xen HVM and KVM hypervisors. A boolean option that specifies if the hypervisor should enable ACPI support for this instance. By default, ACPI is disabled. ACPI should be enabled for user shutdown detection. See ``user_shutdown``. pae Valid for the Xen HVM and KVM hypervisors. A boolean option that specifies if the hypervisor should enable PAE support for this instance. The default is false, disabling PAE support. viridian Valid for the Xen HVM hypervisor. A boolean option that specifies if the hypervisor should enable viridian (Hyper-V) for this instance. The default is false, disabling viridian support. use\_guest\_agent Valid for the KVM hypervisor. A boolean option that specifies if the hypervisor should enable the QEMU Guest Agent protocol for this instance. By default, the Guest Agent is disabled. use\_localtime Valid for the Xen HVM and KVM hypervisors. A boolean option that specifies if the instance should be started with its clock set to the localtime of the machine (when true) or to the UTC (When false). The default is false, which is useful for Linux/Unix machines; for Windows OSes, it is recommended to enable this parameter. kernel\_path Valid for the Xen PVM and KVM hypervisors. This option specifies the path (on the node) to the kernel to boot the instance with. Xen PVM instances always require this, while for KVM if this option is empty, it will cause the machine to load the kernel from its disks (and the boot will be done accordingly to ``boot_order``). kernel\_args Valid for the Xen PVM and KVM hypervisors. This options specifies extra arguments to the kernel that will be loaded. This is always used for Xen PVM, while for KVM it is only used if the ``kernel_path`` option is also specified. The default setting for this value is simply ``"ro"``, which mounts the root disk (initially) in read-only one. For example, setting this to single will cause the instance to start in single-user mode. Note that the hypervisor setting ``serial_console`` appends ``"console=ttyS0,"`` to the end of ``kernel_args`` in KVM. initrd\_path Valid for the Xen PVM and KVM hypervisors. This option specifies the path (on the node) to the initrd to boot the instance with. Xen PVM instances can use this always, while for KVM if this option is only used if the ``kernel_path`` option is also specified. You can pass here either an absolute filename (the path to the initrd) if you want to use an initrd, or use the format no\_initrd\_path for no initrd. root\_path Valid for the Xen PVM and KVM hypervisors. This options specifies the name of the root device. This is always needed for Xen PVM, while for KVM it is only used if the ``kernel_path`` option is also specified. Please note, that if this setting is an empty string and the hypervisor is Xen it will not be written to the Xen configuration file serial\_console Valid for the KVM hypervisor. This boolean option specifies whether to emulate a serial console for the instance. Note that some versions of KVM have a bug that will make an instance hang when configured to use the serial console unless a connection is made to it within about 2 seconds of the instance's startup. For such case it's recommended to disable this option, which is enabled by default. Enabling serial console emulation also appends ``"console=ttyS0,"`` to the end of ``kernel_args`` in KVM and may interfere with previous settings. serial\_speed Valid for the KVM hypervisor. This integer option specifies the speed of the serial console. Common values are 9600, 19200, 38400, 57600 and 115200: choose the one which works on your system. (The default is 38400 for historical reasons, but newer versions of kvm/qemu work with 115200) disk\_cache Valid for the KVM hypervisor. The disk cache mode. It controls QEMUs internal 'cache.writeback', 'cache.direct' and 'cache.no-flush' settings, based on the table found in man **qemu-system-x86_64**\(1). It supports three cache modes: *none* (for direct I/O), *writethrough* (to use the host cache but report completion to the guest only when the host has committed the changes to disk) or *writeback* (to use the host cache and report completion as soon as the data is in the host cache). The value 'default' has been kept in Ganeti for backwards compatibility and corresponds to *writeback*. Note that certain disk templates in Ganeti may override the cache mode to allow for safe live migrations. disk\_aio Valid for the KVM hypervisor. This parameter controls the way in which KVM interactions with the kernel I/O subsystem. It defaults to *threads* for backwards compatibility, but on recent installations *native* or *io\_uring* would be the recommended way. For *io\_uring* to work, QEMU 5.0 and Linux kernel 5.1 are the minimum requirements. disk\_discard Valid for the KVM hypervisor. discard is one of *ignore* or *unmap* and controls whether discard (also known as trim or unmap) requests are ignored or passed to the filesystem. Some machine types may not support discard requests. security\_model Valid for the KVM hypervisor. The security model for kvm. Currently one of *none*, *user* or *pool*. Under *none*, the default, nothing is done and instances are run as the Ganeti daemon user (normally root). Under *user* kvm will drop privileges and become the user specified by the security\_domain parameter. Under *pool* a global cluster pool of users will be used, making sure no two instances share the same user on the same node. (this mode is not implemented yet) security\_domain Valid for the KVM hypervisor. Under security model *user* the username to run the instance under. It must be a valid username existing on the host. Cannot be set under security model *none* or *pool*. kvm\_flag Valid for the KVM hypervisor. If *enabled* accel=kvm is appended to the -machine parameter *Disabling* (or not setting this flag at all) currently does nothing. mem\_path Valid for the KVM hypervisor. This option passes the -mem-path argument to kvm with the path (on the node) to the mount point of the hugetlbfs file system, along with the -mem-prealloc argument too. use\_chroot Valid for the KVM hypervisor. This boolean option determines whether to run the KVM instance in a chroot directory. If it is set to ``true``, an empty directory is created before starting the instance and its path is passed via the -chroot flag to kvm. The directory is removed when the instance is stopped. It is set to ``false`` by default. user\_shutdown Valid for the KVM hypervisor. This boolean option determines whether the KVM instance supports user shutdown detection. This option does not necessarily require ACPI enabled, but ACPI must be enabled for users to poweroff their KVM instances. If it is set to ``true``, the user can shutdown this KVM instance and its status is reported as ``USER_down``. It is set to ``false`` by default. migration\_downtime Valid for the KVM hypervisor. The maximum amount of time (in ms) a KVM instance is allowed to be frozen during a live migration, in order to copy dirty memory pages. Default value is 30ms, but you may need to increase this value for busy instances. This option is only effective with kvm versions >= 87 and qemu-kvm versions >= 0.11.0. cpu\_mask Valid for the Xen, KVM and LXC hypervisors. The processes belonging to the given instance are only scheduled on the specified CPUs. The format of the mask can be given in three forms. First, the word "all", which signifies the common case where all VCPUs can live on any CPU, based on the hypervisor's decisions. Second, a comma-separated list of CPU IDs or CPU ID ranges. The ranges are defined by a lower and higher boundary, separated by a dash, and the boundaries are inclusive. In this form, all VCPUs of the instance will be mapped on the selected list of CPUs. Example: ``0-2,5``, mapping all VCPUs (no matter how many) onto physical CPUs 0, 1, 2 and 5. The last form is used for explicit control of VCPU-CPU pinnings. In this form, the list of VCPU mappings is given as a colon (:) separated list, whose elements are the possible values for the second or first form above. In this form, the number of elements in the colon-separated list _must_ equal the number of VCPUs of the instance. Example: .. code-block:: bash # Map the entire instance to CPUs 0-2 gnt-instance modify -H cpu_mask=0-2 my-inst # Map vCPU 0 to physical CPU 1 and vCPU 1 to CPU 3 (assuming 2 vCPUs) gnt-instance modify -H cpu_mask=1:3 my-inst # Pin vCPU 0 to CPUs 1 or 2, and vCPU 1 to any CPU gnt-instance modify -H cpu_mask=1-2:all my-inst # Pin vCPU 0 to any CPU, vCPU 1 to CPUs 1, 3, 4 or 5, and CPU 2 to # CPU 0 (backslashes for escaping the comma) gnt-instance modify -H cpu_mask=all:1\\,3-5:0 my-inst # Pin entire VM to CPU 0 gnt-instance modify -H cpu_mask=0 my-inst # Turn off CPU pinning (default setting) gnt-instance modify -H cpu_mask=all my-inst cpu\_cap Valid for the Xen hypervisor. Set the maximum amount of cpu usage by the VM. The value is a percentage between 0 and (100 * number of VCPUs). Default cap is 0: unlimited. cpu\_weight Valid for the Xen hypervisor. Set the cpu time ratio to be allocated to the VM. Valid values are between 1 and 65535. Default weight is 256. usb\_mouse Valid for the KVM hypervisor. This option specifies the usb mouse type to be used. It can be "mouse" or "tablet". When using VNC it's recommended to set it to "tablet". keymap Valid for the KVM hypervisor. This option specifies the keyboard mapping to be used. It is only needed when using the VNC console. For example: "fr" or "en-gb". reboot\_behavior Valid for Xen PVM, Xen HVM and KVM hypervisors. Normally if an instance reboots, the hypervisor will restart it. If this option is set to ``exit``, the hypervisor will treat a reboot as a shutdown instead. It is set to ``reboot`` by default. cpu\_cores Valid for the KVM hypervisor. Number of emulated CPU cores. cpu\_threads Valid for the KVM hypervisor. Number of emulated CPU threads. cpu\_sockets Valid for the KVM hypervisor. Number of emulated CPU sockets. soundhw Valid for Xen PVM, Xen HVM and KVM hypervisors. The soundcard to emulate inside your instance. Please consult Qemu (``-audio model=help``) for a list of valid soundcard models. ``hda`` or ``ac97`` are probably the most useful ones. cpuid Valid for the XEN hypervisor. Modify the values returned by CPUID_ instructions run within instances. This allows you to enable migration between nodes with different CPU attributes like cores, threads, hyperthreading or SS4 support by hiding the extra features where needed. See the XEN documentation for syntax and more information. .. _CPUID: http://en.wikipedia.org/wiki/CPUID usb\_devices Valid for the KVM hypervisor. Space separated list of usb devices. These can be emulated devices or passthrough ones, and each one gets passed to kvm with its own ``-usbdevice`` option. See the **qemu**\(1) manpage for the syntax of the possible components. Note that values set with this parameter are split on a space character and currently don't support quoting. For backwards compatibility reasons, the RAPI interface keeps accepting comma separated lists too. vga Valid for the KVM hypervisor. Emulated vga mode, passed the the kvm -vga option. kvm\_extra Valid for the KVM hypervisor. Any other option to the KVM hypervisor, useful tweaking anything that Ganeti doesn't support. Values with a space in the parameter must be be double quoted. machine\_version Valid for the KVM hypervisor. Use in case an instance must be booted with an exact type of machine version (due to e.g. outdated drivers). In case it's not set the default version supported by your version of kvm is used. Ganeti currently only supports the ``pc`` types properly. However, you should not set it to ``pc`` but rather to a specific version, e.g. ``pc-i440fx-9.2`` if you plan to use live migration. You can obtain the default machine version by running: .. code-block:: bash kvm -M ? Look for the line containing (default). Setting it just to ``pc`` will cause problems during rolling cluster upgrades, because your instance will live-migrate into a different kvm version which has a different understanding of what exactly "pc" is. migration\_caps Valid for the KVM hypervisor. Enable specific migration capabilities by providing a ":" separated list of supported capabilities. QEMU version 1.7.0 defines x-rdma-pin-all, auto-converge, zero-blocks, and xbzrle. QEMU version 2.5 defines x-postcopy-ram and 2.6 renames this to postcopy-ram. If x-postcopy-ram or postcopy-ram are enabled, Ganeti will automatically move a migration to postcopy mode after a dirty_sync_count of 2. Other than normal live migration, that can recover from dying destination node, it is not possible to recover from dying source node during active postcopy migration. Please note that while a combination of xbzrle and auto-converge might speed up the migration process significantly, the first may cause BSOD on Windows8r2 instances running on drbd. kvm\_path Valid for the KVM hypervisor. Path to the userspace KVM (or qemu) program. vhost\_net Valid for the KVM hypervisor. This boolean option determines whether the tap devices used by the KVM paravirtual nics (virtio-net) will use accelerated data plane, passing network packets directly between host and guest kernel, without going through userspace emulation layer (qemu). Historically it is set to ``false`` by default. New Clusters created with Ganeti-3.1 and newer defaults to ``true``. Everyone is encouraged to enable it. vnet\_hdr Valid for the KVM hypervisor. This boolean option determines whether the tap devices used by the KVM paravirtual nics (virtio-net) will get created with VNET_HDR (IFF_VNET_HDR) support. If set to false, it effectively disables offloading on the virio-net interfaces, which prevents host kernel tainting and log flooding, when dealing with broken or malicious virtio-net drivers. It is set to ``true`` by default. virtio\_net\_queues Valid for the KVM hypervisor. Set a number of queues (file descriptors) for tap device to parallelize packets sending or receiving. Tap devices will be created with MULTI_QUEUE (IFF_MULTI_QUEUE) support. This only works with KVM paravirtual nics (virtio-net) and the maximum number of queues is limited to ``8``. Technically this is an extension of ``vnet_hdr`` which must be enabled for multiqueue support. If set to ``1`` queue, it effectively disables multiqueue support on the tap and virio-net devices. For instances it is necessary to manually set number of queues (on Linux using: ``ethtool -L ethX combined $queues``). It is set to ``1`` by default. startup\_timeout Valid for the LXC hypervisor. This integer option specifies the number of seconds to wait for the state of an LXC container changes to "RUNNING" after startup, as reported by lxc-wait. Otherwise we assume an error has occurred and report it. It is set to ``30`` by default. extra\_cgroups Valid for the LXC hypervisor. This option specifies the list of cgroup subsystems that will be mounted alongside the needed ones before starting LXC containers. Since LXC version >= 1.0.0, LXC strictly requires all cgroup subsystems to be mounted before starting a container. Users can control the list of desired cgroup subsystems for LXC containers by specifying the lxc.cgroup.use parameter in the LXC system configuration file(see: **lxc.system.conf**\(5)). Its default value is "@kernel" which means all cgroup kernel subsystems. The LXC hypervisor of Ganeti ensures that all cgroup subsystems needed to start an LXC container are mounted, as well as the subsystems specified in this parameter. The needed subsystems are currently ``cpuset``, ``memory``, ``devices``, and ``cpuacct``. The value of this parameter should be a list of cgroup subsystems separated by a comma(e.g., "net_cls,perf_event,blkio"). If this parameter is not specified, a list of subsystems will be taken from /proc/cgroups instead. drop\_capabilities Valid for the LXC hypervisor. This option specifies the list of capabilities which should be dropped for a LXC container. Each value of this option must be in the same form as the lxc.cap.drop configuration parameter of **lxc.container.conf**\(5). It is the lower case of the capability name without the "CAP\_" prefix (e.g., "sys_module,sys_time"). See **capabilities**\(7) for more details about Linux capabilities. Note that some capabilities are required by the LXC container (see: **lxc.container.conf**\(5)). Also note that the CAP_SYS_BOOT is required(should not be dropped) to perform the soft reboot for the LXC container. The default value is ``mac_override,sys_boot,sys_module,sys_time``. devices Valid for the LXC hypervisor. This option specifies the list of devices that can be accessed from inside of the LXC container. Each value of this option must have the same form as the lxc.cgroup.devices.allow configuration parameter of **lxc.container.conf**\(5). It consists of the type(a: all, b: block, c: character), the major-minor pair, and the access type sequence(r: read, w: write, m: mknod), e.g. "c 1:3 rw". If you'd like to allow the LXC container to access /dev/null and /dev/zero with read-write access, you can set this parameter to: "c 1:3 rw,c 1:5 rw". The LXC hypervisor drops all direct device access by default, so if you want to allow the LXC container to access an additional device which is not included in the default value of this parameter, you have to set this parameter manually. By default, this parameter contains (/dev/null, /dev/zero, /dev/full, /dev/random, /dev/urandom, /dev/aio, /dev/tty, /dev/console, /dev/ptmx and first block of Unix98 PTY slaves) with read-write(rw) access. extra\_config Valid for the LXC hypervisor. This option specifies the list of extra config parameters which are not supported by the Ganeti LXC hypervisor natively. Each value of this option must be a valid line of the LXC container config file(see: **lxc.container.conf**\(5)). This parameter is not set by default. num_ttys Valid for the LXC hypervisor. This option specifies the number of ttys(actually ptys) that should be allocated for the LXC container. You can disable pty devices allocation for the LXC container by setting this parameter to 0, but you can't use **gnt-instance console** in this case. It is set to ``6`` by default. The ``-O (--os-parameters)`` option allows customisation of the OS parameters. The actual parameter names and values depend on the OS being used, but the syntax is the same key=value. For example, setting a hypothetical ``dhcp`` parameter to yes can be achieved by:: gnt-instance add -O dhcp=yes ... You can also specify OS parameters that should not be logged but reused at the next reinstall with ``--os-parameters-private`` and OS parameters that should not be logged or saved to configuration with ``--os-parameters-secret``. Bear in mind that: * Launching the daemons in debug mode will cause debug logging to happen, which leaks private and secret parameters to the log files. Do not use the debug mode in production. Daemons will emit a warning on startup if they are in debug mode. * You will have to pass again all ``--os-parameters-secret`` parameters should you want to reinstall this instance. The ``-I (--iallocator)`` option specifies the instance allocator plugin to use (``.`` means the default allocator). If you pass in this option the allocator will select nodes for this instance automatically, so you don't need to pass them with the ``-n`` option. For more information please refer to the instance allocator documentation. The ``-g (--node-group)`` option can be used to create the instance in a particular node group, specified by name. The ``-t (--disk-template)`` options specifies the disk layout type for the instance. If no disk template is specified, the default disk template is used. The default disk template is the first in the list of enabled disk templates, which can be adjusted cluster-wide with ``gnt-cluster modify``. The available choices for disk templates are: diskless This creates an instance with no disks. Its useful for testing only (or other special cases). file Disk devices will be regular files. sharedfile Disk devices will be regular files on a shared directory. plain Disk devices will be logical volumes. drbd Disk devices will be drbd (version 8.x) on top of lvm volumes. rbd Disk devices will be rbd volumes residing inside a RADOS cluster. blockdev Disk devices will be adopted pre-existent block devices. ext Disk devices will be provided by external shared storage, through the ExtStorage Interface using ExtStorage providers. The optional second value of the ``-n (--node)`` is used for the drbd template type and specifies the remote node. If you do not want gnt-instance to wait for the disk mirror to be synced, use the ``--no-wait-for-sync`` option. The ``--file-storage-dir`` specifies the relative path under the cluster-wide file storage directory to store file-based disks. It is useful for having different subdirectories for different instances. The full path of the directory where the disk files are stored will consist of cluster-wide file storage directory + optional subdirectory + instance name. This option is only relevant for instances using the file storage backend. The ``--file-driver`` specifies the driver to use for file-based disks. Note that currently these drivers work with the xen hypervisor only. This option is only relevant for instances using the file storage backend. The available choices are: loop Kernel loopback driver. This driver uses loopback devices to access the filesystem within the file. However, running I/O intensive applications in your instance using the loop driver might result in slowdowns. Furthermore, if you use the loopback driver consider increasing the maximum amount of loopback devices (on most systems it's 8) using the max\_loop param. blktap The blktap driver (for Xen hypervisors). In order to be able to use the blktap driver you should check if the 'blktapctrl' user space disk agent is running (usually automatically started via xend). This user-level disk I/O interface has the advantage of better performance. Especially if you use a network file system (e.g. NFS) to store your instances this is the recommended choice. blktap2 Analogous to the blktap driver, but used by newer versions of Xen. If ``--ignore-ipolicy`` is given any instance policy violations occurring during this operation are ignored. The ``-c`` and ``--communication`` specify whether to enable/disable instance communication, which is a communication mechanism between the instance and the host. The ``--tags`` allows tags to be applied to the instance, typically to influence the allocator. Multiple tags can be separated with commas. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-instance add -t file --disk 0:size=30g -B maxmem=512 -o debian-etch \ -n node1.example.com --file-storage-dir=mysubdir instance1.example.com # gnt-instance add -t plain --disk 0:size=30g -B maxmem=1024,minmem=512 \ -o debian-etch -n node1.example.com instance1.example.com # gnt-instance add -t plain --disk 0:size=30g --disk 1:size=100g,vg=san \ -B maxmem=512 -o debian-etch -n node1.example.com instance1.example.com # gnt-instance add -t drbd --disk 0:size=30g -B maxmem=512 -o debian-etch \ -n node1.example.com:node2.example.com instance2.example.com # gnt-instance add -t rbd --disk 0:size=30g -B maxmem=512 -o debian-etch \ -n node1.example.com instance1.example.com # gnt-instance add -t ext --disk 0:size=30g,provider=pvdr1 -B maxmem=512 \ -o debian-etch -n node1.example.com instance1.example.com # gnt-instance add -t ext --disk 0:size=30g,provider=pvdr1,param1=val1 \ --disk 1:size=40g,provider=pvdr2,param2=val2,param3=val3 -B maxmem=512 \ -o debian-etch -n node1.example.com instance1.example.com BATCH-CREATE ^^^^^^^^^^^^ | **batch-create** | [{-I|\--iallocator} *instance allocator*] | {instances\_file.json} This command (similar to the Ganeti 1.2 **batcher** tool) submits multiple instance creation jobs based on a definition file. This file can contain all options which are valid when adding an instance with the exception of the ``iallocator`` field. The IAllocator is, for optimization purposes, only allowed to be set for the whole batch operation using the ``--iallocator`` parameter. The instance file must be a valid-formed JSON file, containing an array of dictionaries with instance creation parameters. All parameters (except ``iallocator``) which are valid for the instance creation OP code are allowed. The most important ones are: instance\_name The FQDN of the new instance. disk\_template The disk template to use for the instance, the same as in the **add** command. disks Array of disk specifications. Each entry describes one disk as a dictionary of disk parameters. beparams A dictionary of backend parameters. hypervisor The hypervisor for the instance. hvparams A dictionary with the hypervisor options. If not passed, the default hypervisor options will be inherited. nics List of NICs that will be created for the instance. Each entry should be a dict, with mac, ip, mode and link as possible keys. Please don't provide the "mac, ip, mode, link" parent keys if you use this method for specifying NICs. pnode, snode The primary and optionally the secondary node to use for the instance (in case an iallocator script is not used). If those parameters are given, they have to be given consistently for all instances in the batch operation. start whether to start the instance ip\_check Skip the check for already-in-use instance; see the description in the **add** command for details. name\_check Skip the name check for instances; see the description in the **add** command for details. file\_storage\_dir, file\_driver Configuration for the file disk type, see the **add** command for details. A simple definition for one instance can be (with most of the parameters taken from the cluster defaults):: [ { "mode": "create", "instance_name": "instance1.example.com", "disk_template": "drbd", "os_type": "debootstrap", "disks": [{"size":"1024"}], "nics": [{}], "hypervisor": "xen-pvm" }, { "mode": "create", "instance_name": "instance2.example.com", "disk_template": "drbd", "os_type": "debootstrap", "disks": [{"size":"4096", "mode": "rw", "vg": "xenvg"}], "nics": [{}], "hypervisor": "xen-hvm", "hvparams": {"acpi": true}, "beparams": {"maxmem": 512, "minmem": 256} } ] The command will display the job id for each submitted instance, as follows:: # gnt-instance batch-create instances.json Submitted jobs 37, 38 Note: If the allocator is used for computing suitable nodes for the instances, it will only take into account disk information for the default disk template. That means, even if other disk templates are specified for the instances, storage space information of these disk templates will not be considered in the allocation computation. REMOVE ^^^^^^ | **remove** [\--ignore-failures] [\--shutdown-timeout=*N*] [\--submit] | [\--print-jobid] [\--force] {*instance-name*} Remove an instance. This will remove all data from the instance and there is *no way back*. If you are not sure if you use an instance again, use **shutdown** first and leave it in the shutdown state for a while. The ``--ignore-failures`` option will cause the removal to proceed even in the presence of errors during the removal of the instance (e.g. during the shutdown or the disk removal). If this option is not given, the command will stop at the first error. The ``--shutdown-timeout`` is used to specify how much time (in seconds) to wait before forcing the shutdown (e.g. ``xl destroy`` in Xen, killing the kvm process for KVM, etc.). By default two minutes are given to each instance to stop. The ``--force`` option is used to skip the interactive confirmation. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-instance remove instance1.example.com LIST ^^^^ | **list** | [\--no-headers] [\--separator=*SEPARATOR*] [\--units=*UNITS*] [-v] | [{-o|\--output} *[+]FIELD,...*] [\--filter] [*instance-name*...] Shows the currently configured instances with memory usage, disk usage, the node they are running on, and their run status. The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The units used to display the numeric values in the output varies, depending on the options given. By default, the values will be formatted in the most appropriate unit. If the ``--separator`` option is given, then the values are shown in mebibytes to allow parsing by scripts. In both cases, the ``--units`` option can be used to enforce a given output unit. The ``-v`` option activates verbose mode, which changes the display of special field states (see **ganeti**\(7)). The ``-o (--output)`` option takes a comma-separated list of output fields. The available fields and their meaning are: @QUERY_FIELDS_INSTANCE@ If the value of the option starts with the character ``+``, the new field(s) will be added to the default list. This allows one to quickly see the default list plus a few other fields, instead of retyping the entire list of fields. There is a subtle grouping about the available output fields: all fields except for ``oper_state``, ``oper_ram``, ``oper_vcpus`` and ``status`` are configuration value and not run-time values. So if you don't select any of the these fields, the query will be satisfied instantly from the cluster configuration, without having to ask the remote nodes for the data. This can be helpful for big clusters when you only want some data and it makes sense to specify a reduced set of output fields. If exactly one argument is given and it appears to be a query filter (see **ganeti**\(7)), the query result is filtered accordingly. For ambiguous cases (e.g. a single field name as a filter) the ``--filter`` (``-F``) option forces the argument to be treated as a filter (e.g. ``gnt-instance list -F admin_state``). The default output field list is: ``name``, ``os``, ``pnode``, ``admin_state``, ``oper_state``, ``oper_ram``. LIST-FIELDS ^^^^^^^^^^^ **list-fields** [field...] Lists available fields for instances. INFO ^^^^ **info** [-s \| \--static] [\--roman] {\--all \| *instance-name*} Show detailed information about the given instance(s). This is different from **list** as it shows detailed data about the instance's disks (especially useful for the drbd disk template). If the option ``-s`` is used, only information available in the configuration file is returned, without querying nodes, making the operation faster. Use the ``--all`` to get info about all instances, rather than explicitly passing the ones you're interested in. The ``--roman`` option can be used to cause envy among people who like ancient cultures, but are stuck with non-latin-friendly cluster virtualization technologies. MODIFY ^^^^^^ | **modify** | [{-H|\--hypervisor-parameters} *HYPERVISOR\_PARAMETERS*] | [{-B|\--backend-parameters} *BACKEND\_PARAMETERS*] | [{-m|\--runtime-memory} *SIZE*] | [\--net add[:options...] \| | \--net [*N*:]add[,options...] \| | \--net [*ID*:]remove \| | \--net *ID*:modify[,options...]] | [\--disk add:size=*SIZE*[,options...] \| | \--disk *N*:add,size=*SIZE*[,options...] \| | \--disk *N*:add,size=*SIZE*,provider=*PROVIDER*[,options...][,param=*value*... ] \| | \--disk *N*:attach,{name=*NAME* | uuid=*UUID*}\| | \--disk *ID*:modify[,options...] | \--disk [*ID*:]remove] | \--disk [*ID*:]detach] | [\{-t|\--disk-template} { plain | rbd } \| | \{-t|\--disk-template} drbd -n *new_secondary*] [\--no-wait-for-sync] \| | \{-t|\--disk-template} ext {-e|--ext-params} {provider=*PROVIDER*}[,param=*value*... ] \| | \{-t|\--disk-template} { file | sharedfile | gluster } | \| [--file-storage-dir dir_path] [--file-driver {loop | blktap | blktap2}] | [\--new-primary=*node*] | [\--os-type=*OS* [\--force-variant]] | [{-O|\--os-parameters} *param*=*value*... ] | [--os-parameters-private *param*=*value*... ] | [\--offline \| \--online] | [\--submit] [\--print-jobid] | [\--ignore-ipolicy] | [\--no-hotplug] | {*instance-name*} Modifies the memory size, number of vcpus, ip address, MAC address and/or NIC parameters for an instance. It can also add and remove disks and NICs to/from the instance. Note that you need to give at least one of the arguments, otherwise the command complains. The ``-H (--hypervisor-parameters)``, ``-B (--backend-parameters)`` and ``-O (--os-parameters)`` options specifies hypervisor, backend and OS parameter options in the form of name=value[,...]. For details which options can be specified, see the **add** command. The ``-t (--disk-template)`` option will change the disk template of the instance. Currently, conversions between all the available templates are supported, except the ``diskless`` and the ``blockdev`` templates. For the ``blockdev`` disk template, only partial support is provided and acts only as a source template. Since these volumes are adopted pre-existent block devices, conversions targeting this template are not supported. Also, there is no support for conversions to or from the ``diskless`` template. The instance must be stopped before attempting the conversion. When changing from the plain to the drbd disk template, a new secondary node must be specified via the ``-n`` option. The option ``--no-wait-for-sync`` can be used when converting to the ``drbd`` template in order to make the instance available for startup before DRBD has finished resyncing. When changing to a file-based disk template, i.e., ``file``, ``sharedfile`` and ``gluster``, the file storage directory and the file driver can be specified via the ``--file-storage-dir`` and ``--file-driver`` options, respectively. For more details on these options please refer to the **add** command section. When changing to an ``ext`` template, the provider's name must be specified. Also, arbitrary parameters can be passed, as additional comma separated options. Those parameters along with the ExtStorage provider must be passed using either the ``--ext-params`` or ``-e`` option. It is not allowed specifying existing disk parameters such as the size, mode, name, access, adopt, vg, metavg, provider, or spindles options. The ``-m (--runtime-memory)`` option will change an instance's runtime memory to the given size (in MB if a different suffix is not specified), by ballooning it up or down to the new value. The ``--disk add:size=*SIZE*,[options..]`` option adds a disk to the instance, and ``--disk *N*:add,size=*SIZE*,[options..]`` will add a disk to the instance at a specific index. The available options are the same as in the **add** command (``spindles``, ``mode``, ``name``, ``vg``, ``metavg`` and ``access``). By default, gnt-instance waits for the disk mirror to sync. If you do not want this behavior, use the ``--no-wait-for-sync`` option. When adding an ExtStorage disk, the ``provider=*PROVIDER*`` option is also mandatory and specifies the ExtStorage provider. Also, for ExtStorage disks arbitrary parameters can be passed as additional comma separated options, same as in the **add** command. The ``--disk attach:name=*NAME*`` option attaches an existing disk to the instance at the last disk index and ``--disk *N*:attach,name=*NAME*`` will attach a disk to the instance at a specific index. The accepted disk identifiers are its ``name`` or ``uuid``. The ``--disk remove`` option will remove the last disk of the instance. Use ``--disk `` *ID*``:remove`` to remove a disk by its identifier. *ID* can be the index of the disk, the disks's name or the disks's UUID. The above apply also to the ``--disk detach`` option, which removes a disk from an instance but keeps it in the configuration and doesn't destroy it. The ``--disk *ID*:modify[,options...]`` will change the options of the disk. Available options are: mode The access mode. Either ``ro`` (read-only) or the default ``rw`` (read-write). name This option specifies a name for the disk, which can be used as a disk identifier. An instance can not have two disks with the same name. The ``--net *N*:add[,options..]`` will add a new network interface to the instance. The available options are the same as in the **add** command (``mac``, ``ip``, ``link``, ``mode``, ``network``). The ``--net *ID*,remove`` will remove the instances' NIC with *ID* identifier, which can be the index of the NIC, the NIC's name or the NIC's UUID. The ``--net *ID*:modify[,options..]`` option will change the parameters of the instance network interface with the *ID* identifier. The option ``-o (--os-type)`` will change the OS name for the instance (without reinstallation). In case an OS variant is specified that is not found, then by default the modification is refused, unless ``--force-variant`` is passed. An invalid OS will also be refused, unless the ``--force`` option is given. The option ``--new-primary`` will set the new primary node of an instance assuming the disks have already been moved manually. Unless the ``--force`` option is given, it is verified that the instance is no longer running on its current primary node. The ``--online`` and ``--offline`` options are used to transition an instance into and out of the ``offline`` state. An instance can be turned offline only if it was previously down. The ``--online`` option fails if the instance was not in the ``offline`` state, otherwise it changes instance's state to ``down``. These modifications take effect immediately. If ``--ignore-ipolicy`` is given any instance policy violations occurring during this operation are ignored. If ``--no-hotplug`` is given any disk and NIC modifications will not be hot-plugged. The change will take place after the reboot. Without the ``--no-hotplug`` parameter, Ganeti attempts to perform the operation hot if possible. Hotplug is currently supported only for disk and NIC modifications in the KVM hypervisor. If hotplug fails (for any reason) a warning is printed but execution is continued. For existing NIC modification interactive verification is needed unless ``--force`` option is passed. See **ganeti**\(7) for a description of ``--submit`` and other common options. Most of the changes take effect at the next restart. If the instance is running, there is no effect on the instance. REINSTALL ^^^^^^^^^ | **reinstall** [{-o|\--os-type} *os-type*] [\--select-os] [-f *force*] | [\--force-multiple] | [\--instance \| \--node \| \--primary \| \--secondary \| \--all] | [{-O|\--os-parameters} *OS\_PARAMETERS*] | [--os-parameters-private} *OS\_PARAMETERS*] | [--os-parameters-secret} *OS\_PARAMETERS*] | [\--submit] [\--print-jobid] | {*instance*...} Reinstalls the operating system on the given instance(s). The instance(s) must be stopped when running this command. If the ``-o (--os-type)`` is specified, the operating system is changed. The ``--select-os`` option switches to an interactive OS reinstall. The user is prompted to select the OS template from the list of available OS templates. OS parameters can be overridden using ``-O (--os-parameters)`` (more documentation for this option under the **add** command). Since this is a potentially dangerous command, the user will be required to confirm this action, unless the ``-f`` flag is passed. When multiple instances are selected (either by passing multiple arguments or by using the ``--node``, ``--primary``, ``--secondary`` or ``--all`` options), the user must pass the ``--force-multiple`` options to skip the interactive confirmation. See **ganeti**\(7) for a description of ``--submit`` and other common options. RENAME ^^^^^^ | **rename** [\--ip-check] [\--name-check] | [\--submit] [\--print-jobid] | {*instance*} {*new_name*} Renames the given instance. The instance must be stopped when running this command. The ``--name-check`` checks for the new instance name via the resolver (e.g. in DNS or /etc/hosts, depending on your setup) and that the resolved name matches the new name. In addition the ``--ip-check`` can be used to prevent duplicate IPs, by checking that the new name's IP is not already alive (i.e. reachable from the master node). Note that you can rename an instance to its same name, to force re-executing the os-specific rename script for that instance, if needed. See **ganeti**\(7) for a description of ``--submit`` and other common options. Starting/stopping/connecting to console ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ STARTUP ^^^^^^^ | **startup** | [\--force] [\--ignore-offline] | [\--force-multiple] [\--no-remember] | [\--instance \| \--node \| \--primary \| \--secondary \| \--all \| | \--tags \| \--node-tags \| \--pri-node-tags \| \--sec-node-tags] | [{-H|\--hypervisor-parameters} ``key=value...``] | [{-B|\--backend-parameters} ``key=value...``] | [\--submit] [\--print-jobid] [\--paused] | {*instance*...} Starts one or more instances, depending on the following options. The four available modes are: \--instance will start the instances given as arguments (at least one argument required); this is the default selection \--node will start the instances who have the given node as either primary or secondary \--primary will start all instances whose primary node is in the list of nodes passed as arguments (at least one node required) \--secondary will start all instances whose secondary node is in the list of nodes passed as arguments (at least one node required) \--all will start all instances in the cluster (no arguments accepted) \--tags will start all instances in the cluster with the tags given as arguments \--node-tags will start all instances in the cluster on nodes with the tags given as arguments \--pri-node-tags will start all instances in the cluster on primary nodes with the tags given as arguments \--sec-node-tags will start all instances in the cluster on secondary nodes with the tags given as arguments Note that although you can pass more than one selection option, the last one wins, so in order to guarantee the desired result, don't pass more than one such option. Use ``--force`` to start even if secondary disks are failing. ``--ignore-offline`` can be used to ignore offline primary nodes and mark the instance as started even if the primary is not available. The ``--force-multiple`` will skip the interactive confirmation in the case the more than one instance will be affected. The ``--no-remember`` option will perform the startup but not change the state of the instance in the configuration file (if it was stopped before, Ganeti will still think it needs to be stopped). This can be used for testing, or for a one shot-start where you don't want the watcher to restart the instance if it crashes. The ``-H (--hypervisor-parameters)`` and ``-B (--backend-parameters)`` options specify temporary hypervisor and backend parameters that can be used to start an instance with modified parameters. They can be useful for quick testing without having to modify an instance back and forth, e.g.:: # gnt-instance start -H kernel_args="single" instance1 # gnt-instance start -B maxmem=2048 instance2 The first form will start the instance instance1 in single-user mode, and the instance instance2 with 2GB of RAM (this time only, unless that is the actual instance memory size already). Note that the values override the instance parameters (and not extend them): an instance with "kernel\_args=ro" when started with -H kernel\_args=single will result in "single", not "ro single". The ``--paused`` option is only valid for Xen and kvm hypervisors. This pauses the instance at the start of bootup, awaiting ``gnt-instance console`` to unpause it, allowing the entire boot process to be monitored for debugging. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-instance start instance1.example.com # gnt-instance start --node node1.example.com node2.example.com # gnt-instance start --all SHUTDOWN ^^^^^^^^ | **shutdown** | [\--timeout=*N*] | [\--force] [\--force-multiple] [\--ignore-offline] [\--no-remember] | [\--instance \| \--node \| \--primary \| \--secondary \| \--all \| | \--tags \| \--node-tags \| \--pri-node-tags \| \--sec-node-tags] | [\--submit] [\--print-jobid] | {*instance*...} Stops one or more instances. If the instance cannot be cleanly stopped during a hardcoded interval (currently 2 minutes), it will forcibly stop the instance (equivalent to switching off the power on a physical machine). The ``--timeout`` is used to specify how much time (in seconds) to wait before forcing the shutdown (e.g. ``xl destroy`` in Xen, killing the kvm process for KVM, etc.). By default two minutes are given to each instance to stop. The ``--instance``, ``--node``, ``--primary``, ``--secondary``, ``--all``, ``--tags``, ``--node-tags``, ``--pri-node-tags`` and ``--sec-node-tags`` options are similar as for the **startup** command and they influence the actual instances being shutdown. ``--ignore-offline`` can be used to ignore offline primary nodes and force the instance to be marked as stopped. This option should be used with care as it can lead to an inconsistent cluster state. Use ``--force`` to be able to shutdown an instance even when it's marked as offline. This is useful if an offline instance ends up in the ``ERROR_up`` state, for example. The ``--no-remember`` option will perform the shutdown but not change the state of the instance in the configuration file (if it was running before, Ganeti will still thinks it needs to be running). This can be useful for a cluster-wide shutdown, where some instances are marked as up and some as down, and you don't want to change the running state: you just need to disable the watcher, shutdown all instances with ``--no-remember``, and when the watcher is activated again it will restore the correct runtime state for all instances. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-instance shutdown instance1.example.com # gnt-instance shutdown --all REBOOT ^^^^^^ | **reboot** | [{-t|\--type} *REBOOT-TYPE*] | [\--ignore-secondaries] | [\--shutdown-timeout=*N*] | [\--force-multiple] | [\--instance \| \--node \| \--primary \| \--secondary \| \--all \| | \--tags \| \--node-tags \| \--pri-node-tags \| \--sec-node-tags] | [\--submit] [\--print-jobid] | [*instance*...] Reboots one or more instances. The type of reboot depends on the value of ``-t (--type)``. A soft reboot does a hypervisor reboot, a hard reboot does a instance stop, recreates the hypervisor config for the instance and starts the instance. A full reboot does the equivalent of **gnt-instance shutdown && gnt-instance startup**. The default is hard reboot. For the hard reboot the option ``--ignore-secondaries`` ignores errors for the secondary node while re-assembling the instance disks. The ``--instance``, ``--node``, ``--primary``, ``--secondary``, ``--all``, ``--tags``, ``--node-tags``, ``--pri-node-tags`` and ``--sec-node-tags`` options are similar as for the **startup** command and they influence the actual instances being rebooted. The ``--shutdown-timeout`` is used to specify how much time (in seconds) to wait before forcing the shutdown (xl destroy in xen, killing the kvm process, for kvm). By default two minutes are given to each instance to stop. The ``--force-multiple`` will skip the interactive confirmation in the case the more than one instance will be affected. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-instance reboot instance1.example.com # gnt-instance reboot --type=full instance1.example.com CONSOLE ^^^^^^^ **console** [\--show-cmd] {*instance*} Connects to the console of the given instance. If the instance is not up, an error is returned. Use the ``--show-cmd`` option to display the command instead of executing it. For HVM instances, this will attempt to connect to the serial console of the instance. To connect to the virtualized "physical" console of a HVM instance, use a VNC client with the connection info from the **info** command. For Xen/kvm instances, if the instance is paused, this attempts to unpause the instance after waiting a few seconds for the connection to the console to be made. Example:: # gnt-instance console instance1.example.com Disk management ~~~~~~~~~~~~~~~ REPLACE-DISKS ^^^^^^^^^^^^^ | **replace-disks** [\--submit] [\--print-jobid] [\--early-release] | [\--ignore-ipolicy] {-p} [\--disks *idx*] {*instance-name*} | **replace-disks** [\--submit] [\--print-jobid] [\--early-release] | [\--ignore-ipolicy] {-s} [\--disks *idx*] {*instance-name*} | **replace-disks** [\--submit] [\--print-jobid] [\--early-release] | [\--ignore-ipolicy] | {{-I\|\--iallocator} *name* \| {{-n|\--new-secondary} *node* } | {*instance-name*} | **replace-disks** [\--submit] [\--print-jobid] [\--early-release] | [\--ignore-ipolicy] {-a\|\--auto} {*instance-name*} This command is a generalized form for replacing disks. It is currently only valid for the mirrored (DRBD) disk template. The first form (when passing the ``-p`` option) will replace the disks on the primary, while the second form (when passing the ``-s`` option will replace the disks on the secondary node. For these two cases (as the node doesn't change), it is possible to only run the replace for a subset of the disks, using the option ``--disks`` which takes a list of comma-delimited disk indices (zero-based), e.g. 0,2 to replace only the first and third disks. The third form (when passing either the ``--iallocator`` or the ``--new-secondary`` option) is designed to change secondary node of the instance. Specifying ``--iallocator`` makes the new secondary be selected automatically by the specified allocator plugin (use ``.`` to indicate the default allocator), otherwise the new secondary node will be the one chosen manually via the ``--new-secondary`` option. Note that it is not possible to select an offline or drained node as a new secondary. The fourth form (when using ``--auto``) will automatically determine which disks of an instance are faulty and replace them within the same node. The ``--auto`` option works only when an instance has only faulty disks on either the primary or secondary node; it doesn't work when both sides have faulty disks. The ``--early-release`` changes the code so that the old storage on secondary node(s) is removed early (before the resync is completed) and the internal Ganeti locks for the current (and new, if any) secondary node are also released, thus allowing more parallelism in the cluster operation. This should be used only when recovering from a disk failure on the current secondary (thus the old storage is already broken) or when the storage on the primary node is known to be fine (thus we won't need the old storage for potential recovery). The ``--ignore-ipolicy`` let the command ignore instance policy violations if replace-disks changes groups and the instance would violate the new groups instance policy. See **ganeti**\(7) for a description of ``--submit`` and other common options. ACTIVATE-DISKS ^^^^^^^^^^^^^^ | **activate-disks** [\--submit] [\--print-jobid] [\--ignore-size] | [\--wait-for-sync] {*instance-name*} Activates the block devices of the given instance. If successful, the command will show the location and name of the block devices:: node1.example.com:disk/0:/dev/drbd0 node1.example.com:disk/1:/dev/drbd1 In this example, *node1.example.com* is the name of the node on which the devices have been activated. The *disk/0* and *disk/1* are the Ganeti-names of the instance disks; how they are visible inside the instance is hypervisor-specific. */dev/drbd0* and */dev/drbd1* are the actual block devices as visible on the node. The ``--ignore-size`` option can be used to activate disks ignoring the currently configured size in Ganeti. This can be used in cases where the configuration has gotten out of sync with the real-world (e.g. after a partially-failed grow-disk operation or due to rounding in LVM devices). This should not be used in normal cases, but only when activate-disks fails without it. The ``--wait-for-sync`` option will ensure that the command returns only after the instance's disks are synchronised (mostly for DRBD); this can be useful to ensure consistency, as otherwise there are no commands that can wait until synchronisation is done. However when passing this option, the command will have additional output, making it harder to parse the disk information. Note that it is safe to run this command while the instance is already running. See **ganeti**\(7) for a description of ``--submit`` and other common options. DEACTIVATE-DISKS ^^^^^^^^^^^^^^^^ **deactivate-disks** [-f] [\--submit] [\--print-jobid] {*instance-name*} De-activates the block devices of the given instance. Note that if you run this command for an instance with a drbd disk template, while it is running, it will not be able to shutdown the block devices on the primary node, but it will shutdown the block devices on the secondary nodes, thus breaking the replication. The ``-f``/``--force`` option will skip checks that the instance is down; in case the hypervisor is confused and we can't talk to it, normally Ganeti will refuse to deactivate the disks, but with this option passed it will skip this check and directly try to deactivate the disks. This can still fail due to the instance actually running or other issues. See **ganeti**\(7) for a description of ``--submit`` and other common options. GROW-DISK ^^^^^^^^^ | **grow-disk** [\--no-wait-for-sync] [\--submit] [\--print-jobid] | [\--absolute] | {*instance-name*} {*disk*} {*amount*} Grows an instance's disk. This is only possible for instances having a plain, drbd, file, sharedfile, rbd or ext disk template. For the ext template to work, the ExtStorage provider should also support growing. This means having a ``grow`` script that actually grows the volume of the external shared storage. Note that this command only change the block device size; it will not grow the actual filesystems, partitions, etc. that live on that disk. Usually, you will need to: #. use **gnt-instance grow-disk** #. reboot the instance (later, at a convenient time) #. use a filesystem resizer, such as **ext2online**\(8) or **xfs\_growfs**\(8) to resize the filesystem, or use **fdisk**\(8) to change the partition table on the disk The *disk* argument is the index of the instance disk to grow. The *amount* argument is given as a number which can have a suffix (like the disk size in instance create); if the suffix is missing, the value will be interpreted as mebibytes. By default, the *amount* value represents the desired increase in the disk size (e.g. an amount of 1G will take a disk of size 3G to 4G). If the optional ``--absolute`` parameter is passed, then the *amount* argument doesn't represent the delta, but instead the desired final disk size (e.g. an amount of 8G will take a disk of size 4G to 8G). For instances with a drbd template, note that the disk grow operation might complete on one node but fail on the other; this will leave the instance with different-sized LVs on the two nodes, but this will not create problems (except for unused space). If you do not want gnt-instance to wait for the new disk region to be synced, use the ``--no-wait-for-sync`` option. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example (increase the first disk for instance1 by 16GiB):: # gnt-instance grow-disk instance1.example.com 0 16g Example for increasing the disk size to a certain size:: # gnt-instance grow-disk --absolute instance1.example.com 0 32g Also note that disk shrinking is not supported; use **gnt-backup export** and then **gnt-backup import** to reduce the disk size of an instance. RECREATE-DISKS ^^^^^^^^^^^^^^ | **recreate-disks** [\--submit] [\--print-jobid] | [{-n node1:[node2] \| {-I\|\--iallocator *name*}}] | [\--disk=*N*[:[size=*VAL*][,spindles=*VAL*][,mode=*ro\|rw*]]] | {*instance-name*} Recreates all or a subset of disks of the given instance. Note that this functionality should only be used for missing disks; if any of the given disks already exists, the operation will fail. While this is suboptimal, recreate-disks should hopefully not be needed in normal operation and as such the impact of this is low. If only a subset should be recreated, any number of ``disk`` options can be specified. It expects a disk index and an optional list of disk parameters to change. Only ``size``, ``spindles``, and ``mode`` can be changed while recreating disks. To recreate all disks while changing parameters on a subset only, a ``--disk`` option must be given for every disk of the instance. Optionally the instance's disks can be recreated on different nodes. This can be useful if, for example, the original nodes of the instance have gone down (and are marked offline), so we can't recreate on the same nodes. To do this, pass the new node(s) via ``-n`` option, with a syntax similar to the **add** command. The number of nodes passed must equal the number of nodes that the instance currently has. Note that changing nodes is only allowed when all disks are replaced, e.g. when no ``--disk`` option is passed. Another method of choosing which nodes to place the instance on is by using the specified iallocator, passing the ``--iallocator`` option. The primary and secondary nodes will be chosen by the specified iallocator plugin, or by the default allocator if ``.`` is specified. See **ganeti**\(7) for a description of ``--submit`` and other common options. Recovery/moving ~~~~~~~~~~~~~~~ FAILOVER ^^^^^^^^ | **failover** [-f] [\--ignore-consistency] [\--ignore-ipolicy] | [\--shutdown-timeout=*N*] | [{-n|\--target-node} *node* \| {-I|\--iallocator} *name*] | [\--cleanup] | [\--submit] [\--print-jobid] | {*instance-name*} Failover will stop the instance (if running), change its primary node, and if it was originally running it will start it again (on the new primary). This works for instances with drbd template (in which case you can only fail to the secondary node) and for externally mirrored templates (sharedfile, blockdev, rbd and ext) (in which case you can fail to any other node). If the instance's disk template is of type sharedfile, blockdev, rbd or ext, then you can explicitly specify the target node (which can be any node) using the ``-n`` or ``--target-node`` option, or specify an iallocator plugin using the ``-I`` or ``--iallocator`` option. If you omit both, the default iallocator will be used to specify the target node. If the instance's disk template is of type drbd, the target node is automatically selected as the drbd's secondary node. Changing the secondary node is possible with a replace-disks operation. Normally the failover will check the consistency of the disks before failing over the instance. If you are trying to migrate instances off a dead node, this will fail. Use the ``--ignore-consistency`` option for this purpose. Note that this option can be dangerous as errors in shutting down the instance will be ignored, resulting in possibly having the instance running on two machines in parallel (on disconnected DRBD drives). This flag requires the source node to be marked offline first to succeed. The ``--shutdown-timeout`` is used to specify how much time (in seconds) to wait before forcing the shutdown (xl destroy in xen, killing the kvm process, for kvm). By default two minutes are given to each instance to stop. If ``--ignore-ipolicy`` is given any instance policy violations occurring during this operation are ignored. If the ``--cleanup`` option is passed, the operation changes from performing a failover to attempting recovery from a failed previous failover. In this mode, Ganeti checks if the instance runs on the correct node (and updates its configuration if not) and ensures the instance's disks are configured correctly. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-instance failover instance1.example.com For externally mirrored templates also ``-n`` is available:: # gnt-instance failover -n node3.example.com instance1.example.com MIGRATE ^^^^^^^ | **migrate** [-f] [\--allow-failover] [\--non-live] | [\--migration-mode=live\|non-live] [\--ignore-ipolicy] [\--ignore-hvversions] | [\--no-runtime-changes] [\--submit] [\--print-jobid] | [{-n|\--target-node} *node* \| {-I|\--iallocator} *name*] {*instance-name*} | **migrate** [-f] \--cleanup [\--submit] [\--print-jobid] {*instance-name*} Migrate will move the instance to its secondary node without shutdown. As with failover, it works for instances having the drbd disk template or an externally mirrored disk template type such as sharedfile, blockdev, rbd or ext. If the instance's disk template is of type sharedfile, blockdev, rbd or ext, then you can explicitly specify the target node (which can be any node) using the ``-n`` or ``--target-node`` option, or specify an iallocator plugin using the ``-I`` or ``--iallocator`` option. If you omit both, the default iallocator will be used to specify the target node. Alternatively, the default iallocator can be requested by specifying ``.`` as the name of the plugin. If the instance's disk template is of type drbd, the target node is automatically selected as the drbd's secondary node. Changing the secondary node is possible with a replace-disks operation. The migration command needs a perfectly healthy instance for drbd instances, as we rely on the dual-master capability of drbd8 and the disks of the instance are not allowed to be degraded. The ``--non-live`` and ``--migration-mode=non-live`` options will switch (for the hypervisors that support it) between a "fully live" (i.e. the interruption is as minimal as possible) migration and one in which the instance is frozen, its state saved and transported to the remote node, and then resumed there. This all depends on the hypervisor support for two different methods. In any case, it is not an error to pass this parameter (it will just be ignored if the hypervisor doesn't support it). The option ``--migration-mode=live`` option will request a fully-live migration. The default, when neither option is passed, depends on the hypervisor parameters (and can be viewed with the **gnt-cluster info** command). If the ``--cleanup`` option is passed, the operation changes from migration to attempting recovery from a failed previous migration. In this mode, Ganeti checks if the instance runs on the correct node (and updates its configuration if not) and ensures the instances' disks are configured correctly. In this mode, the ``--non-live`` option is ignored. The option ``-f`` will skip the prompting for confirmation. If ``--allow-failover`` is specified it tries to fallback to failover if it already can determine that a migration won't work (e.g. if the instance is shut down). Please note that the fallback will not happen during execution. If a migration fails during execution it still fails. If ``--ignore-ipolicy`` is given any instance policy violations occurring during this operation are ignored. Normally, Ganeti will verify that the hypervisor versions on source and target are compatible and error out if they are not. If ``--ignore-hvversions`` is given, Ganeti will only warn in this case. The ``--no-runtime-changes`` option forbids migrate to alter an instance's runtime before migrating it (eg. ballooning an instance down because the target node doesn't have enough available memory). If an instance has the backend parameter ``always_failover`` set to true, then the migration is automatically converted into a failover. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example (and expected output):: # gnt-instance migrate instance1 Instance instance1 will be migrated. Note that migration might impact the instance if anything goes wrong (e.g. due to bugs in the hypervisor). Continue? y/[n]/?: y Migrating instance instance1.example.com * checking disk consistency between source and target * switching node node2.example.com to secondary mode * changing into standalone mode * changing disks into dual-master mode * wait until resync is done * preparing node2.example.com to accept the instance * migrating instance to node2.example.com * switching node node1.example.com to secondary mode * wait until resync is done * changing into standalone mode * changing disks into single-master mode * wait until resync is done * done # MOVE ^^^^ | **move** [-f] [\--ignore-consistency] | [-n *node*] [\--compress=*compression-mode*] [\--shutdown-timeout=*N*] | [\--submit] [\--print-jobid] [\--ignore-ipolicy] | {*instance-name*} Move will move the instance to an arbitrary node in the cluster. This works only for instances having a plain or file disk template. Note that since this operation is done via data copy, it will take a long time for big disks (similar to replace-disks for a drbd instance). The ``--compress`` option is used to specify which compression mode is used during the move. Valid values are 'none' (the default) and any values specified in the 'compression_tools' cluster parameter. The ``--shutdown-timeout`` is used to specify how much time (in seconds) to wait before forcing the shutdown (e.g. ``xl destroy`` in XEN, killing the kvm process for KVM, etc.). By default two minutes are given to each instance to stop. The ``--ignore-consistency`` option will make Ganeti ignore any errors in trying to shutdown the instance on its node; useful if the hypervisor is broken and you want to recover the data. If ``--ignore-ipolicy`` is given any instance policy violations occurring during this operation are ignored. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-instance move -n node3.example.com instance1.example.com CHANGE-GROUP ^^^^^^^^^^^^ | **change-group** [\--submit] [\--print-jobid] | [\--iallocator *name*] [\--to *group*...] {*instance-name*} This command moves an instance to another node group. The move is calculated by an iallocator, either given on the command line or as a cluster default. Note that the iallocator does only consider disk information of the default disk template, even if the instances' disk templates differ from that. If no specific destination groups are specified using ``--to``, all groups except the one containing the instance are considered. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-instance change-group -I hail --to rack2 inst1.example.com Tags ~~~~ ADD-TAGS ^^^^^^^^ **add-tags** [\--from *file*] {*instance-name*} {*tag*...} Add tags to the given instance. If any of the tags contains invalid characters, the entire operation will abort. If the ``--from`` option is given, the list of tags will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, both sources will be used). A file name of ``-`` will be interpreted as stdin. LIST-TAGS ^^^^^^^^^ **list-tags** {*instance-name*} List the tags of the given instance. REMOVE-TAGS ^^^^^^^^^^^ **remove-tags** [\--from *file*] {*instance-name*} {*tag*...} Remove tags from the given instance. If any of the tags are not existing on the node, the entire operation will abort. If the ``--from`` option is given, the list of tags to be removed will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, tags from both sources will be removed). A file name of ``-`` will be interpreted as stdin. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-job.rst000064400000000000000000000075051476477700300164600ustar00rootroot00000000000000gnt-job(8) Ganeti | Version @GANETI_VERSION@ ============================================ Name ---- gnt-job - Job commands Synopsis -------- **gnt-job** {command} [arguments...] DESCRIPTION ----------- The **gnt-job** is used for examining and manipulating the job queue. COMMANDS -------- ARCHIVE ~~~~~~~ **archive** {job-id...} This command can be used to archive job by their IDs. Only jobs that have finished execution (i.e either *success*, *error* or *canceled* jobs). AUTOARCHIVE ~~~~~~~~~~~ **autoarchive** {*age* | ``all``} Archive jobs by their age. This command can archive jobs older than *age* seconds, or alternatively all finished jobs can be archived if the string all is passed. CANCEL ~~~~~~ | **cancel** | {[\--force] [\--kill] {\--pending | \--queued | \--waiting} | | *job-id* ...} Cancel the job(s) identified by the given *job id*. Only jobs that have not yet started to run can be canceled; that is, jobs in either the *queued* or *waiting* state. To skip a confirmation, pass ``--force``. ``--queued`` and ``waiting`` can be used to cancel all jobs in the respective state, ``--pending`` includes both. If the ``--kill`` option is given, jobs will be killed, even if in *running* state, using SIGKILL in the latter case. This is dangerous, as the job will not have the chance to do any clean up; so it will most likely leave any objects it touched in an inconsistent state. CHANGE-PRIORITY ~~~~~~~~~~~~~~~ | **change-priority** \--priority {low | normal | high} | {[\--force] {\--pending | \--queued | \--waiting} | *job-id*...} Changes the priority of one or multiple pending jobs. Jobs currently running have only the priority of remaining opcodes changed. ``--priority`` must be specified. ``--queued`` and ``waiting`` can be used to re-prioritize all jobs in the respective state, ``--pending`` includes both. To skip a confirmation, pass ``--force``. INFO ~~~~ **info** {*job-id*...} Show detailed information about the given job id(s). If no job id is given, all jobs are examined (warning, this is a lot of information). LIST ~~~~ | **list** [\--no-headers] [\--separator=*SEPARATOR*] | [-o *[+]FIELD,...*] [\--filter] [job-id...] Lists the jobs and their status. By default, the job id, job status, and a small job description is listed, but additional parameters can be selected. The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The ``-o`` option takes a comma-separated list of output fields. The available fields and their meaning are: @QUERY_FIELDS_JOB@ If the value of the option starts with the character ``+``, the new fields will be added to the default list. This allows one to quickly see the default list plus a few other fields, instead of retyping the entire list of fields. To include archived jobs in the list the ``--archived`` option can be used. The following options can be used to show only specific jobs: ``--pending`` Show only jobs pending execution. ``--running`` Show jobs currently running only. ``--error`` Show failed jobs only. ``--finished`` Show finished jobs only. If exactly one argument is given and it appears to be a query filter (see **ganeti**\(7)), the query result is filtered accordingly. For ambiguous cases (e.g. a single field name as a filter) the ``--filter`` (``-F``) option forces the argument to be treated as a filter. LIST-FIELDS ~~~~~~~~~~~ **list-fields** [field...] Lists available fields for jobs. WAIT ~~~~~ **wait** {*job-id*} Wait for the job by the given *job-id* to finish; do not produce any output. WATCH ~~~~~ **watch** {*job-id*} This command follows the output of the job by the given *job-id* and prints it. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-network.rst000064400000000000000000000137321476477700300173760ustar00rootroot00000000000000gnt-network(8) Ganeti | Version @GANETI_VERSION@ ================================================ Name ---- gnt-network - Ganeti network administration Synopsis -------- **gnt-network** {command} [arguments...] DESCRIPTION ----------- The **gnt-network** command is used for network definition and administration in the Ganeti system. Each instance NIC can be connected to a network via the ``network`` NIC parameter. See **gnt-instance**\(8) for more details. BUGS ---- The ``hail`` iallocator hasn't been updated to take networks into account in Ganeti 2.7. The only way to guarantee that it works correctly is having your networks connected to all nodegroups. This will be fixed in a future version. COMMANDS -------- ADD ~~~ | **add** | --network=*network* | [\--gateway=*gateway*] | [\--add-reserved-ips=*reserved-ips*] | [\--network6=*network6*] | [\--gateway6=*gateway6*] | [\--mac-prefix=*macprefix*] | [\--submit] [\--print-jobid] | [\--no-conflicts-check] | {*network-name*} Creates a new network with the given name. The network will be unused initially. To connect it to a node group, use ``gnt-network connect``. ``--network`` option is mandatory. All other are optional. The ``--network`` option allows you to specify the network in a CIDR notation. The ``--gateway`` option allows you to specify the default gateway for this network. IPv6 semantics can be assigned to the network via the ``--network6`` and ``--gateway6`` options. IP pool is meaningless for IPV6 so those two values can be used for EUI64 generation from a NIC's MAC address. The ``--no-conflicts-check`` option can be used to skip the check for conflicting IP addresses. Note that a when connecting a network to a node group (see below) you can specify also the NIC mode and link that will be used by instances on that group to physically connect to this network. This allows the system to work even if the parameters (eg. the VLAN number) change between groups. See **ganeti**\(7) for a description of ``--submit`` and other common options. MODIFY ~~~~~~ | **modify** | [\--gateway=*gateway*] | [\--add-reserved-ips=*reserved-ips*] | [\--remove-reserved-ips=*reserved-ips*] | [\--network6=*network6*] | [\--gateway6=*gateway6*] | [\--mac-prefix=*macprefix*] | [\--submit] [\--print-jobid] | {*network*} Modifies parameters from the network. Unable to modify network (IP address range). Create a new network if you want to do so. All other options are documented in the **add** command above. See **ganeti**\(7) for a description of ``--submit`` and other common options. REMOVE ~~~~~~ | **remove** [\--submit] [\--print-jobid] {*network*} Deletes the indicated network, which must be not connected to any node group. See **ganeti**\(7) for a description of ``--submit`` and other common options. LIST ~~~~ | **list** [\--no-headers] [\--separator=*separator*] [-v] | [-o *[+]field,...*] [network-name...] Lists all existing networks in the cluster. If no group names are given, then all groups are included. Otherwise, only the named groups will be listed. The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The ``-v`` option activates verbose mode, which changes the display of special field states (see **ganeti**\(7)). The ``-o`` option takes a comma-separated list of output fields. If the value of the option starts with the character ``+``, the new fields will be added to the default list. This allows to quickly see the default list plus a few other fields, instead of retyping the entire list of fields. The available fields and their meaning are: @QUERY_FIELDS_NETWORK@ LIST-FIELDS ~~~~~~~~~~~ **list-fields** [field...] List available fields for networks. RENAME ~~~~~~ | **rename** [\--submit] [\--print-jobid] {*oldname*} {*newname*} Renames a given network from *oldname* to *newname*. See **ganeti**\(7) for a description of ``--submit`` and other common options. INFO ~~~~ | **info** [network...] Displays information about a given network. CONNECT ~~~~~~~ | **connect** | [\--no-conflicts-check] | [{-N|\--nic-parameters} *nic-param*=*value*[,*nic-param*=*value*...]] | {*network*} [*groups*...] Connect a network to given node groups (all if not specified) with the network parameters defined via the ``--nic-parameters`` option. Every network interface will inherit those parameters if assigned to a network. The ``--no-conflicts-check`` option can be used to skip the check for conflicting IP addresses. Passing *mode* and *link* as positional arguments along with *network* and *groups* is deprecated and not supported any more. DISCONNECT ~~~~~~~~~~ | **disconnect** {*network*} [*groups*...] Disconnect a network from given node groups (all if not specified). This is possible only if no instance is using the network. Tags ~~~~ ADD-TAGS ^^^^^^^^ **add-tags** [\--from *file*] {*network*} {*tag*...} Add tags to the given network. If any of the tags contains invalid characters, the entire operation will abort. If the ``--from`` option is given, the list of tags will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, both sources will be used). A file name of ``-`` will be interpreted as stdin. LIST-TAGS ^^^^^^^^^ **list-tags** {*network*} List the tags of the given network. REMOVE-TAGS ^^^^^^^^^^^ **remove-tags** [\--from *file*] {*network*} {*tag*...} Remove tags from the given network. If any of the tags are not existing on the network, the entire operation will abort. If the ``--from`` option is given, the list of tags to be removed will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, tags from both sources will be removed). A file name of ``-`` will be interpreted as stdin. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-node.rst000064400000000000000000000567611476477700300166430ustar00rootroot00000000000000gnt-node(8) Ganeti | Version @GANETI_VERSION@ ============================================= Name ---- gnt-node - Node administration Synopsis -------- **gnt-node** {command} [arguments...] DESCRIPTION ----------- The **gnt-node** is used for managing the (physical) nodes in the Ganeti system. COMMANDS -------- ADD ~~~ | **add** [\--readd] [{-s|\--secondary-ip} *secondary\_ip*] | [{-g|\--node-group} *nodegroup*] | [\--master-capable=``yes|no``] [\--vm-capable=``yes|no``] | [\--node-parameters *ndparams*] | [\--disk-state *diskstate*] | [\--hypervisor-state *hvstate*] | [\--no-node-setup] | {*node-name*} Adds the given node to the cluster. This command is used to join a new node to the cluster. You will have to provide credentials to ssh as root to the node to be added. Forwarding of an ssh agent (the ``-A`` option of ssh) works, if an appropriate authorized key is set up on the node to be added. If the other node allows password authentication for root, another way of providing credentials is to provide the root password once asked for it. The command needs to be run on the Ganeti master. Note that the command is potentially destructive, as it will forcibly join the specified host to the cluster, not paying attention to its current status (it could be already in a cluster, etc.) The ``-s (--secondary-ip)`` is used in dual-home clusters and specifies the new node's IP in the secondary network. See the discussion in **gnt-cluster**\(8) for more information. In case you're re-adding a node after hardware failure, you can use the ``--readd`` parameter. In this case, you don't need to pass the secondary IP again, it will be reused from the cluster. Also, the drained and offline flags of the node will be cleared before re-adding it. Note that even for re-added nodes, a new SSH key is generated and distributed and previous Ganeti keys are removed from the machine. The ``-g (--node-group)`` option is used to add the new node into a specific node group, specified by UUID or name. If only one node group exists you can skip this option, otherwise it's mandatory. The ``--no-node-setup`` option that used to prevent Ganeti from performing the initial SSH setup on the new node is no longer valid. Instead, Ganeti considers the ``modify ssh setup`` configuration parameter (which is set using ``--no-ssh-init`` during cluster initialization) to determine whether or not to do the SSH setup on a new node or not. If this parameter is set to ``False``, Ganeti will not touch the SSH keys or the ``authorized_keys`` file of the node at all. Using this option, it lies in the administrators responsibility to ensure SSH connectivity between the hosts by other means. The ``vm_capable``, ``master_capable``, ``ndparams``, ``diskstate`` and ``hvstate`` options are described in **ganeti**\(7), and are used to set the properties of the new node. The command performs some operations that change the state of the master and the new node, like copying certificates and starting the node daemon on the new node, or updating ``/etc/hosts`` on the master node. If the command fails at a later stage, it doesn't undo such changes. This should not be a problem, as a successful run of ``gnt-node add`` will bring everything back in sync. If the node was previously part of another cluster and still has daemons running, the ``node-cleanup`` tool can be run on the machine to be added to clean remains of the previous cluster from the node. Example:: # gnt-node add node5.example.com # gnt-node add -s 192.0.2.5 node5.example.com # gnt-node add -g group2 -s 192.0.2.9 node9.group2.example.com EVACUATE ~~~~~~~~ | **evacuate** [-f] [\--early-release] [\--submit] [\--print-jobid] | [{-I|\--iallocator} *name* \| {-n|\--new-secondary} *destination\_node*] | [--ignore-soft-errors] | [{-p|\--primary-only} \| {-s|\--secondary-only} ] | {*node*} This command will move instances away from the given node. If ``--primary-only`` is given, only primary instances are evacuated, with ``--secondary-only`` only secondaries. If neither is given, all instances are evacuated. It works only for instances having a drbd disk template. The new location for the instances can be specified in two ways: - as a single node for all instances, via the ``-n (--new-secondary)`` option - or via the ``-I (--iallocator)`` option, giving a script name as parameter (or ``.`` to use the default allocator), so each instance will be in turn placed on the (per the script) optimal node The ``--early-release`` changes the code so that the old storage on node being evacuated is removed early (before the resync is completed) and the internal Ganeti locks are also released for both the current secondary and the new secondary, thus allowing more parallelism in the cluster operation. This should be used only when recovering from a disk failure on the current secondary (thus the old storage is already broken) or when the storage on the primary node is known to be fine (thus we won't need the old storage for potential recovery). Note that this command is equivalent to using per-instance commands for each affected instance individually: - ``--primary-only`` is equivalent to performing ``gnt-instance migrate`` for every primary instance running on the node that can be migrated and ``gnt-instance failover`` for every primary instance that cannot be migrated. - ``--secondary-only`` is equivalent to ``gnt-instance replace-disks`` in secondary node change mode (``--new-secondary``) for every DRBD instance that the node is a secondary for. - when neither of the above is done a combination of the two cases is run Note that the iallocator currently only considers disk information of the default disk template, even if the instance's disk templates differ from that. The ``--ignore-soft-errors`` option is passed through to the allocator. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-node evacuate -I hail node3.example.com Note that, due to an issue with the iallocator interface, evacuation of all instances at once is not yet implemented. Full evacuation can currently be achieved by sequentially evacuating primaries and secondaries. :: # gnt-node evacuate -p node3.example.com # gnt-node evacuate -s node3.example.com FAILOVER ~~~~~~~~ **failover** [-f] [\--ignore-consistency] {*node*} This command will fail over all instances having the given node as primary to their secondary nodes. This works only for instances having a drbd disk template. Note that failover will stop any running instances on the given node and restart them again on the new primary. See also FAILOVER in **gnt-instance**\(8). Normally the failover will check the consistency of the disks before failing over the instance. If you are trying to migrate instances off a dead node, this will fail. Use the ``--ignore-consistency`` option for this purpose. Example:: # gnt-node failover node1.example.com INFO ~~~~ **info** [*node*...] Show detailed information about the nodes in the cluster. If you don't give any arguments, all nodes will be shown, otherwise the output will be restricted to the given names. LIST ~~~~ | **list** | [\--no-headers] [\--separator=*SEPARATOR*] | [\--units=*UNITS*] [-v] [{-o|\--output} *[+]FIELD,...*] | [\--filter] | [*node-name*...] Lists the nodes in the cluster. The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The units used to display the numeric values in the output varies, depending on the options given. By default, the values will be formatted in the most appropriate unit. If the ``--separator`` option is given, then the values are shown in mebibytes to allow parsing by scripts. In both cases, the ``--units`` option can be used to enforce a given output unit. Queries of nodes will be done in parallel with any running jobs. This might give inconsistent results for the free disk/memory. The ``-v`` option activates verbose mode, which changes the display of special field states (see **ganeti**\(7)). The ``-o (--output)`` option takes a comma-separated list of output fields. The available fields and their meaning are: @QUERY_FIELDS_NODE@ If the value of the option starts with the character ``+``, the new fields will be added to the default list. This allows one to quickly see the default list plus a few other fields, instead of retyping the entire list of fields. Note that some of these fields are known from the configuration of the cluster (e.g. ``name``, ``pinst``, ``sinst``, ``pip``, ``sip``) and thus the master does not need to contact the node for this data (making the listing fast if only fields from this set are selected), whereas the other fields are "live" fields and require a query to the cluster nodes. Depending on the virtualization type and implementation details, the ``mtotal``, ``mnode`` and ``mfree`` fields may have slightly varying meanings. For example, some solutions share the node memory with the pool of memory used for instances (KVM), whereas others have separate memory for the node and for the instances (Xen). Note that the field 'dtotal' and 'dfree' refer to the storage type that is defined by the default disk template. The default disk template is the first on in the list of cluster-wide enabled disk templates and can be set with ``gnt-cluster modify``. Currently, only the disk templates 'plain', 'drbd', 'file', and 'sharedfile' support storage reporting, for all others '0' is displayed. If exactly one argument is given and it appears to be a query filter (see **ganeti**\(7)), the query result is filtered accordingly. For ambiguous cases (e.g. a single field name as a filter) the ``--filter`` (``-F``) option forces the argument to be treated as a filter (e.g. ``gnt-node list -F master_candidate``). If no node names are given, then all nodes are queried. Otherwise, only the given nodes will be listed. LIST-DRBD ~~~~~~~~~ **list-drbd** [\--no-headers] [\--separator=*SEPARATOR*] *node* Lists the mapping of DRBD minors for a given node. This outputs a static list of fields (it doesn't accept the ``--output`` option), as follows: ``Node`` The (full) name of the node we are querying ``Minor`` The DRBD minor ``Instance`` The instance the DRBD minor belongs to ``Disk`` The disk index that the DRBD minor belongs to ``Role`` Either ``primary`` or ``secondary``, denoting the role of the node for the instance (note: this is not the live status of the DRBD device, but the configuration value) ``PeerNode`` The node that the minor is connected to on the other end This command can be used as a reverse lookup (from node and minor) to a given instance, which can be useful when debugging DRBD issues. Note that this command queries Ganeti via **ganeti-confd**\(8), so it won't be available if support for ``confd`` has not been enabled at build time; furthermore, in Ganeti 2.6 this is only available via the Haskell version of confd (again selected at build time). LIST-FIELDS ~~~~~~~~~~~ **list-fields** [field...] Lists available fields for nodes. MIGRATE ~~~~~~~ | **migrate** [-f] [\--non-live] [\--migration-mode=live\|non-live] | [\--ignore-ipolicy] [\--submit] [\--print-jobid] {*node*} This command will migrate all instances having the given node as primary to their secondary nodes. This works only for instances having a drbd disk template. As for the **gnt-instance migrate** command, the options ``--no-live``, ``--migration-mode`` and ``--no-runtime-changes`` can be given to influence the migration type. If ``--ignore-ipolicy`` is given any instance policy violations occurring during this operation are ignored. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example:: # gnt-node migrate node1.example.com MODIFY ~~~~~~ | **modify** [-f] [\--submit] [\--print-jobid] | [{-C|\--master-candidate} ``yes|no``] | [{-D|\--drained} ``yes|no``] [{-O|\--offline} ``yes|no``] | [\--master-capable=``yes|no``] [\--vm-capable=``yes|no``] [\--auto-promote] | [{-s|\--secondary-ip} *secondary_ip*] | [\--node-parameters *ndparams*] | [\--node-powered=``yes|no``] | [\--hypervisor-state *hvstate*] | [\--disk-state *diskstate*] | {*node-name*} This command changes the role of the node. Each options takes either a literal yes or no, and only one option should be given as yes. The meaning of the roles and flags are described in the manpage **ganeti**\(7). The option ``--node-powered`` can be used to modify state-of-record if it doesn't reflect the reality anymore. In case a node is demoted from the master candidate role, the operation will be refused unless you pass the ``--auto-promote`` option. This option will cause the operation to lock all cluster nodes (thus it will not be able to run in parallel with most other jobs), but it allows automated maintenance of the cluster candidate pool. If locking all cluster node is too expensive, another option is to promote manually another node to master candidate before demoting the current one. Example (setting a node offline, which will demote it from master candidate role if is in that role):: # gnt-node modify --offline=yes node1.example.com The ``-s (--secondary-ip)`` option can be used to change the node's secondary ip. No drbd instances can be running on the node, while this operation is taking place. Remember that the secondary ip must be reachable from the master secondary ip, when being changed, so be sure that the node has the new IP already configured and active. In order to convert a cluster from single homed to multi-homed or vice versa ``--force`` is needed as well, and the target node for the first change must be the master. See **ganeti**\(7) for a description of ``--submit`` and other common options. Example (setting the node back to online and master candidate):: # gnt-node modify --offline=no --master-candidate=yes node1.example.com REMOVE ~~~~~~ **remove** {*node-name*} Removes a node from the cluster. Instances must be removed or migrated to another cluster before. Example:: # gnt-node remove node5.example.com VOLUMES ~~~~~~~ | **volumes** [\--no-headers] [\--human-readable] | [\--separator=*SEPARATOR*] [{-o|\--output} *FIELDS*] | [*node-name*...] Lists all logical volumes and their physical disks from the node(s) provided. The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The units used to display the numeric values in the output varies, depending on the options given. By default, the values will be formatted in the most appropriate unit. If the ``--separator`` option is given, then the values are shown in mebibytes to allow parsing by scripts. In both cases, the ``--units`` option can be used to enforce a given output unit. The ``-o (--output)`` option takes a comma-separated list of output fields. The available fields and their meaning are: node the node name on which the volume exists phys the physical drive (on which the LVM physical volume lives) vg the volume group name name the logical volume name size the logical volume size instance The name of the instance to which this volume belongs, or (in case it's an orphan volume) the character "-" Example:: # gnt-node volumes node5.example.com Node PhysDev VG Name Size Instance node1.example.com /dev/hdc1 xenvg instance1.example.com-sda_11000.meta 128 instance1.example.com node1.example.com /dev/hdc1 xenvg instance1.example.com-sda_11001.data 256 instance1.example.com LIST-STORAGE ~~~~~~~~~~~~ | **list-storage** [\--no-headers] [\--human-readable] | [\--separator=*SEPARATOR*] [\--storage-type=*STORAGE\_TYPE*] | [{-o|\--output} *FIELDS*] | [*node-name*...] Lists the available storage units and their details for the given node(s). The ``--no-headers`` option will skip the initial header line. The ``--separator`` option takes an argument which denotes what will be used between the output fields. Both these options are to help scripting. The units used to display the numeric values in the output varies, depending on the options given. By default, the values will be formatted in the most appropriate unit. If the ``--separator`` option is given, then the values are shown in mebibytes to allow parsing by scripts. In both cases, the ``--units`` option can be used to enforce a given output unit. The ``--storage-type`` option can be used to choose a storage unit type. Possible choices are lvm-pv, lvm-vg, file, sharedfile and gluster. The ``-o (--output)`` option takes a comma-separated list of output fields. The available fields and their meaning are: node the node name on which the volume exists type the type of the storage unit (currently just what is passed in via ``--storage-type``) name the path/identifier of the storage unit size total size of the unit; for the file type see a note below used used space in the unit; for the file type see a note below free available disk space allocatable whether we the unit is available for allocation (only lvm-pv can change this setting, the other types always report true) Note that for the "file" type, the total disk space might not equal to the sum of used and free, due to the method Ganeti uses to compute each of them. The total and free values are computed as the total and free space values for the filesystem to which the directory belongs, but the used space is computed from the used space under that directory *only*, which might not be necessarily the root of the filesystem, and as such there could be files outside the file storage directory using disk space and causing a mismatch in the values. Example:: node1# gnt-node list-storage node2 Node Type Name Size Used Free Allocatable node2 lvm-pv /dev/sda7 673.8G 1.5G 672.3G Y node2 lvm-pv /dev/sdb1 698.6G 0M 698.6G Y MODIFY-STORAGE ~~~~~~~~~~~~~~ | **modify-storage** [\--allocatable={yes|no}] [\--submit] [\--print-jobid] | {*node-name*} {*storage-type*} {*volume-name*} Modifies storage volumes on a node. Only LVM physical volumes can be modified at the moment. They have a storage type of "lvm-pv". Example:: # gnt-node modify-storage --allocatable no node5.example.com lvm-pv /dev/sdb1 REPAIR-STORAGE ~~~~~~~~~~~~~~ | **repair-storage** [\--ignore-consistency] ]\--submit] | {*node-name*} {*storage-type*} {*volume-name*} Repairs a storage volume on a node. Only LVM volume groups can be repaired at this time. They have the storage type "lvm-vg". On LVM volume groups, **repair-storage** runs ``vgreduce --removemissing``. **Caution:** Running this command can lead to data loss. Use it with care. The ``--ignore-consistency`` option will ignore any inconsistent disks (on the nodes paired with this one). Use of this option is most likely to lead to data-loss. Example:: # gnt-node repair-storage node5.example.com lvm-vg xenvg POWERCYCLE ~~~~~~~~~~ **powercycle** [\--yes] [\--force] [\--submit] [\--print-jobid] {*node-name*} This command (tries to) forcefully reboot a node. It is a command that can be used if the node environment is broken, such that the admin can no longer login over SSH, but the Ganeti node daemon is still working. Note that this command is not guaranteed to work; it depends on the hypervisor how effective is the reboot attempt. For Linux, this command requires the kernel option ``CONFIG_MAGIC_SYSRQ`` to be enabled. The ``--yes`` option can be used to skip confirmation, while the ``--force`` option is needed if the target node is the master node. See **ganeti**\(7) for a description of ``--submit`` and other common options. POWER ~~~~~ **power** [``--force``] [``--ignore-status``] [``--all``] [``--power-delay``] on|off|cycle|status [*node-name*...] This command calls out to out-of-band management to change the power state of given node. With ``status`` you get the power status as reported by the out-of-band management script. Note that this command will only work if the out-of-band functionality is configured and enabled on the cluster. If this is not the case, please use the **powercycle** command above. Currently this only has effect for ``off`` and ``cycle``. For safety, Ganeti will not allow either of these operations to be run on the master node. However, it will print a command line which can then be run manually on the master. Note that powering off the master is potentially dangerous, and Ganeti does not support doing this. Providing ``--force`` will skip confirmations for the operation. Providing ``--ignore-status`` will ignore the offline=N state of a node and continue with power off. ``--power-delay`` specifies the time in seconds (factions allowed) waited between powering on the next node. This is by default 2 seconds but can increased if needed with this option. The list of node names is optional. If not provided it will call out for every node in the cluster. Except for the ``off`` and ``cycle`` command where you've to explicit use ``--all`` to select all. HEALTH ~~~~~~ **health** [*node-name*...] This command calls out to out-of-band management to ask for the health status of all or given nodes. The health contains the node name and then the items element with their status in a ``item=status`` manner. Where ``item`` is script specific and ``status`` can be one of ``OK``, ``WARNING``, ``CRITICAL`` or ``UNKNOWN``. Items with status ``WARNING`` or ``CRITICAL`` are logged and annotated in the command line output. RESTRICTED-COMMAND ~~~~~~~~~~~~~~~~~~ | **restricted-command** [-M] [\--sync] | { -g *group* *command* | *command* *node-name*... } Executes a restricted command on the specified nodes. Restricted commands are not arbitrary, but must reside in ``@SYSCONFDIR@/ganeti/restricted-commands`` on a node, either as a regular file or as a symlink. The directory must be owned by root and not be world- or group-writable. If a command fails verification or otherwise fails to start, the node daemon log must be consulted for more detailed information. Example for running a command on two nodes:: # gnt-node restricted-command mycommand \ node1.example.com node2.example.com The ``-g`` option can be used to run a command only on a specific node group, e.g.:: # gnt-node restricted-command -g default mycommand The ``-M`` option can be used to prepend the node name to all command output lines. ``--sync`` forces the opcode to acquire the node lock(s) in exclusive mode. Tags ~~~~ ADD-TAGS ^^^^^^^^ **add-tags** [\--from *file*] {*node-name*} {*tag*...} Add tags to the given node. If any of the tags contains invalid characters, the entire operation will abort. If the ``--from`` option is given, the list of tags will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, both sources will be used). A file name of - will be interpreted as stdin. LIST-TAGS ^^^^^^^^^ **list-tags** {*node-name*} List the tags of the given node. REMOVE-TAGS ^^^^^^^^^^^ **remove-tags** [\--from *file*] {*node-name*} {*tag*...} Remove tags from the given node. If any of the tags are not existing on the node, the entire operation will abort. If the ``--from`` option is given, the list of tags to be removed will be extended with the contents of that file (each line becomes a tag). In this case, there is not need to pass tags on the command line (if you do, tags from both sources will be removed). A file name of - will be interpreted as stdin. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-os.rst000064400000000000000000000065361476477700300163320ustar00rootroot00000000000000gnt-os(8) Ganeti | Version @GANETI_VERSION@ =========================================== Name ---- gnt-os - Instance operating system administration Synopsis -------- **gnt-os** {command} [arguments...] DESCRIPTION ----------- The **gnt-os** is used for managing the list of available operating system flavours for the instances in the Ganeti cluster. COMMANDS -------- LIST ~~~~ **list** [\--no-headers] Gives the list of available/supported OS to use in the instances. When creating the instance you can give the OS-name as an option. Note that hidden or blacklisted OSes are not displayed by this command, use **diagnose** for showing those. DIAGNOSE ~~~~~~~~ **diagnose** This command will help you see why an installed OS is not available in the cluster. The **list** command shows only the OS-es that the cluster sees available on all nodes. It could be that some OS is missing from a node, or is only partially installed, and this command will show the details of all the OSes and the reasons they are or are not valid. INFO ~~~~ **info** {*OS*} This command will list detailed information about each OS available in the cluster, including its validity status, the supported API versions, the supported parameters and variants (if any), and their documentation, etc. If an *OS* name is given, then only the specified OS details will be shown. Note that this command besides the information about the given OS(es), shows detailed information about the given available/supported OS variant(s), in terms of the modified per-OS hypervisor parameters and the modified per-OS parameters passed to the OS install scripts. For the list of the available OSes use **list**. Also, see **modify** for a description of how to modify the parameters for a specific operating system. MODIFY ~~~~~~ | **modify** [\--submit] [\--print-jobid] | [ [ -O | --os-parameters ] =*option*=*value*] | [ --os-parameters-private=*option*=*value*] | [-H *HYPERVISOR*:option=*value*[,...]] | [\--hidden=*yes|no*] [\--blacklisted=*yes|no*] | {*OS*} This command will allow you to modify OS parameters. To modify the per-OS hypervisor parameters (which override the global hypervisor parameters), you can run modify ``-H`` with the same syntax as in **gnt-cluster init** to override default hypervisor parameters of the cluster for specified *OS* argument. To modify the parameters passed to the OS install scripts, use the **--os-parameters** option. If the value of the parameter should not be saved to logs, use **--os-parameters-private** *and* make sure that no Ganeti daemon or program is running in debug mode. **ganeti-luxid** in particular will issue a warning at startup time if ran in debug mode. To modify the hidden and blacklisted states of an OS, pass the options ``--hidden`` *yes|no*, or respectively ``--blacklisted ...``. The 'hidden' state means that an OS won't be listed by default in the OS list, but is available for installation. The 'blacklisted' state means that the OS is not listed and is also not allowed for new instance creations (but can be used for reinstalling old instances). Note: The given operating system doesn't have to exist. This allows preseeding the settings for operating systems not yet known to **gnt-os**. See **ganeti**\(7) for a description of ``--submit`` and other common options. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/gnt-storage.rst000064400000000000000000000036361476477700300173530ustar00rootroot00000000000000gnt-storage(8) Ganeti | Version @GANETI_VERSION@ ================================================ Name ---- gnt-storage - Ganeti storage administration Synopsis -------- **gnt-storage** {command} [arguments...] DESCRIPTION ----------- The **gnt-storage** is used for managing the available storage inside the Ganeti cluster. At the moment, it manages only external storage (ExtStorage). COMMANDS -------- DIAGNOSE ~~~~~~~~ | **diagnose** This command provides detailed information about the state of all ExtStorage providers available in the Ganeti cluster. The state of each provider is calculated per nodegroup. This means that a provider may be valid (meaning usable) for some nodegroups, and invalid (not usable) for some others. This command will help you see why an installed ExtStorage provider is not valid for a specific nodegroup. It could be that it is missing from a node, or is only partially installed. This command will show the details of all ExtStorage providers and the reasons they are or aren't valid for every nodegroup in the cluster. INFO ~~~~ | **info** | [*provider*] This command will list detailed information about each ExtStorage provider found in the cluster, including its nodegroup validity, the supported parameters (if any) and their documentations, etc. For each ExtStorage provider only the valid nodegroups will be listed. If run with no arguments, it will display info for all ExtStorage providers found in the cluster. If given ExtStorage provider's names as arguments it will list info only for providers given. NOTES ----- In the future **gnt-storage** can be extended to also handle internal storage (such as lvm, drbd, etc) and also provide diagnostics for them too. It can also be extended to handle internal and external storage pools, if/when this kind of abstraction is implemented inside Ganeti. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/hail.rst000064400000000000000000000152311476477700300160300ustar00rootroot00000000000000HAIL(1) Ganeti | Version @GANETI_VERSION@ ========================================= NAME ---- hail - Ganeti IAllocator plugin SYNOPSIS -------- **hail** [ **-t** *file* | **\--simulate** *spec* ] [options...] *input-file* **hail** \--version DESCRIPTION ----------- hail is a Ganeti IAllocator plugin that implements the instance placement and movement using the same algorithm as **hbal**\(1). The program takes input via a JSON-file containing current cluster state and the request details, and output (on stdout) a JSON-formatted response. In case of critical failures, the error message is printed on stderr and the exit code is changed to show failure. If the input file name is ``-`` (a single minus sign), then the request data will be read from *stdin*. Apart from input data, hail collects data over the network from all MonDs with the --mond option. Currently it uses only data produced by the CPUload collector. ALGORITHM ~~~~~~~~~ On regular node groups, the program uses a simplified version of the hbal algorithm; for allocation on node groups with exclusive storage see below. For single-node allocations (non-mirrored instances), again we select the node which, when chosen as the primary node, gives the best score. For dual-node allocations (mirrored instances), we chose the best pair; this is the only choice where the algorithm is non-trivial with regard to cluster size. For relocations, we try to change the secondary node of the instance to all the valid other nodes; the node which results in the best cluster score is chosen. For node changes (*change-node* mode), we currently support DRBD instances only, and all three modes (primary changes, secondary changes and all node changes). For group moves (*change-group* mode), again only DRBD is supported, and we compute the correct sequence that will result in a group change; job failure mid-way will result in a split instance. The choice of node(s) on the target group is based on the group score, and the choice of group is based on the same algorithm as allocations (group with lowest score after placement). The deprecated *multi-evacuate* modes is no longer supported. In all cases, the cluster (or group) scoring is identical to the hbal algorithm. For allocation on node groups with exclusive storage, the lost-allocations metrics is used instead to determine which node to allocate an instance on. For a node the allocation vector is the vector of, for each instance policy interval in decreasing order, the number of instances minimally compliant with that interval that still can be placed on that node. The lost-allocations vector for an instance on a node is the difference of the allocation vectors for that node before and after placing the instance on that node. The lost-allocations metrics is the lost allocation vector followed by the remaining disk space on the chosen node, all compared lexicographically. OPTIONS ------- The options that can be passed to the program are as follows: -p, \--print-nodes Prints the before and after node status, in a format designed to allow the user to understand the node's most important parameters. See the man page **htools**\(1) for more details about this option. -t *datafile*, \--text-data=*datafile* The name of the file holding cluster information, to override the data in the JSON request itself. This is mostly used for debugging. The format of the file is described in the man page **htools**\(1). \--mond=*yes|no* If given the program will query all MonDs to fetch data from the supported data collectors over the network. \--mond-data *datafile* The name of the file holding the data provided by MonD, to override querying MonDs over the network. This is mostly used for debugging. The file must be in JSON format and present an array of JSON objects , one for every node, with two members. The first member named ``node`` is the name of the node and the second member named ``reports`` is an array of report objects. The report objects must be in the same format as produced by the monitoring agent. \--ignore-dynu If given, all dynamic utilisation information will be ignored by assuming it to be 0. This option will take precedence over any data passed by the MonDs with the ``--mond`` and the ``--mond-data`` option. \--ignore-soft-errors If given, all checks for soft errors will be omitted when searching for possible allocations. In this way a useful decision can be made even in overloaded clusters. \--no-capacity-checks Normally, hail will only consider those allocations where all instances of a node can immediately restarted should that node fail. With this option given, hail will check only N+1 redundancy for DRBD instances. \--restrict-allocation-to Only consider alloctions on the specified nodes. This overrides any restrictions given in the allocation request. \--simulate *description* Backend specification: similar to the **-t** option, this allows overriding the cluster data with a simulated cluster. For details about the description, see the man page **htools**\(1). -S *filename*, \--save-cluster=*filename* If given, the state of the cluster before and the iallocator run is saved to a file named *filename.pre-ialloc*, respectively *filename.post-ialloc*. This allows re-feeding the cluster state to any of the htools utilities via the ``-t`` option. -v This option increases verbosity and can be used for debugging in order to understand how the IAllocator request is parsed; it can be passed multiple times for successively more information. CONFIGURATION ------------- For the tag-exclusion configuration (see the manpage of hbal for more details), the list of which instance tags to consider as exclusion tags will be read from the cluster tags, configured as follows: - get all cluster tags starting with **htools:iextags:** - use their suffix as the prefix for exclusion tags For example, given a cluster tag like **htools:iextags:service**, all instance tags of the form **service:X** will be considered as exclusion tags, meaning that (e.g.) two instances which both have a tag **service:foo** will not be placed on the same primary node. OPTIONS ------- The options that can be passed to the program are as follows: EXIT STATUS ----------- The exist status of the command will be zero, unless for some reason the algorithm fatally failed (e.g. wrong node or instance data). BUGS ---- Networks (as configured by **gnt-network**\(8)) are not taken into account in Ganeti 2.7. The only way to guarantee that they work correctly is having your networks connected to all nodegroups. This will be fixed in a future version. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/harep.rst000064400000000000000000000103611476477700300162110ustar00rootroot00000000000000HAREP(1) Ganeti | Version @GANETI_VERSION@ ========================================== NAME ---- harep - Ganeti auto-repair tool SYNOPSIS -------- **harep** [ [**-L** | **\--luxi** ] = *socket* ] [ --job-delay = *seconds* ] [ --dry-run ] **harep** \--version DESCRIPTION ----------- Harep is the Ganeti auto-repair tool. It is able to detect that an instance is broken and to generate a sequence of jobs that will fix it, in accordance to the policies set by the administrator. At the moment, only repairs for instances using the disk templates ``plain`` or ``drbd`` are supported. Harep is able to recognize what state an instance is in (healthy, suspended, needs repair, repair disallowed, pending repair, repair failed) and to lead it through a sequence of steps that will bring the instance back to the healthy state. Therefore, harep is mainly meant to be run regularly and frequently using a cron job, so that it can actually follow the instance along all the process. At every run, harep will update the tags it adds to instances that describe its repair status, and will submit jobs that actually perform the required repair operations. By default, harep only reports on the health status of instances, but doesn't perform any action, as they might be potentially dangerous. Therefore, harep will only touch instances that it has been explicitly authorized to work on. The tags enabling harep, can be associated to single instances, or to a nodegroup or to the whole cluster, therefore affecting all the instances they contain. The possible tags share the common structure:: ganeti:watcher:autorepair: where ```` can have the following values: * ``fix-storage``: allow disk replacement or fix the backend without affecting the instance itself (broken DRBD secondary) * ``migrate``: allow instance migration. Note, however, that current harep does not submit migrate jobs; so, currently, this permission level is equivalent to ``fix-storage``. * ``failover``: allow instance reboot on the secondary; this action is taken, if the primary node is offline. * ``reinstall``: allow disks to be recreated and the instance to be reinstalled Each element in the list of tags, includes all the authorizations of the previous one, with ``fix-storage`` being the least powerful and ``reinstall`` being the most powerful. In case multiple autorepair tags act on the same instance, only one can actually be active. The conflict is solved according to the following rules: #. if multiple tags are in the same object, the least destructive takes precedence. #. if the tags are across objects, the nearest tag wins. Example: A cluster has instances I1 and I2, where I1 has the ``failover`` tag, and the cluster has both ``fix-storage`` and ``reinstall``. The I1 instance will be allowed to ``failover``, the I2 instance only to ``fix-storage``. LIMITATIONS ----------- Harep doesn't do any hardware failure detection on its own, it relies on nodes being marked as offline by the administrator. Also harep currently works only for instances with the ``drbd`` and ``plain`` disk templates. Using the data model of **htools**\(1), harep cannot distinguish between drained and offline nodes. In particular, it will (permission provided) failover instances also in situations where a migration would have been enough. In particular, handling of node draining is better done using **hbal**\(1), which will always submit migration jobs, however is the permission to fall back to failover. These issues will be addressed by a new maintenance daemon in future Ganeti versions, which will supersede harep. OPTIONS ------- The options that can be passed to the program are as follows: -L *socket*, \--luxi=*socket* collect data via Luxi, optionally using the given *socket* path. \--job-delay=*seconds* insert this much delay before the execution of repair jobs to allow the tool to continue processing instances. \--dry-run only show which operations would be carried out, but do nothing, even on instances where tags grant the appropriate permissions. Note that harep keeps the state of repair operations in instance tags; therefore, only the operations of the next round of actions can be inspected. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/hbal.rst000064400000000000000000001011411476477700300160150ustar00rootroot00000000000000HBAL(1) Ganeti | Version @GANETI_VERSION@ ========================================= NAME ---- hbal \- Cluster balancer for Ganeti SYNOPSIS -------- **hbal** {backend options...} [algorithm options...] [reporting options...] **hbal** \--version Backend options: { **-m** *cluster* | **-L[** *path* **] [-X]** | **-t** *data-file* | **-I** *path* } Algorithm options: **[ \--max-cpu *cpu-ratio* ]** **[ \--min-disk *disk-ratio* ]** **[ -l *limit* ]** **[ -e *score* ]** **[ -g *delta* ]** **[ \--min-gain-limit *threshold* ]** **[ -O *name...* ]** **[ \--no-disk-moves ]** **[ \--no-instance-moves ]** **[ -U *util-file* ]** **[ \--ignore-dynu ]** **[ \--ignore-soft-errors ]** **[ \--mond *yes|no* ]** **[ \--mond-xen ]** **[ \--exit-on-missing-mond-data ]** **[ \--evac-mode ]** **[ \--restricted-migration ]** **[ \--select-instances *inst...* ]** **[ \--exclude-instances *inst...* ]** Reporting options: **[ -C[ *file* ] ]** **[ -p[ *fields* ] ]** **[ \--print-instances ]** **[ -S *file* ]** **[ -v... | -q ]** DESCRIPTION ----------- hbal is a cluster balancer that looks at the current state of the cluster (nodes with their total and free disk, memory, etc.) and instance placement and computes a series of steps designed to bring the cluster into a better state. The algorithm used is designed to be stable (i.e. it will give you the same results when restarting it from the middle of the solution) and reasonably fast. It is not, however, designed to be a perfect algorithm: it is possible to make it go into a corner from which it can find no improvement, because it looks only one "step" ahead. The program accesses the cluster state via Rapi or Luxi. It also requests data over the network from all MonDs with the --mond option. Currently it uses only data produced by CPUload collector. By default, the program will show the solution incrementally as it is computed, in a somewhat cryptic format; for getting the actual Ganeti command list, use the **-C** option. ALGORITHM ~~~~~~~~~ The program works in independent steps; at each step, we compute the best instance move that lowers the cluster score. The possible move type for an instance are combinations of failover/migrate and replace-disks such that we change one of the instance nodes, and the other one remains (but possibly with changed role, e.g. from primary it becomes secondary). The list is: - failover (f) - replace secondary (r) - replace primary, a composite move (f, r, f) - failover and replace secondary, also composite (f, r) - replace secondary and failover, also composite (r, f) We don't do the only remaining possibility of replacing both nodes (r,f,r,f or the equivalent f,r,f,r) since these move needs an exhaustive search over both candidate primary and secondary nodes, and is O(n*n) in the number of nodes. Furthermore, it doesn't seems to give better scores but will result in more disk replacements. PLACEMENT RESTRICTIONS ~~~~~~~~~~~~~~~~~~~~~~ At each step, we prevent an instance move if it would cause: - a node to go into N+1 failure state - an instance to move onto an offline node (offline nodes are either read from the cluster or declared with *-O*; drained nodes are considered offline) - an exclusion-tag based conflict (exclusion tags are read from the cluster and/or defined via the *\--exclusion-tags* option) - a max vcpu/pcpu ratio to be exceeded (configured via *\--max-cpu*) - min disk free percentage to go below the configured limit (configured via *\--min-disk*) CLUSTER SCORING ~~~~~~~~~~~~~~~ As said before, the algorithm tries to minimise the cluster score at each step. Currently this score is computed as a weighted sum of the following components: - standard deviation of the percent of free memory - standard deviation of the percent of reserved memory - the sum of the percentages of reserved memory - standard deviation of the percent of free disk - count of nodes failing N+1 check - count of instances living (either as primary or secondary) on offline nodes; in the sense of hbal (and the other htools) drained nodes are considered offline - count of instances living (as primary) on offline nodes; this differs from the above metric by helping failover of such instances in 2-node clusters - standard deviation of the ratio of virtual-to-physical cpus (for primary instances of the node) - standard deviation of the fraction of the available spindles (in dedicated mode, spindles represent physical spindles; otherwise this oversubscribable measure for IO load, and the oversubscription factor is taken into account when computing the number of available spindles) - standard deviation of the dynamic load on the nodes, for cpus, memory, disk and network - standard deviation of the CPU load provided by MonD - the count of instances with primary and secondary in the same failure domain - the count of instances sharing the same exclusion tags which primary instances placed in the same failure domain - the overall sum of dissatisfied desired locations among all cluster instances The free memory and free disk values help ensure that all nodes are somewhat balanced in their resource usage. The reserved memory helps to ensure that nodes are somewhat balanced in holding secondary instances, and that no node keeps too much memory reserved for N+1. And finally, the N+1 percentage helps guide the algorithm towards eliminating N+1 failures, if possible. Except for the N+1 failures, offline instances counts, failure domain violation counts and desired locations count, we use the standard deviation since when used with values within a fixed range (we use percents expressed as values between zero and one) it gives consistent results across all metrics (there are some small issues related to different means, but it works generally well). The 'count' type values will have higher score and thus will matter more for balancing; thus these are better for hard constraints (like evacuating nodes and fixing N+1 failures). For example, the offline instances count (i.e. the number of instances living on offline nodes) will cause the algorithm to actively move instances away from offline nodes. This, coupled with the restriction on placement given by offline nodes, will cause evacuation of such nodes. The dynamic load values need to be read from an external file (Ganeti doesn't supply them), and are computed for each node as: sum of primary instance cpu load, sum of primary instance memory load, sum of primary and secondary instance disk load (as DRBD generates write load on secondary nodes too in normal case and in degraded scenarios also read load), and sum of primary instance network load. An example of how to generate these values for input to hbal would be to track ``xl list`` for instances over a day and by computing the delta of the cpu values, and feed that via the *-U* option for all instances (and keep the other metrics as one). For the algorithm to work, all that is needed is that the values are consistent for a metric across all instances (e.g. all instances use cpu% to report cpu usage, and not something related to number of CPU seconds used if the CPUs are different), and that they are normalised to between zero and one. Note that it's recommended to not have zero as the load value for any instance metric since then secondary instances are not well balanced. The CPUload from MonD's data collector will be used only if all MonDs are running, otherwise it won't affect the cluster score. Since we can't find the CPU load of each instance, we can assume that the CPU load of an instance is proportional to the number of its vcpus. With this heuristic, instances from nodes with high CPU load will tend to move to nodes with less CPU load. On a perfectly balanced cluster (all nodes the same size, all instances the same size and spread across the nodes equally, all desired locations satisfied), the values for all metrics would be zero, with the exception of the total percentage of reserved memory. This doesn't happen too often in practice :) OFFLINE INSTANCES ~~~~~~~~~~~~~~~~~ Since current Ganeti versions do not report the memory used by offline (down) instances, ignoring the run status of instances will cause wrong calculations. For this reason, the algorithm subtracts the memory size of down instances from the free node memory of their primary node, in effect simulating the startup of such instances. DESIRED LOCATION TAGS ~~~~~~~~~~~~~~~~~~~~~ Sometimes, administrators want specific instances located in a particular, typically geographic, location. To support this desired location tags are introduced. If the cluster is tagged *htools:desiredlocation:x* then tags starting with *x* are desired location tags. Instances can be assigned tags of the form *x* that means that instance wants to be placed on a node tagged with a location tag *x*. (That means that cluster should be tagged *htools:nlocation:x* too). Instance pinning is just heuristics, not a hard enforced requirement; it will only be achieved by the cluster metrics favouring such placements. EXCLUSION TAGS ~~~~~~~~~~~~~~ The exclusion tags mechanism is designed to prevent instances which run the same workload (e.g. two DNS servers) to land on the same node, which would make the respective node a SPOF for the given service. It works by tagging instances with certain tags and then building exclusion maps based on these. Which tags are actually used is configured either via the command line (option *\--exclusion-tags*) or via adding them to the cluster tags: \--exclusion-tags=a,b This will make all instance tags of the form *a:\**, *b:\** be considered for the exclusion map cluster tags *htools:iextags:a*, *htools:iextags:b* This will make instance tags *a:\**, *b:\** be considered for the exclusion map. More precisely, the suffix of cluster tags starting with *htools:iextags:* will become the prefix of the exclusion tags. Both the above forms mean that two instances both having (e.g.) the tag *a:foo* or *b:bar* won't end on the same node. MIGRATION TAGS ~~~~~~~~~~~~~~ If Ganeti is deployed on a heterogeneous cluster, migration might not be possible between all nodes of a node group. One example of such a situation is upgrading the hypervisor node by node. To make hbal aware of those restrictions, the following cluster tags are used. cluster tags *htools:migration:a*, *htools:migration:b*, etc This make make node tags of the form *a:\**, *b:\**, etc be considered migration restriction. More precisely, the suffix of cluster tags starting with *htools:migration:* will become the prefix of the migration tags. Only those migrations will be taken into consideration where all migration tags of the source node are also present on the target node. cluster tags *htools:allowmigration:x::y* for migration tags *x* and *y* This asserts that a node tagged *y* is able to receive instances in the same way as if they had an *x* tag. So in the simple case of a hypervisor upgrade, tagging all the nodes that have been upgraded with a migration tag suffices. In more complicated situations, it is always possible to use a different migration tag for each hypervisor used and explicitly state the allowed migration directions by means of *htools:allowmigration:* tags. LOCATION TAGS ~~~~~~~~~~~~~ Within a node group, certain nodes might be more likely to fail simultaneously due to a common cause of error (e.g., if they share the same power supply unit). Ganeti can be made aware of those common causes of failure by means of tags. cluster tags *htools:nlocation:a*, *htools:nlocation:b*, etc This make node tags of the form *a:\**, *b:\**, etc be considered to have a common cause of failure. Instances with primary and secondary node having a common cause of failure and instances sharing the same exclusion tag with primary nodes having a common failure are considered badly placed. While such placements are always allowed, they count heavily towards the cluster score. OPTIONS ------- The options that can be passed to the program are as follows: -C, \--print-commands Print the command list at the end of the run. Without this, the program will only show a shorter, but cryptic output. Note that the moves list will be split into independent steps, called "jobsets", but only for visual inspection, not for actually parallelisation. It is not possible to parallelise these directly when executed via "gnt-instance" commands, since a compound command (e.g. failover and replace-disks) must be executed serially. Parallel execution is only possible when using the Luxi backend and the *-L* option. The algorithm for splitting the moves into jobsets is by accumulating moves until the next move is touching nodes already touched by the current moves; this means we can't execute in parallel (due to resource allocation in Ganeti) and thus we start a new jobset. -p, \--print-nodes Prints the before and after node status, in a format designed to allow the user to understand the node's most important parameters. See the man page **htools**\(1) for more details about this option. \--print-instances Prints the before and after instance map. This is less useful as the node status, but it can help in understanding instance moves. -O *name* This option (which can be given multiple times) will mark nodes as being *offline*. This means a couple of things: - instances won't be placed on these nodes, not even temporarily; e.g. the *replace primary* move is not available if the secondary node is offline, since this move requires a failover. - these nodes will not be included in the score calculation (except for the percentage of instances on offline nodes) Note that algorithm will also mark as offline any nodes which are reported by RAPI as such, or that have "?" in file-based input in any numeric fields. -e *score*, \--min-score=*score* This parameter denotes how much above the N+1 bound the cluster score can for us to be happy with and alters the computation in two ways: - if the cluster has the initial score lower than this value, then we don't enter the algorithm at all, and exit with success - during the iterative process, if we reach a score lower than this value, we exit the algorithm The default value of the parameter is currently ``1e-9`` (chosen empirically). -g *delta*, \--min-gain=*delta* Since the balancing algorithm can sometimes result in just very tiny improvements, that bring less gain that they cost in relocation time, this parameter (defaulting to 0.01) represents the minimum gain we require during a step, to continue balancing. \--min-gain-limit=*threshold* The above min-gain option will only take effect if the cluster score is already below *threshold* (defaults to 0.1). The rationale behind this setting is that at high cluster scores (badly balanced clusters), we don't want to abort the rebalance too quickly, as later gains might still be significant. However, under the threshold, the total gain is only the threshold value, so we can exit early. \--no-disk-moves This parameter prevents hbal from using disk move (i.e. "gnt-instance replace-disks") operations. This will result in a much quicker balancing, but of course the improvements are limited. It is up to the user to decide when to use one or another. \--no-instance-moves This parameter prevents hbal from using instance moves (i.e. "gnt-instance migrate/failover") operations. This will only use the slow disk-replacement operations, and will also provide a worse balance, but can be useful if moving instances around is deemed unsafe or not preferred. \--evac-mode This parameter restricts the list of instances considered for moving to the ones living on offline/drained nodes. It can be used as a (bulk) replacement for Ganeti's own *gnt-node evacuate*, with the note that it doesn't guarantee full evacuation. \--restricted-migration This parameter disallows any replace-primary moves (frf), as well as those replace-and-failover moves (rf) where the primary node of the instance is not drained. If used together with the ``--evac-mode`` option, the only migrations that hbal will do are migrations of instances off a drained node. This can be useful if during a reinstall of the base operating system migration is only possible from the old OS to the new OS. Note, however, that usually the use of migration tags is the better choice. \--select-instances=*instances* This parameter marks the given instances (as a comma-separated list) as the only ones being moved during the rebalance. \--exclude-instances=*instances* This parameter marks the given instances (as a comma-separated list) from being moved during the rebalance. -U *util-file* This parameter specifies a file holding instance dynamic utilisation information that will be used to tweak the balancing algorithm to equalise load on the nodes (as opposed to static resource usage). The file is in the format "instance_name cpu_util mem_util disk_util net_util" where the "_util" parameters are interpreted as numbers and the instance name must match exactly the instance as read from Ganeti. In case of unknown instance names, the program will abort. If not given, the default values are one for all metrics and thus dynamic utilisation has only one effect on the algorithm: the equalisation of the secondary instances across nodes (this is the only metric that is not tracked by another, dedicated value, and thus the disk load of instances will cause secondary instance equalisation). Note that value of one will also influence slightly the primary instance count, but that is already tracked via other metrics and thus the influence of the dynamic utilisation will be practically insignificant. \--ignore-dynu If given, all dynamic utilisation information will be ignored by assuming it to be 0. This option will take precedence over any data passed by the ``-U`` option or by the MonDs with the ``--mond`` and the ``--mond-data`` option. \--ignore-soft-errors If given, all checks for soft errors will be omitted when considering balancing moves. In this way, progress can be made in a cluster where all nodes are in a policy-wise bad state, like exceeding oversubscription ratios on CPU or spindles. -S *filename*, \--save-cluster=*filename* If given, the state of the cluster before the balancing is saved to the given file plus the extension "original" (i.e. *filename*.original), and the state at the end of the balancing is saved to the given file plus the extension "balanced" (i.e. *filename*.balanced). This allows re-feeding the cluster state to either hbal itself or for example hspace via the ``-t`` option. -t *datafile*, \--text-data=*datafile* Backend specification: the name of the file holding node and instance information (if not collecting via RAPI or LUXI). This or one of the other backends must be selected. The option is described in the man page **htools**\(1). \--mond=*yes|no* If given the program will query all MonDs to fetch data from the supported data collectors over the network. \--mond-xen If given, also query Xen-specific collectors from MonD, provided that monitoring daemons are queried at all. \--exit-on-missing-mond-data If given, abort if the data obtainable from querying MonDs is incomplete. The default behavior is to continue with a best guess based on the static information. \--mond-data *datafile* The name of the file holding the data provided by MonD, to override querying MonDs over the network. This is mostly used for debugging. The file must be in JSON format and present an array of JSON objects , one for every node, with two members. The first member named ``node`` is the name of the node and the second member named ``reports`` is an array of report objects. The report objects must be in the same format as produced by the monitoring agent. -m *cluster* Backend specification: collect data directly from the *cluster* given as an argument via RAPI. The option is described in the man page **htools**\(1). -L [*path*] Backend specification: collect data directly from the master daemon, which is to be contacted via LUXI (an internal Ganeti protocol). The option is described in the man page **htools**\(1). -X When using the Luxi backend, hbal can also execute the given commands. The execution method is to execute the individual jobsets (see the *-C* option for details) in separate stages, aborting if at any time a jobset doesn't have all jobs successful. Each step in the balancing solution will be translated into exactly one Ganeti job (having between one and three OpCodes), and all the steps in a jobset will be executed in parallel. The jobsets themselves are executed serially. The execution of the job series can be interrupted, see below for signal handling. -l *N*, \--max-length=*N* Restrict the solution to this length. This can be used for example to automate the execution of the balancing. \--max-cpu=*cpu-ratio* The maximum virtual to physical cpu ratio, as a floating point number greater than or equal to one. For example, specifying *cpu-ratio* as **2.5** means that, for a 4-cpu machine, a maximum of 10 virtual cpus should be allowed to be in use for primary instances. A value of exactly one means there will be no over-subscription of CPU (except for the CPU time used by the node itself), and values below one do not make sense, as that means other resources (e.g. disk) won't be fully utilised due to CPU restrictions. \--min-disk=*disk-ratio* The minimum amount of free disk space remaining, as a floating point number. For example, specifying *disk-ratio* as **0.25** means that at least one quarter of disk space should be left free on nodes. -G *uuid*, \--group=*uuid* On an multi-group cluster, select this group for processing. Otherwise hbal will abort, since it cannot balance multiple groups at the same time. -v, \--verbose Increase the output verbosity. Each usage of this option will increase the verbosity (currently more than 2 doesn't make sense) from the default of one. -q, \--quiet Decrease the output verbosity. Each usage of this option will decrease the verbosity (less than zero doesn't make sense) from the default of one. -V, \--version Just show the program version and exit. SIGNAL HANDLING --------------- When executing jobs via LUXI (using the ``-X`` option), normally hbal will execute all jobs until either one errors out or all the jobs finish successfully. Since balancing can take a long time, it is possible to stop hbal early in two ways: - by sending a ``SIGINT`` (``^C``), hbal will register the termination request, and will wait until the currently submitted jobs finish, at which point it will exit (with exit code 0 if all jobs finished correctly, otherwise with exit code 1 as usual) - by sending a ``SIGTERM``, hbal will immediately exit (with exit code 2\); it is the responsibility of the user to follow up with Ganeti and check the result of the currently-executing jobs Note that in any situation, it's perfectly safe to kill hbal, either via the above signals or via any other signal (e.g. ``SIGQUIT``, ``SIGKILL``), since the jobs themselves are processed by Ganeti whereas hbal (after submission) only watches their progression. In this case, the user will have to query Ganeti for job results. EXIT STATUS ----------- The exit status of the command will be zero, unless for some reason the algorithm failed (e.g. wrong node or instance data), invalid command line options, or (in case of job execution) one of the jobs has failed. Once job execution via Luxi has started (``-X``), if the balancing was interrupted early (via *SIGINT*, or via ``--max-length``) but all jobs executed successfully, then the exit status is zero; a non-zero exit code means that the cluster state should be investigated, since a job failed or we couldn't compute its status and this can also point to a problem on the Ganeti side. BUGS ---- The program does not check all its input data for consistency, and sometime aborts with cryptic errors messages with invalid data. The algorithm is not perfect. EXAMPLE ------- Note that these examples are not for the latest version (they don't have full node data). Default output ~~~~~~~~~~~~~~ With the default options, the program shows each individual step and the improvements it brings in cluster score:: $ hbal Loaded 20 nodes, 80 instances Cluster is not N+1 happy, continuing but no guarantee that the cluster will end N+1 happy. Initial score: 0.52329131 Trying to minimize the CV... 1. instance14 node1:node10 => node16:node10 0.42109120 a=f r:node16 f 2. instance54 node4:node15 => node16:node15 0.31904594 a=f r:node16 f 3. instance4 node5:node2 => node2:node16 0.26611015 a=f r:node16 4. instance48 node18:node20 => node2:node18 0.21361717 a=r:node2 f 5. instance93 node19:node18 => node16:node19 0.16166425 a=r:node16 f 6. instance89 node3:node20 => node2:node3 0.11005629 a=r:node2 f 7. instance5 node6:node2 => node16:node6 0.05841589 a=r:node16 f 8. instance94 node7:node20 => node20:node16 0.00658759 a=f r:node16 9. instance44 node20:node2 => node2:node15 0.00438740 a=f r:node15 10. instance62 node14:node18 => node14:node16 0.00390087 a=r:node16 11. instance13 node11:node14 => node11:node16 0.00361787 a=r:node16 12. instance19 node10:node11 => node10:node7 0.00336636 a=r:node7 13. instance43 node12:node13 => node12:node1 0.00305681 a=r:node1 14. instance1 node1:node2 => node1:node4 0.00263124 a=r:node4 15. instance58 node19:node20 => node19:node17 0.00252594 a=r:node17 Cluster score improved from 0.52329131 to 0.00252594 In the above output, we can see: - the input data (here from files) shows a cluster with 20 nodes and 80 instances - the cluster is not initially N+1 compliant - the initial score is 0.52329131 The step list follows, showing the instance, its initial primary/secondary nodes, the new primary secondary, the cluster list, and the actions taken in this step (with 'f' denoting failover/migrate and 'r' denoting replace secondary). Finally, the program shows the improvement in cluster score. A more detailed output is obtained via the *-C* and *-p* options:: $ hbal Loaded 20 nodes, 80 instances Cluster is not N+1 happy, continuing but no guarantee that the cluster will end N+1 happy. Initial cluster status: N1 Name t_mem f_mem r_mem t_dsk f_dsk pri sec p_fmem p_fdsk * node1 32762 1280 6000 1861 1026 5 3 0.03907 0.55179 node2 32762 31280 12000 1861 1026 0 8 0.95476 0.55179 * node3 32762 1280 6000 1861 1026 5 3 0.03907 0.55179 * node4 32762 1280 6000 1861 1026 5 3 0.03907 0.55179 * node5 32762 1280 6000 1861 978 5 5 0.03907 0.52573 * node6 32762 1280 6000 1861 1026 5 3 0.03907 0.55179 * node7 32762 1280 6000 1861 1026 5 3 0.03907 0.55179 node8 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node9 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 * node10 32762 7280 12000 1861 1026 4 4 0.22221 0.55179 node11 32762 7280 6000 1861 922 4 5 0.22221 0.49577 node12 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node13 32762 7280 6000 1861 922 4 5 0.22221 0.49577 node14 32762 7280 6000 1861 922 4 5 0.22221 0.49577 * node15 32762 7280 12000 1861 1131 4 3 0.22221 0.60782 node16 32762 31280 0 1861 1860 0 0 0.95476 1.00000 node17 32762 7280 6000 1861 1106 5 3 0.22221 0.59479 * node18 32762 1280 6000 1396 561 5 3 0.03907 0.40239 * node19 32762 1280 6000 1861 1026 5 3 0.03907 0.55179 node20 32762 13280 12000 1861 689 3 9 0.40535 0.37068 Initial score: 0.52329131 Trying to minimize the CV... 1. instance14 node1:node10 => node16:node10 0.42109120 a=f r:node16 f 2. instance54 node4:node15 => node16:node15 0.31904594 a=f r:node16 f 3. instance4 node5:node2 => node2:node16 0.26611015 a=f r:node16 4. instance48 node18:node20 => node2:node18 0.21361717 a=r:node2 f 5. instance93 node19:node18 => node16:node19 0.16166425 a=r:node16 f 6. instance89 node3:node20 => node2:node3 0.11005629 a=r:node2 f 7. instance5 node6:node2 => node16:node6 0.05841589 a=r:node16 f 8. instance94 node7:node20 => node20:node16 0.00658759 a=f r:node16 9. instance44 node20:node2 => node2:node15 0.00438740 a=f r:node15 10. instance62 node14:node18 => node14:node16 0.00390087 a=r:node16 11. instance13 node11:node14 => node11:node16 0.00361787 a=r:node16 12. instance19 node10:node11 => node10:node7 0.00336636 a=r:node7 13. instance43 node12:node13 => node12:node1 0.00305681 a=r:node1 14. instance1 node1:node2 => node1:node4 0.00263124 a=r:node4 15. instance58 node19:node20 => node19:node17 0.00252594 a=r:node17 Cluster score improved from 0.52329131 to 0.00252594 Commands to run to reach the above solution: echo step 1 echo gnt-instance migrate instance14 echo gnt-instance replace-disks -n node16 instance14 echo gnt-instance migrate instance14 echo step 2 echo gnt-instance migrate instance54 echo gnt-instance replace-disks -n node16 instance54 echo gnt-instance migrate instance54 echo step 3 echo gnt-instance migrate instance4 echo gnt-instance replace-disks -n node16 instance4 echo step 4 echo gnt-instance replace-disks -n node2 instance48 echo gnt-instance migrate instance48 echo step 5 echo gnt-instance replace-disks -n node16 instance93 echo gnt-instance migrate instance93 echo step 6 echo gnt-instance replace-disks -n node2 instance89 echo gnt-instance migrate instance89 echo step 7 echo gnt-instance replace-disks -n node16 instance5 echo gnt-instance migrate instance5 echo step 8 echo gnt-instance migrate instance94 echo gnt-instance replace-disks -n node16 instance94 echo step 9 echo gnt-instance migrate instance44 echo gnt-instance replace-disks -n node15 instance44 echo step 10 echo gnt-instance replace-disks -n node16 instance62 echo step 11 echo gnt-instance replace-disks -n node16 instance13 echo step 12 echo gnt-instance replace-disks -n node7 instance19 echo step 13 echo gnt-instance replace-disks -n node1 instance43 echo step 14 echo gnt-instance replace-disks -n node4 instance1 echo step 15 echo gnt-instance replace-disks -n node17 instance58 Final cluster status: N1 Name t_mem f_mem r_mem t_dsk f_dsk pri sec p_fmem p_fdsk node1 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node2 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node3 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node4 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node5 32762 7280 6000 1861 1078 4 5 0.22221 0.57947 node6 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node7 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node8 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node9 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node10 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node11 32762 7280 6000 1861 1022 4 4 0.22221 0.54951 node12 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node13 32762 7280 6000 1861 1022 4 4 0.22221 0.54951 node14 32762 7280 6000 1861 1022 4 4 0.22221 0.54951 node15 32762 7280 6000 1861 1031 4 4 0.22221 0.55408 node16 32762 7280 6000 1861 1060 4 4 0.22221 0.57007 node17 32762 7280 6000 1861 1006 5 4 0.22221 0.54105 node18 32762 7280 6000 1396 761 4 2 0.22221 0.54570 node19 32762 7280 6000 1861 1026 4 4 0.22221 0.55179 node20 32762 13280 6000 1861 1089 3 5 0.40535 0.58565 Here we see, beside the step list, the initial and final cluster status, with the final one showing all nodes being N+1 compliant, and the command list to reach the final solution. In the initial listing, we see which nodes are not N+1 compliant. The algorithm is stable as long as each step above is fully completed, e.g. in step 8, both the migrate and the replace-disks are done. Otherwise, if only the migrate is done, the input data is changed in a way that the program will output a different solution list (but hopefully will end in the same state). .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/hcheck.rst000064400000000000000000000041451476477700300163420ustar00rootroot00000000000000HCHECK(1) Ganeti | Version @GANETI_VERSION@ =========================================== NAME ---- hcheck \- Cluster checker SYNOPSIS -------- **hcheck** {backend options...} [algorithm options...] [reporting options...] **hcheck** \--version Backend options: { **-m** *cluster* | **-L[** *path* **] | **-t** *data-file* | **-I** *path* } Algorithm options: **[ \--no-simulation ]** **[ \--max-cpu *cpu-ratio* ]** **[ \--min-disk *disk-ratio* ]** **[ -l *limit* ]** **[ -e *score* ]** **[ -g *delta* ]** **[ \--min-gain-limit *threshold* ]** **[ -O *name...* ]** **[ \--no-disk-moves ]** **[ \--no-instance-moves ]** **[ -U *util-file* ]** **[ \--ignore-dynu ]** **[ \--ignore-soft-errors ]** **[ \--evac-mode ]** **[ \--select-instances *inst...* ]** **[ \--exclude-instances *inst...* ]** **[ \--no-capacity-checks ]** Reporting options: **[\--machine-readable**[=*CHOICE*] **]** **[ -p[ *fields* ] ]** **[ \--print-instances ]** **[ -v... | -q ]** DESCRIPTION ----------- hcheck is the cluster checker. It prints information about cluster's health and checks whether a rebalance done using **hbal** would help. This information can be presented in both human-readable and machine-readable way. Note that it does not take any action, only performs a rebalance simulation if necessary. For more information about the algorithm details check **hbal**\(1). Additionally, hcheck also checks if the cluster is globally N+1 redundant. That is, it checks for every node, if after failing over the DRBD instances all instances on that node that with disks externally stored can be restarted on some other node. OPTIONS ------- \--no-simulation Only perform checks based on current cluster state, without trying to simulate rebalancing. \--no-capacity-checks Do not check for global N+1 redundancy, i.e., do not warn if the shared-storage instances of one node cannot be moved to the others should that node fail. For a detailed description about the options listed above have a look at **htools**\(1), **hspace**\(1) and **hbal**\(1). .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/hinfo.rst000064400000000000000000000020171476477700300162140ustar00rootroot00000000000000HINFO(1) Ganeti | Version @GANETI_VERSION@ ========================================== NAME ---- hinfo \- Cluster information printer SYNOPSIS -------- **hinfo** {backend options...} [algorithm options...] [reporting options...] **hinfo** \--version Backend options: { **-m** *cluster* | **-L[** *path* **]** | **-t** *data-file* | **-I** *path* } Algorithm options: **[ -O *name...* ]** Reporting options: **[ -p[ *fields* ] ]** **[ \--print-instances ]** **[ -v... | -q ]** DESCRIPTION ----------- hinfo is the cluster information printer. It prints information about the current cluster state and its residing nodes/instances. It's similar to the output of **hbal** except that it doesn't take any action is just for information purpose. This information might be useful for debugging a certain cluster state. OPTIONS ------- For a detailed description about the options listed above have a look at **htools**\(1) and **hbal**\(1). .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/hroller.rst000064400000000000000000000120751476477700300165650ustar00rootroot00000000000000HROLLER(1) Ganeti | Version @GANETI_VERSION@ ============================================ NAME ---- hroller \- Cluster rolling maintenance scheduler for Ganeti SYNOPSIS -------- **hroller** {backend options...} [algorithm options...] [reporting options...] **hroller** \--version Backend options: { **-m** *cluster* | **-L[** *path* **]** | **-t** *data-file* | **-I** *path* } **[ --force ]** Algorithm options: **[ -G *name* ]** **[ -O *name...* ]** **[ --node-tags** *tag,..* **]** **[ --skip-non-redundant ]** **[ --offline-maintenance ]** **[ --ignore-non-redundant ]** Reporting options: **[ -v... | -q ]** **[ -S *file* ]** **[ --one-step-only ]** **[ --print-moves ]** DESCRIPTION ----------- hroller is a cluster maintenance reboot scheduler. It can calculate which set of nodes can be rebooted at the same time while avoiding having both primary and secondary nodes being rebooted at the same time. For backends that support identifying the master node (currently RAPI and LUXI), the master node is scheduled as the last node in the last reboot group. Apart from this restriction, larger reboot groups are put first. ALGORITHM FOR CALCULATING OFFLINE REBOOT GROUPS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ hroller will view the nodes as vertices of an undirected graph, with two kind of edges. Firstly, there are edges from the primary to the secondary node of every instance. Secondly, two nodes are connected by an edge if they are the primary nodes of two instances that have the same secondary node. It will then color the graph using a few different heuristics, and return the minimum-size color set found. Node with the same color can then simultaneously migrate all instance off to their respective secondary nodes, and it is safe to reboot them simultaneously. OPTIONS ------- For a description of the standard options check **htools**\(1) and **hbal**\(1). \--force Do not fail, even if the master node cannot be determined. \--node-tags *tag,...* Restrict to nodes having at least one of the given tags. \--full-evacuation Also plan moving secondaries out of the nodes to be rebooted. For each instance the move is at most a migrate (if it was primary on that node) followed by a replace secondary. \--skip-non-redundant Restrict to nodes not hosting any non-redundant instance. \--offline-maintenance Pretend that all instances are shutdown before the reboots are carried out. I.e., only edges from the primary to the secondary node of an instance are considered. \--ignore-non-redundant Pretend that the non-redundant instances do not exist, and only take instances with primary and secondary node into account. \--one-step-only Restrict to the first reboot group. Output the group one node per line. \--print-moves After each group list for each affected instance a node where it can be evacuated to. The moves are computed under the assumption that after each reboot group, all instances are moved back to their initial position. BUGS ---- If instances are online the tool should refuse to do offline rolling maintenances, unless explicitly requested. End-to-end shelltests should be provided. EXAMPLES -------- Online Rolling reboots, using tags ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Selecting by tags and getting output for one step only can be used for planing the next maintenance step. :: $ hroller --node-tags needsreboot --one-step-only -L 'First Reboot Group' node1.example.com node3.example.com Typically these nodes would be drained and migrated. :: $ GROUP=`hroller --node-tags needsreboot --one-step-only --no-headers -L` $ for node in $GROUP; do gnt-node modify -D yes $node; done $ for node in $GROUP; do gnt-node migrate -f --submit $node; done After maintenance, the tags would be removed and the nodes undrained. Offline Rolling node reboot output ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If all instances are shut down, usually larger node groups can be found. :: $ hroller --offline-maintenance -L 'Node Reboot Groups' node1.example.com,node3.example.com,node5.example.com node8.example.com,node6.example.com,node2.example.com node7.example.com,node4.example.com Rolling reboots with non-redundant instances ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, hroller plans capacity to move the non-redundant instances out of the nodes to be rebooted. If requested, appropriate locations for the non-redundant instances can be shown. The assumption is that instances are moved back to their original node after each reboot; these back moves are not part of the output. :: $ hroller --print-moves -L 'Node Reboot Groups' node-01-002,node-01-003 inst-20 node-01-001 inst-21 node-01-000 inst-30 node-01-005 inst-31 node-01-004 node-01-004,node-01-005 inst-40 node-01-001 inst-41 node-01-000 inst-50 node-01-003 inst-51 node-01-002 node-01-001,node-01-000 inst-00 node-01-002 inst-01 node-01-003 inst-10 node-01-005 inst-11 node-01-004 .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/hscan.rst000064400000000000000000000054511476477700300162120ustar00rootroot00000000000000HSCAN(1) Ganeti | Version @GANETI_VERSION@ ========================================== NAME ---- hscan - Scan clusters via RAPI and save node/instance data SYNOPSIS -------- **hscan** [-p] [\--no-headers] [-d *path* ] *cluster...* **hscan** \--version DESCRIPTION ----------- hscan is a tool for scanning clusters via RAPI and saving their data in the input format used by **hbal**\(1) and **hspace**\(1). It will also show a one-line score for each cluster scanned or, if desired, the cluster state as show by the **-p** option to the other tools. For each cluster, one file named *cluster***.data** will be generated holding the node and instance data. This file can then be used in **hbal**\(1) or **hspace**\(1) via the *-t* option. In case the cluster name contains slashes (as it can happen when the cluster is a fully-specified URL), these will be replaced with underscores. The one-line output for each cluster will show the following: Name The name of the cluster (or the IP address that was given, etc.) Nodes The number of nodes in the cluster Inst The number of instances in the cluster BNode The number of nodes failing N+1 BInst The number of instances living on N+1-failed nodes t_mem Total memory in the cluster f_mem Free memory in the cluster t_disk Total disk in the cluster f_disk Free disk space in the cluster Score The score of the cluster, as would be reported by **hbal**\(1) if run on the generated data files. In case of errors while collecting data, all fields after the name of the cluster are replaced with the error display. **Note:** this output format is not yet final so it should not be used for scripting yet. OPTIONS ------- The options that can be passed to the program are as follows: -p, \--print-nodes Prints the node status for each cluster after the cluster's one-line status display, in a format designed to allow the user to understand the node's most important parameters. For details, see the man page for **htools**\(1). -d *path* Save the node and instance data for each cluster under *path*, instead of the current directory. -V, \--version Just show the program version and exit. EXIT STATUS ----------- The exist status of the command will be zero, unless for some reason loading the input data failed fatally (e.g. wrong node or instance data). BUGS ---- The program does not check its input data for consistency, and aborts with cryptic errors messages in this case. EXAMPLE ------- :: $ hscan cluster1 Name Nodes Inst BNode BInst t_mem f_mem t_disk f_disk Score cluster1 2 2 0 0 1008 652 255 253 0.24404762 $ ls -l cluster1.data -rw-r--r-- 1 root root 364 2009-03-23 07:26 cluster1.data .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/hspace.rst000064400000000000000000000353611476477700300163640ustar00rootroot00000000000000HSPACE(1) Ganeti | Version @GANETI_VERSION@ =========================================== NAME ---- hspace - Cluster space analyzer for Ganeti SYNOPSIS -------- **hspace** {backend options...} [algorithm options...] [request options...] [output options...] [-v... | -q] **hspace** \--version Backend options: { **-m** *cluster* | **-L[** *path* **]** | **-t** *data-file* | **\--simulate** *spec* | **-I** *path* } Algorithm options: **[ \--max-cpu *cpu-ratio* ]** **[ \--min-disk *disk-ratio* ]** **[ -O *name...* ]** **[ \--independent-groups ]** **[ \--no-capacity-checks ]** Request options: **[\--disk-template** *template* **]** **[\--standard-alloc** *disk,ram,cpu* **]** **[\--tiered-alloc** *disk,ram,cpu* **]** Output options: **[\--machine-readable**[=*CHOICE*] **]** **[-p**[*fields*]**]** DESCRIPTION ----------- hspace computes how many additional instances can be fit on a cluster, while maintaining N+1 status. The program will try to place instances, all of the same size, on the cluster, until the point where we don't have any N+1 possible allocation. It uses the exact same allocation algorithm as the hail iallocator plugin in *allocate* mode. The output of the program is designed either for human consumption (the default) or, when enabled with the ``--machine-readable`` option (described further below), for machine consumption. In the latter case, it is intended to interpreted as a shell fragment (or parsed as a *key=value* file). Options which extend the output (e.g. -p, -v) will output the additional information on stderr (such that the stdout is still parseable). By default, the instance specifications will be read from the cluster; the options ``--standard-alloc`` and ``--tiered-alloc`` can be used to override them. The following keys are available in the machine-readable output of the script (all prefixed with *HTS_*): SPEC_MEM, SPEC_DSK, SPEC_CPU, SPEC_RQN, SPEC_DISK_TEMPLATE, SPEC_SPN These represent the specifications of the instance model used for allocation (the memory, disk, cpu, requested nodes, disk template, spindles). TSPEC_INI_MEM, TSPEC_INI_DSK, TSPEC_INI_CPU, ... Only defined when the tiered mode allocation is enabled, these are similar to the above specifications but show the initial starting spec for tiered allocation. CLUSTER_MEM, CLUSTER_DSK, CLUSTER_CPU, CLUSTER_NODES, CLUSTER_SPN These represent the total memory, disk, CPU count, total nodes, and total spindles in the cluster. INI_SCORE, FIN_SCORE These are the initial (current) and final cluster score (see the hbal man page for details about the scoring algorithm). INI_INST_CNT, FIN_INST_CNT The initial and final instance count. INI_MEM_FREE, FIN_MEM_FREE The initial and final total free memory in the cluster (but this doesn't necessarily mean available for use). INI_MEM_AVAIL, FIN_MEM_AVAIL The initial and final total available memory for allocation in the cluster. If allocating redundant instances, new instances could increase the reserved memory so it doesn't necessarily mean the entirety of this memory can be used for new instance allocations. INI_MEM_RESVD, FIN_MEM_RESVD The initial and final reserved memory (for redundancy/N+1 purposes). INI_MEM_INST, FIN_MEM_INST The initial and final memory used for instances (actual runtime used RAM). INI_MEM_OVERHEAD, FIN_MEM_OVERHEAD The initial and final memory overhead, i.e. memory used for the node itself and unaccounted memory (e.g. due to hypervisor overhead). INI_MEM_EFF, HTS_INI_MEM_EFF The initial and final memory efficiency, represented as instance memory divided by total memory. INI_DSK_FREE, INI_DSK_AVAIL, INI_DSK_RESVD, INI_DSK_INST, INI_DSK_EFF Initial disk stats, similar to the memory ones. FIN_DSK_FREE, FIN_DSK_AVAIL, FIN_DSK_RESVD, FIN_DSK_INST, FIN_DSK_EFF Final disk stats, similar to the memory ones. INI_SPN_FREE, ..., FIN_SPN_FREE, .. Initial and final spindles stats, similar to memory ones. INI_CPU_INST, FIN_CPU_INST Initial and final number of virtual CPUs used by instances. INI_CPU_EFF, FIN_CPU_EFF The initial and final CPU efficiency, represented as the count of virtual instance CPUs divided by the total physical CPU count. INI_MNODE_MEM_AVAIL, FIN_MNODE_MEM_AVAIL The initial and final maximum per-node available memory. This is not very useful as a metric but can give an impression of the status of the nodes; as an example, this value restricts the maximum instance size that can be still created on the cluster. INI_MNODE_DSK_AVAIL, FIN_MNODE_DSK_AVAIL Like the above but for disk. TSPEC This parameter holds the pairs of specifications and counts of instances that can be created in the *tiered allocation* mode. The value of the key is a space-separated list of values; each value is of the form *memory,disk,vcpu,spindles=count* where the memory, disk and vcpu are the values for the current spec, and count is how many instances of this spec can be created. A complete value for this variable could be: **4096,102400,2,1=225 2560,102400,2,1=20 512,102400,2,1=21**. KM_USED_CPU, KM_USED_NPU, KM_USED_MEM, KM_USED_DSK These represents the metrics of used resources at the start of the computation (only for tiered allocation mode). The NPU value is "normalized" CPU count, i.e. the number of virtual CPUs divided by the maximum ratio of the virtual to physical CPUs. KM_POOL_CPU, KM_POOL_NPU, KM_POOL_MEM, KM_POOL_DSK These represents the total resources allocated during the tiered allocation process. In effect, they represent how much is readily available for allocation. KM_UNAV_CPU, KM_POOL_NPU, KM_UNAV_MEM, KM_UNAV_DSK These represents the resources left over (either free as in unallocable or allocable on their own) after the tiered allocation has been completed. They represent better the actual unallocable resources, because some other resource has been exhausted. For example, the cluster might still have 100GiB disk free, but with no memory left for instances, we cannot allocate another instance, so in effect the disk space is unallocable. Note that the CPUs here represent instance virtual CPUs, and in case the *\--max-cpu* option hasn't been specified this will be -1. ALLOC_USAGE The current usage represented as initial number of instances divided per final number of instances. ALLOC_COUNT The number of instances allocated (delta between FIN_INST_CNT and INI_INST_CNT). ALLOC_FAIL*_CNT For the last attempt at allocations (which would have increased FIN_INST_CNT with one, if it had succeeded), this is the count of the failure reasons per failure type; currently defined are FAILMEM, FAILDISK and FAILCPU which represent errors due to not enough memory, disk and CPUs, and FAILN1 which represents a non N+1 compliant cluster on which we can't allocate instances at all. ALLOC_FAIL_REASON The reason for most of the failures, being one of the above FAIL* strings. OK A marker representing the successful end of the computation, and having value "1". If this key is not present in the output it means that the computation failed and any values present should not be relied upon. Many of the ``INI_``/``FIN_`` metrics will be also displayed with a ``TRL_`` prefix, and denote the cluster status at the end of the tiered allocation run. The human output format should be self-explanatory, so it is not described further. OPTIONS ------- The options that can be passed to the program are as follows: \--disk-template *template* Overrides the disk template for the instance read from the cluster; one of the Ganeti disk templates (e.g. plain, drbd, so on) should be passed in. \--spindle-use *spindles* Override the spindle use for the instance read from the cluster. The value can be 0 (for example for instances that use very low I/O), but not negative. For shared storage the value is ignored. \--max-cpu=*cpu-ratio* The maximum virtual to physical cpu ratio, as a floating point number greater than or equal to one. For example, specifying *cpu-ratio* as **2.5** means that, for a 4-cpu machine, a maximum of 10 virtual cpus should be allowed to be in use for primary instances. A value of exactly one means there will be no over-subscription of CPU (except for the CPU time used by the node itself), and values below one do not make sense, as that means other resources (e.g. disk) won't be fully utilised due to CPU restrictions. \--min-disk=*disk-ratio* The minimum amount of free disk space remaining, as a floating point number. For example, specifying *disk-ratio* as **0.25** means that at least one quarter of disk space should be left free on nodes. \--independent-groups Consider all groups independent. That is, if a node that is not N+1 happy is found, ignore its group, but still do allocation in the other groups. The default is to not try allocation at all, if some not N+1 happy node is found. \--accept-existing-errors This is a strengthened form of \--independent-groups. It tells hspace to ignore the presence of not N+1 happy nodes and just allocate on all other nodes without introducing new N+1 violations. Note that this tends to overestimate the capacity, as instances still have to be moved away from the existing not N+1 happy nodes. \--no-capacity-checks Normally, hspace will only consider those allocations where all instances of a node can immediately restarted should that node fail. With this option given, hspace will check only N+1 redundancy for DRBD instances. -l *rounds*, \--max-length=*rounds* Restrict the number of instance allocations to this length. This is not very useful in practice, but can be used for testing hspace itself, or to limit the runtime for very big clusters. -p, \--print-nodes Prints the before and after node status, in a format designed to allow the user to understand the node's most important parameters. See the man page **htools**\(1) for more details about this option. -O *name* This option (which can be given multiple times) will mark nodes as being *offline*. This means a couple of things: - instances won't be placed on these nodes, not even temporarily; e.g. the *replace primary* move is not available if the secondary node is offline, since this move requires a failover. - these nodes will not be included in the score calculation (except for the percentage of instances on offline nodes) Note that the algorithm will also mark as offline any nodes which are reported by RAPI as such, or that have "?" in file-based input in any numeric fields. -S *filename*, \--save-cluster=*filename* If given, the state of the cluster at the end of the allocation is saved to a file named *filename.alloc*, and if tiered allocation is enabled, the state after tiered allocation will be saved to *filename.tiered*. This allows re-feeding the cluster state to either hspace itself (with different parameters) or for example hbal, via the ``-t`` option. -t *datafile*, \--text-data=*datafile* Backend specification: the name of the file holding node and instance information (if not collecting via RAPI or LUXI). This or one of the other backends must be selected. The option is described in the man page **htools**\(1). -m *cluster* Backend specification: collect data directly from the *cluster* given as an argument via RAPI. The option is described in the man page **htools**\(1). -L [*path*] Backend specification: collect data directly from the master daemon, which is to be contacted via LUXI (an internal Ganeti protocol). The option is described in the man page **htools**\(1). \--simulate *description* Backend specification: similar to the **-t** option, this allows overriding the cluster data with a simulated cluster. For details about the description, see the man page **htools**\(1). \--standard-alloc *disk,ram,cpu* This option overrides the instance size read from the cluster for the *standard* allocation mode, where we simply allocate instances of the same, fixed size until the cluster runs out of space. The specification given is similar to the *\--simulate* option and it holds: - the disk size of the instance (units can be used) - the memory size of the instance (units can be used) - the vcpu count for the instance An example description would be *100G,4g,2* describing an instance specification of 100GB of disk space, 4GiB of memory and 2 VCPUs. \--tiered-alloc *disk,ram,cpu* This option overrides the instance size for the *tiered* allocation mode. In this mode, the algorithm starts from the given specification and allocates until there is no more space; then it decreases the specification and tries the allocation again. The decrease is done on the metric that last failed during allocation. The argument should have the same format as for ``--standard-alloc``. Also note that the normal allocation and the tiered allocation are independent, and both start from the initial cluster state; as such, the instance count for these two modes are not related one to another. \--machine-readable[=*choice*] By default, the output of the program is in "human-readable" format, i.e. text descriptions. By passing this flag you can either enable (``--machine-readable`` or ``--machine-readable=yes``) or explicitly disable (``--machine-readable=no``) the machine readable format described above. -v, \--verbose Increase the output verbosity. Each usage of this option will increase the verbosity (currently more than 2 doesn't make sense) from the default of one. -q, \--quiet Decrease the output verbosity. Each usage of this option will decrease the verbosity (less than zero doesn't make sense) from the default of one. -V, \--version Just show the program version and exit. UNITS ~~~~~ By default, all unit-accepting options use mebibytes. Using the lower-case letters of *m*, *g* and *t* (or their longer equivalents of *mib*, *gib*, *tib*, for which case doesn't matter) explicit binary units can be selected. Units in the SI system can be selected using the upper-case letters of *M*, *G* and *T* (or their longer equivalents of *MB*, *GB*, *TB*, for which case doesn't matter). More details about the difference between the SI and binary systems can be read in the **units**\(7) man page. EXIT STATUS ----------- The exist status of the command will be zero, unless for some reason the algorithm fatally failed (e.g. wrong node or instance data). BUGS ---- The algorithm is highly dependent on the number of nodes; its runtime grows exponentially with this number, and as such is impractical for really big clusters. The algorithm doesn't rebalance the cluster or try to get the optimal fit; it just allocates in the best place for the current step, without taking into consideration the impact on future placements. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/hsqueeze.rst000064400000000000000000000063411476477700300167460ustar00rootroot00000000000000HSQUEEZE(1) Ganeti | Version @GANETI_VERSION@ ============================================= NAME ---- hsqueeze \- Dynamic power management SYNOPSIS -------- **hsqueeze** {backend options...} [algorithm options...] [reporting options...] **hsqueeze** \--version Backend options: { **-L[** *path* **]** [-X]** | **-t** *data-file* } Algorithm options: **[ \--minimal-resources=*factor* ]** **[ \--target-resources=*factor* ]** Reporting options: **[ -S *file* ]** **[ -C[*file*] ]** DESCRIPTION ----------- hsqueeze does dynamic power management, by powering up or shutting down nodes, depending on the current load of the cluster. Currently, only suggesting nodes is implemented. ALGORITHM ~~~~~~~~~ hsqueeze considers all online non-master nodes with only externally mirrored instances as candidates for being taken offline. These nodes are iteratively, starting from the node with the least number of instances, added to the set of nodes to be put offline, if possible. A set of nodes is considered as suitable for being taken offline, if, after marking these nodes as offline, balancing the cluster by the algorithm used by **hbal**\(1) yields a situation where all instances are located on online nodes, and each node has at least the target resources free for new instances. All offline nodes with a tag starting with ``htools:standby`` are considered candidates for being taken online. Those nodes are taken online till balancing the cluster by the algorithm used by **hbal**\(1) yields a situation where each node has at least the minimal resources free for new instances. OPTIONS ------- -L [*path*] Backend specification: collect data directly from the master daemon, which is to be contacted via LUXI (an internal Ganeti protocol). The option is described in the man page **htools**\(1). -X When using the Luxi backend, hsqueeze can also execute the given commands. The execution of the job series can be interrupted, see below for signal handling. -S *filename*, \--save-cluster=*filename* If given, the state of the cluster before the squeezing is saved to the given file plus the extension "original" (i.e. *filename*.original), and the state at the end of the squeezing operation is saved to the given file plus the extension "squeezed" (i.e. *filename*.squeezed). -C[*filename*], \--print-commands[=*filename*] If given, a shell script containing the commands to squeeze or unsqueeze the cluster are saved in a file with the given name; if no name is provided, they are printed to stdout. -t *datafile*, \--text-data=*datafile* Backend specification: the name of the file holding node and instance information (if not collecting LUXI). This or one of the other backends must be selected. The option is described in the man page **htools**\(1). \--minimal-resources=*factor* Specify the amount of resources to be free on each node for hsqueeze not to consider onlining additional nodes. The value is reported a multiple of the standard instance specification, as taken from the instance policy. \--target-resources=*factor* Specify the amount of resources to remain free on any node after squeezing. The value is reported a multiple of the standard instance specification, as taken from the instance policy. ganeti-3.1.0~rc2/man/htools.rst000064400000000000000000000247351476477700300164340ustar00rootroot00000000000000HTOOLS(1) Ganeti | Version @GANETI_VERSION@ =========================================== NAME ---- htools - Cluster allocation and placement tools for Ganeti SYNOPSIS -------- **hbal** cluster balancer **hcheck** cluster checker **hspace** cluster capacity computation **hail** IAllocator plugin **hscan** saves cluster state for later reuse **hinfo** cluster information printer **hroller** cluster rolling maintenance scheduler DESCRIPTION ----------- ``htools`` is a suite of tools designed to help with allocation/movement of instances and balancing of Ganeti clusters. ``htools`` is also the generic binary that must be symlinked or hardlinked under each tool's name in order to perform the different functions. Alternatively, the environment variable HTOOLS can be used to set the desired role. Installed as ``hbal``, it computes and optionally executes a suite of instance moves in order to balance the cluster. Installed as ``hcheck``, it preforms cluster checks and optionally simulates rebalancing with all the ``hbal`` options available. Installed as ``hspace``, it computes how many additional instances can be fit on a cluster, while maintaining N+1 status. It can run on models of existing clusters or of simulated clusters. Installed as ``hail``, it acts as an IAllocator plugin, i.e. it is used by Ganeti to compute new instance allocations and instance moves. Installed as ``hscan``, it scans the local or remote cluster state and saves it to files which can later be reused by the other roles. Installed as ``hinfo``, it prints information about the current cluster state. Installed as ``hroller``, it helps scheduling maintenances that require node reboots on a cluster. COMMON OPTIONS -------------- Options behave the same in all program modes, but not all program modes support all options. Some common options are: -p, \--print-nodes Prints the node status, in a format designed to allow the user to understand the node's most important parameters. If the command in question makes a cluster transition (e.g. balancing or allocation), then usually both the initial and final node status is printed. It is possible to customise the listed information by passing a comma-separated list of field names to this option (the field list is currently undocumented), or to extend the default field list by prefixing the additional field list with a plus sign. By default, the node list will contain the following information: F a character denoting the status of the node, with '-' meaning an offline node, '*' meaning N+1 failure and blank meaning a good node Name the node name t_mem the total node memory n_mem the memory used by the node itself i_mem the memory used by instances x_mem amount memory which seems to be in use but cannot be determined why or by which instance; usually this means that the hypervisor has some overhead or that there are other reporting errors f_mem the free node memory r_mem the reserved node memory, which is the amount of free memory needed for N+1 compliance t_dsk total disk f_dsk free disk pcpu the number of physical cpus on the node vcpu the number of virtual cpus allocated to primary instances pcnt number of primary instances scnt number of secondary instances p_fmem percent of free memory p_fdsk percent of free disk r_cpu ratio of virtual to physical cpus lCpu the dynamic CPU load (if the information is available) lMem the dynamic memory load (if the information is available) lDsk the dynamic disk load (if the information is available) lNet the dynamic net load (if the information is available) -t *datafile*, \--text-data=*datafile* Backend specification: the name of the file holding node and instance information (if not collecting via RAPI or LUXI). This or one of the other backends must be selected. The option is described in the man page **htools**\(1). The file should contain text data, line-based, with single empty lines separating sections. In particular, an empty section is described by the empty string followed by the separating empty line, thus yielding two consecutive empty lines. So the number of empty lines does matter and cannot be changed arbitrarily. The lines themselves are column-based, with the pipe symbol (``|``) acting as separator. The first section contains group data, with the following columns: - group name - group uuid - allocation policy - tags (separated by comma) - networks (UUID's, separated by comma) The second sections contains node data, with the following columns: - node name - node total memory - memory used by the node - node free memory - node total disk - node free disk - node physical cores - offline/role field (``Y`` for offline nodes, ``N`` for online non-master nodes, and ``M`` for the master node which is always online) - group UUID - node spindle count - node tags - exclusive storage value (``Y`` if active, ``N`` otherwise) - node free spindles - virtual CPUs used by the node OS - CPU speed relative to that of a ``standard node`` in the node group the node belongs to The third section contains instance data, with the fields: - instance name - instance memory - instance disk size - instance vcpus - instance status (in Ganeti's format, e.g. ``running`` or ``ERROR_down``) - instance ``auto_balance`` flag (see man page **gnt-instance**\(8)) - instance primary node - instance secondary node(s), if any - instance disk type (e.g. ``plain`` or ``drbd``) - instance tags - spindle use back-end parameter - actual disk spindles used by the instance (it can be ``-`` when exclusive storage is not active) The fourth section contains the cluster tags, with one tag per line (no columns/no column processing). The fifth section contains the ipolicies of the cluster and the node groups, in the following format (separated by ``|``): - owner (empty if cluster, group name otherwise) - standard, min, max instance specs; min and max instance specs are separated between them by a semicolon, and can be specified multiple times (min;max;min;max...); each of the specs contains the following values separated by commas: - memory size - cpu count - disk size - disk count - NIC count - disk templates - vcpu ratio - spindle ratio \--mond=*yes|no* If given the program will query all MonDs to fetch data from the supported data collectors over the network. \--mond-data *datafile* The name of the file holding the data provided by MonD, to override querying MonDs over the network. This is mostly used for debugging. The file must be in JSON format and present an array of JSON objects , one for every node, with two members. The first member named ``node`` is the name of the node and the second member named ``reports`` is an array of report objects. The report objects must be in the same format as produced by the monitoring agent. \--ignore-dynu If given, all dynamic utilisation information will be ignored by assuming it to be 0. This option will take precedence over any data passed by the ``-U`` option (available with hbal) or by the MonDs with the ``--mond`` and the ``--mond-data`` option. -m *cluster* Backend specification: collect data directly from the *cluster* given as an argument via RAPI. If the argument doesn't contain a colon (:), then it is converted into a fully-built URL via prepending ``https://`` and appending the default RAPI port, otherwise it is considered a fully-specified URL and used as-is. -L [*path*] Backend specification: collect data directly from the master daemon, which is to be contacted via LUXI (an internal Ganeti protocol). An optional *path* argument is interpreted as the path to the unix socket on which the master daemon listens; otherwise, the default path used by Ganeti (configured at build time) is used. -I|\--ialloc-src *path* Backend specification: load data directly from an iallocator request (as produced by Ganeti when doing an iallocator call). The iallocator request is read from specified path. \--simulate *description* Backend specification: instead of using actual data, build an empty cluster given a node description. The *description* parameter must be a comma-separated list of five elements, describing in order: - the allocation policy for this node group (*preferred*, *allocable* or *unallocable*, or alternatively the short forms *p*, *a* or *u*) - the number of nodes in the cluster - the disk size of the nodes (default in mebibytes, units can be used) - the memory size of the nodes (default in mebibytes, units can be used) - the cpu core count for the nodes - the spindle count for the nodes An example description would be **preferred,20,100G,16g,4,2** describing a 20-node cluster where each node has 100GB of disk space, 16GiB of memory, 4 CPU cores and 2 disk spindles. Note that all nodes must have the same specs currently. This option can be given multiple times, and each new use defines a new node group. Hence different node groups can have different allocation policies and node count/specifications. -v, \--verbose Increase the output verbosity. Each usage of this option will increase the verbosity (currently more than 5 doesn't make sense) from the default of one. -q, \--quiet Decrease the output verbosity. Each usage of this option will decrease the verbosity (less than zero doesn't make sense) from the default of one. -V, \--version Just show the program version and exit. UNITS ~~~~~ Some options accept not simply numerical values, but numerical values together with a unit. By default, such unit-accepting options use mebibytes. Using the lower-case letters of *m*, *g* and *t* (or their longer equivalents of *mib*, *gib*, *tib*, for which case doesn't matter) explicit binary units can be selected. Units in the SI system can be selected using the upper-case letters of *M*, *G* and *T* (or their longer equivalents of *MB*, *GB*, *TB*, for which case doesn't matter). More details about the difference between the SI and binary systems can be read in the **units**\(7) man page. ENVIRONMENT ----------- The environment variable ``HTOOLS`` can be used instead of renaming/symlinking the programs; simply set it to the desired role and then the name of the program is no longer used. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/man/mon-collector.rst000064400000000000000000000077231476477700300176770ustar00rootroot00000000000000mon-collector(7) Ganeti | Version @GANETI_VERSION@ ================================================== NAME ---- mon-collector - Command line interface for the data collectors of the monitoring system SYNOPSIS -------- **mon-collector** {collector} DESCRIPTION ----------- ``mon-collector`` is a suite of tools designed to provide a command line interface to the data collectors implemented by the ganeti monitoring system. ``mon-collector`` is also the generic binary that must be invoked specifying, as the first command line parameter, the name of the actual desired data collector to be run. When executed, ``mon-collector`` will run the specified collector and will print its output to stdout, in JSON format. COLLECTORS ---------- DISKSTATS ~~~~~~~~~ | diskstats [ [ **-f** | **\--file** ] = *input-file* ] Collects the information about the status of the disks of the system, as listed by /proc/diskstats, or by an alternate file with the same syntax specified on the command line. The options that can be passed to the DRBD collector are as follows: -f *input-file*, \--file=*input-file* Where to read the data from. Default if not specified: /proc/diskstats DRBD ~~~~ | drbd [ [ **-s** | **\--drbd-status** ] = *status-file* ] [ [ **-p** | **\--drbd-pairing**] = *pairing-file* ] Collects the information about the version and status of the DRBD kernel module, and of the disks it is managing. If *status-file* and *pairing-file* are specified, the status and the instance-minor paring information will be read from those files. Otherwise, the collector will read them, respectively, from /proc/drbd and from the Confd server. The options that can be passed to the DRBD collector are as follows: -s *status-file*, \--drbd-status=*status-file* Read the DRBD status from the specified file instead of /proc/drbd. -p *pairing-file*, \--drbd-pairing=*pairing-file* Read the information about the pairing between instances and DRBD minors from the specified file instead of asking the Confd servers for them. INSTANCE STATUS ~~~~~~~~~~~~~~~ | inst-status-xen [ [ **-a** | **\--address** ] = *ip-address* ] [ [ **-p** | **\--port** ] = *port-number* ] Collects the information about the status of the instances of the current node. In order to perform this task, it needs to connect to the ConfD daemon to fetch some configuration information. The following parameters allow the user to specify the position where the daemon is listening, in case it's not the default one: -a *ip-address*, \--address=*ip-address* The IP address the ConfD daemon is listening on. -p *port-number*, \--port=*port-number* The port the ConfD daemon is listening on. LOGICAL VOLUMES ~~~~~~~~~~~~~~~ | lv [ [ **-a** | **\--address** ] = *ip-address* ] [ [ **-p** | **\--port** ] = *port-number* ] [ [ **-f** | **\--file** ] = *input-file* ] [ [ **-i** | **\--instances** ] = *instances-file* ] Collects the information about the logical volumes of the current node. In order to perform this task, it needs to interact with the ``lvs`` command line tool and to connect to the ConfD daemon to fetch some configuration information. The following parameters allow the user to specify the position where the daemon is listening, in case it's not the default one: -a *ip-address*, \--address=*ip-address* The IP address the ConfD daemon is listening on. -p *port-number*, \--port=*port-number* The port the ConfD daemon is listening on. Instead of accessing the live data on the cluster, the tool can also read data serialized on files (mainly for testing purposes). Namely: -f *input-file*, \--file *input-file* The name of the file containing a recorded output of the ``lvs`` tool. -i *instances-file*, \--instances=*instances-file* The name of the file containing a JSON serialization of instances the current node is primary and secondary for, listed as:: ([Instance], [Instance]) where the first list contains the instances the node is primary for, the second list those the node is secondary for. ganeti-3.1.0~rc2/pydoctor.ini000064400000000000000000000001371476477700300161510ustar00rootroot00000000000000[pydoctor] projectname = Ganeti projecturl = https://github.com/ganeti/ganeti makehtml = True ganeti-3.1.0~rc2/pylintrc000064400000000000000000000060101476477700300153700ustar00rootroot00000000000000# Configuration file for pylint (http://www.logilab.org/project/pylint). See # http://www.logilab.org/card/pylintfeatures for more detailed variable # descriptions. # # NOTE: Keep this file in sync (as much as possible) with pylintrc-test! [MASTER] profile = no ignore = persistent = no cache-size = 50000 load-plugins = [REPORTS] output-format = colorized include-ids = yes files-output = no reports = no evaluation = 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) comment = yes [BASIC] # disabling docstring checks since we have way too many without (complex # inheritance hierarchies) #no-docstring-rgx = __.*__ no-docstring-rgx = .* module-rgx = (([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # added lower-case names const-rgx = ((_{0,2}[A-Za-z][A-Za-z0-9_]*)|(__.*__))$ class-rgx = _?[A-Z][a-zA-Z0-9]+$ # added lower-case names function-rgx = (_?([A-Z]+[a-z0-9]+([A-Z]+[a-z0-9]*)*)|main|([a-z_][a-z0-9_]*))$ # add lower-case names, since derived classes must obey method names method-rgx = (_{0,2}[A-Z]+[a-z0-9]+([A-Z]+[a-z0-9]*)*|__.*__|([a-z_][a-z0-9_]*))$ attr-rgx = [a-z_][a-z0-9_]{1,30}$ argument-rgx = [a-z_][a-z0-9_]*$ variable-rgx = (_?([a-z_][a-z0-9_]*)|(_?[A-Z0-9_]+))$ inlinevar-rgx = [A-Za-z_][A-Za-z0-9_]*$ good-names = i,j,k,_ bad-names = foo,bar,baz,toto,tutu,tata bad-functions = xrange [TYPECHECK] ignore-mixin-members = yes zope = no acquired-members = ignored-classes = sha1,md5,Popen,ChildProcess [VARIABLES] init-import = no dummy-variables-rgx = _ additional-builtins = [CLASSES] defining-attr-methods = __init__,__new__,setUp valid-classmethod-first-arg = cls,mcs [DESIGN] max-args = 15 max-locals = 50 max-returns = 10 max-branchs = 80 max-statements = 200 max-parents = 7 max-attributes = 20 # zero as struct-like (PODS) classes don't export any methods min-public-methods = 0 max-public-methods = 50 [IMPORTS] deprecated-modules = regsub,string,TERMIOS,Bastion,rexec import-graph = ext-import-graph = int-import-graph = [FORMAT] max-line-length = 80 # TODO if you hit this limit, split the module, and reduce this number to the # next biggest one. max-module-lines = 3600 indent-string = " " indent-after-paren = 2 [MISCELLANEOUS] notes = FIXME,XXX,TODO [SIMILARITIES] min-similarity-lines = 4 ignore-comments = yes ignore-docstrings = yes [MESSAGES CONTROL] # Enable only checker(s) with the given id(s). This option conflicts with the # disable-checker option #enable-checker= # Enable all checker(s) except those with the given id(s). This option # conflicts with the enable-checker option #disable-checker= disable-checker=similarities # Enable all messages in the listed categories (IRCWEF). #enable-msg-cat= # Disable all messages in the listed categories (IRCWEF). disable-msg-cat= # Enable the message(s) with the given id(s). #enable-msg= # Disable the message(s) with the given id(s). disable-msg=W0511,R0922,W0201 # The new pylint 0.21+ style (plus the similarities checker, which is no longer # a separate opiton, but a generic disable control) disable=W0511,R0922,W0201,R0922,R0801,I0011 ganeti-3.1.0~rc2/pylintrc-test000064400000000000000000000055031476477700300163530ustar00rootroot00000000000000# pylint configuration file tailored to test code. # # NOTE: Keep in sync as much as possible with the standard pylintrc file. # Only a few settings had to be adapted for the test code. [MASTER] profile = no ignore = persistent = no cache-size = 50000 load-plugins = [REPORTS] output-format = colorized include-ids = yes files-output = no reports = no evaluation = 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) comment = yes [BASIC] # disabling docstring checks since we have way too many without (complex # inheritance hierarchies) #no-docstring-rgx = __.*__ no-docstring-rgx = .* module-rgx = (([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # added lower-case names const-rgx = ((_{0,2}[A-Za-z][A-Za-z0-9_]*)|(__.*__))$ class-rgx = _?[A-Z][a-zA-Z0-9]+$ # added lower-case names function-rgx = (_?((([A-Z]+[a-z0-9]+)|test|assert)([A-Z]+[a-z0-9]*)*)|main|([a-z_][a-z0-9_]*))$ # add lower-case names, since derived classes must obey method names method-rgx = (_{0,2}(([A-Z]+[a-z0-9]+)|test|assert)([A-Z]+[a-z0-9]*)*|__.*__|runTests|setUp|tearDown|([a-z_][a-z0-9_]*))$ attr-rgx = [a-z_][a-z0-9_]{1,30}$ argument-rgx = [a-z_][a-z0-9_]*$ variable-rgx = (_?([a-z_][a-z0-9_]*)|(_?[A-Z0-9_]+))$ inlinevar-rgx = [A-Za-z_][A-Za-z0-9_]*$ good-names = i,j,k,_ bad-names = foo,bar,baz,toto,tutu,tata bad-functions = xrange [TYPECHECK] ignore-mixin-members = yes zope = no acquired-members = [VARIABLES] init-import = no dummy-variables-rgx = _ additional-builtins = [CLASSES] defining-attr-methods = __init__,__new__,setUp [DESIGN] max-args = 15 max-locals = 50 max-returns = 10 max-branchs = 80 max-statements = 200 max-parents = 7 max-attributes = 20 # zero as struct-like (PODS) classes don't export any methods min-public-methods = 0 max-public-methods = 50 [IMPORTS] deprecated-modules = regsub,string,TERMIOS,Bastion,rexec import-graph = ext-import-graph = int-import-graph = [FORMAT] max-line-length = 80 max-module-lines = 4500 indent-string = " " [MISCELLANEOUS] notes = FIXME,XXX,TODO [SIMILARITIES] min-similarity-lines = 4 ignore-comments = yes ignore-docstrings = yes [MESSAGES CONTROL] # Enable only checker(s) with the given id(s). This option conflicts with the # disable-checker option #enable-checker= # Enable all checker(s) except those with the given id(s). This option # conflicts with the enable-checker option #disable-checker= disable-checker=similarities # Enable all messages in the listed categories (IRCWEF). #enable-msg-cat= # Disable all messages in the listed categories (IRCWEF). disable-msg-cat= # Enable the message(s) with the given id(s). #enable-msg= # Disable the message(s) with the given id(s). disable-msg=W0511,R0922,W0201 # The new pylint 0.21+ style (plus the similarities checker, which is no longer # a separate opiton, but a generic disable control) disable=W0511,R0922,W0201,R0922,R0801,I0011,R0201 ganeti-3.1.0~rc2/qa/000075500000000000000000000000001476477700300142055ustar00rootroot00000000000000ganeti-3.1.0~rc2/qa/__init__.py000064400000000000000000000025731476477700300163250ustar00rootroot00000000000000# # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # empty file for package definition """Ganeti QA scripts""" ganeti-3.1.0~rc2/qa/colors.py000064400000000000000000000060521476477700300160630ustar00rootroot00000000000000#!/usr/bin/python3 -u # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for adding colorized output to Ganeti. Colors are enabled only if the standard output is a proper terminal. (Or call check_for_colors() to make a thorough test using "tput".) See http://en.wikipedia.org/wiki/ANSI_escape_code for more possible additions. """ import os import subprocess import sys DEFAULT = "0" BOLD = "1" UNDERLINE = "4" REVERSE = "7" BLACK = "30" RED = "31" GREEN = "32" YELLOW = "33" BLUE = "34" MAGENTA = "35" CYAN = "36" WHITE = "37" BG_BLACK = "40" BG_RED = "41" BG_GREEN = "42" BG_YELLOW = "43" BG_BLUE = "44" BG_MAGENTA = "45" BG_CYAN = "46" BG_WHITE = "47" _enabled = sys.stdout.isatty() def _escape_one(code): return "\033[" + code + "m" if code else "" def _escape(codes): if hasattr(codes, "__iter__"): return _escape_one(";".join(codes)) else: return _escape_one(codes) def _reset(): return _escape([DEFAULT]) def colorize(line, color=None): """Wraps a given string into ANSI color codes corresponding to given color(s). @param line: a string @param color: a color or a list of colors selected from this module's constants """ if _enabled and color: return _escape(color) + line + _reset() else: return line def check_for_colors(): """Tries to call 'tput' to properly determine, if the terminal has colors. This functions is meant to be run once at the program's start. If not invoked, colors are enabled iff standard output is a terminal. """ colors = 0 if sys.stdout.isatty(): try: p = subprocess.Popen(["tput", "colors"], stdout=subprocess.PIPE) output = p.communicate()[0] if p.returncode == 0: colors = int(output) except (OSError, ValueError): pass global _enabled _enabled = (colors >= 2) ganeti-3.1.0~rc2/qa/ganeti-qa.py000075500000000000000000001200471476477700300164340ustar00rootroot00000000000000#!/usr/bin/python3 -u # # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for doing QA on Ganeti. """ # pylint: disable=C0103 # due to invalid name import copy import datetime import optparse import sys from qa import colors from qa import qa_cluster from qa import qa_config from qa import qa_daemon from qa import qa_env from qa import qa_error from qa import qa_filters from qa import qa_group from qa import qa_instance from qa import qa_iptables from qa import qa_monitoring from qa import qa_network from qa import qa_node from qa import qa_os from qa import qa_performance from qa import qa_job from qa import qa_rapi from qa import qa_tags from qa import qa_utils from ganeti import utils from ganeti import rapi # pylint: disable=W0611 from ganeti import constants from ganeti import netutils from ganeti import pathutils import ganeti.rapi.client # pylint: disable=W0611 from ganeti.rapi.client import UsesRapiClient _QA_PROFILE = pathutils.GetLogFilename("qa-profile") _PROFILE_LOG_INDENT = "" def _FormatHeader(line, end=72, mark="-", color=None): """Fill a line up to the end column. """ line = (mark * 4) + " " + line + " " line += "-" * (end - len(line)) line = line.rstrip() line = colors.colorize(line, color=color) return line def _DescriptionOf(fn): """Computes the description of an item. """ if fn.__doc__: desc = fn.__doc__.splitlines()[0].strip() desc = desc.rstrip(".") if fn.__name__: desc = "[" + fn.__name__ + "] " + desc else: desc = "%r" % fn return desc def RunTest(fn, *args, **kwargs): """Runs a test after printing a header. """ global _PROFILE_LOG_INDENT tstart = datetime.datetime.now() desc = _DescriptionOf(fn) print() print(_FormatHeader("%s start %s" % (tstart, desc), color=colors.YELLOW, mark="<")) with open(_QA_PROFILE, "a") as f: f.write("%sExecuting Function %s\n" % (_PROFILE_LOG_INDENT, fn.__name__)) _PROFILE_LOG_INDENT += " " try: retval = fn(*args, **kwargs) print(_FormatHeader("PASSED %s" % (desc, ), color=colors.GREEN)) return retval except Exception as e: print(_FormatHeader("FAILED %s: %s" % (desc, e), color=colors.RED)) raise finally: tstop = datetime.datetime.now() tdelta = tstop - tstart print(_FormatHeader("%s time=%s %s" % (tstop, tdelta, desc), color=colors.MAGENTA, mark=">")) _PROFILE_LOG_INDENT = _PROFILE_LOG_INDENT[:-1] with open(_QA_PROFILE, "a") as f: f.write("%sFunction %s ran for %s\n" % (_PROFILE_LOG_INDENT, fn.__name__, tdelta)) def ReportTestSkip(desc, testnames): """Reports that tests have been skipped. @type desc: string @param desc: string @type testnames: string or list of string @param testnames: either a single test name in the configuration file, or a list of testnames (which will be AND-ed together) """ tstart = datetime.datetime.now() # TODO: Formatting test names when non-string names are involved print(_FormatHeader("%s skipping %s, test(s) %s disabled" % (tstart, desc, testnames), color=colors.BLUE, mark="*")) def RunTestIf(testnames, fn, *args, **kwargs): """Runs a test conditionally. @param testnames: either a single test name in the configuration file, or a list of testnames (which will be AND-ed together) """ if qa_config.TestEnabled(testnames): RunTest(fn, *args, **kwargs) else: desc = _DescriptionOf(fn) ReportTestSkip(desc, testnames) def RunTestBlock(fn, *args, **kwargs): """Runs a block of tests after printing a header. """ global _PROFILE_LOG_INDENT tstart = datetime.datetime.now() desc = _DescriptionOf(fn) print() print(_FormatHeader("BLOCK %s start %s" % (tstart, desc), color=[colors.YELLOW, colors.BOLD], mark="v")) with open(_QA_PROFILE, "a") as f: f.write("%sExecuting Function %s\n" % (_PROFILE_LOG_INDENT, fn.__name__)) _PROFILE_LOG_INDENT += " " try: return fn(*args, **kwargs) except Exception as e: print(_FormatHeader("BLOCK FAILED %s: %s" % (desc, e), color=[colors.RED, colors.BOLD])) raise finally: tstop = datetime.datetime.now() tdelta = tstop - tstart print(_FormatHeader("BLOCK %s time=%s %s" % (tstop, tdelta, desc), color=[colors.MAGENTA, colors.BOLD], mark="^")) _PROFILE_LOG_INDENT = _PROFILE_LOG_INDENT[:-1] with open(_QA_PROFILE, "a") as f: f.write("%sFunction %s ran for %s\n" % (_PROFILE_LOG_INDENT, fn.__name__, tdelta)) def RunEnvTests(): """Run several environment tests. """ RunTestIf("env", qa_env.TestSshConnection) RunTestIf("env", qa_env.TestIcmpPing) RunTestIf("env", qa_env.TestGanetiCommands) def SetupCluster(): """Initializes the cluster. """ RunTestIf("create-cluster", qa_cluster.TestClusterInit) if not qa_config.TestEnabled("create-cluster"): # If the cluster is already in place, we assume that exclusive-storage is # already set according to the configuration qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False)) qa_rapi.SetupRapi() qa_group.ConfigureGroups() # Test on empty cluster RunTestIf("node-list", qa_node.TestNodeList) RunTestIf("instance-list", qa_instance.TestInstanceList) RunTestIf("job-list", qa_job.TestJobList) RunTestIf("create-cluster", qa_node.TestNodeAddAll) if not qa_config.TestEnabled("create-cluster"): # consider the nodes are already there qa_node.MarkNodeAddedAll() RunTestIf("test-jobqueue", qa_cluster.TestJobqueue) RunTestIf("test-jobqueue", qa_job.TestJobCancellation) # enable the watcher (unconditionally) RunTest(qa_daemon.TestResumeWatcher) RunTestIf("node-list", qa_node.TestNodeList) # Test listing fields RunTestIf("node-list", qa_node.TestNodeListFields) RunTestIf("instance-list", qa_instance.TestInstanceListFields) RunTestIf("job-list", qa_job.TestJobListFields) RunTestIf("instance-export", qa_instance.TestBackupListFields) RunTestIf("node-info", qa_node.TestNodeInfo) def RunClusterTests(): """Runs tests related to gnt-cluster. """ for test, fn in [ ("create-cluster", qa_cluster.TestClusterInitDisk), ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto) ]: RunTestIf(test, fn) for test, fn in [ ("cluster-verify", qa_cluster.TestClusterVerify), ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs), # TODO: add more cluster modify tests ("cluster-modify", qa_cluster.TestClusterModifyEmpty), ("cluster-modify", qa_cluster.TestClusterModifyIPolicy), ("cluster-modify", qa_cluster.TestClusterModifyISpecs), ("cluster-modify", qa_cluster.TestClusterModifyBe), ("cluster-modify", qa_cluster.TestClusterModifyDisk), ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates), ("cluster-modify", qa_cluster.TestClusterModifyFileStorageDir), ("cluster-modify", qa_cluster.TestClusterModifySharedFileStorageDir), ("cluster-modify", qa_cluster.TestClusterModifyInstallImage), ("cluster-modify", qa_cluster.TestClusterModifyUserShutdown), ("cluster-rename", qa_cluster.TestClusterRename), ("cluster-info", qa_cluster.TestClusterVersion), ("cluster-info", qa_cluster.TestClusterInfo), ("cluster-info", qa_cluster.TestClusterGetmaster), ("cluster-redist-conf", qa_cluster.TestClusterRedistConf), (["cluster-copyfile", qa_config.NoVirtualCluster], qa_cluster.TestClusterCopyfile), ("cluster-command", qa_cluster.TestClusterCommand), ("cluster-burnin", qa_cluster.TestClusterBurnin), ("cluster-master-failover", qa_cluster.TestClusterMasterFailover), ("cluster-master-failover", qa_cluster.TestClusterMasterFailoverWithDrainedQueue), (["cluster-oob", qa_config.NoVirtualCluster], qa_cluster.TestClusterOob), ("cluster-instance-communication", qa_cluster.TestInstanceCommunication), (qa_rapi.Enabled, qa_rapi.TestVersion), (qa_rapi.Enabled, qa_rapi.TestEmptyCluster), (qa_rapi.Enabled, qa_rapi.TestRapiQuery), ]: RunTestIf(test, fn) def RunRepairDiskSizes(): """Run the repair disk-sizes test. """ RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes) def RunOsTests(): """Runs all tests related to gnt-os. """ os_enabled = ["os", qa_config.NoVirtualCluster] if qa_config.TestEnabled(qa_rapi.Enabled): rapi_getos = qa_rapi.GetOperatingSystems else: rapi_getos = None for fn in [ qa_os.TestOsList, qa_os.TestOsDiagnose, ]: RunTestIf(os_enabled, fn) for fn in [ qa_os.TestOsValid, qa_os.TestOsInvalid, qa_os.TestOsPartiallyValid, ]: RunTestIf(os_enabled, fn, rapi_getos) for fn in [ qa_os.TestOsModifyValid, qa_os.TestOsModifyInvalid, qa_os.TestOsStatesNonExisting, ]: RunTestIf(os_enabled, fn) def RunCommonInstanceTests(instance, inst_nodes): """Runs a few tests that are common to all disk types. """ RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance) RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled], qa_rapi.TestRapiStoppedInstanceConsole, instance) RunTestIf(["instance-shutdown", "instance-modify"], qa_instance.TestInstanceStoppedModify, instance) RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance) # Test shutdown/start via RAPI RunTestIf(["instance-shutdown", qa_rapi.Enabled], qa_rapi.TestRapiInstanceShutdown, instance) RunTestIf(["instance-shutdown", qa_rapi.Enabled], qa_rapi.TestRapiInstanceStartup, instance) RunTestIf("instance-list", qa_instance.TestInstanceList) RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance) RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance) RunTestIf(["instance-modify", qa_rapi.Enabled], qa_rapi.TestRapiInstanceModify, instance) RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance) RunTestIf(["instance-console", qa_rapi.Enabled], qa_rapi.TestRapiInstanceConsole, instance) RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames, instance) DOWN_TESTS = qa_config.Either([ "instance-reinstall", "instance-rename", "instance-grow-disk", ]) # shutdown instance for any 'down' tests RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance) # now run the 'down' state tests RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance) RunTestIf(["instance-reinstall", qa_rapi.Enabled], qa_rapi.TestRapiInstanceReinstall, instance) if qa_config.TestEnabled("instance-rename"): tgt_instance = qa_config.AcquireInstance() try: rename_source = instance.name rename_target = tgt_instance.name # perform instance rename to the same name RunTest(qa_instance.TestInstanceRenameAndBack, rename_source, rename_source) RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack, rename_source, rename_source) if rename_target is not None: # perform instance rename to a different name, if we have one configured RunTest(qa_instance.TestInstanceRenameAndBack, rename_source, rename_target) RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack, rename_source, rename_target) finally: tgt_instance.Release() RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance) # and now start the instance again RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance) RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance) RunTestIf("tags", qa_tags.TestInstanceTags, instance) if instance.disk_template == constants.DT_DRBD8: RunTestIf("cluster-verify", qa_cluster.TestClusterVerifyDisksBrokenDRBD, instance, inst_nodes) RunTestIf("cluster-verify", qa_cluster.TestClusterVerify) RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance) # Lists instances, too RunTestIf("node-list", qa_node.TestNodeList) # Some jobs have been run, let's test listing them RunTestIf("job-list", qa_job.TestJobList) def RunCommonNodeTests(): """Run a few common node tests. """ RunTestIf("node-volumes", qa_node.TestNodeVolumes) RunTestIf("node-storage", qa_node.TestNodeStorage) RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand) def RunGroupListTests(): """Run tests for listing node groups. """ RunTestIf("group-list", qa_group.TestGroupList) RunTestIf("group-list", qa_group.TestGroupListFields) def RunNetworkTests(): """Run tests for network management. """ RunTestIf("network", qa_network.TestNetworkAddRemove) RunTestIf("network", qa_network.TestNetworkConnect) RunTestIf(["network", "tags"], qa_network.TestNetworkTags) def RunFilterTests(): """Run tests for job filter management. """ RunTestIf("filters", qa_filters.TestFilterList) RunTestIf("filters", qa_filters.TestFilterListFields) RunTestIf("filters", qa_filters.TestFilterAddRemove) RunTestIf("filters", qa_filters.TestFilterReject) RunTestIf("filters", qa_filters.TestFilterOpCode) RunTestIf("filters", qa_filters.TestFilterReasonChain) RunTestIf("filters", qa_filters.TestFilterContinue) RunTestIf("filters", qa_filters.TestFilterAcceptPause) RunTestIf("filters", qa_filters.TestFilterWatermark) RunTestIf("filters", qa_filters.TestFilterRateLimit) RunTestIf("filters", qa_filters.TestAdHocReasonRateLimit) def RunGroupRwTests(): """Run tests for adding/removing/renaming groups. """ RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename) RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions) RunTestIf("group-rwops", qa_group.TestGroupModify) RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups) RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags, qa_group.GetDefaultGroup()) def RunExportImportTests(instance, inodes): """Tries to export and import the instance. @type inodes: list of nodes @param inodes: current nodes of the instance """ # FIXME: export explicitly bails out on file based storage. other non-lvm # based storage types are untested, though. Also note that import could still # work, but is deeply embedded into the "export" case. if qa_config.TestEnabled("instance-export"): RunTest(qa_instance.TestInstanceExportNoTarget, instance) pnode = inodes[0] expnode = qa_config.AcquireNode(exclude=pnode) try: name = RunTest(qa_instance.TestInstanceExport, instance, expnode) RunTest(qa_instance.TestBackupList, expnode) if qa_config.TestEnabled("instance-import"): newinst = qa_config.AcquireInstance() try: RunTest(qa_instance.TestInstanceImport, newinst, pnode, expnode, name) # Check if starting the instance works RunTest(qa_instance.TestInstanceStartup, newinst) RunTest(qa_instance.TestInstanceRemove, newinst) finally: newinst.Release() finally: expnode.Release() # FIXME: inter-cluster-instance-move crashes on file based instances :/ # See Issue 414. if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])): newinst = qa_config.AcquireInstance() try: tnode = qa_config.AcquireNode(exclude=inodes) try: RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst, inodes, tnode) finally: tnode.Release() finally: newinst.Release() def RunDaemonTests(instance): """Test the ganeti-watcher script. """ RunTest(qa_daemon.TestPauseWatcher) RunTestIf("instance-automatic-restart", qa_daemon.TestInstanceAutomaticRestart, instance) RunTestIf("instance-consecutive-failures", qa_daemon.TestInstanceConsecutiveFailures, instance) RunTest(qa_daemon.TestResumeWatcher) def RunHardwareFailureTests(instance, inodes): """Test cluster internal hardware failure recovery. """ RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance) RunTestIf(["instance-failover", qa_rapi.Enabled], qa_rapi.TestRapiInstanceFailover, instance) RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance) RunTestIf(["instance-migrate", qa_rapi.Enabled], qa_rapi.TestRapiInstanceMigrate, instance) if qa_config.TestEnabled("instance-replace-disks"): # We just need alternative secondary nodes, hence "- 1" othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes) try: RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance) RunTest(qa_instance.TestReplaceDisks, instance, inodes, othernodes) finally: qa_config.ReleaseManyNodes(othernodes) del othernodes if qa_config.TestEnabled("instance-recreate-disks"): try: acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes) othernodes = acquirednodes except qa_error.OutOfNodesError: if len(inodes) > 1: # If the cluster is not big enough, let's reuse some of the nodes, but # with different roles. In this way, we can test a DRBD instance even on # a 3-node cluster. acquirednodes = [qa_config.AcquireNode(exclude=inodes)] othernodes = acquirednodes + inodes[:-1] else: raise try: RunTest(qa_instance.TestRecreateDisks, instance, inodes, othernodes) finally: qa_config.ReleaseManyNodes(acquirednodes) if len(inodes) >= 2: RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1]) RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1]) RunTestIf("node-migrate", qa_node.TestNodeMigrate, inodes[0], inodes[1]) def RunExclusiveStorageTests(): """Test exclusive storage.""" if not qa_config.TestEnabled("cluster-exclusive-storage"): return node = qa_config.AcquireNode() try: old_es = qa_cluster.TestSetExclStorCluster(False) qa_node.TestExclStorSingleNode(node) qa_cluster.TestSetExclStorCluster(True) qa_cluster.TestExclStorSharedPv(node) if qa_config.TestEnabled("instance-add-plain-disk"): # Make sure that the cluster doesn't have any pre-existing problem qa_cluster.AssertClusterVerify() # Create and allocate instances instance1 = qa_instance.TestInstanceAddWithPlainDisk([node]) try: instance2 = qa_instance.TestInstanceAddWithPlainDisk([node]) try: # cluster-verify checks that disks are allocated correctly qa_cluster.AssertClusterVerify() # Remove instances qa_instance.TestInstanceRemove(instance2) qa_instance.TestInstanceRemove(instance1) finally: instance2.Release() finally: instance1.Release() if qa_config.TestEnabled("instance-add-drbd-disk"): snode = qa_config.AcquireNode() try: qa_cluster.TestSetExclStorCluster(False) instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode]) try: qa_cluster.TestSetExclStorCluster(True) exp_err = [constants.CV_EINSTANCEUNSUITABLENODE] qa_cluster.AssertClusterVerify(fail=True, errors=exp_err) qa_instance.TestInstanceRemove(instance) finally: instance.Release() finally: snode.Release() qa_cluster.TestSetExclStorCluster(old_es) finally: node.Release() def RunCustomSshPortTests(): """Test accessing nodes with custom SSH ports. This requires removing nodes, adding them to a new group, and then undoing the change. """ if not qa_config.TestEnabled("group-custom-ssh-port"): return std_port = netutils.GetDaemonPort(constants.SSH) port = 211 master = qa_config.GetMasterNode() with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes: # Checks if the node(s) could be contacted through IPv6. # If yes, better skip the whole test. for node in nodes: if qa_utils.UsesIPv6Connection(node.primary, std_port): print("Node %s is likely to be reached using IPv6," "skipping the test" % (node.primary, )) return for node in nodes: qa_node.NodeRemove(node) with qa_iptables.RulesContext() as r: with qa_group.NewGroupCtx() as group: qa_group.ModifyGroupSshPort(r, group, nodes, port) for node in nodes: qa_node.NodeAdd(node, group=group) # Make sure that the cluster doesn't have any pre-existing problem qa_cluster.AssertClusterVerify() # Create and allocate instances instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes) try: instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes) try: # cluster-verify checks that disks are allocated correctly qa_cluster.AssertClusterVerify() # Remove instances qa_instance.TestInstanceRemove(instance2) qa_instance.TestInstanceRemove(instance1) finally: instance2.Release() finally: instance1.Release() for node in nodes: qa_node.NodeRemove(node) for node in nodes: qa_node.NodeAdd(node) qa_cluster.AssertClusterVerify() def _BuildSpecDict(par, mn, st, mx): return { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: {par: mn}, constants.ISPECS_MAX: {par: mx}, }], constants.ISPECS_STD: {par: st}, } def _BuildDoubleSpecDict(index, par, mn, st, mx): new_spec = { constants.ISPECS_MINMAX: [{}, {}], } if st is not None: new_spec[constants.ISPECS_STD] = {par: st} new_spec[constants.ISPECS_MINMAX][index] = { constants.ISPECS_MIN: {par: mn}, constants.ISPECS_MAX: {par: mx}, } return new_spec def TestIPolicyPlainInstance(): """Test instance policy interaction with instances""" params = ["memory-size", "cpu-count", "disk-count", "disk-size", "nic-count"] if not qa_config.IsTemplateSupported(constants.DT_PLAIN): print("Template %s not supported" % constants.DT_PLAIN) return # This test assumes that the group policy is empty (_, old_specs) = qa_cluster.TestClusterSetISpecs() # We also assume to have only one min/max bound assert len(old_specs[constants.ISPECS_MINMAX]) == 1 node = qa_config.AcquireNode() try: # Log of policy changes, list of tuples: # (full_change, incremental_change, policy_violated) history = [] instance = qa_instance.TestInstanceAddWithPlainDisk([node]) try: policyerror = [constants.CV_EINSTANCEPOLICY] for par in params: (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par) # Some specs must be multiple of 4 new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4) history.append((None, new_spec, True)) if iminval > 0: # Some specs must be multiple of 4 if iminval >= 4: upper = iminval - 4 else: upper = iminval - 1 new_spec = _BuildSpecDict(par, 0, upper, upper) history.append((None, new_spec, True)) history.append((old_specs, None, False)) # Test with two instance specs double_specs = copy.deepcopy(old_specs) double_specs[constants.ISPECS_MINMAX] = \ double_specs[constants.ISPECS_MINMAX] * 2 (par1, par2) = params[0:2] (_, imaxval1) = qa_instance.GetInstanceSpec(instance.name, par1) (_, imaxval2) = qa_instance.GetInstanceSpec(instance.name, par2) old_minmax = old_specs[constants.ISPECS_MINMAX][0] history.extend([ (double_specs, None, False), # The first min/max limit is being violated (None, _BuildDoubleSpecDict(0, par1, imaxval1 + 4, imaxval1 + 4, imaxval1 + 4), False), # Both min/max limits are being violated (None, _BuildDoubleSpecDict(1, par2, imaxval2 + 4, None, imaxval2 + 4), True), # The second min/max limit is being violated (None, _BuildDoubleSpecDict(0, par1, old_minmax[constants.ISPECS_MIN][par1], old_specs[constants.ISPECS_STD][par1], old_minmax[constants.ISPECS_MAX][par1]), False), (old_specs, None, False), ]) # Apply the changes, and check policy violations after each change qa_cluster.AssertClusterVerify() for (new_specs, diff_specs, failed) in history: qa_cluster.TestClusterSetISpecs(new_specs=new_specs, diff_specs=diff_specs) if failed: qa_cluster.AssertClusterVerify(warnings=policyerror) else: qa_cluster.AssertClusterVerify() qa_instance.TestInstanceRemove(instance) finally: instance.Release() # Now we replay the same policy changes, and we expect that the instance # cannot be created for the cases where we had a policy violation above for (new_specs, diff_specs, failed) in history: qa_cluster.TestClusterSetISpecs(new_specs=new_specs, diff_specs=diff_specs) if failed: qa_instance.TestInstanceAddWithPlainDisk([node], fail=True) # Instance creation with no policy violation has been tested already finally: node.Release() def IsExclusiveStorageInstanceTestEnabled(): test_name = "exclusive-storage-instance-tests" if qa_config.TestEnabled(test_name): vgname = qa_config.get("vg-name", constants.DEFAULT_VG) vgscmd = utils.ShellQuoteArgs([ "vgs", "--noheadings", "-o", "pv_count", vgname, ]) nodes = qa_config.GetConfig()["nodes"] for node in nodes: try: pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd)) except Exception as e: msg = ("Cannot get the number of PVs on %s, needed by '%s': %s" % (node.primary, test_name, e)) raise qa_error.Error(msg) if pvnum < 2: raise qa_error.Error("Node %s has not enough PVs (%s) to run '%s'" % (node.primary, pvnum, test_name)) res = True else: res = False return res def RunInstanceTests(): """Create and exercise instances.""" requested_conversions = qa_config.get("convert-disk-templates", []) supported_conversions = \ set(requested_conversions).difference(constants.DTS_NOT_CONVERTIBLE_TO) for (test_name, templ, create_fun, num_nodes) in \ qa_instance.available_instance_tests: if (qa_config.TestEnabled(test_name) and qa_config.IsTemplateSupported(templ)): inodes = qa_config.AcquireManyNodes(num_nodes) try: # run instance tests with default hvparams print(_FormatHeader("Starting instance tests with default HVparams")) RunInstanceTestsFull(create_fun, inodes, supported_conversions, templ) # iterate through alternating hvparam values (if enabled) if qa_config.TestEnabled("instance-iterate-hvparams"): hvparam_iterations = qa_cluster.PrepareHvParameterSets() for param, test_data in hvparam_iterations.items(): for value in test_data["values"]: print(_FormatHeader("Starting reduced number of instance tests " "with hypervisor parameter %s=%s" % (param, value))) qa_cluster.AssertClusterHvParameterModify(param, value) RunInstanceTestsReduced(create_fun, inodes) qa_cluster.AssertClusterHvParameterModify( param, test_data["reset_value"]) else: test_desc = "Iterating through hypervisor parameter values" ReportTestSkip(test_desc, "instance-iterate-hvparams") finally: qa_config.ReleaseManyNodes(inodes) else: test_desc = "Creating instances of template %s" % templ if not qa_config.TestEnabled(test_name): ReportTestSkip(test_desc, test_name) else: ReportTestSkip(test_desc, "disk template %s" % templ) def RunInstanceTestsFull(create_fun, inodes, supported_conversions, templ): instance = RunTest(create_fun, inodes) try: RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown, instance) RunTestIf("instance-communication", qa_instance.TestInstanceCommunication, instance, qa_config.GetMasterNode()) RunTestIf("cluster-epo", qa_cluster.TestClusterEpo) RunDaemonTests(instance) for node in inodes: RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node, templ == constants.DT_DRBD8) if len(inodes) > 1: RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit, constants.INITIAL_NODE_GROUP_NAME, inodes[0].primary, inodes[1].primary) # This test will run once but it will cover all the supported # user-provided disk template conversions if qa_config.TestEnabled("instance-convert-disk"): if (len(supported_conversions) > 1 and instance.disk_template in supported_conversions): RunTest(qa_instance.TestInstanceShutdown, instance) RunTest(qa_instance.TestInstanceConvertDiskTemplate, instance, supported_conversions) RunTest(qa_instance.TestInstanceStartup, instance) # At this point we clear the set because the requested conversions # has been tested supported_conversions.clear() else: test_desc = "Converting instance of template %s" % templ ReportTestSkip(test_desc, "conversion feature") RunTestIf("instance-modify-disks", qa_instance.TestInstanceModifyDisks, instance) RunCommonInstanceTests(instance, inodes) if qa_config.TestEnabled("instance-modify-primary"): othernode = qa_config.AcquireNode() RunTest(qa_instance.TestInstanceModifyPrimaryAndBack, instance, inodes[0], othernode) othernode.Release() RunGroupListTests() RunExportImportTests(instance, inodes) RunHardwareFailureTests(instance, inodes) RunRepairDiskSizes() RunTestIf(["rapi", "instance-data-censorship"], qa_rapi.TestInstanceDataCensorship, instance, inodes) RunTest(qa_instance.TestInstanceRemove, instance) finally: instance.Release() del instance qa_cluster.AssertClusterVerify() def RunInstanceTestsReduced(create_fun, inodes): instance = RunTest(create_fun, inodes) try: RunCommonInstanceTests(instance, inodes) RunTest(qa_instance.TestInstanceRemove, instance) finally: instance.Release() del instance qa_cluster.AssertClusterVerify() def RunMonitoringTests(): RunTestIf("mon-collector", qa_monitoring.TestInstStatusCollector) PARALLEL_TEST_DICT = { "parallel-failover": qa_performance.TestParallelInstanceFailover, "parallel-migration": qa_performance.TestParallelInstanceMigration, "parallel-replace-disks": qa_performance.TestParallelInstanceReplaceDisks, "parallel-reboot": qa_performance.TestParallelInstanceReboot, "parallel-reinstall": qa_performance.TestParallelInstanceReinstall, "parallel-rename": qa_performance.TestParallelInstanceRename, } def RunPerformanceTests(): if not qa_config.TestEnabled("performance"): ReportTestSkip("performance related tests", "performance") return # For reproducable performance, run performance tests with the watcher # paused. qa_utils.AssertCommand(["gnt-cluster", "watcher", "pause", "4h"]) if qa_config.TestEnabled("jobqueue-performance"): RunTest(qa_performance.TestParallelMaxInstanceCreationPerformance) RunTest(qa_performance.TestParallelNodeCountInstanceCreationPerformance) instances = qa_performance.CreateAllInstances() RunTest(qa_performance.TestParallelModify, instances) RunTest(qa_performance.TestParallelInstanceOSOperations, instances) RunTest(qa_performance.TestParallelInstanceQueries, instances) qa_performance.RemoveAllInstances(instances) RunTest(qa_performance.TestJobQueueSubmissionPerformance) if qa_config.TestEnabled("parallel-performance"): if qa_config.IsTemplateSupported(constants.DT_DRBD8): RunTest(qa_performance.TestParallelDRBDInstanceCreationPerformance) if qa_config.IsTemplateSupported(constants.DT_PLAIN): RunTest(qa_performance.TestParallelPlainInstanceCreationPerformance) # Preparations need to be made only if some of these tests are enabled if qa_config.IsTemplateSupported(constants.DT_DRBD8) and \ qa_config.TestEnabled(qa_config.Either(list(PARALLEL_TEST_DICT))): inodes = qa_config.AcquireManyNodes(2) try: instance = qa_instance.TestInstanceAddWithDrbdDisk(inodes) try: for (test_name, test_fn) in PARALLEL_TEST_DICT.items(): RunTestIf(test_name, test_fn, instance) finally: instance.Release() qa_instance.TestInstanceRemove(instance) finally: qa_config.ReleaseManyNodes(inodes) qa_utils.AssertCommand(["gnt-cluster", "watcher", "continue"]) def RunQa(): """Main QA body. """ RunTestBlock(RunEnvTests) SetupCluster() RunTestBlock(RunClusterTests) RunTestBlock(RunOsTests) RunTestIf("tags", qa_tags.TestClusterTags) RunTestBlock(RunCommonNodeTests) RunTestBlock(RunGroupListTests) RunTestBlock(RunGroupRwTests) RunTestBlock(RunNetworkTests) RunTestBlock(RunFilterTests) # The master shouldn't be readded or put offline; "delay" needs a non-master # node to test pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode()) try: RunTestIf("node-readd", qa_node.TestNodeReadd, pnode) RunTestIf("node-modify", qa_node.TestNodeModify, pnode) RunTestIf("delay", qa_cluster.TestDelay, pnode) finally: pnode.Release() # Make sure the cluster is clean before running instance tests qa_cluster.AssertClusterVerify() pnode = qa_config.AcquireNode() try: RunTestIf("tags", qa_tags.TestNodeTags, pnode) if qa_rapi.Enabled(): RunTest(qa_rapi.TestNode, pnode) if (qa_config.TestEnabled("instance-add-plain-disk") and qa_config.IsTemplateSupported(constants.DT_PLAIN)): # Normal instance allocation via RAPI for use_client in [True, False]: rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode, use_client) try: if qa_config.TestEnabled("instance-plain-rapi-common-tests"): RunCommonInstanceTests(rapi_instance, [pnode]) RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client) finally: rapi_instance.Release() del rapi_instance # Multi-instance allocation rapi_instance_one, rapi_instance_two = \ RunTest(qa_rapi.TestRapiInstanceMultiAlloc, pnode) try: RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_one, True) RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_two, True) finally: rapi_instance_one.Release() rapi_instance_two.Release() finally: pnode.Release() config_list = [ ("default-instance-tests", lambda: None, lambda _: None), (IsExclusiveStorageInstanceTestEnabled, lambda: qa_cluster.TestSetExclStorCluster(True), qa_cluster.TestSetExclStorCluster), ] for (conf_name, setup_conf_f, restore_conf_f) in config_list: if qa_config.TestEnabled(conf_name): oldconf = setup_conf_f() RunTestBlock(RunInstanceTests) restore_conf_f(oldconf) pnode = qa_config.AcquireNode() try: if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]): for shutdown in [False, True]: instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode]) try: expnode = qa_config.AcquireNode(exclude=pnode) try: if shutdown: # Stop instance before exporting and removing it RunTest(qa_instance.TestInstanceShutdown, instance) RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode) RunTest(qa_instance.TestBackupList, expnode) finally: expnode.Release() finally: instance.Release() del expnode del instance qa_cluster.AssertClusterVerify() finally: pnode.Release() if qa_rapi.Enabled(): RunTestIf("filters", qa_rapi.TestFilters) RunTestIf("cluster-upgrade", qa_cluster.TestUpgrade) RunTestBlock(RunExclusiveStorageTests) RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"], TestIPolicyPlainInstance) RunTestBlock(RunCustomSshPortTests) RunTestIf( "instance-add-restricted-by-disktemplates", qa_instance.TestInstanceCreationRestrictedByDiskTemplates) RunTestIf("instance-add-osparams", qa_instance.TestInstanceAddOsParams) RunTestIf("instance-add-osparams", qa_instance.TestSecretOsParams) # Test removing instance with offline drbd secondary if qa_config.TestEnabled(["instance-remove-drbd-offline", "instance-add-drbd-disk"]): # Make sure the master is not put offline snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode()) try: pnode = qa_config.AcquireNode(exclude=snode) try: instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode]) set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes") set_online = lambda node: qa_node.MakeNodeOffline(node, "no") RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode, set_offline, set_online) finally: pnode.Release() finally: snode.Release() qa_cluster.AssertClusterVerify() RunTestBlock(RunMonitoringTests) RunPerformanceTests() RunTestIf("cluster-destroy", qa_node.TestNodeRemoveAll) RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy) @UsesRapiClient def main(): """Main program. """ colors.check_for_colors() parser = optparse.OptionParser(usage="%prog [options] ") parser.add_option("--yes-do-it", dest="yes_do_it", action="store_true", help="Really execute the tests") (opts, args) = parser.parse_args() if len(args) == 1: (config_file, ) = args else: parser.error("Wrong number of arguments.") if not opts.yes_do_it: print ("Executing this script irreversibly destroys any Ganeti\n" "configuration on all nodes involved. If you really want\n" "to start testing, supply the --yes-do-it option.") sys.exit(1) qa_config.Load(config_file) for node in qa_config.GetAllNodes(): qa_utils.StartMultiplexer(node.primary) print("SSH command for node %s: %s" % (node.primary, utils.ShellQuoteArgs(qa_utils.GetSSHCommand(node.primary, "")))) try: RunQa() finally: qa_utils.CloseMultiplexers() if __name__ == "__main__": main() ganeti-3.1.0~rc2/qa/patch/000075500000000000000000000000001476477700300153045ustar00rootroot00000000000000ganeti-3.1.0~rc2/qa/patch/order000064400000000000000000000000001476477700300163300ustar00rootroot00000000000000ganeti-3.1.0~rc2/qa/qa-patch.json000064400000000000000000000000031476477700300165670ustar00rootroot00000000000000[] ganeti-3.1.0~rc2/qa/qa-sample.json000064400000000000000000000173751476477700300167750ustar00rootroot00000000000000{ "# Note:": null, "# This file is stored in the JSON format and does not support": null, "# comments. As a work-around, comments are keys starting with a hash": null, "# sign (#).": null, "name": "xen-test", "# Name used for renaming cluster": null, "rename": "xen-test-rename", "# Directory versions of the two installed versions of ganeti, for upgrade testing." : null, "# dir-version is the version installed at the beginning, and the majority of tests" : null, "# run in; the other-dir-version, which mus the adjacent, is the version a detour" : null, "# is made through." : null, "dir-version": "2.11", "other-dir-version": "2.10", "# instances of the following types are to remain over upgrades." : null, "upgrade-instances": ["drbd", "plain"], "# Virtual cluster": null, "#vcluster-master": "xen-vcluster", "#vcluster-basedir": "/srv/ganeti/vcluster", "enabled-hypervisors": "xen-pvm", "# Dict of hypervisor name and parameters (like on the cmd line)": null, "hypervisor-parameters": {}, "# Backend parameters (like on the cmd line)": null, "backend-parameters": "", "# Dict of OS name and parameters (like on the cmd line)": null, "os-parameters": {}, "# Dict of OS name and value dict of hypervisor parameters": null, "os-hvp": {}, "primary_ip_version": 4, "# Name of the LVM group for the cluster": null, "vg-name": "xenvg", "# Cluster-level value of the exclusive-storage flag": null, "exclusive-storage": null, "# Only enable disk templates that the QA machines can actually use.": null, "enabled-disk-templates": [ "plain", "drbd", "diskless" ], "# Only test the following disk template conversions": null, "convert-disk-templates": [ "plain", "drbd", "diskless" ], "# Default file storage directories": null, "default-file-storage-dir": "/srv/ganeti/file-storage", "default-shared-file-storage-dir": "/srv/ganeti/shared-file-storage", "default-gluster-storage-dir": "/srv/ganeti/gluster-file-storage", "# Additional arguments for initializing cluster": null, "cluster-init-args": [], "# Network interface for master role": null, "#master-netdev": "xen-br0", "# Default network interface parameters": null, "#default-nicparams": { "mode": "bridged", "link": "xen-br0" }, "os": "debian-etch", "maxmem": "1024M", "minmem": "512M", "# Instance policy specs": null, "#ispec_cpu_count_max": null, "#ispec_cpu_count_min": null, "#ispec_cpu_count_std": null, "#ispec_disk_count_max": null, "#ispec_disk_count_min": null, "#ispec_disk_count_std": null, "#ispec_disk_size_max": null, "ispec_disk_size_min": 512, "#ispec_disk_size_std": null, "ispec_mem_size_max": 1024, "#ispec_mem_size_min": null, "#ispec_mem_size_std": null, "#ispec_nic_count_max": null, "#ispec_nic_count_min": null, "#ispec_nic_count_std": null, "# Lists of disks": null, "disks": [ { "size": "1G", "spindles": 2, "name": "disk0", "growth": "2G", "spindles-growth": 1 }, { "size": "512M", "spindles": 1, "name": "disk1", "growth": "768M", "spindles-growth": 0 } ], "# Script to check instance status": null, "instance-check": null, "# Regular expression to ignore existing tags": null, "ignore-tags-re": null, "# Repository containing additional files for the QA": null, "# qa-storage": "http://example.com", "modify_ssh_setup": true, "nodes": [ { "# Master node": null, "primary": "xen-test-0", "secondary": "192.0.2.1" }, { "primary": "xen-test-1", "secondary": "192.0.2.2" } ], "instances": [ { "name": "xen-test-inst1", "# Static MAC address": null, "#nic.mac/0": "AA:00:00:11:11:11" }, { "name": "xen-test-inst2", "# Static MAC address": null, "#nic.mac/0": "AA:00:00:22:22:22" } ], "groups": { "group-with-nodes": "default", "inexistent-groups": [ "group1", "group2", "group3" ] }, "networks": { "inexistent-networks": [ "network1", "network2", "network3" ] }, "tests": { "# Whether tests are enabled or disabled by default": null, "default": true, "env": true, "os": true, "tags": true, "rapi": true, "performance": true, "test-jobqueue": true, "delay": true, "create-cluster": true, "cluster-verify": true, "cluster-info": true, "cluster-burnin": true, "cluster-command": true, "cluster-copyfile": true, "cluster-master-failover": true, "cluster-renew-crypto": true, "cluster-destroy": true, "cluster-rename": true, "cluster-reserved-lvs": true, "cluster-modify": true, "cluster-oob": true, "cluster-instance-communication": true, "cluster-epo": true, "cluster-redist-conf": true, "cluster-repair-disk-sizes": true, "cluster-exclusive-storage": true, "cluster-instance-policy": true, "cluster-upgrade": true, "haskell-confd": true, "htools": true, "group-list": true, "group-rwops": true, "group-custom-ssh-port": true, "network": false, "node-list": true, "node-info": true, "node-volumes": true, "node-readd": true, "node-storage": true, "node-modify": true, "node-oob": true, "# These tests need at least three nodes": null, "node-evacuate": false, "node-migrate": false, "# This test needs at least two nodes": null, "node-failover": false, "instance-add-plain-disk": true, "instance-add-file": true, "instance-add-shared-file": true, "instance-add-drbd-disk": true, "instance-add-diskless": true, "instance-add-rbd": true, "instance-add-gluster": true, "instance-add-restricted-by-disktemplates": true, "instance-convert-disk": true, "instance-plain-rapi-common-tests": true, "instance-remove-drbd-offline": true, "instance-export": true, "instance-failover": true, "instance-grow-disk": true, "instance-import": true, "instance-info": true, "instance-list": true, "instance-migrate": true, "instance-modify": true, "instance-iterate-hvparams": true, "instance-modify-primary": true, "instance-modify-disks": false, "instance-reboot": true, "instance-reinstall": true, "instance-rename": true, "instance-shutdown": true, "instance-device-names": true, "instance-device-hotplug": false, "instance-user-down": true, "instance-communication": true, "job-list": true, "jobqueue-performance": true, "parallel-performance": true, "# cron/ganeti-watcher should be disabled for these tests": null, "instance-automatic-restart": false, "instance-consecutive-failures": false, "# This test might fail with certain hypervisor types, depending": null, "# on whether they support the `gnt-instance console' command.": null, "instance-console": false, "# Disabled by default because they take rather long": null, "instance-replace-disks": false, "instance-recreate-disks": false, "# Whether to test the tools/move-instance utility": null, "inter-cluster-instance-move": false, "# Run instance tests with different cluster configurations": null, "default-instance-tests": true, "exclusive-storage-instance-tests": false, "mon-collector": true }, "options": { "burnin-instances": 2, "burnin-disk-template": "drbd", "burnin-in-parallel": false, "burnin-check-instances": false, "burnin-rename": "xen-test-rename", "burnin-reboot": true, "reboot-types": ["soft", "hard", "full"], "use-iallocators": false, "# Uncomment if you want to run the whole cluster on a different SSH port": null, "# ssh-port": 222 }, "# vim: set syntax=javascript :": null } ganeti-3.1.0~rc2/qa/qa_cluster.py000064400000000000000000001752261476477700300167360ustar00rootroot00000000000000# # # Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Cluster related QA tests. """ import os import re import tempfile import time from ganeti import _constants from ganeti import constants from ganeti import compat from ganeti import utils from ganeti import pathutils from qa import qa_config from qa import qa_daemon from qa import qa_error from qa import qa_instance from qa import qa_job_utils from qa import qa_logging from qa import qa_rapi from qa import qa_utils from qa_utils import AssertEqual, AssertCommand, AssertRedirectedCommand, \ GetCommandOutput, CheckFileUnmodified # Prefix for LVM volumes created by QA code during tests _QA_LV_PREFIX = "qa-" #: cluster verify command _CLUSTER_VERIFY = ["gnt-cluster", "verify"] def _RemoveFileFromAllNodes(filename): """Removes a file from all nodes. """ for node in qa_config.get("nodes"): AssertCommand(["rm", "-f", filename], node=node) def _CheckFileOnAllNodes(filename, content): """Verifies the content of the given file on all nodes. """ cmd = utils.ShellQuoteArgs(["cat", filename]) for node in qa_config.get("nodes"): AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content) def _GetClusterField(field_path): """Get the value of a cluster field. @type field_path: list of strings @param field_path: Names of the groups/fields to navigate to get the desired value, e.g. C{["Default node parameters", "oob_program"]} @return: The effective value of the field (the actual type depends on the chosen field) """ assert isinstance(field_path, list) assert field_path ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"]) for key in field_path: ret = ret[key] return ret # Cluster-verify errors (date, "ERROR", then error code) _CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):") def _GetCVErrorCodes(cvout): errs = set() warns = set() for l in cvout.splitlines(): m = _CVERROR_RE.match(l) if m: etype = m.group(1) ecode = m.group(2) if etype == "ERROR": errs.add(ecode) elif etype == "WARNING": warns.add(ecode) return (errs, warns) def _CheckVerifyErrors(actual, expected, etype): exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected) if not actual.issuperset(exp_codes): missing = exp_codes.difference(actual) raise qa_error.Error("Cluster-verify didn't return these expected" " %ss: %s" % (etype, utils.CommaJoin(missing))) def _CheckVerifyNoWarnings(actual, expected): exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected) excess = actual.intersection(exp_codes) if excess: raise qa_error.Error("Cluster-verify returned these warnings:" " %s" % (utils.CommaJoin(excess))) def PrepareHvParameterSets(): """Assemble a dictionary of assessable hypervisor parameters @return: dict with hv-params, their values to iterate through and the initial value (to reset after tests) """ default_hv = qa_config.GetDefaultHypervisor() hv_params = _GetClusterField(["Hypervisor parameters", default_hv]) toggle_bool_params = [] toggle_value_params = {} if default_hv == constants.HT_KVM: toggle_bool_params = [ "acpi", "use_chroot", "use_guest_agent", "use_localtime", ] # in general, we toggle through all values known to Ganeti, except: # we cannot use all available disk_types because some require # special backing devices/files (e.g. pflash, mtd) toggle_value_params = { "disk_aio": constants.HT_KVM_VALID_AIO_TYPES, "disk_cache": constants.HT_VALID_CACHE_TYPES, "usb_mouse": constants.HT_KVM_VALID_MOUSE_TYPES, "disk_type": ["ide", "paravirtual"], "soundhw": ["ac97", "hda"], } assembled_tests = {} for param in toggle_bool_params: toggled_value = not hv_params[param] assembled_tests[param] = { "values": [toggled_value], "reset_value": hv_params[param], } for param, values in toggle_value_params.items(): list_values = list(values) if hv_params[param] in list_values: # some HVParams accept 'None' as valid value which is never part # of the list of allowed values, hence we need this check # # if the current value of the HVParam is part of the list of # allowed values, remove it to avoid unnecessary test cycles list_values.remove(hv_params[param]) if hv_params[param] is None: reset_value = "" else: reset_value = hv_params[param] assembled_tests[param] = { "values": list_values, "reset_value": reset_value, } return assembled_tests def AssertClusterHvParameterModify(param, value): """Modify the given hypervisor parameter @type param: string @param param: name of the hv-param to modify @type value: string or bool @param value: value to assign """ default_hv = qa_config.GetDefaultHypervisor() AssertCommand(["gnt-cluster", "modify", "-H", "%s:%s=%s" % (default_hv, param, value)]) def AssertClusterVerify(fail=False, errors=None, warnings=None, no_warnings=None): """Run cluster-verify and check the result, ignoring warnings by default. @type fail: bool @param fail: if cluster-verify is expected to fail instead of succeeding. @type errors: list of tuples @param errors: List of CV_XXX errors that are expected; if specified, all the errors listed must appear in cluster-verify output. A non-empty value implies C{fail=True}. @type warnings: list of tuples @param warnings: List of CV_XXX warnings that are expected to be raised; if specified, all the errors listed must appear in cluster-verify output. @type no_warnings: list of tuples @param no_warnings: List of CV_XXX warnings that we expect NOT to be raised. """ cvcmd = "gnt-cluster verify" mnode = qa_config.GetMasterNode() if errors or warnings or no_warnings: with CheckFileUnmodified(mnode.primary, pathutils.CLUSTER_CONF_FILE): cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes", fail=(fail or errors)) print(cvout) (act_errs, act_warns) = _GetCVErrorCodes(cvout) if errors: _CheckVerifyErrors(act_errs, errors, "error") if warnings: _CheckVerifyErrors(act_warns, warnings, "warning") if no_warnings: _CheckVerifyNoWarnings(act_warns, no_warnings) else: with CheckFileUnmodified(mnode.primary, pathutils.CLUSTER_CONF_FILE): AssertCommand(cvcmd, fail=fail, node=mnode) # data for testing failures due to bad keys/values for disk parameters _FAIL_PARAMS = ["nonexistent:resync-rate=1", "drbd:nonexistent=1", "drbd:resync-rate=invalid", ] def TestClusterInitDisk(): """gnt-cluster init -D""" name = qa_config.get("name") for param in _FAIL_PARAMS: AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True) def TestClusterInit(): """gnt-cluster init""" # If we don't modify the SSH setup by Ganeti, we have to ensure connectivity # before master = qa_config.GetMasterNode() if not qa_config.GetModifySshSetup(): (key_type, _, priv_key_file, pub_key_file, auth_key_path) = \ qa_config.GetSshConfig() AssertCommand("echo -e 'y\n' | ssh-keygen -t %s -f %s -q -N ''" % (key_type, priv_key_file)) AssertCommand("cat %s >> %s" % (pub_key_file, auth_key_path)) for node in qa_config.get("nodes"): if node != master: for key_file in [priv_key_file, pub_key_file]: AssertCommand("scp -oStrictHostKeyChecking=no %s %s:%s" % (key_file, node.primary, key_file)) AssertCommand("ssh %s \'cat %s >> %s\'" % (node.primary, pub_key_file, auth_key_path)) # Initialize cluster enabled_disk_templates = qa_config.GetEnabledDiskTemplates() cmd = [ "gnt-cluster", "init", "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4), "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()), "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates), ] if not qa_config.GetModifySshSetup(): cmd.append("--no-ssh-init") if constants.DT_FILE in enabled_disk_templates: cmd.append( "--file-storage-dir=%s" % qa_config.get("default-file-storage-dir", pathutils.DEFAULT_FILE_STORAGE_DIR)) if constants.DT_SHARED_FILE in enabled_disk_templates: cmd.append( "--shared-file-storage-dir=%s" % qa_config.get("default-shared-file-storage-dir", pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR)) if constants.DT_GLUSTER in enabled_disk_templates: cmd.append( "--gluster-storage-dir=%s" % qa_config.get("default-gluster-storage-dir", pathutils.DEFAULT_GLUSTER_STORAGE_DIR)) for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count", "nic-count"): spec_values = [] for spec_val in ("min", "max", "std"): spec = qa_config.get("ispec_%s_%s" % (spec_type.replace("-", "_"), spec_val), None) if spec is not None: spec_values.append("%s=%d" % (spec_val, spec)) if spec_values: cmd.append("--specs-%s=%s" % (spec_type, ",".join(spec_values))) master = qa_config.GetMasterNode() if master.secondary: cmd.append("--secondary-ip=%s" % master.secondary) if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()): vgname = qa_config.get("vg-name", constants.DEFAULT_VG) if vgname: cmd.append("--vg-name=%s" % vgname) else: raise qa_error.Error("Please specify a volume group if you enable" " lvm-based disk templates in the QA.") master_netdev = qa_config.get("master-netdev", None) if master_netdev: cmd.append("--master-netdev=%s" % master_netdev) nicparams = qa_config.get("default-nicparams", None) if nicparams: cmd.append("--nic-parameters=%s" % ",".join(utils.FormatKeyValue(nicparams))) # Cluster value of the exclusive-storage node parameter e_s = qa_config.get("exclusive-storage") if e_s is not None: cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s]) else: e_s = False qa_config.SetExclusiveStorage(e_s) extra_args = qa_config.get("cluster-init-args") if extra_args: # This option was removed in 2.10, but in order to not break QA of older # branches we remove it from the extra_args if it is in there. opt_drbd_storage = "--no-drbd-storage" if opt_drbd_storage in extra_args: extra_args.remove(opt_drbd_storage) cmd.extend(extra_args) cmd.append(qa_config.get("name")) AssertCommand(cmd) cmd = ["gnt-cluster", "modify"] # hypervisor parameter modifications hvp = qa_config.get("hypervisor-parameters", {}) for k, v in hvp.items(): cmd.extend(["-H", "%s:%s" % (k, v)]) # backend parameter modifications bep = qa_config.get("backend-parameters", "") if bep: cmd.extend(["-B", bep]) if len(cmd) > 2: AssertCommand(cmd) # OS parameters osp = qa_config.get("os-parameters", {}) for k, v in osp.items(): AssertCommand(["gnt-os", "modify", "-O", v, k]) # OS hypervisor parameters os_hvp = qa_config.get("os-hvp", {}) for os_name in os_hvp: for hv, hvp in os_hvp[os_name].items(): AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name]) def TestClusterRename(): """gnt-cluster rename""" cmd = ["gnt-cluster", "rename", "-f"] original_name = qa_config.get("name") rename_target = qa_config.get("rename", None) if rename_target is None: print(qa_logging.FormatError('"rename" entry is missing')) return for data in [ cmd + [rename_target], _CLUSTER_VERIFY, cmd + [original_name], _CLUSTER_VERIFY, ]: AssertCommand(data) def TestClusterOob(): """out-of-band framework""" oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID() AssertCommand(_CLUSTER_VERIFY) AssertCommand(["gnt-cluster", "modify", "--node-parameters", "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" % utils.NewUUID()]) AssertCommand(_CLUSTER_VERIFY, fail=True) AssertCommand(["touch", oob_path_exists]) AssertCommand(["chmod", "0400", oob_path_exists]) AssertCommand(["gnt-cluster", "copyfile", oob_path_exists]) try: AssertCommand(["gnt-cluster", "modify", "--node-parameters", "oob_program=%s" % oob_path_exists]) AssertCommand(_CLUSTER_VERIFY, fail=True) AssertCommand(["chmod", "0500", oob_path_exists]) AssertCommand(["gnt-cluster", "copyfile", oob_path_exists]) AssertCommand(_CLUSTER_VERIFY) finally: AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists]) AssertCommand(["gnt-cluster", "modify", "--node-parameters", "oob_program="]) def TestClusterEpo(): """gnt-cluster epo""" master = qa_config.GetMasterNode() # Assert that OOB is unavailable for all nodes result_output = GetCommandOutput(master.primary, "gnt-node list --verbose --no-headers -o" " powered") AssertEqual(compat.all(powered == "(unavail)" for powered in result_output.splitlines()), True) # Conflicting AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True) # --all doesn't expect arguments AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True) # Unless --all is given master is not allowed to be in the list AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True) with qa_job_utils.PausedWatcher(): # This shouldn't fail AssertCommand(["gnt-cluster", "epo", "-f", "--all"]) # All instances should have been stopped now result_output = GetCommandOutput(master.primary, "gnt-instance list --no-headers -o status") # ERROR_down because the instance is stopped but not recorded as such AssertEqual(compat.all(status == "ERROR_down" for status in result_output.splitlines()), True) # Now start everything again AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"]) # All instances should have been started now result_output = GetCommandOutput(master.primary, "gnt-instance list --no-headers -o status") AssertEqual(compat.all(status == "running" for status in result_output.splitlines()), True) def TestClusterVerify(): """gnt-cluster verify""" AssertCommand(_CLUSTER_VERIFY) AssertCommand(["gnt-cluster", "verify-disks"]) def TestClusterVerifyDisksBrokenDRBD(instance, inst_nodes): """gnt-cluster verify-disks with broken DRBD""" qa_daemon.TestPauseWatcher() try: info = qa_instance.GetInstanceInfo(instance.name) snode = inst_nodes[1] for idx, minor in enumerate(info["drbd-minors"][snode.primary]): if idx % 2 == 0: break_drbd_cmd = \ "(drbdsetup %d down >/dev/null 2>&1;" \ " drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \ (minor, minor) else: break_drbd_cmd = \ "(drbdsetup %d detach >/dev/null 2>&1;" \ " drbdsetup detach %d >/dev/null 2>&1) || /bin/true" % \ (minor, minor) AssertCommand(break_drbd_cmd, node=snode) verify_output = GetCommandOutput(qa_config.GetMasterNode().primary, "gnt-cluster verify-disks") activation_msg = "Activating disks for instance '%s'" % instance.name if activation_msg not in verify_output: raise qa_error.Error("gnt-cluster verify-disks did not activate broken" " DRBD disks:\n%s" % verify_output) verify_output = GetCommandOutput(qa_config.GetMasterNode().primary, "gnt-cluster verify-disks") if activation_msg in verify_output: raise qa_error.Error("gnt-cluster verify-disks wants to activate broken" " DRBD disks on second attempt:\n%s" % verify_output) AssertCommand(_CLUSTER_VERIFY) finally: qa_daemon.TestResumeWatcher() def TestJobqueue(): """gnt-debug test-jobqueue""" AssertCommand(["gnt-debug", "test-jobqueue"]) def TestDelay(node): """gnt-debug delay""" AssertCommand(["gnt-debug", "delay", "1"]) AssertCommand(["gnt-debug", "delay", "--no-master", "1"]) AssertCommand(["gnt-debug", "delay", "--no-master", "-n", node.primary, "1"]) def TestClusterReservedLvs(): """gnt-cluster reserved lvs""" # if no lvm-based templates are supported, skip the test if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG): return vgname = qa_config.get("vg-name", constants.DEFAULT_VG) lvname = _QA_LV_PREFIX + "test" lvfullname = "/".join([vgname, lvname]) # Clean cluster AssertClusterVerify() AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", ""]) AssertCommand(["lvcreate", "-L1G", "-n", lvname, vgname]) AssertClusterVerify(fail=False, warnings=[constants.CV_ENODEORPHANLV]) AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", "%s,.*/other-test" % lvfullname]) AssertClusterVerify(no_warnings=[constants.CV_ENODEORPHANLV]) AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", ".*/%s.*" % _QA_LV_PREFIX]) AssertClusterVerify(no_warnings=[constants.CV_ENODEORPHANLV]) AssertCommand(["gnt-cluster", "modify", "--reserved-lvs", ""]) AssertClusterVerify(fail=False, warnings=[constants.CV_ENODEORPHANLV]) AssertCommand(["lvremove", "-f", lvfullname]) AssertClusterVerify() def TestClusterModifyEmpty(): """gnt-cluster modify""" AssertCommand(["gnt-cluster", "modify"], fail=True) def TestClusterModifyDisk(): """gnt-cluster modify -D""" for param in _FAIL_PARAMS: AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True) def _GetOtherEnabledDiskTemplate(undesired_disk_templates, enabled_disk_templates): """Returns one template that is not in the undesired set. @type undesired_disk_templates: list of string @param undesired_disk_templates: a list of disk templates that we want to exclude when drawing one disk template from the list of enabled disk templates @type enabled_disk_templates: list of string @param enabled_disk_templates: list of enabled disk templates (in QA) """ desired_templates = list(set(enabled_disk_templates) - set(undesired_disk_templates)) if desired_templates: template = desired_templates[0] else: # If no desired disk template is available for QA, choose 'diskless' and # hope for the best. template = constants.ST_DISKLESS return template def TestClusterModifyFileBasedStorageDir( file_disk_template, dir_config_key, default_dir, option_name): """Tests gnt-cluster modify wrt to file-based directory options. @type file_disk_template: string @param file_disk_template: file-based disk template @type dir_config_key: string @param dir_config_key: key for the QA config to retrieve the default directory value @type default_dir: string @param default_dir: default directory, if the QA config does not specify it @type option_name: string @param option_name: name of the option of 'gnt-cluster modify' to change the directory """ enabled_disk_templates = qa_config.GetEnabledDiskTemplates() assert file_disk_template in constants.DTS_FILEBASED if not qa_config.IsTemplateSupported(file_disk_template): return # Get some non-file-based disk template to disable file storage other_disk_template = _GetOtherEnabledDiskTemplate( utils.storage.GetDiskTemplatesOfStorageTypes(constants.ST_FILE, constants.ST_SHARED_FILE), enabled_disk_templates ) file_storage_dir = qa_config.get(dir_config_key, default_dir) invalid_file_storage_dir = "/boot/" for fail, cmd in [ (False, ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % file_disk_template, "--ipolicy-disk-templates=%s" % file_disk_template]), (False, ["gnt-cluster", "modify", "--%s=%s" % (option_name, file_storage_dir)]), (False, ["gnt-cluster", "modify", "--%s=%s" % (option_name, invalid_file_storage_dir)]), # file storage dir is set to an inacceptable path, thus verify # should fail (True, ["gnt-cluster", "verify"]), # unsetting the storage dir while file storage is enabled # should fail (True, ["gnt-cluster", "modify", "--%s=" % option_name]), (False, ["gnt-cluster", "modify", "--%s=%s" % (option_name, file_storage_dir)]), (False, ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % other_disk_template, "--ipolicy-disk-templates=%s" % other_disk_template]), (False, ["gnt-cluster", "modify", "--%s=%s" % (option_name, invalid_file_storage_dir)]), # file storage is set to an inacceptable path, but file storage # is disabled, thus verify should not fail (False, ["gnt-cluster", "verify"]), # unsetting the file storage dir while file storage is not enabled # should be fine (False, ["gnt-cluster", "modify", "--%s=" % option_name]), # resetting everything to sane values (False, ["gnt-cluster", "modify", "--%s=%s" % (option_name, file_storage_dir), "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates), "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)]) ]: AssertCommand(cmd, fail=fail) def TestClusterModifyFileStorageDir(): """gnt-cluster modify --file-storage-dir=...""" TestClusterModifyFileBasedStorageDir( constants.DT_FILE, "default-file-storage-dir", pathutils.DEFAULT_FILE_STORAGE_DIR, "file-storage-dir") def TestClusterModifySharedFileStorageDir(): """gnt-cluster modify --shared-file-storage-dir=...""" TestClusterModifyFileBasedStorageDir( constants.DT_SHARED_FILE, "default-shared-file-storage-dir", pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR, "shared-file-storage-dir") def TestClusterModifyDiskTemplates(): """gnt-cluster modify --enabled-disk-templates=...""" enabled_disk_templates = qa_config.GetEnabledDiskTemplates() default_disk_template = qa_config.GetDefaultDiskTemplate() _TestClusterModifyDiskTemplatesArguments(default_disk_template) _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates) _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates) _RestoreEnabledDiskTemplates() nodes = qa_config.AcquireManyNodes(2) instance_template = enabled_disk_templates[0] instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template) _TestClusterModifyUsedDiskTemplate(instance_template, enabled_disk_templates) qa_instance.TestInstanceRemove(instance) _RestoreEnabledDiskTemplates() def TestClusterModifyInstallImage(): """gnt-cluster modify --install-image=...'""" master = qa_config.GetMasterNode() image = \ GetCommandOutput(master.primary, "mktemp --tmpdir ganeti-install-image.XXXXXX").strip() AssertCommand(["gnt-cluster", "modify", "--install-image=%s" % image]) AssertCommand(["rm", image]) def _RestoreEnabledDiskTemplates(): """Sets the list of enabled disk templates back to the list of enabled disk templates from the QA configuration. This can be used to make sure that the tests that modify the list of disk templates do not interfere with other tests. """ enabled_disk_templates = qa_config.GetEnabledDiskTemplates() cmd = ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates), "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates), ] if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()): vgname = qa_config.get("vg-name", constants.DEFAULT_VG) cmd.append("--vg-name=%s" % vgname) AssertCommand(cmd, fail=False) def _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates): """Tests argument handling of 'gnt-cluster modify' with respect to the parameter '--drbd-usermode-helper'. This test is independent of instances. """ _RestoreEnabledDiskTemplates() if constants.DT_DRBD8 not in enabled_disk_templates: return if constants.DT_PLAIN not in enabled_disk_templates: return drbd_usermode_helper = qa_config.get("drbd-usermode-helper", "/bin/true") bogus_usermode_helper = "/tmp/pinkbunny" for command, fail in [ (["gnt-cluster", "modify", "--enabled-disk-templates=%s" % constants.DT_DRBD8, "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False), (["gnt-cluster", "modify", "--drbd-usermode-helper=%s" % drbd_usermode_helper], False), (["gnt-cluster", "modify", "--drbd-usermode-helper=%s" % bogus_usermode_helper], True), # unsetting helper when DRBD is enabled should not work (["gnt-cluster", "modify", "--drbd-usermode-helper="], True), (["gnt-cluster", "modify", "--enabled-disk-templates=%s" % constants.DT_PLAIN, "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False), (["gnt-cluster", "modify", "--drbd-usermode-helper="], False), (["gnt-cluster", "modify", "--drbd-usermode-helper=%s" % drbd_usermode_helper], False), (["gnt-cluster", "modify", "--drbd-usermode-helper=%s" % drbd_usermode_helper, "--enabled-disk-templates=%s" % constants.DT_DRBD8, "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False), (["gnt-cluster", "modify", "--drbd-usermode-helper=", "--enabled-disk-templates=%s" % constants.DT_PLAIN, "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False), (["gnt-cluster", "modify", "--drbd-usermode-helper=%s" % drbd_usermode_helper, "--enabled-disk-templates=%s" % constants.DT_DRBD8, "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False), ]: AssertCommand(command, fail=fail) _RestoreEnabledDiskTemplates() def _TestClusterModifyDiskTemplatesArguments(default_disk_template): """Tests argument handling of 'gnt-cluster modify' with respect to the parameter '--enabled-disk-templates'. This test is independent of instances. """ _RestoreEnabledDiskTemplates() # bogus templates AssertCommand(["gnt-cluster", "modify", "--enabled-disk-templates=pinkbunny"], fail=True) # duplicate entries do no harm AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s,%s" % (default_disk_template, default_disk_template), "--ipolicy-disk-templates=%s" % default_disk_template], fail=False) def _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates): """Tests argument handling of 'gnt-cluster modify' with respect to the parameter '--enabled-disk-templates' and '--vg-name'. This test is independent of instances. """ if not utils.IsLvmEnabled(enabled_disk_templates): # These tests only make sense if lvm is enabled for QA return # determine an LVM and a non-LVM disk template for the tests non_lvm_template = _GetOtherEnabledDiskTemplate(constants.DTS_LVM, enabled_disk_templates) lvm_template = list(set(enabled_disk_templates) & constants.DTS_LVM)[0] vgname = qa_config.get("vg-name", constants.DEFAULT_VG) # Clean start: unset volume group name, disable lvm storage AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % non_lvm_template, "--ipolicy-disk-templates=%s" % non_lvm_template, "--vg-name="], fail=False) # Try to enable lvm, when no volume group is given AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % lvm_template, "--ipolicy-disk-templates=%s" % lvm_template], fail=True) # Set volume group, with lvm still disabled: just a warning AssertCommand(["gnt-cluster", "modify", "--vg-name=%s" % vgname], fail=False) # Try unsetting vg name and enabling lvm at the same time AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % lvm_template, "--ipolicy-disk-templates=%s" % lvm_template, "--vg-name="], fail=True) # Enable lvm with vg name present AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % lvm_template, "--ipolicy-disk-templates=%s" % lvm_template], fail=False) # Try unsetting vg name with lvm still enabled AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=True) # Disable lvm with vg name still set AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % non_lvm_template, "--ipolicy-disk-templates=%s" % non_lvm_template, ], fail=False) # Try unsetting vg name with lvm disabled AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=False) # Set vg name and enable lvm at the same time AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % lvm_template, "--ipolicy-disk-templates=%s" % lvm_template, "--vg-name=%s" % vgname], fail=False) # Unset vg name and disable lvm at the same time AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % non_lvm_template, "--ipolicy-disk-templates=%s" % non_lvm_template, "--vg-name="], fail=False) _RestoreEnabledDiskTemplates() def _TestClusterModifyUsedDiskTemplate(instance_template, enabled_disk_templates): """Tests that disk templates that are currently in use by instances cannot be disabled on the cluster. """ # If the list of enabled disk templates contains only one template # we need to add some other templates, because the list of enabled disk # templates can only be set to a non-empty list. new_disk_templates = list(set(enabled_disk_templates) - set([instance_template])) if not new_disk_templates: new_disk_templates = list(set([constants.DT_DISKLESS, constants.DT_BLOCK]) - set([instance_template])) AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % ",".join(new_disk_templates), "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)], fail=True) def TestClusterModifyBe(): """gnt-cluster modify -B""" for fail, cmd in [ # max/min mem (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]), (False, ["gnt-cluster", "modify", "-B", "minmem=256"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]), (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]), (True, ["gnt-cluster", "modify", "-B", "minmem=a"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]), (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]), # vcpus (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]), (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]), (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]), # auto_balance (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]), (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]), (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]), (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]), ]: AssertCommand(cmd, fail=fail) # redo the original-requested BE parameters, if any bep = qa_config.get("backend-parameters", "") if bep: AssertCommand(["gnt-cluster", "modify", "-B", bep]) def _GetClusterIPolicy(): """Return the run-time values of the cluster-level instance policy. @rtype: tuple @return: (policy, specs), where: - policy is a dictionary of the policy values, instance specs excluded - specs is a dictionary containing only the specs, using the internal format (see L{constants.IPOLICY_DEFAULTS} for an example) """ info = qa_utils.GetObjectInfo(["gnt-cluster", "info"]) policy = info["Instance policy - limits for instances"] (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy) # Sanity checks assert "minmax" in ret_specs and "std" in ret_specs assert len(ret_specs["minmax"]) > 0 assert len(ret_policy) > 0 return (ret_policy, ret_specs) def TestClusterModifyIPolicy(): """gnt-cluster modify --ipolicy-*""" basecmd = ["gnt-cluster", "modify"] (old_policy, old_specs) = _GetClusterIPolicy() for par in ["vcpu-ratio", "spindle-ratio"]: curr_val = float(old_policy[par]) test_values = [ (True, 1.0), (True, 1.5), (True, 2), (False, "a"), # Restore the old value (True, curr_val), ] for (good, val) in test_values: cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)] AssertCommand(cmd, fail=not good) if good: curr_val = val # Check the affected parameter (eff_policy, eff_specs) = _GetClusterIPolicy() AssertEqual(float(eff_policy[par]), curr_val) # Check everything else AssertEqual(eff_specs, old_specs) for p in eff_policy.keys(): if p == par: continue AssertEqual(eff_policy[p], old_policy[p]) # Allowing disk templates via ipolicy requires them to be # enabled on the cluster. if not (qa_config.IsTemplateSupported(constants.DT_PLAIN) and qa_config.IsTemplateSupported(constants.DT_DRBD8)): return # Disk templates are treated slightly differently par = "disk-templates" disp_str = "allowed disk templates" curr_val = old_policy[disp_str] test_values = [ (True, constants.DT_PLAIN), (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)), (False, "thisisnotadisktemplate"), (False, ""), # Restore the old value (True, curr_val.replace(" ", "")), ] for (good, val) in test_values: cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)] AssertCommand(cmd, fail=not good) if good: curr_val = val # Check the affected parameter (eff_policy, eff_specs) = _GetClusterIPolicy() AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val) # Check everything else AssertEqual(eff_specs, old_specs) for p in eff_policy.keys(): if p == disp_str: continue AssertEqual(eff_policy[p], old_policy[p]) def TestClusterSetISpecs(new_specs=None, diff_specs=None, fail=False, old_values=None): """Change instance specs. At most one of new_specs or diff_specs can be specified. @type new_specs: dict @param new_specs: new complete specs, in the same format returned by L{_GetClusterIPolicy} @type diff_specs: dict @param diff_specs: partial specs, it can be an incomplete specifications, but if min/max specs are specified, their number must match the number of the existing specs @type fail: bool @param fail: if the change is expected to fail @type old_values: tuple @param old_values: (old_policy, old_specs), as returned by L{_GetClusterIPolicy} @return: same as L{_GetClusterIPolicy} """ build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts return qa_utils.TestSetISpecs( new_specs=new_specs, diff_specs=diff_specs, get_policy_fn=_GetClusterIPolicy, build_cmd_fn=build_cmd, fail=fail, old_values=old_values) def TestClusterModifyISpecs(): """gnt-cluster modify --specs-*""" params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"] (cur_policy, cur_specs) = _GetClusterIPolicy() # This test assumes that there is only one min/max bound assert len(cur_specs[constants.ISPECS_MINMAX]) == 1 for par in params: test_values = [ (True, 0, 4, 12), (True, 4, 4, 12), (True, 4, 12, 12), (True, 4, 4, 4), (False, 4, 0, 12), (False, 4, 16, 12), (False, 4, 4, 0), (False, 12, 4, 4), (False, 12, 4, 0), (False, "a", 4, 12), (False, 0, "a", 12), (False, 0, 4, "a"), # This is to restore the old values (True, cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par], cur_specs[constants.ISPECS_STD][par], cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par]) ] for (good, mn, st, mx) in test_values: new_vals = { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: {par: mn}, constants.ISPECS_MAX: {par: mx} }], constants.ISPECS_STD: {par: st} } cur_state = (cur_policy, cur_specs) # We update cur_specs, as we've copied the values to restore already (cur_policy, cur_specs) = TestClusterSetISpecs( diff_specs=new_vals, fail=not good, old_values=cur_state) # Get the ipolicy command mnode = qa_config.GetMasterNode() initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd") modcmd = ["gnt-cluster", "modify"] opts = initcmd.split() assert opts[0:2] == ["gnt-cluster", "init"] for k in range(2, len(opts) - 1): if opts[k].startswith("--ipolicy-"): assert k + 2 <= len(opts) modcmd.extend(opts[k:k + 2]) # Re-apply the ipolicy (this should be a no-op) AssertCommand(modcmd) new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd") AssertEqual(initcmd, new_initcmd) def _TestClusterModifyUserShutdownXen(nodes): """Tests user shutdown cluster wide for the KVM hypervisor. Note that for the Xen hypervisor, the KVM daemon should never run. """ AssertCommand(["gnt-cluster", "modify", "--user-shutdown=true"]) # Give time for kvmd to start and stop on all nodes time.sleep(5) for node in nodes: AssertCommand("pgrep ganeti-kvmd", node=node, fail=True) AssertCommand(["gnt-cluster", "modify", "--user-shutdown=false"]) for node in nodes: AssertCommand("pgrep ganeti-kvmd", node=node, fail=True) def _TestClusterModifyUserShutdownKvm(nodes): """Tests user shutdown cluster wide for the KVM hypervisor. Note that for the KVM hypervisor, the KVM daemon should run according to '--user-shutdown' and whether the node is VM capable. """ # How much time to wait for kvmd to start/stop kvmd_cycle_time = 4 # Start kvmd on all nodes AssertCommand(["gnt-cluster", "modify", "--user-shutdown=true"]) time.sleep(kvmd_cycle_time) for node in nodes: AssertCommand("pgrep ganeti-kvmd", node=node) # Test VM capable node attribute test_node = None for node in nodes: node_info = qa_utils.GetObjectInfo(["gnt-node", "info", node.primary])[0] if "vm_capable" in node_info and node_info["vm_capable"]: test_node = node break if test_node is None: raise qa_error.Error("Failed to find viable node for this test") # Stop kvmd by disabling vm capable AssertCommand(["gnt-node", "modify", "--vm-capable=no", test_node.primary]) time.sleep(kvmd_cycle_time) AssertCommand("pgrep ganeti-kvmd", node=test_node, fail=True) # Start kvmd by enabling vm capable AssertCommand(["gnt-node", "modify", "--vm-capable=yes", test_node.primary]) time.sleep(kvmd_cycle_time) AssertCommand("pgrep ganeti-kvmd", node=test_node) # Stop kvmd on all nodes by removing KVM from the enabled hypervisors enabled_hypervisors = qa_config.GetEnabledHypervisors() AssertCommand(["gnt-cluster", "modify", "--enabled-hypervisors=xen-pvm"]) time.sleep(kvmd_cycle_time) for node in nodes: AssertCommand("pgrep ganeti-kvmd", node=node, fail=True) # Start kvmd on all nodes by restoring KVM to the enabled hypervisors AssertCommand(["gnt-cluster", "modify", "--enabled-hypervisors=%s" % ",".join(enabled_hypervisors)]) time.sleep(kvmd_cycle_time) for node in nodes: AssertCommand("pgrep ganeti-kvmd", node=node) # Stop kvmd on all nodes AssertCommand(["gnt-cluster", "modify", "--user-shutdown=false"]) time.sleep(kvmd_cycle_time) for node in nodes: AssertCommand("pgrep ganeti-kvmd", node=node, fail=True) def TestClusterModifyUserShutdown(): """Tests user shutdown cluster wide. """ enabled_hypervisors = qa_config.GetEnabledHypervisors() nodes = qa_config.get("nodes") for (hv, fn) in [(constants.HT_XEN_PVM, _TestClusterModifyUserShutdownXen), (constants.HT_XEN_HVM, _TestClusterModifyUserShutdownXen), (constants.HT_KVM, _TestClusterModifyUserShutdownKvm)]: if hv in enabled_hypervisors: qa_daemon.TestPauseWatcher() fn(nodes) qa_daemon.TestResumeWatcher() else: print("%s hypervisor is not enabled, skipping test for this hypervisor" \ % hv) def TestClusterInfo(): """gnt-cluster info""" AssertCommand(["gnt-cluster", "info"]) def TestClusterRedistConf(): """gnt-cluster redist-conf""" AssertCommand(["gnt-cluster", "redist-conf"]) def TestClusterGetmaster(): """gnt-cluster getmaster""" AssertCommand(["gnt-cluster", "getmaster"]) def TestClusterVersion(): """gnt-cluster version""" AssertCommand(["gnt-cluster", "version"]) def _AssertSsconfCertFiles(): """This asserts that all ssconf_master_candidate_certs have the same content. """ (vcluster_master, _) = qa_config.GetVclusterSettings() if vcluster_master: print("Skipping asserting SsconfCertFiles for Vcluster") return nodes = qa_config.get("nodes") ssconf_file = "/var/lib/ganeti/ssconf_master_candidates_certs" ssconf_content = {} for node in nodes: cmd = ["cat", ssconf_file] print("Ssconf Master Certificates of node '%s'." % node.primary) result_output = GetCommandOutput(node.primary, utils.ShellQuoteArgs(cmd)) ssconf_content[node] = result_output # Clean up result to make it comparable: # remove trailing whitespace from each line, remove empty lines, sort lines lines_node = result_output.split('\n') lines_node = [line.strip() for line in lines_node if len(line.strip()) > 0] lines_node.sort() ssconf_content[node] = lines_node first_node = nodes[0] for node in nodes[1:]: if not ssconf_content[node] == ssconf_content[first_node]: raise Exception("Cert list of node '%s' differs from the list of node" " '%s'." % (node, first_node)) def _TestSSHKeyChanges(master_node): """Tests a lot of SSH key type- and size- related functionality. @type master_node: L{qa_config._QaNode} @param master_node: The cluster master. """ # Helper fn to avoid specifying base params too many times def _RenewWithParams(new_params, verify=True, fail=False): AssertCommand(["gnt-cluster", "renew-crypto", "--new-ssh-keys", "-f", "--no-ssh-key-check"] + new_params, fail=fail) if not fail and verify: AssertCommand(["gnt-cluster", "verify"]) # First test the simplest change _RenewWithParams([]) # And stop here if vcluster (vcluster_master, _) = qa_config.GetVclusterSettings() if vcluster_master: print("Skipping further SSH key replacement checks for vcluster") return # And the actual tests with qa_config.AcquireManyNodesCtx(1, exclude=[master_node]) as nodes: node_name = nodes[0].primary # Another helper function for checking whether a specific key can log in def _CheckLoginWithKey(key_path, fail=False): AssertCommand(["ssh", "-oIdentityFile=%s" % key_path, "-oBatchMode=yes", "-oStrictHostKeyChecking=no", "-oIdentitiesOnly=yes", "-F/dev/null", node_name, "true"], fail=fail, forward_agent=False) _RenewWithParams(["--ssh-key-type=dsa"]) _CheckLoginWithKey("/root/.ssh/id_dsa") # Stash the key for now old_key_backup = qa_utils.BackupFile(master_node.primary, "/root/.ssh/id_dsa") try: _RenewWithParams(["--ssh-key-type=rsa"]) _CheckLoginWithKey("/root/.ssh/id_rsa") # And check that we cannot log in with the old key _CheckLoginWithKey(old_key_backup, fail=True) finally: AssertCommand(["rm", "-f", old_key_backup]) _RenewWithParams(["--ssh-key-bits=4096"]) _RenewWithParams(["--ssh-key-bits=521"], fail=True) # Restore the cluster to its pristine state, skipping the verify as we did # way too many already _RenewWithParams(["--ssh-key-type=rsa", "--ssh-key-bits=2048"], verify=False) def TestClusterRenewCrypto(): """gnt-cluster renew-crypto""" master = qa_config.GetMasterNode() # Conflicting options cmd = ["gnt-cluster", "renew-crypto", "--force", "--new-cluster-certificate", "--new-confd-hmac-key"] conflicting = [ ["--new-rapi-certificate", "--rapi-certificate=/dev/null"], ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"], ] for i in conflicting: AssertCommand(cmd + i, fail=True) # Invalid RAPI certificate cmd = ["gnt-cluster", "renew-crypto", "--force", "--rapi-certificate=/dev/null"] AssertCommand(cmd, fail=True) rapi_cert_backup = qa_utils.BackupFile(master.primary, pathutils.RAPI_CERT_FILE) try: # Custom RAPI certificate fh = tempfile.NamedTemporaryFile(mode="w") # Ensure certificate doesn't cause "gnt-cluster verify" to complain validity = constants.SSL_CERT_EXPIRATION_WARN * 3 utils.GenerateSelfSignedSslCert(fh.name, 1, validity=validity) tmpcert = qa_utils.UploadFile(master.primary, fh.name) try: AssertCommand(["gnt-cluster", "renew-crypto", "--force", "--rapi-certificate=%s" % tmpcert]) finally: AssertCommand(["rm", "-f", tmpcert]) # Custom cluster domain secret cds_fh = tempfile.NamedTemporaryFile(mode="w") cds_fh.write(utils.GenerateSecret()) cds_fh.write("\n") cds_fh.flush() tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name) try: AssertCommand(["gnt-cluster", "renew-crypto", "--force", "--cluster-domain-secret=%s" % tmpcds]) finally: AssertCommand(["rm", "-f", tmpcds]) # Normal case AssertCommand(["gnt-cluster", "renew-crypto", "--force", "--new-cluster-certificate", "--new-confd-hmac-key", "--new-rapi-certificate", "--new-cluster-domain-secret", "--new-node-certificates", "--new-ssh-keys", "--no-ssh-key-check"]) _AssertSsconfCertFiles() AssertCommand(["gnt-cluster", "verify"]) # Only renew node certificates AssertCommand(["gnt-cluster", "renew-crypto", "--force", "--new-node-certificates"]) _AssertSsconfCertFiles() AssertCommand(["gnt-cluster", "verify"]) # Only renew cluster certificate AssertCommand(["gnt-cluster", "renew-crypto", "--force", "--new-cluster-certificate"]) _AssertSsconfCertFiles() AssertCommand(["gnt-cluster", "verify"]) # Comprehensively test various types of SSH key changes _TestSSHKeyChanges(master) # Restore RAPI certificate AssertCommand(["gnt-cluster", "renew-crypto", "--force", "--rapi-certificate=%s" % rapi_cert_backup]) _AssertSsconfCertFiles() AssertCommand(["gnt-cluster", "verify"]) finally: AssertCommand(["rm", "-f", rapi_cert_backup]) # Since renew-crypto replaced the RAPI cert, reload it. if qa_rapi.Enabled(): qa_rapi.ReloadCertificates() def TestClusterBurnin(): """Burnin""" master = qa_config.GetMasterNode() options = qa_config.get("options", {}) disk_template = options.get("burnin-disk-template", constants.DT_DRBD8) parallel = options.get("burnin-in-parallel", False) check_inst = options.get("burnin-check-instances", False) do_rename = options.get("burnin-rename", "") do_reboot = options.get("burnin-reboot", True) reboot_types = options.get("reboot-types", constants.REBOOT_TYPES) # Get as many instances as we need instances = [] try: try: num = qa_config.get("options", {}).get("burnin-instances", 1) for _ in range(0, num): instances.append(qa_config.AcquireInstance()) except qa_error.OutOfInstancesError: print("Not enough instances, continuing anyway.") if len(instances) < 1: raise qa_error.Error("Burnin needs at least one instance") burnin_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../tools/burnin") script = qa_utils.UploadFile(master.primary, burnin_file) try: disks = qa_config.GetDiskOptions() # Run burnin cmd = ["env", "PYTHONPATH=%s" % _constants.VERSIONEDSHAREDIR, script, "--os=%s" % qa_config.get("os"), "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM), "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM), "--disk-size=%s" % ",".join([d.get("size") for d in disks]), "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]), "--disk-template=%s" % disk_template] if parallel: cmd.append("--parallel") cmd.append("--early-release") if check_inst: cmd.append("--http-check") if do_rename: cmd.append("--rename=%s" % do_rename) if not do_reboot: cmd.append("--no-reboot") else: cmd.append("--reboot-types=%s" % ",".join(reboot_types)) cmd += [inst.name for inst in instances] AssertCommand(cmd) finally: AssertCommand(["rm", "-f", script]) finally: for inst in instances: inst.Release() def TestClusterMasterFailover(): """gnt-cluster master-failover""" master = qa_config.GetMasterNode() failovermaster = qa_config.AcquireNode(exclude=master) # Flush the configuration to prevent race conditions when loading it # on another node print(qa_logging.FormatInfo("Flushing the configuration on the master node")) AssertCommand(["gnt-debug", "wconfd", "flushconfig"]) cmd = ["gnt-cluster", "master-failover"] node_list_cmd = ["gnt-node", "list"] try: AssertCommand(cmd, node=failovermaster) AssertCommand(node_list_cmd, node=failovermaster) # Back to original master node AssertCommand(cmd, node=master) AssertCommand(node_list_cmd, node=master) finally: failovermaster.Release() def TestUpgrade(): """Test gnt-cluster upgrade. This tests the 'gnt-cluster upgrade' command by flipping between the current and a different version of Ganeti. To also recover subtle points in the configuration up/down grades, instances are left over both upgrades. """ this_version = qa_config.get("dir-version") other_version = qa_config.get("other-dir-version") if this_version is None or other_version is None: print(qa_utils.FormatInfo("Test not run, as versions not specified")) return inst_creates = [] upgrade_instances = qa_config.get("upgrade-instances", []) live_instances = [] for (test_name, templ, cf, n) in qa_instance.available_instance_tests: if (qa_config.TestEnabled(test_name) and qa_config.IsTemplateSupported(templ) and templ in upgrade_instances): inst_creates.append((cf, n)) for (cf, n) in inst_creates: nodes = qa_config.AcquireManyNodes(n) live_instances.append(cf(nodes)) # 2.16 only - prior to performing a downgrade, we have to make sure that the # SSH keys used are such that the lower version can still use them, # regardless of cluster defaults. if constants.VERSION_MINOR != 16: raise qa_error.Error("Please remove the key type downgrade code in 2.17") AssertCommand(["gnt-cluster", "renew-crypto", "--no-ssh-key-check", "-f", "--new-ssh-keys", "--ssh-key-type=dsa"]) AssertRedirectedCommand(["gnt-cluster", "upgrade", "--to", other_version]) AssertRedirectedCommand(["gnt-cluster", "verify"]) for instance in live_instances: qa_instance.TestInstanceRemove(instance) instance.Release() live_instances = [] for (cf, n) in inst_creates: nodes = qa_config.AcquireManyNodes(n) live_instances.append(cf(nodes)) AssertRedirectedCommand(["gnt-cluster", "upgrade", "--to", this_version]) AssertRedirectedCommand(["gnt-cluster", "verify"]) for instance in live_instances: qa_instance.TestInstanceRemove(instance) instance.Release() def _NodeQueueDrainFile(node): """Returns path to queue drain file for a node. """ return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE) def _AssertDrainFile(node, **kwargs): """Checks for the queue drain file. """ AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs) def TestClusterMasterFailoverWithDrainedQueue(): """gnt-cluster master-failover with drained queue""" master = qa_config.GetMasterNode() failovermaster = qa_config.AcquireNode(exclude=master) # Ensure queue is not drained for node in [master, failovermaster]: _AssertDrainFile(node, fail=True) # Drain queue on failover master AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)], node=failovermaster) cmd = ["gnt-cluster", "master-failover"] try: _AssertDrainFile(failovermaster) AssertCommand(cmd, node=failovermaster) _AssertDrainFile(master, fail=True) _AssertDrainFile(failovermaster, fail=True) # Back to original master node AssertCommand(cmd, node=master) finally: failovermaster.Release() # Ensure queue is not drained for node in [master, failovermaster]: _AssertDrainFile(node, fail=True) def TestClusterCopyfile(): """gnt-cluster copyfile""" master = qa_config.GetMasterNode() uniqueid = utils.NewUUID() # Create temporary file f = tempfile.NamedTemporaryFile(mode="w") f.write(uniqueid) f.flush() f.seek(0) # Upload file to master node testname = qa_utils.UploadFile(master.primary, f.name) try: # Copy file to all nodes AssertCommand(["gnt-cluster", "copyfile", testname]) _CheckFileOnAllNodes(testname, uniqueid) finally: _RemoveFileFromAllNodes(testname) def TestClusterCommand(): """gnt-cluster command""" uniqueid = utils.NewUUID() rfile = "/tmp/gnt%s" % utils.NewUUID() rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid]) cmd = utils.ShellQuoteArgs(["gnt-cluster", "command", "%s >%s" % (rcmd, rfile)]) try: AssertCommand(cmd) _CheckFileOnAllNodes(rfile, uniqueid) finally: _RemoveFileFromAllNodes(rfile) def TestClusterDestroy(): """gnt-cluster destroy""" AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"]) def TestClusterRepairDiskSizes(): """gnt-cluster repair-disk-sizes""" AssertCommand(["gnt-cluster", "repair-disk-sizes"]) def TestSetExclStorCluster(newvalue): """Set the exclusive_storage node parameter at the cluster level. @type newvalue: bool @param newvalue: New value of exclusive_storage @rtype: bool @return: The old value of exclusive_storage """ es_path = ["Default node parameters", "exclusive_storage"] oldvalue = _GetClusterField(es_path) AssertCommand(["gnt-cluster", "modify", "--node-parameters", "exclusive_storage=%s" % newvalue]) effvalue = _GetClusterField(es_path) if effvalue != newvalue: raise qa_error.Error("exclusive_storage has the wrong value: %s instead" " of %s" % (effvalue, newvalue)) qa_config.SetExclusiveStorage(newvalue) return oldvalue def TestExclStorSharedPv(node): """cluster-verify reports LVs that share the same PV with exclusive_storage. """ vgname = qa_config.get("vg-name", constants.DEFAULT_VG) lvname1 = _QA_LV_PREFIX + "vol1" lvname2 = _QA_LV_PREFIX + "vol2" node_name = node.primary AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name) AssertClusterVerify(fail=False, warnings=[constants.CV_ENODEORPHANLV]) AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name) AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM], warnings=[constants.CV_ENODEORPHANLV]) AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name) AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name) AssertClusterVerify() def TestInstanceCommunication(): """Tests instance communication via 'gnt-cluster modify'""" master = qa_config.GetMasterNode() # Check that the 'default' node group exists cmd = ["gnt-group", "list", "--no-headers", "-o", "name"] result_output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), "default", msg="Checking 'default' group") # Check that no networks exist cmd = ["gnt-network", "list", "--no-headers", "-o", "name"] result_output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), "", msg="Checking networks") # Modify cluster parameter 'instance-communication-network' and # check whether the cluster creates the instance communication # network and connects it to the 'default' node group network_name = "mynetwork" cmd = "gnt-cluster modify --instance-communication-network=%s" % network_name result_output = qa_utils.GetCommandOutput(master.primary, cmd) print(result_output) cmd = ["gnt-network", "list", "--no-headers", "-o", "name", network_name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), "mynetwork", msg="Checking 'mynetwork'") cmd = ["gnt-network", "list", "--no-headers", "-o", "group_list", network_name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) # (, , ), for this test, VLAN is nothing AssertEqual(result_output.strip(), "default (routed, communication_rt, )", msg="Checking network connected groups") # Check that the network has the parameters necessary for instance # communication cmd = ["gnt-network", "list", "--no-headers", "-o", "gateway", network_name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), "-", msg="Checking gateway") cmd = ["gnt-network", "list", "--no-headers", "-o", "gateway6", network_name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), "-", msg="Checking gateway6") cmd = ["gnt-network", "list", "--no-headers", "-o", "network", network_name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), constants.INSTANCE_COMMUNICATION_NETWORK4, msg="Checking network") cmd = ["gnt-network", "list", "--no-headers", "-o", "network6", network_name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), constants.INSTANCE_COMMUNICATION_NETWORK6, msg="Checking network6") # Add a new group and check whether the instance communication # network connects to this new group # # We don't assume any particular group order and allow the output of # 'gnt-network list' to print the 'default' and 'mygroup' groups in # any order. group = "mygroup" cmd = ["gnt-group", "add", group] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) print(result_output) cmd = ["gnt-network", "list", "--no-headers", "-o", "group_list", network_name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) try: r1 = "mygroup (routed, communication_rt, )," \ " default (routed, communication_rt, )" AssertEqual(result_output.strip(), r1, msg="Checking network connected groups") except qa_error.Error: r2 = "default (routed, communication_rt, )," \ " mygroup (routed, communication_rt, )" AssertEqual(result_output.strip(), r2, msg="Checking network connected groups") # Modify cluster parameter 'instance-communication-network' to the # same value and check that nothing happens. cmd = "gnt-cluster modify --instance-communication-network=%s" % network_name result_output = qa_utils.GetCommandOutput(master.primary, cmd) print(result_output) # Disable instance communication network, disconnect the instance # communication network and remove it, and remove the group cmd = "gnt-cluster modify --instance-communication-network=" result_output = qa_utils.GetCommandOutput(master.primary, cmd) print(result_output) cmd = ["gnt-network", "disconnect", network_name] AssertCommand(utils.ShellQuoteArgs(cmd)) cmd = ["gnt-network", "remove", network_name] AssertCommand(utils.ShellQuoteArgs(cmd)) cmd = ["gnt-group", "remove", group] AssertCommand(utils.ShellQuoteArgs(cmd)) # Check that the 'default' node group exists cmd = ["gnt-group", "list", "--no-headers", "-o", "name"] result_output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), "default", msg="Checking 'default' group") # Check that no networks exist cmd = ["gnt-network", "list", "--no-headers", "-o", "name"] result_output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), "", msg="Checking networks") ganeti-3.1.0~rc2/qa/qa_config.py000064400000000000000000000667361476477700300165270ustar00rootroot00000000000000# # # Copyright (C) 2007, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """QA configuration. """ import os from ganeti import constants from ganeti import utils from ganeti import serializer from ganeti import compat from ganeti import ht from qa import qa_error from qa import qa_logging _INSTANCE_CHECK_KEY = "instance-check" _ENABLED_HV_KEY = "enabled-hypervisors" _VCLUSTER_MASTER_KEY = "vcluster-master" _VCLUSTER_BASEDIR_KEY = "vcluster-basedir" _ENABLED_DISK_TEMPLATES_KEY = "enabled-disk-templates" # The constants related to JSON patching (as per RFC6902) that modifies QA's # configuration. _QA_BASE_PATH = os.path.dirname(__file__) _QA_DEFAULT_PATCH = "qa-patch.json" _QA_PATCH_DIR = "patch" _QA_PATCH_ORDER_FILE = "order" #: QA configuration (L{_QaConfig}) _config = None MULTIPLEXERS = {} class _QaInstance(object): __slots__ = [ "name", "nicmac", "_used", "_disk_template", ] def __init__(self, name, nicmac): """Initializes instances of this class. """ self.name = name self.nicmac = nicmac self._used = None self._disk_template = None @classmethod def FromDict(cls, data): """Creates instance object from JSON dictionary. """ nicmac = [] macaddr = data.get("nic.mac/0") if macaddr: nicmac.append(macaddr) return cls(name=data["name"], nicmac=nicmac) def __repr__(self): status = [ "%s.%s" % (self.__class__.__module__, self.__class__.__name__), "name=%s" % self.name, "nicmac=%s" % self.nicmac, "used=%s" % self._used, "disk_template=%s" % self._disk_template, ] return "<%s at %#x>" % (" ".join(status), id(self)) def Use(self): """Marks instance as being in use. """ assert not self._used assert self._disk_template is None self._used = True def Release(self): """Releases instance and makes it available again. """ assert self._used, \ ("Instance '%s' was never acquired or released more than once" % self.name) self._used = False self._disk_template = None def GetNicMacAddr(self, idx, default): """Returns MAC address for NIC. @type idx: int @param idx: NIC index @param default: Default value """ if len(self.nicmac) > idx: return self.nicmac[idx] else: return default def SetDiskTemplate(self, template): """Set the disk template. """ assert template in constants.DISK_TEMPLATES self._disk_template = template @property def used(self): """Returns boolean denoting whether instance is in use. """ return self._used @property def disk_template(self): """Returns the current disk template. """ return self._disk_template class _QaNode(object): __slots__ = [ "primary", "secondary", "_added", "_use_count", ] def __init__(self, primary, secondary): """Initializes instances of this class. @type primary: string @param primary: the primary network address of this node @type secondary: string @param secondary: the secondary network address of this node """ self.primary = primary self.secondary = secondary self._added = False self._use_count = 0 @classmethod def FromDict(cls, data): """Creates node object from JSON dictionary. """ return cls(primary=data["primary"], secondary=data.get("secondary")) def __repr__(self): status = [ "%s.%s" % (self.__class__.__module__, self.__class__.__name__), "primary=%s" % self.primary, "secondary=%s" % self.secondary, "added=%s" % self._added, "use_count=%s" % self._use_count, ] return "<%s at %#x>" % (" ".join(status), id(self)) def Use(self): """Marks a node as being in use. """ assert self._use_count >= 0 self._use_count += 1 return self def Release(self): """Release a node (opposite of L{Use}). """ assert self.use_count > 0 self._use_count -= 1 def MarkAdded(self): """Marks node as having been added to a cluster. """ assert not self._added self._added = True def MarkRemoved(self): """Marks node as having been removed from a cluster. """ assert self._added self._added = False @property def added(self): """Returns whether a node is part of a cluster. """ return self._added @property def use_count(self): """Returns number of current uses (controlled by L{Use} and L{Release}). """ return self._use_count _RESOURCE_CONVERTER = { "instances": _QaInstance.FromDict, "nodes": _QaNode.FromDict, } def _ConvertResources(key_value): """Converts cluster resources in configuration to Python objects. """ (key, value) = key_value fn = _RESOURCE_CONVERTER.get(key, None) if fn: return (key, [fn(v) for v in value]) else: return (key, value) class _QaConfig(object): def __init__(self, data): """Initializes instances of this class. """ self._data = data #: Cluster-wide run-time value of the exclusive storage flag self._exclusive_storage = None @staticmethod def LoadPatch(patch_dict, rel_path): """ Loads a single patch. @type patch_dict: dict of string to dict @param patch_dict: A dictionary storing patches by relative path. @type rel_path: string @param rel_path: The relative path to the patch, might or might not exist. """ try: full_path = os.path.join(_QA_BASE_PATH, rel_path) patch = serializer.LoadJson(utils.ReadFile(full_path)) patch_dict[rel_path] = patch except IOError: pass @staticmethod def LoadPatches(): """ Finds and loads all patches supported by the QA. @rtype: dict of string to dict @return: A dictionary of relative path to patch content. """ patches = {} _QaConfig.LoadPatch(patches, _QA_DEFAULT_PATCH) patch_dir_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR) if os.path.exists(patch_dir_path): for filename in os.listdir(patch_dir_path): if filename.endswith(".json"): _QaConfig.LoadPatch(patches, os.path.join(_QA_PATCH_DIR, filename)) return patches @staticmethod def ApplyPatch(data, patch_module, patches, patch_path): """Applies a single patch. @type data: dict (deserialized json) @param data: The QA configuration to modify @type patch_module: module @param patch_module: The json patch module, loaded dynamically @type patches: dict of string to dict @param patches: The dictionary of patch path to content @type patch_path: string @param patch_path: The path to the patch, relative to the QA directory """ patch_content = patches[patch_path] print(qa_logging.FormatInfo("Applying patch %s" % patch_path)) if not patch_content and patch_path != _QA_DEFAULT_PATCH: print(qa_logging.FormatWarning("The patch %s added by the user is empty" % patch_path)) patch_module.apply_patch(data, patch_content, in_place=True) @staticmethod def ApplyPatches(data, patch_module, patches): """Applies any patches present, and returns the modified QA configuration. First, patches from the patch directory are applied. They are ordered alphabetically, unless there is an ``order`` file present - any patches listed within are applied in that order, and any remaining ones in alphabetical order again. Finally, the default patch residing in the top-level QA directory is applied. @type data: dict (deserialized json) @param data: The QA configuration to modify @type patch_module: module @param patch_module: The json patch module, loaded dynamically @type patches: dict of string to dict @param patches: The dictionary of patch path to content """ ordered_patches = [] order_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR, _QA_PATCH_ORDER_FILE) if os.path.exists(order_path): order_file = open(order_path, 'r') ordered_patches = order_file.read().splitlines() # Removes empty lines ordered_patches = [_f for _f in ordered_patches if _f] # Add the patch dir ordered_patches = [os.path.join(_QA_PATCH_DIR, x) for x in ordered_patches] # First the ordered patches for patch in ordered_patches: if patch not in patches: raise qa_error.Error("Patch %s specified in the ordering file does not " "exist" % patch) _QaConfig.ApplyPatch(data, patch_module, patches, patch) # Then the other non-default ones for patch in sorted(patches): if patch != _QA_DEFAULT_PATCH and patch not in ordered_patches: _QaConfig.ApplyPatch(data, patch_module, patches, patch) # Finally the default one if _QA_DEFAULT_PATCH in patches: _QaConfig.ApplyPatch(data, patch_module, patches, _QA_DEFAULT_PATCH) @classmethod def Load(cls, filename): """Loads a configuration file and produces a configuration object. @type filename: string @param filename: Path to configuration file @rtype: L{_QaConfig} """ data = serializer.LoadJson(utils.ReadFile(filename)) # Patch the document using JSON Patch (RFC6902) in file _PATCH_JSON, if # available try: patches = _QaConfig.LoadPatches() # Try to use the module only if there is a non-empty patch present if any(patches.values()): mod = __import__("jsonpatch", fromlist=[]) _QaConfig.ApplyPatches(data, mod, patches) except IOError: pass except ImportError: raise qa_error.Error("For the QA JSON patching feature to work, you " "need to install Python modules 'jsonpatch' and " "'jsonpointer'.") result = cls(dict(map(_ConvertResources, data.items()))) # pylint: disable=E1103 result.Validate() return result def Validate(self): """Validates loaded configuration data. """ if not self.get("name"): raise qa_error.Error("Cluster name is required") if not self.get("nodes"): raise qa_error.Error("Need at least one node") if not self.get("instances"): raise qa_error.Error("Need at least one instance") disks = self.GetDiskOptions() if disks is None: raise qa_error.Error("Config option 'disks' must exist") else: for d in disks: if d.get("size") is None or d.get("growth") is None: raise qa_error.Error("Config options `size` and `growth` must exist" " for all `disks` items") check = self.GetInstanceCheckScript() if check: try: os.stat(check) except EnvironmentError as err: raise qa_error.Error("Can't find instance check script '%s': %s" % (check, err)) enabled_hv = frozenset(self.GetEnabledHypervisors()) if not enabled_hv: raise qa_error.Error("No hypervisor is enabled") difference = enabled_hv - constants.HYPER_TYPES if difference: raise qa_error.Error("Unknown hypervisor(s) enabled: %s" % utils.CommaJoin(difference)) (vc_master, vc_basedir) = self.GetVclusterSettings() if bool(vc_master) != bool(vc_basedir): raise qa_error.Error("All or none of the config options '%s' and '%s'" " must be set" % (_VCLUSTER_MASTER_KEY, _VCLUSTER_BASEDIR_KEY)) if vc_basedir and not utils.IsNormAbsPath(vc_basedir): raise qa_error.Error("Path given in option '%s' must be absolute and" " normalized" % _VCLUSTER_BASEDIR_KEY) def __getitem__(self, name): """Returns configuration value. @type name: string @param name: Name of configuration entry """ return self._data[name] def __setitem__(self, key, value): """Sets a configuration value. """ self._data[key] = value def __delitem__(self, key): """Deletes a value from the configuration. """ del self._data[key] def __len__(self): """Return the number of configuration items. """ return len(self._data) def get(self, name, default=None): """Returns configuration value. @type name: string @param name: Name of configuration entry @param default: Default value """ return self._data.get(name, default) def GetMasterNode(self): """Returns the default master node for the cluster. """ return self["nodes"][0] def GetAllNodes(self): """Returns the list of nodes. This is not intended to 'acquire' those nodes. For that, C{AcquireManyNodes} is better suited. However, often it is helpful to know the total number of nodes available to adjust cluster parameters and that's where this function is useful. """ return self["nodes"] def GetInstanceCheckScript(self): """Returns path to instance check script or C{None}. """ return self._data.get(_INSTANCE_CHECK_KEY, None) def GetEnabledHypervisors(self): """Returns list of enabled hypervisors. @rtype: list """ return self._GetStringListParameter( _ENABLED_HV_KEY, [constants.DEFAULT_ENABLED_HYPERVISOR]) def GetDefaultHypervisor(self): """Returns the default hypervisor to be used. """ return self.GetEnabledHypervisors()[0] def GetEnabledDiskTemplates(self): """Returns the list of enabled disk templates. @rtype: list """ return self._GetStringListParameter( _ENABLED_DISK_TEMPLATES_KEY, constants.DEFAULT_ENABLED_DISK_TEMPLATES) def GetEnabledStorageTypes(self): """Returns the list of enabled storage types. @rtype: list @returns: the list of storage types enabled for QA """ enabled_disk_templates = self.GetEnabledDiskTemplates() enabled_storage_types = list( set([constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[dt] for dt in enabled_disk_templates])) # Storage type 'lvm-pv' cannot be activated via a disk template, # therefore we add it if 'lvm-vg' is present. if constants.ST_LVM_VG in enabled_storage_types: enabled_storage_types.append(constants.ST_LVM_PV) return enabled_storage_types def GetDefaultDiskTemplate(self): """Returns the default disk template to be used. """ return self.GetEnabledDiskTemplates()[0] def _GetStringListParameter(self, key, default_values): """Retrieves a parameter's value that is supposed to be a list of strings. @rtype: list """ try: value = self._data[key] except KeyError: return default_values else: if value is None: return [] elif isinstance(value, str): return value.split(",") else: return value def SetExclusiveStorage(self, value): """Set the expected value of the C{exclusive_storage} flag for the cluster. """ self._exclusive_storage = bool(value) def GetExclusiveStorage(self): """Get the expected value of the C{exclusive_storage} flag for the cluster. """ value = self._exclusive_storage assert value is not None return value def IsTemplateSupported(self, templ): """Is the given disk template supported by the current configuration? """ enabled = templ in self.GetEnabledDiskTemplates() return enabled and (not self.GetExclusiveStorage() or templ in constants.DTS_EXCL_STORAGE) def IsStorageTypeSupported(self, storage_type): """Is the given storage type supported by the current configuration? This is determined by looking if at least one of the disk templates which is associated with the storage type is enabled in the configuration. """ enabled_disk_templates = self.GetEnabledDiskTemplates() if storage_type == constants.ST_LVM_PV: disk_templates = utils.GetDiskTemplatesOfStorageTypes(constants.ST_LVM_VG) else: disk_templates = utils.GetDiskTemplatesOfStorageTypes(storage_type) return bool(set(enabled_disk_templates).intersection(set(disk_templates))) def AreSpindlesSupported(self): """Are spindles supported by the current configuration? """ return self.GetExclusiveStorage() def GetVclusterSettings(self): """Returns settings for virtual cluster. """ master = self.get(_VCLUSTER_MASTER_KEY) basedir = self.get(_VCLUSTER_BASEDIR_KEY) return (master, basedir) def GetDiskOptions(self): """Return options for the disks of the instances. Get 'disks' parameter from the configuration data. If 'disks' is missing, try to create it from the legacy 'disk' and 'disk-growth' parameters. """ try: return self._data["disks"] except KeyError: pass # Legacy interface sizes = self._data.get("disk") growths = self._data.get("disk-growth") if sizes or growths: if (sizes is None or growths is None or len(sizes) != len(growths)): raise qa_error.Error("Config options 'disk' and 'disk-growth' must" " exist and have the same number of items") disks = [] for (size, growth) in zip(sizes, growths): disks.append({"size": size, "growth": growth}) return disks else: return None def GetModifySshSetup(self): """Return whether to modify the SSH setup or not. This specified whether Ganeti should create and distribute SSH keys for the nodes or not. The default is 'True'. """ if self.get("modify_ssh_setup") is None: return True else: return self.get("modify_ssh_setup") def GetSshConfig(self): """Returns the SSH configuration necessary to create keys etc. @rtype: tuple of (string, string, string, string, string) @return: tuple containing key type ('dsa' or 'rsa'), ssh directory (default '/root/.ssh/'), file path of the private key, file path of the public key, file path of the 'authorized_keys' file """ key_type = "rsa" if self.get("ssh_key_type") is not None: key_type = self.get("ssh_key_type") ssh_dir = "/root/.ssh/" if self.get("ssh_dir") is not None: ssh_dir = self.get("ssh_dir") priv_key_file = os.path.join(ssh_dir, "id_%s" % (key_type)) pub_key_file = "%s.pub" % priv_key_file auth_key_file = os.path.join(ssh_dir, "authorized_keys") return (key_type, ssh_dir, priv_key_file, pub_key_file, auth_key_file) def Load(path): """Loads the passed configuration file. """ global _config # pylint: disable=W0603 _config = _QaConfig.Load(path) def GetConfig(): """Returns the configuration object. """ if _config is None: raise RuntimeError("Configuration not yet loaded") return _config def get(name, default=None): """Wrapper for L{_QaConfig.get}. """ return GetConfig().get(name, default=default) class Either(object): def __init__(self, tests): """Initializes this class. @type tests: list or string @param tests: List of test names @see: L{TestEnabled} for details """ self.tests = tests def _MakeSequence(value): """Make sequence of single argument. If the single argument is not already a list or tuple, a list with the argument as a single item is returned. """ if isinstance(value, (list, tuple)): return value else: return [value] def _TestEnabledInner(check_fn, names, fn): """Evaluate test conditions. @type check_fn: callable @param check_fn: Callback to check whether a test is enabled @type names: sequence or string @param names: Test name(s) @type fn: callable @param fn: Aggregation function @rtype: bool @return: Whether test is enabled """ names = _MakeSequence(names) result = [] for name in names: if isinstance(name, Either): value = _TestEnabledInner(check_fn, name.tests, compat.any) elif isinstance(name, (list, tuple)): value = _TestEnabledInner(check_fn, name, compat.all) elif callable(name): value = name() else: value = check_fn(name) result.append(value) return fn(result) def TestEnabled(tests, _cfg=None): """Returns True if the given tests are enabled. @param tests: A single test as a string, or a list of tests to check; can contain L{Either} for OR conditions, AND is default """ if _cfg is None: cfg = GetConfig() else: cfg = _cfg # Get settings for all tests cfg_tests = cfg.get("tests", {}) # Get default setting default = cfg_tests.get("default", True) return _TestEnabledInner(lambda name: cfg_tests.get(name, default), tests, compat.all) def GetInstanceCheckScript(*args): """Wrapper for L{_QaConfig.GetInstanceCheckScript}. """ return GetConfig().GetInstanceCheckScript(*args) def GetEnabledHypervisors(*args): """Wrapper for L{_QaConfig.GetEnabledHypervisors}. """ return GetConfig().GetEnabledHypervisors(*args) def GetDefaultHypervisor(*args): """Wrapper for L{_QaConfig.GetDefaultHypervisor}. """ return GetConfig().GetDefaultHypervisor(*args) def GetEnabledDiskTemplates(*args): """Wrapper for L{_QaConfig.GetEnabledDiskTemplates}. """ return GetConfig().GetEnabledDiskTemplates(*args) def GetEnabledStorageTypes(*args): """Wrapper for L{_QaConfig.GetEnabledStorageTypes}. """ return GetConfig().GetEnabledStorageTypes(*args) def GetDefaultDiskTemplate(*args): """Wrapper for L{_QaConfig.GetDefaultDiskTemplate}. """ return GetConfig().GetDefaultDiskTemplate(*args) def GetModifySshSetup(*args): """Wrapper for L{_QaConfig.GetModifySshSetup}. """ return GetConfig().GetModifySshSetup(*args) def GetSshConfig(*args): """Wrapper for L{_QaConfig.GetSshConfig}. """ return GetConfig().GetSshConfig(*args) def GetMasterNode(): """Wrapper for L{_QaConfig.GetMasterNode}. """ return GetConfig().GetMasterNode() def GetAllNodes(): """Wrapper for L{_QaConfig.GetAllNodes}. """ return GetConfig().GetAllNodes() def AcquireInstance(_cfg=None): """Returns an instance which isn't in use. """ if _cfg is None: cfg = GetConfig() else: cfg = _cfg # Filter out unwanted instances instances = [inst for inst in cfg["instances"] if not inst.used] if not instances: raise qa_error.OutOfInstancesError("No instances left") instance = instances[0] instance.Use() return instance def AcquireManyInstances(num, _cfg=None): """Return instances that are not in use. @type num: int @param num: Number of instances; can be 0. @rtype: list of instances @return: C{num} different instances """ if _cfg is None: cfg = GetConfig() else: cfg = _cfg # Filter out unwanted instances instances = [inst for inst in cfg["instances"] if not inst.used] if len(instances) < num: raise qa_error.OutOfInstancesError( "Not enough instances left (%d needed, %d remaining)" % (num, len(instances)) ) instances = [] try: for _ in range(0, num): n = AcquireInstance(_cfg=cfg) instances.append(n) except qa_error.OutOfInstancesError: ReleaseManyInstances(instances) raise return instances def ReleaseManyInstances(instances): for instance in instances: instance.Release() def SetExclusiveStorage(value): """Wrapper for L{_QaConfig.SetExclusiveStorage}. """ return GetConfig().SetExclusiveStorage(value) def GetExclusiveStorage(): """Wrapper for L{_QaConfig.GetExclusiveStorage}. """ return GetConfig().GetExclusiveStorage() def IsTemplateSupported(templ): """Wrapper for L{_QaConfig.IsTemplateSupported}. """ return GetConfig().IsTemplateSupported(templ) def IsStorageTypeSupported(storage_type): """Wrapper for L{_QaConfig.IsTemplateSupported}. """ return GetConfig().IsStorageTypeSupported(storage_type) def AreSpindlesSupported(): """Wrapper for L{_QaConfig.AreSpindlesSupported}. """ return GetConfig().AreSpindlesSupported() def _NodeSortKey(node): """Returns sort key for a node. @type node: L{_QaNode} """ return (node.use_count, utils.NiceSortKey(node.primary)) def AcquireNode(exclude=None, _cfg=None): """Returns the least used node. """ if _cfg is None: cfg = GetConfig() else: cfg = _cfg master = cfg.GetMasterNode() # Filter out unwanted nodes # TODO: Maybe combine filters if exclude is None: nodes = cfg["nodes"][:] elif isinstance(exclude, (list, tuple)): nodes = [node for node in cfg["nodes"] if node not in exclude] else: nodes = [node for node in cfg["nodes"] if node != exclude] nodes = [node for node in nodes if node.added or node == master] if not nodes: raise qa_error.OutOfNodesError("No nodes left") # Return node with least number of uses return sorted(nodes, key=_NodeSortKey)[0].Use() class AcquireManyNodesCtx(object): """Returns the least used nodes for use with a `with` block """ def __init__(self, num, exclude=None, cfg=None): self._num = num self._exclude = exclude self._cfg = cfg def __enter__(self): self._nodes = AcquireManyNodes(self._num, exclude=self._exclude, cfg=self._cfg) return self._nodes def __exit__(self, exc_type, exc_value, exc_tb): ReleaseManyNodes(self._nodes) def AcquireManyNodes(num, exclude=None, cfg=None): """Return the least used nodes. @type num: int @param num: Number of nodes; can be 0. @type exclude: list of nodes or C{None} @param exclude: nodes to be excluded from the choice @rtype: list of nodes @return: C{num} different nodes """ nodes = [] if exclude is None: exclude = [] elif isinstance(exclude, (list, tuple)): # Don't modify the incoming argument exclude = list(exclude) else: exclude = [exclude] try: for _ in range(0, num): n = AcquireNode(exclude=exclude, _cfg=cfg) nodes.append(n) exclude.append(n) except qa_error.OutOfNodesError: ReleaseManyNodes(nodes) raise return nodes def ReleaseManyNodes(nodes): for node in nodes: node.Release() def GetVclusterSettings(): """Wrapper for L{_QaConfig.GetVclusterSettings}. """ return GetConfig().GetVclusterSettings() def UseVirtualCluster(_cfg=None): """Returns whether a virtual cluster is used. @rtype: bool """ if _cfg is None: cfg = GetConfig() else: cfg = _cfg (master, _) = cfg.GetVclusterSettings() return bool(master) @ht.WithDesc("No virtual cluster") def NoVirtualCluster(): """Used to disable tests for virtual clusters. """ return not UseVirtualCluster() def GetDiskOptions(): """Wrapper for L{_QaConfig.GetDiskOptions}. """ return GetConfig().GetDiskOptions() ganeti-3.1.0~rc2/qa/qa_daemon.py000064400000000000000000000112611476477700300165040ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Daemon related QA tests. """ import time from ganeti import utils from ganeti import pathutils from qa import qa_config from qa import qa_utils from qa import qa_error from qa_utils import AssertMatch, AssertCommand, StartSSH, GetCommandOutput def _InstanceRunning(name): """Checks whether an instance is running. @param name: full name of the instance """ master = qa_config.GetMasterNode() cmd = (utils.ShellQuoteArgs(["gnt-instance", "list", "-o", "status", name]) + ' | grep running') ret = StartSSH(master.primary, cmd).wait() return ret == 0 def _ShutdownInstance(name): """Shuts down instance without recording state and waits for completion. @param name: full name of the instance """ AssertCommand(["gnt-instance", "shutdown", "--no-remember", name]) if _InstanceRunning(name): raise qa_error.Error("instance shutdown failed") def _StartInstance(name): """Starts instance and waits for completion. @param name: full name of the instance """ AssertCommand(["gnt-instance", "start", name]) if not bool(_InstanceRunning(name)): raise qa_error.Error("instance start failed") def _ResetWatcherDaemon(): """Removes the watcher daemon's state file. """ path = \ qa_utils.MakeNodePath(qa_config.GetMasterNode(), pathutils.WATCHER_GROUP_STATE_FILE % "*-*-*-*") AssertCommand(["bash", "-c", "rm -vf %s" % path]) def RunWatcherDaemon(): """Runs the ganeti-watcher daemon on the master node. """ AssertCommand(["ganeti-watcher", "-d", "--ignore-pause", "--wait-children"]) def TestPauseWatcher(): """Tests and pauses the watcher. """ master = qa_config.GetMasterNode() AssertCommand(["gnt-cluster", "watcher", "pause", "4h"]) cmd = ["gnt-cluster", "watcher", "info"] output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertMatch(output, r"^.*\bis paused\b.*") def TestResumeWatcher(): """Tests and unpauses the watcher. """ master = qa_config.GetMasterNode() AssertCommand(["gnt-cluster", "watcher", "continue"]) cmd = ["gnt-cluster", "watcher", "info"] output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertMatch(output, r"^.*\bis not paused\b.*") def TestInstanceAutomaticRestart(instance): """Test automatic restart of instance by ganeti-watcher. """ inst_name = qa_utils.ResolveInstanceName(instance.name) _ResetWatcherDaemon() _ShutdownInstance(inst_name) RunWatcherDaemon() time.sleep(5) if not _InstanceRunning(inst_name): raise qa_error.Error("Daemon didn't restart instance") AssertCommand(["gnt-instance", "info", inst_name]) def TestInstanceConsecutiveFailures(instance): """Test five consecutive instance failures. """ inst_name = qa_utils.ResolveInstanceName(instance.name) inst_was_running = bool(_InstanceRunning(inst_name)) _ResetWatcherDaemon() for should_start in ([True] * 5) + [False]: _ShutdownInstance(inst_name) RunWatcherDaemon() time.sleep(5) if bool(_InstanceRunning(inst_name)) != should_start: if should_start: msg = "Instance not started when it should" else: msg = "Instance started when it shouldn't" raise qa_error.Error(msg) AssertCommand(["gnt-instance", "info", inst_name]) if inst_was_running: _StartInstance(inst_name) ganeti-3.1.0~rc2/qa/qa_env.py000064400000000000000000000057651476477700300160450ustar00rootroot00000000000000# # # Copyright (C) 2007, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Cluster environment related QA tests. """ from ganeti import utils from qa import qa_config from qa_utils import AssertCommand def TestSshConnection(): """Test SSH connection. """ for node in qa_config.get("nodes"): AssertCommand("exit", node=node) def TestGanetiCommands(): """Test availibility of Ganeti commands. """ cmds = (["gnt-backup", "--version"], ["gnt-cluster", "--version"], ["gnt-debug", "--version"], ["gnt-instance", "--version"], ["gnt-job", "--version"], ["gnt-network", "--version"], ["gnt-node", "--version"], ["gnt-os", "--version"], ["gnt-storage", "--version"], ["gnt-filter", "--version"], ["ganeti-noded", "--version"], ["ganeti-rapi", "--version"], ["ganeti-watcher", "--version"], ["ganeti-confd", "--version"], ["ganeti-luxid", "--version"], ["ganeti-wconfd", "--version"], ) cmd = " && ".join([utils.ShellQuoteArgs(i) for i in cmds]) for node in qa_config.get("nodes"): AssertCommand(cmd, node=node) def TestIcmpPing(): """ICMP ping each node. """ nodes = qa_config.get("nodes") pingprimary = pingsecondary = "fping" if qa_config.get("primary_ip_version") == 6: pingprimary = "fping6" pricmd = [pingprimary, "-e"] seccmd = [pingsecondary, "-e"] for i in nodes: pricmd.append(i.primary) if i.secondary: seccmd.append(i.secondary) pristr = utils.ShellQuoteArgs(pricmd) if seccmd: cmdall = "%s && %s" % (pristr, utils.ShellQuoteArgs(seccmd)) else: cmdall = pristr for node in nodes: AssertCommand(cmdall, node=node) ganeti-3.1.0~rc2/qa/qa_error.py000064400000000000000000000032011476477700300163650ustar00rootroot00000000000000# # # Copyright (C) 2007 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Error definitions for QA. """ class Error(Exception): """An error occurred during Q&A testing. """ pass class OutOfNodesError(Error): """Out of nodes. """ pass class OutOfInstancesError(Error): """Out of instances. """ pass class UnusableNodeError(Error): """Unusable node. """ pass ganeti-3.1.0~rc2/qa/qa_filters.py000064400000000000000000000273361476477700300167230ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """QA tests for job filters. """ import time from ganeti import query from ganeti.utils import retry from qa import qa_job_utils from qa import qa_utils from qa_utils import AssertCommand, AssertEqual, AssertIn, stdout_of def GetJobStatus(job_id): """Queries the status of the job by parsing output of gnt-job info. @type job_id: int @param job_id: ID of the job to query @return: status of the job as lower-case string """ out = stdout_of(["gnt-job", "info", str(job_id)]) # The second line of gnt-job info shows the status. return out.split('\n')[1].strip().lower().split("status: ")[1] def KillWaitJobs(job_ids): """Kills the lists of jobs, then watches them so that when this function returns we can be sure the jobs are all done. This should be called at the end of tests that started jobs with --submit so that following tests have an empty job queue. @type job_ids: list of int @param job_ids: the lists of job IDs to kill and wait for """ # We use fail=None to ignore the exit code, since it can be non-zero # if the job is already terminated. for jid in job_ids: AssertCommand(["gnt-job", "cancel", "--kill", "--yes-do-it", str(jid)], fail=None) for jid in job_ids: AssertCommand(["gnt-job", "watch", str(jid)], fail=None) def AssertStatusRetry(jid, status, interval=1.0, timeout=20.0): """Keeps polling the given job until a given status is reached. @type jid: int @param jid: job ID of the job to poll @type status: string @param status: status to wait for @type interval: float @param interval: polling interval in seconds @type timeout: float @param timeout: polling timeout in seconds @raise retry:RetryTimeout: If the status was not reached within the timeout """ retry_fn = lambda: qa_job_utils.RetryingUntilJobStatus(status, str(jid)) retry.Retry(retry_fn, interval, timeout) def TestFilterList(): """gnt-filter list""" qa_utils.GenericQueryTest("gnt-filter", list(query.FILTER_FIELDS), namefield="uuid", test_unknown=False) def TestFilterListFields(): """gnt-filter list-fields""" qa_utils.GenericQueryFieldsTest("gnt-filter", list(query.FILTER_FIELDS)) def TestFilterAddRemove(): """gnt-filter add/delete""" uuid1 = stdout_of(["gnt-filter", "add", "--reason", "reason1"]) TestFilterList() TestFilterListFields() uuid2 = stdout_of(["gnt-filter", "list", "--no-headers", "--output=uuid"]) AssertEqual(uuid1, uuid2) AssertCommand(["gnt-filter", "delete", uuid1]) TestFilterList() def TestFilterWatermark(): """Tests that the filter watermark is set correctly""" # Check what the current highest job ID is highest_jid1 = int(stdout_of( ["gnt-debug", "delay", "--print-jobid", "0.01"] )) # Add the filter; this sets the watermark uuid = stdout_of(["gnt-filter", "add"]) # Check what the current highest job ID is highest_jid2 = int(stdout_of( ["gnt-debug", "delay", "--print-jobid", "0.01"] )) info_out = stdout_of(["gnt-filter", "info", uuid]) # The second line of gnt-filter info shows the watermark. watermark = int( info_out.split('\n')[1].strip().lower().split("watermark: ")[1] ) # The atermark must be at least as high as the JID of the job we started # just before the creation, and must be lower than the JID of any job # created afterwards. assert highest_jid1 <= watermark < highest_jid2, \ "Watermark not in range: %d <= %d < %d" % (highest_jid1, watermark, highest_jid2) # Clean up. AssertCommand(["gnt-filter", "delete", uuid]) def TestFilterReject(): """Tests that the REJECT filter does reject new jobs and that the "jobid" predicate works. """ # Add a filter that rejects all new jobs. uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=REJECT", ]) # Newly queued jobs must now fail. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) # Clean up. AssertCommand(["gnt-filter", "delete", uuid]) def TestFilterOpCode(): """Tests that filtering with the "opcode" predicate works""" # Check that delay jobs work fine. AssertCommand(["gnt-debug", "delay", "0.01"]) # Add a filter that rejects all new delay jobs. uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["opcode", ["=", "OP_ID", "OP_TEST_DELAY"]]]', "--action=REJECT", ]) # Newly queued delay jobs must now fail. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) # Clean up. AssertCommand(["gnt-filter", "delete", uuid]) def TestFilterContinue(): """Tests that the CONTINUE filter has no effect""" # Add a filter that just passes to the next filter. uuid_cont = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=CONTINUE", "--priority=0", ]) # Add a filter that rejects all new jobs. uuid_reject = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=REJECT", "--priority=1", ]) # Newly queued jobs must now fail. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) # Delete the rejecting filter. AssertCommand(["gnt-filter", "delete", uuid_reject]) # Newly queued jobs must now succeed. AssertCommand(["gnt-debug", "delay", "0.01"]) # Clean up. AssertCommand(["gnt-filter", "delete", uuid_cont]) def TestFilterReasonChain(): """Tests that filters are processed in the right order and that the "reason" predicate works. """ # Add a filter chain that pauses all new jobs apart from those with a # specific reason. # Accept all jobs that have the "allow this" reason. uuid1 = stdout_of([ "gnt-filter", "add", '--predicates=[["reason", ["=", "reason", "allow this"]]]', "--action=ACCEPT", # Default priority 0 ]) # Reject those that haven't (but make the one above run first). uuid2 = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=REJECT", "--priority=1", ]) # This job must now go into queued status. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) AssertCommand(["gnt-debug", "delay", "--reason=allow this", "0.01"]) # Clean up. AssertCommand(["gnt-filter", "delete", uuid1]) AssertCommand(["gnt-filter", "delete", uuid2]) def TestFilterAcceptPause(): """Tests that the PAUSE filter allows scheduling, but prevents starting, and that the ACCEPT filter immediately allows starting. """ AssertCommand(["gnt-cluster", "watcher", "pause", "600"]) # Add a filter chain that pauses all new jobs apart from those with a # specific reason. # When the pausing filter is deleted, paused jobs must be continued. # Accept all jobs that have the "allow this" reason. uuid1 = stdout_of([ "gnt-filter", "add", '--predicates=[["reason", ["=", "reason", "allow this"]]]', "--action=ACCEPT", # Default priority 0 ]) # Pause those that haven't (but make the one above run first). uuid2 = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=PAUSE", "--priority=1", ]) # This job must now go into queued status. jid1 = int(stdout_of([ "gnt-debug", "delay", "--submit", "--print-jobid", "0.01", ])) # This job should run and finish. jid2 = int(stdout_of([ "gnt-debug", "delay", "--submit", "--print-jobid", "--reason=allow this", "0.01", ])) time.sleep(5) # give some time to get queued AssertStatusRetry(jid1, "queued") # job should be paused AssertStatusRetry(jid2, "success") # job should not be paused # Delete the filters. AssertCommand(["gnt-filter", "delete", uuid1]) AssertCommand(["gnt-filter", "delete", uuid2]) # Now the paused job should run through. time.sleep(5) AssertStatusRetry(jid1, "success") AssertCommand(["gnt-cluster", "watcher", "continue"]) def TestFilterRateLimit(): """Tests that the RATE_LIMIT filter does reject new jobs when all rate-limiting buckets are taken. """ # Make sure our test is not constrained by "max-running-jobs" # (simply set it to the default). AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"]) AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"]) AssertCommand(["gnt-cluster", "watcher", "pause", "600"]) # Add a filter that rejects all new jobs. uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=RATE_LIMIT 2", ]) # Now only the first 2 jobs must be scheduled. jid1 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "200" ])) jid2 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "200" ])) jid3 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "200" ])) time.sleep(5) # give the scheduler some time to notice AssertIn(GetJobStatus(jid1), ["running", "waiting"], msg="Job should not be rate-limited") AssertIn(GetJobStatus(jid2), ["running", "waiting"], msg="Job should not be rate-limited") AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited") # Clean up. AssertCommand(["gnt-filter", "delete", uuid]) KillWaitJobs([jid1, jid2, jid3]) AssertCommand(["gnt-cluster", "watcher", "continue"]) def TestAdHocReasonRateLimit(): """Tests that ad-hoc rate limiting using --reason="rate-limit:n:..." works. """ # Make sure our test is not constrained by "max-running-jobs" # (simply set it to the default). AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"]) AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"]) # Only the first 2 jobs must be scheduled. jid1 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "--reason=rate-limit:2:hello", "200", ])) jid2 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "--reason=rate-limit:2:hello", "200", ])) jid3 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "--reason=rate-limit:2:hello", "200", ])) time.sleep(5) # give the scheduler some time to notice AssertIn(GetJobStatus(jid1), ["running", "waiting"], msg="Job should not be rate-limited") AssertIn(GetJobStatus(jid2), ["running", "waiting"], msg="Job should not be rate-limited") AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited") # Clean up. KillWaitJobs([jid1, jid2, jid3]) ganeti-3.1.0~rc2/qa/qa_group.py000064400000000000000000000312531476477700300164000ustar00rootroot00000000000000# # # Copyright (C) 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """QA tests for node groups. """ from ganeti import constants from ganeti import netutils from ganeti import query from ganeti import utils from qa import qa_iptables from qa import qa_config from qa import qa_utils from qa_utils import AssertCommand, AssertEqual, GetCommandOutput def GetDefaultGroup(): """Returns the default node group. """ groups = qa_config.get("groups", {}) return groups.get("group-with-nodes", constants.INITIAL_NODE_GROUP_NAME) def ConfigureGroups(): """Configures groups and nodes for tests such as custom SSH ports. """ defgroup = GetDefaultGroup() nodes = qa_config.get("nodes") options = qa_config.get("options", {}) # Clear any old configuration qa_iptables.CleanRules(nodes) # Custom SSH ports: ssh_port = options.get("ssh-port") default_ssh_port = netutils.GetDaemonPort(constants.SSH) if (ssh_port is not None) and (ssh_port != default_ssh_port): ModifyGroupSshPort(qa_iptables.GLOBAL_RULES, defgroup, nodes, ssh_port) def ModifyGroupSshPort(ipt_rules, group, nodes, ssh_port): """Modifies the node group settings and sets up iptable rules. For each pair of nodes add two rules that affect SSH connections from one to the other one. The first one redirects port 22 to some unused port so that connecting through 22 fails. The second redirects port `ssh_port` to port 22. Together this results in master seeing the SSH daemons on the nodes on `ssh_port` instead of 22. """ default_ssh_port = netutils.GetDaemonPort(constants.SSH) all_nodes = qa_config.get("nodes") AssertCommand(["gnt-group", "modify", "--node-parameters=ssh_port=" + str(ssh_port), group]) for node in nodes: ipt_rules.RedirectPort(node.primary, "localhost", default_ssh_port, 65535) ipt_rules.RedirectPort(node.primary, "localhost", ssh_port, default_ssh_port) for node2 in all_nodes: ipt_rules.RedirectPort(node2.primary, node.primary, default_ssh_port, 65535) ipt_rules.RedirectPort(node2.primary, node.primary, ssh_port, default_ssh_port) def TestGroupAddRemoveRename(): """gnt-group add/remove/rename""" existing_group_with_nodes = GetDefaultGroup() (group1, group2, group3) = qa_utils.GetNonexistentGroups(3) AssertCommand(["gnt-group", "add", group1]) AssertCommand(["gnt-group", "add", group2]) AssertCommand(["gnt-group", "add", group2], fail=True) AssertCommand(["gnt-group", "add", existing_group_with_nodes], fail=True) AssertCommand(["gnt-group", "rename", group1, group2], fail=True) AssertCommand(["gnt-group", "rename", group1, group3]) try: AssertCommand(["gnt-group", "rename", existing_group_with_nodes, group1]) AssertCommand(["gnt-group", "remove", group2]) AssertCommand(["gnt-group", "remove", group3]) AssertCommand(["gnt-group", "remove", group1], fail=True) finally: # Try to ensure idempotency re groups that already existed. AssertCommand(["gnt-group", "rename", group1, existing_group_with_nodes]) def TestGroupAddWithOptions(): """gnt-group add with options""" (group1, ) = qa_utils.GetNonexistentGroups(1) AssertCommand(["gnt-group", "add", "--alloc-policy", "notvalid", group1], fail=True) AssertCommand(["gnt-group", "add", "--alloc-policy", "last_resort", "--node-parameters", "oob_program=/bin/true", group1]) AssertCommand(["gnt-group", "remove", group1]) class NewGroupCtx(object): """Creates a new group and disposes afterwards.""" def __enter__(self): (self._group, ) = qa_utils.GetNonexistentGroups(1) AssertCommand(["gnt-group", "add", self._group]) return self._group def __exit__(self, exc_type, exc_val, exc_tb): AssertCommand(["gnt-group", "remove", self._group]) def _GetGroupIPolicy(groupname): """Return the run-time values of the cluster-level instance policy. @type groupname: string @param groupname: node group name @rtype: tuple @return: (policy, specs), where: - policy is a dictionary of the policy values, instance specs excluded - specs is a dictionary containing only the specs, using the internal format (see L{constants.IPOLICY_DEFAULTS} for an example), but without the standard values """ info = qa_utils.GetObjectInfo(["gnt-group", "info", groupname]) assert len(info) == 1 policy = info[0]["Instance policy"] (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy) # Sanity checks assert "minmax" in ret_specs assert len(ret_specs["minmax"]) > 0 assert len(ret_policy) > 0 return (ret_policy, ret_specs) def _TestGroupSetISpecs(groupname, new_specs=None, diff_specs=None, fail=False, old_values=None): """Change instance specs on a group. At most one of new_specs or diff_specs can be specified. @type groupname: string @param groupname: group name @type new_specs: dict @param new_specs: new complete specs, in the same format returned by L{_GetGroupIPolicy} @type diff_specs: dict @param diff_specs: partial specs, it can be an incomplete specifications, but if min/max specs are specified, their number must match the number of the existing specs @type fail: bool @param fail: if the change is expected to fail @type old_values: tuple @param old_values: (old_policy, old_specs), as returned by L{_GetGroupIPolicy} @return: same as L{_GetGroupIPolicy} """ build_cmd = lambda opts: ["gnt-group", "modify"] + opts + [groupname] get_policy = lambda: _GetGroupIPolicy(groupname) return qa_utils.TestSetISpecs( new_specs=new_specs, diff_specs=diff_specs, get_policy_fn=get_policy, build_cmd_fn=build_cmd, fail=fail, old_values=old_values) def _TestGroupModifyISpecs(groupname): # This test is built on the assumption that the default ipolicy holds for # the node group under test old_values = _GetGroupIPolicy(groupname) samevals = dict((p, 4) for p in constants.ISPECS_PARAMETERS) base_specs = { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: samevals, constants.ISPECS_MAX: samevals, }], } mod_values = _TestGroupSetISpecs(groupname, new_specs=base_specs, old_values=old_values) for par in constants.ISPECS_PARAMETERS: # First make sure that the test works with good values good_specs = { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: {par: 8}, constants.ISPECS_MAX: {par: 8}, }], } mod_values = _TestGroupSetISpecs(groupname, diff_specs=good_specs, old_values=mod_values) bad_specs = { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: {par: 8}, constants.ISPECS_MAX: {par: 4}, }], } _TestGroupSetISpecs(groupname, diff_specs=bad_specs, fail=True, old_values=mod_values) AssertCommand(["gnt-group", "modify", "--ipolicy-bounds-specs", "default", groupname]) AssertEqual(_GetGroupIPolicy(groupname), old_values) # Get the ipolicy command (from the cluster config) mnode = qa_config.GetMasterNode() addcmd = GetCommandOutput(mnode.primary, utils.ShellQuoteArgs([ "gnt-group", "show-ispecs-cmd", "--include-defaults", groupname, ])) modcmd = ["gnt-group", "modify"] opts = addcmd.split() assert opts[0:2] == ["gnt-group", "add"] for k in range(2, len(opts) - 1): if opts[k].startswith("--ipolicy-"): assert k + 2 <= len(opts) modcmd.extend(opts[k:k + 2]) modcmd.append(groupname) # Apply the ipolicy to the group and verify the result AssertCommand(modcmd) new_addcmd = GetCommandOutput(mnode.primary, utils.ShellQuoteArgs([ "gnt-group", "show-ispecs-cmd", groupname, ])) AssertEqual(addcmd, new_addcmd) def _TestGroupModifyIPolicy(groupname): _TestGroupModifyISpecs(groupname) # We assume that the default ipolicy holds (old_policy, old_specs) = _GetGroupIPolicy(groupname) for (par, setval, iname, expval) in [ ("vcpu-ratio", 1.5, None, 1.5), ("spindle-ratio", 1.5, None, 1.5), ("disk-templates", constants.DT_PLAIN, "allowed disk templates", constants.DT_PLAIN) ]: if not iname: iname = par build_cmdline = lambda val: ["gnt-group", "modify", "--ipolicy-" + par, str(val), groupname] AssertCommand(build_cmdline(setval)) (new_policy, new_specs) = _GetGroupIPolicy(groupname) AssertEqual(new_specs, old_specs) for (p, val) in new_policy.items(): if p == iname: AssertEqual(val, expval) else: AssertEqual(val, old_policy[p]) AssertCommand(build_cmdline("default")) (new_policy, new_specs) = _GetGroupIPolicy(groupname) AssertEqual(new_specs, old_specs) AssertEqual(new_policy, old_policy) def TestGroupModify(): """gnt-group modify""" # This tests assumes LVM to be enabled, thus it should skip if # this is not the case if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG): return (group1, ) = qa_utils.GetNonexistentGroups(1) AssertCommand(["gnt-group", "add", group1]) try: _TestGroupModifyIPolicy(group1) AssertCommand(["gnt-group", "modify", "--alloc-policy", "unallocable", "--node-parameters", "oob_program=/bin/false", group1]) AssertCommand(["gnt-group", "modify", "--alloc-policy", "notvalid", group1], fail=True) AssertCommand(["gnt-group", "modify", "--node-parameters", "spindle_count=10", group1]) if qa_config.TestEnabled("htools"): AssertCommand(["hbal", "-L", "-G", group1]) AssertCommand(["gnt-group", "modify", "--node-parameters", "spindle_count=default", group1]) finally: AssertCommand(["gnt-group", "remove", group1]) def TestGroupList(): """gnt-group list""" qa_utils.GenericQueryTest("gnt-group", list(query.GROUP_FIELDS)) def TestGroupListFields(): """gnt-group list-fields""" qa_utils.GenericQueryFieldsTest("gnt-group", list(query.GROUP_FIELDS)) def TestAssignNodesIncludingSplit(orig_group, node1, node2): """gnt-group assign-nodes --force Expects node1 and node2 to be primary and secondary for a common instance. """ assert node1 != node2 (other_group, ) = qa_utils.GetNonexistentGroups(1) master_node = qa_config.GetMasterNode().primary def AssertInGroup(group, nodes): real_output = GetCommandOutput(master_node, "gnt-node list --no-headers -o group " + utils.ShellQuoteArgs(nodes)) AssertEqual(real_output.splitlines(), [group] * len(nodes)) AssertInGroup(orig_group, [node1, node2]) AssertCommand(["gnt-group", "add", other_group]) try: AssertCommand(["gnt-group", "assign-nodes", other_group, node1, node2]) AssertInGroup(other_group, [node1, node2]) # This should fail because moving node1 to orig_group would leave their # common instance split between orig_group and other_group. AssertCommand(["gnt-group", "assign-nodes", orig_group, node1], fail=True) AssertInGroup(other_group, [node1, node2]) AssertCommand(["gnt-group", "assign-nodes", "--force", orig_group, node1]) AssertInGroup(orig_group, [node1]) AssertInGroup(other_group, [node2]) AssertCommand(["gnt-group", "assign-nodes", orig_group, node2]) AssertInGroup(orig_group, [node1, node2]) finally: AssertCommand(["gnt-group", "remove", other_group]) ganeti-3.1.0~rc2/qa/qa_instance.py000064400000000000000000001640571476477700300170610ustar00rootroot00000000000000# # # Copyright (C) 2007, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Instance related QA tests. """ import os import re import time import yaml from ganeti import utils from ganeti import constants from ganeti import pathutils from ganeti import query from ganeti.netutils import IP4Address from qa import qa_config from qa import qa_daemon from qa import qa_utils from qa import qa_error from qa_filters import stdout_of from qa_utils import AssertCommand, AssertEqual, AssertIn from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE from qa_instance_utils import CheckSsconfInstanceList, \ CreateInstanceDrbd8, \ CreateInstanceByDiskTemplate, \ CreateInstanceByDiskTemplateOneNode, \ GetGenericAddParameters def _GetDiskStatePath(disk): return "/sys/block/%s/device/state" % disk def GetInstanceInfo(instance): """Return information about the actual state of an instance. @type instance: string @param instance: the instance name @return: a dictionary with the following keys: - "nodes": instance nodes, a list of strings - "volumes": instance volume IDs, a list of strings - "drbd-minors": DRBD minors used by the instance, a dictionary where keys are nodes, and values are lists of integers (or an empty dictionary for non-DRBD instances) - "disk-template": instance disk template - "storage-type": storage type associated with the instance disk template - "hypervisor-parameters": all hypervisor parameters for this instance """ node_elem = r"([^,()]+)(?:\s+\([^)]+\))?" # re_nodelist matches a list of nodes returned by gnt-instance info, e.g.: # node1.fqdn # node2.fqdn,node3.fqdn # node4.fqdn (group mygroup, group UUID 01234567-abcd-0123-4567-0123456789ab) # FIXME This works with no more than 2 secondaries re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$") info = qa_utils.GetObjectInfo(["gnt-instance", "info", instance])[0] nodes = [] for nodeinfo in info["Nodes"]: if "primary" in nodeinfo: nodes.append(nodeinfo["primary"]) elif "secondaries" in nodeinfo: nodestr = nodeinfo["secondaries"] if nodestr: m = re_nodelist.match(nodestr) if m: nodes.extend([n for n in m.groups() if n]) else: nodes.append(nodestr) re_drbdnode = re.compile(r"^([^\s,]+),\s+minor=([0-9]+)$") vols = [] drbd_min = {} dtypes = [] for (count, diskinfo) in enumerate(info["Disks"]): (dtype, _) = diskinfo["disk/%s" % count].split(",", 1) dtypes.append(dtype) if dtype == constants.DT_DRBD8: for child in diskinfo["child devices"]: vols.append(child["logical_id"]) for key in ["nodeA", "nodeB"]: m = re_drbdnode.match(diskinfo[key]) if not m: raise qa_error.Error("Cannot parse DRBD info: %s" % diskinfo[key]) node = m.group(1) minor = int(m.group(2)) minorlist = drbd_min.setdefault(node, []) minorlist.append(minor) elif dtype == constants.DT_PLAIN: vols.append(diskinfo["logical_id"]) # TODO remove and modify calling sites disk_template = utils.GetDiskTemplateString(dtypes) storage_type = constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[disk_template] hvparams = {} # make sure to not iterate on hypervisors w/o parameters (e.g. fake HV) if isinstance(info["Hypervisor parameters"], dict): for param, value in info["Hypervisor parameters"].items(): if type(value) == str and value.startswith("default ("): result = re.search(r'^default \((.*)\)$', value) try: value = yaml.load(result.group(1), Loader=yaml.SafeLoader) except yaml.YAMLError: value = '' hvparams[param] = value assert nodes assert len(nodes) < 2 or vols return { "nodes": nodes, "volumes": vols, "drbd-minors": drbd_min, "disk-template": disk_template, "storage-type": storage_type, "hypervisor-parameters": hvparams, } def _DestroyInstanceDisks(instance): """Remove all the backend disks of an instance. This is used to simulate HW errors (dead nodes, broken disks...); the configuration of the instance is not affected. @type instance: dictionary @param instance: the instance """ info = GetInstanceInfo(instance.name) # FIXME: destruction/removal should be part of the disk class if info["storage-type"] == constants.ST_LVM_VG: vols = info["volumes"] for node in info["nodes"]: AssertCommand(["lvremove", "-f"] + vols, node=node) elif info["storage-type"] in (constants.ST_FILE, constants.ST_SHARED_FILE): # Note that this works for both file and sharedfile, and this is intended. storage_dir = qa_config.get("file-storage-dir", pathutils.DEFAULT_FILE_STORAGE_DIR) idir = os.path.join(storage_dir, instance.name) for node in info["nodes"]: AssertCommand(["rm", "-rf", idir], node=node) elif info["storage-type"] == constants.ST_DISKLESS: pass def _GetInstanceFields(instance, fields): """Get the value of one or more fields of an instance. @type instance: string @param instance: instance name @type field: list of string @param field: name of the fields @rtype: list of string @return: value of the fields """ master = qa_config.GetMasterNode() infocmd = utils.ShellQuoteArgs(["gnt-instance", "list", "--no-headers", "--separator=:", "--units", "m", "-o", ",".join(fields), instance]) return tuple(qa_utils.GetCommandOutput(master.primary, infocmd) .strip() .split(":")) def _GetInstanceField(instance, field): """Get the value of a field of an instance. @type instance: string @param instance: Instance name @type field: string @param field: Name of the field @rtype: string """ return _GetInstanceFields(instance, [field])[0] def _GetBoolInstanceField(instance, field): """Get the Boolean value of a field of an instance. @type instance: string @param instance: Instance name @type field: string @param field: Name of the field @rtype: bool """ info_out = _GetInstanceField(instance, field) if info_out == "Y": return True elif info_out == "N": return False else: raise qa_error.Error("Field %s of instance %s has a non-Boolean value:" " %s" % (field, instance, info_out)) def _GetNumInstanceField(instance, field): """Get a numeric value of a field of an instance. @type instance: string @param instance: Instance name @type field: string @param field: Name of the field @rtype: int or float """ info_out = _GetInstanceField(instance, field) try: ret = int(info_out) except ValueError: try: ret = float(info_out) except ValueError: raise qa_error.Error("Field %s of instance %s has a non-numeric value:" " %s" % (field, instance, info_out)) return ret def GetInstanceSpec(instance, spec): """Return the current spec for the given parameter. @type instance: string @param instance: Instance name @type spec: string @param spec: one of the supported parameters: "memory-size", "cpu-count", "disk-count", "disk-size", "nic-count" @rtype: tuple @return: (minspec, maxspec); minspec and maxspec can be different only for memory and disk size """ specmap = { "memory-size": ["be/minmem", "be/maxmem"], "cpu-count": ["vcpus"], "disk-count": ["disk.count"], "disk-size": ["disk.size/ "], "nic-count": ["nic.count"], } # For disks, first we need the number of disks if spec == "disk-size": (numdisk, _) = GetInstanceSpec(instance, "disk-count") fields = ["disk.size/%s" % k for k in range(0, numdisk)] else: assert spec in specmap, "%s not in %s" % (spec, specmap) fields = specmap[spec] values = [_GetNumInstanceField(instance, f) for f in fields] return (min(values), max(values)) def IsFailoverSupported(instance): return instance.disk_template in constants.DTS_MIRRORED def IsMigrationSupported(instance): return instance.disk_template in constants.DTS_MIRRORED def IsDiskReplacingSupported(instance): return instance.disk_template == constants.DT_DRBD8 def IsDiskSupported(instance): return instance.disk_template != constants.DT_DISKLESS def TestInstanceAddWithPlainDisk(nodes, fail=False): """gnt-instance add -t plain""" if constants.DT_PLAIN in qa_config.GetEnabledDiskTemplates(): instance = CreateInstanceByDiskTemplateOneNode(nodes, constants.DT_PLAIN, fail=fail) if not fail: qa_utils.RunInstanceCheck(instance, True) return instance @InstanceCheck(None, INST_UP, RETURN_VALUE) def TestInstanceAddWithDrbdDisk(nodes): """gnt-instance add -t drbd""" if constants.DT_DRBD8 in qa_config.GetEnabledDiskTemplates(): return CreateInstanceDrbd8(nodes) @InstanceCheck(None, INST_UP, RETURN_VALUE) def TestInstanceAddFile(nodes): """gnt-instance add -t file""" assert len(nodes) == 1 if constants.DT_FILE in qa_config.GetEnabledDiskTemplates(): return CreateInstanceByDiskTemplateOneNode(nodes, constants.DT_FILE) @InstanceCheck(None, INST_UP, RETURN_VALUE) def TestInstanceAddSharedFile(nodes): """gnt-instance add -t sharedfile""" assert len(nodes) == 1 if constants.DT_SHARED_FILE in qa_config.GetEnabledDiskTemplates(): return CreateInstanceByDiskTemplateOneNode(nodes, constants.DT_SHARED_FILE) @InstanceCheck(None, INST_UP, RETURN_VALUE) def TestInstanceAddDiskless(nodes): """gnt-instance add -t diskless""" assert len(nodes) == 1 if constants.DT_DISKLESS in qa_config.GetEnabledDiskTemplates(): return CreateInstanceByDiskTemplateOneNode(nodes, constants.DT_DISKLESS) @InstanceCheck(None, INST_UP, RETURN_VALUE) def TestInstanceAddRADOSBlockDevice(nodes): """gnt-instance add -t rbd""" assert len(nodes) == 1 if constants.DT_RBD in qa_config.GetEnabledDiskTemplates(): return CreateInstanceByDiskTemplateOneNode(nodes, constants.DT_RBD) @InstanceCheck(None, INST_UP, RETURN_VALUE) def TestInstanceAddGluster(nodes): """gnt-instance add -t gluster""" assert len(nodes) == 1 if constants.DT_GLUSTER in qa_config.GetEnabledDiskTemplates(): return CreateInstanceByDiskTemplateOneNode(nodes, constants.DT_GLUSTER) @InstanceCheck(None, INST_DOWN, FIRST_ARG) def TestInstanceRemove(instance): """gnt-instance remove""" AssertCommand(["gnt-instance", "remove", "-f", instance.name]) @InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG) def TestInstanceStartup(instance): """gnt-instance startup""" AssertCommand(["gnt-instance", "startup", instance.name]) @InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG) def TestInstanceShutdown(instance): """gnt-instance shutdown""" AssertCommand(["gnt-instance", "shutdown", instance.name]) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceReboot(instance): """gnt-instance reboot""" options = qa_config.get("options", {}) reboot_types = options.get("reboot-types", constants.REBOOT_TYPES) name = instance.name for rtype in reboot_types: AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name]) AssertCommand(["gnt-instance", "shutdown", name]) qa_utils.RunInstanceCheck(instance, False) AssertCommand(["gnt-instance", "reboot", name]) master = qa_config.GetMasterNode() cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertEqual(result_output.strip(), constants.INSTST_RUNNING) @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestInstanceReinstall(instance): """gnt-instance reinstall""" if instance.disk_template == constants.DT_DISKLESS: print(qa_utils.FormatInfo("Test not supported for diskless instances")) return qa_storage = qa_config.get("qa-storage") if qa_storage is None: print(qa_utils.FormatInfo("Test not supported because the additional QA" " storage is not available")) else: # Reinstall with OS image from QA storage url = "%s/busybox.img" % qa_storage AssertCommand(["gnt-instance", "reinstall", "--os-parameters", "os-image=" + url, "-f", instance.name]) # Reinstall with OS image as local file on the node pnode = _GetInstanceField(instance.name, "pnode") cmd = ("wget -O busybox.img %s &> /dev/null &&" " echo $(pwd)/busybox.img") % url image = qa_utils.GetCommandOutput(pnode, cmd).strip() AssertCommand(["gnt-instance", "reinstall", "--os-parameters", "os-image=" + image, "-f", instance.name]) # Reinstall non existing local file AssertCommand(["gnt-instance", "reinstall", "--os-parameters", "os-image=NonExistantOsForQa", "-f", instance.name], fail=True) # Reinstall non existing URL AssertCommand(["gnt-instance", "reinstall", "--os-parameters", "os-image=http://NonExistantOsForQa", "-f", instance.name], fail=True) # Reinstall using OS scripts AssertCommand(["gnt-instance", "reinstall", "-f", instance.name]) # Test with non-existant OS definition AssertCommand(["gnt-instance", "reinstall", "-f", "--os-type=NonExistantOsForQa", instance.name], fail=True) # Test with existing OS but invalid variant AssertCommand(["gnt-instance", "reinstall", "-f", "-o", "debootstrap+ola", instance.name], fail=True) # Test with existing OS but invalid variant AssertCommand(["gnt-instance", "reinstall", "-f", "-o", "debian-image+ola", instance.name], fail=True) @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestInstanceRenameAndBack(rename_source, rename_target): """gnt-instance rename This must leave the instance with the original name, not the target name. """ CheckSsconfInstanceList(rename_source) # first do a rename to a different actual name, expecting it to fail qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target]) try: AssertCommand(["gnt-instance", "rename", "--name-check", rename_source, rename_target], fail=True) CheckSsconfInstanceList(rename_source) finally: qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target]) info = GetInstanceInfo(rename_source) # Check instance volume tags correctly updated. Note that this check is lvm # specific, so we skip it for non-lvm-based instances. # FIXME: This will need updating when instances will be able to have # different disks living on storage pools with etherogeneous storage types. # FIXME: This check should be put inside the disk/storage class themselves, # rather than explicitly called here. if info["storage-type"] == constants.ST_LVM_VG: # In the lvm world we can check for tags on the logical volume tags_cmd = ("lvs -o tags --noheadings %s | grep " % (" ".join(info["volumes"]), )) else: # Other storage types don't have tags, so we use an always failing command, # to make sure it never gets executed tags_cmd = "false" # and now rename instance to rename_target... AssertCommand(["gnt-instance", "rename", "--name-check", rename_source, rename_target]) CheckSsconfInstanceList(rename_target) qa_utils.RunInstanceCheck(rename_source, False) qa_utils.RunInstanceCheck(rename_target, False) # NOTE: tags might not be the exactly as the instance name, due to # charset restrictions; hence the test might be flaky if (rename_source != rename_target and info["storage-type"] == constants.ST_LVM_VG): for node in info["nodes"]: AssertCommand(tags_cmd + rename_source, node=node, fail=True) AssertCommand(tags_cmd + rename_target, node=node, fail=False) # and back AssertCommand(["gnt-instance", "rename", "--name-check", rename_target, rename_source]) CheckSsconfInstanceList(rename_source) qa_utils.RunInstanceCheck(rename_target, False) if (rename_source != rename_target and info["storage-type"] == constants.ST_LVM_VG): for node in info["nodes"]: AssertCommand(tags_cmd + rename_source, node=node, fail=False) AssertCommand(tags_cmd + rename_target, node=node, fail=True) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceFailover(instance): """gnt-instance failover""" if not IsFailoverSupported(instance): print(qa_utils.FormatInfo("Instance doesn't support failover, skipping" " test")) return cmd = ["gnt-instance", "failover", "--force", instance.name] # failover ... AssertCommand(cmd) qa_utils.RunInstanceCheck(instance, True) # ... and back AssertCommand(cmd) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceMigrate(instance, toggle_always_failover=True): """gnt-instance migrate""" if not IsMigrationSupported(instance): print(qa_utils.FormatInfo("Instance doesn't support migration, skipping" " test")) return cmd = ["gnt-instance", "migrate", "--force", instance.name] af_par = constants.BE_ALWAYS_FAILOVER af_field = "be/" + constants.BE_ALWAYS_FAILOVER af_init_val = _GetBoolInstanceField(instance.name, af_field) # migrate ... AssertCommand(cmd) # TODO: Verify the choice between failover and migration qa_utils.RunInstanceCheck(instance, True) # ... and back (possibly with always_failover toggled) if toggle_always_failover: AssertCommand(["gnt-instance", "modify", "-B", ("%s=%s" % (af_par, not af_init_val)), instance.name]) AssertCommand(cmd) # TODO: Verify the choice between failover and migration qa_utils.RunInstanceCheck(instance, True) if toggle_always_failover: AssertCommand(["gnt-instance", "modify", "-B", ("%s=%s" % (af_par, af_init_val)), instance.name]) # TODO: Split into multiple tests AssertCommand(["gnt-instance", "shutdown", instance.name]) qa_utils.RunInstanceCheck(instance, False) AssertCommand(cmd, fail=True) AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover", instance.name]) AssertCommand(["gnt-instance", "start", instance.name]) AssertCommand(cmd) # @InstanceCheck enforces the check that the instance is running qa_utils.RunInstanceCheck(instance, True) AssertCommand(["gnt-instance", "modify", "-B", ("%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)), instance.name]) AssertCommand(cmd) qa_utils.RunInstanceCheck(instance, True) # TODO: Verify that a failover has been done instead of a migration # TODO: Verify whether the default value is restored here (not hardcoded) AssertCommand(["gnt-instance", "modify", "-B", ("%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)), instance.name]) AssertCommand(cmd) qa_utils.RunInstanceCheck(instance, True) def TestInstanceInfo(instance): """gnt-instance info""" AssertCommand(["gnt-instance", "info", instance.name]) def _TestKVMHotplug(instance, instance_info): """Tests hotplug modification commands, noting that they """ args_to_try = [ ["--net", "-1:add"], ["--net", "-1:modify,mac=aa:bb:cc:dd:ee:ff", "--force"], ["--net", "-1:remove"] ] if instance_info["hypervisor-parameters"]["disk_type"] != \ constants.HT_DISK_IDE: # hotplugging disks is not supported for IDE-type disks args_to_try.append(["--disk", "-1:add,size=1G"]) args_to_try.append(["--disk", "-1:remove"]) for alist in args_to_try: _, stdout, stderr = \ AssertCommand(["gnt-instance", "modify"] + alist + [instance.name]) if "failed" in stdout or "failed" in stderr: raise qa_error.Error("Hotplugging command failed; please check output" " for further information") @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceModify(instance): """gnt-instance modify""" default_hv = qa_config.GetDefaultHypervisor() # Assume /sbin/init exists on all systems test_kernel = "/sbin/init" test_initrd = test_kernel instance_info = GetInstanceInfo(instance.name) orig_maxmem = qa_config.get(constants.BE_MAXMEM) orig_minmem = qa_config.get(constants.BE_MINMEM) #orig_bridge = qa_config.get("bridge", "xen-br0") args = [ ["-B", "%s=128" % constants.BE_MINMEM], ["-B", "%s=128" % constants.BE_MAXMEM], ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem, constants.BE_MAXMEM, orig_maxmem)], ["-B", "%s=2" % constants.BE_VCPUS], ["-B", "%s=1" % constants.BE_VCPUS], ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)], ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)], ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)], # TODO: bridge tests #["--bridge", "xen-br1"], #["--bridge", orig_bridge], ] # Not all hypervisors support kernel_path(e.g, LXC) if default_hv in (constants.HT_XEN_PVM, constants.HT_XEN_HVM, constants.HT_KVM): args.extend([ ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)], ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)], ]) if default_hv == constants.HT_XEN_PVM: args.extend([ ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)], ["-H", "no_%s" % (constants.HV_INITRD_PATH, )], ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)], ]) elif default_hv == constants.HT_XEN_HVM: args.extend([ ["-H", "%s=acn" % constants.HV_BOOT_ORDER], ["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)], ]) elif default_hv == constants.HT_KVM and \ qa_config.TestEnabled("instance-device-hotplug") and \ instance_info["hypervisor-parameters"]["acpi"]: _TestKVMHotplug(instance, instance_info) elif default_hv == constants.HT_LXC: args.extend([ ["-H", "%s=0" % constants.HV_CPU_MASK], ["-H", "%s=%s" % (constants.HV_CPU_MASK, constants.VALUE_DEFAULT)], ["-H", "%s=0" % constants.HV_LXC_NUM_TTYS], ["-H", "%s=%s" % (constants.HV_LXC_NUM_TTYS, constants.VALUE_DEFAULT)], ]) url = "http://example.com/busybox.img" args.extend([ ["--os-parameters", "os-image=" + url], ["--os-parameters", "os-image=default"] ]) for alist in args: AssertCommand(["gnt-instance", "modify"] + alist + [instance.name]) # check no-modify AssertCommand(["gnt-instance", "modify", instance.name], fail=True) # Marking offline while instance is running must fail... AssertCommand(["gnt-instance", "modify", "--offline", instance.name], fail=True) # ...while making it online fails too (needs to be offline first) AssertCommand(["gnt-instance", "modify", "--online", instance.name], fail=True) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceModifyPrimaryAndBack(instance, currentnode, othernode): """gnt-instance modify --new-primary This will leave the instance on its original primary node, not other node. """ if instance.disk_template != constants.DT_FILE: print(qa_utils.FormatInfo("Test only supported for the file disk template")) return cluster_name = qa_config.get("name") name = instance.name current = currentnode.primary other = othernode.primary filestorage = qa_config.get("file-storage-dir", pathutils.DEFAULT_FILE_STORAGE_DIR) disk = os.path.join(filestorage, name) AssertCommand(["gnt-instance", "modify", "--new-primary=%s" % other, name], fail=True) AssertCommand(["gnt-instance", "shutdown", name]) AssertCommand(["scp", "-oGlobalKnownHostsFile=%s" % pathutils.SSH_KNOWN_HOSTS_FILE, "-oCheckHostIp=no", "-oStrictHostKeyChecking=yes", "-oHashKnownHosts=no", "-oHostKeyAlias=%s" % cluster_name, "-r", disk, "%s:%s" % (other, filestorage)], node=current) AssertCommand(["gnt-instance", "modify", "--new-primary=%s" % other, name]) AssertCommand(["gnt-instance", "startup", name]) # and back AssertCommand(["gnt-instance", "shutdown", name]) AssertCommand(["rm", "-rf", disk], node=other) AssertCommand(["gnt-instance", "modify", "--new-primary=%s" % current, name]) AssertCommand(["gnt-instance", "startup", name]) @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestInstanceStoppedModify(instance): """gnt-instance modify (stopped instance)""" name = instance.name # Instance was not marked offline; try marking it online once more AssertCommand(["gnt-instance", "modify", "--online", name]) # Mark instance as offline AssertCommand(["gnt-instance", "modify", "--offline", name]) # When the instance is offline shutdown should only work with --force, # while start should never work AssertCommand(["gnt-instance", "shutdown", name], fail=True) AssertCommand(["gnt-instance", "shutdown", "--force", name]) AssertCommand(["gnt-instance", "start", name], fail=True) AssertCommand(["gnt-instance", "start", "--force", name], fail=True) # Also do offline to offline AssertCommand(["gnt-instance", "modify", "--offline", name]) # And online again AssertCommand(["gnt-instance", "modify", "--online", name]) @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestInstanceConvertDiskTemplate(instance, requested_conversions): """gnt-instance modify -t""" def _BuildConvertCommand(disk_template, node): cmd = ["gnt-instance", "modify", "-t", disk_template] if disk_template == constants.DT_DRBD8: cmd.extend(["-n", node]) cmd.append(name) return cmd if len(requested_conversions) < 2: print(qa_utils.FormatInfo("You must specify more than one convertible" " disk templates in order to test the conversion" " feature")) return name = instance.name template = instance.disk_template if template in constants.DTS_NOT_CONVERTIBLE_FROM: print(qa_utils.FormatInfo("Unsupported template %s, skipping conversion" " test" % template)) return inodes = qa_config.AcquireManyNodes(2) master = qa_config.GetMasterNode() snode = inodes[0].primary if master.primary == snode: snode = inodes[1].primary enabled_disk_templates = qa_config.GetEnabledDiskTemplates() for templ in requested_conversions: if (templ == template or templ not in enabled_disk_templates or templ in constants.DTS_NOT_CONVERTIBLE_TO): continue AssertCommand(_BuildConvertCommand(templ, snode)) # Before we return, convert to the original template AssertCommand(_BuildConvertCommand(template, snode)) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceModifyDisks(instance): """gnt-instance modify --disk""" if not IsDiskSupported(instance): print(qa_utils.FormatInfo("Instance doesn't support disks, skipping test")) return disk_conf = qa_config.GetDiskOptions()[-1] size = disk_conf.get("size") name = instance.name if qa_config.AreSpindlesSupported(): spindles = disk_conf.get("spindles") spindles_supported = True else: # Any number is good for spindles in this case spindles = 1 spindles_supported = False AssertCommand(["gnt-instance", "modify", "--disk", "add:size=%s,spindles=%s" % (size, spindles), "--no-hotplug", name], fail=not spindles_supported) AssertCommand(["gnt-instance", "modify", "--disk", "add:size=%s,spindles=%s" % (size, spindles), "--no-hotplug", name], fail=spindles_supported) # Exactly one of the above commands has succeded, so we need one remove AssertCommand(["gnt-instance", "modify", "--disk", "remove", "--no-hotplug", name]) @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestInstanceGrowDisk(instance): """gnt-instance grow-disk""" if instance.disk_template == constants.DT_DISKLESS: print(qa_utils.FormatInfo("Test not supported for diskless instances")) return name = instance.name disks = qa_config.GetDiskOptions() all_size = [d.get("size") for d in disks] all_grow = [d.get("growth") for d in disks] if not all_grow: # missing disk sizes but instance grow disk has been enabled, # let's set fixed/nomimal growth all_grow = ["128M" for _ in all_size] for idx, (size, grow) in enumerate(zip(all_size, all_grow)): # succeed in grow by amount AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow]) # fail in grow to the old size AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx), size], fail=True) # succeed to grow to old size + 2 * growth int_size = utils.ParseUnit(size) int_grow = utils.ParseUnit(grow) AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx), str(int_size + 2 * int_grow)]) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceDeviceNames(instance): if instance.disk_template == constants.DT_DISKLESS: print(qa_utils.FormatInfo("Test not supported for diskless instances")) return name = instance.name for dev_type in ["disk", "net"]: if dev_type == "disk": options = ",size=512M" if qa_config.AreSpindlesSupported(): options += ",spindles=1" else: options = "" # succeed in adding a device named 'test_device' AssertCommand(["gnt-instance", "modify", "--%s=-1:add,name=test_device%s" % (dev_type, options), "--no-hotplug", name]) # succeed in removing the 'test_device' AssertCommand(["gnt-instance", "modify", "--%s=test_device:remove" % dev_type, "--no-hotplug", name]) # fail to add two devices with the same name AssertCommand(["gnt-instance", "modify", "--%s=-1:add,name=test_device%s" % (dev_type, options), "--%s=-1:add,name=test_device%s" % (dev_type, options), "--no-hotplug", name], fail=True) # fail to add a device with invalid name AssertCommand(["gnt-instance", "modify", "--%s=-1:add,name=2%s" % (dev_type, options), "--no-hotplug", name], fail=True) # Rename disks disks = qa_config.GetDiskOptions() disk_names = [d.get("name") for d in disks] for idx, disk_name in enumerate(disk_names): # Refer to disk by idx AssertCommand(["gnt-instance", "modify", "--disk=%s:modify,name=renamed" % idx, name]) # Refer to by name and rename to original name AssertCommand(["gnt-instance", "modify", "--disk=renamed:modify,name=%s" % disk_name, name]) if len(disks) >= 2: # fail in renaming to disks to the same name AssertCommand(["gnt-instance", "modify", "--disk=0:modify,name=same_name", "--disk=1:modify,name=same_name", name], fail=True) def TestInstanceList(): """gnt-instance list""" qa_utils.GenericQueryTest("gnt-instance", list(query.INSTANCE_FIELDS)) def TestInstanceListFields(): """gnt-instance list-fields""" qa_utils.GenericQueryFieldsTest("gnt-instance", list(query.INSTANCE_FIELDS)) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceConsole(instance): """gnt-instance console""" AssertCommand(["gnt-instance", "console", "--show-cmd", instance.name]) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestReplaceDisks(instance, curr_nodes, other_nodes): """gnt-instance replace-disks""" def buildcmd(args): cmd = ["gnt-instance", "replace-disks"] cmd.extend(args) cmd.append(instance.name) return cmd if not IsDiskReplacingSupported(instance): print(qa_utils.FormatInfo("Instance doesn't support disk replacing," " skipping test")) return # Currently all supported templates have one primary and one secondary node assert len(curr_nodes) == 2 snode = curr_nodes[1] assert len(other_nodes) == 1 othernode = other_nodes[0] options = qa_config.get("options", {}) use_ialloc = options.get("use-iallocators", True) for data in [ ["-p"], ["-s"], # A placeholder; the actual command choice depends on use_ialloc None, # Restore the original secondary ["--new-secondary=%s" % snode.primary], ]: if data is None: if use_ialloc: data = ["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT] else: data = ["--new-secondary=%s" % othernode.primary] AssertCommand(buildcmd(data)) AssertCommand(buildcmd(["-a"])) AssertCommand(["gnt-instance", "stop", instance.name]) AssertCommand(buildcmd(["-a"]), fail=True) AssertCommand(["gnt-instance", "activate-disks", instance.name]) AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync", instance.name]) AssertCommand(buildcmd(["-a"])) AssertCommand(["gnt-instance", "start", instance.name]) def _AssertRecreateDisks(cmdargs, instance, fail=False, check=True, destroy=True): """Execute gnt-instance recreate-disks and check the result @param cmdargs: Arguments (instance name excluded) @param instance: Instance to operate on @param fail: True if the command is expected to fail @param check: If True and fail is False, check that the disks work @prama destroy: If True, destroy the old disks first """ if destroy: _DestroyInstanceDisks(instance) AssertCommand((["gnt-instance", "recreate-disks"] + cmdargs + [instance.name]), fail) if not fail and check: # Quick check that the disks are there AssertCommand(["gnt-instance", "activate-disks", instance.name]) AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync", instance.name]) AssertCommand(["gnt-instance", "deactivate-disks", instance.name]) def _BuildRecreateDisksOpts(en_disks, with_spindles, with_growth, spindles_supported): if with_spindles: if spindles_supported: if with_growth: build_spindles_opt = (lambda disk: ",spindles=%s" % (disk["spindles"] + disk["spindles-growth"])) else: build_spindles_opt = (lambda disk: ",spindles=%s" % disk["spindles"]) else: build_spindles_opt = (lambda _: ",spindles=1") else: build_spindles_opt = (lambda _: "") if with_growth: build_size_opt = (lambda disk: "size=%s" % (utils.ParseUnit(disk["size"]) + utils.ParseUnit(disk["growth"]))) else: build_size_opt = (lambda disk: "size=%s" % disk["size"]) build_disk_opt = (lambda idx_dsk: "--disk=%s:%s%s" % (idx_dsk[0], build_size_opt(idx_dsk[1]), build_spindles_opt(idx_dsk[1]))) return [build_disk_opt(d) for d in en_disks] @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRecreateDisks(instance, inodes, othernodes): """gnt-instance recreate-disks @param instance: Instance to work on @param inodes: List of the current nodes of the instance @param othernodes: list/tuple of nodes where to temporarily recreate disks """ options = qa_config.get("options", {}) use_ialloc = options.get("use-iallocators", True) other_seq = ":".join([n.primary for n in othernodes]) orig_seq = ":".join([n.primary for n in inodes]) # These fail because the instance is running _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False) if use_ialloc: _AssertRecreateDisks(["-I", "hail"], instance, fail=True, destroy=False) else: _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False) AssertCommand(["gnt-instance", "stop", instance.name]) # Disks exist: this should fail _AssertRecreateDisks([], instance, fail=True, destroy=False) # Unsupported spindles parameters: fail if not qa_config.AreSpindlesSupported(): _AssertRecreateDisks(["--disk=0:spindles=2"], instance, fail=True, destroy=False) # Recreate disks in place _AssertRecreateDisks([], instance) # Move disks away if use_ialloc: _AssertRecreateDisks(["-I", "hail"], instance) # Move disks somewhere else _AssertRecreateDisks(["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT], instance) else: _AssertRecreateDisks(["-n", other_seq], instance) # Move disks back _AssertRecreateDisks(["-n", orig_seq], instance) # Recreate resized disks # One of the two commands fails because either spindles are given when they # should not or vice versa alldisks = qa_config.GetDiskOptions() spindles_supported = qa_config.AreSpindlesSupported() disk_opts = _BuildRecreateDisksOpts(enumerate(alldisks), True, True, spindles_supported) _AssertRecreateDisks(disk_opts, instance, destroy=True, fail=not spindles_supported) disk_opts = _BuildRecreateDisksOpts(enumerate(alldisks), False, True, spindles_supported) _AssertRecreateDisks(disk_opts, instance, destroy=False, fail=spindles_supported) # Recreate the disks one by one (with the original size) for (idx, disk) in enumerate(alldisks): # Only the first call should destroy all the disk destroy = (idx == 0) # Again, one of the two commands is expected to fail disk_opts = _BuildRecreateDisksOpts([(idx, disk)], True, False, spindles_supported) _AssertRecreateDisks(disk_opts, instance, destroy=destroy, check=False, fail=not spindles_supported) disk_opts = _BuildRecreateDisksOpts([(idx, disk)], False, False, spindles_supported) _AssertRecreateDisks(disk_opts, instance, destroy=False, check=False, fail=spindles_supported) # This and InstanceCheck decoration check that the disks are working AssertCommand(["gnt-instance", "reinstall", "-f", instance.name]) AssertCommand(["gnt-instance", "start", instance.name]) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceExport(instance, node): """gnt-backup export -n ...""" name = instance.name options = ["gnt-backup", "export", "-n", node.primary] # For files and shared files, the --long-sleep option should be used if instance.disk_template in [constants.DT_FILE, constants.DT_SHARED_FILE]: options.append("--long-sleep") AssertCommand(options + [name]) return qa_utils.ResolveInstanceName(name) @InstanceCheck(None, INST_DOWN, FIRST_ARG) def TestInstanceExportWithRemove(instance, node): """gnt-backup export --remove-instance""" AssertCommand(["gnt-backup", "export", "-n", node.primary, "--remove-instance", instance.name]) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceExportNoTarget(instance): """gnt-backup export (without target node, should fail)""" AssertCommand(["gnt-backup", "export", instance.name], fail=True) @InstanceCheck(None, INST_DOWN, FIRST_ARG) def TestInstanceImport(newinst, node, expnode, name): """gnt-backup import""" templ = constants.DT_PLAIN if not qa_config.IsTemplateSupported(templ): return cmd = (["gnt-backup", "import", "--disk-template=%s" % templ, "--no-ip-check", "--src-node=%s" % expnode.primary, "--src-dir=%s/%s" % (pathutils.EXPORT_DIR, name), "--node=%s" % node.primary] + GetGenericAddParameters(newinst, templ, force_mac=constants.VALUE_GENERATE)) cmd.append(newinst.name) AssertCommand(cmd) newinst.SetDiskTemplate(templ) def TestBackupList(expnode): """gnt-backup list""" AssertCommand(["gnt-backup", "list", "--node=%s" % expnode.primary]) qa_utils.GenericQueryTest("gnt-backup", list(query.EXPORT_FIELDS), namefield=None, test_unknown=False) def TestBackupListFields(): """gnt-backup list-fields""" qa_utils.GenericQueryFieldsTest("gnt-backup", list(query.EXPORT_FIELDS)) def TestRemoveInstanceOfflineNode(instance, snode, set_offline, set_online): """gnt-instance remove with an off-line node @param instance: instance @param snode: secondary node, to be set offline @param set_offline: function to call to set the node off-line @param set_online: function to call to set the node on-line """ info = GetInstanceInfo(instance.name) set_offline(snode) try: TestInstanceRemove(instance) finally: set_online(snode) # Clean up the disks on the offline node, if necessary if instance.disk_template not in constants.DTS_EXT_MIRROR: # FIXME: abstract the cleanup inside the disks if info["storage-type"] == constants.ST_LVM_VG: for minor in info["drbd-minors"][snode.primary]: # DRBD 8.3 syntax comes first, then DRBD 8.4 syntax. The 8.4 syntax # relies on the fact that we always create a resources for each minor, # and that this resources is always named resource{minor}. # As 'drbdsetup 0 down' does return success (even though that's invalid # syntax), we always have to perform both commands and ignore the # output. drbd_shutdown_cmd = \ "(drbdsetup %d down >/dev/null 2>&1;" \ " drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \ (minor, minor) AssertCommand(drbd_shutdown_cmd, node=snode) AssertCommand(["lvremove", "-f"] + info["volumes"], node=snode) elif info["storage-type"] == constants.ST_FILE: filestorage = qa_config.get("file-storage-dir", pathutils.DEFAULT_FILE_STORAGE_DIR) disk = os.path.join(filestorage, instance.name) AssertCommand(["rm", "-rf", disk], node=snode) def TestInstanceCreationRestrictedByDiskTemplates(): """Test adding instances for disabled disk templates.""" if qa_config.TestEnabled("cluster-exclusive-storage"): # These tests are valid only for non-exclusive storage return enabled_disk_templates = qa_config.GetEnabledDiskTemplates() nodes = qa_config.AcquireManyNodes(2) # Setup the cluster with the enabled_disk_templates AssertCommand( ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates), "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)], fail=False) # Test instance creation for enabled disk templates for disk_template in enabled_disk_templates: instance = CreateInstanceByDiskTemplate(nodes, disk_template, fail=False) TestInstanceRemove(instance) instance.Release() # Test that instance creation fails for disabled disk templates disabled_disk_templates = list(constants.DISK_TEMPLATES - set(enabled_disk_templates)) for disk_template in disabled_disk_templates: instance = CreateInstanceByDiskTemplate(nodes, disk_template, fail=True) # Test instance creation for after disabling enabled disk templates if (len(enabled_disk_templates) > 1): # Partition the disk templates, enable them separately and check if the # disabled ones cannot be used by instances. middle = len(enabled_disk_templates) // 2 templates1 = enabled_disk_templates[:middle] templates2 = enabled_disk_templates[middle:] for (enabled, disabled) in [(templates1, templates2), (templates2, templates1)]: AssertCommand(["gnt-cluster", "modify", "--enabled-disk-templates=%s" % ",".join(enabled), "--ipolicy-disk-templates=%s" % ",".join(enabled)], fail=False) for disk_template in disabled: CreateInstanceByDiskTemplate(nodes, disk_template, fail=True) elif (len(enabled_disk_templates) == 1): # If only one disk template is enabled in the QA config, we have to enable # some other templates in order to test if the disabling the only enabled # disk template prohibits creating instances of that template. other_disk_templates = list( set([constants.DT_DISKLESS, constants.DT_BLOCK]) - set(enabled_disk_templates)) AssertCommand(["gnt-cluster", "modify", "--enabled-disk-templates=%s" % ",".join(other_disk_templates), "--ipolicy-disk-templates=%s" % ",".join(other_disk_templates)], fail=False) CreateInstanceByDiskTemplate(nodes, enabled_disk_templates[0], fail=True) else: raise qa_error.Error("Please enable at least one disk template" " in your QA setup.") # Restore initially enabled disk templates AssertCommand(["gnt-cluster", "modify", "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates), "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)], fail=False) def _AssertInstance(instance, status, admin_state, admin_state_source): x, y, z = \ _GetInstanceFields(instance.name, ["status", "admin_state", "admin_state_source"]) AssertEqual(x, status) AssertEqual(y, admin_state) AssertEqual(z, admin_state_source) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def _TestInstanceUserDown(instance, hv_shutdown_fn): """Test different combinations of user shutdown""" # 1. User shutdown # 2. Instance start hv_shutdown_fn() _AssertInstance(instance, constants.INSTST_USERDOWN, constants.ADMINST_UP, constants.ADMIN_SOURCE) AssertCommand(["gnt-instance", "start", instance.name]) _AssertInstance(instance, constants.INSTST_RUNNING, constants.ADMINST_UP, constants.ADMIN_SOURCE) # 1. User shutdown # 2. Watcher cleanup # 3. Instance start hv_shutdown_fn() _AssertInstance(instance, constants.INSTST_USERDOWN, constants.ADMINST_UP, constants.ADMIN_SOURCE) qa_daemon.RunWatcherDaemon() _AssertInstance(instance, constants.INSTST_USERDOWN, constants.ADMINST_DOWN, constants.USER_SOURCE) AssertCommand(["gnt-instance", "start", instance.name]) _AssertInstance(instance, constants.INSTST_RUNNING, constants.ADMINST_UP, constants.ADMIN_SOURCE) # 1. User shutdown # 2. Watcher cleanup # 3. Instance stop # 4. Instance start hv_shutdown_fn() _AssertInstance(instance, constants.INSTST_USERDOWN, constants.ADMINST_UP, constants.ADMIN_SOURCE) qa_daemon.RunWatcherDaemon() _AssertInstance(instance, constants.INSTST_USERDOWN, constants.ADMINST_DOWN, constants.USER_SOURCE) AssertCommand(["gnt-instance", "shutdown", instance.name]) _AssertInstance(instance, constants.INSTST_ADMINDOWN, constants.ADMINST_DOWN, constants.ADMIN_SOURCE) AssertCommand(["gnt-instance", "start", instance.name]) _AssertInstance(instance, constants.INSTST_RUNNING, constants.ADMINST_UP, constants.ADMIN_SOURCE) # 1. User shutdown # 2. Instance stop # 3. Instance start hv_shutdown_fn() _AssertInstance(instance, constants.INSTST_USERDOWN, constants.ADMINST_UP, constants.ADMIN_SOURCE) AssertCommand(["gnt-instance", "shutdown", instance.name]) _AssertInstance(instance, constants.INSTST_ADMINDOWN, constants.ADMINST_DOWN, constants.ADMIN_SOURCE) AssertCommand(["gnt-instance", "start", instance.name]) _AssertInstance(instance, constants.INSTST_RUNNING, constants.ADMINST_UP, constants.ADMIN_SOURCE) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def _TestInstanceUserDownXen(instance): primary = _GetInstanceField(instance.name, "pnode") fn = lambda: AssertCommand(["xl", "shutdown", "-w", instance.name], node=primary) AssertCommand(["gnt-cluster", "modify", "--user-shutdown=true"]) _TestInstanceUserDown(instance, fn) AssertCommand(["gnt-cluster", "modify", "--user-shutdown=false"]) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def _TestInstanceUserDownKvm(instance): def _StopKVMInstance(): AssertCommand("pkill -f \"\\-name %s\"" % instance.name, node=primary) time.sleep(10) AssertCommand(["gnt-cluster", "modify", "--user-shutdown=true"]) AssertCommand(["gnt-instance", "modify", "-H", "user_shutdown=true", instance.name]) # The instance needs to reboot not because the 'user_shutdown' # parameter was modified but because the KVM daemon need to be # started, given that the instance was first created with user # shutdown disabled. AssertCommand(["gnt-instance", "reboot", instance.name]) primary = _GetInstanceField(instance.name, "pnode") _TestInstanceUserDown(instance, _StopKVMInstance) AssertCommand(["gnt-instance", "modify", "-H", "user_shutdown=false", instance.name]) AssertCommand(["gnt-cluster", "modify", "--user-shutdown=false"]) def TestInstanceUserDown(instance): """Tests user shutdown""" enabled_hypervisors = qa_config.GetEnabledHypervisors() for (hv, fn) in [(constants.HT_XEN_PVM, _TestInstanceUserDownXen), (constants.HT_XEN_HVM, _TestInstanceUserDownXen), (constants.HT_KVM, _TestInstanceUserDownKvm)]: if hv in enabled_hypervisors: qa_daemon.TestPauseWatcher() fn(instance) qa_daemon.TestResumeWatcher() else: print("%s hypervisor is not enabled, skipping test for this hypervisor" \ % hv) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstanceCommunication(instance, master): """Tests instance communication via 'gnt-instance modify'""" # Enable instance communication network at the cluster level network_name = "mynetwork" cmd = ["gnt-cluster", "modify", "--instance-communication-network=%s" % network_name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) print(result_output) # Enable instance communication mechanism for this instance AssertCommand(["gnt-instance", "modify", "-c", "yes", instance.name]) # Reboot instance for changes to NIC to take effect AssertCommand(["gnt-instance", "reboot", instance.name]) # Check if the instance is properly configured for instance # communication. nic_name = "%s%s" % (constants.INSTANCE_COMMUNICATION_NIC_PREFIX, instance.name) ## Check the output of 'gnt-instance list' nic_names = _GetInstanceField(instance.name, "nic.names") nic_names = [x.strip(" '") for x in nic_names.strip("[]").split(",")] AssertIn(nic_name, nic_names, msg="Looking for instance communication TAP interface") nic_n = nic_names.index(nic_name) nic_ip = _GetInstanceField(instance.name, "nic.ip/%d" % nic_n) nic_network = _GetInstanceField(instance.name, "nic.network.name/%d" % nic_n) nic_mode = _GetInstanceField(instance.name, "nic.mode/%d" % nic_n) AssertEqual(IP4Address.InNetwork(constants.INSTANCE_COMMUNICATION_NETWORK4, nic_ip), True, msg="Checking if NIC's IP if part of the expected network") AssertEqual(network_name, nic_network, msg="Checking if NIC's network name matches the expected value") AssertEqual(constants.INSTANCE_COMMUNICATION_NETWORK_MODE, nic_mode, msg="Checking if NIC's mode name matches the expected value") ## Check the output of 'ip route' cmd = ["ip", "route", "show", nic_ip] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) result = result_output.split() AssertEqual(len(result), 5, msg="Checking if the IP route is established") route_ip = result[0] route_dev = result[1] route_tap = result[2] route_scope = result[3] route_link = result[4] AssertEqual(route_ip, nic_ip, msg="Checking if IP route shows the expected IP") AssertEqual(route_dev, "dev", msg="Checking if IP route shows the expected device") AssertEqual(route_scope, "scope", msg="Checking if IP route shows the expected scope") AssertEqual(route_link, "link", msg="Checking if IP route shows the expected link-level scope") ## Check the output of 'ip address' cmd = ["ip", "address", "show", "dev", route_tap] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) result = result_output.splitlines() AssertEqual(len(result), 3, msg="Checking if the IP address is established") result = result.pop().split() AssertEqual(len(result), 7, msg="Checking if the IP address has the expected value") address_ip = result[1] address_netmask = result[3] AssertEqual(address_ip, "169.254.169.254/32", msg="Checking if the TAP interface has the expected IP") AssertEqual(address_netmask, "169.254.255.255", msg="Checking if the TAP interface has the expected netmask") # Disable instance communication mechanism for this instance AssertCommand(["gnt-instance", "modify", "-c", "no", instance.name]) # Reboot instance for changes to NIC to take effect AssertCommand(["gnt-instance", "reboot", instance.name]) # Disable instance communication network at cluster level cmd = ["gnt-cluster", "modify", "--instance-communication-network=%s" % network_name] result_output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) print(result_output) def _TestRedactionOfSecretOsParams(node, cmd, secret_keys): """Tests redaction of secret os parameters """ AssertCommand(["gnt-cluster", "modify", "--max-running-jobs", "1"]) debug_delay_id = int(stdout_of(["gnt-debug", "delay", "--print-jobid", "--submit", "300"])) cmd_jid = int(stdout_of(cmd)) job_file_abspath = "%s/job-%s" % (pathutils.QUEUE_DIR, cmd_jid) job_file = qa_utils.MakeNodePath(node, job_file_abspath) for k in secret_keys: grep_cmd = ["grep", "\"%s\":\"\"" % k, job_file] AssertCommand(grep_cmd) AssertCommand(["gnt-job", "cancel", "--kill", "--yes-do-it", str(debug_delay_id)]) AssertCommand(["gnt-cluster", "modify", "--max-running-jobs", "20"]) AssertCommand(["gnt-job", "wait", str(cmd_jid)]) def TestInstanceAddOsParams(): """Tests instance add with secret os parameters""" if not qa_config.IsTemplateSupported(constants.DT_PLAIN): return master = qa_config.GetMasterNode() instance = qa_config.AcquireInstance() secret_keys = ["param1", "param2"] cmd = (["gnt-instance", "add", "--os-type=%s" % qa_config.get("os"), "--disk-template=%s" % constants.DT_PLAIN, "--os-parameters-secret", "param1=secret1,param2=secret2", "--node=%s" % master.primary] + GetGenericAddParameters(instance, constants.DT_PLAIN)) cmd.append("--submit") cmd.append("--print-jobid") cmd.append(instance.name) _TestRedactionOfSecretOsParams(master.primary, cmd, secret_keys) TestInstanceRemove(instance) instance.Release() def TestSecretOsParams(): """Tests secret os parameter transmission""" master = qa_config.GetMasterNode() secret_keys = ["param1", "param2"] cmd = (["gnt-debug", "test-osparams", "--os-parameters-secret", "param1=secret1,param2=secret2", "--submit", "--print-jobid"]) _TestRedactionOfSecretOsParams(master.primary, cmd, secret_keys) cmd_output = stdout_of(["gnt-debug", "test-osparams", "--os-parameters-secret", "param1=secret1,param2=secret2"]) AssertIn("\'param1\': \'secret1\'", cmd_output) AssertIn("\'param2\': \'secret2\'", cmd_output) available_instance_tests = [ ("instance-add-plain-disk", constants.DT_PLAIN, TestInstanceAddWithPlainDisk, 1), ("instance-add-drbd-disk", constants.DT_DRBD8, TestInstanceAddWithDrbdDisk, 2), ("instance-add-diskless", constants.DT_DISKLESS, TestInstanceAddDiskless, 1), ("instance-add-file", constants.DT_FILE, TestInstanceAddFile, 1), ("instance-add-shared-file", constants.DT_SHARED_FILE, TestInstanceAddSharedFile, 1), ("instance-add-rbd", constants.DT_RBD, TestInstanceAddRADOSBlockDevice, 1), ("instance-add-gluster", constants.DT_GLUSTER, TestInstanceAddGluster, 1), ] ganeti-3.1.0~rc2/qa/qa_instance_utils.py000064400000000000000000000172021476477700300202660ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """QA utility functions for managing instances """ from ganeti import utils from ganeti import constants from ganeti import pathutils from ganeti.utils import retry from qa import qa_config from qa import qa_error from qa import qa_utils from qa_utils import AssertIn, AssertCommand def RemoveInstance(instance): AssertCommand(["gnt-instance", "remove", "-f", instance.name]) def GetGenericAddParameters(inst, disk_template, force_mac=None): params = ["-B"] params.append("%s=%s,%s=%s" % (constants.BE_MINMEM, qa_config.get(constants.BE_MINMEM), constants.BE_MAXMEM, qa_config.get(constants.BE_MAXMEM))) if disk_template != constants.DT_DISKLESS: for idx, disk in enumerate(qa_config.GetDiskOptions()): size = disk.get("size") name = disk.get("name") diskparams = "%s:size=%s" % (idx, size) if name: diskparams += ",name=%s" % name if qa_config.AreSpindlesSupported(): spindles = disk.get("spindles") if spindles is None: raise qa_error.Error("'spindles' is a required parameter for disks" " when you enable exclusive storage tests") diskparams += ",spindles=%s" % spindles params.extend(["--disk", diskparams]) # Set static MAC address if configured if force_mac: nic0_mac = force_mac else: nic0_mac = inst.GetNicMacAddr(0, None) if nic0_mac: params.extend(["--net", "0:mac=%s" % nic0_mac]) return params def _CreateInstanceByDiskTemplateRaw(nodes_spec, disk_template, fail=False): """Creates an instance with the given disk template on the given nodes(s). Note that this function does not check if enough nodes are given for the respective disk template. @type nodes_spec: string @param nodes_spec: string specification of one node (by node name) or several nodes according to the requirements of the disk template @type disk_template: string @param disk_template: the disk template to be used by the instance @return: the created instance """ instance = qa_config.AcquireInstance() try: cmd = (["gnt-instance", "add", "--os-type=%s" % qa_config.get("os"), "--disk-template=%s" % disk_template, "--node=%s" % nodes_spec] + GetGenericAddParameters(instance, disk_template)) cmd.append(instance.name) AssertCommand(cmd, fail=fail) if not fail: CheckSsconfInstanceList(instance.name) instance.SetDiskTemplate(disk_template) return instance except: instance.Release() raise # Handle the case where creation is expected to fail assert fail instance.Release() return None def CreateInstanceDrbd8(nodes, fail=False): """Creates an instance using disk template 'drbd' on the given nodes. @type nodes: list of nodes @param nodes: nodes to be used by the instance @return: the created instance """ assert len(nodes) > 1 return _CreateInstanceByDiskTemplateRaw( ":".join(n.primary for n in nodes), constants.DT_DRBD8, fail=fail) def CreateInstanceByDiskTemplateOneNode(nodes, disk_template, fail=False): """Creates an instance using the given disk template for disk templates for which one given node is sufficient. These templates are for example: plain, diskless, file, sharedfile, blockdev, rados. @type nodes: list of nodes @param nodes: a list of nodes, whose first element is used to create the instance @type disk_template: string @param disk_template: the disk template to be used by the instance @return: the created instance """ assert len(nodes) > 0 return _CreateInstanceByDiskTemplateRaw(nodes[0].primary, disk_template, fail=fail) def CreateInstanceByDiskTemplate(nodes, disk_template, fail=False): """Given a disk template, this function creates an instance using the template. It uses the required number of nodes depending on the disk template. This function is intended to be used by tests that don't care about the specifics of the instance other than that it uses the given disk template. Note: If you use this function, make sure to call 'TestInstanceRemove' at the end of your tests to avoid orphaned instances hanging around and interfering with the following tests. @type nodes: list of nodes @param nodes: the list of the nodes on which the instance will be placed; it needs to have sufficiently many elements for the given disk template @type disk_template: string @param disk_template: the disk template to be used by the instance @return: the created instance """ if disk_template == constants.DT_DRBD8: return CreateInstanceDrbd8(nodes, fail=fail) elif disk_template in [constants.DT_DISKLESS, constants.DT_PLAIN, constants.DT_FILE]: return CreateInstanceByDiskTemplateOneNode(nodes, disk_template, fail=fail) else: # FIXME: This assumes that for all other disk templates, we only need one # node and no disk template specific parameters. This else-branch is # currently only used in cases where we expect failure. Extend it when # QA needs for these templates change. return CreateInstanceByDiskTemplateOneNode(nodes, disk_template, fail=fail) def _ReadSsconfInstanceList(): """Reads ssconf_instance_list from the master node. """ master = qa_config.GetMasterNode() ssconf_path = utils.PathJoin(pathutils.DATA_DIR, "ssconf_%s" % constants.SS_INSTANCE_LIST) cmd = ["cat", qa_utils.MakeNodePath(master, ssconf_path)] return qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)).splitlines() def CheckSsconfInstanceList(instance): """Checks if a certain instance is in the ssconf instance list. Because ssconf is updated in an asynchronous manner, this function will retry reading the ssconf instance list until it either contains the desired instance, or a timeout is reached. @type instance: string @param instance: Instance name """ instance_name = qa_utils.ResolveInstanceName(instance) def _CheckSsconfInstanceList(): if instance_name not in _ReadSsconfInstanceList(): raise retry.RetryAgain() retry.Retry(_CheckSsconfInstanceList, 1, 5) ganeti-3.1.0~rc2/qa/qa_iptables.py000064400000000000000000000071041476477700300170450ustar00rootroot00000000000000#!/usr/bin/python3 -u # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Manipulates nodes using `iptables` to simulate non-standard network conditions. """ import uuid from qa_utils import AssertCommand # String used as a comment for produced `iptables` results IPTABLES_COMMENT_MARKER = "ganeti_qa_script" class RulesContext(object): def __init__(self): self._nodes = set() def __enter__(self): self.marker = IPTABLES_COMMENT_MARKER + "_" + str(uuid.uuid4()) return Rules(self) def __exit__(self, ext_type, exc_val, exc_tb): CleanRules(self._nodes, self.marker) def AddNode(self, node): self._nodes.add(node) class Rules(object): """Allows to introduce iptable rules and dispose them at the end of a block. Don't instantiate this class directly. Use `with RulesContext() as r` instead. """ def __init__(self, ctx=None): self._ctx = ctx if self._ctx is not None: self.marker = self._ctx.marker else: self.marker = IPTABLES_COMMENT_MARKER def AddNode(self, node): if self._ctx is not None: self._ctx.AddNode(node) def AppendRule(self, node, chain, rule, table="filter"): """Appends an `iptables` rule to a given node """ AssertCommand(["iptables", "-t", table, "-A", chain] + rule + ["-m", "comment", "--comment", self.marker], node=node) self.AddNode(node) def RedirectPort(self, node, host, port, new_port): """Adds a rule to a master node that makes a destination host+port visible under a different port number. """ self.AppendRule(node, "OUTPUT", ["--protocol", "tcp", "--destination", host, "--dport", str(port), "--jump", "DNAT", "--to-destination", ":" + str(new_port)], table="nat") GLOBAL_RULES = Rules() def CleanRules(nodes, marker=IPTABLES_COMMENT_MARKER): """Removes all QA `iptables` rules matching a given marker from a given node. If no marker is given, the global default is used, which clean all custom markers. """ if not hasattr(nodes, '__iter__'): nodes = [nodes] for node in nodes: AssertCommand(("iptables-save | grep -v '%s' | iptables-restore" % (marker, )), node=node) ganeti-3.1.0~rc2/qa/qa_job.py000064400000000000000000000077731476477700300160300ustar00rootroot00000000000000# # # Copyright (C) 2012, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Job-related QA tests. """ import functools import re from ganeti.utils import retry from ganeti import constants from ganeti import query from qa import qa_config from qa import qa_error from qa import qa_job_utils from qa import qa_utils from qa_utils import AssertCommand, GetCommandOutput def TestJobList(): """gnt-job list""" qa_utils.GenericQueryTest("gnt-job", list(query.JOB_FIELDS), namefield="id", test_unknown=False) def TestJobListFields(): """gnt-node list-fields""" qa_utils.GenericQueryFieldsTest("gnt-job", list(query.JOB_FIELDS)) def TestJobCancellation(): """gnt-job cancel""" # The delay used for the first command should be large enough for the next # command and the cancellation command to complete before the first job is # done. The second delay should be small enough that not too much time is # spend waiting in the case of a failed cancel and a running command. FIRST_COMMAND_DELAY = 10.0 AssertCommand(["gnt-debug", "delay", "--submit", str(FIRST_COMMAND_DELAY)]) SECOND_COMMAND_DELAY = 3.0 master = qa_config.GetMasterNode() # Forcing tty usage does not work on buildbot, so force all output of this # command to be redirected to stdout job_id_output = GetCommandOutput( master.primary, "gnt-debug delay --submit %s 2>&1" % SECOND_COMMAND_DELAY ) possible_job_ids = re.findall("JobID: ([0-9]+)", job_id_output) if len(possible_job_ids) != 1: raise qa_error.Error("Cannot parse gnt-debug delay output to find job id") job_id = possible_job_ids[0] AssertCommand(["gnt-job", "cancel", job_id]) # Now wait until the second job finishes, and expect the watch to fail due to # job cancellation AssertCommand(["gnt-job", "watch", job_id], fail=True) # Then check for job cancellation job_status = qa_job_utils.GetJobStatus(job_id) if job_status != constants.JOB_STATUS_CANCELED: # Try and see if the job is being cancelled, and wait until the status # changes or we hit a timeout if job_status == constants.JOB_STATUS_CANCELING: retry_fn = functools.partial(qa_job_utils.RetryingWhileJobStatus, constants.JOB_STATUS_CANCELING, job_id) try: # The multiplier to use is arbitrary, setting it higher could prevent # flakiness WAIT_MULTIPLIER = 4.0 job_status = retry.Retry(retry_fn, 2.0, WAIT_MULTIPLIER * FIRST_COMMAND_DELAY) except retry.RetryTimeout: # The job status remains the same pass if job_status != constants.JOB_STATUS_CANCELED: raise qa_error.Error("Job was not successfully cancelled, status " "found: %s" % job_status) ganeti-3.1.0~rc2/qa/qa_job_utils.py000064400000000000000000000350561476477700300172430ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """QA utility functions for testing jobs """ import re import sys import threading import time from ganeti import constants from ganeti import locking from ganeti import utils from ganeti.utils import retry from qa import qa_config from qa import qa_logging from qa import qa_error from qa_utils import AssertCommand, GetCommandOutput, GetObjectInfo, stdout_of AVAILABLE_LOCKS = [locking.LEVEL_NODE, ] def GetOutputFromMaster(cmd, use_multiplexer=True, log_cmd=True): """ Gets the output of a command executed on master. """ if isinstance(cmd, str): cmdstr = cmd else: cmdstr = utils.ShellQuoteArgs(cmd) # Necessary due to the stderr stream not being captured properly on the # buildbot cmdstr += " 2>&1" return GetCommandOutput(qa_config.GetMasterNode().primary, cmdstr, use_multiplexer=use_multiplexer, log_cmd=log_cmd) def ExecuteJobProducingCommand(cmd): """ Executes a command that contains the --submit flag, and returns a job id. @type cmd: list of string @param cmd: The command to execute, broken into constituent components. """ job_id_output = GetOutputFromMaster(cmd) # Usually, the output contains "JobID: ", but for instance related # commands, the output is of the form ": " possible_job_ids = re.findall("JobID: ([0-9]+)", job_id_output) or \ re.findall("([0-9]+): .+", job_id_output) if len(possible_job_ids) != 1: raise qa_error.Error("Cannot parse command output to find job id: output " "is %s" % job_id_output) return int(possible_job_ids[0]) def GetJobStatuses(job_ids=None): """ Invokes gnt-job list and extracts an id to status dictionary. @type job_ids: list @param job_ids: list of job ids to query the status for; if C{None}, the status of all current jobs is returned @rtype: dict of string to string @return: A dictionary mapping job ids to matching statuses """ cmd = ["gnt-job", "list", "--no-headers", "--output=id,status"] if job_ids is not None: cmd.extend([str(id) for id in job_ids]) list_output = GetOutputFromMaster(cmd) return dict([s.split() for s in list_output.splitlines()]) def _RetrieveTerminationInfo(job_id): """ Retrieves the termination info from a job caused by gnt-debug delay. @rtype: dict or None @return: The termination log entry, or None if no entry was found """ job_info = GetObjectInfo(["gnt-job", "info", str(job_id)]) opcodes = job_info[0]["Opcodes"] if not opcodes: raise qa_error.Error("Cannot retrieve a list of opcodes") execution_logs = opcodes[0]["Execution log"] if not execution_logs: return None is_termination_info_fn = \ lambda e: e["Content"][1] == constants.ELOG_DELAY_TEST filtered_logs = [l for l in execution_logs if is_termination_info(l)] no_logs = len(filtered_logs) if no_logs > 1: raise qa_error.Error("Too many interruption information entries found!") elif no_logs == 1: return filtered_logs[0] else: return None def _StartDelayFunction(locks, timeout): """ Starts the gnt-debug delay option with the given locks and timeout. """ # The interruptible switch must be used cmd = ["gnt-debug", "delay", "-i", "--submit", "--no-master"] for node in locks.get(locking.LEVEL_NODE, []): cmd.append("-n%s" % node) cmd.append(str(timeout)) job_id = ExecuteJobProducingCommand(cmd) # Waits until a non-empty result is returned from the function log_entry = retry.SimpleRetry(lambda x: x, _RetrieveTerminationInfo, 2.0, 10.0, args=[job_id]) if not log_entry: raise qa_error.Error("Failure when trying to retrieve delay termination " "information") _, _, (socket_path, ) = log_entry["Content"] return socket_path def _TerminateDelayFunction(termination_socket): """ Terminates the delay function by communicating with the domain socket. """ AssertCommand("echo a | socat -u stdin UNIX-CLIENT:%s" % termination_socket) def _GetNodeUUIDMap(nodes): """ Given a list of nodes, retrieves a mapping of their names to UUIDs. @type nodes: list of string @param nodes: The nodes to retrieve a map for. If empty, returns information for all the nodes. """ cmd = ["gnt-node", "list", "--no-header", "-o", "name,uuid"] cmd.extend(nodes) output = GetOutputFromMaster(cmd) return dict([x.split() for x in output.splitlines()]) def _FindLockNames(locks): """ Finds the ids and descriptions of locks that given locks can block. @type locks: dict of locking level to list @param locks: The locks that gnt-debug delay is holding. @rtype: dict of string to string @return: The lock name to entity name map. For a given set of locks, some internal locks (e.g. ALL_SET locks) can be blocked even though they were not listed explicitly. This function has to take care and list all locks that can be blocked by the locks given as parameters. """ lock_map = {} if locking.LEVEL_NODE in locks: node_locks = locks[locking.LEVEL_NODE] if node_locks == locking.ALL_SET: # Empty list retrieves all info name_uuid_map = _GetNodeUUIDMap([]) else: name_uuid_map = _GetNodeUUIDMap(node_locks) for name in name_uuid_map: lock_map["node/%s" % name_uuid_map[name]] = name # If ALL_SET was requested explicitly, or there is at least one lock # Note that locking.ALL_SET is None and hence the strange form of the if if node_locks == locking.ALL_SET or node_locks: lock_map["node/[lockset]"] = "joint node lock" #TODO add other lock types here when support for these is added return lock_map def _GetBlockingLocks(): """ Finds out which locks are blocking jobs by invoking "gnt-debug locks". @rtype: list of string @return: The names of the locks currently blocking any job. """ # Due to mysterious issues when a SSH multiplexer is being used by two # threads, we turn it off, and block most of the logging to improve the # visibility of the other thread's output locks_output = GetOutputFromMaster("gnt-debug locks", use_multiplexer=False, log_cmd=False) # The first non-empty line is the header, which we do not need lock_lines = locks_output.splitlines()[1:] blocking_locks = [] for lock_line in lock_lines: components = lock_line.split() if len(components) != 4: raise qa_error.Error("Error while parsing gnt-debug locks output, " "line at fault is: %s" % lock_line) lock_name, _, _, pending_jobs = components if pending_jobs != '-': blocking_locks.append(lock_name) return blocking_locks class QAThread(threading.Thread): """ An exception-preserving thread that executes a given function. """ def __init__(self, fn, args, kwargs): """ Constructor accepting the function to be invoked later. """ threading.Thread.__init__(self) self._fn = fn self._args = args self._kwargs = kwargs self._exc_info = None def run(self): """ Executes the function, preserving exception info if necessary. """ # pylint: disable=W0702 # We explicitly want to catch absolutely anything try: self._fn(*self._args, **self._kwargs) except: self._exc_info = sys.exc_info() # pylint: enable=W0702 def reraise(self): """ Reraises any exceptions that might have occured during thread execution. """ if self._exc_info is None: return raise self._exc_info[0](self._exc_info[1]).with_traceback(self._exc_info[2]) class QAThreadGroup(object): """This class manages a list of QAThreads. """ def __init__(self): self._threads = [] def Start(self, thread): """Starts the given thread and adds it to this group. @type thread: qa_job_utils.QAThread @param thread: the thread to start and to add to this group. """ thread.start() self._threads.append(thread) def JoinAndReraise(self): """Joins all threads in this group and calls their C{reraise} method. """ for thread in self._threads: thread.join() thread.reraise() class PausedWatcher(object): """Pauses the watcher for the duration of the inner code """ def __enter__(self): AssertCommand(["gnt-cluster", "watcher", "pause", "12h"]) def __exit__(self, _ex_type, ex_value, _ex_traceback): try: AssertCommand(["gnt-cluster", "watcher", "continue"]) except qa_error.Error as err: # If an exception happens during 'continue', re-raise it only if there # is no exception from the inner block: if ex_value is None: raise else: print(qa_logging.FormatError('Re-enabling watcher failed: %s' % (err, ))) # TODO: Can this be done as a decorator? Implement as needed. def RunWithLocks(fn, locks, timeout, block, *args, **kwargs): """ Runs the given function, acquiring a set of locks beforehand. @type fn: function @param fn: The function to invoke. @type locks: dict of string to list of string @param locks: The locks to acquire, per lock category. @type timeout: number @param timeout: The number of seconds the locks should be held before expiring. @type block: bool @param block: Whether the test should block when locks are used or not. This function allows a set of locks to be acquired in preparation for a QA test, to try and see if the function can run in parallel with other operations. Locks are acquired by invoking a gnt-debug delay operation which can be interrupted as needed. The QA test is then run in a separate thread, with the current thread observing jobs waiting for locks. When a job is spotted waiting for a lock held by the started delay operation, this is noted, and the delay is interrupted, allowing the QA test to continue. A default timeout is not provided by design - the test creator must make a good conservative estimate. """ if [l_type for l_type in locks if l_type not in AVAILABLE_LOCKS]: raise qa_error.Error("Attempted to acquire locks that cannot yet be " "acquired in the course of a QA test.") # The watcher may interfere by issuing its own jobs - therefore pause it # also reject all its jobs and wait for any running jobs to finish. AssertCommand(["gnt-cluster", "watcher", "pause", "12h"]) filter_uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["reason", ["=", "source", "gnt:watcher"]]]', "--action=REJECT" ]) while stdout_of(["gnt-job", "list", "--no-header", "--running"]) != "": time.sleep(1) # Find out the lock names prior to starting the delay function lock_name_map = _FindLockNames(locks) blocking_owned_locks = [] test_blocked = False termination_socket = _StartDelayFunction(locks, timeout) delay_fn_terminated = False try: qa_thread = QAThread(fn, args, kwargs) qa_thread.start() while qa_thread.is_alive(): blocking_locks = _GetBlockingLocks() blocking_owned_locks = \ set(blocking_locks).intersection(set(lock_name_map)) if blocking_owned_locks: # Set the flag first - if the termination attempt fails, we do not want # to redo it in the finally block delay_fn_terminated = True _TerminateDelayFunction(termination_socket) test_blocked = True break time.sleep(5) # Set arbitrarily # The thread should be either finished or unblocked at this point qa_thread.join() # Raise any errors that might have occured in the thread qa_thread.reraise() finally: if not delay_fn_terminated: _TerminateDelayFunction(termination_socket) blocking_lock_names = ", ".join(map(lock_name_map.get, blocking_owned_locks)) if not block and test_blocked: raise qa_error.Error("QA test succeded, but was blocked by locks: %s" % blocking_lock_names) elif block and not test_blocked: raise qa_error.Error("QA test succeded, but was not blocked as it was " "expected to by locks: %s" % blocking_lock_names) else: pass # Revive the watcher AssertCommand(["gnt-filter", "delete", filter_uuid]) AssertCommand(["gnt-cluster", "watcher", "continue"]) def GetJobStatus(job_id): """ Retrieves the status of a job. @type job_id: string @param job_id: The job id, represented as a string. @rtype: string or None @return: The job status, or None if not present. """ return GetJobStatuses([job_id]).get(job_id, None) def RetryingUntilJobStatus(retry_status, job_id): """ Used with C{retry.Retry}, waits for a given status. @type retry_status: string @param retry_status: The job status to wait for. @type job_id: string @param job_id: The job id, represented as a string. """ status = GetJobStatus(job_id) if status != retry_status: raise retry.RetryAgain() def RetryingWhileJobStatus(retry_status, job_id): """ Used with C{retry.Retry}, waits for a status other than the one given. @type retry_status: string @param retry_status: The old job status, expected to change. @type job_id: string @param job_id: The job id, represented as a string. @rtype: string or None @return: The new job status, or None if none could be retrieved. """ status = GetJobStatus(job_id) if status == retry_status: raise retry.RetryAgain() return status ganeti-3.1.0~rc2/qa/qa_logging.py000064400000000000000000000046331476477700300166740ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ Handles the logging of messages with appropriate coloring. """ import sys _INFO_SEQ = None _WARNING_SEQ = None _ERROR_SEQ = None _RESET_SEQ = None def _SetupColours(): """Initializes the colour constants. """ # pylint: disable=W0603 # due to global usage global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ # Don't use colours if stdout isn't a terminal if not sys.stdout.isatty(): return try: import curses except ImportError: # Don't use colours if curses module can't be imported return curses.setupterm() _RESET_SEQ = curses.tigetstr("op") setaf = curses.tigetstr("setaf") _INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN) _WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW) _ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED) _SetupColours() def _FormatWithColor(text, seq): if not seq: return text return "%s%s%s" % (seq, text, _RESET_SEQ) FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ) FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ) FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ) ganeti-3.1.0~rc2/qa/qa_monitoring.py000064400000000000000000000046611476477700300174340ustar00rootroot00000000000000# # # Copyright (C) 2007, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Monitoring related QA tests. """ from ganeti import _constants from ganeti import constants from qa import qa_config from qa_utils import AssertCommand from qa_instance_utils import CreateInstanceByDiskTemplate, \ RemoveInstance MON_COLLECTOR = _constants.PKGLIBDIR + "/mon-collector" def TestInstStatusCollector(): """Test the Xen instance status collector. """ enabled_hypervisors = qa_config.GetEnabledHypervisors() is_xen = (constants.HT_XEN_PVM in enabled_hypervisors or constants.HT_XEN_HVM in enabled_hypervisors) if not is_xen: return # Execute on master on an empty cluster AssertCommand([MON_COLLECTOR, "inst-status-xen"]) #Execute on cluster with instances node1 = qa_config.AcquireNode() node2 = qa_config.AcquireNode() template = qa_config.GetDefaultDiskTemplate() instance = CreateInstanceByDiskTemplate([node1, node2], template) AssertCommand([MON_COLLECTOR, "inst-status-xen"], node=node1) AssertCommand([MON_COLLECTOR, "inst-status-xen"], node=node2) RemoveInstance(instance) node1.Release() node2.Release() ganeti-3.1.0~rc2/qa/qa_network.py000064400000000000000000000074341476477700300167410ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """QA tests for networks. """ from qa import qa_config from qa import qa_tags from qa import qa_utils from ganeti import query from qa_utils import AssertCommand def TestNetworkList(): """gnt-network list""" qa_utils.GenericQueryTest("gnt-network", list(query.NETWORK_FIELDS)) def TestNetworkListFields(): """gnt-network list-fields""" qa_utils.GenericQueryFieldsTest("gnt-network", list(query.NETWORK_FIELDS)) def GetNonexistentNetworks(count): """Gets network names which shouldn't exist on the cluster. @param count: Number of networks to get @rtype: integer """ return qa_utils.GetNonexistentEntityNames(count, "networks", "network") def TestNetworkAddRemove(): """gnt-network add/remove""" (network1, network2) = GetNonexistentNetworks(2) # Add some networks of different sizes. # Note: Using RFC5737 addresses. AssertCommand(["gnt-network", "add", "--network", "192.0.2.0/30", network1]) AssertCommand(["gnt-network", "add", "--network", "198.51.100.0/24", network2]) # Try to add a network with an existing name. AssertCommand(["gnt-network", "add", "--network", "203.0.133.0/24", network2], fail=True) TestNetworkList() TestNetworkListFields() AssertCommand(["gnt-network", "remove", network1]) AssertCommand(["gnt-network", "remove", network2]) TestNetworkList() def TestNetworkTags(): """gnt-network tags""" (network, ) = GetNonexistentNetworks(1) AssertCommand(["gnt-network", "add", "--network", "192.0.2.0/30", network]) qa_tags.TestNetworkTags(network) AssertCommand(["gnt-network", "remove", network]) def TestNetworkConnect(): """gnt-network connect/disconnect""" (group1, ) = qa_utils.GetNonexistentGroups(1) (network1, ) = GetNonexistentNetworks(1) default_mode = "bridged" default_link = "xen-br0" nicparams = qa_config.get("default-nicparams") if nicparams: mode = nicparams.get("mode", default_mode) link = nicparams.get("link", default_link) else: mode = default_mode link = default_link nicparams = "mode=%s,link=%s" % (mode, link) AssertCommand(["gnt-group", "add", group1]) AssertCommand(["gnt-network", "add", "--network", "192.0.2.0/24", network1]) AssertCommand(["gnt-network", "connect", "--nic-parameters", nicparams, network1, group1]) TestNetworkList() AssertCommand(["gnt-network", "disconnect", network1, group1]) AssertCommand(["gnt-group", "remove", group1]) AssertCommand(["gnt-network", "remove", network1]) ganeti-3.1.0~rc2/qa/qa_node.py000064400000000000000000000452101476477700300161670ustar00rootroot00000000000000# # # Copyright (C) 2007, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Node-related QA tests. """ from ganeti import utils from ganeti import constants from ganeti import query from ganeti import serializer from qa import qa_config from qa import qa_error from qa import qa_utils from qa_utils import AssertCommand, AssertRedirectedCommand, AssertEqual, \ AssertIn, GetCommandOutput def NodeAdd(node, readd=False, group=None): if not readd and node.added: raise qa_error.Error("Node %s already in cluster" % node.primary) elif readd and not node.added: raise qa_error.Error("Node %s not yet in cluster" % node.primary) cmd = ["gnt-node", "add", "--no-ssh-key-check"] if node.secondary: cmd.append("--secondary-ip=%s" % node.secondary) if readd: cmd.append("--readd") if group is not None: cmd.extend(["--node-group", group]) if not qa_config.GetModifySshSetup(): cmd.append("--no-node-setup") cmd.append(node.primary) AssertCommand(cmd) if readd: AssertRedirectedCommand(["gnt-cluster", "verify"]) if readd: assert node.added else: node.MarkAdded() def NodeRemove(node): AssertCommand(["gnt-node", "remove", node.primary]) node.MarkRemoved() def MakeNodeOffline(node, value): """gnt-node modify --offline=value""" # value in ["yes", "no"] AssertCommand(["gnt-node", "modify", "--offline", value, node.primary]) def TestNodeAddAll(): """Adding all nodes to cluster.""" master = qa_config.GetMasterNode() for node in qa_config.get("nodes"): if node != master: NodeAdd(node, readd=False) def MarkNodeAddedAll(): """Mark all nodes as added. This is useful if we don't create the cluster ourselves (in qa). """ master = qa_config.GetMasterNode() for node in qa_config.get("nodes"): if node != master: node.MarkAdded() def TestNodeRemoveAll(): """Removing all nodes from cluster.""" master = qa_config.GetMasterNode() for node in qa_config.get("nodes"): if node != master: NodeRemove(node) def TestNodeReadd(node): """gnt-node add --readd""" NodeAdd(node, readd=True) def TestNodeInfo(): """gnt-node info""" AssertCommand(["gnt-node", "info"]) def TestNodeVolumes(): """gnt-node volumes""" AssertCommand(["gnt-node", "volumes"]) def TestNodeStorage(): """gnt-node storage""" master = qa_config.GetMasterNode() # FIXME: test all storage_types in constants.STORAGE_TYPES # as soon as they are implemented. enabled_storage_types = qa_config.GetEnabledStorageTypes() testable_storage_types = list(set(enabled_storage_types).intersection( set([constants.ST_FILE, constants.ST_LVM_VG, constants.ST_LVM_PV]))) for storage_type in testable_storage_types: cmd = ["gnt-node", "list-storage", "--storage-type", storage_type] # Test simple list AssertCommand(cmd) # Test all storage fields cmd = ["gnt-node", "list-storage", "--storage-type", storage_type, "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS))] AssertCommand(cmd) # Get list of valid storage devices cmd = ["gnt-node", "list-storage", "--storage-type", storage_type, "--output=node,name,allocatable", "--separator=|", "--no-headers"] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) # Test with up to two devices testdevcount = 2 for line in output.splitlines()[:testdevcount]: (node_name, st_name, st_allocatable) = line.split("|") # Dummy modification without any changes cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name] AssertCommand(cmd) # Make sure we end up with the same value as before if st_allocatable.lower() == "y": test_allocatable = ["no", "yes"] else: test_allocatable = ["yes", "no"] fail = (constants.SF_ALLOCATABLE not in constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, [])) for i in test_allocatable: AssertCommand(["gnt-node", "modify-storage", "--allocatable", i, node_name, storage_type, st_name], fail=fail) # Verify list output cmd = ["gnt-node", "list-storage", "--storage-type", storage_type, "--output=name,allocatable", "--separator=|", "--no-headers", node_name] listout = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) for line in listout.splitlines(): (vfy_name, vfy_allocatable) = line.split("|") if vfy_name == st_name and not fail: AssertEqual(vfy_allocatable, i[0].upper()) else: AssertEqual(vfy_allocatable, st_allocatable) # Test repair functionality fail = (constants.SO_FIX_CONSISTENCY not in constants.VALID_STORAGE_OPERATIONS.get(storage_type, [])) AssertCommand(["gnt-node", "repair-storage", node_name, storage_type, st_name], fail=fail) def TestNodeFailover(node, node2): """gnt-node failover""" if qa_utils.GetNodeInstances(node2, secondaries=False): raise qa_error.UnusableNodeError("Secondary node has at least one" " primary instance. This test requires" " it to have no primary instances.") # Fail over to secondary node AssertCommand(["gnt-node", "failover", "-f", node.primary]) # ... and back again. AssertCommand(["gnt-node", "failover", "-f", node2.primary]) def TestNodeMigrate(node, node2): """gnt-node migrate""" if qa_utils.GetNodeInstances(node2, secondaries=False): raise qa_error.UnusableNodeError("Secondary node has at least one" " primary instance. This test requires" " it to have no primary instances.") # Migrate to secondary node AssertCommand(["gnt-node", "migrate", "-f", node.primary]) # ... and back again. AssertCommand(["gnt-node", "migrate", "-f", node2.primary]) def TestNodeEvacuate(node, node2): """gnt-node evacuate""" node3 = qa_config.AcquireNode(exclude=[node, node2]) try: if qa_utils.GetNodeInstances(node3, secondaries=True): raise qa_error.UnusableNodeError("Evacuation node has at least one" " secondary instance. This test requires" " it to have no secondary instances.") # Evacuate all secondary instances AssertCommand(["gnt-node", "evacuate", "-f", "--new-secondary=%s" % node3.primary, node2.primary]) # ... and back again. AssertCommand(["gnt-node", "evacuate", "-f", "--new-secondary=%s" % node2.primary, node3.primary]) finally: node3.Release() def TestNodeModify(node): """gnt-node modify""" default_pool_size = 10 nodes = qa_config.GetAllNodes() test_pool_size = len(nodes) - 1 # Reduce the number of master candidates, because otherwise all # subsequent 'gnt-cluster verify' commands fail due to not enough # master candidates. AssertCommand(["gnt-cluster", "modify", "--candidate-pool-size=%s" % test_pool_size]) # make sure enough master candidates will be available by disabling the # master candidate role first with --auto-promote AssertCommand(["gnt-node", "modify", "--master-candidate=no", "--auto-promote", node.primary]) # now it's save to force-remove the master candidate role for flag in ["master-candidate", "drained", "offline"]: for value in ["yes", "no"]: AssertCommand(["gnt-node", "modify", "--force", "--%s=%s" % (flag, value), node.primary]) AssertCommand(["gnt-cluster", "verify"]) AssertCommand(["gnt-node", "modify", "--master-candidate=yes", node.primary]) # Test setting secondary IP address AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary, node.primary]) AssertRedirectedCommand(["gnt-cluster", "verify"]) AssertCommand(["gnt-cluster", "modify", "--candidate-pool-size=%s" % default_pool_size]) # For test clusters with more nodes than the default pool size, # we now have too many master candidates. To readjust to the original # size, manually demote all nodes and rely on auto-promotion to adjust. if len(nodes) > default_pool_size: master = qa_config.GetMasterNode() for n in nodes: if n.primary != master.primary: AssertCommand(["gnt-node", "modify", "--master-candidate=no", "--auto-promote", n.primary]) def _CreateOobScriptStructure(): """Create a simple OOB handling script and its structure.""" master = qa_config.GetMasterNode() data_path = qa_utils.UploadData(master.primary, "") verify_path = qa_utils.UploadData(master.primary, "") exit_code_path = qa_utils.UploadData(master.primary, "") oob_script = (("#!/bin/bash\n" "echo \"$@\" > %s\n" "cat %s\n" "exit $(< %s)\n") % (utils.ShellQuote(verify_path), utils.ShellQuote(data_path), utils.ShellQuote(exit_code_path))) oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0o700) return [oob_path, verify_path, data_path, exit_code_path] def _UpdateOobFile(path, data): """Updates the data file with data.""" master = qa_config.GetMasterNode() qa_utils.UploadData(master.primary, data, filename=path) def _AssertOobCall(verify_path, expected_args): """Assert the OOB call was performed with expetected args.""" master = qa_config.GetMasterNode() verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path]) output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd, tty=False) AssertEqual(expected_args, output.strip()) def TestOutOfBand(): """gnt-node power""" master = qa_config.GetMasterNode() node = qa_config.AcquireNode(exclude=master) master_name = master.primary node_name = node.primary full_node_name = qa_utils.ResolveNodeName(node) (oob_path, verify_path, data_path, exit_code_path) = _CreateOobScriptStructure() try: AssertCommand(["gnt-cluster", "modify", "--node-parameters", "oob_program=%s" % oob_path]) # No data, exit 0 _UpdateOobFile(exit_code_path, "0") AssertCommand(["gnt-node", "power", "on", node_name]) _AssertOobCall(verify_path, "power-on %s" % full_node_name) AssertCommand(["gnt-node", "power", "-f", "off", node_name]) _AssertOobCall(verify_path, "power-off %s" % full_node_name) # Power off on master without options should fail AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True) # With force master it should still fail AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off", master_name], fail=True) # Verify we can't transform back to online when not yet powered on AssertCommand(["gnt-node", "modify", "-O", "no", node_name], fail=True) # Now reset state AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes", node_name]) AssertCommand(["gnt-node", "power", "-f", "cycle", node_name]) _AssertOobCall(verify_path, "power-cycle %s" % full_node_name) # Those commands should fail as they expect output which isn't provided yet # But they should have called the oob helper nevermind AssertCommand(["gnt-node", "power", "status", node_name], fail=True) _AssertOobCall(verify_path, "power-status %s" % full_node_name) AssertCommand(["gnt-node", "health", node_name], fail=True) _AssertOobCall(verify_path, "health %s" % full_node_name) AssertCommand(["gnt-node", "health"], fail=True) # Correct Data, exit 0 _UpdateOobFile(data_path, serializer.DumpJson({"powered": True})) AssertCommand(["gnt-node", "power", "status", node_name]) _AssertOobCall(verify_path, "power-status %s" % full_node_name) _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"], ["disk0", "CRITICAL"]])) AssertCommand(["gnt-node", "health", node_name]) _AssertOobCall(verify_path, "health %s" % full_node_name) AssertCommand(["gnt-node", "health"]) # Those commands should fail as they expect no data regardless of exit 0 AssertCommand(["gnt-node", "power", "on", node_name], fail=True) _AssertOobCall(verify_path, "power-on %s" % full_node_name) try: AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True) _AssertOobCall(verify_path, "power-off %s" % full_node_name) finally: AssertCommand(["gnt-node", "modify", "-O", "no", node_name]) AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True) _AssertOobCall(verify_path, "power-cycle %s" % full_node_name) # Data, exit 1 (all should fail) _UpdateOobFile(exit_code_path, "1") AssertCommand(["gnt-node", "power", "on", node_name], fail=True) _AssertOobCall(verify_path, "power-on %s" % full_node_name) try: AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True) _AssertOobCall(verify_path, "power-off %s" % full_node_name) finally: AssertCommand(["gnt-node", "modify", "-O", "no", node_name]) AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True) _AssertOobCall(verify_path, "power-cycle %s" % full_node_name) AssertCommand(["gnt-node", "power", "status", node_name], fail=True) _AssertOobCall(verify_path, "power-status %s" % full_node_name) AssertCommand(["gnt-node", "health", node_name], fail=True) _AssertOobCall(verify_path, "health %s" % full_node_name) AssertCommand(["gnt-node", "health"], fail=True) # No data, exit 1 (all should fail) _UpdateOobFile(data_path, "") AssertCommand(["gnt-node", "power", "on", node_name], fail=True) _AssertOobCall(verify_path, "power-on %s" % full_node_name) try: AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True) _AssertOobCall(verify_path, "power-off %s" % full_node_name) finally: AssertCommand(["gnt-node", "modify", "-O", "no", node_name]) AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True) _AssertOobCall(verify_path, "power-cycle %s" % full_node_name) AssertCommand(["gnt-node", "power", "status", node_name], fail=True) _AssertOobCall(verify_path, "power-status %s" % full_node_name) AssertCommand(["gnt-node", "health", node_name], fail=True) _AssertOobCall(verify_path, "health %s" % full_node_name) AssertCommand(["gnt-node", "health"], fail=True) # Different OOB script for node verify_path2 = qa_utils.UploadData(master.primary, "") oob_script = ("#!/bin/sh\n" "echo \"$@\" > %s\n") % verify_path2 oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0o700) try: AssertCommand(["gnt-node", "modify", "--node-parameters", "oob_program=%s" % oob_path2, node_name]) AssertCommand(["gnt-node", "power", "on", node_name]) _AssertOobCall(verify_path2, "power-on %s" % full_node_name) finally: AssertCommand(["gnt-node", "modify", "--node-parameters", "oob_program=default", node_name]) AssertCommand(["rm", "-f", oob_path2, verify_path2]) finally: AssertCommand(["gnt-cluster", "modify", "--node-parameters", "oob_program="]) AssertCommand(["rm", "-f", oob_path, verify_path, data_path, exit_code_path]) def TestNodeList(): """gnt-node list""" qa_utils.GenericQueryTest("gnt-node", list(query.NODE_FIELDS)) def TestNodeListFields(): """gnt-node list-fields""" qa_utils.GenericQueryFieldsTest("gnt-node", list(query.NODE_FIELDS)) def TestNodeListDrbd(node, is_drbd): """gnt-node list-drbd""" master = qa_config.GetMasterNode() result_output = GetCommandOutput(master.primary, "gnt-node list-drbd --no-header %s" % node.primary) # Meaningful to note: there is but one instance, and the node is either the # primary or one of the secondaries if is_drbd: # Invoked for both primary and secondary per_disk_info = result_output.splitlines() for line in per_disk_info: try: drbd_node, _, _, _, _, drbd_peer = line.split() except ValueError: raise qa_error.Error("Could not examine list-drbd output: expected a" " single row of 6 entries, found the following:" " %s" % line) AssertIn(node.primary, [drbd_node, drbd_peer], msg="The output %s does not contain the node" % line) else: # Output should be empty, barring newlines AssertEqual(result_output.strip(), "") def _BuildSetESCmd(action, value, node_name): cmd = ["gnt-node"] if action == "add": cmd.extend(["add", "--readd"]) if not qa_config.GetModifySshSetup(): cmd.append("--no-node-setup") else: cmd.append("modify") cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name]) return cmd def TestExclStorSingleNode(node): """gnt-node add/modify cannot change the exclusive_storage flag. """ for action in ["add", "modify"]: for value in (True, False, "default"): AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True) ganeti-3.1.0~rc2/qa/qa_os.py000064400000000000000000000167261476477700300156750ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """OS related QA tests. """ import os import os.path from ganeti import utils from ganeti import constants from ganeti import pathutils from qa import qa_config from qa import qa_utils from qa import qa_error from qa_utils import AssertCommand, AssertIn, AssertNotIn _TEMP_OS_NAME = "TEMP-Ganeti-QA-OS" _TEMP_OS_PATH = os.path.join(pathutils.OS_SEARCH_PATH[0], _TEMP_OS_NAME) (_ALL_VALID, _ALL_INVALID, _PARTIALLY_VALID) = range(1, 4) def TestOsList(): """gnt-os list""" AssertCommand(["gnt-os", "list"]) def TestOsDiagnose(): """gnt-os diagnose""" AssertCommand(["gnt-os", "diagnose"]) def _TestOsModify(hvp_dict, fail=False): """gnt-os modify""" cmd = ["gnt-os", "modify"] for hv_name, hv_params in hvp_dict.items(): cmd.append("-H") options = [] for key, value in hv_params.items(): options.append("%s=%s" % (key, value)) cmd.append("%s:%s" % (hv_name, ",".join(options))) cmd.append(_TEMP_OS_NAME) AssertCommand(cmd, fail=fail) def _TestOsStates(os_name): """gnt-os modify, more stuff""" cmd = ["gnt-os", "modify"] for param in ["hidden", "blacklisted"]: for val in ["yes", "no"]: new_cmd = cmd + ["--%s" % param, val, os_name] AssertCommand(new_cmd) # check that double-running the command is OK AssertCommand(new_cmd) def _SetupTempOs(node, dirname, variant, valid): """Creates a temporary OS definition on the given node. """ sq = utils.ShellQuoteArgs parts = [ sq(["rm", "-rf", dirname]), sq(["mkdir", "-p", dirname]), sq(["cd", dirname]), sq(["ln", "-fs", "/bin/true", "export"]), sq(["ln", "-fs", "/bin/true", "import"]), sq(["ln", "-fs", "/bin/true", "rename"]), sq(["ln", "-fs", "/bin/true", "verify"]), ] if valid: parts.append(sq(["ln", "-fs", "/bin/true", "create"])) parts.append(sq(["echo", str(constants.OS_API_V20)]) + " >ganeti_api_version") parts.append(sq(["echo", variant]) + " >variants.list") parts.append(sq(["echo", "funny this is funny"]) + " >parameters.list") cmd = " && ".join(parts) print(qa_utils.FormatInfo("Setting up %s with %s OS definition" % (node.primary, ["an invalid", "a valid"][int(valid)]))) AssertCommand(cmd, node=node) def _RemoveTempOs(node, dirname): """Removes a temporary OS definition. """ AssertCommand(["rm", "-rf", dirname], node=node) def _TestOs(mode, rapi_cb): """Generic function for OS definition testing """ master = qa_config.GetMasterNode() name = _TEMP_OS_NAME variant = "default" fullname = "%s+%s" % (name, variant) dirname = _TEMP_OS_PATH # Ensure OS is usable cmd = ["gnt-os", "modify", "--hidden=no", "--blacklisted=no", name] AssertCommand(cmd) nodes = [] try: for i, node in enumerate(qa_config.get("nodes")): nodes.append(node) if mode == _ALL_INVALID: valid = False elif mode == _ALL_VALID: valid = True elif mode == _PARTIALLY_VALID: valid = bool(i % 2) else: raise AssertionError("Unknown mode %s" % mode) _SetupTempOs(node, dirname, variant, valid) # TODO: Use Python 2.6's itertools.permutations for (hidden, blacklisted) in [(False, False), (True, False), (False, True), (True, True)]: # Change OS' visibility cmd = ["gnt-os", "modify", "--hidden", ["no", "yes"][int(hidden)], "--blacklisted", ["no", "yes"][int(blacklisted)], name] AssertCommand(cmd) # Diagnose, checking exit status AssertCommand(["gnt-os", "diagnose"], fail=(mode != _ALL_VALID)) # Diagnose again, ignoring exit status output = qa_utils.GetCommandOutput(master.primary, "gnt-os diagnose || :") for line in output.splitlines(): if line.startswith("OS: %s [global status:" % name): break else: raise qa_error.Error("Didn't find OS '%s' in 'gnt-os diagnose'" % name) # Check info for all cmd = ["gnt-os", "info"] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) AssertIn("%s:" % name, output.splitlines()) # Check info for OS cmd = ["gnt-os", "info", name] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)).splitlines() AssertIn("%s:" % name, output) for (field, value) in [("valid", mode == _ALL_VALID), ("hidden", hidden), ("blacklisted", blacklisted)]: AssertIn(" - %s: %s" % (field, value), output) # Only valid OSes should be listed cmd = ["gnt-os", "list", "--no-headers"] output = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) if mode == _ALL_VALID and not (hidden or blacklisted): assert_fn = AssertIn else: assert_fn = AssertNotIn assert_fn(fullname, output.splitlines()) # Check via RAPI if rapi_cb: assert_fn(fullname, rapi_cb()) finally: for node in nodes: _RemoveTempOs(node, dirname) def TestOsValid(rapi_cb): """Testing valid OS definition""" return _TestOs(_ALL_VALID, rapi_cb) def TestOsInvalid(rapi_cb): """Testing invalid OS definition""" return _TestOs(_ALL_INVALID, rapi_cb) def TestOsPartiallyValid(rapi_cb): """Testing partially valid OS definition""" return _TestOs(_PARTIALLY_VALID, rapi_cb) def TestOsModifyValid(): """Testing a valid os modify invocation""" hv_dict = { constants.HT_XEN_PVM: { constants.HV_ROOT_PATH: "/dev/sda5", }, constants.HT_XEN_HVM: { constants.HV_ACPI: False, constants.HV_PAE: True, }, } return _TestOsModify(hv_dict) def TestOsModifyInvalid(): """Testing an invalid os modify invocation""" hv_dict = { "blahblahblubb": {"bar": ""}, } return _TestOsModify(hv_dict, fail=True) def TestOsStatesNonExisting(): """Testing OS states with non-existing OS""" AssertCommand(["test", "-e", _TEMP_OS_PATH], fail=True) return _TestOsStates(_TEMP_OS_NAME) ganeti-3.1.0~rc2/qa/qa_performance.py000064400000000000000000000471211476477700300175460ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Performance testing QA tests. """ import datetime import functools import itertools import threading import time from ganeti import constants from qa import qa_config from qa import qa_error from qa_instance_utils import GetGenericAddParameters from qa import qa_job_utils from qa import qa_logging from qa import qa_utils MAX_JOB_SUBMISSION_DURATION = 15.0 class _JobQueueDriver(object): """This class handles polling of jobs and reacting on status changes. Jobs are added via the L{AddJob} method, and can have callback functions assigned to them. Those are called as soon as the job enters the appropriate state. Callback functions can add new jobs to the driver as needed. A call to L{WaitForCompletion} finally polls Ganeti until all jobs have succeeded. """ _UNKNOWN_STATUS = "unknown" class _JobEntry(object): """Internal class representing a job entry. """ def __init__(self, job_id, running_fn, success_fn): self.job_id = job_id self.running_fn = running_fn self.success_fn = success_fn def __str__(self): return str(self.job_id) def __init__(self): self._jobs = {} self._running_notified = set() self._jobs_per_status = {} self._lock = threading.RLock() def AddJob(self, job_id, running_fn=None, success_fn=None): """Add a job to the driver. @type job_id: of ints @param job_id: job id to add to the driver @type running_fn: function taking a L{_JobQueueDriver} and an int @param running_fn: function called once when a job changes to running state (or success state, if the running state was too short) @type success_fn: function taking a L{_JobQueueDriver} and an int @param success_fn: function called for each successful job id """ with self._lock: self._jobs[job_id] = _JobQueueDriver._JobEntry(job_id, running_fn, success_fn) # the status will be updated on the next call to _FetchJobStatuses self._jobs_per_status.setdefault(self._UNKNOWN_STATUS, []).append(job_id) def _FetchJobStatuses(self): """Retrieves status information of the given jobs. """ job_statuses = qa_job_utils.GetJobStatuses(self._GetJobIds()) new_statuses = {} for job_id, status in job_statuses.items(): new_statuses.setdefault(status, []).append(self._jobs[int(job_id)]) self._jobs_per_status = new_statuses def _GetJobIds(self): return list(self._jobs) def _GetJobsInStatuses(self, statuses): """Returns a list of L{_JobEntry} of all jobs in the given statuses. @type statuses: iterable of strings @param statuses: jobs in those statuses are returned @rtype: list of L{_JobEntry} @return: list of job entries in the requested statuses """ ret = [] for state in statuses: ret.extend(self._jobs_per_status.get(state, [])) return ret def _UpdateJobStatuses(self): """Retrieves job statuses from the cluster and updates internal state. """ self._FetchJobStatuses() error_jobs = self._GetJobsInStatuses([constants.JOB_STATUS_ERROR]) if error_jobs: raise qa_error.Error( "Jobs %s are in error state!" % [job.job_id for job in error_jobs]) for job in self._GetJobsInStatuses([constants.JOB_STATUS_RUNNING, constants.JOB_STATUS_SUCCESS]): if job.job_id not in self._running_notified: if job.running_fn is not None: job.running_fn(self, job.job_id) self._running_notified.add(job.job_id) for job in self._GetJobsInStatuses([constants.JOB_STATUS_SUCCESS]): if job.success_fn is not None: job.success_fn(self, job.job_id) # we're done with this job del self._jobs[job.job_id] def _HasPendingJobs(self): """Checks if there are still jobs pending. @rtype: bool @return: C{True} if there are still jobs which have not succeeded """ with self._lock: self._UpdateJobStatuses() uncompleted_jobs = self._GetJobsInStatuses( constants.JOB_STATUS_ALL - constants.JOBS_FINALIZED) unknown_jobs = self._GetJobsInStatuses([self._UNKNOWN_STATUS]) return len(uncompleted_jobs) > 0 or len(unknown_jobs) > 0 def WaitForCompletion(self): """Wait for the completion of all registered jobs. """ while self._HasPendingJobs(): time.sleep(2) with self._lock: if self._jobs: raise qa_error.Error( "Jobs %s didn't finish in success state!" % self._GetJobIds()) def _AcquireAllInstances(): """Generator for acquiring all instances in the QA config. """ try: while True: instance = qa_config.AcquireInstance() yield instance except qa_error.OutOfInstancesError: pass def _AcquireAllNodes(): """Generator for acquiring all nodes in the QA config. """ exclude = [] try: while True: node = qa_config.AcquireNode(exclude=exclude) exclude.append(node) yield node except qa_error.OutOfNodesError: pass def _ExecuteJobSubmittingCmd(cmd): """Executes a job submitting command and returns the resulting job ID. This will fail if submitting the job takes longer than L{MAX_JOB_SUBMISSION_DURATION}. @type cmd: list of string or string @param cmd: the job producing command to execute on the cluster @rtype: int @return: job-id """ start = datetime.datetime.now() result = qa_job_utils.ExecuteJobProducingCommand(cmd) duration = qa_utils.TimedeltaToTotalSeconds(datetime.datetime.now() - start) if duration > MAX_JOB_SUBMISSION_DURATION: print(qa_logging.FormatWarning( "Executing '%s' took %f seconds, a maximum of %f was expected" % (cmd, duration, MAX_JOB_SUBMISSION_DURATION))) return result def _SubmitInstanceCreationJob(instance, disk_template=None): """Submit an instance creation job. @type instance: L{qa_config._QaInstance} @param instance: instance to submit a create command for @type disk_template: string @param disk_template: disk template for the new instance or C{None} which causes the default disk template to be used @rtype: int @return: job id of the submitted creation job """ if disk_template is None: disk_template = qa_config.GetDefaultDiskTemplate() try: cmd = (["gnt-instance", "add", "--submit", "--opportunistic-locking", "--os-type=%s" % qa_config.get("os"), "--disk-template=%s" % disk_template] + GetGenericAddParameters(instance, disk_template)) cmd.append(instance.name) instance.SetDiskTemplate(disk_template) return _ExecuteJobSubmittingCmd(cmd) except: instance.Release() raise def _SubmitInstanceRemoveJob(instance): """Submit an instance remove job. @type instance: L{qa_config._QaInstance} @param instance: the instance to remove @rtype: int @return: job id of the submitted remove job """ try: cmd = (["gnt-instance", "remove", "--submit", "-f"]) cmd.append(instance.name) return _ExecuteJobSubmittingCmd(cmd) finally: instance.Release() def _TestParallelInstanceCreationAndRemoval(max_instances=None, disk_template=None, custom_job_driver=None): """Tests parallel creation and immediate removal of instances. @type max_instances: int @param max_instances: maximum number of instances to create @type disk_template: string @param disk_template: disk template for the new instances or C{None} which causes the default disk template to be used @type custom_job_driver: _JobQueueDriver @param custom_job_driver: a custom L{_JobQueueDriver} to use if not L{None}. If one is specified, C{WaitForCompletion} is _not_ called on it. """ job_driver = custom_job_driver or _JobQueueDriver() def _CreateSuccessFn(instance, job_driver, _): job_id = _SubmitInstanceRemoveJob(instance) job_driver.AddJob(job_id) instance_generator = _AcquireAllInstances() if max_instances is not None: instance_generator = itertools.islice(instance_generator, max_instances) for instance in instance_generator: job_id = _SubmitInstanceCreationJob(instance, disk_template=disk_template) job_driver.AddJob( job_id, success_fn=functools.partial(_CreateSuccessFn, instance)) if custom_job_driver is None: job_driver.WaitForCompletion() def TestParallelMaxInstanceCreationPerformance(): """PERFORMANCE: Parallel instance creation (instance count = max). """ _TestParallelInstanceCreationAndRemoval() def TestParallelNodeCountInstanceCreationPerformance(): """PERFORMANCE: Parallel instance creation (instance count = node count). """ nodes = list(_AcquireAllNodes()) _TestParallelInstanceCreationAndRemoval(max_instances=len(nodes)) qa_config.ReleaseManyNodes(nodes) def CreateAllInstances(): """Create all instances configured in QA config in the cluster. @rtype: list of L{qa_config._QaInstance} @return: list of instances created in the cluster """ job_driver = _JobQueueDriver() instances = list(_AcquireAllInstances()) for instance in instances: job_id = _SubmitInstanceCreationJob(instance) job_driver.AddJob(job_id) job_driver.WaitForCompletion() return instances def RemoveAllInstances(instances): """Removes all given instances from the cluster. @type instances: list of L{qa_config._QaInstance} @param instances: """ job_driver = _JobQueueDriver() for instance in instances: job_id = _SubmitInstanceRemoveJob(instance) job_driver.AddJob(job_id) job_driver.WaitForCompletion() def TestParallelModify(instances): """PERFORMANCE: Parallel instance modify. @type instances: list of L{qa_config._QaInstance} @param instances: list of instances to issue modify commands against """ job_driver = _JobQueueDriver() # set min mem to same value as max mem new_min_mem = qa_config.get(constants.BE_MAXMEM) for instance in instances: cmd = (["gnt-instance", "modify", "--submit", "-B", "%s=%s" % (constants.BE_MINMEM, new_min_mem)]) cmd.append(instance.name) job_driver.AddJob(_ExecuteJobSubmittingCmd(cmd)) cmd = (["gnt-instance", "modify", "--submit", "-O", "fake_os_param=fake_value"]) cmd.append(instance.name) job_driver.AddJob(_ExecuteJobSubmittingCmd(cmd)) cmd = (["gnt-instance", "modify", "--submit", "-O", "fake_os_param=fake_value", "-B", "%s=%s" % (constants.BE_MINMEM, new_min_mem)]) cmd.append(instance.name) job_driver.AddJob(_ExecuteJobSubmittingCmd(cmd)) job_driver.WaitForCompletion() def TestParallelInstanceOSOperations(instances): """PERFORMANCE: Parallel instance OS operations. Note: This test leaves the instances either running or stopped, there's no guarantee on the actual status. @type instances: list of L{qa_config._QaInstance} @param instances: list of instances to issue lifecycle commands against """ OPS = ["start", "shutdown", "reboot", "reinstall"] job_driver = _JobQueueDriver() def _SubmitNextOperation(instance, start, idx, job_driver, _): if idx == len(OPS): return op_idx = (start + idx) % len(OPS) next_fn = functools.partial(_SubmitNextOperation, instance, start, idx + 1) if OPS[op_idx] == "reinstall" and \ instance.disk_template == constants.DT_DISKLESS: # no reinstall possible with diskless instances next_fn(job_driver, None) return elif OPS[op_idx] == "reinstall": # the instance has to be shut down for reinstall to work shutdown_cmd = ["gnt-instance", "shutdown", "--submit", instance.name] cmd = ["gnt-instance", "reinstall", "--submit", "-f", instance.name] job_driver.AddJob(_ExecuteJobSubmittingCmd(shutdown_cmd), running_fn=lambda _, __: job_driver.AddJob( _ExecuteJobSubmittingCmd(cmd), running_fn=next_fn)) else: cmd = ["gnt-instance", OPS[op_idx], "--submit"] if OPS[op_idx] == "reinstall": cmd.append("-f") cmd.append(instance.name) job_id = _ExecuteJobSubmittingCmd(cmd) job_driver.AddJob(job_id, running_fn=next_fn) for start, instance in enumerate(instances): _SubmitNextOperation(instance, start % len(OPS), 0, job_driver, None) job_driver.WaitForCompletion() def TestParallelInstanceQueries(instances): """PERFORMANCE: Parallel instance queries. @type instances: list of L{qa_config._QaInstance} @param instances: list of instances to issue queries against """ threads = qa_job_utils.QAThreadGroup() for instance in instances: cmd = ["gnt-instance", "info", instance.name] info_thread = qa_job_utils.QAThread(qa_utils.AssertCommand, [cmd], {}) threads.Start(info_thread) cmd = ["gnt-instance", "list"] list_thread = qa_job_utils.QAThread(qa_utils.AssertCommand, [cmd], {}) threads.Start(list_thread) threads.JoinAndReraise() def TestJobQueueSubmissionPerformance(): """PERFORMANCE: Job queue submission performance. This test exercises the job queue and verifies that the job submission time does not increase as more jobs are added. """ MAX_CLUSTER_INFO_SECONDS = 15.0 job_driver = _JobQueueDriver() submission_durations = [] def _VerifySubmissionDuration(duration_seconds): # only start to verify the submission duration once we got data from the # first 10 job submissions if len(submission_durations) >= 10: avg_duration = sum(submission_durations) / len(submission_durations) max_duration = avg_duration * 1.5 if duration_seconds > max_duration: print(qa_logging.FormatWarning( "Submitting a delay job took %f seconds, max %f expected" % (duration_seconds, max_duration))) else: submission_durations.append(duration_seconds) def _SubmitDelayJob(count): for _ in range(count): cmd = ["gnt-debug", "delay", "--submit", "0.1"] start = datetime.datetime.now() job_id = _ExecuteJobSubmittingCmd(cmd) duration_seconds = \ qa_utils.TimedeltaToTotalSeconds(datetime.datetime.now() - start) _VerifySubmissionDuration(duration_seconds) job_driver.AddJob(job_id) threads = qa_job_utils.QAThreadGroup() for _ in range(10): thread = qa_job_utils.QAThread(_SubmitDelayJob, [20], {}) threads.Start(thread) threads.JoinAndReraise() qa_utils.AssertCommand(["gnt-cluster", "info"], max_seconds=MAX_CLUSTER_INFO_SECONDS) job_driver.WaitForCompletion() def TestParallelDRBDInstanceCreationPerformance(): """PERFORMANCE: Parallel DRBD backed instance creation. """ assert qa_config.IsTemplateSupported(constants.DT_DRBD8) nodes = list(_AcquireAllNodes()) _TestParallelInstanceCreationAndRemoval(max_instances=len(nodes) * 2, disk_template=constants.DT_DRBD8) qa_config.ReleaseManyNodes(nodes) def TestParallelPlainInstanceCreationPerformance(): """PERFORMANCE: Parallel plain backed instance creation. """ assert qa_config.IsTemplateSupported(constants.DT_PLAIN) nodes = list(_AcquireAllNodes()) _TestParallelInstanceCreationAndRemoval(max_instances=len(nodes) * 2, disk_template=constants.DT_PLAIN) qa_config.ReleaseManyNodes(nodes) def _TestInstanceOperationInParallelToInstanceCreation(*cmds): """Run the given test command in parallel to an instance creation. @type cmds: list of list of strings @param cmds: commands to execute in parallel to an instance creation. Each command in the list is executed once the previous job starts to run. """ def _SubmitNextCommand(cmd_idx, job_driver, _): if cmd_idx >= len(cmds): return job_id = _ExecuteJobSubmittingCmd(cmds[cmd_idx]) job_driver.AddJob( job_id, success_fn=functools.partial(_SubmitNextCommand, cmd_idx + 1)) assert qa_config.IsTemplateSupported(constants.DT_DRBD8) assert len(cmds) > 0 job_driver = _JobQueueDriver() _SubmitNextCommand(0, job_driver, None) _TestParallelInstanceCreationAndRemoval(max_instances=1, disk_template=constants.DT_DRBD8, custom_job_driver=job_driver) job_driver.WaitForCompletion() def TestParallelInstanceFailover(instance): """PERFORMANCE: Instance failover with parallel instance creation. """ _TestInstanceOperationInParallelToInstanceCreation( ["gnt-instance", "failover", "--submit", "-f", "--shutdown-timeout=0", instance.name]) def TestParallelInstanceMigration(instance): """PERFORMANCE: Instance migration with parallel instance creation. """ _TestInstanceOperationInParallelToInstanceCreation( ["gnt-instance", "migrate", "--submit", "-f", instance.name]) def TestParallelInstanceReplaceDisks(instance): """PERFORMANCE: Instance replace-disks with parallel instance creation. """ _TestInstanceOperationInParallelToInstanceCreation( ["gnt-instance", "replace-disks", "--submit", "--early-release", "-p", instance.name]) def TestParallelInstanceReboot(instance): """PERFORMANCE: Instance reboot with parallel instance creation. """ _TestInstanceOperationInParallelToInstanceCreation( ["gnt-instance", "reboot", "--submit", instance.name]) def TestParallelInstanceReinstall(instance): """PERFORMANCE: Instance reinstall with parallel instance creation. """ # instance reinstall requires the instance to be down qa_utils.AssertCommand(["gnt-instance", "stop", instance.name]) _TestInstanceOperationInParallelToInstanceCreation( ["gnt-instance", "reinstall", "--submit", "-f", instance.name]) qa_utils.AssertCommand(["gnt-instance", "start", instance.name]) def TestParallelInstanceRename(instance): """PERFORMANCE: Instance rename with parallel instance creation. """ # instance rename requires the instance to be down qa_utils.AssertCommand(["gnt-instance", "stop", instance.name]) new_instance = qa_config.AcquireInstance() try: _TestInstanceOperationInParallelToInstanceCreation( ["gnt-instance", "rename", "--submit", instance.name, new_instance.name], ["gnt-instance", "rename", "--submit", new_instance.name, instance.name]) finally: new_instance.Release() qa_utils.AssertCommand(["gnt-instance", "start", instance.name]) ganeti-3.1.0~rc2/qa/qa_rapi.py000064400000000000000000001243161476477700300162020ustar00rootroot00000000000000# # # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Remote API QA tests. """ import copy import itertools import os.path import random import re import tempfile import uuid as uuid_module from ganeti import cli from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import locking from ganeti import objects from ganeti import opcodes from ganeti import pathutils from ganeti import qlang from ganeti import query from ganeti import rapi from ganeti import utils from ganeti.http.auth import ParsePasswordFile import ganeti.rapi.client # pylint: disable=W0611 import ganeti.rapi.client_utils from qa import qa_config from qa import qa_error from qa import qa_logging from qa import qa_utils from qa_instance import GetInstanceInfo from qa_instance import IsDiskReplacingSupported from qa_instance import IsFailoverSupported from qa_instance import IsMigrationSupported from qa_job_utils import RunWithLocks from qa_utils import (AssertEqual, AssertIn, AssertMatch, AssertCommand, StartLocalCommand) from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG _rapi_ca = None _rapi_client = None _rapi_username = None _rapi_password = None # The files to copy if the RAPI files QA config value is set _FILES_TO_COPY = [ pathutils.CLUSTER_DOMAIN_SECRET_FILE, pathutils.RAPI_CERT_FILE, pathutils.RAPI_USERS_FILE, ] def _EnsureRapiFilesPresence(): """Ensures that the specified RAPI files are present on the cluster, if any. """ rapi_files_location = qa_config.get("rapi-files-location", None) if rapi_files_location is None: # No files to be had return print(qa_logging.FormatWarning("Replacing the certificate and users file on" " the node with the ones provided in %s" % rapi_files_location)) # The RAPI files AssertCommand(["mkdir", "-p", pathutils.RAPI_DATA_DIR]) for filename in _FILES_TO_COPY: basename = os.path.split(filename)[-1] AssertCommand(["cp", os.path.join(rapi_files_location, basename), filename]) AssertCommand(["gnt-cluster", "copyfile", filename]) # The certificates have to be reloaded now AssertCommand(["service", "ganeti", "restart"]) def ReloadCertificates(ensure_presence=True): """Reloads the client RAPI certificate with the one present on the node. If the QA is set up to use a specific certificate using the "rapi-files-location" parameter, it will be put in place prior to retrieving it. """ if ensure_presence: _EnsureRapiFilesPresence() if _rapi_username is None or _rapi_password is None: raise qa_error.Error("RAPI username and password have to be set before" " attempting to reload a certificate.") # pylint: disable=W0603 # due to global usage global _rapi_ca global _rapi_client master = qa_config.GetMasterNode() # Load RAPI certificate from master node cmd = ["openssl", "x509", "-in", qa_utils.MakeNodePath(master, pathutils.RAPI_CERT_FILE)] # Write to temporary file _rapi_ca = tempfile.NamedTemporaryFile(mode="w") _rapi_ca.write(qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))) _rapi_ca.flush() port = qa_config.get("rapi-port", default=constants.DEFAULT_RAPI_PORT) cfg_curl = rapi.client.GenericCurlConfig(cafile=_rapi_ca.name, proxy="") if qa_config.UseVirtualCluster(): # TODO: Implement full support for RAPI on virtual clusters print(qa_logging.FormatWarning("RAPI tests are not yet supported on" " virtual clusters and will be disabled")) assert _rapi_client is None else: _rapi_client = rapi.client.GanetiRapiClient(master.primary, port=port, username=_rapi_username, password=_rapi_password, curl_config_fn=cfg_curl) print("RAPI protocol version: %s" % _rapi_client.GetVersion()) #TODO(riba): Remove in 2.13, used just by rapi-workload which disappears there def GetClient(): """Retrieves the RAPI client prepared by this module. """ return _rapi_client def _CreateRapiUser(rapi_user): """RAPI credentials creation, with the secret auto-generated. """ rapi_secret = utils.GenerateSecret() master = qa_config.GetMasterNode() rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE) rapi_dir = os.path.dirname(rapi_users_path) fh = tempfile.NamedTemporaryFile(mode="w") try: fh.write("%s %s write\n" % (rapi_user, rapi_secret)) fh.flush() tmpru = qa_utils.UploadFile(master.primary, fh.name) try: AssertCommand(["mkdir", "-p", rapi_dir]) AssertCommand(["mv", tmpru, rapi_users_path]) finally: AssertCommand(["rm", "-f", tmpru]) finally: fh.close() # The certificates have to be reloaded now AssertCommand(["service", "ganeti", "restart"]) return rapi_secret def _LookupRapiSecret(rapi_user): """Find the RAPI secret for the given user on the QA machines. @param rapi_user: Login user @return: Login secret for the user """ CTEXT = "{CLEARTEXT}" master = qa_config.GetMasterNode() cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)] file_content = qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) users = ParsePasswordFile(file_content) entry = users.get(rapi_user) if not entry: raise qa_error.Error("User %s not found in RAPI users file" % rapi_user) secret = entry.password if secret.upper().startswith(CTEXT): secret = secret[len(CTEXT):] elif secret.startswith("{"): raise qa_error.Error("Unsupported password schema for RAPI user %s:" " not a clear text password" % rapi_user) return secret def _ReadRapiSecret(password_file_path): """Reads a RAPI secret stored locally. @type password_file_path: string @return: Login secret for the user """ try: with open(password_file_path, 'r') as pw_file: return pw_file.readline().strip() except IOError: raise qa_error.Error("Could not open the RAPI password file located at" " %s" % password_file_path) def _GetRapiSecret(rapi_user): """Returns the secret to be used for RAPI access. Where exactly this secret can be found depends on the QA configuration options, and this function invokes additional tools as needed. It can look up a local secret, a remote one, or create a user with a new secret. @param rapi_user: Login user @return: Login secret for the user """ password_file_path = qa_config.get("rapi-password-file", None) if password_file_path is not None: # If the password file is specified, we use the password within. # The file must be present on the QA runner. return _ReadRapiSecret(password_file_path) else: # On an existing cluster, just find out the user's secret return _LookupRapiSecret(rapi_user) def SetupRapi(): """Sets up the RAPI certificate and usernames for the client. """ if not Enabled(): return (None, None) # pylint: disable=W0603 # due to global usage global _rapi_username global _rapi_password _rapi_username = qa_config.get("rapi-user", "ganeti-qa") if qa_config.TestEnabled("create-cluster") and \ qa_config.get("rapi-files-location") is None: # For a new cluster, we have to invent a secret and a user, unless it has # been provided separately _rapi_password = _CreateRapiUser(_rapi_username) else: _EnsureRapiFilesPresence() _rapi_password = _GetRapiSecret(_rapi_username) # Once a username and password have been set, we can fetch the certs and # get all we need for a working RAPI client. ReloadCertificates(ensure_presence=False) INSTANCE_FIELDS = ("name", "os", "pnode", "snodes", "admin_state", "disk_template", "disk.sizes", "disk.spindles", "nic.ips", "nic.macs", "nic.modes", "nic.links", "beparams", "hvparams", "oper_state", "oper_ram", "oper_vcpus", "status", "tags") NODE_FIELDS = ("name", "dtotal", "dfree", "sptotal", "spfree", "mtotal", "mnode", "mfree", "pinst_cnt", "sinst_cnt", "tags") GROUP_FIELDS = compat.UniqueFrozenset([ "name", "uuid", "alloc_policy", "node_cnt", "node_list", ]) JOB_FIELDS = compat.UniqueFrozenset([ "id", "ops", "status", "summary", "opstatus", "opresult", "oplog", "received_ts", "start_ts", "end_ts", ]) FILTER_FIELDS = compat.UniqueFrozenset([ "watermark", "priority", "predicates", "action", "reason_trail", "uuid", ]) LIST_FIELDS = ("id", "uri") def Enabled(): """Return whether remote API tests should be run. """ # TODO: Implement RAPI tests for virtual clusters return (qa_config.TestEnabled("rapi") and not qa_config.UseVirtualCluster()) def _DoTests(uris): # pylint: disable=W0212 # due to _SendRequest usage results = [] for uri, verify, method, body in uris: assert uri.startswith("/") print("%s %s" % (method, uri)) data = _rapi_client._SendRequest(method, uri, None, body) if verify is not None: if callable(verify): verify(data) else: AssertEqual(data, verify) results.append(data) return results # pylint: disable=W0212 # Due to _SendRequest usage def _DoGetPutTests(get_uri, modify_uri, opcode_params, rapi_only_aliases=None, modify_method="PUT", exceptions=None, set_exceptions=None): """ Test if all params of an object can be retrieved, and set as well. @type get_uri: string @param get_uri: The URI from which information about the object can be retrieved. @type modify_uri: string @param modify_uri: The URI which can be used to modify the object. @type opcode_params: list of tuple @param opcode_params: The parameters of the underlying opcode, used to determine which parameters are actually present. @type rapi_only_aliases: list of string or None @param rapi_only_aliases: Aliases for parameters which differ from the opcode, and become renamed before opcode submission. @type modify_method: string @param modify_method: The method to be used in the modification. @type exceptions: list of string or None @param exceptions: The parameters which have not been exposed and should not be tested at all. @type set_exceptions: list of string or None @param set_exceptions: The parameters whose setting should not be tested as a part of this test. """ assert get_uri.startswith("/") assert modify_uri.startswith("/") if exceptions is None: exceptions = [] if set_exceptions is None: set_exceptions = [] print("Testing get/modify symmetry of %s and %s" % (get_uri, modify_uri)) # First we see if all parameters of the opcode are returned through RAPI params_of_interest = [x[0] for x in opcode_params] # The RAPI-specific aliases are to be checked as well if rapi_only_aliases is not None: params_of_interest.extend(rapi_only_aliases) info = _rapi_client._SendRequest("GET", get_uri, None, {}) missing_params = [x for x in params_of_interest if x not in info and x not in exceptions] if missing_params: raise qa_error.Error("The parameters %s which can be set through the " "appropriate opcode are not present in the response " "from %s" % (','.join(missing_params), get_uri)) print("GET successful at %s" % get_uri) # Then if we can perform a set with the same values as received put_payload = {} for param in params_of_interest: if param not in exceptions and param not in set_exceptions: put_payload[param] = info[param] _rapi_client._SendRequest(modify_method, modify_uri, None, put_payload) print("%s successful at %s" % (modify_method, modify_uri)) # pylint: enable=W0212 def _VerifyReturnsJob(data): if not isinstance(data, int): AssertMatch(data, r"^\d+$") def TestVersion(): """Testing remote API version. """ _DoTests([ ("/version", constants.RAPI_VERSION, "GET", None), ]) def TestEmptyCluster(): """Testing remote API on an empty cluster. """ master = qa_config.GetMasterNode() master_full = qa_utils.ResolveNodeName(master) def _VerifyInfo(data): AssertIn("name", data) AssertIn("master", data) AssertEqual(data["master"], master_full) def _VerifyNodes(data): master_entry = { "id": master_full, "uri": "/2/nodes/%s" % master_full, } AssertIn(master_entry, data) def _VerifyNodesBulk(data): for node in data: for entry in NODE_FIELDS: AssertIn(entry, node) def _VerifyGroups(data): default_group = { "name": constants.INITIAL_NODE_GROUP_NAME, "uri": "/2/groups/" + constants.INITIAL_NODE_GROUP_NAME, } AssertIn(default_group, data) def _VerifyGroupsBulk(data): for group in data: for field in GROUP_FIELDS: AssertIn(field, group) def _VerifyFiltersBulk(data): for group in data: for field in FILTER_FIELDS: AssertIn(field, group) _DoTests([ ("/", None, "GET", None), ("/2/info", _VerifyInfo, "GET", None), ("/2/tags", None, "GET", None), ("/2/nodes", _VerifyNodes, "GET", None), ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None), ("/2/groups", _VerifyGroups, "GET", None), ("/2/groups?bulk=1", _VerifyGroupsBulk, "GET", None), ("/2/instances", [], "GET", None), ("/2/instances?bulk=1", [], "GET", None), ("/2/os", None, "GET", None), ("/2/filters", [], "GET", None), ("/2/filters?bulk=1", _VerifyFiltersBulk, "GET", None), ]) # Test HTTP Not Found for method in ["GET", "PUT", "POST", "DELETE"]: try: _DoTests([("/99/resource/not/here/99", None, method, None)]) except rapi.client.GanetiApiError as err: AssertEqual(err.code, 404) else: raise qa_error.Error("Non-existent resource didn't return HTTP 404") # Test HTTP Not Implemented for method in ["PUT", "POST", "DELETE"]: try: _DoTests([("/version", None, method, None)]) except rapi.client.GanetiApiError as err: AssertEqual(err.code, 501) else: raise qa_error.Error("Non-implemented method didn't fail") # Test GET/PUT symmetry LEGITIMATELY_MISSING = [ "force", # Standard option "add_uids", # Modifies UID pool, is not a param itself "remove_uids", # Same as above "osparams_private_cluster", # Should not be returned ] NOT_EXPOSED_YET = ["hv_state", "disk_state", "modify_etc_hosts"] # The nicparams are returned under the default entry, yet accepted as they # are - this is a TODO to fix! DEFAULT_ISSUES = ["nicparams"] # Cannot be set over RAPI due to security issues FORBIDDEN_PARAMS = ["compression_tools"] _DoGetPutTests("/2/info", "/2/modify", opcodes.OpClusterSetParams.OP_PARAMS, exceptions=(LEGITIMATELY_MISSING + NOT_EXPOSED_YET), set_exceptions=DEFAULT_ISSUES + FORBIDDEN_PARAMS) def TestRapiQuery(): """Testing resource queries via remote API. """ # FIXME: the tests are failing if no LVM is enabled, investigate # if it is a bug in the QA or in the code if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG): return master_name = qa_utils.ResolveNodeName(qa_config.GetMasterNode()) rnd = random.Random(7818) for what in constants.QR_VIA_RAPI: namefield = { constants.QR_JOB: "id", constants.QR_EXPORT: "export", constants.QR_FILTER: "uuid", }.get(what, "name") all_fields = list(query.ALL_FIELDS[what]) rnd.shuffle(all_fields) # No fields, should return everything result = _rapi_client.QueryFields(what) qresult = objects.QueryFieldsResponse.FromDict(result) AssertEqual(len(qresult.fields), len(all_fields)) # One field result = _rapi_client.QueryFields(what, fields=[namefield]) qresult = objects.QueryFieldsResponse.FromDict(result) AssertEqual(len(qresult.fields), 1) # Specify all fields, order must be correct result = _rapi_client.QueryFields(what, fields=all_fields) qresult = objects.QueryFieldsResponse.FromDict(result) AssertEqual(len(qresult.fields), len(all_fields)) AssertEqual([fdef.name for fdef in qresult.fields], all_fields) # Unknown field result = _rapi_client.QueryFields(what, fields=["_unknown!"]) qresult = objects.QueryFieldsResponse.FromDict(result) AssertEqual(len(qresult.fields), 1) AssertEqual(qresult.fields[0].name, "_unknown!") AssertEqual(qresult.fields[0].kind, constants.QFT_UNKNOWN) # Try once more, this time without the client _DoTests([ ("/2/query/%s/fields" % what, None, "GET", None), ("/2/query/%s/fields?fields=%s,%s,%s" % (what, namefield, namefield, all_fields[0]), None, "GET", None), ]) # Try missing query argument try: _DoTests([ ("/2/query/%s" % what, None, "GET", None), ]) except rapi.client.GanetiApiError as err: AssertEqual(err.code, 400) else: raise qa_error.Error("Request missing 'fields' parameter didn't fail") def _Check(exp_fields, data): qresult = objects.QueryResponse.FromDict(data) AssertEqual([fdef.name for fdef in qresult.fields], exp_fields) if not isinstance(qresult.data, list): raise qa_error.Error("Query did not return a list") _DoTests([ # Specify fields in query ("/2/query/%s?fields=%s" % (what, ",".join(all_fields)), compat.partial(_Check, all_fields), "GET", None), ("/2/query/%s?fields=%s" % (what, namefield), compat.partial(_Check, [namefield]), "GET", None), # Note the spaces ("/2/query/%s?fields=%s,%%20%s%%09,%s%%20" % (what, namefield, namefield, namefield), compat.partial(_Check, [namefield] * 3), "GET", None)]) if what in constants.QR_VIA_RAPI_PUT: _DoTests([ # PUT with fields in query ("/2/query/%s?fields=%s" % (what, namefield), compat.partial(_Check, [namefield]), "PUT", {}), ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", { "fields": [namefield] * 4, }), ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", { "fields": all_fields, }), ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", { "fields": [namefield] * 4 })]) if what in constants.QR_VIA_RAPI_PUT: trivial_filter = { constants.QR_JOB: [qlang.OP_GE, namefield, 0], }.get(what, [qlang.OP_REGEXP, namefield, ".*"]) _DoTests([ # With filter ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", { "fields": all_fields, "filter": trivial_filter }), ]) if what == constants.QR_NODE: # Test with filter (nodes, ) = _DoTests( [("/2/query/%s" % what, compat.partial(_Check, ["name", "master"]), "PUT", {"fields": ["name", "master"], "filter": [qlang.OP_TRUE, "master"], })]) qresult = objects.QueryResponse.FromDict(nodes) AssertEqual(qresult.data, [ [[constants.RS_NORMAL, master_name], [constants.RS_NORMAL, True]], ]) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstance(instance): """Testing getting instance(s) info via remote API. """ def _VerifyInstance(data): for entry in INSTANCE_FIELDS: AssertIn(entry, data) def _VerifyInstancesList(data): for instance in data: for entry in LIST_FIELDS: AssertIn(entry, instance) def _VerifyInstancesBulk(data): for instance_data in data: _VerifyInstance(instance_data) _DoTests([ ("/2/instances/%s" % instance.name, _VerifyInstance, "GET", None), ("/2/instances", _VerifyInstancesList, "GET", None), ("/2/instances?bulk=1", _VerifyInstancesBulk, "GET", None), ("/2/instances/%s/activate-disks" % instance.name, _VerifyReturnsJob, "PUT", None), ("/2/instances/%s/deactivate-disks" % instance.name, _VerifyReturnsJob, "PUT", None), ]) # Test OpBackupPrepare (job_id, ) = _DoTests([ ("/2/instances/%s/prepare-export?mode=%s" % (instance.name, constants.EXPORT_MODE_REMOTE), _VerifyReturnsJob, "PUT", None), ]) result = _WaitForRapiJob(job_id)[0] AssertEqual(len(result["handshake"]), 3) AssertEqual(result["handshake"][0], constants.RIE_VERSION) AssertEqual(len(result["x509_key_name"]), 3) AssertIn("-----BEGIN CERTIFICATE-----", result["x509_ca"]) def TestNode(node): """Testing getting node(s) info via remote API. """ def _VerifyNode(data): for entry in NODE_FIELDS: AssertIn(entry, data) def _VerifyNodesList(data): for node in data: for entry in LIST_FIELDS: AssertIn(entry, node) def _VerifyNodesBulk(data): for node_data in data: _VerifyNode(node_data) _DoTests([ ("/2/nodes/%s" % node.primary, _VerifyNode, "GET", None), ("/2/nodes", _VerifyNodesList, "GET", None), ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None), ]) # Not parameters of the node, but controlling opcode behavior LEGITIMATELY_MISSING = ["force", "powered"] # Identifying the node - RAPI provides these itself IDENTIFIERS = ["node_name", "node_uuid"] # As the name states, these can be set but not retrieved yet NOT_EXPOSED_YET = ["hv_state", "disk_state", "auto_promote"] _DoGetPutTests("/2/nodes/%s" % node.primary, "/2/nodes/%s/modify" % node.primary, opcodes.OpNodeSetParams.OP_PARAMS, modify_method="POST", exceptions=(LEGITIMATELY_MISSING + NOT_EXPOSED_YET + IDENTIFIERS)) def _FilterTags(seq): """Removes unwanted tags from a sequence. """ ignore_re = qa_config.get("ignore-tags-re", None) if ignore_re: return itertools.filterfalse(re.compile(ignore_re).match, seq) else: return seq def TestTags(kind, name, tags): """Tests .../tags resources. """ if kind == constants.TAG_CLUSTER: uri = "/2/tags" elif kind == constants.TAG_NODE: uri = "/2/nodes/%s/tags" % name elif kind == constants.TAG_INSTANCE: uri = "/2/instances/%s/tags" % name elif kind == constants.TAG_NODEGROUP: uri = "/2/groups/%s/tags" % name elif kind == constants.TAG_NETWORK: uri = "/2/networks/%s/tags" % name else: raise errors.ProgrammerError("Unknown tag kind") def _VerifyTags(data): AssertEqual(sorted(tags), sorted(_FilterTags(data))) queryargs = "&".join("tag=%s" % i for i in tags) # Add tags (job_id, ) = _DoTests([ ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "PUT", None), ]) _WaitForRapiJob(job_id) # Retrieve tags _DoTests([ (uri, _VerifyTags, "GET", None), ]) # Remove tags (job_id, ) = _DoTests([ ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "DELETE", None), ]) _WaitForRapiJob(job_id) def _WaitForRapiJob(job_id): """Waits for a job to finish. """ def _VerifyJob(data): AssertEqual(data["id"], job_id) for field in JOB_FIELDS: AssertIn(field, data) _DoTests([ ("/2/jobs/%s" % job_id, _VerifyJob, "GET", None), ]) return rapi.client_utils.PollJob(_rapi_client, job_id, cli.StdioJobPollReportCb()) def TestRapiNodeGroups(): """Test several node group operations using RAPI. """ (group1, group2, group3) = qa_utils.GetNonexistentGroups(3) # Create a group with no attributes body = { "name": group1, } (job_id, ) = _DoTests([ ("/2/groups", _VerifyReturnsJob, "POST", body), ]) _WaitForRapiJob(job_id) # Create a group specifying alloc_policy body = { "name": group2, "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE, } (job_id, ) = _DoTests([ ("/2/groups", _VerifyReturnsJob, "POST", body), ]) _WaitForRapiJob(job_id) # Modify alloc_policy body = { "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE, } (job_id, ) = _DoTests([ ("/2/groups/%s/modify" % group1, _VerifyReturnsJob, "PUT", body), ]) _WaitForRapiJob(job_id) # Rename a group body = { "new_name": group3, } (job_id, ) = _DoTests([ ("/2/groups/%s/rename" % group2, _VerifyReturnsJob, "PUT", body), ]) _WaitForRapiJob(job_id) # Test for get/set symmetry # Identifying the node - RAPI provides these itself IDENTIFIERS = ["group_name"] # As the name states, not exposed yet NOT_EXPOSED_YET = ["hv_state", "disk_state"] # The parameters we do not want to get and set (as that sets the # group-specific params to the filled ones) FILLED_PARAMS = ["ndparams", "ipolicy", "diskparams"] # The aliases that we can use to perform this test with the group-specific # params CUSTOM_PARAMS = ["custom_ndparams", "custom_ipolicy", "custom_diskparams"] _DoGetPutTests("/2/groups/%s" % group3, "/2/groups/%s/modify" % group3, opcodes.OpGroupSetParams.OP_PARAMS, rapi_only_aliases=CUSTOM_PARAMS, exceptions=(IDENTIFIERS + NOT_EXPOSED_YET), set_exceptions=FILLED_PARAMS) # Delete groups for group in [group1, group3]: (job_id, ) = _DoTests([ ("/2/groups/%s" % group, _VerifyReturnsJob, "DELETE", None), ]) _WaitForRapiJob(job_id) def TestRapiInstanceAdd(node, use_client): """Test adding a new instance via RAPI""" if not qa_config.IsTemplateSupported(constants.DT_PLAIN): return instance = qa_config.AcquireInstance() instance.SetDiskTemplate(constants.DT_PLAIN) try: disks = [{"size": utils.ParseUnit(d.get("size")), "name": str(d.get("name"))} for d in qa_config.GetDiskOptions()] nic0_mac = instance.GetNicMacAddr(0, constants.VALUE_GENERATE) nics = [{ constants.INIC_MAC: nic0_mac, }] beparams = { constants.BE_MAXMEM: utils.ParseUnit(qa_config.get(constants.BE_MAXMEM)), constants.BE_MINMEM: utils.ParseUnit(qa_config.get(constants.BE_MINMEM)), } if use_client: job_id = _rapi_client.CreateInstance(constants.INSTANCE_CREATE, instance.name, constants.DT_PLAIN, disks, nics, os=qa_config.get("os"), pnode=node.primary, beparams=beparams) else: body = { "__version__": 1, "mode": constants.INSTANCE_CREATE, "name": instance.name, "os_type": qa_config.get("os"), "disk_template": constants.DT_PLAIN, "pnode": node.primary, "beparams": beparams, "disks": disks, "nics": nics, } (job_id, ) = _DoTests([ ("/2/instances", _VerifyReturnsJob, "POST", body), ]) _WaitForRapiJob(job_id) return instance except: instance.Release() raise def _GenInstanceAllocationDict(node, instance): """Creates an instance allocation dict to be used with the RAPI""" instance.SetDiskTemplate(constants.DT_PLAIN) disks = [{"size": utils.ParseUnit(d.get("size")), "name": str(d.get("name"))} for d in qa_config.GetDiskOptions()] nic0_mac = instance.GetNicMacAddr(0, constants.VALUE_GENERATE) nics = [{ constants.INIC_MAC: nic0_mac, }] beparams = { constants.BE_MAXMEM: utils.ParseUnit(qa_config.get(constants.BE_MAXMEM)), constants.BE_MINMEM: utils.ParseUnit(qa_config.get(constants.BE_MINMEM)), } return _rapi_client.InstanceAllocation(constants.INSTANCE_CREATE, instance.name, constants.DT_PLAIN, disks, nics, os=qa_config.get("os"), pnode=node.primary, beparams=beparams) def TestRapiInstanceMultiAlloc(node): """Test adding two new instances via the RAPI instance-multi-alloc method""" if not qa_config.IsTemplateSupported(constants.DT_PLAIN): return JOBS_KEY = "jobs" instance_one = qa_config.AcquireInstance() instance_two = qa_config.AcquireInstance() instance_list = [instance_one, instance_two] try: rapi_dicts = [_GenInstanceAllocationDict(node, i) for i in instance_list] job_id = _rapi_client.InstancesMultiAlloc(rapi_dicts) results, = _WaitForRapiJob(job_id) if JOBS_KEY not in results: raise qa_error.Error("RAPI instance-multi-alloc did not deliver " "information about created jobs") if len(results[JOBS_KEY]) != len(instance_list): raise qa_error.Error("RAPI instance-multi-alloc failed to return the " "desired number of jobs!") for success, job in results[JOBS_KEY]: if success: _WaitForRapiJob(job) else: raise qa_error.Error("Failed to create instance in " "instance-multi-alloc call") except: # Note that although released, it may be that some of the instance creations # have in fact succeeded. Handling this in a better way may be possible, but # is not necessary as the QA has already failed at this point. for instance in instance_list: instance.Release() raise return (instance_one, instance_two) @InstanceCheck(None, INST_DOWN, FIRST_ARG) def TestRapiInstanceRemove(instance, use_client): """Test removing instance via RAPI""" # FIXME: this does not work if LVM is not enabled. Find out if this is a bug # in RAPI or in the test if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG): return if use_client: job_id = _rapi_client.DeleteInstance(instance.name) else: (job_id, ) = _DoTests([ ("/2/instances/%s" % instance.name, _VerifyReturnsJob, "DELETE", None), ]) _WaitForRapiJob(job_id) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRapiInstanceMigrate(instance): """Test migrating instance via RAPI""" if not IsMigrationSupported(instance): print(qa_logging.FormatInfo("Instance doesn't support migration, skipping" " test")) return # Move to secondary node _WaitForRapiJob(_rapi_client.MigrateInstance(instance.name)) qa_utils.RunInstanceCheck(instance, True) # And back to previous primary _WaitForRapiJob(_rapi_client.MigrateInstance(instance.name)) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRapiInstanceFailover(instance): """Test failing over instance via RAPI""" if not IsFailoverSupported(instance): print(qa_logging.FormatInfo("Instance doesn't support failover, skipping" " test")) return # Move to secondary node _WaitForRapiJob(_rapi_client.FailoverInstance(instance.name)) qa_utils.RunInstanceCheck(instance, True) # And back to previous primary _WaitForRapiJob(_rapi_client.FailoverInstance(instance.name)) @InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG) def TestRapiInstanceShutdown(instance): """Test stopping an instance via RAPI""" _WaitForRapiJob(_rapi_client.ShutdownInstance(instance.name)) @InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG) def TestRapiInstanceStartup(instance): """Test starting an instance via RAPI""" _WaitForRapiJob(_rapi_client.StartupInstance(instance.name)) @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestRapiInstanceRenameAndBack(rename_source, rename_target): """Test renaming instance via RAPI This must leave the instance with the original name (in the non-failure case). """ _WaitForRapiJob(_rapi_client.RenameInstance(rename_source, rename_target)) qa_utils.RunInstanceCheck(rename_source, False) qa_utils.RunInstanceCheck(rename_target, False) _WaitForRapiJob(_rapi_client.RenameInstance(rename_target, rename_source)) qa_utils.RunInstanceCheck(rename_target, False) @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestRapiInstanceReinstall(instance): """Test reinstalling an instance via RAPI""" if instance.disk_template == constants.DT_DISKLESS: print(qa_logging.FormatInfo("Test not supported for diskless instances")) return _WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name)) # By default, the instance is started again qa_utils.RunInstanceCheck(instance, True) # Reinstall again without starting _WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name, no_startup=True)) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRapiInstanceReplaceDisks(instance): """Test replacing instance disks via RAPI""" if not IsDiskReplacingSupported(instance): print(qa_logging.FormatInfo("Instance doesn't support disk replacing," " skipping test")) return fn = _rapi_client.ReplaceInstanceDisks _WaitForRapiJob(fn(instance.name, mode=constants.REPLACE_DISK_AUTO, disks=[])) _WaitForRapiJob(fn(instance.name, mode=constants.REPLACE_DISK_SEC, disks="0")) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRapiInstanceModify(instance): """Test modifying instance via RAPI""" default_hv = qa_config.GetDefaultHypervisor() def _ModifyInstance(**kwargs): _WaitForRapiJob(_rapi_client.ModifyInstance(instance.name, **kwargs)) _ModifyInstance(beparams={ constants.BE_VCPUS: 3, }) _ModifyInstance(beparams={ constants.BE_VCPUS: constants.VALUE_DEFAULT, }) if default_hv == constants.HT_XEN_PVM: _ModifyInstance(hvparams={ constants.HV_KERNEL_ARGS: "single", }) _ModifyInstance(hvparams={ constants.HV_KERNEL_ARGS: constants.VALUE_DEFAULT, }) elif default_hv == constants.HT_XEN_HVM: _ModifyInstance(hvparams={ constants.HV_BOOT_ORDER: "acn", }) _ModifyInstance(hvparams={ constants.HV_BOOT_ORDER: constants.VALUE_DEFAULT, }) @InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRapiInstanceConsole(instance): """Test getting instance console information via RAPI""" result = _rapi_client.GetInstanceConsole(instance.name) console = objects.InstanceConsole.FromDict(result) AssertEqual(console.Validate(), None) AssertEqual(console.instance, qa_utils.ResolveInstanceName(instance.name)) @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestRapiStoppedInstanceConsole(instance): """Test getting stopped instance's console information via RAPI""" try: _rapi_client.GetInstanceConsole(instance.name) except rapi.client.GanetiApiError as err: AssertEqual(err.code, 503) else: raise qa_error.Error("Getting console for stopped instance didn't" " return HTTP 503") def GetOperatingSystems(): """Retrieves a list of all available operating systems. """ return _rapi_client.GetOperatingSystems() def _InvokeMoveInstance(current_dest_inst, current_src_inst, rapi_pw_filename, joint_master, perform_checks, target_nodes=None): """ Invokes the move-instance tool for testing purposes. """ # Some uses of this test might require that RAPI-only commands are used, # and the checks are command-line based. if perform_checks: qa_utils.RunInstanceCheck(current_dest_inst, False) cmd = [ "../tools/move-instance", "--verbose", "--src-ca-file=%s" % _rapi_ca.name, "--src-username=%s" % _rapi_username, "--src-password-file=%s" % rapi_pw_filename, "--dest-instance-name=%s" % current_dest_inst, ] if target_nodes: pnode, snode = target_nodes cmd.extend([ "--dest-primary-node=%s" % pnode, "--dest-secondary-node=%s" % snode, ]) else: cmd.extend([ "--iallocator=%s" % constants.IALLOC_HAIL, "--opportunistic-tries=1", ]) cmd.extend([ "--net=0:mac=%s" % constants.VALUE_GENERATE, joint_master, joint_master, current_src_inst, ]) AssertEqual(StartLocalCommand(cmd).wait(), 0) if perform_checks: qa_utils.RunInstanceCheck(current_src_inst, False) qa_utils.RunInstanceCheck(current_dest_inst, True) def TestInterClusterInstanceMove(src_instance, dest_instance, inodes, tnode, perform_checks=True): """Test tools/move-instance""" master = qa_config.GetMasterNode() rapi_pw_file = tempfile.NamedTemporaryFile(mode="w") rapi_pw_file.write(_rapi_password) rapi_pw_file.flush() # Needed only if checks are to be performed if perform_checks: dest_instance.SetDiskTemplate(src_instance.disk_template) # TODO: Run some instance tests before moving back if len(inodes) > 1: # No disk template currently requires more than 1 secondary node. If this # changes, either this test must be skipped or the script must be updated. assert len(inodes) == 2 snode = inodes[1] else: # Instance is not redundant, but we still need to pass a node # (which will be ignored) snode = tnode pnode = inodes[0] # pnode:snode are the *current* nodes, and the first move is an # iallocator-guided move outside of pnode. The node lock for the pnode # assures that this happens, and while we cannot be sure where the instance # will land, it is a real move. locks = {locking.LEVEL_NODE: [pnode.primary]} RunWithLocks(_InvokeMoveInstance, locks, 600.0, False, dest_instance.name, src_instance.name, rapi_pw_file.name, master.primary, perform_checks) # And then back to pnode:snode _InvokeMoveInstance(src_instance.name, dest_instance.name, rapi_pw_file.name, master.primary, perform_checks, target_nodes=(pnode.primary, snode.primary)) def TestFilters(): """Testing filter management via the remote API. """ body = { "priority": 10, "predicates": [], "action": "CONTINUE", "reason": [(constants.OPCODE_REASON_SRC_USER, "reason1", utils.EpochNano())], } body1 = copy.deepcopy(body) body1["priority"] = 20 # Query filters _DoTests([("/2/filters", [], "GET", None)]) # Add a filter via POST and delete it again uuid = _DoTests([("/2/filters", None, "POST", body)])[0] uuid_module.UUID(uuid) # Check if uuid is a valid UUID _DoTests([("/2/filters/%s" % uuid, lambda r: r is None, "DELETE", None)]) _DoTests([ # Check PUT-inserting a nonexistent filter with given UUID ("/2/filters/%s" % uuid, lambda u: u == uuid, "PUT", body), # Check PUT-inserting an existent filter with given UUID ("/2/filters/%s" % uuid, lambda u: u == uuid, "PUT", body1), # Check that the update changed the filter ("/2/filters/%s" % uuid, lambda f: f["priority"] == 20, "GET", None), # Delete it again ("/2/filters/%s" % uuid, lambda r: r is None, "DELETE", None), ]) # Add multiple filters, query and delete them uuids = _DoTests([ ("/2/filters", None, "POST", body), ("/2/filters", None, "POST", body), ("/2/filters", None, "POST", body), ]) _DoTests([("/2/filters", lambda rs: [r["uuid"] for r in rs] == uuids, "GET", None)]) for u in uuids: _DoTests([("/2/filters/%s" % u, lambda r: r is None, "DELETE", None)]) _DRBD_SECRET_RE = re.compile('shared-secret.*"([0-9A-Fa-f]+)"') def _RetrieveSecret(instance, pnode): """Retrieves the DRBD secret given an instance object and the primary node. @type instance: L{qa_config._QaInstance} @type pnode: L{qa_config._QaNode} @rtype: string """ instance_info = GetInstanceInfo(instance.name) # We are interested in only the first disk on the primary drbd_minor = instance_info["drbd-minors"][pnode.primary][0] # This form should work for all DRBD versions drbd_command = ("drbdsetup show %d; drbdsetup %d show || true" % (drbd_minor, drbd_minor)) instance_drbd_info = \ qa_utils.GetCommandOutput(pnode.primary, drbd_command) match_obj = _DRBD_SECRET_RE.search(instance_drbd_info) if match_obj is None: raise qa_error.Error("Could not retrieve DRBD secret for instance %s from" " node %s." % (instance.name, pnode.primary)) return match_obj.groups(0)[0] def TestInstanceDataCensorship(instance, inodes): """Test protection of sensitive instance data.""" if instance.disk_template != constants.DT_DRBD8: print(qa_utils.FormatInfo("Only the DRBD secret is a sensitive parameter" " right now, skipping for non-DRBD instance.")) return drbd_secret = _RetrieveSecret(instance, inodes[0]) job_id = _rapi_client.GetInstanceInfo(instance.name) if not _rapi_client.WaitForJobCompletion(job_id): raise qa_error.Error("Could not fetch instance info for instance %s" % instance.name) info_dict = _rapi_client.GetJobStatus(job_id) if drbd_secret in str(info_dict): print(qa_utils.FormatInfo("DRBD secret: %s" % drbd_secret)) print(qa_utils.FormatInfo("Retrieved data\n%s" % str(info_dict))) raise qa_error.Error("Found DRBD secret in contents of RAPI instance info" " call; see above.") ganeti-3.1.0~rc2/qa/qa_tags.py000064400000000000000000000053321476477700300162010ustar00rootroot00000000000000# # # Copyright (C) 2007 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tags related QA tests. """ from ganeti import constants from qa import qa_rapi from qa_utils import AssertCommand _TEMP_TAG_NAMES = ["TEMP-Ganeti-QA-Tag%d" % i for i in range(3)] _TEMP_TAG_RE = r'^TEMP-Ganeti-QA-Tag\d+$' _KIND_TO_COMMAND = { constants.TAG_CLUSTER: "gnt-cluster", constants.TAG_NODE: "gnt-node", constants.TAG_INSTANCE: "gnt-instance", constants.TAG_NODEGROUP: "gnt-group", constants.TAG_NETWORK: "gnt-network", } def _TestTags(kind, name): """Generic function for add-tags. """ def cmdfn(subcmd): cmd = [_KIND_TO_COMMAND[kind], subcmd] if kind != constants.TAG_CLUSTER: cmd.append(name) return cmd for cmd in [ cmdfn("add-tags") + _TEMP_TAG_NAMES, cmdfn("list-tags"), ["gnt-cluster", "search-tags", _TEMP_TAG_RE], cmdfn("remove-tags") + _TEMP_TAG_NAMES, ]: AssertCommand(cmd) if qa_rapi.Enabled(): qa_rapi.TestTags(kind, name, _TEMP_TAG_NAMES) def TestClusterTags(): """gnt-cluster tags""" _TestTags(constants.TAG_CLUSTER, "") def TestNodeTags(node): """gnt-node tags""" _TestTags(constants.TAG_NODE, node.primary) def TestGroupTags(group): """gnt-group tags""" _TestTags(constants.TAG_NODEGROUP, group) def TestInstanceTags(instance): """gnt-instance tags""" _TestTags(constants.TAG_INSTANCE, instance.name) def TestNetworkTags(network): """gnt-network tags""" _TestTags(constants.TAG_NETWORK, network) ganeti-3.1.0~rc2/qa/qa_utils.py000064400000000000000000000734331476477700300164120ustar00rootroot00000000000000# # # Copyright (C) 2007, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utilities for QA tests. """ from __future__ import print_function import contextlib import copy import datetime import operator import os import random import re import socket import subprocess import sys import tempfile import yaml try: import functools except ImportError as err: raise ImportError("Python 2.5 or higher is required: %s" % err) from ganeti import utils from ganeti import compat from ganeti import constants from ganeti import ht from ganeti import pathutils from ganeti import vcluster import colors from qa import qa_config from qa import qa_error from qa_logging import FormatInfo #: Unique ID per QA run _RUN_UUID = utils.NewUUID() #: Path to the QA query output log file _QA_OUTPUT = pathutils.GetLogFilename("qa-output") _RETRIES = 3 (INST_DOWN, INST_UP) = range(500, 502) (FIRST_ARG, RETURN_VALUE) = range(1000, 1002) def _RaiseWithInfo(msg, error_desc): """Raises a QA error with the given content, and adds a message if present. """ if msg: output = "%s: %s" % (msg, error_desc) else: output = error_desc raise qa_error.Error(output) def AssertIn(item, sequence, msg=None): """Raises an error when item is not in sequence. """ if item not in sequence: _RaiseWithInfo(msg, "%r not in %r" % (item, sequence)) def AssertNotIn(item, sequence, msg=None): """Raises an error when item is in sequence. """ if item in sequence: _RaiseWithInfo(msg, "%r in %r" % (item, sequence)) def AssertEqual(first, second, msg=None): """Raises an error when values aren't equal. """ if not first == second: _RaiseWithInfo(msg, "%r == %r" % (first, second)) def AssertMatch(string, pattern, msg=None): """Raises an error when string doesn't match regexp pattern. """ if not re.match(pattern, string): _RaiseWithInfo(msg, "%r doesn't match /%r/" % (string, pattern)) def _GetName(entity, fn): """Tries to get name of an entity. @type entity: string or dict @param fn: Function retrieving name from entity """ if isinstance(entity, str): result = entity else: result = fn(entity) if not ht.TNonEmptyString(result): raise Exception("Invalid name '%s'" % result) return result def _AssertRetCode(rcode, fail, cmdstr, nodename): """Check the return value from a command and possibly raise an exception. """ if fail and rcode == 0: raise qa_error.Error("Command '%s' on node %s was expected to fail but" " didn't" % (cmdstr, nodename)) elif not fail and rcode != 0: raise qa_error.Error("Command '%s' on node %s failed, exit code %s" % (cmdstr, nodename, rcode)) def _PrintCommandOutput(stdout, stderr): """Prints the output of commands, minimizing wasted space. @type stdout: string @type stderr: string """ if stdout: stdout_clean = stdout.rstrip('\n') if stderr: print("Stdout was:\n%s" % stdout_clean) else: print(stdout_clean) if stderr: print("Stderr was:") print(stderr.rstrip('\n'), file=sys.stderr) def AssertCommand(cmd, fail=False, node=None, log_cmd=True, forward_agent=True, max_seconds=None): """Checks that a remote command succeeds. @param cmd: either a string (the command to execute) or a list (to be converted using L{utils.ShellQuoteArgs} into a string) @type fail: boolean or None @param fail: if the command is expected to fail instead of succeeding, or None if we don't care @param node: if passed, it should be the node on which the command should be executed, instead of the master node (can be either a dict or a string) @param log_cmd: if False, the command won't be logged (simply passed to StartSSH) @type forward_agent: boolean @param forward_agent: whether to forward the agent when starting the SSH session or not, sometimes useful for crypto-related operations which can use a key they should not @type max_seconds: double @param max_seconds: fail if the command takes more than C{max_seconds} seconds @return: the return code, stdout and stderr of the command @raise qa_error.Error: if the command fails when it shouldn't or vice versa """ if node is None: node = qa_config.GetMasterNode() nodename = _GetName(node, operator.attrgetter("primary")) if isinstance(cmd, str): cmdstr = cmd else: cmdstr = utils.ShellQuoteArgs(cmd) start = datetime.datetime.now() popen = StartSSH(nodename, cmdstr, log_cmd=log_cmd, forward_agent=forward_agent) # Run the command stdout, stderr = popen.communicate() rcode = popen.returncode duration_seconds = TimedeltaToTotalSeconds(datetime.datetime.now() - start) try: if fail is not None: _AssertRetCode(rcode, fail, cmdstr, nodename) finally: if log_cmd: _PrintCommandOutput(stdout, stderr) if max_seconds is not None: if duration_seconds > max_seconds: raise qa_error.Error( "Cmd '%s' took %f seconds, maximum of %f was exceeded" % (cmdstr, duration_seconds, max_seconds)) return rcode, stdout, stderr def stdout_of(cmd): """Small helper to run a stdout_of. Makes sure the stdout_of returns exit code 0. @type cmd: list of strings @param cmd: the stdout_of to run @return: Captured, stripped stdout. """ _, out, _ = AssertCommand(cmd) return out.strip() def AssertRedirectedCommand(cmd, fail=False, node=None, log_cmd=True): """Executes a command with redirected output. The log will go to the qa-output log file in the ganeti log directory on the node where the command is executed. The fail and node parameters are passed unchanged to AssertCommand. @param cmd: the command to be executed, as a list; a string is not supported """ if not isinstance(cmd, list): raise qa_error.Error("Non-list passed to AssertRedirectedCommand") ofile = utils.ShellQuote(_QA_OUTPUT) cmdstr = utils.ShellQuoteArgs(cmd) AssertCommand("echo ---- $(date) %s ---- >> %s" % (cmdstr, ofile), fail=False, node=node, log_cmd=False) return AssertCommand(cmdstr + " >> %s" % ofile, fail=fail, node=node, log_cmd=log_cmd) def GetSSHCommand(node, cmd, strict=True, opts=None, tty=False, use_multiplexer=True, forward_agent=True): """Builds SSH command to be executed. @type node: string @param node: node the command should run on @type cmd: string @param cmd: command to be executed in the node; if None or empty string, no command will be executed @type strict: boolean @param strict: whether to enable strict host key checking @type opts: list @param opts: list of additional options @type tty: boolean or None @param tty: if we should use tty; if None, will be auto-detected @type use_multiplexer: boolean @param use_multiplexer: if the multiplexer for the node should be used @type forward_agent: boolean @param forward_agent: whether to forward the ssh agent or not """ args = ["ssh", "-oEscapeChar=none", "-oBatchMode=yes", "-lroot"] if tty is None: tty = sys.stdout.isatty() if tty: args.append("-t") # Multiplexers we use right now forward agents, so even if we ought to be # using one, ignore it if agent forwarding is disabled. if not forward_agent: use_multiplexer = False args.append("-oStrictHostKeyChecking=%s" % ("yes" if strict else "no", )) args.append("-oClearAllForwardings=yes") args.append("-oForwardAgent=%s" % ("yes" if forward_agent else "no", )) if opts: args.extend(opts) if node in qa_config.MULTIPLEXERS and use_multiplexer: spath = qa_config.MULTIPLEXERS[node][0] args.append("-oControlPath=%s" % spath) args.append("-oControlMaster=no") (vcluster_master, vcluster_basedir) = \ qa_config.GetVclusterSettings() if vcluster_master: args.append(vcluster_master) args.append("%s/%s/cmd" % (vcluster_basedir, node)) if cmd: # For virtual clusters the whole command must be wrapped using the "cmd" # script, as that script sets a number of environment variables. If the # command contains shell meta characters the whole command needs to be # quoted. args.append(utils.ShellQuote(cmd)) else: args.append(node) if cmd: args.append(cmd) return args def StartLocalCommand(cmd, _nolog_opts=False, log_cmd=True, **kwargs): """Starts a local command. """ if log_cmd: if _nolog_opts: pcmd = [i for i in cmd if not i.startswith("-")] else: pcmd = cmd print("%s %s" % (colors.colorize("Command:", colors.CYAN), utils.ShellQuoteArgs(pcmd))) return subprocess.Popen(cmd, shell=False, encoding="utf-8", **kwargs) def StartSSH(node, cmd, strict=True, log_cmd=True, forward_agent=True): """Starts SSH. """ ssh_command = GetSSHCommand(node, cmd, strict=strict, forward_agent=forward_agent) return StartLocalCommand(ssh_command, _nolog_opts=True, log_cmd=log_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) def StartMultiplexer(node): """Starts a multiplexer command. @param node: the node for which to open the multiplexer """ if node in qa_config.MULTIPLEXERS: return # Note: yes, we only need mktemp, since we'll remove the file anyway sname = tempfile.mktemp(prefix="ganeti-qa-multiplexer.") utils.RemoveFile(sname) opts = ["-N", "-oControlPath=%s" % sname, "-oControlMaster=yes"] print("Created socket at %s" % sname) child = StartLocalCommand(GetSSHCommand(node, None, opts=opts)) qa_config.MULTIPLEXERS[node] = (sname, child) def CloseMultiplexers(): """Closes all current multiplexers and cleans up. """ for node in list(qa_config.MULTIPLEXERS): (sname, child) = qa_config.MULTIPLEXERS.pop(node) utils.KillProcess(child.pid, timeout=10, waitpid=True) utils.RemoveFile(sname) def _GetCommandStdout(proc): """Extract the stored standard error, print it and return it. """ out = proc.stdout.read() sys.stdout.write(out) return out def _NoTimeout(state): """False iff the command timed out.""" rcode, out = state return rcode == 0 or not ('TimeoutError' in out or 'timed out' in out) def GetCommandOutput(node, cmd, tty=False, use_multiplexer=True, log_cmd=True, fail=False): """Returns the output of a command executed on the given node. @type node: string @param node: node the command should run on @type cmd: string @param cmd: command to be executed in the node (cannot be empty or None) @type tty: bool or None @param tty: if we should use tty; if None, it will be auto-detected @type use_multiplexer: bool @param use_multiplexer: if the SSH multiplexer provided by the QA should be used or not @type log_cmd: bool @param log_cmd: if the command should be logged @type fail: bool @param fail: whether the command is expected to fail """ assert cmd def CallCommand(): command = GetSSHCommand(node, cmd, tty=tty, use_multiplexer=use_multiplexer) p = StartLocalCommand(command, stdout=subprocess.PIPE, log_cmd=log_cmd) rcode = p.wait() out = _GetCommandStdout(p) return rcode, out # TODO: make retries configurable rcode, out = utils.CountRetry(_NoTimeout, CallCommand, _RETRIES) _AssertRetCode(rcode, fail, cmd, node) return out def GetObjectInfo(infocmd): """Get and parse information about a Ganeti object. @type infocmd: list of strings @param infocmd: command to be executed, e.g. ["gnt-cluster", "info"] @return: the information parsed, appropriately stored in dictionaries, lists... """ master = qa_config.GetMasterNode() cmdline = utils.ShellQuoteArgs(infocmd) info_out = GetCommandOutput(master.primary, cmdline) return yaml.load(info_out, Loader=yaml.SafeLoader) def UploadFile(node, src): """Uploads a file to a node and returns the filename. Caller needs to remove the returned file on the node when it's not needed anymore. """ # Make sure nobody else has access to it while preserving local permissions mode = os.stat(src).st_mode & 0o700 cmd = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && ' 'chmod %o "${tmp}" && ' '[[ -f "${tmp}" ]] && ' 'cat > "${tmp}" && ' 'echo "${tmp}"') % mode f = open(src, "r") try: p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f, stdout=subprocess.PIPE, encoding="utf-8") AssertEqual(p.wait(), 0) # Return temporary filename return _GetCommandStdout(p).strip() finally: f.close() def UploadData(node, data, mode=0o600, filename=None): """Uploads data to a node and returns the filename. Caller needs to remove the returned file on the node when it's not needed anymore. """ if filename: tmp = "tmp=%s" % utils.ShellQuote(filename) else: tmp = ('tmp=$(mktemp --tmpdir gnt.XXXXXX) && ' 'chmod %o "${tmp}"') % mode cmd = ("%s && " "[[ -f \"${tmp}\" ]] && " "cat > \"${tmp}\" && " "echo \"${tmp}\"") % tmp p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, encoding="utf-8", stdin=subprocess.PIPE, stdout=subprocess.PIPE) p.stdin.write(data) p.stdin.close() AssertEqual(p.wait(), 0) # Return temporary filename return _GetCommandStdout(p).strip() def BackupFile(node, path): """Creates a backup of a file on the node and returns the filename. Caller needs to remove the returned file on the node when it's not needed anymore. """ vpath = MakeNodePath(node, path) cmd = ("tmp=$(mktemp .gnt.XXXXXX --tmpdir=$(dirname %s)) && " "[[ -f \"$tmp\" ]] && " "cp %s $tmp && " "echo $tmp") % (utils.ShellQuote(vpath), utils.ShellQuote(vpath)) # Return temporary filename result = GetCommandOutput(node, cmd).strip() print("Backup filename: %s" % result) return result @contextlib.contextmanager def CheckFileUnmodified(node, filename): """Checks that the content of a given file remains the same after running a wrapped code. @type node: string @param node: node the command should run on @type filename: string @param filename: absolute filename to check """ cmd = utils.ShellQuoteArgs(["sha1sum", MakeNodePath(node, filename)]) def Read(): return GetCommandOutput(node, cmd).strip() # read the configuration before = Read() yield # check that the configuration hasn't changed after = Read() if before != after: raise qa_error.Error("File '%s' has changed unexpectedly on node %s" " during the last operation" % (filename, node)) def ResolveInstanceName(instance): """Gets the full name of an instance. @type instance: string @param instance: Instance name """ info = GetObjectInfo(["gnt-instance", "info", instance]) return info[0]["Instance name"] def ResolveNodeName(node): """Gets the full name of a node. """ info = GetObjectInfo(["gnt-node", "info", node.primary]) return info[0]["Node name"] def GetNodeInstances(node, secondaries=False): """Gets a list of instances on a node. """ master = qa_config.GetMasterNode() node_name = ResolveNodeName(node) # Get list of all instances cmd = ["gnt-instance", "list", "--separator=:", "--no-headers", "--output=name,pnode,snodes"] output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)) instances = [] for line in output.splitlines(): (name, pnode, snodes) = line.split(":", 2) if ((not secondaries and pnode == node_name) or (secondaries and node_name in snodes.split(","))): instances.append(name) return instances def _SelectQueryFields(rnd, fields): """Generates a list of fields for query tests. """ # Create copy for shuffling fields = list(fields) rnd.shuffle(fields) # Check all fields yield fields yield sorted(fields) # Duplicate fields yield fields + fields # Check small groups of fields while fields: yield [fields.pop() for _ in range(rnd.randint(2, 10)) if fields] def _List(listcmd, fields, names): """Runs a list command. """ master = qa_config.GetMasterNode() cmd = [listcmd, "list", "--separator=|", "--no-headers", "--output", ",".join(fields)] if names: cmd.extend(names) return GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd)).splitlines() def GenericQueryTest(cmd, fields, namefield="name", test_unknown=True): """Runs a number of tests on query commands. @param cmd: Command name @param fields: List of field names """ rnd = random.Random(hash(cmd)) fields = list(fields) rnd.shuffle(fields) # Test a number of field combinations for testfields in _SelectQueryFields(rnd, fields): AssertRedirectedCommand([cmd, "list", "--output", ",".join(testfields)]) if namefield is not None: namelist_fn = compat.partial(_List, cmd, [namefield]) # When no names were requested, the list must be sorted names = namelist_fn(None) AssertEqual(names, utils.NiceSort(names)) # When requesting specific names, the order must be kept revnames = list(reversed(names)) AssertEqual(namelist_fn(revnames), revnames) randnames = list(names) rnd.shuffle(randnames) AssertEqual(namelist_fn(randnames), randnames) if test_unknown: # Listing unknown items must fail AssertCommand([cmd, "list", "this.name.certainly.does.not.exist"], fail=True) # Check exit code for listing unknown field rcode, _, _ = AssertRedirectedCommand([cmd, "list", "--output=field/does/not/exist"], fail=True) AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD) def GenericQueryFieldsTest(cmd, fields): master = qa_config.GetMasterNode() # Listing fields AssertRedirectedCommand([cmd, "list-fields"]) AssertRedirectedCommand([cmd, "list-fields"] + fields) # Check listed fields (all, must be sorted) realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"] output = GetCommandOutput(master.primary, utils.ShellQuoteArgs(realcmd)).splitlines() AssertEqual([line.split("|", 1)[0] for line in output], utils.NiceSort(fields)) # Check exit code for listing unknown field rcode, _, _ = AssertCommand([cmd, "list-fields", "field/does/not/exist"], fail=True) AssertEqual(rcode, constants.EXIT_UNKNOWN_FIELD) def AddToEtcHosts(hostnames): """Adds hostnames to /etc/hosts. @param hostnames: List of hostnames first used A records, all other CNAMEs """ master = qa_config.GetMasterNode() tmp_hosts = UploadData(master.primary, "", mode=0o644) data = [] for localhost in ("::1", "127.0.0.1"): data.append("%s %s" % (localhost, " ".join(hostnames))) try: AssertCommand("{ cat %s && echo -e '%s'; } > %s && mv %s %s" % (utils.ShellQuote(pathutils.ETC_HOSTS), "\\n".join(data), utils.ShellQuote(tmp_hosts), utils.ShellQuote(tmp_hosts), utils.ShellQuote(pathutils.ETC_HOSTS))) except Exception: AssertCommand(["rm", "-f", tmp_hosts]) raise def RemoveFromEtcHosts(hostnames): """Remove hostnames from /etc/hosts. @param hostnames: List of hostnames first used A records, all other CNAMEs """ master = qa_config.GetMasterNode() tmp_hosts = UploadData(master.primary, "", mode=0o644) quoted_tmp_hosts = utils.ShellQuote(tmp_hosts) sed_data = " ".join(hostnames) try: AssertCommand((r"sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s" r" && mv %s %s") % (sed_data, utils.ShellQuote(pathutils.ETC_HOSTS), quoted_tmp_hosts, quoted_tmp_hosts, utils.ShellQuote(pathutils.ETC_HOSTS))) except Exception: AssertCommand(["rm", "-f", tmp_hosts]) raise def RunInstanceCheck(instance, running): """Check if instance is running or not. """ instance_name = _GetName(instance, operator.attrgetter("name")) script = qa_config.GetInstanceCheckScript() if not script: return master_node = qa_config.GetMasterNode() # Build command to connect to master node master_ssh = GetSSHCommand(master_node.primary, "--") if running: running_shellval = "1" running_text = "" else: running_shellval = "" running_text = "not " print(FormatInfo("Checking if instance '%s' is %srunning" % (instance_name, running_text))) args = [script, instance_name] env = { "PATH": constants.HOOKS_PATH, "RUN_UUID": _RUN_UUID, "MASTER_SSH": utils.ShellQuoteArgs(master_ssh), "INSTANCE_NAME": instance_name, "INSTANCE_RUNNING": running_shellval, } result = os.spawnve(os.P_WAIT, script, args, env) if result != 0: raise qa_error.Error("Instance check failed with result %s" % result) def _InstanceCheckInner(expected, instarg, args, result): """Helper function used by L{InstanceCheck}. """ if instarg == FIRST_ARG: instance = args[0] elif instarg == RETURN_VALUE: instance = result else: raise Exception("Invalid value '%s' for instance argument" % instarg) if expected in (INST_DOWN, INST_UP): RunInstanceCheck(instance, (expected == INST_UP)) elif expected is not None: raise Exception("Invalid value '%s'" % expected) def InstanceCheck(before, after, instarg): """Decorator to check instance status before and after test. @param before: L{INST_DOWN} if instance must be stopped before test, L{INST_UP} if instance must be running before test, L{None} to not check. @param after: L{INST_DOWN} if instance must be stopped after test, L{INST_UP} if instance must be running after test, L{None} to not check. @param instarg: L{FIRST_ARG} to use first argument to test as instance (a dictionary), L{RETURN_VALUE} to use return value (disallows pre-checks) """ def decorator(fn): @functools.wraps(fn) def wrapper(*args, **kwargs): _InstanceCheckInner(before, instarg, args, NotImplemented) result = fn(*args, **kwargs) _InstanceCheckInner(after, instarg, args, result) return result return wrapper return decorator def GetNonexistentGroups(count): """Gets group names which shouldn't exist on the cluster. @param count: Number of groups to get @rtype: integer """ return GetNonexistentEntityNames(count, "groups", "group") def GetNonexistentEntityNames(count, name_config, name_prefix): """Gets entity names which shouldn't exist on the cluster. The actualy names can refer to arbitrary entities (for example groups, networks). @param count: Number of names to get @rtype: integer @param name_config: name of the leaf in the config containing this entity's configuration, including a 'inexistent-' element @rtype: string @param name_prefix: prefix of the entity's names, used to compose the default values; for example for groups, the prefix is 'group' and the generated names are then group1, group2, ... @rtype: string """ entities = qa_config.get(name_config, {}) default = [name_prefix + str(i) for i in range(count)] assert count <= len(default) name_config_inexistent = "inexistent-" + name_config candidates = entities.get(name_config_inexistent, default)[:count] if len(candidates) < count: raise Exception("At least %s non-existent %s are needed" % (count, name_config)) return candidates def MakeNodePath(node, path): """Builds an absolute path for a virtual node. @type node: string or L{qa_config._QaNode} @param node: Node @type path: string @param path: Path without node-specific prefix """ (_, basedir) = qa_config.GetVclusterSettings() if isinstance(node, str): name = node else: name = node.primary if basedir: assert path.startswith("/") return "%s%s" % (vcluster.MakeNodeRoot(basedir, name), path) else: return path def _GetParameterOptions(specs): """Helper to build policy options.""" values = ["%s=%s" % (par, val) for (par, val) in specs.items()] return ",".join(values) def TestSetISpecs(new_specs=None, diff_specs=None, get_policy_fn=None, build_cmd_fn=None, fail=False, old_values=None): """Change instance specs for an object. At most one of new_specs or diff_specs can be specified. @type new_specs: dict @param new_specs: new complete specs, in the same format returned by L{ParseIPolicy}. @type diff_specs: dict @param diff_specs: partial specs, it can be an incomplete specifications, but if min/max specs are specified, their number must match the number of the existing specs @type get_policy_fn: function @param get_policy_fn: function that returns the current policy as in L{ParseIPolicy} @type build_cmd_fn: function @param build_cmd_fn: function that return the full command line from the options alone @type fail: bool @param fail: if the change is expected to fail @type old_values: tuple @param old_values: (old_policy, old_specs), as returned by L{ParseIPolicy} @return: same as L{ParseIPolicy} """ assert get_policy_fn is not None assert build_cmd_fn is not None assert new_specs is None or diff_specs is None if old_values: (old_policy, old_specs) = old_values else: (old_policy, old_specs) = get_policy_fn() if diff_specs: new_specs = copy.deepcopy(old_specs) if constants.ISPECS_MINMAX in diff_specs: AssertEqual(len(new_specs[constants.ISPECS_MINMAX]), len(diff_specs[constants.ISPECS_MINMAX])) for (new_minmax, diff_minmax) in zip(new_specs[constants.ISPECS_MINMAX], diff_specs[constants.ISPECS_MINMAX]): for (key, parvals) in diff_minmax.items(): for (par, val) in parvals.items(): new_minmax[key][par] = val for (par, val) in diff_specs.get(constants.ISPECS_STD, {}).items(): new_specs[constants.ISPECS_STD][par] = val if new_specs: cmd = [] if (diff_specs is None or constants.ISPECS_MINMAX in diff_specs): minmax_opt_items = [] for minmax in new_specs[constants.ISPECS_MINMAX]: minmax_opts = [] for key in ["min", "max"]: keyopt = _GetParameterOptions(minmax[key]) minmax_opts.append("%s:%s" % (key, keyopt)) minmax_opt_items.append("/".join(minmax_opts)) cmd.extend([ "--ipolicy-bounds-specs", "//".join(minmax_opt_items) ]) if diff_specs is None: std_source = new_specs else: std_source = diff_specs std_opt = _GetParameterOptions(std_source.get("std", {})) if std_opt: cmd.extend(["--ipolicy-std-specs", std_opt]) AssertCommand(build_cmd_fn(cmd), fail=fail) # Check the new state (eff_policy, eff_specs) = get_policy_fn() AssertEqual(eff_policy, old_policy) if fail: AssertEqual(eff_specs, old_specs) else: AssertEqual(eff_specs, new_specs) else: (eff_policy, eff_specs) = (old_policy, old_specs) return (eff_policy, eff_specs) def ParseIPolicy(policy): """Parse and split instance an instance policy. @type policy: dict @param policy: policy, as returned by L{GetObjectInfo} @rtype: tuple @return: (policy, specs), where: - policy is a dictionary of the policy values, instance specs excluded - specs is a dictionary containing only the specs, using the internal format (see L{constants.IPOLICY_DEFAULTS} for an example) """ ret_specs = {} ret_policy = {} for (key, val) in policy.items(): if key == "bounds specs": ret_specs[constants.ISPECS_MINMAX] = [] for minmax in val: ret_minmax = {} for key in minmax: keyparts = key.split("/", 1) assert len(keyparts) > 1 ret_minmax[keyparts[0]] = minmax[key] ret_specs[constants.ISPECS_MINMAX].append(ret_minmax) elif key == constants.ISPECS_STD: ret_specs[key] = val else: ret_policy[key] = val return (ret_policy, ret_specs) def UsesIPv6Connection(host, port): """Returns True if the connection to a given host/port could go through IPv6. """ return any(t[0] == socket.AF_INET6 for t in socket.getaddrinfo(host, port)) def TimedeltaToTotalSeconds(td): """Returns the total seconds in a C{datetime.timedelta} object. This performs the same task as the C{datetime.timedelta.total_seconds()} method which is present in Python 2.7 onwards. @type td: datetime.timedelta @param td: timedelta object to convert @rtype float @return: total seconds in the timedelta object """ return ((td.microseconds + (td.seconds + td.days * 24.0 * 3600.0) * 10 ** 6) / 10 ** 6) ganeti-3.1.0~rc2/regex/000075500000000000000000000000001476477700300147165ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/pcre/000075500000000000000000000000001476477700300156475ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/pcre/Ganeti/000075500000000000000000000000001476477700300170565ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/pcre/Ganeti/Query/000075500000000000000000000000001476477700300201635ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/pcre/Ganeti/Query/RegEx.hs000064400000000000000000000002331476477700300215270ustar00rootroot00000000000000module Ganeti.Query.RegEx ( RegEx.Regex, RegEx.match, RegEx.makeRegexM, (RegEx.=~), ) where import qualified Text.Regex.PCRE as RegEx ganeti-3.1.0~rc2/regex/pcre2/000075500000000000000000000000001476477700300157315ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/pcre2/Ganeti/000075500000000000000000000000001476477700300171405ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/pcre2/Ganeti/Query/000075500000000000000000000000001476477700300202455ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/pcre2/Ganeti/Query/RegEx.hs000064400000000000000000000002341476477700300216120ustar00rootroot00000000000000module Ganeti.Query.RegEx ( RegEx.Regex, RegEx.match, RegEx.makeRegexM, (RegEx.=~), ) where import qualified Text.Regex.PCRE2 as RegEx ganeti-3.1.0~rc2/regex/tdfa/000075500000000000000000000000001476477700300156345ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/tdfa/Ganeti/000075500000000000000000000000001476477700300170435ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/tdfa/Ganeti/Query/000075500000000000000000000000001476477700300201505ustar00rootroot00000000000000ganeti-3.1.0~rc2/regex/tdfa/Ganeti/Query/RegEx.hs000064400000000000000000000002331476477700300215140ustar00rootroot00000000000000module Ganeti.Query.RegEx ( RegEx.Regex, RegEx.match, RegEx.makeRegexM, (RegEx.=~), ) where import qualified Text.Regex.TDFA as RegEx ganeti-3.1.0~rc2/src/000075500000000000000000000000001476477700300143735ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/AutoConf.hs.in000064400000000000000000000123361476477700300170570ustar00rootroot00000000000000{-| Build-time configuration for Ganeti. Note that this file is autogenerated by the Makefile with a header from @AutoConf.hs.in@. -} {- Copyright (C) 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module AutoConf where split :: String -> [String] split str = case span (/= ',') str of (x, []) -> [x] (x, _:xs) -> x:split xs packageVersion :: String packageVersion = "PACKAGE_VERSION" versionMajor :: Int versionMajor = VERSION_MAJOR versionMinor :: Int versionMinor = VERSION_MINOR versionRevision :: Int versionRevision = VERSION_REVISION versionSuffix :: String versionSuffix = "VERSION_SUFFIX" versionFull :: String versionFull = "VERSION_FULL" dirVersion :: String dirVersion = "DIRVERSION" localstatedir :: String localstatedir = "LOCALSTATEDIR" sysconfdir :: String sysconfdir = "SYSCONFDIR" sshConfigDir :: String sshConfigDir = "SSH_CONFIG_DIR" sshLoginUser :: String sshLoginUser = "SSH_LOGIN_USER" sshConsoleUser :: String sshConsoleUser = "SSH_CONSOLE_USER" exportDir :: String exportDir = "EXPORT_DIR" backupDir :: String backupDir = "BACKUP_DIR" osSearchPath :: [String] osSearchPath = split OS_SEARCH_PATH esSearchPath :: [String] esSearchPath = split ES_SEARCH_PATH xenBootloader :: String xenBootloader = "XEN_BOOTLOADER" xenConfigDir :: String xenConfigDir = "XEN_CONFIG_DIR" xenKernel :: String xenKernel = "XEN_KERNEL" xenInitrd :: String xenInitrd = "XEN_INITRD" kvmKernel :: String kvmKernel = "KVM_KERNEL" sharedFileStorageDir :: String sharedFileStorageDir = "SHARED_FILE_STORAGE_DIR" iallocatorSearchPath :: [String] iallocatorSearchPath = split IALLOCATOR_SEARCH_PATH defaultVg :: String defaultVg = "DEFAULT_VG" defaultBridge :: String defaultBridge = "DEFAULT_BRIDGE" kvmPath :: String kvmPath = "KVM_PATH" ipPath :: String ipPath = "IP_PATH" socatPath :: String socatPath = "SOCAT_PATH" pythonPath :: String pythonPath = "PYTHON_PATH" socatUseEscape :: Bool socatUseEscape = SOCAT_USE_ESCAPE socatUseCompress :: Bool socatUseCompress = SOCAT_USE_COMPRESS lvmStripecount :: Int lvmStripecount = LVM_STRIPECOUNT toolsdir :: String toolsdir = "TOOLSDIR" gntScripts :: [String] gntScripts = GNT_SCRIPTS[] htoolsProgs :: [String] htoolsProgs = HS_HTOOLS_PROGS[] pkglibdir :: String pkglibdir = "PKGLIBDIR" sharedir :: String sharedir = "SHAREDIR" versionedsharedir :: String versionedsharedir = "VERSIONEDSHAREDIR" drbdBarriers :: String drbdBarriers = "DRBD_BARRIERS" drbdNoMetaFlush :: Bool drbdNoMetaFlush = DRBD_NO_META_FLUSH syslogUsage :: String syslogUsage = "SYSLOG_USAGE" daemonsGroup :: String daemonsGroup = "DAEMONS_GROUP" adminGroup :: String adminGroup = "ADMIN_GROUP" masterdUser :: String masterdUser = "MASTERD_USER" masterdGroup :: String masterdGroup = "MASTERD_GROUP" metadUser :: String metadUser = "METAD_USER" metadGroup :: String metadGroup = "METAD_GROUP" rapiUser :: String rapiUser = "RAPI_USER" rapiGroup :: String rapiGroup = "RAPI_GROUP" confdUser :: String confdUser = "CONFD_USER" confdGroup :: String confdGroup = "CONFD_GROUP" wconfdUser :: String wconfdUser = "WCONFD_USER" wconfdGroup :: String wconfdGroup = "WCONFD_GROUP" kvmdUser :: String kvmdUser = "KVMD_USER" kvmdGroup :: String kvmdGroup = "KVMD_GROUP" luxidUser :: String luxidUser = "LUXID_USER" luxidGroup :: String luxidGroup = "LUXID_GROUP" nodedUser :: String nodedUser = "NODED_USER" nodedGroup :: String nodedGroup = "NODED_GROUP" mondUser :: String mondUser = "MOND_USER" mondGroup :: String mondGroup = "MOND_GROUP" diskSeparator :: String diskSeparator = "DISK_SEPARATOR" qemuimgPath :: String qemuimgPath = "QEMUIMG_PATH" htools :: Bool htools = True enableRestrictedCommands :: Bool enableRestrictedCommands = ENABLE_RESTRICTED_COMMANDS enableMond :: Bool enableMond = ENABLE_MOND enableMetad :: Bool enableMetad = ENABLE_METADATA hasGnuLn :: Bool hasGnuLn = HAS_GNU_LN -- Write dictionary with man page name as the key and the section -- number as the value manPages :: [(String, Int)] manPages = MAN_PAGES[] pyAfInet4 :: Int pyAfInet4 = AF_INET4 pyAfInet6 :: Int pyAfInet6 = AF_INET6 ganeti-3.1.0~rc2/src/Ganeti/000075500000000000000000000000001476477700300156025ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/BasicTypes.hs000064400000000000000000000414061476477700300202110ustar00rootroot00000000000000{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE DeriveFunctor #-} {-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE CPP #-} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.BasicTypes ( GenericResult(..) , genericResult , Result , ResultT(..) , mkResultT , withError , withErrorT , toError , toErrorBase , toErrorStr , tryError , Error(..) , MonadIO(..) -- re-export from Control.Monad.IO.Class , isOk , isBad , justOk , justBad , eitherToResult , isLeft , isRight , annotateResult , annotateError , failError , catchErrorT , handleErrorT , orElse , iterateOk , select , runListHead , LookupResult(..) , MatchPriority(..) , lookupName , goodLookupResult , goodMatchPriority , prefixMatch , compareNameComponent , ListSet(..) , emptyListSet ) where import System.IO.Error (tryIOError) import Control.Applicative import Control.Exception (IOException) import Control.Monad import Control.Monad.Fail (MonadFail) import Control.Monad.Base import Control.Monad.Except (MonadError, throwError, catchError) import Control.Monad.Trans import Control.Monad.Trans.Control import Data.Function import Data.List import Data.Maybe import Data.Set (Set) import qualified Data.Set as Set (empty) import Text.JSON (JSON) import qualified Text.JSON as JSON (readJSON, showJSON) import qualified Control.Monad.Fail as Fail -- Remove after we require >= 1.8.58 -- See: https://github.com/ndmitchell/hlint/issues/24 {-# ANN module "HLint: ignore Unused LANGUAGE pragma" #-} -- | Generic monad for our error handling mechanisms. data GenericResult a b = Bad a | Ok b deriving (Show, Eq) -- | Sum type structure of GenericResult. genericResult :: (a -> c) -> (b -> c) -> GenericResult a b -> c genericResult f _ (Bad a) = f a genericResult _ g (Ok b) = g b {-# INLINE genericResult #-} -- | Type alias for a string Result. type Result = GenericResult String class Error e where strMsg :: String -> e instance (a ~ Char) => Error [a] where strMsg = id instance Error IOException where strMsg = userError -- | 'Monad' instance for 'GenericResult'. instance (Error a) => Monad (GenericResult a) where (>>=) (Bad x) _ = Bad x (>>=) (Ok x) fn = fn x return = pure #if !MIN_VERSION_base(4,13,0) fail = Bad . strMsg #endif instance Functor (GenericResult a) where fmap _ (Bad msg) = Bad msg fmap fn (Ok val) = Ok (fn val) instance (Error a, Monoid a) => Alternative (GenericResult a) where empty = Bad $ strMsg "zero Result when used as empty" -- for mplus, when we 'add' two Bad values, we concatenate their -- error descriptions (Bad x) <|> (Bad y) = Bad (x `mappend` strMsg "; " `mappend` y) (Bad _) <|> x = x x@(Ok _) <|> _ = x instance (Error a, Monoid a) => MonadPlus (GenericResult a) where mzero = empty mplus = (<|>) instance (Error a) => MonadError a (GenericResult a) where throwError = Bad {-# INLINE throwError #-} catchError x h = genericResult h (const x) x {-# INLINE catchError #-} instance Applicative (GenericResult a) where pure = Ok (Bad f) <*> _ = Bad f _ <*> (Bad x) = Bad x (Ok f) <*> (Ok x) = Ok $ f x instance (Error a) => Fail.MonadFail (GenericResult a) where fail = Bad . strMsg -- | This is a monad transformation for Result. It's implementation is -- based on the implementations of MaybeT and ErrorT. -- -- 'ResultT' is very similar to @ErrorT@, but with one subtle difference: -- If 'mplus' combines two failing operations, errors of both of them -- are combined. newtype ResultT a m b = ResultT {runResultT :: m (GenericResult a b)} deriving (Functor) -- | Eliminates a 'ResultT' value given appropriate continuations elimResultT :: (Monad m) => (a -> ResultT a' m b') -> (b -> ResultT a' m b') -> ResultT a m b -> ResultT a' m b' elimResultT l r = ResultT . (runResultT . result <=< runResultT) where result (Ok x) = r x result (Bad e) = l e {-# INLINE elimResultT #-} instance (Applicative m, Monad m, Error a) => Applicative (ResultT a m) where pure = lift . pure (<*>) = ap instance (Monad m, Error a) => Monad (ResultT a m) where return = pure (>>=) = flip (elimResultT throwError) #if !MIN_VERSION_base(4,13,0) fail err = ResultT (return . Bad $ strMsg err) #endif instance (Monad m, Error a)=> MonadFail (ResultT a m) where fail err = ResultT (return . Bad $ strMsg err) instance (Monad m, Error a) => MonadError a (ResultT a m) where throwError = ResultT . return . Bad catchError = catchErrorT instance (Error a) => MonadTrans (ResultT a) where lift = ResultT . liftM Ok -- | The instance catches any 'IOError' using 'tryIOError' -- and converts it into an error message using 'strMsg'. -- -- This way, monadic code within 'ResultT' that uses solely 'liftIO' to -- include 'IO' actions ensures that all IO exceptions are handled. -- -- Other exceptions (see instances of 'Exception') are not currently handled. -- This might be revised in the future. instance (MonadIO m, Error a) => MonadIO (ResultT a m) where liftIO = ResultT . liftIO . liftM (either (failError . show) return) . tryIOError instance (MonadBase IO m, Error a) => MonadBase IO (ResultT a m) where liftBase = ResultT . liftBase . liftM (either (failError . show) return) . tryIOError instance (Error a) => MonadTransControl (ResultT a) where #if MIN_VERSION_monad_control(1,0,0) -- Needs Undecidable instances type StT (ResultT a) b = GenericResult a b liftWith f = ResultT . liftM return $ f runResultT restoreT = ResultT #else newtype StT (ResultT a) b = StResultT { runStResultT :: GenericResult a b } liftWith f = ResultT . liftM return $ f (liftM StResultT . runResultT) restoreT = ResultT . liftM runStResultT #endif {-# INLINE liftWith #-} {-# INLINE restoreT #-} instance (Error a, MonadBaseControl IO m) => MonadBaseControl IO (ResultT a m) where #if MIN_VERSION_monad_control(1,0,0) -- Needs Undecidable instances type StM (ResultT a m) b = ComposeSt (ResultT a) m b liftBaseWith = defaultLiftBaseWith restoreM = defaultRestoreM #else newtype StM (ResultT a m) b = StMResultT { runStMResultT :: ComposeSt (ResultT a) m b } liftBaseWith = defaultLiftBaseWith StMResultT restoreM = defaultRestoreM runStMResultT #endif {-# INLINE liftBaseWith #-} {-# INLINE restoreM #-} instance (Monad m, Error a, Monoid a) => Alternative (ResultT a m) where empty = ResultT $ return mzero -- Ensure that 'y' isn't run if 'x' contains a value. This makes it a bit -- more complicated than 'mplus' of 'GenericResult'. x <|> y = elimResultT combine return x where combine x' = ResultT $ liftM (mplus (Bad x')) (runResultT y) instance (Monad m, Error a, Monoid a) => MonadPlus (ResultT a m) where mzero = empty mplus = (<|>) -- | Changes the error message of a result value, if present. -- Note that since 'GenericResult' is also a 'MonadError', this function -- is a generalization of -- @(Error e') => (e' -> e) -> GenericResult e' a -> GenericResult e a@ withError :: (MonadError e m) => (e' -> e) -> GenericResult e' a -> m a withError f = genericResult (throwError . f) return -- | Changes the error message of a @ResultT@ value, if present. withErrorT :: (Monad m, Error e) => (e' -> e) -> ResultT e' m a -> ResultT e m a withErrorT f = ResultT . liftM (withError f) . runResultT -- | Lift a 'Result' value to any 'MonadError'. Since 'ResultT' is itself its -- instance, it's a generalization of -- @Monad m => GenericResult a b -> ResultT a m b@. toError :: (MonadError e m) => GenericResult e a -> m a toError = genericResult throwError return {-# INLINE toError #-} -- | Lift a 'ResultT' value into any 'MonadError' with the same base monad. toErrorBase :: (MonadBase b m, MonadError e m) => ResultT e b a -> m a toErrorBase = (toError =<<) . liftBase . runResultT {-# INLINE toErrorBase #-} -- | An alias for @withError strMsg@, which is often used to lift a pure error -- to a monad stack. See also 'annotateResult'. toErrorStr :: (MonadError e m, Error e) => Result a -> m a toErrorStr = withError strMsg -- | Run a given computation and if an error occurs, return it as `Left` of -- `Either`. -- This is a generalized version of 'try'. tryError :: (MonadError e m) => m a -> m (Either e a) tryError = flip catchError (return . Left) . liftM Right {-# INLINE tryError #-} -- | Converts a monadic result with a 'String' message into -- a 'ResultT' with an arbitrary 'Error'. -- -- Expects that the given action has already taken care of any possible -- errors. In particular, if applied on @IO (Result a)@, any exceptions -- should be handled by the given action. -- -- See also 'toErrorStr'. mkResultT :: (Monad m, Error e) => m (Result a) -> ResultT e m a mkResultT = ResultT . liftM toErrorStr -- | Simple checker for whether a 'GenericResult' is OK. isOk :: GenericResult a b -> Bool isOk (Ok _) = True isOk _ = False -- | Simple checker for whether a 'GenericResult' is a failure. isBad :: GenericResult a b -> Bool isBad = not . isOk -- | Simple filter returning only OK values of GenericResult justOk :: [GenericResult a b] -> [b] justOk = mapMaybe (genericResult (const Nothing) Just) -- | Simple filter returning only Bad values of GenericResult justBad :: [GenericResult a b] -> [a] justBad = mapMaybe (genericResult Just (const Nothing)) -- | Converter from Either to 'GenericResult'. eitherToResult :: Either a b -> GenericResult a b eitherToResult (Left s) = Bad s eitherToResult (Right v) = Ok v -- | Check if an either is Left. Equivalent to isLeft from Data.Either -- version 4.7.0.0 or higher. isLeft :: Either a b -> Bool isLeft (Left _) = True isLeft _ = False -- | Check if an either is Right. Equivalent to isRight from Data.Either -- version 4.7.0.0 or higher. isRight :: Either a b -> Bool isRight = not . isLeft -- | Annotate an error with an ownership information, lifting it to a -- 'MonadError'. Since 'Result' is an instance of 'MonadError' itself, -- it's a generalization of type @String -> Result a -> Result a@. -- See also 'toErrorStr'. annotateResult :: (MonadError e m, Error e) => String -> Result a -> m a annotateResult owner = toErrorStr . annotateError owner -- | Annotate an error with an ownership information inside a 'MonadError'. -- See also 'annotateResult'. annotateError :: (MonadError e m, Error e, Monoid e) => String -> m a -> m a annotateError owner = flip catchError (throwError . mappend (strMsg $ owner ++ ": ")) {-# INLINE annotateError #-} -- | Throws a 'String' message as an error in a 'MonadError'. -- This is a generalization of 'Bad'. -- It's similar to 'fail', but works within a 'MonadError', avoiding the -- unsafe nature of 'fail'. failError :: (MonadError e m, Error e) => String -> m a failError = throwError . strMsg -- | A synonym for @flip@ 'catchErrorT'. handleErrorT :: (Monad m, Error e) => (e' -> ResultT e m a) -> ResultT e' m a -> ResultT e m a handleErrorT handler = elimResultT handler return {-# INLINE handleErrorT #-} -- | Catches an error in a @ResultT@ value. This is similar to 'catchError', -- but in addition allows to change the error type. catchErrorT :: (Monad m, Error e) => ResultT e' m a -> (e' -> ResultT e m a) -> ResultT e m a catchErrorT = flip handleErrorT {-# INLINE catchErrorT #-} -- | If the first computation fails, run the second one. -- Unlike 'mplus' instance for 'ResultT', this doesn't require -- the 'Monoid' constrait. orElse :: (MonadError e m) => m a -> m a -> m a orElse x y = catchError x (const y) -- | Iterate while Ok. iterateOk :: (a -> GenericResult b a) -> a -> [a] iterateOk f a = genericResult (const []) ((:) a . iterateOk f) (f a) -- * Misc functionality -- | Return the first result with a True condition, or the default otherwise. select :: a -- ^ default result -> [(Bool, a)] -- ^ list of \"condition, result\" -> a -- ^ first result which has a True condition, or default select def = maybe def snd . find fst -- | Apply a function to the first element of a list, return the default -- value, if the list is empty. This is just a convenient combination of -- maybe and listToMaybe. runListHead :: a -> (b -> a) -> [b] -> a runListHead a f = maybe a f . listToMaybe -- * Lookup of partial names functionality -- | The priority of a match in a lookup result. data MatchPriority = ExactMatch | MultipleMatch | PartialMatch | FailMatch deriving (Show, Enum, Eq, Ord) -- | The result of a name lookup in a list. data LookupResult = LookupResult { lrMatchPriority :: MatchPriority -- ^ The result type -- | Matching value (for ExactMatch, PartialMatch), Lookup string otherwise , lrContent :: String } deriving (Show) -- | Lookup results have an absolute preference ordering. instance Eq LookupResult where (==) = (==) `on` lrMatchPriority instance Ord LookupResult where compare = compare `on` lrMatchPriority -- | Check for prefix matches in names. -- Implemented in Ganeti core utils.text.MatchNameComponent -- as the regexp r"^%s(\..*)?$" % re.escape(key) prefixMatch :: String -- ^ Lookup -> String -- ^ Full name -> Bool -- ^ Whether there is a prefix match prefixMatch = isPrefixOf . (++ ".") -- | Is the lookup priority a "good" one? goodMatchPriority :: MatchPriority -> Bool goodMatchPriority ExactMatch = True goodMatchPriority PartialMatch = True goodMatchPriority _ = False -- | Is the lookup result an actual match? goodLookupResult :: LookupResult -> Bool goodLookupResult = goodMatchPriority . lrMatchPriority -- | Compares a canonical name and a lookup string. compareNameComponent :: String -- ^ Canonical (target) name -> String -- ^ Partial (lookup) name -> LookupResult -- ^ Result of the lookup compareNameComponent cnl lkp = select (LookupResult FailMatch lkp) [ (cnl == lkp , LookupResult ExactMatch cnl) , (prefixMatch lkp cnl , LookupResult PartialMatch cnl) ] -- | Lookup a string and choose the best result. chooseLookupResult :: String -- ^ Lookup key -> String -- ^ String to compare to the lookup key -> LookupResult -- ^ Previous result -> LookupResult -- ^ New result chooseLookupResult lkp cstr old = -- default: use class order to pick the minimum result select (min new old) -- special cases: -- short circuit if the new result is an exact match [ (lrMatchPriority new == ExactMatch, new) -- if both are partial matches generate a multiple match , (partial2, LookupResult MultipleMatch lkp) ] where new = compareNameComponent cstr lkp partial2 = all ((PartialMatch==) . lrMatchPriority) [old, new] -- | Find the canonical name for a lookup string in a list of names. lookupName :: [String] -- ^ List of keys -> String -- ^ Lookup string -> LookupResult -- ^ Result of the lookup lookupName l s = foldr (chooseLookupResult s) (LookupResult FailMatch s) l -- | Wrapper for a Haskell 'Set' -- -- This type wraps a 'Set' and it is used in the Haskell to Python -- opcode generation to transform a Haskell 'Set' into a Python 'list' -- without duplicate elements. newtype ListSet a = ListSet { unListSet :: Set a } deriving (Eq, Show, Ord) instance (Ord a, JSON a) => JSON (ListSet a) where showJSON = JSON.showJSON . unListSet readJSON = liftM ListSet . JSON.readJSON emptyListSet :: ListSet a emptyListSet = ListSet Set.empty ganeti-3.1.0~rc2/src/Ganeti/Codec.hs000064400000000000000000000043431476477700300171570ustar00rootroot00000000000000{-# LANGUAGE CPP, FlexibleContexts #-} {-| Provides interface to the 'zlib' library. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Codec ( compressZlib , decompressZlib ) where import Codec.Compression.Zlib import qualified Codec.Compression.Zlib.Internal as I import Control.Monad.Except import Control.Monad (liftM) import qualified Data.ByteString.Lazy as BL import qualified Data.ByteString.Lazy.Internal as BL -- | Compresses a lazy bytestring. compressZlib :: BL.ByteString -> BL.ByteString compressZlib = compressWith $ defaultCompressParams { compressLevel = CompressionLevel 3 } -- | Decompresses a lazy bytestring, throwing decoding errors using -- 'throwError'. decompressZlib :: (MonadError String m) => BL.ByteString -> m BL.ByteString decompressZlib = I.foldDecompressStreamWithInput (liftM . BL.chunk) return (throwError . (++)"Zlib: " . show) $ I.decompressST I.zlibFormat I.defaultDecompressParams ganeti-3.1.0~rc2/src/Ganeti/Common.hs000064400000000000000000000357401476477700300173770ustar00rootroot00000000000000{-| Base common functionality. This module holds common functionality shared across Ganeti daemons, HTools and any other programs. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Common ( GenericOptType , StandardOptions(..) , OptCompletion(..) , ArgCompletion(..) , PersonalityList , optComplYesNo , oShowHelp , oShowVer , oShowComp , usageHelp , versionInfo , formatCommands , reqWithConversion , parseYesNo , parseOpts , parseOptsInner , parseOptsCmds , genericMainCmds , fillUpList , fillPairFromMaybe , pickPairUnique ) where import Control.Monad (foldM) import Data.Char (toLower) import Data.List (intercalate, stripPrefix, sortBy) import Data.Maybe (fromMaybe) import Data.Ord (comparing) import qualified Data.Version import System.Console.GetOpt import System.Environment import System.Exit import System.Info import System.IO import Text.Printf (printf) import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.Utils (wrap) import qualified Ganeti.Version as Version (version) -- | Parameter type. data OptCompletion = OptComplNone -- ^ No parameter to this option | OptComplFile -- ^ An existing file | OptComplDir -- ^ An existing directory | OptComplHost -- ^ Host name | OptComplInetAddr -- ^ One ipv4\/ipv6 address | OptComplOneNode -- ^ One node | OptComplManyNodes -- ^ Many nodes, comma-sep | OptComplOneInstance -- ^ One instance | OptComplManyInstances -- ^ Many instances, comma-sep | OptComplOneOs -- ^ One OS name | OptComplOneIallocator -- ^ One iallocator | OptComplInstAddNodes -- ^ Either one or two nodes | OptComplOneGroup -- ^ One group | OptComplInteger -- ^ Integer values | OptComplFloat -- ^ Float values | OptComplJobId -- ^ Job Id | OptComplCommand -- ^ Command (executable) | OptComplString -- ^ Arbitrary string | OptComplChoices [String] -- ^ List of string choices | OptComplSuggest [String] -- ^ Suggested choices deriving (Show, Eq) -- | Argument type. This differs from (and wraps) an Option by the -- fact that it can (and usually does) support multiple repetitions of -- the same argument, via a min and max limit. data ArgCompletion = ArgCompletion OptCompletion Int (Maybe Int) deriving (Show, Eq) -- | A personality definition. type Personality a = ( a -> [String] -> IO () -- The main function , IO [GenericOptType a] -- The options , [ArgCompletion] -- The description of args , String -- Description ) -- | Personality lists type, common across all binaries that expose -- multiple personalities. type PersonalityList a = [(String, Personality a)] -- | Yes\/no choices completion. optComplYesNo :: OptCompletion optComplYesNo = OptComplChoices ["yes", "no"] -- | Text serialisation for 'OptCompletion', used on the Python side. complToText :: OptCompletion -> String complToText (OptComplChoices choices) = "choices=" ++ intercalate "," choices complToText (OptComplSuggest choices) = "suggest=" ++ intercalate "," choices complToText compl = let show_compl = show compl stripped = stripPrefix "OptCompl" show_compl in map toLower $ fromMaybe show_compl stripped -- | Text serialisation for 'ArgCompletion'. argComplToText :: ArgCompletion -> String argComplToText (ArgCompletion optc min_cnt max_cnt) = complToText optc ++ " " ++ show min_cnt ++ " " ++ maybe "none" show max_cnt -- | Abbreviation for the option type. type GenericOptType a = (OptDescr (a -> Result a), OptCompletion) -- | Type class for options which support help and version. class StandardOptions a where helpRequested :: a -> Bool verRequested :: a -> Bool compRequested :: a -> Bool requestHelp :: a -> a requestVer :: a -> a requestComp :: a -> a -- | Option to request help output. oShowHelp :: (StandardOptions a) => GenericOptType a oShowHelp = (Option "h" ["help"] (NoArg (Ok . requestHelp)) "show help", OptComplNone) -- | Option to request version information. oShowVer :: (StandardOptions a) => GenericOptType a oShowVer = (Option "V" ["version"] (NoArg (Ok . requestVer)) "show the version of the program", OptComplNone) -- | Option to request completion information oShowComp :: (StandardOptions a) => GenericOptType a oShowComp = (Option "" ["help-completion"] (NoArg (Ok . requestComp) ) "show completion info", OptComplNone) -- | Usage info. usageHelp :: String -> [GenericOptType a] -> String usageHelp progname = usageInfo (printf "%s %s\nUsage: %s [OPTION...]" progname Version.version progname) . map fst -- | Show the program version info. versionInfo :: String -> String versionInfo progname = printf "%s %s\ncompiled with %s %s\nrunning on %s %s\n" progname Version.version compilerName (Data.Version.showVersion compilerVersion) os arch -- | Show completion info. completionInfo :: String -> [GenericOptType a] -> [ArgCompletion] -> String completionInfo _ opts args = unlines $ map (\(Option shorts longs _ _, compinfo) -> let all_opts = map (\c -> ['-', c]) shorts ++ map ("--" ++) longs in intercalate "," all_opts ++ " " ++ complToText compinfo ) opts ++ map argComplToText args -- | Helper for parsing a yes\/no command line flag. parseYesNo :: Bool -- ^ Default value (when we get a @Nothing@) -> Maybe String -- ^ Parameter value -> Result Bool -- ^ Resulting boolean value parseYesNo v Nothing = return v parseYesNo _ (Just "yes") = return True parseYesNo _ (Just "no") = return False parseYesNo _ (Just s) = fail ("Invalid choice '" ++ s ++ "', pass one of 'yes' or 'no'") -- | Helper function for required arguments which need to be converted -- as opposed to stored just as string. reqWithConversion :: (String -> Result a) -> (a -> b -> Result b) -> String -> ArgDescr (b -> Result b) reqWithConversion conversion_fn updater_fn = ReqArg (\string_opt opts -> do parsed_value <- conversion_fn string_opt updater_fn parsed_value opts) -- | Max command length when formatting command list output. maxCmdLen :: Int maxCmdLen = 60 -- | Formats the description of various commands. formatCommands :: (StandardOptions a) => PersonalityList a -> [String] formatCommands personalities = concatMap (\(cmd, (_, _, _, desc)) -> fmtDesc cmd (wrap maxWidth desc) "-" []) $ sortBy (comparing fst) personalities where mlen = min maxCmdLen . maximum $ map (length . fst) personalities maxWidth = 79 - 3 - mlen fmtDesc _ [] _ acc = reverse acc fmtDesc cmd (d : ds) sep acc = fmtDesc "" ds " " (printf " %-*s %s %s" mlen cmd sep d : acc) -- | Formats usage for a multi-personality program. formatCmdUsage :: (StandardOptions a) => String -> PersonalityList a -> String formatCmdUsage prog personalities = let header = [ printf "Usage: %s {command} [options...] [argument...]" prog , printf "%s --help to see details, or man %s" prog prog , "" , "Commands:" ] rows = formatCommands personalities in unlines $ header ++ rows -- | Displays usage for a program and exits. showCmdUsage :: (StandardOptions a) => String -- ^ Program name -> PersonalityList a -- ^ Personality list -> Bool -- ^ Whether the exit code is success or not -> IO b showCmdUsage prog personalities success = do let usage = formatCmdUsage prog personalities putStr usage if success then exitSuccess else exitWith $ ExitFailure C.exitFailure -- | Generates completion information for a multi-command binary. multiCmdCompletion :: (StandardOptions a) => PersonalityList a -> String multiCmdCompletion personalities = argComplToText $ ArgCompletion (OptComplChoices (map fst personalities)) 1 (Just 1) -- | Displays completion information for a multi-command binary and exits. showCmdCompletion :: (StandardOptions a) => PersonalityList a -> IO b showCmdCompletion personalities = putStrLn (multiCmdCompletion personalities) >> exitSuccess -- | Command line parser, using a generic 'Options' structure. parseOpts :: (StandardOptions a) => a -- ^ The default options -> [String] -- ^ The command line arguments -> String -- ^ The program name -> [GenericOptType a] -- ^ The supported command line options -> [ArgCompletion] -- ^ The supported command line arguments -> IO (a, [String]) -- ^ The resulting options and -- leftover arguments parseOpts defaults argv progname options arguments = case parseOptsInner defaults argv progname options arguments of Left (code, msg) -> do hPutStr (if code == ExitSuccess then stdout else stderr) msg exitWith code Right result -> return result -- | Command line parser, for programs with sub-commands. parseOptsCmds :: (StandardOptions a) => a -- ^ The default options -> [String] -- ^ The command line arguments -> String -- ^ The program name -> PersonalityList a -- ^ The supported commands -> [GenericOptType a] -- ^ Generic options -> IO (a, [String], a -> [String] -> IO ()) -- ^ The resulting options and leftover arguments parseOptsCmds defaults argv progname personalities genopts = do let usage = showCmdUsage progname personalities check c = case c of -- hardcoded option strings here! "--version" -> putStrLn (versionInfo progname) >> exitSuccess "--help" -> usage True "--help-completion" -> showCmdCompletion personalities _ -> return c (cmd, cmd_args) <- case argv of cmd:cmd_args -> do cmd' <- check cmd return (cmd', cmd_args) [] -> usage False case cmd `lookup` personalities of Nothing -> usage False Just (mainfn, optdefs, argdefs, _) -> do optdefs' <- optdefs (opts, args) <- parseOpts defaults cmd_args progname (optdefs' ++ genopts) argdefs return (opts, args, mainfn) -- | Inner parse options. The arguments are similar to 'parseOpts', -- but it returns either a 'Left' composed of exit code and message, -- or a 'Right' for the success case. parseOptsInner :: (StandardOptions a) => a -> [String] -> String -> [GenericOptType a] -> [ArgCompletion] -> Either (ExitCode, String) (a, [String]) parseOptsInner defaults argv progname options arguments = case getOpt Permute (map fst options) argv of (opts, args, []) -> case foldM (flip id) defaults opts of Bad msg -> Left (ExitFailure 1, "Error while parsing command line arguments:\n" ++ msg ++ "\n") Ok parsed -> select (Right (parsed, args)) [ (helpRequested parsed, Left (ExitSuccess, usageHelp progname options)) , (verRequested parsed, Left (ExitSuccess, versionInfo progname)) , (compRequested parsed, Left (ExitSuccess, completionInfo progname options arguments)) ] (_, _, errs) -> Left (ExitFailure 2, "Command line error: " ++ concat errs ++ "\n" ++ usageHelp progname options) -- | Parse command line options and execute the main function of a -- multi-personality binary. genericMainCmds :: (StandardOptions a) => a -> PersonalityList a -> [GenericOptType a] -> IO () genericMainCmds defaults personalities genopts = do cmd_args <- getArgs prog <- getProgName (opts, args, fn) <- parseOptsCmds defaults cmd_args prog personalities genopts fn opts args -- | Order a list of pairs in the order of the given list and fill up -- the list for elements that don't have a matching pair fillUpList :: ([(a, b)] -> a -> (a, b)) -> [a] -> [(a, b)] -> [(a, b)] fillUpList fill_fn inputs pairs = map (fill_fn pairs) inputs -- | Fill up a pair with fillup element if no matching pair is present fillPairFromMaybe :: (a -> (a, b)) -> (a -> [(a, b)] -> Maybe (a, b)) -> [(a, b)] -> a -> (a, b) fillPairFromMaybe fill_fn pick_fn pairs element = fromMaybe (fill_fn element) (pick_fn element pairs) -- | Check if the given element matches the given pair isMatchingPair :: (Eq a) => a -> (a, b) -> Bool isMatchingPair element (pair_element, _) = element == pair_element -- | Pick a specific element's pair from the list pickPairUnique :: (Eq a) => a -> [(a, b)] -> Maybe (a, b) pickPairUnique element pairs = let res = filter (isMatchingPair element) pairs in case res of [x] -> Just x -- if we have more than one result, we should get suspcious _ -> Nothing ganeti-3.1.0~rc2/src/Ganeti/Compat.hs000064400000000000000000000074601476477700300173700ustar00rootroot00000000000000{-# LANGUAGE CPP #-} {- | Compatibility helper module. This module holds definitions that help with supporting multiple library versions or transitions between versions. -} {- Copyright (C) 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Compat ( filePath' , maybeFilePath' , toInotifyPath , getPid' , openFd , Posix.closeFd ) where import qualified Data.ByteString.UTF8 as UTF8 import System.FilePath (FilePath) import System.Posix.ByteString.FilePath (RawFilePath) import qualified System.Posix as Posix import qualified System.INotify import qualified Text.JSON import qualified Control.Monad.Fail as Fail import System.Process.Internals import System.Posix.Types (CPid (..)) #if MIN_VERSION_process(1,6,3) import System.Process (getPid) #else import Control.Concurrent.Lifted (readMVar) #endif -- | Wrappers converting ByteString filepaths to Strings and vice versa -- -- hinotify 0.3.10 switched to using RawFilePaths instead of FilePaths, the -- former being Data.ByteString and the latter String. #if MIN_VERSION_hinotify(0,3,10) filePath' :: System.INotify.Event -> FilePath filePath' = UTF8.toString . System.INotify.filePath maybeFilePath' :: System.INotify.Event -> Maybe FilePath maybeFilePath' ev = UTF8.toString <$> System.INotify.maybeFilePath ev toInotifyPath :: FilePath -> RawFilePath toInotifyPath = UTF8.fromString #else filePath' :: System.INotify.Event -> FilePath filePath' = System.INotify.filePath maybeFilePath' :: System.INotify.Event -> Maybe FilePath maybeFilePath' = System.INotify.maybeFilePath toInotifyPath :: FilePath -> FilePath toInotifyPath = id #endif #if !MIN_VERSION_json(0,10,0) -- | MonadFail.Fail instance definitions for JSON results -- -- Required as of GHC 8.6 because MonadFailDesugaring is on by -- default: -- . Added -- upstream in version 0.10. instance Fail.MonadFail Text.JSON.Result where fail = Fail.fail #endif -- | Process 1.6.3. introduced the getPid function, for older versions -- provide an implemention here (https://github.com/haskell/process/pull/109) type Pid = CPid getPid' :: ProcessHandle -> IO (Maybe Pid) #if MIN_VERSION_process(1,6,3) getPid' = getPid #else getPid' (ProcessHandle mh _) = do p_ <- readMVar mh case p_ of OpenHandle pid -> return $ Just pid _ -> return Nothing #endif openFd :: FilePath -> Posix.OpenMode -> Maybe Posix.FileMode -> Posix.OpenFileFlags -> IO Posix.Fd #if MIN_VERSION_unix(2,8,0) openFd path openMode mFileMode fileFlags = Posix.openFd path openMode (fileFlags {Posix.creat = mFileMode}) #else openFd = Posix.openFd #endif ganeti-3.1.0~rc2/src/Ganeti/Confd/000075500000000000000000000000001476477700300166335ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Confd/Client.hs000064400000000000000000000134531476477700300204130ustar00rootroot00000000000000{-| Implementation of the Ganeti Confd client functionality. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Confd.Client ( getConfdClient , query ) where import Control.Concurrent import Control.Exception (bracket) import Control.Monad import qualified Data.ByteString.Char8 as Char8 import Data.List import Data.Maybe import qualified Network.Socket as S import Network.Socket.ByteString (sendTo, recv) import System.Posix.Time import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Confd.Types import Ganeti.Confd.Utils import qualified Ganeti.Constants as C import Ganeti.Hash import Ganeti.Ssconf import Ganeti.Utils -- | Builds a properly initialized ConfdClient. -- The parameters (an IP address and the port number for the Confd client -- to connect to) are mainly meant for testing purposes. If they are not -- provided, the list of master candidates and the default port number will -- be used. getConfdClient :: Maybe String -> Maybe Int -> IO ConfdClient getConfdClient addr portNum = S.withSocketsDo $ do hmac <- getClusterHmac candList <- getMasterCandidatesIps Nothing peerList <- case candList of (Ok p) -> return p (Bad msg) -> fail msg let addrList = maybe peerList (:[]) addr port = fromMaybe C.defaultConfdPort portNum return . ConfdClient hmac addrList $ fromIntegral port -- | Sends a query to all the Confd servers the client is connected to. -- Returns the most up-to-date result according to the serial number, -- chosen between those received before the timeout. query :: ConfdClient -> ConfdRequestType -> ConfdQuery -> IO (Maybe ConfdReply) query client crType cQuery = do semaphore <- newMVar () answer <- newMVar Nothing let dest = [(host, serverPort client) | host <- peers client] hmac = hmacKey client jobs = map (queryOneServer semaphore answer crType cQuery hmac) dest watchdog reqAnswers = do threadDelay $ 1000000 * C.confdClientExpireTimeout _ <- swapMVar reqAnswers 0 putMVar semaphore () waitForResult reqAnswers = do _ <- takeMVar semaphore l <- takeMVar reqAnswers unless (l == 0) $ do putMVar reqAnswers $ l - 1 waitForResult reqAnswers reqAnswers <- newMVar . min C.confdDefaultReqCoverage $ length dest workers <- mapM forkIO jobs watcher <- forkIO $ watchdog reqAnswers waitForResult reqAnswers mapM_ killThread $ watcher:workers takeMVar answer -- | Updates the reply to the query. As per the Confd design document, -- only the reply with the highest serial number is kept. updateConfdReply :: ConfdReply -> Maybe ConfdReply -> Maybe ConfdReply updateConfdReply newValue Nothing = Just newValue updateConfdReply newValue (Just currentValue) = Just $ if confdReplyStatus newValue == ReplyStatusOk && (confdReplyStatus currentValue /= ReplyStatusOk || confdReplySerial newValue > confdReplySerial currentValue) then newValue else currentValue -- | Send a query to a single server, waits for the result and stores it -- in a shared variable. Then, sends a signal on another shared variable -- acting as a semaphore. -- This function is meant to be used as one of multiple threads querying -- multiple servers in parallel. queryOneServer :: MVar () -- ^ The semaphore that will be signalled -> MVar (Maybe ConfdReply) -- ^ The shared variable for the result -> ConfdRequestType -- ^ The type of the query to be sent -> ConfdQuery -- ^ The content of the query -> HashKey -- ^ The hmac key to sign the message -> (String, S.PortNumber) -- ^ The address and port of the server -> IO () queryOneServer semaphore answer crType cQuery hmac (host, port) = do request <- newConfdRequest crType cQuery timestamp <- fmap show epochTime let signedMsg = signMessage hmac timestamp (J.encodeStrict request) completeMsg = C.confdMagicFourcc ++ J.encodeStrict signedMsg addr <- resolveAddr (fromIntegral port) host (af_family, sockaddr) <- exitIfBad "Unable to resolve the IP address" addr replyMsg <- bracket (S.socket af_family S.Datagram S.defaultProtocol) S.close $ \s -> do _ <- sendTo s (Char8.pack completeMsg) sockaddr Char8.unpack <$> recv s C.maxUdpDataSize parsedReply <- if C.confdMagicFourcc `isPrefixOf` replyMsg then return . parseReply hmac (drop 4 replyMsg) $ confdRqRsalt request else fail "Invalid magic code!" reply <- case parsedReply of Ok (_, r) -> return r Bad msg -> fail msg modifyMVar_ answer $! return . updateConfdReply reply putMVar semaphore () ganeti-3.1.0~rc2/src/Ganeti/Confd/ClientFunctions.hs000064400000000000000000000067641476477700300223130ustar00rootroot00000000000000{-| Some utility functions, based on the Confd client, providing data in a ready-to-use way. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Confd.ClientFunctions ( getInstances , getInstanceDisks ) where import Control.Monad (liftM) import qualified Text.JSON as J import Ganeti.BasicTypes as BT import Ganeti.Confd.Types import Ganeti.Confd.Client import Ganeti.Objects -- | Get the list of instances the given node is ([primary], [secondary]) for. -- The server address and the server port parameters are mainly intended -- for testing purposes. If they are Nothing, the default values will be used. getInstances :: String -> Maybe String -> Maybe Int -> BT.ResultT String IO ([Ganeti.Objects.Instance], [Ganeti.Objects.Instance]) getInstances node srvAddr srvPort = do client <- liftIO $ getConfdClient srvAddr srvPort reply <- liftIO . query client ReqNodeInstances $ PlainQuery node case fmap (J.readJSON . confdReplyAnswer) reply of Just (J.Ok instances) -> return instances Just (J.Error msg) -> fail msg Nothing -> fail "No answer from the Confd server" -- | Get the list of disks that belong to a given instance -- The server address and the server port parameters are mainly intended -- for testing purposes. If they are Nothing, the default values will be used. getDisks :: Ganeti.Objects.Instance -> Maybe String -> Maybe Int -> BT.ResultT String IO [Ganeti.Objects.Disk] getDisks inst srvAddr srvPort = do client <- liftIO $ getConfdClient srvAddr srvPort reply <- liftIO . query client ReqInstanceDisks . PlainQuery . uuidOf $ inst case fmap (J.readJSON . confdReplyAnswer) reply of Just (J.Ok disks) -> return disks Just (J.Error msg) -> fail msg Nothing -> fail "No answer from the Confd server" -- | Get the list of instances on the given node along with their disks -- The server address and the server port parameters are mainly intended -- for testing purposes. If they are Nothing, the default values will be used. getInstanceDisks :: String -> Maybe String -> Maybe Int -> BT.ResultT String IO [(Ganeti.Objects.Instance, [Ganeti.Objects.Disk])] getInstanceDisks node srvAddr srvPort = liftM (uncurry (++)) (getInstances node srvAddr srvPort) >>= mapM (\i -> liftM ((,) i) (getDisks i srvAddr srvPort)) ganeti-3.1.0~rc2/src/Ganeti/Confd/Server.hs000064400000000000000000000352701476477700300204440ustar00rootroot00000000000000{-# LANGUAGE TupleSections #-} {-| Implementation of the Ganeti confd server functionality. -} {- Copyright (C) 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Confd.Server ( main , checkMain , prepMain ) where import Control.Concurrent import Control.Monad (forever, liftM) import Data.IORef import Data.List import qualified Data.ByteString.Char8 as Char8 import qualified Data.Map as M import Data.Maybe (fromMaybe) import Network.BSD (getServicePortNumber) import qualified Network.Socket as S import Network.Socket.ByteString (recvFrom, sendTo) import System.Exit import System.IO import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Errors import Ganeti.Daemon import Ganeti.JSON (containerFromList, fromContainer) import Ganeti.Objects import Ganeti.Confd.Types import Ganeti.Confd.Utils import Ganeti.Config import Ganeti.ConfigReader import Ganeti.Hash import Ganeti.Logging import qualified Ganeti.Constants as C import qualified Ganeti.Query.Cluster as QCluster import qualified Ganeti.Utils.Time as Time import Ganeti.Utils import Ganeti.DataCollectors.Types (DataCollector(..)) import Ganeti.DataCollectors (collectors) -- * Types and constants definitions -- | What we store as configuration. type CRef = IORef (Result (ConfigData, LinkIpMap)) -- | A small type alias for readability. type StatusAnswer = (ConfdReplyStatus, J.JSValue, Int) -- | Unknown entry standard response. queryUnknownEntry :: StatusAnswer queryUnknownEntry = (ReplyStatusError, J.showJSON ConfdErrorUnknownEntry, 0) {- not used yet -- | Internal error standard response. queryInternalError :: StatusAnswer queryInternalError = (ReplyStatusError, J.showJSON ConfdErrorInternal) -} -- | Argument error standard response. queryArgumentError :: StatusAnswer queryArgumentError = (ReplyStatusError, J.showJSON ConfdErrorArgument, 0) -- | Converter from specific error to a string format. gntErrorToResult :: ErrorResult a -> Result a gntErrorToResult (Bad err) = Bad (show err) gntErrorToResult (Ok x) = Ok x -- * Confd base functionality -- | Computes the node role nodeRole :: ConfigData -> String -> Result ConfdNodeRole nodeRole cfg name = do cmaster <- errToResult $ QCluster.clusterMasterNodeName cfg mnode <- errToResult $ getNode cfg name let nrole = case mnode of node | cmaster == name -> NodeRoleMaster | nodeDrained node -> NodeRoleDrained | nodeOffline node -> NodeRoleOffline | nodeMasterCandidate node -> NodeRoleCandidate _ -> NodeRoleRegular return nrole -- | Does an instance ip -> instance -> primary node -> primary ip -- transformation. getNodePipByInstanceIp :: ConfigData -> LinkIpMap -> String -> String -> StatusAnswer getNodePipByInstanceIp cfg linkipmap link instip = case M.lookup instip (M.findWithDefault M.empty link linkipmap) of Nothing -> queryUnknownEntry Just instname -> case getInstPrimaryNode cfg instname of Bad _ -> queryUnknownEntry -- either instance or node not found Ok node -> (ReplyStatusOk, J.showJSON (nodePrimaryIp node), clusterSerial $ configCluster cfg) -- | Returns a node name for a given UUID uuidToNodeName :: ConfigData -> String -> Result String uuidToNodeName cfg uuid = gntErrorToResult $ nodeName <$> getNode cfg uuid -- | Encodes a list of minors into a JSON representation, converting UUIDs to -- names in the process encodeMinors :: ConfigData -> (String, Int, String, String, String, String) -> Result J.JSValue encodeMinors cfg (node_uuid, a, b, c, d, peer_uuid) = do node_name <- uuidToNodeName cfg node_uuid peer_name <- uuidToNodeName cfg peer_uuid return . J.JSArray $ [J.showJSON node_name, J.showJSON a, J.showJSON b, J.showJSON c, J.showJSON d, J.showJSON peer_name] -- | Builds the response to a given query. buildResponse :: (ConfigData, LinkIpMap) -> ConfdRequest -> Result StatusAnswer buildResponse (cfg, _) (ConfdRequest { confdRqType = ReqPing }) = return (ReplyStatusOk, J.showJSON (configVersion cfg), 0) buildResponse cdata req@(ConfdRequest { confdRqType = ReqClusterMaster }) = case confdRqQuery req of EmptyQuery -> liftM ((ReplyStatusOk,,serial) . J.showJSON) master_name PlainQuery _ -> return queryArgumentError DictQuery reqq -> do mnode <- gntErrorToResult $ getNode cfg master_uuid mname <- master_name let fvals = map (\field -> case field of ReqFieldName -> mname ReqFieldIp -> clusterMasterIp cluster ReqFieldMNodePip -> nodePrimaryIp mnode ) (confdReqQFields reqq) return (ReplyStatusOk, J.showJSON fvals, serial) where master_uuid = clusterMasterNode cluster master_name = errToResult $ QCluster.clusterMasterNodeName cfg cluster = configCluster cfg cfg = fst cdata serial = clusterSerial $ configCluster cfg buildResponse cdata req@(ConfdRequest { confdRqType = ReqNodeRoleByName }) = do node_name <- case confdRqQuery req of PlainQuery str -> return str _ -> fail $ "Invalid query type " ++ show (confdRqQuery req) nrole <- nodeRole (fst cdata) node_name return (ReplyStatusOk, J.showJSON nrole, clusterSerial . configCluster $ fst cdata) buildResponse cdata (ConfdRequest { confdRqType = ReqNodePipList }) = -- note: we use foldlWithKey because that's present across more -- versions of the library return (ReplyStatusOk, J.showJSON $ M.foldlWithKey (\accu _ n -> nodePrimaryIp n:accu) [] (fromContainer . configNodes . fst $ cdata), clusterSerial . configCluster $ fst cdata) buildResponse cdata (ConfdRequest { confdRqType = ReqMcPipList }) = -- note: we use foldlWithKey because that's present across more -- versions of the library return (ReplyStatusOk, J.showJSON $ M.foldlWithKey (\accu _ n -> if nodeMasterCandidate n then nodePrimaryIp n:accu else accu) [] (fromContainer . configNodes . fst $ cdata), clusterSerial . configCluster $ fst cdata) buildResponse (cfg, linkipmap) req@(ConfdRequest { confdRqType = ReqInstIpsList }) = do link <- case confdRqQuery req of PlainQuery str -> return str EmptyQuery -> return (getDefaultNicLink cfg) _ -> fail "Invalid query type" return (ReplyStatusOk, J.showJSON $ getInstancesIpByLink linkipmap link, clusterSerial $ configCluster cfg) buildResponse cdata (ConfdRequest { confdRqType = ReqNodePipByInstPip , confdRqQuery = DictQuery query}) = let (cfg, linkipmap) = cdata link = fromMaybe (getDefaultNicLink cfg) (confdReqQLink query) in case confdReqQIp query of Just ip -> return $ getNodePipByInstanceIp cfg linkipmap link ip Nothing -> return (ReplyStatusOk, J.showJSON $ map (getNodePipByInstanceIp cfg linkipmap link) (confdReqQIpList query), clusterSerial . configCluster $ fst cdata) buildResponse _ (ConfdRequest { confdRqType = ReqNodePipByInstPip }) = return queryArgumentError buildResponse cdata req@(ConfdRequest { confdRqType = ReqNodeDrbd }) = do let cfg = fst cdata node_name <- case confdRqQuery req of PlainQuery str -> return str _ -> fail $ "Invalid query type " ++ show (confdRqQuery req) node <- gntErrorToResult $ getNode cfg node_name let minors = concatMap (getInstMinorsForNode cfg (uuidOf node)) . M.elems . fromContainer . configInstances $ cfg encoded <- mapM (encodeMinors cfg) minors return (ReplyStatusOk, J.showJSON encoded, nodeSerial node) -- | Return the list of instances for a node (as ([primary], [secondary])) given -- the node name. buildResponse cdata req@(ConfdRequest { confdRqType = ReqNodeInstances }) = do let cfg = fst cdata node_name <- case confdRqQuery req of PlainQuery str -> return str _ -> fail $ "Invalid query type " ++ show (confdRqQuery req) node <- case getNode cfg node_name of Ok n -> return n Bad e -> fail $ "Node not found in the configuration: " ++ show e let node_uuid = uuidOf node instances = getNodeInstances cfg node_uuid return (ReplyStatusOk, J.showJSON instances, nodeSerial node) -- | Return the list of disks for an instance given the instance uuid. buildResponse cdata req@(ConfdRequest { confdRqType = ReqInstanceDisks }) = do let cfg = fst cdata inst_name <- case confdRqQuery req of PlainQuery str -> return str _ -> fail $ "Invalid query type " ++ show (confdRqQuery req) inst <- case getInstance cfg inst_name of Ok i -> return i Bad e -> fail $ "Instance not found in the configuration: " ++ show e case getInstDisks cfg . uuidOf $ inst of Ok disks -> return (ReplyStatusOk, J.showJSON disks, instSerial inst) Bad e -> fail $ "Could not retrieve disks: " ++ show e -- | Return arbitrary configuration value given by a path. buildResponse cdata req@(ConfdRequest { confdRqType = ReqConfigQuery , confdRqQuery = pathQ }) = do let cfg = fst cdata path <- case pathQ of PlainQuery path -> return path _ -> fail $ "Invalid query type " ++ show (confdRqQuery req) let configValue = extractJSONPath path cfg case configValue of J.Ok jsvalue -> return (ReplyStatusOk, jsvalue, clusterSerial $ configCluster cfg) J.Error _ -> return queryArgumentError -- | Return activation state of data collectors buildResponse (cdata,_) (ConfdRequest { confdRqType = ReqDataCollectors }) = do let mkConfig col = (dName col, DataCollectorConfig (dActive col (dName col) cdata) (dInterval col (dName col) cdata)) datacollectors = containerFromList $ map mkConfig collectors return (ReplyStatusOk, J.showJSON datacollectors, clusterSerial . configCluster $ cdata) -- | Creates a ConfdReply from a given answer. serializeResponse :: Result StatusAnswer -> ConfdReply serializeResponse r = let (status, result, serial) = case r of Bad err -> (ReplyStatusError, J.showJSON err, 0) Ok (code, val, ser) -> (code, val, ser) in ConfdReply { confdReplyProtocol = 1 , confdReplyStatus = status , confdReplyAnswer = result , confdReplySerial = serial } -- ** Client input/output handlers -- | Main loop for a given client. responder :: CRef -> S.Socket -> HashKey -> String -> S.SockAddr -> IO () responder cfgref socket hmac msg peer = do ctime <- Time.getCurrentTime case parseRequest hmac msg ctime of Ok (origmsg, rq) -> do logDebug $ "Processing request: " ++ rStripSpace origmsg mcfg <- readIORef cfgref let response = respondInner mcfg hmac rq _ <- sendTo socket (Char8.pack response) peer logDebug $ "Response sent: " ++ response return () Bad err -> logInfo $ "Failed to parse incoming message: " ++ err return () -- | Inner helper function for a given client. This generates the -- final encoded message (as a string), ready to be sent out to the -- client. respondInner :: Result (ConfigData, LinkIpMap) -> HashKey -> ConfdRequest -> String respondInner cfg hmac rq = let rsalt = confdRqRsalt rq innermsg = serializeResponse (cfg >>= flip buildResponse rq) innerserialised = J.encodeStrict innermsg outermsg = signMessage hmac rsalt innerserialised outerserialised = C.confdMagicFourcc ++ J.encodeStrict outermsg in outerserialised -- | Main listener loop. listener :: S.Socket -> HashKey -> (S.Socket -> HashKey -> String -> S.SockAddr -> IO ()) -> IO () listener s hmac resp = do (msg, peer) <- (\(m, p) -> (Char8.unpack m, p)) <$> recvFrom s 4096 if C.confdMagicFourcc `isPrefixOf` msg then forkIO (resp s hmac (drop 4 msg) peer) >> return () else logDebug "Invalid magic code!" >> return () return () -- | Type alias for prepMain results type PrepResult = (S.Socket, IORef (Result (ConfigData, LinkIpMap))) -- | Check function for confd. checkMain :: CheckFn (S.Family, S.SockAddr) checkMain opts = do defaultPort <- withDefaultOnIOError C.defaultConfdPort . liftM fromIntegral $ getServicePortNumber C.confd parseresult <- parseAddress opts defaultPort case parseresult of Bad msg -> do hPutStrLn stderr $ "parsing bind address: " ++ msg return . Left $ ExitFailure 1 Ok v -> return $ Right v -- | Prepare function for confd. prepMain :: PrepFn (S.Family, S.SockAddr) PrepResult prepMain _ (af_family, bindaddr) = do s <- S.socket af_family S.Datagram S.defaultProtocol S.setSocketOption s S.ReuseAddr 1 S.bind s bindaddr cref <- newIORef (Bad "Configuration not yet loaded") return (s, cref) -- | Main function. main :: MainFn (S.Family, S.SockAddr) PrepResult main _ _ (s, cref) = do let cfg_transform :: Result ConfigData -> Result (ConfigData, LinkIpMap) cfg_transform = liftM (\cfg -> (cfg, buildLinkIpInstnameMap cfg)) initConfigReader (writeIORef cref . cfg_transform) hmac <- getClusterHmac -- enter the responder loop forever $ listener s hmac (responder cref) ganeti-3.1.0~rc2/src/Ganeti/Confd/Types.hs000064400000000000000000000135771476477700300203100ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Implementation of the Ganeti confd types. -} {- Copyright (C) 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Confd.Types ( ConfdClient(..) , ConfdRequestType(..) , confdRequestTypeToRaw , ConfdReqField(..) , confdReqFieldToRaw , ConfdReqQ(..) , ConfdReplyStatus(..) , confdReplyStatusToRaw , ConfdNodeRole(..) , confdNodeRoleToRaw , ConfdErrorType(..) , confdErrorTypeToRaw , ConfdRequest(..) , newConfdRequest , ConfdReply(..) , ConfdQuery(..) , SignedMessage(..) ) where import Text.JSON import qualified Network.Socket as S import qualified Ganeti.ConstantUtils as ConstantUtils import Ganeti.Hash import Ganeti.THH import Ganeti.Utils (newUUID) $(declareILADT "ConfdRequestType" [ ("ReqPing", 0) , ("ReqNodeRoleByName", 1) , ("ReqNodePipByInstPip", 2) , ("ReqClusterMaster", 3) , ("ReqNodePipList", 4) , ("ReqMcPipList", 5) , ("ReqInstIpsList", 6) , ("ReqNodeDrbd", 7) , ("ReqNodeInstances", 8) , ("ReqInstanceDisks", 9) , ("ReqConfigQuery", 10) , ("ReqDataCollectors", 11) ]) $(makeJSONInstance ''ConfdRequestType) $(declareILADT "ConfdReqField" [ ("ReqFieldName", 0) , ("ReqFieldIp", 1) , ("ReqFieldMNodePip", 2) ]) $(makeJSONInstance ''ConfdReqField) -- Confd request query fields. These are used to narrow down queries. -- These must be strings rather than integers, because json-encoding -- converts them to strings anyway, as they're used as dict-keys. $(buildObject "ConfdReqQ" "confdReqQ" [ renameField "Ip" . optionalField $ simpleField ConstantUtils.confdReqqIp [t| String |] , renameField "IpList" . defaultField [| [] |] $ simpleField ConstantUtils.confdReqqIplist [t| [String] |] , renameField "Link" . optionalField $ simpleField ConstantUtils.confdReqqLink [t| String |] , renameField "Fields" . defaultField [| [] |] $ simpleField ConstantUtils.confdReqqFields [t| [ConfdReqField] |] ]) -- | Confd query type. This is complex enough that we can't -- automatically derive it via THH. data ConfdQuery = EmptyQuery | PlainQuery String | DictQuery ConfdReqQ deriving (Show, Eq) instance JSON ConfdQuery where readJSON o = case o of JSNull -> return EmptyQuery JSString s -> return . PlainQuery . fromJSString $ s JSObject _ -> fmap DictQuery (readJSON o::Result ConfdReqQ) _ -> fail $ "Cannot deserialise into ConfdQuery\ \ the value '" ++ show o ++ "'" showJSON cq = case cq of EmptyQuery -> JSNull PlainQuery s -> showJSON s DictQuery drq -> showJSON drq $(declareILADT "ConfdReplyStatus" [ ("ReplyStatusOk", 0) , ("ReplyStatusError", 1) , ("ReplyStatusNotImpl", 2) ]) $(makeJSONInstance ''ConfdReplyStatus) $(declareILADT "ConfdNodeRole" [ ("NodeRoleMaster", 0) , ("NodeRoleCandidate", 1) , ("NodeRoleOffline", 2) , ("NodeRoleDrained", 3) , ("NodeRoleRegular", 4) ]) $(makeJSONInstance ''ConfdNodeRole) -- Note that the next item is not a frozenset in Python, but we make -- it a separate type for safety $(declareILADT "ConfdErrorType" [ ("ConfdErrorUnknownEntry", 0) , ("ConfdErrorInternal", 1) , ("ConfdErrorArgument", 2) ]) $(makeJSONInstance ''ConfdErrorType) $(buildObject "ConfdRequest" "confdRq" [ simpleField "protocol" [t| Int |] , simpleField "type" [t| ConfdRequestType |] , defaultField [| EmptyQuery |] $ simpleField "query" [t| ConfdQuery |] , simpleField "rsalt" [t| String |] ]) -- | Client side helper function for creating requests. It automatically fills -- in some default values. newConfdRequest :: ConfdRequestType -> ConfdQuery -> IO ConfdRequest newConfdRequest reqType query = do rsalt <- newUUID return $ ConfdRequest ConstantUtils.confdProtocolVersion reqType query rsalt $(buildObject "ConfdReply" "confdReply" [ simpleField "protocol" [t| Int |] , simpleField "status" [t| ConfdReplyStatus |] , simpleField "answer" [t| JSValue |] , simpleField "serial" [t| Int |] ]) $(buildObject "SignedMessage" "signedMsg" [ simpleField "hmac" [t| String |] , simpleField "msg" [t| String |] , simpleField "salt" [t| String |] ]) -- | Data type containing information used by the Confd client. data ConfdClient = ConfdClient { hmacKey :: HashKey -- ^ The hmac used for authentication , peers :: [String] -- ^ The list of nodes to query , serverPort :: S.PortNumber -- ^ The port where confd server is listening } ganeti-3.1.0~rc2/src/Ganeti/Confd/Utils.hs000064400000000000000000000136741476477700300203020ustar00rootroot00000000000000{-| Implementation of the Ganeti confd utilities. This holds a few utility functions that could be useful in both clients and servers. -} {- Copyright (C) 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Confd.Utils ( getClusterHmac , parseSignedMessage , parseRequest , parseReply , signMessage , Time.getCurrentTime , extractJSONPath ) where import qualified Data.Attoparsec.Text as P import qualified Data.ByteString as B import Data.Text (pack) import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Confd.Types import Ganeti.Hash import qualified Ganeti.Constants as C import qualified Ganeti.Path as Path import qualified Ganeti.Utils.Time as Time import Ganeti.JSON (fromJResult) import Ganeti.Utils -- | Type-adjusted max clock skew constant. maxClockSkew :: Integer maxClockSkew = fromIntegral C.confdMaxClockSkew -- | Returns the HMAC key. getClusterHmac :: IO HashKey getClusterHmac = Path.confdHmacKey >>= fmap B.unpack . B.readFile -- | Parses a signed message. parseSignedMessage :: (J.JSON a) => HashKey -> String -> Result (String, String, a) parseSignedMessage key str = do (SignedMessage hmac msg salt) <- fromJResult "parsing signed message" $ J.decode str parsedMsg <- if verifyMac key (Just salt) msg hmac then fromJResult "parsing message" $ J.decode msg else Bad "HMAC verification failed" return (salt, msg, parsedMsg) -- | Message parsing. This can either result in a good, valid request -- message, or fail in the Result monad. parseRequest :: HashKey -> String -> Integer -> Result (String, ConfdRequest) parseRequest hmac msg curtime = do (salt, origmsg, request) <- parseSignedMessage hmac msg ts <- tryRead "Parsing timestamp" salt::Result Integer if abs (ts - curtime) > maxClockSkew then fail "Too old/too new timestamp or clock skew" else return (origmsg, request) -- | Message parsing. This can either result in a good, valid reply -- message, or fail in the Result monad. -- It also checks that the salt in the message corresponds to the one -- that is expected parseReply :: HashKey -> String -> String -> Result (String, ConfdReply) parseReply hmac msg expSalt = do (salt, origmsg, reply) <- parseSignedMessage hmac msg if salt /= expSalt then fail "The received salt differs from the expected salt" else return (origmsg, reply) -- | Signs a message with a given key and salt. signMessage :: HashKey -> String -> String -> SignedMessage signMessage key salt msg = SignedMessage { signedMsgMsg = msg , signedMsgSalt = salt , signedMsgHmac = hmac } where hmac = computeMac key (Just salt) msg data Pointer = Pointer [String] deriving (Show, Eq) -- | Parse a fixed size Int. readInteger :: String -> J.Result Int readInteger = either J.Error J.Ok . P.parseOnly P.decimal . pack -- | Parse a path for a JSON structure. pointerFromString :: String -> J.Result Pointer pointerFromString s = either J.Error J.Ok . P.parseOnly parser $ pack s where parser = do _ <- P.char '/' tokens <- token `P.sepBy1` P.char '/' return $ Pointer tokens token = P.choice [P.many1 (P.choice [ escaped , P.satisfy $ P.notInClass "~/"]) , P.endOfInput *> return ""] escaped = P.choice [escapedSlash, escapedTilde] escapedSlash = P.string (pack "~1") *> return '/' escapedTilde = P.string (pack "~0") *> return '~' -- | Use a Pointer to access any value nested in a JSON object. extractValue :: J.JSON a => Pointer -> a -> J.Result J.JSValue extractValue (Pointer l) json = getJSValue l $ J.showJSON json where indexWithString x (J.JSObject object) = J.valFromObj x object indexWithString x (J.JSArray list) = do i <- readInteger x if 0 <= i && i < length list then return $ list !! i else J.Error ("list index " ++ show i ++ " out of bounds") indexWithString _ _ = J.Error "Atomic value was indexed" getJSValue :: [String] -> J.JSValue -> J.Result J.JSValue getJSValue [] js = J.Ok js getJSValue (x:xs) js = do value <- indexWithString x js getJSValue xs value -- | Extract a 'JSValue' from an object at the position defined by the path. -- -- The path syntax follows RCF6901. Error is returned if the path doesn't -- exist, Ok if the path leads to an valid value. -- -- JSON pointer syntax according to RFC6901: -- -- > "/path/0/x" => Pointer ["path", "0", "x"] -- -- This accesses 1 in the following JSON: -- -- > { "path": { "0": { "x": 1 } } } -- -- or the following: -- -- > { "path": [{"x": 1}] } extractJSONPath :: J.JSON a => String -> a -> J.Result J.JSValue extractJSONPath path obj = do pointer <- pointerFromString path extractValue pointer obj ganeti-3.1.0~rc2/src/Ganeti/Config.hs000064400000000000000000000567471476477700300173660ustar00rootroot00000000000000{-# LANGUAGE ViewPatterns #-} {-| Implementation of the Ganeti configuration database. -} {- Copyright (C) 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Config ( LinkIpMap , NdParamObject(..) , loadConfig , saveConfig , getNodeInstances , getNodeRole , getNodeNdParams , getDefaultNicLink , getDefaultHypervisor , getInstancesIpByLink , getMasterNodes , getMasterCandidates , getMasterOrCandidates , getMasterNetworkParameters , getOnlineNodes , getNode , getInstance , getDisk , getFilterRule , getGroup , getGroupNdParams , getGroupIpolicy , getGroupDiskParams , getGroupNodes , getGroupInstances , getGroupOfNode , getInstPrimaryNode , getInstMinorsForNode , getInstAllNodes , getInstDisks , getInstDisksFromObj , getDrbdMinorsForDisk , getDrbdMinorsForInstance , getFilledInstHvParams , getFilledInstBeParams , getFilledInstOsParams , getNetwork , MAC , getAllMACs , getAllDrbdSecrets , NodeLVsMap , getInstanceLVsByNode , getAllLVs , buildLinkIpInstnameMap , instNodes ) where import Control.Arrow ((&&&)) import Control.Monad import Control.Monad.State import qualified Data.ByteString as BS import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Foldable as F import Data.List (foldl', nub) import Data.Maybe (fromMaybe, mapMaybe) import Data.Semigroup ((<>)) import qualified Data.Map as M import qualified Data.Set as S import qualified Text.JSON as J import System.IO import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.Errors import Ganeti.JSON (fromJResult, fromContainer, GenericContainer(..)) import Ganeti.Objects import Ganeti.Types import qualified Ganeti.Utils.MultiMap as MM -- | Type alias for the link and ip map. type LinkIpMap = M.Map String (M.Map String String) -- * Operations on the whole configuration -- | Reads the config file. readConfig :: FilePath -> IO (Result String) readConfig = runResultT . liftIO . readFile -- | Parses the configuration file. parseConfig :: String -> Result ConfigData parseConfig = fromJResult "parsing configuration" . J.decodeStrict -- | Encodes the configuration file. encodeConfig :: ConfigData -> String encodeConfig = J.encodeStrict -- | Wrapper over 'readConfig' and 'parseConfig'. loadConfig :: FilePath -> IO (Result ConfigData) loadConfig = fmap (>>= parseConfig) . readConfig -- | Wrapper over 'hPutStr' and 'encodeConfig'. saveConfig :: Handle -> ConfigData -> IO () saveConfig fh = hPutStr fh . encodeConfig -- * Query functions -- | Annotate Nothing as missing parameter and apply the given -- transformation otherwise withMissingParam :: String -> (a -> ErrorResult b) -> Maybe a -> ErrorResult b withMissingParam = maybe . Bad . ParameterError -- | Computes the nodes covered by a disk. computeDiskNodes :: Disk -> S.Set String computeDiskNodes dsk = case diskLogicalId dsk of Just (LIDDrbd8 nodeA nodeB _ _ _ _) -> S.fromList [nodeA, nodeB] _ -> S.empty -- | Computes all disk-related nodes of an instance. For non-DRBD, -- this will be empty, for DRBD it will contain both the primary and -- the secondaries. instDiskNodes :: ConfigData -> Instance -> S.Set String instDiskNodes cfg inst = case getInstDisksFromObj cfg inst of Ok disks -> S.unions $ map computeDiskNodes disks Bad _ -> S.empty -- | Computes all nodes of an instance. instNodes :: ConfigData -> Instance -> S.Set String instNodes cfg inst = maybe id S.insert (instPrimaryNode inst) $ instDiskNodes cfg inst -- | Computes the secondary node UUID for a DRBD disk computeDiskSecondaryNode :: Disk -> String -> Maybe String computeDiskSecondaryNode dsk primary = case diskLogicalId dsk of Just (LIDDrbd8 a b _ _ _ _) -> Just $ if primary == a then b else a _ -> Nothing -- | Get instances of a given node. -- The node is specified through its UUID. -- The secondary calculation is expensive and frequently called, so optimise -- this to allocate fewer temporary values getNodeInstances :: ConfigData -> String -> ([Instance], [Instance]) getNodeInstances cfg nname = let all_insts = M.elems . fromContainer . configInstances $ cfg all_disks = fromContainer . configDisks $ cfg pri_inst = filter ((== Just nname) . instPrimaryNode) all_insts find_disk :: String -> Maybe Disk find_disk d_uuid = M.lookup (UTF8.fromString d_uuid) all_disks inst_disks :: [(Instance, [Disk])] inst_disks = [(i, mapMaybe find_disk $ instDisks i) | i <- all_insts] sec_insts :: [Instance] sec_insts = [inst | (inst, disks) <- inst_disks, any ((==) nname) $ mapMaybe (\d -> instPrimaryNode inst >>= computeDiskSecondaryNode d) disks] in (pri_inst, sec_insts) -- | Computes the role of a node. getNodeRole :: ConfigData -> Node -> NodeRole getNodeRole cfg node | uuidOf node == clusterMasterNode (configCluster cfg) = NRMaster | nodeMasterCandidate node = NRCandidate | nodeDrained node = NRDrained | nodeOffline node = NROffline | otherwise = NRRegular -- | Get the list of the master nodes (usually one). getMasterNodes :: ConfigData -> [Node] getMasterNodes cfg = filter ((==) NRMaster . getNodeRole cfg) . F.toList . configNodes $ cfg -- | Get the list of master candidates, /not including/ the master itself. getMasterCandidates :: ConfigData -> [Node] getMasterCandidates cfg = filter ((==) NRCandidate . getNodeRole cfg) . F.toList . configNodes $ cfg -- | Get the list of master candidates, /including/ the master. getMasterOrCandidates :: ConfigData -> [Node] getMasterOrCandidates cfg = let isMC r = (r == NRCandidate) || (r == NRMaster) in filter (isMC . getNodeRole cfg) . F.toList . configNodes $ cfg -- | Get the network parameters for the master IP address. getMasterNetworkParameters :: ConfigData -> MasterNetworkParameters getMasterNetworkParameters cfg = let cluster = configCluster cfg in MasterNetworkParameters { masterNetworkParametersUuid = clusterMasterNode cluster , masterNetworkParametersIp = clusterMasterIp cluster , masterNetworkParametersNetmask = clusterMasterNetmask cluster , masterNetworkParametersNetdev = clusterMasterNetdev cluster , masterNetworkParametersIpFamily = clusterPrimaryIpFamily cluster } -- | Get the list of online nodes. getOnlineNodes :: ConfigData -> [Node] getOnlineNodes = filter (not . nodeOffline) . F.toList . configNodes -- | Returns the default cluster link. getDefaultNicLink :: ConfigData -> String getDefaultNicLink = let ppDefault = UTF8.fromString C.ppDefault in nicpLink . (M.! ppDefault) . fromContainer . clusterNicparams . configCluster -- | Returns the default cluster hypervisor. getDefaultHypervisor :: ConfigData -> Hypervisor getDefaultHypervisor cfg = case clusterEnabledHypervisors $ configCluster cfg of -- FIXME: this case shouldn't happen (configuration broken), but -- for now we handle it here because we're not authoritative for -- the config [] -> XenPvm x:_ -> x -- | Returns instances of a given link. getInstancesIpByLink :: LinkIpMap -> String -> [String] getInstancesIpByLink linkipmap link = M.keys $ M.findWithDefault M.empty link linkipmap -- | Generic lookup function that converts from a possible abbreviated -- name to a full name. getItem :: String -> String -> M.Map String a -> ErrorResult a getItem kind name allitems = do let lresult = lookupName (M.keys allitems) name err msg = Bad $ OpPrereqError (kind ++ " name " ++ name ++ " " ++ msg) ECodeNoEnt fullname <- case lrMatchPriority lresult of PartialMatch -> Ok $ lrContent lresult ExactMatch -> Ok $ lrContent lresult MultipleMatch -> err "has multiple matches" FailMatch -> err "not found" maybe (err "not found after successfull match?!") Ok $ M.lookup fullname allitems -- | Simple lookup function, insisting on exact matches and using -- byte strings. getItem' :: String -> String -> M.Map BS.ByteString a -> ErrorResult a getItem' kind name allitems = let name' = UTF8.fromString name err = Bad $ OpPrereqError (kind ++ " uuid " ++ name ++ " not found") ECodeNoEnt in maybe err Ok $ M.lookup name' allitems -- | Looks up a node by name or uuid. getNode :: ConfigData -> String -> ErrorResult Node getNode cfg name = let nodes = fromContainer (configNodes cfg) in case getItem' "Node" name nodes of -- if not found by uuid, we need to look it up by name Ok node -> Ok node Bad _ -> let by_name = M.mapKeys (nodeName . (M.!) nodes) nodes in getItem "Node" name by_name -- | Looks up an instance by name or uuid. getInstance :: ConfigData -> String -> ErrorResult Instance getInstance cfg name = let instances = fromContainer (configInstances cfg) in case getItem' "Instance" name instances of -- if not found by uuid, we need to look it up by name Ok inst -> Ok inst Bad _ -> let by_name = M.delete "" . M.mapKeys (fromMaybe "" . instName . (M.!) instances) $ instances in getItem "Instance" name by_name -- | Looks up an instance by exact name match getInstanceByName :: ConfigData -> String -> ErrorResult Instance getInstanceByName cfg name = let instances = M.elems . fromContainer . configInstances $ cfg matching = F.find (maybe False (== name) . instName) instances in case matching of Just inst -> Ok inst Nothing -> Bad $ OpPrereqError ("Instance name " ++ name ++ " not found") ECodeNoEnt -- | Looks up a disk by uuid. getDisk :: ConfigData -> String -> ErrorResult Disk getDisk cfg name = let disks = fromContainer (configDisks cfg) in getItem' "Disk" name disks -- | Looks up a filter by uuid. getFilterRule :: ConfigData -> String -> ErrorResult FilterRule getFilterRule cfg name = let filters = fromContainer (configFilters cfg) in getItem' "Filter" name filters -- | Looks up a node group by name or uuid. getGroup :: ConfigData -> String -> ErrorResult NodeGroup getGroup cfg name = let groups = fromContainer (configNodegroups cfg) in case getItem' "NodeGroup" name groups of -- if not found by uuid, we need to look it up by name, slow Ok grp -> Ok grp Bad _ -> let by_name = M.mapKeys (groupName . (M.!) groups) groups in getItem "NodeGroup" name by_name -- | Computes a node group's node params. getGroupNdParams :: ConfigData -> NodeGroup -> FilledNDParams getGroupNdParams cfg ng = fillParams (clusterNdparams $ configCluster cfg) (groupNdparams ng) -- | Computes a node group's ipolicy. getGroupIpolicy :: ConfigData -> NodeGroup -> FilledIPolicy getGroupIpolicy cfg ng = fillParams (clusterIpolicy $ configCluster cfg) (groupIpolicy ng) -- | Computes a group\'s (merged) disk params. getGroupDiskParams :: ConfigData -> NodeGroup -> GroupDiskParams getGroupDiskParams cfg ng = GenericContainer $ fillDict (fromContainer . clusterDiskparams $ configCluster cfg) (fromContainer $ groupDiskparams ng) [] -- | Get nodes of a given node group. getGroupNodes :: ConfigData -> String -> [Node] getGroupNodes cfg gname = let all_nodes = M.elems . fromContainer . configNodes $ cfg in filter ((==gname) . nodeGroup) all_nodes -- | Get (primary, secondary) instances of a given node group. getGroupInstances :: ConfigData -> String -> ([Instance], [Instance]) getGroupInstances cfg gname = let gnodes = map uuidOf (getGroupNodes cfg gname) ginsts = map (getNodeInstances cfg) gnodes in (concatMap fst ginsts, concatMap snd ginsts) -- | Retrieves the instance hypervisor params, missing values filled with -- cluster defaults. getFilledInstHvParams :: [String] -> ConfigData -> Instance -> HvParams getFilledInstHvParams globals cfg inst = -- First get the defaults of the parent let maybeHvName = instHypervisor inst hvParamMap = fromContainer . clusterHvparams $ configCluster cfg parentHvParams = maybe M.empty fromContainer (maybeHvName >>= flip M.lookup hvParamMap) -- Then the os defaults for the given hypervisor maybeOsName = UTF8.fromString <$> instOs inst osParamMap = fromContainer . clusterOsHvp $ configCluster cfg osHvParamMap = maybe M.empty (maybe M.empty fromContainer . flip M.lookup osParamMap) maybeOsName osHvParams = maybe M.empty (maybe M.empty fromContainer . flip M.lookup osHvParamMap) maybeHvName -- Then the child childHvParams = fromContainer . instHvparams $ inst -- Helper function fillFn con val = fillDict con val $ fmap UTF8.fromString globals in GenericContainer $ fillFn (fillFn parentHvParams osHvParams) childHvParams -- | Retrieves the instance backend params, missing values filled with cluster -- defaults. getFilledInstBeParams :: ConfigData -> Instance -> ErrorResult FilledBeParams getFilledInstBeParams cfg inst = do let beParamMap = fromContainer . clusterBeparams . configCluster $ cfg parentParams <- getItem' "FilledBeParams" C.ppDefault beParamMap return $ fillParams parentParams (instBeparams inst) -- | Retrieves the instance os params, missing values filled with cluster -- defaults. This does NOT include private and secret parameters. getFilledInstOsParams :: ConfigData -> Instance -> OsParams getFilledInstOsParams cfg inst = let maybeOsLookupName = liftM (takeWhile (/= '+')) (instOs inst) osParamMap = fromContainer . clusterOsparams $ configCluster cfg childOsParams = instOsparams inst in case withMissingParam "Instance without OS" (flip (getItem' "OsParams") osParamMap) maybeOsLookupName of Ok parentOsParams -> GenericContainer $ fillDict (fromContainer parentOsParams) (fromContainer childOsParams) [] Bad _ -> childOsParams -- | Looks up an instance's primary node. getInstPrimaryNode :: ConfigData -> String -> ErrorResult Node getInstPrimaryNode cfg name = getInstanceByName cfg name >>= withMissingParam "Instance without primary node" return . instPrimaryNode >>= getNode cfg -- | Retrieves all nodes hosting a DRBD disk getDrbdDiskNodes :: ConfigData -> Disk -> [Node] getDrbdDiskNodes cfg disk = let retrieved = case diskLogicalId disk of Just (LIDDrbd8 nodeA nodeB _ _ _ _) -> justOk [getNode cfg nodeA, getNode cfg nodeB] _ -> [] in retrieved ++ concatMap (getDrbdDiskNodes cfg) (diskChildren disk) -- | Retrieves all the nodes of the instance. -- -- As instances not using DRBD can be sent as a parameter as well, -- the primary node has to be appended to the results. getInstAllNodes :: ConfigData -> String -> ErrorResult [Node] getInstAllNodes cfg name = do inst <- getInstanceByName cfg name inst_disks <- getInstDisksFromObj cfg inst let disk_nodes = concatMap (getDrbdDiskNodes cfg) inst_disks pNode <- getInstPrimaryNode cfg name return . nub $ pNode:disk_nodes -- | Get disks for a given instance. -- The instance is specified by name or uuid. getInstDisks :: ConfigData -> String -> ErrorResult [Disk] getInstDisks cfg iname = getInstance cfg iname >>= mapM (getDisk cfg) . instDisks -- | Get disks for a given instance object. getInstDisksFromObj :: ConfigData -> Instance -> ErrorResult [Disk] getInstDisksFromObj cfg = getInstDisks cfg . uuidOf -- | Collects a value for all DRBD disks collectFromDrbdDisks :: (Monoid a) => (String -> String -> Int -> Int -> Int -> Private DRBDSecret -> a) -- ^ NodeA, NodeB, Port, MinorA, MinorB, Secret -> Disk -> a collectFromDrbdDisks f = col where col (diskLogicalId &&& diskChildren -> (Just (LIDDrbd8 nA nB port mA mB secret), ch)) = f nA nB port mA mB secret <> F.foldMap col ch col d = F.foldMap col (diskChildren d) -- | Returns the DRBD secrets of a given 'Disk' getDrbdSecretsForDisk :: Disk -> [DRBDSecret] getDrbdSecretsForDisk = collectFromDrbdDisks (\_ _ _ _ _ (Private secret) -> [secret]) -- | Returns the DRBD minors of a given 'Disk' getDrbdMinorsForDisk :: Disk -> [(Int, String)] getDrbdMinorsForDisk = collectFromDrbdDisks (\nA nB _ mnA mnB _ -> [(mnA, nA), (mnB, nB)]) -- | Filters DRBD minors for a given node. getDrbdMinorsForNode :: String -> Disk -> [(Int, String)] getDrbdMinorsForNode node disk = let child_minors = concatMap (getDrbdMinorsForNode node) (diskChildren disk) this_minors = case diskLogicalId disk of Just (LIDDrbd8 nodeA nodeB _ minorA minorB _) | nodeA == node -> [(minorA, nodeB)] | nodeB == node -> [(minorB, nodeA)] _ -> [] in this_minors ++ child_minors -- | Returns the DRBD minors of a given instance getDrbdMinorsForInstance :: ConfigData -> Instance -> ErrorResult [(Int, String)] getDrbdMinorsForInstance cfg = liftM (concatMap getDrbdMinorsForDisk) . getInstDisksFromObj cfg -- | String for primary role. rolePrimary :: String rolePrimary = "primary" -- | String for secondary role. roleSecondary :: String roleSecondary = "secondary" -- | Gets the list of DRBD minors for an instance that are related to -- a given node. getInstMinorsForNode :: ConfigData -> String -- ^ The UUID of a node. -> Instance -> [(String, Int, String, String, String, String)] getInstMinorsForNode cfg node inst = let nrole = if Just node == instPrimaryNode inst then rolePrimary else roleSecondary iname = fromMaybe "" $ instName inst inst_disks = case getInstDisksFromObj cfg inst of Ok disks -> disks Bad _ -> [] -- FIXME: the disk/ build there is hack-ish; unify this in a -- separate place, or reuse the iv_name (but that is deprecated on -- the Python side) in concatMap (\(idx, dsk) -> [(node, minor, iname, "disk/" ++ show idx, nrole, peer) | (minor, peer) <- getDrbdMinorsForNode node dsk]) . zip [(0::Int)..] $ inst_disks -- | Builds link -> ip -> instname map. -- For instances without a name, we insert the uuid instead. -- -- TODO: improve this by splitting it into multiple independent functions: -- -- * abstract the \"fetch instance with filled params\" functionality -- -- * abstsract the [instance] -> [(nic, instance_name)] part -- -- * etc. buildLinkIpInstnameMap :: ConfigData -> LinkIpMap buildLinkIpInstnameMap cfg = let cluster = configCluster cfg instances = M.elems . fromContainer . configInstances $ cfg defparams = (M.!) (fromContainer $ clusterNicparams cluster) $ UTF8.fromString C.ppDefault nics = concatMap (\i -> [(fromMaybe (uuidOf i) $ instName i, nic) | nic <- instNics i]) instances in foldl' (\accum (iname, nic) -> let pparams = nicNicparams nic fparams = fillParams defparams pparams link = nicpLink fparams in case nicIp nic of Nothing -> accum Just ip -> let oldipmap = M.findWithDefault M.empty link accum newipmap = M.insert ip iname oldipmap in M.insert link newipmap accum ) M.empty nics -- | Returns a node's group, with optional failure if we can't find it -- (configuration corrupt). getGroupOfNode :: ConfigData -> Node -> Maybe NodeGroup getGroupOfNode cfg node = M.lookup (UTF8.fromString $ nodeGroup node) (fromContainer . configNodegroups $ cfg) -- | Returns a node's ndparams, filled. getNodeNdParams :: ConfigData -> Node -> Maybe FilledNDParams getNodeNdParams cfg node = do group <- getGroupOfNode cfg node let gparams = getGroupNdParams cfg group return $ fillParams gparams (nodeNdparams node) -- * Network -- | Looks up a network. If looking up by uuid fails, we look up -- by name. getNetwork :: ConfigData -> String -> ErrorResult Network getNetwork cfg name = let networks = fromContainer (configNetworks cfg) in case getItem' "Network" name networks of Ok net -> Ok net Bad _ -> let by_name = M.mapKeys (fromNonEmpty . networkName . (M.!) networks) networks in getItem "Network" name by_name -- ** MACs type MAC = String -- | Returns all MAC addresses used in the cluster. getAllMACs :: ConfigData -> [MAC] getAllMACs = F.foldMap (map nicMac . instNics) . configInstances -- ** DRBD secrets getAllDrbdSecrets :: ConfigData -> [DRBDSecret] getAllDrbdSecrets = F.foldMap getDrbdSecretsForDisk . configDisks -- ** LVs -- | A map from node UUIDs to -- -- FIXME: After adding designated types for UUIDs, -- use them to replace 'String' here. type NodeLVsMap = MM.MultiMap String LogicalVolume getInstanceLVsByNode :: ConfigData -> Instance -> ErrorResult NodeLVsMap getInstanceLVsByNode cd inst = withMissingParam "Instance without Primary Node" (\i -> return $ MM.fromList . lvsByNode i) (instPrimaryNode inst) <*> getInstDisksFromObj cd inst where lvsByNode :: String -> [Disk] -> [(String, LogicalVolume)] lvsByNode node = concatMap (lvsByNode1 node) lvsByNode1 :: String -> Disk -> [(String, LogicalVolume)] lvsByNode1 _ (diskLogicalId &&& diskChildren -> (Just (LIDDrbd8 nA nB _ _ _ _), ch)) = lvsByNode nA ch ++ lvsByNode nB ch lvsByNode1 node (diskLogicalId -> (Just (LIDPlain lv))) = [(node, lv)] lvsByNode1 node (diskChildren -> ch) = lvsByNode node ch getAllLVs :: ConfigData -> ErrorResult (S.Set LogicalVolume) getAllLVs cd = mconcat <$> mapM (liftM MM.values . getInstanceLVsByNode cd) (F.toList $ configInstances cd) -- * ND params -- | Type class denoting objects which have node parameters. class NdParamObject a where getNdParamsOf :: ConfigData -> a -> Maybe FilledNDParams instance NdParamObject Node where getNdParamsOf = getNodeNdParams instance NdParamObject NodeGroup where getNdParamsOf cfg = Just . getGroupNdParams cfg instance NdParamObject Cluster where getNdParamsOf _ = Just . clusterNdparams ganeti-3.1.0~rc2/src/Ganeti/ConfigReader.hs000064400000000000000000000263721476477700300205000ustar00rootroot00000000000000{-# LANGUAGE BangPatterns #-} {-| Implementation of configuration reader with watching support. -} {- Copyright (C) 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.ConfigReader ( ConfigReader , initConfigReader ) where import Control.Concurrent import Control.Exception import Control.Monad (unless) import System.IO.Error (catchIOError) import System.INotify import Ganeti.BasicTypes import Ganeti.Objects import Ganeti.Compat import Ganeti.Config import Ganeti.Logging import qualified Ganeti.Constants as C import qualified Ganeti.Path as Path import Ganeti.Utils.Time (getCurrentTime, getCurrentTimeUSec) import Ganeti.Utils -- | A type for functions that can return the configuration when -- executed. type ConfigReader = IO (Result ConfigData) -- | Reload model data type. data ReloadModel = ReloadNotify -- ^ We are using notifications | ReloadPoll Int -- ^ We are using polling deriving (Eq, Show) -- | Server state data type. data ServerState = ServerState { reloadModel :: ReloadModel , reloadTime :: Integer -- ^ Reload time (epoch) in microseconds , reloadFStat :: FStat } -- | Maximum no-reload poll rounds before reverting to inotify. maxIdlePollRounds :: Int maxIdlePollRounds = 3 -- | Reload timeout in microseconds. watchInterval :: Int watchInterval = C.confdConfigReloadTimeout * 1000000 -- | Ratelimit timeout in microseconds. pollInterval :: Int pollInterval = C.confdConfigReloadRatelimit -- | Ratelimit timeout in microseconds, as an 'Integer'. reloadRatelimit :: Integer reloadRatelimit = fromIntegral C.confdConfigReloadRatelimit -- | Initial poll round. initialPoll :: ReloadModel initialPoll = ReloadPoll 0 -- | Reload status data type. data ConfigReload = ConfigToDate -- ^ No need to reload | ConfigReloaded -- ^ Configuration reloaded | ConfigIOError -- ^ Error during configuration reload deriving (Eq) -- * Configuration handling -- ** Helper functions -- | Helper function for logging transition into polling mode. moveToPolling :: String -> INotify -> FilePath -> (Result ConfigData -> IO ()) -> MVar ServerState -> IO ReloadModel moveToPolling msg inotify path save_fn mstate = do logInfo $ "Moving to polling mode: " ++ msg let inotiaction = addNotifier inotify path save_fn mstate _ <- forkIO $ onPollTimer inotiaction path save_fn mstate return initialPoll -- | Helper function for logging transition into inotify mode. moveToNotify :: IO ReloadModel moveToNotify = do logInfo "Moving to inotify mode" return ReloadNotify -- ** Configuration loading -- | (Re)loads the configuration. updateConfig :: FilePath -> (Result ConfigData -> IO ()) -> IO () updateConfig path save_fn = do newcfg <- loadConfig path let !newdata = case newcfg of Ok !cfg -> Ok cfg Bad msg -> Bad $ "Cannot load configuration from " ++ path ++ ": " ++ msg save_fn newdata case newcfg of Ok cfg -> logInfo ("Loaded new config, serial " ++ show (configSerial cfg)) Bad msg -> logError $ "Failed to load config: " ++ msg return () -- | Wrapper over 'updateConfig' that handles IO errors. safeUpdateConfig :: FilePath -> FStat -> (Result ConfigData -> IO ()) -> IO (FStat, ConfigReload) safeUpdateConfig path oldfstat save_fn = Control.Exception.catch (do nt <- needsReload oldfstat path case nt of Nothing -> return (oldfstat, ConfigToDate) Just nt' -> do updateConfig path save_fn return (nt', ConfigReloaded) ) (\e -> do let msg = "Failure during configuration update: " ++ show (e::IOError) save_fn $ Bad msg return (nullFStat, ConfigIOError) ) -- ** Watcher threads -- $watcher -- We have three threads/functions that can mutate the server state: -- -- 1. the long-interval watcher ('onWatcherTimer') -- -- 2. the polling watcher ('onPollTimer') -- -- 3. the inotify event handler ('onInotify') -- -- All of these will mutate the server state under 'modifyMVar' or -- 'modifyMVar_', so that server transitions are more or less -- atomic. The inotify handler remains active during polling mode, but -- checks for polling mode and doesn't do anything in this case (this -- check is needed even if we would unregister the event handler due -- to how events are serialised). -- | Long-interval reload watcher. -- -- This is on top of the inotify-based triggered reload. onWatcherTimer :: FilePath -> (Result ConfigData -> IO ()) -> MVar ServerState -> IO () onWatcherTimer path save_fn state = do threadDelay watchInterval logDebug "Config-reader watcher timer fired" modifyMVar_ state (onWatcherInner path save_fn) onWatcherTimer path save_fn state -- | Inner onWatcher handler. -- -- This mutates the server state under a modifyMVar_ call. It never -- changes the reload model, just does a safety reload and tried to -- re-establish the inotify watcher. onWatcherInner :: FilePath -> (Result ConfigData -> IO ()) -> ServerState -> IO ServerState onWatcherInner path save_fn state = do (newfstat, _) <- safeUpdateConfig path (reloadFStat state) save_fn return state { reloadFStat = newfstat } -- | Short-interval (polling) reload watcher. -- -- This is only active when we're in polling mode; it will -- automatically exit when it detects that the state has changed to -- notification. onPollTimer :: IO Bool -> FilePath -> (Result ConfigData -> IO ()) -> MVar ServerState -> IO () onPollTimer inotiaction path save_fn state = do threadDelay pollInterval logDebug $ "Poll timer fired for " ++ path continue <- modifyMVar state (onPollInner inotiaction path save_fn) if continue then onPollTimer inotiaction path save_fn state else logDebug "Inotify watch active, polling thread exiting" -- | Inner onPoll handler. -- -- This again mutates the state under a modifyMVar call, and also -- returns whether the thread should continue or not. onPollInner :: IO Bool -> FilePath -> (Result ConfigData -> IO ()) -> ServerState -> IO (ServerState, Bool) onPollInner _ _ _ state@(ServerState { reloadModel = ReloadNotify } ) = return (state, False) onPollInner inotiaction path save_fn state@(ServerState { reloadModel = ReloadPoll pround } ) = do (newfstat, reload) <- safeUpdateConfig path (reloadFStat state) save_fn let state' = state { reloadFStat = newfstat } -- compute new poll model based on reload data; however, failure to -- re-establish the inotifier means we stay on polling newmode <- case reload of ConfigToDate -> if pround >= maxIdlePollRounds then do -- try to switch to notify result <- inotiaction if result then moveToNotify else return initialPoll else return (ReloadPoll (pround + 1)) _ -> return initialPoll let continue = case newmode of ReloadNotify -> False _ -> True return (state' { reloadModel = newmode }, continue) -- | Setup inotify watcher. -- -- This tries to setup the watch descriptor; in case of any IO errors, -- it will return False. addNotifier :: INotify -> FilePath -> (Result ConfigData -> IO ()) -> MVar ServerState -> IO Bool addNotifier inotify path save_fn mstate = catchIOError (addWatch inotify [CloseWrite] (toInotifyPath path) (onInotify inotify path save_fn mstate) >> return True) (\_e -> return False) -- | Inotify event handler. onInotify :: INotify -> String -> (Result ConfigData -> IO ()) -> MVar ServerState -> Event -> IO () onInotify inotify path save_fn mstate Ignored = do logDebug $ "File lost, trying to re-establish notifier for " ++ path modifyMVar_ mstate $ \state -> do result <- addNotifier inotify path save_fn mstate (newfstat, _) <- safeUpdateConfig path (reloadFStat state) save_fn let state' = state { reloadFStat = newfstat } if result then return state' -- keep notify else do mode <- moveToPolling "cannot re-establish inotify watch" inotify path save_fn mstate return state' { reloadModel = mode } onInotify inotify path save_fn mstate _ = do logDebug $ "onInotify fired for " ++ path modifyMVar_ mstate $ \state -> if reloadModel state == ReloadNotify then do ctime <- getCurrentTimeUSec (newfstat, _) <- safeUpdateConfig path (reloadFStat state) save_fn let state' = state { reloadFStat = newfstat, reloadTime = ctime } if abs (reloadTime state - ctime) < reloadRatelimit then do mode <- moveToPolling "too many reloads" inotify path save_fn mstate return state' { reloadModel = mode } else return state' else return state initConfigReader :: (Result ConfigData -> IO ()) -> IO () initConfigReader save_fn = do -- Inotify setup inotify <- initINotify -- try to load the configuration, if possible conf_file <- Path.clusterConfFile (fstat, reloaded) <- safeUpdateConfig conf_file nullFStat save_fn ctime <- getCurrentTime statemvar <- newMVar $ ServerState ReloadNotify ctime fstat let inotiaction = addNotifier inotify conf_file save_fn statemvar has_inotify <- if reloaded == ConfigReloaded then inotiaction else return False if has_inotify then logInfo "Starting up in inotify mode" else do -- inotify was not enabled, we need to update the reload model logInfo "Starting up in polling mode" modifyMVar_ statemvar (\state -> return state { reloadModel = initialPoll }) -- fork the timeout timer _ <- forkIO $ onWatcherTimer conf_file save_fn statemvar -- fork the polling timer unless has_inotify $ do _ <- forkIO $ onPollTimer inotiaction conf_file save_fn statemvar return () ganeti-3.1.0~rc2/src/Ganeti/ConstantUtils.hs000064400000000000000000000142331476477700300207530ustar00rootroot00000000000000{-| ConstantUtils contains the helper functions for constants This module cannot be merged with 'Ganeti.Utils' because it would create a circular dependency if imported, for example, from 'Ganeti.Constants'. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.ConstantUtils where import Data.Char (ord) import Data.Set (Set) import qualified Data.Set as Set (difference, fromList, toList, union) import qualified Data.Semigroup as Sem import Ganeti.PyValue -- | 'PythonChar' wraps a Python 'char' newtype PythonChar = PythonChar { unPythonChar :: Char } deriving (Show) instance PyValue PythonChar where showValue c = "chr(" ++ show (ord (unPythonChar c)) ++ ")" -- | 'PythonNone' wraps Python 'None' data PythonNone = PythonNone instance PyValue PythonNone where showValue _ = "None" -- | FrozenSet wraps a Haskell 'Set' -- -- See 'PyValue' instance for 'FrozenSet'. newtype FrozenSet a = FrozenSet { unFrozenSet :: Set a } deriving (Eq, Ord, Show) instance (Ord a) => Sem.Semigroup (FrozenSet a) where (FrozenSet s) <> (FrozenSet t) = FrozenSet (mappend s t) instance (Ord a) => Monoid (FrozenSet a) where mempty = FrozenSet mempty mappend = (Sem.<>) -- | Converts a Haskell 'Set' into a Python 'frozenset' -- -- This instance was supposed to be for 'Set' instead of 'FrozenSet'. -- However, 'ghc-6.12.1' seems to be crashing with 'segmentation -- fault' due to the presence of more than one instance of 'Set', -- namely, this one and the one in 'Ganeti.OpCodes'. For this reason, -- we wrap 'Set' into 'FrozenSet'. instance PyValue a => PyValue (FrozenSet a) where showValue s = "frozenset(" ++ showValue (Set.toList (unFrozenSet s)) ++ ")" mkSet :: Ord a => [a] -> FrozenSet a mkSet = FrozenSet . Set.fromList toList :: FrozenSet a -> [a] toList = Set.toList . unFrozenSet union :: Ord a => FrozenSet a -> FrozenSet a -> FrozenSet a union x y = FrozenSet (unFrozenSet x `Set.union` unFrozenSet y) difference :: Ord a => FrozenSet a -> FrozenSet a -> FrozenSet a difference x y = FrozenSet (unFrozenSet x `Set.difference` unFrozenSet y) -- | 'Protocol' represents the protocols used by the daemons data Protocol = Tcp | Udp deriving (Show) -- | 'PyValue' instance of 'Protocol' -- -- This instance is used by the Haskell to Python constants instance PyValue Protocol where showValue Tcp = "\"tcp\"" showValue Udp = "\"udp\"" -- | Failure exit code -- -- These are defined here and not in 'Ganeti.Constants' together with -- the other exit codes in order to avoid a circular dependency -- between 'Ganeti.Constants' and 'Ganeti.Runtime' exitFailure :: Int exitFailure = 1 -- | Console device -- -- This is defined here and not in 'Ganeti.Constants' order to avoid a -- circular dependency between 'Ganeti.Constants' and 'Ganeti.Logging' devConsole :: String devConsole = "/dev/console" -- | Random uuid generator -- -- This is defined here and not in 'Ganeti.Constants' order to avoid a -- circular dependendy between 'Ganeti.Constants' and 'Ganeti.Types' randomUuidFile :: String randomUuidFile = "/proc/sys/kernel/random/uuid" -- * Priority levels -- -- This is defined here and not in 'Ganeti.Types' in order to avoid a -- GHC stage restriction and because there is no suitable 'declareADT' -- variant that handles integer values directly. priorityLow :: Int priorityLow = 10 priorityNormal :: Int priorityNormal = 0 priorityHigh :: Int priorityHigh = -10 -- | Calculates int version number from major, minor and revision -- numbers. buildVersion :: Int -> Int -> Int -> Int buildVersion major minor revision = 1000000 * major + 10000 * minor + 1 * revision -- | Confd protocol version -- -- This is defined here in order to avoid a circular dependency -- between 'Ganeti.Confd.Types' and 'Ganeti.Constants'. confdProtocolVersion :: Int confdProtocolVersion = 1 -- * Confd request query fields -- -- These are defined here and not in 'Ganeti.Types' due to GHC stage -- restrictions concerning Template Haskell. They are also not -- defined in 'Ganeti.Constants' in order to avoid a circular -- dependency between that module and 'Ganeti.Types'. confdReqqLink :: String confdReqqLink = "0" confdReqqIp :: String confdReqqIp = "1" confdReqqIplist :: String confdReqqIplist = "2" confdReqqFields :: String confdReqqFields = "3" -- * ISpec ispecMemSize :: String ispecMemSize = "memory-size" ispecCpuCount :: String ispecCpuCount = "cpu-count" ispecDiskCount :: String ispecDiskCount = "disk-count" ispecDiskSize :: String ispecDiskSize = "disk-size" ispecNicCount :: String ispecNicCount = "nic-count" ispecSpindleUse :: String ispecSpindleUse = "spindle-use" ispecsMinmax :: String ispecsMinmax = "minmax" ispecsStd :: String ispecsStd = "std" ipolicyDts :: String ipolicyDts = "disk-templates" ipolicyVcpuRatio :: String ipolicyVcpuRatio = "vcpu-ratio" ipolicySpindleRatio :: String ipolicySpindleRatio = "spindle-ratio" ipolicyDefaultsVcpuRatio :: Double ipolicyDefaultsVcpuRatio = 4.0 ipolicyDefaultsSpindleRatio :: Double ipolicyDefaultsSpindleRatio = 32.0 ganeti-3.1.0~rc2/src/Ganeti/Constants.hs000064400000000000000000004324561476477700300201300ustar00rootroot00000000000000{-# OPTIONS -fno-warn-type-defaults #-} {-| Constants contains the Haskell constants The constants in this module are used in Haskell and are also converted to Python. Do not write any definitions in this file other than constants. Do not even write helper functions. The definitions in this module are automatically stripped to build the Makefile.am target 'ListConstants.hs'. If there are helper functions in this module, they will also be dragged and it will cause compilation to fail. Therefore, all helper functions should go to a separate module and imported. -} {- Copyright (C) 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Constants where import Control.Arrow ((***),(&&&)) import qualified Data.ByteString.Char8 as BC import Data.List ((\\)) import Data.Map (Map) import qualified Data.Map as Map (empty, fromList, keys, insert) import Data.Monoid import qualified AutoConf import Ganeti.ConstantUtils (FrozenSet, Protocol(..), buildVersion) import qualified Ganeti.ConstantUtils as ConstantUtils import qualified Ganeti.HTools.Types as Types import Ganeti.HTools.Types (AutoRepairResult(..), AutoRepairType(..)) import Ganeti.Logging (SyslogUsage(..)) import qualified Ganeti.Logging as Logging (syslogUsageToRaw) import qualified Ganeti.Runtime as Runtime import Ganeti.Runtime (GanetiDaemon(..), MiscGroup(..), GanetiGroup(..), ExtraLogReason(..)) import Ganeti.THH (PyValueEx(..)) import Ganeti.Types import qualified Ganeti.Types as Types import Ganeti.Confd.Types (ConfdRequestType(..), ConfdReqField(..), ConfdReplyStatus(..), ConfdNodeRole(..), ConfdErrorType(..)) import qualified Ganeti.Confd.Types as Types import qualified Ganeti.HTools.Tags.Constants as Tags {-# ANN module "HLint: ignore Use camelCase" #-} -- * 'autoconf' constants for Python only ('autotools/build-bash-completion') htoolsProgs :: [String] htoolsProgs = AutoConf.htoolsProgs -- * 'autoconf' constants for Python only ('lib/constants.py') drbdBarriers :: String drbdBarriers = AutoConf.drbdBarriers drbdNoMetaFlush :: Bool drbdNoMetaFlush = AutoConf.drbdNoMetaFlush lvmStripecount :: Int lvmStripecount = AutoConf.lvmStripecount hasGnuLn :: Bool hasGnuLn = AutoConf.hasGnuLn -- * 'autoconf' constants for Python only ('lib/pathutils.py') -- ** Build-time constants exportDir :: String exportDir = AutoConf.exportDir backupDir :: String backupDir = AutoConf.backupDir osSearchPath :: [String] osSearchPath = AutoConf.osSearchPath esSearchPath :: [String] esSearchPath = AutoConf.esSearchPath sshConfigDir :: String sshConfigDir = AutoConf.sshConfigDir xenConfigDir :: String xenConfigDir = AutoConf.xenConfigDir sysconfdir :: String sysconfdir = AutoConf.sysconfdir toolsdir :: String toolsdir = AutoConf.toolsdir localstatedir :: String localstatedir = AutoConf.localstatedir -- ** Paths which don't change for a virtual cluster pkglibdir :: String pkglibdir = AutoConf.pkglibdir sharedir :: String sharedir = AutoConf.sharedir -- * 'autoconf' constants for Python only ('lib/build/sphinx_ext.py') manPages :: Map String Int manPages = Map.fromList AutoConf.manPages -- * 'autoconf' constants for QA cluster only ('qa/qa_cluster.py') versionedsharedir :: String versionedsharedir = AutoConf.versionedsharedir -- * 'autoconf' constants for Python only ('tests/py/docs_unittest.py') gntScripts :: [String] gntScripts = AutoConf.gntScripts -- * Various versions releaseVersion :: String releaseVersion = AutoConf.packageVersion versionMajor :: Int versionMajor = AutoConf.versionMajor versionMinor :: Int versionMinor = AutoConf.versionMinor versionRevision :: Int versionRevision = AutoConf.versionRevision dirVersion :: String dirVersion = AutoConf.dirVersion osApiV10 :: Int osApiV10 = 10 osApiV15 :: Int osApiV15 = 15 osApiV20 :: Int osApiV20 = 20 osApiVersions :: FrozenSet Int osApiVersions = ConstantUtils.mkSet [osApiV10, osApiV15, osApiV20] -- | The version of the backup/export instance description file format we are -- producing when exporting and accepting when importing. The two are currently -- tightly intertwined. exportVersion :: Int exportVersion = 0 rapiVersion :: Int rapiVersion = 2 configMajor :: Int configMajor = AutoConf.versionMajor configMinor :: Int configMinor = AutoConf.versionMinor -- | The configuration is supposed to remain stable across -- revisions. Therefore, the revision number is cleared to '0'. configRevision :: Int configRevision = 0 configVersion :: Int configVersion = buildVersion configMajor configMinor configRevision -- | Similarly to the configuration (see 'configRevision'), the -- protocols are supposed to remain stable across revisions. protocolVersion :: Int protocolVersion = buildVersion configMajor configMinor configRevision -- * User separation daemonsGroup :: String daemonsGroup = Runtime.daemonGroup (ExtraGroup DaemonsGroup) adminGroup :: String adminGroup = Runtime.daemonGroup (ExtraGroup AdminGroup) masterdUser :: String masterdUser = Runtime.daemonUser GanetiMasterd masterdGroup :: String masterdGroup = Runtime.daemonGroup (DaemonGroup GanetiMasterd) metadUser :: String metadUser = Runtime.daemonUser GanetiMetad metadGroup :: String metadGroup = Runtime.daemonGroup (DaemonGroup GanetiMetad) rapiUser :: String rapiUser = Runtime.daemonUser GanetiRapi rapiGroup :: String rapiGroup = Runtime.daemonGroup (DaemonGroup GanetiRapi) confdUser :: String confdUser = Runtime.daemonUser GanetiConfd confdGroup :: String confdGroup = Runtime.daemonGroup (DaemonGroup GanetiConfd) wconfdUser :: String wconfdUser = Runtime.daemonUser GanetiWConfd wconfdGroup :: String wconfdGroup = Runtime.daemonGroup (DaemonGroup GanetiWConfd) kvmdUser :: String kvmdUser = Runtime.daemonUser GanetiKvmd kvmdGroup :: String kvmdGroup = Runtime.daemonGroup (DaemonGroup GanetiKvmd) luxidUser :: String luxidUser = Runtime.daemonUser GanetiLuxid luxidGroup :: String luxidGroup = Runtime.daemonGroup (DaemonGroup GanetiLuxid) nodedUser :: String nodedUser = Runtime.daemonUser GanetiNoded nodedGroup :: String nodedGroup = Runtime.daemonGroup (DaemonGroup GanetiNoded) mondUser :: String mondUser = Runtime.daemonUser GanetiMond mondGroup :: String mondGroup = Runtime.daemonGroup (DaemonGroup GanetiMond) sshLoginUser :: String sshLoginUser = AutoConf.sshLoginUser sshConsoleUser :: String sshConsoleUser = AutoConf.sshConsoleUser -- * Cpu pinning separators and constants cpuPinningSep :: String cpuPinningSep = ":" cpuPinningAll :: String cpuPinningAll = "all" -- | Internal representation of "all" cpuPinningAllVal :: Int cpuPinningAllVal = -1 -- | One "all" entry in a CPU list means CPU pinning is off cpuPinningOff :: [Int] cpuPinningOff = [cpuPinningAllVal] -- | A Xen-specific implementation detail is that there is no way to -- actually say "use any cpu for pinning" in a Xen configuration file, -- as opposed to the command line, where you can say -- @ -- xm vcpu-pin all -- @ -- -- The workaround used in Xen is "0-63" (see source code function -- "xm_vcpu_pin" in @/tools/python/xen/xm/main.py@). -- -- To support future changes, the following constant is treated as a -- blackbox string that simply means "use any cpu for pinning under -- xen". cpuPinningAllXen :: String cpuPinningAllXen = "0-63" -- * Image and wipe ddCmd :: String ddCmd = "dd" -- | 1 MiB -- The default block size for the 'dd' command ddBlockSize :: Int ddBlockSize = 1024^2 -- | 1GB maxWipeChunk :: Int maxWipeChunk = 1024 minWipeChunkPercent :: Int minWipeChunkPercent = 10 -- * Directories runDirsMode :: Int runDirsMode = 0o775 secureDirMode :: Int secureDirMode = 0o700 secureFileMode :: Int secureFileMode = 0o600 adoptableBlockdevRoot :: String adoptableBlockdevRoot = "/dev/disk/" -- * 'autoconf' enable/disable enableMond :: Bool enableMond = AutoConf.enableMond enableMetad :: Bool enableMetad = AutoConf.enableMetad enableRestrictedCommands :: Bool enableRestrictedCommands = AutoConf.enableRestrictedCommands -- * SSH constants ssh :: String ssh = "ssh" scp :: String scp = "scp" -- * Daemons confd :: String confd = Runtime.daemonName GanetiConfd masterd :: String masterd = Runtime.daemonName GanetiMasterd metad :: String metad = Runtime.daemonName GanetiMetad mond :: String mond = Runtime.daemonName GanetiMond noded :: String noded = Runtime.daemonName GanetiNoded wconfd :: String wconfd = Runtime.daemonName GanetiWConfd luxid :: String luxid = Runtime.daemonName GanetiLuxid rapi :: String rapi = Runtime.daemonName GanetiRapi kvmd :: String kvmd = Runtime.daemonName GanetiKvmd -- Set of daemons which only run on the master. -- Keep in sync with the 'daemon-util' script. daemonsMaster :: FrozenSet String daemonsMaster = ConstantUtils.mkSet [wconfd, luxid, rapi] daemons :: FrozenSet String daemons = ConstantUtils.mkSet $ map Runtime.daemonName [minBound .. maxBound] defaultConfdPort :: Int defaultConfdPort = 1814 defaultMondPort :: Int defaultMondPort = 1815 defaultMetadPort :: Int defaultMetadPort = 80 defaultNodedPort :: Int defaultNodedPort = 1811 defaultRapiPort :: Int defaultRapiPort = 5080 daemonsPorts :: Map String (Protocol, Int) daemonsPorts = Map.fromList [ (confd, (Udp, defaultConfdPort)) , (metad, (Tcp, defaultMetadPort)) , (mond, (Tcp, defaultMondPort)) , (noded, (Tcp, defaultNodedPort)) , (rapi, (Tcp, defaultRapiPort)) , (ssh, (Tcp, 22)) ] firstDrbdPort :: Int firstDrbdPort = 11000 lastDrbdPort :: Int lastDrbdPort = 14999 daemonsLogbase :: Map String String daemonsLogbase = Map.fromList [ (Runtime.daemonName d, Runtime.daemonLogBase d) | d <- [minBound..] ] daemonsExtraLogbase :: Map String (Map String String) daemonsExtraLogbase = Map.fromList $ map (Runtime.daemonName *** id) [ (GanetiMond, Map.fromList [ ("access", Runtime.daemonsExtraLogbase GanetiMond AccessLog) , ("error", Runtime.daemonsExtraLogbase GanetiMond ErrorLog) ]) , (GanetiMetad, Map.fromList [ ("access", Runtime.daemonsExtraLogbase GanetiMetad AccessLog) , ("error", Runtime.daemonsExtraLogbase GanetiMetad ErrorLog) ]) ] extraLogreasonAccess :: String extraLogreasonAccess = Runtime.daemonsExtraLogbase GanetiMond AccessLog extraLogreasonError :: String extraLogreasonError = Runtime.daemonsExtraLogbase GanetiMond ErrorLog devConsole :: String devConsole = ConstantUtils.devConsole procMounts :: String procMounts = "/proc/mounts" -- * Luxi (Local UniX Interface) related constants luxiEom :: BC.ByteString luxiEom = BC.pack "\x03" -- | Environment variable for the luxi override socket luxiOverride :: String luxiOverride = "FORCE_LUXI_SOCKET" luxiOverrideMaster :: String luxiOverrideMaster = "master" luxiOverrideQuery :: String luxiOverrideQuery = "query" luxiVersion :: Int luxiVersion = configVersion -- * Syslog syslogUsage :: String syslogUsage = AutoConf.syslogUsage syslogNo :: String syslogNo = Logging.syslogUsageToRaw SyslogNo syslogYes :: String syslogYes = Logging.syslogUsageToRaw SyslogYes syslogOnly :: String syslogOnly = Logging.syslogUsageToRaw SyslogOnly syslogSocket :: String syslogSocket = "/dev/log" exportConfFile :: String exportConfFile = "config.ini" -- * Xen xenBootloader :: String xenBootloader = AutoConf.xenBootloader xenInitrd :: String xenInitrd = AutoConf.xenInitrd xenKernel :: String xenKernel = AutoConf.xenKernel xlSocatCmd :: String xlSocatCmd = "socat -b524288 - TCP:%s:%d #" xlMigrationPidfile :: String xlMigrationPidfile = "socat.pid" -- * KVM and socat kvmPath :: String kvmPath = AutoConf.kvmPath kvmKernel :: String kvmKernel = AutoConf.kvmKernel socatEscapeCode :: String socatEscapeCode = "0x1d" socatPath :: String socatPath = AutoConf.socatPath socatUseCompress :: Bool socatUseCompress = AutoConf.socatUseCompress socatUseEscape :: Bool socatUseEscape = AutoConf.socatUseEscape -- * LXC -- If you are trying to change the value of these default constants, you also -- need to edit the default value declaration in man/gnt-instance.rst. lxcDevicesDefault :: String lxcDevicesDefault = "c 1:3 rw" -- /dev/null ++ ",c 1:5 rw" -- /dev/zero ++ ",c 1:7 rw" -- /dev/full ++ ",c 1:8 rw" -- /dev/random ++ ",c 1:9 rw" -- /dev/urandom ++ ",c 1:10 rw" -- /dev/aio ++ ",c 5:0 rw" -- /dev/tty ++ ",c 5:1 rw" -- /dev/console ++ ",c 5:2 rw" -- /dev/ptmx ++ ",c 136:* rw" -- first block of Unix98 PTY slaves lxcDropCapabilitiesDefault :: String lxcDropCapabilitiesDefault = "mac_override" -- Allow MAC configuration or state changes ++ ",sys_boot" -- Use reboot(2) and kexec_load(2) ++ ",sys_module" -- Load and unload kernel modules ++ ",sys_time" -- Set system clock, set real-time (hardware) clock ++ ",sys_admin" -- Various system administration operations lxcStateRunning :: String lxcStateRunning = "RUNNING" -- * Console types -- | Display a message for console access consMessage :: String consMessage = "msg" -- | Console as SPICE server consSpice :: String consSpice = "spice" -- | Console as SSH command consSsh :: String consSsh = "ssh" -- | Console as VNC server consVnc :: String consVnc = "vnc" consAll :: FrozenSet String consAll = ConstantUtils.mkSet [consMessage, consSpice, consSsh, consVnc] -- | RSA key bit length -- -- For RSA keys more bits are better, but they also make operations -- more expensive. NIST SP 800-131 recommends a minimum of 2048 bits -- from the year 2010 on. rsaKeyBits :: Int rsaKeyBits = 2048 -- | Ciphers allowed for SSL connections. -- -- For the format, see ciphers(1). A better way to disable ciphers -- would be to use the exclamation mark (!), but socat versions below -- 1.5 can't parse exclamation marks in options properly. When -- modifying the ciphers, ensure not to accidentially add something -- after it's been removed. Use the "openssl" utility to check the -- allowed ciphers, e.g. "openssl ciphers -v HIGH:-DES". opensslCiphers :: String opensslCiphers = "HIGH:-DES:-3DES:-EXPORT:-DH" -- * X509 -- | commonName (CN) used in certificates x509CertCn :: String x509CertCn = "ganeti.example.com" -- | Default validity of certificates in days x509CertDefaultValidity :: Int x509CertDefaultValidity = 365 * 5 x509CertSignatureHeader :: String x509CertSignatureHeader = "X-Ganeti-Signature" -- | Digest used to sign certificates x509CertSignDigest :: String x509CertSignDigest = "SHA256" -- * Import/export daemon mode iemExport :: String iemExport = "export" iemImport :: String iemImport = "import" -- * Import/export transport compression iecGzip :: String iecGzip = "gzip" iecGzipFast :: String iecGzipFast = "gzip-fast" iecGzipSlow :: String iecGzipSlow = "gzip-slow" iecLzop :: String iecLzop = "lzop" iecNone :: String iecNone = "none" iecAll :: [String] iecAll = [iecGzip, iecGzipFast, iecGzipSlow, iecLzop, iecNone] iecDefaultTools :: [String] iecDefaultTools = [iecGzip, iecGzipFast, iecGzipSlow] iecCompressionUtilities :: Map String String iecCompressionUtilities = Map.fromList [ (iecGzipFast, iecGzip) , (iecGzipSlow, iecGzip) ] ieCustomSize :: String ieCustomSize = "fd" -- * Import/export I/O -- | Direct file I/O, equivalent to a shell's I/O redirection using -- '<' or '>' ieioFile :: String ieioFile = "file" -- | Raw block device I/O using "dd" ieioRawDisk :: String ieioRawDisk = "raw" -- | OS definition import/export script ieioScript :: String ieioScript = "script" -- * Values valueDefault :: String valueDefault = "default" valueAuto :: String valueAuto = "auto" valueGenerate :: String valueGenerate = "generate" valueNone :: String valueNone = "none" valueTrue :: String valueTrue = "true" valueFalse :: String valueFalse = "false" -- * Hooks hooksNameCfgupdate :: String hooksNameCfgupdate = "config-update" hooksNameWatcher :: String hooksNameWatcher = "watcher" hooksPath :: String hooksPath = "/sbin:/bin:/usr/sbin:/usr/bin" hooksPhasePost :: String hooksPhasePost = "post" hooksPhasePre :: String hooksPhasePre = "pre" hooksVersion :: Int hooksVersion = 2 -- * Hooks subject type (what object type does the LU deal with) htypeCluster :: String htypeCluster = "CLUSTER" htypeGroup :: String htypeGroup = "GROUP" htypeInstance :: String htypeInstance = "INSTANCE" htypeNetwork :: String htypeNetwork = "NETWORK" htypeNode :: String htypeNode = "NODE" -- * Hkr hkrSkip :: Int hkrSkip = 0 hkrFail :: Int hkrFail = 1 hkrSuccess :: Int hkrSuccess = 2 -- * Storage types stBlock :: String stBlock = Types.storageTypeToRaw StorageBlock stDiskless :: String stDiskless = Types.storageTypeToRaw StorageDiskless stExt :: String stExt = Types.storageTypeToRaw StorageExt stFile :: String stFile = Types.storageTypeToRaw StorageFile stSharedFile :: String stSharedFile = Types.storageTypeToRaw StorageSharedFile stGluster :: String stGluster = Types.storageTypeToRaw StorageGluster stLvmPv :: String stLvmPv = Types.storageTypeToRaw StorageLvmPv stLvmVg :: String stLvmVg = Types.storageTypeToRaw StorageLvmVg stRados :: String stRados = Types.storageTypeToRaw StorageRados storageTypes :: FrozenSet String storageTypes = ConstantUtils.mkSet $ map Types.storageTypeToRaw [minBound..] -- | The set of storage types for which full storage reporting is available stsReport :: FrozenSet String stsReport = ConstantUtils.mkSet [stFile, stLvmPv, stLvmVg] -- | The set of storage types for which node storage reporting is available -- | (as used by LUQueryNodeStorage) stsReportNodeStorage :: FrozenSet String stsReportNodeStorage = ConstantUtils.union stsReport $ ConstantUtils.mkSet [ stSharedFile , stGluster ] -- * Storage fields -- ** First two are valid in LU context only, not passed to backend sfNode :: String sfNode = "node" sfType :: String sfType = "type" -- ** and the rest are valid in backend sfAllocatable :: String sfAllocatable = Types.storageFieldToRaw SFAllocatable sfFree :: String sfFree = Types.storageFieldToRaw SFFree sfName :: String sfName = Types.storageFieldToRaw SFName sfSize :: String sfSize = Types.storageFieldToRaw SFSize sfUsed :: String sfUsed = Types.storageFieldToRaw SFUsed validStorageFields :: FrozenSet String validStorageFields = ConstantUtils.mkSet $ map Types.storageFieldToRaw [minBound..] ++ [sfNode, sfType] modifiableStorageFields :: Map String (FrozenSet String) modifiableStorageFields = Map.fromList [(Types.storageTypeToRaw StorageLvmPv, ConstantUtils.mkSet [sfAllocatable])] -- * Storage operations soFixConsistency :: String soFixConsistency = "fix-consistency" validStorageOperations :: Map String (FrozenSet String) validStorageOperations = Map.fromList [(Types.storageTypeToRaw StorageLvmVg, ConstantUtils.mkSet [soFixConsistency])] -- * Volume fields vfDev :: String vfDev = "dev" vfInstance :: String vfInstance = "instance" vfName :: String vfName = "name" vfNode :: String vfNode = "node" vfPhys :: String vfPhys = "phys" vfSize :: String vfSize = "size" vfVg :: String vfVg = "vg" -- * Local disk status ldsFaulty :: Int ldsFaulty = Types.localDiskStatusToRaw DiskStatusFaulty ldsOkay :: Int ldsOkay = Types.localDiskStatusToRaw DiskStatusOk ldsUnknown :: Int ldsUnknown = Types.localDiskStatusToRaw DiskStatusUnknown ldsSync :: Int ldsSync = Types.localDiskStatusToRaw DiskStatusSync ldsNames :: Map Int String ldsNames = Map.fromList [ (Types.localDiskStatusToRaw ds, localDiskStatusName ds) | ds <- [minBound..] ] -- * Disk template types dtDiskless :: String dtDiskless = Types.diskTemplateToRaw DTDiskless dtFile :: String dtFile = Types.diskTemplateToRaw DTFile dtSharedFile :: String dtSharedFile = Types.diskTemplateToRaw DTSharedFile dtPlain :: String dtPlain = Types.diskTemplateToRaw DTPlain dtBlock :: String dtBlock = Types.diskTemplateToRaw DTBlock dtDrbd8 :: String dtDrbd8 = Types.diskTemplateToRaw DTDrbd8 dtRbd :: String dtRbd = Types.diskTemplateToRaw DTRbd dtExt :: String dtExt = Types.diskTemplateToRaw DTExt dtGluster :: String dtGluster = Types.diskTemplateToRaw DTGluster dtMixed :: String dtMixed = "mixed" -- | This is used to order determine the default disk template when -- the list of enabled disk templates is inferred from the current -- state of the cluster. This only happens on an upgrade from a -- version of Ganeti that did not support the 'enabled_disk_templates' -- so far. diskTemplatePreference :: [String] diskTemplatePreference = map Types.diskTemplateToRaw [DTBlock, DTDiskless, DTDrbd8, DTExt, DTFile, DTPlain, DTRbd, DTSharedFile, DTGluster] diskTemplates :: FrozenSet String diskTemplates = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [minBound..] -- | Disk templates that are enabled by default defaultEnabledDiskTemplates :: [String] defaultEnabledDiskTemplates = map Types.diskTemplateToRaw [DTDrbd8, DTPlain] -- | Mapping of disk templates to storage types mapDiskTemplateStorageType :: Map String String mapDiskTemplateStorageType = Map.fromList $ map ( Types.diskTemplateToRaw &&& Types.storageTypeToRaw . diskTemplateToStorageType) [minBound..maxBound] -- | The set of network-mirrored disk templates dtsIntMirror :: FrozenSet String dtsIntMirror = ConstantUtils.mkSet [dtDrbd8] -- | 'DTDiskless' is 'trivially' externally mirrored dtsExtMirror :: FrozenSet String dtsExtMirror = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTDiskless, DTBlock, DTExt, DTSharedFile, DTRbd, DTGluster] -- | The set of non-lvm-based disk templates dtsNotLvm :: FrozenSet String dtsNotLvm = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTSharedFile, DTDiskless, DTBlock, DTExt, DTFile, DTRbd, DTGluster] -- | The set of disk templates which can be grown dtsGrowable :: FrozenSet String dtsGrowable = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTSharedFile, DTDrbd8, DTPlain, DTExt, DTFile, DTRbd, DTGluster] -- | The set of disk templates that allow adoption dtsMayAdopt :: FrozenSet String dtsMayAdopt = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTBlock, DTPlain] -- | The set of disk templates that *must* use adoption dtsMustAdopt :: FrozenSet String dtsMustAdopt = ConstantUtils.mkSet [Types.diskTemplateToRaw DTBlock] -- | The set of disk templates that allow migrations dtsMirrored :: FrozenSet String dtsMirrored = dtsIntMirror `ConstantUtils.union` dtsExtMirror -- | The set of file based disk templates dtsFilebased :: FrozenSet String dtsFilebased = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTSharedFile, DTFile, DTGluster] -- | The set of file based disk templates whose path is tied to the instance -- name dtsInstanceDependentPath :: FrozenSet String dtsInstanceDependentPath = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTSharedFile, DTFile] -- | The set of disk templates that can be moved by copying -- -- Note: a requirement is that they're not accessed externally or -- shared between nodes; in particular, sharedfile is not suitable. dtsCopyable :: FrozenSet String dtsCopyable = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTPlain, DTFile] -- | The set of disk templates which can be snapshot. dtsSnapshotCapable :: FrozenSet String dtsSnapshotCapable = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTPlain, DTDrbd8, DTExt] -- | The set of disk templates that are supported by exclusive_storage dtsExclStorage :: FrozenSet String dtsExclStorage = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTPlain] -- | Templates for which we don't perform checks on free space dtsNoFreeSpaceCheck :: FrozenSet String dtsNoFreeSpaceCheck = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTExt, DTSharedFile, DTFile, DTRbd, DTGluster] dtsBlock :: FrozenSet String dtsBlock = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTPlain, DTDrbd8, DTBlock, DTRbd, DTExt] -- | The set of lvm-based disk templates dtsLvm :: FrozenSet String dtsLvm = diskTemplates `ConstantUtils.difference` dtsNotLvm -- | The set of lvm-based disk templates dtsHaveAccess :: FrozenSet String dtsHaveAccess = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTRbd, DTGluster, DTExt] -- | The set of disk templates that cannot convert from dtsNotConvertibleFrom :: FrozenSet String dtsNotConvertibleFrom = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTDiskless] -- | The set of disk templates that cannot convert to dtsNotConvertibleTo :: FrozenSet String dtsNotConvertibleTo = ConstantUtils.mkSet $ map Types.diskTemplateToRaw [DTDiskless, DTBlock] -- * Drbd drbdHmacAlg :: String drbdHmacAlg = "md5" drbdDefaultNetProtocol :: String drbdDefaultNetProtocol = "C" drbdMigrationNetProtocol :: String drbdMigrationNetProtocol = "C" drbdStatusFile :: String drbdStatusFile = "/proc/drbd" -- | The length of generated DRBD secrets (see also TempRes module). drbdSecretLength :: Int drbdSecretLength = 20 -- | Size of DRBD meta block device drbdMetaSize :: Int drbdMetaSize = 128 -- * Drbd barrier types drbdBDiskBarriers :: String drbdBDiskBarriers = "b" drbdBDiskDrain :: String drbdBDiskDrain = "d" drbdBDiskFlush :: String drbdBDiskFlush = "f" drbdBNone :: String drbdBNone = "n" -- | Valid barrier combinations: "n" or any non-null subset of "bfd" drbdValidBarrierOpt :: FrozenSet (FrozenSet String) drbdValidBarrierOpt = ConstantUtils.mkSet [ ConstantUtils.mkSet [drbdBNone] , ConstantUtils.mkSet [drbdBDiskBarriers] , ConstantUtils.mkSet [drbdBDiskDrain] , ConstantUtils.mkSet [drbdBDiskFlush] , ConstantUtils.mkSet [drbdBDiskDrain, drbdBDiskFlush] , ConstantUtils.mkSet [drbdBDiskBarriers, drbdBDiskDrain] , ConstantUtils.mkSet [drbdBDiskBarriers, drbdBDiskFlush] , ConstantUtils.mkSet [drbdBDiskBarriers, drbdBDiskFlush, drbdBDiskDrain] ] -- | Rbd tool command rbdCmd :: String rbdCmd = "rbd" -- * File backend driver fdBlktap :: String fdBlktap = Types.fileDriverToRaw FileBlktap fdBlktap2 :: String fdBlktap2 = Types.fileDriverToRaw FileBlktap2 fdLoop :: String fdLoop = Types.fileDriverToRaw FileLoop fdDefault :: String fdDefault = fdLoop fileDriver :: FrozenSet String fileDriver = ConstantUtils.mkSet $ map Types.fileDriverToRaw [minBound..] -- | The set of drbd-like disk types dtsDrbd :: FrozenSet String dtsDrbd = ConstantUtils.mkSet [Types.diskTemplateToRaw DTDrbd8] -- * Disk access mode diskRdonly :: String diskRdonly = Types.diskModeToRaw DiskRdOnly diskRdwr :: String diskRdwr = Types.diskModeToRaw DiskRdWr diskAccessSet :: FrozenSet String diskAccessSet = ConstantUtils.mkSet $ map Types.diskModeToRaw [minBound..] -- * Disk replacement mode replaceDiskAuto :: String replaceDiskAuto = Types.replaceDisksModeToRaw ReplaceAuto replaceDiskChg :: String replaceDiskChg = Types.replaceDisksModeToRaw ReplaceNewSecondary replaceDiskPri :: String replaceDiskPri = Types.replaceDisksModeToRaw ReplaceOnPrimary replaceDiskSec :: String replaceDiskSec = Types.replaceDisksModeToRaw ReplaceOnSecondary replaceModes :: FrozenSet String replaceModes = ConstantUtils.mkSet $ map Types.replaceDisksModeToRaw [minBound..] -- * Instance export mode exportModeLocal :: String exportModeLocal = Types.exportModeToRaw ExportModeLocal exportModeRemote :: String exportModeRemote = Types.exportModeToRaw ExportModeRemote exportModes :: FrozenSet String exportModes = ConstantUtils.mkSet $ map Types.exportModeToRaw [minBound..] -- * Instance creation modes instanceCreate :: String instanceCreate = Types.instCreateModeToRaw InstCreate instanceImport :: String instanceImport = Types.instCreateModeToRaw InstImport instanceRemoteImport :: String instanceRemoteImport = Types.instCreateModeToRaw InstRemoteImport instanceCreateModes :: FrozenSet String instanceCreateModes = ConstantUtils.mkSet $ map Types.instCreateModeToRaw [minBound..] -- * Remote import/export handshake message and version rieHandshake :: String rieHandshake = "Hi, I'm Ganeti" rieVersion :: Int rieVersion = 0 -- | Remote import/export certificate validity (seconds) rieCertValidity :: Int rieCertValidity = 24 * 60 * 60 -- | Export only: how long to wait per connection attempt (seconds) rieConnectAttemptTimeout :: Int rieConnectAttemptTimeout = 20 -- | Export only: number of attempts to connect rieConnectRetries :: Int rieConnectRetries = 10 -- | Overall timeout for establishing connection rieConnectTimeout :: Int rieConnectTimeout = 180 -- | Give child process up to 5 seconds to exit after sending a signal childLingerTimeout :: Double childLingerTimeout = 5.0 -- * Import/export config options inisectBep :: String inisectBep = "backend" inisectExp :: String inisectExp = "export" inisectHyp :: String inisectHyp = "hypervisor" inisectIns :: String inisectIns = "instance" inisectOsp :: String inisectOsp = "os" inisectOspPrivate :: String inisectOspPrivate = "os_private" -- * Dynamic device modification ddmAdd :: String ddmAdd = Types.ddmFullToRaw DdmFullAdd ddmAttach :: String ddmAttach = Types.ddmFullToRaw DdmFullAttach ddmModify :: String ddmModify = Types.ddmFullToRaw DdmFullModify ddmRemove :: String ddmRemove = Types.ddmFullToRaw DdmFullRemove ddmDetach :: String ddmDetach = Types.ddmFullToRaw DdmFullDetach ddmsValues :: FrozenSet String ddmsValues = ConstantUtils.mkSet [ddmAdd, ddmAttach, ddmRemove, ddmDetach] ddmsValuesWithModify :: FrozenSet String ddmsValuesWithModify = ConstantUtils.mkSet $ map Types.ddmFullToRaw [minBound..] -- * Common exit codes exitSuccess :: Int exitSuccess = 0 exitFailure :: Int exitFailure = ConstantUtils.exitFailure exitNotcluster :: Int exitNotcluster = 5 exitNotmaster :: Int exitNotmaster = 11 exitNodesetupError :: Int exitNodesetupError = 12 -- | Need user confirmation exitConfirmation :: Int exitConfirmation = 13 -- | Exit code for query operations with unknown fields exitUnknownField :: Int exitUnknownField = 14 -- * Tags tagCluster :: String tagCluster = Types.tagKindToRaw TagKindCluster tagInstance :: String tagInstance = Types.tagKindToRaw TagKindInstance tagNetwork :: String tagNetwork = Types.tagKindToRaw TagKindNetwork tagNode :: String tagNode = Types.tagKindToRaw TagKindNode tagNodegroup :: String tagNodegroup = Types.tagKindToRaw TagKindGroup validTagTypes :: FrozenSet String validTagTypes = ConstantUtils.mkSet $ map Types.tagKindToRaw [minBound..] maxTagLen :: Int maxTagLen = 128 maxTagsPerObj :: Int maxTagsPerObj = 4096 -- * Others defaultBridge :: String defaultBridge = AutoConf.defaultBridge defaultOvs :: String defaultOvs = "switch1" -- | 60 MiB/s, expressed in KiB/s classicDrbdSyncSpeed :: Int classicDrbdSyncSpeed = 60 * 1024 ip4AddressAny :: String ip4AddressAny = "0.0.0.0" ip4AddressLocalhost :: String ip4AddressLocalhost = "127.0.0.1" ip6AddressAny :: String ip6AddressAny = "::" ip6AddressLocalhost :: String ip6AddressLocalhost = "::1" ip4Version :: Int ip4Version = 4 ip6Version :: Int ip6Version = 6 validIpVersions :: FrozenSet Int validIpVersions = ConstantUtils.mkSet [ip4Version, ip6Version] tcpPingTimeout :: Int tcpPingTimeout = 10 defaultVg :: String defaultVg = AutoConf.defaultVg defaultDrbdHelper :: String defaultDrbdHelper = "/bin/true" minVgSize :: Int minVgSize = 20480 defaultMacPrefix :: String defaultMacPrefix = "aa:00:00" -- | Default maximum instance wait time (seconds) defaultShutdownTimeout :: Int defaultShutdownTimeout = 120 -- | Node clock skew (seconds) nodeMaxClockSkew :: Int nodeMaxClockSkew = 150 -- | Time for an intra-cluster disk transfer to wait for a connection diskTransferConnectTimeout :: Int diskTransferConnectTimeout = 60 -- | Disk index separator diskSeparator :: String diskSeparator = AutoConf.diskSeparator ipCommandPath :: String ipCommandPath = AutoConf.ipPath -- | Key for job IDs in opcode result jobIdsKey :: String jobIdsKey = "jobs" -- * Runparts results runpartsErr :: Int runpartsErr = 2 runpartsRun :: Int runpartsRun = 1 runpartsSkip :: Int runpartsSkip = 0 runpartsStatus :: [Int] runpartsStatus = [runpartsErr, runpartsRun, runpartsSkip] -- * RPC rpcEncodingNone :: Int rpcEncodingNone = 0 rpcEncodingZlibBase64 :: Int rpcEncodingZlibBase64 = 1 -- * Timeout table -- -- Various time constants for the timeout table rpcTmoUrgent :: Int rpcTmoUrgent = Types.rpcTimeoutToRaw Urgent rpcTmoFast :: Int rpcTmoFast = Types.rpcTimeoutToRaw Fast rpcTmoNormal :: Int rpcTmoNormal = Types.rpcTimeoutToRaw Normal rpcTmoSlow :: Int rpcTmoSlow = Types.rpcTimeoutToRaw Slow -- | 'rpcTmo_4hrs' contains an underscore to circumvent a limitation -- in the 'Ganeti.THH.deCamelCase' function and generate the correct -- Python name. rpcTmo_4hrs :: Int rpcTmo_4hrs = Types.rpcTimeoutToRaw FourHours -- | 'rpcTmo_1day' contains an underscore to circumvent a limitation -- in the 'Ganeti.THH.deCamelCase' function and generate the correct -- Python name. rpcTmo_1day :: Int rpcTmo_1day = Types.rpcTimeoutToRaw OneDay -- | Timeout for connecting to nodes (seconds) rpcConnectTimeout :: Int rpcConnectTimeout = 5 -- OS osScriptCreate :: String osScriptCreate = "create" osScriptCreateUntrusted :: String osScriptCreateUntrusted = "create_untrusted" osScriptExport :: String osScriptExport = "export" osScriptImport :: String osScriptImport = "import" osScriptRename :: String osScriptRename = "rename" osScriptVerify :: String osScriptVerify = "verify" osScripts :: [String] osScripts = [osScriptCreate, osScriptCreateUntrusted, osScriptExport, osScriptImport, osScriptRename, osScriptVerify] osApiFile :: String osApiFile = "ganeti_api_version" osVariantsFile :: String osVariantsFile = "variants.list" osParametersFile :: String osParametersFile = "parameters.list" osValidateParameters :: String osValidateParameters = "parameters" osValidateCalls :: FrozenSet String osValidateCalls = ConstantUtils.mkSet [osValidateParameters] -- | External Storage (ES) related constants esActionAttach :: String esActionAttach = "attach" esActionCreate :: String esActionCreate = "create" esActionDetach :: String esActionDetach = "detach" esActionGrow :: String esActionGrow = "grow" esActionRemove :: String esActionRemove = "remove" esActionSetinfo :: String esActionSetinfo = "setinfo" esActionVerify :: String esActionVerify = "verify" esActionSnapshot :: String esActionSnapshot = "snapshot" esActionOpen :: String esActionOpen = "open" esActionClose :: String esActionClose = "close" esScriptCreate :: String esScriptCreate = esActionCreate esScriptRemove :: String esScriptRemove = esActionRemove esScriptGrow :: String esScriptGrow = esActionGrow esScriptAttach :: String esScriptAttach = esActionAttach esScriptDetach :: String esScriptDetach = esActionDetach esScriptSetinfo :: String esScriptSetinfo = esActionSetinfo esScriptVerify :: String esScriptVerify = esActionVerify esScriptSnapshot :: String esScriptSnapshot = esActionSnapshot esScriptOpen :: String esScriptOpen = esActionOpen esScriptClose :: String esScriptClose = esActionClose esScripts :: FrozenSet String esScripts = ConstantUtils.mkSet [esScriptAttach, esScriptCreate, esScriptDetach, esScriptGrow, esScriptRemove, esScriptSetinfo, esScriptVerify, esScriptSnapshot, esScriptOpen, esScriptClose] esParametersFile :: String esParametersFile = "parameters.list" -- * Reboot types instanceRebootSoft :: String instanceRebootSoft = Types.rebootTypeToRaw RebootSoft instanceRebootHard :: String instanceRebootHard = Types.rebootTypeToRaw RebootHard instanceRebootFull :: String instanceRebootFull = Types.rebootTypeToRaw RebootFull rebootTypes :: FrozenSet String rebootTypes = ConstantUtils.mkSet $ map Types.rebootTypeToRaw [minBound..] -- * Instance reboot behaviors instanceRebootAllowed :: String instanceRebootAllowed = "reboot" instanceRebootExit :: String instanceRebootExit = "exit" rebootBehaviors :: [String] rebootBehaviors = [instanceRebootAllowed, instanceRebootExit] -- * VTypes vtypeBool :: VType vtypeBool = VTypeBool vtypeInt :: VType vtypeInt = VTypeInt vtypeFloat :: VType vtypeFloat = VTypeFloat vtypeMaybeString :: VType vtypeMaybeString = VTypeMaybeString -- | Size in MiBs vtypeSize :: VType vtypeSize = VTypeSize vtypeString :: VType vtypeString = VTypeString enforceableTypes :: FrozenSet VType enforceableTypes = ConstantUtils.mkSet [minBound..] -- | Constant representing that the user does not specify any IP version ifaceNoIpVersionSpecified :: Int ifaceNoIpVersionSpecified = 0 validSerialSpeeds :: [Int] validSerialSpeeds = [75, 110, 300, 600, 1200, 1800, 2400, 4800, 9600, 14400, 19200, 28800, 38400, 57600, 115200, 230400, 345600, 460800] -- * HV parameter names (global namespace) hvAcpi :: String hvAcpi = "acpi" hvBlockdevPrefix :: String hvBlockdevPrefix = "blockdev_prefix" hvBootloaderArgs :: String hvBootloaderArgs = "bootloader_args" hvBootloaderPath :: String hvBootloaderPath = "bootloader_path" hvBootOrder :: String hvBootOrder = "boot_order" hvCdromImagePath :: String hvCdromImagePath = "cdrom_image_path" hvCpuCap :: String hvCpuCap = "cpu_cap" hvCpuCores :: String hvCpuCores = "cpu_cores" hvCpuMask :: String hvCpuMask = "cpu_mask" hvCpuSockets :: String hvCpuSockets = "cpu_sockets" hvCpuThreads :: String hvCpuThreads = "cpu_threads" hvCpuType :: String hvCpuType = "cpu_type" hvCpuWeight :: String hvCpuWeight = "cpu_weight" hvDeviceModel :: String hvDeviceModel = "device_model" hvDiskCache :: String hvDiskCache = "disk_cache" hvDiskDiscard :: String hvDiskDiscard = "disk_discard" hvDiskType :: String hvDiskType = "disk_type" hvInitrdPath :: String hvInitrdPath = "initrd_path" hvInitScript :: String hvInitScript = "init_script" hvKernelArgs :: String hvKernelArgs = "kernel_args" hvKernelPath :: String hvKernelPath = "kernel_path" hvKeymap :: String hvKeymap = "keymap" hvKvmCdrom2ImagePath :: String hvKvmCdrom2ImagePath = "cdrom2_image_path" hvKvmCdromDiskType :: String hvKvmCdromDiskType = "cdrom_disk_type" hvKvmExtra :: String hvKvmExtra = "kvm_extra" hvKvmFlag :: String hvKvmFlag = "kvm_flag" hvKvmFloppyImagePath :: String hvKvmFloppyImagePath = "floppy_image_path" hvKvmMachineVersion :: String hvKvmMachineVersion = "machine_version" hvKvmMigrationCaps :: String hvKvmMigrationCaps = "migration_caps" hvKvmPath :: String hvKvmPath = "kvm_path" hvKvmDiskAio :: String hvKvmDiskAio = "disk_aio" hvKvmScsiControllerType :: String hvKvmScsiControllerType = "scsi_controller_type" hvKvmPciReservations :: String hvKvmPciReservations = "kvm_pci_reservations" hvKvmSpiceAudioCompr :: String hvKvmSpiceAudioCompr = "spice_playback_compression" hvKvmSpiceBind :: String hvKvmSpiceBind = "spice_bind" hvKvmSpiceIpVersion :: String hvKvmSpiceIpVersion = "spice_ip_version" hvKvmSpiceJpegImgCompr :: String hvKvmSpiceJpegImgCompr = "spice_jpeg_wan_compression" hvKvmSpiceLosslessImgCompr :: String hvKvmSpiceLosslessImgCompr = "spice_image_compression" hvKvmSpicePasswordFile :: String hvKvmSpicePasswordFile = "spice_password_file" hvKvmSpiceStreamingVideoDetection :: String hvKvmSpiceStreamingVideoDetection = "spice_streaming_video" hvKvmSpiceTlsCiphers :: String hvKvmSpiceTlsCiphers = "spice_tls_ciphers" hvKvmSpiceUseTls :: String hvKvmSpiceUseTls = "spice_use_tls" hvKvmSpiceUseVdagent :: String hvKvmSpiceUseVdagent = "spice_use_vdagent" hvKvmSpiceZlibGlzImgCompr :: String hvKvmSpiceZlibGlzImgCompr = "spice_zlib_glz_wan_compression" hvKvmUseChroot :: String hvKvmUseChroot = "use_chroot" hvKvmUserShutdown :: String hvKvmUserShutdown = "user_shutdown" hvLxcStartupTimeout :: String hvLxcStartupTimeout = "startup_timeout" hvLxcExtraCgroups :: String hvLxcExtraCgroups = "extra_cgroups" hvLxcDevices :: String hvLxcDevices = "devices" hvLxcDropCapabilities :: String hvLxcDropCapabilities = "drop_capabilities" hvLxcExtraConfig :: String hvLxcExtraConfig = "extra_config" hvLxcNumTtys :: String hvLxcNumTtys = "num_ttys" hvMemPath :: String hvMemPath = "mem_path" hvMigrationBandwidth :: String hvMigrationBandwidth = "migration_bandwidth" hvMigrationDowntime :: String hvMigrationDowntime = "migration_downtime" hvMigrationMode :: String hvMigrationMode = "migration_mode" hvMigrationPort :: String hvMigrationPort = "migration_port" hvNicType :: String hvNicType = "nic_type" hvPae :: String hvPae = "pae" hvPassthrough :: String hvPassthrough = "pci_pass" hvRebootBehavior :: String hvRebootBehavior = "reboot_behavior" hvRootPath :: String hvRootPath = "root_path" hvSecurityDomain :: String hvSecurityDomain = "security_domain" hvSecurityModel :: String hvSecurityModel = "security_model" hvSerialConsole :: String hvSerialConsole = "serial_console" hvSerialSpeed :: String hvSerialSpeed = "serial_speed" hvSoundhw :: String hvSoundhw = "soundhw" hvUsbDevices :: String hvUsbDevices = "usb_devices" hvUsbMouse :: String hvUsbMouse = "usb_mouse" hvUseBootloader :: String hvUseBootloader = "use_bootloader" hvUseGuestAgent :: String hvUseGuestAgent = "use_guest_agent" hvUseLocaltime :: String hvUseLocaltime = "use_localtime" hvVga :: String hvVga = "vga" hvVhostNet :: String hvVhostNet = "vhost_net" hvVirtioNetQueues :: String hvVirtioNetQueues = "virtio_net_queues" hvVifScript :: String hvVifScript = "vif_script" hvVifType :: String hvVifType = "vif_type" hvViridian :: String hvViridian = "viridian" hvVncBindAddress :: String hvVncBindAddress = "vnc_bind_address" hvVncPasswordFile :: String hvVncPasswordFile = "vnc_password_file" hvVncTls :: String hvVncTls = "vnc_tls" hvVncX509 :: String hvVncX509 = "vnc_x509_path" hvVncX509Verify :: String hvVncX509Verify = "vnc_x509_verify" hvVnetHdr :: String hvVnetHdr = "vnet_hdr" hvXenCpuid :: String hvXenCpuid = "cpuid" hvsParameterTitles :: Map String String hvsParameterTitles = Map.fromList [(hvAcpi, "ACPI"), (hvBootOrder, "Boot_order"), (hvCdromImagePath, "CDROM_image_path"), (hvCpuType, "cpu_type"), (hvDiskType, "Disk_type"), (hvInitrdPath, "Initrd_path"), (hvKernelPath, "Kernel_path"), (hvNicType, "NIC_type"), (hvPae, "PAE"), (hvPassthrough, "pci_pass"), (hvVncBindAddress, "VNC_bind_address")] hvsParameters :: FrozenSet String hvsParameters = ConstantUtils.mkSet $ Map.keys hvsParameterTypes hvsParameterTypes :: Map String VType hvsParameterTypes = Map.fromList [ (hvAcpi, VTypeBool) , (hvBlockdevPrefix, VTypeString) , (hvBootloaderArgs, VTypeString) , (hvBootloaderPath, VTypeString) , (hvBootOrder, VTypeString) , (hvCdromImagePath, VTypeString) , (hvCpuCap, VTypeInt) , (hvCpuCores, VTypeInt) , (hvCpuMask, VTypeString) , (hvCpuSockets, VTypeInt) , (hvCpuThreads, VTypeInt) , (hvCpuType, VTypeString) , (hvCpuWeight, VTypeInt) , (hvDeviceModel, VTypeString) , (hvDiskCache, VTypeString) , (hvDiskDiscard, VTypeString) , (hvDiskType, VTypeString) , (hvInitrdPath, VTypeString) , (hvInitScript, VTypeString) , (hvKernelArgs, VTypeString) , (hvKernelPath, VTypeString) , (hvKeymap, VTypeString) , (hvKvmCdrom2ImagePath, VTypeString) , (hvKvmCdromDiskType, VTypeString) , (hvKvmExtra, VTypeString) , (hvKvmFlag, VTypeString) , (hvKvmFloppyImagePath, VTypeString) , (hvKvmMachineVersion, VTypeString) , (hvKvmMigrationCaps, VTypeString) , (hvKvmPath, VTypeString) , (hvKvmDiskAio, VTypeString) , (hvKvmScsiControllerType, VTypeString) , (hvKvmPciReservations, VTypeInt) , (hvKvmSpiceAudioCompr, VTypeBool) , (hvKvmSpiceBind, VTypeString) , (hvKvmSpiceIpVersion, VTypeInt) , (hvKvmSpiceJpegImgCompr, VTypeString) , (hvKvmSpiceLosslessImgCompr, VTypeString) , (hvKvmSpicePasswordFile, VTypeString) , (hvKvmSpiceStreamingVideoDetection, VTypeString) , (hvKvmSpiceTlsCiphers, VTypeString) , (hvKvmSpiceUseTls, VTypeBool) , (hvKvmSpiceUseVdagent, VTypeBool) , (hvKvmSpiceZlibGlzImgCompr, VTypeString) , (hvKvmUseChroot, VTypeBool) , (hvKvmUserShutdown, VTypeBool) , (hvLxcDevices, VTypeString) , (hvLxcDropCapabilities, VTypeString) , (hvLxcExtraCgroups, VTypeString) , (hvLxcExtraConfig, VTypeString) , (hvLxcNumTtys, VTypeInt) , (hvLxcStartupTimeout, VTypeInt) , (hvMemPath, VTypeString) , (hvMigrationBandwidth, VTypeInt) , (hvMigrationDowntime, VTypeInt) , (hvMigrationMode, VTypeString) , (hvMigrationPort, VTypeInt) , (hvNicType, VTypeString) , (hvPae, VTypeBool) , (hvPassthrough, VTypeString) , (hvRebootBehavior, VTypeString) , (hvRootPath, VTypeMaybeString) , (hvSecurityDomain, VTypeString) , (hvSecurityModel, VTypeString) , (hvSerialConsole, VTypeBool) , (hvSerialSpeed, VTypeInt) , (hvSoundhw, VTypeString) , (hvUsbDevices, VTypeString) , (hvUsbMouse, VTypeString) , (hvUseBootloader, VTypeBool) , (hvUseGuestAgent, VTypeBool) , (hvUseLocaltime, VTypeBool) , (hvVga, VTypeString) , (hvVhostNet, VTypeBool) , (hvVirtioNetQueues, VTypeInt) , (hvVifScript, VTypeString) , (hvVifType, VTypeString) , (hvViridian, VTypeBool) , (hvVncBindAddress, VTypeString) , (hvVncPasswordFile, VTypeString) , (hvVncTls, VTypeBool) , (hvVncX509, VTypeString) , (hvVncX509Verify, VTypeBool) , (hvVnetHdr, VTypeBool) , (hvXenCpuid, VTypeString) ] -- * Migration statuses hvMigrationActive :: String hvMigrationActive = "active" hvMigrationCancelled :: String hvMigrationCancelled = "cancelled" hvMigrationCompleted :: String hvMigrationCompleted = "completed" hvMigrationFailed :: String hvMigrationFailed = "failed" hvMigrationValidStatuses :: FrozenSet String hvMigrationValidStatuses = ConstantUtils.mkSet [hvMigrationActive, hvMigrationCancelled, hvMigrationCompleted, hvMigrationFailed] hvMigrationFailedStatuses :: FrozenSet String hvMigrationFailedStatuses = ConstantUtils.mkSet [hvMigrationFailed, hvMigrationCancelled] -- | KVM-specific statuses -- hvKvmMigrationPostcopyActive :: String hvKvmMigrationPostcopyActive = "postcopy-active" hvKvmMigrationValidStatuses :: FrozenSet String hvKvmMigrationValidStatuses = ConstantUtils.union hvMigrationValidStatuses (ConstantUtils.mkSet [hvKvmMigrationPostcopyActive]) hvKvmMigrationActiveStatuses :: FrozenSet String hvKvmMigrationActiveStatuses = ConstantUtils.mkSet [hvMigrationActive, hvKvmMigrationPostcopyActive] -- | Node info keys hvNodeinfoKeyVersion :: String hvNodeinfoKeyVersion = "hv_version" -- * Hypervisor state hvstCpuNode :: String hvstCpuNode = "cpu_node" hvstCpuTotal :: String hvstCpuTotal = "cpu_total" hvstMemoryHv :: String hvstMemoryHv = "mem_hv" hvstMemoryNode :: String hvstMemoryNode = "mem_node" hvstMemoryTotal :: String hvstMemoryTotal = "mem_total" hvstsParameters :: FrozenSet String hvstsParameters = ConstantUtils.mkSet [hvstCpuNode, hvstCpuTotal, hvstMemoryHv, hvstMemoryNode, hvstMemoryTotal] hvstDefaults :: Map String Int hvstDefaults = Map.fromList [(hvstCpuNode, 1), (hvstCpuTotal, 1), (hvstMemoryHv, 0), (hvstMemoryTotal, 0), (hvstMemoryNode, 0)] hvstsParameterTypes :: Map String VType hvstsParameterTypes = Map.fromList [(hvstMemoryTotal, VTypeInt), (hvstMemoryNode, VTypeInt), (hvstMemoryHv, VTypeInt), (hvstCpuTotal, VTypeInt), (hvstCpuNode, VTypeInt)] -- * Disk state dsDiskOverhead :: String dsDiskOverhead = "disk_overhead" dsDiskReserved :: String dsDiskReserved = "disk_reserved" dsDiskTotal :: String dsDiskTotal = "disk_total" dsDefaults :: Map String Int dsDefaults = Map.fromList [(dsDiskTotal, 0), (dsDiskReserved, 0), (dsDiskOverhead, 0)] dssParameterTypes :: Map String VType dssParameterTypes = Map.fromList [(dsDiskTotal, VTypeInt), (dsDiskReserved, VTypeInt), (dsDiskOverhead, VTypeInt)] dssParameters :: FrozenSet String dssParameters = ConstantUtils.mkSet [dsDiskTotal, dsDiskReserved, dsDiskOverhead] dsValidTypes :: FrozenSet String dsValidTypes = ConstantUtils.mkSet [Types.diskTemplateToRaw DTPlain] -- Backend parameter names beAlwaysFailover :: String beAlwaysFailover = "always_failover" beAutoBalance :: String beAutoBalance = "auto_balance" beMaxmem :: String beMaxmem = "maxmem" -- | Deprecated and replaced by max and min mem beMemory :: String beMemory = "memory" beMinmem :: String beMinmem = "minmem" beSpindleUse :: String beSpindleUse = "spindle_use" beVcpus :: String beVcpus = "vcpus" besParameterTypes :: Map String VType besParameterTypes = Map.fromList [(beAlwaysFailover, VTypeBool), (beAutoBalance, VTypeBool), (beMaxmem, VTypeSize), (beMinmem, VTypeSize), (beSpindleUse, VTypeInt), (beVcpus, VTypeInt)] besParameterTitles :: Map String String besParameterTitles = Map.fromList [(beAutoBalance, "Auto_balance"), (beMinmem, "ConfigMinMem"), (beVcpus, "ConfigVCPUs"), (beMaxmem, "ConfigMaxMem")] besParameterCompat :: Map String VType besParameterCompat = Map.insert beMemory VTypeSize besParameterTypes besParameters :: FrozenSet String besParameters = ConstantUtils.mkSet [beAlwaysFailover, beAutoBalance, beMaxmem, beMinmem, beSpindleUse, beVcpus] -- | Instance specs -- -- FIXME: these should be associated with 'Ganeti.HTools.Types.ISpec' ispecMemSize :: String ispecMemSize = ConstantUtils.ispecMemSize ispecCpuCount :: String ispecCpuCount = ConstantUtils.ispecCpuCount ispecDiskCount :: String ispecDiskCount = ConstantUtils.ispecDiskCount ispecDiskSize :: String ispecDiskSize = ConstantUtils.ispecDiskSize ispecNicCount :: String ispecNicCount = ConstantUtils.ispecNicCount ispecSpindleUse :: String ispecSpindleUse = ConstantUtils.ispecSpindleUse ispecsParameterTypes :: Map String VType ispecsParameterTypes = Map.fromList [(ConstantUtils.ispecDiskSize, VTypeInt), (ConstantUtils.ispecCpuCount, VTypeInt), (ConstantUtils.ispecSpindleUse, VTypeInt), (ConstantUtils.ispecMemSize, VTypeInt), (ConstantUtils.ispecNicCount, VTypeInt), (ConstantUtils.ispecDiskCount, VTypeInt)] ispecsParameters :: FrozenSet String ispecsParameters = ConstantUtils.mkSet [ConstantUtils.ispecCpuCount, ConstantUtils.ispecDiskCount, ConstantUtils.ispecDiskSize, ConstantUtils.ispecMemSize, ConstantUtils.ispecNicCount, ConstantUtils.ispecSpindleUse] ispecsMinmax :: String ispecsMinmax = ConstantUtils.ispecsMinmax ispecsMax :: String ispecsMax = "max" ispecsMin :: String ispecsMin = "min" ispecsStd :: String ispecsStd = ConstantUtils.ispecsStd ipolicyDts :: String ipolicyDts = ConstantUtils.ipolicyDts ipolicyVcpuRatio :: String ipolicyVcpuRatio = ConstantUtils.ipolicyVcpuRatio ipolicySpindleRatio :: String ipolicySpindleRatio = ConstantUtils.ipolicySpindleRatio ispecsMinmaxKeys :: FrozenSet String ispecsMinmaxKeys = ConstantUtils.mkSet [ispecsMax, ispecsMin] ipolicyParameters :: FrozenSet String ipolicyParameters = ConstantUtils.mkSet [ConstantUtils.ipolicyVcpuRatio, ConstantUtils.ipolicySpindleRatio] ipolicyAllKeys :: FrozenSet String ipolicyAllKeys = ConstantUtils.union ipolicyParameters $ ConstantUtils.mkSet [ConstantUtils.ipolicyDts, ConstantUtils.ispecsMinmax, ispecsStd] -- | Node parameter names ndExclusiveStorage :: String ndExclusiveStorage = "exclusive_storage" ndOobProgram :: String ndOobProgram = "oob_program" ndSpindleCount :: String ndSpindleCount = "spindle_count" ndOvs :: String ndOvs = "ovs" ndOvsLink :: String ndOvsLink = "ovs_link" ndOvsName :: String ndOvsName = "ovs_name" ndSshPort :: String ndSshPort = "ssh_port" ndCpuSpeed :: String ndCpuSpeed = "cpu_speed" ndsParameterTypes :: Map String VType ndsParameterTypes = Map.fromList [(ndExclusiveStorage, VTypeBool), (ndOobProgram, VTypeString), (ndOvs, VTypeBool), (ndOvsLink, VTypeMaybeString), (ndOvsName, VTypeMaybeString), (ndSpindleCount, VTypeInt), (ndSshPort, VTypeInt), (ndCpuSpeed, VTypeFloat)] ndsParameters :: FrozenSet String ndsParameters = ConstantUtils.mkSet (Map.keys ndsParameterTypes) ndsParameterTitles :: Map String String ndsParameterTitles = Map.fromList [(ndExclusiveStorage, "ExclusiveStorage"), (ndOobProgram, "OutOfBandProgram"), (ndOvs, "OpenvSwitch"), (ndOvsLink, "OpenvSwitchLink"), (ndOvsName, "OpenvSwitchName"), (ndSpindleCount, "SpindleCount")] -- * Logical Disks parameters ldpAccess :: String ldpAccess = "access" ldpBarriers :: String ldpBarriers = "disabled-barriers" ldpDefaultMetavg :: String ldpDefaultMetavg = "default-metavg" ldpDelayTarget :: String ldpDelayTarget = "c-delay-target" ldpDiskCustom :: String ldpDiskCustom = "disk-custom" ldpDynamicResync :: String ldpDynamicResync = "dynamic-resync" ldpFillTarget :: String ldpFillTarget = "c-fill-target" ldpMaxRate :: String ldpMaxRate = "c-max-rate" ldpMinRate :: String ldpMinRate = "c-min-rate" ldpNetCustom :: String ldpNetCustom = "net-custom" ldpNoMetaFlush :: String ldpNoMetaFlush = "disable-meta-flush" ldpPlanAhead :: String ldpPlanAhead = "c-plan-ahead" ldpPool :: String ldpPool = "pool" ldpProtocol :: String ldpProtocol = "protocol" ldpResyncRate :: String ldpResyncRate = "resync-rate" ldpStripes :: String ldpStripes = "stripes" diskLdTypes :: Map String VType diskLdTypes = Map.fromList [(ldpAccess, VTypeString), (ldpResyncRate, VTypeInt), (ldpStripes, VTypeInt), (ldpBarriers, VTypeString), (ldpNoMetaFlush, VTypeBool), (ldpDefaultMetavg, VTypeString), (ldpDiskCustom, VTypeString), (ldpNetCustom, VTypeString), (ldpProtocol, VTypeString), (ldpDynamicResync, VTypeBool), (ldpPlanAhead, VTypeInt), (ldpFillTarget, VTypeInt), (ldpDelayTarget, VTypeInt), (ldpMaxRate, VTypeInt), (ldpMinRate, VTypeInt), (ldpPool, VTypeString)] diskLdParameters :: FrozenSet String diskLdParameters = ConstantUtils.mkSet (Map.keys diskLdTypes) -- * Disk template parameters -- -- Disk template parameters can be set/changed by the user via -- gnt-cluster and gnt-group) drbdResyncRate :: String drbdResyncRate = "resync-rate" drbdDataStripes :: String drbdDataStripes = "data-stripes" drbdMetaStripes :: String drbdMetaStripes = "meta-stripes" drbdDiskBarriers :: String drbdDiskBarriers = "disk-barriers" drbdMetaBarriers :: String drbdMetaBarriers = "meta-barriers" drbdDefaultMetavg :: String drbdDefaultMetavg = "metavg" drbdDiskCustom :: String drbdDiskCustom = "disk-custom" drbdNetCustom :: String drbdNetCustom = "net-custom" drbdProtocol :: String drbdProtocol = "protocol" drbdDynamicResync :: String drbdDynamicResync = "dynamic-resync" drbdPlanAhead :: String drbdPlanAhead = "c-plan-ahead" drbdFillTarget :: String drbdFillTarget = "c-fill-target" drbdDelayTarget :: String drbdDelayTarget = "c-delay-target" drbdMaxRate :: String drbdMaxRate = "c-max-rate" drbdMinRate :: String drbdMinRate = "c-min-rate" lvStripes :: String lvStripes = "stripes" rbdAccess :: String rbdAccess = "access" rbdPool :: String rbdPool = "pool" diskDtTypes :: Map String VType diskDtTypes = Map.fromList [(drbdResyncRate, VTypeInt), (drbdDataStripes, VTypeInt), (drbdMetaStripes, VTypeInt), (drbdDiskBarriers, VTypeString), (drbdMetaBarriers, VTypeBool), (drbdDefaultMetavg, VTypeString), (drbdDiskCustom, VTypeString), (drbdNetCustom, VTypeString), (drbdProtocol, VTypeString), (drbdDynamicResync, VTypeBool), (drbdPlanAhead, VTypeInt), (drbdFillTarget, VTypeInt), (drbdDelayTarget, VTypeInt), (drbdMaxRate, VTypeInt), (drbdMinRate, VTypeInt), (lvStripes, VTypeInt), (rbdAccess, VTypeString), (rbdPool, VTypeString), (glusterHost, VTypeString), (glusterVolume, VTypeString), (glusterPort, VTypeInt) ] diskDtParameters :: FrozenSet String diskDtParameters = ConstantUtils.mkSet (Map.keys diskDtTypes) -- * Dynamic disk parameters ddpLocalIp :: String ddpLocalIp = "local-ip" ddpRemoteIp :: String ddpRemoteIp = "remote-ip" ddpPort :: String ddpPort = "port" ddpLocalMinor :: String ddpLocalMinor = "local-minor" ddpRemoteMinor :: String ddpRemoteMinor = "remote-minor" -- * OOB supported commands oobPowerOn :: String oobPowerOn = Types.oobCommandToRaw OobPowerOn oobPowerOff :: String oobPowerOff = Types.oobCommandToRaw OobPowerOff oobPowerCycle :: String oobPowerCycle = Types.oobCommandToRaw OobPowerCycle oobPowerStatus :: String oobPowerStatus = Types.oobCommandToRaw OobPowerStatus oobHealth :: String oobHealth = Types.oobCommandToRaw OobHealth oobCommands :: FrozenSet String oobCommands = ConstantUtils.mkSet $ map Types.oobCommandToRaw [minBound..] oobPowerStatusPowered :: String oobPowerStatusPowered = "powered" -- | 60 seconds oobTimeout :: Int oobTimeout = 60 -- | 2 seconds oobPowerDelay :: Double oobPowerDelay = 2.0 oobStatusCritical :: String oobStatusCritical = Types.oobStatusToRaw OobStatusCritical oobStatusOk :: String oobStatusOk = Types.oobStatusToRaw OobStatusOk oobStatusUnknown :: String oobStatusUnknown = Types.oobStatusToRaw OobStatusUnknown oobStatusWarning :: String oobStatusWarning = Types.oobStatusToRaw OobStatusWarning oobStatuses :: FrozenSet String oobStatuses = ConstantUtils.mkSet $ map Types.oobStatusToRaw [minBound..] -- | Instance Parameters Profile ppDefault :: String ppDefault = "default" -- * nic* constants are used inside the ganeti config nicLink :: String nicLink = "link" nicMode :: String nicMode = "mode" nicVlan :: String nicVlan = "vlan" nicsParameterTypes :: Map String VType nicsParameterTypes = Map.fromList [(nicMode, vtypeString), (nicLink, vtypeString), (nicVlan, vtypeString)] nicsParameters :: FrozenSet String nicsParameters = ConstantUtils.mkSet (Map.keys nicsParameterTypes) nicModeBridged :: String nicModeBridged = Types.nICModeToRaw NMBridged nicModeRouted :: String nicModeRouted = Types.nICModeToRaw NMRouted nicModeOvs :: String nicModeOvs = Types.nICModeToRaw NMOvs nicIpPool :: String nicIpPool = Types.nICModeToRaw NMPool nicValidModes :: FrozenSet String nicValidModes = ConstantUtils.mkSet $ map Types.nICModeToRaw [minBound..] releaseAction :: String releaseAction = "release" reserveAction :: String reserveAction = "reserve" -- * idisk* constants are used in opcodes, to create/change disks idiskAdopt :: String idiskAdopt = "adopt" idiskMetavg :: String idiskMetavg = "metavg" idiskMode :: String idiskMode = "mode" idiskName :: String idiskName = "name" idiskSize :: String idiskSize = "size" idiskSpindles :: String idiskSpindles = "spindles" idiskVg :: String idiskVg = "vg" idiskProvider :: String idiskProvider = "provider" idiskAccess :: String idiskAccess = "access" idiskType :: String idiskType = "dev_type" idiskParamsTypes :: Map String VType idiskParamsTypes = Map.fromList [ (idiskSize, VTypeSize) , (idiskSpindles, VTypeInt) , (idiskMode, VTypeString) , (idiskAdopt, VTypeString) , (idiskVg, VTypeString) , (idiskMetavg, VTypeString) , (idiskProvider, VTypeString) , (idiskAccess, VTypeString) , (idiskName, VTypeMaybeString) , (idiskType, VTypeString) ] idiskParams :: FrozenSet String idiskParams = ConstantUtils.mkSet (Map.keys idiskParamsTypes) modifiableIdiskParamsTypes :: Map String VType modifiableIdiskParamsTypes = Map.fromList [(idiskMode, VTypeString), (idiskName, VTypeString)] modifiableIdiskParams :: FrozenSet String modifiableIdiskParams = ConstantUtils.mkSet (Map.keys modifiableIdiskParamsTypes) -- * inic* constants are used in opcodes, to create/change nics inicBridge :: String inicBridge = "bridge" inicIp :: String inicIp = "ip" inicLink :: String inicLink = "link" inicMac :: String inicMac = "mac" inicMode :: String inicMode = "mode" inicName :: String inicName = "name" inicNetwork :: String inicNetwork = "network" inicVlan :: String inicVlan = "vlan" inicParamsTypes :: Map String VType inicParamsTypes = Map.fromList [(inicBridge, VTypeMaybeString), (inicIp, VTypeMaybeString), (inicLink, VTypeString), (inicMac, VTypeString), (inicMode, VTypeString), (inicName, VTypeMaybeString), (inicNetwork, VTypeMaybeString), (inicVlan, VTypeMaybeString)] inicParams :: FrozenSet String inicParams = ConstantUtils.mkSet (Map.keys inicParamsTypes) -- * Hypervisor constants htXenPvm :: String htXenPvm = Types.hypervisorToRaw XenPvm htFake :: String htFake = Types.hypervisorToRaw Fake htXenHvm :: String htXenHvm = Types.hypervisorToRaw XenHvm htKvm :: String htKvm = Types.hypervisorToRaw Kvm htChroot :: String htChroot = Types.hypervisorToRaw Chroot htLxc :: String htLxc = Types.hypervisorToRaw Lxc hyperTypes :: FrozenSet String hyperTypes = ConstantUtils.mkSet $ map Types.hypervisorToRaw [minBound..] htsReqPort :: FrozenSet String htsReqPort = ConstantUtils.mkSet [htXenHvm, htKvm] vncBasePort :: Int vncBasePort = 5900 vncDefaultBindAddress :: String vncDefaultBindAddress = ip4AddressAny qemuPciSlots :: Int qemuPciSlots = 32 qemuDefaultPciReservations :: Int qemuDefaultPciReservations = 12 -- * NIC types htNicE1000 :: String htNicE1000 = "e1000" htNicI82551 :: String htNicI82551 = "i82551" htNicI8259er :: String htNicI8259er = "i82559er" htNicI85557b :: String htNicI85557b = "i82557b" htNicNe2kIsa :: String htNicNe2kIsa = "ne2k_isa" htNicNe2kPci :: String htNicNe2kPci = "ne2k_pci" htNicParavirtual :: String htNicParavirtual = "paravirtual" htNicPcnet :: String htNicPcnet = "pcnet" htNicRtl8139 :: String htNicRtl8139 = "rtl8139" htHvmValidNicTypes :: FrozenSet String htHvmValidNicTypes = ConstantUtils.mkSet [htNicE1000, htNicNe2kIsa, htNicNe2kPci, htNicParavirtual, htNicRtl8139] htKvmValidNicTypes :: FrozenSet String htKvmValidNicTypes = ConstantUtils.mkSet [htNicE1000, htNicI82551, htNicI8259er, htNicI85557b, htNicNe2kIsa, htNicNe2kPci, htNicParavirtual, htNicPcnet, htNicRtl8139] -- * Vif types -- | Default vif type in xen-hvm htHvmVifIoemu :: String htHvmVifIoemu = "ioemu" htHvmVifVif :: String htHvmVifVif = "vif" htHvmValidVifTypes :: FrozenSet String htHvmValidVifTypes = ConstantUtils.mkSet [htHvmVifIoemu, htHvmVifVif] -- * Disk types htDiskIde :: String htDiskIde = "ide" htDiskIoemu :: String htDiskIoemu = "ioemu" htDiskMtd :: String htDiskMtd = "mtd" htDiskParavirtual :: String htDiskParavirtual = "paravirtual" htDiskPflash :: String htDiskPflash = "pflash" htDiskScsi :: String htDiskScsi = "scsi" htDiskSd :: String htDiskSd = "sd" htDiskScsiGeneric :: String htDiskScsiGeneric = "scsi-generic" htDiskScsiBlock :: String htDiskScsiBlock = "scsi-block" htDiskScsiCd :: String htDiskScsiCd = "scsi-cd" htDiskScsiHd :: String htDiskScsiHd = "scsi-hd" htScsiDeviceTypes :: FrozenSet String htScsiDeviceTypes = ConstantUtils.mkSet [htDiskScsiGeneric, htDiskScsiBlock, htDiskScsiCd, htDiskScsiHd] htHvmValidDiskTypes :: FrozenSet String htHvmValidDiskTypes = ConstantUtils.mkSet [htDiskIoemu, htDiskParavirtual] htKvmValidDiskTypes :: FrozenSet String htKvmValidDiskTypes = ConstantUtils.mkSet [htDiskIde, htDiskMtd, htDiskParavirtual, htDiskPflash, htDiskScsi, htDiskSd, htDiskScsiGeneric, htDiskScsiBlock, htDiskScsiHd, htDiskScsiCd] -- * SCSI controller types htScsiControllerLsi :: String htScsiControllerLsi = "lsi" htScsiControllerVirtio :: String htScsiControllerVirtio = "virtio-scsi-pci" htScsiControllerMegasas :: String htScsiControllerMegasas = "megasas" htKvmValidScsiControllerTypes :: FrozenSet String htKvmValidScsiControllerTypes = ConstantUtils.mkSet [htScsiControllerLsi, htScsiControllerVirtio, htScsiControllerMegasas] htCacheDefault :: String htCacheDefault = "default" htCacheNone :: String htCacheNone = "none" htCacheWback :: String htCacheWback = "writeback" htCacheWthrough :: String htCacheWthrough = "writethrough" htValidCacheTypes :: FrozenSet String htValidCacheTypes = ConstantUtils.mkSet [htCacheDefault, htCacheNone, htCacheWback, htCacheWthrough] htDiscardIgnore :: String htDiscardIgnore = "ignore" htDiscardUnmap :: String htDiscardUnmap = "unmap" htValidDiscardTypes :: FrozenSet String htValidDiscardTypes = ConstantUtils.mkSet [htDiscardIgnore, htDiscardUnmap] htKvmAioThreads :: String htKvmAioThreads = "threads" htKvmAioNative :: String htKvmAioNative = "native" htKvmAioIoUring :: String htKvmAioIoUring = "io_uring" htKvmValidAioTypes :: FrozenSet String htKvmValidAioTypes = ConstantUtils.mkSet [htKvmAioThreads, htKvmAioNative, htKvmAioIoUring] -- * Mouse types htMouseMouse :: String htMouseMouse = "mouse" htMouseTablet :: String htMouseTablet = "tablet" htKvmValidMouseTypes :: FrozenSet String htKvmValidMouseTypes = ConstantUtils.mkSet [htMouseMouse, htMouseTablet] -- * Boot order htBoCdrom :: String htBoCdrom = "cdrom" htBoDisk :: String htBoDisk = "disk" htBoFloppy :: String htBoFloppy = "floppy" htBoNetwork :: String htBoNetwork = "network" htKvmValidBoTypes :: FrozenSet String htKvmValidBoTypes = ConstantUtils.mkSet [htBoCdrom, htBoDisk, htBoFloppy, htBoNetwork] -- * SPICE lossless image compression options htKvmSpiceLosslessImgComprAutoGlz :: String htKvmSpiceLosslessImgComprAutoGlz = "auto_glz" htKvmSpiceLosslessImgComprAutoLz :: String htKvmSpiceLosslessImgComprAutoLz = "auto_lz" htKvmSpiceLosslessImgComprGlz :: String htKvmSpiceLosslessImgComprGlz = "glz" htKvmSpiceLosslessImgComprLz :: String htKvmSpiceLosslessImgComprLz = "lz" htKvmSpiceLosslessImgComprOff :: String htKvmSpiceLosslessImgComprOff = "off" htKvmSpiceLosslessImgComprQuic :: String htKvmSpiceLosslessImgComprQuic = "quic" htKvmSpiceValidLosslessImgComprOptions :: FrozenSet String htKvmSpiceValidLosslessImgComprOptions = ConstantUtils.mkSet [htKvmSpiceLosslessImgComprAutoGlz, htKvmSpiceLosslessImgComprAutoLz, htKvmSpiceLosslessImgComprGlz, htKvmSpiceLosslessImgComprLz, htKvmSpiceLosslessImgComprOff, htKvmSpiceLosslessImgComprQuic] htKvmSpiceLossyImgComprAlways :: String htKvmSpiceLossyImgComprAlways = "always" htKvmSpiceLossyImgComprAuto :: String htKvmSpiceLossyImgComprAuto = "auto" htKvmSpiceLossyImgComprNever :: String htKvmSpiceLossyImgComprNever = "never" htKvmSpiceValidLossyImgComprOptions :: FrozenSet String htKvmSpiceValidLossyImgComprOptions = ConstantUtils.mkSet [htKvmSpiceLossyImgComprAlways, htKvmSpiceLossyImgComprAuto, htKvmSpiceLossyImgComprNever] -- * SPICE video stream detection htKvmSpiceVideoStreamDetectionAll :: String htKvmSpiceVideoStreamDetectionAll = "all" htKvmSpiceVideoStreamDetectionFilter :: String htKvmSpiceVideoStreamDetectionFilter = "filter" htKvmSpiceVideoStreamDetectionOff :: String htKvmSpiceVideoStreamDetectionOff = "off" htKvmSpiceValidVideoStreamDetectionOptions :: FrozenSet String htKvmSpiceValidVideoStreamDetectionOptions = ConstantUtils.mkSet [htKvmSpiceVideoStreamDetectionAll, htKvmSpiceVideoStreamDetectionFilter, htKvmSpiceVideoStreamDetectionOff] -- * Security models htSmNone :: String htSmNone = "none" htSmPool :: String htSmPool = "pool" htSmUser :: String htSmUser = "user" htKvmValidSmTypes :: FrozenSet String htKvmValidSmTypes = ConstantUtils.mkSet [htSmNone, htSmPool, htSmUser] -- * Kvm flag values htKvmDisabled :: String htKvmDisabled = "disabled" htKvmEnabled :: String htKvmEnabled = "enabled" htKvmFlagValues :: FrozenSet String htKvmFlagValues = ConstantUtils.mkSet [htKvmDisabled, htKvmEnabled] -- * Migration type htMigrationLive :: String htMigrationLive = Types.migrationModeToRaw MigrationLive htMigrationNonlive :: String htMigrationNonlive = Types.migrationModeToRaw MigrationNonLive htMigrationModes :: FrozenSet String htMigrationModes = ConstantUtils.mkSet $ map Types.migrationModeToRaw [minBound..] -- * Cluster verify steps verifyNplusoneMem :: String verifyNplusoneMem = Types.verifyOptionalChecksToRaw VerifyNPlusOneMem verifyHvparamAssessment:: String verifyHvparamAssessment = Types.verifyOptionalChecksToRaw VerifyHVParamAssessment verifyOptionalChecks :: FrozenSet String verifyOptionalChecks = ConstantUtils.mkSet $ map Types.verifyOptionalChecksToRaw [minBound..] -- * Cluster Verify error classes cvTcluster :: String cvTcluster = "cluster" cvTgroup :: String cvTgroup = "group" cvTnode :: String cvTnode = "node" cvTinstance :: String cvTinstance = "instance" -- * Cluster Verify error levels cvWarning :: String cvWarning = "WARNING" cvError :: String cvError = "ERROR" -- * Cluster Verify error codes and documentation cvEclustercert :: (String, String, String) cvEclustercert = ("cluster", Types.cVErrorCodeToRaw CvECLUSTERCERT, "Cluster certificate files verification failure") cvEclusterclientcert :: (String, String, String) cvEclusterclientcert = ("cluster", Types.cVErrorCodeToRaw CvECLUSTERCLIENTCERT, "Cluster client certificate files verification failure") cvEclustercfg :: (String, String, String) cvEclustercfg = ("cluster", Types.cVErrorCodeToRaw CvECLUSTERCFG, "Cluster configuration verification failure") cvEclusterdanglinginst :: (String, String, String) cvEclusterdanglinginst = ("node", Types.cVErrorCodeToRaw CvECLUSTERDANGLINGINST, "Some instances have a non-existing primary node") cvEclusterdanglingnodes :: (String, String, String) cvEclusterdanglingnodes = ("node", Types.cVErrorCodeToRaw CvECLUSTERDANGLINGNODES, "Some nodes belong to non-existing groups") cvEclusterfilecheck :: (String, String, String) cvEclusterfilecheck = ("cluster", Types.cVErrorCodeToRaw CvECLUSTERFILECHECK, "Cluster configuration verification failure") cvEgroupdifferentpvsize :: (String, String, String) cvEgroupdifferentpvsize = ("group", Types.cVErrorCodeToRaw CvEGROUPDIFFERENTPVSIZE, "PVs in the group have different sizes") cvEinstancebadnode :: (String, String, String) cvEinstancebadnode = ("instance", Types.cVErrorCodeToRaw CvEINSTANCEBADNODE, "Instance marked as running lives on an offline node") cvEinstancedown :: (String, String, String) cvEinstancedown = ("instance", Types.cVErrorCodeToRaw CvEINSTANCEDOWN, "Instance not running on its primary node") cvEinstancefaultydisk :: (String, String, String) cvEinstancefaultydisk = ("instance", Types.cVErrorCodeToRaw CvEINSTANCEFAULTYDISK, "Impossible to retrieve status for a disk") cvEinstancelayout :: (String, String, String) cvEinstancelayout = ("instance", Types.cVErrorCodeToRaw CvEINSTANCELAYOUT, "Instance has multiple secondary nodes") cvEinstancemissingcfgparameter :: (String, String, String) cvEinstancemissingcfgparameter = ("instance", Types.cVErrorCodeToRaw CvEINSTANCEMISSINGCFGPARAMETER, "A configuration parameter for an instance is missing") cvEinstancemissingdisk :: (String, String, String) cvEinstancemissingdisk = ("instance", Types.cVErrorCodeToRaw CvEINSTANCEMISSINGDISK, "Missing volume on an instance") cvEinstancepolicy :: (String, String, String) cvEinstancepolicy = ("instance", Types.cVErrorCodeToRaw CvEINSTANCEPOLICY, "Instance does not meet policy") cvEinstancesplitgroups :: (String, String, String) cvEinstancesplitgroups = ("instance", Types.cVErrorCodeToRaw CvEINSTANCESPLITGROUPS, "Instance with primary and secondary nodes in different groups") cvEinstanceunsuitablenode :: (String, String, String) cvEinstanceunsuitablenode = ("instance", Types.cVErrorCodeToRaw CvEINSTANCEUNSUITABLENODE, "Instance running on nodes that are not suitable for it") cvEinstancewrongnode :: (String, String, String) cvEinstancewrongnode = ("instance", Types.cVErrorCodeToRaw CvEINSTANCEWRONGNODE, "Instance running on the wrong node") cvEnodedrbd :: (String, String, String) cvEnodedrbd = ("node", Types.cVErrorCodeToRaw CvENODEDRBD, "Error parsing the DRBD status file") cvEnodedrbdhelper :: (String, String, String) cvEnodedrbdhelper = ("node", Types.cVErrorCodeToRaw CvENODEDRBDHELPER, "Error caused by the DRBD helper") cvEnodedrbdversion :: (String, String, String) cvEnodedrbdversion = ("node", Types.cVErrorCodeToRaw CvENODEDRBDVERSION, "DRBD version mismatch within a node group") cvEnodefilecheck :: (String, String, String) cvEnodefilecheck = ("node", Types.cVErrorCodeToRaw CvENODEFILECHECK, "Error retrieving the checksum of the node files") cvEnodefilestoragepaths :: (String, String, String) cvEnodefilestoragepaths = ("node", Types.cVErrorCodeToRaw CvENODEFILESTORAGEPATHS, "Detected bad file storage paths") cvEnodefilestoragepathunusable :: (String, String, String) cvEnodefilestoragepathunusable = ("node", Types.cVErrorCodeToRaw CvENODEFILESTORAGEPATHUNUSABLE, "File storage path unusable") cvEnodehooks :: (String, String, String) cvEnodehooks = ("node", Types.cVErrorCodeToRaw CvENODEHOOKS, "Communication failure in hooks execution") cvEnodehv :: (String, String, String) cvEnodehv = ("node", Types.cVErrorCodeToRaw CvENODEHV, "Hypervisor parameters verification failure") cvEnodelvm :: (String, String, String) cvEnodelvm = ("node", Types.cVErrorCodeToRaw CvENODELVM, "LVM-related node error") cvEnoden1 :: (String, String, String) cvEnoden1 = ("node", Types.cVErrorCodeToRaw CvENODEN1, "Not enough memory to accommodate instance failovers") cvEextags :: (String, String, String) cvEextags = ("node", Types.cVErrorCodeToRaw CvEEXTAGS, "Instances with same exclusion tag on the same node") cvEnodenet :: (String, String, String) cvEnodenet = ("node", Types.cVErrorCodeToRaw CvENODENET, "Network-related node error") cvEnodeoobpath :: (String, String, String) cvEnodeoobpath = ("node", Types.cVErrorCodeToRaw CvENODEOOBPATH, "Invalid Out Of Band path") cvEnodeorphaninstance :: (String, String, String) cvEnodeorphaninstance = ("node", Types.cVErrorCodeToRaw CvENODEORPHANINSTANCE, "Unknown intance running on a node") cvEnodeorphanlv :: (String, String, String) cvEnodeorphanlv = ("node", Types.cVErrorCodeToRaw CvENODEORPHANLV, "Unknown LVM logical volume") cvEnodeos :: (String, String, String) cvEnodeos = ("node", Types.cVErrorCodeToRaw CvENODEOS, "OS-related node error") cvEnoderpc :: (String, String, String) cvEnoderpc = ("node", Types.cVErrorCodeToRaw CvENODERPC, "Error during connection to the primary node of an instance") cvEnodesetup :: (String, String, String) cvEnodesetup = ("node", Types.cVErrorCodeToRaw CvENODESETUP, "Node setup error") cvEnodesharedfilestoragepathunusable :: (String, String, String) cvEnodesharedfilestoragepathunusable = ("node", Types.cVErrorCodeToRaw CvENODESHAREDFILESTORAGEPATHUNUSABLE, "Shared file storage path unusable") cvEnodeglusterstoragepathunusable :: (String, String, String) cvEnodeglusterstoragepathunusable = ("node", Types.cVErrorCodeToRaw CvENODEGLUSTERSTORAGEPATHUNUSABLE, "Gluster storage path unusable") cvEnodessh :: (String, String, String) cvEnodessh = ("node", Types.cVErrorCodeToRaw CvENODESSH, "SSH-related node error") cvEnodetime :: (String, String, String) cvEnodetime = ("node", Types.cVErrorCodeToRaw CvENODETIME, "Node returned invalid time") cvEnodeuserscripts :: (String, String, String) cvEnodeuserscripts = ("node", Types.cVErrorCodeToRaw CvENODEUSERSCRIPTS, "User scripts not present or not executable") cvEnodeversion :: (String, String, String) cvEnodeversion = ("node", Types.cVErrorCodeToRaw CvENODEVERSION, "Protocol version mismatch or Ganeti version mismatch") cvAllEcodes :: FrozenSet (String, String, String) cvAllEcodes = ConstantUtils.mkSet [cvEclustercert, cvEclustercfg, cvEclusterdanglinginst, cvEclusterdanglingnodes, cvEclusterfilecheck, cvEgroupdifferentpvsize, cvEinstancebadnode, cvEinstancedown, cvEinstancefaultydisk, cvEinstancelayout, cvEinstancemissingcfgparameter, cvEinstancemissingdisk, cvEinstancepolicy, cvEinstancesplitgroups, cvEinstanceunsuitablenode, cvEinstancewrongnode, cvEnodedrbd, cvEnodedrbdhelper, cvEnodedrbdversion, cvEnodefilecheck, cvEnodefilestoragepaths, cvEnodefilestoragepathunusable, cvEnodehooks, cvEnodehv, cvEnodelvm, cvEnoden1, cvEnodenet, cvEnodeoobpath, cvEnodeorphaninstance, cvEnodeorphanlv, cvEnodeos, cvEnoderpc, cvEnodesetup, cvEnodesharedfilestoragepathunusable, cvEnodeglusterstoragepathunusable, cvEnodessh, cvEnodetime, cvEnodeuserscripts, cvEnodeversion] cvAllEcodesStrings :: FrozenSet String cvAllEcodesStrings = ConstantUtils.mkSet $ map Types.cVErrorCodeToRaw [minBound..] -- * Node verify constants nvBridges :: String nvBridges = "bridges" nvClientCert :: String nvClientCert = "client-cert" nvDrbdhelper :: String nvDrbdhelper = "drbd-helper" nvDrbdversion :: String nvDrbdversion = "drbd-version" nvDrbdlist :: String nvDrbdlist = "drbd-list" nvExclusivepvs :: String nvExclusivepvs = "exclusive-pvs" nvFilelist :: String nvFilelist = "filelist" nvAcceptedStoragePaths :: String nvAcceptedStoragePaths = "allowed-file-storage-paths" nvFileStoragePath :: String nvFileStoragePath = "file-storage-path" nvSharedFileStoragePath :: String nvSharedFileStoragePath = "shared-file-storage-path" nvGlusterStoragePath :: String nvGlusterStoragePath = "gluster-storage-path" nvHvinfo :: String nvHvinfo = "hvinfo" nvHvparams :: String nvHvparams = "hvparms" nvHypervisor :: String nvHypervisor = "hypervisor" nvInstancelist :: String nvInstancelist = "instancelist" nvLvlist :: String nvLvlist = "lvlist" nvMasterip :: String nvMasterip = "master-ip" nvNodelist :: String nvNodelist = "nodelist" nvNodenettest :: String nvNodenettest = "node-net-test" nvNodesetup :: String nvNodesetup = "nodesetup" nvOobPaths :: String nvOobPaths = "oob-paths" nvOslist :: String nvOslist = "oslist" nvPvlist :: String nvPvlist = "pvlist" nvTime :: String nvTime = "time" nvUserscripts :: String nvUserscripts = "user-scripts" nvVersion :: String nvVersion = "version" nvVglist :: String nvVglist = "vglist" nvNonvmnodes :: String nvNonvmnodes = "nonvmnodes" nvSshSetup :: String nvSshSetup = "ssh-setup" nvSshClutter :: String nvSshClutter = "ssh-clutter" -- * Instance status inststAdmindown :: String inststAdmindown = Types.instanceStatusToRaw StatusDown inststAdminoffline :: String inststAdminoffline = Types.instanceStatusToRaw StatusOffline inststErrordown :: String inststErrordown = Types.instanceStatusToRaw ErrorDown inststErrorup :: String inststErrorup = Types.instanceStatusToRaw ErrorUp inststNodedown :: String inststNodedown = Types.instanceStatusToRaw NodeDown inststNodeoffline :: String inststNodeoffline = Types.instanceStatusToRaw NodeOffline inststRunning :: String inststRunning = Types.instanceStatusToRaw Running inststUserdown :: String inststUserdown = Types.instanceStatusToRaw UserDown inststWrongnode :: String inststWrongnode = Types.instanceStatusToRaw WrongNode inststAll :: FrozenSet String inststAll = ConstantUtils.mkSet $ map Types.instanceStatusToRaw [minBound..] -- * Admin states adminstDown :: String adminstDown = Types.adminStateToRaw AdminDown adminstOffline :: String adminstOffline = Types.adminStateToRaw AdminOffline adminstUp :: String adminstUp = Types.adminStateToRaw AdminUp adminstAll :: FrozenSet String adminstAll = ConstantUtils.mkSet $ map Types.adminStateToRaw [minBound..] -- * Admin state sources adminSource :: AdminStateSource adminSource = AdminSource userSource :: AdminStateSource userSource = UserSource adminStateSources :: FrozenSet AdminStateSource adminStateSources = ConstantUtils.mkSet [minBound..] -- * Node roles nrDrained :: String nrDrained = Types.nodeRoleToRaw NRDrained nrMaster :: String nrMaster = Types.nodeRoleToRaw NRMaster nrMcandidate :: String nrMcandidate = Types.nodeRoleToRaw NRCandidate nrOffline :: String nrOffline = Types.nodeRoleToRaw NROffline nrRegular :: String nrRegular = Types.nodeRoleToRaw NRRegular nrAll :: FrozenSet String nrAll = ConstantUtils.mkSet $ map Types.nodeRoleToRaw [minBound..] -- * SSL certificate check constants (in days) sslCertExpirationError :: Int sslCertExpirationError = 7 sslCertExpirationWarn :: Int sslCertExpirationWarn = 30 -- * Allocator framework constants iallocatorVersion :: Int iallocatorVersion = 2 iallocatorDirIn :: String iallocatorDirIn = Types.iAllocatorTestDirToRaw IAllocatorDirIn iallocatorDirOut :: String iallocatorDirOut = Types.iAllocatorTestDirToRaw IAllocatorDirOut validIallocatorDirections :: FrozenSet String validIallocatorDirections = ConstantUtils.mkSet $ map Types.iAllocatorTestDirToRaw [minBound..] iallocatorModeAlloc :: String iallocatorModeAlloc = Types.iAllocatorModeToRaw IAllocatorAlloc iallocatorModeAllocateSecondary :: String iallocatorModeAllocateSecondary = Types.iAllocatorModeToRaw IAllocatorAllocateSecondary iallocatorModeChgGroup :: String iallocatorModeChgGroup = Types.iAllocatorModeToRaw IAllocatorChangeGroup iallocatorModeMultiAlloc :: String iallocatorModeMultiAlloc = Types.iAllocatorModeToRaw IAllocatorMultiAlloc iallocatorModeNodeEvac :: String iallocatorModeNodeEvac = Types.iAllocatorModeToRaw IAllocatorNodeEvac iallocatorModeReloc :: String iallocatorModeReloc = Types.iAllocatorModeToRaw IAllocatorReloc validIallocatorModes :: FrozenSet String validIallocatorModes = ConstantUtils.mkSet $ map Types.iAllocatorModeToRaw [minBound..] iallocatorSearchPath :: [String] iallocatorSearchPath = AutoConf.iallocatorSearchPath defaultIallocatorShortcut :: String defaultIallocatorShortcut = "." -- * Opportunistic allocator usage -- | Time delay in seconds between repeated opportunistic instance creations. -- Rather than failing with an informative error message if the opportunistic -- creation cannot grab enough nodes, for some uses it is better to retry the -- creation with an interval between attempts. This is a reasonable default. defaultOpportunisticRetryInterval :: Int defaultOpportunisticRetryInterval = 30 -- * Node evacuation nodeEvacPri :: String nodeEvacPri = Types.evacModeToRaw ChangePrimary nodeEvacSec :: String nodeEvacSec = Types.evacModeToRaw ChangeSecondary nodeEvacAll :: String nodeEvacAll = Types.evacModeToRaw ChangeAll nodeEvacModes :: FrozenSet String nodeEvacModes = ConstantUtils.mkSet $ map Types.evacModeToRaw [minBound..] -- * Job queue jobQueueVersion :: Int jobQueueVersion = 1 jobQueueSizeHardLimit :: Int jobQueueSizeHardLimit = 5000 jobQueueFilesPerms :: Int jobQueueFilesPerms = 0o640 -- * Unchanged job return jobNotchanged :: String jobNotchanged = "nochange" -- * Job status jobStatusQueued :: String jobStatusQueued = Types.jobStatusToRaw JOB_STATUS_QUEUED jobStatusWaiting :: String jobStatusWaiting = Types.jobStatusToRaw JOB_STATUS_WAITING jobStatusCanceling :: String jobStatusCanceling = Types.jobStatusToRaw JOB_STATUS_CANCELING jobStatusRunning :: String jobStatusRunning = Types.jobStatusToRaw JOB_STATUS_RUNNING jobStatusCanceled :: String jobStatusCanceled = Types.jobStatusToRaw JOB_STATUS_CANCELED jobStatusSuccess :: String jobStatusSuccess = Types.jobStatusToRaw JOB_STATUS_SUCCESS jobStatusError :: String jobStatusError = Types.jobStatusToRaw JOB_STATUS_ERROR jobsPending :: FrozenSet String jobsPending = ConstantUtils.mkSet [jobStatusQueued, jobStatusWaiting, jobStatusCanceling] jobsFinalized :: FrozenSet String jobsFinalized = ConstantUtils.mkSet $ map Types.finalizedJobStatusToRaw [minBound..] jobStatusAll :: FrozenSet String jobStatusAll = ConstantUtils.mkSet $ map Types.jobStatusToRaw [minBound..] -- * OpCode status -- ** Not yet finalized opcodes opStatusCanceling :: String opStatusCanceling = "canceling" opStatusQueued :: String opStatusQueued = "queued" opStatusRunning :: String opStatusRunning = "running" opStatusWaiting :: String opStatusWaiting = "waiting" -- ** Finalized opcodes opStatusCanceled :: String opStatusCanceled = "canceled" opStatusError :: String opStatusError = "error" opStatusSuccess :: String opStatusSuccess = "success" opsFinalized :: FrozenSet String opsFinalized = ConstantUtils.mkSet [opStatusCanceled, opStatusError, opStatusSuccess] -- * OpCode priority opPrioLowest :: Int opPrioLowest = 19 opPrioHighest :: Int opPrioHighest = -20 opPrioLow :: Int opPrioLow = Types.opSubmitPriorityToRaw OpPrioLow opPrioNormal :: Int opPrioNormal = Types.opSubmitPriorityToRaw OpPrioNormal opPrioHigh :: Int opPrioHigh = Types.opSubmitPriorityToRaw OpPrioHigh opPrioSubmitValid :: FrozenSet Int opPrioSubmitValid = ConstantUtils.mkSet [opPrioLow, opPrioNormal, opPrioHigh] opPrioDefault :: Int opPrioDefault = opPrioNormal -- * Lock recalculate mode locksAppend :: String locksAppend = "append" locksReplace :: String locksReplace = "replace" -- * Lock timeout -- -- The lock timeout (sum) before we transition into blocking acquire -- (this can still be reset by priority change). Computed as max time -- (10 hours) before we should actually go into blocking acquire, -- given that we start from the default priority level. lockAttemptsMaxwait :: Double lockAttemptsMaxwait = 75.0 lockAttemptsMinwait :: Double lockAttemptsMinwait = 5.0 lockAttemptsTimeout :: Int lockAttemptsTimeout = (10 * 3600) `div` (opPrioDefault - opPrioHighest) -- * Execution log types elogMessage :: String elogMessage = Types.eLogTypeToRaw ELogMessage -- ELogMessageList is internal to Python code only, used to distinguish -- calling conventions in Feedback(), but is never serialized or loaded -- in Luxi. elogMessageList :: String elogMessageList = Types.eLogTypeToRaw ELogMessageList elogRemoteImport :: String elogRemoteImport = Types.eLogTypeToRaw ELogRemoteImport elogJqueueTest :: String elogJqueueTest = Types.eLogTypeToRaw ELogJqueueTest elogDelayTest :: String elogDelayTest = Types.eLogTypeToRaw ELogDelayTest -- * /etc/hosts modification etcHostsAdd :: String etcHostsAdd = "add" etcHostsRemove :: String etcHostsRemove = "remove" -- * Job queue test jqtMsgprefix :: String jqtMsgprefix = "TESTMSG=" jqtExec :: String jqtExec = "exec" jqtExpandnames :: String jqtExpandnames = "expandnames" jqtLogmsg :: String jqtLogmsg = "logmsg" jqtStartmsg :: String jqtStartmsg = "startmsg" jqtAll :: FrozenSet String jqtAll = ConstantUtils.mkSet [jqtExec, jqtExpandnames, jqtLogmsg, jqtStartmsg] -- * Query resources qrCluster :: String qrCluster = "cluster" qrExport :: String qrExport = "export" qrExtstorage :: String qrExtstorage = "extstorage" qrGroup :: String qrGroup = "group" qrInstance :: String qrInstance = "instance" qrJob :: String qrJob = "job" qrLock :: String qrLock = "lock" qrNetwork :: String qrNetwork = "network" qrFilter :: String qrFilter = "filter" qrNode :: String qrNode = "node" qrOs :: String qrOs = "os" -- | List of resources which can be queried using 'Ganeti.OpCodes.OpQuery' qrViaOp :: FrozenSet String qrViaOp = ConstantUtils.mkSet [qrCluster, qrOs, qrExtstorage] -- | List of resources which can be queried using Local UniX Interface qrViaLuxi :: FrozenSet String qrViaLuxi = ConstantUtils.mkSet [qrGroup, qrExport, qrInstance, qrJob, qrLock, qrNetwork, qrNode, qrFilter] -- | List of resources which can be queried using RAPI qrViaRapi :: FrozenSet String qrViaRapi = qrViaLuxi -- | List of resources which can be queried via RAPI including PUT requests qrViaRapiPut :: FrozenSet String qrViaRapiPut = ConstantUtils.mkSet [qrLock, qrJob, qrFilter] -- * Query field types qftBool :: String qftBool = "bool" qftNumber :: String qftNumber = "number" qftNumberFloat :: String qftNumberFloat = "float" qftOther :: String qftOther = "other" qftText :: String qftText = "text" qftTimestamp :: String qftTimestamp = "timestamp" qftUnit :: String qftUnit = "unit" qftUnknown :: String qftUnknown = "unknown" qftAll :: FrozenSet String qftAll = ConstantUtils.mkSet [qftBool, qftNumber, qftNumberFloat, qftOther, qftText, qftTimestamp, qftUnit, qftUnknown] -- * Query result field status -- -- Don't change or reuse values as they're used by clients. -- -- FIXME: link with 'Ganeti.Query.Language.ResultStatus' -- | No data (e.g. RPC error), can be used instead of 'rsOffline' rsNodata :: Int rsNodata = 2 rsNormal :: Int rsNormal = 0 -- | Resource marked offline rsOffline :: Int rsOffline = 4 -- | Value unavailable/unsupported for item; if this field is -- supported but we cannot get the data for the moment, 'rsNodata' or -- 'rsOffline' should be used rsUnavail :: Int rsUnavail = 3 rsUnknown :: Int rsUnknown = 1 rsAll :: FrozenSet Int rsAll = ConstantUtils.mkSet [rsNodata, rsNormal, rsOffline, rsUnavail, rsUnknown] -- | Special field cases and their verbose/terse formatting rssDescription :: Map Int (String, String) rssDescription = Map.fromList [(rsUnknown, ("(unknown)", "??")), (rsNodata, ("(nodata)", "?")), (rsOffline, ("(offline)", "*")), (rsUnavail, ("(unavail)", "-"))] -- * Max dynamic devices maxDisks :: Int maxDisks = Types.maxDisks maxNics :: Int maxNics = Types.maxNics -- | SSCONF file prefix ssconfFileprefix :: String ssconfFileprefix = "ssconf_" -- * SSCONF keys ssClusterName :: String ssClusterName = "cluster_name" ssClusterTags :: String ssClusterTags = "cluster_tags" ssFileStorageDir :: String ssFileStorageDir = "file_storage_dir" ssSharedFileStorageDir :: String ssSharedFileStorageDir = "shared_file_storage_dir" ssGlusterStorageDir :: String ssGlusterStorageDir = "gluster_storage_dir" ssMasterCandidates :: String ssMasterCandidates = "master_candidates" ssMasterCandidatesIps :: String ssMasterCandidatesIps = "master_candidates_ips" ssMasterCandidatesCerts :: String ssMasterCandidatesCerts = "master_candidates_certs" ssMasterIp :: String ssMasterIp = "master_ip" ssMasterNetdev :: String ssMasterNetdev = "master_netdev" ssMasterNetmask :: String ssMasterNetmask = "master_netmask" ssMasterNode :: String ssMasterNode = "master_node" ssNodeList :: String ssNodeList = "node_list" ssNodePrimaryIps :: String ssNodePrimaryIps = "node_primary_ips" ssNodeSecondaryIps :: String ssNodeSecondaryIps = "node_secondary_ips" ssNodeVmCapable :: String ssNodeVmCapable = "node_vm_capable" ssOfflineNodes :: String ssOfflineNodes = "offline_nodes" ssOnlineNodes :: String ssOnlineNodes = "online_nodes" ssPrimaryIpFamily :: String ssPrimaryIpFamily = "primary_ip_family" ssInstanceList :: String ssInstanceList = "instance_list" ssReleaseVersion :: String ssReleaseVersion = "release_version" ssHypervisorList :: String ssHypervisorList = "hypervisor_list" ssMaintainNodeHealth :: String ssMaintainNodeHealth = "maintain_node_health" ssUidPool :: String ssUidPool = "uid_pool" ssNodegroups :: String ssNodegroups = "nodegroups" ssNetworks :: String ssNetworks = "networks" -- | This is not a complete SSCONF key, but the prefix for the -- hypervisor keys ssHvparamsPref :: String ssHvparamsPref = "hvparams_" -- * Hvparams keys ssHvparamsXenChroot :: String ssHvparamsXenChroot = ssHvparamsPref ++ htChroot ssHvparamsXenFake :: String ssHvparamsXenFake = ssHvparamsPref ++ htFake ssHvparamsXenHvm :: String ssHvparamsXenHvm = ssHvparamsPref ++ htXenHvm ssHvparamsXenKvm :: String ssHvparamsXenKvm = ssHvparamsPref ++ htKvm ssHvparamsXenLxc :: String ssHvparamsXenLxc = ssHvparamsPref ++ htLxc ssHvparamsXenPvm :: String ssHvparamsXenPvm = ssHvparamsPref ++ htXenPvm validSsHvparamsKeys :: FrozenSet String validSsHvparamsKeys = ConstantUtils.mkSet [ssHvparamsXenChroot, ssHvparamsXenLxc, ssHvparamsXenFake, ssHvparamsXenHvm, ssHvparamsXenKvm, ssHvparamsXenPvm] ssFilePerms :: Int ssFilePerms = 0o444 ssEnabledUserShutdown :: String ssEnabledUserShutdown = "enabled_user_shutdown" ssSshPorts :: String ssSshPorts = "ssh_ports" validSsKeys :: FrozenSet String validSsKeys = ConstantUtils.mkSet [ ssClusterName , ssClusterTags , ssFileStorageDir , ssSharedFileStorageDir , ssGlusterStorageDir , ssMasterCandidates , ssMasterCandidatesIps , ssMasterCandidatesCerts , ssMasterIp , ssMasterNetdev , ssMasterNetmask , ssMasterNode , ssNodeList , ssNodePrimaryIps , ssNodeSecondaryIps , ssNodeVmCapable , ssOfflineNodes , ssOnlineNodes , ssPrimaryIpFamily , ssInstanceList , ssReleaseVersion , ssHypervisorList , ssMaintainNodeHealth , ssUidPool , ssNodegroups , ssNetworks , ssEnabledUserShutdown , ssSshPorts ] <> validSsHvparamsKeys -- | Cluster wide default parameters defaultEnabledHypervisor :: String defaultEnabledHypervisor = htXenPvm hvcDefaults :: Map Hypervisor (Map String PyValueEx) hvcDefaults = Map.fromList [ (XenPvm, Map.fromList [ (hvUseBootloader, PyValueEx False) , (hvBootloaderPath, PyValueEx xenBootloader) , (hvBootloaderArgs, PyValueEx "") , (hvKernelPath, PyValueEx xenKernel) , (hvInitrdPath, PyValueEx "") , (hvRootPath, PyValueEx "/dev/xvda1") , (hvKernelArgs, PyValueEx "ro") , (hvMigrationPort, PyValueEx (8002 :: Int)) , (hvMigrationMode, PyValueEx htMigrationLive) , (hvBlockdevPrefix, PyValueEx "sd") , (hvRebootBehavior, PyValueEx instanceRebootAllowed) , (hvCpuMask, PyValueEx cpuPinningAll) , (hvCpuCap, PyValueEx (0 :: Int)) , (hvCpuWeight, PyValueEx (256 :: Int)) , (hvVifScript, PyValueEx "") , (hvXenCpuid, PyValueEx "") , (hvSoundhw, PyValueEx "") ]) , (XenHvm, Map.fromList [ (hvBootOrder, PyValueEx "cd") , (hvCdromImagePath, PyValueEx "") , (hvNicType, PyValueEx htNicRtl8139) , (hvDiskType, PyValueEx htDiskParavirtual) , (hvVncBindAddress, PyValueEx ip4AddressAny) , (hvAcpi, PyValueEx True) , (hvPae, PyValueEx True) , (hvKernelPath, PyValueEx "/usr/lib/xen/boot/hvmloader") , (hvDeviceModel, PyValueEx "/usr/lib/xen/bin/qemu-dm") , (hvMigrationPort, PyValueEx (8002 :: Int)) , (hvMigrationMode, PyValueEx htMigrationNonlive) , (hvUseLocaltime, PyValueEx False) , (hvBlockdevPrefix, PyValueEx "hd") , (hvPassthrough, PyValueEx "") , (hvRebootBehavior, PyValueEx instanceRebootAllowed) , (hvCpuMask, PyValueEx cpuPinningAll) , (hvCpuCap, PyValueEx (0 :: Int)) , (hvCpuWeight, PyValueEx (256 :: Int)) , (hvVifType, PyValueEx htHvmVifIoemu) , (hvVifScript, PyValueEx "") , (hvViridian, PyValueEx False) , (hvXenCpuid, PyValueEx "") , (hvSoundhw, PyValueEx "") ]) , (Kvm, Map.fromList [ (hvKvmPath, PyValueEx kvmPath) , (hvKernelPath, PyValueEx kvmKernel) , (hvInitrdPath, PyValueEx "") , (hvKernelArgs, PyValueEx "ro") , (hvRootPath, PyValueEx "/dev/vda1") , (hvAcpi, PyValueEx True) , (hvSerialConsole, PyValueEx True) , (hvSerialSpeed, PyValueEx (38400 :: Int)) , (hvVncBindAddress, PyValueEx "") , (hvVncTls, PyValueEx False) , (hvVncX509, PyValueEx "") , (hvVncX509Verify, PyValueEx False) , (hvVncPasswordFile, PyValueEx "") , (hvKvmScsiControllerType, PyValueEx htScsiControllerLsi) , (hvKvmPciReservations, PyValueEx qemuDefaultPciReservations) , (hvKvmSpiceBind, PyValueEx "") , (hvKvmSpiceIpVersion, PyValueEx ifaceNoIpVersionSpecified) , (hvKvmSpicePasswordFile, PyValueEx "") , (hvKvmSpiceLosslessImgCompr, PyValueEx "") , (hvKvmSpiceJpegImgCompr, PyValueEx "") , (hvKvmSpiceZlibGlzImgCompr, PyValueEx "") , (hvKvmSpiceStreamingVideoDetection, PyValueEx "") , (hvKvmSpiceAudioCompr, PyValueEx True) , (hvKvmSpiceUseTls, PyValueEx False) , (hvKvmSpiceTlsCiphers, PyValueEx opensslCiphers) , (hvKvmSpiceUseVdagent, PyValueEx True) , (hvKvmFloppyImagePath, PyValueEx "") , (hvCdromImagePath, PyValueEx "") , (hvKvmCdrom2ImagePath, PyValueEx "") , (hvBootOrder, PyValueEx htBoDisk) , (hvNicType, PyValueEx htNicParavirtual) , (hvDiskType, PyValueEx htDiskParavirtual) , (hvKvmCdromDiskType, PyValueEx "") , (hvKvmDiskAio, PyValueEx htKvmAioThreads) , (hvUsbMouse, PyValueEx "") , (hvKeymap, PyValueEx "") , (hvMigrationPort, PyValueEx (8102 :: Int)) , (hvMigrationBandwidth, PyValueEx (32 :: Int)) , (hvMigrationDowntime, PyValueEx (30 :: Int)) , (hvMigrationMode, PyValueEx htMigrationLive) , (hvUseGuestAgent, PyValueEx False) , (hvUseLocaltime, PyValueEx False) , (hvDiskCache, PyValueEx htCacheDefault) , (hvDiskDiscard, PyValueEx htDiscardIgnore) , (hvSecurityModel, PyValueEx htSmNone) , (hvSecurityDomain, PyValueEx "") , (hvKvmFlag, PyValueEx "") , (hvVhostNet, PyValueEx True) , (hvVirtioNetQueues, PyValueEx (1 :: Int)) , (hvKvmUseChroot, PyValueEx False) , (hvKvmUserShutdown, PyValueEx False) , (hvMemPath, PyValueEx "") , (hvRebootBehavior, PyValueEx instanceRebootAllowed) , (hvCpuMask, PyValueEx cpuPinningAll) , (hvCpuType, PyValueEx "") , (hvCpuCores, PyValueEx (0 :: Int)) , (hvCpuThreads, PyValueEx (0 :: Int)) , (hvCpuSockets, PyValueEx (0 :: Int)) , (hvSoundhw, PyValueEx "") , (hvUsbDevices, PyValueEx "") , (hvVga, PyValueEx "") , (hvKvmExtra, PyValueEx "") , (hvKvmMachineVersion, PyValueEx "") , (hvKvmMigrationCaps, PyValueEx "") , (hvVnetHdr, PyValueEx True)]) , (Fake, Map.fromList [(hvMigrationMode, PyValueEx htMigrationLive)]) , (Chroot, Map.fromList [(hvInitScript, PyValueEx "/ganeti-chroot")]) , (Lxc, Map.fromList [ (hvCpuMask, PyValueEx "") , (hvLxcDevices, PyValueEx lxcDevicesDefault) , (hvLxcDropCapabilities, PyValueEx lxcDropCapabilitiesDefault) , (hvLxcExtraCgroups, PyValueEx "") , (hvLxcExtraConfig, PyValueEx "") , (hvLxcNumTtys, PyValueEx (6 :: Int)) , (hvLxcStartupTimeout, PyValueEx (30 :: Int)) ]) ] hvcGlobals :: FrozenSet String hvcGlobals = ConstantUtils.mkSet [hvMigrationBandwidth, hvMigrationMode, hvMigrationPort] becDefaults :: Map String PyValueEx becDefaults = Map.fromList [ (beMinmem, PyValueEx (128 :: Int)) , (beMaxmem, PyValueEx (128 :: Int)) , (beVcpus, PyValueEx (1 :: Int)) , (beAutoBalance, PyValueEx True) , (beAlwaysFailover, PyValueEx False) , (beSpindleUse, PyValueEx (1 :: Int)) ] ndcDefaults :: Map String PyValueEx ndcDefaults = Map.fromList [ (ndOobProgram, PyValueEx "") , (ndSpindleCount, PyValueEx (1 :: Int)) , (ndExclusiveStorage, PyValueEx False) , (ndOvs, PyValueEx False) , (ndOvsName, PyValueEx defaultOvs) , (ndOvsLink, PyValueEx "") , (ndSshPort, PyValueEx (22 :: Int)) , (ndCpuSpeed, PyValueEx (1 :: Double)) ] ndcGlobals :: FrozenSet String ndcGlobals = ConstantUtils.mkSet [ndExclusiveStorage] -- | Default delay target measured in sectors defaultDelayTarget :: Int defaultDelayTarget = 1 defaultDiskCustom :: String defaultDiskCustom = "" defaultDiskResync :: Bool defaultDiskResync = False -- | Default fill target measured in sectors defaultFillTarget :: Int defaultFillTarget = 0 -- | Default mininum rate measured in KiB/s defaultMinRate :: Int defaultMinRate = 4 * 1024 defaultNetCustom :: String defaultNetCustom = "" -- | Default plan ahead measured in sectors -- -- The default values for the DRBD dynamic resync speed algorithm are -- taken from the drbsetup 8.3.11 man page, except for c-plan-ahead -- (that we don't need to set to 0, because we have a separate option -- to enable it) and for c-max-rate, that we cap to the default value -- for the static resync rate. defaultPlanAhead :: Int defaultPlanAhead = 20 defaultRbdPool :: String defaultRbdPool = "rbd" diskLdDefaults :: Map DiskTemplate (Map String PyValueEx) diskLdDefaults = Map.fromList [ (DTBlock, Map.empty) , (DTDrbd8, Map.fromList [ (ldpBarriers, PyValueEx drbdBarriers) , (ldpDefaultMetavg, PyValueEx defaultVg) , (ldpDelayTarget, PyValueEx defaultDelayTarget) , (ldpDiskCustom, PyValueEx defaultDiskCustom) , (ldpDynamicResync, PyValueEx defaultDiskResync) , (ldpFillTarget, PyValueEx defaultFillTarget) , (ldpMaxRate, PyValueEx classicDrbdSyncSpeed) , (ldpMinRate, PyValueEx defaultMinRate) , (ldpNetCustom, PyValueEx defaultNetCustom) , (ldpNoMetaFlush, PyValueEx drbdNoMetaFlush) , (ldpPlanAhead, PyValueEx defaultPlanAhead) , (ldpProtocol, PyValueEx drbdDefaultNetProtocol) , (ldpResyncRate, PyValueEx classicDrbdSyncSpeed) ]) , (DTExt, Map.fromList [ (ldpAccess, PyValueEx diskKernelspace) ]) , (DTFile, Map.empty) , (DTPlain, Map.fromList [(ldpStripes, PyValueEx lvmStripecount)]) , (DTRbd, Map.fromList [ (ldpPool, PyValueEx defaultRbdPool) , (ldpAccess, PyValueEx diskKernelspace) ]) , (DTSharedFile, Map.empty) , (DTGluster, Map.fromList [ (rbdAccess, PyValueEx diskKernelspace) , (glusterHost, PyValueEx glusterHostDefault) , (glusterVolume, PyValueEx glusterVolumeDefault) , (glusterPort, PyValueEx glusterPortDefault) ]) ] diskDtDefaults :: Map DiskTemplate (Map String PyValueEx) diskDtDefaults = Map.fromList [ (DTBlock, Map.empty) , (DTDiskless, Map.empty) , (DTDrbd8, Map.fromList [ (drbdDataStripes, PyValueEx lvmStripecount) , (drbdDefaultMetavg, PyValueEx defaultVg) , (drbdDelayTarget, PyValueEx defaultDelayTarget) , (drbdDiskBarriers, PyValueEx drbdBarriers) , (drbdDiskCustom, PyValueEx defaultDiskCustom) , (drbdDynamicResync, PyValueEx defaultDiskResync) , (drbdFillTarget, PyValueEx defaultFillTarget) , (drbdMaxRate, PyValueEx classicDrbdSyncSpeed) , (drbdMetaBarriers, PyValueEx drbdNoMetaFlush) , (drbdMetaStripes, PyValueEx lvmStripecount) , (drbdMinRate, PyValueEx defaultMinRate) , (drbdNetCustom, PyValueEx defaultNetCustom) , (drbdPlanAhead, PyValueEx defaultPlanAhead) , (drbdProtocol, PyValueEx drbdDefaultNetProtocol) , (drbdResyncRate, PyValueEx classicDrbdSyncSpeed) ]) , (DTExt, Map.fromList [ (rbdAccess, PyValueEx diskKernelspace) ]) , (DTFile, Map.empty) , (DTPlain, Map.fromList [(lvStripes, PyValueEx lvmStripecount)]) , (DTRbd, Map.fromList [ (rbdPool, PyValueEx defaultRbdPool) , (rbdAccess, PyValueEx diskKernelspace) ]) , (DTSharedFile, Map.empty) , (DTGluster, Map.fromList [ (rbdAccess, PyValueEx diskKernelspace) , (glusterHost, PyValueEx glusterHostDefault) , (glusterVolume, PyValueEx glusterVolumeDefault) , (glusterPort, PyValueEx glusterPortDefault) ]) ] niccDefaults :: Map String PyValueEx niccDefaults = Map.fromList [ (nicMode, PyValueEx nicModeBridged) , (nicLink, PyValueEx defaultBridge) , (nicVlan, PyValueEx "") ] -- | All of the following values are quite arbitrary - there are no -- "good" defaults, these must be customised per-site ispecsMinmaxDefaults :: Map String (Map String Int) ispecsMinmaxDefaults = Map.fromList [(ispecsMin, Map.fromList [(ConstantUtils.ispecMemSize, Types.iSpecMemorySize Types.defMinISpec), (ConstantUtils.ispecCpuCount, Types.iSpecCpuCount Types.defMinISpec), (ConstantUtils.ispecDiskCount, Types.iSpecDiskCount Types.defMinISpec), (ConstantUtils.ispecDiskSize, Types.iSpecDiskSize Types.defMinISpec), (ConstantUtils.ispecNicCount, Types.iSpecNicCount Types.defMinISpec), (ConstantUtils.ispecSpindleUse, Types.iSpecSpindleUse Types.defMinISpec)]), (ispecsMax, Map.fromList [(ConstantUtils.ispecMemSize, Types.iSpecMemorySize Types.defMaxISpec), (ConstantUtils.ispecCpuCount, Types.iSpecCpuCount Types.defMaxISpec), (ConstantUtils.ispecDiskCount, Types.iSpecDiskCount Types.defMaxISpec), (ConstantUtils.ispecDiskSize, Types.iSpecDiskSize Types.defMaxISpec), (ConstantUtils.ispecNicCount, Types.iSpecNicCount Types.defMaxISpec), (ConstantUtils.ispecSpindleUse, Types.iSpecSpindleUse Types.defMaxISpec)])] ipolicyDefaults :: Map String PyValueEx ipolicyDefaults = Map.fromList [ (ispecsMinmax, PyValueEx [ispecsMinmaxDefaults]) , (ispecsStd, PyValueEx (Map.fromList [ (ispecMemSize, 128) , (ispecCpuCount, 1) , (ispecDiskCount, 1) , (ispecDiskSize, 1024) , (ispecNicCount, 1) , (ispecSpindleUse, 1) ] :: Map String Int)) , (ipolicyDts, PyValueEx (ConstantUtils.toList diskTemplates)) , (ipolicyVcpuRatio, PyValueEx (4.0 :: Double)) , (ipolicySpindleRatio, PyValueEx (32.0 :: Double)) ] masterPoolSizeDefault :: Int masterPoolSizeDefault = 10 -- * Exclusive storage -- | Error margin used to compare physical disks partMargin :: Double partMargin = 0.01 -- | Space reserved when creating instance disks partReserved :: Double partReserved = 0.02 -- * Luxid job scheduling -- | Time intervall in seconds for polling updates on the job queue. This -- intervall is only relevant if the number of running jobs reaches the maximal -- allowed number, as otherwise new jobs will be started immediately anyway. -- Also, as jobs are watched via inotify, scheduling usually works independent -- of polling. Therefore we chose a sufficiently large interval, in the order of -- 5 minutes. As with the interval for reloading the configuration, we chose a -- prime number to avoid accidental 'same wakeup' with other processes. luxidJobqueuePollInterval :: Int luxidJobqueuePollInterval = 307 -- | The default value for the maximal number of jobs to be running at the same -- time. Once the maximal number is reached, new jobs will just be queued and -- only started, once some of the other jobs have finished. luxidMaximalRunningJobsDefault :: Int luxidMaximalRunningJobsDefault = 20 -- | The default value for the maximal number of jobs that luxid tracks via -- inotify. If the number of running jobs exceeds this limit (which only happens -- if the user increases the default value of maximal running jobs), new forked -- jobs are no longer tracked by inotify; progress will still be noticed on the -- regular polls. luxidMaximalTrackedJobsDefault :: Int luxidMaximalTrackedJobsDefault = 25 -- * Luxid job death testing -- | The number of attempts to prove that a job is dead after sending it a -- KILL signal. luxidJobDeathDetectionRetries :: Int luxidJobDeathDetectionRetries = 3 -- | Time to delay (in /us/) after unsucessfully verifying the death of a -- job we believe to be dead. This is best choosen to be the average time -- sending a SIGKILL to take effect. luxidJobDeathDelay :: Int luxidJobDeathDelay = 100000 -- * WConfD -- | Time itnervall in seconds between checks that all lock owners are still -- alive, and cleaning up the resources for the dead ones. As jobs dying without -- releasing resources is the exception, not the rule, we don't want this task -- to take up too many cycles itself. Hence we choose a sufficiently large -- intervall, in the order of 5 minutes. To avoid accidental 'same wakeup' -- with other tasks, we choose the next unused prime number. wconfdDeathdetectionIntervall :: Int wconfdDeathdetectionIntervall = 311 wconfdDefCtmo :: Int wconfdDefCtmo = 10 wconfdDefRwto :: Int wconfdDefRwto = 60 -- | The prefix of the WConfD livelock file name. wconfLivelockPrefix :: String wconfLivelockPrefix = "wconf-daemon" -- * Confd confdProtocolVersion :: Int confdProtocolVersion = ConstantUtils.confdProtocolVersion -- Confd request type confdReqPing :: Int confdReqPing = Types.confdRequestTypeToRaw ReqPing confdReqNodeRoleByname :: Int confdReqNodeRoleByname = Types.confdRequestTypeToRaw ReqNodeRoleByName confdReqNodePipByInstanceIp :: Int confdReqNodePipByInstanceIp = Types.confdRequestTypeToRaw ReqNodePipByInstPip confdReqClusterMaster :: Int confdReqClusterMaster = Types.confdRequestTypeToRaw ReqClusterMaster confdReqNodePipList :: Int confdReqNodePipList = Types.confdRequestTypeToRaw ReqNodePipList confdReqMcPipList :: Int confdReqMcPipList = Types.confdRequestTypeToRaw ReqMcPipList confdReqInstancesIpsList :: Int confdReqInstancesIpsList = Types.confdRequestTypeToRaw ReqInstIpsList confdReqNodeDrbd :: Int confdReqNodeDrbd = Types.confdRequestTypeToRaw ReqNodeDrbd confdReqNodeInstances :: Int confdReqNodeInstances = Types.confdRequestTypeToRaw ReqNodeInstances confdReqInstanceDisks :: Int confdReqInstanceDisks = Types.confdRequestTypeToRaw ReqInstanceDisks confdReqConfigQuery :: Int confdReqConfigQuery = Types.confdRequestTypeToRaw ReqConfigQuery confdReqDataCollectors :: Int confdReqDataCollectors = Types.confdRequestTypeToRaw ReqDataCollectors confdReqs :: FrozenSet Int confdReqs = ConstantUtils.mkSet . map Types.confdRequestTypeToRaw $ [minBound..] \\ [ReqNodeInstances] -- * Confd request type confdReqfieldName :: Int confdReqfieldName = Types.confdReqFieldToRaw ReqFieldName confdReqfieldIp :: Int confdReqfieldIp = Types.confdReqFieldToRaw ReqFieldIp confdReqfieldMnodePip :: Int confdReqfieldMnodePip = Types.confdReqFieldToRaw ReqFieldMNodePip -- * Confd repl status confdReplStatusOk :: Int confdReplStatusOk = Types.confdReplyStatusToRaw ReplyStatusOk confdReplStatusError :: Int confdReplStatusError = Types.confdReplyStatusToRaw ReplyStatusError confdReplStatusNotimplemented :: Int confdReplStatusNotimplemented = Types.confdReplyStatusToRaw ReplyStatusNotImpl confdReplStatuses :: FrozenSet Int confdReplStatuses = ConstantUtils.mkSet $ map Types.confdReplyStatusToRaw [minBound..] -- * Confd node role confdNodeRoleMaster :: Int confdNodeRoleMaster = Types.confdNodeRoleToRaw NodeRoleMaster confdNodeRoleCandidate :: Int confdNodeRoleCandidate = Types.confdNodeRoleToRaw NodeRoleCandidate confdNodeRoleOffline :: Int confdNodeRoleOffline = Types.confdNodeRoleToRaw NodeRoleOffline confdNodeRoleDrained :: Int confdNodeRoleDrained = Types.confdNodeRoleToRaw NodeRoleDrained confdNodeRoleRegular :: Int confdNodeRoleRegular = Types.confdNodeRoleToRaw NodeRoleRegular -- * A few common errors for confd confdErrorUnknownEntry :: Int confdErrorUnknownEntry = Types.confdErrorTypeToRaw ConfdErrorUnknownEntry confdErrorInternal :: Int confdErrorInternal = Types.confdErrorTypeToRaw ConfdErrorInternal confdErrorArgument :: Int confdErrorArgument = Types.confdErrorTypeToRaw ConfdErrorArgument -- * Confd request query fields confdReqqLink :: String confdReqqLink = ConstantUtils.confdReqqLink confdReqqIp :: String confdReqqIp = ConstantUtils.confdReqqIp confdReqqIplist :: String confdReqqIplist = ConstantUtils.confdReqqIplist confdReqqFields :: String confdReqqFields = ConstantUtils.confdReqqFields -- | Each request is "salted" by the current timestamp. -- -- This constant decides how many seconds of skew to accept. -- -- TODO: make this a default and allow the value to be more -- configurable confdMaxClockSkew :: Int confdMaxClockSkew = 2 * nodeMaxClockSkew -- | When we haven't reloaded the config for more than this amount of -- seconds, we force a test to see if inotify is betraying us. Using a -- prime number to ensure we get less chance of 'same wakeup' with -- other processes. confdConfigReloadTimeout :: Int confdConfigReloadTimeout = 17 -- | If we receive more than one update in this amount of -- microseconds, we move to polling every RATELIMIT seconds, rather -- than relying on inotify, to be able to serve more requests. confdConfigReloadRatelimit :: Int confdConfigReloadRatelimit = 250000 -- | Magic number prepended to all confd queries. -- -- This allows us to distinguish different types of confd protocols -- and handle them. For example by changing this we can move the whole -- payload to be compressed, or move away from json. confdMagicFourcc :: String confdMagicFourcc = "plj0" -- | The confd magic encoded in bytes confdMagicFourccBytes :: BC.ByteString confdMagicFourccBytes = BC.pack confdMagicFourcc -- | By default a confd request is sent to the minimum between this -- number and all MCs. 6 was chosen because even in the case of a -- disastrous 50% response rate, we should have enough answers to be -- able to compare more than one. confdDefaultReqCoverage :: Int confdDefaultReqCoverage = 6 -- | Timeout in seconds to expire pending query request in the confd -- client library. We don't actually expect any answer more than 10 -- seconds after we sent a request. confdClientExpireTimeout :: Int confdClientExpireTimeout = 10 -- | Maximum UDP datagram size. -- -- On IPv4: 64K - 20 (ip header size) - 8 (udp header size) = 65507 -- On IPv6: 64K - 40 (ip6 header size) - 8 (udp header size) = 65487 -- (assuming we can't use jumbo frames) -- We just set this to 60K, which should be enough maxUdpDataSize :: Int maxUdpDataSize = 61440 -- * User-id pool minimum/maximum acceptable user-ids uidpoolUidMin :: Int uidpoolUidMin = 0 -- | Assuming 32 bit user-ids uidpoolUidMax :: Integer uidpoolUidMax = 2 ^ 32 - 1 -- | Name or path of the pgrep command pgrep :: String pgrep = "pgrep" -- | Name of the node group that gets created at cluster init or -- upgrade initialNodeGroupName :: String initialNodeGroupName = "default" -- * Possible values for NodeGroup.alloc_policy allocPolicyLastResort :: String allocPolicyLastResort = Types.allocPolicyToRaw AllocLastResort allocPolicyPreferred :: String allocPolicyPreferred = Types.allocPolicyToRaw AllocPreferred allocPolicyUnallocable :: String allocPolicyUnallocable = Types.allocPolicyToRaw AllocUnallocable validAllocPolicies :: [String] validAllocPolicies = map Types.allocPolicyToRaw [minBound..] -- | Temporary external/shared storage parameters blockdevDriverManual :: String blockdevDriverManual = Types.blockDriverToRaw BlockDrvManual -- | 'qemu-img' path, required for 'ovfconverter' qemuimgPath :: String qemuimgPath = AutoConf.qemuimgPath -- | The hail iallocator iallocHail :: String iallocHail = "hail" -- * Fake opcodes for functions that have hooks attached to them via -- backend.RunLocalHooks fakeOpMasterTurndown :: String fakeOpMasterTurndown = "OP_CLUSTER_IP_TURNDOWN" fakeOpMasterTurnup :: String fakeOpMasterTurnup = "OP_CLUSTER_IP_TURNUP" -- * Crypto Types -- Types of cryptographic tokens used in node communication cryptoTypeSslDigest :: String cryptoTypeSslDigest = "ssl" cryptoTypeSsh :: String cryptoTypeSsh = "ssh" -- So far only ssl keys are used in the context of this constant cryptoTypes :: FrozenSet String cryptoTypes = ConstantUtils.mkSet [cryptoTypeSslDigest] -- * Crypto Actions -- Actions that can be performed on crypto tokens cryptoActionGet :: String cryptoActionGet = "get" cryptoActionCreate :: String cryptoActionCreate = "create" cryptoActionDelete :: String cryptoActionDelete = "delete" cryptoActions :: FrozenSet String cryptoActions = ConstantUtils.mkSet [ cryptoActionCreate , cryptoActionGet , cryptoActionDelete] -- Key word for master candidate cert list for bootstrapping. cryptoBootstrap :: String cryptoBootstrap = "bootstrap" -- * Options for CryptoActions -- Filename of the certificate cryptoOptionCertFile :: String cryptoOptionCertFile = "cert_file" -- Serial number of the certificate cryptoOptionSerialNo :: String cryptoOptionSerialNo = "serial_no" -- * SSH key types sshkDsa :: String sshkDsa = Types.sshKeyTypeToRaw DSA sshkEcdsa :: String sshkEcdsa = Types.sshKeyTypeToRaw ECDSA sshkRsa :: String sshkRsa = Types.sshKeyTypeToRaw RSA sshkAll :: FrozenSet String sshkAll = ConstantUtils.mkSet [sshkRsa, sshkDsa, sshkEcdsa] -- * SSH authorized key types sshakDss :: String sshakDss = "ssh-dss" sshakRsa :: String sshakRsa = "ssh-rsa" sshakAll :: FrozenSet String sshakAll = ConstantUtils.mkSet [sshakDss, sshakRsa] -- * SSH key default values -- Document the change in gnt-cluster.rst when changing these sshDefaultKeyType :: String sshDefaultKeyType = sshkRsa sshDefaultKeyBits :: Int sshDefaultKeyBits = 2048 -- * SSH setup sshsClusterName :: String sshsClusterName = "cluster_name" sshsSshHostKey :: String sshsSshHostKey = "ssh_host_key" sshsSshRootKey :: String sshsSshRootKey = "ssh_root_key" sshsSshAuthorizedKeys :: String sshsSshAuthorizedKeys = "authorized_keys" sshsSshPublicKeys :: String sshsSshPublicKeys = "public_keys" sshsNodeDaemonCertificate :: String sshsNodeDaemonCertificate = "node_daemon_certificate" sshsSshKeyType :: String sshsSshKeyType = "ssh_key_type" sshsSshKeyBits :: String sshsSshKeyBits = "ssh_key_bits" -- Number of maximum retries when contacting nodes per SSH -- during SSH update operations. sshsMaxRetries :: Integer sshsMaxRetries = 3 sshsAdd :: String sshsAdd = "add" sshsReplaceOrAdd :: String sshsReplaceOrAdd = "replace_or_add" sshsRemove :: String sshsRemove = "remove" sshsOverride :: String sshsOverride = "override" sshsClear :: String sshsClear = "clear" sshsGenerate :: String sshsGenerate = "generate" sshsSuffix :: String sshsSuffix = "suffix" sshsMasterSuffix :: String sshsMasterSuffix = "_master_tmp" sshsActions :: FrozenSet String sshsActions = ConstantUtils.mkSet [ sshsAdd , sshsRemove , sshsOverride , sshsClear , sshsReplaceOrAdd] -- * Key files for SSH daemon sshHostDsaPriv :: String sshHostDsaPriv = sshConfigDir ++ "/ssh_host_dsa_key" sshHostDsaPub :: String sshHostDsaPub = sshHostDsaPriv ++ ".pub" sshHostEcdsaPriv :: String sshHostEcdsaPriv = sshConfigDir ++ "/ssh_host_ecdsa_key" sshHostEcdsaPub :: String sshHostEcdsaPub = sshHostEcdsaPriv ++ ".pub" sshHostRsaPriv :: String sshHostRsaPriv = sshConfigDir ++ "/ssh_host_rsa_key" sshHostRsaPub :: String sshHostRsaPub = sshHostRsaPriv ++ ".pub" sshDaemonKeyfiles :: Map String (String, String) sshDaemonKeyfiles = Map.fromList [ (sshkRsa, (sshHostRsaPriv, sshHostRsaPub)) , (sshkDsa, (sshHostDsaPriv, sshHostDsaPub)) , (sshkEcdsa, (sshHostEcdsaPriv, sshHostEcdsaPub)) ] -- * Node daemon setup ndsClusterName :: String ndsClusterName = "cluster_name" ndsNodeDaemonCertificate :: String ndsNodeDaemonCertificate = "node_daemon_certificate" ndsSsconf :: String ndsSsconf = "ssconf" ndsHmac :: String ndsHmac = "hmac_key" ndsStartNodeDaemon :: String ndsStartNodeDaemon = "start_node_daemon" ndsNodeName :: String ndsNodeName = "node_name" ndsAction :: String ndsAction = "action" -- * VCluster related constants vClusterEtcHosts :: String vClusterEtcHosts = "/etc/hosts" vClusterVirtPathPrefix :: String vClusterVirtPathPrefix = "/###-VIRTUAL-PATH-###," vClusterRootdirEnvname :: String vClusterRootdirEnvname = "GANETI_ROOTDIR" vClusterHostnameEnvname :: String vClusterHostnameEnvname = "GANETI_HOSTNAME" vClusterVpathWhitelist :: FrozenSet String vClusterVpathWhitelist = ConstantUtils.mkSet [ vClusterEtcHosts ] -- * The source reasons for the execution of an OpCode opcodeReasonSrcClient :: String opcodeReasonSrcClient = "gnt:client" _opcodeReasonSrcDaemon :: String _opcodeReasonSrcDaemon = "gnt:daemon" _opcodeReasonSrcMasterd :: String _opcodeReasonSrcMasterd = _opcodeReasonSrcDaemon ++ ":masterd" opcodeReasonSrcNoded :: String opcodeReasonSrcNoded = _opcodeReasonSrcDaemon ++ ":noded" opcodeReasonSrcOpcode :: String opcodeReasonSrcOpcode = "gnt:opcode" opcodeReasonSrcPickup :: String opcodeReasonSrcPickup = _opcodeReasonSrcMasterd ++ ":pickup" opcodeReasonSrcWatcher :: String opcodeReasonSrcWatcher = "gnt:watcher" opcodeReasonSrcRlib2 :: String opcodeReasonSrcRlib2 = "gnt:library:rlib2" opcodeReasonSrcUser :: String opcodeReasonSrcUser = "gnt:user" opcodeReasonSources :: FrozenSet String opcodeReasonSources = ConstantUtils.mkSet [opcodeReasonSrcClient, opcodeReasonSrcNoded, opcodeReasonSrcOpcode, opcodeReasonSrcPickup, opcodeReasonSrcWatcher, opcodeReasonSrcRlib2, opcodeReasonSrcUser] -- | Path generating random UUID randomUuidFile :: String randomUuidFile = ConstantUtils.randomUuidFile -- * Auto-repair levels autoRepairFailover :: String autoRepairFailover = Types.autoRepairTypeToRaw ArFailover autoRepairFixStorage :: String autoRepairFixStorage = Types.autoRepairTypeToRaw ArFixStorage autoRepairMigrate :: String autoRepairMigrate = Types.autoRepairTypeToRaw ArMigrate autoRepairReinstall :: String autoRepairReinstall = Types.autoRepairTypeToRaw ArReinstall autoRepairAllTypes :: FrozenSet String autoRepairAllTypes = ConstantUtils.mkSet [autoRepairFailover, autoRepairFixStorage, autoRepairMigrate, autoRepairReinstall] -- * Auto-repair results autoRepairEnoperm :: String autoRepairEnoperm = Types.autoRepairResultToRaw ArEnoperm autoRepairFailure :: String autoRepairFailure = Types.autoRepairResultToRaw ArFailure autoRepairSuccess :: String autoRepairSuccess = Types.autoRepairResultToRaw ArSuccess autoRepairAllResults :: FrozenSet String autoRepairAllResults = ConstantUtils.mkSet [autoRepairEnoperm, autoRepairFailure, autoRepairSuccess] -- | The version identifier for builtin data collectors builtinDataCollectorVersion :: String builtinDataCollectorVersion = "B" -- | The reason trail opcode parameter name opcodeReason :: String opcodeReason = "reason" -- | The reason trail opcode parameter name opcodeSequential :: String opcodeSequential = "sequential" diskstatsFile :: String diskstatsFile = "/proc/diskstats" -- * CPU load collector statFile :: String statFile = "/proc/stat" cpuavgloadBufferSize :: Int cpuavgloadBufferSize = 150 -- | Window size for averaging in seconds. cpuavgloadWindowSize :: Int cpuavgloadWindowSize = 600 -- * Xen cpu load collector xentopCommand :: String xentopCommand = "xentop" -- | Minimal observation time in seconds, the xen cpu load collector -- can report load averages for the first time. xentopAverageThreshold :: Int xentopAverageThreshold = 100 -- * Monitoring daemon -- | Mond's variable for periodical data collection mondTimeInterval :: Int mondTimeInterval = 5 -- | Mond's waiting time for requesting the current configuration. mondConfigTimeInterval :: Int mondConfigTimeInterval = 15 -- | Mond's latest API version mondLatestApiVersion :: Int mondLatestApiVersion = 1 mondDefaultCategory :: String mondDefaultCategory = "default" -- * Disk access modes diskUserspace :: String diskUserspace = Types.diskAccessModeToRaw DiskUserspace diskKernelspace :: String diskKernelspace = Types.diskAccessModeToRaw DiskKernelspace diskValidAccessModes :: FrozenSet String diskValidAccessModes = ConstantUtils.mkSet $ map Types.diskAccessModeToRaw [minBound..] -- | Timeout for queue draining in upgrades upgradeQueueDrainTimeout :: Int upgradeQueueDrainTimeout = 36 * 60 * 60 -- 1.5 days -- | Intervall at which the queue is polled during upgrades upgradeQueuePollInterval :: Int upgradeQueuePollInterval = 10 -- * Hotplug Actions hotplugActionAdd :: String hotplugActionAdd = Types.hotplugActionToRaw HAAdd hotplugActionRemove :: String hotplugActionRemove = Types.hotplugActionToRaw HARemove hotplugActionModify :: String hotplugActionModify = Types.hotplugActionToRaw HAMod hotplugAllActions :: FrozenSet String hotplugAllActions = ConstantUtils.mkSet $ map Types.hotplugActionToRaw [minBound..] -- * Hotplug Device Targets hotplugTargetNic :: String hotplugTargetNic = Types.hotplugTargetToRaw HTNic hotplugTargetDisk :: String hotplugTargetDisk = Types.hotplugTargetToRaw HTDisk hotplugAllTargets :: FrozenSet String hotplugAllTargets = ConstantUtils.mkSet $ map Types.hotplugTargetToRaw [minBound..] -- | Timeout for disk removal (seconds) diskRemoveRetryTimeout :: Int diskRemoveRetryTimeout = 30 -- | Interval between disk removal retries (seconds) diskRemoveRetryInterval :: Int diskRemoveRetryInterval = 3 -- * UUID regex uuidRegex :: String uuidRegex = "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" -- * Luxi constants luxiSocketPerms :: Int luxiSocketPerms = 0o660 luxiKeyMethod :: String luxiKeyMethod = "method" luxiKeyArgs :: String luxiKeyArgs = "args" luxiKeySuccess :: String luxiKeySuccess = "success" luxiKeyResult :: String luxiKeyResult = "result" luxiKeyVersion :: String luxiKeyVersion = "version" luxiReqSubmitJob :: String luxiReqSubmitJob = "SubmitJob" luxiReqSubmitJobToDrainedQueue :: String luxiReqSubmitJobToDrainedQueue = "SubmitJobToDrainedQueue" luxiReqSubmitManyJobs :: String luxiReqSubmitManyJobs = "SubmitManyJobs" luxiReqWaitForJobChange :: String luxiReqWaitForJobChange = "WaitForJobChange" luxiReqPickupJob :: String luxiReqPickupJob = "PickupJob" luxiReqCancelJob :: String luxiReqCancelJob = "CancelJob" luxiReqArchiveJob :: String luxiReqArchiveJob = "ArchiveJob" luxiReqChangeJobPriority :: String luxiReqChangeJobPriority = "ChangeJobPriority" luxiReqAutoArchiveJobs :: String luxiReqAutoArchiveJobs = "AutoArchiveJobs" luxiReqQuery :: String luxiReqQuery = "Query" luxiReqQueryFields :: String luxiReqQueryFields = "QueryFields" luxiReqQueryJobs :: String luxiReqQueryJobs = "QueryJobs" luxiReqQueryFilters :: String luxiReqQueryFilters = "QueryFilters" luxiReqReplaceFilter :: String luxiReqReplaceFilter = "ReplaceFilter" luxiReqDeleteFilter :: String luxiReqDeleteFilter = "DeleteFilter" luxiReqQueryInstances :: String luxiReqQueryInstances = "QueryInstances" luxiReqQueryNodes :: String luxiReqQueryNodes = "QueryNodes" luxiReqQueryGroups :: String luxiReqQueryGroups = "QueryGroups" luxiReqQueryNetworks :: String luxiReqQueryNetworks = "QueryNetworks" luxiReqQueryExports :: String luxiReqQueryExports = "QueryExports" luxiReqQueryConfigValues :: String luxiReqQueryConfigValues = "QueryConfigValues" luxiReqQueryClusterInfo :: String luxiReqQueryClusterInfo = "QueryClusterInfo" luxiReqQueryTags :: String luxiReqQueryTags = "QueryTags" luxiReqSetDrainFlag :: String luxiReqSetDrainFlag = "SetDrainFlag" luxiReqSetWatcherPause :: String luxiReqSetWatcherPause = "SetWatcherPause" luxiReqAll :: FrozenSet String luxiReqAll = ConstantUtils.mkSet [ luxiReqArchiveJob , luxiReqAutoArchiveJobs , luxiReqCancelJob , luxiReqChangeJobPriority , luxiReqQuery , luxiReqQueryClusterInfo , luxiReqQueryConfigValues , luxiReqQueryExports , luxiReqQueryFields , luxiReqQueryGroups , luxiReqQueryInstances , luxiReqQueryJobs , luxiReqQueryNodes , luxiReqQueryNetworks , luxiReqQueryTags , luxiReqSetDrainFlag , luxiReqSetWatcherPause , luxiReqSubmitJob , luxiReqSubmitJobToDrainedQueue , luxiReqSubmitManyJobs , luxiReqWaitForJobChange , luxiReqPickupJob , luxiReqQueryFilters , luxiReqReplaceFilter , luxiReqDeleteFilter ] luxiDefCtmo :: Int luxiDefCtmo = 30 luxiDefRwto :: Int luxiDefRwto = 180 -- | Luxi 'WaitForJobChange' timeout luxiWfjcTimeout :: Int luxiWfjcTimeout = (luxiDefRwto - 1) `div` 2 -- | The prefix of the LUXI livelock file name luxiLivelockPrefix :: String luxiLivelockPrefix = "luxi-daemon" -- | The LUXI daemon waits this number of seconds for ensuring that a canceled -- job terminates before giving up. luxiCancelJobTimeout :: Int luxiCancelJobTimeout = (luxiDefRwto - 1) `div` 4 -- * Master voting constants -- | Number of retries to carry out if nodes do not answer masterVotingRetries :: Int masterVotingRetries = 6 -- | Retry interval (in seconds) in master voting, if not enough answers -- could be gathered. masterVotingRetryIntervall :: Int masterVotingRetryIntervall = 10 -- * Query language constants -- ** Logic operators with one or more operands, each of which is a -- filter on its own qlangOpAnd :: String qlangOpAnd = "&" qlangOpOr :: String qlangOpOr = "|" -- ** Unary operators with exactly one operand qlangOpNot :: String qlangOpNot = "!" qlangOpTrue :: String qlangOpTrue = "?" -- ** Binary operators with exactly two operands, the field name and -- an operator-specific value qlangOpContains :: String qlangOpContains = "=[]" qlangOpEqual :: String qlangOpEqual = "==" qlangOpEqualLegacy :: String qlangOpEqualLegacy = "=" qlangOpGe :: String qlangOpGe = ">=" qlangOpGt :: String qlangOpGt = ">" qlangOpLe :: String qlangOpLe = "<=" qlangOpLt :: String qlangOpLt = "<" qlangOpNotEqual :: String qlangOpNotEqual = "!=" qlangOpRegexp :: String qlangOpRegexp = "=~" -- | Characters used for detecting user-written filters (see -- L{_CheckFilter}) qlangFilterDetectionChars :: FrozenSet String qlangFilterDetectionChars = ConstantUtils.mkSet ["!", " ", "\"", "\'", ")", "(", "\x0b", "\n", "\r", "\x0c", "/", "<", "\t", ">", "=", "\\", "~"] -- | Characters used to detect globbing filters qlangGlobDetectionChars :: FrozenSet String qlangGlobDetectionChars = ConstantUtils.mkSet ["*", "?"] -- * Error related constants -- -- 'OpPrereqError' failure types -- | Environment error (e.g. node disk error) errorsEcodeEnviron :: String errorsEcodeEnviron = "environment_error" -- | Entity already exists errorsEcodeExists :: String errorsEcodeExists = "already_exists" -- | Internal cluster error errorsEcodeFault :: String errorsEcodeFault = "internal_error" -- | Wrong arguments (at syntax level) errorsEcodeInval :: String errorsEcodeInval = "wrong_input" -- | Entity not found errorsEcodeNoent :: String errorsEcodeNoent = "unknown_entity" -- | Not enough resources (iallocator failure, disk space, memory, etc) errorsEcodeNores :: String errorsEcodeNores = "insufficient_resources" -- | Resource not unique (e.g. MAC or IP duplication) errorsEcodeNotunique :: String errorsEcodeNotunique = "resource_not_unique" -- | Resolver errors errorsEcodeResolver :: String errorsEcodeResolver = "resolver_error" -- | Wrong entity state errorsEcodeState :: String errorsEcodeState = "wrong_state" -- | Temporarily out of resources; operation can be tried again errorsEcodeTempNores :: String errorsEcodeTempNores = "temp_insufficient_resources" errorsEcodeAll :: FrozenSet String errorsEcodeAll = ConstantUtils.mkSet [ errorsEcodeNores , errorsEcodeExists , errorsEcodeState , errorsEcodeNotunique , errorsEcodeTempNores , errorsEcodeNoent , errorsEcodeFault , errorsEcodeResolver , errorsEcodeInval , errorsEcodeEnviron ] -- * Jstore related constants jstoreJobsPerArchiveDirectory :: Int jstoreJobsPerArchiveDirectory = 10000 -- * Gluster settings -- | Name of the Gluster host setting glusterHost :: String glusterHost = "host" -- | Default value of the Gluster host setting glusterHostDefault :: String glusterHostDefault = "127.0.0.1" -- | Name of the Gluster volume setting glusterVolume :: String glusterVolume = "volume" -- | Default value of the Gluster volume setting glusterVolumeDefault :: String glusterVolumeDefault = "gv0" -- | Name of the Gluster port setting glusterPort :: String glusterPort = "port" -- | Default value of the Gluster port setting glusterPortDefault :: Int glusterPortDefault = 24007 -- * Instance communication -- -- The instance communication attaches an additional NIC, named -- @instanceCommunicationNicPrefix@:@instanceName@ with MAC address -- prefixed by @instanceCommunicationMacPrefix@, to the instances that -- have instance communication enabled. This NIC is part of the -- instance communication network which is supplied by the user via -- -- gnt-cluster modify --instance-communication=mynetwork -- -- This network is defined as @instanceCommunicationNetwork4@ for IPv4 -- and @instanceCommunicationNetwork6@ for IPv6. instanceCommunicationDoc :: String instanceCommunicationDoc = "Enable or disable the communication mechanism for an instance" instanceCommunicationMacPrefix :: String instanceCommunicationMacPrefix = "52:54:00" -- | The instance communication network is a link-local IPv4/IPv6 -- network because the communication is meant to be exclusive between -- the host and the guest and not routed outside the node. instanceCommunicationNetwork4 :: String instanceCommunicationNetwork4 = "169.254.0.0/16" -- | See 'instanceCommunicationNetwork4'. instanceCommunicationNetwork6 :: String instanceCommunicationNetwork6 = "fe80::/10" instanceCommunicationNetworkLink :: String instanceCommunicationNetworkLink = "communication_rt" instanceCommunicationNetworkMode :: String instanceCommunicationNetworkMode = nicModeRouted instanceCommunicationNicPrefix :: String instanceCommunicationNicPrefix = "ganeti:communication:" -- | Parameters that should be protected -- -- Python does not have a type system and can't automatically infer what should -- be the resulting type of a JSON request. As a result, it must rely on this -- list of parameter names to protect values correctly. -- -- Names ending in _cluster will be treated as dicts of dicts of private values. -- Otherwise they are considered dicts of private values. privateParametersBlacklist :: [String] privateParametersBlacklist = [ "osparams_private" , "osparams_secret" , "osparams_private_cluster" ] -- | Warn the user that the logging level is too low for production use. debugModeConfidentialityWarning :: String debugModeConfidentialityWarning = "ALERT: %s started in debug mode.\n\ \ Private and secret parameters WILL be logged!\n" -- | Use to hide secret parameter value redacted :: String redacted = Types.redacted -- * Stat dictionary entries -- -- The get_file_info RPC returns a number of values as a dictionary, and the -- following constants are both descriptions and means of accessing them. -- | The size of the file statSize :: String statSize = "size" -- * Helper VM-related timeouts -- | The default fixed timeout needed to startup the helper VM. helperVmStartup :: Int helperVmStartup = 5 * 60 -- | The default fixed timeout needed until the helper VM is finally -- shutdown, for example, after installing the OS. helperVmShutdown :: Int helperVmShutdown = 2 * 60 * 60 -- | The zeroing timeout per MiB of disks to zero -- -- Determined by estimating that a disk writes at a relatively slow -- speed of 1/5 of the max speed of current drives. zeroingTimeoutPerMib :: Double zeroingTimeoutPerMib = 1.0 / (100.0 / 5.0) -- * Networking -- The minimum size of a network. ipv4NetworkMinSize :: Int ipv4NetworkMinSize = 30 -- The maximum size of a network. -- -- FIXME: This limit is for performance reasons. Remove when refactoring -- for performance tuning was successful. ipv4NetworkMaxSize :: Int ipv4NetworkMaxSize = 30 -- * Data Collectors dataCollectorCPULoad :: String dataCollectorCPULoad = "cpu-avg-load" dataCollectorXenCpuLoad :: String dataCollectorXenCpuLoad = "xen-cpu-avg-load" dataCollectorDiskStats :: String dataCollectorDiskStats = "diskstats" dataCollectorDrbd :: String dataCollectorDrbd = "drbd" dataCollectorLv :: String dataCollectorLv = "lv" dataCollectorInstStatus :: String dataCollectorInstStatus = "inst-status-xen" dataCollectorParameterInterval :: String dataCollectorParameterInterval = "interval" dataCollectorNames :: FrozenSet String dataCollectorNames = ConstantUtils.mkSet [ dataCollectorCPULoad , dataCollectorDiskStats , dataCollectorDrbd , dataCollectorLv , dataCollectorInstStatus , dataCollectorXenCpuLoad ] dataCollectorStateActive :: String dataCollectorStateActive = "active" dataCollectorsEnabledName :: String dataCollectorsEnabledName = "enabled_data_collectors" dataCollectorsIntervalName :: String dataCollectorsIntervalName = "data_collector_interval" -- * HTools tag prefixes exTagsPrefix :: String exTagsPrefix = Tags.exTagsPrefix -- | The polling frequency to wait for a job status change cliWfjcFrequency :: Int cliWfjcFrequency = 20 -- | Default 'WaitForJobChange' timeout in seconds defaultWfjcTimeout :: Int defaultWfjcTimeout = 60 ganeti-3.1.0~rc2/src/Ganeti/Cpu/000075500000000000000000000000001476477700300163315ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Cpu/LoadParser.hs000064400000000000000000000053421476477700300207250ustar00rootroot00000000000000{-# LANGUAGE OverloadedStrings #-} {-| /proc/stat file parser This module holds the definition of the parser that extracts information about the CPU load of the system from the @/proc/stat@ file. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Cpu.LoadParser (cpustatParser) where import Control.Applicative ((<|>)) import qualified Data.Attoparsec.Text as A import qualified Data.Attoparsec.Combinator as AC import Data.Attoparsec.Text (Parser) import Ganeti.Parsers import Ganeti.Cpu.Types -- * Parser implementation -- | The parser for one line of the CPU status file. oneCPUstatParser :: Parser CPUstat oneCPUstatParser = let nameP = stringP userP = numberP niceP = numberP systemP = numberP idleP = numberP iowaitP = numberP irqP = numberP softirqP = numberP stealP = numberP guestP = numberP guest_niceP = numberP in CPUstat <$> nameP <*> userP <*> niceP <*> systemP <*> idleP <*> iowaitP <*> irqP <*> softirqP <*> stealP <*> guestP <*> guest_niceP <* A.endOfLine -- | When this is satisfied all the lines containing information about -- the CPU load are parsed. intrFound :: Parser () intrFound = (A.string "intr" *> return ()) <|> (A.string "page" *> return ()) <|> (A.string "swap" *> return ()) -- | The parser for the fragment of CPU status file containing -- information about the CPU load. cpustatParser :: Parser [CPUstat] cpustatParser = oneCPUstatParser `AC.manyTill` intrFound ganeti-3.1.0~rc2/src/Ganeti/Cpu/Types.hs000064400000000000000000000045041476477700300177740ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| CPUload data types This module holds the definition of the data types describing the CPU load according to information collected periodically from @/proc/stat@. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Cpu.Types ( CPUstat(..) , CPUavgload(..) ) where import Ganeti.THH -- | This is the format of the report produced by the cpu load -- collector. $(buildObject "CPUavgload" "cav" [ simpleField "cpu_number" [t| Int |] , simpleField "cpus" [t| [Double] |] , simpleField "cpu_total" [t| Double |] ]) -- | This is the format of the data parsed by the input file. $(buildObject "CPUstat" "cs" [ simpleField "name" [t| String |] , simpleField "user" [t| Int |] , simpleField "nice" [t| Int |] , simpleField "system" [t| Int |] , simpleField "idle" [t| Int |] , simpleField "iowait" [t| Int |] , simpleField "irq" [t| Int |] , simpleField "softirq" [t| Int |] , simpleField "steal" [t| Int |] , simpleField "guest" [t| Int |] , simpleField "guest_nice" [t| Int |] ]) ganeti-3.1.0~rc2/src/Ganeti/Curl/000075500000000000000000000000001476477700300165075ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Curl/Internal.hsc000064400000000000000000000116161476477700300207670ustar00rootroot00000000000000{-# LANGUAGE ForeignFunctionInterface #-} {-# OPTIONS_GHC -fno-warn-deprecated-flags #-} -- the above is needed due to the fact that hsc2hs generates code also -- compatible with older compilers; see -- http://hackage.haskell.org/trac/ghc/ticket/3844 {-| Hsc2hs definitions for 'Storable' interfaces. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Curl.Internal ( CurlMsgCode(..) , toMsgCode , fromMsgCode , CurlMsg(..) , errorBufferSize , CurlMCode(..) , toMCode ) where import Foreign import Foreign.C.Types import Network.Curl #include -- | Data representing a @CURLMSG@ enum. data CurlMsgCode = CurlMsgNone | CurlMsgDone | CurlMsgUnknown CInt -- ^ Haskell specific code for -- unknown codes deriving (Show, Eq) -- | Data representing a @struct CURLMsg@. data CurlMsg = CurlMsg { cmMessage :: CurlMsgCode -- ^ The message type , cmHandle :: CurlH -- ^ The internal curl handle to which it applies , cmResult :: CurlCode -- ^ The message-specific result } -- | Partial 'Storable' instance for 'CurlMsg'; we do not extract all -- fields, only the one we are interested in. instance Storable CurlMsg where sizeOf _ = (#size CURLMsg) alignment _ = alignment (undefined :: CInt) peek ptr = do msg <- (#peek CURLMsg, msg) ptr handle <- (#peek CURLMsg, easy_handle) ptr result <- (#peek CURLMsg, data.result) ptr return $ CurlMsg (toMsgCode msg) handle (toCode result) poke ptr (CurlMsg msg handle result) = do (#poke CURLMsg, msg) ptr (fromMsgCode msg) (#poke CURLMsg, easy_handle) ptr handle (#poke CURLMsg, data.result) ptr ((fromIntegral $ fromEnum result)::CInt) -- | Minimum buffer size for 'CurlErrorBuffer'. errorBufferSize :: Int errorBufferSize = (#const CURL_ERROR_SIZE) -- | Multi interface error codes. data CurlMCode = CurlmCallMultiPerform | CurlmOK | CurlmBadHandle | CurlmBadEasyHandle | CurlmOutOfMemory | CurlmInternalError | CurlmBadSocket | CurlmUnknownOption | CurlmUnknown CInt -- ^ Haskell specific code denoting -- undefined codes (e.g. when -- libcurl has defined new codes -- that are not implemented yet) deriving (Show, Eq) -- | Convert a CInt CURLMSG code (as returned by the C library) to a -- 'CurlMsgCode'. When an unknown code is received, the special -- 'CurlMsgUnknown' constructor will be used. toMsgCode :: CInt -> CurlMsgCode toMsgCode (#const CURLMSG_NONE) = CurlMsgNone toMsgCode (#const CURLMSG_DONE) = CurlMsgDone toMsgCode v = CurlMsgUnknown v -- | Convert a CurlMsgCode to a CInt. fromMsgCode :: CurlMsgCode -> CInt fromMsgCode CurlMsgNone = (#const CURLMSG_NONE) fromMsgCode CurlMsgDone = (#const CURLMSG_DONE) fromMsgCode (CurlMsgUnknown v) = v -- | Convert a CInt CURLMcode (as returned by the C library) to a -- 'CurlMCode'. When an unknown code is received, the special -- 'CurlmUnknown' constructor will be used. toMCode :: CInt -> CurlMCode toMCode (#const CURLM_CALL_MULTI_PERFORM) = CurlmCallMultiPerform toMCode (#const CURLM_OK) = CurlmOK toMCode (#const CURLM_BAD_HANDLE) = CurlmBadHandle toMCode (#const CURLM_BAD_EASY_HANDLE) = CurlmBadEasyHandle toMCode (#const CURLM_OUT_OF_MEMORY) = CurlmOutOfMemory toMCode (#const CURLM_INTERNAL_ERROR) = CurlmInternalError toMCode (#const CURLM_BAD_SOCKET) = CurlmBadSocket toMCode (#const CURLM_UNKNOWN_OPTION) = CurlmUnknownOption toMCode v = CurlmUnknown v ganeti-3.1.0~rc2/src/Ganeti/Curl/Multi.hs000064400000000000000000000203331476477700300201360ustar00rootroot00000000000000{-# LANGUAGE ForeignFunctionInterface, EmptyDataDecls #-} {-| Ganeti-specific implementation of the Curl multi interface (). TODO: Evaluate implementing and switching to curl_multi_socket_action(3) interface, which is deemed to be more performant for high-numbers of connections (but this is not the case for Ganeti). -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Curl.Multi where import Control.Concurrent import Control.Monad import Data.IORef import qualified Data.Map as Map import Foreign.C.String import Foreign.C.Types import Foreign.Marshal import Foreign.Ptr import Foreign.Storable import Network.Curl import Ganeti.Curl.Internal import Ganeti.Logging -- * Data types -- | Empty data type denoting a Curl multi handle. Naming is similar to -- "Network.Curl" types. data CurlM_ -- | Type alias for a pointer to a Curl multi handle. type CurlMH = Ptr CurlM_ -- | Our type alias for maps indexing 'CurlH' handles to the 'IORef' -- for the Curl code. type HandleMap = Map.Map CurlH (IORef CurlCode) -- * FFI declarations foreign import ccall "curl_multi_init" curl_multi_init :: IO CurlMH foreign import ccall "curl_multi_cleanup" curl_multi_cleanup :: CurlMH -> IO CInt foreign import ccall "curl_multi_add_handle" curl_multi_add_handle :: CurlMH -> CurlH -> IO CInt foreign import ccall "curl_multi_remove_handle" curl_multi_remove_handle :: CurlMH -> CurlH -> IO CInt foreign import ccall "curl_multi_perform" curl_multi_perform :: CurlMH -> Ptr CInt -> IO CInt foreign import ccall "curl_multi_info_read" curl_multi_info_read :: CurlMH -> Ptr CInt -> IO (Ptr CurlMsg) -- * Wrappers over FFI functions -- | Adds an easy handle to a multi handle. This is a nicer wrapper -- over 'curl_multi_add_handle' that fails for wrong codes. curlMultiAddHandle :: CurlMH -> Curl -> IO () curlMultiAddHandle multi easy = do r <- curlPrim easy $ \_ x -> curl_multi_add_handle multi x when (toMCode r /= CurlmOK) . fail $ "Failed adding easy handle to multi handle: " ++ show r -- | Nice wrapper over 'curl_multi_info_read' that massages the -- results into Haskell types. curlMultiInfoRead :: CurlMH -> IO (Maybe CurlMsg, CInt) curlMultiInfoRead multi = alloca $ \ppending -> do pmsg <- curl_multi_info_read multi ppending pending <- peek ppending msg <- if pmsg == nullPtr then return Nothing else Just `fmap` peek pmsg return (msg, pending) -- | Nice wrapper over 'curl_multi_perform'. curlMultiPerform :: CurlMH -> IO (CurlMCode, CInt) curlMultiPerform multi = alloca $ \running -> do mcode <- curl_multi_perform multi running running' <- peek running return (toMCode mcode, running') -- * Helper functions -- | Magical constant for the polling delay. This needs to be chosen such that: -- -- * we don't poll too often; a slower poll allows the RTS to schedule -- other threads, and let them work -- -- * we don't want to pool too slow, so that Curl gets to act on the -- handles that need it pollDelayInterval :: Int pollDelayInterval = 10000 -- | Writes incoming curl data to a list of strings, stored in an 'IORef'. writeHandle :: IORef [String] -> Ptr CChar -> CInt -> CInt -> Ptr () -> IO CInt writeHandle bufref cstr sz nelems _ = do let full_sz = sz * nelems hs_str <- peekCStringLen (cstr, fromIntegral full_sz) modifyIORef bufref (hs_str:) return full_sz -- | Loops and extracts all pending messages from a Curl multi handle. readMessages :: CurlMH -> HandleMap -> IO () readMessages mh hmap = do (cmsg, pending) <- curlMultiInfoRead mh case cmsg of Nothing -> return () Just (CurlMsg msg eh res) -> do logDebug $ "Got msg! msg " ++ show msg ++ " res " ++ show res ++ ", " ++ show pending ++ " messages left" let cref = (Map.!) hmap eh writeIORef cref res _ <- curl_multi_remove_handle mh eh when (pending > 0) $ readMessages mh hmap -- | Loops and polls curl until there are no more remaining handles. performMulti :: CurlMH -> HandleMap -> CInt -> IO () performMulti mh hmap expected = do (mcode, running) <- curlMultiPerform mh delay <- case mcode of CurlmCallMultiPerform -> return $ return () CurlmOK -> return $ threadDelay pollDelayInterval code -> error $ "Received bad return code from" ++ "'curl_multi_perform': " ++ show code logDebug $ "mcode: " ++ show mcode ++ ", remaining: " ++ show running -- check if any handles are done and then retrieve their messages when (expected /= running) $ readMessages mh hmap -- and if we still have handles running, loop when (running > 0) $ delay >> performMulti mh hmap running -- | Template for the Curl error buffer. errorBuffer :: String errorBuffer = replicate errorBufferSize '\0' -- | Allocate a NULL-initialised error buffer. mallocErrorBuffer :: IO CString mallocErrorBuffer = fst `fmap` newCStringLen errorBuffer -- | Initialise a curl handle. This is just a wrapper over the -- "Network.Curl" function 'initialize', plus adding our options. makeEasyHandle :: (IORef [String], Ptr CChar, ([CurlOption], URLString)) -> IO Curl makeEasyHandle (f, e, (opts, url)) = do h <- initialize setopts h opts setopts h [ CurlWriteFunction (writeHandle f) , CurlErrorBuffer e , CurlURL url , CurlFailOnError True , CurlNoSignal True , CurlProxy "" ] return h -- * Main multi-call work function -- | Perform a multi-call against a list of nodes. execMultiCall :: [([CurlOption], String)] -> IO [(CurlCode, String)] execMultiCall ous = do -- error buffers errorbufs <- mapM (const mallocErrorBuffer) ous -- result buffers outbufs <- mapM (\_ -> newIORef []) ous -- handles ehandles <- mapM makeEasyHandle $ zip3 outbufs errorbufs ous -- data.map holding handles to error code iorefs hmap <- foldM (\m h -> curlPrim h (\_ hnd -> do ccode <- newIORef CurlOK return $ Map.insert hnd ccode m )) Map.empty ehandles mh <- curl_multi_init mapM_ (curlMultiAddHandle mh) ehandles performMulti mh hmap (fromIntegral $ length ehandles) -- dummy code to keep the handles alive until here mapM_ (\h -> curlPrim h (\_ _ -> return ())) ehandles -- cleanup the multi handle mh_cleanup <- toMCode `fmap` curl_multi_cleanup mh when (mh_cleanup /= CurlmOK) . logError $ "Non-OK return from multi_cleanup: " ++ show mh_cleanup -- and now extract the data from the IORefs mapM (\(e, b, h) -> do s <- peekCString e free e cref <- curlPrim h (\_ hnd -> return $ (Map.!) hmap hnd) ccode <- readIORef cref result <- if ccode == CurlOK then (concat . reverse) `fmap` readIORef b else return s return (ccode, result) ) $ zip3 errorbufs outbufs ehandles ganeti-3.1.0~rc2/src/Ganeti/Daemon.hs000064400000000000000000000454271476477700300173550ustar00rootroot00000000000000{-| Implementation of the generic daemon functionality. -} {-# LANGUAGE CPP #-} {- Copyright (C) 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Daemon ( DaemonOptions(..) , OptType , CheckFn , PrepFn , MainFn , defaultOptions , oShowHelp , oShowVer , oNoDaemonize , oNoUserChecks , oDebug , oPort , oBindAddress , oSyslogUsage , oForceNode , oNoVoting , oYesDoIt , parseArgs , parseAddress , cleanupSocket , describeError , genericMain , getFQDN ) where import Control.Concurrent import Control.Exception import Control.Monad import Control.Monad.Fail (MonadFail) import Data.Maybe (fromMaybe, listToMaybe) import Text.Printf import Data.Word import GHC.IO.Handle (hDuplicateTo) import Network.BSD (getHostName) import qualified Network.Socket as Socket import Network.Socket import System.Console.GetOpt import System.Directory import System.Exit import System.Environment import System.IO import System.IO.Error (isDoesNotExistError, modifyIOError, annotateIOError) import qualified System.Posix as Posix import System.Posix.Directory import System.Posix.Files import System.Posix.Process import System.Posix.Types import System.Posix.Signals import Ganeti.Common as Common import Ganeti.Compat (openFd, closeFd) import Ganeti.Logging import Ganeti.Runtime import Ganeti.BasicTypes import Ganeti.Utils import qualified Ganeti.Constants as C import qualified Ganeti.Ssconf as Ssconf -- * Constants -- | \/dev\/null path. devNull :: FilePath devNull = "/dev/null" -- | Error message prefix, used in two separate paths (when forking -- and when not). daemonStartupErr :: String -> String daemonStartupErr = ("Error when starting the daemon process: " ++) -- * Data types -- | Command line options structure. data DaemonOptions = DaemonOptions { optShowHelp :: Bool -- ^ Just show the help , optShowVer :: Bool -- ^ Just show the program version , optShowComp :: Bool -- ^ Just show the completion info , optDaemonize :: Bool -- ^ Whether to daemonize or not , optPort :: Maybe Word16 -- ^ Override for the network port , optDebug :: Bool -- ^ Enable debug messages , optNoUserChecks :: Bool -- ^ Ignore user checks , optBindAddress :: Maybe String -- ^ Listen on a custom address , optSyslogUsage :: Maybe SyslogUsage -- ^ Override for Syslog usage , optForceNode :: Bool -- ^ Ignore node checks , optNoVoting :: Bool -- ^ skip voting for master , optYesDoIt :: Bool -- ^ force dangerous options } -- | Default values for the command line options. defaultOptions :: DaemonOptions defaultOptions = DaemonOptions { optShowHelp = False , optShowVer = False , optShowComp = False , optDaemonize = True , optPort = Nothing , optDebug = False , optNoUserChecks = False , optBindAddress = Nothing , optSyslogUsage = Nothing , optForceNode = False , optNoVoting = False , optYesDoIt = False } instance StandardOptions DaemonOptions where helpRequested = optShowHelp verRequested = optShowVer compRequested = optShowComp requestHelp o = o { optShowHelp = True } requestVer o = o { optShowVer = True } requestComp o = o { optShowComp = True } -- | Abrreviation for the option type. type OptType = GenericOptType DaemonOptions -- | Check function type. type CheckFn a = DaemonOptions -> IO (Either ExitCode a) -- | Prepare function type. type PrepFn a b = DaemonOptions -> a -> IO b -- | Main execution function type. type MainFn a b = DaemonOptions -> a -> b -> IO () -- * Command line options oNoDaemonize :: OptType oNoDaemonize = (Option "f" ["foreground"] (NoArg (\ opts -> Ok opts { optDaemonize = False })) "Don't detach from the current terminal", OptComplNone) oDebug :: OptType oDebug = (Option "d" ["debug"] (NoArg (\ opts -> Ok opts { optDebug = True })) "Enable debug messages", OptComplNone) oNoUserChecks :: OptType oNoUserChecks = (Option "" ["no-user-checks"] (NoArg (\ opts -> Ok opts { optNoUserChecks = True })) "Ignore user checks", OptComplNone) oPort :: Int -> OptType oPort def = (Option "p" ["port"] (reqWithConversion (tryRead "reading port") (\port opts -> Ok opts { optPort = Just port }) "PORT") ("Network port (default: " ++ show def ++ ")"), OptComplInteger) oBindAddress :: OptType oBindAddress = (Option "b" ["bind"] (ReqArg (\addr opts -> Ok opts { optBindAddress = Just addr }) "ADDR") ("Bind address (default is 'any' on either IPv4 or IPv6, " ++ "depending on cluster configuration)"), OptComplInetAddr) oSyslogUsage :: OptType oSyslogUsage = (Option "" ["syslog"] (reqWithConversion syslogUsageFromRaw (\su opts -> Ok opts { optSyslogUsage = Just su }) "SYSLOG") ("Enable logging to syslog (except debug messages); " ++ "one of 'no', 'yes' or 'only' [" ++ C.syslogUsage ++ "]"), OptComplChoices ["yes", "no", "only"]) oForceNode :: OptType oForceNode = (Option "" ["force-node"] (NoArg (\ opts -> Ok opts { optForceNode = True })) "Force the daemon to run on a different node than the master", OptComplNone) oNoVoting :: OptType oNoVoting = (Option "" ["no-voting"] (NoArg (\ opts -> Ok opts { optNoVoting = True })) "Skip node agreement check (dangerous)", OptComplNone) oYesDoIt :: OptType oYesDoIt = (Option "" ["yes-do-it"] (NoArg (\ opts -> Ok opts { optYesDoIt = True })) "Force a dangerous operation", OptComplNone) -- | Generic options. genericOpts :: [OptType] genericOpts = [ oShowHelp , oShowVer , oShowComp ] -- | Annotates and transforms IOErrors into a Result type. This can be -- used in the error handler argument to 'catch', for example. ioErrorToResult :: String -> IOError -> IO (Result a) ioErrorToResult description exc = return . Bad $ description ++ ": " ++ show exc -- | Small wrapper over getArgs and 'parseOpts'. parseArgs :: String -> [OptType] -> IO (DaemonOptions, [String]) parseArgs cmd options = do cmd_args <- getArgs parseOpts defaultOptions cmd_args cmd (options ++ genericOpts) [] -- * Daemon-related functions -- | PID file mode. pidFileMode :: FileMode pidFileMode = Posix.unionFileModes Posix.ownerReadMode Posix.ownerWriteMode -- | PID file open flags. pidFileFlags :: Posix.OpenFileFlags pidFileFlags = Posix.defaultFileFlags { Posix.noctty = True, Posix.trunc = False } -- | Writes a PID file and locks it. writePidFile :: FilePath -> IO Fd writePidFile path = do fd <- openFd path Posix.ReadWrite (Just pidFileMode) pidFileFlags Posix.setLock fd (Posix.WriteLock, AbsoluteSeek, 0, 0) my_pid <- getProcessID _ <- Posix.fdWrite fd (show my_pid ++ "\n") return fd -- | Helper function to ensure a socket doesn't exist. Should only be -- called once we have locked the pid file successfully. cleanupSocket :: FilePath -> IO () cleanupSocket socketPath = catchJust (guard . isDoesNotExistError) (removeLink socketPath) (const $ return ()) -- | Sets up a daemon's environment. setupDaemonEnv :: FilePath -> FileMode -> IO () setupDaemonEnv cwd umask = do changeWorkingDirectory cwd _ <- setFileCreationMask umask _ <- createSession return () -- | Cleanup function, performing all the operations that need to be done prior -- to shutting down a daemon. finalCleanup :: FilePath -> IO () finalCleanup = removeFile -- | Signal handler for the termination signal. handleSigTerm :: ThreadId -> IO () handleSigTerm mainTID = -- Throw termination exception to the main thread, so that the daemon is -- actually stopped in the proper way, executing all the functions waiting on -- "finally" statement. Control.Exception.throwTo mainTID ExitSuccess -- | Signal handler for reopening log files. handleSigHup :: FilePath -> IO () handleSigHup path = do setupDaemonFDs (Just path) logInfo "Reopening log files after receiving SIGHUP" -- | Sets up a daemon's standard file descriptors. setupDaemonFDs :: Maybe FilePath -> IO () setupDaemonFDs logfile = do null_in_handle <- openFile devNull ReadMode null_out_handle <- openFile (fromMaybe devNull logfile) AppendMode hDuplicateTo null_in_handle stdin hDuplicateTo null_out_handle stdout hDuplicateTo null_out_handle stderr hClose null_in_handle hClose null_out_handle -- | Computes the default bind address for a given family. defaultBindAddr :: Int -- ^ The port we want -> Result Socket.Family -- ^ The cluster IP family -> IO (Result (Socket.Family, Socket.SockAddr)) #if MIN_VERSION_network(2,7,0) defaultBindAddr _ (Bad m) = return (Bad m) defaultBindAddr port (Ok fam) = do addrs <- getAddrInfo (Just defaultHints { addrFamily = fam , addrFlags = [AI_PASSIVE] , addrSocketType = Stream }) Nothing (Just (show port)) return $ case addrs of a:_ -> Ok $ (fam, addrAddress a) [] -> Bad $ "Cannot resolve default listening addres?!" #else defaultBindAddr port (Ok Socket.AF_INET) = return $ Ok (Socket.AF_INET, Socket.SockAddrInet (fromIntegral port) Socket.iNADDR_ANY) defaultBindAddr port (Ok Socket.AF_INET6) = return $ Ok (Socket.AF_INET6, Socket.SockAddrInet6 (fromIntegral port) 0 Socket.iN6ADDR_ANY 0) defaultBindAddr _ fam = return $ Bad $ "Unsupported address family: " ++ show fam #endif -- | Based on the options, compute the socket address to use for the -- daemon. parseAddress :: DaemonOptions -- ^ Command line options -> Int -- ^ Default port for this daemon -> IO (Result (Socket.Family, Socket.SockAddr)) parseAddress opts defport = do let port = maybe defport fromIntegral $ optPort opts def_family <- Ssconf.getPrimaryIPFamily Nothing case optBindAddress opts of Nothing -> defaultBindAddr port def_family Just saddr -> Control.Exception.catch (resolveAddr port saddr) (ioErrorToResult $ "Invalid address " ++ saddr) -- | Environment variable to override the assumed host name of the -- current node. vClusterHostNameEnvVar :: String vClusterHostNameEnvVar = "GANETI_HOSTNAME" -- | Get the real full qualified host name. getFQDN' :: Maybe Socket.AddrInfo -> IO String getFQDN' hints = do hostname <- getHostName addrInfos <- Socket.getAddrInfo hints (Just hostname) Nothing let address = listToMaybe addrInfos >>= (Just . Socket.addrAddress) case address of Just a -> do fqdn <- liftM fst $ Socket.getNameInfo [] True False a return (fromMaybe hostname fqdn) Nothing -> return hostname -- | Return the full qualified host name, honoring the vcluster setup -- and hints on the preferred socket type or protocol. getFQDNwithHints :: Maybe Socket.AddrInfo -> IO String getFQDNwithHints hints = do let ioErrorToNothing :: IOError -> IO (Maybe String) ioErrorToNothing _ = return Nothing vcluster_node <- Control.Exception.catch (liftM Just (getEnv vClusterHostNameEnvVar)) ioErrorToNothing case vcluster_node of Just node_name -> return node_name Nothing -> getFQDN' hints -- | Return the full qualified host name, honoring the vcluster setup. getFQDN :: IO String getFQDN = do familyresult <- Ssconf.getPrimaryIPFamily Nothing getFQDNwithHints $ genericResult (const Nothing) (\family -> Just $ Socket.defaultHints { Socket.addrFamily = family }) familyresult -- | Returns if the current node is the master node. isMaster :: IO Bool isMaster = do curNode <- getFQDN masterNode <- Ssconf.getMasterNode Nothing case masterNode of Ok n -> return (curNode == n) Bad _ -> return False -- | Ensures that the daemon runs on the right node (and exits -- gracefully if it doesnt) ensureNode :: GanetiDaemon -> DaemonOptions -> IO () ensureNode daemon opts = do is_master <- isMaster when (daemonOnlyOnMaster daemon && not is_master && not (optForceNode opts)) $ do putStrLn "Not master, exiting." exitWith (ExitFailure C.exitNotmaster) -- | Run an I\/O action that might throw an I\/O error, under a -- handler that will simply annotate and re-throw the exception. describeError :: String -> Maybe Handle -> Maybe FilePath -> IO a -> IO a describeError descr hndl fpath = modifyIOError (\e -> annotateIOError e descr hndl fpath) -- | Run an I\/O action as a daemon. -- -- WARNING: this only works in single-threaded mode (either using the -- single-threaded runtime, or using the multi-threaded one but with -- only one OS thread, i.e. -N1). daemonize :: FilePath -> (Maybe Fd -> IO ()) -> IO () daemonize logfile action = do (rpipe, wpipe) <- Posix.createPipe -- first fork _ <- forkProcess $ do -- in the child closeFd rpipe let wpipe' = Just wpipe setupDaemonEnv "/" (Posix.unionFileModes Posix.groupModes Posix.otherModes) setupDaemonFDs (Just logfile) `Control.Exception.catch` handlePrepErr False wpipe' -- second fork, launches the actual child code; standard -- double-fork technique _ <- forkProcess (action wpipe') exitImmediately ExitSuccess closeFd wpipe hndl <- Posix.fdToHandle rpipe errors <- hGetContents hndl ecode <- if null errors then return ExitSuccess else do hPutStrLn stderr $ daemonStartupErr errors return $ ExitFailure C.exitFailure exitImmediately ecode -- | Generic daemon startup. genericMain :: GanetiDaemon -- ^ The daemon we're running -> [OptType] -- ^ The available options -> CheckFn a -- ^ Check function -> PrepFn a b -- ^ Prepare function -> MainFn a b -- ^ Execution function -> IO () genericMain daemon options check_fn prep_fn exec_fn = do let progname = daemonName daemon (opts, args) <- parseArgs progname options -- Modify handleClient in Ganeti.UDSServer to remove this logging from luxid. when (optDebug opts && daemon == GanetiLuxid) . hPutStrLn stderr $ printf C.debugModeConfidentialityWarning (daemonName daemon) ensureNode daemon opts exitUnless (null args) "This program doesn't take any arguments" unless (optNoUserChecks opts) $ do runtimeEnts <- runResultT getEnts ents <- exitIfBad "Can't find required user/groups" runtimeEnts verifyDaemonUser daemon ents syslog <- case optSyslogUsage opts of Nothing -> exitIfBad "Invalid cluster syslog setting" $ syslogUsageFromRaw C.syslogUsage Just v -> return v log_file <- daemonLogFile daemon -- run the check function and optionally exit if it returns an exit code check_result <- check_fn opts check_result' <- case check_result of Left code -> exitWith code Right v -> return v let processFn = if optDaemonize opts then daemonize log_file else \action -> action Nothing _ <- installHandler lostConnection (Catch (handleSigHup log_file)) Nothing processFn $ innerMain daemon opts syslog check_result' prep_fn exec_fn -- | Full prepare function. -- -- This is executed after daemonization, and sets up both the log -- files (a generic functionality) and the custom prepare function of -- the daemon. fullPrep :: GanetiDaemon -- ^ The daemon we're running -> DaemonOptions -- ^ The options structure, filled from the cmdline -> SyslogUsage -- ^ Syslog mode -> a -- ^ Check results -> PrepFn a b -- ^ Prepare function -> IO (FilePath, b) fullPrep daemon opts syslog check_result prep_fn = do logfile <- if optDaemonize opts then return Nothing else liftM Just $ daemonLogFile daemon pidfile <- daemonPidFile daemon let dname = daemonName daemon setupLogging logfile dname (optDebug opts) True False syslog _ <- describeError "writing PID file; already locked?" Nothing (Just pidfile) $ writePidFile pidfile logNotice $ dname ++ " daemon startup" prep_res <- prep_fn opts check_result tid <- myThreadId _ <- installHandler sigTERM (Catch $ handleSigTerm tid) Nothing return (pidfile, prep_res) -- | Inner daemon function. -- -- This is executed after daemonization. innerMain :: GanetiDaemon -- ^ The daemon we're running -> DaemonOptions -- ^ The options structure, filled from the cmdline -> SyslogUsage -- ^ Syslog mode -> a -- ^ Check results -> PrepFn a b -- ^ Prepare function -> MainFn a b -- ^ Execution function -> Maybe Fd -- ^ Error reporting function -> IO () innerMain daemon opts syslog check_result prep_fn exec_fn fd = do (pidFile, prep_result) <- fullPrep daemon opts syslog check_result prep_fn `Control.Exception.catch` handlePrepErr True fd -- no error reported, we should now close the fd maybeCloseFd fd finally (exec_fn opts check_result prep_result) (finalCleanup pidFile >> logNotice (daemonName daemon ++ " daemon shutdown")) -- | Daemon prepare error handling function. handlePrepErr :: Bool -> Maybe Fd -> IOError -> IO a handlePrepErr logging_setup fd err = do let msg = show err case fd of -- explicitly writing to the fd directly, since when forking it's -- better (safer) than trying to convert this into a full handle Just fd' -> Posix.fdWrite fd' msg >> return () Nothing -> hPutStrLn stderr (daemonStartupErr msg) when logging_setup $ logError msg exitWith $ ExitFailure 1 -- | Close a file descriptor. maybeCloseFd :: Maybe Fd -> IO () maybeCloseFd Nothing = return () maybeCloseFd (Just fd) = closeFd fd ganeti-3.1.0~rc2/src/Ganeti/Daemon/000075500000000000000000000000001476477700300170055ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Daemon/Utils.hs000064400000000000000000000111341476477700300204410ustar00rootroot00000000000000{-| Utility functions for complex operations carried out by several daemons. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Daemon.Utils ( verifyMaster , handleMasterVerificationOptions ) where import Control.Concurrent (threadDelay) import Control.Monad (unless) import Data.Either (rights) import qualified Data.Foldable as F import Data.List (partition) import System.Exit (ExitCode(..)) import Ganeti.BasicTypes import qualified Ganeti.Config as Config import qualified Ganeti.Constants as C import Ganeti.Daemon (getFQDN, DaemonOptions, optNoVoting, optYesDoIt) import Ganeti.Logging import Ganeti.Objects import qualified Ganeti.Path as Path import Ganeti.Utils (frequency) import Ganeti.Rpc -- | Gather votes from all nodes and verify that we we are -- the master. Return True if the voting is won, False if -- not enough verifyMasterVotes :: IO (Result Bool) verifyMasterVotes = runResultT $ do liftIO $ logDebug "Gathering votes for the master node" myName <- liftIO getFQDN liftIO . logDebug $ "My hostname is " ++ myName conf_file <- liftIO Path.clusterConfFile config <- mkResultT $ Config.loadConfig conf_file let nodes = F.toList $ configNodes config votes <- liftIO . executeRpcCall nodes $ RpcCallMasterNodeName let (missing, valid) = partition (isLeft . snd) votes noDataNodes = map (nodeName . fst) missing validVotes = map rpcResultMasterNodeNameMaster . rights $ map snd valid inFavor = length $ filter (== myName) validVotes voters = length nodes unknown = length missing liftIO . unless (null noDataNodes) . logWarning . (++) "No voting RPC result from " $ show noDataNodes liftIO . logDebug . (++) "Valid votes: " $ show (frequency validVotes) if 2 * inFavor > voters then return True else if 2 * (inFavor + unknown) > voters then return False else fail $ "Voting cannot be won by " ++ myName ++ ", valid votes of " ++ show voters ++ " are " ++ show (frequency validVotes) -- | Verify, by voting, that this node is the master. Bad if we're not. -- Allow the given number of retries to wait for not available nodes. verifyMaster :: Int -> IO (Result ()) verifyMaster retries = runResultT $ do won <- mkResultT verifyMasterVotes unless won $ if retries <= 0 then fail "Couldn't gather voting results of enough nodes" else do liftIO $ logDebug "Voting not final due to missing votes." liftIO . threadDelay $ C.masterVotingRetryIntervall * 1000000 mkResultT $ verifyMaster (retries - 1) -- | Verify master position according to the options provided, usually -- by carrying out a voting. Either return unit on success, or a suggested -- exit code. handleMasterVerificationOptions :: DaemonOptions -> IO (Either ExitCode ()) handleMasterVerificationOptions opts = if optNoVoting opts then if optYesDoIt opts then return $ Right () else do logError "The no-voting option is dangerous and cannot be\ \ given without providing yes-do-it as well." return . Left $ ExitFailure C.exitFailure else do masterStatus <- verifyMaster C.masterVotingRetries case masterStatus of Bad s -> do logError $ "Failed to verify master status: " ++ s return . Left $ ExitFailure C.exitFailure Ok _ -> return $ Right () ganeti-3.1.0~rc2/src/Ganeti/DataCollectors.hs000064400000000000000000000073421476477700300210470ustar00rootroot00000000000000{-| Definition of the data collectors used by MonD. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors( collectors ) where import qualified Data.ByteString.UTF8 as UTF8 import Data.Map (findWithDefault) import qualified Ganeti.DataCollectors.CPUload as CPUload import qualified Ganeti.DataCollectors.Diskstats as Diskstats import qualified Ganeti.DataCollectors.Drbd as Drbd import qualified Ganeti.DataCollectors.InstStatus as InstStatus import qualified Ganeti.DataCollectors.Lv as Lv import qualified Ganeti.DataCollectors.XenCpuLoad as XenCpuLoad import Ganeti.DataCollectors.Types (DataCollector(..),ReportBuilder(..)) import Ganeti.JSON (GenericContainer(..)) import Ganeti.Objects import Ganeti.Types -- | The list of available builtin data collectors. collectors :: [DataCollector] collectors = [ cpuLoadCollector , xenCpuLoadCollector , diskStatsCollector , drdbCollector , instStatusCollector , lvCollector ] where f .&&. g = \x y -> f x y && g x y xenHypervisor = flip elem [XenPvm, XenHvm] xenCluster _ cfg = any xenHypervisor . clusterEnabledHypervisors $ configCluster cfg collectorConfig name cfg = let config = fromContainer . clusterDataCollectors $ configCluster cfg in findWithDefault mempty (UTF8.fromString name) config updateInterval name cfg = dataCollectorInterval $ collectorConfig name cfg activeConfig name cfg = dataCollectorActive $ collectorConfig name cfg diskStatsCollector = DataCollector Diskstats.dcName Diskstats.dcCategory Diskstats.dcKind (StatelessR Diskstats.dcReport) Nothing activeConfig updateInterval drdbCollector = DataCollector Drbd.dcName Drbd.dcCategory Drbd.dcKind (StatelessR Drbd.dcReport) Nothing activeConfig updateInterval instStatusCollector = DataCollector InstStatus.dcName InstStatus.dcCategory InstStatus.dcKind (StatelessR InstStatus.dcReport) Nothing (xenCluster .&&. activeConfig) updateInterval lvCollector = DataCollector Lv.dcName Lv.dcCategory Lv.dcKind (StatelessR Lv.dcReport) Nothing activeConfig updateInterval cpuLoadCollector = DataCollector CPUload.dcName CPUload.dcCategory CPUload.dcKind (StatefulR CPUload.dcReport) (Just CPUload.dcUpdate) activeConfig updateInterval xenCpuLoadCollector = DataCollector XenCpuLoad.dcName XenCpuLoad.dcCategory XenCpuLoad.dcKind (StatefulR XenCpuLoad.dcReport) (Just XenCpuLoad.dcUpdate) activeConfig updateInterval ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/000075500000000000000000000000001476477700300205055ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/CLI.hs000064400000000000000000000124321476477700300214520ustar00rootroot00000000000000{-| Implementation of DataCollectors CLI functions. This module holds the common command-line related functions for the collector binaries. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.CLI ( Options(..) , OptType , defaultOptions -- * The options , oShowHelp , oShowVer , oShowComp , oDrbdPairing , oDrbdStatus , oNode , oConfdAddr , oConfdPort , oInputFile , oInstances , genericOptions ) where import System.Console.GetOpt import Ganeti.BasicTypes import Ganeti.Common as Common import Ganeti.Utils -- * Data types -- | Command line options structure. data Options = Options { optShowHelp :: Bool -- ^ Just show the help , optShowComp :: Bool -- ^ Just show the completion info , optShowVer :: Bool -- ^ Just show the program version , optDrbdStatus :: Maybe FilePath -- ^ Path to the file containing DRBD -- status information , optDrbdPairing :: Maybe FilePath -- ^ Path to the file containing pairings -- between instances and DRBD minors , optNode :: Maybe String -- ^ Info are requested for this node , optConfdAddr :: Maybe String -- ^ IP address of the Confd server , optConfdPort :: Maybe Int -- ^ The port of the Confd server to -- connect to , optInputFile :: Maybe FilePath -- ^ Path to the file containing the -- information to be parsed , optInstances :: Maybe FilePath -- ^ Path to the file contained a -- serialized list of instances as in: -- ([Primary], [Secondary]) } deriving Show -- | Default values for the command line options. defaultOptions :: Options defaultOptions = Options { optShowHelp = False , optShowComp = False , optShowVer = False , optDrbdStatus = Nothing , optDrbdPairing = Nothing , optNode = Nothing , optConfdAddr = Nothing , optConfdPort = Nothing , optInputFile = Nothing , optInstances = Nothing } -- | Abbreviation for the option type. type OptType = GenericOptType Options instance StandardOptions Options where helpRequested = optShowHelp verRequested = optShowVer compRequested = optShowComp requestHelp o = o { optShowHelp = True } requestVer o = o { optShowVer = True } requestComp o = o { optShowComp = True } -- * Command line options oDrbdPairing :: OptType oDrbdPairing = ( Option "p" ["drbd-pairing"] (ReqArg (\ f o -> Ok o { optDrbdPairing = Just f}) "FILE") "the FILE containing pairings between instances and DRBD minors", OptComplFile) oDrbdStatus :: OptType oDrbdStatus = ( Option "s" ["drbd-status"] (ReqArg (\ f o -> Ok o { optDrbdStatus = Just f }) "FILE") "the DRBD status FILE", OptComplFile) oNode :: OptType oNode = ( Option "n" ["node"] (ReqArg (\ n o -> Ok o { optNode = Just n }) "NODE") "the FQDN of the NODE about which information is requested", OptComplFile) oConfdAddr :: OptType oConfdAddr = ( Option "a" ["address"] (ReqArg (\ a o -> Ok o { optConfdAddr = Just a }) "IP_ADDR") "the IP address of the Confd server to connect to", OptComplFile) oConfdPort :: OptType oConfdPort = (Option "p" ["port"] (reqWithConversion (tryRead "reading port") (\port opts -> Ok opts { optConfdPort = Just port }) "PORT") "Network port of the Confd server to connect to", OptComplInteger) oInputFile :: OptType oInputFile = ( Option "f" ["file"] (ReqArg (\ f o -> Ok o { optInputFile = Just f }) "FILE") "the input FILE", OptComplFile) oInstances :: OptType oInstances = ( Option "i" ["instances"] (ReqArg (\ f o -> Ok o { optInstances = Just f}) "FILE") "the FILE containing serialized instances", OptComplFile) -- | Generic options. genericOptions :: [GenericOptType Options] genericOptions = [ oShowVer , oShowHelp , oShowComp ] ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/CPUload.hs000064400000000000000000000157761476477700300223500ustar00rootroot00000000000000{-# OPTIONS_GHC -fno-warn-overlapping-patterns #-} {-| @/proc/stat@ data collector. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.CPUload ( dcName , dcVersion , dcFormatVersion , dcCategory , dcKind , dcReport , dcUpdate ) where import Control.Arrow (first) import qualified Control.Exception as E import Data.Attoparsec.Text.Lazy as A import Data.Maybe (fromMaybe) import Data.Text.Lazy (pack, unpack) import qualified Text.JSON as J import qualified Data.Sequence as Seq import System.Posix.Unistd (getSysVar, SysVar(ClockTick)) import System.Time (ClockTime(..), getClockTime) import qualified Ganeti.BasicTypes as BT import qualified Ganeti.Constants as C import Ganeti.Cpu.LoadParser(cpustatParser) import Ganeti.DataCollectors.Types import qualified Ganeti.JSON as GJ import Ganeti.Utils.Time (clockTimeToUSec) import Ganeti.Utils import Ganeti.Cpu.Types -- | The default path of the CPU status file. -- It is hardcoded because it is not likely to change. defaultFile :: FilePath defaultFile = C.statFile -- | The buffer size of the values kept in the map. bufferSize :: Int bufferSize = C.cpuavgloadBufferSize -- | The window size of the values that will export the average load. windowSize :: Integer windowSize = toInteger C.cpuavgloadWindowSize -- | The default setting for the maximum amount of not parsed character to -- print in case of error. -- It is set to use most of the screen estate on a standard 80x25 terminal. -- TODO: add the possibility to set this with a command line parameter. defaultCharNum :: Int defaultCharNum = 80*20 -- | The name of this data collector. dcName :: String dcName = C.dataCollectorCPULoad -- | The version of this data collector. dcVersion :: DCVersion dcVersion = DCVerBuiltin -- | The version number for the data format of this data collector. dcFormatVersion :: Int dcFormatVersion = 1 -- | The category of this data collector. dcCategory :: Maybe DCCategory dcCategory = Nothing -- | The kind of this data collector. dcKind :: DCKind dcKind = DCKPerf -- | The data exported by the data collector, taken from the default location. dcReport :: Maybe CollectorData -> IO DCReport dcReport colData = let extractColData c = case c of (CPULoadData v) -> Just v _ -> Nothing cpuLoadData = fromMaybe Seq.empty $ colData >>= extractColData in buildDCReport cpuLoadData -- | Data stored by the collector in mond's memory. type Buffer = Seq.Seq (ClockTime, [Int]) -- | Compute the load from a CPU. computeLoad :: CPUstat -> Int computeLoad cpuData = csUser cpuData + csNice cpuData + csSystem cpuData + csIowait cpuData + csIrq cpuData + csSoftirq cpuData + csSteal cpuData + csGuest cpuData + csGuestNice cpuData -- | Reads and Computes the load for each CPU. dcCollectFromFile :: FilePath -> IO (ClockTime, [Int]) dcCollectFromFile inputFile = do contents <- ((E.try $ readFile inputFile) :: IO (Either IOError String)) >>= exitIfBad "reading from file" . either (BT.Bad . show) BT.Ok cpustatData <- case A.parse cpustatParser $ pack contents of A.Fail unparsedText contexts errorMessage -> exitErr $ show (Prelude.take defaultCharNum $ unpack unparsedText) ++ "\n" ++ show contexts ++ "\n" ++ errorMessage A.Done _ cpustatD -> return cpustatD now <- getClockTime return (now, map computeLoad cpustatData) -- | Returns the collected data in the appropriate type. dcCollect :: IO Buffer dcCollect = do l <- dcCollectFromFile defaultFile return (Seq.singleton l) -- | Formats data for JSON transformation. formatData :: [Double] -> CPUavgload formatData [] = CPUavgload (0 :: Int) [] (0 :: Double) formatData l@(x:xs) = CPUavgload (length l - 1) xs x -- | Update a Map Entry. updateEntry :: Buffer -> Buffer -> Buffer updateEntry newBuffer mapEntry = (Seq.><) newBuffer (if Seq.length mapEntry < bufferSize then mapEntry else Seq.drop 1 mapEntry) -- | Updates the given Collector data. dcUpdate :: Maybe CollectorData -> IO CollectorData dcUpdate mcd = do v <- dcCollect let new_v = fromMaybe v $ do cd <- mcd case cd of CPULoadData old_v -> return $ updateEntry v old_v _ -> Nothing new_v `seq` return $ CPULoadData new_v -- | Computes the average load for every CPU and the overall from data read -- from the map. Returns Bad if there are not enough values to compute it. computeAverage :: Buffer -> Integer -> Integer -> BT.Result [Double] computeAverage s w ticks = let inUSec = fmap (first clockTimeToUSec) s window = Seq.takeWhileL ((> w) . fst) inUSec go Seq.EmptyL _ = BT.Bad "Empty buffer" go _ Seq.EmptyR = BT.Bad "Empty buffer" go (leftmost Seq.:< _) (_ Seq.:> rightmost) = do let (timestampL, listL) = leftmost (timestampR, listR) = rightmost workInWindow = zipWith (-) listL listR timediff = timestampL - timestampR overall = fromInteger (timediff * ticks) / 1000000 :: Double if overall > 0 then BT.Ok $ map (flip (/) overall . fromIntegral) workInWindow else BT.Bad $ "Time covered by data is not sufficient." ++ "The window considered is " ++ show w in go (Seq.viewl window) (Seq.viewr window) -- | This function computes the JSON representation of the CPU load. buildJsonReport :: Buffer -> IO J.JSValue buildJsonReport v = do ticks <- getSysVar ClockTick let res = computeAverage v windowSize ticks showError s = J.showJSON $ GJ.containerFromList [("error", s)] return $ BT.genericResult showError (J.showJSON . formatData) res -- | This function computes the DCReport for the CPU load. buildDCReport :: Buffer -> IO DCReport buildDCReport v = buildJsonReport v >>= buildReport dcName dcVersion dcFormatVersion dcCategory dcKind ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/Diskstats.hs000064400000000000000000000105611476477700300230150ustar00rootroot00000000000000{-| @/proc/diskstats@ data collector. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.Diskstats ( main , options , arguments , dcName , dcVersion , dcFormatVersion , dcCategory , dcKind , dcReport ) where import qualified Control.Exception as E import Control.Monad import Data.Attoparsec.Text.Lazy as A import Data.Maybe import Data.Text.Lazy (pack, unpack) import qualified Text.JSON as J import qualified Ganeti.BasicTypes as BT import qualified Ganeti.Constants as C import Ganeti.Storage.Diskstats.Parser(diskstatsParser) import Ganeti.Common import Ganeti.DataCollectors.CLI import Ganeti.DataCollectors.Types import Ganeti.Utils -- | The default path of the diskstats status file. -- It is hardcoded because it is not likely to change. defaultFile :: FilePath defaultFile = C.diskstatsFile -- | The default setting for the maximum amount of not parsed character to -- print in case of error. -- It is set to use most of the screen estate on a standard 80x25 terminal. -- TODO: add the possibility to set this with a command line parameter. defaultCharNum :: Int defaultCharNum = 80*20 -- | The name of this data collector. dcName :: String dcName = C.dataCollectorDiskStats -- | The version of this data collector. dcVersion :: DCVersion dcVersion = DCVerBuiltin -- | The version number for the data format of this data collector. dcFormatVersion :: Int dcFormatVersion = 1 -- | The category of this data collector. dcCategory :: Maybe DCCategory dcCategory = Just DCStorage -- | The kind of this data collector. dcKind :: DCKind dcKind = DCKPerf -- | The data exported by the data collector, taken from the default location. dcReport :: IO DCReport dcReport = buildDCReport defaultFile -- * Command line options options :: IO [OptType] options = return [ oInputFile ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [ArgCompletion OptComplFile 0 (Just 0)] -- | This function computes the JSON representation of the diskstats status. buildJsonReport :: FilePath -> IO J.JSValue buildJsonReport inputFile = do contents <- ((E.try $ readFile inputFile) :: IO (Either IOError String)) >>= exitIfBad "reading from file" . either (BT.Bad . show) BT.Ok diskstatsData <- case A.parse diskstatsParser $ pack contents of A.Fail unparsedText contexts errorMessage -> exitErr $ show (Prelude.take defaultCharNum $ unpack unparsedText) ++ "\n" ++ show contexts ++ "\n" ++ errorMessage A.Done _ diskstatsD -> return diskstatsD return $ J.showJSON diskstatsData -- | This function computes the DCReport for the diskstats status. buildDCReport :: FilePath -> IO DCReport buildDCReport inputFile = buildJsonReport inputFile >>= buildReport dcName dcVersion dcFormatVersion dcCategory dcKind -- | Main function. main :: Options -> [String] -> IO () main opts args = do let inputFile = fromMaybe defaultFile $ optInputFile opts unless (null args) . exitErr $ "This program takes exactly zero" ++ " arguments, got '" ++ unwords args ++ "'" report <- buildDCReport inputFile putStrLn $ J.encode report ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/Drbd.hs000064400000000000000000000177561476477700300217340ustar00rootroot00000000000000{-| DRBD data collector. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.Drbd ( main , options , arguments , dcName , dcVersion , dcFormatVersion , dcCategory , dcKind , dcReport ) where import qualified Control.Exception as E import Control.Monad import Data.Attoparsec.Text.Lazy as A import Data.List import Data.Maybe import Data.Text.Lazy (pack, unpack) import Network.BSD (getHostName) import qualified Text.JSON as J import qualified Ganeti.BasicTypes as BT import qualified Ganeti.Constants as C import Ganeti.Storage.Drbd.Parser(drbdStatusParser) import Ganeti.Storage.Drbd.Types import Ganeti.Common import Ganeti.Confd.Client import Ganeti.Confd.Types import Ganeti.DataCollectors.CLI import Ganeti.DataCollectors.Types import Ganeti.Utils -- | The default path of the DRBD status file. -- It is hardcoded because it is not likely to change. defaultFile :: FilePath defaultFile = C.drbdStatusFile -- | The default setting for the maximum amount of not parsed character to -- print in case of error. -- It is set to use most of the screen estate on a standard 80x25 terminal. -- TODO: add the possibility to set this with a command line parameter. defaultCharNum :: Int defaultCharNum = 80*20 -- | The name of this data collector. dcName :: String dcName = C.dataCollectorDrbd -- | The version of this data collector. dcVersion :: DCVersion dcVersion = DCVerBuiltin -- | The version number for the data format of this data collector. dcFormatVersion :: Int dcFormatVersion = 1 -- | The category of this data collector. dcCategory :: Maybe DCCategory dcCategory = Just DCStorage -- | The kind of this data collector. dcKind :: DCKind dcKind = DCKStatus -- | The data exported by the data collector, taken from the default location. dcReport :: IO DCReport dcReport = buildDCReport defaultFile Nothing -- * Command line options options :: IO [OptType] options = return [ oDrbdStatus , oDrbdPairing ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [ArgCompletion OptComplFile 0 (Just 0)] -- | Get information about the pairing of DRBD minors and Ganeti instances -- on the current node. The information is taken from the Confd client -- or, if a filename is specified, from a JSON encoded file (for testing -- purposes). getPairingInfo :: Maybe String -> IO (BT.Result [DrbdInstMinor]) getPairingInfo Nothing = do curNode <- getHostName client <- getConfdClient Nothing Nothing reply <- query client ReqNodeDrbd $ PlainQuery curNode return $ case fmap (J.readJSONs . confdReplyAnswer) reply of Just (J.Ok instMinor) -> BT.Ok instMinor Just (J.Error msg) -> BT.Bad msg Nothing -> BT.Bad "No answer from the Confd server" getPairingInfo (Just filename) = do content <- readFile filename return $ case J.decode content of J.Ok instMinor -> BT.Ok instMinor J.Error msg -> BT.Bad msg -- | Compute the status code and message, given the current DRBD data -- The final state will have the code corresponding to the worst code of -- all the devices, and the error message given from the concatenation of the -- non-empty error messages. computeStatus :: DRBDStatus -> DCStatus computeStatus (DRBDStatus _ devInfos) = let statuses = map computeDevStatus devInfos (code, strList) = foldr mergeStatuses (DCSCOk, [""]) statuses in DCStatus code $ intercalate "\n" strList -- | Compute the status of a DRBD device and its error message. computeDevStatus :: DeviceInfo -> (DCStatusCode, String) computeDevStatus (UnconfiguredDevice _) = (DCSCOk, "") computeDevStatus dev = let errMsg s = show (minorNumber dev) ++ ": " ++ s compute_helper StandAlone = (DCSCBad, errMsg "No network config available") compute_helper Disconnecting = (DCSCBad, errMsg "The peer is being disconnected") compute_helper Unconnected = (DCSCTempBad, errMsg "Trying to establish a network connection") compute_helper Timeout = (DCSCTempBad, errMsg "Communication problems between the peers") compute_helper BrokenPipe = (DCSCTempBad, errMsg "Communication problems between the peers") compute_helper NetworkFailure = (DCSCTempBad, errMsg "Communication problems between the peers") compute_helper ProtocolError = (DCSCTempBad, errMsg "Communication problems between the peers") compute_helper TearDown = (DCSCBad, errMsg "The peer is closing the connection") compute_helper WFConnection = (DCSCTempBad, errMsg "Trying to establish a network connection") compute_helper WFReportParams = (DCSCTempBad, errMsg "Trying to establish a network connection") compute_helper Connected = (DCSCOk, "") compute_helper StartingSyncS = (DCSCOk, "") compute_helper StartingSyncT = (DCSCOk, "") compute_helper WFBitMapS = (DCSCOk, "") compute_helper WFBitMapT = (DCSCOk, "") compute_helper WFSyncUUID = (DCSCOk, "") compute_helper SyncSource = (DCSCOk, "") compute_helper SyncTarget = (DCSCOk, "") compute_helper PausedSyncS = (DCSCOk, "") compute_helper PausedSyncT = (DCSCOk, "") compute_helper VerifyS = (DCSCOk, "") compute_helper VerifyT = (DCSCOk, "") compute_helper Unconfigured = (DCSCOk, "") in compute_helper $ connectionState dev -- | This function computes the JSON representation of the DRBD status. buildJsonReport :: FilePath -> Maybe FilePath -> IO J.JSValue buildJsonReport statusFile pairingFile = do contents <- ((E.try $ readFile statusFile) :: IO (Either IOError String)) >>= exitIfBad "reading from file" . either (BT.Bad . show) BT.Ok pairingResult <- getPairingInfo pairingFile pairing <- logWarningIfBad "Can't get pairing info" [] pairingResult drbdData <- case A.parse (drbdStatusParser pairing) $ pack contents of A.Fail unparsedText contexts errorMessage -> exitErr $ show (Prelude.take defaultCharNum $ unpack unparsedText) ++ "\n" ++ show contexts ++ "\n" ++ errorMessage A.Done _ drbdS -> return drbdS let status = computeStatus drbdData return . addStatus status $ J.showJSON drbdData -- | This function computes the DCReport for the DRBD status. buildDCReport :: FilePath -> Maybe FilePath -> IO DCReport buildDCReport statusFile pairingFile = buildJsonReport statusFile pairingFile >>= buildReport dcName dcVersion dcFormatVersion dcCategory dcKind -- | Main function. main :: Options -> [String] -> IO () main opts args = do let statusFile = fromMaybe defaultFile $ optDrbdStatus opts pairingFile = optDrbdPairing opts unless (null args) . exitErr $ "This program takes exactly zero" ++ " arguments, got '" ++ unwords args ++ "'" report <- buildDCReport statusFile pairingFile putStrLn $ J.encode report ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/InstStatus.hs000064400000000000000000000164551476477700300231750ustar00rootroot00000000000000{-| Instance status data collector. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.InstStatus ( main , options , arguments , dcName , dcVersion , dcFormatVersion , dcCategory , dcKind , dcReport ) where import Control.Exception.Base import qualified Data.ByteString.UTF8 as UTF8 import Data.List import Data.Maybe import qualified Data.Map as Map import Network.BSD (getHostName) import qualified Text.JSON as J import Ganeti.BasicTypes as BT import Ganeti.Confd.ClientFunctions import Ganeti.Common import qualified Ganeti.Constants as C import Ganeti.DataCollectors.CLI import Ganeti.DataCollectors.InstStatusTypes import Ganeti.DataCollectors.Types import Ganeti.Hypervisor.Xen import Ganeti.Hypervisor.Xen.Types import Ganeti.Logging import Ganeti.Objects import Ganeti.Path import Ganeti.Types import Ganeti.Utils -- | The name of this data collector. dcName :: String dcName = C.dataCollectorInstStatus -- | The version of this data collector. dcVersion :: DCVersion dcVersion = DCVerBuiltin -- | The version number for the data format of this data collector. dcFormatVersion :: Int dcFormatVersion = 1 -- | The category of this data collector. dcCategory :: Maybe DCCategory dcCategory = Just DCInstance -- | The kind of this data collector. dcKind :: DCKind dcKind = DCKStatus -- | The report of this data collector. dcReport :: IO DCReport dcReport = buildInstStatusReport Nothing Nothing -- * Command line options options :: IO [OptType] options = return [ oConfdAddr , oConfdPort ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [] -- | Try to get the reason trail for an instance. In case it is not possible, -- log the failure and return an empty list instead. getReasonTrail :: String -> IO ReasonTrail getReasonTrail instanceName = do fileName <- getInstReasonFilename instanceName content <- try $ readFile fileName case content of Left e -> do logWarning $ "Unable to open the reason trail for instance " ++ instanceName ++ " expected at " ++ fileName ++ ": " ++ show (e :: IOException) return [] Right trailString -> case J.decode trailString of J.Ok t -> return t J.Error msg -> do logWarning $ "Unable to parse the reason trail: " ++ msg return [] -- | Determine the value of the status field for the report of one instance computeStatusField :: AdminState -> ActualState -> DCStatus computeStatusField AdminDown actualState = if actualState `notElem` [ActualShutdown, ActualDying] then DCStatus DCSCBad "The instance is not stopped as it should be" else DCStatus DCSCOk "" computeStatusField AdminUp ActualHung = DCStatus DCSCUnknown "Instance marked as running, but it appears to be hung" computeStatusField AdminUp actualState = if actualState `notElem` [ActualRunning, ActualBlocked] then DCStatus DCSCBad "The instance is not running as it should be" else DCStatus DCSCOk "" computeStatusField AdminOffline _ = -- FIXME: The "offline" status seems not to be used anywhere in the source -- code, but it is defined, so we have to consider it anyway here. DCStatus DCSCUnknown "The instance is marked as offline" -- Builds the status of an instance using runtime information about the Xen -- Domains, their uptime information and the static information provided by -- the ConfD server. buildStatus :: Map.Map String Domain -> Map.Map Int UptimeInfo -> RealInstanceData -> IO InstStatus buildStatus domains uptimes inst = do let name = realInstName inst currDomain = Map.lookup name domains idNum = fmap domId currDomain currUInfo = idNum >>= (`Map.lookup` uptimes) uptime = fmap uInfoUptime currUInfo adminState = realInstAdminState inst actualState = if adminState == AdminDown && isNothing currDomain then ActualShutdown else case currDomain of (Just dom@(Domain _ _ _ _ (Just isHung))) -> if isHung then ActualHung else domState dom _ -> ActualUnknown status = computeStatusField adminState actualState trail <- getReasonTrail name return $ InstStatus name (UTF8.toString $ realInstUuid inst) adminState actualState uptime (realInstMtime inst) trail status -- | Compute the status code and message, given the current DRBD data -- The final state will have the code corresponding to the worst code of -- all the devices, and the error message given from the concatenation of the -- non-empty error messages. computeGlobalStatus :: [InstStatus] -> DCStatus computeGlobalStatus instStatusList = let dcstatuses = map iStatStatus instStatusList statuses = map (\s -> (dcStatusCode s, dcStatusMessage s)) dcstatuses (code, strList) = foldr mergeStatuses (DCSCOk, [""]) statuses in DCStatus code $ intercalate "\n" strList -- | Build the report of this data collector, containing all the information -- about the status of the instances. buildInstStatusReport :: Maybe String -> Maybe Int -> IO DCReport buildInstStatusReport srvAddr srvPort = do node <- getHostName answer <- runResultT $ getInstances node srvAddr srvPort inst <- exitIfBad "Can't get instance info from ConfD" answer d <- getInferredDomInfo let toReal (RealInstance i) = Just i toReal _ = Nothing reportData <- case d of BT.Ok domains -> do uptimes <- getUptimeInfo let primaryInst = mapMaybe toReal $ fst inst iStatus <- mapM (buildStatus domains uptimes) primaryInst let globalStatus = computeGlobalStatus iStatus return $ ReportData iStatus globalStatus BT.Bad m -> return . ReportData [] . DCStatus DCSCBad $ "Unable to receive the list of instances: " ++ m let jsonReport = J.showJSON reportData buildReport dcName dcVersion dcFormatVersion dcCategory dcKind jsonReport -- | Main function. main :: Options -> [String] -> IO () main opts _ = do report <- buildInstStatusReport (optConfdAddr opts) (optConfdPort opts) putStrLn $ J.encode report ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/InstStatusTypes.hs000064400000000000000000000043321476477700300242110ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Type declarations specific for the instance status data collector. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.InstStatusTypes ( InstStatus(..) , ReportData(..) ) where import Ganeti.DataCollectors.Types import Ganeti.Hypervisor.Xen.Types import Ganeti.THH import Ganeti.THH.Field import Ganeti.Types -- | Data type representing the status of an instance to be returned. $(buildObject "InstStatus" "iStat" [ simpleField "name" [t| String |] , simpleField "uuid" [t| String |] , simpleField "adminState" [t| AdminState |] , simpleField "actualState" [t| ActualState |] , optionalNullSerField $ simpleField "uptime" [t| String |] , timeAsDoubleField "mtime" , simpleField "state_reason" [t| ReasonTrail |] , simpleField "status" [t| DCStatus |] ]) $(buildObject "ReportData" "rData" [ simpleField "instances" [t| [InstStatus] |] , simpleField "status" [t| DCStatus |] ]) ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/Lv.hs000064400000000000000000000155511476477700300214310ustar00rootroot00000000000000{-| Logical Volumes data collector. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.Lv ( main , options , arguments , dcName , dcVersion , dcFormatVersion , dcCategory , dcKind , dcReport ) where import qualified Control.Exception as E import Control.Monad import Data.Attoparsec.Text.Lazy as A import Data.List import Data.Maybe (mapMaybe) import Data.Text.Lazy (pack, unpack) import Network.BSD (getHostName) import System.Process import qualified Text.JSON as J import qualified Ganeti.BasicTypes as BT import Ganeti.Common import qualified Ganeti.Constants as C import Ganeti.Confd.ClientFunctions import Ganeti.DataCollectors.CLI import Ganeti.DataCollectors.Types import Ganeti.JSON (fromJResult) import Ganeti.Objects import Ganeti.Storage.Lvm.LVParser import Ganeti.Storage.Lvm.Types import Ganeti.Utils -- | The default setting for the maximum amount of not parsed character to -- print in case of error. -- It is set to use most of the screen estate on a standard 80x25 terminal. -- TODO: add the possibility to set this with a command line parameter. defaultCharNum :: Int defaultCharNum = 80*20 -- | The name of this data collector. dcName :: String dcName = C.dataCollectorLv -- | The version of this data collector. dcVersion :: DCVersion dcVersion = DCVerBuiltin -- | The version number for the data format of this data collector. dcFormatVersion :: Int dcFormatVersion = 1 -- | The category of this data collector. dcCategory :: Maybe DCCategory dcCategory = Just DCStorage -- | The kind of this data collector. dcKind :: DCKind dcKind = DCKPerf -- | The data exported by the data collector, taken from the default location. dcReport :: IO DCReport dcReport = buildDCReport defaultOptions -- * Command line options options :: IO [OptType] options = return [ oInputFile , oConfdAddr , oConfdPort , oInstances ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [ArgCompletion OptComplFile 0 (Just 0)] -- | Get information about logical volumes from file (if specified) or -- by actually running the command to get it from a live cluster. getLvInfo :: Maybe FilePath -> IO [LVInfo] getLvInfo inputFile = do let cmd = lvCommand params = lvParams fromLvs = ((E.try $ readProcess cmd params "") :: IO (Either IOError String)) >>= exitIfBad "running command" . either (BT.Bad . show) BT.Ok contents <- maybe fromLvs (\fn -> ((E.try $ readFile fn) :: IO (Either IOError String)) >>= exitIfBad "reading from file" . either (BT.Bad . show) BT.Ok) inputFile case A.parse lvParser $ pack contents of A.Fail unparsedText contexts errorMessage -> exitErr $ show (Prelude.take defaultCharNum $ unpack unparsedText) ++ "\n" ++ show contexts ++ "\n" ++ errorMessage A.Done _ lvinfoD -> return lvinfoD -- | Get the list of real instances on the current node along with their disks, -- either from a provided file or by querying Confd. getInstDiskList :: Options -> IO [(RealInstanceData, [Disk])] getInstDiskList opts = do instances <- maybe fromConfd fromFile $ optInstances opts exitIfBad "Unable to obtain the list of instances" instances where fromConfdUnchecked :: IO (BT.Result [(RealInstanceData, [Disk])]) fromConfdUnchecked = do let srvAddr = optConfdAddr opts srvPort = optConfdPort opts toReal (RealInstance i, dsks) = Just (i, dsks) toReal _ = Nothing getHostName >>= \n -> BT.runResultT . liftM (mapMaybe toReal) $ getInstanceDisks n srvAddr srvPort fromConfd :: IO (BT.Result [(RealInstanceData, [Disk])]) fromConfd = liftM (either (BT.Bad . show) id) (E.try fromConfdUnchecked :: IO (Either IOError (BT.Result [(RealInstanceData, [Disk])]))) fromFile :: FilePath -> IO (BT.Result [(RealInstanceData, [Disk])]) fromFile inputFile = do contents <- ((E.try $ readFile inputFile) :: IO (Either IOError String)) >>= exitIfBad "reading from file" . either (BT.Bad . show) BT.Ok return . fromJResult "Not a list of instances" $ J.decode contents -- | Adds the name of the instance to the information about one logical volume. addInstNameToOneLv :: [(RealInstanceData, [Disk])] -> LVInfo -> LVInfo addInstNameToOneLv instDiskList lvInfo = let lv = LogicalVolume (lviVgName lvInfo) (lviName lvInfo) instanceHasDisk = any (includesLogicalId lv) . snd rightInstance = find instanceHasDisk instDiskList in case rightInstance of Nothing -> lvInfo Just (i, _) -> lvInfo { lviInstance = Just $ realInstName i } -- | Adds the name of the instance to the information about logical volumes. addInstNameToLv :: [(RealInstanceData, [Disk])] -> [LVInfo] -> [LVInfo] addInstNameToLv instDisksList = map (addInstNameToOneLv instDisksList) -- | This function computes the JSON representation of the LV status. buildJsonReport :: Options -> IO J.JSValue buildJsonReport opts = do let inputFile = optInputFile opts lvInfo <- getLvInfo inputFile instDiskList <- getInstDiskList opts return . J.showJSON $ addInstNameToLv instDiskList lvInfo -- | This function computes the DCReport for the logical volumes. buildDCReport :: Options -> IO DCReport buildDCReport opts = buildJsonReport opts >>= buildReport dcName dcVersion dcFormatVersion dcCategory dcKind -- | Main function. main :: Options -> [String] -> IO () main opts args = do unless (null args) . exitErr $ "This program takes exactly zero" ++ " arguments, got '" ++ unwords args ++ "'" report <- buildDCReport opts putStrLn $ J.encode report ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/Program.hs000064400000000000000000000052611476477700300224540ustar00rootroot00000000000000{-| Small module holding program definitions for data collectors. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.Program (personalities) where import Ganeti.Common (PersonalityList) import Ganeti.DataCollectors.CLI (Options) import qualified Ganeti.DataCollectors.Diskstats as Diskstats import qualified Ganeti.DataCollectors.Drbd as Drbd import qualified Ganeti.DataCollectors.InstStatus as InstStatus import qualified Ganeti.DataCollectors.Lv as Lv -- | Supported binaries. personalities :: PersonalityList Options personalities = [ (Drbd.dcName, (Drbd.main, Drbd.options, Drbd.arguments, "gathers and displays DRBD statistics in JSON\ \ format")) , (InstStatus.dcName, (InstStatus.main, InstStatus.options, InstStatus.arguments, "gathers and displays the status of the\ \ instances in JSON format")) , (Diskstats.dcName, (Diskstats.main, Diskstats.options, Diskstats.arguments, "gathers and displays the disk usage\ \ statistics in JSON format")) , (Lv.dcName, (Lv.main, Lv.options, Lv.arguments, "gathers and\ \ displays info about logical volumes")) ] ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/Types.hs000064400000000000000000000213671476477700300221560ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, CPP #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Implementation of the Ganeti data collector types. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.Types ( addStatus , DCCategory(..) , DCKind(..) , DCReport(..) , DCStatus(..) , DCStatusCode(..) , DCVersion(..) , CollectorData(..) , CollectorMap , buildReport , mergeStatuses , getCategoryName , ReportBuilder(..) , DataCollector(..) ) where import Control.DeepSeq (NFData, rnf) #if !MIN_VERSION_containers(0,5,0) import Control.Seq (using, seqFoldable, rdeepseq) #endif import Data.Char import Data.Ratio import qualified Data.Map as Map import qualified Data.Sequence as Seq import System.Time (ClockTime(..)) import Text.JSON import Ganeti.Constants as C import Ganeti.Objects (ConfigData) import Ganeti.THH import Ganeti.Utils.Time (getCurrentTimeUSec) -- | The possible classes a data collector can belong to. data DCCategory = DCInstance | DCStorage | DCDaemon | DCHypervisor deriving (Show, Eq, Read, Enum, Bounded) -- | Get the category name and return it as a string. getCategoryName :: DCCategory -> String getCategoryName dcc = map toLower . drop 2 . show $ dcc categoryNames :: Map.Map String DCCategory categoryNames = let l = [minBound ..] in Map.fromList $ zip (map getCategoryName l) l -- | The JSON instance for DCCategory. instance JSON DCCategory where showJSON = showJSON . getCategoryName readJSON (JSString s) = let s' = fromJSString s in case Map.lookup s' categoryNames of Just category -> Ok category Nothing -> fail $ "Invalid category name " ++ s' ++ " for type" ++ " DCCategory" readJSON v = fail $ "Invalid JSON value " ++ show v ++ " for type DCCategory" -- | The possible status codes of a data collector. data DCStatusCode = DCSCOk -- ^ Everything is OK | DCSCTempBad -- ^ Bad, but being automatically fixed | DCSCUnknown -- ^ Unable to determine the status | DCSCBad -- ^ Bad. External intervention required deriving (Show, Eq, Ord) -- | The JSON instance for CollectorStatus. instance JSON DCStatusCode where showJSON DCSCOk = showJSON (0 :: Int) showJSON DCSCTempBad = showJSON (1 :: Int) showJSON DCSCUnknown = showJSON (2 :: Int) showJSON DCSCBad = showJSON (4 :: Int) readJSON = error "JSON read instance not implemented for type DCStatusCode" -- | The status of a \"status reporting data collector\". $(buildObject "DCStatus" "dcStatus" [ simpleField "code" [t| DCStatusCode |] , simpleField "message" [t| String |] ]) -- | The type representing the kind of the collector. data DCKind = DCKPerf -- ^ Performance reporting collector | DCKStatus -- ^ Status reporting collector deriving (Show, Eq) -- | The JSON instance for CollectorKind. instance JSON DCKind where showJSON DCKPerf = showJSON (0 :: Int) showJSON DCKStatus = showJSON (1 :: Int) readJSON (JSRational _ x) = if denominator x /= 1 then fail $ "Invalid JSON value " ++ show x ++ " for type DCKind" else let x' = (fromIntegral . numerator $ x) :: Int in if x' == 0 then Ok DCKPerf else if x' == 1 then Ok DCKStatus else fail $ "Invalid JSON value " ++ show x' ++ " for type DCKind" readJSON v = fail $ "Invalid JSON value " ++ show v ++ " for type DCKind" -- | Type representing the version number of a data collector. data DCVersion = DCVerBuiltin | DCVersion String deriving (Show, Eq) -- | The JSON instance for DCVersion. instance JSON DCVersion where showJSON DCVerBuiltin = showJSON C.builtinDataCollectorVersion showJSON (DCVersion v) = showJSON v readJSON (JSString s) = if fromJSString s == C.builtinDataCollectorVersion then Ok DCVerBuiltin else Ok . DCVersion $ fromJSString s readJSON v = fail $ "Invalid JSON value " ++ show v ++ " for type DCVersion" -- | Type for the value field of the `CollectorMap` below. data CollectorData = CPULoadData (Seq.Seq (ClockTime, [Int])) | InstanceCpuLoad (Map.Map String (Seq.Seq (ClockTime, Double))) instance NFData ClockTime where rnf (TOD x y) = rnf x `seq` rnf y #if MIN_VERSION_containers(0,5,0) instance NFData CollectorData where rnf (CPULoadData x) = rnf x rnf (InstanceCpuLoad x) = rnf x #else {- In older versions of the containers library, Seq is not an instance of NFData, so use a generic way to reduce to normal form -} instance NFData CollectorData where rnf (CPULoadData x) = (x `using` seqFoldable rdeepseq) `seq` () rnf (InstanceCpuLoad x) = (x `using` seqFoldable (seqFoldable rdeepseq)) `seq` () #endif -- | Type for the map storing the data of the statefull DataCollectors. type CollectorMap = Map.Map String CollectorData -- | This is the format of the report produced by each data collector. $(buildObject "DCReport" "dcReport" [ simpleField "name" [t| String |] , simpleField "version" [t| DCVersion |] , simpleField "format_version" [t| Int |] , simpleField "timestamp" [t| Integer |] , optionalNullSerField $ simpleField "category" [t| DCCategory |] , simpleField "kind" [t| DCKind |] , simpleField "data" [t| JSValue |] ]) -- | Add the data collector status information to the JSON representation of -- the collector data. addStatus :: DCStatus -> JSValue -> JSValue addStatus dcStatus (JSObject obj) = makeObj $ ("status", showJSON dcStatus) : fromJSObject obj addStatus dcStatus value = makeObj [ ("status", showJSON dcStatus) , ("data", value) ] -- | Helper function for merging statuses. mergeStatuses :: (DCStatusCode, String) -> (DCStatusCode, [String]) -> (DCStatusCode, [String]) mergeStatuses (newStat, newStr) (storedStat, storedStrs) = let resStat = max newStat storedStat resStrs = if newStr == "" then storedStrs else storedStrs ++ [newStr] in (resStat, resStrs) -- | Utility function for building a report automatically adding the current -- timestamp (rounded up to seconds). -- If the version is not specified, it will be set to the value indicating -- a builtin collector. buildReport :: String -> DCVersion -> Int -> Maybe DCCategory -> DCKind -> JSValue -> IO DCReport buildReport name version format_version category kind jsonData = do usecs <- getCurrentTimeUSec let timestamp = usecs * 1000 :: Integer return $ DCReport name version format_version timestamp category kind jsonData -- | A report of a data collector might be stateful or stateless. data ReportBuilder = StatelessR (IO DCReport) | StatefulR (Maybe CollectorData -> IO DCReport) type Name = String -- | Type describing a data collector basic information data DataCollector = DataCollector { dName :: Name -- ^ Name of the data collector , dCategory :: Maybe DCCategory -- ^ Category (storage, instance, ecc) -- of the collector , dKind :: DCKind -- ^ Kind (performance or status reporting) of -- the data collector , dReport :: ReportBuilder -- ^ Report produced by the collector , dUpdate :: Maybe (Maybe CollectorData -> IO CollectorData) -- ^ Update operation for stateful collectors. , dActive :: Name -> ConfigData -> Bool -- ^ Checks if the collector applies for the cluster. , dInterval :: Name -> ConfigData -> Integer -- ^ Interval between collection in microseconds } ganeti-3.1.0~rc2/src/Ganeti/DataCollectors/XenCpuLoad.hs000064400000000000000000000147261476477700300230550ustar00rootroot00000000000000{-| xentop CPU data collector -} {- Copyright (C) 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.DataCollectors.XenCpuLoad ( dcName , dcVersion , dcFormatVersion , dcCategory , dcKind , dcReport , dcUpdate ) where import Control.Applicative (liftA2) import Control.Arrow ((***)) import Control.Monad (liftM, when) import Control.Monad.IO.Class (liftIO) import qualified Data.Map as Map import Data.Maybe (mapMaybe) import qualified Data.Sequence as Seq import System.Process (readProcess) import qualified Text.JSON as J import System.Time (ClockTime, getClockTime) import Ganeti.Utils.Time (addToClockTime, diffClockTimes, clockTimeToUSec) import Ganeti.BasicTypes (GenericResult(..), Result, genericResult, runResultT) import qualified Ganeti.Constants as C import Ganeti.DataCollectors.Types import Ganeti.Utils (readMaybe) -- | The name of this data collector. dcName :: String dcName = C.dataCollectorXenCpuLoad -- | The version of this data collector. dcVersion :: DCVersion dcVersion = DCVerBuiltin -- | The version number for the data format of this data collector. dcFormatVersion :: Int dcFormatVersion = 1 -- | The category of this data collector. dcCategory :: Maybe DCCategory dcCategory = Nothing -- | The kind of this data collector. dcKind :: DCKind dcKind = DCKPerf -- | Read xentop output, if this program is available. readXentop :: IO (Result String) readXentop = runResultT . liftIO $ readProcess C.xentopCommand ["-f", "-b", "-i", "1"] "" -- | Parse output of xentop command. parseXentop :: String -> Result (Map.Map String Double) parseXentop s = do let values = map words $ lines s case values of [] -> Bad "No output received" (name_header:_:cpu_header:_):vals -> do when (name_header /= "NAME" || cpu_header /= "CPU(sec)") $ Bad "Unexpected data format" return . Map.fromList $ mapMaybe (\ dom -> case dom of name:_:cpu:_ -> if name /= "Domain-0" then liftM ((,) name) $ readMaybe cpu else Nothing _ -> Nothing ) vals _ -> Bad "Insufficient number of output columns" -- | Add a new value to a sequence of observations, taking into account -- counter rollovers. In case of a rollover, we drop the joining interval -- so that we do not have to make assumptions about the value at which is -- rolled over, but we do keep the right sequence, appropriately moved. combineWithRollover :: Seq.Seq (ClockTime, Double) -> Seq.Seq (ClockTime, Double) -> Seq.Seq (ClockTime, Double) combineWithRollover new old | Seq.null new || Seq.null old = new Seq.>< old combineWithRollover new old = let (t2, x2) = Seq.index new $ Seq.length new - 1 (t1, x1) = Seq.index old 0 in if x2 >= x1 then new Seq.>< old else let delta_t = diffClockTimes t2 t1 deltax = x2 - x1 old' = (addToClockTime delta_t *** (+ deltax)) <$> Seq.drop 1 old in new Seq.>< old' -- | Updates the given Collector data. dcUpdate :: Maybe CollectorData -> IO CollectorData dcUpdate maybeCollector = do let oldData = case maybeCollector of Just (InstanceCpuLoad x) -> x _ -> Map.empty now <- getClockTime newResult <- liftM (>>= parseXentop) readXentop let newValues = Map.map (Seq.singleton . (,) now) $ genericResult (const Map.empty) id newResult sampleSizeUSec = fromIntegral C.cpuavgloadWindowSize * 1000000 combinedValues = Map.unionWith combineWithRollover newValues oldData withinRange = Map.map (Seq.dropWhileR ((<) sampleSizeUSec . (clockTimeToUSec now -) . clockTimeToUSec . fst)) combinedValues withoutOld = Map.filter (liftA2 (&&) (not . Seq.null) $ (>) (fromIntegral $ C.xentopAverageThreshold * 1000000) . (clockTimeToUSec now -) . clockTimeToUSec . fst . flip Seq.index 0) withinRange return $ InstanceCpuLoad withoutOld -- | From a list of timestamps and cumulative CPU data, compute the -- average CPU activity in vCPUs. loadAverage :: Seq.Seq (ClockTime, Double) -> Maybe Double loadAverage observations = do when (Seq.null observations) Nothing let (t2, cpu2) = Seq.index observations 0 (t1, cpu1) = Seq.index observations $ Seq.length observations - 1 tUsec2 = clockTimeToUSec t2 tUsec1 = clockTimeToUSec t1 when (tUsec2 - tUsec1 < (fromIntegral C.xentopAverageThreshold * 1000000)) Nothing return $ 1000000 * (cpu2 - cpu1) / fromIntegral (tUsec2 - tUsec1) -- | The data exported by the data collector, taken from the default location. dcReport :: Maybe CollectorData -> IO DCReport dcReport maybeCollector = let collectedData = case maybeCollector of Just (InstanceCpuLoad x) -> x _ -> Map.empty loads = Map.mapMaybe loadAverage collectedData in buildReport dcName dcVersion dcFormatVersion dcCategory dcKind . J.JSObject . J.toJSObject . Map.toAscList $ Map.map J.showJSON loads ganeti-3.1.0~rc2/src/Ganeti/Errors.hs000064400000000000000000000165301476477700300174170ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Implementation of the Ganeti error types. This module implements our error hierarchy. Currently we implement one identical to the Python one; later we might one to have separate ones for frontend (clients), master and backend code. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Errors ( ErrorCode(..) , GanetiException(..) , ErrorResult , errToResult , errorExitCode , excName , formatError , ResultG , maybeToError ) where import Text.JSON hiding (Result, Ok) import System.Exit import Ganeti.THH import Ganeti.BasicTypes import Ganeti.Compat() import qualified Ganeti.Constants as C -- | Error code types for 'OpPrereqError'. $(declareSADT "ErrorCode" [ ("ECodeResolver", 'C.errorsEcodeResolver) , ("ECodeNoRes", 'C.errorsEcodeNores) , ("ECodeTempNoRes", 'C.errorsEcodeTempNores) , ("ECodeInval", 'C.errorsEcodeInval) , ("ECodeState", 'C.errorsEcodeState) , ("ECodeNoEnt", 'C.errorsEcodeNoent) , ("ECodeExists", 'C.errorsEcodeExists) , ("ECodeNotUnique", 'C.errorsEcodeNotunique) , ("ECodeFault", 'C.errorsEcodeFault) , ("ECodeEnviron", 'C.errorsEcodeEnviron) ]) $(makeJSONInstance ''ErrorCode) $(genException "GanetiException" [ ("GenericError", [excErrMsg]) , ("LockError", [excErrMsg]) , ("PidFileLockError", [excErrMsg]) , ("HypervisorError", [excErrMsg]) , ("ProgrammerError", [excErrMsg]) , ("BlockDeviceError", [excErrMsg]) , ("ConfigurationError", [excErrMsg]) , ("ConfigVerifyError", [excErrMsg, ("allErrors", [t| [String] |])]) , ("ConfigVersionMismatch", [ ("expVer", [t| Int |]) , ("actVer", [t| Int |])]) , ("ReservationError", [excErrMsg]) , ("RemoteError", [excErrMsg]) , ("SignatureError", [excErrMsg]) , ("ParameterError", [excErrMsg]) , ("ResultValidationError", [excErrMsg]) , ("OpPrereqError", [excErrMsg, ("errCode", [t| ErrorCode |])]) , ("OpExecError", [excErrMsg]) , ("OpResultError", [excErrMsg]) , ("OpCodeUnknown", [excErrMsg]) , ("JobLost", [excErrMsg]) , ("JobFileCorrupted", [excErrMsg]) , ("ResolverError", [ ("errHostname", [t| String |]) , ("errResolverCode", [t| Int |]) , ("errResolverMsg", [t| String |])]) , ("HooksFailure", [excErrMsg]) , ("HooksAbort", [("errs", [t| [(String, String, String)] |])]) , ("UnitParseError", [excErrMsg]) , ("ParseError", [excErrMsg]) , ("TypeEnforcementError", [excErrMsg]) , ("X509CertError", [ ("certFileName", [t| String |]) , excErrMsg ]) , ("TagError", [excErrMsg]) , ("CommandError", [excErrMsg]) , ("StorageError", [excErrMsg]) , ("InotifyError", [excErrMsg]) , ("JobQueueError", [excErrMsg]) , ("JobQueueDrainError", [excErrMsg]) , ("JobQueueFull", []) , ("ConfdMagicError", [excErrMsg]) , ("ConfdClientError", [excErrMsg]) , ("UdpDataSizeError", [excErrMsg]) , ("NoCtypesError", [excErrMsg]) , ("IPAddressError", [excErrMsg]) , ("LuxiError", [excErrMsg]) , ("QueryFilterParseError", [excErrMsg]) -- not consistent with Python , ("RapiTestResult", [excErrMsg]) , ("FileStoragePathError", [excErrMsg]) ]) instance Error GanetiException where strMsg = GenericError instance JSON GanetiException where showJSON = saveGanetiException readJSON = loadGanetiException -- | Error monad using 'GanetiException' type alias. type ErrorResult = GenericResult GanetiException $(genStrOfOp ''GanetiException "excName") -- | Returns the exit code of a program that should be used if we got -- back an exception from masterd. errorExitCode :: GanetiException -> ExitCode errorExitCode (ConfigurationError {}) = ExitFailure 2 errorExitCode (ConfigVerifyError {}) = ExitFailure 2 errorExitCode _ = ExitFailure 1 -- | Formats an exception. formatError :: GanetiException -> String formatError (ConfigurationError msg) = "Corrupt configuration file: " ++ msg ++ "\nAborting." formatError (ConfigVerifyError msg es) = "Corrupt configuration file: " ++ msg ++ "\nAborting. Details:\n" ++ unlines es formatError (HooksAbort errs) = unlines $ "Failure: hooks execution failed:": map (\(node, script, out) -> " node: " ++ node ++ ", script: " ++ script ++ if null out then " (no output)" else ", output: " ++ out ) errs formatError (HooksFailure msg) = "Failure: hooks general failure: " ++ msg formatError (ResolverError host _ _) = -- FIXME: in Python, this uses the system hostname to format the -- error differently if we are failing to resolve our own hostname "Failure: can't resolve hostname " ++ host formatError (OpPrereqError msg code) = "Failure: prerequisites not met for this" ++ " operation:\nerror type: " ++ show code ++ ", error details:\n" ++ msg formatError (OpExecError msg) = "Failure: command execution error:\n" ++ msg formatError (TagError msg) = "Failure: invalid tag(s) given:\n" ++ msg formatError (JobQueueDrainError _)= "Failure: the job queue is marked for drain and doesn't accept new requests" formatError JobQueueFull = "Failure: the job queue is full and doesn't accept new" ++ " job submissions until old jobs are archived" formatError (TypeEnforcementError msg) = "Parameter Error: " ++ msg formatError (ParameterError msg) = "Failure: unknown/wrong parameter name '" ++ msg ++ "'" formatError (JobLost msg) = "Error checking job status: " ++ msg formatError (QueryFilterParseError msg) = -- FIXME: in Python, this has a more complex error message "Error while parsing query filter: " ++ msg formatError (GenericError msg) = "Unhandled Ganeti error: " ++ msg formatError err = "Unhandled exception: " ++ show err -- | A type for IO actions with errors properly handled as -- 'GanetiException's. -- TODO: Move to Errors.hs type ResultG = ResultT GanetiException IO -- | Convert from an 'ErrorResult' to a standard 'Result'. errToResult :: ErrorResult a -> Result a errToResult (Ok a) = Ok a errToResult (Bad e) = Bad $ formatError e -- | Convert from a 'Maybe' to a an 'ErrorResult'. maybeToError :: String -> Maybe a -> ErrorResult a maybeToError _ (Just a) = Ok a maybeToError m Nothing = Bad $ GenericError m ganeti-3.1.0~rc2/src/Ganeti/HTools/000075500000000000000000000000001476477700300170125ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/HTools/AlgorithmParams.hs000064400000000000000000000065321476477700300224460ustar00rootroot00000000000000{-| Algorithm Options for HTools This module describes the parameters that influence the balancing algorithm in htools. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.AlgorithmParams ( AlgorithmOptions(..) , defaultOptions , fromCLIOptions ) where import qualified Ganeti.HTools.CLI as CLI import qualified Ganeti.HTools.Types as T data AlgorithmOptions = AlgorithmOptions { algDiskMoves :: Bool -- ^ Whether disk moves are allowed , algInstanceMoves :: Bool -- ^ Whether instance moves are allowed , algRestrictedMigration :: Bool -- ^ Whether migration is restricted , algIgnoreSoftErrors :: Bool -- ^ Whether to always ignore soft errors , algEvacMode :: Bool -- ^ Consider only eavacation moves , algMinGain :: Double -- ^ Minimal gain per balancing step , algMinGainLimit :: Double -- ^ Limit below which minimal gain is used , algCapacity :: Bool -- ^ Whether to check capacity properties, -- like global N+1 redundancy , algCapacityIgnoreGroups :: [T.Gdx] -- ^ Groups to ignore in capacity checks , algRestrictToNodes :: Maybe [String] -- ^ nodes to restrict allocation to , algAcceptExisting :: Bool -- ^ accept existing violations in capacity -- checks } -- | Obtain the relevant algorithmic option from the commandline options fromCLIOptions :: CLI.Options -> AlgorithmOptions fromCLIOptions opts = AlgorithmOptions { algDiskMoves = CLI.optDiskMoves opts , algInstanceMoves = CLI.optInstMoves opts , algRestrictedMigration = CLI.optRestrictedMigrate opts , algIgnoreSoftErrors = CLI.optIgnoreSoftErrors opts , algEvacMode = CLI.optEvacMode opts , algMinGain = CLI.optMinGain opts , algMinGainLimit = CLI.optMinGainLim opts , algCapacity = CLI.optCapacity opts , algCapacityIgnoreGroups = [] , algRestrictToNodes = CLI.optRestrictToNodes opts , algAcceptExisting = CLI.optAcceptExisting opts } -- | Default options for the balancing algorithm defaultOptions :: AlgorithmOptions defaultOptions = fromCLIOptions CLI.defaultOptions ganeti-3.1.0~rc2/src/Ganeti/HTools/Backend/000075500000000000000000000000001476477700300203415ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/HTools/Backend/IAlloc.hs000064400000000000000000000564271476477700300220560ustar00rootroot00000000000000{-| Implementation of the iallocator interface. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Backend.IAlloc ( readRequest , runIAllocator , processRelocate , loadData , formatAllocate , formatIAllocResult , formatMultiAlloc ) where import Data.Either () import Data.Maybe (fromMaybe, isJust, fromJust) import Data.List import Control.Monad import System.Time import Text.JSON (JSObject, JSValue(JSArray), makeObj, encodeStrict, decodeStrict, fromJSObject, showJSON) import Ganeti.BasicTypes import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.AllocationSolution as AllocSol import qualified Ganeti.HTools.Cluster.AllocateSecondary as AllocSecondary import qualified Ganeti.HTools.Cluster.Evacuate as Evacuate import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Nic as Nic import qualified Ganeti.Constants as C import Ganeti.HTools.AlgorithmParams (AlgorithmOptions(algRestrictToNodes)) import Ganeti.HTools.CLI import Ganeti.HTools.Loader import Ganeti.HTools.Types import Ganeti.JSON (maybeFromObj, JSRecord, tryFromObj, toArray, asObjectList, readEitherString, fromJResult, fromObj, fromObjWithDefault, asJSObject) import Ganeti.Types ( EvacMode(ChangePrimary, ChangeSecondary) , adminStateFromRaw, AdminState(..)) import Ganeti.Utils {-# ANN module "HLint: ignore Eta reduce" #-} -- | Type alias for the result of an IAllocator call. type IAllocResult = (String, JSValue, Node.List, Instance.List) -- | Parse a NIC within an instance (in a creation request) parseNic :: String -> JSRecord -> Result Nic.Nic parseNic n a = do mac <- maybeFromObj a "mac" ip <- maybeFromObj a "ip" mode <- maybeFromObj a "mode" >>= \m -> case m of Just "bridged" -> Ok $ Just Nic.Bridged Just "routed" -> Ok $ Just Nic.Routed Just "openvswitch" -> Ok $ Just Nic.OpenVSwitch Nothing -> Ok Nothing _ -> Bad $ "invalid NIC mode in instance " ++ n link <- maybeFromObj a "link" bridge <- maybeFromObj a "bridge" network <- maybeFromObj a "network" return (Nic.create mac ip mode link bridge network) -- | Parse the basic specifications of an instance. -- -- Instances in the cluster instance list and the instance in an -- 'Allocate' request share some common properties, which are read by -- this function. parseBaseInstance :: String -> JSRecord -> Result (String, Instance.Instance) parseBaseInstance n a = do let errorMessage = "invalid data for instance '" ++ n ++ "'" let extract x = tryFromObj errorMessage a x disk <- extract "disk_space_total" jsdisks <- extract "disks" >>= toArray >>= asObjectList dsizes <- mapM (flip (tryFromObj errorMessage) "size" . fromJSObject) jsdisks dspindles <- mapM (annotateResult errorMessage . flip maybeFromObj "spindles" . fromJSObject) jsdisks let disks = zipWith Instance.Disk dsizes dspindles mem <- extract "memory" vcpus <- extract "vcpus" tags <- extract "tags" dt <- extract "disk_template" su <- extract "spindle_use" nics <- extract "nics" >>= toArray >>= asObjectList >>= mapM (parseNic n . fromJSObject) state <- (tryFromObj errorMessage a "admin_state" >>= adminStateFromRaw) `mplus` Ok AdminUp let getRunSt AdminOffline = StatusOffline getRunSt AdminDown = StatusDown getRunSt AdminUp = Running -- Not forthcoming by default. forthcoming <- extract "forthcoming" `orElse` Ok False return (n, Instance.create n mem disk disks vcpus (getRunSt state) tags True 0 0 dt su nics forthcoming) -- | Parses an instance as found in the cluster instance list. parseInstance :: NameAssoc -- ^ The node name-to-index association list -> String -- ^ The name of the instance -> JSRecord -- ^ The JSON object -> Result (String, Instance.Instance) parseInstance ktn n a = do base <- parseBaseInstance n a nodes <- fromObj a "nodes" (pnode, snodes) <- case nodes of [] -> Bad $ "empty node list for instance " ++ n x:xs -> readEitherString x >>= \x' -> return (x', xs) pidx <- lookupNode ktn n pnode sidx <- case snodes of [] -> return Node.noSecondary x:_ -> readEitherString x >>= lookupNode ktn n return (n, Instance.setBoth (snd base) pidx sidx) -- | Parses a node as found in the cluster node list. parseNode :: NameAssoc -- ^ The group association -> String -- ^ The node's name -> JSRecord -- ^ The JSON object -> Result (String, Node.Node) parseNode ktg n a = do let desc = "invalid data for node '" ++ n ++ "'" extract x = tryFromObj desc a x extractDef def key = fromObjWithDefault a key def offline <- extract "offline" drained <- extract "drained" guuid <- extract "group" vm_capable <- annotateResult desc $ maybeFromObj a "vm_capable" let vm_capable' = fromMaybe True vm_capable gidx <- lookupGroup ktg n guuid ndparams <- extract "ndparams" >>= asJSObject -- Despite the fact that tags field is reported by iallocator.py, -- some tests don't contain tags field tags <- extractDef [] "tags" excl_stor <- tryFromObj desc (fromJSObject ndparams) "exclusive_storage" let live = not offline && vm_capable' lvextract def = eitherLive live def . extract sptotal <- if excl_stor then lvextract 0 "total_spindles" else tryFromObj desc (fromJSObject ndparams) "spindle_count" spfree <- lvextract 0 "free_spindles" mtotal <- lvextract 0.0 "total_memory" mnode <- lvextract 0 "reserved_memory" mfree <- lvextract 0 "free_memory" dtotal <- lvextract 0.0 "total_disk" dfree <- lvextract 0 "free_disk" ctotal <- lvextract 0.0 "total_cpus" cnos <- lvextract 0 "reserved_cpus" let node = flip Node.setNodeTags tags $ Node.create n mtotal mnode mfree dtotal dfree ctotal cnos (not live || drained) sptotal spfree gidx excl_stor return (n, node) -- | Parses a group as found in the cluster group list. parseGroup :: String -- ^ The group UUID -> JSRecord -- ^ The JSON object -> Result (String, Group.Group) parseGroup u a = do let extract x = tryFromObj ("invalid data for group '" ++ u ++ "'") a x name <- extract "name" apol <- extract "alloc_policy" nets <- extract "networks" ipol <- extract "ipolicy" tags <- extract "tags" return (u, Group.create name u apol nets ipol tags) -- | Top-level parser. -- -- The result is a tuple of eventual warning messages and the parsed -- request; if parsing the input data fails, we'll return a 'Bad' -- value. parseData :: ClockTime -- ^ The current time -> String -- ^ The JSON message as received from Ganeti -> Int -- ^ Static node memory size, see optStaticKvmNodeMemory -> Result ([String], Request) -- ^ Result tuple parseData now body static_n_mem = do decoded <- fromJResult "Parsing input IAllocator message" (decodeStrict body) let obj = fromJSObject decoded extrObj x = tryFromObj "invalid iallocator message" obj x -- request parser request <- liftM fromJSObject (extrObj "request") let extrFromReq r x = tryFromObj "invalid request dict" r x let extrReq x = extrFromReq request x -- existing group parsing glist <- liftM fromJSObject (extrObj "nodegroups") gobj <- mapM (\(x, y) -> asJSObject y >>= parseGroup x . fromJSObject) glist let (ktg, gl) = assignIndices gobj -- existing node parsing nlist <- liftM fromJSObject (extrObj "nodes") nobj <- mapM (\(x,y) -> asJSObject y >>= parseNode ktg x . fromJSObject) nlist let (ktn, nl) = assignIndices nobj -- existing instance parsing ilist <- extrObj "instances" let idata = fromJSObject ilist iobj <- mapM (\(x,y) -> asJSObject y >>= parseInstance ktn x . fromJSObject) idata let (kti, il) = assignIndices iobj -- cluster tags ctags <- extrObj "cluster_tags" let ex_tags = extractExTags ctags dsrd_loc_tags = extractDesiredLocations ctags updateTags = updateExclTags ex_tags . updateDesiredLocationTags dsrd_loc_tags -- hypervisor enabled_hypervisors <- extrObj "enabled_hypervisors" -- This is ugly, unfortunately the IAllocator "API" (which is a CLI not an -- API to be precise) is different from other htools backends that receive -- the 'default_hypervisor' field. Normally there is no reason to have -- more than one hypervisor so in practice this should be fine. Ganeti 2.17 -- exports the hv_stat per node field with the missing information. -- TODO: Remove, once the caller can get the correct nodes size on KVM. let nl2 = case enabled_hypervisors of hv : _ -> Container.map (`Node.setHypervisor` hv) nl _ -> nl cdata1 <- mergeData [] [] [] [] now (ClusterData gl nl2 il ctags defIPolicy) let (msgs, fix_nl) = updateMissing (cdNodes cdata1) (cdInstances cdata1) static_n_mem cdata = cdata1 { cdNodes = fix_nl } map_n = cdNodes cdata map_i = cdInstances cdata map_g = cdGroups cdata optype <- extrReq "type" rqtype <- case () of _ | optype == C.iallocatorModeAlloc -> do rname <- extrReq "name" rgn <- maybeFromObj request "group_name" rest_nodes <- maybeFromObj request "restrict-to-nodes" req_nodes <- extrReq "required_nodes" inew <- parseBaseInstance rname request let io = updateTags $ snd inew return $ Allocate io (Cluster.AllocDetails req_nodes rgn) rest_nodes | optype == C.iallocatorModeReloc -> do rname <- extrReq "name" ridx <- lookupInstance kti rname req_nodes <- extrReq "required_nodes" ex_nodes <- extrReq "relocate_from" ex_idex <- mapM (Container.findByName map_n) ex_nodes return $ Relocate ridx req_nodes (map Node.idx ex_idex) | optype == C.iallocatorModeChgGroup -> do rl_names <- extrReq "instances" rl_insts <- mapM (liftM Instance.idx . Container.findByName map_i) rl_names gr_uuids <- extrReq "target_groups" gr_idxes <- mapM (liftM Group.idx . Container.findByName map_g) gr_uuids return $ ChangeGroup rl_insts gr_idxes | optype == C.iallocatorModeNodeEvac -> do rl_names <- extrReq "instances" rl_insts <- mapM (Container.findByName map_i) rl_names let rl_idx = map Instance.idx rl_insts rl_mode <- extrReq "evac_mode" return $ NodeEvacuate rl_idx rl_mode | optype == C.iallocatorModeMultiAlloc -> do arry <- extrReq "instances" :: Result [JSObject JSValue] let inst_reqs = map fromJSObject arry prqs <- forM inst_reqs (\r -> do rname <- extrFromReq r "name" rgn <- maybeFromObj request "group_name" req_nodes <- extrFromReq r "required_nodes" inew <- parseBaseInstance rname r let io = updateTags $ snd inew return (io, Cluster.AllocDetails req_nodes rgn)) return $ MultiAllocate prqs | optype == C.iallocatorModeAllocateSecondary -> do rname <- extrReq "name" ridx <- lookupInstance kti rname return $ AllocateSecondary ridx | otherwise -> fail ("Invalid request type '" ++ optype ++ "'") return (msgs, Request rqtype cdata) -- | Formats the result into a valid IAllocator response message. formatResponse :: Bool -- ^ Whether the request was successful -> String -- ^ Information text -> JSValue -- ^ The JSON encoded result -> String -- ^ The full JSON-formatted message formatResponse success info result = let e_success = ("success", showJSON success) e_info = ("info", showJSON info) e_result = ("result", result) in encodeStrict $ makeObj [e_success, e_info, e_result] -- | Flatten the log of a solution into a string. describeSolution :: AllocSol.GenericAllocSolution a -> String describeSolution = intercalate ", " . AllocSol.asLog -- | Convert allocation/relocation results into the result format. formatAllocate :: Instance.List -> AllocSol.GenericAllocSolution a -> Result IAllocResult formatAllocate il as = do let info = describeSolution as case AllocSol.asSolution as of Nothing -> fail info Just (nl, inst, nodes, _) -> do let il' = Container.add (Instance.idx inst) inst il return (info, showJSON $ map Node.name nodes, nl, il') -- | Convert allocation/relocation results into the result format. formatAllocateSecondary :: Instance.List -> AllocSol.GenericAllocSolution a -> Result IAllocResult formatAllocateSecondary il as = do let info = describeSolution as case AllocSol.asSolution as of Nothing -> fail info Just (nl, inst, [_, snode], _) -> do let il' = Container.add (Instance.idx inst) inst il return (info, showJSON $ Node.name snode, nl, il') _ -> fail $ "Internal error (not a DRBD allocation); info was: " ++ info -- | Convert multi allocation results into the result format. formatMultiAlloc :: ( Node.List, Instance.List , Cluster.GenericAllocSolutionList a) -> Result IAllocResult formatMultiAlloc (fin_nl, fin_il, ars) = let rars = reverse ars (allocated, failed) = partition (isJust . AllocSol.asSolution . snd) rars aars = map (\(_, ar) -> let (_, inst, nodes, _) = fromJust $ AllocSol.asSolution ar iname = Instance.name inst nnames = map Node.name nodes in (iname, nnames)) allocated fars = map (\(inst, ar) -> let iname = Instance.name inst in (iname, describeSolution ar)) failed info = show (length failed) ++ " instances failed to allocate and " ++ show (length allocated) ++ " were allocated successfully" in return (info, showJSON (aars, fars), fin_nl, fin_il) -- | Convert a node-evacuation/change group result. formatNodeEvac :: Group.List -> Node.List -> Instance.List -> (Node.List, Instance.List, Evacuate.EvacSolution) -> Result IAllocResult formatNodeEvac gl nl il (fin_nl, fin_il, es) = let iname = Instance.name . flip Container.find il nname = Node.name . flip Container.find nl gname = Group.name . flip Container.find gl fes = map (\(idx, msg) -> (iname idx, msg)) $ Evacuate.esFailed es mes = map (\(idx, gdx, ndxs) -> (iname idx, gname gdx, map nname ndxs)) $ Evacuate.esMoved es failed = length fes moved = length mes info = show failed ++ " instances failed to move and " ++ show moved ++ " were moved successfully" in Ok (info, showJSON (mes, fes, Evacuate.esOpCodes es), fin_nl, fin_il) -- | Runs relocate for a single instance. -- -- This is wrapper over the 'Cluster.tryNodeEvac' function that is run -- with a single instance (ours), and further it checks that the -- result it got (in the nodes field) is actually consistent, as -- tryNodeEvac is designed to output primarily an opcode list, not a -- node list. processRelocate :: AlgorithmOptions -> Group.List -- ^ The group list -> Node.List -- ^ The node list -> Instance.List -- ^ The instance list -> Idx -- ^ The index of the instance to move -> Int -- ^ The number of nodes required -> [Ndx] -- ^ Nodes which should not be used -> Result (Node.List, Instance.List, [Ndx]) -- ^ Solution list processRelocate opts gl nl il idx 1 exndx = do let orig = Container.find idx il sorig = Instance.sNode orig porig = Instance.pNode orig mir_type = Instance.mirrorType orig (exp_node, node_type, reloc_type) <- case mir_type of MirrorNone -> fail "Can't relocate non-mirrored instances" MirrorInternal -> return (sorig, "secondary", ChangeSecondary) MirrorExternal -> return (porig, "primary", ChangePrimary) when (exndx /= [exp_node]) . -- FIXME: we can't use the excluded nodes here; the logic is -- already _but only partially_ implemented in tryNodeEvac... fail $ "Unsupported request: excluded nodes not equal to\ \ instance's " ++ node_type ++ "(" ++ show exp_node ++ " versus " ++ show exndx ++ ")" (nl', il', esol) <- Evacuate.tryNodeEvac opts gl nl il reloc_type [idx] nodes <- case lookup idx (Evacuate.esFailed esol) of Just msg -> fail msg Nothing -> case lookup idx (map (\(a, _, b) -> (a, b)) (Evacuate.esMoved esol)) of Nothing -> fail "Internal error: lost instance idx during move" Just n -> return n let inst = Container.find idx il' pnode = Instance.pNode inst snode = Instance.sNode inst nodes' <- case mir_type of MirrorNone -> fail "Internal error: mirror type none after relocation?!" MirrorInternal -> do when (snode == sorig) $ fail "Internal error: instance didn't change secondary node?!" when (snode == pnode) $ fail "Internal error: selected primary as new secondary?!" if nodes == [pnode, snode] then return [snode] -- only the new secondary is needed else fail $ "Internal error: inconsistent node list (" ++ show nodes ++ ") versus instance nodes (" ++ show pnode ++ "," ++ show snode ++ ")" MirrorExternal -> do when (pnode == porig) $ fail "Internal error: instance didn't change primary node?!" if nodes == [pnode] then return nodes else fail $ "Internal error: inconsistent node list (" ++ show nodes ++ ") versus instance node (" ++ show pnode ++ ")" return (nl', il', nodes') processRelocate _ _ _ _ _ reqn _ = fail $ "Exchange " ++ show reqn ++ " nodes mode is not implemented" formatRelocate :: (Node.List, Instance.List, [Ndx]) -> Result IAllocResult formatRelocate (nl, il, ndxs) = let nodes = map (`Container.find` nl) ndxs names = map Node.name nodes in Ok ("success", showJSON names, nl, il) -- | Process a request and return new node lists. processRequest :: AlgorithmOptions -> Request -> Result IAllocResult processRequest opts request = let Request rqtype (ClusterData gl nl il _ _) = request in case rqtype of Allocate xi (Cluster.AllocDetails reqn Nothing) rest_nodes -> let opts' = opts { algRestrictToNodes = algRestrictToNodes opts `mplus` rest_nodes } in Cluster.tryMGAlloc opts' gl nl il xi reqn >>= formatAllocate il Allocate xi (Cluster.AllocDetails reqn (Just gn)) rest_nodes -> let opts' = opts { algRestrictToNodes = algRestrictToNodes opts `mplus` rest_nodes } in Cluster.tryGroupAlloc opts' gl nl il gn xi reqn >>= formatAllocate il Relocate idx reqn exnodes -> processRelocate opts gl nl il idx reqn exnodes >>= formatRelocate ChangeGroup gdxs idxs -> Cluster.tryChangeGroup opts gl nl il idxs gdxs >>= formatNodeEvac gl nl il NodeEvacuate xi mode -> Evacuate.tryNodeEvac opts gl nl il mode xi >>= formatNodeEvac gl nl il MultiAllocate xies -> Cluster.allocList opts gl nl il xies [] >>= formatMultiAlloc AllocateSecondary xi -> AllocSecondary.tryAllocateSecondary opts gl nl il xi >>= formatAllocateSecondary il -- | Reads the request from the data file(s). readRequest :: FilePath -- ^ Path to IAllocator input file -> Int -- ^ Static node memory size, see optStaticKvmNodeMemory -> IO Request readRequest fp static_n_mem = do now <- getClockTime input_data <- case fp of "-" -> getContents _ -> readFile fp case parseData now input_data static_n_mem of Bad err -> exitErr err Ok (fix_msgs, rq) -> maybeShowWarnings fix_msgs >> return rq -- | Format an IAlloc result to maybe the new cluster and a response. formatIAllocResult :: Result IAllocResult -> (Maybe (Node.List, Instance.List), String) formatIAllocResult iallocResult = let (ok, info, result, cdata) = case iallocResult of Ok (msg, r, nl, il) -> (True, "Request successful: " ++ msg, r, Just (nl, il)) Bad msg -> (False, "Request failed: " ++ msg, JSArray [], Nothing) rstring = formatResponse ok info result in (cdata, rstring) -- | Main iallocator pipeline. runIAllocator :: AlgorithmOptions -> Request -> (Maybe (Node.List, Instance.List), String) runIAllocator opts request = formatIAllocResult $ processRequest opts request -- | Load the data from an iallocation request file loadData :: FilePath -- ^ Path to IAllocator input file -> Int -- ^ Static node memory size, see optStaticKvmNodeMemory -> IO (Result ClusterData) loadData fp static_n_mem = do Request _ cdata <- readRequest fp static_n_mem return $ Ok cdata ganeti-3.1.0~rc2/src/Ganeti/HTools/Backend/Luxi.hs000064400000000000000000000321521476477700300216210ustar00rootroot00000000000000{-| Implementation of the LUXI loader. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Backend.Luxi ( loadData , parseData ) where import qualified Control.Exception as E import Control.Monad (liftM) import Control.Monad.Fail (MonadFail) import Text.JSON.Types import qualified Text.JSON import Ganeti.BasicTypes import Ganeti.Errors import qualified Ganeti.Luxi as L import qualified Ganeti.Query.Language as Qlang import Ganeti.Types (Hypervisor(..)) import Ganeti.HTools.Loader import Ganeti.HTools.Types import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance import Ganeti.JSON (fromObj, fromJVal, tryFromObj, arrayMaybeFromJVal) {-# ANN module "HLint: ignore Eta reduce" #-} -- * Utility functions -- | Get values behind \"data\" part of the result. getData :: (MonadFail m) => JSValue -> m JSValue getData (JSObject o) = fromObj (fromJSObject o) "data" getData x = fail $ "Invalid input, expected dict entry but got " ++ show x -- | Converts a (status, value) into m value, if possible. parseQueryField :: (MonadFail m) => JSValue -> m (JSValue, JSValue) parseQueryField (JSArray [status, result]) = return (status, result) parseQueryField o = fail $ "Invalid query field, expected (status, value) but got " ++ show o -- | Parse a result row. parseQueryRow :: (MonadFail m) => JSValue -> m [(JSValue, JSValue)] parseQueryRow (JSArray arr) = mapM parseQueryField arr parseQueryRow o = fail $ "Invalid query row result, expected array but got " ++ show o -- | Parse an overall query result and get the [(status, value)] list -- for each element queried. parseQueryResult :: (MonadFail m) => JSValue -> m [[(JSValue, JSValue)]] parseQueryResult (JSArray arr) = mapM parseQueryRow arr parseQueryResult o = fail $ "Invalid query result, expected array but got " ++ show o -- | Prepare resulting output as parsers expect it. extractArray :: (MonadFail m) => JSValue -> m [[(JSValue, JSValue)]] extractArray v = getData v >>= parseQueryResult -- | Testing result status for more verbose error message. fromJValWithStatus :: (Text.JSON.JSON a, MonadFail m) => (JSValue, JSValue) -> m a fromJValWithStatus (st, v) = do st' <- fromJVal st Qlang.checkRS st' v >>= fromJVal annotateConvert :: String -> String -> String -> Result a -> Result a annotateConvert otype oname oattr = annotateResult $ otype ++ " '" ++ oname ++ "', error while reading attribute '" ++ oattr ++ "'" -- | Annotate errors when converting values with owner/attribute for -- better debugging. genericConvert :: (Text.JSON.JSON a) => String -- ^ The object type -> String -- ^ The object name -> String -- ^ The attribute we're trying to convert -> (JSValue, JSValue) -- ^ The value we're trying to convert -> Result a -- ^ The annotated result genericConvert otype oname oattr = annotateConvert otype oname oattr . fromJValWithStatus convertArrayMaybe :: (Text.JSON.JSON a) => String -- ^ The object type -> String -- ^ The object name -> String -- ^ The attribute we're trying to convert -> (JSValue, JSValue) -- ^ The value we're trying to convert -> Result [Maybe a] -- ^ The annotated result convertArrayMaybe otype oname oattr (st, v) = do st' <- fromJVal st Qlang.checkRS st' v >>= annotateConvert otype oname oattr . arrayMaybeFromJVal -- * Data querying functionality -- | The input data for node query. queryNodesMsg :: L.LuxiOp queryNodesMsg = L.Query (Qlang.ItemTypeOpCode Qlang.QRNode) ["name", "mtotal", "mnode", "mfree", "dtotal", "dfree", "ctotal", "cnos", "offline", "drained", "vm_capable", "ndp/spindle_count", "group.uuid", "tags", "ndp/exclusive_storage", "sptotal", "spfree", "ndp/cpu_speed"] Qlang.EmptyFilter -- | The input data for instance query. queryInstancesMsg :: L.LuxiOp queryInstancesMsg = L.Query (Qlang.ItemTypeOpCode Qlang.QRInstance) ["name", "disk_usage", "be/memory", "be/vcpus", "status", "pnode", "snodes", "tags", "be/auto_balance", "disk_template", "be/spindle_use", "disk.sizes", "disk.spindles", "forthcoming"] Qlang.EmptyFilter -- | The input data for cluster query. queryClusterInfoMsg :: L.LuxiOp queryClusterInfoMsg = L.QueryClusterInfo -- | The input data for node group query. queryGroupsMsg :: L.LuxiOp queryGroupsMsg = L.Query (Qlang.ItemTypeOpCode Qlang.QRGroup) ["uuid", "name", "alloc_policy", "ipolicy", "tags"] Qlang.EmptyFilter -- | Wraper over 'callMethod' doing node query. queryNodes :: L.Client -> IO (Result JSValue) queryNodes = liftM errToResult . L.callMethod queryNodesMsg -- | Wraper over 'callMethod' doing instance query. queryInstances :: L.Client -> IO (Result JSValue) queryInstances = liftM errToResult . L.callMethod queryInstancesMsg -- | Wrapper over 'callMethod' doing cluster information query. queryClusterInfo :: L.Client -> IO (Result JSValue) queryClusterInfo = liftM errToResult . L.callMethod queryClusterInfoMsg -- | Wrapper over callMethod doing group query. queryGroups :: L.Client -> IO (Result JSValue) queryGroups = liftM errToResult . L.callMethod queryGroupsMsg -- | Parse a instance list in JSON format. getInstances :: NameAssoc -> JSValue -> Result [(String, Instance.Instance)] getInstances ktn arr = extractArray arr >>= mapM (parseInstance ktn) -- | Construct an instance from a JSON object. parseInstance :: NameAssoc -> [(JSValue, JSValue)] -> Result (String, Instance.Instance) parseInstance ktn [ name, disk, mem, vcpus , status, pnode, snodes, tags , auto_balance, disk_template, su , dsizes, dspindles, forthcoming ] = do xname <- annotateResult "Parsing new instance" (fromJValWithStatus name) let convert a = genericConvert "Instance" xname a xdisk <- convert "disk_usage" disk xmem <- convert "be/memory" mem xvcpus <- convert "be/vcpus" vcpus xpnode <- convert "pnode" pnode >>= lookupNode ktn xname xsnodes <- convert "snodes" snodes::Result [String] snode <- case xsnodes of [] -> return Node.noSecondary x:_ -> lookupNode ktn xname x xrunning <- convert "status" status xtags <- convert "tags" tags xauto_balance <- convert "auto_balance" auto_balance xdt <- convert "disk_template" disk_template xsu <- convert "be/spindle_use" su xdsizes <- convert "disk.sizes" dsizes xdspindles <- convertArrayMaybe "Instance" xname "disk.spindles" dspindles xforthcoming <- convert "forthcoming" forthcoming let disks = zipWith Instance.Disk xdsizes xdspindles inst = Instance.create xname xmem xdisk disks xvcpus xrunning xtags xauto_balance xpnode snode xdt xsu [] xforthcoming return (xname, inst) parseInstance _ v = fail ("Invalid instance query result: " ++ show v) -- | Parse a node list in JSON format. getNodes :: NameAssoc -> JSValue -> Result [(String, Node.Node)] getNodes ktg arr = extractArray arr >>= mapM (parseNode ktg) -- | Construct a node from a JSON object. parseNode :: NameAssoc -> [(JSValue, JSValue)] -> Result (String, Node.Node) parseNode ktg [ name, mtotal, mnode, mfree, dtotal, dfree , ctotal, cnos, offline, drained, vm_capable, spindles, g_uuid , tags, excl_stor, sptotal, spfree, cpu_speed ] = do xname <- annotateResult "Parsing new node" (fromJValWithStatus name) let convert a = genericConvert "Node" xname a xoffline <- convert "offline" offline xdrained <- convert "drained" drained xvm_capable <- convert "vm_capable" vm_capable xgdx <- convert "group.uuid" g_uuid >>= lookupGroup ktg xname xtags <- convert "tags" tags xexcl_stor <- convert "exclusive_storage" excl_stor xcpu_speed <- convert "cpu_speed" cpu_speed let live = not xoffline && xvm_capable lvconvert def n d = eitherLive live def $ convert n d xsptotal <- if xexcl_stor then lvconvert 0 "sptotal" sptotal else convert "spindles" spindles let xspfree = genericResult (const (0 :: Int)) id $ lvconvert 0 "spfree" spfree -- "spfree" might be missing, if sharedfile is the only -- supported disk template xmtotal <- lvconvert 0.0 "mtotal" mtotal xmnode <- lvconvert 0 "mnode" mnode xmfree <- lvconvert 0 "mfree" mfree let xdtotal = genericResult (const 0.0) id $ lvconvert 0.0 "dtotal" dtotal xdfree = genericResult (const 0) id $ lvconvert 0 "dfree" dfree -- "dtotal" and "dfree" might be missing, e.g., if sharedfile -- is the only supported disk template xctotal <- lvconvert 0.0 "ctotal" ctotal xcnos <- lvconvert 0 "cnos" cnos let node = flip Node.setCpuSpeed xcpu_speed . flip Node.setNodeTags xtags $ Node.create xname xmtotal xmnode xmfree xdtotal xdfree xctotal xcnos (not live || xdrained) xsptotal xspfree xgdx xexcl_stor return (xname, node) parseNode _ v = fail ("Invalid node query result: " ++ show v) -- | Parses the cluster tags. getClusterData :: JSValue -> Result ([String], IPolicy, String, Hypervisor) getClusterData (JSObject obj) = do let errmsg = "Parsing cluster info" obj' = fromJSObject obj ctags <- tryFromObj errmsg obj' "tags" cpol <- tryFromObj errmsg obj' "ipolicy" master <- tryFromObj errmsg obj' "master" hypervisor <- tryFromObj errmsg obj' "default_hypervisor" return (ctags, cpol, master, hypervisor) getClusterData _ = Bad "Cannot parse cluster info, not a JSON record" -- | Parses the cluster groups. getGroups :: JSValue -> Result [(String, Group.Group)] getGroups jsv = extractArray jsv >>= mapM parseGroup -- | Parses a given group information. parseGroup :: [(JSValue, JSValue)] -> Result (String, Group.Group) parseGroup [uuid, name, apol, ipol, tags] = do xname <- annotateResult "Parsing new group" (fromJValWithStatus name) let convert a = genericConvert "Group" xname a xuuid <- convert "uuid" uuid xapol <- convert "alloc_policy" apol xipol <- convert "ipolicy" ipol xtags <- convert "tags" tags -- TODO: parse networks to which this group is connected return (xuuid, Group.create xname xuuid xapol [] xipol xtags) parseGroup v = fail ("Invalid group query result: " ++ show v) -- * Main loader functionality -- | Builds the cluster data by querying a given socket name. readData :: String -- ^ Unix socket to use as source -> IO (Result JSValue, Result JSValue, Result JSValue, Result JSValue) readData master = E.bracket (L.getLuxiClient master) L.closeClient (\s -> do nodes <- queryNodes s instances <- queryInstances s cinfo <- queryClusterInfo s groups <- queryGroups s return (groups, nodes, instances, cinfo) ) -- | Converts the output of 'readData' into the internal cluster -- representation. parseData :: (Result JSValue, Result JSValue, Result JSValue, Result JSValue) -> Result ClusterData parseData (groups, nodes, instances, cinfo) = do group_data <- groups >>= getGroups let (group_names, group_idx) = assignIndices group_data node_data <- nodes >>= getNodes group_names let (node_names, node_idx) = assignIndices node_data inst_data <- instances >>= getInstances node_names let (_, inst_idx) = assignIndices inst_data (ctags, cpol, master, hypervisor) <- cinfo >>= getClusterData node_idx' <- setMaster node_names node_idx master let node_idx'' = Container.map (`Node.setHypervisor` hypervisor) node_idx' return (ClusterData group_idx node_idx'' inst_idx ctags cpol) -- | Top level function for data loading. loadData :: String -- ^ Unix socket to use as source -> IO (Result ClusterData) loadData = fmap parseData . readData ganeti-3.1.0~rc2/src/Ganeti/HTools/Backend/MonD.hs000064400000000000000000000264751476477700300215500ustar00rootroot00000000000000{-# LANGUAGE BangPatterns #-} {-| Monitoring daemon backend This module holds implements the querying of the monitoring daemons for dynamic utilisation data. -} {- Copyright (C) 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Backend.MonD ( queryAllMonDDCs , pMonDData ) where import Control.Monad import Control.Monad.Writer import qualified Data.List as L import qualified Data.IntMap as IntMap import qualified Data.Map as Map import Data.Maybe (catMaybes, mapMaybe) import Data.Monoid (All(All)) import qualified Data.Set as Set import Network.Curl import qualified Text.JSON as J import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.Cpu.Types import qualified Ganeti.DataCollectors.XenCpuLoad as XenCpuLoad import qualified Ganeti.DataCollectors.CPUload as CPUload import Ganeti.DataCollectors.Types ( DCReport, DCCategory , dcReportData, dcReportName , getCategoryName ) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance import Ganeti.HTools.Loader (ClusterData(..)) import Ganeti.HTools.Types import Ganeti.HTools.CLI import Ganeti.JSON (fromJVal, tryFromObj, JSRecord, loadJSArray, maybeParseMap) import Ganeti.Logging.Lifted (logWarning) import Ganeti.Utils (exitIfBad) -- * General definitions -- | The actual data types for MonD's Data Collectors. data Report = CPUavgloadReport CPUavgload | InstanceCpuReport (Map.Map String Double) -- | Type describing a data collector basic information. data DataCollector = DataCollector { dName :: String -- ^ Name of the data collector , dCategory :: Maybe DCCategory -- ^ The name of the category , dMkReport :: DCReport -> Maybe Report -- ^ How to parse a monitor report , dUse :: [(Node.Node, Report)] -> (Node.List, Instance.List) -> Result (Node.List, Instance.List) -- ^ How the collector reports are to be used to bring dynamic -- data into a cluster } -- * Node-total CPU load average data collector -- | Parse a DCReport for the node-total CPU collector. mkCpuReport :: DCReport -> Maybe Report mkCpuReport dcr = case fromJVal (dcReportData dcr) :: Result CPUavgload of Ok cav -> Just $ CPUavgloadReport cav Bad _ -> Nothing -- | Take reports of node CPU values and update a node accordingly. updateNodeCpuFromReport :: (Node.Node, Report) -> Node.Node updateNodeCpuFromReport (node, CPUavgloadReport cav) = let ct = cavCpuTotal cav du = Node.utilLoad node du' = du {cpuWeight = ct} in node { Node.utilLoad = du' } updateNodeCpuFromReport (node, _) = node -- | Update the instance CPU-utilization data, asuming that each virtual -- CPU contributes equally to the node CPU load. updateCpuUtilDataFromNode :: Instance.List -> Node.Node -> Instance.List updateCpuUtilDataFromNode il node = let ct = cpuWeight (Node.utilLoad node) n_uCpu = Node.uCpu node upd inst = if Node.idx node == Instance.pNode inst then let i_vcpus = Instance.vcpus inst i_util = ct / fromIntegral n_uCpu * fromIntegral i_vcpus i_du = Instance.util inst i_du' = i_du {cpuWeight = i_util} in inst {Instance.util = i_du'} else inst in Container.map upd il -- | Update cluster data from node CPU load reports. useNodeTotalCPU :: [(Node.Node, Report)] -> (Node.List, Instance.List) -> Result (Node.List, Instance.List) useNodeTotalCPU reports (nl, il) = let newnodes = map updateNodeCpuFromReport reports il' = foldl updateCpuUtilDataFromNode il newnodes nl' = zip (Container.keys nl) newnodes in return (Container.fromList nl', il') -- | The node-total CPU collector. totalCPUCollector :: DataCollector totalCPUCollector = DataCollector { dName = CPUload.dcName , dCategory = CPUload.dcCategory , dMkReport = mkCpuReport , dUse = useNodeTotalCPU } -- * Xen instance CPU-usage collector -- | Parse results of the Xen-Cpu-load data collector. mkXenCpuReport :: DCReport -> Maybe Report mkXenCpuReport = liftM InstanceCpuReport . maybeParseMap . dcReportData -- | Update cluster data based on the per-instance CPU usage -- reports useInstanceCpuData :: [(Node.Node, Report)] -> (Node.List, Instance.List) -> Result (Node.List, Instance.List) useInstanceCpuData reports (nl, il) = do let toMap (InstanceCpuReport m) = Just m toMap _ = Nothing let usage = Map.unions $ mapMaybe (toMap . snd) reports missingData = (Set.fromList . map Instance.name $ IntMap.elems il) Set.\\ Map.keysSet usage unless (Set.null missingData) . Bad . (++) "No CPU information available for " . show $ Set.elems missingData let updateInstance inst = let cpu = Map.lookup (Instance.name inst) usage dynU = Instance.util inst dynU' = maybe dynU (\c -> dynU { cpuWeight = c }) cpu in inst { Instance.util = dynU' } let il' = IntMap.map updateInstance il let updateNode node = let cpu = sum . map (\ idx -> maybe 0 (cpuWeight . Instance.util) $ IntMap.lookup idx il') $ Node.pList node dynU = Node.utilLoad node dynU' = dynU { cpuWeight = cpu } in node { Node.utilLoad = dynU' } let nl' = IntMap.map updateNode nl return (nl', il') -- | Collector for per-instance CPU data as observed by Xen xenCPUCollector :: DataCollector xenCPUCollector = DataCollector { dName = XenCpuLoad.dcName , dCategory = XenCpuLoad.dcCategory , dMkReport = mkXenCpuReport , dUse = useInstanceCpuData } -- * Collector choice -- | The list of Data Collectors used by hail and hbal. collectors :: Options -> [DataCollector] collectors opts | optIgnoreDynu opts = [] | optMonDXen opts = [ xenCPUCollector ] | otherwise = [ totalCPUCollector ] -- * Querying infrastructure -- | Return the data from correct combination of a Data Collector -- and a DCReport. mkReport :: DataCollector -> Maybe DCReport -> Maybe Report mkReport dc = (>>= dMkReport dc) -- | MonDs Data parsed by a mock file. Representing (node name, list of reports -- produced by MonDs Data Collectors). type MonDData = (String, [DCReport]) -- | A map storing MonDs data. type MapMonDData = Map.Map String [DCReport] -- | Get data report for the specified Data Collector and Node from the map. fromFile :: DataCollector -> Node.Node -> MapMonDData -> Maybe DCReport fromFile dc node m = let matchDCName dcr = dName dc == dcReportName dcr in maybe Nothing (L.find matchDCName) $ Map.lookup (Node.name node) m -- | Get Category Name. getDCCName :: Maybe DCCategory -> String getDCCName dcc = case dcc of Nothing -> "default" Just c -> getCategoryName c -- | Prepare url to query a single collector. prepareUrl :: DataCollector -> Node.Node -> URLString prepareUrl dc node = Node.name node ++ ":" ++ show C.defaultMondPort ++ "/" ++ show C.mondLatestApiVersion ++ "/report/" ++ getDCCName (dCategory dc) ++ "/" ++ dName dc -- | Query a specified MonD for a Data Collector. fromCurl :: DataCollector -> Node.Node -> IO (Maybe DCReport) fromCurl dc node = do (code, !body) <- curlGetString (prepareUrl dc node) [] case code of CurlOK -> case J.decodeStrict body :: J.Result DCReport of J.Ok r -> return $ Just r J.Error _ -> return Nothing _ -> do logWarning $ "Failed to contact node's " ++ Node.name node ++ " MonD for DC " ++ dName dc return Nothing -- | Parse a node's JSON record. pMonDN :: JSRecord -> Result MonDData pMonDN a = do node <- tryFromObj "Parsing node's name" a "node" reports <- tryFromObj "Parsing node's reports" a "reports" return (node, reports) -- | Parse MonD data file contents. pMonDData :: String -> Result [MonDData] pMonDData input = loadJSArray "Parsing MonD's answer" input >>= mapM (pMonDN . J.fromJSObject) -- | Query a single MonD for a single Data Collector. queryAMonD :: Maybe MapMonDData -> DataCollector -> Node.Node -> IO (Maybe Report) queryAMonD m dc node = liftM (mkReport dc) $ case m of Nothing -> fromCurl dc node Just m' -> return $ fromFile dc node m' -- | Query all MonDs for a single Data Collector. Return the updated -- cluster, as well as a bit inidicating wether the collector succeeded. queryAllMonDs :: Maybe MapMonDData -> (Node.List, Instance.List) -> DataCollector -> WriterT All IO (Node.List, Instance.List) queryAllMonDs m (nl, il) dc = do elems <- liftIO $ mapM (queryAMonD m dc) (Container.elems nl) let elems' = catMaybes elems if length elems == length elems' then let results = zip (Container.elems nl) elems' in case dUse dc results (nl, il) of Ok (nl', il') -> return (nl', il') Bad s -> do logWarning s tell $ All False return (nl, il) else do logWarning $ "Didn't receive an answer by all MonDs, " ++ dName dc ++ "'s data will be ignored." tell $ All False return (nl,il) -- | Query all MonDs for all Data Collector. Return the cluster enriched -- by dynamic data, as well as a bit indicating wether all collectors -- could be queried successfully. queryAllMonDDCs :: ClusterData -> Options -> WriterT All IO ClusterData queryAllMonDDCs cdata opts = do map_mDD <- case optMonDFile opts of Nothing -> return Nothing Just fp -> do monDData_contents <- liftIO $ readFile fp monDData <- liftIO . exitIfBad "can't parse MonD data" . pMonDData $ monDData_contents return . Just $ Map.fromList monDData let (ClusterData _ nl il _ _) = cdata (nl', il') <- foldM (queryAllMonDs map_mDD) (nl, il) (collectors opts) return $ cdata {cdNodes = nl', cdInstances = il'} ganeti-3.1.0~rc2/src/Ganeti/HTools/Backend/Rapi.hs000064400000000000000000000246571476477700300216060ustar00rootroot00000000000000{-| Implementation of the RAPI client interface. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} {-# LANGUAGE BangPatterns, CPP #-} module Ganeti.HTools.Backend.Rapi ( loadData , parseData ) where import Control.Exception import Data.List (isPrefixOf) import Data.Maybe (fromMaybe) import Network.Curl import Network.Curl.Types () import Control.Monad import Control.Monad.Fail (MonadFail) import Text.JSON (JSObject, fromJSObject, decodeStrict) import Text.JSON.Types (JSValue(..)) import Text.Printf (printf) import System.FilePath import Ganeti.BasicTypes import Ganeti.Types (Hypervisor(..)) import Ganeti.HTools.Loader import Ganeti.HTools.Types import Ganeti.JSON (loadJSArray, JSRecord, tryFromObj, fromJVal, maybeFromObj, fromJResult, tryArrayMaybeFromObj, readEitherString, fromObjWithDefault, asJSObject) import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Container as Container import qualified Ganeti.Constants as C {-# ANN module "HLint: ignore Eta reduce" #-} -- | File method prefix. filePrefix :: String filePrefix = "file://" -- | Read an URL via curl and return the body if successful. getUrl :: (MonadFail m) => String -> IO (m String) -- | Connection timeout (when using non-file methods). connTimeout :: Long connTimeout = 15 -- | The default timeout for queries (when using non-file methods). queryTimeout :: Long queryTimeout = 60 -- | The curl options we use. curlOpts :: [CurlOption] curlOpts = [ CurlSSLVerifyPeer False , CurlSSLVerifyHost 0 , CurlTimeout queryTimeout , CurlConnectTimeout connTimeout ] getUrl url = do (code, !body) <- curlGetString url curlOpts return (case code of CurlOK -> return body _ -> fail $ printf "Curl error for '%s', error %s" url (show code)) -- | Helper to convert I/O errors in 'Bad' values. ioErrToResult :: IO a -> IO (Result a) ioErrToResult ioaction = Control.Exception.catch (liftM Ok ioaction) (\e -> return . Bad . show $ (e::IOException)) -- | Append the default port if not passed in. formatHost :: String -> String formatHost master = if ':' `elem` master then master else "https://" ++ master ++ ":" ++ show C.defaultRapiPort -- | Parse a instance list in JSON format. getInstances :: NameAssoc -> String -> Result [(String, Instance.Instance)] getInstances ktn body = loadJSArray "Parsing instance data" body >>= mapM (parseInstance ktn . fromJSObject) -- | Parse a node list in JSON format. getNodes :: NameAssoc -> String -> Result [(String, Node.Node)] getNodes ktg body = loadJSArray "Parsing node data" body >>= mapM (parseNode ktg . fromJSObject) -- | Parse a group list in JSON format. getGroups :: String -> Result [(String, Group.Group)] getGroups body = loadJSArray "Parsing group data" body >>= mapM (parseGroup . fromJSObject) -- | Construct an instance from a JSON object. parseInstance :: NameAssoc -> JSRecord -> Result (String, Instance.Instance) parseInstance ktn a = do name <- tryFromObj "Parsing new instance" a "name" let owner_name = "Instance '" ++ name ++ "', error while parsing data" let extract s x = tryFromObj owner_name x s disk <- extract "disk_usage" a dsizes <- extract "disk.sizes" a dspindles <- tryArrayMaybeFromObj owner_name a "disk.spindles" beparams <- liftM fromJSObject (extract "beparams" a) omem <- extract "oper_ram" a mem <- case omem of JSRational _ _ -> annotateResult owner_name (fromJVal omem) _ -> extract "memory" beparams `mplus` extract "maxmem" beparams vcpus <- extract "vcpus" beparams pnode <- extract "pnode" a >>= lookupNode ktn name snodes <- extract "snodes" a snode <- case snodes of [] -> return Node.noSecondary x:_ -> readEitherString x >>= lookupNode ktn name running <- extract "status" a tags <- extract "tags" a auto_balance <- extract "auto_balance" beparams dt <- extract "disk_template" a su <- extract "spindle_use" beparams -- Not forthcoming by default. forthcoming <- extract "forthcoming" a `orElse` Ok False let disks = zipWith Instance.Disk dsizes dspindles let inst = Instance.create name mem disk disks vcpus running tags auto_balance pnode snode dt su [] forthcoming return (name, inst) -- | Construct a node from a JSON object. parseNode :: NameAssoc -> JSRecord -> Result (String, Node.Node) parseNode ktg a = do name <- tryFromObj "Parsing new node" a "name" let desc = "Node '" ++ name ++ "', error while parsing data" extract key = tryFromObj desc a key extractDef def key = fromObjWithDefault a key def offline <- extract "offline" drained <- extract "drained" vm_cap <- annotateResult desc $ maybeFromObj a "vm_capable" let vm_cap' = fromMaybe True vm_cap ndparams <- extract "ndparams" >>= asJSObject excl_stor <- tryFromObj desc (fromJSObject ndparams) "exclusive_storage" guuid <- annotateResult desc $ maybeFromObj a "group.uuid" guuid' <- lookupGroup ktg name (fromMaybe defaultGroupID guuid) let live = not offline && vm_cap' lvextract def = eitherLive live def . extract lvextractDef def = eitherLive live def . extractDef def sptotal <- if excl_stor then lvextract 0 "sptotal" else tryFromObj desc (fromJSObject ndparams) "spindle_count" spfree <- lvextractDef 0 "spfree" mtotal <- lvextract 0.0 "mtotal" mnode <- lvextract 0 "mnode" mfree <- lvextract 0 "mfree" dtotal <- lvextractDef 0.0 "dtotal" dfree <- lvextractDef 0 "dfree" ctotal <- lvextract 0.0 "ctotal" cnos <- lvextract 0 "cnos" tags <- extract "tags" let node = flip Node.setNodeTags tags $ Node.create name mtotal mnode mfree dtotal dfree ctotal cnos (not live || drained) sptotal spfree guuid' excl_stor return (name, node) -- | Construct a group from a JSON object. parseGroup :: JSRecord -> Result (String, Group.Group) parseGroup a = do name <- tryFromObj "Parsing new group" a "name" let extract s = tryFromObj ("Group '" ++ name ++ "'") a s uuid <- extract "uuid" apol <- extract "alloc_policy" ipol <- extract "ipolicy" tags <- extract "tags" -- TODO: parse networks to which this group is connected return (uuid, Group.create name uuid apol [] ipol tags) -- | Parse cluster data from the info resource. parseCluster :: JSObject JSValue -> Result ([String], IPolicy, String, Hypervisor) parseCluster obj = do let obj' = fromJSObject obj extract s = tryFromObj "Parsing cluster data" obj' s master <- extract "master" tags <- extract "tags" ipolicy <- extract "ipolicy" hypervisor <- extract "default_hypervisor" return (tags, ipolicy, master, hypervisor) -- | Loads the raw cluster data from an URL. readDataHttp :: String -- ^ Cluster or URL to use as source -> IO (Result String, Result String, Result String, Result String) readDataHttp master = do let url = formatHost master group_body <- getUrl $ printf "%s/2/groups?bulk=1" url node_body <- getUrl $ printf "%s/2/nodes?bulk=1" url inst_body <- getUrl $ printf "%s/2/instances?bulk=1" url info_body <- getUrl $ printf "%s/2/info" url return (group_body, node_body, inst_body, info_body) -- | Loads the raw cluster data from the filesystem. readDataFile:: String -- ^ Path to the directory containing the files -> IO (Result String, Result String, Result String, Result String) readDataFile path = do group_body <- ioErrToResult . readFile $ path "groups.json" node_body <- ioErrToResult . readFile $ path "nodes.json" inst_body <- ioErrToResult . readFile $ path "instances.json" info_body <- ioErrToResult . readFile $ path "info.json" return (group_body, node_body, inst_body, info_body) -- | Loads data via either 'readDataFile' or 'readDataHttp'. readData :: String -- ^ URL to use as source -> IO (Result String, Result String, Result String, Result String) readData url = if filePrefix `isPrefixOf` url then readDataFile (drop (length filePrefix) url) else readDataHttp url -- | Builds the cluster data from the raw Rapi content. parseData :: (Result String, Result String, Result String, Result String) -> Result ClusterData parseData (group_body, node_body, inst_body, info_body) = do group_data <- group_body >>= getGroups let (group_names, group_idx) = assignIndices group_data node_data <- node_body >>= getNodes group_names let (node_names, node_idx) = assignIndices node_data inst_data <- inst_body >>= getInstances node_names let (_, inst_idx) = assignIndices inst_data (tags, ipolicy, master, hypervisor) <- info_body >>= (fromJResult "Parsing cluster info" . decodeStrict) >>= parseCluster node_idx' <- setMaster node_names node_idx master let node_idx'' = Container.map (`Node.setHypervisor` hypervisor) node_idx' return (ClusterData group_idx node_idx'' inst_idx tags ipolicy) -- | Top level function for data loading. loadData :: String -- ^ Cluster or URL to use as source -> IO (Result ClusterData) loadData = fmap parseData . readData ganeti-3.1.0~rc2/src/Ganeti/HTools/Backend/Simu.hs000064400000000000000000000115361476477700300216200ustar00rootroot00000000000000{-| Parsing data from a simulated description of the cluster. This module holds the code for parsing a cluster description. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Backend.Simu ( loadData , parseData ) where import Control.Monad (mplus, zipWithM) import Text.Printf (printf) import Ganeti.BasicTypes import Ganeti.Utils import Ganeti.HTools.Types import Ganeti.HTools.Loader import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Node as Node -- | Parse a shortened policy string (for command line usage). apolAbbrev :: String -> Result AllocPolicy apolAbbrev c | c == "p" = return AllocPreferred | c == "a" = return AllocLastResort | c == "u" = return AllocUnallocable | otherwise = fail $ "Cannot parse AllocPolicy abbreviation '" ++ c ++ "'" -- | Parse the string description into nodes. parseDesc :: String -> [String] -> Result (AllocPolicy, Int, Int, Int, Int, Int, Bool) parseDesc _ [a, n, d, m, c, s, exstor] = do apol <- allocPolicyFromRaw a `mplus` apolAbbrev a ncount <- tryRead "node count" n disk <- annotateResult "disk size" (parseUnit d) mem <- annotateResult "memory size" (parseUnit m) cpu <- tryRead "cpu count" c spindles <- tryRead "spindles" s excl_stor <- tryRead "exclusive storage" exstor return (apol, ncount, disk, mem, cpu, spindles, excl_stor) parseDesc desc [a, n, d, m, c] = parseDesc desc [a, n, d, m, c, "1"] parseDesc desc [a, n, d, m, c, s] = parseDesc desc [a, n, d, m, c, s, "False"] parseDesc desc es = fail $ printf "Invalid cluster specification, expected 6 comma-separated\ \ sections (allocation policy, node count, disk size,\ \ memory size, number of CPUs, spindles) but got %d: '%s'" (length es) desc -- | Creates a node group with the given specifications. createGroup :: Int -- ^ The group index -> String -- ^ The group specification -> Result (Group.Group, [Node.Node]) createGroup grpIndex spec = do (apol, ncount, disk, mem, cpu, spindles, excl_stor) <- parseDesc spec $ sepSplit ',' spec let nodes = map (\idx -> flip Node.setMaster (grpIndex == 1 && idx == 1) $ Node.create (printf "node-%02d-%03d" grpIndex idx) (fromIntegral mem) 0 mem (fromIntegral disk) disk (fromIntegral cpu) 1 False spindles 0 grpIndex excl_stor ) [1..ncount] -- TODO: parse networks to which this group is connected grp = Group.create (printf "group-%02d" grpIndex) (printf "fake-uuid-%02d" grpIndex) apol [] defIPolicy [] return (Group.setIdx grp grpIndex, nodes) -- | Builds the cluster data from node\/instance files. parseData :: [String] -- ^ Cluster description in text format -> Result ClusterData parseData ndata = do grpNodeData <- zipWithM createGroup [1..] ndata let (groups, nodes) = unzip grpNodeData nodes' = concat nodes let ktn = map (\(idx, n) -> (idx, Node.setIdx n idx)) $ zip [1..] nodes' ktg = map (\g -> (Group.idx g, g)) groups return (ClusterData (Container.fromList ktg) (Container.fromList ktn) Container.empty [] defIPolicy) -- | Builds the cluster data from node\/instance files. loadData :: [String] -- ^ Cluster description in text format -> IO (Result ClusterData) loadData = -- IO monad, just for consistency with the other loaders return . parseData ganeti-3.1.0~rc2/src/Ganeti/HTools/Backend/Text.hs000064400000000000000000000446111476477700300216270ustar00rootroot00000000000000{-# LANGUAGE TupleSections #-} {-| Parsing data from text-files. This module holds the code for loading the cluster state from text files, as produced by @gnt-node@ and @gnt-instance@ @list@ command. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Backend.Text ( loadData , parseData , loadInst , loadNode , loadISpec , loadMultipleMinMaxISpecs , loadIPolicy , serializeInstance , serializeInstances , serializeNode , serializeNodes , serializeGroup , serializeISpec , serializeMultipleMinMaxISpecs , serializeIPolicy , serializeCluster ) where import Control.Monad import Control.Monad.Fail (MonadFail) import Data.List import Text.Printf (printf) import Ganeti.BasicTypes import Ganeti.Utils import Ganeti.HTools.Loader import Ganeti.HTools.Types import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance -- * Helper functions -- | Simple wrapper over sepSplit commaSplit :: String -> [String] commaSplit = sepSplit ',' -- * Serialisation functions -- | Serialize a single group. serializeGroup :: Group.Group -> String serializeGroup grp = printf "%s|%s|%s|%s|%s" (Group.name grp) (Group.uuid grp) (allocPolicyToRaw (Group.allocPolicy grp)) (intercalate "," (Group.allTags grp)) (intercalate "," (Group.networks grp)) -- | Generate group file data from a group list. serializeGroups :: Group.List -> String serializeGroups = unlines . map serializeGroup . Container.elems -- | Serialize a single node. serializeNode :: Group.List -- ^ The list of groups (needed for group uuid) -> Node.Node -- ^ The node to be serialised -> String serializeNode gl node = printf "%s|%.0f|%d|%d|%.0f|%d|%.0f|%c|%s|%d|%s|%s|%d|%d|%f" (Node.name node) (Node.tMem node) (Node.nMem node) (Node.fMem node) (Node.tDsk node) (Node.fDsk node) (Node.tCpu node) (if Node.offline node then 'Y' else if Node.isMaster node then 'M' else 'N') (Group.uuid grp) (Node.tSpindles node) (intercalate "," (Node.nTags node)) (if Node.exclStorage node then "Y" else "N") (Node.fSpindles node) (Node.nCpu node) (Node.tCpuSpeed node) where grp = Container.find (Node.group node) gl -- | Generate node file data from node objects. serializeNodes :: Group.List -> Node.List -> String serializeNodes gl = unlines . map (serializeNode gl) . Container.elems -- | Serialize a single instance. serializeInstance :: Node.List -- ^ The node list (needed for -- node names) -> Instance.Instance -- ^ The instance to be serialised -> String serializeInstance nl inst = let iname = Instance.name inst pnode = Container.nameOf nl (Instance.pNode inst) sidx = Instance.sNode inst snode = (if sidx == Node.noSecondary then "" else Container.nameOf nl sidx) in printf "%s|%d|%d|%d|%s|%s|%s|%s|%s|%s|%d|%s|%s" iname (Instance.mem inst) (Instance.dsk inst) (Instance.vcpus inst) (instanceStatusToRaw (Instance.runSt inst)) (if Instance.autoBalance inst then "Y" else "N") pnode snode (diskTemplateToRaw (Instance.diskTemplate inst)) (intercalate "," (Instance.allTags inst)) (Instance.spindleUse inst) -- disk spindles are summed together, as it's done for disk size (case Instance.getTotalSpindles inst of Nothing -> "-" Just x -> show x) (if Instance.forthcoming inst then "Y" else "N") -- | Generate instance file data from instance objects. serializeInstances :: Node.List -> Instance.List -> String serializeInstances nl = unlines . map (serializeInstance nl) . Container.elems -- | Separator between ISpecs (in MinMaxISpecs). iSpecsSeparator :: Char iSpecsSeparator = ';' -- | Generate a spec data from a given ISpec object. serializeISpec :: ISpec -> String serializeISpec ispec = -- this needs to be kept in sync with the object definition let ISpec mem_s cpu_c disk_s disk_c nic_c su = ispec strings = [show mem_s, show cpu_c, show disk_s, show disk_c, show nic_c, show su] in intercalate "," strings -- | Generate disk template data. serializeDiskTemplates :: [DiskTemplate] -> String serializeDiskTemplates = intercalate "," . map diskTemplateToRaw -- | Generate min/max instance specs data. serializeMultipleMinMaxISpecs :: [MinMaxISpecs] -> String serializeMultipleMinMaxISpecs minmaxes = intercalate [iSpecsSeparator] $ foldr serialpair [] minmaxes where serialpair (MinMaxISpecs minspec maxspec) acc = serializeISpec minspec : serializeISpec maxspec : acc -- | Generate policy data from a given policy object. serializeIPolicy :: String -> IPolicy -> String serializeIPolicy owner ipol = let IPolicy minmax stdspec dts vcpu_ratio spindle_ratio = ipol strings = [ owner , serializeISpec stdspec , serializeMultipleMinMaxISpecs minmax , serializeDiskTemplates dts , show vcpu_ratio , show spindle_ratio ] in intercalate "|" strings -- | Generates the entire ipolicy section from the cluster and group -- objects. serializeAllIPolicies :: IPolicy -> Group.List -> String serializeAllIPolicies cpol gl = let groups = Container.elems gl allpolicies = ("", cpol) : map (\g -> (Group.name g, Group.iPolicy g)) groups strings = map (uncurry serializeIPolicy) allpolicies in unlines strings -- | Generate complete cluster data from node and instance lists. serializeCluster :: ClusterData -> String serializeCluster (ClusterData gl nl il ctags cpol) = let gdata = serializeGroups gl ndata = serializeNodes gl nl idata = serializeInstances nl il pdata = serializeAllIPolicies cpol gl -- note: not using 'unlines' as that adds too many newlines in intercalate "\n" [gdata, ndata, idata, unlines ctags, pdata] -- * Parsing functions -- | Load a group from a field list. loadGroup :: (MonadFail m) => [String] -> m (String, Group.Group) -- ^ The result, a tuple of group -- UUID and group object loadGroup (name:gid:apol:tags:nets:_) = do xapol <- allocPolicyFromRaw apol let xtags = commaSplit tags let xnets = commaSplit nets return (gid, Group.create name gid xapol xnets defIPolicy xtags) loadGroup [name, gid, apol, tags] = loadGroup [name, gid, apol, tags, ""] loadGroup s = fail $ "Invalid/incomplete group data: '" ++ show s ++ "'" -- | Load a node from a field list. loadNode :: (MonadFail m) => NameAssoc -- ^ Association list with current groups -> [String] -- ^ Input data as a list of fields -> m (String, Node.Node) -- ^ The result, a tuple o node name -- and node object loadNode ktg (name:tm:nm:fm:td:fd:tc:fo:gu:spindles:tags: excl_stor:free_spindles:nos_cpu:cpu_speed:_) = do gdx <- lookupGroup ktg name gu new_node <- if "?" `elem` [tm,nm,fm,td,fd,tc] then return $ Node.create name 0 0 0 0 0 0 0 True 0 0 gdx False else do let vtags = commaSplit tags vtm <- tryRead name tm vnm <- tryRead name nm vfm <- tryRead name fm vtd <- tryRead name td vfd <- tryRead name fd vtc <- tryRead name tc vnc <- tryRead name nos_cpu vspindles <- tryRead name spindles vcpu_speed <- tryRead name cpu_speed vfree_spindles <- tryRead name free_spindles vexcl_stor <- case excl_stor of "Y" -> return True "N" -> return False _ -> fail $ "Invalid exclusive_storage value for node '" ++ name ++ "': " ++ excl_stor return . flip Node.setMaster (fo == "M") . flip Node.setNodeTags vtags . flip Node.setCpuSpeed vcpu_speed $ Node.create name vtm vnm vfm vtd vfd vtc vnc (fo == "Y") vspindles vfree_spindles gdx vexcl_stor return (name, new_node) loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu] = loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, "1"] loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles] = loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles, ""] loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles, tags] = loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles, tags, "N"] loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles, tags, excl_stor] = loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles, tags, excl_stor, "0"] loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles, tags, excl_stor, free_spindles] = loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles, tags, excl_stor, free_spindles, "1"] loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles, tags, excl_stor, free_spindles, nos_cpu] = loadNode ktg [name, tm, nm, fm, td, fd, tc, fo, gu, spindles, tags, excl_stor, free_spindles, nos_cpu, "1.0"] loadNode _ s = fail $ "Invalid/incomplete node data: '" ++ show s ++ "'" -- | Load an instance from a field list. loadInst :: NameAssoc -- ^ Association list with the current nodes -> [String] -- ^ Input data as a list of fields -> Result (String, Instance.Instance) -- ^ A tuple of -- instance name and -- the instance object loadInst ktn (name:mem:dsk:vcpus:status:auto_bal:pnode:snode :dt:tags:su:spindles:forthcoming_yn:_) = do pidx <- lookupNode ktn name pnode sidx <- if null snode then return Node.noSecondary else lookupNode ktn name snode vmem <- tryRead name mem dsize <- tryRead name dsk vvcpus <- tryRead name vcpus vstatus <- instanceStatusFromRaw status auto_balance <- case auto_bal of "Y" -> return True "N" -> return False _ -> fail $ "Invalid auto_balance value '" ++ auto_bal ++ "' for instance " ++ name disk_template <- annotateResult ("Instance " ++ name) (diskTemplateFromRaw dt) spindle_use <- tryRead name su vspindles <- case spindles of "-" -> return Nothing _ -> liftM Just (tryRead name spindles) forthcoming <- case forthcoming_yn of "Y" -> return True "N" -> return False x -> fail $ "Invalid forthcoming value '" ++ x ++ "' for instance " ++ name let disk = Instance.Disk dsize vspindles let vtags = commaSplit tags newinst = Instance.create name vmem dsize [disk] vvcpus vstatus vtags auto_balance pidx sidx disk_template spindle_use [] forthcoming when (Instance.hasSecondary newinst && sidx == pidx) . fail $ "Instance " ++ name ++ " has same primary and secondary node - " ++ pnode return (name, newinst) loadInst ktn [ name, mem, dsk, vcpus, status, auto_bal, pnode, snode , dt, tags, su, spindles ] = loadInst ktn [ name, mem, dsk, vcpus, status, auto_bal, pnode, snode , dt, tags, su, spindles, "N" ] -- older versions were not -- forthcoming loadInst ktn [ name, mem, dsk, vcpus, status, auto_bal, pnode, snode , dt, tags ] = loadInst ktn [ name, mem, dsk, vcpus, status, auto_bal, pnode, snode, dt, tags, "1" ] loadInst ktn [ name, mem, dsk, vcpus, status, auto_bal, pnode, snode , dt, tags, su ] = loadInst ktn [ name, mem, dsk, vcpus, status, auto_bal, pnode, snode, dt , tags, su, "-" ] loadInst _ s = fail $ "Invalid/incomplete instance data: '" ++ show s ++ "'" -- | Loads a spec from a field list. loadISpec :: String -> [String] -> Result ISpec loadISpec owner (mem_s:cpu_c:dsk_s:dsk_c:nic_c:su:_) = do xmem_s <- tryRead (owner ++ "/memsize") mem_s xcpu_c <- tryRead (owner ++ "/cpucount") cpu_c xdsk_s <- tryRead (owner ++ "/disksize") dsk_s xdsk_c <- tryRead (owner ++ "/diskcount") dsk_c xnic_c <- tryRead (owner ++ "/niccount") nic_c xsu <- tryRead (owner ++ "/spindleuse") su return $ ISpec xmem_s xcpu_c xdsk_s xdsk_c xnic_c xsu loadISpec owner s = fail $ "Invalid ispec data for " ++ owner ++ ": " ++ show s -- | Load a single min/max ISpec pair loadMinMaxISpecs :: String -> String -> String -> Result MinMaxISpecs loadMinMaxISpecs owner minspec maxspec = do xminspec <- loadISpec (owner ++ "/minspec") (commaSplit minspec) xmaxspec <- loadISpec (owner ++ "/maxspec") (commaSplit maxspec) return $ MinMaxISpecs xminspec xmaxspec -- | Break a list of ispecs strings into a list of (min/max) ispecs pairs breakISpecsPairs :: String -> [String] -> Result [(String, String)] breakISpecsPairs _ [] = return [] breakISpecsPairs owner (x:y:xs) = do rest <- breakISpecsPairs owner xs return $ (x, y) : rest breakISpecsPairs owner _ = fail $ "Odd number of min/max specs for " ++ owner -- | Load a list of min/max ispecs pairs loadMultipleMinMaxISpecs :: String -> [String] -> Result [MinMaxISpecs] loadMultipleMinMaxISpecs owner ispecs = do pairs <- breakISpecsPairs owner ispecs mapM (uncurry $ loadMinMaxISpecs owner) pairs -- | Loads an ipolicy from a field list. loadIPolicy :: [String] -> Result (String, IPolicy) loadIPolicy (owner:stdspec:minmaxspecs:dtemplates: vcpu_ratio:spindle_ratio:_) = do xstdspec <- loadISpec (owner ++ "/stdspec") (commaSplit stdspec) xminmaxspecs <- loadMultipleMinMaxISpecs owner $ sepSplit iSpecsSeparator minmaxspecs xdts <- mapM diskTemplateFromRaw $ commaSplit dtemplates xvcpu_ratio <- tryRead (owner ++ "/vcpu_ratio") vcpu_ratio xspindle_ratio <- tryRead (owner ++ "/spindle_ratio") spindle_ratio return (owner, IPolicy xminmaxspecs xstdspec xdts xvcpu_ratio xspindle_ratio) loadIPolicy s = fail $ "Invalid ipolicy data: '" ++ show s ++ "'" loadOnePolicy :: (IPolicy, Group.List) -> String -> Result (IPolicy, Group.List) loadOnePolicy (cpol, gl) line = do (owner, ipol) <- loadIPolicy (sepSplit '|' line) case owner of "" -> return (ipol, gl) -- this is a cluster policy (no owner) _ -> do grp <- Container.findByName gl owner let grp' = grp { Group.iPolicy = ipol } gl' = Container.add (Group.idx grp') grp' gl return (cpol, gl') -- | Loads all policies from the policy section loadAllIPolicies :: Group.List -> [String] -> Result (IPolicy, Group.List) loadAllIPolicies gl = foldM loadOnePolicy (defIPolicy, gl) -- | Convert newline and delimiter-separated text. -- -- This function converts a text in tabular format as generated by -- @gnt-instance list@ and @gnt-node list@ to a list of objects using -- a supplied conversion function. loadTabular :: (MonadFail m, Element a) => [String] -- ^ Input data, as a list of lines -> ([String] -> m (String, a)) -- ^ Conversion function -> m ( NameAssoc , Container.Container a ) -- ^ A tuple of an -- association list (name -- to object) and a set as -- used in -- "Ganeti.HTools.Container" loadTabular lines_data convert_fn = do let rows = map (sepSplit '|') lines_data kerows <- mapM convert_fn rows return $ assignIndices kerows -- | Load the cluser data from disk. -- -- This is an alias to 'readFile' just for consistency with the other -- modules. readData :: String -- ^ Path to the text file -> IO String -- ^ Contents of the file readData = readFile -- | Builds the cluster data from text input. parseData :: String -- ^ Text data -> Result ClusterData parseData fdata = do let flines = lines fdata (glines, nlines, ilines, ctags, pollines) <- case sepSplit "" flines of -- Ignore all additional fields a:b:c:d:e:_ -> Ok (a, b, c, d, e) [a, b, c, d] -> Ok (a, b, c, d, []) xs -> Bad $ printf "Invalid format of the input file: %d sections\ \ instead of 4 or more" (length xs) {- group file: name uuid alloc_policy -} (ktg, gl) <- loadTabular glines loadGroup {- node file: name t_mem n_mem f_mem t_disk f_disk t_cpu offline grp_uuid spindles tags -} (ktn, nl) <- loadTabular nlines (loadNode ktg) {- instance file: name mem disk vcpus status auto_bal pnode snode disk_template tags spindle_use -} (_, il) <- loadTabular ilines (loadInst ktn) {- the tags are simply line-based, no processing needed -} {- process policies -} (cpol, gl') <- loadAllIPolicies gl pollines return (ClusterData gl' nl il ctags cpol) -- | Top level function for data loading. loadData :: String -- ^ Path to the text file -> IO (Result ClusterData) loadData = fmap parseData . readData ganeti-3.1.0~rc2/src/Ganeti/HTools/CLI.hs000064400000000000000000000771011476477700300177630ustar00rootroot00000000000000{-| Implementation of command-line functions. This module holds the common command-line related functions for the binaries, separated into this module since "Ganeti.Utils" is used in many other places and this is more IO oriented. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.CLI ( Options(..) , OptType , defaultOptions , Ganeti.HTools.CLI.parseOpts , parseOptsInner , parseYesNo , parseISpecString , shTemplate , maybeSaveCommands , maybePrintNodes , maybePrintInsts , maybeShowWarnings , printKeys , printFinal , setNodeStatus -- * The options , oDataFile , oDiskMoves , oDiskTemplate , oDryRun , oSpindleUse , oDynuFile , oMonD , oMonDDataFile , oMonDXen , oEvacMode , oMonDExitMissing , oFirstJobGroup , oRestrictedMigrate , oExInst , oExTags , oExecJobs , oForce , oFullEvacuation , oGroup , oIAllocSrc , oIgnoreDyn , oIgnoreNonRedundant , oIgnoreSoftErrors , oIndependentGroups , oAcceptExisting , oInstMoves , oJobDelay , genOLuxiSocket , oLuxiSocket , oMachineReadable , oMaxCpu , oMaxSolLength , oMinDisk , oMinGain , oMinGainLim , oMinResources , oMinScore , oNoHeaders , oNoSimulation , oNodeSim , oNodeTags , oOfflineMaintenance , oOfflineNode , oOneStepOnly , oOutputDir , oPrintCommands , oPrintInsts , oPrintMoves , oPrintNodes , oQuiet , oRapiMaster , oReason , oRestrictToNodes , oSaveCluster , oSelInst , oShowHelp , oShowVer , oShowComp , oSkipNonRedundant , oStaticKvmNodeMemory , oStdSpec , oTargetResources , oTieredSpec , oVerbose , oPriority , oNoCapacityChecks , genericOpts ) where import Control.Monad import Data.Char (toUpper) import Data.Maybe (fromMaybe) import System.Console.GetOpt import System.IO import Text.Printf (printf) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Node as Node import qualified Ganeti.Path as Path import Ganeti.HTools.Types import Ganeti.BasicTypes import Ganeti.Common as Common import Ganeti.Types import Ganeti.Utils -- * Data types -- | Command line options structure. data Options = Options { optDataFile :: Maybe FilePath -- ^ Path to the cluster data file , optDiskMoves :: Bool -- ^ Allow disk moves , optInstMoves :: Bool -- ^ Allow instance moves , optDiskTemplate :: Maybe DiskTemplate -- ^ Override for the disk template , optSpindleUse :: Maybe Int -- ^ Override for the spindle usage , optDynuFile :: Maybe FilePath -- ^ Optional file with dynamic use data , optIgnoreDynu :: Bool -- ^ Do not use dynamic use data , optIgnoreSoftErrors :: Bool -- ^ Ignore soft errors in balancing moves , optIndependentGroups :: Bool -- ^ consider groups independently , optAcceptExisting :: Bool -- ^ accept existing N+1 violations , optMonD :: Bool -- ^ Query MonDs , optMonDFile :: Maybe FilePath -- ^ Optional file with data provided -- by MonDs , optMonDXen :: Bool -- ^ Should Xen-specific collectors be -- considered (only if MonD is queried) , optMonDExitMissing :: Bool -- ^ If the program should exit on missing -- MonD data , optEvacMode :: Bool -- ^ Enable evacuation mode , optRestrictedMigrate :: Bool -- ^ Disallow replace-primary moves , optExInst :: [String] -- ^ Instances to be excluded , optExTags :: Maybe [String] -- ^ Tags to use for exclusion , optExecJobs :: Bool -- ^ Execute the commands via Luxi , optDryRun :: Bool -- ^ Only do a dry run , optFirstJobGroup :: Bool -- ^ Only execute the first group of jobs , optForce :: Bool -- ^ Force the execution , optFullEvacuation :: Bool -- ^ Fully evacuate nodes to be rebooted , optGroup :: Maybe GroupID -- ^ The UUID of the group to process , optIAllocSrc :: Maybe FilePath -- ^ The iallocation spec , optIgnoreNonRedundant :: Bool -- ^ Ignore non-redundant instances , optSelInst :: [String] -- ^ Instances to be excluded , optLuxi :: Maybe FilePath -- ^ Collect data from Luxi , optJobDelay :: Double -- ^ Delay before executing first job , optMachineReadable :: Bool -- ^ Output machine-readable format , optMaster :: String -- ^ Collect data from RAPI , optMaxLength :: Int -- ^ Stop after this many steps , optMcpu :: Maybe Double -- ^ Override max cpu ratio for nodes , optMdsk :: Double -- ^ Max disk usage ratio for nodes , optMinGain :: Score -- ^ Min gain we aim for in a step , optMinGainLim :: Score -- ^ Limit below which we apply mingain , optMinResources :: Double -- ^ Minimal resources for hsqueeze , optMinScore :: Score -- ^ The minimum score we aim for , optNoHeaders :: Bool -- ^ Do not show a header line , optNoSimulation :: Bool -- ^ Skip the rebalancing dry-run , optNodeSim :: [String] -- ^ Cluster simulation mode , optNodeTags :: Maybe [String] -- ^ List of node tags to restrict to , optOffline :: [String] -- ^ Names of offline nodes , optRestrictToNodes :: Maybe [String] -- ^ if not Nothing, restrict -- allocation to those nodes , optOfflineMaintenance :: Bool -- ^ Pretend all instances are offline , optOneStepOnly :: Bool -- ^ Only do the first step , optOutPath :: FilePath -- ^ Path to the output directory , optPrintMoves :: Bool -- ^ Whether to show the instance moves , optReason :: Maybe String -- ^ The reason to be passed when -- submitting jobs , optSaveCluster :: Maybe FilePath -- ^ Save cluster state to this file , optShowCmds :: Maybe FilePath -- ^ Whether to show the command list , optShowHelp :: Bool -- ^ Just show the help , optShowComp :: Bool -- ^ Just show the completion info , optShowInsts :: Bool -- ^ Whether to show the instance map , optShowNodes :: Maybe [String] -- ^ Whether to show node status , optShowVer :: Bool -- ^ Just show the program version , optSkipNonRedundant :: Bool -- ^ Skip nodes with non-redundant instance , optStaticKvmNodeMemory :: Int -- ^ Use static value for node memory -- ^ on KVM , optStdSpec :: Maybe RSpec -- ^ Requested standard specs , optTargetResources :: Double -- ^ Target resources for squeezing , optTestCount :: Maybe Int -- ^ Optional test count override , optTieredSpec :: Maybe RSpec -- ^ Requested specs for tiered mode , optReplay :: Maybe String -- ^ Unittests: RNG state , optVerbose :: Int -- ^ Verbosity level , optPriority :: Maybe OpSubmitPriority -- ^ OpCode submit priority , optCapacity :: Bool -- ^ Also do capacity-related checks } deriving Show -- | Default values for the command line options. defaultOptions :: Options defaultOptions = Options { optDataFile = Nothing , optDiskMoves = True , optInstMoves = True , optIndependentGroups = False , optAcceptExisting = False , optDiskTemplate = Nothing , optSpindleUse = Nothing , optIgnoreDynu = False , optIgnoreSoftErrors = False , optDynuFile = Nothing , optMonD = False , optMonDFile = Nothing , optMonDXen = False , optMonDExitMissing = False , optEvacMode = False , optRestrictedMigrate = False , optExInst = [] , optExTags = Nothing , optExecJobs = False , optDryRun = False , optFirstJobGroup = False , optForce = False , optFullEvacuation = False , optGroup = Nothing , optIAllocSrc = Nothing , optIgnoreNonRedundant = False , optSelInst = [] , optLuxi = Nothing , optJobDelay = 10 , optMachineReadable = False , optMaster = "" , optMaxLength = -1 , optMcpu = Nothing , optMdsk = defReservedDiskRatio , optMinGain = 1e-2 , optMinGainLim = 1e-1 , optMinResources = 2.0 , optMinScore = 1e-9 , optNoHeaders = False , optNoSimulation = False , optNodeSim = [] , optNodeTags = Nothing , optSkipNonRedundant = False , optStaticKvmNodeMemory = 4096 , optOffline = [] , optRestrictToNodes = Nothing , optOfflineMaintenance = False , optOneStepOnly = False , optOutPath = "." , optPrintMoves = False , optReason = Nothing , optSaveCluster = Nothing , optShowCmds = Nothing , optShowHelp = False , optShowComp = False , optShowInsts = False , optShowNodes = Nothing , optShowVer = False , optStdSpec = Nothing , optTargetResources = 2.0 , optTestCount = Nothing , optTieredSpec = Nothing , optReplay = Nothing , optVerbose = 1 , optPriority = Nothing , optCapacity = True } -- | Abbreviation for the option type. type OptType = GenericOptType Options instance StandardOptions Options where helpRequested = optShowHelp verRequested = optShowVer compRequested = optShowComp requestHelp o = o { optShowHelp = True } requestVer o = o { optShowVer = True } requestComp o = o { optShowComp = True } -- * Helper functions parseISpecString :: String -> String -> Result RSpec parseISpecString descr inp = do let sp = sepSplit ',' inp err = Bad ("Invalid " ++ descr ++ " specification: '" ++ inp ++ "', expected disk,ram,cpu") when (length sp < 3 || length sp > 4) err prs <- mapM (\(fn, val) -> fn val) $ zip [ annotateResult (descr ++ " specs disk") . parseUnit , annotateResult (descr ++ " specs memory") . parseUnit , tryRead (descr ++ " specs cpus") , tryRead (descr ++ " specs spindles") ] sp case prs of {- Spindles are optional, so that they are not needed when exclusive storage is disabled. When exclusive storage is disabled, spindles are ignored, so the actual value doesn't matter. We use 1 as a default so that in case someone forgets and exclusive storage is enabled, we don't run into weird situations. -} [dsk, ram, cpu] -> return $ RSpec cpu ram dsk 1 [dsk, ram, cpu, spn] -> return $ RSpec cpu ram dsk spn _ -> err -- | Disk template choices. optComplDiskTemplate :: OptCompletion optComplDiskTemplate = OptComplChoices $ map diskTemplateToRaw [minBound..maxBound] -- * Command line options oDataFile :: OptType oDataFile = (Option "t" ["text-data"] (ReqArg (\ f o -> Ok o { optDataFile = Just f }) "FILE") "the cluster data FILE", OptComplFile) oDiskMoves :: OptType oDiskMoves = (Option "" ["no-disk-moves"] (NoArg (\ opts -> Ok opts { optDiskMoves = False})) "disallow disk moves from the list of allowed instance changes,\ \ thus allowing only the 'cheap' failover/migrate operations", OptComplNone) oMonD :: OptType oMonD = (Option "" ["mond"] (OptArg (\ f opts -> do flag <- parseYesNo True f return $ opts { optMonD = flag }) "CHOICE") "pass either 'yes' or 'no' to query all monDs", optComplYesNo) oMonDDataFile :: OptType oMonDDataFile = (Option "" ["mond-data"] (ReqArg (\ f opts -> Ok opts { optMonDFile = Just f }) "FILE") "Import data provided by MonDs from the given FILE", OptComplFile) oMonDXen :: OptType oMonDXen = (Option "" ["mond-xen"] (NoArg (\ opts -> Ok opts { optMonDXen = True })) "also consider xen-specific collectors in MonD queries", OptComplNone) oMonDExitMissing :: OptType oMonDExitMissing = (Option "" ["exit-on-missing-mond-data"] (NoArg (\ opts -> Ok opts { optMonDExitMissing = True })) "abort if the data available from the monitoring daemons is incomplete", OptComplNone) oDiskTemplate :: OptType oDiskTemplate = (Option "" ["disk-template"] (reqWithConversion diskTemplateFromRaw (\dt opts -> Ok opts { optDiskTemplate = Just dt }) "TEMPLATE") "select the desired disk template", optComplDiskTemplate) oSpindleUse :: OptType oSpindleUse = (Option "" ["spindle-use"] (reqWithConversion (tryRead "parsing spindle-use") (\su opts -> do when (su < 0) $ fail "Invalid value of the spindle-use (expected >= 0)" return $ opts { optSpindleUse = Just su }) "SPINDLES") "select how many virtual spindle instances use\ \ [default read from cluster]", OptComplFloat) oSelInst :: OptType oSelInst = (Option "" ["select-instances"] (ReqArg (\ f opts -> Ok opts { optSelInst = sepSplit ',' f }) "INSTS") "only select given instances for any moves", OptComplManyInstances) oInstMoves :: OptType oInstMoves = (Option "" ["no-instance-moves"] (NoArg (\ opts -> Ok opts { optInstMoves = False})) "disallow instance (primary node) moves from the list of allowed,\ \ instance changes, thus allowing only slower, but sometimes\ \ safer, drbd secondary changes", OptComplNone) oDynuFile :: OptType oDynuFile = (Option "U" ["dynu-file"] (ReqArg (\ f opts -> Ok opts { optDynuFile = Just f }) "FILE") "Import dynamic utilisation data from the given FILE", OptComplFile) oIgnoreDyn :: OptType oIgnoreDyn = (Option "" ["ignore-dynu"] (NoArg (\ opts -> Ok opts {optIgnoreDynu = True})) "Ignore any dynamic utilisation information", OptComplNone) oIgnoreSoftErrors :: OptType oIgnoreSoftErrors = (Option "" ["ignore-soft-errors"] (NoArg (\ opts -> Ok opts {optIgnoreSoftErrors = True})) "Ignore any soft restrictions in balancing", OptComplNone) oIndependentGroups :: OptType oIndependentGroups = (Option "" ["independent-groups"] (NoArg (\ opts -> Ok opts {optIndependentGroups = True})) "Consider groups independently", OptComplNone) oAcceptExisting :: OptType oAcceptExisting = (Option "" ["accept-existing-errors"] (NoArg (\ opts -> Ok opts {optAcceptExisting = True})) "Accept existing N+1 violations; just don't add new ones", OptComplNone) oEvacMode :: OptType oEvacMode = (Option "E" ["evac-mode"] (NoArg (\opts -> Ok opts { optEvacMode = True })) "enable evacuation mode, where the algorithm only moves\ \ instances away from offline and drained nodes", OptComplNone) oRestrictedMigrate :: OptType oRestrictedMigrate = (Option "" ["restricted-migration"] (NoArg (\opts -> Ok opts { optRestrictedMigrate = True })) "disallow replace-primary moves (aka frf-moves); in evacuation mode, this\ \ will ensure that the only migrations are off the drained nodes", OptComplNone) oExInst :: OptType oExInst = (Option "" ["exclude-instances"] (ReqArg (\ f opts -> Ok opts { optExInst = sepSplit ',' f }) "INSTS") "exclude given instances from any moves", OptComplManyInstances) oExTags :: OptType oExTags = (Option "" ["exclusion-tags"] (ReqArg (\ f opts -> Ok opts { optExTags = Just $ sepSplit ',' f }) "TAG,...") "Enable instance exclusion based on given tag prefix", OptComplString) oExecJobs :: OptType oExecJobs = (Option "X" ["exec"] (NoArg (\ opts -> Ok opts { optExecJobs = True})) "execute the suggested moves via Luxi (only available when using\ \ it for data gathering)", OptComplNone) oDryRun :: OptType oDryRun = (Option "" ["dry-run"] (NoArg (\ opts -> Ok opts { optDryRun = True})) "do not execute any commands and just report what would be done", OptComplNone) oReason :: OptType oReason = (Option "" ["reason"] (ReqArg (\ f opts -> Ok opts { optReason = Just f }) "REASON") "The reason to pass to the submitted jobs", OptComplNone) oFirstJobGroup :: OptType oFirstJobGroup = (Option "" ["first-job-group"] (NoArg (\ opts -> Ok opts {optFirstJobGroup = True})) "only execute the first group of jobs", OptComplNone) oForce :: OptType oForce = (Option "f" ["force"] (NoArg (\ opts -> Ok opts {optForce = True})) "force the execution of this program, even if warnings would\ \ otherwise prevent it", OptComplNone) oFullEvacuation :: OptType oFullEvacuation = (Option "" ["full-evacuation"] (NoArg (\ opts -> Ok opts { optFullEvacuation = True})) "fully evacuate the nodes to be rebooted", OptComplNone) oGroup :: OptType oGroup = (Option "G" ["group"] (ReqArg (\ f o -> Ok o { optGroup = Just f }) "ID") "the target node group (name or UUID)", OptComplOneGroup) oIAllocSrc :: OptType oIAllocSrc = (Option "I" ["ialloc-src"] (ReqArg (\ f opts -> Ok opts { optIAllocSrc = Just f }) "FILE") "Specify an iallocator spec as the cluster data source", OptComplFile) oIgnoreNonRedundant :: OptType oIgnoreNonRedundant = (Option "" ["ignore-non-redundant"] (NoArg (\ opts -> Ok opts { optIgnoreNonRedundant = True })) "Pretend that there are no non-redundant instances in the cluster", OptComplNone) oJobDelay :: OptType oJobDelay = (Option "" ["job-delay"] (reqWithConversion (tryRead "job delay") (\d opts -> Ok opts { optJobDelay = d }) "SECONDS") "insert this much delay before the execution of repair jobs\ \ to allow the tool to continue processing instances", OptComplFloat) genOLuxiSocket :: String -> OptType genOLuxiSocket defSocket = (Option "L" ["luxi"] (OptArg ((\ f opts -> Ok opts { optLuxi = Just f }) . fromMaybe defSocket) "SOCKET") ("collect data via Luxi, optionally using the given SOCKET path [" ++ defSocket ++ "]"), OptComplFile) oLuxiSocket :: IO OptType oLuxiSocket = liftM genOLuxiSocket Path.defaultQuerySocket oMachineReadable :: OptType oMachineReadable = (Option "" ["machine-readable"] (OptArg (\ f opts -> do flag <- parseYesNo True f return $ opts { optMachineReadable = flag }) "CHOICE") "enable machine readable output (pass either 'yes' or 'no' to\ \ explicitly control the flag, or without an argument defaults to\ \ yes)", optComplYesNo) oMaxCpu :: OptType oMaxCpu = (Option "" ["max-cpu"] (reqWithConversion (tryRead "parsing max-cpu") (\mcpu opts -> do when (mcpu <= 0) $ fail "Invalid value of the max-cpu ratio, expected >0" return $ opts { optMcpu = Just mcpu }) "RATIO") "maximum virtual-to-physical cpu ratio for nodes (from 0\ \ upwards) [default read from cluster]", OptComplFloat) oMaxSolLength :: OptType oMaxSolLength = (Option "l" ["max-length"] (reqWithConversion (tryRead "max solution length") (\i opts -> Ok opts { optMaxLength = i }) "N") "cap the solution at this many balancing or allocation\ \ rounds (useful for very unbalanced clusters or empty\ \ clusters)", OptComplInteger) oMinDisk :: OptType oMinDisk = (Option "" ["min-disk"] (reqWithConversion (tryRead "min free disk space") (\n opts -> Ok opts { optMdsk = n }) "RATIO") "minimum free disk space for nodes (between 0 and 1) [0]", OptComplFloat) oMinGain :: OptType oMinGain = (Option "g" ["min-gain"] (reqWithConversion (tryRead "min gain") (\g opts -> Ok opts { optMinGain = g }) "DELTA") "minimum gain to aim for in a balancing step before giving up", OptComplFloat) oMinGainLim :: OptType oMinGainLim = (Option "" ["min-gain-limit"] (reqWithConversion (tryRead "min gain limit") (\g opts -> Ok opts { optMinGainLim = g }) "SCORE") "minimum cluster score for which we start checking the min-gain", OptComplFloat) oMinResources :: OptType oMinResources = (Option "" ["minimal-resources"] (reqWithConversion (tryRead "minimal resources") (\d opts -> Ok opts { optMinResources = d}) "FACTOR") "minimal resources to be present on each in multiples of\ \ the standard allocation for not onlining standby nodes", OptComplFloat) oMinScore :: OptType oMinScore = (Option "e" ["min-score"] (reqWithConversion (tryRead "min score") (\e opts -> Ok opts { optMinScore = e }) "EPSILON") "mininum excess to the N+1 limit to aim for", OptComplFloat) oNoHeaders :: OptType oNoHeaders = (Option "" ["no-headers"] (NoArg (\ opts -> Ok opts { optNoHeaders = True })) "do not show a header line", OptComplNone) oNoSimulation :: OptType oNoSimulation = (Option "" ["no-simulation"] (NoArg (\opts -> Ok opts {optNoSimulation = True})) "do not perform rebalancing simulation", OptComplNone) oNodeSim :: OptType oNodeSim = (Option "" ["simulate"] (ReqArg (\ f o -> Ok o { optNodeSim = f:optNodeSim o }) "SPEC") "simulate an empty cluster, given as\ \ 'alloc_policy,num_nodes,disk,ram,cpu'", OptComplString) oNodeTags :: OptType oNodeTags = (Option "" ["node-tags"] (ReqArg (\ f opts -> Ok opts { optNodeTags = Just $ sepSplit ',' f }) "TAG,...") "Restrict to nodes with the given tags", OptComplString) oOfflineMaintenance :: OptType oOfflineMaintenance = (Option "" ["offline-maintenance"] (NoArg (\ opts -> Ok opts {optOfflineMaintenance = True})) "Schedule offline maintenance, i.e., pretend that all instance are\ \ offline.", OptComplNone) oOfflineNode :: OptType oOfflineNode = (Option "O" ["offline"] (ReqArg (\ n o -> Ok o { optOffline = n:optOffline o }) "NODE") "set node as offline", OptComplOneNode) oRestrictToNodes :: OptType oRestrictToNodes = (Option "" ["restrict-allocation-to"] (ReqArg (\ ns o -> Ok o { optRestrictToNodes = Just $ sepSplit ',' ns }) "NODE,...") "Restrict allocations to the given set of nodes", OptComplManyNodes) oOneStepOnly :: OptType oOneStepOnly = (Option "" ["one-step-only"] (NoArg (\ opts -> Ok opts {optOneStepOnly = True})) "Only do the first step", OptComplNone) oOutputDir :: OptType oOutputDir = (Option "d" ["output-dir"] (ReqArg (\ d opts -> Ok opts { optOutPath = d }) "PATH") "directory in which to write output files", OptComplDir) oPrintCommands :: OptType oPrintCommands = (Option "C" ["print-commands"] (OptArg ((\ f opts -> Ok opts { optShowCmds = Just f }) . fromMaybe "-") "FILE") "print the ganeti command list for reaching the solution,\ \ if an argument is passed then write the commands to a\ \ file named as such", OptComplNone) oPrintInsts :: OptType oPrintInsts = (Option "" ["print-instances"] (NoArg (\ opts -> Ok opts { optShowInsts = True })) "print the final instance map", OptComplNone) oPrintMoves :: OptType oPrintMoves = (Option "" ["print-moves"] (NoArg (\ opts -> Ok opts { optPrintMoves = True })) "print the moves of the instances", OptComplNone) oPrintNodes :: OptType oPrintNodes = (Option "p" ["print-nodes"] (OptArg ((\ f opts -> let (prefix, realf) = case f of '+':rest -> (["+"], rest) _ -> ([], f) splitted = prefix ++ sepSplit ',' realf in Ok opts { optShowNodes = Just splitted }) . fromMaybe []) "FIELDS") "print the final node list", OptComplNone) oQuiet :: OptType oQuiet = (Option "q" ["quiet"] (NoArg (\ opts -> Ok opts { optVerbose = optVerbose opts - 1 })) "decrease the verbosity level", OptComplNone) oRapiMaster :: OptType oRapiMaster = (Option "m" ["master"] (ReqArg (\ m opts -> Ok opts { optMaster = m }) "ADDRESS") "collect data via RAPI at the given ADDRESS", OptComplHost) oSaveCluster :: OptType oSaveCluster = (Option "S" ["save"] (ReqArg (\ f opts -> Ok opts { optSaveCluster = Just f }) "FILE") "Save cluster state at the end of the processing to FILE", OptComplNone) oSkipNonRedundant :: OptType oSkipNonRedundant = (Option "" ["skip-non-redundant"] (NoArg (\ opts -> Ok opts { optSkipNonRedundant = True })) "Skip nodes that host a non-redundant instance", OptComplNone) oStaticKvmNodeMemory :: OptType oStaticKvmNodeMemory = (Option "" ["static-kvm-node-memory"] (reqWithConversion (tryRead "static node memory") (\i opts -> Ok opts { optStaticKvmNodeMemory = i }) "N") "use static node memory [in MB] on KVM instead of value reported by \ \hypervisor. Use 0 to take value reported from hypervisor.", OptComplInteger) oStdSpec :: OptType oStdSpec = (Option "" ["standard-alloc"] (ReqArg (\ inp opts -> do tspec <- parseISpecString "standard" inp return $ opts { optStdSpec = Just tspec } ) "STDSPEC") "enable standard specs allocation, given as 'disk,ram,cpu'", OptComplString) oTargetResources :: OptType oTargetResources = (Option "" ["target-resources"] (reqWithConversion (tryRead "target resources") (\d opts -> Ok opts { optTargetResources = d}) "FACTOR") "target resources to be left on each node after squeezing in\ \ multiples of the standard allocation", OptComplFloat) oTieredSpec :: OptType oTieredSpec = (Option "" ["tiered-alloc"] (ReqArg (\ inp opts -> do tspec <- parseISpecString "tiered" inp return $ opts { optTieredSpec = Just tspec } ) "TSPEC") "enable tiered specs allocation, given as 'disk,ram,cpu'", OptComplString) oVerbose :: OptType oVerbose = (Option "v" ["verbose"] (NoArg (\ opts -> Ok opts { optVerbose = optVerbose opts + 1 })) "increase the verbosity level", OptComplNone) oPriority :: OptType oPriority = (Option "" ["priority"] (ReqArg (\ inp opts -> do prio <- parseSubmitPriority inp Ok opts { optPriority = Just prio }) "PRIO") "set the priority of submitted jobs", OptComplChoices (map fmtSubmitPriority [minBound..maxBound])) oNoCapacityChecks :: OptType oNoCapacityChecks = (Option "" ["no-capacity-checks"] (NoArg (\ opts -> Ok opts { optCapacity = False})) "disable capacity checks (like global N+1 redundancy)", OptComplNone) -- | Generic options. genericOpts :: [GenericOptType Options] genericOpts = [ oShowVer , oShowHelp , oShowComp ] -- * Functions -- | Wrapper over 'Common.parseOpts' with our custom options. parseOpts :: [String] -- ^ The command line arguments -> String -- ^ The program name -> [OptType] -- ^ The supported command line options -> [ArgCompletion] -- ^ The supported command line arguments -> IO (Options, [String]) -- ^ The resulting options and leftover -- arguments parseOpts = Common.parseOpts defaultOptions -- | A shell script template for autogenerated scripts. shTemplate :: String shTemplate = printf "#!/bin/sh\n\n\ \# Auto-generated script for executing cluster rebalancing\n\n\ \# To stop, touch the file /tmp/stop-htools\n\n\ \set -e\n\n\ \check() {\n\ \ if [ -f /tmp/stop-htools ]; then\n\ \ echo 'Stop requested, exiting'\n\ \ exit 0\n\ \ fi\n\ \}\n\n" -- | Optionally show or save a list of commands maybeSaveCommands :: String -- ^ Informal description -> Options -> String -- ^ commands -> IO () maybeSaveCommands msg opts cmds = case optShowCmds opts of Nothing -> return () Just "-" -> do putStrLn "" putStrLn msg putStr . unlines . map (" " ++) . filter (/= " check") . lines $ cmds Just out_path -> do writeFile out_path (shTemplate ++ cmds) printf "The commands have been written to file '%s'\n" out_path -- | Optionally print the node list. maybePrintNodes :: Maybe [String] -- ^ The field list -> String -- ^ Informational message -> ([String] -> String) -- ^ Function to generate the listing -> IO () maybePrintNodes Nothing _ _ = return () maybePrintNodes (Just fields) msg fn = do hPutStrLn stderr "" hPutStrLn stderr (msg ++ " status:") hPutStrLn stderr $ fn fields -- | Optionally print the instance list. maybePrintInsts :: Bool -- ^ Whether to print the instance list -> String -- ^ Type of the instance map (e.g. initial) -> String -- ^ The instance data -> IO () maybePrintInsts do_print msg instdata = when do_print $ do hPutStrLn stderr "" hPutStrLn stderr $ msg ++ " instance map:" hPutStr stderr instdata -- | Function to display warning messages from parsing the cluster -- state. maybeShowWarnings :: [String] -- ^ The warning messages -> IO () maybeShowWarnings fix_msgs = unless (null fix_msgs) $ do hPutStrLn stderr "Warning: cluster has inconsistent data:" hPutStrLn stderr . unlines . map (printf " - %s") $ fix_msgs -- | Format a list of key, value as a shell fragment. printKeys :: String -- ^ Prefix to printed variables -> [(String, String)] -- ^ List of (key, value) pairs to be printed -> IO () printKeys prefix = mapM_ (\(k, v) -> printf "%s_%s=%s\n" prefix (map toUpper k) (ensureQuoted v)) -- | Prints the final @OK@ marker in machine readable output. printFinal :: String -- ^ Prefix to printed variable -> Bool -- ^ Whether output should be machine readable; -- note: if not, there is nothing to print -> IO () printFinal prefix True = -- this should be the final entry printKeys prefix [("OK", "1")] printFinal _ False = return () -- | Potentially set the node as offline based on passed offline list. setNodeOffline :: [Ndx] -> Node.Node -> Node.Node setNodeOffline offline_indices n = if Node.idx n `elem` offline_indices then Node.setOffline n True else n -- | Set node properties based on command line options. setNodeStatus :: Options -> Node.List -> IO Node.List setNodeStatus opts fixed_nl = do let offline_passed = optOffline opts all_nodes = Container.elems fixed_nl offline_lkp = map (lookupName (map Node.name all_nodes)) offline_passed offline_wrong = filter (not . goodLookupResult) offline_lkp offline_names = map lrContent offline_lkp offline_indices = map Node.idx $ filter (\n -> Node.name n `elem` offline_names) all_nodes m_cpu = optMcpu opts m_dsk = optMdsk opts unless (null offline_wrong) . exitErr $ printf "wrong node name(s) set as offline: %s\n" (commaJoin (map lrContent offline_wrong)) let setMCpuFn = case m_cpu of Nothing -> id Just new_mcpu -> flip Node.setMcpu new_mcpu let nm = Container.map (setNodeOffline offline_indices . flip Node.setMdsk m_dsk . setMCpuFn) fixed_nl return nm ganeti-3.1.0~rc2/src/Ganeti/HTools/Cluster.hs000064400000000000000000001353541476477700300210020ustar00rootroot00000000000000{-| Implementation of cluster-wide logic. This module holds all pure cluster-logic; I\/O related functionality goes into the /Main/ module for the individual binaries. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Cluster ( -- * Types AllocDetails(..) , Table(..) , CStats(..) , AllocNodes , AllocResult , AllocMethod , GenericAllocSolutionList , AllocSolutionList -- * Generic functions , totalResources , computeAllocationDelta , hasRequiredNetworks -- * First phase functions , computeBadItems -- * Second phase functions , printSolutionLine , formatCmds , involvedNodes , getMoves , splitJobs -- * Display functions , printNodes , printInsts -- * Balacing functions , doNextBalance , tryBalance , iMoveToJob -- * IAllocator functions , genAllocNodes , tryAlloc , tryGroupAlloc , tryMGAlloc , filterMGResults , sortMGResults , tryChangeGroup , allocList -- * Allocation functions , iterateAlloc , tieredAlloc -- * Node group functions , instanceGroup , findSplitInstances ) where import Control.Applicative (liftA2) import Control.Arrow ((&&&)) import Control.Monad (unless) import Control.Monad.Fail (MonadFail) import Control.Parallel.Strategies (rseq, parMap) import qualified Data.IntSet as IntSet import Data.List import Data.Maybe (fromJust, fromMaybe, isJust, isNothing) import Data.Ord (comparing) import Text.Printf (printf) import Ganeti.BasicTypes import Ganeti.HTools.AlgorithmParams (AlgorithmOptions(..), defaultOptions) import qualified Ganeti.HTools.Container as Container import Ganeti.HTools.Cluster.AllocatePrimitives ( allocateOnSingle , allocateOnPair) import Ganeti.HTools.Cluster.AllocationSolution ( GenericAllocSolution(..) , AllocSolution, emptyAllocSolution , sumAllocs, extractNl, updateIl , annotateSolution, solutionDescription, collapseFailures , emptyAllocCollection, concatAllocCollections, collectionToSolution ) import Ganeti.HTools.Cluster.Evacuate ( EvacSolution(..), emptyEvacSolution , updateEvacSolution, reverseEvacSolution , nodeEvacInstance) import Ganeti.HTools.Cluster.Metrics (compCV, compClusterStatistics) import Ganeti.HTools.Cluster.Moves (applyMoveEx) import Ganeti.HTools.Cluster.Utils (splitCluster, instancePriGroup , availableGroupNodes, iMoveToJob) import Ganeti.HTools.GlobalN1 (allocGlobalN1, redundant) import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Nic as Nic import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Group as Group import Ganeti.HTools.Types import Ganeti.Utils import Ganeti.Types (EvacMode(..)) -- * Types -- | Allocation details for an instance, specifying -- required number of nodes, and -- an optional group (name) to allocate to data AllocDetails = AllocDetails Int (Maybe String) deriving (Show) -- | Allocation results, as used in 'iterateAlloc' and 'tieredAlloc'. type AllocResult = (FailStats, Node.List, Instance.List, [Instance.Instance], [CStats]) -- | Type alias for easier handling. type GenericAllocSolutionList a = [(Instance.Instance, GenericAllocSolution a)] type AllocSolutionList = GenericAllocSolutionList Score -- | A type denoting the valid allocation mode/pairs. -- -- For a one-node allocation, this will be a @Left ['Ndx']@, whereas -- for a two-node allocation, this will be a @Right [('Ndx', -- ['Ndx'])]@. In the latter case, the list is basically an -- association list, grouped by primary node and holding the potential -- secondary nodes in the sub-list. type AllocNodes = Either [Ndx] [(Ndx, [Ndx])] -- | The complete state for the balancing solution. data Table = Table Node.List Instance.List Score [Placement] deriving (Show) -- | Cluster statistics data type. data CStats = CStats { csFmem :: Integer -- ^ Cluster free mem , csFdsk :: Integer -- ^ Cluster free disk , csFspn :: Integer -- ^ Cluster free spindles , csAmem :: Integer -- ^ Cluster allocatable mem , csAdsk :: Integer -- ^ Cluster allocatable disk , csAcpu :: Integer -- ^ Cluster allocatable cpus , csMmem :: Integer -- ^ Max node allocatable mem , csMdsk :: Integer -- ^ Max node allocatable disk , csMcpu :: Integer -- ^ Max node allocatable cpu , csImem :: Integer -- ^ Instance used mem , csIdsk :: Integer -- ^ Instance used disk , csIspn :: Integer -- ^ Instance used spindles , csIcpu :: Integer -- ^ Instance used cpu , csTmem :: Double -- ^ Cluster total mem , csTdsk :: Double -- ^ Cluster total disk , csTspn :: Double -- ^ Cluster total spindles , csTcpu :: Double -- ^ Cluster total cpus , csVcpu :: Integer -- ^ Cluster total virtual cpus , csNcpu :: Double -- ^ Equivalent to 'csIcpu' but in terms of -- physical CPUs, i.e. normalised used phys CPUs , csXmem :: Integer -- ^ Unnacounted for mem , csNmem :: Integer -- ^ Node own memory , csScore :: Score -- ^ The cluster score , csNinst :: Int -- ^ The total number of instances } deriving (Show) -- | A simple type for allocation functions. type AllocMethod = Node.List -- ^ Node list -> Instance.List -- ^ Instance list -> Maybe Int -- ^ Optional allocation limit -> Instance.Instance -- ^ Instance spec for allocation -> AllocNodes -- ^ Which nodes we should allocate on -> [Instance.Instance] -- ^ Allocated instances -> [CStats] -- ^ Running cluster stats -> Result AllocResult -- ^ Allocation result -- * Utility functions -- | Verifies the N+1 status and return the affected nodes. verifyN1 :: [Node.Node] -> [Node.Node] verifyN1 = filter Node.failN1 {-| Computes the pair of bad nodes and instances. The bad node list is computed via a simple 'verifyN1' check, and the bad instance list is the list of primary and secondary instances of those nodes. -} computeBadItems :: Node.List -> Instance.List -> ([Node.Node], [Instance.Instance]) computeBadItems nl il = let bad_nodes = verifyN1 $ getOnline nl bad_instances = map (`Container.find` il) . sort . nub $ concatMap (\ n -> Node.sList n ++ Node.pList n) bad_nodes in (bad_nodes, bad_instances) -- | Zero-initializer for the CStats type. emptyCStats :: CStats emptyCStats = CStats 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 -- | Update stats with data from a new node. updateCStats :: CStats -> Node.Node -> CStats updateCStats cs node = let CStats { csFmem = x_fmem, csFdsk = x_fdsk, csAmem = x_amem, csAcpu = x_acpu, csAdsk = x_adsk, csMmem = x_mmem, csMdsk = x_mdsk, csMcpu = x_mcpu, csImem = x_imem, csIdsk = x_idsk, csIcpu = x_icpu, csTmem = x_tmem, csTdsk = x_tdsk, csTcpu = x_tcpu, csVcpu = x_vcpu, csNcpu = x_ncpu, csXmem = x_xmem, csNmem = x_nmem, csNinst = x_ninst, csFspn = x_fspn, csIspn = x_ispn, csTspn = x_tspn } = cs inc_amem = Node.availMem node inc_adsk = Node.availDisk node inc_imem = Node.iMem node inc_icpu = Node.uCpu node inc_idsk = truncate (Node.tDsk node) - Node.fDsk node inc_ispn = Node.tSpindles node - Node.fSpindles node inc_vcpu = Node.hiCpu node inc_acpu = Node.availCpu node inc_ncpu = fromIntegral (Node.uCpu node) / iPolicyVcpuRatio (Node.iPolicy node) in cs { csFmem = x_fmem + fromIntegral (Node.unallocatedMem node) , csFdsk = x_fdsk + fromIntegral (Node.fDsk node) , csFspn = x_fspn + fromIntegral (Node.fSpindles node) , csAmem = x_amem + fromIntegral inc_amem , csAdsk = x_adsk + fromIntegral inc_adsk , csAcpu = x_acpu + fromIntegral inc_acpu , csMmem = max x_mmem (fromIntegral inc_amem) , csMdsk = max x_mdsk (fromIntegral inc_adsk) , csMcpu = max x_mcpu (fromIntegral inc_acpu) , csImem = x_imem + fromIntegral inc_imem , csIdsk = x_idsk + fromIntegral inc_idsk , csIspn = x_ispn + fromIntegral inc_ispn , csIcpu = x_icpu + fromIntegral inc_icpu , csTmem = x_tmem + Node.tMem node , csTdsk = x_tdsk + Node.tDsk node , csTspn = x_tspn + fromIntegral (Node.tSpindles node) , csTcpu = x_tcpu + Node.tCpu node , csVcpu = x_vcpu + fromIntegral inc_vcpu , csNcpu = x_ncpu + inc_ncpu , csXmem = x_xmem + fromIntegral (Node.xMem node) , csNmem = x_nmem + fromIntegral (Node.nMem node) , csNinst = x_ninst + length (Node.pList node) } -- | Compute the total free disk and memory in the cluster. totalResources :: Node.List -> CStats totalResources nl = let cs = foldl' updateCStats emptyCStats . Container.elems $ nl in cs { csScore = compCV nl } -- | Compute the delta between two cluster state. -- -- This is used when doing allocations, to understand better the -- available cluster resources. The return value is a triple of the -- current used values, the delta that was still allocated, and what -- was left unallocated. computeAllocationDelta :: CStats -> CStats -> AllocStats computeAllocationDelta cini cfin = let CStats {csImem = i_imem, csIdsk = i_idsk, csIcpu = i_icpu, csNcpu = i_ncpu, csIspn = i_ispn } = cini CStats {csImem = f_imem, csIdsk = f_idsk, csIcpu = f_icpu, csTmem = t_mem, csTdsk = t_dsk, csVcpu = f_vcpu, csNcpu = f_ncpu, csTcpu = f_tcpu, csIspn = f_ispn, csTspn = t_spn } = cfin rini = AllocInfo { allocInfoVCpus = fromIntegral i_icpu , allocInfoNCpus = i_ncpu , allocInfoMem = fromIntegral i_imem , allocInfoDisk = fromIntegral i_idsk , allocInfoSpn = fromIntegral i_ispn } rfin = AllocInfo { allocInfoVCpus = fromIntegral (f_icpu - i_icpu) , allocInfoNCpus = f_ncpu - i_ncpu , allocInfoMem = fromIntegral (f_imem - i_imem) , allocInfoDisk = fromIntegral (f_idsk - i_idsk) , allocInfoSpn = fromIntegral (f_ispn - i_ispn) } runa = AllocInfo { allocInfoVCpus = fromIntegral (f_vcpu - f_icpu) , allocInfoNCpus = f_tcpu - f_ncpu , allocInfoMem = truncate t_mem - fromIntegral f_imem , allocInfoDisk = truncate t_dsk - fromIntegral f_idsk , allocInfoSpn = truncate t_spn - fromIntegral f_ispn } in (rini, rfin, runa) -- | Compute online nodes from a 'Node.List'. getOnline :: Node.List -> [Node.Node] getOnline = filter (not . Node.offline) . Container.elems -- * Balancing functions -- | Compute best table. Note that the ordering of the arguments is important. compareTables :: Table -> Table -> Table compareTables a@(Table _ _ a_cv _) b@(Table _ _ b_cv _ ) = if a_cv > b_cv then b else a -- | Tries to perform an instance move and returns the best table -- between the original one and the new one. checkSingleStep :: Bool -- ^ Whether to unconditionally ignore soft errors -> Table -- ^ The original table -> Instance.Instance -- ^ The instance to move -> Table -- ^ The current best table -> IMove -- ^ The move to apply -> Table -- ^ The final best table checkSingleStep force ini_tbl target cur_tbl move = let Table ini_nl ini_il _ ini_plc = ini_tbl tmp_resu = applyMoveEx force ini_nl target move in case tmp_resu of Bad _ -> cur_tbl Ok (upd_nl, new_inst, pri_idx, sec_idx) -> let tgt_idx = Instance.idx target upd_cvar = compCV upd_nl upd_il = Container.add tgt_idx new_inst ini_il upd_plc = (tgt_idx, pri_idx, sec_idx, move, upd_cvar):ini_plc upd_tbl = Table upd_nl upd_il upd_cvar upd_plc in compareTables cur_tbl upd_tbl -- | Given the status of the current secondary as a valid new node and -- the current candidate target node, generate the possible moves for -- a instance. possibleMoves :: MirrorType -- ^ The mirroring type of the instance -> Bool -- ^ Whether the secondary node is a valid new node -> Bool -- ^ Whether we can change the primary node -> Bool -- ^ Whether we alowed to move disks -> (Bool, Bool) -- ^ Whether migration is restricted and whether -- the instance primary is offline -> Ndx -- ^ Target node candidate -> [IMove] -- ^ List of valid result moves possibleMoves MirrorNone _ _ _ _ _ = [] possibleMoves MirrorExternal _ False _ _ _ = [] possibleMoves MirrorExternal _ True _ _ tdx = [ FailoverToAny tdx ] possibleMoves MirrorInternal _ _ False _ _ = [] possibleMoves MirrorInternal _ False True _ tdx = [ ReplaceSecondary tdx ] possibleMoves MirrorInternal _ _ True (True, False) tdx = [ ReplaceSecondary tdx ] possibleMoves MirrorInternal True True True (False, _) tdx = [ ReplaceSecondary tdx , ReplaceAndFailover tdx , ReplacePrimary tdx , FailoverAndReplace tdx ] possibleMoves MirrorInternal True True True (True, True) tdx = [ ReplaceSecondary tdx , ReplaceAndFailover tdx , FailoverAndReplace tdx ] possibleMoves MirrorInternal False True True _ tdx = [ ReplaceSecondary tdx , ReplaceAndFailover tdx ] -- | Compute the best move for a given instance. checkInstanceMove :: AlgorithmOptions -- ^ Algorithmic options for balancing -> [Ndx] -- ^ Allowed target node indices -> Table -- ^ Original table -> Instance.Instance -- ^ Instance to move -> Table -- ^ Best new table for this instance checkInstanceMove opts nodes_idx ini_tbl@(Table nl _ _ _) target = let force = algIgnoreSoftErrors opts disk_moves = algDiskMoves opts inst_moves = algInstanceMoves opts rest_mig = algRestrictedMigration opts opdx = Instance.pNode target osdx = Instance.sNode target bad_nodes = [opdx, osdx] nodes = filter (`notElem` bad_nodes) nodes_idx mir_type = Instance.mirrorType target use_secondary = elem osdx nodes_idx && inst_moves aft_failover = if mir_type == MirrorInternal && use_secondary -- if drbd and allowed to failover then checkSingleStep force ini_tbl target ini_tbl Failover else ini_tbl primary_drained = Node.offline . flip Container.find nl $ Instance.pNode target all_moves = concatMap (possibleMoves mir_type use_secondary inst_moves disk_moves (rest_mig, primary_drained)) nodes in -- iterate over the possible nodes for this instance foldl' (checkSingleStep force ini_tbl target) aft_failover all_moves -- | Compute the best next move. checkMove :: AlgorithmOptions -- ^ Algorithmic options for balancing -> [Ndx] -- ^ Allowed target node indices -> Table -- ^ The current solution -> [Instance.Instance] -- ^ List of instances still to move -> Table -- ^ The new solution checkMove opts nodes_idx ini_tbl victims = let Table _ _ _ ini_plc = ini_tbl -- we're using rseq from the Control.Parallel.Strategies -- package; we don't need to use rnf as that would force too -- much evaluation in single-threaded cases, and in -- multi-threaded case the weak head normal form is enough to -- spark the evaluation tables = parMap rseq (checkInstanceMove opts nodes_idx ini_tbl) victims -- iterate over all instances, computing the best move best_tbl = foldl' compareTables ini_tbl tables Table _ _ _ best_plc = best_tbl in if length best_plc == length ini_plc then ini_tbl -- no advancement else best_tbl -- | Check if we are allowed to go deeper in the balancing. doNextBalance :: Table -- ^ The starting table -> Int -- ^ Remaining length -> Score -- ^ Score at which to stop -> Bool -- ^ The resulting table and commands doNextBalance ini_tbl max_rounds min_score = let Table _ _ ini_cv ini_plc = ini_tbl ini_plc_len = length ini_plc in (max_rounds < 0 || ini_plc_len < max_rounds) && ini_cv > min_score -- | Run a balance move. tryBalance :: AlgorithmOptions -- ^ Algorithmic options for balancing -> Table -- ^ The starting table -> Maybe Table -- ^ The resulting table and commands tryBalance opts ini_tbl = let evac_mode = algEvacMode opts mg_limit = algMinGainLimit opts min_gain = algMinGain opts Table ini_nl ini_il ini_cv _ = ini_tbl all_inst = Container.elems ini_il all_nodes = Container.elems ini_nl (offline_nodes, online_nodes) = partition Node.offline all_nodes all_inst' = if evac_mode then let bad_nodes = map Node.idx offline_nodes in filter (any (`elem` bad_nodes) . Instance.allNodes) all_inst else all_inst reloc_inst = filter (\i -> Instance.movable i && Instance.autoBalance i) all_inst' node_idx = map Node.idx online_nodes fin_tbl = checkMove opts node_idx ini_tbl reloc_inst (Table _ _ fin_cv _) = fin_tbl in if fin_cv < ini_cv && (ini_cv > mg_limit || ini_cv - fin_cv >= min_gain) then Just fin_tbl -- this round made success, return the new table else Nothing -- * Allocation functions -- | Generate the valid node allocation singles or pairs for a new instance. genAllocNodes :: AlgorithmOptions -- ^ algorithmic options to honor -> Group.List -- ^ Group list -> Node.List -- ^ The node map -> Int -- ^ The number of nodes required -> Bool -- ^ Whether to drop or not -- unallocable nodes -> Result AllocNodes -- ^ The (monadic) result genAllocNodes opts gl nl count drop_unalloc = let filter_fn = if drop_unalloc then filter (Group.isAllocable . flip Container.find gl . Node.group) else id restrict_fn = maybe id (\ns -> filter (flip elem ns . Node.name)) $ algRestrictToNodes opts all_nodes = restrict_fn . filter_fn $ getOnline nl all_pairs = [(Node.idx p, [Node.idx s | s <- all_nodes, Node.idx p /= Node.idx s, Node.group p == Node.group s]) | p <- all_nodes] in case count of 1 -> Ok (Left (map Node.idx all_nodes)) 2 -> Ok (Right (filter (not . null . snd) all_pairs)) _ -> Bad "Unsupported number of nodes, only one or two supported" -- | Try to allocate an instance on the cluster. tryAlloc :: (MonadFail m) => AlgorithmOptions -> Node.List -- ^ The node list -> Instance.List -- ^ The instance list -> Instance.Instance -- ^ The instance to allocate -> AllocNodes -- ^ The allocation targets -> m AllocSolution -- ^ Possible solution list tryAlloc _ _ _ _ (Right []) = fail "Not enough online nodes" tryAlloc opts nl il inst (Right ok_pairs) = let cstat = compClusterStatistics $ Container.elems nl n1pred = if algCapacity opts then allocGlobalN1 opts nl il else const True psols = parMap rseq (\(p, ss) -> collectionToSolution FailN1 n1pred $ foldl (\cstate -> concatAllocCollections cstate . allocateOnPair opts cstat nl inst p) emptyAllocCollection ss) ok_pairs sols = foldl' sumAllocs emptyAllocSolution psols in return $ annotateSolution sols tryAlloc _ _ _ _ (Left []) = fail "No online nodes" tryAlloc opts nl il inst (Left all_nodes) = let sols = foldl (\cstate -> concatAllocCollections cstate . allocateOnSingle opts nl inst ) emptyAllocCollection all_nodes n1pred = if algCapacity opts then allocGlobalN1 opts nl il else const True in return . annotateSolution $ collectionToSolution FailN1 n1pred sols -- | From a list of possibly bad and possibly empty solutions, filter -- only the groups with a valid result. Note that the result will be -- reversed compared to the original list. filterMGResults :: [(Group.Group, Result (GenericAllocSolution a))] -> [(Group.Group, GenericAllocSolution a)] filterMGResults = foldl' fn [] where unallocable = not . Group.isAllocable fn accu (grp, rasol) = case rasol of Bad _ -> accu Ok sol | isNothing (asSolution sol) -> accu | unallocable grp -> accu | otherwise -> (grp, sol):accu -- | Sort multigroup results based on policy and score. sortMGResults :: Ord a => [(Group.Group, GenericAllocSolution a)] -> [(Group.Group, GenericAllocSolution a)] sortMGResults sols = let extractScore (_, _, _, x) = x solScore (grp, sol) = (Group.allocPolicy grp, (extractScore . fromJust . asSolution) sol) in sortBy (comparing solScore) sols -- | Determines if a group is connected to the networks required by the -- | instance. hasRequiredNetworks :: Group.Group -> Instance.Instance -> Bool hasRequiredNetworks ng = all hasNetwork . Instance.nics where hasNetwork = maybe True (`elem` Group.networks ng) . Nic.network -- | Removes node groups which can't accommodate the instance filterValidGroups :: [(Group.Group, (Node.List, Instance.List))] -> Instance.Instance -> ([(Group.Group, (Node.List, Instance.List))], [String]) filterValidGroups [] _ = ([], []) filterValidGroups (ng:ngs) inst = let (valid_ngs, msgs) = filterValidGroups ngs inst in if hasRequiredNetworks (fst ng) inst then (ng:valid_ngs, msgs) else (valid_ngs, ("group " ++ Group.name (fst ng) ++ " is not connected to a network required by instance " ++ Instance.name inst):msgs) -- | Finds an allocation solution for an instance on a group findAllocation :: AlgorithmOptions -> Group.List -- ^ The group list -> Node.List -- ^ The node list -> Instance.List -- ^ The instance list -> Gdx -- ^ The group to allocate to -> Instance.Instance -- ^ The instance to allocate -> Int -- ^ Required number of nodes -> Result (AllocSolution, [String]) findAllocation opts mggl mgnl mgil gdx inst cnt = do let belongsTo nl' nidx = nidx `elem` map Node.idx (Container.elems nl') nl = Container.filter ((== gdx) . Node.group) mgnl il = Container.filter (belongsTo nl . Instance.pNode) mgil group' = Container.find gdx mggl unless (hasRequiredNetworks group' inst) . failError $ "The group " ++ Group.name group' ++ " is not connected to\ \ a network required by instance " ++ Instance.name inst solution <- genAllocNodes opts mggl nl cnt False >>= tryAlloc opts nl il inst return (solution, solutionDescription (group', return solution)) -- | Finds the best group for an instance on a multi-group cluster. -- -- Only solutions in @preferred@ and @last_resort@ groups will be -- accepted as valid, and additionally if the allowed groups parameter -- is not null then allocation will only be run for those group -- indices. findBestAllocGroup :: AlgorithmOptions -> Group.List -- ^ The group list -> Node.List -- ^ The node list -> Instance.List -- ^ The instance list -> Maybe [Gdx] -- ^ The allowed groups -> Instance.Instance -- ^ The instance to allocate -> Int -- ^ Required number of nodes -> Result (Group.Group, AllocSolution, [String]) findBestAllocGroup opts mggl mgnl mgil allowed_gdxs inst cnt = let groups_by_idx = splitCluster mgnl mgil groups = map (\(gid, d) -> (Container.find gid mggl, d)) groups_by_idx groups' = maybe groups (\gs -> filter ((`elem` gs) . Group.idx . fst) groups) allowed_gdxs (groups'', filter_group_msgs) = filterValidGroups groups' inst sols = map (\(gr, (nl, _)) -> (gr, genAllocNodes opts mggl nl cnt False >>= tryAlloc opts mgnl mgil inst)) groups''::[(Group.Group, Result AllocSolution)] all_msgs = filter_group_msgs ++ concatMap solutionDescription sols goodSols = filterMGResults sols sortedSols = sortMGResults goodSols in case sortedSols of [] -> Bad $ if null groups' then "no groups for evacuation: allowed groups was " ++ show allowed_gdxs ++ ", all groups: " ++ show (map fst groups) else intercalate ", " all_msgs (final_group, final_sol):_ -> return (final_group, final_sol, all_msgs) -- | Try to allocate an instance on a multi-group cluster. tryMGAlloc :: AlgorithmOptions -> Group.List -- ^ The group list -> Node.List -- ^ The node list -> Instance.List -- ^ The instance list -> Instance.Instance -- ^ The instance to allocate -> Int -- ^ Required number of nodes -> Result AllocSolution -- ^ Possible solution list tryMGAlloc opts mggl mgnl mgil inst cnt = do (best_group, solution, all_msgs) <- findBestAllocGroup opts mggl mgnl mgil Nothing inst cnt let group_name = Group.name best_group selmsg = "Selected group: " ++ group_name return $ solution { asLog = selmsg:all_msgs } -- | Try to allocate an instance to a group. tryGroupAlloc :: AlgorithmOptions -> Group.List -- ^ The group list -> Node.List -- ^ The node list -> Instance.List -- ^ The instance list -> String -- ^ The allocation group (name) -> Instance.Instance -- ^ The instance to allocate -> Int -- ^ Required number of nodes -> Result AllocSolution -- ^ Solution tryGroupAlloc opts mggl mgnl ngil gn inst cnt = do gdx <- Group.idx <$> Container.findByName mggl gn (solution, msgs) <- findAllocation opts mggl mgnl ngil gdx inst cnt return $ solution { asLog = msgs } -- | Try to allocate a list of instances on a multi-group cluster. allocList :: AlgorithmOptions -> Group.List -- ^ The group list -> Node.List -- ^ The node list -> Instance.List -- ^ The instance list -> [(Instance.Instance, AllocDetails)] -- ^ The instance to -- allocate -> AllocSolutionList -- ^ Possible solution -- list -> Result (Node.List, Instance.List, AllocSolutionList) -- ^ The final solution -- list allocList _ _ nl il [] result = Ok (nl, il, result) allocList opts gl nl il ((xi, AllocDetails xicnt mgn):xies) result = do ares <- case mgn of Nothing -> tryMGAlloc opts gl nl il xi xicnt Just gn -> tryGroupAlloc opts gl nl il gn xi xicnt let sol = asSolution ares nl' = extractNl nl il sol il' = updateIl il sol allocList opts gl nl' il' xies ((xi, ares):result) -- | Change-group IAllocator mode main function. -- -- This is very similar to 'tryNodeEvac', the only difference is that -- we don't choose as target group the current instance group, but -- instead: -- -- 1. at the start of the function, we compute which are the target -- groups; either no groups were passed in, in which case we choose -- all groups out of which we don't evacuate instance, or there were -- some groups passed, in which case we use those -- -- 2. for each instance, we use 'findBestAllocGroup' to choose the -- best group to hold the instance, and then we do what -- 'tryNodeEvac' does, except for this group instead of the current -- instance group. -- -- Note that the correct behaviour of this function relies on the -- function 'nodeEvacInstance' to be able to do correctly both -- intra-group and inter-group moves when passed the 'ChangeAll' mode. tryChangeGroup :: AlgorithmOptions -> Group.List -- ^ The cluster groups -> Node.List -- ^ The node list (cluster-wide) -> Instance.List -- ^ Instance list (cluster-wide) -> [Gdx] -- ^ Target groups; if empty, any -- groups not being evacuated -> [Idx] -- ^ List of instance (indices) to be evacuated -> Result (Node.List, Instance.List, EvacSolution) tryChangeGroup opts gl ini_nl ini_il gdxs idxs = let evac_gdxs = nub $ map (instancePriGroup ini_nl . flip Container.find ini_il) idxs target_gdxs = (if null gdxs then Container.keys gl else gdxs) \\ evac_gdxs offline = map Node.idx . filter Node.offline $ Container.elems ini_nl excl_ndx = foldl' (flip IntSet.insert) IntSet.empty offline group_ndx = map (\(gdx, (nl, _)) -> (gdx, map Node.idx (Container.elems nl))) $ splitCluster ini_nl ini_il (fin_nl, fin_il, esol) = foldl' (\state@(nl, il, _) inst -> let solution = do let ncnt = Instance.requiredNodes $ Instance.diskTemplate inst (grp, _, _) <- findBestAllocGroup opts gl nl il (Just target_gdxs) inst ncnt let gdx = Group.idx grp av_nodes <- availableGroupNodes group_ndx excl_ndx gdx nodeEvacInstance defaultOptions nl il ChangeAll inst gdx av_nodes in updateEvacSolution state (Instance.idx inst) solution ) (ini_nl, ini_il, emptyEvacSolution) (map (`Container.find` ini_il) idxs) in return (fin_nl, fin_il, reverseEvacSolution esol) -- | Standard-sized allocation method. -- -- This places instances of the same size on the cluster until we're -- out of space. The result will be a list of identically-sized -- instances. iterateAllocSmallStep :: AlgorithmOptions -> AllocMethod iterateAllocSmallStep opts nl il limit newinst allocnodes ixes cstats = let depth = length ixes newname = printf "new-%d" depth::String newidx = Container.size il newi2 = Instance.setIdx (Instance.setName newinst newname) newidx newlimit = fmap (flip (-) 1) limit opts' = if Instance.diskTemplate newi2 == DTDrbd8 then opts { algCapacity = False } else opts in case tryAlloc opts' nl il newi2 allocnodes of Bad s -> Bad s Ok (AllocSolution { asFailures = errs, asSolution = sols3 }) -> let newsol = Ok (collapseFailures errs, nl, il, ixes, cstats) in case sols3 of Nothing -> newsol Just (xnl, xi, _, _) -> if limit == Just 0 then newsol else iterateAllocSmallStep opts xnl (Container.add newidx xi il) newlimit newinst allocnodes (xi:ixes) (totalResources xnl:cstats) -- | Guess a number of machines worth trying to put on the cluster in one step. -- The goal is to guess a number close to the actual capacity of the cluster but -- preferrably not bigger, unless it is quite small (as we don't want to do -- big steps smaller than 20). guessBigstepSize :: Node.List -> Instance.Instance -> Int guessBigstepSize nl inst = let nodes = Container.elems nl totalAvail = sum $ map Node.availMem nodes capacity = totalAvail `div` Instance.mem inst -- however, at every node we might lose almost an instance if it just -- doesn't fit by a tiny margin guess = capacity - Container.size nl in if guess < 20 then 20 else guess -- | A speed-up version of `iterateAllocSmallStep`. -- -- This function returns precisely the same result as `iterateAllocSmallStep`. -- However the computation is speed up by the following heuristic: allocate -- a group of instances iteratively without considering global N+1 redundancy; -- if the result of this is globally N+1 redundant, then everything was OK -- inbetween and we can continue from there. Only if that fails, do a -- step-by-step iterative allocation. -- In order to further speed up the computation while keeping it robust, we -- first try (if the first argument is True) a number of steps guessed from -- the node capacity, then, if that failed, a fixed step size and only as last -- restort step-by-step iterative allocation. iterateAlloc' :: Bool -> AlgorithmOptions -> AllocMethod iterateAlloc' tryHugestep opts nl il limit newinst allocnodes ixes cstats = if not $ algCapacity opts then iterateAllocSmallStep opts nl il limit newinst allocnodes ixes cstats else let bigstepsize = if tryHugestep then guessBigstepSize nl newinst else 10 (limit', newlimit) = maybe (Just bigstepsize, Nothing) (Just . min bigstepsize &&& Just . max 0 . flip (-) bigstepsize) limit opts' = opts { algCapacity = False } in case iterateAllocSmallStep opts' nl il limit' newinst allocnodes ixes cstats of Bad s -> Bad s Ok res@(_, nl', il', ixes', cstats') | redundant opts nl' il' -> if newlimit == Just 0 || length ixes' == length ixes then return res else iterateAlloc' tryHugestep opts nl' il' newlimit newinst allocnodes ixes' cstats' _ -> if tryHugestep then iterateAlloc' False opts nl il limit newinst allocnodes ixes cstats else iterateAllocSmallStep opts nl il limit newinst allocnodes ixes cstats -- | A speed-up version of `iterateAllocSmallStep`. iterateAlloc :: AlgorithmOptions -> AllocMethod iterateAlloc = iterateAlloc' True -- | Predicate whether shrinking a single resource can lead to a valid -- allocation. sufficesShrinking :: (Instance.Instance -> AllocSolution) -> Instance.Instance -> FailMode -> Maybe Instance.Instance sufficesShrinking allocFn inst fm = case dropWhile (isNothing . asSolution . fst) . takeWhile (liftA2 (||) (elem fm . asFailures . fst) (isJust . asSolution . fst)) . map (allocFn &&& id) $ iterateOk (`Instance.shrinkByType` fm) inst of x:_ -> Just . snd $ x _ -> Nothing -- | Tiered allocation method. -- -- This places instances on the cluster, and decreases the spec until -- we can allocate again. The result will be a list of decreasing -- instance specs. tieredAlloc :: AlgorithmOptions -> AllocMethod tieredAlloc opts nl il limit newinst allocnodes ixes cstats = case iterateAlloc opts nl il limit newinst allocnodes ixes cstats of Bad s -> Bad s Ok (errs, nl', il', ixes', cstats') -> let newsol = Ok (errs, nl', il', ixes', cstats') ixes_cnt = length ixes' (stop, newlimit) = case limit of Nothing -> (False, Nothing) Just n -> (n <= ixes_cnt, Just (n - ixes_cnt)) sortedErrs = map fst $ sortBy (comparing snd) errs suffShrink = sufficesShrinking (fromMaybe emptyAllocSolution . flip (tryAlloc opts nl' il') allocnodes) newinst bigSteps = filter isJust . map suffShrink . reverse $ sortedErrs progress (Ok (_, _, _, newil', _)) (Ok (_, _, _, newil, _)) = length newil' > length newil progress _ _ = False in if stop then newsol else let newsol' = case Instance.shrinkByType newinst . last $ sortedErrs of Bad _ -> newsol Ok newinst' -> tieredAlloc opts nl' il' newlimit newinst' allocnodes ixes' cstats' in if progress newsol' newsol then newsol' else case bigSteps of Just newinst':_ -> tieredAlloc opts nl' il' newlimit newinst' allocnodes ixes' cstats' _ -> newsol -- * Formatting functions -- | Given the original and final nodes, computes the relocation description. computeMoves :: Instance.Instance -- ^ The instance to be moved -> String -- ^ The instance name -> IMove -- ^ The move being performed -> String -- ^ New primary -> String -- ^ New secondary -> (String, [String]) -- ^ Tuple of moves and commands list; moves is containing -- either @/f/@ for failover or @/r:name/@ for replace -- secondary, while the command list holds gnt-instance -- commands (without that prefix), e.g \"@failover instance1@\" computeMoves i inam mv c d = case mv of Failover -> ("f", [mig]) FailoverToAny _ -> (printf "fa:%s" c, [mig_any]) FailoverAndReplace _ -> (printf "f r:%s" d, [mig, rep d]) ReplaceSecondary _ -> (printf "r:%s" d, [rep d]) ReplaceAndFailover _ -> (printf "r:%s f" c, [rep c, mig]) ReplacePrimary _ -> (printf "f r:%s f" c, [mig, rep c, mig]) where morf = if Instance.isRunning i then "migrate" else "failover" mig = printf "%s -f %s" morf inam::String mig_any = printf "%s -f -n %s %s" morf c inam::String rep n = printf "replace-disks -n %s %s" n inam::String -- | Converts a placement to string format. printSolutionLine :: Node.List -- ^ The node list -> Instance.List -- ^ The instance list -> Int -- ^ Maximum node name length -> Int -- ^ Maximum instance name length -> Placement -- ^ The current placement -> Int -- ^ The index of the placement in -- the solution -> (String, [String]) printSolutionLine nl il nmlen imlen plc pos = let pmlen = (2*nmlen + 1) (i, p, s, mv, c) = plc old_sec = Instance.sNode inst inst = Container.find i il inam = Instance.alias inst npri = Node.alias $ Container.find p nl nsec = Node.alias $ Container.find s nl opri = Node.alias $ Container.find (Instance.pNode inst) nl osec = Node.alias $ Container.find old_sec nl (moves, cmds) = computeMoves inst inam mv npri nsec -- FIXME: this should check instead/also the disk template ostr = if old_sec == Node.noSecondary then printf "%s" opri::String else printf "%s:%s" opri osec::String nstr = if s == Node.noSecondary then printf "%s" npri::String else printf "%s:%s" npri nsec::String in (printf " %3d. %-*s %-*s => %-*s %12.8f a=%s" pos imlen inam pmlen ostr pmlen nstr c moves, cmds) -- | Return the instance and involved nodes in an instance move. -- -- Note that the output list length can vary, and is not required nor -- guaranteed to be of any specific length. involvedNodes :: Instance.List -- ^ Instance list, used for retrieving -- the instance from its index; note -- that this /must/ be the original -- instance list, so that we can -- retrieve the old nodes -> Placement -- ^ The placement we're investigating, -- containing the new nodes and -- instance index -> [Ndx] -- ^ Resulting list of node indices involvedNodes il plc = let (i, np, ns, _, _) = plc inst = Container.find i il in nub . filter (>= 0) $ [np, ns] ++ Instance.allNodes inst -- | From two adjacent cluster tables get the list of moves that transitions -- from to the other getMoves :: (Table, Table) -> [MoveJob] getMoves (Table _ initial_il _ initial_plc, Table final_nl _ _ final_plc) = let plctoMoves (plc@(idx, p, s, mv, _)) = let inst = Container.find idx initial_il inst_name = Instance.name inst affected = involvedNodes initial_il plc np = Node.alias $ Container.find p final_nl ns = Node.alias $ Container.find s final_nl (_, cmds) = computeMoves inst inst_name mv np ns in (affected, idx, mv, cmds) in map plctoMoves . reverse . drop (length initial_plc) $ reverse final_plc -- | Inner function for splitJobs, that either appends the next job to -- the current jobset, or starts a new jobset. mergeJobs :: ([JobSet], [Ndx]) -> MoveJob -> ([JobSet], [Ndx]) mergeJobs ([], _) n@(ndx, _, _, _) = ([[n]], ndx) mergeJobs (cjs@(j:js), nbuf) n@(ndx, _, _, _) | null (ndx `intersect` nbuf) = ((n:j):js, ndx ++ nbuf) | otherwise = ([n]:cjs, ndx) -- | Break a list of moves into independent groups. Note that this -- will reverse the order of jobs. splitJobs :: [MoveJob] -> [JobSet] splitJobs = fst . foldl mergeJobs ([], []) -- | Given a list of commands, prefix them with @gnt-instance@ and -- also beautify the display a little. formatJob :: Int -> Int -> (Int, MoveJob) -> [String] formatJob jsn jsl (sn, (_, _, _, cmds)) = let out = printf " echo job %d/%d" jsn sn: printf " check": map (" gnt-instance " ++) cmds in if sn == 1 then ["", printf "echo jobset %d, %d jobs" jsn jsl] ++ out else out -- | Given a list of commands, prefix them with @gnt-instance@ and -- also beautify the display a little. formatCmds :: [JobSet] -> String formatCmds = unlines . concatMap (\(jsn, js) -> concatMap (formatJob jsn (length js)) (zip [1..] js)) . zip [1..] -- | Print the node list. printNodes :: Node.List -> [String] -> String printNodes nl fs = let fields = case fs of [] -> Node.defaultFields "+":rest -> Node.defaultFields ++ rest _ -> fs snl = sortBy (comparing Node.idx) (Container.elems nl) (header, isnum) = unzip $ map Node.showHeader fields in printTable "" header (map (Node.list fields) snl) isnum -- | Print the instance list. printInsts :: Node.List -> Instance.List -> String printInsts nl il = let sil = sortBy (comparing Instance.idx) (Container.elems il) helper inst = [ if Instance.isRunning inst then "R" else " " , Instance.name inst , Container.nameOf nl (Instance.pNode inst) , let sdx = Instance.sNode inst in if sdx == Node.noSecondary then "" else Container.nameOf nl sdx , if Instance.autoBalance inst then "Y" else "N" , printf "%3d" $ Instance.vcpus inst , printf "%5d" $ Instance.mem inst , printf "%5d" $ Instance.dsk inst `div` 1024 , printf "%5.3f" lC , printf "%5.3f" lM , printf "%5.3f" lD , printf "%5.3f" lN ] where DynUtil lC lM lD lN = Instance.util inst header = [ "F", "Name", "Pri_node", "Sec_node", "Auto_bal" , "vcpu", "mem" , "dsk", "lCpu", "lMem", "lDsk", "lNet" ] isnum = False:False:False:False:False:repeat True in printTable "" header (map helper sil) isnum -- * Node group functions -- | Computes the group of an instance. instanceGroup :: Node.List -> Instance.Instance -> Result Gdx instanceGroup nl i = let sidx = Instance.sNode i pnode = Container.find (Instance.pNode i) nl snode = if sidx == Node.noSecondary then pnode else Container.find sidx nl pgroup = Node.group pnode sgroup = Node.group snode in if pgroup /= sgroup then fail ("Instance placed accross two node groups, primary " ++ show pgroup ++ ", secondary " ++ show sgroup) else return pgroup -- | Compute the list of badly allocated instances (split across node -- groups). findSplitInstances :: Node.List -> Instance.List -> [Instance.Instance] findSplitInstances nl = filter (not . isOk . instanceGroup nl) . Container.elems ganeti-3.1.0~rc2/src/Ganeti/HTools/Cluster/000075500000000000000000000000001476477700300204335ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/HTools/Cluster/AllocatePrimitives.hs000064400000000000000000000071211476477700300245700ustar00rootroot00000000000000{-| Implementation of the primitives of instance allocation -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013, 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Cluster.AllocatePrimitives ( allocateOnSingle , allocateOnPair ) where import Ganeti.HTools.AlgorithmParams (AlgorithmOptions(..)) import Ganeti.HTools.Cluster.AllocationSolution (AllocElement) import Ganeti.HTools.Cluster.Metrics ( compCV, compCVfromStats , updateClusterStatisticsTwice) import Ganeti.HTools.Cluster.Moves (setInstanceLocationScore) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import Ganeti.HTools.Types import Ganeti.Utils.Statistics -- | Tries to allocate an instance on one given node. allocateOnSingle :: AlgorithmOptions -> Node.List -> Instance.Instance -> Ndx -> OpResult AllocElement allocateOnSingle opts nl inst new_pdx = let p = Container.find new_pdx nl new_inst = Instance.setBoth inst new_pdx Node.noSecondary force = algIgnoreSoftErrors opts in do Instance.instMatchesPolicy inst (Node.iPolicy p) (Node.exclStorage p) new_p <- Node.addPriEx force p inst let new_nl = Container.add new_pdx new_p nl new_score = compCV new_nl return (new_nl, new_inst, [new_p], new_score) -- | Tries to allocate an instance on a given pair of nodes. allocateOnPair :: AlgorithmOptions -> [Statistics] -> Node.List -> Instance.Instance -> Ndx -> Ndx -> OpResult AllocElement allocateOnPair opts stats nl inst new_pdx new_sdx = let tgt_p = Container.find new_pdx nl tgt_s = Container.find new_sdx nl force = algIgnoreSoftErrors opts in do Instance.instMatchesPolicy inst (Node.iPolicy tgt_p) (Node.exclStorage tgt_p) let new_inst = Instance.setBoth (setInstanceLocationScore inst tgt_p (Just tgt_s)) new_pdx new_sdx new_p <- Node.addPriEx force tgt_p new_inst new_s <- Node.addSec tgt_s new_inst new_pdx let new_nl = Container.addTwo new_pdx new_p new_sdx new_s nl new_stats = updateClusterStatisticsTwice stats (tgt_p, new_p) (tgt_s, new_s) return (new_nl, new_inst, [new_p, new_s], compCVfromStats new_stats) ganeti-3.1.0~rc2/src/Ganeti/HTools/Cluster/AllocateSecondary.hs000064400000000000000000000054471476477700300243750ustar00rootroot00000000000000{-| Implementation of finding a secondary for disk template conversion -} {- Copyright (C) 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Cluster.AllocateSecondary ( tryAllocateSecondary ) where import Control.Monad (unless) import Ganeti.BasicTypes import Ganeti.HTools.AlgorithmParams (AlgorithmOptions(..)) import qualified Ganeti.HTools.Cluster as Cluster import Ganeti.HTools.Cluster.AllocationSolution (AllocSolution) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import Ganeti.HTools.Types tryAllocateSecondary :: AlgorithmOptions -> Group.List -- ^ The cluster groups -> Node.List -- ^ The node list (cluster-wide, -- not per group) -> Instance.List -- ^ Instance list (cluster-wide) -> Idx -> Result AllocSolution tryAllocateSecondary opts _ nl il idx = do let inst = Container.find idx il unless (Instance.sNode inst < 0) $ fail "Instance already has a secondary" let pidx = Instance.pNode inst pnode = Container.find pidx nl pnode' = Node.removePri pnode inst nl' = Container.add pidx pnode' nl inst' = inst { Instance.diskTemplate = DTDrbd8 } gidx = Node.group pnode' sidxs = filter (/= pidx) . Container.keys $ Container.filter ((==) gidx . Node.group) nl' Cluster.tryAlloc opts nl' il inst' $ Right [(pidx, sidxs)] ganeti-3.1.0~rc2/src/Ganeti/HTools/Cluster/AllocationSolution.hs000064400000000000000000000251551476477700300246210ustar00rootroot00000000000000{-| Implementation of handling of Allocation Solutions -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013, 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Cluster.AllocationSolution ( GenericAllocElement , AllocElement , GenericAllocSolution(..) , AllocSolution , emptyAllocSolution , sumAllocs , concatAllocs , updateIl , extractNl , collapseFailures , genericAnnotateSolution , annotateSolution , solutionDescription , AllocSolutionCollection , emptyAllocCollection , concatAllocCollections , collectionToSolution ) where import Data.Ord (comparing) import Data.List (intercalate, foldl', sortBy) import Data.Maybe (listToMaybe) import Text.Printf (printf) import Ganeti.BasicTypes (GenericResult(..), Result) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Types as T -- | A simple name for an allocation element (here just for logistic -- reasons), generic in the type of the metric. type GenericAllocElement a = (Node.List, Instance.Instance, [Node.Node], a) -- | Obtain the metric of a GenericAllocElement. allocMetric :: GenericAllocElement a -> a allocMetric (_, _, _, a) = a -- | A simple name for an allocation element (here just for logistic -- reasons). type AllocElement = GenericAllocElement T.Score -- | Allocation\/relocation solution. data GenericAllocSolution a = AllocSolution { asFailures :: [T.FailMode] -- ^ Failure counts , asAllocs :: Int -- ^ Good allocation count , asSolution :: Maybe (GenericAllocElement a) -- ^ The actual allocation -- result , asLog :: [String] -- ^ Informational messages } type AllocSolution = GenericAllocSolution T.Score -- | The empty solution we start with when computing allocations. emptyAllocSolution :: GenericAllocSolution a emptyAllocSolution = AllocSolution { asFailures = [], asAllocs = 0 , asSolution = Nothing, asLog = [] } -- | Calculate the new instance list after allocation solution. updateIl :: Instance.List -- ^ The original instance list -> Maybe (GenericAllocElement a) -- ^ The result of -- the allocation attempt -> Instance.List -- ^ The updated instance list updateIl il Nothing = il updateIl il (Just (_, xi, _, _)) = Container.add (Container.size il) xi il -- | Extract the the new node list from the allocation solution. extractNl :: Node.List -- ^ The original node list -> Instance.List -- ^ The original instance list -> Maybe (GenericAllocElement a) -- ^ The result of the -- allocation attempt -> Node.List -- ^ The new node list extractNl nl _ Nothing = nl extractNl _ il (Just (xnl, _, ns, _)) = let newIndex = Container.size il fixIndex = map (\i -> if i < 0 then newIndex else i) fixIndices nodes node = let nidx = Node.idx node n = Container.find nidx nodes n' = n { Node.pList = fixIndex $ Node.pList n , Node.sList = fixIndex $ Node.sList n } in Container.add nidx n' nodes in foldl fixIndices xnl ns -- | Compares two Maybe AllocElement and chooses the best score. bestAllocElement :: Ord a => Maybe (GenericAllocElement a) -> Maybe (GenericAllocElement a) -> Maybe (GenericAllocElement a) bestAllocElement a Nothing = a bestAllocElement Nothing b = b bestAllocElement a@(Just (_, _, _, ascore)) b@(Just (_, _, _, bscore)) = if ascore < bscore then a else b -- | Update current Allocation solution and failure stats with new -- elements. concatAllocs :: Ord a => GenericAllocSolution a -> T.OpResult (GenericAllocElement a) -> GenericAllocSolution a concatAllocs as (Bad reason) = as { asFailures = reason : asFailures as } concatAllocs as (Ok ns) = let -- Choose the old or new solution, based on the cluster score cntok = asAllocs as osols = asSolution as nsols = bestAllocElement osols (Just ns) nsuc = cntok + 1 -- Note: we force evaluation of nsols here in order to keep the -- memory profile low - we know that we will need nsols for sure -- in the next cycle, so we force evaluation of nsols, since the -- foldl' in the caller will only evaluate the tuple, but not the -- elements of the tuple in nsols `seq` nsuc `seq` as { asAllocs = nsuc, asSolution = nsols } -- | Sums two 'AllocSolution' structures. sumAllocs :: Ord a => GenericAllocSolution a -> GenericAllocSolution a -> GenericAllocSolution a sumAllocs (AllocSolution aFails aAllocs aSols aLog) (AllocSolution bFails bAllocs bSols bLog) = -- note: we add b first, since usually it will be smaller; when -- fold'ing, a will grow and grow whereas b is the per-group -- result, hence smaller let nFails = bFails ++ aFails nAllocs = aAllocs + bAllocs nSols = bestAllocElement aSols bSols nLog = bLog ++ aLog in AllocSolution nFails nAllocs nSols nLog -- | Build failure stats out of a list of failures. collapseFailures :: [T.FailMode] -> T.FailStats collapseFailures flst = map (\k -> (k, foldl' (\a e -> if e == k then a + 1 else a) 0 flst)) [minBound..maxBound] -- | Given a solution, generates a reasonable description for it. genericDescribeSolution :: (a -> String) -> GenericAllocSolution a -> String genericDescribeSolution formatMetrics as = let fcnt = asFailures as sols = asSolution as freasons = intercalate ", " . map (\(a, b) -> printf "%s: %d" (show a) b) . filter ((> 0) . snd) . collapseFailures $ fcnt in case sols of Nothing -> "No valid allocation solutions, failure reasons: " ++ (if null fcnt then "unknown reasons" else freasons) Just (_, _, nodes, cv) -> printf ("score: %s, successes %d, failures %d (%s)" ++ " for node(s) %s") (formatMetrics cv) (asAllocs as) (length fcnt) freasons (intercalate "/" . map Node.name $ nodes) -- | Annotates a solution with the appropriate string. genericAnnotateSolution :: (a -> String) ->GenericAllocSolution a -> GenericAllocSolution a genericAnnotateSolution formatMetrics as = as { asLog = genericDescribeSolution formatMetrics as : asLog as } -- | Annotate a solution based on the standard metrics annotateSolution :: AllocSolution -> AllocSolution annotateSolution = genericAnnotateSolution (printf "%.8f") -- | Given a group/result, describe it as a nice (list of) messages. solutionDescription :: (Group.Group, Result (GenericAllocSolution a)) -> [String] solutionDescription (grp, result) = case result of Ok solution -> map (printf "Group %s (%s): %s" gname pol) (asLog solution) Bad message -> [printf "Group %s: error %s" gname message] where gname = Group.name grp pol = T.allocPolicyToRaw (Group.allocPolicy grp) -- * Collection of Allocation Solutions for later filtering -- | Collection of Allocation Solution data AllocSolutionCollection a = AllocSolutionCollection { ascFailures :: [T.FailMode] -- ^ Failure counts , ascAllocs :: Int -- ^ Good allocation count , ascSolutions :: [GenericAllocElement a] -- ^ The actual allocation results , ascLog :: [String] -- ^ Informational messages } -- | Empty collection of allocation solutions. emptyAllocCollection :: AllocSolutionCollection a emptyAllocCollection = AllocSolutionCollection { ascFailures = [] , ascAllocs = 0 , ascSolutions = [] , ascLog = [] } -- | Update current collection of solution and failure stats with new -- elements. concatAllocCollections :: Ord a => AllocSolutionCollection a -> T.OpResult (GenericAllocElement a) -> AllocSolutionCollection a concatAllocCollections asc (Bad reason) = asc { ascFailures = reason : ascFailures asc } concatAllocCollections asc (Ok ns) = asc { ascAllocs = ascAllocs asc + 1, ascSolutions = ns : ascSolutions asc } -- | From a collection of solutions collapse to a single one by chosing the best -- that fulfills a given predicate. collectionToSolution :: Ord a => T.FailMode -- ^ Failure mode to assign to solutions -- filtered out in this step -> (GenericAllocElement a -> Bool) -- ^ predicate -- to restrict to -> AllocSolutionCollection a -> GenericAllocSolution a collectionToSolution failmode isgood asc = let sols = sortBy (comparing allocMetric) $ ascSolutions asc (dropped, good) = break isgood sols dropcount = length dropped nsols = ascAllocs asc - dropcount failures = replicate dropcount failmode ++ ascFailures asc sol = listToMaybe good in AllocSolution { asFailures = failures , asAllocs = nsols , asSolution = sol , asLog = ascLog asc } ganeti-3.1.0~rc2/src/Ganeti/HTools/Cluster/Evacuate.hs000064400000000000000000000451071476477700300225330ustar00rootroot00000000000000{-| Implementation of node evacuation -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Cluster.Evacuate ( EvacSolution(..) , nodeEvacInstance , tryNodeEvac , emptyEvacSolution , updateEvacSolution , reverseEvacSolution ) where import Control.Monad.Fail (MonadFail) import qualified Data.IntSet as IntSet import qualified Data.List as List import Data.Maybe (fromJust) import Ganeti.BasicTypes import Ganeti.HTools.AlgorithmParams (AlgorithmOptions(..)) import Ganeti.HTools.Cluster.Metrics (compCVNodes) import Ganeti.HTools.Cluster.Moves (applyMoveEx) import Ganeti.HTools.Cluster.Utils ( splitCluster, iMoveToJob , instancePriGroup, availableGroupNodes) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import Ganeti.HTools.Types import qualified Ganeti.OpCodes as OpCodes import Ganeti.Types -- | Node evacuation/group change iallocator result type. This result -- type consists of actual opcodes (a restricted subset) that are -- transmitted back to Ganeti. data EvacSolution = EvacSolution { esMoved :: [(Idx, Gdx, [Ndx])] -- ^ Instances moved successfully , esFailed :: [(Idx, String)] -- ^ Instances which were not -- relocated , esOpCodes :: [[OpCodes.OpCode]] -- ^ List of jobs } deriving (Show) -- | The empty evac solution. emptyEvacSolution :: EvacSolution emptyEvacSolution = EvacSolution { esMoved = [] , esFailed = [] , esOpCodes = [] } -- | Reverses an evacuation solution. -- -- Rationale: we always concat the results to the top of the lists, so -- for proper jobset execution, we should reverse all lists. reverseEvacSolution :: EvacSolution -> EvacSolution reverseEvacSolution (EvacSolution f m o) = EvacSolution (reverse f) (reverse m) (reverse o) -- | A simple type for the running solution of evacuations. type EvacInnerState = Either String (Node.List, Instance.Instance, Score, Ndx) -- | Function which fails if the requested mode is change secondary. -- -- This is useful since except DRBD, no other disk template can -- execute change secondary; thus, we can just call this function -- instead of always checking for secondary mode. After the call to -- this function, whatever mode we have is just a primary change. failOnSecondaryChange :: (MonadFail m) => EvacMode -> DiskTemplate -> m () failOnSecondaryChange ChangeSecondary dt = fail $ "Instances with disk template '" ++ diskTemplateToRaw dt ++ "' can't execute change secondary" failOnSecondaryChange _ _ = return () -- | Inner fold function for changing one node of an instance. -- -- Depending on the instance disk template, this will either change -- the secondary (for DRBD) or the primary node (for shared -- storage). However, the operation is generic otherwise. -- -- The running solution is either a @Left String@, which means we -- don't have yet a working solution, or a @Right (...)@, which -- represents a valid solution; it holds the modified node list, the -- modified instance (after evacuation), the score of that solution, -- and the new secondary node index. evacOneNodeInner :: AlgorithmOptions -> Node.List -- ^ Cluster node list -> Instance.Instance -- ^ Instance being evacuated -> Gdx -- ^ The group index of the instance -> (Ndx -> IMove) -- ^ Operation constructor -> EvacInnerState -- ^ Current best solution -> Ndx -- ^ Node we're evaluating as target -> EvacInnerState -- ^ New best solution evacOneNodeInner opts nl inst gdx op_fn accu ndx = case applyMoveEx (algIgnoreSoftErrors opts) nl inst (op_fn ndx) of Bad fm -> let fail_msg = " Node " ++ Container.nameOf nl ndx ++ " failed: " ++ show fm ++ ";" in either (Left . (++ fail_msg)) Right accu Ok (nl', inst', _, _) -> let nodes = Container.elems nl' -- The fromJust below is ugly (it can fail nastily), but -- at this point we should have any internal mismatches, -- and adding a monad here would be quite involved grpnodes = fromJust (gdx `lookup` Node.computeGroups nodes) new_cv = compCVNodes grpnodes new_accu = Right (nl', inst', new_cv, ndx) in case accu of Left _ -> new_accu Right (_, _, old_cv, _) -> if old_cv < new_cv then accu else new_accu -- | Generic function for changing one node of an instance. -- -- This is similar to 'nodeEvacInstance' but will be used in a few of -- its sub-patterns. It folds the inner function 'evacOneNodeInner' -- over the list of available nodes, which results in the best choice -- for relocation. evacOneNodeOnly :: AlgorithmOptions -> Node.List -- ^ The node list (cluster-wide) -> Instance.List -- ^ Instance list (cluster-wide) -> Instance.Instance -- ^ The instance to be evacuated -> Gdx -- ^ The group we're targetting -> [Ndx] -- ^ The list of available nodes -- for allocation -> Result (Node.List, Instance.List, [OpCodes.OpCode]) evacOneNodeOnly opts nl il inst gdx avail_nodes = do op_fn <- case Instance.mirrorType inst of MirrorNone -> Bad "Can't relocate/evacuate non-mirrored instances" MirrorInternal -> Ok ReplaceSecondary MirrorExternal -> Ok FailoverToAny (nl', inst', _, ndx) <- annotateResult "Can't find any good node" . eitherToResult $ List.foldl' (evacOneNodeInner opts nl inst gdx op_fn) (Left "") avail_nodes let idx = Instance.idx inst il' = Container.add idx inst' il ops = iMoveToJob nl' il' idx (op_fn ndx) return (nl', il', ops) -- | Compute result of changing all nodes of a DRBD instance. -- -- Given the target primary and secondary node (which might be in a -- different group or not), this function will 'execute' all the -- required steps and assuming all operations succceed, will return -- the modified node and instance lists, the opcodes needed for this -- and the new group score. evacDrbdAllInner :: AlgorithmOptions -> Node.List -- ^ Cluster node list -> Instance.List -- ^ Cluster instance list -> Instance.Instance -- ^ The instance to be moved -> Gdx -- ^ The target group index -- (which can differ from the -- current group of the -- instance) -> (Ndx, Ndx) -- ^ Tuple of new -- primary\/secondary nodes -> Result (Node.List, Instance.List, [OpCodes.OpCode], Score) evacDrbdAllInner opts nl il inst gdx (t_pdx, t_sdx) = do let primary = Container.find (Instance.pNode inst) nl idx = Instance.idx inst apMove = applyMoveEx $ algIgnoreSoftErrors opts -- if the primary is offline, then we first failover (nl1, inst1, ops1) <- if Node.offline primary then do (nl', inst', _, _) <- annotateResult "Failing over to the secondary" . opToResult $ apMove nl inst Failover return (nl', inst', [Failover]) else return (nl, inst, []) let (o1, o2, o3) = (ReplaceSecondary t_pdx, Failover, ReplaceSecondary t_sdx) -- we now need to execute a replace secondary to the future -- primary node (nl2, inst2, _, _) <- annotateResult "Changing secondary to new primary" . opToResult $ apMove nl1 inst1 o1 let ops2 = o1:ops1 -- we now execute another failover, the primary stays fixed now (nl3, inst3, _, _) <- annotateResult "Failing over to new primary" . opToResult $ apMove nl2 inst2 o2 let ops3 = o2:ops2 -- and finally another replace secondary, to the final secondary (nl4, inst4, _, _) <- annotateResult "Changing secondary to final secondary" . opToResult $ apMove nl3 inst3 o3 let ops4 = o3:ops3 il' = Container.add idx inst4 il ops = concatMap (iMoveToJob nl4 il' idx) $ reverse ops4 let nodes = Container.elems nl4 -- The fromJust below is ugly (it can fail nastily), but -- at this point we should have any internal mismatches, -- and adding a monad here would be quite involved grpnodes = fromJust (gdx `lookup` Node.computeGroups nodes) new_cv = compCVNodes grpnodes return (nl4, il', ops, new_cv) -- | Run evacuation for a single instance. -- -- /Note:/ this function should correctly execute both intra-group -- evacuations (in all modes) and inter-group evacuations (in the -- 'ChangeAll' mode). Of course, this requires that the correct list -- of target nodes is passed. nodeEvacInstance :: AlgorithmOptions -> Node.List -- ^ The node list (cluster-wide) -> Instance.List -- ^ Instance list (cluster-wide) -> EvacMode -- ^ The evacuation mode -> Instance.Instance -- ^ The instance to be evacuated -> Gdx -- ^ The group we're targetting -> [Ndx] -- ^ The list of available nodes -- for allocation -> Result (Node.List, Instance.List, [OpCodes.OpCode]) nodeEvacInstance opts nl il mode inst@(Instance.Instance {Instance.diskTemplate = dt@DTDiskless}) gdx avail_nodes = failOnSecondaryChange mode dt >> evacOneNodeOnly opts nl il inst gdx avail_nodes nodeEvacInstance _ _ _ _ (Instance.Instance {Instance.diskTemplate = DTPlain}) _ _ = fail "Instances of type plain cannot be relocated" nodeEvacInstance _ _ _ _ (Instance.Instance {Instance.diskTemplate = DTFile}) _ _ = fail "Instances of type file cannot be relocated" nodeEvacInstance opts nl il mode inst@(Instance.Instance {Instance.diskTemplate = dt@DTSharedFile}) gdx avail_nodes = failOnSecondaryChange mode dt >> evacOneNodeOnly opts nl il inst gdx avail_nodes nodeEvacInstance opts nl il mode inst@(Instance.Instance {Instance.diskTemplate = dt@DTBlock}) gdx avail_nodes = failOnSecondaryChange mode dt >> evacOneNodeOnly opts nl il inst gdx avail_nodes nodeEvacInstance opts nl il mode inst@(Instance.Instance {Instance.diskTemplate = dt@DTRbd}) gdx avail_nodes = failOnSecondaryChange mode dt >> evacOneNodeOnly opts nl il inst gdx avail_nodes nodeEvacInstance opts nl il mode inst@(Instance.Instance {Instance.diskTemplate = dt@DTExt}) gdx avail_nodes = failOnSecondaryChange mode dt >> evacOneNodeOnly opts nl il inst gdx avail_nodes nodeEvacInstance opts nl il mode inst@(Instance.Instance {Instance.diskTemplate = dt@DTGluster}) gdx avail_nodes = failOnSecondaryChange mode dt >> evacOneNodeOnly opts nl il inst gdx avail_nodes nodeEvacInstance opts nl il ChangePrimary inst@(Instance.Instance {Instance.diskTemplate = DTDrbd8}) _ _ = do (nl', inst', _, _) <- opToResult $ applyMoveEx (algIgnoreSoftErrors opts) nl inst Failover let idx = Instance.idx inst il' = Container.add idx inst' il ops = iMoveToJob nl' il' idx Failover return (nl', il', ops) nodeEvacInstance opts nl il ChangeSecondary inst@(Instance.Instance {Instance.diskTemplate = DTDrbd8}) gdx avail_nodes = evacOneNodeOnly opts nl il inst gdx avail_nodes -- The algorithm for ChangeAll is as follows: -- -- * generate all (primary, secondary) node pairs for the target groups -- * for each pair, execute the needed moves (r:s, f, r:s) and compute -- the final node list state and group score -- * select the best choice via a foldl that uses the same Either -- String solution as the ChangeSecondary mode nodeEvacInstance opts nl il ChangeAll inst@(Instance.Instance {Instance.diskTemplate = DTDrbd8}) gdx avail_nodes = do let no_nodes = Left "no nodes available" node_pairs = [(p,s) | p <- avail_nodes, s <- avail_nodes, p /= s] (nl', il', ops, _) <- annotateResult "Can't find any good nodes for relocation" . eitherToResult $ List.foldl' (\accu nodes -> case evacDrbdAllInner opts nl il inst gdx nodes of Bad msg -> case accu of Right _ -> accu -- we don't need more details (which -- nodes, etc.) as we only selected -- this group if we can allocate on -- it, hence failures will not -- propagate out of this fold loop Left _ -> Left $ "Allocation failed: " ++ msg Ok result@(_, _, _, new_cv) -> let new_accu = Right result in case accu of Left _ -> new_accu Right (_, _, _, old_cv) -> if old_cv < new_cv then accu else new_accu ) no_nodes node_pairs return (nl', il', ops) -- | Updates the evac solution with the results of an instance -- evacuation. updateEvacSolution :: (Node.List, Instance.List, EvacSolution) -> Idx -> Result (Node.List, Instance.List, [OpCodes.OpCode]) -> (Node.List, Instance.List, EvacSolution) updateEvacSolution (nl, il, es) idx (Bad msg) = (nl, il, es { esFailed = (idx, msg):esFailed es}) updateEvacSolution (_, _, es) idx (Ok (nl, il, opcodes)) = (nl, il, es { esMoved = new_elem:esMoved es , esOpCodes = opcodes:esOpCodes es }) where inst = Container.find idx il new_elem = (idx, instancePriGroup nl inst, Instance.allNodes inst) -- | Compute the list of nodes that are to be evacuated, given a list -- of instances and an evacuation mode. nodesToEvacuate :: Instance.List -- ^ The cluster-wide instance list -> EvacMode -- ^ The evacuation mode we're using -> [Idx] -- ^ List of instance indices being evacuated -> IntSet.IntSet -- ^ Set of node indices nodesToEvacuate il mode = IntSet.delete Node.noSecondary . List.foldl' (\ns idx -> let i = Container.find idx il pdx = Instance.pNode i sdx = Instance.sNode i dt = Instance.diskTemplate i withSecondary = case dt of DTDrbd8 -> IntSet.insert sdx ns _ -> ns in case mode of ChangePrimary -> IntSet.insert pdx ns ChangeSecondary -> withSecondary ChangeAll -> IntSet.insert pdx withSecondary ) IntSet.empty -- | Node-evacuation IAllocator mode main function. tryNodeEvac :: AlgorithmOptions -> Group.List -- ^ The cluster groups -> Node.List -- ^ The node list (cluster-wide, not per group) -> Instance.List -- ^ Instance list (cluster-wide) -> EvacMode -- ^ The evacuation mode -> [Idx] -- ^ List of instance (indices) to be evacuated -> Result (Node.List, Instance.List, EvacSolution) tryNodeEvac opts _ ini_nl ini_il mode idxs = let evac_ndx = nodesToEvacuate ini_il mode idxs offline = map Node.idx . filter Node.offline $ Container.elems ini_nl excl_ndx = List.foldl' (flip IntSet.insert) evac_ndx offline group_ndx = map (\(gdx, (nl, _)) -> (gdx, map Node.idx (Container.elems nl))) $ splitCluster ini_nl ini_il (fin_nl, fin_il, esol) = List.foldl' (\state@(nl, il, _) inst -> let gdx = instancePriGroup nl inst pdx = Instance.pNode inst in updateEvacSolution state (Instance.idx inst) $ availableGroupNodes group_ndx (IntSet.insert pdx excl_ndx) gdx >>= nodeEvacInstance opts nl il mode inst gdx ) (ini_nl, ini_il, emptyEvacSolution) (map (`Container.find` ini_il) idxs) in return (fin_nl, fin_il, reverseEvacSolution esol) ganeti-3.1.0~rc2/src/Ganeti/HTools/Cluster/Metrics.hs000064400000000000000000000233211476477700300223760ustar00rootroot00000000000000{-| Implementation of the cluster metric -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Cluster.Metrics ( compCV , compCVfromStats , compCVNodes , compClusterStatistics , updateClusterStatisticsTwice , optimalCVScore , printStats ) where import Control.Monad (guard) import Data.List (partition, transpose) import Data.Maybe (fromMaybe) import Text.Printf (printf) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.PeerMap as P import Ganeti.HTools.Types import Ganeti.Utils (printTable) import Ganeti.Utils.Statistics -- | Coefficient for the total reserved memory in the cluster metric. We -- use a (local) constant here, as it is also used in the computation of -- the best possible cluster score. reservedMemRtotalCoeff :: Double reservedMemRtotalCoeff = 0.25 -- | The names and weights of the individual elements in the CV list, together -- with their statistical accumulation function and a bit to decide whether it -- is a statistics for online nodes. detailedCVInfoExt :: [((Double, String) , ([AggregateComponent] -> Statistics, Bool))] detailedCVInfoExt = [ ((0.5, "free_mem_cv"), (getStdDevStatistics, True)) , ((0.5, "free_disk_cv"), (getStdDevStatistics, True)) , ((1, "n1_cnt"), (getSumStatistics, True)) , ((1, "reserved_mem_cv"), (getStdDevStatistics, True)) , ((4, "offline_all_cnt"), (getSumStatistics, False)) , ((16, "offline_pri_cnt"), (getSumStatistics, False)) , ( (0.5, "vcpu_ratio_cv") , (getStdDevStatistics, True)) , ((1, "cpu_load_cv"), (getStdDevStatistics, True)) , ((1, "mem_load_cv"), (getStdDevStatistics, True)) , ((1, "disk_load_cv"), (getStdDevStatistics, True)) , ((1, "net_load_cv"), (getStdDevStatistics, True)) , ((2, "pri_tags_score"), (getSumStatistics, True)) , ((0.5, "spindles_cv"), (getStdDevStatistics, True)) , ((0.5, "free_mem_cv_forth"), (getStdDevStatistics, True)) , ( (0.5, "free_disk_cv_forth") , (getStdDevStatistics, True)) , ( (0.5, "vcpu_ratio_cv_forth") , (getStdDevStatistics, True)) , ((0.5, "spindles_cv_forth"), (getStdDevStatistics, True)) , ((1, "location_score"), (getSumStatistics, True)) , ( (1, "location_exclusion_score") , (getMapStatistics, True)) , ( (reservedMemRtotalCoeff, "reserved_mem_rtotal") , (getSumStatistics, True)) ] -- | Compute the lower bound of the cluster score, i.e., the sum of the minimal -- values for all cluster score values that are not 0 on a perfectly balanced -- cluster. optimalCVScore :: Node.List -> Double optimalCVScore nodelist = fromMaybe 0 $ do let nodes = Container.elems nodelist guard $ length nodes > 1 let nodeMems = map Node.tMem nodes totalMem = sum nodeMems totalMemOneLessNode = totalMem - maximum nodeMems guard $ totalMemOneLessNode > 0 let totalDrbdMem = fromIntegral . sum $ map (P.sumElems . Node.peers) nodes optimalUsage = totalDrbdMem / totalMem optimalUsageOneLessNode = totalDrbdMem / totalMemOneLessNode relativeReserved = optimalUsageOneLessNode - optimalUsage return $ reservedMemRtotalCoeff * relativeReserved -- | The names and weights of the individual elements in the CV list. detailedCVInfo :: [(Double, String)] detailedCVInfo = map fst detailedCVInfoExt -- | Holds the weights used by 'compCVNodes' for each metric. detailedCVWeights :: [Double] detailedCVWeights = map fst detailedCVInfo -- | The aggregation functions for the weights detailedCVAggregation :: [([AggregateComponent] -> Statistics, Bool)] detailedCVAggregation = map snd detailedCVInfoExt -- | The bit vector describing which parts of the statistics are -- for online nodes. detailedCVOnlineStatus :: [Bool] detailedCVOnlineStatus = map snd detailedCVAggregation -- | Compute statistical measures of a single node. compDetailedCVNode :: Node.Node -> [AggregateComponent] compDetailedCVNode node = let mem = Node.pMem node memF = Node.pMemForth node dsk = Node.pDsk node dskF = Node.pDskForth node n1 = fromIntegral $ if Node.failN1 node then length (Node.sList node) + length (Node.pList node) else 0 res = Node.pRem node ipri = fromIntegral . length $ Node.pList node isec = fromIntegral . length $ Node.sList node ioff = ipri + isec cpu = Node.pCpuEff node cpuF = Node.pCpuEffForth node DynUtil c1 m1 d1 nn1 = Node.utilLoad node DynUtil c2 m2 d2 nn2 = Node.utilPool node (c_load, m_load, d_load, n_load) = (c1/c2, m1/m2, d1/d2, nn1/nn2) pri_tags = fromIntegral $ Node.conflictingPrimaries node spindles = Node.instSpindles node / Node.hiSpindles node spindlesF = Node.instSpindlesForth node / Node.hiSpindles node location_score = fromIntegral $ Node.locationScore node location_exclusion_score = Node.instanceMap node in [ SimpleNumber mem, SimpleNumber dsk, SimpleNumber n1, SimpleNumber res , SimpleNumber ioff, SimpleNumber ipri, SimpleNumber cpu , SimpleNumber c_load, SimpleNumber m_load, SimpleNumber d_load , SimpleNumber n_load , SimpleNumber pri_tags, SimpleNumber spindles , SimpleNumber memF, SimpleNumber dskF, SimpleNumber cpuF , SimpleNumber spindlesF , SimpleNumber location_score , SpreadValues location_exclusion_score , SimpleNumber res ] -- | Compute the statistics of a cluster. compClusterStatistics :: [Node.Node] -> [Statistics] compClusterStatistics all_nodes = let (offline, nodes) = partition Node.offline all_nodes offline_values = transpose (map compDetailedCVNode offline) ++ repeat [] -- transpose of an empty list is empty and not k times the empty list, as -- would be the transpose of a 0 x k matrix online_values = transpose $ map compDetailedCVNode nodes aggregate (f, True) (onNodes, _) = f onNodes aggregate (f, False) (_, offNodes) = f offNodes in zipWith aggregate detailedCVAggregation $ zip online_values offline_values -- | Update a cluster statistics by replacing the contribution of one -- node by that of another. updateClusterStatistics :: [Statistics] -> (Node.Node, Node.Node) -> [Statistics] updateClusterStatistics stats (old, new) = let update = zip (compDetailedCVNode old) (compDetailedCVNode new) online = not $ Node.offline old updateStat forOnline stat upd = if forOnline == online then updateStatistics stat upd else stat in zipWith3 updateStat detailedCVOnlineStatus stats update -- | Update a cluster statistics twice. updateClusterStatisticsTwice :: [Statistics] -> (Node.Node, Node.Node) -> (Node.Node, Node.Node) -> [Statistics] updateClusterStatisticsTwice s a = updateClusterStatistics (updateClusterStatistics s a) -- | Compute cluster statistics compDetailedCV :: [Node.Node] -> [Double] compDetailedCV = map getStatisticValue . compClusterStatistics -- | Compute the cluster score from its statistics compCVfromStats :: [Statistics] -> Double compCVfromStats = sum . zipWith (*) detailedCVWeights . map getStatisticValue -- | Compute the /total/ variance. compCVNodes :: [Node.Node] -> Double compCVNodes = sum . zipWith (*) detailedCVWeights . compDetailedCV -- | Wrapper over 'compCVNodes' for callers that have a 'Node.List'. compCV :: Node.List -> Double compCV = compCVNodes . Container.elems -- | Shows statistics for a given node list. printStats :: String -> Node.List -> String printStats lp nl = let dcvs = compDetailedCV $ Container.elems nl (weights, names) = unzip detailedCVInfo hd = zip3 (weights ++ repeat 1) (names ++ repeat "unknown") dcvs header = [ "Field", "Value", "Weight" ] formatted = map (\(w, h, val) -> [ h , printf "%.8f" val , printf "x%.2f" w ]) hd in printTable lp header formatted $ False:repeat True ganeti-3.1.0~rc2/src/Ganeti/HTools/Cluster/Moves.hs000064400000000000000000000205341476477700300220640ustar00rootroot00000000000000{-| Implementation of instance moves in a cluster. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Cluster.Moves ( applyMoveEx , setInstanceLocationScore , move ) where import qualified Data.Set as Set import Ganeti.HTools.Types import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node -- | Extracts the node pairs for an instance. This can fail if the -- instance is single-homed. FIXME: this needs to be improved, -- together with the general enhancement for handling non-DRBD moves. instanceNodes :: Node.List -> Instance.Instance -> (Ndx, Ndx, Node.Node, Node.Node) instanceNodes nl inst = let old_pdx = Instance.pNode inst old_sdx = Instance.sNode inst old_p = Container.find old_pdx nl old_s = Container.find old_sdx nl in (old_pdx, old_sdx, old_p, old_s) -- | Sets the location score of an instance, given its primary -- and secondary node. setInstanceLocationScore :: Instance.Instance -- ^ the original instance -> Node.Node -- ^ the primary node of the -- ^ instance -> Maybe Node.Node -- ^ the secondary node of the -- ^ instance -> Instance.Instance -- ^ the instance with the -- location score updated setInstanceLocationScore t _ Nothing = t { Instance.locationScore = 0 } setInstanceLocationScore t p (Just s) = t { Instance.locationScore = Set.size $ Node.locationTags p `Set.intersection` Node.locationTags s } -- | Applies an instance move to a given node list and instance. applyMoveEx :: Bool -- ^ whether to ignore soft errors -> Node.List -> Instance.Instance -> IMove -> OpResult (Node.List, Instance.Instance, Ndx, Ndx) -- Failover (f) applyMoveEx force nl inst Failover = let (old_pdx, old_sdx, old_p, old_s) = instanceNodes nl inst int_p = Node.removePri old_p inst int_s = Node.removeSec old_s inst new_nl = do -- OpResult Node.checkMigration old_p old_s new_p <- Node.addPriEx (Node.offline old_p || force) int_s inst new_s <- Node.addSecExEx (Node.offline old_p) (Node.offline old_p || force) int_p inst old_sdx let new_inst = Instance.setBoth inst old_sdx old_pdx return (Container.addTwo old_pdx new_s old_sdx new_p nl, new_inst, old_sdx, old_pdx) in new_nl -- Failover to any (fa) applyMoveEx force nl inst (FailoverToAny new_pdx) = do let (old_pdx, old_sdx, old_pnode, _) = instanceNodes nl inst new_pnode = Container.find new_pdx nl force_failover = Node.offline old_pnode || force Node.checkMigration old_pnode new_pnode new_pnode' <- Node.addPriEx force_failover new_pnode inst let old_pnode' = Node.removePri old_pnode inst inst' = Instance.setPri inst new_pdx nl' = Container.addTwo old_pdx old_pnode' new_pdx new_pnode' nl return (nl', inst', new_pdx, old_sdx) -- Replace the primary (f:, r:np, f) applyMoveEx force nl inst (ReplacePrimary new_pdx) = let (old_pdx, old_sdx, old_p, old_s) = instanceNodes nl inst tgt_n = Container.find new_pdx nl int_p = Node.removePri old_p inst int_s = Node.removeSec old_s inst new_inst = Instance.setPri (setInstanceLocationScore inst tgt_n (Just int_s)) new_pdx force_p = Node.offline old_p || force new_nl = do -- OpResult -- check that the current secondary can host the instance -- during the migration Node.checkMigration old_p old_s Node.checkMigration old_s tgt_n tmp_s <- Node.addPriEx force_p int_s new_inst let tmp_s' = Node.removePri tmp_s new_inst new_p <- Node.addPriEx force_p tgt_n new_inst new_s <- Node.addSecEx force_p tmp_s' new_inst new_pdx return (Container.add new_pdx new_p $ Container.addTwo old_pdx int_p old_sdx new_s nl, new_inst, new_pdx, old_sdx) in new_nl -- Replace the secondary (r:ns) applyMoveEx force nl inst (ReplaceSecondary new_sdx) = let old_pdx = Instance.pNode inst old_sdx = Instance.sNode inst old_s = Container.find old_sdx nl tgt_n = Container.find new_sdx nl pnode = Container.find old_pdx nl pnode' = Node.removePri pnode inst int_s = Node.removeSec old_s inst force_s = Node.offline old_s || force new_inst = Instance.setSec (setInstanceLocationScore inst pnode (Just tgt_n)) new_sdx new_nl = do new_s <- Node.addSecEx force_s tgt_n new_inst old_pdx pnode'' <- Node.addPriEx True pnode' new_inst return (Container.add old_pdx pnode'' $ Container.addTwo new_sdx new_s old_sdx int_s nl, new_inst, old_pdx, new_sdx) in new_nl -- Replace the secondary and failover (r:np, f) applyMoveEx force nl inst (ReplaceAndFailover new_pdx) = let (old_pdx, old_sdx, old_p, old_s) = instanceNodes nl inst tgt_n = Container.find new_pdx nl int_p = Node.removePri old_p inst int_s = Node.removeSec old_s inst new_inst = Instance.setBoth (setInstanceLocationScore inst tgt_n (Just int_p)) new_pdx old_pdx force_s = Node.offline old_s || force new_nl = do -- OpResult Node.checkMigration old_p tgt_n new_p <- Node.addPriEx force tgt_n new_inst new_s <- Node.addSecEx force_s int_p new_inst new_pdx return (Container.add new_pdx new_p $ Container.addTwo old_pdx new_s old_sdx int_s nl, new_inst, new_pdx, old_pdx) in new_nl -- Failver and replace the secondary (f, r:ns) applyMoveEx force nl inst (FailoverAndReplace new_sdx) = let (old_pdx, old_sdx, old_p, old_s) = instanceNodes nl inst tgt_n = Container.find new_sdx nl int_p = Node.removePri old_p inst int_s = Node.removeSec old_s inst force_p = Node.offline old_p || force new_inst = Instance.setBoth (setInstanceLocationScore inst int_s (Just tgt_n)) old_sdx new_sdx new_nl = do -- OpResult Node.checkMigration old_p old_s new_p <- Node.addPriEx force_p int_s new_inst new_s <- Node.addSecEx force_p tgt_n new_inst old_sdx return (Container.add new_sdx new_s $ Container.addTwo old_sdx new_p old_pdx int_p nl, new_inst, old_sdx, new_sdx) in new_nl -- | Apply a move to an instance, ignoring soft errors. This is a -- variant of `applyMoveEx True` suitable for folding. move :: (Node.List, Instance.List) -> (Idx, IMove) -> OpResult (Node.List, Instance.List) move (nl, il) (idx, mv) = do let inst = Container.find idx il (nl', inst', _, _) <- applyMoveEx True nl inst mv return (nl', Container.add idx inst' il) ganeti-3.1.0~rc2/src/Ganeti/HTools/Cluster/Utils.hs000064400000000000000000000150021476477700300220650ustar00rootroot00000000000000{-| Utility functions for cluster operations -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Cluster.Utils ( splitCluster , iMoveToJob , instancePriGroup , availableGroupNodes ) where import Data.Maybe (fromJust) import qualified Data.IntSet as IntSet import Ganeti.BasicTypes import qualified Ganeti.Constants as C import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import Ganeti.HTools.Types import qualified Ganeti.OpCodes as OpCodes import Ganeti.Types (mkNonEmpty, mkNonNegative) -- | Splits a cluster into the component node groups. splitCluster :: Node.List -> Instance.List -> [(Gdx, (Node.List, Instance.List))] splitCluster nl il = let ngroups = Node.computeGroups (Container.elems nl) in map (\(gdx, nodes) -> let nidxs = map Node.idx nodes nodes' = zip nidxs nodes instances = Container.filter ((`elem` nidxs) . Instance.pNode) il in (gdx, (Container.fromList nodes', instances))) ngroups -- | Convert a placement into a list of OpCodes (basically a job). iMoveToJob :: Node.List -- ^ The node list; only used for node -- names, so any version is good -- (before or after the operation) -> Instance.List -- ^ The instance list; also used for -- names only -> Idx -- ^ The index of the instance being -- moved -> IMove -- ^ The actual move to be described -> [OpCodes.OpCode] -- ^ The list of opcodes equivalent to -- the given move iMoveToJob nl il idx move = let inst = Container.find idx il iname = Instance.name inst lookNode n = case mkNonEmpty (Container.nameOf nl n) of -- FIXME: convert htools codebase to non-empty strings Bad msg -> error $ "Empty node name for idx " ++ show n ++ ": " ++ msg ++ "??" Ok ne -> Just ne opF' = OpCodes.OpInstanceMigrate { OpCodes.opInstanceName = iname , OpCodes.opInstanceUuid = Nothing , OpCodes.opMigrationMode = Nothing -- default , OpCodes.opOldLiveMode = Nothing -- default as well , OpCodes.opTargetNode = Nothing -- this is drbd , OpCodes.opTargetNodeUuid = Nothing , OpCodes.opAllowRuntimeChanges = False , OpCodes.opIgnoreIpolicy = False , OpCodes.opMigrationCleanup = False , OpCodes.opIallocator = Nothing , OpCodes.opAllowFailover = True , OpCodes.opIgnoreHvversions = True } opFA n = opF { OpCodes.opTargetNode = lookNode n } -- not drbd opFforced = OpCodes.OpInstanceFailover { OpCodes.opInstanceName = iname , OpCodes.opInstanceUuid = Nothing , OpCodes.opShutdownTimeout = fromJust $ mkNonNegative C.defaultShutdownTimeout , OpCodes.opIgnoreConsistency = False , OpCodes.opTargetNode = Nothing , OpCodes.opTargetNodeUuid = Nothing , OpCodes.opIgnoreIpolicy = False , OpCodes.opIallocator = Nothing , OpCodes.opMigrationCleanup = False } opF = if Instance.forthcoming inst then opFforced else opF' opR n = OpCodes.OpInstanceReplaceDisks { OpCodes.opInstanceName = iname , OpCodes.opInstanceUuid = Nothing , OpCodes.opEarlyRelease = False , OpCodes.opIgnoreIpolicy = False , OpCodes.opReplaceDisksMode = OpCodes.ReplaceNewSecondary , OpCodes.opReplaceDisksList = [] , OpCodes.opRemoteNode = lookNode n , OpCodes.opRemoteNodeUuid = Nothing , OpCodes.opIallocator = Nothing } in case move of Failover -> [ opF ] FailoverToAny np -> [ opFA np ] ReplacePrimary np -> [ opF, opR np, opF ] ReplaceSecondary ns -> [ opR ns ] ReplaceAndFailover np -> [ opR np, opF ] FailoverAndReplace ns -> [ opF, opR ns ] -- | Computes the group of an instance per the primary node. instancePriGroup :: Node.List -> Instance.Instance -> Gdx instancePriGroup nl i = let pnode = Container.find (Instance.pNode i) nl in Node.group pnode -- | Computes the nodes in a given group which are available for -- allocation. availableGroupNodes :: [(Gdx, [Ndx])] -- ^ Group index/node index assoc list -> IntSet.IntSet -- ^ Nodes that are excluded -> Gdx -- ^ The group for which we -- query the nodes -> Result [Ndx] -- ^ List of available node indices availableGroupNodes group_nodes excl_ndx gdx = do local_nodes <- maybe (Bad $ "Can't find group with index " ++ show gdx) Ok (lookup gdx group_nodes) let avail_nodes = filter (not . flip IntSet.member excl_ndx) local_nodes return avail_nodes ganeti-3.1.0~rc2/src/Ganeti/HTools/Container.hs000064400000000000000000000057551476477700300213040ustar00rootroot00000000000000{-| Module abstracting the node and instance container implementation. This is currently implemented on top of an 'IntMap', which seems to give the best performance for our workload. -} {- Copyright (C) 2009, 2010, 2011 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Container ( -- * Types Container , Key -- * Creation , IntMap.empty , IntMap.singleton , IntMap.fromList -- * Query , IntMap.size , IntMap.null , find , IntMap.findMax , IntMap.member , IntMap.lookup -- * Update , add , addTwo , IntMap.map , IntMap.mapAccum , IntMap.filter -- * Conversion , IntMap.elems , IntMap.keys -- * Element functions , nameOf , findByName ) where import Control.Monad.Fail (MonadFail) import qualified Data.IntMap as IntMap import qualified Ganeti.HTools.Types as T -- | Our key type. type Key = IntMap.Key -- | Our container type. type Container = IntMap.IntMap -- | Locate a key in the map (must exist). find :: Key -> Container a -> a find k = (IntMap.! k) -- | Add or update one element to the map. add :: Key -> a -> Container a -> Container a add = IntMap.insert -- | Add or update two elements of the map. addTwo :: Key -> a -> Key -> a -> Container a -> Container a addTwo k1 v1 k2 v2 = add k1 v1 . add k2 v2 -- | Compute the name of an element in a container. nameOf :: (T.Element a) => Container a -> Key -> String nameOf c k = T.nameOf $ find k c -- | Find an element by name in a Container; this is a very slow function. findByName :: (T.Element a, MonadFail m) => Container a -> String -> m a findByName c n = let all_elems = IntMap.elems c result = filter ((n `elem`) . T.allNames) all_elems in case result of [item] -> return item _ -> fail $ "Wrong number of elems found with name " ++ n ganeti-3.1.0~rc2/src/Ganeti/HTools/Dedicated.hs000064400000000000000000000261721476477700300212240ustar00rootroot00000000000000{-| Implementation of special handling of dedicated clusters. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Dedicated ( isDedicated , testInstances , allocationVector , Metric , lostAllocationsMetric , allocateOnSingle , allocateOnPair , findAllocation , runDedicatedAllocation ) where import Control.Applicative (liftA2) import Control.Arrow ((&&&)) import Control.Monad (unless, liftM, foldM, mplus) import qualified Data.Foldable as F import Data.Function (on) import qualified Data.IntMap as IntMap import qualified Data.IntSet as IntSet import Data.List (sortBy, intercalate) import Ganeti.BasicTypes (iterateOk, Result, failError) import qualified Ganeti.HTools.AlgorithmParams as Alg import qualified Ganeti.HTools.Backend.IAlloc as IAlloc import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.AllocationSolution as AllocSol import qualified Ganeti.HTools.Cluster.Utils as ClusterUtils import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Loader as Loader import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Types as T -- | Given a cluster description and maybe a group name, decide -- if that group, or all allocatable groups if no group is given, -- is dedicated. isDedicated :: Loader.ClusterData -> Maybe String -> Bool isDedicated cdata maybeGroup = let groups = IntMap.keysSet . IntMap.filter (maybe ((/=) T.AllocUnallocable . Group.allocPolicy) (\name -> (==) name . Group.name) maybeGroup) $ Loader.cdGroups cdata in F.all (liftA2 (||) Node.exclStorage $ not . (`IntSet.member` groups) . Node.group) $ Loader.cdNodes cdata -- | Given a specification interval, create an instance minimally fitting -- into that interval. In other words create an instance from the lower bounds -- of the specified interval. minimallyCompliantInstance :: T.ISpec -> Instance.Instance minimallyCompliantInstance spec = Instance.create "minimalspecinstance" (T.iSpecMemorySize spec) (T.iSpecDiskSize spec) [] (T.iSpecCpuCount spec) T.Running [] False Node.noSecondary Node.noSecondary T.DTPlain (T.iSpecSpindleUse spec) [] False -- | From an instance policy get the list of test instances, in correct order, -- for which the allocation count has to be determined for the lost allocations -- metrics. testInstances :: T.IPolicy -> [Instance.Instance] testInstances = map minimallyCompliantInstance . sortBy (flip compare `on` T.iSpecDiskSize) . map T.minMaxISpecsMinSpec . T.iPolicyMinMaxISpecs -- | Given the test instances, compute the allocations vector of a node allocationVector :: [Instance.Instance] -> Node.Node -> [Int] allocationVector insts node = map (\ inst -> length $ iterateOk (`Node.addPri` inst) node) insts -- | The metric do be used in dedicated allocation. type Metric = ([Int], Int) -- | Given the test instances and an instance to be placed, compute -- the lost allocations metrics for that node, together with the -- modified node. Return Bad if it is not possible to place the -- instance on that node. lostAllocationsMetric :: Alg.AlgorithmOptions -> [Instance.Instance] -> Instance.Instance -> Node.Node -> T.OpResult (Metric, Node.Node) lostAllocationsMetric opts insts inst node = do let allocVec = allocationVector insts before = allocVec node force = Alg.algIgnoreSoftErrors opts node' <- Node.addPriEx force node inst let after = allocVec node' disk = Node.fDsk node' return ((zipWith (-) before after, disk), node') -- | Allocate an instance on a given node. allocateOnSingle :: Alg.AlgorithmOptions -> Node.List -> Instance.Instance -> T.Ndx -> T.OpResult (AllocSol.GenericAllocElement Metric) allocateOnSingle opts nl inst new_pdx = do let primary = Container.find new_pdx nl policy = Node.iPolicy primary testInst = testInstances policy excl = Node.exclStorage primary new_inst = Instance.setBoth inst new_pdx Node.noSecondary Instance.instMatchesPolicy inst policy excl (metrics, new_p) <- lostAllocationsMetric opts testInst inst primary let new_nl = Container.add new_pdx new_p nl return (new_nl, new_inst, [new_p], metrics) -- | Allocate an instance on a given pair of nodes. allocateOnPair :: Alg.AlgorithmOptions -> Node.List -> Instance.Instance -> T.Ndx -> T.Ndx -> T.OpResult (AllocSol.GenericAllocElement Metric) allocateOnPair opts nl inst pdx sdx = do let primary = Container.find pdx nl secondary = Container.find sdx nl policy = Node.iPolicy primary testInst = testInstances policy inst' = Instance.setBoth inst pdx sdx Instance.instMatchesPolicy inst policy (Node.exclStorage primary) ((lAllP, dskP), primary') <- lostAllocationsMetric opts testInst inst' primary secondary' <- Node.addSec secondary inst' pdx let lAllS = zipWith (-) (allocationVector testInst secondary) (allocationVector testInst secondary') dskS = Node.fDsk secondary' metric = (zipWith (+) lAllP lAllS, dskP + dskS) nl' = Container.addTwo pdx primary' sdx secondary' nl return (nl', inst', [primary', secondary'], metric) -- | Find an allocation for an instance on a group. findAllocation :: Alg.AlgorithmOptions -> Group.List -> Node.List -> T.Gdx -> Instance.Instance -> Int -> Result (AllocSol.GenericAllocSolution Metric, [String]) findAllocation opts mggl mgnl gdx inst count = do let nl = Container.filter ((== gdx) . Node.group) mgnl group = Container.find gdx mggl unless (Cluster.hasRequiredNetworks group inst) . failError $ "The group " ++ Group.name group ++ " is not connected to\ \ a network required by instance " ++ Instance.name inst allocNodes <- Cluster.genAllocNodes opts mggl nl count False solution <- case allocNodes of (Right []) -> fail "Not enough online nodes" (Right pairs) -> let sols = foldl AllocSol.sumAllocs AllocSol.emptyAllocSolution $ map (\(p, ss) -> foldl (\cstate -> AllocSol.concatAllocs cstate . allocateOnPair opts nl inst p) AllocSol.emptyAllocSolution ss) pairs in return $ AllocSol.genericAnnotateSolution show sols (Left []) -> fail "No online nodes" (Left nodes) -> let sols = foldl (\cstate -> AllocSol.concatAllocs cstate . allocateOnSingle opts nl inst) AllocSol.emptyAllocSolution nodes in return $ AllocSol.genericAnnotateSolution show sols return (solution, AllocSol.solutionDescription (group, return solution)) -- | Find an allocation in a suitable group. findMGAllocation :: Alg.AlgorithmOptions -> Group.List -> Node.List -> Instance.List -> Instance.Instance -> Int -> Result (AllocSol.GenericAllocSolution Metric) findMGAllocation opts gl nl il inst count = do let groups_by_idx = ClusterUtils.splitCluster nl il genSol (gdx, (nl', _)) = liftM fst $ findAllocation opts gl nl' gdx inst count sols = map (flip Container.find gl . fst &&& genSol) groups_by_idx goodSols = Cluster.sortMGResults $ Cluster.filterMGResults sols all_msgs = concatMap AllocSol.solutionDescription sols case goodSols of [] -> fail $ intercalate ", " all_msgs (final_group, final_sol):_ -> let sel_msg = "Selected group: " ++ Group.name final_group in return $ final_sol { AllocSol.asLog = sel_msg : all_msgs } -- | Handle allocation requests in the dedicated scenario. runDedicatedAllocation :: Alg.AlgorithmOptions -> Loader.Request -> (Maybe (Node.List, Instance.List), String) runDedicatedAllocation opts request = let Loader.Request rqtype (Loader.ClusterData gl nl il _ _) = request allocresult = case rqtype of Loader.Allocate inst (Cluster.AllocDetails count (Just gn)) rNds -> do gdx <- Group.idx <$> Container.findByName gl gn let opts' = opts { Alg.algRestrictToNodes = Alg.algRestrictToNodes opts `mplus` rNds } (solution, msgs) <- findAllocation opts' gl nl gdx inst count IAlloc.formatAllocate il $ solution { AllocSol.asLog = msgs } Loader.Allocate inst (Cluster.AllocDetails count Nothing) rNds -> let opts' = opts { Alg.algRestrictToNodes = Alg.algRestrictToNodes opts `mplus` rNds } in findMGAllocation opts' gl nl il inst count >>= IAlloc.formatAllocate il Loader.MultiAllocate insts -> IAlloc.formatMultiAlloc =<< foldM (\(nl', il', res) (inst, Cluster.AllocDetails count maybeGroup) -> do ares <- maybe (findMGAllocation opts gl nl' il' inst count) (\gn -> do gdx <- Group.idx <$> Container.findByName gl gn liftM fst $ findAllocation opts gl nl gdx inst count) maybeGroup let sol = AllocSol.asSolution ares nl'' = AllocSol.extractNl nl' il' sol il'' = AllocSol.updateIl il' sol return (nl'', il'', (inst, ares):res)) (nl, il, []) insts _ -> fail "Dedicated Allocation only for proper allocation requests" in IAlloc.formatIAllocResult allocresult ganeti-3.1.0~rc2/src/Ganeti/HTools/ExtLoader.hs000064400000000000000000000141721476477700300212420ustar00rootroot00000000000000{-| External data loader. This module holds the external data loading, and thus is the only one depending (via the specialized Text\/Rapi\/Luxi modules) on the actual libraries implementing the low-level protocols. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.ExtLoader ( loadExternalData , commonSuffix , maybeSaveData ) where import Control.Monad import Control.Monad.Writer (runWriterT) import Control.Exception import Data.Maybe (isJust, fromJust) import Data.Monoid (getAll) import System.FilePath import System.IO import System.Time (getClockTime) import Text.Printf (hPrintf) import Ganeti.BasicTypes import qualified Ganeti.HTools.Backend.Luxi as Luxi import qualified Ganeti.HTools.Backend.Rapi as Rapi import qualified Ganeti.HTools.Backend.Simu as Simu import qualified Ganeti.HTools.Backend.Text as Text import qualified Ganeti.HTools.Backend.IAlloc as IAlloc import qualified Ganeti.HTools.Backend.MonD as MonD import Ganeti.HTools.CLI import Ganeti.HTools.Loader (mergeData, updateMissing, ClusterData(..) , commonSuffix, clearDynU) import Ganeti.HTools.Types import Ganeti.Utils (sepSplit, tryRead, exitIfBad, exitWhen) -- | Error beautifier. wrapIO :: IO (Result a) -> IO (Result a) wrapIO = handle (\e -> return . Bad . show $ (e::IOException)) -- | Parses a user-supplied utilisation string. parseUtilisation :: String -> Result (String, DynUtil) parseUtilisation line = case sepSplit ' ' line of [name, cpu, mem, dsk, net] -> do rcpu <- tryRead name cpu rmem <- tryRead name mem rdsk <- tryRead name dsk rnet <- tryRead name net let du = DynUtil { cpuWeight = rcpu, memWeight = rmem , dskWeight = rdsk, netWeight = rnet } return (name, du) _ -> Bad $ "Cannot parse line " ++ line -- | External tool data loader from a variety of sources. loadExternalData :: Options -> IO ClusterData loadExternalData opts = do let mhost = optMaster opts lsock = optLuxi opts tfile = optDataFile opts simdata = optNodeSim opts iallocsrc = optIAllocSrc opts setRapi = mhost /= "" setLuxi = isJust lsock setSim = (not . null) simdata setFile = isJust tfile setIAllocSrc = isJust iallocsrc allSet = filter id [setRapi, setLuxi, setFile] exTags = case optExTags opts of Nothing -> [] Just etl -> map (++ ":") etl selInsts = optSelInst opts exInsts = optExInst opts exitWhen (length allSet > 1) "Only one of the rapi, luxi, and data\ \ files options should be given." util_contents <- maybe (return "") readFile (optDynuFile opts) util_data <- exitIfBad "can't parse utilisation data" . mapM parseUtilisation $ lines util_contents input_data <- case () of _ | setRapi -> wrapIO $ Rapi.loadData mhost | setLuxi -> wrapIO . Luxi.loadData $ fromJust lsock | setSim -> Simu.loadData simdata | setFile -> wrapIO . Text.loadData $ fromJust tfile -- IAlloc.loadData calls updateMissing internally because Hail does not -- loadExternalData for loading the JSON config (see wrapReadRequest). -- Here we just pass a 0 as the 'generic' call to updateMissing follows. | setIAllocSrc -> wrapIO . flip IAlloc.loadData 0 $ fromJust iallocsrc | otherwise -> return $ Bad "No backend selected! Exiting." now <- getClockTime let ignoreDynU = optIgnoreDynu opts staticNodeMem = optStaticKvmNodeMemory opts eff_u = if ignoreDynU then [] else util_data ldresult = input_data >>= (if ignoreDynU then clearDynU else return) >>= mergeData eff_u exTags selInsts exInsts now cdata <- exitIfBad "failed to load data, aborting" ldresult (cdata', ok) <- runWriterT $ if optMonD opts then MonD.queryAllMonDDCs cdata opts else return cdata exitWhen (optMonDExitMissing opts && not (getAll ok)) "Not all required data available" let (fix_msgs, nl) = updateMissing (cdNodes cdata') (cdInstances cdata') staticNodeMem unless (optVerbose opts == 0) $ maybeShowWarnings fix_msgs return cdata' {cdNodes = nl} -- | Function to save the cluster data to a file. maybeSaveData :: Maybe FilePath -- ^ The file prefix to save to -> String -- ^ The suffix (extension) to add -> String -- ^ Informational message -> ClusterData -- ^ The cluster data -> IO () maybeSaveData Nothing _ _ _ = return () maybeSaveData (Just path) ext msg cdata = do let adata = Text.serializeCluster cdata out_path = path <.> ext writeFile out_path adata hPrintf stderr "The cluster state %s has been written to file '%s'\n" msg out_path ganeti-3.1.0~rc2/src/Ganeti/HTools/GlobalN1.hs000064400000000000000000000137741476477700300207610ustar00rootroot00000000000000{-| Implementation of global N+1 redundancy -} {- Copyright (C) 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.GlobalN1 ( canEvacuateNode , redundant , redundantGrp , allocGlobalN1 ) where import Control.Monad (foldM, foldM_) import qualified Data.Foldable as Foldable import Data.Function (on) import Data.List (partition, sortBy) import Ganeti.BasicTypes (isOk, Result) import Ganeti.HTools.AlgorithmParams (AlgorithmOptions(..), defaultOptions) import Ganeti.HTools.Cluster.AllocatePrimitives (allocateOnSingle) import qualified Ganeti.HTools.Cluster.AllocationSolution as AllocSol import qualified Ganeti.HTools.Cluster.Evacuate as Evacuate import Ganeti.HTools.Cluster.Moves (move) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import Ganeti.HTools.Types ( IMove(Failover), Ndx, Gdx, Idx, opToResult, FailMode(FailN1) ) import Ganeti.Types ( DiskTemplate(DTDrbd8), diskTemplateMovable , EvacMode(ChangePrimary)) -- | Foldable function describing how a non-DRBD instance -- is to be evacuated. evac :: Gdx -> [Ndx] -> (Node.List, Instance.List) -> Idx -> Result (Node.List, Instance.List) evac gdx ndxs (nl, il) idx = do let opts = defaultOptions { algIgnoreSoftErrors = True, algEvacMode = True } inst = Container.find idx il (nl', il', _) <- Evacuate.nodeEvacInstance opts nl il ChangePrimary inst gdx ndxs return (nl', il') -- | Foldable function describing how a non-movable instance is to -- be recreated on one of the given nodes. recreate :: [Ndx] -> (Node.List, Instance.List) -> Instance.Instance -> Result (Node.List, Instance.List) recreate targetnodes (nl, il) inst = do let opts = defaultOptions { algIgnoreSoftErrors = True, algEvacMode = True } sols = foldl (\cstate -> AllocSol.concatAllocCollections cstate . allocateOnSingle opts nl inst ) AllocSol.emptyAllocCollection targetnodes sol = AllocSol.collectionToSolution FailN1 (const True) sols alloc <- maybe (fail "No solution found") return $ AllocSol.asSolution sol let il' = AllocSol.updateIl il $ Just alloc nl' = AllocSol.extractNl nl il $ Just alloc return (nl', il') -- | Decide if a node can be evacuated, i.e., all DRBD instances -- failed over and all shared/external storage instances moved off -- to other nodes. canEvacuateNode :: (Node.List, Instance.List) -> Node.Node -> Bool canEvacuateNode (nl, il) n = isOk $ do let (drbdIdxs, otherIdxs) = partition ((==) DTDrbd8 . Instance.diskTemplate . flip Container.find il) $ Node.pList n (sharedIdxs, nonMoveIdxs) = partition (diskTemplateMovable . Instance.diskTemplate . flip Container.find il) otherIdxs -- failover all DRBD instances with primaries on n (nl', il') <- opToResult . foldM move (nl, il) $ map (flip (,) Failover) drbdIdxs -- evacuate other instances let grp = Node.group n escapenodes = filter (/= Node.idx n) . map Node.idx . filter ((== grp) . Node.group) $ Container.elems nl' (nl'', il'') <- foldM (evac grp escapenodes) (nl',il') sharedIdxs let recreateInstances = sortBy (flip compare `on` Instance.mem) $ map (`Container.find` il'') nonMoveIdxs foldM_ (recreate escapenodes) (nl'', il'') recreateInstances -- | Predicate on wheter a given situation is globally N+1 redundant. redundant :: AlgorithmOptions -> Node.List -> Instance.List -> Bool redundant opts nl il = let filterFun = if algAcceptExisting opts then Container.filter (not . Node.offline) else id in Foldable.all (canEvacuateNode (nl, il)) . Container.filter (not . (`elem` algCapacityIgnoreGroups opts) . Node.group) $ filterFun nl -- | Predicate on wheter a given group is globally N+1 redundant. redundantGrp :: AlgorithmOptions -> Node.List -> Instance.List -> Gdx -> Bool redundantGrp opts nl il gdx = redundant opts (Container.filter ((==) gdx . Node.group) nl) il -- | Predicate on wheter an allocation element leads to a globally N+1 redundant -- state. allocGlobalN1 :: AlgorithmOptions -> Node.List -- ^ the original list of nodes -> Instance.List -- ^ the original list of instances -> AllocSol.GenericAllocElement a -> Bool allocGlobalN1 opts nl il alloc = let il' = AllocSol.updateIl il $ Just alloc nl' = AllocSol.extractNl nl il $ Just alloc in redundant opts nl' il' ganeti-3.1.0~rc2/src/Ganeti/HTools/Graph.hs000064400000000000000000000205531476477700300204140ustar00rootroot00000000000000{-| Algorithms on Graphs. This module contains a few graph algorithms and the transoformations needed for them to be used on nodes. For more information about Graph Coloring see: LF-coloring is described in: Welsh, D. J. A.; Powell, M. B. (1967), "An upper bound for the chromatic number of a graph and its application to timetabling problems", The Computer Journal 10 (1): 85-86, doi:10.1093/comjnl/10.1.85 DSatur is described in: Brelaz, D. (1979), "New methods to color the vertices of a graph", Communications of the ACM 22 (4): 251-256, doi:10.1145/359094.359101 Also interesting: Klotz, W. (2002). Graph coloring algorithms. Mathematics Report, Technical University Clausthal, 1-9. -} {- Copyright (C) 2012, 2013, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Graph ( -- * Types Color , VertColorMap , ColorVertMap -- * Creation , emptyVertColorMap -- * Coloring , colorInOrder , colorLF , colorDsatur , colorDcolor , isColorable -- * Color map transformations , colorVertMap -- * Vertex characteristics , verticesByDegreeDesc , verticesByDegreeAsc , neighbors , hasLoop , isUndirected ) where import Data.Maybe import Data.Ord import Data.List import qualified Data.IntMap as IntMap import qualified Data.IntSet as IntSet import qualified Data.Graph as Graph import qualified Data.Array as Array -- * Type declarations -- | Node colors. type Color = Int -- | Saturation: number of colored neighbors. type Satur = Int -- | Vertex to Color association. type VertColorMap = IntMap.IntMap Color -- | Color to Vertex association. type ColorVertMap = IntMap.IntMap [Int] -- * Vertices characteristics -- | (vertex, degree) tuples on a graph. verticesDegree :: Graph.Graph -> [(Graph.Vertex, Int)] verticesDegree g = Array.assocs $ Graph.outdegree g -- | vertices of a graph, sorted by ascending degree. verticesByDegreeDesc :: Graph.Graph -> [Graph.Vertex] verticesByDegreeDesc g = map fst . sortBy (flip (comparing snd)) $ verticesDegree g -- | vertices of a graph, sorted by descending degree. verticesByDegreeAsc :: Graph.Graph -> [Graph.Vertex] verticesByDegreeAsc g = map fst . sortBy (comparing snd) $ verticesDegree g -- | Get the neighbors of a vertex. neighbors :: Graph.Graph -> Graph.Vertex -> [Graph.Vertex] neighbors g v = g Array.! v -- | Check whether a graph has no loops. -- (vertices connected to themselves) hasLoop :: Graph.Graph -> Bool hasLoop g = any vLoops $ Graph.vertices g where vLoops v = v `elem` neighbors g v -- | Check whether a graph is undirected isUndirected :: Graph.Graph -> Bool isUndirected g = (sort . Graph.edges) g == (sort . Graph.edges . Graph.transposeG) g -- * Coloring -- | Empty color map. emptyVertColorMap :: VertColorMap emptyVertColorMap = IntMap.empty -- | Check whether a graph is colorable. isColorable :: Graph.Graph -> Bool isColorable g = isUndirected g && not (hasLoop g) -- | Get the colors of a list of vertices. -- Any uncolored vertices are ignored. verticesColors :: VertColorMap -> [Graph.Vertex] -> [Color] verticesColors cMap = mapMaybe (`IntMap.lookup` cMap) -- | Get the set of colors of a list of vertices. -- Any uncolored vertices are ignored. verticesColorSet :: VertColorMap -> [Graph.Vertex] -> IntSet.IntSet verticesColorSet cMap = IntSet.fromList . verticesColors cMap -- | Get the colors of the neighbors of a vertex. neighColors :: Graph.Graph -> VertColorMap -> Graph.Vertex -> [Color] neighColors g cMap v = verticesColors cMap $ neighbors g v -- | Color one node. colorNode :: Graph.Graph -> VertColorMap -> Graph.Vertex -> Color colorNode g cMap v = case filter (`notElem` neighColors g cMap v) [0..] of c:_ -> c [] -> error $ "colorNode: " ++ "we excluded finitely many colors from an infinite list, " ++ "thus the remaining list should be non-empty" -- | Color a node returning the updated color map. colorNodeInMap :: Graph.Graph -> Graph.Vertex -> VertColorMap -> VertColorMap colorNodeInMap g v cMap = IntMap.insert v newcolor cMap where newcolor = colorNode g cMap v -- | Color greedily all nodes in the given order. colorInOrder :: Graph.Graph -> [Graph.Vertex] -> VertColorMap colorInOrder g = foldr (colorNodeInMap g) emptyVertColorMap -- | Color greedily all nodes, larger first. colorLF :: Graph.Graph -> VertColorMap colorLF g = colorInOrder g $ verticesByDegreeAsc g -- | (vertex, (saturation, degree)) for a vertex. vertexSaturation :: Graph.Graph -> VertColorMap -> Graph.Vertex -> (Graph.Vertex, (Satur, Int)) vertexSaturation g cMap v = (v, (IntSet.size (verticesColorSet cMap neigh), length neigh)) where neigh = neighbors g v -- | (vertex, (colordegree, degree)) for a vertex. vertexColorDegree :: Graph.Graph -> VertColorMap -> Graph.Vertex -> (Graph.Vertex, (Int, Int)) vertexColorDegree g cMap v = (v, (length (verticesColors cMap neigh), length neigh)) where neigh = neighbors g v -- | Color all nodes in a dynamic order. -- We have a list of vertices still uncolored, and at each round we -- choose&delete one vertex among the remaining ones. A helper function -- is used to induce an order so that the next vertex can be chosen. colorDynamicOrder :: Ord a => (Graph.Graph -> VertColorMap -> Graph.Vertex -> (Graph.Vertex, a)) -- ^ Helper to induce the choice -> Graph.Graph -- ^ Target graph -> VertColorMap -- ^ Accumulating vertex color map -> [Graph.Vertex] -- ^ List of remaining vertices -> VertColorMap -- ^ Output vertex color map colorDynamicOrder _ _ cMap [] = cMap colorDynamicOrder ordind g cMap l = colorDynamicOrder ordind g newmap newlist where newmap = colorNodeInMap g choosen cMap choosen = fst . maximumBy (comparing snd) $ ordlist ordlist = map (ordind g cMap) l newlist = delete choosen l -- | Color greedily all nodes, highest number of colored neighbors, then -- highest degree. This is slower than "colorLF" as we must dynamically -- recalculate which node to color next among all remaining ones but -- produces better results. colorDcolor :: Graph.Graph -> VertColorMap colorDcolor g = colorDynamicOrder vertexColorDegree g emptyVertColorMap $ Graph.vertices g -- | Color greedily all nodes, highest saturation, then highest degree. -- This is slower than "colorLF" as we must dynamically recalculate -- which node to color next among all remaining ones but produces better -- results. colorDsatur :: Graph.Graph -> VertColorMap colorDsatur g = colorDynamicOrder vertexSaturation g emptyVertColorMap $ Graph.vertices g -- | ColorVertMap from VertColorMap. colorVertMap :: VertColorMap -> ColorVertMap colorVertMap = IntMap.foldrWithKey (flip (IntMap.insertWith (++)) . replicate 1) IntMap.empty ganeti-3.1.0~rc2/src/Ganeti/HTools/Group.hs000064400000000000000000000073651476477700300204550ustar00rootroot00000000000000{-| Module describing a node group. -} {- Copyright (C) 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Group ( Group(..) , List , AssocList -- * Constructor , create , setIdx , isAllocable , setUnallocable ) where import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Types as T -- * Type declarations -- | The node group type. data Group = Group { name :: String -- ^ The node name , uuid :: T.GroupID -- ^ The UUID of the group , idx :: T.Gdx -- ^ Internal index for book-keeping , allocPolicy :: T.AllocPolicy -- ^ The allocation policy for this group , networks :: [T.NetworkID] -- ^ The networks connected to this group , iPolicy :: T.IPolicy -- ^ The instance policy for this group , allTags :: [String] -- ^ The tags for this group } deriving (Show, Eq) -- Note: we use the name as the alias, and the UUID as the official -- name instance T.Element Group where nameOf = uuid idxOf = idx setAlias = setName setIdx = setIdx allNames n = [name n, uuid n] -- | A simple name for the int, node association list. type AssocList = [(T.Gdx, Group)] -- | A simple name for a node map. type List = Container.Container Group -- * Initialization functions -- | Create a new group. create :: String -- ^ The node name -> T.GroupID -- ^ The UUID of the group -> T.AllocPolicy -- ^ The allocation policy for this group -> [T.NetworkID] -- ^ The networks connected to this group -> T.IPolicy -- ^ The instance policy for this group -> [String] -- ^ The tags for this group -> Group create name_init id_init apol_init nets_init ipol_init tags_init = Group { name = name_init , uuid = id_init , allocPolicy = apol_init , networks = nets_init , iPolicy = ipol_init , allTags = tags_init , idx = -1 } -- | Sets the group index. -- -- This is used only during the building of the data structures. setIdx :: Group -> T.Gdx -> Group setIdx t i = t {idx = i} -- | Changes the alias. -- -- This is used only during the building of the data structures. setName :: Group -> String -> Group setName t s = t { name = s } -- | Checks if a group is allocable. isAllocable :: Group -> Bool isAllocable = (/= T.AllocUnallocable) . allocPolicy -- | Makes the group unallocatable setUnallocable :: Group -> Group setUnallocable t = t { allocPolicy = T.AllocUnallocable } ganeti-3.1.0~rc2/src/Ganeti/HTools/Instance.hs000064400000000000000000000354431476477700300211230ustar00rootroot00000000000000{-| Module describing an instance. The instance data type holds very few fields, the algorithm intelligence is in the "Node" and "Cluster" modules. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Instance ( Instance(..) , Disk(..) , AssocList , List , create , isRunning , isOffline , notOffline , instanceDown , usesSecMem , applyIfOnline , setIdx , setName , setAlias , setPri , setSec , setBoth , setMovable , specOf , getTotalSpindles , instBelowISpec , instAboveISpec , instMatchesPolicy , shrinkByType , localStorageTemplates , hasSecondary , requiredNodes , allNodes , usesLocalStorage , mirrorType , usesMemory ) where import Control.Monad (liftM2) import qualified Data.Set as Set import Ganeti.BasicTypes import qualified Ganeti.HTools.Types as T import qualified Ganeti.HTools.Container as Container import Ganeti.HTools.Nic (Nic) import Ganeti.Utils -- * Type declarations data Disk = Disk { dskSize :: Int -- ^ Size in bytes , dskSpindles :: Maybe Int -- ^ Number of spindles } deriving (Show, Eq) -- | The instance type. data Instance = Instance { name :: String -- ^ The instance name , alias :: String -- ^ The shortened name , mem :: Int -- ^ Memory of the instance , dsk :: Int -- ^ Total disk usage of the instance , disks :: [Disk] -- ^ Sizes of the individual disks , vcpus :: Int -- ^ Number of VCPUs , runSt :: T.InstanceStatus -- ^ Original run status , pNode :: T.Ndx -- ^ Original primary node , sNode :: T.Ndx -- ^ Original secondary node , idx :: T.Idx -- ^ Internal index , util :: T.DynUtil -- ^ Dynamic resource usage , movable :: Bool -- ^ Can and should the instance be moved? , autoBalance :: Bool -- ^ Is the instance auto-balanced? , diskTemplate :: T.DiskTemplate -- ^ The disk template of the instance , spindleUse :: Int -- ^ The numbers of used spindles , allTags :: [String] -- ^ List of all instance tags , exclTags :: [String] -- ^ List of instance exclusion tags , dsrdLocTags :: Set.Set String -- ^ Instance desired location tags , locationScore :: Int -- ^ The number of common-failures between -- primary and secondary node of the instance , arPolicy :: T.AutoRepairPolicy -- ^ Instance's auto-repair policy , nics :: [Nic] -- ^ NICs of the instance , forthcoming :: Bool -- ^ Is the instance is forthcoming? } deriving (Show, Eq) instance T.Element Instance where nameOf = name idxOf = idx setAlias = setAlias setIdx = setIdx allNames n = [name n, alias n] -- | Check if instance is running. isRunning :: Instance -> Bool isRunning (Instance {runSt = T.Running}) = True isRunning (Instance {runSt = T.ErrorUp}) = True isRunning _ = False -- | Check if instance is offline. isOffline :: Instance -> Bool isOffline (Instance {runSt = T.StatusOffline}) = True isOffline _ = False -- | Helper to check if the instance is not offline. notOffline :: Instance -> Bool notOffline = not . isOffline -- | Check if instance is down. instanceDown :: Instance -> Bool instanceDown inst | isRunning inst = False instanceDown inst | isOffline inst = False instanceDown _ = True -- | Apply the function if the instance is online. Otherwise use -- the initial value applyIfOnline :: Instance -> (a -> a) -> a -> a applyIfOnline = applyIf . notOffline -- | Helper for determining whether an instance's memory needs to be -- taken into account for secondary memory reservation. usesSecMem :: Instance -> Bool usesSecMem inst = notOffline inst && autoBalance inst -- | Constant holding the local storage templates. -- -- /Note:/ Currently Ganeti only exports node total/free disk space -- for LVM-based storage; file-based storage is ignored in this model, -- so even though file-based storage uses in reality disk space on the -- node, in our model it won't affect it and we can't compute whether -- there is enough disk space for a file-based instance. Therefore we -- will treat this template as \'foreign\' storage. localStorageTemplates :: [T.DiskTemplate] localStorageTemplates = [ T.DTDrbd8, T.DTPlain ] -- | Constant holding the movable disk templates. -- -- This only determines the initial 'movable' state of the -- instance. Further the movable state can be restricted more due to -- user choices, etc. movableDiskTemplates :: [T.DiskTemplate] movableDiskTemplates = [ T.DTDrbd8 , T.DTBlock , T.DTSharedFile , T.DTGluster , T.DTRbd , T.DTExt ] -- | A simple name for the int, instance association list. type AssocList = [(T.Idx, Instance)] -- | A simple name for an instance map. type List = Container.Container Instance -- * Initialization -- | Create an instance. -- -- Some parameters are not initialized by function, and must be set -- later (via 'setIdx' for example). create :: String -> Int -> Int -> [Disk] -> Int -> T.InstanceStatus -> [String] -> Bool -> T.Ndx -> T.Ndx -> T.DiskTemplate -> Int -> [Nic] -> Bool -> Instance create name_init mem_init dsk_init disks_init vcpus_init run_init tags_init auto_balance_init pn sn dt su nics_init forthcoming_init = Instance { name = name_init , alias = name_init , mem = mem_init , dsk = dsk_init , disks = disks_init , vcpus = vcpus_init , runSt = run_init , pNode = pn , sNode = sn , idx = -1 , util = T.baseUtil , movable = supportsMoves dt , autoBalance = auto_balance_init , diskTemplate = dt , spindleUse = su , allTags = tags_init , exclTags = [] , dsrdLocTags = Set.empty , locationScore = 0 , arPolicy = T.ArNotEnabled , nics = nics_init , forthcoming = forthcoming_init } -- | Changes the index. -- -- This is used only during the building of the data structures. setIdx :: Instance -- ^ The original instance -> T.Idx -- ^ New index -> Instance -- ^ The modified instance setIdx t i = t { idx = i } -- | Changes the name. -- -- This is used only during the building of the data structures. setName :: Instance -- ^ The original instance -> String -- ^ New name -> Instance -- ^ The modified instance setName t s = t { name = s, alias = s } -- | Changes the alias. -- -- This is used only during the building of the data structures. setAlias :: Instance -- ^ The original instance -> String -- ^ New alias -> Instance -- ^ The modified instance setAlias t s = t { alias = s } -- * Update functions -- | Changes the primary node of the instance. setPri :: Instance -- ^ the original instance -> T.Ndx -- ^ the new primary node -> Instance -- ^ the modified instance setPri t p = t { pNode = p } -- | Changes the secondary node of the instance. setSec :: Instance -- ^ the original instance -> T.Ndx -- ^ the new secondary node -> Instance -- ^ the modified instance setSec t s = t { sNode = s } -- | Changes both nodes of the instance. setBoth :: Instance -- ^ the original instance -> T.Ndx -- ^ new primary node index -> T.Ndx -- ^ new secondary node index -> Instance -- ^ the modified instance setBoth t p s = t { pNode = p, sNode = s } -- | Sets the movable flag on an instance. setMovable :: Instance -- ^ The original instance -> Bool -- ^ New movable flag -> Instance -- ^ The modified instance setMovable t m = t { movable = m } -- | Try to shrink the instance based on the reason why we can't -- allocate it. shrinkByType :: Instance -> T.FailMode -> Result Instance shrinkByType inst T.FailMem = let v = mem inst - T.unitMem in if v < T.unitMem then Bad "out of memory" else Ok inst { mem = v } shrinkByType inst T.FailDisk = let newdisks = [d {dskSize = dskSize d - T.unitDsk}| d <- disks inst] v = dsk inst - (length . disks $ inst) * T.unitDsk in if any (< T.unitDsk) $ map dskSize newdisks then Bad "out of disk" else Ok inst { dsk = v, disks = newdisks } shrinkByType inst T.FailCPU = let v = vcpus inst - T.unitCpu in if v < T.unitCpu then Bad "out of vcpus" else Ok inst { vcpus = v } shrinkByType inst T.FailSpindles = case disks inst of [Disk ds sp] -> case sp of Nothing -> Bad "No spindles, shouldn't have happened" Just sp' -> let v = sp' - T.unitSpindle in if v < T.unitSpindle then Bad "out of spindles" else Ok inst { disks = [Disk ds (Just v)] } d -> Bad $ "Expected one disk, but found " ++ show d shrinkByType _ f = Bad $ "Unhandled failure mode " ++ show f -- | Get the number of disk spindles getTotalSpindles :: Instance -> Maybe Int getTotalSpindles inst = foldr (liftM2 (+) . dskSpindles ) (Just 0) (disks inst) -- | Return the spec of an instance. specOf :: Instance -> T.RSpec specOf Instance { mem = m, dsk = d, vcpus = c, disks = dl } = let sp = case dl of [Disk _ (Just sp')] -> sp' _ -> 0 in T.RSpec { T.rspecCpu = c, T.rspecMem = m, T.rspecDsk = d, T.rspecSpn = sp } -- | Checks if an instance is smaller/bigger than a given spec. Returns -- OpGood for a correct spec, otherwise Bad one of the possible -- failure modes. instCompareISpec :: Ordering -> Instance-> T.ISpec -> Bool -> T.OpResult () instCompareISpec which inst ispec exclstor | which == mem inst `compare` T.iSpecMemorySize ispec = Bad T.FailMem | which `elem` map ((`compare` T.iSpecDiskSize ispec) . dskSize) (disks inst) = Bad T.FailDisk | which == vcpus inst `compare` T.iSpecCpuCount ispec = Bad T.FailCPU | exclstor && case getTotalSpindles inst of Nothing -> True Just sp_sum -> which == sp_sum `compare` T.iSpecSpindleUse ispec = Bad T.FailSpindles | not exclstor && which == spindleUse inst `compare` T.iSpecSpindleUse ispec = Bad T.FailSpindles | diskTemplate inst /= T.DTDiskless && which == length (disks inst) `compare` T.iSpecDiskCount ispec = Bad T.FailDiskCount | otherwise = Ok () -- | Checks if an instance is smaller than a given spec. instBelowISpec :: Instance -> T.ISpec -> Bool -> T.OpResult () instBelowISpec = instCompareISpec GT -- | Checks if an instance is bigger than a given spec. instAboveISpec :: Instance -> T.ISpec -> Bool -> T.OpResult () instAboveISpec = instCompareISpec LT -- | Checks if an instance matches a min/max specs pair instMatchesMinMaxSpecs :: Instance -> T.MinMaxISpecs -> Bool -> T.OpResult () instMatchesMinMaxSpecs inst minmax exclstor = do instAboveISpec inst (T.minMaxISpecsMinSpec minmax) exclstor instBelowISpec inst (T.minMaxISpecsMaxSpec minmax) exclstor -- | Checks if an instance matches any specs of a policy instMatchesSpecs :: Instance -> [T.MinMaxISpecs] -> Bool -> T.OpResult () -- Return Ok for no constraints, though this should never happen instMatchesSpecs _ [] _ = Ok () instMatchesSpecs inst minmaxes exclstor = -- The initial "Bad" should be always replaced by a real result foldr eithermatch (Bad T.FailInternal) minmaxes where eithermatch mm (Bad _) = instMatchesMinMaxSpecs inst mm exclstor eithermatch _ y@(Ok ()) = y -- | Checks if an instance matches a policy. instMatchesPolicy :: Instance -> T.IPolicy -> Bool -> T.OpResult () instMatchesPolicy inst ipol exclstor = do instMatchesSpecs inst (T.iPolicyMinMaxISpecs ipol) exclstor if diskTemplate inst `elem` T.iPolicyDiskTemplates ipol then Ok () else Bad T.FailDisk -- | Checks whether the instance uses a secondary node. -- -- /Note:/ This should be reconciled with @'sNode' == -- 'Node.noSecondary'@. hasSecondary :: Instance -> Bool hasSecondary = (== T.DTDrbd8) . diskTemplate -- | Computed the number of nodes for a given disk template. requiredNodes :: T.DiskTemplate -> Int requiredNodes T.DTDrbd8 = 2 requiredNodes _ = 1 -- | Computes all nodes of an instance. allNodes :: Instance -> [T.Ndx] allNodes inst = case diskTemplate inst of T.DTDrbd8 -> [pNode inst, sNode inst] _ -> [pNode inst] -- | Checks whether a given disk template uses local storage. usesLocalStorage :: Instance -> Bool usesLocalStorage = (`elem` localStorageTemplates) . diskTemplate -- | Checks whether a given disk template supported moves. supportsMoves :: T.DiskTemplate -> Bool supportsMoves = (`elem` movableDiskTemplates) -- | A simple wrapper over 'T.templateMirrorType'. mirrorType :: Instance -> T.MirrorType mirrorType = T.templateMirrorType . diskTemplate -- | Whether the instance uses memory on its host node. -- Depends on the `InstanceStatus` and on whether the instance is forthcoming; -- instances that aren't running or existent don't use memory. usesMemory :: Instance -> Bool usesMemory inst | forthcoming inst = False | otherwise = case runSt inst of T.StatusDown -> False T.StatusOffline -> False T.ErrorDown -> False T.ErrorUp -> True T.NodeDown -> True -- value has little meaning when node is down T.NodeOffline -> True -- value has little meaning when node is offline T.Running -> True T.UserDown -> False T.WrongNode -> True ganeti-3.1.0~rc2/src/Ganeti/HTools/Loader.hs000064400000000000000000000443061476477700300205630ustar00rootroot00000000000000{-| Generic data loader. This module holds the common code for parsing the input data after it has been loaded from external sources. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Loader ( mergeData , clearDynU , updateMissing , updateMemStat , assignIndices , setMaster , lookupNode , lookupInstance , lookupGroup , eitherLive , commonSuffix , extractExTags , updateExclTags , RqType(..) , Request(..) , ClusterData(..) , isAllocationRequest , emptyCluster , extractDesiredLocations , updateDesiredLocationTags ) where import Control.Monad import Control.Monad.Fail (MonadFail) import Data.List import qualified Data.Map as M import Data.Maybe import qualified Data.Set as Set import Text.Printf (printf) import System.Time (ClockTime(..)) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.Moves as Moves import Ganeti.BasicTypes import qualified Ganeti.HTools.Tags as Tags import qualified Ganeti.HTools.Tags.Constants as TagsC import Ganeti.HTools.Types import Ganeti.Utils import Ganeti.Types (EvacMode, Hypervisor(..)) -- * Types {-| The iallocator request type. This type denotes what request we got from Ganeti and also holds request-specific fields. -} data RqType = Allocate Instance.Instance Cluster.AllocDetails (Maybe [String]) -- ^ A new instance allocation, maybe with allocation restrictions | AllocateSecondary Idx -- ^ Find a suitable -- secondary node for disk -- conversion | Relocate Idx Int [Ndx] -- ^ Choose a new -- secondary node | NodeEvacuate [Idx] EvacMode -- ^ node-evacuate mode | ChangeGroup [Gdx] [Idx] -- ^ Multi-relocate mode | MultiAllocate [(Instance.Instance, Cluster.AllocDetails)] -- ^ Multi-allocate mode deriving (Show) -- | A complete request, as received from Ganeti. data Request = Request RqType ClusterData deriving (Show) -- | Decide whether a request asks to allocate new instances; if so, also -- return the desired node group, if a unique node group is specified. -- That is, return `Nothing` if the request is not an allocation request, -- `Just Nothing`, if it is an Allocation request, but there is no unique -- group specified, and return `Just (Just g)` if it is an allocation request -- uniquely requesting Group `g`. isAllocationRequest :: RqType -> Maybe (Maybe String) isAllocationRequest (Allocate _ (Cluster.AllocDetails _ grp) _) = Just grp isAllocationRequest (MultiAllocate reqs) = Just $ case ordNub . catMaybes $ map (\(_, Cluster.AllocDetails _ grp) -> grp) reqs of [grp] -> Just grp _ -> Nothing isAllocationRequest _ = Nothing -- | The cluster state. data ClusterData = ClusterData { cdGroups :: Group.List -- ^ The node group list , cdNodes :: Node.List -- ^ The node list , cdInstances :: Instance.List -- ^ The instance list , cdTags :: [String] -- ^ The cluster tags , cdIPolicy :: IPolicy -- ^ The cluster instance policy } deriving (Show, Eq) -- | An empty cluster. emptyCluster :: ClusterData emptyCluster = ClusterData Container.empty Container.empty Container.empty [] defIPolicy -- * Functions -- | Lookups a node into an assoc list. lookupNode :: (MonadFail m) => NameAssoc -> String -> String -> m Ndx lookupNode ktn inst node = maybe (fail $ "Unknown node '" ++ node ++ "' for instance " ++ inst) return $ M.lookup node ktn -- | Lookups an instance into an assoc list. lookupInstance :: (MonadFail m) => NameAssoc -> String -> m Idx lookupInstance kti inst = maybe (fail $ "Unknown instance '" ++ inst ++ "'") return $ M.lookup inst kti -- | Lookups a group into an assoc list. lookupGroup :: (MonadFail m) => NameAssoc -> String -> String -> m Gdx lookupGroup ktg nname gname = maybe (fail $ "Unknown group '" ++ gname ++ "' for node " ++ nname) return $ M.lookup gname ktg -- | Given a list of elements (and their names), assign indices to them. assignIndices :: (Element a) => [(String, a)] -> (NameAssoc, Container.Container a) assignIndices name_element = let (name_idx, idx_element) = unzip . map (\ (idx, (k, v)) -> ((k, idx), (idx, setIdx v idx))) . zip [0..] $ name_element in (M.fromList name_idx, Container.fromList idx_element) -- | Given am indexed node list, and the name of the master, mark it as such. setMaster :: (MonadFail m) => NameAssoc -> Node.List -> String -> m Node.List setMaster node_names node_idx master = do kmaster <- maybe (fail $ "Master node " ++ master ++ " unknown") return $ M.lookup master node_names let mnode = Container.find kmaster node_idx return $ Container.add kmaster (Node.setMaster mnode True) node_idx -- | Given the nodes with the location tags already set correctly, compute -- the location score for an instance. setLocationScore :: Node.List -> Instance.Instance -> Instance.Instance setLocationScore nl inst = let pnode = Container.find (Instance.pNode inst) nl snode = Container.lookup (Instance.sNode inst) nl in Moves.setInstanceLocationScore inst pnode snode -- | For each instance, add its index to its primary and secondary nodes. fixNodes :: Node.List -> Instance.Instance -> Node.List fixNodes accu inst = let pdx = Instance.pNode inst sdx = Instance.sNode inst pold = Container.find pdx accu pnew = Node.setPri pold inst ac2 = Container.add pdx pnew accu in if sdx /= Node.noSecondary then let sold = Container.find sdx accu snew = Node.setSec sold inst in Container.add sdx snew ac2 else ac2 -- | Set the node's policy to its group one. Note that this requires -- the group to exist (should have been checked before), otherwise it -- will abort with a runtime error. setNodePolicy :: Group.List -> Node.Node -> Node.Node setNodePolicy gl node = let grp = Container.find (Node.group node) gl gpol = Group.iPolicy grp in Node.setPolicy gpol node -- | Update instance with exclusion tags list. updateExclTags :: [String] -> Instance.Instance -> Instance.Instance updateExclTags tl inst = let allTags = Instance.allTags inst exclTags = filter (\tag -> any (`isPrefixOf` tag) tl) allTags in inst { Instance.exclTags = exclTags } -- | Update instance with desired location tags list. updateDesiredLocationTags :: [String] -> Instance.Instance -> Instance.Instance updateDesiredLocationTags tl inst = let allTags = Instance.allTags inst dsrdLocTags = filter (\tag -> any (`isPrefixOf` tag) tl) allTags in inst { Instance.dsrdLocTags = Set.fromList dsrdLocTags } -- | Update the movable attribute. updateMovable :: [String] -- ^ Selected instances (if not empty) -> [String] -- ^ Excluded instances -> Instance.Instance -- ^ Target Instance -> Instance.Instance -- ^ Target Instance with updated attribute updateMovable selinsts exinsts inst = if Instance.name inst `elem` exinsts || not (null selinsts || Instance.name inst `elem` selinsts) then Instance.setMovable inst False else inst -- | Disables moves for instances with a split group. disableSplitMoves :: Node.List -> Instance.Instance -> Instance.Instance disableSplitMoves nl inst = if not . isOk . Cluster.instanceGroup nl $ inst then Instance.setMovable inst False else inst -- | Set the auto-repair policy for an instance. setArPolicy :: [String] -- ^ Cluster tags -> Group.List -- ^ List of node groups -> Node.List -- ^ List of nodes -> Instance.List -- ^ List of instances -> ClockTime -- ^ Current timestamp, to evaluate ArSuspended -> Instance.List -- ^ Updated list of instances setArPolicy ctags gl nl il time = let getArPolicy' = flip getArPolicy time cpol = fromMaybe ArNotEnabled $ getArPolicy' ctags gpols = Container.map (fromMaybe cpol . getArPolicy' . Group.allTags) gl ipolfn = getArPolicy' . Instance.allTags nlookup = flip Container.find nl . Instance.pNode glookup = flip Container.find gpols . Node.group . nlookup updateInstance inst = inst { Instance.arPolicy = fromMaybe (glookup inst) $ ipolfn inst } in Container.map updateInstance il -- | Get the auto-repair policy from a list of tags. -- -- This examines the ganeti:watcher:autorepair and -- ganeti:watcher:autorepair:suspend tags to determine the policy. If none of -- these tags are present, Nothing (and not ArNotEnabled) is returned. getArPolicy :: [String] -> ClockTime -> Maybe AutoRepairPolicy getArPolicy tags time = let enabled = mapMaybe (autoRepairTypeFromRaw <=< chompPrefix TagsC.autoRepairTagEnabled) tags suspended = mapMaybe (chompPrefix TagsC.autoRepairTagSuspended) tags futureTs = filter (> time) . map (flip TOD 0) $ mapMaybe (tryRead "auto-repair suspend time") suspended in case () of -- Note how we must return ArSuspended even if "enabled" is empty, so that -- node groups or instances can suspend repairs that were enabled at an -- upper scope (cluster or node group). _ | "" `elem` suspended -> Just $ ArSuspended Forever | not $ null futureTs -> Just . ArSuspended . Until . maximum $ futureTs | not $ null enabled -> Just $ ArEnabled (minimum enabled) | otherwise -> Nothing -- | Compute the longest common suffix of a list of strings that -- starts with a dot. longestDomain :: [String] -> String longestDomain [] = "" longestDomain (x:xs) = foldr (\ suffix accu -> if all (isSuffixOf suffix) xs then suffix else accu) "" $ filter (isPrefixOf ".") (tails x) -- | Extracts the exclusion tags from the cluster configuration. extractExTags :: [String] -> [String] extractExTags = filter (not . null) . mapMaybe (chompPrefix TagsC.exTagsPrefix) -- | Extracts the desired locations from the instance tags. extractDesiredLocations :: [String] -> [String] extractDesiredLocations = filter (not . null) . mapMaybe (chompPrefix TagsC.desiredLocationPrefix) -- | Extracts the common suffix from node\/instance names. commonSuffix :: Node.List -> Instance.List -> String commonSuffix nl il = let node_names = map Node.name $ Container.elems nl inst_names = map Instance.name $ Container.elems il in longestDomain (node_names ++ inst_names) -- | Set the migration-related tags on a node given the cluster tags; -- this assumes that the node tags are already set on that node. addMigrationTags :: [String] -- ^ cluster tags -> Node.Node -> Node.Node addMigrationTags ctags node = let ntags = Node.nTags node migTags = Tags.getMigRestrictions ctags ntags rmigTags = Tags.getRecvMigRestrictions ctags ntags in Node.setRecvMigrationTags (Node.setMigrationTags node migTags) rmigTags -- | Set the location tags on a node given the cluster tags; -- this assumes that the node tags are already set on that node. addLocationTags :: [String] -- ^ cluster tags -> Node.Node -> Node.Node addLocationTags ctags node = let ntags = Node.nTags node in Node.setLocationTags node $ Tags.getLocations ctags ntags -- | Initializer function that loads the data from a node and instance -- list and massages it into the correct format. mergeData :: [(String, DynUtil)] -- ^ Instance utilisation data -> [String] -- ^ Exclusion tags -> [String] -- ^ Selected instances (if not empty) -> [String] -- ^ Excluded instances -> ClockTime -- ^ The current timestamp -> ClusterData -- ^ Data from backends -> Result ClusterData -- ^ Fixed cluster data mergeData um extags selinsts exinsts time cdata@(ClusterData gl nl il ctags _) = let il2 = setArPolicy ctags gl nl il time il3 = foldl' (\im (name, n_util) -> case Container.findByName im name of Nothing -> im -- skipping unknown instance Just inst -> let new_i = inst { Instance.util = n_util } in Container.add (Instance.idx inst) new_i im ) il2 um allextags = extags ++ extractExTags ctags dsrdLocTags = extractDesiredLocations ctags inst_names = map Instance.name $ Container.elems il3 selinst_lkp = map (lookupName inst_names) selinsts exinst_lkp = map (lookupName inst_names) exinsts lkp_unknown = filter (not . goodLookupResult) (selinst_lkp ++ exinst_lkp) selinst_names = map lrContent selinst_lkp exinst_names = map lrContent exinst_lkp node_names = map Node.name (Container.elems nl) common_suffix = longestDomain (node_names ++ inst_names) il4 = Container.map (computeAlias common_suffix . updateExclTags allextags . updateDesiredLocationTags dsrdLocTags . updateMovable selinst_names exinst_names) il3 nl2 = Container.map (addLocationTags ctags) nl il5 = Container.map (setLocationScore nl2) il4 nl3 = foldl' fixNodes nl2 (Container.elems il5) nl4 = Container.map (setNodePolicy gl . computeAlias common_suffix . (`Node.buildPeers` il4)) nl3 il6 = Container.map (disableSplitMoves nl3) il5 nl5 = Container.map (addMigrationTags ctags) nl4 in if' (null lkp_unknown) (Ok cdata { cdNodes = nl5, cdInstances = il6 }) (Bad $ "Unknown instance(s): " ++ show(map lrContent lkp_unknown)) -- | In a cluster description, clear dynamic utilisation information. clearDynU :: ClusterData -> Result ClusterData clearDynU cdata@(ClusterData _ _ il _ _) = let il2 = Container.map (\ inst -> inst {Instance.util = zeroUtil }) il in Ok cdata { cdInstances = il2 } -- | Update cluster data to use static node memory on KVM. setStaticKvmNodeMem :: Node.List -- ^ Nodes to update -> Int -- ^ Static node size -> Node.List -- ^ Updated nodes setStaticKvmNodeMem nl static_node_mem = let updateNM n | Node.hypervisor n == Just Kvm = n { Node.nMem = static_node_mem } | otherwise = n in if static_node_mem > 0 then Container.map updateNM nl else nl -- | Update node memory stat based on instance list. updateMemStat :: Node.Node -> Instance.List -> Node.Node updateMemStat node il = let node2 = node { Node.iMem = nodeImem node il } node3 = node2 { Node.xMem = Node.missingMem node2 } in node3 { Node.pMem = fromIntegral (Node.unallocatedMem node3) / Node.tMem node3 } -- | Check the cluster for memory/disk allocation consistency and update stats. updateMissing :: Node.List -- ^ All nodes in the cluster -> Instance.List -- ^ All instances in the cluster -> Int -- ^ Static node memory for KVM -> ([String], Node.List) -- ^ Pair of errors, update node list updateMissing nl il static_node_mem = -- This overrides node mem on KVM as loaded from backend. Ganeti 2.17 -- handles this using obtainNodeMemory. let nl2 = setStaticKvmNodeMem nl static_node_mem updateSingle msgs node = let nname = Node.name node newn = updateMemStat node il delta_mem = Node.xMem newn delta_dsk = truncate (Node.tDsk node) - Node.fDsk node - nodeIdsk node il umsg1 = if delta_mem > 512 || delta_dsk > 1024 then printf "node %s is missing %d MB ram and %d GB disk" nname delta_mem (delta_dsk `div` 1024):msgs else msgs in (umsg1, newn) in Container.mapAccum updateSingle [] nl2 -- | Compute the amount of memory used by primary instances on a node. nodeImem :: Node.Node -> Instance.List -> Int nodeImem node il = let rfind = flip Container.find il il' = map rfind $ Node.pList node oil' = filter Instance.usesMemory il' in sum . map Instance.mem $ oil' -- | Compute the amount of disk used by instances on a node (either primary -- or secondary). nodeIdsk :: Node.Node -> Instance.List -> Int nodeIdsk node il = let rfind = flip Container.find il in sum . map (Instance.dsk . rfind) $ Node.pList node ++ Node.sList node -- | Get live information or a default value eitherLive :: (MonadFail m) => Bool -> a -> m a -> m a eitherLive True _ live_data = live_data eitherLive False def_data _ = return def_data ganeti-3.1.0~rc2/src/Ganeti/HTools/Nic.hs000064400000000000000000000065341476477700300200670ustar00rootroot00000000000000{-| Module describing an NIC. The NIC data type only holds data about a NIC, but does not provide any logic. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Nic ( Nic(..) , Mode(..) , List , create ) where import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Types as T -- * Type declarations data Mode = Bridged | Routed | OpenVSwitch deriving (Show, Eq) -- | The NIC type. -- -- It holds the data for a NIC as it is provided via the IAllocator protocol -- for an instance creation request. All data in those request is optional, -- that's why all fields are Maybe's. -- -- TODO: Another name might be more appropriate for this type, as for example -- RequestedNic. But this type is used as a field in the Instance type, which -- is not named RequestedInstance, so such a name would be weird. PartialNic -- already exists in Objects, but doesn't fit the bill here, as it contains -- a required field (mac). Objects and the types therein are subject to being -- reworked, so until then this type is left as is. data Nic = Nic { mac :: Maybe String -- ^ MAC address of the NIC , ip :: Maybe String -- ^ IP address of the NIC , mode :: Maybe Mode -- ^ the mode the NIC operates in , link :: Maybe String -- ^ the link of the NIC , bridge :: Maybe String -- ^ the bridge this NIC is connected to if -- the mode is Bridged , network :: Maybe T.NetworkID -- ^ network UUID if this NIC is connected -- to a network } deriving (Show, Eq) -- | A simple name for an instance map. type List = Container.Container Nic -- * Initialization -- | Create a NIC. -- create :: Maybe String -> Maybe String -> Maybe Mode -> Maybe String -> Maybe String -> Maybe T.NetworkID -> Nic create mac_init ip_init mode_init link_init bridge_init network_init = Nic { mac = mac_init , ip = ip_init , mode = mode_init , link = link_init , bridge = bridge_init , network = network_init } ganeti-3.1.0~rc2/src/Ganeti/HTools/Node.hs000064400000000000000000001574661476477700300202560ustar00rootroot00000000000000{-| Module describing a node. All updates are functional (copy-based) and return a new node with updated value. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Node ( Node(..) , List , pCpuEff , pCpuEffForth -- * Constructor , create -- ** Finalization after data loading , buildPeers , setIdx , setAlias , setOffline , setPri , calcFmemOfflineOrForthcoming , setSec , setMaster , setNodeTags , setMdsk , setMcpu , setPolicy , setCpuSpeed , setMigrationTags , setRecvMigrationTags , setLocationTags , setHypervisor -- * Tag maps , addTags , delTags , rejectAddTags -- * Diagnostic commands , getPolicyHealth -- * Instance (re)location , removePri , removeSec , addPri , addPriEx , addSec , addSecEx , addSecExEx , checkMigration -- * Stats , availDisk , availMem , missingMem , unallocatedMem , recordedFreeMem , availCpu , iDsk , conflictingPrimaries -- * Generate OpCodes , genPowerOnOpCodes , genPowerOffOpCodes , genAddTagsOpCode -- * Formatting , defaultFields , showHeader , showField , list -- * Misc stuff , AssocList , noSecondary , computeGroups , mkNodeGraph , mkRebootNodeGraph , haveExclStorage ) where import Control.Monad (liftM, liftM2) import Control.Monad.Fail (MonadFail) import qualified Data.Foldable as Foldable import Data.Function (on) import qualified Data.Graph as Graph import qualified Data.IntMap as IntMap import qualified Data.List.NonEmpty as NonEmpty import qualified Data.List as List import qualified Data.Map as Map import Data.Ord (comparing) import qualified Data.Set as Set import Text.Printf (printf) import qualified Ganeti.Constants as C import qualified Ganeti.OpCodes as OpCodes import Ganeti.Types (Hypervisor(..), OobCommand(..), TagKind(..), mkNonEmpty) import Ganeti.HTools.Container (Container) import qualified Ganeti.HTools.Container as Container import Ganeti.HTools.Instance (Instance) import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.PeerMap as P import Ganeti.BasicTypes import qualified Ganeti.HTools.Types as T -- * Type declarations -- | The tag map type. type TagMap = Map.Map String Int -- | The node type. data Node = Node { name :: String -- ^ The node name , alias :: String -- ^ The shortened name (for display purposes) , tMem :: Double -- ^ Total memory (MiB) (state-of-world) , nMem :: Int -- ^ Node memory (MiB) (state-of-record) , iMem :: Int -- ^ Instance memory (MiB) (state-of-record) , fMem :: Int -- ^ Free memory (MiB) (state-of-world) , fMemForth :: Int -- ^ Free memory (MiB) including forthcoming -- instances TODO: Use state of record calculations -- for forthcoming instances (see unallocatedMem) , xMem :: Int -- ^ Unaccounted memory (MiB) , tDsk :: Double -- ^ Total disk space (MiB) , fDsk :: Int -- ^ Free disk space (MiB) , fDskForth :: Int -- ^ Free disk space (MiB) including forthcoming -- instances , tCpu :: Double -- ^ Total CPU count , tCpuSpeed :: Double -- ^ Relative CPU speed , nCpu :: Int -- ^ VCPUs used by the node OS , uCpu :: Int -- ^ Used VCPU count , uCpuForth :: Int -- ^ Used VCPU count including forthcoming instances , tSpindles :: Int -- ^ Node spindles (spindle_count node parameter, -- or actual spindles, see note below) , fSpindles :: Int -- ^ Free spindles (see note below) , fSpindlesForth :: Int -- ^ Free spindles (see note below) including -- forthcoming instances , pList :: [T.Idx] -- ^ List of primary instance indices , pListForth :: [T.Idx] -- ^ List of primary instance indices including -- forthcoming instances , sList :: [T.Idx] -- ^ List of secondary instance indices , sListForth :: [T.Idx] -- ^ List of secondary instance indices including -- forthcoming instances , idx :: T.Ndx -- ^ Internal index for book-keeping , peers :: P.PeerMap -- ^ Pnode to instance mapping , failN1 :: Bool -- ^ Whether the node has failed n1 , failN1Forth :: Bool -- ^ Whether the node has failed n1, including -- forthcoming instances , rMem :: Int -- ^ Maximum memory needed for failover by -- primaries of this node , rMemForth :: Int -- ^ Maximum memory needed for failover by -- primaries of this node, including forthcoming -- instances , pMem :: Double -- ^ Percent of free memory , pMemForth :: Double -- ^ Percent of free memory including forthcoming -- instances , pDsk :: Double -- ^ Percent of free disk , pDskForth :: Double -- ^ Percent of free disk including forthcoming -- instances , pRem :: Double -- ^ Percent of reserved memory , pRemForth :: Double -- ^ Percent of reserved memory including -- forthcoming instances , pCpu :: Double -- ^ Ratio of virtual to physical CPUs , pCpuForth :: Double -- ^ Ratio of virtual to physical CPUs including -- forthcoming instances , mDsk :: Double -- ^ Minimum free disk ratio , loDsk :: Int -- ^ Autocomputed from mDsk low disk -- threshold , hiCpu :: Int -- ^ Autocomputed from mCpu high cpu -- threshold , hiSpindles :: Double -- ^ Limit auto-computed from policy spindle_ratio -- and the node spindle count (see note below) , instSpindles :: Double -- ^ Spindles used by instances (see note below) , instSpindlesForth :: Double -- ^ Spindles used by instances (see note -- below) including forthcoming instances , offline :: Bool -- ^ Whether the node should not be used for -- allocations and skipped from score -- computations , isMaster :: Bool -- ^ Whether the node is the master node , nTags :: [String] -- ^ The node tags for this node , utilPool :: T.DynUtil -- ^ Total utilisation capacity , utilLoad :: T.DynUtil -- ^ Sum of instance utilisation , utilLoadForth :: T.DynUtil -- ^ Sum of instance utilisation, including -- forthcoming instances , pTags :: TagMap -- ^ Primary instance exclusion tags and their -- count, including forthcoming instances , group :: T.Gdx -- ^ The node's group (index) , iPolicy :: T.IPolicy -- ^ The instance policy (of the node's group) , exclStorage :: Bool -- ^ Effective value of exclusive_storage , migTags :: Set.Set String -- ^ migration-relevant tags , rmigTags :: Set.Set String -- ^ migration tags able to receive , locationTags :: Set.Set String -- ^ common-failure domains the node belongs -- to , locationScore :: Int -- ^ Sum of instance location and desired location -- scores , instanceMap :: Map.Map (String, String) Int -- ^ Number of instances with -- each exclusion/location tags -- pair , hypervisor :: Maybe Hypervisor -- ^ Active hypervisor on the node } deriving (Show, Eq) {- A note on how we handle spindles With exclusive storage spindles is a resource, so we track the number of spindles still available (fSpindles). This is the only reliable way, as some spindles could be used outside of Ganeti. When exclusive storage is off, spindles are a way to represent disk I/O pressure, and hence we track the amount used by the instances. We compare it against 'hiSpindles', computed from the instance policy, to avoid policy violations. In both cases we store the total spindles in 'tSpindles'. -} instance T.Element Node where nameOf = name idxOf = idx setAlias = setAlias setIdx = setIdx allNames n = [name n, alias n] -- | Derived parameter: ratio of virutal to physical CPUs, weighted -- by CPU speed. pCpuEff :: Node -> Double pCpuEff n = pCpu n / tCpuSpeed n -- | Derived parameter: ratio of virutal to physical CPUs, weighted -- by CPU speed and taking forthcoming instances into account. pCpuEffForth :: Node -> Double pCpuEffForth n = pCpuForth n / tCpuSpeed n -- | A simple name for the int, node association list. type AssocList = [(T.Ndx, Node)] -- | A simple name for a node map. type List = Container.Container Node -- | Constant node index for a non-moveable instance. noSecondary :: T.Ndx noSecondary = -1 -- * Helper functions -- | Add a value to a map. addTag :: (Ord k) => Map.Map k Int -> k -> Map.Map k Int addTag t s = Map.insertWith (+) s 1 t -- | Add multiple values. addTags :: (Ord k) => Map.Map k Int -> [k] -> Map.Map k Int addTags = List.foldl' addTag -- | Adjust or delete a value from a map. delTag :: (Ord k) => Map.Map k Int -> k -> Map.Map k Int delTag t s = Map.update (\v -> if v > 1 then Just (v-1) else Nothing) s t -- | Remove multiple value. delTags :: (Ord k) => Map.Map k Int -> [k] -> Map.Map k Int delTags = List.foldl' delTag -- | Check if we can add a list of tags to a tagmap. rejectAddTags :: TagMap -> [String] -> Bool rejectAddTags t = any (`Map.member` t) -- | Check how many primary instances have conflicting tags. The -- algorithm to compute this is to sum the count of all tags, then -- subtract the size of the tag map (since each tag has at least one, -- non-conflicting instance); this is equivalent to summing the -- values in the tag map minus one. conflictingPrimaries :: Node -> Int conflictingPrimaries (Node { pTags = t }) = Foldable.sum t - Map.size t -- | Helper function to increment a base value depending on the passed -- boolean argument. incIf :: (Num a) => Bool -> a -> a -> a incIf True base delta = base + delta incIf False base _ = base -- | Helper function to decrement a base value depending on the passed -- boolean argument. decIf :: (Num a) => Bool -> a -> a -> a decIf True base delta = base - delta decIf False base _ = base -- | Is exclusive storage enabled on any node? haveExclStorage :: List -> Bool haveExclStorage nl = any exclStorage $ Container.elems nl -- * Initialization functions -- | Create a new node. -- -- The index and the peers maps are empty, and will be need to be -- update later via the 'setIdx' and 'buildPeers' functions. create :: String -> Double -> Int -> Int -> Double -> Int -> Double -> Int -> Bool -> Int -> Int -> T.Gdx -> Bool -> Node create name_init mem_t_init mem_n_init mem_f_init dsk_t_init dsk_f_init cpu_t_init cpu_n_init offline_init spindles_t_init spindles_f_init group_init excl_stor = Node { name = name_init , alias = name_init , tMem = mem_t_init , nMem = mem_n_init , iMem = 0 -- updated after instances are loaded , xMem = 0 -- updated after instances are loaded , pMem = 0 -- updated after instances are loaded , fMem = mem_f_init , fMemForth = mem_f_init , tDsk = dsk_t_init , fDsk = dsk_f_init , fDskForth = dsk_f_init , tCpu = cpu_t_init , tCpuSpeed = 1 , nCpu = cpu_n_init , uCpu = cpu_n_init , uCpuForth = cpu_n_init , tSpindles = spindles_t_init , fSpindles = spindles_f_init , fSpindlesForth = spindles_f_init , pList = [] , pListForth = [] , sList = [] , sListForth = [] , failN1 = True , failN1Forth = True , idx = -1 , peers = P.empty , rMem = 0 , rMemForth = 0 , pMemForth = fromIntegral mem_f_init / mem_t_init , pDsk = if excl_stor then computePDsk spindles_f_init $ fromIntegral spindles_t_init else computePDsk dsk_f_init dsk_t_init , pDskForth = if excl_stor then computePDsk spindles_f_init $ fromIntegral spindles_t_init else computePDsk dsk_f_init dsk_t_init , pRem = 0 , pRemForth = 0 , pCpu = fromIntegral cpu_n_init / cpu_t_init , pCpuForth = fromIntegral cpu_n_init / cpu_t_init , offline = offline_init , isMaster = False , nTags = [] , mDsk = T.defReservedDiskRatio , loDsk = mDskToloDsk T.defReservedDiskRatio dsk_t_init , hiCpu = mCpuTohiCpu (T.iPolicyVcpuRatio T.defIPolicy) cpu_t_init , hiSpindles = computeHiSpindles (T.iPolicySpindleRatio T.defIPolicy) spindles_t_init , instSpindles = 0 , instSpindlesForth = 0 , utilPool = T.baseUtil , utilLoad = T.zeroUtil , utilLoadForth = T.zeroUtil , pTags = Map.empty , group = group_init , iPolicy = T.defIPolicy , exclStorage = excl_stor , migTags = Set.empty , rmigTags = Set.empty , locationTags = Set.empty , locationScore = 0 , instanceMap = Map.empty , hypervisor = Nothing } -- | Conversion formula from mDsk\/tDsk to loDsk. mDskToloDsk :: Double -> Double -> Int mDskToloDsk mval = floor . (mval *) -- | Conversion formula from mCpu\/tCpu to hiCpu. mCpuTohiCpu :: Double -> Double -> Int mCpuTohiCpu mval = floor . (mval *) -- | Conversiojn formula from spindles and spindle ratio to hiSpindles. computeHiSpindles :: Double -> Int -> Double computeHiSpindles spindle_ratio = (spindle_ratio *) . fromIntegral -- | Changes the index. -- -- This is used only during the building of the data structures. setIdx :: Node -> T.Ndx -> Node setIdx t i = t {idx = i} -- | Changes the alias. -- -- This is used only during the building of the data structures. setAlias :: Node -> String -> Node setAlias t s = t { alias = s } -- | Sets the offline attribute. setOffline :: Node -> Bool -> Node setOffline t val = t { offline = val } -- | Sets the master attribute setMaster :: Node -> Bool -> Node setMaster t val = t { isMaster = val } -- | Sets the node tags attribute setNodeTags :: Node -> [String] -> Node setNodeTags t val = t { nTags = val } -- | Set migration tags setMigrationTags :: Node -> Set.Set String -> Node setMigrationTags t val = t { migTags = val } -- | Set the migration tags a node is able to receive setRecvMigrationTags :: Node -> Set.Set String -> Node setRecvMigrationTags t val = t { rmigTags = val } -- | Set the location tags setLocationTags :: Node -> Set.Set String -> Node setLocationTags t val = t { locationTags = val } -- | Sets the hypervisor attribute. setHypervisor :: Node -> Hypervisor -> Node setHypervisor t val = t { hypervisor = Just val } -- | Sets the max disk usage ratio. setMdsk :: Node -> Double -> Node setMdsk t val = t { mDsk = val, loDsk = mDskToloDsk val (tDsk t) } -- | Sets the max cpu usage ratio. This will update the node's -- ipolicy, losing sharing (but it should be a seldomly done operation). setMcpu :: Node -> Double -> Node setMcpu t val = let new_ipol = (iPolicy t) { T.iPolicyVcpuRatio = val } in t { hiCpu = mCpuTohiCpu val (tCpu t), iPolicy = new_ipol } -- | Sets the policy. setPolicy :: T.IPolicy -> Node -> Node setPolicy pol node = node { iPolicy = pol , hiCpu = mCpuTohiCpu (T.iPolicyVcpuRatio pol) (tCpu node) , hiSpindles = computeHiSpindles (T.iPolicySpindleRatio pol) (tSpindles node) } -- | Computes the maximum reserved memory for peers from a peer map. computeMaxRes :: P.PeerMap -> P.Elem computeMaxRes = P.maxElem -- | Builds the peer map for a given node. buildPeers :: Node -> Instance.List -> Node buildPeers t il = let mdata = map (\i_idx -> let inst = Container.find i_idx il mem = if Instance.usesSecMem inst -- TODO Use usesMemory here, or change -- usesSecMem to return False on -- forthcoming instances? && not (Instance.forthcoming inst) then Instance.mem inst else 0 in (Instance.pNode inst, mem)) (sList t) pmap = P.accumArray (+) mdata new_rmem = computeMaxRes pmap new_failN1 = fMem t < new_rmem new_prem = fromIntegral new_rmem / tMem t in t { peers = pmap , failN1 = new_failN1 , rMem = new_rmem , pRem = new_prem -- TODO Set failN1Forth, rMemForth, pRemForth and peersForth. -- Calculate it from an mdata_forth here that doesn't have the -- `not (Instance.forthcoming inst)` filter. } -- | Calculate the new spindle usage calcSpindleUse :: Bool -- Action: True = adding instance, False = removing it -> Node -> Instance.Instance -> Double calcSpindleUse _ (Node {exclStorage = True}) _ = 0.0 calcSpindleUse act n@(Node {exclStorage = False}) i = f (Instance.usesLocalStorage i) (instSpindles n) (fromIntegral $ Instance.spindleUse i) where f :: Bool -> Double -> Double -> Double -- avoid monomorphism restriction f = if act then incIf else decIf -- | Calculate the new spindle usage including forthcoming instances. calcSpindleUseForth :: Bool -- Action: True = adding instance, False = removing -> Node -> Instance.Instance -> Double calcSpindleUseForth _ (Node {exclStorage = True}) _ = 0.0 calcSpindleUseForth act n@(Node {exclStorage = False}) i = f (Instance.usesLocalStorage i) (instSpindlesForth n) (fromIntegral $ Instance.spindleUse i) where f :: Bool -> Double -> Double -> Double -- avoid monomorphism restriction f = if act then incIf else decIf -- | Calculate the new number of free spindles calcNewFreeSpindles :: Bool -- Action: True = adding instance, False = removing -> Node -> Instance.Instance -> Int calcNewFreeSpindles _ (Node {exclStorage = False}) _ = 0 calcNewFreeSpindles act n@(Node {exclStorage = True}) i = case Instance.getTotalSpindles i of Nothing -> if act then -1 -- Force a spindle error, so the instance don't go here else fSpindles n -- No change, as we aren't sure Just s -> (if act then (-) else (+)) (fSpindles n) s -- | Calculate the new number of free spindles including forthcoming instances calcNewFreeSpindlesForth :: Bool -- Action: True = adding instance, -- False = removing -> Node -> Instance.Instance -> Int calcNewFreeSpindlesForth _ (Node {exclStorage = False}) _ = 0 calcNewFreeSpindlesForth act n@(Node {exclStorage = True}) i = case Instance.getTotalSpindles i of Nothing -> if act then -1 -- Force a spindle error, so the instance don't go here else fSpindlesForth n -- No change, as we aren't sure Just s -> (if act then (-) else (+)) (fSpindlesForth n) s calcFmemOfflineOrForthcoming :: Node -> Container Instance -> Int calcFmemOfflineOrForthcoming node allInstances = let nodeInstances = map (`Container.find` allInstances) (pList node) in sum . map Instance.mem . filter (not . Instance.usesMemory) $ nodeInstances -- | Calculates the desired location score of an instance, given its primary -- node. getInstanceDsrdLocScore :: Node -- ^ the primary node of the instance -> Instance.Instance -- ^ the original instance -> Int -- ^ the desired location score of the instance getInstanceDsrdLocScore p t = desiredLocationScore (Instance.dsrdLocTags t) (locationTags p) where desiredLocationScore instTags nodeTags = Set.size instTags - Set.size ( instTags `Set.intersection` nodeTags ) -- this way we get the number of unsatisfied desired locations -- | Returns list of all pairs of node location and instance -- exclusion tags. getLocationExclusionPairs :: Node -- ^ the primary node of the instance -> Instance.Instance -- ^ the instance -> [(String, String)] getLocationExclusionPairs p inst = [(loc, excl) | loc <- Set.toList (locationTags p) , excl <- Instance.exclTags inst] -- | Assigns an instance to a node as primary and update the used VCPU -- count, utilisation data, tags map and desired location score. setPri :: Node -> Instance.Instance -> Node setPri t inst -- Real instance, update real fields and forthcoming fields. | not (Instance.forthcoming inst) = updateForthcomingFields $ t { pList = Instance.idx inst:pList t , uCpu = new_count , pCpu = fromIntegral new_count / tCpu t , utilLoad = utilLoad t `T.addUtil` Instance.util inst , instSpindles = calcSpindleUse True t inst , locationScore = locationScore t + Instance.locationScore inst + getInstanceDsrdLocScore t inst , instanceMap = new_instance_map } -- Forthcoming instance, update forthcoming fields only. | otherwise = updateForthcomingOnlyFields $ updateForthcomingFields t where new_count = Instance.applyIfOnline inst (+ Instance.vcpus inst) (uCpu t) new_count_forth = Instance.applyIfOnline inst (+ Instance.vcpus inst) (uCpuForth t) new_instance_map = addTags (instanceMap t) $ getLocationExclusionPairs t inst uses_disk = Instance.usesLocalStorage inst -- Updates the *Forth fields that include real and forthcoming instances. updateForthcomingFields node = let new_fMemForth = decIf (not $ Instance.usesMemory inst) (fMemForth node) (Instance.mem inst) new_pMemForth = fromIntegral new_fMemForth / tMem node in node { pTags = addTags (pTags node) (Instance.exclTags inst) , pListForth = Instance.idx inst:pListForth node , uCpuForth = new_count_forth , pCpuForth = fromIntegral new_count_forth / tCpu node , utilLoadForth = utilLoadForth node `T.addUtil` Instance.util inst , fMemForth = new_fMemForth , pMemForth = new_pMemForth -- TODO Should this be in updateForthcomingOnlyFields? , instSpindlesForth = calcSpindleUseForth True node inst -- TODO Set failN1Forth, rMemForth, pRemForth } -- This updates the fields that we do not want to update if the instance -- is real (not forthcoming), in contrast to `updateForthcomingFields` -- which deals with the fields that have to be updated in either case. updateForthcomingOnlyFields node = let new_fDskForth = decIf uses_disk (fDskForth node) (Instance.dsk inst) new_free_sp_forth = calcNewFreeSpindlesForth True node inst new_pDskForth = computeNewPDsk node new_free_sp_forth new_fDskForth in node { fDskForth = new_fDskForth , pDskForth = new_pDskForth , fSpindlesForth = new_free_sp_forth } -- | Assigns an instance to a node as secondary and updates disk utilisation. setSec :: Node -> Instance.Instance -> Node setSec t inst -- Real instance, update real fields and forthcoming fields. | not (Instance.forthcoming inst) = updateForthcomingFields $ t { sList = Instance.idx inst:sList t , utilLoad = old_load { T.dskWeight = T.dskWeight old_load + T.dskWeight (Instance.util inst) } , instSpindles = calcSpindleUse True t inst } -- Forthcoming instance, update forthcoming fields only. | otherwise = updateForthcomingOnlyFields $ updateForthcomingFields t where old_load = utilLoad t uses_disk = Instance.usesLocalStorage inst -- Updates the *Forth fields that include real and forthcoming instances. updateForthcomingFields node = let old_load_forth = utilLoadForth node in node { sListForth = Instance.idx inst:sListForth node , utilLoadForth = old_load_forth { T.dskWeight = T.dskWeight old_load_forth + T.dskWeight (Instance.util inst) } -- TODO Should this be in updateForthcomingOnlyFields? , instSpindlesForth = calcSpindleUseForth True node inst -- TODO Set failN1Forth, rMemForth, pRemForth and peersForth } updateForthcomingOnlyFields node = let new_fDskForth = decIf uses_disk (fDskForth node) (Instance.dsk inst) new_free_sp_forth = calcNewFreeSpindlesForth True node inst new_pDskForth = computeNewPDsk node new_free_sp_forth new_fDskForth in node { fDskForth = new_fDskForth , pDskForth = new_pDskForth , fSpindlesForth = new_free_sp_forth } -- | Computes the new 'pDsk' value, handling nodes without local disk -- storage (we consider all their disk unused). computePDsk :: Int -> Double -> Double computePDsk _ 0 = 1 computePDsk free total = fromIntegral free / total -- | Computes the new 'pDsk' value, handling the exclusive storage state. computeNewPDsk :: Node -> Int -> Int -> Double computeNewPDsk node new_free_sp new_free_dsk = if exclStorage node then computePDsk new_free_sp . fromIntegral $ tSpindles node else computePDsk new_free_dsk $ tDsk node -- * Diagnostic functions -- | For a node diagnose whether it conforms with all policies. The type -- is chosen to represent that of a no-op node operation. getPolicyHealth :: Node -> T.OpResult () getPolicyHealth n = case () of _ | instSpindles n > hiSpindles n -> Bad T.FailDisk | pCpu n > T.iPolicyVcpuRatio (iPolicy n) -> Bad T.FailCPU | otherwise -> Ok () -- * Update functions -- | Set the CPU speed setCpuSpeed :: Node -> Double -> Node setCpuSpeed n f = n { tCpuSpeed = f } -- | Removes a primary instance. removePri :: Node -> Instance.Instance -> Node removePri t inst = let iname = Instance.idx inst forthcoming = Instance.forthcoming inst i_online = Instance.notOffline inst uses_disk = Instance.usesLocalStorage inst updateForthcomingFields n = let new_plist_forth = List.delete iname (pListForth n) new_mem_forth = fMemForth n + Instance.mem inst new_dsk_forth = incIf uses_disk (fDskForth n) (Instance.dsk inst) new_free_sp_forth = calcNewFreeSpindlesForth False n inst new_inst_sp_forth = calcSpindleUseForth False n inst new_mp_forth = fromIntegral new_mem_forth / tMem n new_dp_forth = computeNewPDsk n new_free_sp_forth new_dsk_forth new_ucpu_forth = decIf i_online (uCpuForth n) (Instance.vcpus inst) new_rcpu_forth = fromIntegral new_ucpu_forth / tCpu n new_load_forth = utilLoadForth n `T.subUtil` Instance.util inst in n { pTags = delTags (pTags t) (Instance.exclTags inst) , pListForth = new_plist_forth , fMemForth = new_mem_forth , fDskForth = new_dsk_forth , pMemForth = new_mp_forth , pDskForth = new_dp_forth , uCpuForth = new_ucpu_forth , pCpuForth = new_rcpu_forth , utilLoadForth = new_load_forth , instSpindlesForth = new_inst_sp_forth , fSpindlesForth = new_free_sp_forth -- TODO Set failN1Forth, rMemForth, pRemForth } in if forthcoming then updateForthcomingFields t else let new_plist = List.delete iname (pList t) (new_i_mem, new_free_mem) = prospectiveMem t inst False new_p_mem = fromIntegral new_free_mem / tMem t new_failn1 = new_free_mem <= rMem t new_dsk = incIf uses_disk (fDsk t) (Instance.dsk inst) new_free_sp = calcNewFreeSpindles False t inst new_inst_sp = calcSpindleUse False t inst new_dp = computeNewPDsk t new_free_sp new_dsk new_ucpu = decIf i_online (uCpu t) (Instance.vcpus inst) new_rcpu = fromIntegral new_ucpu / tCpu t new_load = utilLoad t `T.subUtil` Instance.util inst new_instance_map = delTags (instanceMap t) $ getLocationExclusionPairs t inst in updateForthcomingFields $ t { pList = new_plist, iMem = new_i_mem, fDsk = new_dsk , failN1 = new_failn1, pMem = new_p_mem, pDsk = new_dp , uCpu = new_ucpu, pCpu = new_rcpu, utilLoad = new_load , instSpindles = new_inst_sp, fSpindles = new_free_sp , locationScore = locationScore t - Instance.locationScore inst - getInstanceDsrdLocScore t inst , instanceMap = new_instance_map } -- | Removes a secondary instance. removeSec :: Node -> Instance.Instance -> Node removeSec t inst = let iname = Instance.idx inst forthcoming = Instance.forthcoming inst uses_disk = Instance.usesLocalStorage inst cur_dsk = fDsk t pnode = Instance.pNode inst updateForthcomingFields n = let new_slist_forth = List.delete iname (sListForth n) new_dsk_forth = incIf uses_disk (fDskForth n) (Instance.dsk inst) new_free_sp_forth = calcNewFreeSpindlesForth False n inst new_inst_sp_forth = calcSpindleUseForth False n inst new_dp_forth = computeNewPDsk n new_free_sp_forth new_dsk_forth old_load_forth = utilLoadForth n new_load_forth = old_load_forth { T.dskWeight = T.dskWeight old_load_forth - T.dskWeight (Instance.util inst) } in n { sListForth = new_slist_forth , fDskForth = new_dsk_forth , pDskForth = new_dp_forth , utilLoadForth = new_load_forth , instSpindlesForth = new_inst_sp_forth , fSpindlesForth = new_free_sp_forth -- TODO Set failN1Forth, rMemForth, pRemForth } in if forthcoming then updateForthcomingFields t else let new_slist = List.delete iname (sList t) new_dsk = incIf uses_disk cur_dsk (Instance.dsk inst) new_free_sp = calcNewFreeSpindles False t inst new_inst_sp = calcSpindleUse False t inst old_peers = peers t old_peem = P.find pnode old_peers new_peem = decIf (Instance.usesSecMem inst) old_peem (Instance.mem inst) new_peers = if new_peem > 0 then P.add pnode new_peem old_peers else P.remove pnode old_peers old_rmem = rMem t new_rmem = if old_peem < old_rmem then old_rmem else computeMaxRes new_peers new_prem = fromIntegral new_rmem / tMem t new_failn1 = unallocatedMem t <= new_rmem new_dp = computeNewPDsk t new_free_sp new_dsk old_load = utilLoad t new_load = old_load { T.dskWeight = T.dskWeight old_load - T.dskWeight (Instance.util inst) } in updateForthcomingFields $ t { sList = new_slist, fDsk = new_dsk, peers = new_peers , failN1 = new_failn1, rMem = new_rmem, pDsk = new_dp , pRem = new_prem, utilLoad = new_load , instSpindles = new_inst_sp, fSpindles = new_free_sp } -- | Adds a primary instance (basic version). addPri :: Node -> Instance.Instance -> T.OpResult Node addPri = addPriEx False -- | Adds a primary instance (extended version). addPriEx :: Bool -- ^ Whether to override the N+1 and -- other /soft/ checks, useful if we -- come from a worse status (e.g. offline). -- If this is True, forthcoming instances -- may exceed available Node resources. -> Node -- ^ The target node -> Instance.Instance -- ^ The instance to add -> T.OpResult Node -- ^ The result of the operation, -- either the new version of the node -- or a failure mode addPriEx force t inst = let iname = Instance.idx inst forthcoming = Instance.forthcoming inst i_online = Instance.notOffline inst uses_disk = Instance.usesLocalStorage inst l_cpu = T.iPolicyVcpuRatio $ iPolicy t old_tags = pTags t strict = not force inst_tags = Instance.exclTags inst new_mem_forth = fMemForth t - Instance.mem inst new_mp_forth = fromIntegral new_mem_forth / tMem t new_dsk_forth = decIf uses_disk (fDskForth t) (Instance.dsk inst) new_free_sp_forth = calcNewFreeSpindlesForth True t inst new_inst_sp_forth = calcSpindleUseForth True t inst new_ucpu_forth = incIf i_online (uCpuForth t) (Instance.vcpus inst) new_pcpu_forth = fromIntegral new_ucpu_forth / tCpu t new_dp_forth = computeNewPDsk t new_free_sp_forth new_dsk_forth new_load_forth = utilLoadForth t `T.addUtil` Instance.util inst new_plist_forth = iname:pListForth t updateForthcomingFields n = n { pTags = addTags old_tags inst_tags , pListForth = new_plist_forth , fMemForth = new_mem_forth , fDskForth = new_dsk_forth , pMemForth = new_mp_forth , pDskForth = new_dp_forth , uCpuForth = new_ucpu_forth , pCpuForth = new_pcpu_forth , utilLoadForth = new_load_forth , instSpindlesForth = new_inst_sp_forth , fSpindlesForth = new_free_sp_forth -- TODO Set failN1Forth, rMemForth, pRemForth } checkForthcomingViolation | new_mem_forth <= 0 = Bad T.FailMem | uses_disk && new_dsk_forth <= 0 = Bad T.FailDisk | uses_disk && new_dsk_forth < loDsk t = Bad T.FailDisk | uses_disk && exclStorage t && new_free_sp_forth < 0 = Bad T.FailSpindles | uses_disk && new_inst_sp_forth > hiSpindles t = Bad T.FailDisk -- TODO Check failN1 including forthcoming instances | l_cpu >= 0 && l_cpu < new_pcpu_forth = Bad T.FailCPU | otherwise = Ok () in if forthcoming then case strict of True | Bad err <- checkForthcomingViolation -> Bad err _ -> Ok $ updateForthcomingFields t else let (new_i_mem, new_free_mem) = prospectiveMem t inst True new_p_mem = fromIntegral new_free_mem / tMem t new_failn1 = new_free_mem <= rMem t new_dsk = decIf uses_disk (fDsk t) (Instance.dsk inst) new_free_sp = calcNewFreeSpindles True t inst new_inst_sp = calcSpindleUse True t inst new_ucpu = incIf i_online (uCpu t) (Instance.vcpus inst) new_pcpu = fromIntegral new_ucpu / tCpu t new_dp = computeNewPDsk t new_free_sp new_dsk new_load = utilLoad t `T.addUtil` Instance.util inst new_plist = iname:pList t new_instance_map = addTags (instanceMap t) $ getLocationExclusionPairs t inst in case () of _ | new_free_mem <= 0 -> Bad T.FailMem | uses_disk && new_dsk <= 0 -> Bad T.FailDisk | strict && uses_disk && new_dsk < loDsk t -> Bad T.FailDisk | uses_disk && exclStorage t && new_free_sp < 0 -> Bad T.FailSpindles | strict && uses_disk && new_inst_sp > hiSpindles t -> Bad T.FailDisk | strict && new_failn1 && not (failN1 t) -> Bad T.FailMem | strict && l_cpu >= 0 && l_cpu < new_pcpu -> Bad T.FailCPU | strict && rejectAddTags old_tags inst_tags -> Bad T.FailTags -- When strict also check forthcoming limits, but after normal checks | strict, Bad err <- checkForthcomingViolation -> Bad err | otherwise -> Ok . updateForthcomingFields $ t { pList = new_plist , iMem = new_i_mem , fDsk = new_dsk , failN1 = new_failn1 , pMem = new_p_mem , pDsk = new_dp , uCpu = new_ucpu , pCpu = new_pcpu , utilLoad = new_load , instSpindles = new_inst_sp , fSpindles = new_free_sp , locationScore = locationScore t + Instance.locationScore inst + getInstanceDsrdLocScore t inst , instanceMap = new_instance_map } -- | Adds a secondary instance (basic version). addSec :: Node -> Instance.Instance -> T.Ndx -> T.OpResult Node addSec = addSecEx False -- | Adds a secondary instance (extended version). addSecEx :: Bool -> Node -> Instance.Instance -> T.Ndx -> T.OpResult Node addSecEx = addSecExEx False -- | Adds a secondary instance (doubly extended version). The first parameter -- tells `addSecExEx` to ignore disks completly. There is only one legitimate -- use case for this, and this is failing over a DRBD instance where the primary -- node is offline (and hence will become the secondary afterwards). addSecExEx :: Bool -> Bool -> Node -> Instance.Instance -> T.Ndx -> T.OpResult Node addSecExEx ignore_disks force t inst pdx = let iname = Instance.idx inst forthcoming = Instance.forthcoming inst old_peers = peers t strict = not force secondary_needed_mem = if Instance.usesSecMem inst then Instance.mem inst else 0 new_peem = P.find pdx old_peers + secondary_needed_mem new_peers = P.add pdx new_peem old_peers old_mem_forth = fMemForth t new_dsk_forth = fDskForth t - Instance.dsk inst new_free_sp_forth = calcNewFreeSpindlesForth True t inst new_inst_sp_forth = calcSpindleUseForth True t inst new_dp_forth = computeNewPDsk t new_free_sp_forth new_dsk_forth old_load_forth = utilLoadForth t new_load_forth = old_load_forth { T.dskWeight = T.dskWeight old_load_forth + T.dskWeight (Instance.util inst) } new_slist_forth = iname:sListForth t updateForthcomingFields n = n { sListForth = new_slist_forth , fDskForth = new_dsk_forth , pDskForth = new_dp_forth , utilLoadForth = new_load_forth , instSpindlesForth = new_inst_sp_forth , fSpindlesForth = new_free_sp_forth -- TODO Set failN1Forth, rMemForth, pRemForth } checkForthcomingViolation | not (Instance.hasSecondary inst) = Bad T.FailDisk | new_dsk_forth <= 0 = Bad T.FailDisk | new_dsk_forth < loDsk t = Bad T.FailDisk | exclStorage t && new_free_sp_forth < 0 = Bad T.FailSpindles | new_inst_sp_forth > hiSpindles t = Bad T.FailDisk | secondary_needed_mem >= old_mem_forth = Bad T.FailMem -- TODO Check failN1 including forthcoming instances | otherwise = Ok () in if forthcoming then case strict of True | Bad err <- checkForthcomingViolation -> Bad err _ -> Ok $ updateForthcomingFields t else let old_mem = unallocatedMem t new_dsk = fDsk t - Instance.dsk inst new_free_sp = calcNewFreeSpindles True t inst new_inst_sp = calcSpindleUse True t inst new_rmem = max (rMem t) new_peem new_prem = fromIntegral new_rmem / tMem t new_failn1 = old_mem <= new_rmem new_dp = computeNewPDsk t new_free_sp new_dsk old_load = utilLoad t new_load = old_load { T.dskWeight = T.dskWeight old_load + T.dskWeight (Instance.util inst) } new_slist = iname:sList t in case () of _ | not (Instance.hasSecondary inst) -> Bad T.FailDisk | not ignore_disks && new_dsk <= 0 -> Bad T.FailDisk | strict && new_dsk < loDsk t -> Bad T.FailDisk | exclStorage t && new_free_sp < 0 -> Bad T.FailSpindles | strict && new_inst_sp > hiSpindles t -> Bad T.FailDisk | strict && secondary_needed_mem >= old_mem -> Bad T.FailMem | strict && new_failn1 && not (failN1 t) -> Bad T.FailMem -- When strict also check forthcoming limits, but after normal checks | strict, Bad err <- checkForthcomingViolation -> Bad err | otherwise -> Ok . updateForthcomingFields $ t { sList = new_slist, fDsk = new_dsk , peers = new_peers, failN1 = new_failn1 , rMem = new_rmem, pDsk = new_dp , pRem = new_prem, utilLoad = new_load , instSpindles = new_inst_sp , fSpindles = new_free_sp } -- | Predicate on whether migration is supported between two nodes. checkMigration :: Node -> Node -> T.OpResult () checkMigration nsrc ntarget = if migTags nsrc `Set.isSubsetOf` rmigTags ntarget then Ok () else Bad T.FailMig -- * Stats functions -- | Computes the amount of available disk on a given node. availDisk :: Node -> Int availDisk t = let _f = fDsk t -- TODO Shall we use fDiskForth here? _l = loDsk t in if _f < _l then 0 else _f - _l -- | Computes the amount of used disk on a given node. iDsk :: Node -> Int iDsk t = truncate (tDsk t) - fDsk t -- | Returns state-of-world free memory on the node. -- | NOTE: This value is valid only before placement simulations. -- | TODO: Redefine this for memoy overcommitment. reportedFreeMem :: Node -> Int reportedFreeMem = fMem -- | Computes state-of-record free memory on the node. -- | TODO: Redefine this for memory overcommitment. recordedFreeMem :: Node -> Int recordedFreeMem t = let total = tMem t node = nMem t inst = iMem t in truncate total - node - inst -- | Computes the amount of missing memory on the node. -- NOTE: This formula uses free memory for calculations as opposed to -- used_memory in the definition, that's why it is the inverse. -- Explanations for missing memory (+) positive, (-) negative: -- (+) instances are using more memory that state-of-record -- - on KVM this might be due to the overhead per qemu process -- - on Xen manually upsized domains (xen mem-set) -- (+) on KVM non-qemu processes might be using more memory than what is -- reserved for node (no isolation) -- (-) on KVM qemu processes allocate memory on demand, thus an instance grows -- over its lifetime until it reaches state-of-record (+overhead) -- (-) on KVM KSM might be active -- (-) on Xen manually downsized domains (xen mem-set) missingMem :: Node -> Int missingMem t = recordedFreeMem t - reportedFreeMem t -- | Computes the 'guaranteed' free memory, that is the minimum of what -- is reported by the node (available bytes) and our calculation based on -- instance sizes (our records), thus considering missing memory. -- NOTE 1: During placement simulations, the recorded memory changes, as -- instances are added/removed from the node, thus we have to calculate the -- missingMem (correction) before altering state-of-record and then -- use that correction to estimate state-of-world memory usage _after_ -- the placements are done rather than doing min(record, world). -- NOTE 2: This is still only an approximation on KVM. As we shuffle instances -- during the simulation we are considering their state-of-record size, but -- in the real world the moves would shuffle parts of missing memory as well. -- Unfortunately as long as we don't have a more finegrained model that can -- better explain missing memory (split down based on root causes), we can't -- do better. -- NOTE 3: This is a hard limit based on available bytes and our bookkeeping. -- In case of memory overcommitment, both recordedFreeMem and reportedFreeMem -- would be extended by swap size on KVM or baloon size on Xen (their nominal -- and reported values). unallocatedMem :: Node -> Int unallocatedMem t = let state_of_record = recordedFreeMem t in state_of_record - max 0 (xMem t) -- | Computes the amount of available memory on a given node. -- Compared to unallocatedMem, this takes into account also memory reserved for -- secondary instances. -- NOTE: In case of memory overcommitment, there would be also an additional -- soft limit based on RAM size dedicated for instances and sum of -- state-of-record instance sizes (iMem): (tMem - nMem)*overcommit_ratio - iMem availMem :: Node -> Int availMem t = let reserved = rMem t unallocated = unallocatedMem t in max 0 (unallocated - reserved) -- | Prospective memory stats after instance operation. prospectiveMem :: Node -> Instance -> Bool -- ^ Operation: True if add, False for remove. -> (Int, Int) -- ^ Tuple (used_by_instances, guaranteed_free_mem) prospectiveMem node inst add = let uses_mem = (Instance.usesMemory inst) condOp = if add then incIf else decIf new_i_mem = condOp uses_mem (iMem node) (Instance.mem inst) new_node = node { iMem = new_i_mem } new_free_mem = unallocatedMem new_node in (new_i_mem, new_free_mem) -- | Computes the amount of available memory on a given node. availCpu :: Node -> Int availCpu t = let _u = uCpu t _l = hiCpu t in if _l >= _u then _l - _u else 0 -- * Node graph functions -- These functions do the transformations needed so that nodes can be -- represented as a graph connected by the instances that are replicated -- on them. -- * Making of a Graph from a node/instance list -- | Transform an instance into a list of edges on the node graph instanceToEdges :: Instance.Instance -> [Graph.Edge] instanceToEdges i | Instance.hasSecondary i = [(pnode,snode), (snode,pnode)] | otherwise = [] where pnode = Instance.pNode i snode = Instance.sNode i -- | Transform the list of instances into list of destination edges instancesToEdges :: Instance.List -> [Graph.Edge] instancesToEdges = concatMap instanceToEdges . Container.elems -- | Transform the list of nodes into vertices bounds. -- Returns Nothing is the list is empty. nodesToBounds :: List -> Maybe Graph.Bounds nodesToBounds nl = liftM2 (,) nmin nmax where nmin = fmap (fst . fst) (IntMap.minViewWithKey nl) nmax = fmap (fst . fst) (IntMap.maxViewWithKey nl) -- | The clique of the primary nodes of the instances with a given secondary. -- Return the full graph of those nodes that are primary node of at least one -- instance that has the given node as secondary. nodeToSharedSecondaryEdge :: Instance.List -> Node -> [Graph.Edge] nodeToSharedSecondaryEdge il n = (,) <$> primaries <*> primaries where primaries = map (Instance.pNode . flip Container.find il) $ sList n -- | Predicate of an edge having both vertices in a set of nodes. filterValid :: List -> [Graph.Edge] -> [Graph.Edge] filterValid nl = filter $ \(x,y) -> IntMap.member x nl && IntMap.member y nl -- | Transform a Node + Instance list into a NodeGraph type. -- Returns Nothing if the node list is empty. mkNodeGraph :: List -> Instance.List -> Maybe Graph.Graph mkNodeGraph nl il = liftM (`Graph.buildG` (filterValid nl . instancesToEdges $ il)) (nodesToBounds nl) -- | Transform a Nodes + Instances into a NodeGraph with all reboot exclusions. -- This includes edges between nodes that are the primary nodes of instances -- that have the same secondary node. Nodes not in the node list will not be -- part of the graph, but they are still considered for the edges arising from -- two instances having the same secondary node. -- Return Nothing if the node list is empty. mkRebootNodeGraph :: List -> List -> Instance.List -> Maybe Graph.Graph mkRebootNodeGraph allnodes nl il = liftM (`Graph.buildG` filterValid nl edges) (nodesToBounds nl) where edges = instancesToEdges il `List.union` (Container.elems allnodes >>= nodeToSharedSecondaryEdge il) -- * Display functions -- | Return a field for a given node. showField :: Node -- ^ Node which we're querying -> String -- ^ Field name -> String -- ^ Field value as string showField t field = case field of "idx" -> printf "%4d" $ idx t "name" -> alias t "fqdn" -> name t "status" -> case () of _ | offline t -> "-" | failN1 t -> "*" | otherwise -> " " "tmem" -> printf "%5.0f" $ tMem t "nmem" -> printf "%5d" $ nMem t "xmem" -> printf "%5d" $ xMem t "fmem" -> printf "%5d" $ fMem t "umem" -> printf "%5d" $ unallocatedMem t "imem" -> printf "%5d" $ iMem t "rmem" -> printf "%5d" $ rMem t "amem" -> printf "%5d" $ availMem t "tdsk" -> printf "%5.0f" $ tDsk t / 1024 "fdsk" -> printf "%5d" $ fDsk t `div` 1024 "tcpu" -> printf "%4.0f" $ tCpu t "ucpu" -> printf "%4d" $ uCpu t "pcnt" -> printf "%3d" $ length (pList t) "scnt" -> printf "%3d" $ length (sList t) "plist" -> show $ pList t "slist" -> show $ sList t "pfmem" -> printf "%6.4f" $ pMem t "pfdsk" -> printf "%6.4f" $ pDsk t "rcpu" -> printf "%5.2f" $ pCpu t "cload" -> printf "%5.3f" uC "mload" -> printf "%5.3f" uM "dload" -> printf "%5.3f" uD "nload" -> printf "%5.3f" uN "ptags" -> List.intercalate "," . map (uncurry (printf "%s=%d")) . Map.toList $ pTags t "peermap" -> show $ peers t "spindle_count" -> show $ tSpindles t "hi_spindles" -> show $ hiSpindles t "inst_spindles" -> show $ instSpindles t _ -> T.unknownField where T.DynUtil { T.cpuWeight = uC, T.memWeight = uM, T.dskWeight = uD, T.netWeight = uN } = utilLoad t -- | Returns the header and numeric propery of a field. showHeader :: String -> (String, Bool) showHeader field = case field of "idx" -> ("Index", True) "name" -> ("Name", False) "fqdn" -> ("Name", False) "status" -> ("F", False) "tmem" -> ("t_mem", True) "nmem" -> ("n_mem", True) "xmem" -> ("x_mem", True) "fmem" -> ("f_mem", True) "umem" -> ("u_mem", True) "amem" -> ("a_mem", True) "imem" -> ("i_mem", True) "rmem" -> ("r_mem", True) "tdsk" -> ("t_dsk", True) "fdsk" -> ("f_dsk", True) "tcpu" -> ("pcpu", True) "ucpu" -> ("vcpu", True) "pcnt" -> ("pcnt", True) "scnt" -> ("scnt", True) "plist" -> ("primaries", True) "slist" -> ("secondaries", True) "pfmem" -> ("p_fmem", True) "pfdsk" -> ("p_fdsk", True) "rcpu" -> ("r_cpu", True) "cload" -> ("lCpu", True) "mload" -> ("lMem", True) "dload" -> ("lDsk", True) "nload" -> ("lNet", True) "ptags" -> ("PrimaryTags", False) "peermap" -> ("PeerMap", False) "spindle_count" -> ("NodeSpindles", True) "hi_spindles" -> ("MaxSpindles", True) "inst_spindles" -> ("InstSpindles", True) -- TODO: add node fields (group.uuid, group) _ -> (T.unknownField, False) -- | String converter for the node list functionality. list :: [String] -> Node -> [String] list fields t = map (showField t) fields -- | Generate OpCode for setting a node's offline status genOpSetOffline :: (MonadFail m) => Node -> Bool -> m OpCodes.OpCode genOpSetOffline node offlineStatus = do nodeName <- mkNonEmpty (name node) return OpCodes.OpNodeSetParams { OpCodes.opNodeName = nodeName , OpCodes.opNodeUuid = Nothing , OpCodes.opForce = False , OpCodes.opHvState = Nothing , OpCodes.opDiskState = Nothing , OpCodes.opMasterCandidate = Nothing , OpCodes.opOffline = Just offlineStatus , OpCodes.opDrained = Nothing , OpCodes.opAutoPromote = False , OpCodes.opMasterCapable = Nothing , OpCodes.opVmCapable = Nothing , OpCodes.opSecondaryIp = Nothing , OpCodes.opgenericNdParams = Nothing , OpCodes.opPowered = Nothing } -- | Generate OpCode for applying a OobCommand to the given nodes genOobCommand :: (MonadFail m) => [Node] -> OobCommand -> m OpCodes.OpCode genOobCommand nodes command = do names <- mapM (mkNonEmpty . name) nodes return OpCodes.OpOobCommand { OpCodes.opNodeNames = names , OpCodes.opNodeUuids = Nothing , OpCodes.opOobCommand = command , OpCodes.opOobTimeout = C.oobTimeout , OpCodes.opIgnoreStatus = False , OpCodes.opPowerDelay = C.oobPowerDelay } -- | Generate OpCode for powering on a list of nodes genPowerOnOpCodes :: (MonadFail m) => [Node] -> m [OpCodes.OpCode] genPowerOnOpCodes nodes = do opSetParams <- mapM (`genOpSetOffline` False) nodes oobCommand <- genOobCommand nodes OobPowerOn return $ opSetParams ++ [oobCommand] -- | Generate OpCodes for powering off a list of nodes genPowerOffOpCodes :: (MonadFail m) => [Node] -> m [OpCodes.OpCode] genPowerOffOpCodes nodes = do opSetParams <- mapM (`genOpSetOffline` True) nodes oobCommand <- genOobCommand nodes OobPowerOff return $ opSetParams ++ [oobCommand] -- | Generate OpCodes for adding tags to a node genAddTagsOpCode :: Node -> [String] -> OpCodes.OpCode genAddTagsOpCode node tags = OpCodes.OpTagsSet { OpCodes.opKind = TagKindNode , OpCodes.opTagsList = tags , OpCodes.opTagsGetName = Just $ name node } -- | Constant holding the fields we're displaying by default. defaultFields :: [String] defaultFields = [ "status", "name", "tmem", "nmem", "imem", "xmem", "fmem", "umem" , "rmem", "tdsk", "fdsk", "tcpu", "ucpu", "pcnt", "scnt" , "pfmem", "pfdsk", "rcpu" , "cload", "mload", "dload", "nload" ] -- | Split a list of nodes into a list of (node group UUID, list of -- associated nodes). computeGroups :: [Node] -> [(T.Gdx, [Node])] computeGroups nodes = let nodes' = List.sortBy (comparing group) nodes nodes'' = NonEmpty.groupBy ((==) `on` group) nodes' in map (\nl -> (group (NonEmpty.head nl), NonEmpty.toList nl)) nodes'' ganeti-3.1.0~rc2/src/Ganeti/HTools/PeerMap.hs000064400000000000000000000070161476477700300207030ustar00rootroot00000000000000{-| Module abstracting the peer map implementation. This is abstracted separately since the speed of peermap updates can be a significant part of the total runtime, and as such changing the implementation should be easy in case it's needed. -} {- Copyright (C) 2009, 2011 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.PeerMap ( PeerMap , Key , Elem , empty , accumArray , Ganeti.HTools.PeerMap.find , add , remove , maxElem , sumElems ) where import Data.Maybe (fromMaybe) import Data.List import Data.Ord (comparing) import Ganeti.HTools.Types -- * Type definitions -- | Our key type. type Key = Ndx -- | Our element type. type Elem = Int -- | The definition of a peer map. type PeerMap = [(Key, Elem)] -- * Initialization functions -- | Create a new empty map. empty :: PeerMap empty = [] -- | Our reverse-compare function. pmCompare :: (Key, Elem) -> (Key, Elem) -> Ordering pmCompare a b = comparing snd b a -- | Add or update (via a custom function) an element. addWith :: (Elem -> Elem -> Elem) -> Key -> Elem -> PeerMap -> PeerMap addWith fn k v lst = case lookup k lst of Nothing -> insertBy pmCompare (k, v) lst Just o -> insertBy pmCompare (k, fn o v) (remove k lst) -- | Create a PeerMap from an association list, with possible duplicates. accumArray :: (Elem -> Elem -> Elem) -- ^ function used to merge the elements -> [(Key, Elem)] -- ^ source data -> PeerMap -- ^ results accumArray _ [] = empty accumArray fn ((k, v):xs) = addWith fn k v $ accumArray fn xs -- * Basic operations -- | Returns either the value for a key or zero if not found. find :: Key -> PeerMap -> Elem find k = fromMaybe 0 . lookup k -- | Add an element to a peermap, overwriting the previous value. add :: Key -> Elem -> PeerMap -> PeerMap add = addWith (flip const) -- | Remove an element from a peermap. remove :: Key -> PeerMap -> PeerMap remove _ [] = [] remove k ((x@(x', _)):xs) = if k == x' then xs else x:remove k xs -- | Find the maximum element. -- -- Since this is a sorted list, we just get the value at the head of -- the list, or zero for a null list maxElem :: PeerMap -> Elem maxElem (x:_) = snd x maxElem _ = 0 -- | Sum of all peers. sumElems :: PeerMap -> Elem sumElems = sum . map snd ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/000075500000000000000000000000001476477700300204215ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Hail.hs000064400000000000000000000115511476477700300216350ustar00rootroot00000000000000{-| IAllocator plugin for Ganeti. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Hail ( main , options , arguments ) where import Control.Monad import Control.Monad.Writer (runWriterT) import Data.Maybe (fromMaybe, isJust) import System.IO import qualified Ganeti.HTools.AlgorithmParams as Alg import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Dedicated as Dedicated import Ganeti.Common import Ganeti.HTools.CLI import Ganeti.HTools.Backend.IAlloc import qualified Ganeti.HTools.Backend.MonD as MonD import Ganeti.HTools.Loader (Request(..), ClusterData(..), isAllocationRequest) import Ganeti.HTools.ExtLoader (maybeSaveData, loadExternalData) import Ganeti.Utils -- | Options list and functions. options :: IO [OptType] options = return [ oPrintNodes , oSaveCluster , oDataFile , oNodeSim , oVerbose , oIgnoreDyn , oIgnoreSoftErrors , oNoCapacityChecks , oRestrictToNodes , oMonD , oMonDXen , oStaticKvmNodeMemory ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [ArgCompletion OptComplFile 1 (Just 1)] wrapReadRequest :: Options -> [String] -> IO Request wrapReadRequest opts args = do let static_n_mem = optStaticKvmNodeMemory opts r1 <- case args of [] -> exitErr "This program needs an input file." _:_:_ -> exitErr "Only one argument is accepted (the input file)" x:_ -> readRequest x static_n_mem if isJust (optDataFile opts) || (not . null . optNodeSim) opts then do -- TODO: Cleanup this mess. ClusterData is loaded first in -- IAlloc.readRequest, then the data part is dropped and replaced with -- ExtLoader.loadExternalData that uses IAlloc.loadData to load the same -- data again. This codepath is executed only with a manually specified -- cluster data file or simulation (i.e. not under'normal' operation.) cdata <- loadExternalData opts let Request rqt _ = r1 return $ Request rqt cdata else do let Request rqt cdata = r1 (cdata', _) <- runWriterT $ if optMonD opts then MonD.queryAllMonDDCs cdata opts else return cdata return $ Request rqt cdata' -- | Main function. main :: Options -> [String] -> IO () main opts args = do let shownodes = optShowNodes opts verbose = optVerbose opts savecluster = optSaveCluster opts request <- wrapReadRequest opts args let Request rq cdata = request when (verbose > 1) . hPutStrLn stderr $ "Received request: " ++ show rq when (verbose > 2) . hPutStrLn stderr $ "Received cluster data: " ++ show cdata let dedicatedAlloc = maybe False (Dedicated.isDedicated cdata) $ isAllocationRequest rq when (verbose > 1 && dedicatedAlloc) $ hPutStrLn stderr "Allocation on a dedicated cluster;\ \ using lost-allocations metrics." maybePrintNodes shownodes "Initial cluster" (Cluster.printNodes (cdNodes cdata)) maybeSaveData savecluster "pre-ialloc" "before iallocator run" cdata let runAlloc = if dedicatedAlloc then Dedicated.runDedicatedAllocation else runIAllocator (maybe_ni, resp) = runAlloc (Alg.fromCLIOptions opts) request (fin_nl, fin_il) = fromMaybe (cdNodes cdata, cdInstances cdata) maybe_ni putStrLn resp maybePrintNodes shownodes "Final cluster" (Cluster.printNodes fin_nl) maybeSaveData savecluster "post-ialloc" "after iallocator run" (cdata { cdNodes = fin_nl, cdInstances = fin_il}) ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Harep.hs000064400000000000000000000535731476477700300220310ustar00rootroot00000000000000{-# LANGUAGE TupleSections #-} {-| Auto-repair tool for Ganeti. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Harep ( main , arguments , options) where import Control.Exception (bracket) import Control.Lens (over) import Control.Monad import Data.Function import Data.List import Data.Maybe import Data.Ord import System.Time import qualified Data.Map as Map import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Common import Ganeti.Errors import Ganeti.JQueue (currentTimestamp, reasonTrailTimestamp) import Ganeti.JQueue.Objects (Timestamp) import Ganeti.Jobs import Ganeti.OpCodes import Ganeti.OpCodes.Lens (metaParamsL, opReasonL) import Ganeti.OpParams import Ganeti.Types import Ganeti.Utils import qualified Ganeti.Constants as C import qualified Ganeti.Luxi as L import qualified Ganeti.Path as Path import qualified Ganeti.Utils.Time as Time import Ganeti.HTools.CLI import Ganeti.HTools.Loader import Ganeti.HTools.ExtLoader import qualified Ganeti.HTools.Tags.Constants as Tags import Ganeti.HTools.Types import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import Ganeti.Version (version) -- | Options list and functions. options :: IO [OptType] options = do luxi <- oLuxiSocket return [ luxi , oJobDelay , oReason , oDryRun ] arguments :: [ArgCompletion] arguments = [] -- | Wraps an 'OpCode' in a 'MetaOpCode' while also adding a comment -- about what generated the opcode. annotateOpCode :: Maybe String -> Timestamp -> OpCode -> MetaOpCode annotateOpCode reason ts = over (metaParamsL . opReasonL) (++ [( "harep", fromMaybe ("harep " ++ version ++ " called") reason , reasonTrailTimestamp ts)]) . setOpComment ("automated repairs by harep " ++ version) . wrapOpCode data InstanceData = InstanceData { arInstance :: Instance.Instance , arState :: AutoRepairStatus , tagsToRemove :: [String] } deriving (Eq, Show) -- | Parse a tag into an 'AutoRepairData' record. -- -- @Nothing@ is returned if the tag is not an auto-repair tag, or if it's -- malformed. parseInitTag :: String -> Maybe AutoRepairData parseInitTag tag = let parsePending = do subtag <- chompPrefix Tags.autoRepairTagPending tag case sepSplit ':' subtag of [rtype, uuid, ts, jobs] -> makeArData rtype uuid ts jobs _ -> fail ("Invalid tag: " ++ show tag) parseResult = do subtag <- chompPrefix Tags.autoRepairTagResult tag case sepSplit ':' subtag of [rtype, uuid, ts, result, jobs] -> do arData <- makeArData rtype uuid ts jobs result' <- autoRepairResultFromRaw result return arData { arResult = Just result' } _ -> fail ("Invalid tag: " ++ show tag) makeArData rtype uuid ts jobs = do rtype' <- autoRepairTypeFromRaw rtype ts' <- tryRead "auto-repair time" ts jobs' <- mapM makeJobIdS $ sepSplit '+' jobs return AutoRepairData { arType = rtype' , arUuid = uuid , arTime = TOD ts' 0 , arJobs = jobs' , arResult = Nothing , arTag = tag } in parsePending `mplus` parseResult -- | Return the 'AutoRepairData' element of an 'AutoRepairStatus' type. getArData :: AutoRepairStatus -> Maybe AutoRepairData getArData status = case status of ArHealthy (Just d) -> Just d ArFailedRepair d -> Just d ArPendingRepair d -> Just d ArNeedsRepair d -> Just d _ -> Nothing -- | Return a short name for each auto-repair status. -- -- This is a more concise representation of the status, because the default -- "Show" formatting includes all the accompanying auto-repair data. arStateName :: AutoRepairStatus -> String arStateName status = case status of ArHealthy _ -> "Healthy" ArFailedRepair _ -> "Failure" ArPendingRepair _ -> "Pending repair" ArNeedsRepair _ -> "Needs repair" -- | Return a new list of tags to remove that includes @arTag@ if present. delCurTag :: InstanceData -> [String] delCurTag instData = let arData = getArData $ arState instData rmTags = tagsToRemove instData in case arData of Just d -> arTag d : rmTags Nothing -> rmTags -- | Set the initial auto-repair state of an instance from its auto-repair tags. -- -- The rules when there are multiple tags is: -- -- * the earliest failure result always wins -- -- * two or more pending repairs results in a fatal error -- -- * a pending result from id X and a success result from id Y result in error -- if Y is newer than X -- -- * if there are no pending repairs, the newest success result wins, -- otherwise the pending result is used. setInitialState :: Instance.Instance -> Result InstanceData setInitialState inst = let arData = mapMaybe parseInitTag $ Instance.allTags inst -- Group all the AutoRepairData records by id (i.e. by repair task), and -- present them from oldest to newest. arData' = sortBy (comparing arUuid) arData arGroups = groupBy ((==) `on` arUuid) arData' arGroups' = sortBy (comparing $ minimum . map arTime) arGroups in foldM arStatusCmp (InstanceData inst (ArHealthy Nothing) []) arGroups' -- | Update the initial status of an instance with new repair task tags. -- -- This function gets called once per repair group in an instance's tag, and it -- determines whether to set the status of the instance according to this new -- group, or to keep the existing state. See the documentation for -- 'setInitialState' for the rules to be followed when determining this. arStatusCmp :: InstanceData -> [AutoRepairData] -> Result InstanceData arStatusCmp instData arData = let curSt = arState instData arData' = sortBy (comparing keyfn) arData keyfn d = (arResult d, arTime d) newData = last arData' newSt = case arResult newData of Just ArSuccess -> ArHealthy $ Just newData Just ArEnoperm -> ArHealthy $ Just newData Just ArFailure -> ArFailedRepair newData Nothing -> ArPendingRepair newData in case curSt of ArFailedRepair _ -> Ok instData -- Always keep the earliest failure. ArHealthy _ -> Ok instData { arState = newSt , tagsToRemove = delCurTag instData } ArPendingRepair d -> Bad ( "An unfinished repair was found in instance " ++ Instance.name (arInstance instData) ++ ": found tag " ++ show (arTag newData) ++ ", but older pending tag " ++ show (arTag d) ++ "exists.") ArNeedsRepair _ -> Bad "programming error: ArNeedsRepair found as an initial state" -- | Query jobs of a pending repair, returning the new instance data. processPending :: Options -> L.Client -> InstanceData -> IO InstanceData processPending opts client instData = case arState instData of (ArPendingRepair arData) -> do sts <- L.queryJobsStatus client $ arJobs arData time <- getClockTime case sts of Bad e -> exitErr $ "could not check job status: " ++ formatError e Ok sts' -> if any (<= JOB_STATUS_RUNNING) sts' then return instData -- (no change) else do let iname = Instance.name $ arInstance instData srcSt = arStateName $ arState instData destSt = arStateName arState' putStrLn ("Moving " ++ iname ++ " from " ++ show srcSt ++ " to " ++ show destSt) commitChange opts client instData' where instData' = instData { arState = arState' , tagsToRemove = delCurTag instData } arState' = if all (== JOB_STATUS_SUCCESS) sts' then ArHealthy $ Just (updateTag $ arData { arResult = Just ArSuccess , arTime = time }) else ArFailedRepair (updateTag $ arData { arResult = Just ArFailure , arTime = time }) _ -> return instData -- | Update the tag of an 'AutoRepairData' record to match all the other fields. updateTag :: AutoRepairData -> AutoRepairData updateTag arData = let ini = [autoRepairTypeToRaw $ arType arData, arUuid arData, Time.clockTimeToString $ arTime arData] end = [intercalate "+" . map (show . fromJobId) $ arJobs arData] (pfx, middle) = case arResult arData of Nothing -> (Tags.autoRepairTagPending, []) Just rs -> (Tags.autoRepairTagResult, [autoRepairResultToRaw rs]) in arData { arTag = pfx ++ intercalate ":" (ini ++ middle ++ end) } -- | Apply and remove tags from an instance as indicated by 'InstanceData'. -- -- If the /arState/ of the /InstanceData/ record has an associated -- 'AutoRepairData', add its tag to the instance object. Additionally, if -- /tagsToRemove/ is not empty, remove those tags from the instance object. The -- returned /InstanceData/ object always has an empty /tagsToRemove/. commitChange :: Options -> L.Client -> InstanceData -> IO InstanceData commitChange opts client instData = do now <- currentTimestamp let iname = Instance.name $ arInstance instData arData = getArData $ arState instData rmTags = tagsToRemove instData execJobsWaitOk' opcodes = unless (optDryRun opts) $ do res <- execJobsWaitOk [map (annotateOpCode (optReason opts) now) opcodes] client case res of Ok _ -> return () Bad e -> exitErr e when (isJust arData) $ do let tag = arTag $ fromJust arData putStrLn (">>> Adding the following tag to " ++ iname ++ ":\n" ++ show tag) execJobsWaitOk' [OpTagsSet TagKindInstance [tag] (Just iname)] unless (null rmTags) $ do putStr (">>> Removing the following tags from " ++ iname ++ ":\n" ++ unlines (map show rmTags)) execJobsWaitOk' [OpTagsDel TagKindInstance rmTags (Just iname)] return instData { tagsToRemove = [] } -- | Detect brokenness with an instance and suggest repair type and jobs to run. detectBroken :: Node.List -> Instance.Instance -> Maybe (AutoRepairType, [OpCode]) detectBroken nl inst = let disk = Instance.diskTemplate inst iname = Instance.name inst offPri = Node.offline $ Container.find (Instance.pNode inst) nl offSec = Node.offline $ Container.find (Instance.sNode inst) nl in case disk of DTDrbd8 | offPri && offSec -> Just ( ArReinstall, [ OpInstanceRecreateDisks { opInstanceName = iname , opInstanceUuid = Nothing , opRecreateDisksInfo = RecreateDisksAll , opNodes = [] -- FIXME: there should be a better way to -- specify opcode parameters than abusing -- mkNonEmpty in this way (using the fact -- that Maybe is used both for optional -- fields, and to express failure). , opNodeUuids = Nothing , opIallocator = mkNonEmpty "hail" } , OpInstanceReinstall { opInstanceName = iname , opInstanceUuid = Nothing , opOsType = Nothing , opTempOsParams = Nothing , opOsparamsPrivate = Nothing , opOsparamsSecret = Nothing , opForceVariant = False } ]) | offPri -> Just ( ArFailover, [ OpInstanceFailover { opInstanceName = iname , opInstanceUuid = Nothing -- FIXME: ditto, see above. , opShutdownTimeout = fromJust $ mkNonNegative C.defaultShutdownTimeout , opIgnoreConsistency = False , opTargetNode = Nothing , opTargetNodeUuid = Nothing , opIgnoreIpolicy = False , opIallocator = Nothing , opMigrationCleanup = False } ]) | offSec -> Just ( ArFixStorage, [ OpInstanceReplaceDisks { opInstanceName = iname , opInstanceUuid = Nothing , opReplaceDisksMode = ReplaceNewSecondary , opReplaceDisksList = [] , opRemoteNode = Nothing -- FIXME: ditto, see above. , opRemoteNodeUuid = Nothing , opIallocator = mkNonEmpty "hail" , opEarlyRelease = False , opIgnoreIpolicy = False } ]) | otherwise -> Nothing DTPlain | offPri -> Just ( ArReinstall, [ OpInstanceRecreateDisks { opInstanceName = iname , opInstanceUuid = Nothing , opRecreateDisksInfo = RecreateDisksAll , opNodes = [] -- FIXME: ditto, see above. , opNodeUuids = Nothing , opIallocator = mkNonEmpty "hail" } , OpInstanceReinstall { opInstanceName = iname , opInstanceUuid = Nothing , opOsType = Nothing , opTempOsParams = Nothing , opOsparamsPrivate = Nothing , opOsparamsSecret = Nothing , opForceVariant = False } ]) | otherwise -> Nothing _ -> Nothing -- Other cases are unimplemented for now: DTDiskless, -- DTFile, DTSharedFile, DTBlock, DTRbd, DTExt. -- | Submit jobs, unless a dry-run is requested; in this case, just report -- the job that would be submitted. submitJobs' :: Options -> [[MetaOpCode]] -> L.Client -> IO (Result [JobId]) submitJobs' opts jobs client = if optDryRun opts then do putStrLn . (++) "jobs: " . J.encode $ map (map metaOpCode) jobs return $ Ok [] else submitJobs jobs client -- | Perform the suggested repair on an instance if its policy allows it. doRepair :: Options -> L.Client -- ^ The Luxi client -> Double -- ^ Delay to insert before the first repair opcode -> InstanceData -- ^ The instance data -> (AutoRepairType, [OpCode]) -- ^ The repair job to perform -> IO InstanceData -- ^ The updated instance data doRepair opts client delay instData (rtype, opcodes) = let inst = arInstance instData ipol = Instance.arPolicy inst iname = Instance.name inst in case ipol of ArEnabled maxtype -> if rtype > maxtype then do uuid <- newUUID time <- getClockTime let arState' = ArNeedsRepair ( updateTag $ AutoRepairData rtype uuid time [] (Just ArEnoperm) "") instData' = instData { arState = arState' , tagsToRemove = delCurTag instData } putStrLn ("Not performing a repair of type " ++ show rtype ++ " on " ++ iname ++ " because only repairs up to " ++ show maxtype ++ " are allowed") commitChange opts client instData' -- Adds "enoperm" result label. else do now <- currentTimestamp putStrLn ("Executing " ++ show rtype ++ " repair on " ++ iname) -- After submitting the job, we must write an autorepair:pending tag, -- that includes the repair job IDs so that they can be checked later. -- One problem we run into is that the repair job immediately grabs -- locks for the affected instance, and the subsequent TAGS_SET job is -- blocked, introducing an unnecessary delay for the end-user. One -- alternative would be not to wait for the completion of the TAGS_SET -- job, contrary to what commitChange normally does; but we insist on -- waiting for the tag to be set so as to abort in case of failure, -- because the cluster is left in an invalid state in that case. -- -- The proper solution (in 2.9+) would be not to use tags for storing -- autorepair data, or make the TAGS_SET opcode not grab an instance's -- locks (if that's deemed safe). In the meantime, we introduce an -- artificial delay in the repair job (via a TestDelay opcode) so that -- once we have the job ID, the TAGS_SET job can complete before the -- repair job actually grabs the locks. (Please note that this is not -- about synchronization, but merely about speeding up the execution of -- the harep tool. If this TestDelay opcode is removed, the program is -- still correct.) let opcodes' = if delay > 0 then OpTestDelay { opDelayDuration = delay , opDelayOnMaster = True , opDelayOnNodes = [] , opDelayOnNodeUuids = Nothing , opDelayRepeat = fromJust $ mkNonNegative 0 , opDelayInterruptible = False , opDelayNoLocks = False } : opcodes else opcodes uuid <- newUUID time <- getClockTime jids <- submitJobs' opts [map (annotateOpCode (optReason opts) now) opcodes'] client case jids of Bad e -> exitErr e Ok jids' -> let arState' = ArPendingRepair ( updateTag $ AutoRepairData rtype uuid time jids' Nothing "") instData' = instData { arState = arState' , tagsToRemove = delCurTag instData } in commitChange opts client instData' -- Adds "pending" label. otherSt -> do putStrLn ("Not repairing " ++ iname ++ " because it's in state " ++ show otherSt) return instData -- | Main function. main :: Options -> [String] -> IO () main opts args = do unless (null args) $ exitErr "this program doesn't take any arguments." luxiDef <- Path.defaultQuerySocket let master = fromMaybe luxiDef $ optLuxi opts opts' = opts { optLuxi = Just master } (ClusterData _ nl il _ _) <- loadExternalData opts' let iniDataRes = mapM setInitialState $ Container.elems il iniData <- exitIfBad "when parsing auto-repair tags" iniDataRes -- First step: check all pending repairs, see if they are completed. iniData' <- bracket (L.getLuxiClient master) L.closeClient $ forM iniData . processPending opts -- Second step: detect any problems. let repairs = map (detectBroken nl . arInstance) iniData' -- Third step: create repair jobs for broken instances that are in ArHealthy. let maybeRepair c (i, r) = maybe (return i) (repairHealthy c i) r jobDelay = optJobDelay opts repairHealthy c i = case arState i of ArHealthy _ -> doRepair opts c jobDelay i _ -> const (return i) repairDone <- bracket (L.getLuxiClient master) L.closeClient $ forM (zip iniData' repairs) . maybeRepair -- Print some stats and exit. let states = map ((, 1 :: Int) . arStateName . arState) repairDone counts = Map.fromListWith (+) states putStrLn "---------------------" putStrLn "Instance status count" putStrLn "---------------------" putStr . unlines . Map.elems $ Map.mapWithKey (\k v -> k ++ ": " ++ show v) counts ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Hbal.hs000064400000000000000000000337061476477700300216340ustar00rootroot00000000000000{-| Cluster rebalancer. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Hbal ( main , options , arguments , iterateDepth ) where import Control.Arrow ((&&&)) import Control.Lens (over) import Control.Monad import Data.List import Data.Maybe (isNothing, fromMaybe) import System.Exit import System.IO import Text.Printf (printf) import Ganeti.HTools.AlgorithmParams (AlgorithmOptions(..), fromCLIOptions) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.Metrics as Metrics import qualified Ganeti.HTools.Cluster.Utils as ClusterUtils import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance import Ganeti.BasicTypes import Ganeti.Common import Ganeti.HTools.CLI import Ganeti.HTools.ExtLoader import Ganeti.HTools.Types import Ganeti.HTools.Loader import Ganeti.OpCodes (wrapOpCode, setOpComment, setOpPriority) import Ganeti.OpCodes.Lens (metaParamsL, opReasonL) import Ganeti.JQueue (currentTimestamp, reasonTrailTimestamp) import Ganeti.JQueue.Objects (Timestamp) import Ganeti.Jobs as Jobs import Ganeti.Utils import Ganeti.Version (version) -- | Options list and functions. options :: IO [OptType] options = do luxi <- oLuxiSocket return [ oPrintNodes , oPrintInsts , oPrintCommands , oDataFile , oEvacMode , oRestrictedMigrate , oRapiMaster , luxi , oIAllocSrc , oExecJobs , oFirstJobGroup , oReason , oGroup , oMaxSolLength , oVerbose , oQuiet , oOfflineNode , oStaticKvmNodeMemory , oMinScore , oMaxCpu , oMinDisk , oMinGain , oMinGainLim , oDiskMoves , oSelInst , oInstMoves , oIgnoreSoftErrors , oDynuFile , oIgnoreDyn , oMonD , oMonDDataFile , oMonDExitMissing , oMonDXen , oExTags , oExInst , oSaveCluster , oPriority ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [] -- | Wraps an 'OpCode' in a 'MetaOpCode' while also adding a comment -- about what generated the opcode. annotateOpCode :: Maybe String -> Timestamp -> Jobs.Annotator annotateOpCode reason ts = over (metaParamsL . opReasonL) (++ [( "hbal", fromMaybe ("hbal " ++ version ++ " called") reason , reasonTrailTimestamp ts)]) . setOpComment ("rebalancing via hbal " ++ version) . wrapOpCode {- | Start computing the solution at the given depth and recurse until we find a valid solution or we exceed the maximum depth. -} iterateDepth :: Bool -- ^ Whether to print moves -> AlgorithmOptions -- ^ Algorithmic options to apply -> Cluster.Table -- ^ The starting table -> Int -- ^ Remaining length -> Int -- ^ Max node name len -> Int -- ^ Max instance name len -> [MoveJob] -- ^ Current command list -> Score -- ^ Score at which to stop -> IO (Cluster.Table, [MoveJob]) -- ^ The resulting table -- and commands iterateDepth printmove algOpts ini_tbl max_rounds nmlen imlen cmd_strs min_score = let Cluster.Table ini_nl ini_il _ _ = ini_tbl allowed_next = Cluster.doNextBalance ini_tbl max_rounds min_score m_fin_tbl = if allowed_next then Cluster.tryBalance algOpts ini_tbl else Nothing in case m_fin_tbl of Just fin_tbl -> do let (Cluster.Table _ _ _ fin_plc) = fin_tbl cur_plc@(idx, _, _, move, _) <- exitIfEmpty "Empty placement list returned for solution?!" fin_plc let fin_plc_len = length fin_plc (sol_line, cmds) = Cluster.printSolutionLine ini_nl ini_il nmlen imlen cur_plc fin_plc_len afn = Cluster.involvedNodes ini_il cur_plc upd_cmd_strs = (afn, idx, move, cmds):cmd_strs when printmove $ do putStrLn sol_line hFlush stdout iterateDepth printmove algOpts fin_tbl max_rounds nmlen imlen upd_cmd_strs min_score Nothing -> return (ini_tbl, cmd_strs) -- | Displays the cluster stats. printStats :: Node.List -> Node.List -> IO () printStats ini_nl fin_nl = do let ini_cs = Cluster.totalResources ini_nl fin_cs = Cluster.totalResources fin_nl printf "Original: mem=%d disk=%d\n" (Cluster.csFmem ini_cs) (Cluster.csFdsk ini_cs) :: IO () printf "Final: mem=%d disk=%d\n" (Cluster.csFmem fin_cs) (Cluster.csFdsk fin_cs) -- | Executes the jobs, if possible and desired. maybeExecJobs :: Options -> [a] -> Node.List -> Instance.List -> [JobSet] -> IO (Result ()) maybeExecJobs opts ord_plc fin_nl il cmd_jobs = if optExecJobs opts && not (null ord_plc) then (case optLuxi opts of Nothing -> return $ Bad "Execution of commands possible only on LUXI" Just master -> do ts <- currentTimestamp let annotator = maybe id setOpPriority (optPriority opts) . annotateOpCode (optReason opts) ts execWithCancel annotator master $ zip (map toOpcodes cmd_jobs) (map toDescr cmd_jobs)) else return $ Ok () where toOpcodes = map (\(_, idx, move, _) -> Cluster.iMoveToJob fin_nl il idx move) toDescr job = "Executing jobset for instances " ++ commaJoin (map (\(_, idx, _, _) -> Container.nameOf il idx) job) -- | Select the target node group. selectGroup :: Options -> Group.List -> Node.List -> Instance.List -> IO (String, (Node.List, Instance.List)) selectGroup opts gl nlf ilf = do let ngroups = ClusterUtils.splitCluster nlf ilf when (length ngroups > 1 && isNothing (optGroup opts)) $ do hPutStrLn stderr "Found multiple node groups:" mapM_ (hPutStrLn stderr . (" " ++) . Group.name . flip Container.find gl . fst) ngroups exitErr "Aborting." case optGroup opts of Nothing -> do (gidx, cdata) <- exitIfEmpty "No groups found by splitCluster?!" ngroups let grp = Container.find gidx gl return (Group.name grp, cdata) Just g -> case Container.findByName gl g of Nothing -> do hPutStrLn stderr $ "Node group " ++ g ++ " not found. Node group list is:" mapM_ (hPutStrLn stderr . (" " ++) . Group.name ) (Container.elems gl) exitErr "Aborting." Just grp -> case lookup (Group.idx grp) ngroups of Nothing -> -- This will only happen if there are no nodes assigned -- to this group return (Group.name grp, (Container.empty, Container.empty)) Just cdata -> return (Group.name grp, cdata) -- | Do a few checks on the cluster data. checkCluster :: Int -> Node.List -> Instance.List -> IO () checkCluster verbose nl il = do -- nothing to do on an empty cluster when (Container.null il) $ do printf "Cluster is empty, exiting.\n"::IO () exitSuccess -- hbal doesn't currently handle split clusters let split_insts = Cluster.findSplitInstances nl il unless (null split_insts || verbose <= 1) $ do hPutStrLn stderr "Found instances belonging to multiple node groups:" mapM_ (\i -> hPutStrLn stderr $ " " ++ Instance.name i) split_insts hPutStrLn stderr "These instances will not be moved." printf "Loaded %d nodes, %d instances\n" (Container.size nl) (Container.size il)::IO () let csf = commonSuffix nl il when (not (null csf) && verbose > 1) $ printf "Note: Stripping common suffix of '%s' from names\n" csf -- | Do a few checks on the selected group data. checkGroup :: Bool -> Int -> String -> Node.List -> Instance.List -> IO () checkGroup force verbose gname nl il = do printf "Group size %d nodes, %d instances\n" (Container.size nl) (Container.size il)::IO () putStrLn $ "Selected node group: " ++ gname let (bad_nodes, bad_instances) = Cluster.computeBadItems nl il unless (verbose < 1) $ printf "Initial check done: %d bad nodes, %d bad instances.\n" (length bad_nodes) (length bad_instances) let other_nodes = filter (not . (`elem` bad_nodes)) $ Container.elems nl node_status = map (Node.name &&& Node.getPolicyHealth) other_nodes policy_bad = filter (isBad . snd) node_status when (verbose > 4) $ do printf "Bad nodes: %s\n" . show $ map Node.name bad_nodes :: IO () printf "N+1 happy nodes: %s\n" . show $ map Node.name other_nodes :: IO () printf "Node policy status: %s\n" $ show node_status :: IO () unless (null bad_nodes) $ putStrLn "Cluster is not N+1 happy, continuing but no guarantee \ \that the cluster will end N+1 happy." unless (null policy_bad) $ do printf "The cluster contains %d policy-violating nodes.\n" $ length policy_bad :: IO () putStrLn $ if force then "Continuing, ignoring soft errors." else "Continuing, but the set of moves might be too restricted;\ \ consider using the --ignore-soft-errors option." -- | Check that we actually need to rebalance. checkNeedRebalance :: Options -> Score -> Score -> IO () checkNeedRebalance opts ini_cv opt_cv = do let min_cv = optMinScore opts when (ini_cv - opt_cv < min_cv) $ do printf "Cluster is already well balanced (initial score %.6g,\n\ \optimum score due to N+1 reservations %.6g,\n\ \minimum score %.6g).\nNothing to do, exiting\n" ini_cv opt_cv min_cv:: IO () exitSuccess -- | Main function. main :: Options -> [String] -> IO () main opts args = do unless (null args) $ exitErr "This program doesn't take any arguments." let verbose = optVerbose opts shownodes = optShowNodes opts showinsts = optShowInsts opts force = optIgnoreSoftErrors opts ini_cdata@(ClusterData gl fixed_nl ilf ctags ipol) <- loadExternalData opts when (verbose > 1) $ do putStrLn $ "Loaded cluster tags: " ++ intercalate "," ctags putStrLn $ "Loaded cluster ipolicy: " ++ show ipol nlf <- setNodeStatus opts fixed_nl checkCluster verbose nlf ilf maybeSaveData (optSaveCluster opts) "original" "before balancing" ini_cdata (gname, (nl, il)) <- selectGroup opts gl nlf ilf checkGroup force verbose gname nl il maybePrintInsts showinsts "Initial" (Cluster.printInsts nl il) maybePrintNodes shownodes "Initial cluster" (Cluster.printNodes nl) let ini_cv = Metrics.compCV nl opt_cv = Metrics.optimalCVScore nl ini_tbl = Cluster.Table nl il ini_cv [] min_cv = optMinScore opts if verbose > 2 then printf "Initial coefficients: overall %.8f\n%s" ini_cv (Metrics.printStats " " nl)::IO () else printf "Initial score: %.8f\n" ini_cv checkNeedRebalance opts ini_cv opt_cv putStrLn "Trying to minimize the CV..." let imlen = maximum . map (length . Instance.alias) $ Container.elems il nmlen = maximum . map (length . Node.alias) $ Container.elems nl (fin_tbl, cmd_strs) <- iterateDepth True (fromCLIOptions opts) ini_tbl (optMaxLength opts) nmlen imlen [] (opt_cv + min_cv) let (Cluster.Table fin_nl fin_il fin_cv fin_plc) = fin_tbl ord_plc = reverse fin_plc sol_msg = case () of _ | null fin_plc -> printf "No solution found\n" | verbose > 2 -> printf "Final coefficients: overall %.8f\n%s" fin_cv (Metrics.printStats " " fin_nl) | otherwise -> printf "Cluster score improved from %.8f to %.8f\n" ini_cv fin_cv ::String putStr sol_msg unless (verbose < 1) $ printf "Solution length=%d\n" (length ord_plc) let cmd_jobs = (if optFirstJobGroup opts then take 1 else id) $ Cluster.splitJobs cmd_strs maybeSaveCommands (if optFirstJobGroup opts then "First set of jobs:" else "Commands to run to reach the above solution:") opts $ Cluster.formatCmds cmd_jobs maybeSaveData (optSaveCluster opts) "balanced" "after balancing" ini_cdata { cdNodes = fin_nl, cdInstances = fin_il } maybePrintInsts showinsts "Final" (Cluster.printInsts fin_nl fin_il) maybePrintNodes shownodes "Final cluster" (Cluster.printNodes fin_nl) when (verbose > 3) $ printStats nl fin_nl exitIfBad "hbal" =<< maybeExecJobs opts ord_plc fin_nl il cmd_jobs ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Hcheck.hs000064400000000000000000000311721476477700300221460ustar00rootroot00000000000000{-| Cluster checker. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Hcheck ( main , options , arguments ) where import Control.Monad import qualified Data.IntMap as IntMap import Data.List (transpose) import System.Exit import Text.Printf (printf) import Ganeti.HTools.AlgorithmParams (fromCLIOptions) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.Metrics as Metrics import qualified Ganeti.HTools.Cluster.Utils as ClusterUtils import qualified Ganeti.HTools.GlobalN1 as GlobalN1 import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Program.Hbal as Hbal import Ganeti.Common import Ganeti.HTools.CLI import Ganeti.HTools.ExtLoader import Ganeti.HTools.Loader import Ganeti.HTools.Types import Ganeti.Utils -- | Options list and functions. options :: IO [OptType] options = do luxi <- oLuxiSocket return [ oDataFile , oDiskMoves , oDynuFile , oIgnoreDyn , oEvacMode , oExInst , oExTags , oIAllocSrc , oInstMoves , luxi , oMachineReadable , oMaxCpu , oMaxSolLength , oMinDisk , oMinGain , oMinGainLim , oMinScore , oIgnoreSoftErrors , oNoSimulation , oOfflineNode , oQuiet , oRapiMaster , oSelInst , oNoCapacityChecks , oVerbose ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [] -- | Check phase - are we before (initial) or after rebalance. data Phase = Initial | Rebalanced -- | Level of presented statistics. data Level = GroupLvl String -- ^ Group level, with name | ClusterLvl -- ^ Cluster level -- | A type alias for a group index and node\/instance lists. type GroupInfo = (Gdx, (Node.List, Instance.List)) -- | A type alias for group stats. type GroupStats = ((Group.Group, Double), [Int]) -- | Prefix for machine readable names. htcPrefix :: String htcPrefix = "HCHECK" -- | Data showed both per group and per cluster. commonData :: Options -> [(String, String)] commonData opts = [ ("N1_FAIL", "Nodes not N+1 happy") , ("CONFLICT_TAGS", "Nodes with conflicting instances") , ("OFFLINE_PRI", "Instances having the primary node offline") , ("OFFLINE_SEC", "Instances having a secondary node offline") ] ++ [ ("GN1_FAIL", "Nodes not directly evacuateable") | optCapacity opts ] -- | Data showed per group. groupData :: Options -> [(String, String)] groupData opts = commonData opts ++ [("SCORE", "Group score")] -- | Data showed per cluster. clusterData :: Options -> [(String, String)] clusterData opts = commonData opts ++ [ ("NEED_REBALANCE", "Cluster is not healthy") ] -- | Phase-specific prefix for machine readable version. phasePrefix :: Phase -> String phasePrefix Initial = "INIT" phasePrefix Rebalanced = "FINAL" -- | Level-specific prefix for machine readable version. levelPrefix :: Level -> String levelPrefix GroupLvl {} = "GROUP" levelPrefix ClusterLvl = "CLUSTER" -- | Machine-readable keys to show depending on given level. keysData :: Options -> Level -> [String] keysData opts GroupLvl {} = map fst $ groupData opts keysData opts ClusterLvl = map fst $ clusterData opts -- | Description of phases for human readable version. phaseDescr :: Phase -> String phaseDescr Initial = "initially" phaseDescr Rebalanced = "after rebalancing" -- | Description to show depending on given level. descrData :: Options -> Level -> [String] descrData opts GroupLvl {} = map snd $ groupData opts descrData opts ClusterLvl = map snd $ clusterData opts -- | Human readable prefix for statistics. phaseLevelDescr :: Phase -> Level -> String phaseLevelDescr phase (GroupLvl name) = printf "Statistics for group %s %s\n" name $ phaseDescr phase phaseLevelDescr phase ClusterLvl = printf "Cluster statistics %s\n" $ phaseDescr phase -- | Format a list of key, value as a shell fragment. printKeysHTC :: [(String, String)] -> IO () printKeysHTC = printKeys htcPrefix -- | Prepare string from boolean value. printBool :: Bool -- ^ Whether the result should be machine readable -> Bool -- ^ Value to be converted to string -> String printBool True True = "1" printBool True False = "0" printBool False b = show b -- | Print mapping from group idx to group uuid (only in machine -- readable mode). printGroupsMappings :: Group.List -> IO () printGroupsMappings gl = do let extract_vals g = (printf "GROUP_UUID_%d" $ Group.idx g :: String, Group.uuid g) printpairs = map extract_vals (Container.elems gl) printKeysHTC printpairs -- | Prepare a single key given a certain level and phase of simulation. prepareKey :: Level -> Phase -> String -> String prepareKey level@ClusterLvl phase suffix = printf "%s_%s_%s" (phasePrefix phase) (levelPrefix level) suffix prepareKey level@(GroupLvl idx) phase suffix = printf "%s_%s_%s_%s" (phasePrefix phase) (levelPrefix level) idx suffix -- | Print all the statistics for given level and phase. printStats :: Options -> Bool -- ^ If the output should be machine readable -> Level -- ^ Level on which we are printing -> Phase -- ^ Current phase of simulation -> [String] -- ^ Values to print -> IO () printStats opts True level phase values = do let keys = map (prepareKey level phase) (keysData opts level) printKeysHTC $ zip keys values printStats opts False level phase values = do let prefix = phaseLevelDescr phase level descr = descrData opts level unless (optVerbose opts < 1) $ do putStrLn "" putStr prefix mapM_ (uncurry (printf " %s: %s\n")) (zip descr values) -- | Extract name or idx from group. extractGroupData :: Bool -> Group.Group -> String extractGroupData True grp = show $ Group.idx grp extractGroupData False grp = Group.name grp -- | Prepare values for group. prepareGroupValues :: [Int] -> Double -> [String] prepareGroupValues stats score = map show stats ++ [printf "%.8f" score] -- | Prepare values for cluster. prepareClusterValues :: Bool -> [Int] -> [Bool] -> [String] prepareClusterValues machineread stats bstats = map show stats ++ map (printBool machineread) bstats -- | Print all the statistics on a group level. printGroupStats :: Options -> Bool -> Phase -> GroupStats -> IO () printGroupStats opts machineread phase ((grp, score), stats) = do let values = prepareGroupValues stats score extradata = extractGroupData machineread grp printStats opts machineread (GroupLvl extradata) phase values -- | Print all the statistics on a cluster (global) level. printClusterStats :: Options -> Bool -> Phase -> [Int] -> Bool -> IO () printClusterStats opts machineread phase stats needhbal = do let values = prepareClusterValues machineread stats [needhbal] printStats opts machineread ClusterLvl phase values -- | Check if any of cluster metrics is non-zero. clusterNeedsRebalance :: [Int] -> Bool clusterNeedsRebalance stats = sum stats > 0 {- | Check group for N+1 hapiness, conflicts of primaries on nodes and instances residing on offline nodes. -} perGroupChecks :: Options -> Group.List -> GroupInfo -> GroupStats perGroupChecks opts gl (gidx, (nl, il)) = let grp = Container.find gidx gl offnl = filter Node.offline (Container.elems nl) n1violated = length . fst $ Cluster.computeBadItems nl il gn1fail = length . filter (not . GlobalN1.canEvacuateNode (nl, il)) $ IntMap.elems nl conflicttags = length $ filter (>0) (map Node.conflictingPrimaries (Container.elems nl)) offline_pri = sum . map length $ map Node.pList offnl offline_sec = length $ map Node.sList offnl score = Metrics.compCV nl groupstats = [ n1violated , conflicttags , offline_pri , offline_sec ] ++ [ gn1fail | optCapacity opts ] in ((grp, score), groupstats) -- | Use Hbal's iterateDepth to simulate group rebalance. executeSimulation :: Options -> Cluster.Table -> Double -> Gdx -> Node.List -> Instance.List -> IO GroupInfo executeSimulation opts ini_tbl min_cv gidx nl il = do let imlen = maximum . map (length . Instance.alias) $ Container.elems il nmlen = maximum . map (length . Node.alias) $ Container.elems nl (fin_tbl, _) <- Hbal.iterateDepth False (fromCLIOptions opts) ini_tbl (optMaxLength opts) nmlen imlen [] min_cv let (Cluster.Table fin_nl fin_il _ _) = fin_tbl return (gidx, (fin_nl, fin_il)) -- | Simulate group rebalance if group's score is not good maybeSimulateGroupRebalance :: Options -> GroupInfo -> IO GroupInfo maybeSimulateGroupRebalance opts (gidx, (nl, il)) = do let ini_cv = Metrics.compCV nl ini_tbl = Cluster.Table nl il ini_cv [] min_cv = optMinScore opts + Metrics.optimalCVScore nl if ini_cv < min_cv then return (gidx, (nl, il)) else executeSimulation opts ini_tbl min_cv gidx nl il -- | Decide whether to simulate rebalance. maybeSimulateRebalance :: Bool -- ^ Whether to simulate rebalance -> Options -- ^ Command line options -> [GroupInfo] -- ^ Group data -> IO [GroupInfo] maybeSimulateRebalance True opts cluster = mapM (maybeSimulateGroupRebalance opts) cluster maybeSimulateRebalance False _ cluster = return cluster -- | Prints the final @OK@ marker in machine readable output. printFinalHTC :: Bool -> IO () printFinalHTC = printFinal htcPrefix -- | Main function. main :: Options -> [String] -> IO () main opts args = do unless (null args) $ exitErr "This program doesn't take any arguments." let verbose = optVerbose opts machineread = optMachineReadable opts nosimulation = optNoSimulation opts (ClusterData gl fixed_nl ilf _ _) <- loadExternalData opts nlf <- setNodeStatus opts fixed_nl let splitcluster = ClusterUtils.splitCluster nlf ilf when machineread $ printGroupsMappings gl let groupsstats = map (perGroupChecks opts gl) splitcluster clusterstats = map sum . transpose . map snd $ groupsstats needrebalance = clusterNeedsRebalance clusterstats unless (verbose < 1 || machineread) . putStrLn $ if nosimulation then "Running in no-simulation mode." else if needrebalance then "Cluster needs rebalancing." else "No need to rebalance cluster, no problems found." mapM_ (printGroupStats opts machineread Initial) groupsstats printClusterStats opts machineread Initial clusterstats needrebalance let exitOK = nosimulation || not needrebalance simulate = not nosimulation && needrebalance rebalancedcluster <- maybeSimulateRebalance simulate opts splitcluster when (simulate || machineread) $ do let newgroupstats = map (perGroupChecks opts gl) rebalancedcluster newclusterstats = map sum . transpose . map snd $ newgroupstats newneedrebalance = clusterNeedsRebalance clusterstats mapM_ (printGroupStats opts machineread Rebalanced) newgroupstats printClusterStats opts machineread Rebalanced newclusterstats newneedrebalance printFinalHTC machineread unless exitOK . exitWith $ ExitFailure 1 ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Hinfo.hs000064400000000000000000000146451476477700300220320ustar00rootroot00000000000000{-| Cluster information printer. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Hinfo ( main , options , arguments ) where import Control.Monad import Data.List import System.IO import Text.Printf (printf) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.Utils as ClusterUtils import qualified Ganeti.HTools.Cluster.Metrics as Metrics import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Instance as Instance import Ganeti.Common import Ganeti.HTools.CLI import Ganeti.HTools.ExtLoader import Ganeti.HTools.Loader import Ganeti.Utils -- | Options list and functions. options :: IO [OptType] options = do luxi <- oLuxiSocket return [ oPrintNodes , oPrintInsts , oDataFile , oRapiMaster , luxi , oIAllocSrc , oVerbose , oQuiet , oOfflineNode , oIgnoreDyn , oMonD , oMonDDataFile , oStaticKvmNodeMemory ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [] -- | Group information data-type. data GroupInfo = GroupInfo { giName :: String , giNodeCount :: Int , giInstCount :: Int , giBadNodes :: Int , giBadInsts :: Int , giN1Status :: Bool , giScore :: Double } -- | Node group statistics. calcGroupInfo :: Group.Group -> Node.List -> Instance.List -> GroupInfo calcGroupInfo g nl il = let nl_size = Container.size nl il_size = Container.size il (bad_nodes, bad_instances) = Cluster.computeBadItems nl il bn_size = length bad_nodes bi_size = length bad_instances n1h = bn_size == 0 score = Metrics.compCV nl in GroupInfo (Group.name g) nl_size il_size bn_size bi_size n1h score -- | Helper to format one group row result. groupRowFormatHelper :: GroupInfo -> [String] groupRowFormatHelper gi = [ giName gi , printf "%d" $ giNodeCount gi , printf "%d" $ giInstCount gi , printf "%d" $ giBadNodes gi , printf "%d" $ giBadInsts gi , show $ giN1Status gi , printf "%.8f" $ giScore gi ] -- | Print node group information. showGroupInfo :: Int -> Group.List -> Node.List -> Instance.List -> IO () showGroupInfo verbose gl nl il = do let cgrs = map (\(gdx, (gnl, gil)) -> calcGroupInfo (Container.find gdx gl) gnl gil) $ ClusterUtils.splitCluster nl il cn1h = all giN1Status cgrs grs = map groupRowFormatHelper cgrs header = ["Group", "Nodes", "Instances", "Bad_Nodes", "Bad_Instances", "N+1", "Score"] when (verbose > 1) $ printf "Node group information:\n%s" (printTable " " header grs [False, True, True, True, True, False, True]) printf "Cluster is N+1 %s\n" $ if cn1h then "happy" else "unhappy" -- | Gather and print split instances. splitInstancesInfo :: Int -> Node.List -> Instance.List -> IO () splitInstancesInfo verbose nl il = do let split_insts = Cluster.findSplitInstances nl il if null split_insts then when (verbose > 1) $ putStrLn "No split instances found"::IO () else do putStrLn "Found instances belonging to multiple node groups:" mapM_ (\i -> hPutStrLn stderr $ " " ++ Instance.name i) split_insts -- | Print common (interesting) information. commonInfo :: Int -> Group.List -> Node.List -> Instance.List -> IO () commonInfo verbose gl nl il = do when (Container.null il && verbose > 1) $ printf "Cluster is empty.\n"::IO () let nl_size = Container.size nl il_size = Container.size il gl_size = Container.size gl printf "Loaded %d %s, %d %s, %d %s\n" nl_size (plural nl_size "node" "nodes") il_size (plural il_size "instance" "instances") gl_size (plural gl_size "node group" "node groups")::IO () let csf = commonSuffix nl il when (not (null csf) && verbose > 2) $ printf "Note: Stripping common suffix of '%s' from names\n" csf -- | Main function. main :: Options -> [String] -> IO () main opts args = do unless (null args) $ exitErr "This program doesn't take any arguments." let verbose = optVerbose opts shownodes = optShowNodes opts showinsts = optShowInsts opts (ClusterData gl fixed_nl ilf ctags ipol) <- loadExternalData opts putStrLn $ "Loaded cluster tags: " ++ intercalate "," ctags when (verbose > 2) . putStrLn $ "Loaded cluster ipolicy: " ++ show ipol nlf <- setNodeStatus opts fixed_nl commonInfo verbose gl nlf ilf splitInstancesInfo verbose nlf ilf showGroupInfo verbose gl nlf ilf maybePrintInsts showinsts "Instances" (Cluster.printInsts nlf ilf) maybePrintNodes shownodes "Cluster" (Cluster.printNodes nlf) printf "Cluster coefficients:\n%s" (Metrics.printStats " " nlf)::IO () printf "Cluster score: %.8f\n" (Metrics.compCV nlf) ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Hroller.hs000064400000000000000000000445101476477700300223700ustar00rootroot00000000000000{-| Cluster rolling maintenance helper. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Hroller ( main , options , arguments ) where import Control.Applicative import Control.Arrow import Control.Monad import Data.Function import Data.List import Data.Ord import Text.Printf import qualified Data.IntMap as IntMap import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Group as Group import Ganeti.BasicTypes import Ganeti.Common import Ganeti.HTools.CLI import Ganeti.HTools.ExtLoader import Ganeti.HTools.Graph import Ganeti.HTools.Loader import Ganeti.HTools.Types import Ganeti.Utils -- | Options list and functions. options :: IO [OptType] options = do luxi <- oLuxiSocket return [ luxi , oRapiMaster , oDataFile , oIAllocSrc , oOfflineNode , oOfflineMaintenance , oVerbose , oQuiet , oNoHeaders , oNodeTags , oSaveCluster , oGroup , oPrintMoves , oFullEvacuation , oSkipNonRedundant , oIgnoreNonRedundant , oForce , oOneStepOnly , oStaticKvmNodeMemory ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [] -- | Compute the result of moving an instance to a different node. move :: Idx -> Ndx -> (Node.List, Instance.List) -> OpResult (Node.List, Instance.List) move idx new_ndx (nl, il) = do let new_node = Container.find new_ndx nl inst = Container.find idx il old_ndx = Instance.pNode inst old_node = Container.find old_ndx nl new_node' <- Node.addPriEx True new_node inst let old_node' = Node.removePri old_node inst inst' = Instance.setPri inst new_ndx nl' = Container.addTwo old_ndx old_node' new_ndx new_node' nl il' = Container.add idx inst' il return (nl', il') -- | Move a non-redundant instance to one of the candidate nodes mentioned. locateInstance :: Idx -> [Ndx] -> (Node.List, Instance.List) -> Result (Node.List, Instance.List) locateInstance idx ndxs conf = msum $ map (opToResult . flip (move idx) conf) ndxs -- | Move a list of non-redundant instances to some of the nodes mentioned. locateInstances :: [Idx] -> [Ndx] -> (Node.List, Instance.List) -> Result (Node.List, Instance.List) locateInstances idxs ndxs conf = foldM (\ cf idx -> locateInstance idx ndxs cf) conf idxs -- | Greedily clear a node of a kind of instances by a given relocation method. -- The arguments are a function providing the list of instances to be cleared, -- the relocation function, the list of nodes to be cleared, a list of nodes -- that can be relocated to, and the initial configuration. Returned is a list -- of nodes that can be cleared simultaneously and the configuration after -- clearing these nodes. greedyClearNodes :: ((Node.List, Instance.List) -> Ndx -> [Idx]) -> ([Idx] -> [Ndx] -> (Node.List, Instance.List) -> Result (Node.List, Instance.List)) -> [Ndx] -> [Ndx] -> (Node.List, Instance.List) -> Result ([Ndx], (Node.List, Instance.List)) greedyClearNodes _ _ [] _ conf = return ([], conf) greedyClearNodes getInstances relocate (ndx:ndxs) targets conf@(nl, _) = withFirst `mplus` withoutFirst where withFirst = do let othernodes = delete ndx targets grp = Node.group $ Container.find ndx nl othernodesSameGroup = filter ((==) grp . Node.group . flip Container.find nl) othernodes conf' <- relocate (getInstances conf ndx) othernodesSameGroup conf (ndxs', conf'') <- greedyClearNodes getInstances relocate ndxs othernodes conf' return (ndx:ndxs', conf'') withoutFirst = greedyClearNodes getInstances relocate ndxs targets conf -- | Greedily move the non-redundant instances away from a list of nodes. -- Returns a list of ndoes that can be cleared simultaneously and the -- configuration after clearing these nodes. clearNodes :: [Ndx] -> [Ndx] -> (Node.List, Instance.List) -> Result ([Ndx], (Node.List, Instance.List)) clearNodes = greedyClearNodes nonRedundant locateInstances -- | Partition nodes according to some clearing strategy. -- Arguments are the clearing strategy, the list of nodes to be cleared, -- the list of nodes that instances can be moved to, and the initial -- configuration. Returned is a partion of the nodes to be cleared with the -- configuration in that clearing situation. partitionNodes :: ([Ndx] -> [Ndx] -> (Node.List, Instance.List) -> Result ([Ndx], (Node.List, Instance.List))) -> [Ndx] -> [Ndx] -> (Node.List, Instance.List) -> Result [([Ndx], (Node.List, Instance.List))] partitionNodes _ [] _ _ = return [] partitionNodes clear ndxs targets conf = do (grp, conf') <- clear ndxs targets conf guard . not . null $ grp let remaining = ndxs \\ grp part <- partitionNodes clear remaining targets conf return $ (grp, conf') : part -- | Parition a list of nodes into chunks according cluster capacity. partitionNonRedundant :: [Ndx] -> [Ndx] -> (Node.List, Instance.List) -> Result [([Ndx], (Node.List, Instance.List))] partitionNonRedundant = partitionNodes clearNodes -- | Compute the result of migrating an instance. migrate :: Idx -> (Node.List, Instance.List) -> OpResult (Node.List, Instance.List) migrate idx (nl, il) = do let inst = Container.find idx il pdx = Instance.pNode inst sdx = Instance.sNode inst pNode = Container.find pdx nl sNode = Container.find sdx nl pNode' = Node.removePri pNode inst sNode' = Node.removeSec sNode inst sNode'' <- Node.addPriEx True sNode' inst pNode'' <- Node.addSecEx True pNode' inst sdx let inst' = Instance.setBoth inst sdx pdx nl' = Container.addTwo pdx pNode'' sdx sNode'' nl il' = Container.add idx inst' il return (nl', il') -- | Obtain the list of primaries for a given node. -- This restricts to those instances that have a secondary node. primaries :: (Node.List, Instance.List) -> Ndx -> [Idx] primaries (nl, il) = filter (Instance.hasSecondary . flip Container.find il) . Node.pList . flip Container.find nl -- | Migrate all instances of a given list of nodes. -- The list of nodes is repeated as first argument in the result. migrateOffNodes :: ([Ndx], (Node.List, Instance.List)) -> OpResult ([Ndx], (Node.List, Instance.List)) migrateOffNodes (ndxs, conf) = do let instances = ndxs >>= primaries conf conf' <- foldM (flip migrate) conf instances return (ndxs, conf') -- | Compute the result of replacing the secondary node of an instance. replaceSecondary :: Idx -> Ndx -> (Node.List, Instance.List) -> OpResult (Node.List, Instance.List) replaceSecondary idx new_ndx (nl, il) = do let new_secondary = Container.find new_ndx nl inst = Container.find idx il old_ndx = Instance.sNode inst pdx = Instance.pNode inst old_secondary = Container.find pdx nl if pdx == new_ndx then Bad FailInternal else Ok () new_secondary' <- Node.addSecEx True new_secondary inst pdx let old_secondary' = Node.removeSec old_secondary inst inst' = Instance.setSec inst new_ndx nl' = Container.addTwo old_ndx old_secondary' new_ndx new_secondary' nl il' = Container.add idx inst' il return (nl', il') -- | Find a suitable secondary node for the given instance from a list of nodes. findSecondary :: Idx -> [Ndx] -> (Node.List, Instance.List) -> Result (Node.List, Instance.List) findSecondary idx ndxs conf = msum $ map (opToResult . flip (replaceSecondary idx) conf) ndxs -- | Find suitable secondary nodes from the given nodes for a list of instances. findSecondaries :: [Idx] -> [Ndx] -> (Node.List, Instance.List) -> Result (Node.List, Instance.List) findSecondaries idxs ndxs conf = foldM (\ cf idx -> findSecondary idx ndxs cf) conf idxs -- | Obtain the list of secondaries for a given node. secondaries :: (Node.List, Instance.List) -> Ndx -> [Idx] secondaries (nl, _) = Node.sList . flip Container.find nl -- | Greedily move secondaries away from a list of nodes. -- Returns a list of nodes that can be cleared simultaneously, -- and the configuration after these nodes are cleared. clearSecondaries :: [Ndx] -> [Ndx] -> (Node.List, Instance.List) -> Result ([Ndx], (Node.List, Instance.List)) clearSecondaries = greedyClearNodes secondaries findSecondaries -- | Partition a list of nodes into chunks according to the ability to find -- suitable replacement secondary nodes. partitionSecondaries :: [Ndx] -> [Ndx] -> (Node.List, Instance.List) -> Result [([Ndx], (Node.List, Instance.List))] partitionSecondaries = partitionNodes clearSecondaries -- | Gather statistics for the coloring algorithms. -- Returns a string with a summary on how each algorithm has performed, -- in order of non-decreasing effectiveness, and whether it tied or lost -- with the previous one. getStats :: [(String, ColorVertMap)] -> String getStats colorings = snd . foldr helper (0,"") $ algBySize colorings where algostat (algo, cmap) = algo ++ ": " ++ size cmap ++ grpsizes cmap size cmap = show (IntMap.size cmap) ++ " " grpsizes cmap = "(" ++ commaJoin (map (show.length) (IntMap.elems cmap)) ++ ")" algBySize = sortBy (flip (comparing (IntMap.size.snd))) helper :: (String, ColorVertMap) -> (Int, String) -> (Int, String) helper el (0, _) = ((IntMap.size.snd) el, algostat el) helper el (old, str) | old == elsize = (elsize, str ++ " TIE " ++ algostat el) | otherwise = (elsize, str ++ " LOOSE " ++ algostat el) where elsize = (IntMap.size.snd) el -- | Predicate of belonging to a given group restriction. hasGroup :: Maybe Group.Group -> Node.Node -> Bool hasGroup Nothing _ = True hasGroup (Just grp) node = Node.group node == Group.idx grp -- | Predicate of having at least one tag in a given set. hasTag :: Maybe [String] -> Node.Node -> Bool hasTag Nothing _ = True hasTag (Just tags) node = not . null $ Node.nTags node `intersect` tags -- | From a cluster configuration, get the list of non-redundant instances -- of a node. nonRedundant :: (Node.List, Instance.List) -> Ndx -> [Idx] nonRedundant (nl, il) ndx = filter (not . Instance.hasSecondary . flip Container.find il) $ Node.pList (Container.find ndx nl) -- | Within a cluster configuration, decide if the node hosts non-redundant -- Instances. noNonRedundant :: (Node.List, Instance.List) -> Node.Node -> Bool noNonRedundant conf = null . nonRedundant conf . Node.idx -- | Put the master node last. -- Reorder a list groups of nodes (with additional information) such that the -- master node (if present) is the last node of the last group. masterLast :: [([Node.Node], a)] -> [([Node.Node], a)] masterLast rebootgroups = map (first $ uncurry (++)) . uncurry (++) . partition (null . snd . fst) $ map (first $ partition (not . Node.isMaster)) rebootgroups -- | From two configurations compute the list of moved instances. -- Do not show instances where only primary and secondary switched their -- role, as here the instance is not moved in a proper sense. getMoves :: (Node.List, Instance.List) -> (Node.List, Instance.List) -> [(Instance.Instance, (Node.Node, Maybe Node.Node))] getMoves (_, il) (nl', il') = do ix <- Container.keys il let inst = Container.find ix il inst' = Container.find ix il' hasSec = Instance.hasSecondary inst guard $ Instance.pNode inst /= Instance.pNode inst' || (hasSec && Instance.sNode inst /= Instance.sNode inst') guard . not $ Instance.pNode inst' == Instance.sNode inst && Instance.sNode inst' == Instance.pNode inst return (inst', (Container.find (Instance.pNode inst') nl', if hasSec then Just $ Container.find (Instance.sNode inst') nl' else Nothing)) -- | Main function. main :: Options -> [String] -> IO () main opts args = do unless (null args) $ exitErr "This program doesn't take any arguments." let verbose = optVerbose opts maybeExit = if optForce opts then warn else exitErr -- Load cluster data. The last two arguments, cluster tags and ipolicy, are -- currently not used by this tool. ini_cdata@(ClusterData gl fixed_nl ilf _ _) <- loadExternalData opts let master_names = map Node.name . filter Node.isMaster . IntMap.elems $ fixed_nl case master_names of [] -> maybeExit "No master node found (maybe not supported by backend)." [ _ ] -> return () _ -> exitErr $ "Found more than one master node: " ++ show master_names nlf <- setNodeStatus opts fixed_nl maybeSaveData (optSaveCluster opts) "original" "before hroller run" ini_cdata -- Find the wanted node group, if any. wantedGroup <- case optGroup opts of Nothing -> return Nothing Just name -> case Container.findByName gl name of Nothing -> exitErr "Cannot find target group." Just grp -> return (Just grp) let nodes = IntMap.filter (foldl (liftA2 (&&)) (const True) [ not . Node.offline , if optSkipNonRedundant opts then noNonRedundant (nlf, ilf) else const True , hasTag $ optNodeTags opts , hasGroup wantedGroup ]) nlf mkGraph = if optOfflineMaintenance opts then Node.mkNodeGraph else Node.mkRebootNodeGraph nlf nodeGraph <- case mkGraph nodes ilf of Nothing -> exitErr "Cannot create node graph" Just g -> return g when (verbose > 2) . putStrLn $ "Node Graph: " ++ show nodeGraph let colorAlgorithms = [ ("LF", colorLF) , ("Dsatur", colorDsatur) , ("Dcolor", colorDcolor) ] colorings = map (\(v,a) -> (v,(colorVertMap.a) nodeGraph)) colorAlgorithms smallestColoring = IntMap.elems $ (snd . minimumBy (comparing (IntMap.size . snd))) colorings allNdx = map Node.idx . filter (not . Node.offline) . Container.elems $ nlf splitted = mapM (\ grp -> partitionNonRedundant grp allNdx (nlf,ilf)) smallestColoring rebootGroups <- if optIgnoreNonRedundant opts then return $ zip smallestColoring (repeat (nlf, ilf)) else case splitted of Ok splitgroups -> return $ concat splitgroups Bad _ -> exitErr "Not enough capacity to move\ \ non-redundant instances" let migrated = mapM migrateOffNodes rebootGroups rebootGroups' <- if not . optFullEvacuation $ opts then return rebootGroups else case migrated of Ok migratedGroup -> return migratedGroup Bad _ -> exitErr "Failed to migrate instances\ \ off nodes" let splitted' = mapM (\(grp, conf) -> partitionSecondaries grp allNdx conf) rebootGroups' rebootGroups'' <- if optFullEvacuation opts then case splitted' of Ok splitgroups -> return $ concat splitgroups Bad _ -> exitErr "Not enough capacity to move\ \ secondaries" else return rebootGroups' let idToNode = (`Container.find` nodes) nodesRebootGroups = map (first $ map idToNode . filter (`IntMap.member` nodes)) rebootGroups'' outputRebootGroups = masterLast . sortBy (flip compare `on` length . fst) $ nodesRebootGroups confToMoveNames = map (Instance.name *** (Node.name *** (=<<) (return . Node.name))) . getMoves (nlf, ilf) namesAndMoves = map (map Node.name *** confToMoveNames) outputRebootGroups when (verbose > 1) . putStrLn $ getStats colorings let showGroup = if optOneStepOnly opts then mapM_ putStrLn else putStrLn . commaJoin showMoves :: [(String, (String, Maybe String))] -> IO () showMoves = if optPrintMoves opts then mapM_ $ putStrLn . \(a,(b,c)) -> maybe (printf " %s %s" a b) (printf " %s %s %s" a b) c else const $ return () showBoth = liftM2 (>>) (showGroup . fst) (showMoves . snd) if optOneStepOnly opts then do unless (optNoHeaders opts) $ putStrLn "'First Reboot Group'" case namesAndMoves of [] -> return () y : _ -> showBoth y else do unless (optNoHeaders opts) $ putStrLn "'Node Reboot Groups'" mapM_ showBoth namesAndMoves ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Hscan.hs000064400000000000000000000134131476477700300220130ustar00rootroot00000000000000{-| Scan clusters via RAPI or LUXI and write state data files. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Hscan ( main , options , arguments ) where import Control.Monad import Data.Maybe (isJust, fromJust, fromMaybe) import System.Exit import System.IO import System.FilePath import System.Time import Text.Printf (printf) import Ganeti.BasicTypes import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.Metrics as Metrics import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Backend.Rapi as Rapi import qualified Ganeti.HTools.Backend.Luxi as Luxi import qualified Ganeti.Path as Path import Ganeti.HTools.Loader (updateMissing, mergeData, ClusterData(..)) import Ganeti.HTools.Backend.Text (serializeCluster) import Ganeti.Common import Ganeti.HTools.CLI -- | Options list and functions. options :: IO [OptType] options = do luxi <- oLuxiSocket return [ oPrintNodes , oOutputDir , luxi , oVerbose , oNoHeaders , oStaticKvmNodeMemory ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [ArgCompletion OptComplHost 0 Nothing] -- | Return a one-line summary of cluster state. printCluster :: Node.List -> Instance.List -> String printCluster nl il = let (bad_nodes, bad_instances) = Cluster.computeBadItems nl il ccv = Metrics.compCV nl nodes = Container.elems nl insts = Container.elems il t_ram = sum . map Node.tMem $ nodes t_dsk = sum . map Node.tDsk $ nodes f_ram = sum . map Node.fMem $ nodes f_dsk = sum . map Node.fDsk $ nodes in printf "%5d %5d %5d %5d %6.0f %6d %6.0f %6d %.8f" (length nodes) (length insts) (length bad_nodes) (length bad_instances) t_ram f_ram (t_dsk / 1024) (f_dsk `div` 1024) ccv -- | Replace slashes with underscore for saving to filesystem. fixSlash :: String -> String fixSlash = map (\x -> if x == '/' then '_' else x) -- | Generates serialized data from loader input. processData :: ClockTime -> ClusterData -> Int -> Result ClusterData processData now input_data static_n_mem = do cdata@(ClusterData _ nl il _ _) <- mergeData [] [] [] [] now input_data let (_, fix_nl) = updateMissing nl il static_n_mem return cdata { cdNodes = fix_nl } -- | Writes cluster data out. writeData :: Int -> String -> Options -> Result ClusterData -> IO Bool writeData _ name _ (Bad err) = printf "\nError for %s: failed to load data. Details:\n%s\n" name err >> return False writeData nlen name opts (Ok cdata) = do now <- getClockTime let static_n_mem = optStaticKvmNodeMemory opts fixdata = processData now cdata static_n_mem case fixdata of Bad err -> printf "\nError for %s: failed to process data. Details:\n%s\n" name err >> return False Ok processed -> writeDataInner nlen name opts cdata processed -- | Inner function for writing cluster data to disk. writeDataInner :: Int -> String -> Options -> ClusterData -> ClusterData -> IO Bool writeDataInner nlen name opts cdata fixdata = do let (ClusterData _ nl il _ _) = fixdata printf "%-*s " nlen name :: IO () hFlush stdout let shownodes = optShowNodes opts odir = optOutPath opts oname = odir fixSlash name putStrLn $ printCluster nl il hFlush stdout when (isJust shownodes) . putStr $ Cluster.printNodes nl (fromJust shownodes) writeFile (oname <.> "data") (serializeCluster cdata) return True -- | Main function. main :: Options -> [String] -> IO () main opts clusters = do let local = "LOCAL" let nlen = if null clusters then length local else maximum . map length $ clusters unless (optNoHeaders opts) $ printf "%-*s %5s %5s %5s %5s %6s %6s %6s %6s %10s\n" nlen "Name" "Nodes" "Inst" "BNode" "BInst" "t_mem" "f_mem" "t_disk" "f_disk" "Score" when (null clusters) $ do def_socket <- Path.defaultQuerySocket let lsock = fromMaybe def_socket (optLuxi opts) let name = local input_data <- Luxi.loadData lsock result <- writeData nlen name opts input_data unless result . exitWith $ ExitFailure 2 results <- mapM (\name -> Rapi.loadData name >>= writeData nlen name opts) clusters unless (and results) $ exitWith (ExitFailure 2) ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Hspace.hs000064400000000000000000000535121476477700300221660ustar00rootroot00000000000000{-| Cluster space sizing -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Hspace (main , options , arguments ) where import Control.Monad import Data.Char (toUpper, toLower) import Data.Function (on) import qualified Data.IntMap as IntMap import qualified Data.List.NonEmpty as NonEmpty import Data.List import Data.Maybe (fromMaybe) import Data.Ord (comparing) import System.IO import Text.Printf (printf, hPrintf) import qualified Ganeti.HTools.AlgorithmParams as Alg import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.Metrics as Metrics import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Instance as Instance import Ganeti.BasicTypes import Ganeti.Common import Ganeti.HTools.AlgorithmParams (AlgorithmOptions(algCapacityIgnoreGroups)) import Ganeti.HTools.GlobalN1 (redundantGrp) import Ganeti.HTools.Types import Ganeti.HTools.CLI import Ganeti.HTools.ExtLoader import Ganeti.HTools.Loader import Ganeti.Utils -- | Options list and functions. options :: IO [OptType] options = do luxi <- oLuxiSocket return [ oPrintNodes , oDataFile , oDiskTemplate , oSpindleUse , oNodeSim , oRapiMaster , luxi , oIAllocSrc , oVerbose , oQuiet , oIndependentGroups , oAcceptExisting , oOfflineNode , oNoCapacityChecks , oMachineReadable , oMaxCpu , oMaxSolLength , oMinDisk , oStdSpec , oTieredSpec , oSaveCluster , oStaticKvmNodeMemory ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [] -- | The allocation phase we're in (initial, after tiered allocs, or -- after regular allocation). data Phase = PInitial | PFinal | PTiered -- | The kind of instance spec we print. data SpecType = SpecNormal | SpecTiered -- | Prefix for machine readable names htsPrefix :: String htsPrefix = "HTS" -- | What we prefix a spec with. specPrefix :: SpecType -> String specPrefix SpecNormal = "SPEC" specPrefix SpecTiered = "TSPEC_INI" -- | The description of a spec. specDescription :: SpecType -> String specDescription SpecNormal = "Standard (fixed-size)" specDescription SpecTiered = "Tiered (initial size)" -- | The \"name\" of a 'SpecType'. specName :: SpecType -> String specName SpecNormal = "Standard" specName SpecTiered = "Tiered" -- | Efficiency generic function. effFn :: (Cluster.CStats -> Integer) -> (Cluster.CStats -> Double) -> Cluster.CStats -> Double effFn fi ft cs = fromIntegral (fi cs) / ft cs -- | Memory efficiency. memEff :: Cluster.CStats -> Double memEff = effFn Cluster.csImem Cluster.csTmem -- | Disk efficiency. dskEff :: Cluster.CStats -> Double dskEff = effFn Cluster.csIdsk Cluster.csTdsk -- | Cpu efficiency. cpuEff :: Cluster.CStats -> Double cpuEff = effFn Cluster.csIcpu (fromIntegral . Cluster.csVcpu) -- | Spindles efficiency. spnEff :: Cluster.CStats -> Double spnEff = effFn Cluster.csIspn Cluster.csTspn -- | Holds data for converting a 'Cluster.CStats' structure into -- detailed statistics. statsData :: [(String, Cluster.CStats -> String)] statsData = [ ("SCORE", printf "%.8f" . Cluster.csScore) , ("INST_CNT", printf "%d" . Cluster.csNinst) , ("MEM_FREE", printf "%d" . Cluster.csFmem) , ("MEM_AVAIL", printf "%d" . Cluster.csAmem) , ("MEM_RESVD", \cs -> printf "%d" (Cluster.csFmem cs - Cluster.csAmem cs)) , ("MEM_INST", printf "%d" . Cluster.csImem) , ("MEM_OVERHEAD", \cs -> printf "%d" (Cluster.csXmem cs + Cluster.csNmem cs)) , ("MEM_EFF", printf "%.8f" . memEff) , ("DSK_FREE", printf "%d" . Cluster.csFdsk) , ("DSK_AVAIL", printf "%d". Cluster.csAdsk) , ("DSK_RESVD", \cs -> printf "%d" (Cluster.csFdsk cs - Cluster.csAdsk cs)) , ("DSK_INST", printf "%d" . Cluster.csIdsk) , ("DSK_EFF", printf "%.8f" . dskEff) , ("SPN_FREE", printf "%d" . Cluster.csFspn) , ("SPN_INST", printf "%d" . Cluster.csIspn) , ("SPN_EFF", printf "%.8f" . spnEff) , ("CPU_INST", printf "%d" . Cluster.csIcpu) , ("CPU_EFF", printf "%.8f" . cpuEff) , ("MNODE_MEM_AVAIL", printf "%d" . Cluster.csMmem) , ("MNODE_DSK_AVAIL", printf "%d" . Cluster.csMdsk) ] -- | List holding 'RSpec' formatting information. specData :: [(String, RSpec -> String)] specData = [ ("MEM", printf "%d" . rspecMem) , ("DSK", printf "%d" . rspecDsk) , ("CPU", printf "%d" . rspecCpu) ] -- | 'RSpec' formatting information including spindles. specDataSpn :: [(String, RSpec -> String)] specDataSpn = specData ++ [("SPN", printf "%d" . rspecSpn)] -- | List holding 'Cluster.CStats' formatting information. clusterData :: [(String, Cluster.CStats -> String)] clusterData = [ ("MEM", printf "%.0f" . Cluster.csTmem) , ("DSK", printf "%.0f" . Cluster.csTdsk) , ("CPU", printf "%.0f" . Cluster.csTcpu) , ("VCPU", printf "%d" . Cluster.csVcpu) ] -- | 'Cluster.CStats' formatting information including spindles clusterDataSpn :: [(String, Cluster.CStats -> String)] clusterDataSpn = clusterData ++ [("SPN", printf "%.0f" . Cluster.csTspn)] -- | Function to print stats for a given phase. printStats :: Phase -> Cluster.CStats -> [(String, String)] printStats ph cs = map (\(s, fn) -> (printf "%s_%s" kind s, fn cs)) statsData where kind = case ph of PInitial -> "INI" PFinal -> "FIN" PTiered -> "TRL" -- | Print failure reason and scores printFRScores :: Node.List -> Node.List -> [(FailMode, Int)] -> IO () printFRScores ini_nl fin_nl sreason = do printf " - most likely failure reason: %s\n" $ failureReason sreason::IO () printClusterScores ini_nl fin_nl printClusterEff (Cluster.totalResources fin_nl) (Node.haveExclStorage fin_nl) -- | Print final stats and related metrics. printResults :: Bool -> Node.List -> Node.List -> Int -> Int -> [(FailMode, Int)] -> IO () printResults True _ fin_nl num_instances allocs sreason = do let fin_stats = Cluster.totalResources fin_nl fin_instances = num_instances + allocs exitWhen (num_instances + allocs /= Cluster.csNinst fin_stats) $ printf "internal inconsistency, allocated (%d)\ \ != counted (%d)\n" (num_instances + allocs) (Cluster.csNinst fin_stats) main_reason <- exitIfEmpty "Internal error, no failure reasons?!" sreason printKeysHTS $ printStats PFinal fin_stats printKeysHTS [ ("ALLOC_USAGE", printf "%.8f" ((fromIntegral num_instances::Double) / fromIntegral fin_instances)) , ("ALLOC_INSTANCES", printf "%d" allocs) , ("ALLOC_FAIL_REASON", map toUpper . show . fst $ main_reason) ] printKeysHTS $ map (\(x, y) -> (printf "ALLOC_%s_CNT" (show x), printf "%d" y)) sreason printResults False ini_nl fin_nl _ allocs sreason = do putStrLn "Normal (fixed-size) allocation results:" printf " - %3d instances allocated\n" allocs :: IO () printFRScores ini_nl fin_nl sreason -- | Prints the final @OK@ marker in machine readable output. printFinalHTS :: Bool -> IO () printFinalHTS = printFinal htsPrefix -- | Compute the tiered spec counts from a list of allocated instances. tieredSpecMap :: [Instance.Instance] -> [(RSpec, Int)] tieredSpecMap trl_ixes = let fin_trl_ixes = reverse trl_ixes ix_byspec = NonEmpty.groupBy ((==) `on` Instance.specOf) fin_trl_ixes spec_map = map (\ixs -> (Instance.specOf $ NonEmpty.head ixs, NonEmpty.length ixs)) ix_byspec in spec_map -- | Formats a spec map to strings. formatSpecMap :: [(RSpec, Int)] -> [String] formatSpecMap = map (\(spec, cnt) -> printf "%d,%d,%d,%d=%d" (rspecMem spec) (rspecDsk spec) (rspecCpu spec) (rspecSpn spec) cnt) -- | Formats \"key-metrics\" values. formatRSpec :: String -> AllocInfo -> [(String, String)] formatRSpec s r = [ ("KM_" ++ s ++ "_CPU", show $ allocInfoVCpus r) , ("KM_" ++ s ++ "_NPU", show $ allocInfoNCpus r) , ("KM_" ++ s ++ "_MEM", show $ allocInfoMem r) , ("KM_" ++ s ++ "_DSK", show $ allocInfoDisk r) , ("KM_" ++ s ++ "_SPN", show $ allocInfoSpn r) ] -- | Shows allocations stats. printAllocationStats :: Node.List -> Node.List -> IO () printAllocationStats ini_nl fin_nl = do let ini_stats = Cluster.totalResources ini_nl fin_stats = Cluster.totalResources fin_nl (rini, ralo, runa) = Cluster.computeAllocationDelta ini_stats fin_stats printKeysHTS $ formatRSpec "USED" rini printKeysHTS $ formatRSpec "POOL" ralo printKeysHTS $ formatRSpec "UNAV" runa -- | Format a list of key\/values as a shell fragment. printKeysHTS :: [(String, String)] -> IO () printKeysHTS = printKeys htsPrefix -- | Converts instance data to a list of strings. printInstance :: Node.List -> Instance.Instance -> [String] printInstance nl i = [ Instance.name i , Container.nameOf nl $ Instance.pNode i , let sdx = Instance.sNode i in if sdx == Node.noSecondary then "" else Container.nameOf nl sdx , show (Instance.mem i) , show (Instance.dsk i) , show (Instance.vcpus i) , if Node.haveExclStorage nl then case Instance.getTotalSpindles i of Nothing -> "?" Just sp -> show sp else "" ] -- | Optionally print the allocation map. printAllocationMap :: Int -> String -> Node.List -> [Instance.Instance] -> IO () printAllocationMap verbose msg nl ixes = when (verbose > 1) $ do hPutStrLn stderr (msg ++ " map") hPutStr stderr . unlines . map ((:) ' ' . unwords) $ formatTable (map (printInstance nl) (reverse ixes)) -- This is the numberic-or-not field -- specification; the first three fields are -- strings, whereas the rest are numeric [False, False, False, True, True, True, True] -- | Formats nicely a list of resources. formatResources :: a -> [(String, a->String)] -> String formatResources res = intercalate ", " . map (\(a, fn) -> a ++ " " ++ fn res) -- | Print the cluster resources. printCluster :: Bool -> Cluster.CStats -> Int -> Bool -> IO () printCluster True ini_stats node_count _ = do printKeysHTS $ map (\(a, fn) -> ("CLUSTER_" ++ a, fn ini_stats)) clusterDataSpn printKeysHTS [("CLUSTER_NODES", printf "%d" node_count)] printKeysHTS $ printStats PInitial ini_stats printCluster False ini_stats node_count print_spn = do let cldata = if print_spn then clusterDataSpn else clusterData printf "The cluster has %d nodes and the following resources:\n %s.\n" node_count (formatResources ini_stats cldata)::IO () printf "There are %s initial instances on the cluster.\n" (if inst_count > 0 then show inst_count else "no" ) where inst_count = Cluster.csNinst ini_stats -- | Prints the normal instance spec. printISpec :: Bool -> RSpec -> SpecType -> DiskTemplate -> Bool -> IO () printISpec True ispec spec disk_template _ = do printKeysHTS $ map (\(a, fn) -> (prefix ++ "_" ++ a, fn ispec)) specDataSpn printKeysHTS [ (prefix ++ "_RQN", printf "%d" req_nodes) ] printKeysHTS [ (prefix ++ "_DISK_TEMPLATE", diskTemplateToRaw disk_template) ] where req_nodes = Instance.requiredNodes disk_template prefix = specPrefix spec printISpec False ispec spec disk_template print_spn = let spdata = if print_spn then specDataSpn else specData in printf "%s instance spec is:\n %s, using disk\ \ template '%s'.\n" (specDescription spec) (formatResources ispec spdata) (diskTemplateToRaw disk_template) -- | Prints the tiered results. printTiered :: Bool -> [(RSpec, Int)] -> Node.List -> Node.List -> [(FailMode, Int)] -> IO () printTiered True spec_map nl trl_nl _ = do printKeysHTS $ printStats PTiered (Cluster.totalResources trl_nl) printKeysHTS [("TSPEC", unwords (formatSpecMap spec_map))] printAllocationStats nl trl_nl printTiered False spec_map ini_nl fin_nl sreason = do _ <- printf "Tiered allocation results:\n" let spdata = if Node.haveExclStorage ini_nl then specDataSpn else specData if null spec_map then putStrLn " - no instances allocated" else mapM_ (\(ispec, cnt) -> printf " - %3d instances of spec %s\n" cnt (formatResources ispec spdata)) spec_map printFRScores ini_nl fin_nl sreason -- | Displays the initial/final cluster scores. printClusterScores :: Node.List -> Node.List -> IO () printClusterScores ini_nl fin_nl = do printf " - initial cluster score: %.8f\n" $ Metrics.compCV ini_nl::IO () printf " - final cluster score: %.8f\n" $ Metrics.compCV fin_nl -- | Displays the cluster efficiency. printClusterEff :: Cluster.CStats -> Bool -> IO () printClusterEff cs print_spn = do let format = [("memory", memEff), ("disk", dskEff), ("vcpu", cpuEff)] ++ [("spindles", spnEff) | print_spn] len = maximum $ map (length . fst) format mapM_ (\(s, fn) -> printf " - %*s usage efficiency: %5.2f%%\n" len s (fn cs * 100)) format -- | Computes the most likely failure reason. failureReason :: [(FailMode, Int)] -> String failureReason = show . fst . head -- | Sorts the failure reasons. sortReasons :: [(FailMode, Int)] -> [(FailMode, Int)] sortReasons = sortBy (flip $ comparing snd) -- | Runs an allocation algorithm and saves cluster state. runAllocation :: ClusterData -- ^ Cluster data -> Maybe Cluster.AllocResult -- ^ Optional stop-allocation -> Result Cluster.AllocResult -- ^ Allocation result -> RSpec -- ^ Requested instance spec -> DiskTemplate -- ^ Requested disk template -> SpecType -- ^ Allocation type -> Options -- ^ CLI options -> IO (FailStats, Node.List, Int, [(RSpec, Int)]) runAllocation cdata stop_allocation actual_result spec dt mode opts = do (reasons, new_nl, new_il, new_ixes, _) <- case stop_allocation of Just result_noalloc -> return result_noalloc Nothing -> exitIfBad "failure during allocation" actual_result let name = specName mode descr = name ++ " allocation" ldescr = "after " ++ map toLower descr excstor = Node.haveExclStorage new_nl printISpec (optMachineReadable opts) spec mode dt excstor printAllocationMap (optVerbose opts) descr new_nl new_ixes maybePrintNodes (optShowNodes opts) descr (Cluster.printNodes new_nl) maybeSaveData (optSaveCluster opts) (map toLower name) ldescr (cdata { cdNodes = new_nl, cdInstances = new_il}) return (sortReasons reasons, new_nl, length new_ixes, tieredSpecMap new_ixes) -- | Create an instance from a given spec. -- For values not implied by the resorce specification (like distribution of -- of the disk space to individual disks), sensible defaults are guessed (e.g., -- having a single disk). instFromSpec :: RSpec -> DiskTemplate -> Int -> Instance.Instance instFromSpec spx dt su = Instance.create "new" (rspecMem spx) (rspecDsk spx) [Instance.Disk (rspecDsk spx) (Just $ rspecSpn spx)] (rspecCpu spx) Running [] True (-1) (-1) dt su [] False combineTiered :: AlgorithmOptions -> Maybe Int -> Cluster.AllocNodes -> Cluster.AllocResult -> Instance.Instance -> Result Cluster.AllocResult combineTiered algOpts limit allocnodes result inst = do let (_, nl, il, ixes, cstats) = result ixes_cnt = length ixes (stop, newlimit) = case limit of Nothing -> (False, Nothing) Just n -> (n <= ixes_cnt, Just (n - ixes_cnt)) if stop then return result else Cluster.tieredAlloc algOpts nl il newlimit inst allocnodes ixes cstats -- | Main function. main :: Options -> [String] -> IO () main opts args = do exitUnless (null args) "This program doesn't take any arguments." let verbose = optVerbose opts machine_r = optMachineReadable opts independent_grps = optIndependentGroups opts accept_existing = optAcceptExisting opts algOpts = Alg.fromCLIOptions opts orig_cdata@(ClusterData gl fixed_nl il _ ipol) <- loadExternalData opts nl <- setNodeStatus opts fixed_nl cluster_disk_template <- case iPolicyDiskTemplates ipol of first_templ:_ -> return first_templ _ -> exitErr "null list of disk templates received from cluster" let num_instances = Container.size il all_nodes = Container.elems fixed_nl cdata = orig_cdata { cdNodes = fixed_nl } disk_template = fromMaybe cluster_disk_template (optDiskTemplate opts) req_nodes = Instance.requiredNodes disk_template csf = commonSuffix fixed_nl il su = fromMaybe (iSpecSpindleUse $ iPolicyStdSpec ipol) (optSpindleUse opts) when (not (null csf) && verbose > 1) $ hPrintf stderr "Note: Stripping common suffix of '%s' from names\n" csf maybePrintNodes (optShowNodes opts) "Initial cluster" (Cluster.printNodes nl) when (verbose > 2) $ hPrintf stderr "Initial coefficients: overall %.8f\n%s" (Metrics.compCV nl) (Metrics.printStats " " nl) printCluster machine_r (Cluster.totalResources nl) (length all_nodes) (Node.haveExclStorage nl) let (bad_nodes, _) = Cluster.computeBadItems nl il bad_grp_idxs = filter (not . redundantGrp algOpts nl il) $ Container.keys gl when ((verbose > 3) && (not . null $ bad_nodes)) . hPrintf stderr "Bad nodes: %s\n" . show . map Node.name $ bad_nodes when ((verbose > 3) && not (null bad_grp_idxs)) . hPrintf stderr "Bad groups: %s\n" . show $ map (Group.name . flip Container.find gl) bad_grp_idxs let markGrpsUnalloc = foldl (flip $ IntMap.adjust Group.setUnallocable) gl' = if accept_existing then gl else markGrpsUnalloc gl $ map Node.group bad_nodes (gl'', algOpts') = if independent_grps then ( markGrpsUnalloc gl' bad_grp_idxs , algOpts { algCapacityIgnoreGroups = bad_grp_idxs } ) else (gl', algOpts) grps_remaining = any Group.isAllocable $ IntMap.elems gl'' stop_allocation = case () of _ | accept_existing-> Nothing _ | independent_grps && grps_remaining -> Nothing _ | null bad_nodes -> Nothing _ -> Just ([(FailN1, 1)]::FailStats, nl, il, [], []) alloclimit = if optMaxLength opts == -1 then Nothing else Just (optMaxLength opts) allocnodes <- exitIfBad "failure during allocation" $ Cluster.genAllocNodes algOpts' gl'' nl req_nodes True when (verbose > 3) . hPrintf stderr "Allocatable nodes: %s\n" $ show allocnodes -- Run the tiered allocation let minmaxes = iPolicyMinMaxISpecs ipol tspecs = case optTieredSpec opts of Nothing -> map (rspecFromISpec . minMaxISpecsMaxSpec) minmaxes Just t -> [t] tinsts = map (\ts -> instFromSpec ts disk_template su) tspecs tspec <- case tspecs of [] -> exitErr "Empty list of specs received from the cluster" t:_ -> return t (treason, trl_nl, _, spec_map) <- runAllocation cdata stop_allocation (foldM (combineTiered algOpts' alloclimit allocnodes) ([], nl, il, [], []) tinsts ) tspec disk_template SpecTiered opts printTiered machine_r spec_map nl trl_nl treason -- Run the standard (avg-mode) allocation let ispec = fromMaybe (rspecFromISpec (iPolicyStdSpec ipol)) (optStdSpec opts) (sreason, fin_nl, allocs, _) <- runAllocation cdata stop_allocation (Cluster.iterateAlloc algOpts' nl il alloclimit (instFromSpec ispec disk_template su) allocnodes [] []) ispec disk_template SpecNormal opts printResults machine_r nl fin_nl num_instances allocs sreason -- Print final result printFinalHTS machine_r ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Hsqueeze.hs000064400000000000000000000344141476477700300225540ustar00rootroot00000000000000{-| Node freeing scheduler -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Hsqueeze (main , options , arguments ) where import Control.Applicative import Control.Lens (over) import Control.Monad import Data.Function import Data.List import Data.Maybe import qualified Data.IntMap as IntMap import Text.Printf (printf) import Ganeti.BasicTypes import Ganeti.Common import qualified Ganeti.HTools.AlgorithmParams as Alg import Ganeti.HTools.CLI import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.Metrics as Metrics import Ganeti.HTools.ExtLoader import qualified Ganeti.HTools.Instance as Instance import Ganeti.HTools.Loader import qualified Ganeti.HTools.Node as Node import Ganeti.HTools.Tags (hasStandbyTag) import Ganeti.HTools.Tags.Constants (standbyAuto) import Ganeti.HTools.Types import Ganeti.JQueue (currentTimestamp, reasonTrailTimestamp) import Ganeti.JQueue.Objects (Timestamp) import qualified Ganeti.Jobs as Jobs import Ganeti.OpCodes import Ganeti.OpCodes.Lens (metaParamsL, opReasonL) import Ganeti.Utils import Ganeti.Version (version) -- | Options list and functions. options :: IO [OptType] options = do luxi <- oLuxiSocket return [ luxi , oDataFile , oExecJobs , oMinResources , oTargetResources , oSaveCluster , oPrintCommands , oVerbose , oNoHeaders , oStaticKvmNodeMemory ] -- | The list of arguments supported by the program. arguments :: [ArgCompletion] arguments = [] -- | Wraps an 'OpCode' in a 'MetaOpCode' while also adding a comment -- about what generated the opcode. annotateOpCode :: Timestamp -> String -> Jobs.Annotator annotateOpCode ts comment = over (metaParamsL . opReasonL) (++ [("hsqueeze" , "hsqueeze " ++ version ++ " called" , reasonTrailTimestamp ts )]) . setOpComment (comment ++ " " ++ version) . wrapOpCode -- | Within a cluster configuration, decide if the node hosts only -- externally-mirrored instances. onlyExternal :: (Node.List, Instance.List) -> Node.Node -> Bool onlyExternal (_, il) nd = not . any (Instance.usesLocalStorage . flip Container.find il) $ Node.pList nd -- | Predicate of not being secondary node for any instance noSecondaries :: Node.Node -> Bool noSecondaries = null . Node.sList -- | Predicate whether, in a configuration, all running instances are on -- online nodes. allInstancesOnOnlineNodes :: (Node.List, Instance.List) -> Bool allInstancesOnOnlineNodes (nl, il) = all (not . Node.offline . flip Container.find nl . Instance.pNode) . IntMap.elems $ il -- | Predicate whether, in a configuration, each node has enough resources -- to additionally host the given instance. allNodesCapacityFor :: Instance.Instance -> (Node.List, Instance.List) -> Bool allNodesCapacityFor inst (nl, _) = all (isOk . flip Node.addPri inst) . IntMap.elems $ nl -- | Balance a configuration, possible for 0 steps, till no further improvement -- is possible. balance :: (Node.List, Instance.List) -> ((Node.List, Instance.List), [MoveJob]) balance (nl, il) = let ini_cv = Metrics.compCV nl ini_tbl = Cluster.Table nl il ini_cv [] balanceStep = Cluster.tryBalance (Alg.defaultOptions { Alg.algMinGain = 0.0 , Alg.algMinGainLimit = 0.0}) bTables = map fromJust . takeWhile isJust $ iterate (>>= balanceStep) (Just ini_tbl) (Cluster.Table nl' il' _ _) = last bTables moves = zip bTables (drop 1 bTables) >>= Cluster.getMoves in ((nl', il'), reverse moves) -- | In a configuration, mark a node as online or offline. onlineOfflineNode :: Bool -> (Node.List, Instance.List) -> Ndx -> (Node.List, Instance.List) onlineOfflineNode offline (nl, il) ndx = let nd = Container.find ndx nl nd' = Node.setOffline nd offline nl' = Container.add ndx nd' nl in (nl', il) -- | Offline or online a list nodes, and return the state after a balancing -- attempt together with the sequence of moves that lead there. onlineOfflineNodes :: Bool -> [Ndx] -> (Node.List, Instance.List) -> ((Node.List, Instance.List), [MoveJob]) onlineOfflineNodes offline ndxs conf = let conf' = foldl (onlineOfflineNode offline) conf ndxs in balance conf' -- | Offline a list of nodes, and return the state after balancing with -- the sequence of moves that lead there. offlineNodes :: [Ndx] -> (Node.List, Instance.List) -> ((Node.List, Instance.List), [MoveJob]) offlineNodes = onlineOfflineNodes True -- | Online a list of nodes, and return the state after balancing with -- the sequence of moves that lead there. onlineNodes :: [Ndx] -> (Node.List, Instance.List) -> ((Node.List, Instance.List), [MoveJob]) onlineNodes = onlineOfflineNodes False -- | Predicate on whether a list of nodes can be offlined or onlined -- simultaneously in a given configuration, while still leaving enough -- capacity on every node for the given instance. canOnlineOffline :: Bool -> Instance.Instance -> (Node.List, Instance.List) -> [Node.Node] ->Bool canOnlineOffline offline inst conf nds = let conf' = fst $ onlineOfflineNodes offline (map Node.idx nds) conf in allInstancesOnOnlineNodes conf' && allNodesCapacityFor inst conf' -- | Predicate on whether a list of nodes can be offlined simultaneously. canOffline :: Instance.Instance -> (Node.List, Instance.List) -> [Node.Node] -> Bool canOffline = canOnlineOffline True -- | Predicate on whether onlining a list of nodes suffices to get enough -- free resources for given instance. sufficesOnline :: Instance.Instance -> (Node.List, Instance.List) -> [Node.Node] -> Bool sufficesOnline = canOnlineOffline False -- | Greedily offline the nodes, starting from the last element, and return -- the list of nodes that could simultaneously be offlined, while keeping -- the resources specified by an instance. greedyOfflineNodes :: Instance.Instance -> (Node.List, Instance.List) -> [Node.Node] -> [Node.Node] greedyOfflineNodes _ _ [] = [] greedyOfflineNodes inst conf (nd:nds) = let nds' = greedyOfflineNodes inst conf nds in if canOffline inst conf (nd:nds') then nd:nds' else nds' -- | Try to provide enough resources by onlining an initial segment of -- a list of nodes. Return Nothing, if even onlining all of them is not -- enough. tryOnline :: Instance.Instance -> (Node.List, Instance.List) -> [Node.Node] -> Maybe [Node.Node] tryOnline inst conf = listToMaybe . filter (sufficesOnline inst conf) . inits -- | From a specification, name, and factor create an instance that uses that -- factor times the specification, rounded down. instanceFromSpecAndFactor :: String -> Double -> ISpec -> Instance.Instance instanceFromSpecAndFactor name f spec = Instance.create name (floor (f * fromIntegral (iSpecMemorySize spec))) 0 [] (floor (f * fromIntegral (iSpecCpuCount spec))) Running [] False Node.noSecondary Node.noSecondary DTExt (floor (f * fromIntegral (iSpecSpindleUse spec))) [] False -- | Get opcodes for the given move job. getMoveOpCodes :: Node.List -> Instance.List -> [JobSet] -> Result [([[OpCode]], String)] getMoveOpCodes nl il js = return $ zip (map opcodes js) (map descr js) where opcodes = map (\(_, idx, move, _) -> Cluster.iMoveToJob nl il idx move) descr job = "Moving instances " ++ commaJoin (map (\(_, idx, _, _) -> Container.nameOf il idx) job) -- | Get opcodes for tagging nodes with standby. getTagOpCodes :: [Node.Node] -> Result [([[OpCode]], String)] getTagOpCodes nl = return $ zip (map opCode nl) (map descr nl) where opCode node = [[Node.genAddTagsOpCode node [standbyAuto]]] descr node = "Tagging node " ++ Node.name node ++ " with standby" -- | Get opcodes for powering off nodes getPowerOffOpCodes :: [Node.Node] -> Result [([[OpCode]], String)] getPowerOffOpCodes nl = do opcodes <- Node.genPowerOffOpCodes nl return [([opcodes], "Powering off nodes")] -- | Get opcodes for powering on nodes getPowerOnOpCodes :: [Node.Node] -> Result [([[OpCode]], String)] getPowerOnOpCodes nl = do opcodes <- Node.genPowerOnOpCodes nl return [([opcodes], "Powering on nodes")] maybeExecJobs :: Options -> String -> Result [([[OpCode]], String)] -> IO (Result ()) maybeExecJobs opts comment opcodes = if optExecJobs opts then (case optLuxi opts of Nothing -> return $ Bad "Execution of commands possible only on LUXI" Just master -> do ts <- currentTimestamp let annotator = maybe id setOpPriority (optPriority opts) . annotateOpCode ts comment case opcodes of Bad msg -> error msg Ok codes -> Jobs.execWithCancel annotator master codes) else return $ Ok () -- | Main function. main :: Options -> [String] -> IO () main opts args = do unless (null args) $ exitErr "This program doesn't take any arguments." let verbose = optVerbose opts targetf = optTargetResources opts minf = optMinResources opts ini_cdata@(ClusterData _ nlf ilf _ ipol) <- loadExternalData opts maybeSaveData (optSaveCluster opts) "original" "before hsqueeze run" ini_cdata let nodelist = IntMap.elems nlf offlineCandidates = sortBy (flip compare `on` length . Node.pList) . filter (foldl (liftA2 (&&)) (const True) [ not . Node.offline , not . Node.isMaster , noSecondaries , onlyExternal (nlf, ilf) ]) $ nodelist onlineCandidates = filter (liftA2 (&&) Node.offline hasStandbyTag) nodelist conf = (nlf, ilf) std = iPolicyStdSpec ipol targetInstance = instanceFromSpecAndFactor "targetInstance" targetf std minInstance = instanceFromSpecAndFactor "targetInstance" minf std toOffline = greedyOfflineNodes targetInstance conf offlineCandidates ((fin_off_nl, fin_off_il), off_mvs) = offlineNodes (map Node.idx toOffline) conf final_off_cdata = ini_cdata { cdNodes = fin_off_nl, cdInstances = fin_off_il } off_jobs = Cluster.splitJobs off_mvs off_opcodes = liftM concat $ sequence [ getMoveOpCodes nlf ilf off_jobs , getTagOpCodes toOffline , getPowerOffOpCodes toOffline ] off_cmd = Cluster.formatCmds off_jobs ++ "\necho Tagging Commands\n" ++ (toOffline >>= (printf " gnt-node add-tags %s %s\n" `flip` standbyAuto) . Node.alias) ++ "\necho Power Commands\n" ++ (toOffline >>= printf " gnt-node power -f off %s\n" . Node.alias) toOnline = tryOnline minInstance conf onlineCandidates nodesToOnline = fromMaybe onlineCandidates toOnline ((fin_on_nl, fin_on_il), on_mvs) = onlineNodes (map Node.idx nodesToOnline) conf final_on_cdata = ini_cdata { cdNodes = fin_on_nl, cdInstances = fin_on_il } on_jobs = Cluster.splitJobs on_mvs on_opcodes = liftM2 (++) (getPowerOnOpCodes nodesToOnline) (getMoveOpCodes nlf ilf on_jobs) on_cmd = "echo Power Commands\n" ++ (nodesToOnline >>= printf " gnt-node power -f on %s\n" . Node.alias) ++ Cluster.formatCmds on_jobs when (verbose > 1) . putStrLn $ "Offline candidates: " ++ commaJoin (map Node.name offlineCandidates) when (verbose > 1) . putStrLn $ "Online candidates: " ++ commaJoin (map Node.name onlineCandidates) if not (allNodesCapacityFor minInstance conf) then do unless (optNoHeaders opts) $ putStrLn "'Nodes to online'" mapM_ (putStrLn . Node.name) nodesToOnline when (verbose > 1 && isNothing toOnline) . putStrLn $ "Onlining all nodes will not yield enough capacity" maybeSaveCommands "Commands to run:" opts on_cmd let comment = printf "expanding by %d nodes" (length nodesToOnline) exitIfBad "hsqueeze" =<< maybeExecJobs opts comment on_opcodes maybeSaveData (optSaveCluster opts) "squeezed" "after hsqueeze expansion" final_on_cdata else if null toOffline then do unless (optNoHeaders opts) $ putStrLn "'No action'" maybeSaveCommands "Commands to run:" opts "echo Nothing to do" maybeSaveData (optSaveCluster opts) "squeezed" "after hsqueeze doing nothing" ini_cdata else do unless (optNoHeaders opts) $ putStrLn "'Nodes to offline'" mapM_ (putStrLn . Node.name) toOffline maybeSaveCommands "Commands to run:" opts off_cmd let comment = printf "condensing by %d nodes" (length toOffline) exitIfBad "hsqueeze" =<< maybeExecJobs opts comment off_opcodes maybeSaveData (optSaveCluster opts) "squeezed" "after hsqueeze run" final_off_cdata ganeti-3.1.0~rc2/src/Ganeti/HTools/Program/Main.hs000064400000000000000000000123321476477700300216420ustar00rootroot00000000000000{-| Small module holding program definitions. -} {- Copyright (C) 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Program.Main ( personalities , main ) where import Control.Exception import Control.Monad (guard) import Data.Char (toLower) import System.Environment import System.IO import System.IO.Error (isDoesNotExistError) import Ganeti.Common (formatCommands, PersonalityList) import Ganeti.HTools.CLI (Options, parseOpts, genericOpts) import qualified Ganeti.HTools.Program.Hail as Hail import qualified Ganeti.HTools.Program.Harep as Harep import qualified Ganeti.HTools.Program.Hbal as Hbal import qualified Ganeti.HTools.Program.Hcheck as Hcheck import qualified Ganeti.HTools.Program.Hscan as Hscan import qualified Ganeti.HTools.Program.Hspace as Hspace import qualified Ganeti.HTools.Program.Hsqueeze as Hsqueeze import qualified Ganeti.HTools.Program.Hinfo as Hinfo import qualified Ganeti.HTools.Program.Hroller as Hroller import Ganeti.Utils -- | Supported binaries. personalities :: PersonalityList Options personalities = [ ("hail", (Hail.main, Hail.options, Hail.arguments, "Ganeti IAllocator plugin that implements the instance\ \ placement and movement using the same algorithm as\ \ hbal(1)")) , ("harep", (Harep.main, Harep.options, Harep.arguments, "auto-repair tool that detects certain kind of problems\ \ with instances and applies the allowed set of solutions")) , ("hbal", (Hbal.main, Hbal.options, Hbal.arguments, "cluster balancer that looks at the current state of\ \ the cluster and computes a series of steps designed\ \ to bring the cluster into a better state")) , ("hcheck", (Hcheck.main, Hcheck.options, Hcheck.arguments, "cluster checker; prints information about cluster's\ \ health and checks whether a rebalance done using\ \ hbal would help")) , ("hscan", (Hscan.main, Hscan.options, Hscan.arguments, "tool for scanning clusters via RAPI and saving their\ \ data in the input format used by hbal(1) and hspace(1)")) , ("hspace", (Hspace.main, Hspace.options, Hspace.arguments, "computes how many additional instances can be fit on a\ \ cluster, while maintaining N+1 status.")) , ("hinfo", (Hinfo.main, Hinfo.options, Hinfo.arguments, "cluster information printer; it prints information\ \ about the current cluster state and its residing\ \ nodes/instances")) , ("hroller", (Hroller.main, Hroller.options, Hroller.arguments, "cluster rolling maintenance helper; it helps scheduling\ \ node reboots in a manner that doesn't conflict with the\ \ instances' topology")) , ("hsqueeze", (Hsqueeze.main, Hsqueeze.options, Hsqueeze.arguments, "cluster dynamic power management; it powers up and down\ \ nodes to keep the amount of free online resources in a\ \ given range")) ] -- | Display usage and exit. usage :: String -> IO () usage name = do hPutStrLn stderr $ "Unrecognised personality '" ++ name ++ "'." hPutStrLn stderr "This program must be installed under one of the following\ \ names:" hPutStrLn stderr . unlines $ formatCommands personalities exitErr "Please either rename/symlink the program or set\n\ \the environment variable HTOOLS to the desired role." main :: IO () main = do binary <- catchJust (guard . isDoesNotExistError) (getEnv "HTOOLS") (const getProgName) let name = map toLower binary case name `lookup` personalities of Nothing -> usage name Just (fn, options, arguments, _) -> do cmd_args <- getArgs real_options <- options (opts, args) <- parseOpts cmd_args name (real_options ++ genericOpts) arguments fn opts args ganeti-3.1.0~rc2/src/Ganeti/HTools/Tags.hs000064400000000000000000000073501476477700300202510ustar00rootroot00000000000000{-| Tags This module holds all the tag interpretation done by htools. -} {- Copyright (C) 2014, 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Tags ( hasStandbyTag , getMigRestrictions , getRecvMigRestrictions , getLocations ) where import Control.Monad (guard, (>=>)) import Data.List (isPrefixOf, isInfixOf, stripPrefix) import Data.Maybe (mapMaybe) import qualified Data.Set as S import qualified Ganeti.HTools.Node as Node import Ganeti.HTools.Tags.Constants ( standbyPrefix , migrationPrefix, allowMigrationPrefix , locationPrefix ) -- * Predicates -- | Predicate of having a standby tag. hasStandbyTag :: Node.Node -> Bool hasStandbyTag = any (standbyPrefix `isPrefixOf`) . Node.nTags -- * Utility functions -- | Htools standard tag extraction. Given a set of cluster tags, -- take those starting with a specific prefix, strip the prefix -- and append a colon, and then take those node tags starting with -- one of those strings. getTags :: String -> [String] -> [String] -> S.Set String getTags prefix ctags ntags = S.fromList (mapMaybe (stripPrefix prefix) ctags >>= \ p -> filter ((p ++ ":") `isPrefixOf`) ntags) -- * Migration restriction tags -- | Given the cluster tags extract the migration restrictions -- from a node tag. getMigRestrictions :: [String] -> [String] -> S.Set String getMigRestrictions = getTags migrationPrefix -- | Maybe split a string on the first single occurence of "::" return -- the parts before and after. splitAtColons :: String -> Maybe (String, String) splitAtColons (':':':':xs) = do guard $ not ("::" `isInfixOf` xs) return ("", xs) splitAtColons (x:xs) = do (as, bs) <- splitAtColons xs return (x:as, bs) splitAtColons _ = Nothing -- | Get the pairs of allowed migrations from a set of cluster tags. migrations :: [String] -> [(String, String)] migrations = mapMaybe $ stripPrefix allowMigrationPrefix >=> splitAtColons -- | Given the cluster tags, extract the set of migration restrictions -- a node is able to receive from its node tags. getRecvMigRestrictions :: [String] -> [String] -> S.Set String getRecvMigRestrictions ctags ntags = let migs = migrations ctags closure tag = (:) tag . map fst $ filter ((==) tag . snd) migs in S.fromList $ S.elems (getMigRestrictions ctags ntags) >>= closure -- * Location tags -- | Given the cluster tags, extract the node location tags -- from the node tags. getLocations :: [String] -> [String] -> S.Set String getLocations = getTags locationPrefix ganeti-3.1.0~rc2/src/Ganeti/HTools/Tags/000075500000000000000000000000001476477700300177105ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/HTools/Tags/Constants.hs000064400000000000000000000062561476477700300222310ustar00rootroot00000000000000{-| Tag constants This module holds all the special tag prefixes honored by Ganeti's htools. The module itself does not depend on anything Ganeti specific so that it can be imported anywhere. -} {- Copyright (C) 2014, 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Tags.Constants ( exTagsPrefix , standbyPrefix , migrationPrefix , allowMigrationPrefix , locationPrefix , desiredLocationPrefix , standbyAuto , autoRepairTagPrefix , autoRepairTagEnabled , autoRepairTagPending , autoRepairTagResult , autoRepairTagSuspended ) where -- | The exclusion tag prefix. Instance tags starting with this prefix -- describe a service provided by the instance. Instances providing the -- same service at not places on the same node. exTagsPrefix :: String exTagsPrefix = "htools:iextags:" -- | The tag-prefix indicating that hsqueeze should consider a node -- as being standby. standbyPrefix :: String standbyPrefix = "htools:standby:" -- | The prefix for migration tags migrationPrefix :: String migrationPrefix = "htools:migration:" -- | Prefix of tags allowing migration allowMigrationPrefix :: String allowMigrationPrefix = "htools:allowmigration:" -- | The prefix for node location tags. locationPrefix :: String locationPrefix = "htools:nlocation:" -- | The prefix for instance desired location tags. desiredLocationPrefix :: String desiredLocationPrefix = "htools:desiredlocation:" -- | The tag to be added to nodes that were shutdown by hsqueeze. standbyAuto :: String standbyAuto = "htools:standby:auto" -- | Auto-repair tag prefix autoRepairTagPrefix :: String autoRepairTagPrefix = "ganeti:watcher:autorepair:" autoRepairTagEnabled :: String autoRepairTagEnabled = autoRepairTagPrefix autoRepairTagPending :: String autoRepairTagPending = autoRepairTagPrefix ++ "pending:" autoRepairTagResult :: String autoRepairTagResult = autoRepairTagPrefix ++ "result:" autoRepairTagSuspended :: String autoRepairTagSuspended = autoRepairTagPrefix ++ "suspend:" ganeti-3.1.0~rc2/src/Ganeti/HTools/Types.hs000064400000000000000000000361361476477700300204630ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Some common types. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.HTools.Types ( Idx , Ndx , Gdx , NameAssoc , Score , Weight , GroupID , defaultGroupID , AllocPolicy(..) , allocPolicyFromRaw , allocPolicyToRaw , NetworkID , InstanceStatus(..) , instanceStatusFromRaw , instanceStatusToRaw , RSpec(..) , AllocInfo(..) , AllocStats , DynUtil(..) , zeroUtil , baseUtil , addUtil , subUtil , defReservedDiskRatio , unitMem , unitCpu , unitDsk , unitSpindle , unknownField , Placement , IMove(..) , DiskTemplate(..) , diskTemplateToRaw , diskTemplateFromRaw , MirrorType(..) , templateMirrorType , MoveJob , JobSet , Element(..) , FailMode(..) , FailStats , OpResult , opToResult , ISpec(..) , defMinISpec , defStdISpec , maxDisks , maxNics , defMaxISpec , MinMaxISpecs(..) , IPolicy(..) , defIPolicy , rspecFromISpec , AutoRepairType(..) , autoRepairTypeToRaw , autoRepairTypeFromRaw , AutoRepairResult(..) , autoRepairResultToRaw , autoRepairResultFromRaw , AutoRepairPolicy(..) , AutoRepairSuspendTime(..) , AutoRepairData(..) , AutoRepairStatus(..) ) where import qualified Data.Map as M import System.Time (ClockTime) import qualified Ganeti.ConstantUtils as ConstantUtils import qualified Ganeti.THH as THH import Ganeti.BasicTypes import Ganeti.Types -- | The instance index type. type Idx = Int -- | The node index type. type Ndx = Int -- | The group index type. type Gdx = Int -- | The type used to hold name-to-idx mappings. type NameAssoc = M.Map String Int -- | A separate name for the cluster score type. type Score = Double -- | A separate name for a weight metric. type Weight = Double -- | The Group UUID type. type GroupID = String -- | Default group UUID (just a string, not a real UUID). defaultGroupID :: GroupID defaultGroupID = "00000000-0000-0000-0000-000000000000" -- | Mirroring type. data MirrorType = MirrorNone -- ^ No mirroring/movability | MirrorInternal -- ^ DRBD-type mirroring | MirrorExternal -- ^ Shared-storage type mirroring deriving (Eq, Show) -- | Correspondence between disk template and mirror type. templateMirrorType :: DiskTemplate -> MirrorType templateMirrorType DTDiskless = MirrorExternal templateMirrorType DTFile = MirrorNone templateMirrorType DTSharedFile = MirrorExternal templateMirrorType DTPlain = MirrorNone templateMirrorType DTBlock = MirrorExternal templateMirrorType DTDrbd8 = MirrorInternal templateMirrorType DTRbd = MirrorExternal templateMirrorType DTExt = MirrorExternal templateMirrorType DTGluster = MirrorExternal -- | The resource spec type. data RSpec = RSpec { rspecCpu :: Int -- ^ Requested VCPUs , rspecMem :: Int -- ^ Requested memory , rspecDsk :: Int -- ^ Requested disk , rspecSpn :: Int -- ^ Requested spindles } deriving (Show, Eq) -- | Allocation stats type. This is used instead of 'RSpec' (which was -- used at first), because we need to track more stats. The actual -- data can refer either to allocated, or available, etc. values -- depending on the context. See also -- 'Cluster.computeAllocationDelta'. data AllocInfo = AllocInfo { allocInfoVCpus :: Int -- ^ VCPUs , allocInfoNCpus :: Double -- ^ Normalised CPUs , allocInfoMem :: Int -- ^ Memory , allocInfoDisk :: Int -- ^ Disk , allocInfoSpn :: Int -- ^ Spindles } deriving (Show, Eq) -- | Currently used, possibly to allocate, unallocable. type AllocStats = (AllocInfo, AllocInfo, AllocInfo) -- | The network UUID type. type NetworkID = String -- | Instance specification type. $(THH.buildObject "ISpec" "iSpec" [ THH.renameField "MemorySize" $ THH.simpleField ConstantUtils.ispecMemSize [t| Int |] , THH.renameField "CpuCount" $ THH.simpleField ConstantUtils.ispecCpuCount [t| Int |] , THH.renameField "DiskSize" $ THH.simpleField ConstantUtils.ispecDiskSize [t| Int |] , THH.renameField "DiskCount" $ THH.simpleField ConstantUtils.ispecDiskCount [t| Int |] , THH.renameField "NicCount" $ THH.simpleField ConstantUtils.ispecNicCount [t| Int |] , THH.renameField "SpindleUse" $ THH.simpleField ConstantUtils.ispecSpindleUse [t| Int |] ]) -- | The default minimum ispec. defMinISpec :: ISpec defMinISpec = ISpec { iSpecMemorySize = 128 , iSpecCpuCount = 1 , iSpecDiskCount = 1 , iSpecDiskSize = 1024 , iSpecNicCount = 1 , iSpecSpindleUse = 1 } -- | The default standard ispec. defStdISpec :: ISpec defStdISpec = ISpec { iSpecMemorySize = 128 , iSpecCpuCount = 1 , iSpecDiskCount = 1 , iSpecDiskSize = 1024 , iSpecNicCount = 1 , iSpecSpindleUse = 1 } maxDisks :: Int maxDisks = 16 maxNics :: Int maxNics = 8 -- | The default max ispec. defMaxISpec :: ISpec defMaxISpec = ISpec { iSpecMemorySize = 32768 , iSpecCpuCount = 8 , iSpecDiskCount = maxDisks , iSpecDiskSize = 1024 * 1024 , iSpecNicCount = maxNics , iSpecSpindleUse = 12 } -- | Minimum and maximum instance specs type. $(THH.buildObject "MinMaxISpecs" "minMaxISpecs" [ THH.renameField "MinSpec" $ THH.simpleField "min" [t| ISpec |] , THH.renameField "MaxSpec" $ THH.simpleField "max" [t| ISpec |] ]) -- | Defult minimum and maximum instance specs. defMinMaxISpecs :: [MinMaxISpecs] defMinMaxISpecs = [MinMaxISpecs { minMaxISpecsMinSpec = defMinISpec , minMaxISpecsMaxSpec = defMaxISpec }] -- | Instance policy type. $(THH.buildObject "IPolicy" "iPolicy" [ THH.renameField "MinMaxISpecs" $ THH.simpleField ConstantUtils.ispecsMinmax [t| [MinMaxISpecs] |] , THH.renameField "StdSpec" $ THH.simpleField ConstantUtils.ispecsStd [t| ISpec |] , THH.renameField "DiskTemplates" $ THH.simpleField ConstantUtils.ipolicyDts [t| [DiskTemplate] |] , THH.renameField "VcpuRatio" $ THH.simpleField ConstantUtils.ipolicyVcpuRatio [t| Double |] , THH.renameField "SpindleRatio" $ THH.simpleField ConstantUtils.ipolicySpindleRatio [t| Double |] ]) -- | Converts an ISpec type to a RSpec one. rspecFromISpec :: ISpec -> RSpec rspecFromISpec ispec = RSpec { rspecCpu = iSpecCpuCount ispec , rspecMem = iSpecMemorySize ispec , rspecDsk = iSpecDiskSize ispec , rspecSpn = iSpecSpindleUse ispec } -- | The default instance policy. defIPolicy :: IPolicy defIPolicy = IPolicy { iPolicyMinMaxISpecs = defMinMaxISpecs , iPolicyStdSpec = defStdISpec -- hardcoding here since Constants.hs exports the -- string values, not the actual type; and in -- htools, we are mostly looking at DRBD , iPolicyDiskTemplates = [minBound..maxBound] , iPolicyVcpuRatio = ConstantUtils.ipolicyDefaultsVcpuRatio , iPolicySpindleRatio = ConstantUtils.ipolicyDefaultsSpindleRatio } -- | The dynamic resource specs of a machine (i.e. load or load -- capacity, as opposed to size). data DynUtil = DynUtil { cpuWeight :: Weight -- ^ Standardised CPU usage , memWeight :: Weight -- ^ Standardised memory load , dskWeight :: Weight -- ^ Standardised disk I\/O usage , netWeight :: Weight -- ^ Standardised network usage } deriving (Show, Eq) -- | Initial empty utilisation. zeroUtil :: DynUtil zeroUtil = DynUtil { cpuWeight = 0, memWeight = 0 , dskWeight = 0, netWeight = 0 } -- | Base utilisation (used when no actual utilisation data is -- supplied). baseUtil :: DynUtil baseUtil = DynUtil { cpuWeight = 1, memWeight = 1 , dskWeight = 1, netWeight = 1 } -- | Sum two utilisation records. addUtil :: DynUtil -> DynUtil -> DynUtil addUtil (DynUtil a1 a2 a3 a4) (DynUtil b1 b2 b3 b4) = DynUtil (a1+b1) (a2+b2) (a3+b3) (a4+b4) -- | Substracts one utilisation record from another. subUtil :: DynUtil -> DynUtil -> DynUtil subUtil (DynUtil a1 a2 a3 a4) (DynUtil b1 b2 b3 b4) = DynUtil (a1-b1) (a2-b2) (a3-b3) (a4-b4) -- | The description of an instance placement. It contains the -- instance index, the new primary and secondary node, the move being -- performed and the score of the cluster after the move. type Placement = (Idx, Ndx, Ndx, IMove, Score) -- | An instance move definition. data IMove = Failover -- ^ Failover the instance (f) | FailoverToAny Ndx -- ^ Failover to a random node -- (fa:np), for shared storage | ReplacePrimary Ndx -- ^ Replace primary (f, r:np, f) | ReplaceSecondary Ndx -- ^ Replace secondary (r:ns) | ReplaceAndFailover Ndx -- ^ Replace secondary, failover (r:np, f) | FailoverAndReplace Ndx -- ^ Failover, replace secondary (f, r:ns) deriving (Show) -- | Formatted solution output for one move (involved nodes and -- commands. type MoveJob = ([Ndx], Idx, IMove, [String]) -- | Unknown field in table output. unknownField :: String unknownField = "" -- | A list of command elements. type JobSet = [MoveJob] -- | Default max disk usage ratio. defReservedDiskRatio :: Double defReservedDiskRatio = 0 -- | Base memory unit. unitMem :: Int unitMem = 64 -- | Base disk unit. unitDsk :: Int unitDsk = 256 -- | Base vcpus unit. unitCpu :: Int unitCpu = 1 -- | Base spindles unit. unitSpindle :: Int unitSpindle = 1 -- | Reason for an operation's falure. data FailMode = FailMem -- ^ Failed due to not enough RAM | FailDisk -- ^ Failed due to not enough disk | FailCPU -- ^ Failed due to not enough CPU capacity | FailN1 -- ^ Failed due to not passing N1 checks | FailTags -- ^ Failed due to tag exclusion | FailMig -- ^ Failed due to migration restrictions | FailDiskCount -- ^ Failed due to wrong number of disks | FailSpindles -- ^ Failed due to wrong/missing spindles | FailInternal -- ^ Internal error deriving (Eq, Enum, Bounded, Show) -- | List with failure statistics. type FailStats = [(FailMode, Int)] -- | Either-like data-type customized for our failure modes. -- -- The failure values for this monad track the specific allocation -- failures, so this is not a general error-monad (compare with the -- 'Result' data type). One downside is that this type cannot encode a -- generic failure mode, hence our way to build a FailMode from string -- will instead raise an exception. type OpResult = GenericResult FailMode -- | 'Error' instance for 'FailMode' designed to catch unintended -- use as a general monad. instance Error FailMode where strMsg v = error $ "Programming error: OpResult used as generic monad" ++ v -- | Conversion from 'OpResult' to 'Result'. opToResult :: OpResult a -> Result a opToResult (Bad f) = Bad $ show f opToResult (Ok v) = Ok v -- | A generic class for items that have updateable names and indices. class Element a where -- | Returns the name of the element nameOf :: a -> String -- | Returns all the known names of the element allNames :: a -> [String] -- | Returns the index of the element idxOf :: a -> Int -- | Updates the alias of the element setAlias :: a -> String -> a -- | Compute the alias by stripping a given suffix (domain) from -- the name computeAlias :: String -> a -> a computeAlias dom e = setAlias e alias where alias = take (length name - length dom) name name = nameOf e -- | Updates the index of the element setIdx :: a -> Int -> a -- | The repair modes for the auto-repair tool. $(THH.declareLADT ''String "AutoRepairType" -- Order is important here: from least destructive to most. [ ("ArFixStorage", "fix-storage") , ("ArMigrate", "migrate") , ("ArFailover", "failover") , ("ArReinstall", "reinstall") ]) -- | The possible auto-repair results. $(THH.declareLADT ''String "AutoRepairResult" -- Order is important here: higher results take precedence when an object -- has several result annotations attached. [ ("ArEnoperm", "enoperm") , ("ArSuccess", "success") , ("ArFailure", "failure") ]) -- | The possible auto-repair policy for a given instance. data AutoRepairPolicy = ArEnabled AutoRepairType -- ^ Auto-repair explicitly enabled | ArSuspended AutoRepairSuspendTime -- ^ Suspended temporarily, or forever | ArNotEnabled -- ^ Auto-repair not explicitly enabled deriving (Eq, Show) -- | The suspend timeout for 'ArSuspended'. data AutoRepairSuspendTime = Forever -- ^ Permanently suspended | Until ClockTime -- ^ Suspended up to a certain time deriving (Eq, Show) -- | The possible auto-repair states for any given instance. data AutoRepairStatus = ArHealthy (Maybe AutoRepairData) -- ^ No problems detected with the instance | ArNeedsRepair AutoRepairData -- ^ Instance has problems, no action taken | ArPendingRepair AutoRepairData -- ^ Repair jobs ongoing for the instance | ArFailedRepair AutoRepairData -- ^ Some repair jobs for the instance failed deriving (Eq, Show) -- | The data accompanying a repair operation (future, pending, or failed). data AutoRepairData = AutoRepairData { arType :: AutoRepairType , arUuid :: String , arTime :: ClockTime , arJobs :: [JobId] , arResult :: Maybe AutoRepairResult , arTag :: String } deriving (Eq, Show) ganeti-3.1.0~rc2/src/Ganeti/Hash.hs000064400000000000000000000041501476477700300170210ustar00rootroot00000000000000{-| Crypto-related helper functions. -} {- Copyright (C) 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Hash ( computeMac , verifyMac , HashKey ) where import qualified Data.ByteString as B import qualified Data.ByteString.UTF8 as BU import Crypto.Hash.Algorithms import Crypto.MAC.HMAC import Data.Char import Data.Word -- | Type alias for the hash key. This depends on the library being -- used. type HashKey = [Word8] -- | Computes the HMAC for a given key/test and salt. computeMac :: HashKey -> Maybe String -> String -> String computeMac key salt text = let hashable = maybe text (++ text) salt in show . hmacGetDigest $ (hmac (B.pack key) (BU.fromString hashable) :: HMAC SHA1) -- | Verifies the HMAC for a given message. verifyMac :: HashKey -> Maybe String -> String -> String -> Bool verifyMac key salt text digest = map toLower digest == computeMac key salt text ganeti-3.1.0~rc2/src/Ganeti/Hs2Py/000075500000000000000000000000001476477700300165475ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Hs2Py/GenConstants.hs000064400000000000000000000044421476477700300215150ustar00rootroot00000000000000{-| Template Haskell code for Haskell to Python constants. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} {-# LANGUAGE TemplateHaskell #-} module Ganeti.Hs2Py.GenConstants (genPyConstants) where import Language.Haskell.TH import Ganeti.THH fileFromModule :: Maybe String -> String fileFromModule Nothing = "" fileFromModule (Just name) = "src/" ++ map dotToSlash name ++ ".hs" where dotToSlash '.' = '/' dotToSlash c = c comment :: Name -> String comment name = "# Generated automatically from Haskell constant '" ++ nameBase name ++ "' in file '" ++ fileFromModule (nameModule name) ++ "'" genList :: Name -> [Name] -> Q [Dec] genList name consNames = do let cons = listE $ map (\n -> tupE [mkString n, mkPyValueEx n]) consNames sig <- sigD name [t| [(String, String)] |] fun <- funD name [clause [] (normalB cons) []] return [sig, fun] where mkString n = stringE (comment n ++ "\n" ++ deCamelCase (nameBase n)) mkPyValueEx n = [| showValue $(varE n) |] genPyConstants :: String -> [Name] -> Q [Dec] genPyConstants name = genList (mkName name) ganeti-3.1.0~rc2/src/Ganeti/Hs2Py/GenOpCodes.hs000064400000000000000000000065031476477700300210750ustar00rootroot00000000000000{-| GenOpCodes handles Python opcode generation. GenOpCodes contains the helper functions that generate the Python opcodes as strings from the Haskell opcode description. -} {- Copyright (C) 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Hs2Py.GenOpCodes (showPyClasses) where import Data.List (intercalate) import Ganeti.OpCodes import Ganeti.THH -- | Generates the Python class docstring. pyClassDoc :: String -> String pyClassDoc doc | length (lines doc) > 1 = " \"\"\"" ++ doc ++ "\n\n" ++ " \"\"\"" ++ "\n" | otherwise = " \"\"\"" ++ doc ++ "\"\"\"" ++ "\n" -- | Generates an opcode parameter in Python. pyClassField :: OpCodeField -> String pyClassField (OpCodeField name typ Nothing doc) = "(" ++ intercalate ", " [show name, "None", showValue typ, show doc] ++ ")" pyClassField (OpCodeField name typ (Just def) doc) = "(" ++ intercalate ", " [show name, showValue def, showValue typ, show doc] ++ ")" -- | Comma intercalates and indents opcode parameters in Python. intercalateIndent :: [String] -> String intercalateIndent xs = intercalate "," (map ("\n " ++) xs) -- | Generates an opcode as a Python class. showPyClass :: OpCodeDescriptor -> String showPyClass (OpCodeDescriptor name typ doc fields dsc) = let baseclass | name == "OpInstanceMultiAlloc" = "OpInstanceMultiAllocBase" | otherwise = "OpCode" opDscField | null dsc = "" | otherwise = " OP_DSC_FIELD = " ++ show dsc ++ "\n" opDscFormatter | name == "OpTestDelay" = " def OP_DSC_FORMATTER(self, value):\n" ++ " return FormatDuration(value)\n" | otherwise = "" withLU | name == "OpTestDummy" = "\n WITH_LU = False" | otherwise = "" in "class " ++ name ++ "(" ++ baseclass ++ "):" ++ "\n" ++ pyClassDoc doc ++ opDscField ++ " OP_PARAMS = [" ++ intercalateIndent (map pyClassField fields) ++ "\n ]" ++ "\n" ++ opDscFormatter ++ " OP_RESULT = " ++ showValue typ ++ withLU ++ "\n\n" -- | Generates all opcodes as Python classes. showPyClasses :: String showPyClasses = concatMap showPyClass pyClasses ganeti-3.1.0~rc2/src/Ganeti/Hs2Py/ListConstants.hs.in000064400000000000000000000033571476477700300223300ustar00rootroot00000000000000{-| Contains the list of the Haskell to Python constants. Note that this file is autogenerated by the Makefile with a header from @ListConstants.hs.in@. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} {-# LANGUAGE TemplateHaskell #-} module Ganeti.Hs2Py.ListConstants where import Ganeti.Constants import Ganeti.Hs2Py.GenConstants import Ganeti.PyValue () $(genPyConstants "pyConstants" ( PY_CONSTANT_NAMES[])) putConstants :: IO () putConstants = sequence_ [ putStrLn (k ++ " = " ++ v) | (k, v) <- pyConstants ] ganeti-3.1.0~rc2/src/Ganeti/Hs2Py/OpDoc.hs000064400000000000000000000361271476477700300201200ustar00rootroot00000000000000{-| Implementation of the doc strings for the opcodes. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Hs2Py.OpDoc where opClusterPostInit :: String opClusterPostInit = "Post cluster initialization.\n\ \\n\ \ This opcode does not touch the cluster at all. Its purpose is to run hooks\n\ \ after the cluster has been initialized." opClusterDestroy :: String opClusterDestroy = "Destroy the cluster.\n\ \\n\ \ This opcode has no other parameters. All the state is irreversibly\n\ \ lost after the execution of this opcode." opClusterQuery :: String opClusterQuery = "Query cluster information." opClusterVerify :: String opClusterVerify = "Submits all jobs necessary to verify the cluster." opClusterVerifyConfig :: String opClusterVerifyConfig = "Verify the cluster config." opClusterVerifyGroup :: String opClusterVerifyGroup = "Run verify on a node group from the cluster.\n\ \\n\ \ @type skip_checks: C{list}\n\ \ @ivar skip_checks: steps to be skipped from the verify process; this\n\ \ needs to be a subset of\n\ \ L{constants.VERIFY_OPTIONAL_CHECKS}; currently\n\ \ only L{constants.VERIFY_NPLUSONE_MEM} can be passed" opClusterVerifyDisks :: String opClusterVerifyDisks = "Verify the cluster disks." opGroupVerifyDisks :: String opGroupVerifyDisks = "Verifies the status of all disks in a node group.\n\ \\n\ \ Result: a tuple of three elements:\n\ \ - dict of node names with issues (values: error msg)\n\ \ - list of instances with degraded disks (that should be activated)\n\ \ - dict of instances with missing logical volumes (values: (node, vol)\n\ \ pairs with details about the missing volumes)\n\ \\n\ \ In normal operation, all lists should be empty. A non-empty instance\n\ \ list (3rd element of the result) is still ok (errors were fixed) but\n\ \ non-empty node list means some node is down, and probably there are\n\ \ unfixable drbd errors.\n\ \\n\ \ Note that only instances that are drbd-based are taken into\n\ \ consideration. This might need to be revisited in the future." opClusterRepairDiskSizes :: String opClusterRepairDiskSizes = "Verify the disk sizes of the instances and fixes configuration\n\ \ mismatches.\n\ \\n\ \ Parameters: optional instances list, in case we want to restrict the\n\ \ checks to only a subset of the instances.\n\ \\n\ \ Result: a list of tuples, (instance, disk, parameter, new-size) for\n\ \ changed configurations.\n\ \\n\ \ In normal operation, the list should be empty.\n\ \\n\ \ @type instances: list\n\ \ @ivar instances: the list of instances to check, or empty for all instances" opClusterConfigQuery :: String opClusterConfigQuery = "Query cluster configuration values." opClusterRename :: String opClusterRename = "Rename the cluster.\n\ \\n\ \ @type name: C{str}\n\ \ @ivar name: The new name of the cluster. The name and/or the master IP\n\ \ address will be changed to match the new name and its IP\n\ \ address." opClusterSetParams :: String opClusterSetParams = "Change the parameters of the cluster.\n\ \\n\ \ @type vg_name: C{str} or C{None}\n\ \ @ivar vg_name: The new volume group name or None to disable LVM usage." opClusterRedistConf :: String opClusterRedistConf = "Force a full push of the cluster configuration." opClusterActivateMasterIp :: String opClusterActivateMasterIp = "Activate the master IP on the master node." opClusterDeactivateMasterIp :: String opClusterDeactivateMasterIp = "Deactivate the master IP on the master node." opClusterRenewCrypto :: String opClusterRenewCrypto = "Renews the cluster node's SSL client certificates." opQuery :: String opQuery = "Query for resources/items.\n\ \\n\ \ @ivar what: Resources to query for, must be one of L{constants.QR_VIA_OP}\n\ \ @ivar fields: List of fields to retrieve\n\ \ @ivar qfilter: Query filter" opQueryFields :: String opQueryFields = "Query for available resource/item fields.\n\ \\n\ \ @ivar what: Resources to query for, must be one of L{constants.QR_VIA_OP}\n\ \ @ivar fields: List of fields to retrieve" opOobCommand :: String opOobCommand = "Interact with OOB." opRestrictedCommand :: String opRestrictedCommand = "Runs a restricted command on node(s)." opNodeRemove :: String opNodeRemove = "Remove a node.\n\ \\n\ \ @type node_name: C{str}\n\ \ @ivar node_name: The name of the node to remove. If the node still has\n\ \ instances on it, the operation will fail." opNodeAdd :: String opNodeAdd = "Add a node to the cluster.\n\ \\n\ \ @type node_name: C{str}\n\ \ @ivar node_name: The name of the node to add. This can be a short name,\n\ \ but it will be expanded to the FQDN.\n\ \ @type primary_ip: IP address\n\ \ @ivar primary_ip: The primary IP of the node. This will be ignored when\n\ \ the opcode is submitted, but will be filled during the\n\ \ node add (so it will be visible in the job query).\n\ \ @type secondary_ip: IP address\n\ \ @ivar secondary_ip: The secondary IP of the node. This needs to be passed\n\ \ if the cluster has been initialized in 'dual-network'\n\ \ mode, otherwise it must not be given.\n\ \ @type readd: C{bool}\n\ \ @ivar readd: Whether to re-add an existing node to the cluster. If\n\ \ this is not passed, then the operation will abort if the node\n\ \ name is already in the cluster; use this parameter to\n\ \ 'repair' a node that had its configuration broken, or was\n\ \ reinstalled without removal from the cluster.\n\ \ @type group: C{str}\n\ \ @ivar group: The node group to which this node will belong.\n\ \ @type vm_capable: C{bool}\n\ \ @ivar vm_capable: The vm_capable node attribute\n\ \ @type master_capable: C{bool}\n\ \ @ivar master_capable: The master_capable node attribute" opNodeQuery :: String opNodeQuery = "Compute the list of nodes." opNodeQueryvols :: String opNodeQueryvols = "Get list of volumes on node." opNodeQueryStorage :: String opNodeQueryStorage = "Get information on storage for node(s)." opNodeModifyStorage :: String opNodeModifyStorage = "Modifies the properies of a storage unit" opRepairNodeStorage :: String opRepairNodeStorage = "Repairs the volume group on a node." opNodeSetParams :: String opNodeSetParams = "Change the parameters of a node." opNodePowercycle :: String opNodePowercycle = "Tries to powercycle a node." opNodeMigrate :: String opNodeMigrate = "Migrate all instances from a node." opNodeEvacuate :: String opNodeEvacuate = "Evacuate instances off a number of nodes." opInstanceCreate :: String opInstanceCreate = "Create an instance.\n\ \\n\ \ @ivar instance_name: Instance name\n\ \ @ivar mode: Instance creation mode (one of\ \ L{constants.INSTANCE_CREATE_MODES})\n\ \ @ivar source_handshake: Signed handshake from source (remote import only)\n\ \ @ivar source_x509_ca: Source X509 CA in PEM format (remote import only)\n\ \ @ivar source_instance_name: Previous name of instance (remote import only)\n\ \ @ivar source_shutdown_timeout: Shutdown timeout used for source instance\n\ \ (remote import only)" opInstanceMultiAlloc :: String opInstanceMultiAlloc = "Allocates multiple instances." opInstanceReinstall :: String opInstanceReinstall = "Reinstall an instance's OS." opInstanceRemove :: String opInstanceRemove = "Remove an instance." opInstanceRename :: String opInstanceRename = "Rename an instance." opInstanceStartup :: String opInstanceStartup = "Startup an instance." opInstanceShutdown :: String opInstanceShutdown = "Shutdown an instance." opInstanceReboot :: String opInstanceReboot = "Reboot an instance." opInstanceReplaceDisks :: String opInstanceReplaceDisks = "Replace the disks of an instance." opInstanceFailover :: String opInstanceFailover = "Failover an instance." opInstanceMigrate :: String opInstanceMigrate = "Migrate an instance.\n\ \\n\ \ This migrates (without shutting down an instance) to its secondary\n\ \ node.\n\ \\n\ \ @ivar instance_name: the name of the instance\n\ \ @ivar mode: the migration mode (live, non-live or None for auto)" opInstanceMove :: String opInstanceMove = "Move an instance.\n\ \\n\ \ This move (with shutting down an instance and data copying) to an\n\ \ arbitrary node.\n\ \\n\ \ @ivar instance_name: the name of the instance\n\ \ @ivar target_node: the destination node" opInstanceConsole :: String opInstanceConsole = "Connect to an instance's console." opInstanceActivateDisks :: String opInstanceActivateDisks = "Activate an instance's disks." opInstanceDeactivateDisks :: String opInstanceDeactivateDisks = "Deactivate an instance's disks." opInstanceRecreateDisks :: String opInstanceRecreateDisks = "Recreate an instance's disks." opInstanceQuery :: String opInstanceQuery = "Compute the list of instances." opInstanceQueryData :: String opInstanceQueryData = "Compute the run-time status of instances." opInstanceSetParams :: String opInstanceSetParams = "Change the parameters of an instance." opInstanceGrowDisk :: String opInstanceGrowDisk = "Grow a disk of an instance." opInstanceChangeGroup :: String opInstanceChangeGroup = "Moves an instance to another node group." opGroupAdd :: String opGroupAdd = "Add a node group to the cluster." opGroupAssignNodes :: String opGroupAssignNodes = "Assign nodes to a node group." opGroupQuery :: String opGroupQuery = "Compute the list of node groups." opGroupSetParams :: String opGroupSetParams = "Change the parameters of a node group." opGroupRemove :: String opGroupRemove = "Remove a node group from the cluster." opGroupRename :: String opGroupRename = "Rename a node group in the cluster." opGroupEvacuate :: String opGroupEvacuate = "Evacuate a node group in the cluster." opOsDiagnose :: String opOsDiagnose = "Compute the list of guest operating systems." opExtStorageDiagnose :: String opExtStorageDiagnose = "Compute the list of external storage providers." opBackupQuery :: String opBackupQuery = "Compute the list of exported images." opBackupPrepare :: String opBackupPrepare = "Prepares an instance export.\n\ \\n\ \ @ivar instance_name: Instance name\n\ \ @ivar mode: Export mode (one of L{constants.EXPORT_MODES})" opBackupExport :: String opBackupExport = "Export an instance.\n\ \\n\ \ For local exports, the export destination is the node name. For\n\ \ remote exports, the export destination is a list of tuples, each\n\ \ consisting of hostname/IP address, port, magic, HMAC and HMAC\n\ \ salt. The HMAC is calculated using the cluster domain secret over\n\ \ the value \"${index}:${hostname}:${port}\". The destination X509 CA\n\ \ must be a signed certificate.\n\ \\n\ \ @ivar mode: Export mode (one of L{constants.EXPORT_MODES})\n\ \ @ivar target_node: Export destination\n\ \ @ivar x509_key_name: X509 key to use (remote export only)\n\ \ @ivar destination_x509_ca: Destination X509 CA in PEM format (remote\n\ \ export only)" opBackupRemove :: String opBackupRemove = "Remove an instance's export." opTagsGet :: String opTagsGet = "Returns the tags of the given object." opTagsSearch :: String opTagsSearch = "Searches the tags in the cluster for a given pattern." opTagsSet :: String opTagsSet = "Add a list of tags on a given object." opTagsDel :: String opTagsDel = "Remove a list of tags from a given object." opTestDelay :: String opTestDelay = "Sleeps for a configured amount of time.\n\ \\n\ \ This is used just for debugging and testing.\n\ \\n\ \ Parameters:\n\ \ - duration: the time to sleep, in seconds\n\ \ - on_master: if true, sleep on the master\n\ \ - on_nodes: list of nodes in which to sleep\n\ \\n\ \ If the on_master parameter is true, it will execute a sleep on the\n\ \ master (before any node sleep).\n\ \\n\ \ If the on_nodes list is not empty, it will sleep on those nodes\n\ \ (after the sleep on the master, if that is enabled).\n\ \\n\ \ As an additional feature, the case of duration < 0 will be reported\n\ \ as an execution error, so this opcode can be used as a failure\n\ \ generator. The case of duration == 0 will not be treated specially." opTestAllocator :: String opTestAllocator = "Allocator framework testing.\n\ \\n\ \ This opcode has two modes:\n\ \ - gather and return allocator input for a given mode (allocate new\n\ \ or replace secondary) and a given instance definition (direction\n\ \ 'in')\n\ \ - run a selected allocator for a given operation (as above) and\n\ \ return the allocator output (direction 'out')" opTestJqueue :: String opTestJqueue = "Utility opcode to test some aspects of the job queue." opTestOsParams :: String opTestOsParams = "Utility opcode to test secret os parameter transmission." opTestDummy :: String opTestDummy = "Utility opcode used by unittests." opNetworkAdd :: String opNetworkAdd = "Add an IP network to the cluster." opNetworkRemove :: String opNetworkRemove = "Remove an existing network from the cluster.\n\ \ Must not be connected to any nodegroup." opNetworkRename :: String opNetworkRename = "Rename a network in the cluster." opNetworkSetParams :: String opNetworkSetParams = "Modify Network's parameters except for IPv4 subnet" opNetworkConnect :: String opNetworkConnect = "Connect a Network to a specific Nodegroup with the defined netparams\n\ \ (mode, link). Nics in this Network will inherit those params.\n\ \ Produce errors if a NIC (that its not already assigned to a network)\n\ \ has an IP that is contained in the Network this will produce error\ \ unless\n\ \ --no-conflicts-check is passed." opNetworkDisconnect :: String opNetworkDisconnect = "Disconnect a Network from a Nodegroup. Produce errors if NICs are\n\ \ present in the Network unless --no-conficts-check option is passed." opNetworkQuery :: String opNetworkQuery = "Compute the list of networks." ganeti-3.1.0~rc2/src/Ganeti/Hypervisor/000075500000000000000000000000001476477700300177545ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Hypervisor/Xen.hs000064400000000000000000000077721476477700300210570ustar00rootroot00000000000000{-| Module to access the information provided by the Xen hypervisor. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Hypervisor.Xen ( getDomainsInfo , getInferredDomInfo , getUptimeInfo --Data types to be re-exported from here , Domain(..) , UptimeInfo(..) ) where import qualified Control.Exception as E import Data.Attoparsec.Text as A import qualified Data.Map as Map import Data.Text (pack) import System.Process import qualified Ganeti.BasicTypes as BT import Ganeti.Hypervisor.Xen.Types import Ganeti.Hypervisor.Xen.XlParser import Ganeti.Logging import Ganeti.Utils -- | Get information about the current Xen domains as a map where the domain -- name is the key. This only includes the information made available by Xen -- itself. getDomainsInfo :: IO (BT.Result (Map.Map String Domain)) getDomainsInfo = do contents <- (E.try $ readProcess "xl" ["list", "--long"] "") :: IO (Either IOError String) return $ either (BT.Bad . show) ( \c -> case A.parseOnly xlListParser $ pack c of Left msg -> BT.Bad msg Right dom -> BT.Ok dom ) contents -- | Given a domain and a map containing information about multiple domains, -- infer additional information about that domain (specifically, whether it is -- hung). inferDomInfos :: Map.Map String Domain -> Domain -> Domain inferDomInfos domMap dom1 = case Map.lookup (domName dom1) domMap of Just dom2 -> dom1 { domIsHung = Just $ domCpuTime dom1 == domCpuTime dom2 } Nothing -> dom1 { domIsHung = Nothing } -- | Get information about the current Xen domains as a map where the domain -- name is the key. This includes information made available by Xen itself as -- well as further information that can be inferred by querying Xen multiple -- times and comparing the results. getInferredDomInfo :: IO (BT.Result (Map.Map String Domain)) getInferredDomInfo = do domMap1 <- getDomainsInfo domMap2 <- getDomainsInfo case (domMap1, domMap2) of (BT.Bad m1, BT.Bad m2) -> return . BT.Bad $ m1 ++ "\n" ++ m2 (BT.Bad m, BT.Ok d) -> do logWarning $ "Unable to retrieve domains info the first time" ++ m return $ BT.Ok d (BT.Ok d, BT.Bad m) -> do logWarning $ "Unable to retrieve domains info the second time" ++ m return $ BT.Ok d (BT.Ok d1, BT.Ok d2) -> return . BT.Ok $ fmap (inferDomInfos d2) d1 -- | Get information about the uptime of domains, as a map where the domain ID -- is the key. getUptimeInfo :: IO (Map.Map Int UptimeInfo) getUptimeInfo = do contents <- ((E.try $ readProcess "xl" ["uptime"] "") :: IO (Either IOError String)) >>= exitIfBad "running command" . either (BT.Bad . show) BT.Ok case A.parseOnly xlUptimeParser $ pack contents of Left msg -> exitErr msg Right uInfo -> return uInfo ganeti-3.1.0~rc2/src/Ganeti/Hypervisor/Xen/000075500000000000000000000000001476477700300205065ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Hypervisor/Xen/Types.hs000064400000000000000000000107351476477700300221540ustar00rootroot00000000000000{-# LANGUAGE FlexibleInstances, TypeSynonymInstances #-} {-| Data types for Xen-specific hypervisor functionalities. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Hypervisor.Xen.Types ( LispConfig(..) , Domain(..) , FromLispConfig(..) , UptimeInfo(..) , ActualState(..) ) where import qualified Text.JSON as J import Ganeti.BasicTypes -- | Data type representing configuration data as produced by the -- @xl list --long@ command. data LispConfig = LCList [LispConfig] | LCString String | LCDouble Double deriving (Eq, Show) -- | Data type representing a Xen Domain. data Domain = Domain { domId :: Int , domName :: String , domCpuTime :: Double , domState :: ActualState , domIsHung :: Maybe Bool } deriving (Show, Eq) -- | Class representing all the types that can be extracted from LispConfig. class FromLispConfig a where fromLispConfig :: LispConfig -> Result a -- | Instance of FromLispConfig for Int. instance FromLispConfig Int where fromLispConfig (LCDouble d) = Ok $ floor d fromLispConfig (LCList [LCString _, LCDouble d]) = Ok $ floor d fromLispConfig c = Bad $ "Unable to extract a Int from this configuration: " ++ show c -- | Instance of FromLispConfig for Double. instance FromLispConfig Double where fromLispConfig (LCDouble d) = Ok d fromLispConfig (LCList [LCString _, LCDouble d]) = Ok d fromLispConfig c = Bad $ "Unable to extract a Double from this configuration: " ++ show c -- | Instance of FromLispConfig for String instance FromLispConfig String where fromLispConfig (LCString s) = Ok s fromLispConfig (LCList [LCString _, LCString s]) = Ok s fromLispConfig c = Bad $ "Unable to extract a String from this configuration: " ++ show c -- | Instance of FromLispConfig for [LispConfig] instance FromLispConfig [LispConfig] where fromLispConfig (LCList l) = Ok l fromLispConfig c = Bad $ "Unable to extract a List from this configuration: " ++ show c -- Data type representing the information that can be obtained from @xm uptime@ data UptimeInfo = UptimeInfo { uInfoName :: String , uInfoID :: Int , uInfoUptime :: String } deriving (Eq, Show) data ActualState = ActualRunning -- ^ The instance is running | ActualBlocked -- ^ The instance is not running or runnable | ActualPaused -- ^ The instance has been paused | ActualShutdown -- ^ The instance is shut down | ActualCrashed -- ^ The instance has crashed | ActualDying -- ^ The instance is in process of dying | ActualHung -- ^ The instance is hung | ActualUnknown -- ^ Unknown state. Parsing error. deriving (Show, Eq) instance J.JSON ActualState where showJSON ActualRunning = J.showJSON "running" showJSON ActualBlocked = J.showJSON "blocked" showJSON ActualPaused = J.showJSON "paused" showJSON ActualShutdown = J.showJSON "shutdown" showJSON ActualCrashed = J.showJSON "crashed" showJSON ActualDying = J.showJSON "dying" showJSON ActualHung = J.showJSON "hung" showJSON ActualUnknown = J.showJSON "unknown" readJSON = error "JSON read instance not implemented for type ActualState" ganeti-3.1.0~rc2/src/Ganeti/Hypervisor/Xen/XlParser.hs000064400000000000000000000147241476477700300226120ustar00rootroot00000000000000{-# LANGUAGE OverloadedStrings #-} {-| Parser for the output of the @xl list --long@ command of Xen -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Hypervisor.Xen.XlParser ( xlListParser , lispConfigParser , xlUptimeParser , uptimeLineParser ) where import Control.Applicative import Control.Monad import qualified Data.Attoparsec.Combinator as AC import qualified Data.Attoparsec.Text as A import Data.Attoparsec.Text (Parser) import Data.Char import Data.List import Data.Text (unpack) import qualified Data.Map as Map import Ganeti.BasicTypes import Ganeti.Hypervisor.Xen.Types -- | A parser for parsing generic config files written in the (LISP-like) -- format that is the output of the @xl list --long@ command. -- This parser only takes care of the syntactic parse, but does not care -- about the semantics. -- Note: parsing the double requires checking for the next character in order -- to prevent string like "9a" to be recognized as the number 9. lispConfigParser :: Parser LispConfig lispConfigParser = A.skipSpace *> ( listConfigP <|> doubleP <|> stringP ) <* A.skipSpace where listConfigP = LCList <$> (A.char '(' *> liftA2 (++) (many middleP) (((:[]) <$> finalP) <|> (rparen *> pure []))) doubleP = LCDouble <$> A.rational <* A.skipSpace <* A.endOfInput innerDoubleP = LCDouble <$> A.rational stringP = LCString . unpack <$> A.takeWhile1 (not . (\c -> isSpace c || c `elem` ("()" :: String))) wspace = AC.many1 A.space rparen = A.skipSpace *> A.char ')' finalP = listConfigP <* rparen <|> innerDoubleP <* rparen <|> stringP <* rparen middleP = listConfigP <* wspace <|> innerDoubleP <* wspace <|> stringP <* wspace -- | Find a configuration having the given string as its first element, -- from a list of configurations. findConf :: String -> [LispConfig] -> Result LispConfig findConf key configs = case find (isNamed key) configs of (Just c) -> Ok c _ -> Bad "Configuration not found" -- | Get the value of of a configuration having the given string as its -- first element. -- The value is the content of the configuration, discarding the name itself. getValue :: (FromLispConfig a) => String -> [LispConfig] -> Result a getValue key configs = findConf key configs >>= fromLispConfig -- | Extract the values of a configuration containing a list of them. extractValues :: LispConfig -> Result [LispConfig] extractValues c = tail `fmap` fromLispConfig c -- | Verify whether the given configuration has a certain name or not.fmap -- The name of a configuration is its first parameter, if it is a string. isNamed :: String -> LispConfig -> Bool isNamed key (LCList (LCString x:_)) = x == key isNamed _ _ = False -- | Parser for recognising the current state of a Xen domain. parseState :: String -> ActualState parseState s = case s of "r-----" -> ActualRunning "-b----" -> ActualBlocked "--p---" -> ActualPaused "---s--" -> ActualShutdown "----c-" -> ActualCrashed "-----d" -> ActualDying _ -> ActualUnknown -- | Extract the configuration data of a Xen domain from a generic LispConfig -- data structure. Fail if the LispConfig does not represent a domain. getDomainConfig :: LispConfig -> Result Domain getDomainConfig configData = do domainConf <- if isNamed "domain" configData then extractValues configData else Bad $ "Not a domain configuration: " ++ show configData domid <- getValue "domid" domainConf name <- getValue "name" domainConf cpuTime <- getValue "cpu_time" domainConf state <- getValue "state" domainConf let actualState = parseState state return $ Domain domid name cpuTime actualState Nothing -- | A parser for parsing the output of the @xl list --long@ command. -- It adds the semantic layer on top of lispConfigParser. -- It returns a map of domains, with their name as the key. -- FIXME: This is efficient under the assumption that only a few fields of the -- domain configuration are actually needed. If many of them are required, a -- parser able to directly extract the domain config would actually be better. xlListParser :: Parser (Map.Map String Domain) xlListParser = do configs <- lispConfigParser `AC.manyTill` A.endOfInput let domains = map getDomainConfig configs foldResult m (Ok val) = Ok $ Map.insert (domName val) val m foldResult _ (Bad msg) = Bad msg case foldM foldResult Map.empty domains of Ok d -> return d Bad msg -> fail msg -- | A parser for parsing the output of the @xl uptime@ command. xlUptimeParser :: Parser (Map.Map Int UptimeInfo) xlUptimeParser = do _ <- headerParser uptimes <- uptimeLineParser `AC.manyTill` A.endOfInput return $ Map.fromList [(uInfoID u, u) | u <- uptimes] where headerParser = A.string "Name" <* A.skipSpace <* A.string "ID" <* A.skipSpace <* A.string "Uptime" <* A.skipSpace -- | A helper for parsing a single line of the @xl uptime@ output. uptimeLineParser :: Parser UptimeInfo uptimeLineParser = do name <- A.takeTill isSpace <* A.skipSpace idNum <- A.decimal <* A.skipSpace uptime <- A.takeTill (`elem` ("\n\r" :: String)) <* A.skipSpace return . UptimeInfo (unpack name) idNum $ unpack uptime ganeti-3.1.0~rc2/src/Ganeti/JQScheduler.hs000064400000000000000000000601611476477700300203130ustar00rootroot00000000000000{-# LANGUAGE Rank2Types #-} {-| Implementation of a reader for the job queue. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.JQScheduler ( JQStatus , jqLivelock , emptyJQStatus , selectJobsToRun , scheduleSomeJobs , initJQScheduler , enqueueNewJobs , dequeueJob , setJobPriority , cleanupIfDead , updateStatusAndScheduleSomeJobs , configChangeNeedsRescheduling ) where import Control.Applicative (liftA2) import Control.Arrow import Control.Concurrent import Control.Exception import Control.Monad import Control.Monad.IO.Class import Data.Function (on) import Data.IORef (IORef, atomicModifyIORef, atomicModifyIORef', newIORef, readIORef) import Data.List import Data.Maybe import qualified Data.Map as Map import Data.Ord (comparing) import Data.Set (Set) import qualified Data.Set as S import System.INotify import Ganeti.BasicTypes import Ganeti.Compat import Ganeti.Constants as C import Ganeti.Errors import Ganeti.JQScheduler.Filtering (applyingFilter, jobFiltering) import Ganeti.JQScheduler.Types import Ganeti.JQScheduler.ReasonRateLimiting (reasonRateLimit) import Ganeti.JQueue as JQ import Ganeti.JSON (fromContainer) import Ganeti.Lens hiding (chosen) import Ganeti.Logging import Ganeti.Objects import Ganeti.Path import Ganeti.Types import Ganeti.Utils import Ganeti.Utils.Livelock import Ganeti.Utils.MVarLock {-| Representation of the job queue We keep two lists of jobs (together with information about the last fstat result observed): the jobs that are enqueued, but not yet handed over for execution, and the jobs already handed over for execution. They are kept together in a single IORef, so that we can atomically update both, in particular when scheduling jobs to be handed over for execution. -} data JQStatus = JQStatus { jqJobs :: IORef Queue , jqConfig :: IORef (Result ConfigData) , jqLivelock :: Livelock , jqForkLock :: Lock } emptyJQStatus :: IORef (Result ConfigData) -> IO JQStatus emptyJQStatus config = do jqJ <- newIORef Queue { qEnqueued = [], qRunning = [], qManipulated = [] } (_, livelock) <- mkLivelockFile C.luxiLivelockPrefix forkLock <- newLock return JQStatus { jqJobs = jqJ, jqConfig = config, jqLivelock = livelock , jqForkLock = forkLock } -- When updating the job lists, force the elements to WHNF, otherwise it is -- easy to leak the resources held onto by the lazily parsed job file. -- This can happen, eg, if updateJob is called, but the resulting QueuedJob -- isn't used by the scheduler, for example when the inotify watcher or the -- the polling loop re-reads a job with a new message appended to it. -- | Apply a function on the running jobs. onRunningJobs :: ([JobWithStat] -> [JobWithStat]) -> Queue -> Queue onRunningJobs f q@Queue { qRunning = qr } = let qr' = (foldr seq () qr) `seq` f qr -- force list els to WHNF in q { qRunning = qr' } -- | Apply a function on the queued jobs. onQueuedJobs :: ([JobWithStat] -> [JobWithStat]) -> Queue -> Queue onQueuedJobs f q@Queue { qEnqueued = qe } = let qe' = (foldr seq () qe) `seq` f qe -- force list els to WHNF in q { qEnqueued = qe' } -- | Obtain a JobWithStat from a QueuedJob. unreadJob :: QueuedJob -> JobWithStat unreadJob job = JobWithStat {jJob=job, jStat=nullFStat, jINotify=Nothing} -- | Reload interval for polling the running jobs for updates in microseconds. watchInterval :: Int watchInterval = C.luxidJobqueuePollInterval * 1000000 -- | Read a cluster parameter from the configuration, using a default if the -- configuration is not available. getConfigValue :: (Cluster -> a) -> a -> JQStatus -> IO a getConfigValue param defaultvalue = liftM (genericResult (const defaultvalue) (param . configCluster)) . readIORef . jqConfig -- | Get the maximual number of jobs to be run simultaneously from the -- configuration. If the configuration is not available, be conservative -- and use the smallest possible value, i.e., 1. getMaxRunningJobs :: JQStatus -> IO Int getMaxRunningJobs = getConfigValue clusterMaxRunningJobs 1 -- | Get the maximual number of jobs to be tracked simultaneously from the -- configuration. If the configuration is not available, be conservative -- and use the smallest possible value, i.e., 1. getMaxTrackedJobs :: JQStatus -> IO Int getMaxTrackedJobs = getConfigValue clusterMaxTrackedJobs 1 -- | Get the number of jobs currently running. getRQL :: JQStatus -> IO Int getRQL = liftM (length . qRunning) . readIORef . jqJobs -- | Wrapper function to atomically update the jobs in the queue status. modifyJobs :: JQStatus -> (Queue -> Queue) -> IO () modifyJobs qstat f = atomicModifyIORef' (jqJobs qstat) (flip (,) () . f) -- | Reread a job from disk, if the file has changed. readJobStatus :: JobWithStat -> IO (Maybe JobWithStat) readJobStatus jWS@(JobWithStat {jStat=fstat, jJob=job}) = do let jid = qjId job qdir <- queueDir let fpath = liveJobFile qdir jid logDebug $ "Checking if " ++ fpath ++ " changed on disk." changedResult <- try $ needsReload fstat fpath :: IO (Either IOError (Maybe FStat)) let changed = either (const $ Just nullFStat) id changedResult case changed of Nothing -> do logDebug $ "File " ++ fpath ++ " not changed on disk." return Nothing Just fstat' -> do let jids = show $ fromJobId jid logDebug $ "Rereading job " ++ jids readResult <- loadJobFromDisk qdir True jid case readResult of Bad s -> do logWarning $ "Failed to read job " ++ jids ++ ": " ++ s return Nothing Ok (job', _) -> do logDebug $ "Read job " ++ jids ++ ", status is " ++ show (calcJobStatus job') return . Just $ jWS {jStat=fstat', jJob=job'} -- jINotify unchanged -- | Update a job in the job queue, if it is still there. This is the -- pure function for inserting a previously read change into the queue. -- as the change contains its time stamp, we don't have to worry about a -- later read change overwriting a newer read state. If this happens, the -- fstat value will be outdated, so the next poller run will fix this. updateJobStatus :: JobWithStat -> [JobWithStat] -> [JobWithStat] updateJobStatus job' = let jid = qjId $ jJob job' in map (\job -> if qjId (jJob job) == jid then job' else job) -- | Update a single job by reading it from disk, if necessary. updateJob :: JQStatus -> JobWithStat -> IO () updateJob state jb = do jb' <- readJobStatus jb maybe (return ()) (modifyJobs state . onRunningJobs . updateJobStatus) jb' when (maybe True (jobFinalized . jJob) jb') . (>> return ()) . forkIO $ do logDebug "Scheduler noticed a job to have finished." cleanupFinishedJobs state scheduleSomeJobs state -- | Move a job from one part of the queue to another. -- Return the job that was moved, or 'Nothing' if it wasn't found in -- the queue. moveJob :: Lens' Queue [JobWithStat] -- ^ from queue -> Lens' Queue [JobWithStat] -- ^ to queue -> JobId -> Queue -> (Queue, Maybe JobWithStat) moveJob fromQ toQ jid queue = -- traverse over the @(,) [JobWithStats]@ functor to extract the job case traverseOf fromQ (partition ((== jid) . qjId . jJob)) queue of (job : _, queue') -> (over toQ (++ [job]) queue', Just job) _ -> (queue, Nothing) -- | Atomically move a job from one part of the queue to another. -- Return the job that was moved, or 'Nothing' if it wasn't found in -- the queue. moveJobAtomic :: Lens' Queue [JobWithStat] -- ^ from queue -> Lens' Queue [JobWithStat] -- ^ to queue -> JobId -> JQStatus -> IO (Maybe JobWithStat) moveJobAtomic fromQ toQ jid qstat = atomicModifyIORef (jqJobs qstat) (moveJob fromQ toQ jid) -- | Manipulate a running job by atomically moving it from 'qRunning' -- into 'qManipulated', running a given IO action and then atomically -- returning it back. -- -- Returns the result of the IO action, or 'Nothing', if the job wasn't found -- in the queue. manipulateRunningJob :: JQStatus -> JobId -> IO a -> IO (Maybe a) manipulateRunningJob qstat jid k = do jobOpt <- moveJobAtomic qRunningL qManipulatedL jid qstat case jobOpt of Nothing -> return Nothing Just _ -> (Just `liftM` k) `finally` moveJobAtomic qManipulatedL qRunningL jid qstat -- | Sort out the finished jobs from the monitored part of the queue. -- This is the pure part, splitting the queue into a remaining queue -- and the jobs that were removed. sortoutFinishedJobs :: Queue -> (Queue, [JobWithStat]) sortoutFinishedJobs queue = let (fin, run') = partition (jobFinalized . jJob) . qRunning $ queue in (queue {qRunning=run'}, fin) -- | Actually clean up the finished jobs. This is the IO wrapper around -- the pure `sortoutFinishedJobs`. cleanupFinishedJobs :: JQStatus -> IO () cleanupFinishedJobs qstate = do finished <- atomicModifyIORef (jqJobs qstate) sortoutFinishedJobs let showJob = show . ((fromJobId . qjId) &&& calcJobStatus) . jJob jlist = commaJoin $ map showJob finished unless (null finished) . logInfo $ "Finished jobs: " ++ jlist mapM_ (maybe (return ()) killINotify . jINotify) finished -- | Watcher task for a job, to update it on file changes. It also -- reinstantiates itself upon receiving an Ignored event. jobWatcher :: JQStatus -> JobWithStat -> Event -> IO () jobWatcher state jWS e = do let jid = qjId $ jJob jWS jids = show $ fromJobId jid logInfo $ "Scheduler notified of change of job " ++ jids logDebug $ "Scheduler notify event for " ++ jids ++ ": " ++ show e let inotify = jINotify jWS when (e == Ignored && isJust inotify) $ do qdir <- queueDir let fpath = toInotifyPath $ liveJobFile qdir jid _ <- addWatch (fromJust inotify) [Modify, Delete] fpath (jobWatcher state jWS) return () updateJob state jWS -- | Attach the job watcher to a running job. attachWatcher :: JQStatus -> JobWithStat -> IO () attachWatcher state jWS = when (isNothing $ jINotify jWS) $ do max_watch <- getMaxTrackedJobs state rql <- getRQL state if rql < max_watch then do inotify <- initINotify qdir <- queueDir let fpath = liveJobFile qdir . qjId $ jJob jWS jWS' = jWS { jINotify=Just inotify } logDebug $ "Attaching queue watcher for " ++ fpath _ <- addWatch inotify [Modify, Delete] (toInotifyPath fpath) $ jobWatcher state jWS' modifyJobs state . onRunningJobs $ updateJobStatus jWS' else logDebug $ "Not attaching watcher for job " ++ (show . fromJobId . qjId $ jJob jWS) ++ ", run queue length is " ++ show rql -- | For a queued job, determine whether it is eligible to run, i.e., -- if no jobs it depends on are either enqueued or running. jobEligible :: Queue -> JobWithStat -> Bool jobEligible queue jWS = let jdeps = getJobDependencies $ jJob jWS blocks = flip elem jdeps . qjId . jJob in not . any blocks . liftA2 (++) qRunning qEnqueued $ queue -- | Decide on which jobs to schedule next for execution. This is the -- pure function doing the scheduling. selectJobsToRun :: Int -- ^ How many jobs are allowed to run at the -- same time. -> Set FilterRule -- ^ Filter rules to respect for scheduling -> Queue -> (Queue, [JobWithStat]) selectJobsToRun count filters queue = let n = count - length (qRunning queue) - length (qManipulated queue) chosen = take n . jobFiltering queue filters . reasonRateLimit queue . sortBy (comparing (calcJobPriority . jJob)) . filter (jobEligible queue) $ qEnqueued queue remain = deleteFirstsBy ((==) `on` (qjId . jJob)) (qEnqueued queue) chosen in (queue {qEnqueued=remain, qRunning=qRunning queue ++ chosen}, chosen) -- | Logs errors of failed jobs and returns the set of job IDs. logFailedJobs :: (MonadLog m) => [(JobWithStat, GanetiException)] -> m (S.Set JobId) logFailedJobs [] = return S.empty logFailedJobs jobs = do let jids = S.fromList . map (qjId . jJob . fst) $ jobs jidsString = commaJoin . map (show . fromJobId) . S.toList $ jids logWarning $ "Starting jobs " ++ jidsString ++ " failed: " ++ show (map snd jobs) return jids -- | Fail jobs that were previously selected for execution -- but couldn't be started. failJobs :: ConfigData -> JQStatus -> [(JobWithStat, GanetiException)] -> IO () failJobs cfg qstate jobs = do qdir <- queueDir now <- currentTimestamp jids <- logFailedJobs jobs let sjobs = intercalate "." . map (show . fromJobId) $ S.toList jids let rmJobs = filter ((`S.notMember` jids) . qjId . jJob) logWarning $ "Failing jobs " ++ sjobs modifyJobs qstate $ onRunningJobs rmJobs let trySaveJob :: JobWithStat -> ResultT String IO () trySaveJob = (() <$) . writeAndReplicateJob cfg qdir . jJob reason jid msg = ( "gnt:daemon:luxid:startjobs" , "job " ++ show (fromJobId jid) ++ " failed to start: " ++ msg , reasonTrailTimestamp now ) failJob err job = failQueuedJob (reason (qjId job) (show err)) now job failAndSaveJobWithStat (jws, err) = trySaveJob . over jJobL (failJob err) $ jws mapM_ (runResultT . failAndSaveJobWithStat) jobs logDebug $ "Failed jobs " ++ sjobs -- | Checks if any jobs match a REJECT filter rule, and cancels them. cancelRejectedJobs :: JQStatus -> ConfigData -> Set FilterRule -> IO () cancelRejectedJobs qstate cfg filters = do enqueuedJobs <- map jJob . qEnqueued <$> readIORef (jqJobs qstate) -- Determine which jobs are rejected. let jobsToCancel = [ (job, fr) | job <- enqueuedJobs , Just fr <- [applyingFilter filters job] , frAction fr == Reject ] -- Cancel them. qDir <- queueDir forM_ jobsToCancel $ \(job, fr) -> do let jid = qjId job logDebug $ "Cancelling job " ++ show (fromJobId jid) ++ " because it was REJECTed by filter rule " ++ uuidOf fr -- First dequeue, then cancel. dequeueResult <- dequeueJob qstate jid case dequeueResult of Ok True -> do now <- currentTimestamp r <- runResultT $ writeAndReplicateJob cfg qDir (cancelQueuedJob now job) case r of Ok _ -> return () Bad err -> logError $ "Failed to write config when cancelling job: " ++ err Ok False -> do logDebug $ "Job " ++ show (fromJobId jid) ++ " not queued; trying to cancel directly" _ <- cancelJob False (jqLivelock qstate) jid -- sigTERM-kill only return () Bad s -> logError s -- passing a nonexistent job ID is an error here -- | Schedule jobs to be run. This is the IO wrapper around the -- pure `selectJobsToRun`. scheduleSomeJobs :: JQStatus -> IO () scheduleSomeJobs qstate = do cfgR <- readIORef (jqConfig qstate) case cfgR of Bad err -> do let msg = "Configuration unavailable: " ++ err logError msg Ok cfg -> do let filters = S.fromList . Map.elems . fromContainer $ configFilters cfg -- Check if jobs are rejected by a REJECT filter, and cancel them. cancelRejectedJobs qstate cfg filters -- Select the jobs to run. count <- getMaxRunningJobs qstate chosen <- atomicModifyIORef (jqJobs qstate) (selectJobsToRun count filters) let jobs = map jJob chosen unless (null chosen) . logInfo . (++) "Starting jobs: " . commaJoin $ map (show . fromJobId . qjId) jobs -- Attach the watcher. mapM_ (attachWatcher qstate) chosen -- Start the jobs. result <- JQ.startJobs (jqLivelock qstate) (jqForkLock qstate) jobs let badWith (x, Bad y) = Just (x, y) badWith _ = Nothing let failed = mapMaybe badWith $ zip chosen result unless (null failed) $ failJobs cfg qstate failed -- | Format the job queue status in a compact, human readable way. showQueue :: Queue -> String showQueue (Queue {qEnqueued=waiting, qRunning=running}) = let showids = show . map (fromJobId . qjId . jJob) in "Waiting jobs: " ++ showids waiting ++ "; running jobs: " ++ showids running -- | Check if a job died, and clean up if so. Return True, if -- the job was found dead. checkForDeath :: JQStatus -> JobWithStat -> IO Bool checkForDeath state jobWS = do let job = jJob jobWS jid = qjId job sjid = show $ fromJobId jid livelock = qjLivelock job logDebug $ "Livelock of job " ++ sjid ++ " is " ++ show livelock died <- maybe (return False) isDead . mfilter (/= jqLivelock state) $ livelock logDebug $ "Death of " ++ sjid ++ ": " ++ show died when died $ do logInfo $ "Detected death of job " ++ sjid -- if we manage to remove the job from the queue, we own the job file -- and can manipulate it. void . manipulateRunningJob state jid . runResultT $ do jobWS' <- mkResultT $ readJobFromDisk jid :: ResultG JobWithStat unless (jobFinalized . jJob $ jobWS') . void $ do -- If the job isn't finalized, but dead, add a corresponding -- failed status. now <- liftIO currentTimestamp qDir <- liftIO queueDir let reason = ( "gnt:daemon:luxid:deathdetection" , "detected death of job " ++ sjid , reasonTrailTimestamp now ) failedJob = failQueuedJob reason now $ jJob jobWS' cfg <- mkResultT . readIORef $ jqConfig state writeAndReplicateJob cfg qDir failedJob return died -- | Trigger job detection for the job with the given job id. -- Return True, if the job is dead. cleanupIfDead :: JQStatus -> JobId -> IO Bool cleanupIfDead state jid = do logDebug $ "Extra job-death detection for " ++ show (fromJobId jid) jobs <- readIORef (jqJobs state) let jobWS = find ((==) jid . qjId . jJob) $ qRunning jobs maybe (return True) (checkForDeath state) jobWS -- | Force the queue to check the state of all jobs. updateStatusAndScheduleSomeJobs :: JQStatus -> IO () updateStatusAndScheduleSomeJobs qstate = do jobs <- readIORef (jqJobs qstate) mapM_ (checkForDeath qstate) $ qRunning jobs jobs' <- readIORef (jqJobs qstate) mapM_ (updateJob qstate) $ qRunning jobs' cleanupFinishedJobs qstate jobs'' <- readIORef (jqJobs qstate) logInfo $ showQueue jobs'' scheduleSomeJobs qstate -- | Time-based watcher for updating the job queue. onTimeWatcher :: JQStatus -> IO () onTimeWatcher qstate = forever $ do threadDelay watchInterval logDebug "Job queue watcher timer fired" updateStatusAndScheduleSomeJobs qstate logDebug "Job queue watcher cycle finished" -- | Read a single, non-archived, job, specified by its id, from disk. readJobFromDisk :: JobId -> IO (Result JobWithStat) readJobFromDisk jid = do qdir <- queueDir let fpath = liveJobFile qdir jid logDebug $ "Reading " ++ fpath tryFstat <- try $ getFStat fpath :: IO (Either IOError FStat) let fstat = either (const nullFStat) id tryFstat loadResult <- JQ.loadJobFromDisk qdir False jid return $ liftM (JobWithStat Nothing fstat . fst) loadResult -- | Read all non-finalized jobs from disk. readJobsFromDisk :: IO [JobWithStat] readJobsFromDisk = do logInfo "Loading job queue" qdir <- queueDir eitherJids <- JQ.getJobIDs [qdir] let jids = genericResult (const []) JQ.sortJobIDs eitherJids jidsstring = commaJoin $ map (show . fromJobId) jids logInfo $ "Non-archived jobs on disk: " ++ jidsstring jobs <- mapM readJobFromDisk jids return $ justOk jobs -- | Set up the job scheduler. This will also start the monitoring -- of changes to the running jobs. initJQScheduler :: JQStatus -> IO () initJQScheduler qstate = do alljobs <- readJobsFromDisk let jobs = filter (not . jobFinalized . jJob) alljobs (running, queued) = partition (jobStarted . jJob) jobs modifyJobs qstate (onQueuedJobs (++ queued) . onRunningJobs (++ running)) jqjobs <- readIORef (jqJobs qstate) logInfo $ showQueue jqjobs scheduleSomeJobs qstate logInfo "Starting time-based job queue watcher" _ <- forkIO $ onTimeWatcher qstate return () -- | Enqueue new jobs. This will guarantee that the jobs will be executed -- eventually. enqueueNewJobs :: JQStatus -> [QueuedJob] -> IO () enqueueNewJobs state jobs = do logInfo . (++) "New jobs enqueued: " . commaJoin $ map (show . fromJobId . qjId) jobs let jobs' = map unreadJob jobs insertFn = insertBy (compare `on` fromJobId . qjId . jJob) addJobs oldjobs = foldl (flip insertFn) oldjobs jobs' modifyJobs state (onQueuedJobs addJobs) scheduleSomeJobs state -- | Pure function for removing a queued job from the job queue by -- atomicModifyIORef. The answer is Just the job if the job could be removed -- before being handed over to execution, Nothing if it already was started -- and a Bad result if the job is not found in the queue. rmJob :: JobId -> Queue -> (Queue, Result (Maybe QueuedJob)) rmJob jid q = let isJid = (jid ==) . qjId . jJob (found, queued') = partition isJid $ qEnqueued q isRunning = any isJid $ qRunning q sJid = (++) "Job " . show $ fromJobId jid in case (found, isRunning) of ([job], _) -> (q {qEnqueued = queued'}, Ok . Just $ jJob job) (_:_, _) -> (q, Bad $ "Queue in inconsistent state." ++ sJid ++ " queued multiple times") (_, True) -> (q, Ok Nothing) _ -> (q, Bad $ sJid ++ " not found in queue") -- | Try to remove a queued job from the job queue. Return True, if -- the job could be removed from the queue before being handed over -- to execution, False if the job already started, and a Bad result -- if the job is unknown. dequeueJob :: JQStatus -> JobId -> IO (Result Bool) dequeueJob state jid = do result <- atomicModifyIORef (jqJobs state) $ rmJob jid let result' = fmap isJust result logDebug $ "Result of dequeing job " ++ show (fromJobId jid) ++ " is " ++ show result' return result' -- | Change the priority of a queued job (once the job is handed over -- to execution, the job itself needs to be informed). To avoid the -- job being started unmodified, it is temporarily unqueued during the -- change. Return the modified job, if the job's priority was sucessfully -- modified, Nothing, if the job already started, and a Bad value, if the job -- is unkown. setJobPriority :: JQStatus -> JobId -> Int -> IO (Result (Maybe QueuedJob)) setJobPriority state jid prio = runResultT $ do maybeJob <- mkResultT . atomicModifyIORef (jqJobs state) $ rmJob jid case maybeJob of Nothing -> return Nothing Just job -> do let job' = changeJobPriority prio job qDir <- liftIO queueDir mkResultT $ writeJobToDisk qDir job' liftIO $ enqueueNewJobs state [job'] return $ Just job' -- | Given old and new configs, determines if the changes between them should -- trigger the scheduler to run. configChangeNeedsRescheduling :: ConfigData -> ConfigData -> Bool configChangeNeedsRescheduling old new = -- Trigger rescheduling if any of the following change: (((/=) `on` configFilters) old new || -- filters ((/=) `on` clusterMaxRunningJobs . configCluster) old new -- run queue length ) ganeti-3.1.0~rc2/src/Ganeti/JQScheduler/000075500000000000000000000000001476477700300177535ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/JQScheduler/Filtering.hs000064400000000000000000000224161476477700300222370ustar00rootroot00000000000000{-# LANGUAGE TupleSections, NamedFieldPuns, ScopedTypeVariables, Rank2Types, GADTs #-} {-| Filtering of jobs for the Ganeti job queue. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.JQScheduler.Filtering ( applyingFilter , jobFiltering -- * For testing only , matchPredicate , matches ) where import qualified Data.ByteString as BS import Data.List import Data.Maybe import qualified Data.Map as Map import Data.Set (Set) import qualified Data.Set as Set import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Errors import Ganeti.Lens hiding (chosen) import Ganeti.JQScheduler.Types import Ganeti.JQueue (QueuedJob(..)) import Ganeti.JQueue.Lens import Ganeti.JSON (nestedAccessByKeyDotted) import Ganeti.Objects (FilterRule(..), FilterAction(..), FilterPredicate(..), filterRuleOrder) import Ganeti.OpCodes (OpCode) import Ganeti.OpCodes.Lens import Ganeti.Query.Language import Ganeti.Query.Filter (evaluateFilterM, evaluateFilterJSON, Comparator, FilterOp(..), toCompFun) import Ganeti.SlotMap import Ganeti.Types (JobId(..), ReasonElem) -- | Accesses a field of the JSON representation of an `OpCode` using a dotted -- accessor (like @"a.b.c"@). accessOpCodeField :: OpCode -> String -> ErrorResult J.JSValue accessOpCodeField opc s = case nestedAccessByKeyDotted s (J.showJSON opc) of J.Ok x -> Ok x J.Error e -> Bad . ParameterError $ e -- | All `OpCode`s of a job. opCodesOf :: QueuedJob -> [OpCode] opCodesOf job = job ^.. qjOpsL . traverse . qoInputL . validOpCodeL . metaOpCodeL -- | All `ReasonElem`s of a job. reasonsOf :: QueuedJob -> [ReasonElem] reasonsOf job = job ^.. qjOpsL . traverse . qoInputL . validOpCodeL . metaParamsL . opReasonL . traverse -- | Like `evaluateFilterM`, but allowing only `Comparator` operations; -- all other filter language operations are evaluated as `False`. -- -- The passed function is supposed to return `Just True/False` depending -- on whether the comparing operation succeeds or not, and `Nothing` if -- the comparison itself is invalid (e.g. comparing to a field that doesn't -- exist). evaluateFilterComparator :: (Ord field) => Filter field -> (Comparator -> field -> FilterValue -> Maybe Bool) -> Bool evaluateFilterComparator fil opFun = fromMaybe False $ evaluateFilterM (\filterOp -> case filterOp of Comp cmp -> opFun (toCompFun cmp) _ -> \_ _ -> Nothing -- non-comparisons (become False) ) fil -- | Whether a `FilterPredicate` is true for a job. matchPredicate :: QueuedJob -> JobId -- ^ the watermark to compare against -- if the predicate references it -> FilterPredicate -> Bool matchPredicate job watermark predicate = case predicate of FPJobId fil -> let jid = qjId job jidInt = fromIntegral (fromJobId jid) in evaluateFilterComparator fil $ \comp field val -> case field of "id" -> case val of NumericValue i -> Just $ jidInt `comp` i QuotedString "watermark" -> Just $ jid `comp` watermark QuotedString _ -> Nothing _ -> Nothing FPOpCode fil -> let opMatches opc = genericResult (const False) id $ do jsonFilter <- traverse (accessOpCodeField opc) fil evaluateFilterJSON jsonFilter in any opMatches (opCodesOf job) FPReason fil -> let reasonMatches (source, reason, timestamp) = evaluateFilterComparator fil $ \comp field val -> case field of "source" -> Just $ QuotedString source `comp` val "reason" -> Just $ QuotedString reason `comp` val "timestamp" -> Just $ NumericValue timestamp `comp` val _ -> Nothing in any reasonMatches (reasonsOf job) -- | Whether all predicates of the filter rule are true for the job. matches :: QueuedJob -> FilterRule -> Bool matches job FilterRule{ frPredicates, frWatermark } = all (matchPredicate job frWatermark) frPredicates -- | Filters need to be processed in the order as given by the spec; -- see `filterRuleOrder`. orderFilters :: Set FilterRule -> [FilterRule] orderFilters = sortBy filterRuleOrder . Set.toList -- | Finds the first filter whose predicates all match the job and whose -- action is not `Continue`. This is the /applying/ filter. applyingFilter :: Set FilterRule -> QueuedJob -> Maybe FilterRule applyingFilter filters job = -- Skip over all `Continue`s, to the first filter that matches. find ((Continue /=) . frAction) . filter (matches job) . orderFilters $ filters -- | SlotMap for filter rule rate limiting, having `FilterRule` UUIDs as keys. type RateLimitSlotMap = SlotMap BS.ByteString -- We would prefer FilterRule here but that has no Ord instance (yet). -- | State to be accumulated while traversing filters. data FilterChainState = FilterChainState { rateLimitSlotMap :: RateLimitSlotMap -- ^ counts } deriving (Eq, Ord, Show) -- | Update a `FilterChainState` if the given `CountMap` fits into its -- filtering SlotsMap. tryFitSlots :: FilterChainState -> CountMap BS.ByteString -> Maybe FilterChainState tryFitSlots st@FilterChainState{ rateLimitSlotMap = slotMap } countMap = if slotMap `hasSlotsFor` countMap then Just st{ rateLimitSlotMap = slotMap `occupySlots` countMap } else Nothing -- | For a given job queue and set of filters, calculates how many rate -- limiting filter slots are available and how many are taken by running jobs -- in the queue. queueRateLimitSlotMap :: Queue -> Set FilterRule -> RateLimitSlotMap queueRateLimitSlotMap queue filters = let -- Rate limiting slots for each filter, with 0 occupied count each -- (limits only). emptyFilterSlots = Map.fromList [ (uuid, Slot 0 n) | FilterRule{ frUuid = uuid , frAction = RateLimit n } <- Set.toList filters ] -- How many rate limiting slots are taken by the jobs currently running -- in the queue jobs (counts only). -- A job takes a slot of a RateLimit filter if that filter is the first -- one that matches for the job. runningJobSlots = Map.fromListWith (+) [ (frUuid, 1) | Just FilterRule{ frUuid, frAction = RateLimit _ } <- map (applyingFilter filters . jJob) $ qRunning queue ++ qManipulated queue ] in -- Fill limits from above with counts from above. emptyFilterSlots `occupySlots` runningJobSlots -- | Implements job filtering as specified in `doc/design-optables.rst`. -- -- Importantly, the filter that *applies* is the first one of which all -- predicates match; this is implemented in `applyingFilter`. -- -- The initial `FilterChainState` is currently not cached across -- `selectJobsToRun` invocations because the number of running jobs is -- typically small (< 100). jobFiltering :: Queue -> Set FilterRule -> [JobWithStat] -> [JobWithStat] jobFiltering queue filters = let processFilters :: FilterChainState -> JobWithStat -> (FilterChainState, Maybe JobWithStat) processFilters state job = case applyingFilter filters (jJob job) of Nothing -> (state, Just job) -- no filter applies, accept job Just FilterRule{ frUuid, frAction } -> case frAction of Accept -> (state, Just job) Continue -> (state, Just job) Pause -> (state, Nothing) Reject -> (state, Nothing) RateLimit _ -> -- A matching job takes 1 slot. let jobSlots = Map.fromList [(frUuid, 1)] in case tryFitSlots state jobSlots of Nothing -> (state, Nothing) Just state' -> (state', Just job) in catMaybes . snd . mapAccumL processFilters FilterChainState { rateLimitSlotMap = queueRateLimitSlotMap queue filters } ganeti-3.1.0~rc2/src/Ganeti/JQScheduler/ReasonRateLimiting.hs000064400000000000000000000137471476477700300240630ustar00rootroot00000000000000{-# LANGUAGE TupleSections #-} {-| Ad-hoc rate limiting for the JQScheduler based on reason trails. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.JQScheduler.ReasonRateLimiting ( reasonRateLimit -- * For testing only , parseReasonRateLimit , countMapFromJob , slotMapFromJobs ) where import Control.Monad.Fail (MonadFail) import Data.List import Data.Maybe import qualified Data.Map as Map import Ganeti.Lens hiding (chosen) import Ganeti.JQScheduler.Types import Ganeti.JQueue (QueuedJob(..)) import Ganeti.JQueue.Lens import Ganeti.OpCodes.Lens import Ganeti.SlotMap import Ganeti.Utils -- | Ad-hoc rate limiting buckets are identified by the /combination/ -- `REASONSTRING:n`, so "mybucket:3" and "mybucket:4" are /different/ buckets. type AdHocReasonKey = String -- | Parses an ad-hoc rate limit from a reason trail, as defined under -- "Ad-Hoc Rate Limiting" in `doc/design-optables.rst`. -- -- The parse succeeds only on reasons of form `rate-limit:n:REASONSTRING` -- where `n` is a positive integer and `REASONSTRING` is an arbitrary -- string (may include spaces). parseReasonRateLimit :: (MonadFail m) => String -> m (String, Int) parseReasonRateLimit reason = case sepSplit ':' reason of "rate-limit":nStr:rest | Just n <- readMaybe nStr , n > 0 -> return (intercalate ":" (nStr:rest), n) _ -> fail $ "'" ++ reason ++ "' is not a valid ad-hoc rate limit reason" -- | Computes the bucket slots required by a job, also extracting how many -- slots are available from the reason rate limits in the job reason trails. -- -- A job can have multiple `OpCode`s, and the `ReasonTrail`s -- can be different for each `OpCode`. The `OpCode`s of a job are -- run sequentially, so a job can only take 1 slot. -- Thus a job takes part in a set of buckets, requiring 1 slot in -- each of them. labelCountMapFromJob :: QueuedJob -> CountMap (String, Int) labelCountMapFromJob job = let reasonsStrings = job ^.. qjOpsL . traverse . qoInputL . validOpCodeL . metaParamsL . opReasonL . traverse . _2 buckets = ordNub . mapMaybe parseReasonRateLimit $ reasonsStrings -- Buckets are already unique from `ordNub`. in Map.fromList $ map (, 1) buckets -- | Computes the bucket slots required by a job. countMapFromJob :: QueuedJob -> CountMap AdHocReasonKey countMapFromJob = Map.mapKeys (\(str, n) -> str ++ ":" ++ show n) . labelCountMapFromJob -- | Map of how many slots are in use for a given bucket, for a list of jobs. -- The slot limits are taken from the ad-hoc reason rate limiting strings. slotMapFromJobs :: [QueuedJob] -> SlotMap AdHocReasonKey slotMapFromJobs jobs = Map.mapKeys (\(str, n) -> str ++ ":" ++ show n) . Map.mapWithKey (\(_str, limit) occup -> Slot occup limit) . Map.unionsWith (+) . map labelCountMapFromJob $ jobs -- | Like `slotMapFromJobs`, but setting all occupation counts to 0. -- Useful to find what the bucket limits of a set of jobs are. unoccupiedSlotMapFromJobs :: [QueuedJob] -> SlotMap AdHocReasonKey unoccupiedSlotMapFromJobs = Map.map (\s -> s{ slotOccupied = 0 }) . slotMapFromJobs -- | Implements ad-hoc rate limiting using the reason trail as specified -- in `doc/design-optables.rst`. -- -- Reasons of form `rate-limit:n:REASONSTRING` define buckets that limit -- how many jobs with that reason can be running at the same time to -- a positive integer n of available slots. -- -- The used buckets map is currently not cached across `selectJobsToRun` -- invocations because the number of running jobs is typically small -- (< 100). reasonRateLimit :: Queue -> [JobWithStat] -> [JobWithStat] reasonRateLimit queue jobs = let -- For the purpose of rate limiting, manipulated jobs count as running. running = map jJob $ qRunning queue ++ qManipulated queue candidates = map jJob jobs -- Reason rate limiting slot map of the jobs running in the queue. -- All jobs determine the reason buckets, but only running jobs count -- to the initial limits. initSlotMap = unoccupiedSlotMapFromJobs (running ++ candidates) `occupySlots` toCountMap (slotMapFromJobs running) -- A job can be run (fits) if all buckets it takes part in have -- a free slot. If yes, accept the job and update the slotMap. -- Note: If the slotMap is overfull in some slots, but the job -- doesn't take part in any of those, it is to be accepted. accumFittingJobs slotMap job = let jobBuckets = countMapFromJob (jJob job) in if slotMap `hasSlotsFor` jobBuckets then (slotMap `occupySlots` jobBuckets, Just job) -- job fits else (slotMap, Nothing) -- job doesn't fit in catMaybes . snd . mapAccumL accumFittingJobs initSlotMap $ jobs ganeti-3.1.0~rc2/src/Ganeti/JQScheduler/Types.hs000064400000000000000000000044031476477700300214140ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, BangPatterns #-} {-| Types for the JQScheduler. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.JQScheduler.Types where import System.INotify import Ganeti.JQueue as JQ import Ganeti.Lens hiding (chosen) import Ganeti.Utils data JobWithStat = JobWithStat { jINotify :: Maybe INotify , jStat :: FStat , jJob :: !QueuedJob } deriving (Eq, Show) $(makeCustomLenses' ''JobWithStat ['jJob]) -- | A job without `INotify` and `FStat`. nullJobWithStat :: QueuedJob -> JobWithStat nullJobWithStat = JobWithStat Nothing nullFStat data Queue = Queue { qEnqueued :: ![JobWithStat] , qRunning :: ![JobWithStat] , qManipulated :: ![JobWithStat] -- ^ running jobs that are -- being manipulated by -- some thread } deriving (Eq, Show) $(makeCustomLenses ''Queue) ganeti-3.1.0~rc2/src/Ganeti/JQueue.hs000064400000000000000000000733741476477700300173520ustar00rootroot00000000000000{-| Implementation of the job queue. -} {- Copyright (C) 2010, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.JQueue ( queuedOpCodeFromMetaOpCode , queuedJobFromOpCodes , changeOpCodePriority , changeJobPriority , cancelQueuedJob , failQueuedJob , fromClockTime , noTimestamp , currentTimestamp , advanceTimestamp , reasonTrailTimestamp , setReceivedTimestamp , extendJobReasonTrail , getJobDependencies , opStatusFinalized , extractOpSummary , calcJobStatus , jobStarted , jobFinalized , jobArchivable , calcJobPriority , jobFileName , liveJobFile , archivedJobFile , determineJobDirectories , getJobIDs , sortJobIDs , loadJobFromDisk , noSuchJob , readSerialFromDisk , allocateJobIds , allocateJobId , writeJobToDisk , replicateManyJobs , writeAndReplicateJob , isQueueOpen , startJobs , cancelJob , tellJobPriority , notifyJob , waitUntilJobExited , queueDirPermissions , archiveJobs -- re-export , Timestamp , InputOpCode(..) , QueuedOpCode(..) , QueuedJob(..) ) where import Control.Applicative (liftA2, (<|>)) import Control.Arrow (first, second) import Control.Concurrent (forkIO, threadDelay) import Control.Exception import Control.Lens (over) import Control.Monad import Control.Monad.Fail (MonadFail) import Control.Monad.Fix import Control.Monad.IO.Class import Control.Monad.Trans (lift) import Control.Monad.Trans.Maybe import Data.List import Data.Maybe import Data.Ord (comparing) -- workaround what seems to be a bug in ghc 7.4's TH shadowing code import Prelude hiding (id, log) import System.Directory import System.FilePath import System.IO.Error (isDoesNotExistError) import System.Posix.Files import System.Posix.Signals (sigHUP, sigTERM, sigUSR1, sigKILL, signalProcess) import System.Posix.Types (ProcessID) import System.Time (ClockTime(TOD), getClockTime) import Ganeti.Utils.Time (TimeDiff(..), addToClockTime, noTimeDiff) import qualified System.Time as Time import qualified Text.JSON import Text.JSON.Types import Ganeti.BasicTypes import qualified Ganeti.Config as Config import qualified Ganeti.Constants as C import Ganeti.Errors (ErrorResult, ResultG) import Ganeti.JQueue.Lens (qoInputL, validOpCodeL) import Ganeti.JQueue.Objects import Ganeti.JSON (fromJResult, fromObjWithDefault) import Ganeti.Logging import Ganeti.Luxi import Ganeti.Objects (ConfigData, Node) import Ganeti.OpCodes import Ganeti.OpCodes.Lens (metaParamsL, opReasonL) import Ganeti.Path import Ganeti.Query.Exec as Exec import Ganeti.Rpc (executeRpcCall, ERpcError, logRpcErrors, RpcCallJobqueueUpdate(..), RpcCallJobqueueRename(..)) import Ganeti.Runtime (GanetiDaemon(..), GanetiGroup(..), MiscGroup(..)) import Ganeti.Types import Ganeti.Utils import Ganeti.Utils.Atomic import Ganeti.Utils.Livelock (Livelock, isDead) import Ganeti.Utils.MVarLock import Ganeti.VCluster (makeVirtualPath) -- * Data types -- | Missing timestamp type. noTimestamp :: Timestamp noTimestamp = (-1, -1) -- | Obtain a Timestamp from a given clock time fromClockTime :: ClockTime -> Timestamp fromClockTime (TOD ctime pico) = (fromIntegral ctime, fromIntegral $ pico `div` 1000000) -- | Get the current time in the job-queue timestamp format. currentTimestamp :: IO Timestamp currentTimestamp = fromClockTime `liftM` getClockTime -- | From a given timestamp, obtain the timestamp of the -- time that is the given number of seconds later. advanceTimestamp :: Int -> Timestamp -> Timestamp advanceTimestamp = first . (+) -- | From an InputOpCode obtain the MetaOpCode, if any. toMetaOpCode :: InputOpCode -> [MetaOpCode] toMetaOpCode (ValidOpCode mopc) = [mopc] toMetaOpCode _ = [] -- | Invalid opcode summary. invalidOp :: String invalidOp = "INVALID_OP" -- | Tries to extract the opcode summary from an 'InputOpCode'. This -- duplicates some functionality from the 'opSummary' function in -- "Ganeti.OpCodes". extractOpSummary :: InputOpCode -> String extractOpSummary (ValidOpCode metaop) = opSummary $ metaOpCode metaop extractOpSummary (InvalidOpCode (JSObject o)) = case fromObjWithDefault (fromJSObject o) "OP_ID" ("OP_" ++ invalidOp) of Just s -> drop 3 s -- drop the OP_ prefix Nothing -> invalidOp extractOpSummary _ = invalidOp -- | Convenience function to obtain a QueuedOpCode from a MetaOpCode queuedOpCodeFromMetaOpCode :: MetaOpCode -> QueuedOpCode queuedOpCodeFromMetaOpCode op = QueuedOpCode { qoInput = ValidOpCode op , qoStatus = OP_STATUS_QUEUED , qoPriority = opSubmitPriorityToRaw . opPriority . metaParams $ op , qoLog = [] , qoResult = JSNull , qoStartTimestamp = Nothing , qoEndTimestamp = Nothing , qoExecTimestamp = Nothing } -- | From a job-id and a list of op-codes create a job. This is -- the pure part of job creation, as allocating a new job id -- lives in IO. queuedJobFromOpCodes :: (MonadFail m) => JobId -> [MetaOpCode] -> m QueuedJob queuedJobFromOpCodes jobid ops = do ops' <- mapM (`resolveDependencies` jobid) ops return QueuedJob { qjId = jobid , qjOps = map queuedOpCodeFromMetaOpCode ops' , qjReceivedTimestamp = Nothing , qjStartTimestamp = Nothing , qjEndTimestamp = Nothing , qjLivelock = Nothing , qjProcessId = Nothing } -- | Attach a received timestamp to a Queued Job. setReceivedTimestamp :: Timestamp -> QueuedJob -> QueuedJob setReceivedTimestamp ts job = job { qjReceivedTimestamp = Just ts } -- | Build a timestamp in the format expected by the reason trail (nanoseconds) -- starting from a JQueue Timestamp. reasonTrailTimestamp :: Timestamp -> Integer reasonTrailTimestamp (sec, micro) = let sec' = toInteger sec micro' = toInteger micro in sec' * 1000000000 + micro' * 1000 -- | Append an element to the reason trail of an input opcode. extendInputOpCodeReasonTrail :: JobId -> Timestamp -> Int -> InputOpCode -> InputOpCode extendInputOpCodeReasonTrail _ _ _ op@(InvalidOpCode _) = op extendInputOpCodeReasonTrail jid ts i (ValidOpCode vOp) = let metaP = metaParams vOp op = metaOpCode vOp trail = opReason metaP reasonSrc = opReasonSrcID op reasonText = "job=" ++ show (fromJobId jid) ++ ";index=" ++ show i reason = (reasonSrc, reasonText, reasonTrailTimestamp ts) trail' = trail ++ [reason] in ValidOpCode $ vOp { metaParams = metaP { opReason = trail' } } -- | Append an element to the reason trail of a queued opcode. extendOpCodeReasonTrail :: JobId -> Timestamp -> Int -> QueuedOpCode -> QueuedOpCode extendOpCodeReasonTrail jid ts i op = let inOp = qoInput op in op { qoInput = extendInputOpCodeReasonTrail jid ts i inOp } -- | Append an element to the reason trail of all the OpCodes of a queued job. extendJobReasonTrail :: QueuedJob -> QueuedJob extendJobReasonTrail job = let jobId = qjId job mTimestamp = qjReceivedTimestamp job -- This function is going to be called on QueuedJobs that already contain -- a timestamp. But for safety reasons we cannot assume mTimestamp will -- be (Just timestamp), so we use the value 0 in the extremely unlikely -- case this is not true. timestamp = fromMaybe (0, 0) mTimestamp in job { qjOps = zipWith (extendOpCodeReasonTrail jobId timestamp) [0..] $ qjOps job } -- | From a queued job obtain the list of jobs it depends on. getJobDependencies :: QueuedJob -> [JobId] getJobDependencies job = do op <- qjOps job mopc <- toMetaOpCode $ qoInput op dep <- fromMaybe [] . opDepends $ metaParams mopc getJobIdFromDependency dep -- | Change the priority of a QueuedOpCode, if it is not already -- finalized. changeOpCodePriority :: Int -> QueuedOpCode -> QueuedOpCode changeOpCodePriority prio op = if qoStatus op > OP_STATUS_RUNNING then op else op { qoPriority = prio } -- | Set the state of a QueuedOpCode to canceled. cancelOpCode :: Timestamp -> QueuedOpCode -> QueuedOpCode cancelOpCode now op = op { qoStatus = OP_STATUS_CANCELED, qoEndTimestamp = Just now } -- | Change the priority of a job, i.e., change the priority of the -- non-finalized opcodes. changeJobPriority :: Int -> QueuedJob -> QueuedJob changeJobPriority prio job = job { qjOps = map (changeOpCodePriority prio) $ qjOps job } -- | Transform a QueuedJob that has not been started into its canceled form. cancelQueuedJob :: Timestamp -> QueuedJob -> QueuedJob cancelQueuedJob now job = let ops' = map (cancelOpCode now) $ qjOps job in job { qjOps = ops', qjEndTimestamp = Just now } -- | Set the state of a QueuedOpCode to failed -- and set the Op result using the given reason message. failOpCode :: ReasonElem -> Timestamp -> QueuedOpCode -> QueuedOpCode failOpCode reason@(_, msg, _) now op = over (qoInputL . validOpCodeL . metaParamsL . opReasonL) (++ [reason]) op { qoStatus = OP_STATUS_ERROR , qoResult = Text.JSON.JSString . Text.JSON.toJSString $ msg , qoEndTimestamp = Just now } -- | Transform a QueuedJob that has not been started into its failed form. failQueuedJob :: ReasonElem -> Timestamp -> QueuedJob -> QueuedJob failQueuedJob reason now job = let ops' = map (failOpCode reason now) $ qjOps job in job { qjOps = ops', qjEndTimestamp = Just now } -- | Job file prefix. jobFilePrefix :: String jobFilePrefix = "job-" -- | Computes the filename for a given job ID. jobFileName :: JobId -> FilePath jobFileName jid = jobFilePrefix ++ show (fromJobId jid) -- | Parses a job ID from a file name. parseJobFileId :: (MonadFail m) => FilePath -> m JobId parseJobFileId path = case stripPrefix jobFilePrefix path of Nothing -> fail $ "Job file '" ++ path ++ "' doesn't have the correct prefix" Just suffix -> makeJobIdS suffix -- | Computes the full path to a live job. liveJobFile :: FilePath -> JobId -> FilePath liveJobFile rootdir jid = rootdir jobFileName jid -- | Computes the full path to an archives job. BROKEN. archivedJobFile :: FilePath -> JobId -> FilePath archivedJobFile rootdir jid = let subdir = show (fromJobId jid `div` C.jstoreJobsPerArchiveDirectory) in rootdir jobQueueArchiveSubDir subdir jobFileName jid -- | Map from opcode status to job status. opStatusToJob :: OpStatus -> JobStatus opStatusToJob OP_STATUS_QUEUED = JOB_STATUS_QUEUED opStatusToJob OP_STATUS_WAITING = JOB_STATUS_WAITING opStatusToJob OP_STATUS_SUCCESS = JOB_STATUS_SUCCESS opStatusToJob OP_STATUS_RUNNING = JOB_STATUS_RUNNING opStatusToJob OP_STATUS_CANCELING = JOB_STATUS_CANCELING opStatusToJob OP_STATUS_CANCELED = JOB_STATUS_CANCELED opStatusToJob OP_STATUS_ERROR = JOB_STATUS_ERROR -- | Computes a queued job's status. calcJobStatus :: QueuedJob -> JobStatus calcJobStatus QueuedJob { qjOps = ops } = extractOpSt (map qoStatus ops) JOB_STATUS_QUEUED True where terminalStatus OP_STATUS_ERROR = True terminalStatus OP_STATUS_CANCELING = True terminalStatus OP_STATUS_CANCELED = True terminalStatus _ = False softStatus OP_STATUS_SUCCESS = True softStatus OP_STATUS_QUEUED = True softStatus _ = False extractOpSt [] _ True = JOB_STATUS_SUCCESS extractOpSt [] d False = d extractOpSt (x:xs) d old_all | terminalStatus x = opStatusToJob x -- abort recursion | softStatus x = extractOpSt xs d new_all -- continue unchanged | otherwise = extractOpSt xs (opStatusToJob x) new_all where new_all = x == OP_STATUS_SUCCESS && old_all -- | Determine if a job has started jobStarted :: QueuedJob -> Bool jobStarted = (> JOB_STATUS_QUEUED) . calcJobStatus -- | Determine if a job is finalised. jobFinalized :: QueuedJob -> Bool jobFinalized = (> JOB_STATUS_RUNNING) . calcJobStatus -- | Determine if a job is finalized and its timestamp is before -- a given time. jobArchivable :: Timestamp -> QueuedJob -> Bool jobArchivable ts = liftA2 (&&) jobFinalized $ maybe False (< ts) . liftA2 (<|>) qjEndTimestamp qjStartTimestamp -- | Determine whether an opcode status is finalized. opStatusFinalized :: OpStatus -> Bool opStatusFinalized = (> OP_STATUS_RUNNING) -- | Compute a job's priority. calcJobPriority :: QueuedJob -> Int calcJobPriority QueuedJob { qjOps = ops } = helper . map qoPriority $ filter (not . opStatusFinalized . qoStatus) ops where helper [] = C.opPrioDefault helper ps = minimum ps -- | Log but ignore an 'IOError'. ignoreIOError :: a -> Bool -> String -> IOError -> IO a ignoreIOError a ignore_noent msg e = do unless (isDoesNotExistError e && ignore_noent) . logWarning $ msg ++ ": " ++ show e return a -- | Compute the list of existing archive directories. Note that I/O -- exceptions are swallowed and ignored. allArchiveDirs :: FilePath -> IO [FilePath] allArchiveDirs rootdir = do let adir = rootdir jobQueueArchiveSubDir contents <- getDirectoryContents adir `Control.Exception.catch` ignoreIOError [] False ("Failed to list queue directory " ++ adir) let fpaths = map (adir ) $ filter (not . ("." `isPrefixOf`)) contents filterM (\path -> liftM isDirectory (getFileStatus (adir path)) `Control.Exception.catch` ignoreIOError False True ("Failed to stat archive path " ++ path)) fpaths -- | Build list of directories containing job files. Note: compared to -- the Python version, this doesn't ignore a potential lost+found -- file. determineJobDirectories :: FilePath -> Bool -> IO [FilePath] determineJobDirectories rootdir archived = do other <- if archived then allArchiveDirs rootdir else return [] return $ rootdir:other -- | Computes the list of all jobs in the given directories. getJobIDs :: [FilePath] -> IO (GenericResult IOError [JobId]) getJobIDs = runResultT . liftM concat . mapM getDirJobIDs -- | Sorts the a list of job IDs. sortJobIDs :: [JobId] -> [JobId] sortJobIDs = sortBy (comparing fromJobId) -- | Computes the list of jobs in a given directory. getDirJobIDs :: FilePath -> ResultT IOError IO [JobId] getDirJobIDs path = withErrorLogAt WARNING ("Failed to list job directory " ++ path) . liftM (mapMaybe parseJobFileId) $ liftIO (getDirectoryContents path) -- | Reads the job data from disk. readJobDataFromDisk :: FilePath -> Bool -> JobId -> IO (Maybe (String, Bool)) readJobDataFromDisk rootdir archived jid = do let live_path = liveJobFile rootdir jid archived_path = archivedJobFile rootdir jid all_paths = if archived then [(live_path, False), (archived_path, True)] else [(live_path, False)] foldM (\state (path, isarchived) -> liftM (\r -> Just (r, isarchived)) (readFile path) `Control.Exception.catch` ignoreIOError state True ("Failed to read job file " ++ path)) Nothing all_paths -- | Failed to load job error. noSuchJob :: Result (QueuedJob, Bool) noSuchJob = Bad "Can't load job file" -- | Loads a job from disk. loadJobFromDisk :: FilePath -> Bool -> JobId -> IO (Result (QueuedJob, Bool)) loadJobFromDisk rootdir archived jid = do raw <- readJobDataFromDisk rootdir archived jid -- note: we need some stricness below, otherwise the wrapping in a -- Result will create too much lazyness, and not close the file -- descriptors for the individual jobs return $! case raw of Nothing -> noSuchJob Just (str, arch) -> liftM (\qj -> (qj, arch)) . fromJResult "Parsing job file" $ Text.JSON.decode str -- | Write a job to disk. writeJobToDisk :: FilePath -> QueuedJob -> IO (Result ()) writeJobToDisk rootdir job = do let filename = liveJobFile rootdir . qjId $ job content = Text.JSON.encode . Text.JSON.showJSON $ job tryAndLogIOError (atomicWriteFile filename content) ("Failed to write " ++ filename) Ok -- | Replicate a job to all master candidates. replicateJob :: FilePath -> [Node] -> QueuedJob -> IO [(Node, ERpcError ())] replicateJob rootdir mastercandidates job = do let filename = liveJobFile rootdir . qjId $ job content = Text.JSON.encode . Text.JSON.showJSON $ job filename' <- makeVirtualPath filename callresult <- executeRpcCall mastercandidates $ RpcCallJobqueueUpdate filename' content let result = map (second (() <$)) callresult _ <- logRpcErrors result return result -- | Replicate many jobs to all master candidates. replicateManyJobs :: FilePath -> [Node] -> [QueuedJob] -> IO () replicateManyJobs rootdir mastercandidates = mapM_ (replicateJob rootdir mastercandidates) -- | Writes a job to a file and replicates it to master candidates. writeAndReplicateJob :: (Error e) => ConfigData -> FilePath -> QueuedJob -> ResultT e IO [(Node, ERpcError ())] writeAndReplicateJob cfg rootdir job = do mkResultT $ writeJobToDisk rootdir job liftIO $ replicateJob rootdir (Config.getMasterCandidates cfg) job -- | Read the job serial number from disk. readSerialFromDisk :: IO (Result JobId) readSerialFromDisk = do filename <- jobQueueSerialFile tryAndLogIOError (readFile filename) "Failed to read serial file" (makeJobIdS . rStripSpace) -- | Allocate new job ids. -- To avoid races while accessing the serial file, the threads synchronize -- over a lock, as usual provided by a Lock. allocateJobIds :: [Node] -> Lock -> Int -> IO (Result [JobId]) allocateJobIds mastercandidates lock n = if n <= 0 then if n == 0 then return $ Ok [] else return . Bad $ "Can only allocate non-negative number of job ids" else withLock lock $ do rjobid <- readSerialFromDisk case rjobid of Bad s -> return . Bad $ s Ok jid -> do let current = fromJobId jid serial_content = show (current + n) ++ "\n" serial <- jobQueueSerialFile write_result <- try $ atomicWriteFile serial serial_content :: IO (Either IOError ()) case write_result of Left e -> do let msg = "Failed to write serial file: " ++ show e logError msg return . Bad $ msg Right () -> do serial' <- makeVirtualPath serial _ <- executeRpcCall mastercandidates $ RpcCallJobqueueUpdate serial' serial_content return $ mapM makeJobId [(current+1)..(current+n)] -- | Allocate one new job id. allocateJobId :: [Node] -> Lock -> IO (Result JobId) allocateJobId mastercandidates lock = do jids <- allocateJobIds mastercandidates lock 1 return (jids >>= monadicThe "Failed to allocate precisely one Job ID") -- | Decide if job queue is open isQueueOpen :: IO Bool isQueueOpen = liftM not (jobQueueDrainFile >>= doesFileExist) -- | Start enqueued jobs by executing the Python code. startJobs :: Livelock -- ^ Luxi's livelock path -> Lock -- ^ lock for forking new processes -> [QueuedJob] -- ^ the list of jobs to start -> IO [ErrorResult QueuedJob] startJobs luxiLivelock forkLock jobs = do qdir <- queueDir let updateJob job llfile = void . mkResultT . writeJobToDisk qdir $ job { qjLivelock = Just llfile } let runJob job = withLock forkLock $ do (llfile, _) <- Exec.forkJobProcess job luxiLivelock (updateJob job) return $ job { qjLivelock = Just llfile } mapM (runResultT . runJob) jobs -- | Try to prove that a queued job is dead. This function needs to know -- the livelock of the caller (i.e., luxid) to avoid considering a job dead -- that is in the process of forking off. isQueuedJobDead :: MonadIO m => Livelock -> QueuedJob -> m Bool isQueuedJobDead ownlivelock = maybe (return False) (liftIO . isDead) . mfilter (/= ownlivelock) . qjLivelock -- | Waits for a job's process to exit waitUntilJobExited :: Livelock -- ^ LuxiD's own livelock -> QueuedJob -- ^ the job to wait for -> Int -- ^ timeout in milliseconds -> ResultG (Bool, String) waitUntilJobExited ownlivelock job tmout = do let sleepDelay = 100 :: Int -- ms jName = ("Job " ++) . show . fromJobId . qjId $ job logDebug $ "Waiting for " ++ jName ++ " to exit" start <- liftIO getClockTime let tmoutPicosec :: Integer tmoutPicosec = fromIntegral $ tmout * 1000 * 1000 * 1000 wait = noTimeDiff { tdPicosec = tmoutPicosec } deadline = addToClockTime wait start liftIO . fix $ \loop -> do -- fail if the job is in the startup phase or has no livelock dead <- maybe (fail $ jName ++ " has not yet started up") (liftIO . isDead) . mfilter (/= ownlivelock) . qjLivelock $ job curtime <- getClockTime let elapsed = Time.timeDiffToString $ Time.diffClockTimes curtime start case dead of True -> return (True, jName ++ " process exited after " ++ elapsed) _ | curtime < deadline -> threadDelay (sleepDelay * 1000) >> loop _ -> fail $ jName ++ " still running after " ++ elapsed -- | Waits for a job ordered to cancel to react, and returns whether it was -- canceled, and a user-intended description of the reason. waitForJobCancelation :: JobId -> Int -> ResultG (Bool, String) waitForJobCancelation jid tmout = do qDir <- liftIO queueDir let jobfile = liveJobFile qDir jid load = liftM fst <$> loadJobFromDisk qDir False jid finalizedR = genericResult (const False) jobFinalized jobR <- liftIO $ watchFileBy jobfile tmout finalizedR load case calcJobStatus <$> jobR of Ok s | s == JOB_STATUS_CANCELED -> return (True, "Job successfully cancelled") | finalizedR jobR -> return (False, "Job exited before it could have been canceled,\ \ status " ++ show s) | otherwise -> return (False, "Job could not be canceled, status " ++ show s) Bad e -> failError $ "Can't read job status: " ++ e -- | Try to cancel a job that has already been handed over to execution, -- by terminating the process. cancelJob :: Bool -- ^ if True, use sigKILL instead of sigTERM -> Livelock -- ^ Luxi's livelock path -> JobId -- ^ the job to cancel -> IO (ErrorResult (Bool, String)) cancelJob kill luxiLivelock jid = runResultT $ do -- we can't terminate the job if it's just being started, so -- retry several times in such a case result <- runMaybeT . msum . flip map [0..5 :: Int] $ \tryNo -> do -- if we're retrying, sleep for some time when (tryNo > 0) . liftIO . threadDelay $ 100000 * (2 ^ tryNo) -- first check if the job is alive so that we don't kill some other -- process by accident qDir <- liftIO queueDir (job, _) <- lift . mkResultT $ loadJobFromDisk qDir True jid let jName = ("Job " ++) . show . fromJobId . qjId $ job dead <- isQueuedJobDead luxiLivelock job case qjProcessId job of _ | dead -> return (True, jName ++ " has been already dead") Just pid -> do liftIO $ signalProcess (if kill then sigKILL else sigTERM) pid if not kill then if calcJobStatus job > JOB_STATUS_WAITING then return (False, "Job no longer waiting, can't cancel\ \ (informed it anyway)") else lift $ waitForJobCancelation jid C.luxiCancelJobTimeout else return (True, "SIGKILL send to the process") _ -> do logDebug $ jName ++ " in its startup phase, retrying" mzero return $ fromMaybe (False, "Timeout: job still in its startup phase") result -- | Inform a job that it is requested to change its priority. This is done -- by writing the new priority to a file and sending SIGUSR1. tellJobPriority :: Livelock -- ^ Luxi's livelock path -> JobId -- ^ the job to inform -> Int -- ^ the new priority -> IO (ErrorResult (Bool, String)) tellJobPriority luxiLivelock jid prio = runResultT $ do let jidS = show $ fromJobId jid jName = "Job " ++ jidS mDir <- liftIO luxidMessageDir let prioFile = mDir jidS ++ ".prio" liftIO . atomicWriteFile prioFile $ show prio qDir <- liftIO queueDir (job, _) <- mkResultT $ loadJobFromDisk qDir True jid dead <- isQueuedJobDead luxiLivelock job case qjProcessId job of _ | dead -> do liftIO $ removeFile prioFile return (False, jName ++ " is dead") Just pid -> do liftIO $ signalProcess sigUSR1 pid return (True, jName ++ " with pid " ++ show pid ++ " signaled") _ -> return (False, jName ++ "'s pid unknown") -- | Notify a job that something relevant happened, e.g., a lock became -- available. We do this by sending sigHUP to the process. notifyJob :: ProcessID -> IO (ErrorResult ()) notifyJob pid = runResultT $ do logDebug $ "Signalling process " ++ show pid liftIO $ signalProcess sigHUP pid -- | Permissions for the archive directories. queueDirPermissions :: FilePermissions queueDirPermissions = FilePermissions { fpOwner = Just GanetiMasterd , fpGroup = Just $ ExtraGroup DaemonsGroup , fpPermissions = 0o0750 } -- | Try, at most until the given endtime, to archive some of the given -- jobs, if they are older than the specified cut-off time; also replicate -- archival of the additional jobs. Return the pair of the number of jobs -- archived, and the number of jobs remaining int he queue, asuming the -- given numbers about the not considered jobs. archiveSomeJobsUntil :: ([JobId] -> IO ()) -- ^ replication function -> FilePath -- ^ queue root directory -> ClockTime -- ^ Endtime -> Timestamp -- ^ cut-off time for archiving jobs -> Int -- ^ number of jobs alread archived -> [JobId] -- ^ Additional jobs to replicate -> [JobId] -- ^ List of job-ids still to consider -> IO (Int, Int) archiveSomeJobsUntil replicateFn _ _ _ arch torepl [] = do unless (null torepl) . (>> return ()) . forkIO $ replicateFn torepl return (arch, 0) archiveSomeJobsUntil replicateFn qDir endt cutt arch torepl (jid:jids) = do let archiveMore = archiveSomeJobsUntil replicateFn qDir endt cutt continue = archiveMore arch torepl jids jidname = show $ fromJobId jid time <- getClockTime if time >= endt then do _ <- forkIO $ replicateFn torepl return (arch, length (jid:jids)) else do logDebug $ "Inspecting job " ++ jidname ++ " for archival" loadResult <- loadJobFromDisk qDir False jid case loadResult of Bad _ -> continue Ok (job, _) -> if jobArchivable cutt job then do let live = liveJobFile qDir jid archive = archivedJobFile qDir jid renameResult <- safeRenameFile queueDirPermissions live archive case renameResult of Bad s -> do logWarning $ "Renaming " ++ live ++ " to " ++ archive ++ " failed unexpectedly: " ++ s continue Ok () -> do let torepl' = jid:torepl if length torepl' >= 10 then do _ <- forkIO $ replicateFn torepl' archiveMore (arch + 1) [] jids else archiveMore (arch + 1) torepl' jids else continue -- | Archive jobs older than the given time, but do not exceed the timeout for -- carrying out this task. archiveJobs :: ConfigData -- ^ cluster configuration -> Int -- ^ time the job has to be in the past in order -- to be archived -> Int -- ^ timeout -> [JobId] -- ^ jobs to consider -> IO (Int, Int) archiveJobs cfg age timeout jids = do now <- getClockTime qDir <- queueDir let endtime = addToClockTime (noTimeDiff { tdSec = fromIntegral timeout }) now cuttime = if age < 0 then noTimestamp else advanceTimestamp (- age) (fromClockTime now) mcs = Config.getMasterCandidates cfg replicateFn jobs = do let olds = map (liveJobFile qDir) jobs news = map (archivedJobFile qDir) jobs _ <- executeRpcCall mcs . RpcCallJobqueueRename $ zip olds news return () archiveSomeJobsUntil replicateFn qDir endtime cuttime 0 [] jids ganeti-3.1.0~rc2/src/Ganeti/JQueue/000075500000000000000000000000001476477700300170005ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/JQueue/Lens.hs000064400000000000000000000033531476477700300202410ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Lenses for job-queue objects -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.JQueue.Lens where import Control.Lens.Prism (Prism', prism') import Ganeti.JQueue.Objects import Ganeti.Lens (makeCustomLenses) import Ganeti.OpCodes (MetaOpCode) validOpCodeL :: Prism' InputOpCode MetaOpCode validOpCodeL = prism' ValidOpCode $ \op -> case op of ValidOpCode mop -> Just mop _ -> Nothing $(makeCustomLenses ''QueuedOpCode) $(makeCustomLenses ''QueuedJob) ganeti-3.1.0~rc2/src/Ganeti/JQueue/Objects.hs000064400000000000000000000071431476477700300207320ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, StandaloneDeriving #-} {-| Objects in the job queue. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.JQueue.Objects ( Timestamp , InputOpCode(..) , QueuedOpCode(..) , QueuedJob(..) ) where import Prelude hiding (id, log) import qualified Text.JSON import Text.JSON.Types import Ganeti.THH import Ganeti.THH.Field import Ganeti.Types import Ganeti.OpCodes -- | The ganeti queue timestamp type. It represents the time as the pair -- of seconds since the epoch and microseconds since the beginning of the -- second. type Timestamp = (Int, Int) -- | An input opcode. data InputOpCode = ValidOpCode MetaOpCode -- ^ OpCode was parsed successfully | InvalidOpCode JSValue -- ^ Invalid opcode deriving (Show, Eq, Ord) -- | JSON instance for 'InputOpCode', trying to parse it and if -- failing, keeping the original JSValue. instance Text.JSON.JSON InputOpCode where showJSON (ValidOpCode mo) = Text.JSON.showJSON mo showJSON (InvalidOpCode inv) = inv readJSON v = case Text.JSON.readJSON v of Text.JSON.Error _ -> return $ InvalidOpCode v Text.JSON.Ok mo -> return $ ValidOpCode mo $(buildObject "QueuedOpCode" "qo" [ simpleField "input" [t| InputOpCode |] , simpleField "status" [t| OpStatus |] , simpleField "result" [t| JSValue |] , defaultField [| [] |] $ simpleField "log" [t| [(Int, Timestamp, ELogType, JSValue)] |] , simpleField "priority" [t| Int |] , optionalNullSerField $ simpleField "start_timestamp" [t| Timestamp |] , optionalNullSerField $ simpleField "exec_timestamp" [t| Timestamp |] , optionalNullSerField $ simpleField "end_timestamp" [t| Timestamp |] ]) deriving instance Ord QueuedOpCode $(buildObject "QueuedJob" "qj" [ simpleField "id" [t| JobId |] , simpleField "ops" [t| [QueuedOpCode] |] , optionalNullSerField $ simpleField "received_timestamp" [t| Timestamp |] , optionalNullSerField $ simpleField "start_timestamp" [t| Timestamp |] , optionalNullSerField $ simpleField "end_timestamp" [t| Timestamp |] , optionalField $ simpleField "livelock" [t| FilePath |] , optionalField $ processIdField "process_id" ]) deriving instance Ord QueuedJob ganeti-3.1.0~rc2/src/Ganeti/JSON.hs000064400000000000000000000477011476477700300167200ustar00rootroot00000000000000{-# LANGUAGE TypeSynonymInstances, FlexibleInstances, TupleSections, GeneralizedNewtypeDeriving, DeriveTraversable #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| JSON utility functions. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.JSON ( fromJResult , fromJResultE , readJSONWithDesc , readEitherString , JSRecord , loadJSArray , fromObj , maybeFromObj , fromObjWithDefault , fromKeyValue , fromJVal , fromJValE , jsonHead , getMaybeJsonHead , getMaybeJsonElem , asJSObject , asObjectList , tryFromObj , arrayMaybeFromJVal , tryArrayMaybeFromObj , toArray , optionalJSField , optFieldsToObj , containerFromList , lookupContainer , alterContainerL , readContainer , mkUsedKeys , allUsedKeys , DictObject(..) , showJSONtoDict , readJSONfromDict , ArrayObject(..) , HasStringRepr(..) , GenericContainer(..) , emptyContainer , Container , MaybeForJSON(..) , TimeAsDoubleJSON(..) , Tuple5(..) , nestedAccessByKey , nestedAccessByKeyDotted , branchOnField , addField , maybeParseMap ) where import Control.Applicative import Control.DeepSeq import Control.Monad.Except (MonadError, throwError) import Control.Monad.Fail (MonadFail) import Control.Monad (liftM, (<=<)) import Control.Monad.Writer import qualified Data.ByteString as BS import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Foldable as F import qualified Data.Text as T import qualified Data.Traversable as F import Data.Maybe (fromMaybe, catMaybes) import qualified Data.Map as Map import qualified Data.Set as Set import qualified Data.Semigroup as Sem import System.Time (ClockTime(..)) import Text.Printf (printf) import qualified Text.JSON as J import qualified Text.JSON.Types as JT import Text.JSON.Pretty (pp_value) -- Note: this module should not import any Ganeti-specific modules -- beside BasicTypes and Compact, since it's used in THH which is used -- itself to build many other modules. import Ganeti.BasicTypes import Ganeti.Compat () -- Remove after we require >= 1.8.58 -- See: https://github.com/ndmitchell/hlint/issues/24 {-# ANN module "HLint: ignore Unused LANGUAGE pragma" #-} -- * JSON-related functions instance NFData J.JSValue where rnf J.JSNull = () rnf (J.JSBool b) = rnf b rnf (J.JSRational b r) = rnf b `seq` rnf r rnf (J.JSString s) = rnf $ J.fromJSString s rnf (J.JSArray a) = rnf a rnf (J.JSObject o) = rnf o instance (NFData a) => NFData (J.JSObject a) where rnf = rnf . J.fromJSObject -- | A type alias for a field of a JSRecord. type JSField = (String, J.JSValue) -- | A type alias for the list-based representation of J.JSObject. type JSRecord = [JSField] -- | Annotate @readJSON@ error messages with descriptions of what -- is being parsed into what. readJSONWithDesc :: (J.JSON a) => String -- ^ description of @a@ -> J.JSValue -- ^ input value -> J.Result a readJSONWithDesc name input = case J.readJSON input of J.Ok r -> J.Ok r J.Error e -> J.Error $ "Can't parse value for '" ++ name ++ "': " ++ e -- | Converts a JSON Result into a monadic value. fromJResult :: (Monad m, MonadFail m) => String -> J.Result a -> m a fromJResult s (J.Error x) = fail (s ++ ": " ++ x) fromJResult _ (J.Ok x) = return x -- | Converts a JSON Result into a MonadError value. fromJResultE :: (Error e, MonadError e m) => String -> J.Result a -> m a fromJResultE s (J.Error x) = throwError . strMsg $ s ++ ": " ++ x fromJResultE _ (J.Ok x) = return x -- | Tries to read a string from a JSON value. -- -- In case the value was not a string, we fail the read (in the -- context of the current monad. readEitherString :: (MonadFail m) => J.JSValue -> m String readEitherString v = case v of J.JSString s -> return $ J.fromJSString s _ -> fail "Wrong JSON type" -- | Converts a JSON message into an array of JSON objects. loadJSArray :: MonadFail m => String -- ^ Operation description (for error reporting) -> String -- ^ Input message -> m [J.JSObject J.JSValue] loadJSArray s = fromJResult s . J.decodeStrict -- | Helper function for missing-key errors buildNoKeyError :: JSRecord -> String -> String buildNoKeyError o k = printf "key '%s' not found, object contains only %s" k (show (map fst o)) -- | Reads the value of a key in a JSON object. fromObj :: (J.JSON a, MonadFail m) => JSRecord -> String -> m a fromObj o k = case lookup k o of Nothing -> fail $ buildNoKeyError o k Just val -> fromKeyValue k val -- | Reads the value of an optional key in a JSON object. Missing -- keys, or keys that have a \'null\' value, will be returned as -- 'Nothing', otherwise we attempt deserialisation and return a 'Just' -- value. maybeFromObj :: (J.JSON a, MonadFail m) => JSRecord -> String -> m (Maybe a) maybeFromObj o k = case lookup k o of Nothing -> return Nothing -- a optional key with value JSNull is the same as missing, since -- we can't convert it meaningfully anyway to a Haskell type, and -- the Python code can emit 'null' for optional values (depending -- on usage), and finally our encoding rules treat 'null' values -- as 'missing' Just J.JSNull -> return Nothing Just val -> liftM Just (fromKeyValue k val) -- | Reads the value of a key in a JSON object with a default if -- missing. Note that both missing keys and keys with value \'null\' -- will cause the default value to be returned. fromObjWithDefault :: (J.JSON a, MonadFail m) => JSRecord -> String -> a -> m a fromObjWithDefault o k d = liftM (fromMaybe d) $ maybeFromObj o k arrayMaybeFromJVal :: (J.JSON a, Monad m, MonadFail m) => J.JSValue -> m [Maybe a] arrayMaybeFromJVal (J.JSArray xs) = mapM parse xs where parse J.JSNull = return Nothing parse x = liftM Just $ fromJVal x arrayMaybeFromJVal v = fail $ "Expecting array, got '" ++ show (pp_value v) ++ "'" -- | Reads an array of optional items arrayMaybeFromObj :: (J.JSON a, MonadFail m) => JSRecord -> String -> m [Maybe a] arrayMaybeFromObj o k = case lookup k o of Just a -> arrayMaybeFromJVal a _ -> fail $ buildNoKeyError o k -- | Wrapper for arrayMaybeFromObj with better diagnostic tryArrayMaybeFromObj :: (J.JSON a) => String -- ^ Textual "owner" in error messages -> JSRecord -- ^ The object array -> String -- ^ The desired key from the object -> Result [Maybe a] tryArrayMaybeFromObj t o = annotateResult t . arrayMaybeFromObj o -- | Reads a JValue, that originated from an object key. fromKeyValue :: (J.JSON a, MonadFail m) => String -- ^ The key name -> J.JSValue -- ^ The value to read -> m a fromKeyValue k val = fromJResult (printf "key '%s'" k) (J.readJSON val) -- | Small wrapper over readJSON. fromJVal :: (Monad m, MonadFail m, J.JSON a) => J.JSValue -> m a fromJVal v = case J.readJSON v of J.Error s -> fail ("Cannot convert value '" ++ show (pp_value v) ++ "', error: " ++ s) J.Ok x -> return x -- | Small wrapper over 'readJSON' for 'MonadError'. fromJValE :: (Error e, MonadError e m, J.JSON a) => J.JSValue -> m a fromJValE v = case J.readJSON v of J.Error s -> throwError . strMsg $ "Cannot convert value '" ++ show (pp_value v) ++ "', error: " ++ s J.Ok x -> return x -- | Helper function that returns Null or first element of the list. jsonHead :: (J.JSON b) => [a] -> (a -> b) -> J.JSValue jsonHead [] _ = J.JSNull jsonHead (x:_) f = J.showJSON $ f x -- | Helper for extracting Maybe values from a possibly empty list. getMaybeJsonHead :: (J.JSON b) => [a] -> (a -> Maybe b) -> J.JSValue getMaybeJsonHead [] _ = J.JSNull getMaybeJsonHead (x:_) f = maybe J.JSNull J.showJSON (f x) -- | Helper for extracting Maybe values from a list that might be too short. getMaybeJsonElem :: (J.JSON b) => [a] -> Int -> (a -> Maybe b) -> J.JSValue getMaybeJsonElem [] _ _ = J.JSNull getMaybeJsonElem xs 0 f = getMaybeJsonHead xs f getMaybeJsonElem (_:xs) n f | n < 0 = J.JSNull | otherwise = getMaybeJsonElem xs (n - 1) f -- | Converts a JSON value into a JSON object. asJSObject :: (Monad m, MonadFail m) => J.JSValue -> m (J.JSObject J.JSValue) asJSObject (J.JSObject a) = return a asJSObject _ = fail "not an object" -- | Coneverts a list of JSON values into a list of JSON objects. asObjectList :: (Monad m, MonadFail m) => [J.JSValue] -> m [J.JSObject J.JSValue] asObjectList = mapM asJSObject -- | Try to extract a key from an object with better error reporting -- than fromObj. tryFromObj :: (J.JSON a) => String -- ^ Textual "owner" in error messages -> JSRecord -- ^ The object array -> String -- ^ The desired key from the object -> Result a tryFromObj t o = annotateResult t . fromObj o -- | Ensure a given JSValue is actually a JSArray. toArray :: (Monad m, MonadFail m) => J.JSValue -> m [J.JSValue] toArray (J.JSArray arr) = return arr toArray o = fail $ "Invalid input, expected array but got " ++ show (pp_value o) -- | Creates a Maybe JSField. If the value string is Nothing, the JSField -- will be Nothing as well. optionalJSField :: (J.JSON a) => String -> Maybe a -> Maybe JSField optionalJSField name (Just value) = Just (name, J.showJSON value) optionalJSField _ Nothing = Nothing -- | Creates an object with all the non-Nothing fields of the given list. optFieldsToObj :: [Maybe JSField] -> J.JSValue optFieldsToObj = J.makeObj . catMaybes -- * Container type (special type for JSON serialisation) -- | Class of types that can be converted from Strings. This is -- similar to the 'Read' class, but it's using a different -- serialisation format, so we have to define a separate class. Mostly -- useful for custom key types in JSON dictionaries, which have to be -- backed by strings. class HasStringRepr a where fromStringRepr :: (MonadFail m) => String -> m a toStringRepr :: a -> String -- | Trivial instance 'HasStringRepr' for 'String'. instance HasStringRepr String where fromStringRepr = return toStringRepr = id -- | The container type, a wrapper over Data.Map newtype GenericContainer a b = GenericContainer { fromContainer :: Map.Map a b } deriving (Show, Eq, Ord, Functor, F.Foldable, F.Traversable) instance (NFData a, NFData b) => NFData (GenericContainer a b) where rnf = rnf . Map.toList . fromContainer -- | The empty container. emptyContainer :: GenericContainer a b emptyContainer = GenericContainer Map.empty -- | Type alias for string keys. type Container = GenericContainer BS.ByteString instance HasStringRepr BS.ByteString where fromStringRepr = return . UTF8.fromString toStringRepr = UTF8.toString -- | Creates a GenericContainer from a list of key-value pairs. containerFromList :: Ord a => [(a,b)] -> GenericContainer a b containerFromList = GenericContainer . Map.fromList -- | Looks up a value in a container with a default value. -- If a key has no value, a given monadic default is returned. -- This allows simple error handling, as the default can be -- 'mzero', 'failError' etc. lookupContainer :: (Monad m, Ord a) => m b -> a -> GenericContainer a b -> m b lookupContainer dflt k = maybe dflt return . Map.lookup k . fromContainer -- | Updates a value inside a container. -- The signature of the function is crafted so that it can be directly -- used as a lens. alterContainerL :: (Functor f, Ord a) => a -> (Maybe b -> f (Maybe b)) -> GenericContainer a b -> f (GenericContainer a b) alterContainerL key f (GenericContainer m) = fmap (\v -> GenericContainer $ Map.alter (const v) key m) (f $ Map.lookup key m) -- | Container loader. readContainer :: (MonadFail m, HasStringRepr a, Ord a, J.JSON b) => J.JSObject J.JSValue -> m (GenericContainer a b) readContainer obj = do let kjvlist = J.fromJSObject obj kalist <- mapM (\(k, v) -> do k' <- fromStringRepr k v' <- fromKeyValue k v return (k', v')) kjvlist return $ GenericContainer (Map.fromList kalist) {-# ANN showContainer "HLint: ignore Use ***" #-} -- | Container dumper. showContainer :: (HasStringRepr a, J.JSON b) => GenericContainer a b -> J.JSValue showContainer = J.makeObj . map (\(k, v) -> (toStringRepr k, J.showJSON v)) . Map.toList . fromContainer instance (HasStringRepr a, Ord a, J.JSON b) => J.JSON (GenericContainer a b) where showJSON = showContainer readJSON (J.JSObject o) = readContainer o readJSON v = fail $ "Failed to load container, expected object but got " ++ show (pp_value v) -- * Types that (de)serialize in a special form of JSON newtype UsedKeys = UsedKeys (Maybe (Set.Set T.Text)) instance Sem.Semigroup UsedKeys where (UsedKeys xs) <> (UsedKeys ys) = UsedKeys $ liftA2 Set.union xs ys instance Monoid UsedKeys where mempty = UsedKeys (Just Set.empty) mappend = (Sem.<>) mkUsedKeys :: Set.Set T.Text -> UsedKeys mkUsedKeys = UsedKeys . Just allUsedKeys :: UsedKeys allUsedKeys = UsedKeys Nothing -- | Class of objects that can be converted from and to 'JSObject' -- lists-format. class DictObject a where toDict :: a -> [(String, J.JSValue)] fromDictWKeys :: [(String, J.JSValue)] -> WriterT UsedKeys J.Result a fromDict :: [(String, J.JSValue)] -> J.Result a fromDict = liftM fst . runWriterT . fromDictWKeys -- | A default implementation of 'showJSON' using 'toDict'. showJSONtoDict :: (DictObject a) => a -> J.JSValue showJSONtoDict = J.makeObj . toDict -- | A default implementation of 'readJSON' using 'fromDict'. -- Checks that the input value is a JSON object and -- converts it using 'fromDict'. -- Also checks the input contains only the used keys returned by 'fromDict'. readJSONfromDict :: (DictObject a) => J.JSValue -> J.Result a readJSONfromDict jsv = do dict <- liftM J.fromJSObject $ J.readJSON jsv (r, UsedKeys keys) <- runWriterT $ fromDictWKeys dict -- check that no superfluous dictionary keys are present case keys of Just allowedSet | not (Set.null superfluous) -> fail $ "Superfluous dictionary keys: " ++ show (Set.toAscList superfluous) ++ ", but only " ++ show (Set.toAscList allowedSet) ++ " allowed." where superfluous = Set.fromList (map (T.pack . fst) dict) Set.\\ allowedSet _ -> return () return r -- | Class of objects that can be converted from and to @[JSValue]@ with -- a fixed length and order. class ArrayObject a where toJSArray :: a -> [J.JSValue] fromJSArray :: [J.JSValue] -> J.Result a -- * General purpose data types for working with JSON -- | A Maybe newtype that allows for serialization more appropriate to the -- semantics of Maybe and JSON in our calls. Does not produce needless -- and confusing dictionaries. -- -- In particular, `J.JSNull` corresponds to `Nothing`. -- This also means that this `Maybe a` newtype should not be used with `a` -- values that themselves can serialize to `null`. newtype MaybeForJSON a = MaybeForJSON { unMaybeForJSON :: Maybe a } deriving (Show, Eq, Ord) instance (J.JSON a) => J.JSON (MaybeForJSON a) where readJSON J.JSNull = return $ MaybeForJSON Nothing readJSON x = (MaybeForJSON . Just) `liftM` J.readJSON x showJSON (MaybeForJSON (Just x)) = J.showJSON x showJSON (MaybeForJSON Nothing) = J.JSNull newtype TimeAsDoubleJSON = TimeAsDoubleJSON { unTimeAsDoubleJSON :: ClockTime } deriving (Show, Eq, Ord) instance J.JSON TimeAsDoubleJSON where readJSON v = do t <- J.readJSON v :: J.Result Double return . TimeAsDoubleJSON . uncurry TOD $ divMod (round $ t * pico) (pico :: Integer) where pico :: (Num a) => a pico = 10^(12 :: Int) showJSON (TimeAsDoubleJSON (TOD ss ps)) = J.showJSON (fromIntegral ss + fromIntegral ps / 10^(12 :: Int) :: Double) -- Text.JSON from the JSON package only has instances for tuples up to size 4. -- We use these newtypes so that we don't get a breakage once the 'json' -- package adds instances for larger tuples (or have to resort to CPP). newtype Tuple5 a b c d e = Tuple5 { unTuple5 :: (a, b, c, d, e) } instance (J.JSON a, J.JSON b, J.JSON c, J.JSON d, J.JSON e) => J.JSON (Tuple5 a b c d e) where readJSON (J.JSArray [a,b,c,d,e]) = Tuple5 <$> ((,,,,) <$> J.readJSON a <*> J.readJSON b <*> J.readJSON c <*> J.readJSON d <*> J.readJSON e) readJSON _ = fail "Unable to read Tuple5" showJSON (Tuple5 (a, b, c, d, e)) = J.JSArray [ J.showJSON a , J.showJSON b , J.showJSON c , J.showJSON d , J.showJSON e ] -- | Look up a value in a JSON object. Accessing @["a", "b", "c"]@ on an -- object is equivalent as accessing @myobject.a.b.c@ on a JavaScript object. -- -- An error is returned if the object doesn't have such an accessor or if -- any value during the nested access is not an object at all. nestedAccessByKey :: [String] -> J.JSValue -> J.Result J.JSValue nestedAccessByKey keys json = case keys of [] -> return json k:ks -> case json of J.JSObject obj -> J.valFromObj k obj >>= nestedAccessByKey ks _ -> J.Error $ "Cannot access non-object with key '" ++ k ++ "'" -- | Same as `nestedAccessByKey`, but accessing with a dotted string instead -- (like @nestedAccessByKeyDotted "a.b.c"@). nestedAccessByKeyDotted :: String -> J.JSValue -> J.Result J.JSValue nestedAccessByKeyDotted s = nestedAccessByKey (map T.unpack . T.splitOn (T.pack ".") . T.pack $ s) -- | Branch decoding on a field in a JSON object. branchOnField :: String -- ^ fieldname to branch on -> (J.JSValue -> J.Result a) -- ^ decoding function if field is present and @true@; field -- will already be removed in the input -> (J.JSValue -> J.Result a) -- ^ decoding function otherwise -> J.JSValue -> J.Result a branchOnField k ifTrue ifFalse (J.JSObject jobj) = let fields = J.fromJSObject jobj jobj' = J.JSObject . J.toJSObject $ filter ((/=) k . fst) fields in if lookup k fields == Just (J.JSBool True) then ifTrue jobj' else ifFalse jobj' branchOnField k _ _ _ = J.Error $ "Need an object to branch on key " ++ k -- | Add a field to a JSON object; to nothing, if the argument is not an object. addField :: (String, J.JSValue) -> J.JSValue -> J.JSValue addField (n,v) (J.JSObject obj) = J.JSObject $ JT.set_field obj n v addField _ jsval = jsval -- | Maybe obtain a map from a JSON object. maybeParseMap :: J.JSON a => J.JSValue -> Maybe (Map.Map String a) maybeParseMap =liftM fromContainer . readContainer <=< asJSObject ganeti-3.1.0~rc2/src/Ganeti/Jobs.hs000064400000000000000000000155261476477700300170440ustar00rootroot00000000000000{-| Generic code to work with jobs, e.g. submit jobs and check their status. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Jobs ( submitJobs , Annotator , execWithCancel , execJobsWait , execJobsWaitOk , waitForJobs ) where import Control.Concurrent (threadDelay) import Control.Exception (bracket) import Data.List import Data.Tuple import Data.IORef import System.Exit import System.Posix.Process import System.Posix.Signals import Ganeti.BasicTypes import Ganeti.Errors import qualified Ganeti.Luxi as L import Ganeti.OpCodes import Ganeti.Types import Ganeti.Utils -- | A simple type alias for clearer signature. type Annotator = OpCode -> MetaOpCode --- | Wrapper over execJobSet checking for early termination via an IORef. execCancelWrapper :: Annotator -> String -> IORef Int -> [([[OpCode]], String)] -> [(String, [(Int, JobStatus)])]-> IO (Result ()) execCancelWrapper _ _ _ [] _ = return $ Ok () execCancelWrapper anno master cref jobs submitted = do cancel <- readIORef cref if cancel > 0 then do putStrLn "Exiting early due to user request, " putStrLn $ show (length submitted) ++ "jobset(s) submitted: " print submitted putStrLn $ show (length jobs) ++ " jobset(s) not submitted:" print $ map swap jobs return $ Ok () else execJobSet anno master cref jobs submitted --- | Execute an entire jobset. execJobSet :: Annotator -> String -> IORef Int -> [([[OpCode]], String)] -> [(String, [(Int, JobStatus)])] -> IO (Result ()) execJobSet _ _ _ [] _ = return $ Ok () execJobSet anno master cref ((opcodes, descr):jobs) submitted = do jrs <- bracket (L.getLuxiClient master) L.closeClient $ execJobsWait metaopcodes logfn case jrs of Bad x -> return $ Bad x Ok x -> let failures = filter ((/= JOB_STATUS_SUCCESS) . snd) x in if null failures then execCancelWrapper anno master cref jobs $ submitted ++ [(descr, jobs_info x)] else return . Bad . unlines $ [ "Not all jobs completed successfully: " ++ show failures, "Aborting."] where metaopcodes = map (map anno) opcodes logfn = putStrLn . ("Got job IDs " ++) . commaJoin . map (show . fromJobId) jobs_info ji = zip (map (fromJobId . fst) ji) $ map snd ji -- | Signal handler for graceful termination. handleSigInt :: IORef Int -> IO () handleSigInt cref = do writeIORef cref 1 putStrLn ("Cancel request registered, will exit at" ++ " the end of the current job set...") -- | Signal handler for immediate termination. handleSigTerm :: IORef Int -> IO () handleSigTerm cref = do -- update the cref to 2, just for consistency writeIORef cref 2 putStrLn "Double cancel request, exiting now..." exitImmediately $ ExitFailure 2 -- | Prepares to run a set of jobsets with handling of signals and early -- termination. execWithCancel :: Annotator -> String -> [([[OpCode]], String)] -> IO (Result ()) execWithCancel anno master cmd_jobs = do cref <- newIORef 0 mapM_ (\(hnd, sig) -> installHandler sig (Catch (hnd cref)) Nothing) [(handleSigTerm, softwareTermination), (handleSigInt, keyboardSignal)] execCancelWrapper anno master cref cmd_jobs [] -- | Submits a set of jobs and returns their job IDs without waiting for -- completion. submitJobs :: [[MetaOpCode]] -> L.Client -> IO (Result [L.JobId]) submitJobs opcodes client = do jids <- L.submitManyJobs client opcodes return (case jids of Bad e -> Bad $ "Job submission error: " ++ formatError e Ok jids' -> Ok jids') -- | Executes a set of jobs and waits for their completion, returning their -- status. execJobsWait :: [[MetaOpCode]] -- ^ The list of jobs -> ([L.JobId] -> IO ()) -- ^ Post-submission callback -> L.Client -- ^ The Luxi client -> IO (Result [(L.JobId, JobStatus)]) execJobsWait opcodes callback client = do jids <- submitJobs opcodes client case jids of Bad e -> return $ Bad e Ok jids' -> do callback jids' waitForJobs jids' client -- | Polls a set of jobs at an increasing interval until all are finished one -- way or another. waitForJobs :: [L.JobId] -> L.Client -> IO (Result [(L.JobId, JobStatus)]) waitForJobs jids client = waitForJobs' 500000 15000000 where waitForJobs' delay maxdelay = do -- TODO: this should use WaitForJobChange once it's available in Haskell -- land, instead of a fixed schedule of sleeping intervals. threadDelay delay sts <- L.queryJobsStatus client jids case sts of Bad e -> return . Bad $ "Checking job status: " ++ formatError e Ok sts' -> if any (<= JOB_STATUS_RUNNING) sts' then waitForJobs' (min (delay * 2) maxdelay) maxdelay else return . Ok $ zip jids sts' -- | Execute jobs and return @Ok@ only if all of them succeeded. execJobsWaitOk :: [[MetaOpCode]] -> L.Client -> IO (Result ()) execJobsWaitOk opcodes client = do let nullog = const (return () :: IO ()) failed = filter ((/=) JOB_STATUS_SUCCESS . snd) fmtfail (i, s) = show (fromJobId i) ++ "=>" ++ jobStatusToRaw s sts <- execJobsWait opcodes nullog client case sts of Bad e -> return $ Bad e Ok sts' -> return (if null $ failed sts' then Ok () else Bad ("The following jobs failed: " ++ (intercalate ", " . map fmtfail $ failed sts'))) ganeti-3.1.0~rc2/src/Ganeti/Kvmd.hs000064400000000000000000000320061476477700300170400ustar00rootroot00000000000000{-| KVM daemon The KVM daemon is responsible for determining whether a given KVM instance was shutdown by an administrator or a user. For more information read the design document on the KVM daemon. The KVM daemon design is split in 2 parts, namely, monitors for Qmp sockets and directory/file watching. The monitors are spawned in lightweight Haskell threads and are reponsible for handling the communication between the KVM daemon and the KVM instance using the Qmp protocol. During the communcation, the monitor parses the Qmp messages and if powerdown or shutdown is received, then the shutdown file is written in the KVM control directory. Otherwise, when the communication terminates, that same file is removed. The communication terminates when the KVM instance stops or crashes. The directory and file watching uses inotify to track down events on the KVM control directory and its parents. There is a directory crawler that will try to add a watch to the KVM control directory if available or its parents, thus replacing watches until the KVM control directory becomes available. When this happens, a monitor for the Qmp socket is spawned. Given that the KVM daemon might stop or crash, the directory watching also simulates events for the Qmp sockets that already exist in the KVM control directory when the KVM daemon starts. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Kvmd where import Prelude hiding (rem) import Control.Exception (try) import Control.Concurrent import Control.Monad (unless, when) import Data.List import Data.Set (Set) import qualified Data.Set as Set (delete, empty, insert, member) import System.Directory import System.FilePath import System.IO import System.IO.Error (isEOFError) import System.INotify import qualified AutoConf import qualified Ganeti.BasicTypes as BasicTypes import Ganeti.Compat import qualified Ganeti.Constants as Constants import qualified Ganeti.Daemon as Daemon (getFQDN) import qualified Ganeti.Logging as Logging import qualified Ganeti.UDSServer as UDSServer import qualified Ganeti.Ssconf as Ssconf import qualified Ganeti.Types as Types type Lock = MVar () type Monitors = MVar (Set FilePath) -- * Utils -- | @isPrefixPath x y@ determines whether @x@ is a 'FilePath' prefix -- of 'FilePath' @y@. isPrefixPath :: FilePath -> FilePath -> Bool isPrefixPath x y = (splitPath x `isPrefixOf` splitPath y) || (splitPath (x ++ "/") `isPrefixOf` splitPath y) monitorGreeting :: String monitorGreeting = "{\"execute\": \"qmp_capabilities\"}" -- | KVM control directory containing the Qmp sockets. monitorDir :: String monitorDir = AutoConf.localstatedir "run/ganeti/kvm-hypervisor/ctrl/" monitorExtension :: String monitorExtension = ".kvmd" isMonitorPath :: FilePath -> Bool isMonitorPath = (== monitorExtension) . takeExtension shutdownExtension :: String shutdownExtension = ".shutdown" shutdownPath :: String -> String shutdownPath = (`replaceExtension` shutdownExtension) touchFile :: FilePath -> IO () touchFile file = withFile file WriteMode (const . return $ ()) -- * Monitors for Qmp sockets -- | @parseQmp isPowerdown isShutdown isStop str@ parses the packet -- @str@ and returns whether a powerdown, shutdown, or stop event is -- contained in that packet, defaulting to the values @isPowerdown@, -- @isShutdown@, and @isStop@, otherwise. parseQmp :: Bool -> Bool -> Bool -> String -> (Bool, Bool, Bool) parseQmp isPowerdown isShutdown isStop str = let isPowerdown' | "\"POWERDOWN\"" `isInfixOf` str = True | otherwise = isPowerdown isShutdown' | "\"SHUTDOWN\"" `isInfixOf` str = True | otherwise = isShutdown isStop' | "\"STOP\"" `isInfixOf` str = True | otherwise = isStop in (isPowerdown', isShutdown', isStop') -- | @receiveQmp handle@ listens for Qmp events on @handle@ and, when -- @handle@ is closed, it returns 'True' if a user shutdown event was -- received, and 'False' otherwise. receiveQmp :: Handle -> IO Bool receiveQmp handle = isUserShutdown <$> receive False False False where -- | A user shutdown consists of a shutdown event with no -- prior powerdown event and no stop event. isUserShutdown (isShutdown, isPowerdown, isStop) = isPowerdown && not isShutdown && not isStop receive isPowerdown isShutdown isStop = do res <- try $ hGetLine handle case res of Left err -> do unless (isEOFError err) $ hPrint stderr err return (isPowerdown, isShutdown, isStop) Right str -> do let (isPowerdown', isShutdown', isStop') = parseQmp isPowerdown isShutdown isStop str Logging.logDebug $ "Receive QMP message: " ++ str receive isPowerdown' isShutdown' isStop' -- | @detectMonitor monitorFile handle@ listens for Qmp events on -- @handle@ for Qmp socket @monitorFile@ and, when communcation -- terminates, it either creates the shutdown file, if a user shutdown -- was detected, or it deletes that same file, if an administrator -- shutdown was detected. detectMonitor :: FilePath -> Handle -> IO () detectMonitor monitorFile handle = do let shutdownFile = shutdownPath monitorFile res <- receiveQmp handle if res then do Logging.logInfo $ "Detect user shutdown, creating file " ++ show shutdownFile touchFile shutdownFile else do Logging.logInfo $ "Detect admin shutdown, removing file " ++ show shutdownFile (try (removeFile shutdownFile) :: IO (Either IOError ())) >> return () -- | @runMonitor monitorFile@ creates a monitor for the Qmp socket -- @monitorFile@ and calls 'detectMonitor'. runMonitor :: FilePath -> IO () runMonitor monitorFile = do handle <- UDSServer.openClientSocket Constants.luxiDefRwto monitorFile hPutStrLn handle monitorGreeting hFlush handle detectMonitor monitorFile handle UDSServer.closeClientSocket handle -- | @ensureMonitor monitors monitorFile@ ensures that there is -- exactly one monitor running for the Qmp socket @monitorFile@, given -- the existing set of monitors @monitors@. ensureMonitor :: Monitors -> FilePath -> IO () ensureMonitor monitors monitorFile = modifyMVar_ monitors $ \files -> if monitorFile `Set.member` files then return files else do forkIO tryMonitor >> return () return $ monitorFile `Set.insert` files where tryMonitor = do Logging.logInfo $ "Start monitor " ++ show monitorFile res <- try (runMonitor monitorFile) :: IO (Either IOError ()) case res of Left err -> Logging.logError $ "Catch monitor exception: " ++ show err _ -> return () Logging.logInfo $ "Stop monitor " ++ show monitorFile modifyMVar_ monitors (return . Set.delete monitorFile) -- * Directory and file watching -- | Handles an inotify event outside the target directory. -- -- Tracks events on the parent directory of the KVM control directory -- until one of its parents becomes available. handleGenericEvent :: Lock -> String -> String -> Event -> IO () handleGenericEvent lock curDir tarDir ev@Created {} | isDirectory ev && curDir /= tarDir && (curDir filePath' ev) `isPrefixPath` tarDir = putMVar lock () handleGenericEvent lock _ _ event | event == DeletedSelf || event == Unmounted = putMVar lock () handleGenericEvent _ _ _ _ = return () -- | Handles an inotify event in the target directory. -- -- Upon a create or open event inside the KVM control directory, it -- ensures that there is a monitor running for the new Qmp socket. handleTargetEvent :: Lock -> Monitors -> String -> Event -> IO () handleTargetEvent _ monitors tarDir ev@Created {} | not (isDirectory ev) && isMonitorPath (filePath' ev) = ensureMonitor monitors $ tarDir filePath' ev handleTargetEvent lock monitors tarDir ev@Opened {} | not (isDirectory ev) = case maybeFilePath' ev of Just p | isMonitorPath p -> ensureMonitor monitors $ tarDir filePath' ev _ -> handleGenericEvent lock tarDir tarDir ev handleTargetEvent _ _ tarDir ev@Created {} | not (isDirectory ev) && takeExtension (filePath' ev) == shutdownExtension = Logging.logInfo $ "User shutdown file opened " ++ show (tarDir filePath' ev) handleTargetEvent _ _ tarDir ev@Deleted {} | not (isDirectory ev) && takeExtension (filePath' ev) == shutdownExtension = Logging.logInfo $ "User shutdown file deleted " ++ show (tarDir filePath' ev) handleTargetEvent lock _ tarDir ev = handleGenericEvent lock tarDir tarDir ev -- | Dispatches inotify events depending on the directory they occur in. handleDir :: Lock -> Monitors -> String -> String -> Event -> IO () handleDir lock monitors curDir tarDir event = do Logging.logDebug $ "Handle event " ++ show event if curDir == tarDir then handleTargetEvent lock monitors tarDir event else handleGenericEvent lock curDir tarDir event -- | Simulates file creation events for the Qmp sockets that already -- exist in @dir@. recapDir :: Lock -> Monitors -> FilePath -> IO () recapDir lock monitors dir = do files <- getDirectoryContents dir let files' = map toInotifyPath $ filter isMonitorPath files mapM_ sendEvent files' where sendEvent file = handleTargetEvent lock monitors dir Created { isDirectory = False , filePath = file } -- | Crawls @tarDir@, or its parents until @tarDir@ becomes available, -- always listening for inotify events. -- -- Used for crawling the KVM control directory and its parents, as -- well as simulating file creation events. watchDir :: Lock -> FilePath -> INotify -> IO () watchDir lock tarDir inotify = watchDir' tarDir where watchDirEvents dir | dir == tarDir = [AllEvents] | otherwise = [Create, DeleteSelf] watchDir' dir = do add <- doesDirectoryExist dir if add then do let events = watchDirEvents dir Logging.logInfo $ "Watch directory " ++ show dir monitors <- newMVar Set.empty wd <- addWatch inotify events (toInotifyPath dir) (handleDir lock monitors dir tarDir) when (dir == tarDir) $ recapDir lock monitors dir () <- takeMVar lock rem <- doesDirectoryExist dir if rem then do Logging.logInfo $ "Unwatch directory " ++ show dir removeWatch wd else Logging.logInfo $ "Throw away watch from directory " ++ show dir else watchDir' (takeDirectory dir) rewatchDir :: Lock -> FilePath -> INotify -> IO () rewatchDir lock tarDir inotify = do watchDir lock tarDir inotify rewatchDir lock tarDir inotify -- * Starting point startWith :: FilePath -> IO () startWith dir = do lock <- newEmptyMVar withINotify (rewatchDir lock dir) start :: IO () start = do fqdn <- Daemon.getFQDN hypervisors <- Ssconf.getHypervisorList Nothing userShutdown <- Ssconf.getEnabledUserShutdown Nothing vmCapable <- Ssconf.getNodesVmCapable Nothing BasicTypes.genericResult Logging.logInfo (const $ startWith monitorDir) $ do isKvm =<< hypervisors isUserShutdown =<< userShutdown isVmCapable fqdn =<< vmCapable where isKvm hs | Types.Kvm `elem` hs = return () | otherwise = fail "KVM not enabled, exiting" isUserShutdown True = return () isUserShutdown _ = fail "User shutdown not enabled, exiting" isVmCapable node vmCapables = case lookup node vmCapables of Just True -> return () _ -> fail $ "Node " ++ show node ++ " is not VM capable, exiting" ganeti-3.1.0~rc2/src/Ganeti/Lens.hs000064400000000000000000000101201476477700300170310ustar00rootroot00000000000000{-# LANGUAGE Rank2Types, CPP #-} {-# LANGUAGE NoPolyKinds #-} {-| Provides all lens-related functions. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Lens ( module Control.Lens , lensWith , makeCustomLenses , makeCustomLenses' , traverseOf2 , mapMOf2 , atSet ) where import Control.Applicative (WrappedMonad(..)) import Control.Lens import Control.Monad import Data.Functor.Compose (Compose(..)) import qualified Data.Set as S import Language.Haskell.TH -- | Creates an optimized lens where the setter also gets the original value -- from the getter. lensWith :: (s -> a) -> (s -> a -> b -> t) -> Lens s t a b lensWith sa sbt f s = uncurry (sbt s) <$> (\a -> fmap ((,) a) (f a)) (sa s) lensFieldName :: String -> String lensFieldName = (++ "L") -- | Internal helper method for constructing partial set of lenses. makeCustomLensesFiltered :: (String -> Bool) -> Name -> Q [Dec] makeCustomLensesFiltered f = makeLensesWith customRules where customRules :: LensRules customRules = set lensField nameFun lensRules #if MIN_VERSION_lens(4,5,0) nameFun :: Name -> [Name] -> Name -> [DefName] nameFun _ _ = liftM (TopName . mkName) . nameFilter . nameBase #elif MIN_VERSION_lens(4,4,0) nameFun :: [Name] -> Name -> [DefName] nameFun _ = liftM (TopName . mkName) . nameFilter . nameBase #else nameFun :: String -> Maybe String nameFun = nameFilter #endif nameFilter :: (MonadPlus m) => String -> m String nameFilter = liftM lensFieldName . mfilter f . return -- | Create lenses for all fields of a given data type. makeCustomLenses :: Name -> Q [Dec] makeCustomLenses = makeCustomLensesFiltered (const True) -- | Create lenses for some fields of a given data type. makeCustomLenses' :: Name -> [Name] -> Q [Dec] makeCustomLenses' name lst = makeCustomLensesFiltered f name where allowed = S.fromList . map nameBase $ lst f = flip S.member allowed -- | Traverses over a composition of two functors. -- Most often the @g@ functor is @(,) r@ and 'traverseOf2' is used to -- traverse an effectful computation that also returns an additional output -- value. traverseOf2 :: LensLike (Compose f g) s t a b -> (a -> f (g b)) -> s -> f (g t) traverseOf2 k f = getCompose . traverseOf k (Compose . f) -- | Traverses over a composition of a monad and a functor. -- See 'traverseOf2'. mapMOf2 :: LensLike (Compose (WrappedMonad m) g) s t a b -> (a -> m (g b)) -> s -> m (g t) mapMOf2 k f = unwrapMonad . traverseOf2 k (WrapMonad . f) -- | A helper lens over sets. -- While a similar lens exists in the package (as @Lens' Set (Maybe ())@), -- it's available only in most recent versions. -- And using @Bool@ instead of @Maybe ()@ is more convenient. atSet :: (Ord a) => a -> Lens' (S.Set a) Bool atSet k = lensWith (S.member k) f where f s True False = S.delete k s f s False True = S.insert k s f s _ _ = s ganeti-3.1.0~rc2/src/Ganeti/Locking/000075500000000000000000000000001476477700300171705ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Locking/Allocation.hs000064400000000000000000000426671476477700300216300ustar00rootroot00000000000000{-# LANGUAGE BangPatterns #-} {-| Implementation of lock allocation. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Locking.Allocation ( LockAllocation , emptyAllocation , OwnerState(..) , lockOwners , listLocks , listAllLocks , listAllLocksOwners , holdsLock , LockRequest(..) , requestExclusive , requestShared , requestRelease , updateLocks , freeLocks ) where import Control.Applicative (liftA2) import Control.Arrow (second, (***)) import Control.Monad import Data.Foldable (for_, find) import Data.List (foldl') import qualified Data.Map as M import Data.Maybe (fromMaybe) import qualified Data.Set as S import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.JSON (toArray) import Ganeti.Locking.Types {- This module is parametric in the type of locks and lock owners. While we only state minimal requirements for the types, we will consistently use the type variable 'a' for the type of locks and the variable 'b' for the type of the lock owners throughout this module. -} -- | Data type describing the way a lock can be owned. data OwnerState = OwnShared | OwnExclusive deriving (Ord, Eq, Show) -- | Type describing indirect ownership on a lock. We keep the set -- of all (lock, owner)-pairs for locks that are implied in the given -- lock, annotated with the type of ownership (shared or exclusive). type IndirectOwners a b = M.Map (a, b) OwnerState -- | The state of a lock that is taken. Besides the state of the lock -- itself, we also keep track of all other lock allocation that affect -- the given lock by means of implication. data AllocationState a b = Exclusive b (IndirectOwners a b) | Shared (S.Set b) (IndirectOwners a b) deriving (Eq, Show) -- | Compute the set of indirect owners from the information about -- indirect ownership. indirectOwners :: (Ord a, Ord b) => M.Map (a, b) OwnerState -> S.Set b indirectOwners = S.map snd . M.keysSet -- | Compute the (zero or one-elment) set of exclusive indirect owners. indirectExclusives :: (Ord a, Ord b) => M.Map (a, b) OwnerState -> S.Set b indirectExclusives = indirectOwners . M.filter (== OwnExclusive) {-| Representation of a Lock allocation To keep queries for locks efficient, we keep two associations, with the invariant that they fit together: the association from locks to their allocation state, and the association from an owner to the set of locks owned. As we do not export the constructor, the problem of keeping this invariant reduces to only exporting functions that keep the invariant. -} data LockAllocation a b = LockAllocation { laLocks :: M.Map a (AllocationState a b) , laOwned :: M.Map b (M.Map a OwnerState) } deriving (Eq, Show) -- | A state with all locks being free. emptyAllocation :: (Ord a, Ord b) => LockAllocation a b emptyAllocation = LockAllocation { laLocks = M.empty , laOwned = M.empty } -- | Obtain the list of all owners holding at least a single lock. lockOwners :: Ord b => LockAllocation a b -> [b] lockOwners = M.keys . laOwned -- | Obtain the locks held by a given owner. The locks are reported -- as a map from the owned locks to the form of ownership (OwnShared -- or OwnExclusive). listLocks :: Ord b => b -> LockAllocation a b -> M.Map a OwnerState listLocks owner = fromMaybe M.empty . M.lookup owner . laOwned -- | List all locks currently (directly or indirectly) owned by someone. listAllLocks :: Ord b => LockAllocation a b -> [a] listAllLocks = M.keys . laLocks -- | Map an AllocationState to a list of pairs of owners and type of -- ownership, showing the direct owners only. toOwnersList :: AllocationState a b -> [(b, OwnerState)] toOwnersList (Exclusive owner _) = [(owner, OwnExclusive)] toOwnersList (Shared owners _) = map (flip (,) OwnShared) . S.elems $ owners -- | List all locks currently (directly of indirectly) in use together -- with the direct owners. listAllLocksOwners :: LockAllocation a b -> [(a, [(b, OwnerState)])] listAllLocksOwners = M.toList . M.map toOwnersList . laLocks -- | Returns 'True' if the given owner holds the given lock at the given -- ownership level or higher. This means that querying for a shared lock -- returns 'True' of the owner holds the lock in shared or exlusive mode. holdsLock :: (Ord a, Ord b) => b -> a -> OwnerState -> LockAllocation a b -> Bool holdsLock owner lock state = (>= Just state) . M.lookup lock . listLocks owner -- | Data Type describing a change request on a single lock. data LockRequest a = LockRequest { lockAffected :: a , lockRequestType :: Maybe OwnerState } deriving (Eq, Show, Ord) instance J.JSON a => J.JSON (LockRequest a) where showJSON (LockRequest a Nothing) = J.showJSON (a, "release") showJSON (LockRequest a (Just OwnShared)) = J.showJSON (a, "shared") showJSON (LockRequest a (Just OwnExclusive)) = J.showJSON (a, "exclusive") readJSON (J.JSArray [a, J.JSString tp]) = case J.fromJSString tp of "release" -> LockRequest <$> J.readJSON a <*> pure Nothing "shared" -> LockRequest <$> J.readJSON a <*> pure (Just OwnShared) "exclusive" -> LockRequest <$> J.readJSON a <*> pure (Just OwnExclusive) _ -> J.Error $ "malformed request type: " ++ J.fromJSString tp readJSON x = J.Error $ "malformed lock request: " ++ show x -- | Lock request for an exclusive lock. requestExclusive :: a -> LockRequest a requestExclusive lock = LockRequest { lockAffected = lock , lockRequestType = Just OwnExclusive } -- | Lock request for a shared lock. requestShared :: a -> LockRequest a requestShared lock = LockRequest { lockAffected = lock , lockRequestType = Just OwnShared } -- | Request to release a lock. requestRelease :: a -> LockRequest a requestRelease lock = LockRequest { lockAffected = lock , lockRequestType = Nothing } -- | Update the Allocation state of a lock according to a given -- function. updateAllocState :: (Ord a, Ord b) => (Maybe (AllocationState a b) -> AllocationState a b) -> LockAllocation a b -> a -> LockAllocation a b updateAllocState f state lock = let !locks' = M.alter (find (/= Shared S.empty M.empty) . Just . f) lock (laLocks state) in state { laLocks = locks' } -- | Internal function to update the state according to a single -- lock request, assuming all prerequisites are met. updateLock :: (Ord a, Ord b) => b -> LockAllocation a b -> LockRequest a -> LockAllocation a b updateLock owner state (LockRequest lock (Just OwnExclusive)) = let locks = laLocks state lockstate' = case M.lookup lock locks of Just (Exclusive _ i) -> Exclusive owner i Just (Shared _ i) -> Exclusive owner i Nothing -> Exclusive owner M.empty !locks' = M.insert lock lockstate' locks ownersLocks' = M.insert lock OwnExclusive $ listLocks owner state !owned' = M.insert owner ownersLocks' $ laOwned state in state { laLocks = locks', laOwned = owned' } updateLock owner state (LockRequest lock (Just OwnShared)) = let ownersLocks' = M.insert lock OwnShared $ listLocks owner state !owned' = M.insert owner ownersLocks' $ laOwned state locks = laLocks state lockState' = case M.lookup lock locks of Just (Exclusive _ i) -> Shared (S.singleton owner) i Just (Shared s i) -> Shared (S.insert owner s) i _ -> Shared (S.singleton owner) M.empty !locks' = M.insert lock lockState' locks in state { laLocks = locks', laOwned = owned' } updateLock owner state (LockRequest lock Nothing) = let ownersLocks' = M.delete lock $ listLocks owner state owned = laOwned state owned' = if M.null ownersLocks' then M.delete owner owned else M.insert owner ownersLocks' owned update (Just (Exclusive x i)) = if x == owner then Shared S.empty i else Exclusive x i update (Just (Shared s i)) = Shared (S.delete owner s) i update Nothing = Shared S.empty M.empty in updateAllocState update (state { laOwned = owned' }) lock -- | Update the set of indirect ownerships of a lock by the given function. updateIndirectSet :: (Ord a, Ord b) => (IndirectOwners a b -> IndirectOwners a b) -> LockAllocation a b -> a -> LockAllocation a b updateIndirectSet f = let update (Just (Exclusive x i)) = Exclusive x (f i) update (Just (Shared s i)) = Shared s (f i) update Nothing = Shared S.empty (f M.empty) in updateAllocState update -- | Update all indirect onwerships of a given lock. updateIndirects :: (Lock a, Ord b) => b -> LockAllocation a b -> LockRequest a -> LockAllocation a b updateIndirects owner state req = let lock = lockAffected req fn = case lockRequestType req of Nothing -> M.delete (lock, owner) Just tp -> M.insert (lock, owner) tp in foldl' (updateIndirectSet fn) state $ lockImplications lock -- | Update the locks of an owner according to the given request. Return -- the pair of the new state and the result of the operation, which is the -- the set of owners on which the operation was blocked on. so an empty set is -- success, and the state is updated if, and only if, the returned set is emtpy. -- In that way, it can be used in atomicModifyIORef. updateLocks :: (Lock a, Ord b) => b -> [LockRequest a] -> LockAllocation a b -> (LockAllocation a b, Result (S.Set b)) updateLocks owner reqs state = genericResult ((,) state . Bad) (second Ok) $ do unless ((==) (length reqs) . S.size . S.fromList $ map lockAffected reqs) . runListHead (return ()) (fail . (++) "Inconsitent requests for lock " . show) $ do r <- reqs r' <- reqs guard $ r /= r' guard $ lockAffected r == lockAffected r' return $ lockAffected r let current = listLocks owner state unless (M.null current) $ do let (highest, _) = M.findMax current notHolding = not . any (uncurry (==) . ((M.lookup `flip` current) *** Just)) orderViolation l = fail $ "Order violation: requesting " ++ show l ++ " while holding " ++ show highest for_ reqs $ \req -> case req of LockRequest lock (Just OwnExclusive) | lock < highest && notHolding ((,) <$> lock : lockImplications lock <*> [OwnExclusive]) -> orderViolation lock LockRequest lock (Just OwnShared) | lock < highest && notHolding ((,) <$> lock : lockImplications lock <*> [OwnExclusive, OwnShared]) -> orderViolation lock _ -> Ok () let sharedsHeld = M.keysSet $ M.filter (== OwnShared) current exclusivesRequested = map lockAffected . filter ((== Just OwnExclusive) . lockRequestType) $ reqs runListHead (return ()) fail $ do x <- exclusivesRequested i <- lockImplications x guard $ S.member i sharedsHeld return $ "Order violation: requesting exclusively " ++ show x ++ " while holding a shared lock on the group lock " ++ show i ++ " it belongs to." let blockedOn (LockRequest _ Nothing) = S.empty blockedOn (LockRequest lock (Just OwnExclusive)) = case M.lookup lock (laLocks state) of Just (Exclusive x i) -> S.singleton x `S.union` indirectOwners i Just (Shared xs i) -> xs `S.union` indirectOwners i _ -> S.empty blockedOn (LockRequest lock (Just OwnShared)) = case M.lookup lock (laLocks state) of Just (Exclusive x i) -> S.singleton x `S.union` indirectExclusives i Just (Shared _ i) -> indirectExclusives i _ -> S.empty let indirectBlocked Nothing _ = S.empty indirectBlocked (Just OwnShared) lock = case M.lookup lock (laLocks state) of Just (Exclusive x _) -> S.singleton x _ -> S.empty indirectBlocked (Just OwnExclusive) lock = case M.lookup lock (laLocks state) of Just (Exclusive x _) -> S.singleton x Just (Shared xs _) -> xs _ -> S.empty let direct = S.unions $ map blockedOn reqs indirect = reqs >>= \req -> map (indirectBlocked (lockRequestType req)) . lockImplications $ lockAffected req let blocked = S.delete owner . S.unions $ direct:indirect let state' = foldl' (updateLock owner) state reqs state'' = foldl' (updateIndirects owner) state' reqs return (if S.null blocked then state'' else state, blocked) -- | Manipluate all locks of the owner with a given property. manipulateLocksPredicate :: (Lock a, Ord b) => (a -> LockRequest a) -> (a -> Bool) -> b -> LockAllocation a b -> LockAllocation a b manipulateLocksPredicate req prop owner state = fst . flip (updateLocks owner) state . map req . filter prop . M.keys $ listLocks owner state -- | Compute the state after an owner releases all its locks that -- satisfy a certain property. freeLocksPredicate :: (Lock a, Ord b) => (a -> Bool) -> LockAllocation a b -> b -> LockAllocation a b freeLocksPredicate prop = flip $ manipulateLocksPredicate requestRelease prop -- | Compute the state after an onwer releases all its locks. freeLocks :: (Lock a, Ord b) => LockAllocation a b -> b -> LockAllocation a b freeLocks = freeLocksPredicate (const True) {-| Serializaiton of Lock Allocations To serialize a lock allocation, we only remember which owner holds which locks at which level (shared or exclusive). From this information, everything else can be reconstructed, simply using updateLocks. -} instance J.JSON OwnerState where showJSON OwnShared = J.showJSON "shared" showJSON OwnExclusive = J.showJSON "exclusive" readJSON (J.JSString x) = let s = J.fromJSString x in case s of "shared" -> J.Ok OwnShared "exclusive" -> J.Ok OwnExclusive _ -> J.Error $ "Unknown owner type " ++ s readJSON _ = J.Error "Owner type not encoded as a string" -- | Read a lock-ownerstate pair from JSON. readLockOwnerstate :: (J.JSON a) => J.JSValue -> J.Result (a, OwnerState) readLockOwnerstate (J.JSArray [x, y]) = liftA2 (,) (J.readJSON x) (J.readJSON y) readLockOwnerstate x = fail $ "lock-ownerstate pairs are encoded as arrays" ++ " of length 2, but found " ++ show x -- | Read an owner-lock pair from JSON. readOwnerLock :: (J.JSON a, J.JSON b) => J.JSValue -> J.Result (b, [(a, OwnerState)]) readOwnerLock (J.JSArray [x, J.JSArray ys]) = liftA2 (,) (J.readJSON x) (mapM readLockOwnerstate ys) readOwnerLock x = fail $ "Expected pair of owner and list of owned locks," ++ " but found " ++ show x -- | Transform a lock-ownerstate pair into a LockRequest. toRequest :: (a, OwnerState) -> LockRequest a toRequest (a, OwnExclusive) = requestExclusive a toRequest (a, OwnShared) = requestShared a -- | Obtain a LockAllocation from a given owner-locks list. -- The obtained allocation is the one obtained if the respective owners -- requested their locks sequentially. allocationFromOwners :: (Lock a, Ord b, Show b) => [(b, [(a, OwnerState)])] -> J.Result (LockAllocation a b) allocationFromOwners = let allocateOneOwner s (o, req) = do let (s', result) = updateLocks o (map toRequest req) s when (result /= Ok S.empty) . fail . (++) ("Inconsistent lock status for " ++ show o ++ ": ") $ case result of Bad err -> err Ok blocked -> "blocked on " ++ show (S.toList blocked) return s' in foldM allocateOneOwner emptyAllocation instance (Lock a, J.JSON a, Ord b, J.JSON b, Show b) => J.JSON (LockAllocation a b) where showJSON = J.showJSON . M.toList . M.map M.toList . laOwned readJSON x = do xs <- toArray x owned <- mapM readOwnerLock xs allocationFromOwners owned ganeti-3.1.0~rc2/src/Ganeti/Locking/Locks.hs000064400000000000000000000204641476477700300206050ustar00rootroot00000000000000{-# LANGUAGE ViewPatterns, FlexibleContexts #-} {-| Ganeti lock structure -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Locking.Locks ( GanetiLocks(..) , lockName , ClientType(..) , ClientId(..) , GanetiLockWaiting , LockLevel(..) , lockLevel ) where import Control.Monad ((>=>), liftM) import Data.List (stripPrefix) import System.Posix.Types (ProcessID) import qualified Text.JSON as J import Ganeti.JSON (readEitherString) import Ganeti.Locking.Types import Ganeti.Locking.Waiting import Ganeti.Types -- | The type of Locks available in Ganeti. The order of this type -- is the lock oder. data GanetiLocks = ClusterLockSet | BGL | InstanceLockSet | Instance String | NodeGroupLockSet | NodeGroup String | NodeLockSet | Node String | NodeResLockSet | NodeRes String | NetworkLockSet | Network String -- | A lock used for a transitional period when WConfd -- keeps the state of the configuration, but all the -- operations are still performed on the Python side. | ConfigLock deriving (Ord, Eq, Show) -- | Provide the String representation of a lock lockName :: GanetiLocks -> String lockName BGL = "cluster/BGL" lockName ClusterLockSet = "cluster/[lockset]" lockName InstanceLockSet = "instance/[lockset]" lockName (Instance uuid) = "instance/" ++ uuid lockName NodeGroupLockSet = "nodegroup/[lockset]" lockName (NodeGroup uuid) = "nodegroup/" ++ uuid lockName NodeLockSet = "node/[lockset]" lockName (Node uuid) = "node/" ++ uuid lockName NodeResLockSet = "node-res/[lockset]" lockName (NodeRes uuid) = "node-res/" ++ uuid lockName NetworkLockSet = "network/[lockset]" lockName (Network uuid) = "network/" ++ uuid lockName ConfigLock = "cluster/config" -- | Obtain a lock from its name. lockFromName :: String -> J.Result GanetiLocks lockFromName "cluster/BGL" = return BGL lockFromName "cluster/[lockset]" = return ClusterLockSet lockFromName "instance/[lockset]" = return InstanceLockSet lockFromName (stripPrefix "instance/" -> Just uuid) = return $ Instance uuid lockFromName "nodegroup/[lockset]" = return NodeGroupLockSet lockFromName (stripPrefix "nodegroup/" -> Just uuid) = return $ NodeGroup uuid lockFromName "node-res/[lockset]" = return NodeResLockSet lockFromName (stripPrefix "node-res/" -> Just uuid) = return $ NodeRes uuid lockFromName "node/[lockset]" = return NodeLockSet lockFromName (stripPrefix "node/" -> Just uuid) = return $ Node uuid lockFromName "network/[lockset]" = return NetworkLockSet lockFromName (stripPrefix "network/" -> Just uuid) = return $ Network uuid lockFromName "cluster/config" = return ConfigLock lockFromName n = fail $ "Unknown lock name '" ++ n ++ "'" instance J.JSON GanetiLocks where showJSON = J.JSString . J.toJSString . lockName readJSON = readEitherString >=> lockFromName -- | The levels, the locks belong to. data LockLevel = LevelCluster | LevelInstance | LevelNodeGroup | LevelNode | LevelNodeRes | LevelNetwork -- | A transitional level for internal configuration locks | LevelConfig deriving (Eq, Show, Enum) -- | Provide the names of the lock levels. lockLevelName :: LockLevel -> String lockLevelName LevelCluster = "cluster" lockLevelName LevelInstance = "instance" lockLevelName LevelNodeGroup = "nodegroup" lockLevelName LevelNode = "node" lockLevelName LevelNodeRes = "node-res" lockLevelName LevelNetwork = "network" lockLevelName LevelConfig = "config" -- | Obtain a lock level from its name/ lockLevelFromName :: String -> J.Result LockLevel lockLevelFromName "cluster" = return LevelCluster lockLevelFromName "instance" = return LevelInstance lockLevelFromName "nodegroup" = return LevelNodeGroup lockLevelFromName "node" = return LevelNode lockLevelFromName "node-res" = return LevelNodeRes lockLevelFromName "network" = return LevelNetwork lockLevelFromName "config" = return LevelConfig lockLevelFromName n = fail $ "Unknown lock-level name '" ++ n ++ "'" instance J.JSON LockLevel where showJSON = J.JSString . J.toJSString . lockLevelName readJSON = readEitherString >=> lockLevelFromName -- | For a lock, provide its level. lockLevel :: GanetiLocks -> LockLevel lockLevel BGL = LevelCluster lockLevel ClusterLockSet = LevelCluster lockLevel InstanceLockSet = LevelInstance lockLevel (Instance _) = LevelInstance lockLevel NodeGroupLockSet = LevelNodeGroup lockLevel (NodeGroup _) = LevelNodeGroup lockLevel NodeLockSet = LevelNode lockLevel (Node _) = LevelNode lockLevel NodeResLockSet = LevelNodeRes lockLevel (NodeRes _) = LevelNodeRes lockLevel NetworkLockSet = LevelNetwork lockLevel (Network _) = LevelNetwork lockLevel ConfigLock = LevelConfig instance Lock GanetiLocks where lockImplications BGL = [ClusterLockSet] lockImplications (Instance _) = [InstanceLockSet] lockImplications (NodeGroup _) = [NodeGroupLockSet] lockImplications (NodeRes _) = [NodeResLockSet] lockImplications (Node _) = [NodeLockSet] lockImplications (Network _) = [NetworkLockSet] -- the ConfigLock is idependent of everything, it only synchronizes -- access to the configuration lockImplications ConfigLock = [] lockImplications _ = [] -- | Type of entities capable of owning locks. Usually, locks are owned -- by jobs. However, occassionally other tasks need locks (currently, e.g., -- to lock the configuration). These are identified by a unique name, -- reported to WConfD as a strig. data ClientType = ClientOther String | ClientJob JobId deriving (Ord, Eq, Show) instance J.JSON ClientType where showJSON (ClientOther s) = J.showJSON s showJSON (ClientJob jid) = J.showJSON jid readJSON (J.JSString s) = J.Ok . ClientOther $ J.fromJSString s readJSON jids = J.readJSON jids >>= \jid -> J.Ok (ClientJob jid) -- | A client is identified as a job id, thread id, a path to its process -- identifier file, and its process id. -- -- The JobId isn't enough to identify a client as the master daemon -- also handles client calls that aren't jobs, but which use the configuration. -- These taks are identified by a unique name, reported to WConfD as a string. data ClientId = ClientId { ciIdentifier :: ClientType , ciLockFile :: FilePath , ciPid :: ProcessID } deriving (Ord, Eq, Show) -- | Obtain the ClientID from its JSON representation. clientIdFromJSON :: J.JSValue -> J.Result ClientId clientIdFromJSON (J.JSArray [clienttp, J.JSString lf, pid]) = ClientId <$> J.readJSON clienttp <*> pure (J.fromJSString lf) <*> liftM fromIntegral (J.readJSON pid :: J.Result Integer) clientIdFromJSON x = J.Error $ "malformed client id: " ++ show x instance J.JSON ClientId where showJSON (ClientId client lf pid) = J.showJSON (client, lf, fromIntegral pid :: Integer) readJSON = clientIdFromJSON -- | The type of lock Allocations in Ganeti. In Ganeti, the owner of -- locks are jobs. type GanetiLockWaiting = LockWaiting GanetiLocks ClientId Integer ganeti-3.1.0~rc2/src/Ganeti/Locking/Types.hs000064400000000000000000000041611476477700300206320ustar00rootroot00000000000000{-| Ganeti lock-related types and type classes -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Locking.Types ( Lock , lockImplications ) where {-| The type class of being a lock As usual, locks need to come with an order, the lock order, and be an instance of Show, so that malformed requests can meaningfully be reported. Additionally, in Ganeti we also have group locks, like a lock for all nodes. While those group locks contain infinitely many locks, the set of locks a single lock is included in is always finite, and usually very small. So we take this association from a lock to the locks it is (strictly) included in as additional data of the type class. It is a prerequisite that whenever 'a' is implied in 'b', then all locks that are in the lock order between 'a' and 'b' are also implied in 'b'. -} class (Ord a, Show a) => Lock a where lockImplications :: a -> [a] ganeti-3.1.0~rc2/src/Ganeti/Locking/Waiting.hs000064400000000000000000000441001476477700300211250ustar00rootroot00000000000000{-# LANGUAGE BangPatterns #-} {-| Implementation of a priority waiting structure for locks. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Locking.Waiting ( LockWaiting , ExtWaiting , emptyWaiting , updateLocks , updateLocksWaiting , safeUpdateLocksWaiting , getAllocation , getPendingOwners , hasPendingRequest , removePendingRequest , releaseResources , getPendingRequests , extRepr , fromExtRepr , freeLocksPredicate , downGradeLocksPredicate , intersectLocks , opportunisticLockUnion , guardedOpportunisticLockUnion ) where import Control.Arrow ((&&&), (***), second) import Control.Monad (liftM) import Data.List (sort, foldl') import qualified Data.Map as M import Data.Maybe (fromMaybe) import qualified Data.Set as S import qualified Text.JSON as J import Ganeti.BasicTypes import qualified Ganeti.Locking.Allocation as L import Ganeti.Locking.Types (Lock) {- This module is parametric in the type of locks, lock owners, and priorities of the request. While we state only minimal requirements for the types, we will consistently use the type variable 'a' for the type of locks, the variable 'b' for the type of the lock owners, and 'c' for the type of priorities throughout this module. The type 'c' will have to instance Ord, and the smallest value indicate the most important priority. -} {-| Representation of the waiting structure For any request we cannot fullfill immediately, we have a set of lock owners it is blocked on. We can pick one of the owners, the smallest say; then we know that this request cannot possibly be fulfilled until this owner does something. So we can index the pending requests by such a chosen owner and only revisit them once the owner acts. For the requests to revisit we need to do so in order of increasing priority; this order can be maintained by the Set data structure, where we make use of the fact that tuples are ordered lexicographically. Additionally, we keep track of which owners have pending requests, to disallow them any other lock tasks till their request is fulfilled. To allow canceling of pending requests, we also keep track on which owner their request is pending on and what the request was. -} data LockWaiting a b c = LockWaiting { lwAllocation :: L.LockAllocation a b , lwPending :: M.Map b (S.Set (c, b, [L.LockRequest a])) , lwPendingOwners :: M.Map b (b, (c, b, [L.LockRequest a])) } deriving Show -- | A state without locks and pending requests. emptyWaiting :: (Ord a, Ord b, Ord c) => LockWaiting a b c emptyWaiting = LockWaiting { lwAllocation = L.emptyAllocation , lwPending = M.empty , lwPendingOwners = M.empty } -- | Get the set of owners with pending lock requests. getPendingOwners :: LockWaiting a b c -> S.Set b getPendingOwners = M.keysSet . lwPendingOwners -- | Predicate on whether an owner has a pending lock request. hasPendingRequest :: Ord b => b -> LockWaiting a b c -> Bool hasPendingRequest owner = M.member owner . lwPendingOwners -- | Get the allocation state from the waiting state getAllocation :: LockWaiting a b c -> L.LockAllocation a b getAllocation = lwAllocation -- | Get the list of all pending requests. getPendingRequests :: (Ord a, Ord b, Ord c) => LockWaiting a b c -> S.Set (c, b, [L.LockRequest a]) getPendingRequests = S.unions . M.elems . lwPending -- | Type of the extensional representation of a LockWaiting. type ExtWaiting a b c = (L.LockAllocation a b, S.Set (c, b, [L.LockRequest a])) -- | Get a representation, comparable by (==), that captures the extensional -- behaviour. In other words, @(==) `on` extRepr@ is a bisumlation. extRepr :: (Ord a, Ord b, Ord c) => LockWaiting a b c -> ExtWaiting a b c extRepr = getAllocation &&& getPendingRequests -- | Internal function to fulfill one request if possible, and keep track of -- the owners to be notified. The type is chosen to be suitable as fold -- operation. -- -- This function calls the later defined updateLocksWaiting', as they are -- mutually recursive. tryFulfillRequest :: (Lock a, Ord b, Ord c) => (LockWaiting a b c, S.Set b) -> (c, b, [L.LockRequest a]) -> (LockWaiting a b c, S.Set b) tryFulfillRequest (waiting, toNotify) (prio, owner, req) = let (waiting', (_, newNotify)) = updateLocksWaiting' prio owner req waiting in (waiting', toNotify `S.union` newNotify) -- | Internal function to recursively follow the consequences of a change. revisitRequests :: (Lock a, Ord b, Ord c) => S.Set b -- ^ the owners where the requests keyed by them -- already have been revisited -> S.Set b -- ^ the owners where requests keyed by them need -- to be revisited -> LockWaiting a b c -- ^ state before revisiting -> (S.Set b, LockWaiting a b c) -- ^ owners visited and state -- after revisiting revisitRequests notify todo state = let getRequests (pending, reqs) owner = (M.delete owner pending , fromMaybe S.empty (M.lookup owner pending) `S.union` reqs) (pending', requests) = S.foldl' getRequests (lwPending state, S.empty) todo revisitedOwners = S.map (\(_, o, _) -> o) requests pendingOwners' = S.foldl' (flip M.delete) (lwPendingOwners state) revisitedOwners state' = state { lwPending = pending', lwPendingOwners = pendingOwners' } (!state'', !notify') = S.foldl' tryFulfillRequest (state', notify) requests done = notify `S.union` todo !newTodo = notify' S.\\ done in if S.null todo then (notify, state) else revisitRequests done newTodo state'' -- | Update the locks on an onwer according to the given request, if possible. -- Additionally (if the request succeeds) fulfill any pending requests that -- became possible through this request. Return the new state of the waiting -- structure, the result of the operation, and a list of owner whose requests -- have been fulfilled. The result is, as for lock allocation, the set of owners -- the request is blocked on. Again, the type is chosen to be suitable for use -- in atomicModifyIORef. updateLocks' :: (Lock a, Ord b, Ord c) => b -> [L.LockRequest a] -> LockWaiting a b c -> (LockWaiting a b c, (Result (S.Set b), S.Set b)) updateLocks' owner reqs state = let (!allocation', !result) = L.updateLocks owner reqs (lwAllocation state) state' = state { lwAllocation = allocation' } (!notify, !state'') = revisitRequests S.empty (S.singleton owner) state' in if M.member owner $ lwPendingOwners state then ( state , (Bad "cannot update locks while having pending requests", S.empty) ) else if result /= Ok S.empty -- skip computation if request could not -- be handled anyway then (state, (result, S.empty)) else let pendingOwners' = lwPendingOwners state'' toNotify = S.filter (not . flip M.member pendingOwners') notify in (state'', (result, toNotify)) -- | Update locks as soon as possible. If the request cannot be fulfilled -- immediately add the request to the waiting queue. The first argument is -- the priority at which the owner is waiting, the remaining are as for -- updateLocks', and so is the output. updateLocksWaiting' :: (Lock a, Ord b, Ord c) => c -> b -> [L.LockRequest a] -> LockWaiting a b c -> (LockWaiting a b c, (Result (S.Set b), S.Set b)) updateLocksWaiting' prio owner reqs state = let (state', (result, notify)) = updateLocks' owner reqs state !state'' = case result of Bad _ -> state' -- bad requests cannot be queued Ok empty | S.null empty -> state' Ok blocked -> let blocker = S.findMin blocked owners = M.insert owner (blocker, (prio, owner, reqs)) $ lwPendingOwners state pendingEntry = S.insert (prio, owner, reqs) . fromMaybe S.empty . M.lookup blocker $ lwPending state pending = M.insert blocker pendingEntry $ lwPending state in state' { lwPendingOwners = owners , lwPending = pending } in (state'', (result, notify)) -- | Predicate whether a request is already fulfilled in a given state -- and no requests for that owner are pending. requestFulfilled :: (Ord a, Ord b) => b -> [L.LockRequest a] -> LockWaiting a b c -> Bool requestFulfilled owner req state = let locks = L.listLocks owner $ lwAllocation state isFulfilled r = M.lookup (L.lockAffected r) locks == L.lockRequestType r in not (hasPendingRequest owner state) && all isFulfilled req -- | Update the locks on an onwer according to the given request, if possible. -- Additionally (if the request succeeds) fulfill any pending requests that -- became possible through this request. Return the new state of the waiting -- structure, the result of the operation, and a list of owners to be notified. -- The result is, as for lock allocation, the set of owners the request is -- blocked on. Again, the type is chosen to be suitable for use in -- atomicModifyIORef. -- For convenience, fulfilled requests are always accepted. updateLocks :: (Lock a, Ord b, Ord c) => b -> [L.LockRequest a] -> LockWaiting a b c -> (LockWaiting a b c, (Result (S.Set b), S.Set b)) updateLocks owner req state = if requestFulfilled owner req state then (state, (Ok S.empty, S.empty)) else second (second $ S.delete owner) $ updateLocks' owner req state -- | Update locks as soon as possible. If the request cannot be fulfilled -- immediately add the request to the waiting queue. The first argument is -- the priority at which the owner is waiting, the remaining are as for -- updateLocks, and so is the output. -- For convenience, fulfilled requests are always accepted. updateLocksWaiting :: (Lock a, Ord b, Ord c) => c -> b -> [L.LockRequest a] -> LockWaiting a b c -> (LockWaiting a b c, (Result (S.Set b), S.Set b)) updateLocksWaiting prio owner req state = if requestFulfilled owner req state then (state, (Ok S.empty, S.empty)) else second (second $ S.delete owner) $ updateLocksWaiting' prio owner req state -- | Compute the state of a waiting after an owner gives up -- on his pending request. removePendingRequest :: (Lock a, Ord b, Ord c) => b -> LockWaiting a b c -> LockWaiting a b c removePendingRequest owner state = let pendingOwners = lwPendingOwners state pending = lwPending state in case M.lookup owner pendingOwners of Nothing -> state Just (blocker, entry) -> let byBlocker = fromMaybe S.empty . M.lookup blocker $ pending byBlocker' = S.delete entry byBlocker pending' = if S.null byBlocker' then M.delete blocker pending else M.insert blocker byBlocker' pending in state { lwPendingOwners = M.delete owner pendingOwners , lwPending = pending' } -- | A repeatable version of `updateLocksWaiting`. If the owner has a pending -- request and the pending request is equal to the current one, do nothing; -- otherwise call updateLocksWaiting. safeUpdateLocksWaiting :: (Lock a, Ord b, Ord c) => c -> b -> [L.LockRequest a] -> LockWaiting a b c -> (LockWaiting a b c, (Result (S.Set b), S.Set b)) safeUpdateLocksWaiting prio owner req state = if hasPendingRequest owner state && S.singleton req == (S.map (\(_, _, r) -> r) . S.filter (\(_, b, _) -> b == owner) $ getPendingRequests state) then let (_, answer) = updateLocksWaiting prio owner req $ removePendingRequest owner state in (state, answer) else updateLocksWaiting prio owner req state -- | Convenience function to release all pending requests and locks -- of a given owner. Return the new configuration and the owners to -- notify. releaseResources :: (Lock a, Ord b, Ord c) => b -> LockWaiting a b c -> (LockWaiting a b c, S.Set b) releaseResources owner state = let state' = removePendingRequest owner state request = map L.requestRelease . M.keys . L.listLocks owner $ getAllocation state' (state'', (_, notify)) = updateLocks owner request state' in (state'', notify) -- | Obtain a LockWaiting from its extensional representation. fromExtRepr :: (Lock a, Ord b, Ord c) => ExtWaiting a b c -> LockWaiting a b c fromExtRepr (alloc, pending) = S.foldl' (\s (prio, owner, req) -> fst $ updateLocksWaiting prio owner req s) (emptyWaiting { lwAllocation = alloc }) pending instance (Lock a, J.JSON a, Ord b, J.JSON b, Show b, Ord c, J.JSON c) => J.JSON (LockWaiting a b c) where showJSON = J.showJSON . extRepr readJSON = liftM fromExtRepr . J.readJSON -- | Manipulate a all locks of an owner that have a given property. Also -- drop all pending requests. manipulateLocksPredicate :: (Lock a, Ord b, Ord c) => (a -> L.LockRequest a) -> (a -> Bool) -> b -> LockWaiting a b c -> (LockWaiting a b c, S.Set b) manipulateLocksPredicate req prop owner state = second snd . flip (updateLocks owner) (removePendingRequest owner state) . map req . filter prop . M.keys . L.listLocks owner $ getAllocation state -- | Free all Locks of a given owner satisfying a given predicate. As this -- operation is supposed to unconditionally suceed, all pending requests -- are dropped as well. freeLocksPredicate :: (Lock a, Ord b, Ord c) => (a -> Bool) -> b -> LockWaiting a b c -> (LockWaiting a b c, S.Set b) freeLocksPredicate = manipulateLocksPredicate L.requestRelease -- | Downgrade all locks of a given owner that satisfy a given predicate. As -- this operation is supposed to unconditionally suceed, all pending requests -- are dropped as well. downGradeLocksPredicate :: (Lock a, Ord b, Ord c) => (a -> Bool) -> b -> LockWaiting a b c -> (LockWaiting a b c, S.Set b) downGradeLocksPredicate = manipulateLocksPredicate L.requestShared -- | Intersect locks to a given set. intersectLocks :: (Lock a, Ord b, Ord c) => [a] -> b -> LockWaiting a b c -> (LockWaiting a b c, S.Set b) intersectLocks locks = freeLocksPredicate (not . flip elem locks) -- | Opprotunistically allocate locks for a given owner; return the set -- of newly actually acquired locks (i.e., locks already held before are -- not mentioned). opportunisticLockUnion :: (Lock a, Ord b, Ord c) => b -> [(a, L.OwnerState)] -> LockWaiting a b c -> (LockWaiting a b c, ([a], S.Set b)) opportunisticLockUnion owner reqs state = let locks = L.listLocks owner $ getAllocation state reqs' = sort $ filter (uncurry (<) . (flip M.lookup locks *** Just)) reqs maybeAllocate (s, success) (lock, ownstate) = let (s', (result, _)) = updateLocks owner [(if ownstate == L.OwnShared then L.requestShared else L.requestExclusive) lock] s in (s', if result == Ok S.empty then lock:success else success) in second (flip (,) S.empty) $ foldl' maybeAllocate (state, []) reqs' -- | A guarded version of opportunisticLockUnion; if the number of fulfilled -- requests is not at least the given amount, then do not change anything. guardedOpportunisticLockUnion :: (Lock a, Ord b, Ord c) => Int -> b -> [(a, L.OwnerState)] -> LockWaiting a b c -> (LockWaiting a b c, ([a], S.Set b)) guardedOpportunisticLockUnion count owner reqs state = let (state', (acquired, toNotify)) = opportunisticLockUnion owner reqs state in if length acquired < count then (state, ([], S.empty)) else (state', (acquired, toNotify)) ganeti-3.1.0~rc2/src/Ganeti/Logging.hs000064400000000000000000000154661476477700300175400ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, StandaloneDeriving, GeneralizedNewtypeDeriving #-} {-| Implementation of the Ganeti logging functionality. This currently lacks the following (FIXME): - log file reopening Note that this requires the hslogger library version 1.1 and above. -} {- Copyright (C) 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Logging ( setupLogging , MonadLog(..) , Priority(..) , logDebug , logInfo , logNotice , logWarning , logError , logCritical , logAlert , logEmergency , SyslogUsage(..) , syslogUsageToRaw , syslogUsageFromRaw , withErrorLogAt , isDebugMode ) where import Control.Monad import Control.Monad.Except (MonadError(..), catchError) import Control.Monad.Reader import qualified Control.Monad.RWS.Strict as RWSS import qualified Control.Monad.State.Strict as SS import Control.Monad.Trans.Identity import Control.Monad.Trans.Maybe import System.Log.Logger import System.Log.Handler.Simple import System.Log.Handler.Syslog import System.Log.Handler (setFormatter, LogHandler) import System.Log.Formatter import System.IO import Ganeti.BasicTypes (Error(..), ResultT(..)) import Ganeti.THH import qualified Ganeti.ConstantUtils as ConstantUtils -- | Syslog usage type. $(declareLADT ''String "SyslogUsage" [ ("SyslogNo", "no") , ("SyslogYes", "yes") , ("SyslogOnly", "only") ]) -- | Builds the log formatter. logFormatter :: String -- ^ Program -> Bool -- ^ Multithreaded -> Bool -- ^ Syslog -> LogFormatter a logFormatter prog mt syslog = let parts = [ if syslog then "[$pid]:" else "$time: " ++ prog ++ " pid=$pid" , if mt then if syslog then " ($tid)" else "/$tid" else "" , " $prio $msg" ] in tfLogFormatter "%F %X,%q %Z" $ concat parts -- | Helper to open and set the formatter on a log if enabled by a -- given condition, otherwise returning an empty list. openFormattedHandler :: (LogHandler a) => Bool -> LogFormatter a -> IO a -> IO [a] openFormattedHandler False _ _ = return [] openFormattedHandler True fmt opener = do handler <- opener return [setFormatter handler fmt] -- | Sets up the logging configuration. setupLogging :: Maybe String -- ^ Log file -> String -- ^ Program name -> Bool -- ^ Debug level -> Bool -- ^ Log to stderr -> Bool -- ^ Log to console -> SyslogUsage -- ^ Syslog usage -> IO () setupLogging logf program debug stderr_logging console syslog = do let level = if debug then DEBUG else INFO destf = if console then Just ConstantUtils.devConsole else logf fmt = logFormatter program True False file_logging = syslog /= SyslogOnly updateGlobalLogger rootLoggerName (setLevel level) stderr_handlers <- openFormattedHandler stderr_logging fmt $ streamHandler stderr level file_handlers <- case destf of Nothing -> return [] Just path -> openFormattedHandler file_logging fmt $ fileHandler path level let handlers = file_handlers ++ stderr_handlers updateGlobalLogger rootLoggerName $ setHandlers handlers -- syslog handler is special (another type, still instance of the -- typeclass, and has a built-in formatter), so we can't pass it in -- the above list when (syslog /= SyslogNo) $ do syslog_handler <- openlog program [PID] DAEMON INFO updateGlobalLogger rootLoggerName $ addHandler syslog_handler -- * Logging function aliases -- | A monad that allows logging. class Monad m => MonadLog m where -- | Log at a given level. logAt :: Priority -> String -> m () instance MonadLog IO where logAt = logM rootLoggerName deriving instance (MonadLog m) => MonadLog (IdentityT m) instance (MonadLog m) => MonadLog (MaybeT m) where logAt p = lift . logAt p instance (MonadLog m) => MonadLog (ReaderT r m) where logAt p = lift . logAt p instance (MonadLog m) => MonadLog (SS.StateT s m) where logAt p = lift . logAt p instance (MonadLog m, Monoid w) => MonadLog (RWSS.RWST r w s m) where logAt p = lift . logAt p instance (MonadLog m, Error e) => MonadLog (ResultT e m) where logAt p = lift . logAt p -- | Log at debug level. logDebug :: (MonadLog m) => String -> m () logDebug = logAt DEBUG -- | Log at info level. logInfo :: (MonadLog m) => String -> m () logInfo = logAt INFO -- | Log at notice level. logNotice :: (MonadLog m) => String -> m () logNotice = logAt NOTICE -- | Log at warning level. logWarning :: (MonadLog m) => String -> m () logWarning = logAt WARNING -- | Log at error level. logError :: (MonadLog m) => String -> m () logError = logAt ERROR -- | Log at critical level. logCritical :: (MonadLog m) => String -> m () logCritical = logAt CRITICAL -- | Log at alert level. logAlert :: (MonadLog m) => String -> m () logAlert = logAt ALERT -- | Log at emergency level. logEmergency :: (MonadLog m) => String -> m () logEmergency = logAt EMERGENCY -- | Check if the logging is at DEBUG level. -- DEBUG logging is unacceptable for production. isDebugMode :: IO Bool isDebugMode = (Just DEBUG ==) . getLevel <$> getRootLogger -- * Logging in an error monad with rethrowing errors -- | If an error occurs within a given computation, it annotated -- with a given message and logged and the error is re-thrown. withErrorLogAt :: (MonadLog m, MonadError e m, Show e) => Priority -> String -> m a -> m a withErrorLogAt prio msg = flip catchError $ \e -> do logAt prio (msg ++ ": " ++ show e) throwError e ganeti-3.1.0~rc2/src/Ganeti/Logging/000075500000000000000000000000001476477700300171705ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Logging/Lifted.hs000064400000000000000000000054631476477700300207430ustar00rootroot00000000000000{-| Ganeti logging functions expressed using MonadBase This allows to use logging functions without having instances for all possible transformers. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Logging.Lifted ( MonadLog() , Priority(..) , L.withErrorLogAt , L.isDebugMode , logAt , logDebug , logInfo , logNotice , logWarning , logError , logCritical , logAlert , logEmergency ) where import Control.Monad.Base import Ganeti.Logging (MonadLog, Priority(..)) import qualified Ganeti.Logging as L -- * Logging function aliases for MonadBase -- | A monad that allows logging. logAt :: (MonadLog b, MonadBase b m) => Priority -> String -> m () logAt p = liftBase . L.logAt p -- | Log at debug level. logDebug :: (MonadLog b, MonadBase b m) => String -> m () logDebug = logAt DEBUG -- | Log at info level. logInfo :: (MonadLog b, MonadBase b m) => String -> m () logInfo = logAt INFO -- | Log at notice level. logNotice :: (MonadLog b, MonadBase b m) => String -> m () logNotice = logAt NOTICE -- | Log at warning level. logWarning :: (MonadLog b, MonadBase b m) => String -> m () logWarning = logAt WARNING -- | Log at error level. logError :: (MonadLog b, MonadBase b m) => String -> m () logError = logAt ERROR -- | Log at critical level. logCritical :: (MonadLog b, MonadBase b m) => String -> m () logCritical = logAt CRITICAL -- | Log at alert level. logAlert :: (MonadLog b, MonadBase b m) => String -> m () logAlert = logAt ALERT -- | Log at emergency level. logEmergency :: (MonadLog b, MonadBase b m) => String -> m () logEmergency = logAt EMERGENCY ganeti-3.1.0~rc2/src/Ganeti/Logging/WriterLog.hs000064400000000000000000000114341476477700300214450ustar00rootroot00000000000000{-# LANGUAGE FlexibleInstances, FlexibleContexts, TypeFamilies, MultiParamTypeClasses, GeneralizedNewtypeDeriving, StandaloneDeriving, UndecidableInstances, CPP #-} {-| A pure implementation of MonadLog using MonadWriter -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Logging.WriterLog ( WriterLogT , WriterLog , runWriterLogT , runWriterLog , dumpLogSeq , execWriterLogT , execWriterLog ) where import Control.Applicative import Control.Monad import Control.Monad.Base import Control.Monad.IO.Class import Control.Monad.Trans.Control import Control.Monad.Writer import qualified Data.Foldable as F import Data.Functor.Identity import Data.Sequence import Ganeti.Logging -- * The data type of the monad transformer type LogSeq = Seq (Priority, String) type WriterSeq = WriterT LogSeq -- | A monad transformer that adds pure logging capability. newtype WriterLogT m a = WriterLogT { unwrapWriterLogT :: WriterSeq m a } deriving (Functor, Applicative, Alternative, Monad, MonadPlus, MonadIO, MonadTrans) deriving instance (MonadBase IO m) => MonadBase IO (WriterLogT m) type WriterLog = WriterLogT Identity -- Runs a 'WriterLogT', returning the result and accumulated messages. runWriterLogT :: WriterLogT m a -> m (a, LogSeq) runWriterLogT = runWriterT . unwrapWriterLogT -- Runs a 'WriterLog', returning the result and accumulated messages. runWriterLog :: WriterLog a -> (a, LogSeq) runWriterLog = runIdentity . runWriterLogT -- | Runs a 'WriterLogT', and when it finishes, resends all log messages -- to the underlying monad that implements 'MonadLog'. -- -- This can be used to delay logging messages, by accumulating them -- in 'WriterLogT', and resending them at the end to the underlying monad. execWriterLogT :: (MonadLog m) => WriterLogT m a -> m a execWriterLogT k = do (r, msgs) <- runWriterLogT k F.mapM_ (uncurry logAt) msgs return r -- | Sends all log messages to the a monad that implements 'MonadLog'. dumpLogSeq :: (MonadLog m) => LogSeq -> m () dumpLogSeq = F.mapM_ (uncurry logAt) -- | Runs a 'WriterLog', and when it finishes, resends all log messages -- to the a monad that implements 'MonadLog'. execWriterLog :: (MonadLog m) => WriterLog a -> m a execWriterLog k = do let (r, msgs) = runWriterLog k dumpLogSeq msgs return r instance (Monad m) => MonadLog (WriterLogT m) where logAt = curry (WriterLogT . tell . singleton) instance MonadTransControl WriterLogT where #if MIN_VERSION_monad_control(1,0,0) -- Needs Undecidable instances type StT WriterLogT a = (a, LogSeq) liftWith f = WriterLogT . WriterT $ liftM (\x -> (x, mempty)) (f runWriterLogT) restoreT = WriterLogT . WriterT #else newtype StT WriterLogT a = StWriterLog { unStWriterLog :: (a, LogSeq) } liftWith f = WriterLogT . WriterT $ liftM (\x -> (x, mempty)) (f $ liftM StWriterLog . runWriterLogT) restoreT = WriterLogT . WriterT . liftM unStWriterLog #endif {-# INLINE liftWith #-} {-# INLINE restoreT #-} instance (MonadBaseControl IO m) => MonadBaseControl IO (WriterLogT m) where #if MIN_VERSION_monad_control(1,0,0) -- Needs Undecidable instances type StM (WriterLogT m) a = ComposeSt WriterLogT m a liftBaseWith = defaultLiftBaseWith restoreM = defaultRestoreM #else newtype StM (WriterLogT m) a = StMWriterLog { runStMWriterLog :: ComposeSt WriterLogT m a } liftBaseWith = defaultLiftBaseWith StMWriterLog restoreM = defaultRestoreM runStMWriterLog #endif {-# INLINE liftBaseWith #-} {-# INLINE restoreM #-} ganeti-3.1.0~rc2/src/Ganeti/Luxi.hs000064400000000000000000000324471476477700300170710ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Implementation of the Ganeti LUXI interface. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Luxi ( LuxiOp(..) , LuxiReq(..) , Client , Server , JobId , fromJobId , makeJobId , RecvResult(..) , strOfOp , opToArgs , getLuxiClient , getLuxiServer , acceptClient , closeClient , closeServer , callMethod , submitManyJobs , queryJobsStatus , buildCall , buildResponse , decodeLuxiCall , recvMsg , recvMsgExt , sendMsg , allLuxiCalls ) where import Data.Maybe (listToMaybe) import Control.Applicative (optional, liftA, (<|>)) import Control.Monad import qualified Text.JSON as J import Text.JSON.Pretty (pp_value) import Text.JSON.Types import Ganeti.BasicTypes import Ganeti.Constants import Ganeti.Errors import Ganeti.JSON (fromJResult, fromJVal, Tuple5(..), MaybeForJSON(..), TimeAsDoubleJSON(..)) import Ganeti.UDSServer import Ganeti.Objects import Ganeti.OpParams (pTagsObject) import Ganeti.OpCodes import qualified Ganeti.Query.Language as Qlang import Ganeti.Runtime (GanetiDaemon(..), GanetiGroup(..), MiscGroup(..)) import Ganeti.THH import Ganeti.THH.Field import Ganeti.THH.Types (getOneTuple) import Ganeti.Types import Ganeti.Utils -- | Currently supported Luxi operations and JSON serialization. $(genLuxiOp "LuxiOp" [ (luxiReqQuery, [ simpleField "what" [t| Qlang.ItemType |] , simpleField "fields" [t| [String] |] , simpleField "qfilter" [t| Qlang.Filter Qlang.FilterField |] ]) , (luxiReqQueryFields, [ simpleField "what" [t| Qlang.ItemType |] , simpleField "fields" [t| [String] |] ]) , (luxiReqQueryNodes, [ simpleField "names" [t| [String] |] , simpleField "fields" [t| [String] |] , simpleField "lock" [t| Bool |] ]) , (luxiReqQueryGroups, [ simpleField "names" [t| [String] |] , simpleField "fields" [t| [String] |] , simpleField "lock" [t| Bool |] ]) , (luxiReqQueryNetworks, [ simpleField "names" [t| [String] |] , simpleField "fields" [t| [String] |] , simpleField "lock" [t| Bool |] ]) , (luxiReqQueryInstances, [ simpleField "names" [t| [String] |] , simpleField "fields" [t| [String] |] , simpleField "lock" [t| Bool |] ]) , (luxiReqQueryFilters, [ simpleField "uuids" [t| [String] |] , simpleField "fields" [t| [String] |] ]) , (luxiReqReplaceFilter, -- UUID is missing for insert, present for upsert [ optionalNullSerField $ simpleField "uuid" [t| String |] , simpleField "priority" [t| NonNegative Int |] , simpleField "predicates" [t| [FilterPredicate] |] , simpleField "action" [t| FilterAction |] , simpleField "reason" [t| ReasonTrail |] ]) , (luxiReqDeleteFilter, [ simpleField "uuid" [t| String |] ]) , (luxiReqQueryJobs, [ simpleField "ids" [t| [JobId] |] , simpleField "fields" [t| [String] |] ]) , (luxiReqQueryExports, [ simpleField "nodes" [t| [String] |] , simpleField "lock" [t| Bool |] ]) , (luxiReqQueryConfigValues, [ simpleField "fields" [t| [String] |] ] ) , (luxiReqQueryClusterInfo, []) , (luxiReqQueryTags, [ pTagsObject , simpleField "name" [t| String |] ]) , (luxiReqSubmitJob, [ simpleField "job" [t| [MetaOpCode] |] ] ) , (luxiReqSubmitJobToDrainedQueue, [ simpleField "job" [t| [MetaOpCode] |] ] ) , (luxiReqSubmitManyJobs, [ simpleField "ops" [t| [[MetaOpCode]] |] ] ) , (luxiReqWaitForJobChange, [ simpleField "job" [t| JobId |] , simpleField "fields" [t| [String]|] , simpleField "prev_job" [t| JSValue |] , simpleField "prev_log" [t| JSValue |] , simpleField "tmout" [t| Int |] ]) , (luxiReqPickupJob, [ simpleField "job" [t| JobId |] ] ) , (luxiReqArchiveJob, [ simpleField "job" [t| JobId |] ] ) , (luxiReqAutoArchiveJobs, [ simpleField "age" [t| Int |] , simpleField "tmout" [t| Int |] ]) , (luxiReqCancelJob, [ simpleField "job" [t| JobId |] , simpleField "kill" [t| Bool |] ]) , (luxiReqChangeJobPriority, [ simpleField "job" [t| JobId |] , simpleField "priority" [t| Int |] ] ) , (luxiReqSetDrainFlag, [ simpleField "flag" [t| Bool |] ] ) , (luxiReqSetWatcherPause, [ optionalNullSerField $ timeAsDoubleField "duration" ] ) ]) $(makeJSONInstance ''LuxiReq) -- | List of all defined Luxi calls. $(genAllConstr (drop 3) ''LuxiReq "allLuxiCalls") -- | The serialisation of LuxiOps into strings in messages. $(genStrOfOp ''LuxiOp "strOfOp") luxiConnectConfig :: ServerConfig luxiConnectConfig = ServerConfig -- The rapi daemon talks to the luxi one, and for this -- purpose we need group rw permissions. FilePermissions { fpOwner = Just GanetiLuxid , fpGroup = Just $ ExtraGroup DaemonsGroup , fpPermissions = 0o0660 } ConnectConfig { recvTmo = luxiDefRwto , sendTmo = luxiDefRwto } -- | Connects to the master daemon and returns a luxi Client. getLuxiClient :: String -> IO Client getLuxiClient = connectClient (connConfig luxiConnectConfig) luxiDefCtmo -- | Creates and returns a server endpoint. getLuxiServer :: Bool -> FilePath -> IO Server getLuxiServer = connectServer luxiConnectConfig -- | Converts Luxi call arguments into a 'LuxiOp' data structure. -- This is used for building a Luxi 'Handler'. -- -- This is currently hand-coded until we make it more uniform so that -- it can be generated using TH. decodeLuxiCall :: JSValue -> JSValue -> Result LuxiOp decodeLuxiCall method args = do call <- fromJResult "Unable to parse LUXI request method" $ J.readJSON method case call of ReqQueryFilters -> do (uuids, fields) <- fromJVal args uuids' <- case uuids of JSNull -> return [] _ -> fromJVal uuids return $ QueryFilters uuids' fields ReqReplaceFilter -> do Tuple5 ( uuid , priority , predicates , action , reason) <- fromJVal args return $ ReplaceFilter (unMaybeForJSON uuid) priority predicates action reason ReqDeleteFilter -> do [uuid] <- fromJVal args return $ DeleteFilter uuid ReqQueryJobs -> do (jids, jargs) <- fromJVal args jids' <- case jids of JSNull -> return [] _ -> fromJVal jids return $ QueryJobs jids' jargs ReqQueryInstances -> do (names, fields, locking) <- fromJVal args return $ QueryInstances names fields locking ReqQueryNodes -> do (names, fields, locking) <- fromJVal args return $ QueryNodes names fields locking ReqQueryGroups -> do (names, fields, locking) <- fromJVal args return $ QueryGroups names fields locking ReqQueryClusterInfo -> return QueryClusterInfo ReqQueryNetworks -> do (names, fields, locking) <- fromJVal args return $ QueryNetworks names fields locking ReqQuery -> do (what, fields, qfilter) <- fromJVal args return $ Query what fields qfilter ReqQueryFields -> do (what, fields) <- fromJVal args fields' <- case fields of JSNull -> return [] _ -> fromJVal fields return $ QueryFields what fields' ReqSubmitJob -> do [ops1] <- fromJVal args ops2 <- mapM (fromJResult (luxiReqToRaw call) . J.readJSON) ops1 return $ SubmitJob ops2 ReqSubmitJobToDrainedQueue -> do [ops1] <- fromJVal args ops2 <- mapM (fromJResult (luxiReqToRaw call) . J.readJSON) ops1 return $ SubmitJobToDrainedQueue ops2 ReqSubmitManyJobs -> do [ops1] <- fromJVal args ops2 <- mapM (fromJResult (luxiReqToRaw call) . J.readJSON) ops1 return $ SubmitManyJobs ops2 ReqWaitForJobChange -> do (jid, fields, pinfo, pidx, wtmout) <- -- No instance for 5-tuple, code copied from the -- json sources and adapted fromJResult "Parsing WaitForJobChange message" $ case args of JSArray [a, b, c, d, e] -> (,,,,) `fmap` J.readJSON a `ap` J.readJSON b `ap` J.readJSON c `ap` J.readJSON d `ap` J.readJSON e _ -> J.Error "Not enough values" return $ WaitForJobChange jid fields pinfo pidx wtmout ReqPickupJob -> do [jid] <- fromJVal args return $ PickupJob jid ReqArchiveJob -> do [jid] <- fromJVal args return $ ArchiveJob jid ReqAutoArchiveJobs -> do (age, tmout) <- fromJVal args return $ AutoArchiveJobs age tmout ReqQueryExports -> do (nodes, lock) <- fromJVal args return $ QueryExports nodes lock ReqQueryConfigValues -> do [fields] <- fromJVal args return $ QueryConfigValues fields ReqQueryTags -> do (kind, name) <- fromJVal args return $ QueryTags kind name ReqCancelJob -> do (jid, kill) <- fromJVal args <|> liftA (flip (,) False . getOneTuple) (fromJVal args) return $ CancelJob jid kill ReqChangeJobPriority -> do (jid, priority) <- fromJVal args return $ ChangeJobPriority jid priority ReqSetDrainFlag -> do [flag] <- fromJVal args return $ SetDrainFlag flag ReqSetWatcherPause -> do duration <- optional $ do [x] <- fromJVal args liftM unTimeAsDoubleJSON $ fromJVal x return $ SetWatcherPause duration -- | Generic luxi method call callMethod :: LuxiOp -> Client -> IO (ErrorResult JSValue) callMethod method s = do sendMsg s $ buildCall (strOfOp method) (opToArgs method) result <- recvMsg s return $ parseResponse result -- | Parse job submission result. parseSubmitJobResult :: JSValue -> ErrorResult JobId parseSubmitJobResult (JSArray [JSBool True, v]) = case J.readJSON v of J.Error msg -> Bad $ LuxiError msg J.Ok v' -> Ok v' parseSubmitJobResult (JSArray [JSBool False, JSString x]) = Bad . LuxiError $ fromJSString x parseSubmitJobResult v = Bad . LuxiError $ "Unknown result from the master daemon: " ++ show (pp_value v) -- | Specialized submitManyJobs call. submitManyJobs :: Client -> [[MetaOpCode]] -> IO (ErrorResult [JobId]) submitManyJobs s jobs = do rval <- callMethod (SubmitManyJobs jobs) s -- map each result (status, payload) pair into a nice Result ADT return $ case rval of Bad x -> Bad x Ok (JSArray r) -> mapM parseSubmitJobResult r x -> Bad . LuxiError $ "Cannot parse response from Ganeti: " ++ show x -- | Custom queryJobs call. queryJobsStatus :: Client -> [JobId] -> IO (ErrorResult [JobStatus]) queryJobsStatus s jids = do rval <- callMethod (QueryJobs jids ["status"]) s return $ case rval of Bad x -> Bad x Ok y -> case J.readJSON y::(J.Result [[JobStatus]]) of J.Ok vals -> case mapM listToMaybe vals of Nothing -> Bad $ LuxiError "Missing job status field" Just headVals -> Ok headVals J.Error x -> Bad $ LuxiError x ganeti-3.1.0~rc2/src/Ganeti/Metad/000075500000000000000000000000001476477700300166345ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Metad/Config.hs000064400000000000000000000144551476477700300204060ustar00rootroot00000000000000{-# LANGUAGE FlexibleContexts #-} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Metad.Config where import Control.Arrow (second) import Control.Monad ((>=>), mzero) import Control.Monad.Trans import Control.Monad.Trans.Maybe import qualified Data.List as List (isPrefixOf) import qualified Data.Map as Map import Text.JSON import qualified Text.JSON as JSON import Ganeti.Constants as Constants import Ganeti.Metad.Types -- | Merges two instance configurations into one. -- -- In the case where instance IPs (i.e., map keys) are repeated, the -- old instance configuration is thrown away by 'Map.union' and -- replaced by the new configuration. As a result, the old private -- and secret OS parameters are completely lost. mergeConfig :: InstanceParams -> InstanceParams -> InstanceParams mergeConfig cfg1 cfg2 = cfg2 `Map.union` cfg1 -- | Extracts the OS parameters (public, private, secret) from a JSON -- object. -- -- This function checks whether the OS parameters are in fact a JSON -- object. getOsParams :: String -> String -> JSObject JSValue -> Result (JSObject JSValue) getOsParams key msg jsonObj = case lookup key (fromJSObject jsonObj) of Nothing -> Error $ "Could not find " ++ msg ++ " OS parameters" Just x -> readJSON x getPublicOsParams :: JSObject JSValue -> Result (JSObject JSValue) getPublicOsParams = getOsParams "osparams" "public" getPrivateOsParams :: JSObject JSValue -> Result (JSObject JSValue) getPrivateOsParams = getOsParams "osparams_private" "private" getSecretOsParams :: JSObject JSValue -> Result (JSObject JSValue) getSecretOsParams = getOsParams "osparams_secret" "secret" -- | Merges the OS parameters (public, private, secret) in a single -- data structure containing all parameters and their visibility. -- -- Example: -- -- > { "os-image": ["http://example.com/disk.img", "public"], -- > "os-password": ["mypassword", "secret"] } makeInstanceParams :: JSObject JSValue -> JSObject JSValue -> JSObject JSValue -> JSValue makeInstanceParams pub priv sec = JSObject . JSON.toJSObject $ addVisibility "public" pub ++ addVisibility "private" priv ++ addVisibility "secret" sec where key = JSString . JSON.toJSString addVisibility param params = map (second (JSArray . (:[key param]))) (JSON.fromJSObject params) getOsParamsWithVisibility :: JSValue -> Result JSValue getOsParamsWithVisibility json = do obj <- readJSON json publicOsParams <- getPublicOsParams obj privateOsParams <- getPrivateOsParams obj secretOsParams <- getSecretOsParams obj Ok $ makeInstanceParams publicOsParams privateOsParams secretOsParams -- | Finds the IP address of the instance communication NIC in the -- instance's NICs. If the corresponding NIC isn't found, 'Nothing' is returned. getInstanceCommunicationIp :: JSObject JSValue -> Result (Maybe String) getInstanceCommunicationIp = runMaybeT . (getNics >=> getInstanceCommunicationNic >=> getIp) where getIp :: JSObject JSValue -> MaybeT Result String getIp nic = case lookup "ip" (fromJSObject nic) of Nothing -> failErrorT "Could not find instance communication IP" Just (JSString ip) -> return (JSON.fromJSString ip) _ -> failErrorT "Instance communication IP is not a string" getInstanceCommunicationNic :: [JSValue] -> MaybeT Result (JSObject JSValue) getInstanceCommunicationNic [] = mzero -- no communication NIC found getInstanceCommunicationNic (JSObject nic : nics) = case lookup "name" (fromJSObject nic) of Just (JSString name) | Constants.instanceCommunicationNicPrefix `List.isPrefixOf` JSON.fromJSString name -> return nic _ -> getInstanceCommunicationNic nics getInstanceCommunicationNic (n : _) = failErrorT $ "Found wrong data in instance NICs: " ++ show n getNics :: JSObject JSValue -> MaybeT Result [JSValue] getNics jsonObj = case lookup "nics" (fromJSObject jsonObj) of Nothing -> failErrorT "Could not find OS parameters key 'nics'" Just (JSArray nics) -> return nics _ -> failErrorT "Instance nics is not an array" -- | A helper function for failing a 'Result' wrapped in a monad -- transformer. failErrorT :: (MonadTrans t) => String -> t Result a failErrorT = lift . JSON.Error -- | Extracts the OS parameters from the instance's parameters and -- returns a data structure containing all the OS parameters and their -- visibility indexed by the instance's IP address which is used in -- the instance communication NIC. getInstanceParams :: JSValue -> Result (String, Maybe InstanceParams) getInstanceParams json = case json of JSObject jsonObj -> do name <- case lookup "name" (fromJSObject jsonObj) of Nothing -> failError "Could not find instance name" Just (JSString x) -> return (JSON.fromJSString x) _ -> failError "Name is not a string" m'ip <- getInstanceCommunicationIp jsonObj return (name, fmap (\ip -> Map.fromList [(ip, json)]) m'ip) _ -> failError "Expecting a dictionary" where failError = JSON.Error ganeti-3.1.0~rc2/src/Ganeti/Metad/ConfigCore.hs000064400000000000000000000112701476477700300212070ustar00rootroot00000000000000{-# LANGUAGE TupleSections, TemplateHaskell, CPP, UndecidableInstances, MultiParamTypeClasses, TypeFamilies, GeneralizedNewtypeDeriving, ImpredicativeTypes #-} {-| Functions of the metadata daemon exported for RPC -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Metad.ConfigCore where import Control.Concurrent.MVar.Lifted import Control.Monad.Base import Control.Monad.IO.Class import Control.Monad.Reader import Control.Monad.Trans.Control import Language.Haskell.TH (Name) import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Errors import qualified Ganeti.JSON as J import Ganeti.Logging as L import Ganeti.Metad.Config as Config import Ganeti.Metad.Types (InstanceParams) -- * The monad in which all the Metad functions execute data MetadHandle = MetadHandle { mhInstParams :: MVar InstanceParams } -- | A type alias for easier referring to the actual content of the monad -- when implementing its instances. type MetadMonadIntType = ReaderT MetadHandle IO -- | The internal part of the monad without error handling. newtype MetadMonadInt a = MetadMonadInt { getMetadMonadInt :: MetadMonadIntType a } deriving ( Functor, Applicative, Monad, MonadIO, MonadBase IO , L.MonadLog ) instance MonadBaseControl IO MetadMonadInt where #if MIN_VERSION_monad_control(1,0,0) -- Needs Undecidable instances type StM MetadMonadInt b = StM MetadMonadIntType b liftBaseWith f = MetadMonadInt $ liftBaseWith $ \r -> f (r . getMetadMonadInt) restoreM = MetadMonadInt . restoreM #else newtype StM MetadMonadInt b = StMMetadMonadInt { runStMMetadMonadInt :: StM MetadMonadIntType b } liftBaseWith f = MetadMonadInt . liftBaseWith $ \r -> f (liftM StMMetadMonadInt . r . getMetadMonadInt) restoreM = MetadMonadInt . restoreM . runStMMetadMonadInt #endif -- | Runs the internal part of the MetadMonad monad on a given daemon -- handle. runMetadMonadInt :: MetadMonadInt a -> MetadHandle -> IO a runMetadMonadInt (MetadMonadInt k) = runReaderT k -- | The complete monad with error handling. type MetadMonad = ResultT GanetiException MetadMonadInt -- * Basic functions in the monad metadHandle :: MetadMonad MetadHandle metadHandle = lift . MetadMonadInt $ ask instParams :: MetadMonad InstanceParams instParams = readMVar . mhInstParams =<< metadHandle modifyInstParams :: (InstanceParams -> MetadMonad (InstanceParams, a)) -> MetadMonad a modifyInstParams f = do h <- metadHandle modifyMVar (mhInstParams h) f -- * Functions available to the RPC module -- Just a debugging function echo :: String -> MetadMonad String echo = return -- | Update the configuration with the received instance parameters. updateConfig :: J.JSValue -> MetadMonad () updateConfig input = do (name, m'instanceParams) <- J.fromJResultE "Could not get instance parameters" $ Config.getInstanceParams input case m'instanceParams of Nothing -> L.logInfo $ "No communication NIC for instance " ++ name ++ ", skipping" Just instanceParams -> do cfg' <- modifyInstParams $ \cfg -> let cfg' = mergeConfig cfg instanceParams in return (cfg', cfg') L.logInfo $ "Updated instance " ++ name ++ " configuration" L.logDebug $ "Instance configuration: " ++ show cfg' -- * The list of all functions exported to RPC. exportedFunctions :: [Name] exportedFunctions = [ 'echo , 'updateConfig ] ganeti-3.1.0~rc2/src/Ganeti/Metad/ConfigServer.hs000064400000000000000000000056471476477700300216000ustar00rootroot00000000000000{-# LANGUAGE TupleSections, TemplateHaskell #-} {-| Configuration server for the metadata daemon. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Metad.ConfigServer where import Control.Exception (finally) import Control.Monad (forever) import Ganeti.Path as Path import Ganeti.Daemon (DaemonOptions, cleanupSocket, describeError) import Ganeti.Runtime (GanetiDaemon(..), GanetiGroup(..), MiscGroup(..)) import Ganeti.THH.RPC import Ganeti.UDSServer (ConnectConfig(..), ServerConfig(..)) import qualified Ganeti.UDSServer as UDSServer import Ganeti.Utils (FilePermissions(..)) import Ganeti.Metad.ConfigCore -- * The handler that converts RPCs to calls to the above functions handler :: RpcServer MetadMonadInt handler = $( mkRpcM exportedFunctions ) -- * The main server code start :: DaemonOptions -> MetadHandle -> IO () start _ config = do socket_path <- Path.defaultMetadSocket cleanupSocket socket_path server <- describeError "binding to the socket" Nothing (Just socket_path) $ UDSServer.connectServer metadConfig True socket_path finally (forever $ runMetadMonadInt (UDSServer.listener handler server) config) (UDSServer.closeServer server) where metadConfig = ServerConfig -- The permission 0600 is completely acceptable because only the node -- daemon talks to the metadata daemon, and the node daemon runs as -- root. FilePermissions { fpOwner = Just GanetiMetad , fpGroup = Just $ ExtraGroup DaemonsGroup , fpPermissions = 0o0600 } ConnectConfig { recvTmo = 60 , sendTmo = 60 } ganeti-3.1.0~rc2/src/Ganeti/Metad/Server.hs000064400000000000000000000035061476477700300204420ustar00rootroot00000000000000{-| Metadata daemon server, which controls the configuration and web servers. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Metad.Server (start) where import Control.Concurrent import qualified Data.Map (empty) import Ganeti.Daemon (DaemonOptions) import Ganeti.Metad.ConfigCore (MetadHandle(..)) import qualified Ganeti.Metad.ConfigServer as ConfigServer import qualified Ganeti.Metad.WebServer as WebServer start :: DaemonOptions -> IO () start opts = do config <- newMVar Data.Map.empty _ <- forkIO $ WebServer.start opts config ConfigServer.start opts (MetadHandle config) ganeti-3.1.0~rc2/src/Ganeti/Metad/Types.hs000064400000000000000000000026431476477700300203010ustar00rootroot00000000000000{-| Metadata daemon types. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Metad.Types where import Data.Map (Map) import Text.JSON type InstanceParams = Map String JSValue ganeti-3.1.0~rc2/src/Ganeti/Metad/WebServer.hs000064400000000000000000000221761476477700300211040ustar00rootroot00000000000000{-# LANGUAGE FlexibleContexts, OverloadedStrings #-} {-| Web server for the metadata daemon. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Metad.WebServer (start) where import Control.Applicative import Control.Concurrent (MVar, readMVar) import Control.Monad.Base (MonadBase) import Control.Monad.IO.Class (liftIO) import Control.Exception.Lifted (catch, throwIO) import Control.Exception.Base (Exception) import Data.Typeable (Typeable) import qualified Data.CaseInsensitive as CI import Data.List (intercalate) import Data.Map (Map) import qualified Data.Map as Map import qualified Data.ByteString.Char8 as ByteString (pack, unpack) import Snap.Core import Snap.Util.FileServe import Snap.Http.Server import Text.JSON (JSValue, Result(..), JSObject) import qualified Text.JSON as JSON import System.FilePath (()) import Ganeti.Daemon import qualified Ganeti.Constants as Constants import qualified Ganeti.Logging as Logging import Ganeti.Runtime (GanetiDaemon(..), ExtraLogReason(..)) import qualified Ganeti.Runtime as Runtime import Ganeti.Metad.Config as Config import Ganeti.Metad.Types (InstanceParams) type MetaM = Snap () data MetaMExc = MetaMExc String deriving (Show, Typeable) instance Exception MetaMExc throwError :: MonadBase IO m => String -> m a throwError = throwIO . MetaMExc split :: String -> [String] split str = case span (/= '/') str of (x, []) -> [x] (x, _:xs) -> x:split xs lookupInstanceParams :: MonadBase IO m => String -> Map String b -> m b lookupInstanceParams inst params = case Map.lookup inst params of Nothing -> throwError $ "Could not get instance params for " ++ show inst Just x -> return x -- | The 404 "not found" error. error404 :: MetaM error404 = do modifyResponse $ setResponseStatus 404 "Not found" writeBS "Resource not found" -- | The 405 "method not allowed error", including the list of allowed methods. error405 :: [Method] -> MetaM error405 ms = modifyResponse $ addHeader (CI.mk "Allow") (ByteString.pack . intercalate ", " $ map show ms) . setResponseStatus 405 "Method not allowed" maybeResult :: MonadBase IO m => Result t -> (t -> m a) -> m a maybeResult (Error err) _ = throwError err maybeResult (Ok x) f = f x serveOsParams :: String -> Map String JSValue -> MetaM serveOsParams inst params = do instParams <- lookupInstanceParams inst params maybeResult (Config.getOsParamsWithVisibility instParams) $ \osParams -> writeBS . ByteString.pack . JSON.encode $ osParams serveOsPackage :: String -> Map String JSValue -> String -> MetaM serveOsPackage inst params key = do instParams <- lookupInstanceParams inst params maybeResult (JSON.readJSON instParams >>= Config.getPublicOsParams >>= getOsPackage) $ \package -> serveFile package `catch` \err -> throwError $ "Could not serve OS package: " ++ show (err :: IOError) where getOsPackage osParams = case lookup key (JSON.fromJSObject osParams) of Nothing -> Error $ "Could not find OS package for " ++ show inst Just x -> JSON.readJSON x serveOsScript :: String -> Map String JSValue -> String -> MetaM serveOsScript inst params script = do instParams <- lookupInstanceParams inst params maybeResult (getOsType instParams) $ \os -> if null os then throwError $ "There is no OS for " ++ show inst else serveScript os Constants.osSearchPath where getOsType instParams = do obj <- JSON.readJSON instParams :: Result (JSObject JSValue) case lookup "os" (JSON.fromJSObject obj) of Nothing -> Error $ "Could not find OS for " ++ show inst Just x -> JSON.readJSON x :: Result String serveScript :: String -> [String] -> MetaM serveScript os [] = throwError $ "Could not find OS script " ++ show (os script) serveScript os (d:ds) = serveFile (d os script) `catch` \err -> do let _ = err :: IOError serveScript os ds handleMetadata :: MVar InstanceParams -> Method -> String -> String -> String -> MetaM handleMetadata _ GET "ganeti" "latest" "meta_data.json" = liftIO $ Logging.logInfo "ganeti metadata" handleMetadata params GET "ganeti" "latest" "os/os-install-package" = do remoteAddr <- ByteString.unpack . rqClientAddr <$> getRequest instanceParams <- liftIO $ do Logging.logInfo $ "OS install package for " ++ show remoteAddr readMVar params serveOsPackage remoteAddr instanceParams "os-install-package" `catch` \err -> do let MetaMExc e = err liftIO . Logging.logWarning $ "Could not serve OS install package: " ++ e error404 handleMetadata params GET "ganeti" "latest" "os/package" = do remoteAddr <- ByteString.unpack . rqClientAddr <$> getRequest instanceParams <- liftIO $ do Logging.logInfo $ "OS package for " ++ show remoteAddr readMVar params serveOsPackage remoteAddr instanceParams "os-package" handleMetadata params GET "ganeti" "latest" "os/parameters.json" = do remoteAddr <- ByteString.unpack . rqClientAddr <$> getRequest instanceParams <- liftIO $ do Logging.logInfo $ "OS parameters for " ++ show remoteAddr readMVar params serveOsParams remoteAddr instanceParams `catch` \err -> do let MetaMExc e = err liftIO . Logging.logWarning $ "Could not serve OS parameters: " ++ e error404 handleMetadata params GET "ganeti" "latest" script | isScript script = do remoteAddr <- ByteString.unpack . rqClientAddr <$> getRequest instanceParams <- liftIO $ do Logging.logInfo $ "OS package for " ++ show remoteAddr readMVar params serveOsScript remoteAddr instanceParams (last $ split script) `catch` \err -> do let MetaMExc e = err liftIO . Logging.logWarning $ "Could not serve OS scripts: " ++ e error404 where isScript = (`elem` [ "os/scripts/create" , "os/scripts/export" , "os/scripts/import" , "os/scripts/rename" , "os/scripts/verify" ]) handleMetadata _ GET "ganeti" "latest" "read" = liftIO $ Logging.logInfo "ganeti READ" handleMetadata _ _ "ganeti" "latest" "read" = error405 [GET] handleMetadata _ POST "ganeti" "latest" "write" = liftIO $ Logging.logInfo "ganeti WRITE" handleMetadata _ _ "ganeti" "latest" "write" = error405 [POST] handleMetadata _ _ _ _ _ = error404 routeMetadata :: MVar InstanceParams -> MetaM routeMetadata params = route [ (providerRoute1, dispatchMetadata) , (providerRoute2, dispatchMetadata) ] <|> dispatchMetadata where provider = "provider" version = "version" providerRoute1 = ByteString.pack $ ':':provider ++ "/" ++ ':':version providerRoute2 = ByteString.pack $ ':':version getParamString :: String -> Snap String getParamString = fmap (maybe "" ByteString.unpack) . getParam . ByteString.pack dispatchMetadata = do m <- rqMethod <$> getRequest p <- getParamString provider v <- getParamString version r <- ByteString.unpack . rqPathInfo <$> getRequest handleMetadata params m p v r defaultHttpConf :: DaemonOptions -> FilePath -> FilePath -> Config Snap () defaultHttpConf opts accessLog errorLog = maybe id (setBind . ByteString.pack) (optBindAddress opts) . setAccessLog (ConfigFileLog accessLog) . setCompression False . setErrorLog (ConfigFileLog errorLog) . setPort (maybe Constants.defaultMetadPort fromIntegral (optPort opts)) . setVerbose False $ emptyConfig start :: DaemonOptions -> MVar InstanceParams -> IO () start opts params = do accessLog <- Runtime.daemonsExtraLogFile GanetiMetad AccessLog errorLog <- Runtime.daemonsExtraLogFile GanetiMetad ErrorLog httpServe (defaultHttpConf opts accessLog errorLog) (routeMetadata params) ganeti-3.1.0~rc2/src/Ganeti/Monitoring/000075500000000000000000000000001476477700300177275ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Monitoring/Server.hs000064400000000000000000000260771476477700300215450ustar00rootroot00000000000000{-# LANGUAGE OverloadedStrings #-} {-| Implementation of the Ganeti confd server functionality. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Monitoring.Server ( main , checkMain , prepMain , DataCollector(..) ) where import Control.Applicative import Control.DeepSeq (force) import Control.Exception.Base (evaluate) import Control.Monad import Control.Monad.IO.Class import Data.ByteString.Char8 (pack, unpack) import qualified Data.ByteString.UTF8 as UTF8 import Data.Maybe (fromMaybe) import Data.List (find) import qualified Data.Map as Map import qualified Data.PSQueue as Queue import Network.BSD (getServicePortNumber) import Snap.Core import Snap.Http.Server import qualified Text.JSON as J import Control.Concurrent import qualified Ganeti.BasicTypes as BT import Ganeti.Confd.Client import Ganeti.Confd.Types import qualified Ganeti.Confd.Types as CT import Ganeti.Daemon import qualified Ganeti.DataCollectors as DC import Ganeti.DataCollectors.Types import qualified Ganeti.JSON as GJ import Ganeti.Objects (DataCollectorConfig(..)) import qualified Ganeti.Constants as C import qualified Ganeti.ConstantUtils as CU import Ganeti.Runtime import Ganeti.Utils.Time (getCurrentTimeUSec) import Ganeti.Utils (withDefaultOnIOError) -- * Types and constants definitions type ConfigAccess = String -> DataCollectorConfig -- | Type alias for checkMain results. type CheckResult = () -- | Type alias for prepMain results. type PrepResult = Config Snap () -- | Version of the latest supported http API. latestAPIVersion :: Int latestAPIVersion = C.mondLatestApiVersion -- * Configuration handling -- | The default configuration for the HTTP server. defaultHttpConf :: FilePath -> FilePath -> Config Snap () defaultHttpConf accessLog errorLog = setAccessLog (ConfigFileLog accessLog) . setCompression False . setErrorLog (ConfigFileLog errorLog) $ setVerbose False emptyConfig -- * Helper functions -- | Check function for the monitoring agent. checkMain :: CheckFn CheckResult checkMain _ = return $ Right () -- | Prepare function for monitoring agent. prepMain :: PrepFn CheckResult PrepResult prepMain opts _ = do accessLog <- daemonsExtraLogFile GanetiMond AccessLog errorLog <- daemonsExtraLogFile GanetiMond ErrorLog defaultPort <- withDefaultOnIOError C.defaultMondPort . liftM fromIntegral $ getServicePortNumber C.mond return . setPort (maybe defaultPort fromIntegral (optPort opts)) . maybe id (setBind . pack) (optBindAddress opts) $ defaultHttpConf accessLog errorLog -- * Query answers -- | Reply to the supported API version numbers query. versionQ :: Snap () versionQ = writeBS . pack $ J.encode [latestAPIVersion] -- | Version 1 of the monitoring HTTP API. version1Api :: MVar CollectorMap -> MVar ConfigAccess -> Snap () version1Api mvar mvarConfig = let returnNull = writeBS . pack $ J.encode J.JSNull :: Snap () in ifTop returnNull <|> route [ ("list", listHandler mvarConfig) , ("report", reportHandler mvar mvarConfig) ] -- | Gives a lookup function for DataCollectorConfig that corresponds to the -- configuration known to RConfD. collectorConfigs :: ConfdClient -> IO ConfigAccess collectorConfigs confdClient = do response <- query confdClient CT.ReqDataCollectors CT.EmptyQuery return $ lookupConfig response where lookupConfig :: Maybe ConfdReply -> String -> DataCollectorConfig lookupConfig response name = fromMaybe (mempty :: DataCollectorConfig) $ do confdReply <- response let answer = CT.confdReplyAnswer confdReply case J.readJSON answer :: J.Result (GJ.Container DataCollectorConfig) of J.Error _ -> Nothing J.Ok container -> GJ.lookupContainer Nothing (UTF8.fromString name) container activeCollectors :: MVar ConfigAccess -> IO [DataCollector] activeCollectors mvarConfig = do configs <- readMVar mvarConfig return $ filter (dataCollectorActive . configs . dName) DC.collectors -- | Get the JSON representation of a data collector to be used in the collector -- list. dcListItem :: DataCollector -> J.JSValue dcListItem dc = J.JSArray [ J.showJSON $ dName dc , maybe defaultCategory J.showJSON $ dCategory dc , J.showJSON $ dKind dc ] where defaultCategory = J.showJSON C.mondDefaultCategory -- | Handler for returning lists. listHandler :: MVar ConfigAccess -> Snap () listHandler mvarConfig = dir "collectors" $ do collectors' <- liftIO $ activeCollectors mvarConfig writeBS . pack . J.encode $ map dcListItem collectors' -- | Handler for returning data collector reports. reportHandler :: MVar CollectorMap -> MVar ConfigAccess -> Snap () reportHandler mvar mvarConfig = route [ ("all", allReports mvar mvarConfig) , (":category/:collector", oneReport mvar mvarConfig) ] <|> errorReport -- | Return the report of all the available collectors. allReports :: MVar CollectorMap -> MVar ConfigAccess -> Snap () allReports mvar mvarConfig = do collectors' <- liftIO $ activeCollectors mvarConfig reports <- mapM (liftIO . getReport mvar) collectors' writeBS . pack . J.encode $ reports -- | Takes the CollectorMap and a DataCollector and returns the report for this -- collector. getReport :: MVar CollectorMap -> DataCollector -> IO DCReport getReport mvar collector = case dReport collector of StatelessR r -> r StatefulR r -> do colData <- getColData (dName collector) mvar r colData -- | Returns the data for the corresponding collector. getColData :: String -> MVar CollectorMap -> IO (Maybe CollectorData) getColData name mvar = do m <- readMVar mvar return $ Map.lookup name m -- | Returns a category given its name. -- If "collector" is given as the name, the collector has no category, and -- Nothing will be returned. catFromName :: String -> BT.Result (Maybe DCCategory) catFromName "instance" = BT.Ok $ Just DCInstance catFromName "storage" = BT.Ok $ Just DCStorage catFromName "daemon" = BT.Ok $ Just DCDaemon catFromName "hypervisor" = BT.Ok $ Just DCHypervisor catFromName "default" = BT.Ok Nothing catFromName _ = BT.Bad "No such category" errorReport :: Snap () errorReport = do modifyResponse $ setResponseStatus 404 "Not found" writeBS "Unable to produce a report for the requested resource" error404 :: Snap () error404 = do modifyResponse $ setResponseStatus 404 "Not found" writeBS "Resource not found" -- | Return the report of one collector. oneReport :: MVar CollectorMap -> MVar ConfigAccess -> Snap () oneReport mvar mvarConfig = do collectors' <- liftIO $ activeCollectors mvarConfig categoryName <- maybe mzero unpack <$> getParam "category" collectorName <- maybe mzero unpack <$> getParam "collector" category <- case catFromName categoryName of BT.Ok cat -> return cat BT.Bad msg -> fail msg collector <- case find (\col -> collectorName == dName col) $ filter (\c -> category == dCategory c) collectors' of Just col -> return col Nothing -> fail "Unable to find the requested collector" dcr <- liftIO $ getReport mvar collector writeBS . pack . J.encode $ dcr -- | The function implementing the HTTP API of the monitoring agent. monitoringApi :: MVar CollectorMap -> MVar ConfigAccess -> Snap () monitoringApi mvar mvarConfig = ifTop versionQ <|> dir "1" (version1Api mvar mvarConfig) <|> error404 -- | The function collecting data for each data collector providing a dcUpdate -- function. collect :: CollectorMap -> DataCollector -> IO CollectorMap collect m collector = case dUpdate collector of Nothing -> return m Just update -> do let name = dName collector existing = Map.lookup name m new_data <- update existing _ <- evaluate $ force new_data return $ Map.insert name new_data m -- | Invokes collect for each data collector. collection :: CollectorMap -> MVar ConfigAccess -> IO CollectorMap collection m mvarConfig = do collectors <- activeCollectors mvarConfig foldM collect m collectors -- | Convert seconds to microseconds seconds :: Int -> Integer seconds = (* 1000000) . fromIntegral -- | The thread responsible for the periodical collection of data for each data -- data collector. Note that even though the collectors might be deactivated, -- they will still be collected to provide a complete history. collectord :: MVar CollectorMap -> MVar ConfigAccess -> IO () collectord mvar mvarConfig = do let queue = Queue.fromAscList . map (Queue.:-> 0) $ CU.toList C.dataCollectorNames foldM_ update queue [0::Integer ..] where resetTimer configs = Queue.adjustWithKey ((+) . dataCollectorInterval . configs) resetAll configs = foldr (resetTimer configs) keyInList = flip . const . flip elem update q _ = do t <- getCurrentTimeUSec configs <- readMVar mvarConfig m <- takeMVar mvar let dueNames = map Queue.key $ Queue.atMost t q dueEntries = Map.filterWithKey (keyInList dueNames) m m' <- collection dueEntries mvarConfig let m'' = m' `Map.union` m putMVar mvar m'' let q' = resetAll configs q dueNames maxSleep = seconds C.mondTimeInterval nextWakeup = fromMaybe maxSleep . liftM Queue.prio $ Queue.findMin q' delay = min maxSleep nextWakeup threadDelay $ fromInteger delay return q' -- | Main function. main :: MainFn CheckResult PrepResult main _ _ httpConf = do mvarCollectorMap <- newMVar Map.empty mvarConfig <- newEmptyMVar confdClient <- getConfdClient Nothing Nothing void . forkIO . forever $ do configs <- collectorConfigs confdClient putMVar mvarConfig configs threadDelay . fromInteger $ seconds C.mondConfigTimeInterval takeMVar mvarConfig void . forkIO $ collectord mvarCollectorMap mvarConfig httpServe httpConf . method GET $ monitoringApi mvarCollectorMap mvarConfig ganeti-3.1.0~rc2/src/Ganeti/Network.hs000064400000000000000000000210311476477700300175640ustar00rootroot00000000000000{-# LANGUAGE Rank2Types #-} {-| Implementation of the Ganeti network objects. This is does not (yet) cover all methods that are provided in the corresponding python implementation (network.py). -} {- Copyright (C) 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Network ( PoolPart(..) , netIpv4NumHosts , ip4BaseAddr , getReservedCount , getFreeCount , isFull , getMap , isReserved , reserve , release , findFree , allReservations , reservations , extReservations ) where import Control.Monad import Control.Monad.Except import Control.Monad.State import Data.Bits ((.&.)) import Data.Function (on) import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.Lens import Ganeti.Objects import Ganeti.Objects.Lens import qualified Ganeti.Objects.BitArray as BA ip4BaseAddr :: Ip4Network -> Ip4Address ip4BaseAddr net = let m = ip4netMask net mask = 2^(32 :: Integer) - 2^(32 - m) in ip4AddressFromNumber . (.&.) mask . ip4AddressToNumber $ ip4netAddr net ipv4NumHosts :: (Integral n) => n -> Integer ipv4NumHosts mask = 2^(32 - mask) ipv4NetworkMinNumHosts :: Integer ipv4NetworkMinNumHosts = ipv4NumHosts C.ipv4NetworkMinSize ipv4NetworkMaxNumHosts :: Integer ipv4NetworkMaxNumHosts = ipv4NumHosts C.ipv4NetworkMaxSize data PoolPart = PoolInstances | PoolExt addressPoolIso :: Iso' AddressPool BA.BitArray addressPoolIso = iso apReservations AddressPool poolLens :: PoolPart -> Lens' Network (Maybe AddressPool) poolLens PoolInstances = networkReservationsL poolLens PoolExt = networkExtReservationsL poolArrayLens :: PoolPart -> Lens' Network (Maybe BA.BitArray) poolArrayLens part = poolLens part . mapping addressPoolIso netIpv4NumHosts :: Network -> Integer netIpv4NumHosts = ipv4NumHosts . ip4netMask . networkNetwork -- | Creates a new bit array pool of the appropriate size newPoolArray :: (MonadError e m, Error e) => Network -> m BA.BitArray newPoolArray net = do let numhosts = netIpv4NumHosts net when (numhosts > ipv4NetworkMaxNumHosts) . failError $ "A big network with " ++ show numhosts ++ " host(s) is currently" ++ " not supported, please specify at most a /" ++ show ipv4NetworkMaxNumHosts ++ " network" when (numhosts < ipv4NetworkMinNumHosts) . failError $ "A network with only " ++ show numhosts ++ " host(s) is too small," ++ " please specify at least a /" ++ show ipv4NetworkMinNumHosts ++ " network" return $ BA.zeroes (fromInteger numhosts) -- | Creates a new bit array pool of the appropriate size newPool :: (MonadError e m, Error e) => Network -> m AddressPool newPool = liftM AddressPool . newPoolArray -- | A helper function that creates a bit array pool, of it's missing. orNewPool :: (MonadError e m, Error e) => Network -> Maybe AddressPool -> m AddressPool orNewPool net = maybe (newPool net) return withPool :: (MonadError e m, Error e) => PoolPart -> (Network -> BA.BitArray -> m (a, BA.BitArray)) -> StateT Network m a withPool part f = StateT $ \n -> mapMOf2 (poolLens part) (f' n) n where f' net = liftM (over _2 Just) . mapMOf2 addressPoolIso (f net) <=< orNewPool net withPool_ :: (MonadError e m, Error e) => PoolPart -> (Network -> BA.BitArray -> m BA.BitArray) -> Network -> m Network withPool_ part f = execStateT $ withPool part ((liftM ((,) ()) .) . f) readPool :: PoolPart -> Network -> Maybe BA.BitArray readPool part = view (poolArrayLens part) readPoolE :: (MonadError e m, Error e) => PoolPart -> Network -> m BA.BitArray readPoolE part net = liftM apReservations $ orNewPool net (view (poolLens part) net) readAllE :: (MonadError e m, Error e) => Network -> m BA.BitArray readAllE net = do let toRes = liftM apReservations . orNewPool net res <- toRes $ networkReservations net ext <- toRes $ networkExtReservations net return $ res BA.-|- ext reservations :: Network -> Maybe BA.BitArray reservations = readPool PoolInstances extReservations :: Network -> Maybe BA.BitArray extReservations = readPool PoolExt -- | Get a bit vector of all reservations (internal and external) combined. allReservations :: Network -> Maybe BA.BitArray allReservations a = (BA.-|-) `liftM` reservations a `ap` extReservations a -- | Get the count of reserved addresses. getReservedCount :: Network -> Int getReservedCount = maybe 0 BA.count1 . allReservations -- | Get the count of free addresses. getFreeCount :: Network -> Int getFreeCount = maybe 0 BA.count0 . allReservations -- | Check whether the network is full. isFull :: Network -> Bool isFull = (0 ==) . getFreeCount -- | Return a textual representation of the network's occupation status. getMap :: Network -> String getMap = maybe "" (BA.asString '.' 'X') . allReservations -- * Functions used for manipulating the reservations -- | Returns an address index wrt a network. -- Fails if the address isn't in the network range. addrIndex :: (MonadError e m, Error e) => Ip4Address -> Network -> m Int addrIndex addr net = do let n = networkNetwork net i = on (-) ip4AddressToNumber addr (ip4BaseAddr n) when ((i < 0) || (i >= ipv4NumHosts (ip4netMask n))) . failError $ "Address '" ++ show addr ++ "' not in the network '" ++ show net ++ "'" return $ fromInteger i -- | Returns an address of a given index wrt a network. -- Fails if the index isn't in the network range. addrAt :: (MonadError e m, Error e) => Int -> Network -> m Ip4Address addrAt i net | (i' < 0) || (i' >= ipv4NumHosts (ip4netMask n)) = failError $ "Requested index " ++ show i ++ " outside the range of network '" ++ show net ++ "'" | otherwise = return $ ip4AddressFromNumber (ip4AddressToNumber (ip4BaseAddr n) + i') where n = networkNetwork net i' = toInteger i -- | Checks if a given address is reserved. -- Fails if the address isn't in the network range. isReserved :: (MonadError e m, Error e) => PoolPart -> Ip4Address -> Network -> m Bool isReserved part addr net = (BA.!) `liftM` readPoolE part net `ap` addrIndex addr net -- | Marks an address as used. reserve :: (MonadError e m, Error e) => PoolPart -> Ip4Address -> Network -> m Network reserve part addr = withPool_ part $ \net ba -> do idx <- addrIndex addr net let addrs = show addr when (ba BA.! idx) . failError $ case part of PoolExt -> "IP " ++ addrs ++ " is already externally reserved" PoolInstances -> "IP " ++ addrs ++ " is already used by an instance" BA.setAt idx True ba -- | Marks an address as unused. release :: (MonadError e m, Error e) => PoolPart -> Ip4Address -> Network -> m Network release part addr = withPool_ part $ \net ba -> do idx <- addrIndex addr net let addrs = show addr unless (ba BA.! idx) . failError $ case part of PoolExt -> "IP " ++ addrs ++ " is not externally reserved" PoolInstances -> "IP " ++ addrs ++ " is not used by an instance" BA.setAt idx False ba -- | Get the first free address in the network -- that satisfies a given predicate. findFree :: (MonadError e m, Error e) => (Ip4Address -> Bool) -> Network -> m (Maybe Ip4Address) findFree p net = readAllE net >>= BA.foldr f (return Nothing) where addrAtEither = addrAt :: Int -> Network -> Either String Ip4Address f False i _ | Right a <- addrAtEither i net, p a = return (Just a) f _ _ x = x ganeti-3.1.0~rc2/src/Ganeti/Objects.hs000064400000000000000000000655521476477700300175440ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, FunctionalDependencies #-} {-| Implementation of the Ganeti config objects. -} {- Copyright (C) 2011, 2012, 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Objects ( HvParams , OsParams , OsParamsPrivate , PartialNicParams(..) , FilledNicParams(..) , allNicParamFields , PartialNic(..) , FileDriver(..) , DataCollectorConfig(..) , DiskTemplate(..) , PartialBeParams(..) , FilledBeParams(..) , PartialNDParams(..) , FilledNDParams(..) , allNDParamFields , Node(..) , AllocPolicy(..) , FilledISpecParams(..) , PartialISpecParams(..) , allISpecParamFields , MinMaxISpecs(..) , FilledIPolicy(..) , PartialIPolicy(..) , GroupDiskParams , NodeGroup(..) , FilterAction(..) , FilterPredicate(..) , FilterRule(..) , filterRuleOrder , IpFamily(..) , ipFamilyToRaw , ipFamilyToVersion , fillDict , ClusterHvParams , OsHvParams , ClusterBeParams , ClusterOsParams , ClusterOsParamsPrivate , ClusterNicParams , UidPool , formatUidRange , UidRange , Cluster(..) , ConfigData(..) , TimeStampObject(..) -- re-exported from Types , UuidObject(..) -- re-exported from Types , SerialNoObject(..) -- re-exported from Types , TagsObject(..) -- re-exported from Types , DictObject(..) -- re-exported from THH , TagSet(..) -- re-exported from THH , emptyTagSet -- re-exported from THH , Network(..) , AddressPool(..) , Ip4Address() , mkIp4Address , Ip4Network() , mkIp4Network , ip4netAddr , ip4netMask , readIp4Address , ip4AddressToList , ip4AddressToNumber , ip4AddressFromNumber , nextIp4Address , IAllocatorParams , MasterNetworkParameters(..) , module Ganeti.PartialParams , module Ganeti.Objects.Disk , module Ganeti.Objects.Instance ) where import Control.Arrow (first) import Control.Monad.State import Control.Monad.Fail (MonadFail) import Control.Monad (liftM) import qualified Data.ByteString.UTF8 as UTF8 import Data.List (foldl', intercalate) import Data.Maybe import qualified Data.Map as Map import Data.Ord (comparing) import Data.Ratio (numerator, denominator) import qualified Data.Semigroup as Sem import Data.Tuple (swap) import Data.Word import Text.JSON (showJSON, readJSON, JSON, JSValue(..), fromJSString, toJSString) import qualified Text.JSON as J import qualified AutoConf import qualified Ganeti.Constants as C import qualified Ganeti.ConstantUtils as ConstantUtils import Ganeti.JSON (DictObject(..), Container, emptyContainer, GenericContainer) import Ganeti.Objects.BitArray (BitArray) import Ganeti.Objects.Disk import Ganeti.Objects.Nic import Ganeti.Objects.Instance import Ganeti.Query.Language import Ganeti.PartialParams import Ganeti.Types import Ganeti.THH import Ganeti.THH.Field import Ganeti.Utils (sepSplit, tryRead) -- * Generic definitions -- | Fills one map with keys from the other map, if not already -- existing. Mirrors objects.py:FillDict. fillDict :: (Ord k) => Map.Map k v -> Map.Map k v -> [k] -> Map.Map k v fillDict defaults custom skip_keys = let updated = Map.union custom defaults in foldl' (flip Map.delete) updated skip_keys -- * Network definitions -- ** Ipv4 types data Ip4Address = Ip4Address Word8 Word8 Word8 Word8 deriving (Eq, Ord) mkIp4Address :: (Word8, Word8, Word8, Word8) -> Ip4Address mkIp4Address (a, b, c, d) = Ip4Address a b c d instance Show Ip4Address where show (Ip4Address a b c d) = intercalate "." $ map show [a, b, c, d] readIp4Address :: (Applicative m, MonadFail m) => String -> m Ip4Address readIp4Address s = case sepSplit '.' s of [a, b, c, d] -> Ip4Address <$> tryRead "first octect" a <*> tryRead "second octet" b <*> tryRead "third octet" c <*> tryRead "fourth octet" d _ -> fail $ "Can't parse IPv4 address from string " ++ s instance JSON Ip4Address where showJSON = showJSON . show readJSON (JSString s) = readIp4Address (fromJSString s) readJSON v = fail $ "Invalid JSON value " ++ show v ++ " for an IPv4 address" -- Converts an address to a list of numbers ip4AddressToList :: Ip4Address -> [Word8] ip4AddressToList (Ip4Address a b c d) = [a, b, c, d] -- | Converts an address into its ordinal number. -- This is needed for indexing IP adresses in reservation pools. ip4AddressToNumber :: Ip4Address -> Integer ip4AddressToNumber = foldl (\n i -> 256 * n + toInteger i) 0 . ip4AddressToList -- | Converts a number into an address. -- This is needed for indexing IP adresses in reservation pools. ip4AddressFromNumber :: Integer -> Ip4Address ip4AddressFromNumber n = let s = state $ first fromInteger . swap . (`divMod` 256) (d, c, b, a) = evalState ((,,,) <$> s <*> s <*> s <*> s) n in Ip4Address a b c d nextIp4Address :: Ip4Address -> Ip4Address nextIp4Address = ip4AddressFromNumber . (+ 1) . ip4AddressToNumber -- | Custom type for an IPv4 network. data Ip4Network = Ip4Network { ip4netAddr :: Ip4Address , ip4netMask :: Word8 } deriving (Eq) mkIp4Network :: Ip4Address -> Word8 -> Ip4Network mkIp4Network = Ip4Network instance Show Ip4Network where show (Ip4Network ip netmask) = show ip ++ "/" ++ show netmask -- | JSON instance for 'Ip4Network'. instance JSON Ip4Network where showJSON = showJSON . show readJSON (JSString s) = case sepSplit '/' (fromJSString s) of [ip, nm] -> do ip' <- readIp4Address ip nm' <- tryRead "parsing netmask" nm if nm' >= 0 && nm' <= 32 then return $ Ip4Network ip' nm' else fail $ "Invalid netmask " ++ show nm' ++ " from string " ++ fromJSString s _ -> fail $ "Can't parse IPv4 network from string " ++ fromJSString s readJSON v = fail $ "Invalid JSON value " ++ show v ++ " for an IPv4 network" -- ** Address pools -- | Currently address pools just wrap a reservation 'BitArray'. -- -- In future, 'Network' might be extended to include several address pools -- and address pools might include their own ranges of addresses. newtype AddressPool = AddressPool { apReservations :: BitArray } deriving (Eq, Ord, Show) instance JSON AddressPool where showJSON = showJSON . apReservations readJSON = liftM AddressPool . readJSON -- ** Ganeti \"network\" config object. -- FIXME: Not all types might be correct here, since they -- haven't been exhaustively deduced from the python code yet. -- -- FIXME: When parsing, check that the ext_reservations and reservations -- have the same length $(buildObject "Network" "network" $ [ simpleField "name" [t| NonEmptyString |] , optionalField $ simpleField "mac_prefix" [t| String |] , simpleField "network" [t| Ip4Network |] , optionalField $ simpleField "network6" [t| String |] , optionalField $ simpleField "gateway" [t| Ip4Address |] , optionalField $ simpleField "gateway6" [t| String |] , optionalField $ simpleField "reservations" [t| AddressPool |] , optionalField $ simpleField "ext_reservations" [t| AddressPool |] ] ++ uuidFields ++ timeStampFields ++ serialFields ++ tagsFields) instance SerialNoObject Network where serialOf = networkSerial instance TagsObject Network where tagsOf = networkTags instance UuidObject Network where uuidOf = UTF8.toString . networkUuid instance TimeStampObject Network where cTimeOf = networkCtime mTimeOf = networkMtime -- * Datacollector definitions type MicroSeconds = Integer -- | The configuration regarding a single data collector. $(buildObject "DataCollectorConfig" "dataCollector" [ simpleField "active" [t| Bool|], simpleField "interval" [t| MicroSeconds |] ]) -- | Central default values of the data collector config. instance Sem.Semigroup DataCollectorConfig where _ <> a = a instance Monoid DataCollectorConfig where mempty = DataCollectorConfig { dataCollectorActive = True , dataCollectorInterval = 10^(6::Integer) * fromIntegral C.mondTimeInterval } mappend = (Sem.<>) -- * IPolicy definitions $(buildParam "ISpec" "ispec" [ simpleField ConstantUtils.ispecMemSize [t| Int |] , simpleField ConstantUtils.ispecDiskSize [t| Int |] , simpleField ConstantUtils.ispecDiskCount [t| Int |] , simpleField ConstantUtils.ispecCpuCount [t| Int |] , simpleField ConstantUtils.ispecNicCount [t| Int |] , simpleField ConstantUtils.ispecSpindleUse [t| Int |] ]) $(buildObject "MinMaxISpecs" "mmis" [ renameField "MinSpec" $ simpleField "min" [t| FilledISpecParams |] , renameField "MaxSpec" $ simpleField "max" [t| FilledISpecParams |] ]) -- | Custom partial ipolicy. This is not built via buildParam since it -- has a special 2-level inheritance mode. $(buildObject "PartialIPolicy" "ipolicy" [ optionalField . renameField "MinMaxISpecsP" $ simpleField ConstantUtils.ispecsMinmax [t| [MinMaxISpecs] |] , optionalField . renameField "StdSpecP" $ simpleField "std" [t| PartialISpecParams |] , optionalField . renameField "SpindleRatioP" $ simpleField "spindle-ratio" [t| Double |] , optionalField . renameField "VcpuRatioP" $ simpleField "vcpu-ratio" [t| Double |] , optionalField . renameField "DiskTemplatesP" $ simpleField "disk-templates" [t| [DiskTemplate] |] ]) -- | Custom filled ipolicy. This is not built via buildParam since it -- has a special 2-level inheritance mode. $(buildObject "FilledIPolicy" "ipolicy" [ renameField "MinMaxISpecs" $ simpleField ConstantUtils.ispecsMinmax [t| [MinMaxISpecs] |] , renameField "StdSpec" $ simpleField "std" [t| FilledISpecParams |] , simpleField "spindle-ratio" [t| Double |] , simpleField "vcpu-ratio" [t| Double |] , simpleField "disk-templates" [t| [DiskTemplate] |] ]) -- | Custom filler for the ipolicy types. instance PartialParams FilledIPolicy PartialIPolicy where fillParams (FilledIPolicy { ipolicyMinMaxISpecs = fminmax , ipolicyStdSpec = fstd , ipolicySpindleRatio = fspindleRatio , ipolicyVcpuRatio = fvcpuRatio , ipolicyDiskTemplates = fdiskTemplates}) (PartialIPolicy { ipolicyMinMaxISpecsP = pminmax , ipolicyStdSpecP = pstd , ipolicySpindleRatioP = pspindleRatio , ipolicyVcpuRatioP = pvcpuRatio , ipolicyDiskTemplatesP = pdiskTemplates}) = FilledIPolicy { ipolicyMinMaxISpecs = fromMaybe fminmax pminmax , ipolicyStdSpec = maybe fstd (fillParams fstd) pstd , ipolicySpindleRatio = fromMaybe fspindleRatio pspindleRatio , ipolicyVcpuRatio = fromMaybe fvcpuRatio pvcpuRatio , ipolicyDiskTemplates = fromMaybe fdiskTemplates pdiskTemplates } toPartial (FilledIPolicy { ipolicyMinMaxISpecs = fminmax , ipolicyStdSpec = fstd , ipolicySpindleRatio = fspindleRatio , ipolicyVcpuRatio = fvcpuRatio , ipolicyDiskTemplates = fdiskTemplates}) = PartialIPolicy { ipolicyMinMaxISpecsP = Just fminmax , ipolicyStdSpecP = Just $ toPartial fstd , ipolicySpindleRatioP = Just fspindleRatio , ipolicyVcpuRatioP = Just fvcpuRatio , ipolicyDiskTemplatesP = Just fdiskTemplates } toFilled (PartialIPolicy { ipolicyMinMaxISpecsP = pminmax , ipolicyStdSpecP = pstd , ipolicySpindleRatioP = pspindleRatio , ipolicyVcpuRatioP = pvcpuRatio , ipolicyDiskTemplatesP = pdiskTemplates}) = FilledIPolicy <$> pminmax <*> (toFilled =<< pstd) <*> pspindleRatio <*> pvcpuRatio <*> pdiskTemplates -- * Node definitions $(buildParam "ND" "ndp" [ simpleField "oob_program" [t| String |] , simpleField "spindle_count" [t| Int |] , simpleField "exclusive_storage" [t| Bool |] , simpleField "ovs" [t| Bool |] , simpleField "ovs_name" [t| String |] , simpleField "ovs_link" [t| String |] , simpleField "ssh_port" [t| Int |] , simpleField "cpu_speed" [t| Double |] ]) -- | Disk state parameters. -- -- As according to the documentation this option is unused by Ganeti, -- the content is just a 'JSValue'. type DiskState = Container JSValue -- | Hypervisor state parameters. -- -- As according to the documentation this option is unused by Ganeti, -- the content is just a 'JSValue'. type HypervisorState = Container JSValue $(buildObject "Node" "node" $ [ simpleField "name" [t| String |] , simpleField "primary_ip" [t| String |] , simpleField "secondary_ip" [t| String |] , simpleField "master_candidate" [t| Bool |] , simpleField "offline" [t| Bool |] , simpleField "drained" [t| Bool |] , simpleField "group" [t| String |] , simpleField "master_capable" [t| Bool |] , simpleField "vm_capable" [t| Bool |] , simpleField "ndparams" [t| PartialNDParams |] , simpleField "powered" [t| Bool |] , notSerializeDefaultField [| emptyContainer |] $ simpleField "hv_state_static" [t| HypervisorState |] , notSerializeDefaultField [| emptyContainer |] $ simpleField "disk_state_static" [t| DiskState |] ] ++ timeStampFields ++ uuidFields ++ serialFields ++ tagsFields) instance TimeStampObject Node where cTimeOf = nodeCtime mTimeOf = nodeMtime instance UuidObject Node where uuidOf = UTF8.toString . nodeUuid instance SerialNoObject Node where serialOf = nodeSerial instance TagsObject Node where tagsOf = nodeTags -- * NodeGroup definitions -- | The cluster/group disk parameters type. type GroupDiskParams = Container DiskParams -- | A mapping from network UUIDs to nic params of the networks. type Networks = Container PartialNicParams $(buildObject "NodeGroup" "group" $ [ simpleField "name" [t| String |] , defaultField [| [] |] $ simpleField "members" [t| [String] |] , simpleField "ndparams" [t| PartialNDParams |] , simpleField "alloc_policy" [t| AllocPolicy |] , simpleField "ipolicy" [t| PartialIPolicy |] , simpleField "diskparams" [t| GroupDiskParams |] , simpleField "networks" [t| Networks |] , notSerializeDefaultField [| emptyContainer |] $ simpleField "hv_state_static" [t| HypervisorState |] , notSerializeDefaultField [| emptyContainer |] $ simpleField "disk_state_static" [t| DiskState |] ] ++ timeStampFields ++ uuidFields ++ serialFields ++ tagsFields) instance TimeStampObject NodeGroup where cTimeOf = groupCtime mTimeOf = groupMtime instance UuidObject NodeGroup where uuidOf = UTF8.toString . groupUuid instance SerialNoObject NodeGroup where serialOf = groupSerial instance TagsObject NodeGroup where tagsOf = groupTags -- * Job scheduler filtering definitions -- | Actions that can be performed when a filter matches. data FilterAction = Accept | Pause | Reject | Continue | RateLimit Int deriving (Eq, Ord, Show) instance JSON FilterAction where showJSON fa = case fa of Accept -> JSString (toJSString "ACCEPT") Pause -> JSString (toJSString "PAUSE") Reject -> JSString (toJSString "REJECT") Continue -> JSString (toJSString "CONTINUE") RateLimit n -> JSArray [ JSString (toJSString "RATE_LIMIT") , JSRational False (fromIntegral n) ] readJSON v = case v of -- `FilterAction`s are case-sensitive. JSString s | fromJSString s == "ACCEPT" -> return Accept JSString s | fromJSString s == "PAUSE" -> return Pause JSString s | fromJSString s == "REJECT" -> return Reject JSString s | fromJSString s == "CONTINUE" -> return Continue JSArray (JSString s : rest) | fromJSString s == "RATE_LIMIT" -> case rest of [JSRational False n] | denominator n == 1 && numerator n > 0 -> return . RateLimit . fromIntegral $ numerator n _ -> fail "RATE_LIMIT argument must be a positive integer" x -> fail $ "malformed FilterAction JSON: " ++ J.showJSValue x "" data FilterPredicate = FPJobId (Filter FilterField) | FPOpCode (Filter FilterField) | FPReason (Filter FilterField) deriving (Eq, Ord, Show) instance JSON FilterPredicate where showJSON fp = case fp of FPJobId expr -> JSArray [string "jobid", showJSON expr] FPOpCode expr -> JSArray [string "opcode", showJSON expr] FPReason expr -> JSArray [string "reason", showJSON expr] where string = JSString . toJSString readJSON v = case v of -- Predicate names are case-sensitive. JSArray [JSString name, expr] | name == toJSString "jobid" -> FPJobId <$> readJSON expr | name == toJSString "opcode" -> FPOpCode <$> readJSON expr | name == toJSString "reason" -> FPReason <$> readJSON expr JSArray (JSString name:params) -> fail $ "malformed FilterPredicate: bad parameter list for\ \ '" ++ fromJSString name ++ "' predicate: " ++ J.showJSArray params "" _ -> fail "malformed FilterPredicate: must be a list with the first\ \ entry being a string describing the predicate type" $(buildObject "FilterRule" "fr" $ [ simpleField "watermark" [t| JobId |] , simpleField "priority" [t| NonNegative Int |] , simpleField "predicates" [t| [FilterPredicate] |] , simpleField "action" [t| FilterAction |] , simpleField "reason_trail" [t| ReasonTrail |] ] ++ uuidFields) instance UuidObject FilterRule where uuidOf = UTF8.toString . frUuid -- | Order in which filter rules are evaluated, according to -- `doc/design-optables.rst`. -- For `FilterRule` fields not specified as important for the order, -- we choose an arbitrary ordering effect (after the ones from the spec). -- -- The `Ord` instance for `FilterRule` agrees with this function. -- Yet it is recommended to use this function instead of `compare` to be -- explicit that the spec order is used. filterRuleOrder :: FilterRule -> FilterRule -> Ordering filterRuleOrder = compare instance Ord FilterRule where -- It is important that the Ord instance respects the ordering given in -- `doc/design-optables.rst` for the fields defined in there. The other -- fields may be ordered arbitrarily. -- Use `filterRuleOrder` when relying on the spec order. compare = comparing $ \(FilterRule watermark prio predicates action reason uuid) -> ( prio, watermark, uuid -- spec part , predicates, action, reason -- arbitrary part ) -- | IP family type $(declareIADT "IpFamily" [ ("IpFamilyV4", 'AutoConf.pyAfInet4) , ("IpFamilyV6", 'AutoConf.pyAfInet6) ]) $(makeJSONInstance ''IpFamily) -- | Conversion from IP family to IP version. This is needed because -- Python uses both, depending on context. ipFamilyToVersion :: IpFamily -> Int ipFamilyToVersion IpFamilyV4 = C.ip4Version ipFamilyToVersion IpFamilyV6 = C.ip6Version -- | Cluster HvParams (hvtype to hvparams mapping). type ClusterHvParams = GenericContainer Hypervisor HvParams -- | Cluster Os-HvParams (os to hvparams mapping). type OsHvParams = Container ClusterHvParams -- | Cluser BeParams. type ClusterBeParams = Container FilledBeParams -- | Cluster OsParams. type ClusterOsParams = Container OsParams type ClusterOsParamsPrivate = Container (Private OsParams) -- | Cluster NicParams. type ClusterNicParams = Container FilledNicParams -- | A low-high UID ranges. type UidRange = (Int, Int) formatUidRange :: UidRange -> String formatUidRange (lower, higher) | lower == higher = show lower | otherwise = show lower ++ "-" ++ show higher -- | Cluster UID Pool, list (low, high) UID ranges. type UidPool = [UidRange] -- | The iallocator parameters type. type IAllocatorParams = Container JSValue -- | The master candidate client certificate digests type CandidateCertificates = Container String -- * Cluster definitions $(buildObject "Cluster" "cluster" $ [ simpleField "rsahostkeypub" [t| String |] , optionalField $ simpleField "dsahostkeypub" [t| String |] , simpleField "highest_used_port" [t| Int |] , simpleField "tcpudp_port_pool" [t| [Int] |] , simpleField "mac_prefix" [t| String |] , optionalField $ simpleField "volume_group_name" [t| String |] , simpleField "reserved_lvs" [t| [String] |] , optionalField $ simpleField "drbd_usermode_helper" [t| String |] , simpleField "master_node" [t| String |] , simpleField "master_ip" [t| String |] , simpleField "master_netdev" [t| String |] , simpleField "master_netmask" [t| Int |] , simpleField "use_external_mip_script" [t| Bool |] , simpleField "cluster_name" [t| String |] , simpleField "file_storage_dir" [t| String |] , simpleField "shared_file_storage_dir" [t| String |] , simpleField "gluster_storage_dir" [t| String |] , simpleField "enabled_hypervisors" [t| [Hypervisor] |] , simpleField "hvparams" [t| ClusterHvParams |] , simpleField "os_hvp" [t| OsHvParams |] , simpleField "beparams" [t| ClusterBeParams |] , simpleField "osparams" [t| ClusterOsParams |] , simpleField "osparams_private_cluster" [t| ClusterOsParamsPrivate |] , simpleField "nicparams" [t| ClusterNicParams |] , simpleField "ndparams" [t| FilledNDParams |] , simpleField "diskparams" [t| GroupDiskParams |] , simpleField "candidate_pool_size" [t| Int |] , simpleField "modify_etc_hosts" [t| Bool |] , simpleField "modify_ssh_setup" [t| Bool |] , simpleField "maintain_node_health" [t| Bool |] , simpleField "uid_pool" [t| UidPool |] , simpleField "default_iallocator" [t| String |] , simpleField "default_iallocator_params" [t| IAllocatorParams |] , simpleField "hidden_os" [t| [String] |] , simpleField "blacklisted_os" [t| [String] |] , simpleField "primary_ip_family" [t| IpFamily |] , simpleField "prealloc_wipe_disks" [t| Bool |] , simpleField "ipolicy" [t| FilledIPolicy |] , defaultField [| emptyContainer |] $ simpleField "hv_state_static" [t| HypervisorState |] , defaultField [| emptyContainer |] $ simpleField "disk_state_static" [t| DiskState |] , simpleField "enabled_disk_templates" [t| [DiskTemplate] |] , simpleField "candidate_certs" [t| CandidateCertificates |] , simpleField "max_running_jobs" [t| Int |] , simpleField "max_tracked_jobs" [t| Int |] , simpleField "install_image" [t| String |] , simpleField "instance_communication_network" [t| String |] , simpleField "zeroing_image" [t| String |] , simpleField "compression_tools" [t| [String] |] , simpleField "enabled_user_shutdown" [t| Bool |] , simpleField "data_collectors" [t| Container DataCollectorConfig |] , simpleField "ssh_key_type" [t| SshKeyType |] , simpleField "ssh_key_bits" [t| Int |] ] ++ timeStampFields ++ uuidFields ++ serialFields ++ tagsFields) instance TimeStampObject Cluster where cTimeOf = clusterCtime mTimeOf = clusterMtime instance UuidObject Cluster where uuidOf = UTF8.toString . clusterUuid instance SerialNoObject Cluster where serialOf = clusterSerial instance TagsObject Cluster where tagsOf = clusterTags -- * ConfigData definitions $(buildObject "ConfigData" "config" $ -- timeStampFields ++ [ simpleField "version" [t| Int |] , simpleField "cluster" [t| Cluster |] , simpleField "nodes" [t| Container Node |] , simpleField "nodegroups" [t| Container NodeGroup |] , simpleField "instances" [t| Container Instance |] , simpleField "networks" [t| Container Network |] , simpleField "disks" [t| Container Disk |] , simpleField "filters" [t| Container FilterRule |] ] ++ timeStampFields ++ serialFields) instance SerialNoObject ConfigData where serialOf = configSerial instance TimeStampObject ConfigData where cTimeOf = configCtime mTimeOf = configMtime -- * Master network parameters $(buildObject "MasterNetworkParameters" "masterNetworkParameters" [ simpleField "uuid" [t| String |] , simpleField "ip" [t| String |] , simpleField "netmask" [t| Int |] , simpleField "netdev" [t| String |] , simpleField "ip_family" [t| IpFamily |] ]) ganeti-3.1.0~rc2/src/Ganeti/Objects/000075500000000000000000000000001476477700300171735ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Objects/BitArray.hs000064400000000000000000000120401476477700300212410ustar00rootroot00000000000000{-# LANGUAGE BangPatterns, Rank2Types #-} {-| Space efficient bit arrays The module is meant to be imported qualified (as it is common with collection libraries). -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Objects.BitArray ( BitArray , size , empty , zeroes , count0 , count1 , foldr , (!) , setAt , (-&-) , (-|-) , subset , asString , fromList , toList ) where import Prelude hiding (foldr) import Control.Monad import Control.Monad.Except import qualified Data.IntSet as IS import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.JSON (readEitherString) -- | A fixed-size, space-efficient array of bits. data BitArray = BitArray { size :: !Int , _bitArrayBits :: !IS.IntSet -- ^ Must not contain elements outside [0..size-1]. } deriving (Eq, Ord) instance Show BitArray where show = asString '0' '1' empty :: BitArray empty = BitArray 0 IS.empty zeroes :: Int -> BitArray zeroes s = BitArray s IS.empty -- | Right fold over the set, including indexes of each value. foldr :: (Bool -> Int -> a -> a) -> a -> BitArray -> a foldr f z (BitArray s bits) = let (j, x) = IS.foldr loop (s, z) bits in feed0 (-1) j x where loop i (!l, x) = (i, f True i (feed0 i l x)) feed0 !i !j x | i >= j' = x | otherwise = feed0 i j' (f False j' x) where j' = j - 1 -- | Converts a bit array into a string, given characters -- for @0@ and @1@/ asString :: Char -> Char -> BitArray -> String asString c0 c1 = foldr f [] where f b _ = ((if b then c1 else c0) :) -- | Computes the number of zeroes in the array. count0 :: BitArray -> Int count0 ba@(BitArray s _) = s - count1 ba -- | Computes the number of ones in the array. count1 :: BitArray -> Int count1 (BitArray _ bits) = IS.size bits infixl 9 ! -- | Test a given bit in an array. -- If it's outside its scope, it's always @False@. (!) :: BitArray -> Int -> Bool (!) (BitArray s bits) i | (i >= 0) && (i < s) = IS.member i bits | otherwise = False -- | Sets or removes an element from a bit array. -- | Sets a given bit in an array. Fails if the index is out of bounds. setAt :: (MonadError e m, Error e) => Int -> Bool -> BitArray -> m BitArray setAt i False (BitArray s bits) = return $ BitArray s (IS.delete i bits) setAt i True (BitArray s bits) | (i >= 0) && (i < s) = return $ BitArray s (IS.insert i bits) setAt i True _ = failError $ "Index out of bounds: " ++ show i infixl 7 -&- -- | An intersection of two bit arrays. -- The length of the result is the minimum length of the two. (-&-) :: BitArray -> BitArray -> BitArray BitArray xs xb -&- BitArray ys yb = BitArray (min xs ys) (xb `IS.intersection` yb) infixl 5 -|- -- | A union of two bit arrays. -- The length of the result is the maximum length of the two. (-|-) :: BitArray -> BitArray -> BitArray BitArray xs xb -|- BitArray ys yb = BitArray (max xs ys) (xb `IS.union` yb) -- | Checks if the first array is a subset of the other. subset :: BitArray -> BitArray -> Bool subset (BitArray _ xs) (BitArray _ ys) = IS.isSubsetOf xs ys -- | Converts a bit array into a list of booleans. toList :: BitArray -> [Bool] toList = foldr (\b _ -> (b :)) [] -- | Converts a list of booleans to a 'BitArray'. fromList :: [Bool] -> BitArray fromList xs = -- Note: This traverses the list twice. It'd be better to compute everything -- in one pass. BitArray (length xs) (IS.fromList . map fst . filter snd . zip [0..] $ xs) instance J.JSON BitArray where showJSON = J.JSString . J.toJSString . show readJSON j = do let parseBit '0' = return False parseBit '1' = return True parseBit c = fail $ "Neither '0' nor '1': '" ++ [c] ++ "'" str <- readEitherString j fromList `liftM` mapM parseBit str ganeti-3.1.0~rc2/src/Ganeti/Objects/Disk.hs000064400000000000000000000245501476477700300204270ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, FunctionalDependencies #-} {-| Implementation of the Ganeti Disk config object. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Objects.Disk where import qualified Data.ByteString.UTF8 as UTF8 import Data.Char (isAsciiLower, isAsciiUpper, isDigit) import Data.List (isPrefixOf, isInfixOf) import Language.Haskell.TH.Syntax import Text.JSON (showJSON, readJSON, JSValue(..)) import qualified Text.JSON as J import Ganeti.JSON (Container, fromObj) import Ganeti.THH import Ganeti.THH.Field import Ganeti.Types import Ganeti.Utils.Validate -- | Constant for the dev_type key entry in the disk config. devType :: String devType = "dev_type" -- | The disk parameters type. type DiskParams = Container JSValue -- | An alias for DRBD secrets type DRBDSecret = String -- Represents a group name and a volume name. -- -- From @man lvm@: -- -- The following characters are valid for VG and LV names: a-z A-Z 0-9 + _ . - -- -- VG and LV names cannot begin with a hyphen. There are also various reserved -- names that are used internally by lvm that can not be used as LV or VG names. -- A VG cannot be called anything that exists in /dev/ at the time of -- creation, nor can it be called '.' or '..'. A LV cannot be called '.' '..' -- 'snapshot' or 'pvmove'. The LV name may also not contain the strings '_mlog' -- or '_mimage' data LogicalVolume = LogicalVolume { lvGroup :: String , lvVolume :: String } deriving (Eq, Ord) instance Show LogicalVolume where showsPrec _ (LogicalVolume g v) = showString g . showString "/" . showString v -- | Check the constraints for VG\/LV names (except the @\/dev\/@ check). instance Validatable LogicalVolume where validate (LogicalVolume g v) = do let vgn = "Volume group name" -- Group name checks nonEmpty vgn g validChars vgn g notStartsDash vgn g notIn vgn g [".", ".."] -- Volume name checks let lvn = "Volume name" nonEmpty lvn v validChars lvn v notStartsDash lvn v notIn lvn v [".", "..", "snapshot", "pvmove"] reportIf ("_mlog" `isInfixOf` v) $ lvn ++ " must not contain '_mlog'." reportIf ("_mimage" `isInfixOf` v) $ lvn ++ "must not contain '_mimage'." where nonEmpty prefix x = reportIf (null x) $ prefix ++ " must be non-empty" notIn prefix x = mapM_ (\y -> reportIf (x == y) $ prefix ++ " must not be '" ++ y ++ "'") notStartsDash prefix x = reportIf ("-" `isPrefixOf` x) $ prefix ++ " must not start with '-'" validChars prefix x = reportIf (not . all validChar $ x) $ prefix ++ " must consist only of [a-z][A-Z][0-9][+_.-]" validChar c = isAsciiLower c || isAsciiUpper c || isDigit c || (c `elem` "+_.-") instance J.JSON LogicalVolume where showJSON = J.showJSON . show readJSON (J.JSString s) | (g, _ : l) <- break (== '/') (J.fromJSString s) = either fail return . evalValidate . validate' $ LogicalVolume g l readJSON v = fail $ "Invalid JSON value " ++ show v ++ " for a logical volume" -- | The disk configuration type. This includes the disk type itself, -- for a more complete consistency. Note that since in the Python -- code-base there's no authoritative place where we document the -- logical id, this is probably a good reference point. There is a bijective -- correspondence between the 'DiskLogicalId' constructors and 'DiskTemplate'. data DiskLogicalId = LIDPlain LogicalVolume -- ^ Volume group, logical volume | LIDDrbd8 String String Int Int Int (Private DRBDSecret) -- ^ NodeA, NodeB, Port, MinorA, MinorB, Secret | LIDFile FileDriver String -- ^ Driver, path | LIDSharedFile FileDriver String -- ^ Driver, path | LIDGluster FileDriver String -- ^ Driver, path | LIDBlockDev BlockDriver String -- ^ Driver, path (must be under /dev) | LIDRados String String -- ^ Unused, path | LIDExt String String -- ^ ExtProvider, unique name deriving (Show, Eq) -- | Mapping from a logical id to a disk type. lidDiskType :: DiskLogicalId -> DiskTemplate lidDiskType (LIDPlain {}) = DTPlain lidDiskType (LIDDrbd8 {}) = DTDrbd8 lidDiskType (LIDFile {}) = DTFile lidDiskType (LIDSharedFile {}) = DTSharedFile lidDiskType (LIDGluster {}) = DTGluster lidDiskType (LIDBlockDev {}) = DTBlock lidDiskType (LIDRados {}) = DTRbd lidDiskType (LIDExt {}) = DTExt -- | Builds the extra disk_type field for a given logical id. lidEncodeType :: DiskLogicalId -> [(String, JSValue)] lidEncodeType v = [(devType, showJSON . lidDiskType $ v)] -- | Custom encoder for DiskLogicalId (logical id only). encodeDLId :: DiskLogicalId -> JSValue encodeDLId (LIDPlain (LogicalVolume vg lv)) = JSArray [showJSON vg, showJSON lv] encodeDLId (LIDDrbd8 nodeA nodeB port minorA minorB key) = JSArray [ showJSON nodeA, showJSON nodeB, showJSON port , showJSON minorA, showJSON minorB, showJSON key ] encodeDLId (LIDRados pool name) = JSArray [showJSON pool, showJSON name] encodeDLId (LIDFile driver name) = JSArray [showJSON driver, showJSON name] encodeDLId (LIDSharedFile driver name) = JSArray [showJSON driver, showJSON name] encodeDLId (LIDGluster driver name) = JSArray [showJSON driver, showJSON name] encodeDLId (LIDBlockDev driver name) = JSArray [showJSON driver, showJSON name] encodeDLId (LIDExt extprovider name) = JSArray [showJSON extprovider, showJSON name] -- | Custom encoder for DiskLogicalId, composing both the logical id -- and the extra disk_type field. encodeFullDLId :: DiskLogicalId -> (JSValue, [(String, JSValue)]) encodeFullDLId v = (encodeDLId v, lidEncodeType v) -- | Custom decoder for DiskLogicalId. This is manual for now, since -- we don't have yet automation for separate-key style fields. decodeDLId :: [(String, JSValue)] -> JSValue -> J.Result DiskLogicalId decodeDLId obj lid = do dtype <- fromObj obj devType case dtype of DTDrbd8 -> case lid of JSArray [nA, nB, p, mA, mB, k] -> LIDDrbd8 <$> readJSON nA <*> readJSON nB <*> readJSON p <*> readJSON mA <*> readJSON mB <*> readJSON k _ -> fail "Can't read logical_id for DRBD8 type" DTPlain -> case lid of JSArray [vg, lv] -> LIDPlain <$> (LogicalVolume <$> readJSON vg <*> readJSON lv) _ -> fail "Can't read logical_id for plain type" DTFile -> case lid of JSArray [driver, path] -> LIDFile <$> readJSON driver <*> readJSON path _ -> fail "Can't read logical_id for file type" DTSharedFile -> case lid of JSArray [driver, path] -> LIDSharedFile <$> readJSON driver <*> readJSON path _ -> fail "Can't read logical_id for shared file type" DTGluster -> case lid of JSArray [driver, path] -> LIDGluster <$> readJSON driver <*> readJSON path _ -> fail "Can't read logical_id for shared file type" DTBlock -> case lid of JSArray [driver, path] -> LIDBlockDev <$> readJSON driver <*> readJSON path _ -> fail "Can't read logical_id for blockdev type" DTRbd -> case lid of JSArray [driver, path] -> LIDRados <$> readJSON driver <*> readJSON path _ -> fail "Can't read logical_id for rdb type" DTExt -> case lid of JSArray [extprovider, name] -> LIDExt <$> readJSON extprovider <*> readJSON name _ -> fail "Can't read logical_id for extstorage type" DTDiskless -> fail "Retrieved 'diskless' disk." -- | Disk data structure. $(buildObjectWithForthcoming "Disk" "disk" $ [ customField 'decodeDLId 'encodeFullDLId ["dev_type"] $ simpleField "logical_id" [t| DiskLogicalId |] , defaultField [| [] |] $ simpleField "children" (return . AppT ListT . ConT $ mkName "Disk") , defaultField [| [] |] $ simpleField "nodes" [t| [String] |] , defaultField [| "" |] $ simpleField "iv_name" [t| String |] , simpleField "size" [t| Int |] , defaultField [| DiskRdWr |] $ simpleField "mode" [t| DiskMode |] , optionalField $ simpleField "name" [t| String |] , optionalField $ simpleField "spindles" [t| Int |] , optionalField $ simpleField "params" [t| DiskParams |] ] ++ uuidFields ++ serialFields ++ timeStampFields) instance TimeStampObject Disk where cTimeOf = diskCtime mTimeOf = diskMtime instance UuidObject Disk where uuidOf = UTF8.toString . diskUuid instance SerialNoObject Disk where serialOf = diskSerial instance ForthcomingObject Disk where isForthcoming = diskForthcoming -- | Determines whether a disk or one of his children has the given logical id -- (determined by the volume group name and by the logical volume name). -- This can be true only for DRBD or LVM disks. includesLogicalId :: LogicalVolume -> Disk -> Bool includesLogicalId lv disk = case diskLogicalId disk of Just (LIDPlain lv') -> lv' == lv Just (LIDDrbd8 {}) -> any (includesLogicalId lv) $ diskChildren disk _ -> False ganeti-3.1.0~rc2/src/Ganeti/Objects/Instance.hs000064400000000000000000000073071476477700300213020ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, FunctionalDependencies #-} {-# OPTIONS_GHC -O0 #-} -- We have to disable optimisation here, as some versions of ghc otherwise -- fail to compile this code, at least within reasonable memory limits (40g). {-| Implementation of the Ganeti Instance config object. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Objects.Instance where import qualified Data.ByteString.UTF8 as UTF8 import Ganeti.JSON (emptyContainer) import Ganeti.Objects.Nic import Ganeti.THH import Ganeti.THH.Field import Ganeti.Types import Ganeti.Utils (parseUnitAssumeBinary) $(buildParam "Be" "bep" [ specialNumericalField 'parseUnitAssumeBinary $ simpleField "minmem" [t| Int |] , specialNumericalField 'parseUnitAssumeBinary $ simpleField "maxmem" [t| Int |] , simpleField "vcpus" [t| Int |] , simpleField "auto_balance" [t| Bool |] , simpleField "always_failover" [t| Bool |] , simpleField "spindle_use" [t| Int |] ]) $(buildObjectWithForthcoming "Instance" "inst" $ [ simpleField "name" [t| String |] , simpleField "primary_node" [t| String |] , simpleField "os" [t| String |] , simpleField "hypervisor" [t| Hypervisor |] , defaultField [| emptyContainer |] $ simpleField "hvparams" [t| HvParams |] , defaultField [| mempty |] $ simpleField "beparams" [t| PartialBeParams |] , defaultField [| emptyContainer |] $ simpleField "osparams" [t| OsParams |] , defaultField [| emptyContainer |] $ simpleField "osparams_private" [t| OsParamsPrivate |] , simpleField "admin_state" [t| AdminState |] , simpleField "admin_state_source" [t| AdminStateSource |] , defaultField [| [] |] $ simpleField "nics" [t| [PartialNic] |] , defaultField [| [] |] $ simpleField "disks" [t| [String] |] , simpleField "disks_active" [t| Bool |] , optionalField $ simpleField "network_port" [t| Int |] ] ++ timeStampFields ++ uuidFields ++ serialFields ++ tagsFields) instance TimeStampObject Instance where cTimeOf = instCtime mTimeOf = instMtime instance UuidObject Instance where uuidOf = UTF8.toString . instUuid instance SerialNoObject Instance where serialOf = instSerial instance TagsObject Instance where tagsOf = instTags instance ForthcomingObject Instance where isForthcoming = instForthcoming ganeti-3.1.0~rc2/src/Ganeti/Objects/Lens.hs000064400000000000000000000103051476477700300204270ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE LiberalTypeSynonyms #-} {-| Lenses for Ganeti config objects -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Objects.Lens where import qualified Data.ByteString as BS import qualified Data.ByteString.UTF8 as UTF8 import Control.Lens (Simple) import Control.Lens.Iso (Iso, iso) import System.Time (ClockTime(..)) import Ganeti.Lens (makeCustomLenses, Lens') import Ganeti.Objects -- | Isomorphism between Strings and bytestrings stringL :: Simple Iso BS.ByteString String stringL = iso UTF8.toString UTF8.fromString -- | Class of objects that have timestamps. class TimeStampObject a => TimeStampObjectL a where mTimeL :: Lens' a ClockTime -- | Class of objects that have an UUID. class UuidObject a => UuidObjectL a where uuidL :: Lens' a String -- | Class of object that have a serial number. class SerialNoObject a => SerialNoObjectL a where serialL :: Lens' a Int -- | Class of objects that have tags. class TagsObject a => TagsObjectL a where tagsL :: Lens' a TagSet $(makeCustomLenses ''AddressPool) $(makeCustomLenses ''Network) instance SerialNoObjectL Network where serialL = networkSerialL instance TagsObjectL Network where tagsL = networkTagsL instance UuidObjectL Network where uuidL = networkUuidL . stringL instance TimeStampObjectL Network where mTimeL = networkMtimeL $(makeCustomLenses ''PartialNic) $(makeCustomLenses ''Disk) instance TimeStampObjectL Disk where mTimeL = diskMtimeL instance UuidObjectL Disk where uuidL = diskUuidL . stringL instance SerialNoObjectL Disk where serialL = diskSerialL $(makeCustomLenses ''Instance) instance TimeStampObjectL Instance where mTimeL = instMtimeL instance UuidObjectL Instance where uuidL = instUuidL . stringL instance SerialNoObjectL Instance where serialL = instSerialL instance TagsObjectL Instance where tagsL = instTagsL $(makeCustomLenses ''MinMaxISpecs) $(makeCustomLenses ''PartialIPolicy) $(makeCustomLenses ''FilledIPolicy) $(makeCustomLenses ''Node) instance TimeStampObjectL Node where mTimeL = nodeMtimeL instance UuidObjectL Node where uuidL = nodeUuidL . stringL instance SerialNoObjectL Node where serialL = nodeSerialL instance TagsObjectL Node where tagsL = nodeTagsL $(makeCustomLenses ''NodeGroup) instance TimeStampObjectL NodeGroup where mTimeL = groupMtimeL instance UuidObjectL NodeGroup where uuidL = groupUuidL . stringL instance SerialNoObjectL NodeGroup where serialL = groupSerialL instance TagsObjectL NodeGroup where tagsL = groupTagsL $(makeCustomLenses ''Cluster) instance TimeStampObjectL Cluster where mTimeL = clusterMtimeL instance UuidObjectL Cluster where uuidL = clusterUuidL . stringL instance SerialNoObjectL Cluster where serialL = clusterSerialL instance TagsObjectL Cluster where tagsL = clusterTagsL $(makeCustomLenses ''ConfigData) instance SerialNoObjectL ConfigData where serialL = configSerialL instance TimeStampObjectL ConfigData where mTimeL = configMtimeL ganeti-3.1.0~rc2/src/Ganeti/Objects/Nic.hs000064400000000000000000000040431476477700300202410ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, FunctionalDependencies #-} {-| Implementation of the Ganeti Instance config object. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Objects.Nic where import qualified Data.ByteString.UTF8 as UTF8 import Ganeti.THH import Ganeti.THH.Field import Ganeti.Types $(buildParam "Nic" "nicp" [ simpleField "mode" [t| NICMode |] , simpleField "link" [t| String |] , simpleField "vlan" [t| String |] ]) $(buildObject "PartialNic" "nic" $ [ simpleField "mac" [t| String |] , optionalField $ simpleField "ip" [t| String |] , simpleField "nicparams" [t| PartialNicParams |] , optionalField $ simpleField "network" [t| String |] , optionalField $ simpleField "name" [t| String |] ] ++ uuidFields) instance UuidObject PartialNic where uuidOf = UTF8.toString . nicUuid ganeti-3.1.0~rc2/src/Ganeti/OpCodes.hs000064400000000000000000000727111476477700300175020ustar00rootroot00000000000000{-# LANGUAGE ExistentialQuantification, TemplateHaskell, StandaloneDeriving #-} {-# OPTIONS_GHC -fno-warn-orphans -O0 #-} -- We have to disable optimisation here, as some versions of ghc otherwise -- fail to compile this code, at least within reasonable memory limits (40g). {-| Implementation of the opcodes. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.OpCodes ( pyClasses , OpCode(..) , ReplaceDisksMode(..) , DiskIndex , mkDiskIndex , unDiskIndex , opID , opReasonSrcID , allOpIDs , allOpFields , opSummary , CommonOpParams(..) , defOpParams , MetaOpCode(..) , resolveDependencies , wrapOpCode , setOpComment , setOpPriority ) where import Control.Monad.Fail (MonadFail) import Data.List (intercalate) import Data.Map (Map) import qualified Text.JSON import Text.JSON (readJSON, JSObject, JSON, JSValue(..), fromJSObject) import Text.Printf (printf) import qualified Ganeti.Constants as C import qualified Ganeti.Hs2Py.OpDoc as OpDoc import Ganeti.JSON (DictObject(..), readJSONfromDict, showJSONtoDict) import Ganeti.OpParams import Ganeti.PyValue () import Ganeti.Query.Language (queryTypeOpToRaw) import Ganeti.THH import Ganeti.Types instance PyValue DiskIndex where showValue = showValue . unDiskIndex instance PyValue IDiskParams where showValue _ = error "OpCodes.showValue(IDiskParams): unhandled case" instance PyValue RecreateDisksInfo where showValue RecreateDisksAll = "[]" showValue (RecreateDisksIndices is) = showValue is showValue (RecreateDisksParams is) = showValue is instance PyValue a => PyValue (SetParamsMods a) where showValue SetParamsEmpty = "[]" showValue _ = error "OpCodes.showValue(SetParamsMods): unhandled case" instance PyValue a => PyValue (NonNegative a) where showValue = showValue . fromNonNegative instance PyValue a => PyValue (NonEmpty a) where showValue = showValue . fromNonEmpty -- FIXME: should use the 'toRaw' function instead of being harcoded or -- perhaps use something similar to the NonNegative type instead of -- using the declareSADT instance PyValue ExportMode where showValue ExportModeLocal = show C.exportModeLocal showValue ExportModeRemote = show C.exportModeLocal instance PyValue CVErrorCode where showValue = cVErrorCodeToRaw instance PyValue VerifyOptionalChecks where showValue = verifyOptionalChecksToRaw instance PyValue INicParams where showValue = error "instance PyValue INicParams: not implemented" instance PyValue a => PyValue (JSObject a) where showValue obj = "{" ++ intercalate ", " (map showPair (fromJSObject obj)) ++ "}" where showPair (k, v) = show k ++ ":" ++ showValue v instance PyValue JSValue where showValue (JSObject obj) = showValue obj showValue x = show x type JobIdListOnly = Map String [(Bool, Either String JobId)] type InstanceMultiAllocResponse = ([(Bool, Either String JobId)], NonEmptyString) type QueryFieldDef = (NonEmptyString, NonEmptyString, TagKind, NonEmptyString) type QueryResponse = ([QueryFieldDef], [[(QueryResultCode, JSValue)]]) type QueryFieldsResponse = [QueryFieldDef] -- | OpCode representation. -- -- We only implement a subset of Ganeti opcodes: those which are actually used -- in the htools codebase. $(genOpCode "OpCode" [ ("OpClusterPostInit", [t| Bool |], OpDoc.opClusterPostInit, [], []) , ("OpClusterDestroy", [t| NonEmptyString |], OpDoc.opClusterDestroy, [], []) , ("OpClusterQuery", [t| JSObject JSValue |], OpDoc.opClusterQuery, [], []) , ("OpClusterVerify", [t| JobIdListOnly |], OpDoc.opClusterVerify, [ pDebugSimulateErrors , pErrorCodes , pSkipChecks , pIgnoreErrors , pVerbose , pOptGroupName , pVerifyClutter ], []) , ("OpClusterVerifyConfig", [t| Bool |], OpDoc.opClusterVerifyConfig, [ pDebugSimulateErrors , pErrorCodes , pIgnoreErrors , pVerbose ], []) , ("OpClusterVerifyGroup", [t| Bool |], OpDoc.opClusterVerifyGroup, [ pGroupName , pDebugSimulateErrors , pErrorCodes , pSkipChecks , pIgnoreErrors , pVerbose , pVerifyClutter ], "group_name") , ("OpClusterVerifyDisks", [t| JobIdListOnly |], OpDoc.opClusterVerifyDisks, [ pOptGroupName , pIsStrict ], []) , ("OpGroupVerifyDisks", [t| (Map String String, [String], Map String [[String]]) |], OpDoc.opGroupVerifyDisks, [ pGroupName , pIsStrict ], "group_name") , ("OpClusterRepairDiskSizes", [t| [(NonEmptyString, NonNegative Int, NonEmptyString, NonNegative Int)]|], OpDoc.opClusterRepairDiskSizes, [ pInstances ], []) , ("OpClusterConfigQuery", [t| [JSValue] |], OpDoc.opClusterConfigQuery, [ pOutputFields ], []) , ("OpClusterRename", [t| NonEmptyString |], OpDoc.opClusterRename, [ pName ], "name") , ("OpClusterSetParams", [t| Either () JobIdListOnly |], OpDoc.opClusterSetParams, [ pForce , pHvState , pDiskState , pVgName , pEnabledHypervisors , pClusterHvParams , pClusterBeParams , pOsHvp , pClusterOsParams , pClusterOsParamsPrivate , pGroupDiskParams , pCandidatePoolSize , pMaxRunningJobs , pMaxTrackedJobs , pUidPool , pAddUids , pRemoveUids , pMaintainNodeHealth , pPreallocWipeDisks , pNicParams , withDoc "Cluster-wide node parameter defaults" pNdParams , withDoc "Cluster-wide ipolicy specs" pIpolicy , pDrbdHelper , pDefaultIAllocator , pDefaultIAllocatorParams , pNetworkMacPrefix , pMasterNetdev , pMasterNetmask , pReservedLvs , pHiddenOs , pBlacklistedOs , pUseExternalMipScript , pEnabledDiskTemplates , pModifyEtcHosts , pClusterFileStorageDir , pClusterSharedFileStorageDir , pClusterGlusterStorageDir , pInstallImage , pInstanceCommunicationNetwork , pZeroingImage , pCompressionTools , pEnabledUserShutdown , pEnabledDataCollectors , pDataCollectorInterval ], []) , ("OpClusterRedistConf", [t| () |], OpDoc.opClusterRedistConf, [], []) , ("OpClusterActivateMasterIp", [t| () |], OpDoc.opClusterActivateMasterIp, [], []) , ("OpClusterDeactivateMasterIp", [t| () |], OpDoc.opClusterDeactivateMasterIp, [], []) , ("OpClusterRenewCrypto", [t| () |], OpDoc.opClusterRenewCrypto, [ pNodeSslCerts , pRenewSshKeys , pSshKeyType , pSshKeyBits , pVerbose , pDebug ], []) , ("OpQuery", [t| QueryResponse |], OpDoc.opQuery, [ pQueryWhat , pUseLocking , pQueryFields , pQueryFilter ], "what") , ("OpQueryFields", [t| QueryFieldsResponse |], OpDoc.opQueryFields, [ pQueryWhat , pQueryFieldsFields ], "what") , ("OpOobCommand", [t| [[(QueryResultCode, JSValue)]] |], OpDoc.opOobCommand, [ pNodeNames , withDoc "List of node UUIDs to run the OOB command against" pNodeUuids , pOobCommand , pOobTimeout , pIgnoreStatus , pPowerDelay ], []) , ("OpRestrictedCommand", [t| [(Bool, String)] |], OpDoc.opRestrictedCommand, [ pUseLocking , withDoc "Nodes on which the command should be run (at least one)" pRequiredNodes , withDoc "Node UUIDs on which the command should be run (at least one)" pRequiredNodeUuids , pRestrictedCommand ], []) , ("OpNodeRemove", [t| () |], OpDoc.opNodeRemove, [ pNodeName , pNodeUuid ], "node_name") , ("OpNodeAdd", [t| () |], OpDoc.opNodeAdd, [ pNodeName , pHvState , pDiskState , pPrimaryIp , pSecondaryIp , pReadd , pNodeGroup , pMasterCapable , pVmCapable , pNdParams , pNodeSetup ], "node_name") , ("OpNodeQueryvols", [t| [JSValue] |], OpDoc.opNodeQueryvols, [ pOutputFields , withDoc "Empty list to query all nodes, node names otherwise" pNodes ], []) , ("OpNodeQueryStorage", [t| [[JSValue]] |], OpDoc.opNodeQueryStorage, [ pOutputFields , pOptStorageType , withDoc "Empty list to query all, list of names to query otherwise" pNodes , pStorageName ], []) , ("OpNodeModifyStorage", [t| () |], OpDoc.opNodeModifyStorage, [ pNodeName , pNodeUuid , pStorageType , pStorageName , pStorageChanges ], "node_name") , ("OpRepairNodeStorage", [t| () |], OpDoc.opRepairNodeStorage, [ pNodeName , pNodeUuid , pStorageType , pStorageName , pIgnoreConsistency ], "node_name") , ("OpNodeSetParams", [t| [(NonEmptyString, JSValue)] |], OpDoc.opNodeSetParams, [ pNodeName , pNodeUuid , pForce , pHvState , pDiskState , pMasterCandidate , withDoc "Whether to mark the node offline" pOffline , pDrained , pAutoPromote , pMasterCapable , pVmCapable , pSecondaryIp , pNdParams , pPowered ], "node_name") , ("OpNodePowercycle", [t| Maybe NonEmptyString |], OpDoc.opNodePowercycle, [ pNodeName , pNodeUuid , pForce ], "node_name") , ("OpNodeMigrate", [t| JobIdListOnly |], OpDoc.opNodeMigrate, [ pNodeName , pNodeUuid , pMigrationMode , pMigrationLive , pMigrationTargetNode , pMigrationTargetNodeUuid , pAllowRuntimeChgs , pIgnoreIpolicy , pIallocator ], "node_name") , ("OpNodeEvacuate", [t| JobIdListOnly |], OpDoc.opNodeEvacuate, [ pEarlyRelease , pNodeName , pNodeUuid , pRemoteNode , pRemoteNodeUuid , pIallocator , pEvacMode , pIgnoreSoftErrors ], "node_name") , ("OpInstanceCreate", [t| [NonEmptyString] |], OpDoc.opInstanceCreate, [ pInstanceName , pForceVariant , pWaitForSync , pNameCheck , pIgnoreIpolicy , pOpportunisticLocking , pInstBeParams , pInstDisks , pOptDiskTemplate , pOptGroupName , pFileDriver , pFileStorageDir , pInstHvParams , pHypervisor , pIallocator , pResetDefaults , pIpCheck , pIpConflictsCheck , pInstCreateMode , pInstNics , pNoInstall , pInstOsParams , pInstOsParamsPrivate , pInstOsParamsSecret , pInstOs , pPrimaryNode , pPrimaryNodeUuid , pSecondaryNode , pSecondaryNodeUuid , pSourceHandshake , pSourceInstance , pSourceShutdownTimeout , pSourceX509Ca , pSrcNode , pSrcNodeUuid , pSrcPath , pBackupCompress , pStartInstance , pForthcoming , pCommit , pInstTags , pInstanceCommunication , pHelperStartupTimeout , pHelperShutdownTimeout ], "instance_name") , ("OpInstanceMultiAlloc", [t| InstanceMultiAllocResponse |], OpDoc.opInstanceMultiAlloc, [ pOpportunisticLocking , pIallocator , pMultiAllocInstances ], []) , ("OpInstanceReinstall", [t| () |], OpDoc.opInstanceReinstall, [ pInstanceName , pInstanceUuid , pForceVariant , pInstOs , pTempOsParams , pTempOsParamsPrivate , pTempOsParamsSecret ], "instance_name") , ("OpInstanceRemove", [t| () |], OpDoc.opInstanceRemove, [ pInstanceName , pInstanceUuid , pShutdownTimeout , pIgnoreFailures ], "instance_name") , ("OpInstanceRename", [t| NonEmptyString |], OpDoc.opInstanceRename, [ pInstanceName , pInstanceUuid , withDoc "New instance name" pNewName , pNameCheck , pIpCheck ], []) , ("OpInstanceStartup", [t| () |], OpDoc.opInstanceStartup, [ pInstanceName , pInstanceUuid , pForce , pIgnoreOfflineNodes , pTempHvParams , pTempBeParams , pNoRemember , pStartupPaused -- timeout to cleanup a user down instance , pShutdownTimeout ], "instance_name") , ("OpInstanceShutdown", [t| () |], OpDoc.opInstanceShutdown, [ pInstanceName , pInstanceUuid , pForce , pIgnoreOfflineNodes , pShutdownTimeout' , pNoRemember , pAdminStateSource ], "instance_name") , ("OpInstanceReboot", [t| () |], OpDoc.opInstanceReboot, [ pInstanceName , pInstanceUuid , pShutdownTimeout , pIgnoreSecondaries , pRebootType ], "instance_name") , ("OpInstanceReplaceDisks", [t| () |], OpDoc.opInstanceReplaceDisks, [ pInstanceName , pInstanceUuid , pEarlyRelease , pIgnoreIpolicy , pReplaceDisksMode , pReplaceDisksList , pRemoteNode , pRemoteNodeUuid , pIallocator ], "instance_name") , ("OpInstanceFailover", [t| () |], OpDoc.opInstanceFailover, [ pInstanceName , pInstanceUuid , pShutdownTimeout , pIgnoreConsistency , pMigrationTargetNode , pMigrationTargetNodeUuid , pIgnoreIpolicy , pMigrationCleanup , pIallocator ], "instance_name") , ("OpInstanceMigrate", [t| () |], OpDoc.opInstanceMigrate, [ pInstanceName , pInstanceUuid , pMigrationMode , pMigrationLive , pMigrationTargetNode , pMigrationTargetNodeUuid , pAllowRuntimeChgs , pIgnoreIpolicy , pMigrationCleanup , pIallocator , pAllowFailover , pIgnoreHVVersions ], "instance_name") , ("OpInstanceMove", [t| () |], OpDoc.opInstanceMove, [ pInstanceName , pInstanceUuid , pShutdownTimeout , pIgnoreIpolicy , pMoveTargetNode , pMoveTargetNodeUuid , pMoveCompress , pIgnoreConsistency ], "instance_name") , ("OpInstanceConsole", [t| JSObject JSValue |], OpDoc.opInstanceConsole, [ pInstanceName , pInstanceUuid ], "instance_name") , ("OpInstanceActivateDisks", [t| [(NonEmptyString, NonEmptyString, Maybe NonEmptyString)] |], OpDoc.opInstanceActivateDisks, [ pInstanceName , pInstanceUuid , pIgnoreDiskSize , pWaitForSyncFalse ], "instance_name") , ("OpInstanceDeactivateDisks", [t| () |], OpDoc.opInstanceDeactivateDisks, [ pInstanceName , pInstanceUuid , pForce ], "instance_name") , ("OpInstanceRecreateDisks", [t| () |], OpDoc.opInstanceRecreateDisks, [ pInstanceName , pInstanceUuid , pRecreateDisksInfo , withDoc "New instance nodes, if relocation is desired" pNodes , withDoc "New instance node UUIDs, if relocation is desired" pNodeUuids , pIallocator ], "instance_name") , ("OpInstanceQueryData", [t| JSObject (JSObject JSValue) |], OpDoc.opInstanceQueryData, [ pUseLocking , pInstances , pStatic ], []) , ("OpInstanceSetParams", [t| [(NonEmptyString, JSValue)] |], OpDoc.opInstanceSetParams, [ pInstanceName , pInstanceUuid , pForce , pForceVariant , pIgnoreIpolicy , pInstParamsNicChanges , pInstParamsDiskChanges , pInstBeParams , pRuntimeMem , pInstHvParams , pOptDiskTemplate , pExtParams , pFileDriver , pFileStorageDir , pPrimaryNode , pPrimaryNodeUuid , withDoc "Secondary node (used when changing disk template)" pRemoteNode , withDoc "Secondary node UUID (used when changing disk template)" pRemoteNodeUuid , pIallocator , pOsNameChange , pInstOsParams , pInstOsParamsPrivate , pWaitForSync , withDoc "Whether to mark the instance as offline" pOffline , pIpConflictsCheck , pHotplug , pOptInstanceCommunication ], "instance_name") , ("OpInstanceGrowDisk", [t| () |], OpDoc.opInstanceGrowDisk, [ pInstanceName , pInstanceUuid , pWaitForSync , pDiskIndex , pDiskChgAmount , pDiskChgAbsolute , pIgnoreIpolicy ], "instance_name") , ("OpInstanceChangeGroup", [t| JobIdListOnly |], OpDoc.opInstanceChangeGroup, [ pInstanceName , pInstanceUuid , pEarlyRelease , pIallocator , pTargetGroups ], "instance_name") , ("OpGroupAdd", [t| Either () JobIdListOnly |], OpDoc.opGroupAdd, [ pGroupName , pNodeGroupAllocPolicy , pGroupNodeParams , pGroupDiskParams , pHvState , pDiskState , withDoc "Group-wide ipolicy specs" pIpolicy ], "group_name") , ("OpGroupAssignNodes", [t| () |], OpDoc.opGroupAssignNodes, [ pGroupName , pForce , withDoc "List of nodes to assign" pRequiredNodes , withDoc "List of node UUIDs to assign" pRequiredNodeUuids ], "group_name") , ("OpGroupSetParams", [t| [(NonEmptyString, JSValue)] |], OpDoc.opGroupSetParams, [ pGroupName , pNodeGroupAllocPolicy , pGroupNodeParams , pGroupDiskParams , pHvState , pDiskState , withDoc "Group-wide ipolicy specs" pIpolicy ], "group_name") , ("OpGroupRemove", [t| () |], OpDoc.opGroupRemove, [ pGroupName ], "group_name") , ("OpGroupRename", [t| NonEmptyString |], OpDoc.opGroupRename, [ pGroupName , withDoc "New group name" pNewName ], []) , ("OpGroupEvacuate", [t| JobIdListOnly |], OpDoc.opGroupEvacuate, [ pGroupName , pEarlyRelease , pIallocator , pTargetGroups , pSequential , pForceFailover ], "group_name") , ("OpOsDiagnose", [t| [[JSValue]] |], OpDoc.opOsDiagnose, [ pOutputFields , withDoc "Which operating systems to diagnose" pNames ], []) , ("OpExtStorageDiagnose", [t| [[JSValue]] |], OpDoc.opExtStorageDiagnose, [ pOutputFields , withDoc "Which ExtStorage Provider to diagnose" pNames ], []) , ("OpBackupPrepare", [t| Maybe (JSObject JSValue) |], OpDoc.opBackupPrepare, [ pInstanceName , pInstanceUuid , pExportMode ], "instance_name") , ("OpBackupExport", [t| (Bool, [Bool]) |], OpDoc.opBackupExport, [ pInstanceName , pInstanceUuid , pBackupCompress , pShutdownTimeout , pExportTargetNode , pExportTargetNodeUuid , pShutdownInstance , pRemoveInstance , pIgnoreRemoveFailures , defaultField [| ExportModeLocal |] pExportMode , pX509KeyName , pX509DestCA , pZeroFreeSpace , pZeroingTimeoutFixed , pZeroingTimeoutPerMiB , pLongSleep ], "instance_name") , ("OpBackupRemove", [t| () |], OpDoc.opBackupRemove, [ pInstanceName , pInstanceUuid ], "instance_name") , ("OpTagsGet", [t| [NonEmptyString] |], OpDoc.opTagsGet, [ pTagsObject , pUseLocking , withDoc "Name of object to retrieve tags from" pTagsName ], "name") , ("OpTagsSearch", [t| [(NonEmptyString, NonEmptyString)] |], OpDoc.opTagsSearch, [ pTagSearchPattern ], "pattern") , ("OpTagsSet", [t| () |], OpDoc.opTagsSet, [ pTagsObject , pTagsList , withDoc "Name of object where tag(s) should be added" pTagsName ], []) , ("OpTagsDel", [t| () |], OpDoc.opTagsDel, [ pTagsObject , pTagsList , withDoc "Name of object where tag(s) should be deleted" pTagsName ], []) , ("OpTestDelay", [t| () |], OpDoc.opTestDelay, [ pDelayDuration , pDelayOnMaster , pDelayOnNodes , pDelayOnNodeUuids , pDelayRepeat , pDelayInterruptible , pDelayNoLocks ], "duration") , ("OpTestAllocator", [t| String |], OpDoc.opTestAllocator, [ pIAllocatorDirection , pIAllocatorMode , pIAllocatorReqName , pIAllocatorNics , pIAllocatorDisks , pHypervisor , pIallocator , pInstTags , pIAllocatorMemory , pIAllocatorVCpus , pIAllocatorOs , pOptDiskTemplate , pIAllocatorInstances , pIAllocatorEvacMode , pTargetGroups , pIAllocatorSpindleUse , pIAllocatorCount , pOptGroupName ], "iallocator") , ("OpTestJqueue", [t| Bool |], OpDoc.opTestJqueue, [ pJQueueNotifyWaitLock , pJQueueNotifyExec , pJQueueLogMessages , pJQueueFail ], []) , ("OpTestOsParams", [t| () |], OpDoc.opTestOsParams, [ pInstOsParamsSecret ], []) , ("OpTestDummy", [t| () |], OpDoc.opTestDummy, [ pTestDummyResult , pTestDummyMessages , pTestDummyFail , pTestDummySubmitJobs ], []) , ("OpNetworkAdd", [t| () |], OpDoc.opNetworkAdd, [ pNetworkName , pNetworkAddress4 , pNetworkGateway4 , pNetworkAddress6 , pNetworkGateway6 , pNetworkMacPrefix , pNetworkAddRsvdIps , pIpConflictsCheck , withDoc "Network tags" pInstTags ], "network_name") , ("OpNetworkRemove", [t| () |], OpDoc.opNetworkRemove, [ pNetworkName , pForce ], "network_name") , ("OpNetworkRename", [t| NonEmptyString |], OpDoc.opNetworkRename, [ pNetworkName , withDoc "New network name" pNewName ], []) , ("OpNetworkSetParams", [t| () |], OpDoc.opNetworkSetParams, [ pNetworkName , pNetworkGateway4 , pNetworkAddress6 , pNetworkGateway6 , pNetworkMacPrefix , withDoc "Which external IP addresses to reserve" pNetworkAddRsvdIps , pNetworkRemoveRsvdIps ], "network_name") , ("OpNetworkConnect", [t| () |], OpDoc.opNetworkConnect, [ pGroupName , pNetworkName , pNetworkMode , pNetworkLink , pNetworkVlan , pIpConflictsCheck ], "network_name") , ("OpNetworkDisconnect", [t| () |], OpDoc.opNetworkDisconnect, [ pGroupName , pNetworkName ], "network_name") ]) deriving instance Ord OpCode -- | Returns the OP_ID for a given opcode value. $(genOpID ''OpCode "opID") -- | A list of all defined/supported opcode IDs. $(genAllOpIDs ''OpCode "allOpIDs") -- | Convert the opcode name to lowercase with underscores and strip -- the @Op@ prefix. $(genOpLowerStrip (C.opcodeReasonSrcOpcode ++ ":") ''OpCode "opReasonSrcID") instance JSON OpCode where readJSON = readJSONfromDict showJSON = showJSONtoDict -- | Generates the summary value for an opcode. opSummaryVal :: OpCode -> Maybe String opSummaryVal OpClusterVerifyGroup { opGroupName = s } = Just (fromNonEmpty s) opSummaryVal OpGroupVerifyDisks { opGroupName = s } = Just (fromNonEmpty s) opSummaryVal OpClusterRename { opName = s } = Just (fromNonEmpty s) opSummaryVal OpQuery { opWhat = s } = Just (queryTypeOpToRaw s) opSummaryVal OpQueryFields { opWhat = s } = Just (queryTypeOpToRaw s) opSummaryVal OpNodeRemove { opNodeName = s } = Just (fromNonEmpty s) opSummaryVal OpNodeAdd { opNodeName = s } = Just (fromNonEmpty s) opSummaryVal OpNodeModifyStorage { opNodeName = s } = Just (fromNonEmpty s) opSummaryVal OpRepairNodeStorage { opNodeName = s } = Just (fromNonEmpty s) opSummaryVal OpNodeSetParams { opNodeName = s } = Just (fromNonEmpty s) opSummaryVal OpNodePowercycle { opNodeName = s } = Just (fromNonEmpty s) opSummaryVal OpNodeMigrate { opNodeName = s } = Just (fromNonEmpty s) opSummaryVal OpNodeEvacuate { opNodeName = s } = Just (fromNonEmpty s) opSummaryVal OpInstanceCreate { opInstanceName = s } = Just s opSummaryVal OpInstanceReinstall { opInstanceName = s } = Just s opSummaryVal OpInstanceRemove { opInstanceName = s } = Just s -- FIXME: instance rename should show both names; currently it shows none -- opSummaryVal OpInstanceRename { opInstanceName = s } = Just s opSummaryVal OpInstanceStartup { opInstanceName = s } = Just s opSummaryVal OpInstanceShutdown { opInstanceName = s } = Just s opSummaryVal OpInstanceReboot { opInstanceName = s } = Just s opSummaryVal OpInstanceReplaceDisks { opInstanceName = s } = Just s opSummaryVal OpInstanceFailover { opInstanceName = s } = Just s opSummaryVal OpInstanceMigrate { opInstanceName = s } = Just s opSummaryVal OpInstanceMove { opInstanceName = s } = Just s opSummaryVal OpInstanceConsole { opInstanceName = s } = Just s opSummaryVal OpInstanceActivateDisks { opInstanceName = s } = Just s opSummaryVal OpInstanceDeactivateDisks { opInstanceName = s } = Just s opSummaryVal OpInstanceRecreateDisks { opInstanceName = s } = Just s opSummaryVal OpInstanceSetParams { opInstanceName = s } = Just s opSummaryVal OpInstanceGrowDisk { opInstanceName = s } = Just s opSummaryVal OpInstanceChangeGroup { opInstanceName = s } = Just s opSummaryVal OpGroupAdd { opGroupName = s } = Just (fromNonEmpty s) opSummaryVal OpGroupAssignNodes { opGroupName = s } = Just (fromNonEmpty s) opSummaryVal OpGroupSetParams { opGroupName = s } = Just (fromNonEmpty s) opSummaryVal OpGroupRemove { opGroupName = s } = Just (fromNonEmpty s) opSummaryVal OpGroupEvacuate { opGroupName = s } = Just (fromNonEmpty s) opSummaryVal OpBackupPrepare { opInstanceName = s } = Just s opSummaryVal OpBackupExport { opInstanceName = s } = Just s opSummaryVal OpBackupRemove { opInstanceName = s } = Just s opSummaryVal OpTagsGet { opKind = s } = Just (show s) opSummaryVal OpTagsSearch { opTagSearchPattern = s } = Just (fromNonEmpty s) opSummaryVal OpTestDelay { opDelayDuration = d } = Just $ printf "%d ns" (round(d * 1e9) :: Integer) opSummaryVal OpTestAllocator { opIallocator = s } = -- FIXME: Python doesn't handle None fields well, so we have behave the same Just $ maybe "None" fromNonEmpty s opSummaryVal OpNetworkAdd { opNetworkName = s} = Just (fromNonEmpty s) opSummaryVal OpNetworkRemove { opNetworkName = s} = Just (fromNonEmpty s) opSummaryVal OpNetworkSetParams { opNetworkName = s} = Just (fromNonEmpty s) opSummaryVal OpNetworkConnect { opNetworkName = s} = Just (fromNonEmpty s) opSummaryVal OpNetworkDisconnect { opNetworkName = s} = Just (fromNonEmpty s) opSummaryVal _ = Nothing -- | Computes the summary of the opcode. opSummary :: OpCode -> String opSummary op = case opSummaryVal op of Nothing -> op_suffix Just s -> op_suffix ++ "(" ++ s ++ ")" where op_suffix = drop 3 $ opID op -- | Generic\/common opcode parameters. $(buildObject "CommonOpParams" "op" [ pDryRun , pDebugLevel , pOpPriority , pDependencies , pComment , pReason ]) deriving instance Ord CommonOpParams -- | Default common parameter values. defOpParams :: CommonOpParams defOpParams = CommonOpParams { opDryRun = Nothing , opDebugLevel = Nothing , opPriority = OpPrioNormal , opDepends = Nothing , opComment = Nothing , opReason = [] } -- | Resolve relative dependencies to absolute ones, given the job ID. resolveDependsCommon :: (MonadFail m) => CommonOpParams -> JobId -> m CommonOpParams resolveDependsCommon p@(CommonOpParams { opDepends = Just deps}) jid = do deps' <- mapM (`absoluteJobDependency` jid) deps return p { opDepends = Just deps' } resolveDependsCommon p _ = return p -- | The top-level opcode type. data MetaOpCode = MetaOpCode { metaParams :: CommonOpParams , metaOpCode :: OpCode } deriving (Show, Eq, Ord) -- | Resolve relative dependencies to absolute ones, given the job Id. resolveDependencies :: (MonadFail m) => MetaOpCode -> JobId -> m MetaOpCode resolveDependencies mopc jid = do mpar <- resolveDependsCommon (metaParams mopc) jid return (mopc { metaParams = mpar }) instance DictObject MetaOpCode where toDict (MetaOpCode meta op) = toDict meta ++ toDict op fromDictWKeys dict = MetaOpCode <$> fromDictWKeys dict <*> fromDictWKeys dict instance JSON MetaOpCode where readJSON = readJSONfromDict showJSON = showJSONtoDict -- | Wraps an 'OpCode' with the default parameters to build a -- 'MetaOpCode'. wrapOpCode :: OpCode -> MetaOpCode wrapOpCode = MetaOpCode defOpParams -- | Sets the comment on a meta opcode. setOpComment :: String -> MetaOpCode -> MetaOpCode setOpComment comment (MetaOpCode common op) = MetaOpCode (common { opComment = Just comment}) op -- | Sets the priority on a meta opcode. setOpPriority :: OpSubmitPriority -> MetaOpCode -> MetaOpCode setOpPriority prio (MetaOpCode common op) = MetaOpCode (common { opPriority = prio }) op ganeti-3.1.0~rc2/src/Ganeti/OpCodes/000075500000000000000000000000001476477700300171365ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/OpCodes/Lens.hs000064400000000000000000000027661476477700300204060ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Lenses for OpCodes -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.OpCodes.Lens where import Ganeti.Lens (makeCustomLenses) import Ganeti.OpCodes $(makeCustomLenses ''MetaOpCode) $(makeCustomLenses ''CommonOpParams) ganeti-3.1.0~rc2/src/Ganeti/OpParams.hs000064400000000000000000001556271476477700300177000ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, StandaloneDeriving #-} {-| Implementation of opcodes parameters. These are defined in a separate module only due to TemplateHaskell stage restrictions - expressions defined in the current module can't be passed to splices. So we have to either parameters/repeat each parameter definition multiple times, or separate them into this module. -} {- Copyright (C) 2012, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.OpParams ( ReplaceDisksMode(..) , DiskIndex , mkDiskIndex , unDiskIndex , DiskAccess(..) , INicParams(..) , IDiskParams(..) , RecreateDisksInfo(..) , DdmOldChanges(..) , SetParamsMods(..) , ExportTarget(..) , pInstanceName , pInstallImage , pInstanceCommunication , pOptInstanceCommunication , pInstanceUuid , pInstances , pName , pTagsList , pTagsObject , pTagsName , pOutputFields , pShutdownTimeout , pShutdownTimeout' , pShutdownInstance , pForce , pIgnoreOfflineNodes , pIgnoreHVVersions , pNodeName , pNodeUuid , pNodeNames , pNodeUuids , pGroupName , pMigrationMode , pMigrationLive , pMigrationCleanup , pForceVariant , pWaitForSync , pWaitForSyncFalse , pIgnoreConsistency , pStorageName , pUseLocking , pOpportunisticLocking , pNameCheck , pNodeGroupAllocPolicy , pGroupNodeParams , pQueryWhat , pEarlyRelease , pIpCheck , pIpConflictsCheck , pNoRemember , pMigrationTargetNode , pMigrationTargetNodeUuid , pMoveTargetNode , pMoveTargetNodeUuid , pMoveCompress , pBackupCompress , pStartupPaused , pVerbose , pDebug , pDebugSimulateErrors , pErrorCodes , pSkipChecks , pIgnoreErrors , pOptGroupName , pGroupDiskParams , pHvState , pDiskState , pIgnoreIpolicy , pHotplug , pAllowRuntimeChgs , pInstDisks , pDiskTemplate , pOptDiskTemplate , pExtParams , pFileDriver , pFileStorageDir , pClusterFileStorageDir , pClusterSharedFileStorageDir , pClusterGlusterStorageDir , pInstanceCommunicationNetwork , pZeroingImage , pCompressionTools , pVgName , pEnabledHypervisors , pHypervisor , pClusterHvParams , pInstHvParams , pClusterBeParams , pInstBeParams , pResetDefaults , pOsHvp , pClusterOsParams , pClusterOsParamsPrivate , pInstOsParams , pInstOsParamsPrivate , pInstOsParamsSecret , pCandidatePoolSize , pMaxRunningJobs , pMaxTrackedJobs , pUidPool , pAddUids , pRemoveUids , pMaintainNodeHealth , pModifyEtcHosts , pPreallocWipeDisks , pNicParams , pInstNics , pNdParams , pIpolicy , pDrbdHelper , pDefaultIAllocator , pDefaultIAllocatorParams , pMasterNetdev , pMasterNetmask , pReservedLvs , pHiddenOs , pBlacklistedOs , pUseExternalMipScript , pQueryFields , pQueryFilter , pQueryFieldsFields , pOobCommand , pOobTimeout , pIgnoreStatus , pPowerDelay , pPrimaryIp , pSecondaryIp , pReadd , pNodeGroup , pMasterCapable , pVmCapable , pNames , pNodes , pRequiredNodes , pRequiredNodeUuids , pStorageType , pOptStorageType , pStorageChanges , pMasterCandidate , pOffline , pDrained , pAutoPromote , pPowered , pIallocator , pRemoteNode , pRemoteNodeUuid , pEvacMode , pIgnoreSoftErrors , pInstCreateMode , pNoInstall , pInstOs , pPrimaryNode , pPrimaryNodeUuid , pSecondaryNode , pSecondaryNodeUuid , pSourceHandshake , pSourceInstance , pSourceShutdownTimeout , pSourceX509Ca , pSrcNode , pSrcNodeUuid , pSrcPath , pStartInstance , pForthcoming , pCommit , pInstTags , pMultiAllocInstances , pTempOsParams , pTempOsParamsPrivate , pTempOsParamsSecret , pTempHvParams , pTempBeParams , pIgnoreFailures , pNewName , pIgnoreSecondaries , pRebootType , pIgnoreDiskSize , pRecreateDisksInfo , pStatic , pInstParamsNicChanges , pInstParamsDiskChanges , pRuntimeMem , pOsNameChange , pDiskIndex , pDiskChgAmount , pDiskChgAbsolute , pTargetGroups , pExportMode , pExportTargetNode , pExportTargetNodeUuid , pRemoveInstance , pIgnoreRemoveFailures , pX509KeyName , pX509DestCA , pZeroFreeSpace , pHelperStartupTimeout , pHelperShutdownTimeout , pZeroingTimeoutFixed , pZeroingTimeoutPerMiB , pTagSearchPattern , pRestrictedCommand , pReplaceDisksMode , pReplaceDisksList , pAllowFailover , pForceFailover , pDelayDuration , pDelayOnMaster , pDelayOnNodes , pDelayOnNodeUuids , pDelayRepeat , pDelayInterruptible , pDelayNoLocks , pIAllocatorDirection , pIAllocatorMode , pIAllocatorReqName , pIAllocatorNics , pIAllocatorDisks , pIAllocatorMemory , pIAllocatorVCpus , pIAllocatorOs , pIAllocatorInstances , pIAllocatorEvacMode , pIAllocatorSpindleUse , pIAllocatorCount , pJQueueNotifyWaitLock , pJQueueNotifyExec , pJQueueLogMessages , pJQueueFail , pTestDummyResult , pTestDummyMessages , pTestDummyFail , pTestDummySubmitJobs , pNetworkName , pNetworkAddress4 , pNetworkGateway4 , pNetworkAddress6 , pNetworkGateway6 , pNetworkMacPrefix , pNetworkAddRsvdIps , pNetworkRemoveRsvdIps , pNetworkMode , pNetworkLink , pNetworkVlan , pDryRun , pDebugLevel , pOpPriority , pDependencies , pComment , pReason , pSequential , pEnabledDiskTemplates , pEnabledUserShutdown , pAdminStateSource , pEnabledDataCollectors , pDataCollectorInterval , pNodeSslCerts , pSshKeyBits , pSshKeyType , pRenewSshKeys , pNodeSetup , pVerifyClutter , pLongSleep , pIsStrict ) where import Control.Monad (liftM, mplus) import Control.Monad.Fail (MonadFail) import Text.JSON (JSON, JSValue(..), JSObject (..), readJSON, showJSON, fromJSString, toJSObject) import qualified Text.JSON import Text.JSON.Pretty (pp_value) import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.THH import Ganeti.THH.Field import Ganeti.Utils import Ganeti.JSON (GenericContainer) import Ganeti.Types import qualified Ganeti.Query.Language as Qlang -- * Helper functions and types -- | Build a boolean field. booleanField :: String -> Field booleanField = flip simpleField [t| Bool |] -- | Default a field to 'False'. defaultFalse :: String -> Field defaultFalse = defaultField [| False |] . booleanField -- | Default a field to 'True'. defaultTrue :: String -> Field defaultTrue = defaultField [| True |] . booleanField -- | An alias for a 'String' field. stringField :: String -> Field stringField = flip simpleField [t| String |] -- | An alias for an optional string field. optionalStringField :: String -> Field optionalStringField = optionalField . stringField -- | An alias for an optional non-empty string field. optionalNEStringField :: String -> Field optionalNEStringField = optionalField . flip simpleField [t| NonEmptyString |] -- | Function to force a non-negative value, without returning via a -- monad. This is needed for, and should be used /only/ in the case of -- forcing constants. In case the constant is wrong (< 0), this will -- become a runtime error. forceNonNeg :: (Num a, Ord a, Show a) => a -> NonNegative a forceNonNeg i = case mkNonNegative i of Ok n -> n Bad msg -> error msg -- ** Disks -- | Disk index type (embedding constraints on the index value via a -- smart constructor). newtype DiskIndex = DiskIndex { unDiskIndex :: Int } deriving (Show, Eq, Ord) -- | Smart constructor for 'DiskIndex'. mkDiskIndex :: (MonadFail m) => Int -> m DiskIndex mkDiskIndex i | i >= 0 && i < C.maxDisks = return (DiskIndex i) | otherwise = fail $ "Invalid value for disk index '" ++ show i ++ "', required between 0 and " ++ show C.maxDisks instance JSON DiskIndex where readJSON v = readJSON v >>= mkDiskIndex showJSON = showJSON . unDiskIndex -- ** I* param types -- | Type holding disk access modes. $(declareSADT "DiskAccess" [ ("DiskReadOnly", 'C.diskRdonly) , ("DiskReadWrite", 'C.diskRdwr) ]) $(makeJSONInstance ''DiskAccess) -- | NIC modification definition. $(buildObject "INicParams" "inic" [ optionalField $ simpleField C.inicMac [t| NonEmptyString |] , optionalField $ simpleField C.inicIp [t| String |] , optionalField $ simpleField C.inicMode [t| NonEmptyString |] , optionalField $ simpleField C.inicLink [t| NonEmptyString |] , optionalField $ simpleField C.inicName [t| NonEmptyString |] , optionalField $ simpleField C.inicVlan [t| String |] , optionalField $ simpleField C.inicBridge [t| NonEmptyString |] , optionalField $ simpleField C.inicNetwork [t| NonEmptyString |] ]) deriving instance Ord INicParams -- | Disk modification definition. $(buildObject "IDiskParams" "idisk" [ specialNumericalField 'parseUnitAssumeBinary . optionalField $ simpleField C.idiskSize [t| Int |] , optionalField $ simpleField C.idiskMode [t| DiskAccess |] , optionalField $ simpleField C.idiskAdopt [t| NonEmptyString |] , optionalField $ simpleField C.idiskVg [t| NonEmptyString |] , optionalField $ simpleField C.idiskMetavg [t| NonEmptyString |] , optionalField $ simpleField C.idiskName [t| NonEmptyString |] , optionalField $ simpleField C.idiskProvider [t| NonEmptyString |] , optionalField $ simpleField C.idiskSpindles [t| Int |] , optionalField $ simpleField C.idiskAccess [t| NonEmptyString |] , andRestArguments "opaque" ]) deriving instance Ord IDiskParams -- | Disk changes type for OpInstanceRecreateDisks. This is a bit -- strange, because the type in Python is something like Either -- [DiskIndex] [DiskChanges], but we can't represent the type of an -- empty list in JSON, so we have to add a custom case for the empty -- list. data RecreateDisksInfo = RecreateDisksAll | RecreateDisksIndices (NonEmpty DiskIndex) | RecreateDisksParams (NonEmpty (DiskIndex, IDiskParams)) deriving (Eq, Show, Ord) readRecreateDisks :: JSValue -> Text.JSON.Result RecreateDisksInfo readRecreateDisks (JSArray []) = return RecreateDisksAll readRecreateDisks v = case readJSON v::Text.JSON.Result [DiskIndex] of Text.JSON.Ok indices -> liftM RecreateDisksIndices (mkNonEmpty indices) _ -> case readJSON v::Text.JSON.Result [(DiskIndex, IDiskParams)] of Text.JSON.Ok params -> liftM RecreateDisksParams (mkNonEmpty params) _ -> fail $ "Can't parse disk information as either list of disk" ++ " indices or list of disk parameters; value received:" ++ show (pp_value v) instance JSON RecreateDisksInfo where readJSON = readRecreateDisks showJSON RecreateDisksAll = showJSON () showJSON (RecreateDisksIndices idx) = showJSON idx showJSON (RecreateDisksParams params) = showJSON params -- | Simple type for old-style ddm changes. data DdmOldChanges = DdmOldIndex (NonNegative Int) | DdmOldMod DdmSimple deriving (Eq, Show, Ord) readDdmOldChanges :: JSValue -> Text.JSON.Result DdmOldChanges readDdmOldChanges v = case readJSON v::Text.JSON.Result (NonNegative Int) of Text.JSON.Ok nn -> return $ DdmOldIndex nn _ -> case readJSON v::Text.JSON.Result DdmSimple of Text.JSON.Ok ddms -> return $ DdmOldMod ddms _ -> fail $ "Can't parse value '" ++ show (pp_value v) ++ "' as" ++ " either index or modification" instance JSON DdmOldChanges where showJSON (DdmOldIndex i) = showJSON i showJSON (DdmOldMod m) = showJSON m readJSON = readDdmOldChanges -- | Instance disk or nic modifications. data SetParamsMods a = SetParamsEmpty | SetParamsDeprecated (NonEmpty (DdmOldChanges, a)) | SetParamsNew (NonEmpty (DdmFull, Int, a)) | SetParamsNewName (NonEmpty (DdmFull, String, a)) deriving (Eq, Show, Ord) -- | Custom deserialiser for 'SetParamsMods'. readSetParams :: (JSON a) => JSValue -> Text.JSON.Result (SetParamsMods a) readSetParams (JSArray []) = return SetParamsEmpty readSetParams v = liftM SetParamsDeprecated (readJSON v) `mplus` liftM SetParamsNew (readJSON v) `mplus` liftM SetParamsNewName (readJSON v) instance (JSON a) => JSON (SetParamsMods a) where showJSON SetParamsEmpty = showJSON () showJSON (SetParamsDeprecated v) = showJSON v showJSON (SetParamsNew v) = showJSON v showJSON (SetParamsNewName v) = showJSON v readJSON = readSetParams -- | Custom type for target_node parameter of OpBackupExport, which -- varies depending on mode. FIXME: this uses an [JSValue] since -- we don't care about individual rows (just like the Python code -- tests). But the proper type could be parsed if we wanted. data ExportTarget = ExportTargetLocal NonEmptyString | ExportTargetRemote [JSValue] deriving (Eq, Show, Ord) -- | Custom reader for 'ExportTarget'. readExportTarget :: JSValue -> Text.JSON.Result ExportTarget readExportTarget (JSString s) = liftM ExportTargetLocal $ mkNonEmpty (fromJSString s) readExportTarget (JSArray arr) = return $ ExportTargetRemote arr readExportTarget v = fail $ "Invalid value received for 'target_node': " ++ show (pp_value v) instance JSON ExportTarget where showJSON (ExportTargetLocal s) = showJSON s showJSON (ExportTargetRemote l) = showJSON l readJSON = readExportTarget -- * Common opcode parameters pDryRun :: Field pDryRun = withDoc "Run checks only, don't execute" . optionalField $ booleanField "dry_run" pDebugLevel :: Field pDebugLevel = withDoc "Debug level" . optionalField $ simpleField "debug_level" [t| NonNegative Int |] pOpPriority :: Field pOpPriority = withDoc "Opcode priority. Note: python uses a separate constant,\ \ we're using the actual value we know it's the default" . defaultField [| OpPrioNormal |] $ simpleField "priority" [t| OpSubmitPriority |] pDependencies :: Field pDependencies = withDoc "Job dependencies" . optionalNullSerField $ simpleField "depends" [t| [JobDependency] |] pComment :: Field pComment = withDoc "Comment field" . optionalNullSerField $ stringField "comment" pReason :: Field pReason = withDoc "Reason trail field" $ simpleField C.opcodeReason [t| ReasonTrail |] pSequential :: Field pSequential = withDoc "Sequential job execution" $ defaultFalse C.opcodeSequential -- * Parameters pDebugSimulateErrors :: Field pDebugSimulateErrors = withDoc "Whether to simulate errors (useful for debugging)" $ defaultFalse "debug_simulate_errors" pErrorCodes :: Field pErrorCodes = withDoc "Error codes" $ defaultFalse "error_codes" pSkipChecks :: Field pSkipChecks = withDoc "Which checks to skip" . defaultField [| emptyListSet |] $ simpleField "skip_checks" [t| ListSet VerifyOptionalChecks |] pIgnoreErrors :: Field pIgnoreErrors = withDoc "List of error codes that should be treated as warnings" . defaultField [| emptyListSet |] $ simpleField "ignore_errors" [t| ListSet CVErrorCode |] pVerbose :: Field pVerbose = withDoc "Verbose mode" $ defaultFalse "verbose" pDebug :: Field pDebug = withDoc "Debug mode" $ defaultFalse "debug" pOptGroupName :: Field pOptGroupName = withDoc "Optional group name" . renameField "OptGroupName" . optionalField $ simpleField "group_name" [t| NonEmptyString |] pGroupName :: Field pGroupName = withDoc "Group name" $ simpleField "group_name" [t| NonEmptyString |] -- | Whether to hotplug device. pHotplug :: Field pHotplug = defaultTrue "hotplug" pInstances :: Field pInstances = withDoc "List of instances" . defaultField [| [] |] $ simpleField "instances" [t| [NonEmptyString] |] pOutputFields :: Field pOutputFields = withDoc "Selected output fields" $ simpleField "output_fields" [t| [NonEmptyString] |] pName :: Field pName = withDoc "A generic name" $ simpleField "name" [t| NonEmptyString |] pForce :: Field pForce = withDoc "Whether to force the operation" $ defaultFalse "force" pHvState :: Field pHvState = withDoc "Set hypervisor states" . optionalField $ simpleField "hv_state" [t| JSObject JSValue |] pDiskState :: Field pDiskState = withDoc "Set disk states" . optionalField $ simpleField "disk_state" [t| JSObject JSValue |] -- | Cluster-wide default directory for storing file-backed disks. pClusterFileStorageDir :: Field pClusterFileStorageDir = renameField "ClusterFileStorageDir" $ optionalStringField "file_storage_dir" -- | Cluster-wide default directory for storing shared-file-backed disks. pClusterSharedFileStorageDir :: Field pClusterSharedFileStorageDir = renameField "ClusterSharedFileStorageDir" $ optionalStringField "shared_file_storage_dir" -- | Cluster-wide default directory for storing Gluster-backed disks. pClusterGlusterStorageDir :: Field pClusterGlusterStorageDir = renameField "ClusterGlusterStorageDir" $ optionalStringField "gluster_storage_dir" pInstallImage :: Field pInstallImage = withDoc "OS image for running OS scripts in a safe environment" $ optionalStringField "install_image" pInstanceCommunicationNetwork :: Field pInstanceCommunicationNetwork = optionalStringField "instance_communication_network" -- | The OS to use when zeroing instance disks. pZeroingImage :: Field pZeroingImage = optionalStringField "zeroing_image" -- | The additional tools that can be used to compress data in transit pCompressionTools :: Field pCompressionTools = withDoc "List of enabled compression tools" . optionalField $ simpleField "compression_tools" [t| [NonEmptyString] |] -- | Volume group name. pVgName :: Field pVgName = withDoc "Volume group name" $ optionalStringField "vg_name" pEnabledHypervisors :: Field pEnabledHypervisors = withDoc "List of enabled hypervisors" . optionalField $ simpleField "enabled_hypervisors" [t| [Hypervisor] |] pClusterHvParams :: Field pClusterHvParams = withDoc "Cluster-wide hypervisor parameters, hypervisor-dependent" . renameField "ClusterHvParams" . optionalField $ simpleField "hvparams" [t| GenericContainer String (JSObject JSValue) |] pClusterBeParams :: Field pClusterBeParams = withDoc "Cluster-wide backend parameter defaults" . renameField "ClusterBeParams" . optionalField $ simpleField "beparams" [t| JSObject JSValue |] pOsHvp :: Field pOsHvp = withDoc "Cluster-wide per-OS hypervisor parameter defaults" . optionalField $ simpleField "os_hvp" [t| GenericContainer String (JSObject JSValue) |] pClusterOsParams :: Field pClusterOsParams = withDoc "Cluster-wide OS parameter defaults" . renameField "ClusterOsParams" . optionalField $ simpleField "osparams" [t| GenericContainer String (JSObject JSValue) |] pClusterOsParamsPrivate :: Field pClusterOsParamsPrivate = withDoc "Cluster-wide private OS parameter defaults" . renameField "ClusterOsParamsPrivate" . optionalField $ -- This field needs an unique name to aid Python deserialization simpleField "osparams_private_cluster" [t| GenericContainer String (JSObject (Private JSValue)) |] pGroupDiskParams :: Field pGroupDiskParams = withDoc "Disk templates' parameter defaults" . optionalField $ simpleField "diskparams" [t| GenericContainer DiskTemplate (JSObject JSValue) |] pCandidatePoolSize :: Field pCandidatePoolSize = withDoc "Master candidate pool size" . optionalField $ simpleField "candidate_pool_size" [t| Positive Int |] pMaxRunningJobs :: Field pMaxRunningJobs = withDoc "Maximal number of jobs to run simultaneously" . optionalField $ simpleField "max_running_jobs" [t| Positive Int |] pMaxTrackedJobs :: Field pMaxTrackedJobs = withDoc "Maximal number of jobs tracked in the job queue" . optionalField $ simpleField "max_tracked_jobs" [t| Positive Int |] pUidPool :: Field pUidPool = withDoc "Set UID pool, must be list of lists describing UID ranges\ \ (two items, start and end inclusive)" . optionalField $ simpleField "uid_pool" [t| [(Int, Int)] |] pAddUids :: Field pAddUids = withDoc "Extend UID pool, must be list of lists describing UID\ \ ranges (two items, start and end inclusive)" . optionalField $ simpleField "add_uids" [t| [(Int, Int)] |] pRemoveUids :: Field pRemoveUids = withDoc "Shrink UID pool, must be list of lists describing UID\ \ ranges (two items, start and end inclusive) to be removed" . optionalField $ simpleField "remove_uids" [t| [(Int, Int)] |] pMaintainNodeHealth :: Field pMaintainNodeHealth = withDoc "Whether to automatically maintain node health" . optionalField $ booleanField "maintain_node_health" -- | Whether to modify and keep in sync the @/etc/hosts@ files of nodes. pModifyEtcHosts :: Field pModifyEtcHosts = optionalField $ booleanField "modify_etc_hosts" -- | Whether to wipe disks before allocating them to instances. pPreallocWipeDisks :: Field pPreallocWipeDisks = withDoc "Whether to wipe disks before allocating them to instances" . optionalField $ booleanField "prealloc_wipe_disks" pNicParams :: Field pNicParams = withDoc "Cluster-wide NIC parameter defaults" . optionalField $ simpleField "nicparams" [t| INicParams |] pIpolicy :: Field pIpolicy = withDoc "Ipolicy specs" . optionalField $ simpleField "ipolicy" [t| JSObject JSValue |] pDrbdHelper :: Field pDrbdHelper = withDoc "DRBD helper program" $ optionalStringField "drbd_helper" pDefaultIAllocator :: Field pDefaultIAllocator = withDoc "Default iallocator for cluster" $ optionalStringField "default_iallocator" pDefaultIAllocatorParams :: Field pDefaultIAllocatorParams = withDoc "Default iallocator parameters for cluster" . optionalField $ simpleField "default_iallocator_params" [t| JSObject JSValue |] pMasterNetdev :: Field pMasterNetdev = withDoc "Master network device" $ optionalStringField "master_netdev" pMasterNetmask :: Field pMasterNetmask = withDoc "Netmask of the master IP" . optionalField $ simpleField "master_netmask" [t| NonNegative Int |] pReservedLvs :: Field pReservedLvs = withDoc "List of reserved LVs" . optionalField $ simpleField "reserved_lvs" [t| [NonEmptyString] |] pHiddenOs :: Field pHiddenOs = withDoc "Modify list of hidden operating systems: each modification\ \ must have two items, the operation and the OS name; the operation\ \ can be add or remove" . optionalField $ simpleField "hidden_os" [t| [(DdmSimple, NonEmptyString)] |] pBlacklistedOs :: Field pBlacklistedOs = withDoc "Modify list of blacklisted operating systems: each\ \ modification must have two items, the operation and the OS name;\ \ the operation can be add or remove" . optionalField $ simpleField "blacklisted_os" [t| [(DdmSimple, NonEmptyString)] |] pUseExternalMipScript :: Field pUseExternalMipScript = withDoc "Whether to use an external master IP address setup script" . optionalField $ booleanField "use_external_mip_script" pEnabledDiskTemplates :: Field pEnabledDiskTemplates = withDoc "List of enabled disk templates" . optionalField $ simpleField "enabled_disk_templates" [t| [DiskTemplate] |] pEnabledUserShutdown :: Field pEnabledUserShutdown = withDoc "Whether user shutdown is enabled cluster wide" . optionalField $ simpleField "enabled_user_shutdown" [t| Bool |] pQueryWhat :: Field pQueryWhat = withDoc "Resource(s) to query for" $ simpleField "what" [t| Qlang.QueryTypeOp |] pUseLocking :: Field pUseLocking = withDoc "Whether to use synchronization" $ defaultFalse "use_locking" pQueryFields :: Field pQueryFields = withDoc "Requested fields" $ simpleField "fields" [t| [NonEmptyString] |] pQueryFilter :: Field pQueryFilter = withDoc "Query filter" . optionalField $ simpleField "qfilter" [t| [JSValue] |] pQueryFieldsFields :: Field pQueryFieldsFields = withDoc "Requested fields; if not given, all are returned" . renameField "QueryFieldsFields" $ optionalField pQueryFields pNodeNames :: Field pNodeNames = withDoc "List of node names to run the OOB command against" . defaultField [| [] |] $ simpleField "node_names" [t| [NonEmptyString] |] pNodeUuids :: Field pNodeUuids = withDoc "List of node UUIDs" . optionalField $ simpleField "node_uuids" [t| [NonEmptyString] |] pOobCommand :: Field pOobCommand = withDoc "OOB command to run" . renameField "OobCommand" $ simpleField "command" [t| OobCommand |] pOobTimeout :: Field pOobTimeout = withDoc "Timeout before the OOB helper will be terminated" . defaultField [| C.oobTimeout |] . renameField "OobTimeout" $ simpleField "timeout" [t| Int |] pIgnoreStatus :: Field pIgnoreStatus = withDoc "Ignores the node offline status for power off" $ defaultFalse "ignore_status" pPowerDelay :: Field pPowerDelay = -- FIXME: we can't use the proper type "NonNegative Double", since -- the default constant is a plain Double, not a non-negative one. -- And trying to fix the constant introduces a cyclic import. withDoc "Time in seconds to wait between powering on nodes" . defaultField [| C.oobPowerDelay |] $ simpleField "power_delay" [t| Double |] pRequiredNodes :: Field pRequiredNodes = withDoc "Required list of node names" . renameField "ReqNodes" $ simpleField "nodes" [t| [NonEmptyString] |] pRequiredNodeUuids :: Field pRequiredNodeUuids = withDoc "Required list of node UUIDs" . renameField "ReqNodeUuids" . optionalField $ simpleField "node_uuids" [t| [NonEmptyString] |] pRestrictedCommand :: Field pRestrictedCommand = withDoc "Restricted command name" . renameField "RestrictedCommand" $ simpleField "command" [t| NonEmptyString |] pNodeName :: Field pNodeName = withDoc "A required node name (for single-node LUs)" $ simpleField "node_name" [t| NonEmptyString |] pNodeUuid :: Field pNodeUuid = withDoc "A node UUID (for single-node LUs)" . optionalField $ simpleField "node_uuid" [t| NonEmptyString |] pPrimaryIp :: Field pPrimaryIp = withDoc "Primary IP address" . optionalField $ simpleField "primary_ip" [t| NonEmptyString |] pSecondaryIp :: Field pSecondaryIp = withDoc "Secondary IP address" $ optionalNEStringField "secondary_ip" pReadd :: Field pReadd = withDoc "Whether node is re-added to cluster" $ defaultFalse "readd" pNodeGroup :: Field pNodeGroup = withDoc "Initial node group" $ optionalNEStringField "group" pMasterCapable :: Field pMasterCapable = withDoc "Whether node can become master or master candidate" . optionalField $ booleanField "master_capable" pVmCapable :: Field pVmCapable = withDoc "Whether node can host instances" . optionalField $ booleanField "vm_capable" pNdParams :: Field pNdParams = withDoc "Node parameters" . renameField "genericNdParams" . optionalField $ simpleField "ndparams" [t| JSObject JSValue |] pNames :: Field pNames = withDoc "List of names" . defaultField [| [] |] $ simpleField "names" [t| [NonEmptyString] |] pNodes :: Field pNodes = withDoc "List of nodes" . defaultField [| [] |] $ simpleField "nodes" [t| [NonEmptyString] |] pStorageType :: Field pStorageType = withDoc "Storage type" $ simpleField "storage_type" [t| StorageType |] pOptStorageType :: Field pOptStorageType = withDoc "Storage type" . renameField "OptStorageType" . optionalField $ simpleField "storage_type" [t| StorageType |] pStorageName :: Field pStorageName = withDoc "Storage name" . renameField "StorageName" . optionalField $ simpleField "name" [t| NonEmptyString |] pStorageChanges :: Field pStorageChanges = withDoc "Requested storage changes" $ simpleField "changes" [t| JSObject JSValue |] pIgnoreConsistency :: Field pIgnoreConsistency = withDoc "Whether to ignore disk consistency" $ defaultFalse "ignore_consistency" pIgnoreHVVersions :: Field pIgnoreHVVersions = withDoc "Whether to ignore incompatible Hypervisor versions" $ defaultFalse "ignore_hvversions" pMasterCandidate :: Field pMasterCandidate = withDoc "Whether the node should become a master candidate" . optionalField $ booleanField "master_candidate" pOffline :: Field pOffline = withDoc "Whether to mark the node or instance offline" . optionalField $ booleanField "offline" pDrained ::Field pDrained = withDoc "Whether to mark the node as drained" . optionalField $ booleanField "drained" pAutoPromote :: Field pAutoPromote = withDoc "Whether node(s) should be promoted to master candidate if\ \ necessary" $ defaultFalse "auto_promote" pPowered :: Field pPowered = withDoc "Whether the node should be marked as powered" . optionalField $ booleanField "powered" pMigrationMode :: Field pMigrationMode = withDoc "Migration type (live/non-live)" . renameField "MigrationMode" . optionalField $ simpleField "mode" [t| MigrationMode |] pMigrationLive :: Field pMigrationLive = withDoc "Obsolete \'live\' migration mode (do not use)" . renameField "OldLiveMode" . optionalField $ booleanField "live" pMigrationTargetNode :: Field pMigrationTargetNode = withDoc "Target node for instance migration/failover" $ optionalNEStringField "target_node" pMigrationTargetNodeUuid :: Field pMigrationTargetNodeUuid = withDoc "Target node UUID for instance migration/failover" $ optionalNEStringField "target_node_uuid" pAllowRuntimeChgs :: Field pAllowRuntimeChgs = withDoc "Whether to allow runtime changes while migrating" $ defaultTrue "allow_runtime_changes" pIgnoreIpolicy :: Field pIgnoreIpolicy = withDoc "Whether to ignore ipolicy violations" $ defaultFalse "ignore_ipolicy" pIallocator :: Field pIallocator = withDoc "Iallocator for deciding the target node for shared-storage\ \ instances" $ optionalNEStringField "iallocator" pEarlyRelease :: Field pEarlyRelease = withDoc "Whether to release locks as soon as possible" $ defaultFalse "early_release" pRemoteNode :: Field pRemoteNode = withDoc "New secondary node" $ optionalNEStringField "remote_node" pRemoteNodeUuid :: Field pRemoteNodeUuid = withDoc "New secondary node UUID" $ optionalNEStringField "remote_node_uuid" pEvacMode :: Field pEvacMode = withDoc "Node evacuation mode" . renameField "EvacMode" $ simpleField "mode" [t| EvacMode |] pIgnoreSoftErrors :: Field pIgnoreSoftErrors = withDoc "Ignore soft htools errors" . optionalField $ booleanField "ignore_soft_errors" pInstanceName :: Field pInstanceName = withDoc "A required instance name (for single-instance LUs)" $ simpleField "instance_name" [t| String |] pInstanceCommunication :: Field pInstanceCommunication = withDoc C.instanceCommunicationDoc $ defaultFalse "instance_communication" pOptInstanceCommunication :: Field pOptInstanceCommunication = withDoc C.instanceCommunicationDoc . renameField "OptInstanceCommunication" . optionalField $ booleanField "instance_communication" pForceVariant :: Field pForceVariant = withDoc "Whether to force an unknown OS variant" $ defaultFalse "force_variant" pWaitForSync :: Field pWaitForSync = withDoc "Whether to wait for the disk to synchronize" $ defaultTrue "wait_for_sync" pNameCheck :: Field pNameCheck = withDoc "Whether to check name" $ defaultFalse "name_check" pInstBeParams :: Field pInstBeParams = withDoc "Backend parameters for instance" . renameField "InstBeParams" . defaultField [| toJSObject [] |] $ simpleField "beparams" [t| JSObject JSValue |] pInstDisks :: Field pInstDisks = withDoc "List of instance disks" . renameField "instDisks" $ simpleField "disks" [t| [IDiskParams] |] pDiskTemplate :: Field pDiskTemplate = withDoc "Disk template" $ simpleField "disk_template" [t| DiskTemplate |] pExtParams :: Field pExtParams = withDoc "List of ExtStorage parameters" . renameField "InstExtParams" . defaultField [| toJSObject [] |] $ simpleField "ext_params" [t| JSObject JSValue |] pFileDriver :: Field pFileDriver = withDoc "Driver for file-backed disks" . optionalField $ simpleField "file_driver" [t| FileDriver |] pFileStorageDir :: Field pFileStorageDir = withDoc "Directory for storing file-backed disks" $ optionalNEStringField "file_storage_dir" pInstHvParams :: Field pInstHvParams = withDoc "Hypervisor parameters for instance, hypervisor-dependent" . renameField "InstHvParams" . defaultField [| toJSObject [] |] $ simpleField "hvparams" [t| JSObject JSValue |] pHypervisor :: Field pHypervisor = withDoc "Selected hypervisor for an instance" . optionalField $ simpleField "hypervisor" [t| Hypervisor |] pResetDefaults :: Field pResetDefaults = withDoc "Reset instance parameters to default if equal" $ defaultFalse "identify_defaults" pIpCheck :: Field pIpCheck = withDoc "Whether to ensure instance's IP address is inactive" $ defaultFalse "ip_check" pIpConflictsCheck :: Field pIpConflictsCheck = withDoc "Whether to check for conflicting IP addresses" $ defaultTrue "conflicts_check" pInstCreateMode :: Field pInstCreateMode = withDoc "Instance creation mode" . renameField "InstCreateMode" $ simpleField "mode" [t| InstCreateMode |] pInstNics :: Field pInstNics = withDoc "List of NIC (network interface) definitions" $ simpleField "nics" [t| [INicParams] |] pNoInstall :: Field pNoInstall = withDoc "Do not install the OS (will disable automatic start)" . optionalField $ booleanField "no_install" pInstOs :: Field pInstOs = withDoc "OS type for instance installation" $ optionalNEStringField "os_type" pInstOsParams :: Field pInstOsParams = withDoc "OS parameters for instance" . renameField "InstOsParams" . defaultField [| toJSObject [] |] $ simpleField "osparams" [t| JSObject JSValue |] pInstOsParamsPrivate :: Field pInstOsParamsPrivate = withDoc "Private OS parameters for instance" . optionalField $ simpleField "osparams_private" [t| JSObject (Private JSValue) |] pInstOsParamsSecret :: Field pInstOsParamsSecret = withDoc "Secret OS parameters for instance" . optionalField $ simpleField "osparams_secret" [t| JSObject (Secret JSValue) |] pPrimaryNode :: Field pPrimaryNode = withDoc "Primary node for an instance" $ optionalNEStringField "pnode" pPrimaryNodeUuid :: Field pPrimaryNodeUuid = withDoc "Primary node UUID for an instance" $ optionalNEStringField "pnode_uuid" pSecondaryNode :: Field pSecondaryNode = withDoc "Secondary node for an instance" $ optionalNEStringField "snode" pSecondaryNodeUuid :: Field pSecondaryNodeUuid = withDoc "Secondary node UUID for an instance" $ optionalNEStringField "snode_uuid" pSourceHandshake :: Field pSourceHandshake = withDoc "Signed handshake from source (remote import only)" . optionalField $ simpleField "source_handshake" [t| [JSValue] |] pSourceInstance :: Field pSourceInstance = withDoc "Source instance name (remote import only)" $ optionalNEStringField "source_instance_name" -- FIXME: non-negative int, whereas the constant is a plain int. pSourceShutdownTimeout :: Field pSourceShutdownTimeout = withDoc "How long source instance was given to shut down (remote import\ \ only)" . defaultField [| forceNonNeg C.defaultShutdownTimeout |] $ simpleField "source_shutdown_timeout" [t| NonNegative Int |] pSourceX509Ca :: Field pSourceX509Ca = withDoc "Source X509 CA in PEM format (remote import only)" $ optionalNEStringField "source_x509_ca" pSrcNode :: Field pSrcNode = withDoc "Source node for import" $ optionalNEStringField "src_node" pSrcNodeUuid :: Field pSrcNodeUuid = withDoc "Source node UUID for import" $ optionalNEStringField "src_node_uuid" pSrcPath :: Field pSrcPath = withDoc "Source directory for import" $ optionalNEStringField "src_path" pStartInstance :: Field pStartInstance = withDoc "Whether to start instance after creation" $ defaultTrue "start" pForthcoming :: Field pForthcoming = withDoc "Whether to only reserve resources" $ defaultFalse "forthcoming" pCommit :: Field pCommit = withDoc "Commit the already reserved instance" $ defaultFalse "commit" -- FIXME: unify/simplify with pTags, once that migrates to NonEmpty String" pInstTags :: Field pInstTags = withDoc "Instance tags" . renameField "InstTags" . defaultField [| [] |] $ simpleField "tags" [t| [NonEmptyString] |] pMultiAllocInstances :: Field pMultiAllocInstances = withDoc "List of instance create opcodes describing the instances to\ \ allocate" . renameField "InstMultiAlloc" . defaultField [| [] |] $ simpleField "instances"[t| [JSValue] |] pOpportunisticLocking :: Field pOpportunisticLocking = withDoc "Whether to employ opportunistic locking for nodes, meaning\ \ nodes already locked by another opcode won't be considered for\ \ instance allocation (only when an iallocator is used)" $ defaultFalse "opportunistic_locking" pInstanceUuid :: Field pInstanceUuid = withDoc "An instance UUID (for single-instance LUs)" . optionalField $ simpleField "instance_uuid" [t| NonEmptyString |] pTempOsParams :: Field pTempOsParams = withDoc "Temporary OS parameters (currently only in reinstall, might be\ \ added to install as well)" . renameField "TempOsParams" . optionalField $ simpleField "osparams" [t| JSObject JSValue |] pTempOsParamsPrivate :: Field pTempOsParamsPrivate = withDoc "Private OS parameters for instance reinstalls" . optionalField $ simpleField "osparams_private" [t| JSObject (Private JSValue) |] pTempOsParamsSecret :: Field pTempOsParamsSecret = withDoc "Secret OS parameters for instance reinstalls" . optionalField $ simpleField "osparams_secret" [t| JSObject (Secret JSValue) |] pShutdownTimeout :: Field pShutdownTimeout = withDoc "How long to wait for instance to shut down" . defaultField [| forceNonNeg C.defaultShutdownTimeout |] $ simpleField "shutdown_timeout" [t| NonNegative Int |] -- | Another name for the shutdown timeout, because we like to be -- inconsistent. pShutdownTimeout' :: Field pShutdownTimeout' = withDoc "How long to wait for instance to shut down" . renameField "InstShutdownTimeout" . defaultField [| forceNonNeg C.defaultShutdownTimeout |] $ simpleField "timeout" [t| NonNegative Int |] pIgnoreFailures :: Field pIgnoreFailures = withDoc "Whether to ignore failures during removal" $ defaultFalse "ignore_failures" pNewName :: Field pNewName = withDoc "New group or instance name" $ simpleField "new_name" [t| NonEmptyString |] pIgnoreOfflineNodes :: Field pIgnoreOfflineNodes = withDoc "Whether to ignore offline nodes" $ defaultFalse "ignore_offline_nodes" pTempHvParams :: Field pTempHvParams = withDoc "Temporary hypervisor parameters, hypervisor-dependent" . renameField "TempHvParams" . defaultField [| toJSObject [] |] $ simpleField "hvparams" [t| JSObject JSValue |] pTempBeParams :: Field pTempBeParams = withDoc "Temporary backend parameters" . renameField "TempBeParams" . defaultField [| toJSObject [] |] $ simpleField "beparams" [t| JSObject JSValue |] pNoRemember :: Field pNoRemember = withDoc "Do not remember instance state changes" $ defaultFalse "no_remember" pStartupPaused :: Field pStartupPaused = withDoc "Pause instance at startup" $ defaultFalse "startup_paused" pIgnoreSecondaries :: Field pIgnoreSecondaries = withDoc "Whether to start the instance even if secondary disks are failing" $ defaultFalse "ignore_secondaries" pRebootType :: Field pRebootType = withDoc "How to reboot the instance" $ simpleField "reboot_type" [t| RebootType |] pReplaceDisksMode :: Field pReplaceDisksMode = withDoc "Replacement mode" . renameField "ReplaceDisksMode" $ simpleField "mode" [t| ReplaceDisksMode |] pReplaceDisksList :: Field pReplaceDisksList = withDoc "List of disk indices" . renameField "ReplaceDisksList" . defaultField [| [] |] $ simpleField "disks" [t| [DiskIndex] |] pMigrationCleanup :: Field pMigrationCleanup = withDoc "Whether a previously failed migration should be cleaned up" . renameField "MigrationCleanup" $ defaultFalse "cleanup" pAllowFailover :: Field pAllowFailover = withDoc "Whether we can fallback to failover if migration is not possible" $ defaultFalse "allow_failover" pForceFailover :: Field pForceFailover = withDoc "Disallow migration moves and always use failovers" $ defaultFalse "force_failover" pMoveTargetNode :: Field pMoveTargetNode = withDoc "Target node for instance move" . renameField "MoveTargetNode" $ simpleField "target_node" [t| NonEmptyString |] pMoveTargetNodeUuid :: Field pMoveTargetNodeUuid = withDoc "Target node UUID for instance move" . renameField "MoveTargetNodeUuid" . optionalField $ simpleField "target_node_uuid" [t| NonEmptyString |] pMoveCompress :: Field pMoveCompress = withDoc "Compression mode to use during instance moves" . defaultField [| C.iecNone |] $ simpleField "compress" [t| String |] pBackupCompress :: Field pBackupCompress = withDoc "Compression mode to use for moves during backups/imports" . defaultField [| C.iecNone |] $ simpleField "compress" [t| String |] pIgnoreDiskSize :: Field pIgnoreDiskSize = withDoc "Whether to ignore recorded disk size" $ defaultFalse "ignore_size" pWaitForSyncFalse :: Field pWaitForSyncFalse = withDoc "Whether to wait for the disk to synchronize (defaults to false)" $ defaultField [| False |] pWaitForSync pRecreateDisksInfo :: Field pRecreateDisksInfo = withDoc "Disk list for recreate disks" . renameField "RecreateDisksInfo" . defaultField [| RecreateDisksAll |] $ simpleField "disks" [t| RecreateDisksInfo |] pStatic :: Field pStatic = withDoc "Whether to only return configuration data without querying nodes" $ defaultFalse "static" pInstParamsNicChanges :: Field pInstParamsNicChanges = withDoc "List of NIC changes" . renameField "InstNicChanges" . defaultField [| SetParamsEmpty |] $ simpleField "nics" [t| SetParamsMods INicParams |] pInstParamsDiskChanges :: Field pInstParamsDiskChanges = withDoc "List of disk changes" . renameField "InstDiskChanges" . defaultField [| SetParamsEmpty |] $ simpleField "disks" [t| SetParamsMods IDiskParams |] pRuntimeMem :: Field pRuntimeMem = withDoc "New runtime memory" . optionalField $ simpleField "runtime_mem" [t| Positive Int |] pOptDiskTemplate :: Field pOptDiskTemplate = withDoc "Instance disk template" . optionalField . renameField "OptDiskTemplate" $ simpleField "disk_template" [t| DiskTemplate |] pOsNameChange :: Field pOsNameChange = withDoc "Change the instance's OS without reinstalling the instance" $ optionalNEStringField "os_name" pDiskIndex :: Field pDiskIndex = withDoc "Disk index for e.g. grow disk" . renameField "DiskIndex" $ simpleField "disk" [t| DiskIndex |] pDiskChgAmount :: Field pDiskChgAmount = withDoc "Disk amount to add or grow to" . renameField "DiskChgAmount" $ simpleField "amount" [t| NonNegative Int |] pDiskChgAbsolute :: Field pDiskChgAbsolute = withDoc "Whether the amount parameter is an absolute target or a relative one" . renameField "DiskChkAbsolute" $ defaultFalse "absolute" pTargetGroups :: Field pTargetGroups = withDoc "Destination group names or UUIDs (defaults to \"all but current group\")" . optionalField $ simpleField "target_groups" [t| [NonEmptyString] |] pNodeGroupAllocPolicy :: Field pNodeGroupAllocPolicy = withDoc "Instance allocation policy" . optionalField $ simpleField "alloc_policy" [t| AllocPolicy |] pGroupNodeParams :: Field pGroupNodeParams = withDoc "Default node parameters for group" . optionalField $ simpleField "ndparams" [t| JSObject JSValue |] pExportMode :: Field pExportMode = withDoc "Export mode" . renameField "ExportMode" $ simpleField "mode" [t| ExportMode |] -- FIXME: Rename target_node as it changes meaning for different -- export modes (e.g. "destination") pExportTargetNode :: Field pExportTargetNode = withDoc "Target node (depends on export mode)" . renameField "ExportTarget" $ simpleField "target_node" [t| ExportTarget |] pExportTargetNodeUuid :: Field pExportTargetNodeUuid = withDoc "Target node UUID (if local export)" . renameField "ExportTargetNodeUuid" . optionalField $ simpleField "target_node_uuid" [t| NonEmptyString |] pShutdownInstance :: Field pShutdownInstance = withDoc "Whether to shutdown the instance before export" $ defaultTrue "shutdown" pRemoveInstance :: Field pRemoveInstance = withDoc "Whether to remove instance after export" $ defaultFalse "remove_instance" pIgnoreRemoveFailures :: Field pIgnoreRemoveFailures = withDoc "Whether to ignore failures while removing instances" $ defaultFalse "ignore_remove_failures" pX509KeyName :: Field pX509KeyName = withDoc "Name of X509 key (remote export only)" . optionalField $ simpleField "x509_key_name" [t| [JSValue] |] pX509DestCA :: Field pX509DestCA = withDoc "Destination X509 CA (remote export only)" $ optionalNEStringField "destination_x509_ca" pZeroFreeSpace :: Field pZeroFreeSpace = withDoc "Whether to zero the free space on the disks of the instance" $ defaultFalse "zero_free_space" pHelperStartupTimeout :: Field pHelperStartupTimeout = withDoc "Startup timeout for the helper VM" . optionalField $ simpleField "helper_startup_timeout" [t| Int |] pHelperShutdownTimeout :: Field pHelperShutdownTimeout = withDoc "Shutdown timeout for the helper VM" . optionalField $ simpleField "helper_shutdown_timeout" [t| Int |] pZeroingTimeoutFixed :: Field pZeroingTimeoutFixed = withDoc "The fixed part of time to wait before declaring the zeroing\ \ operation to have failed" . optionalField $ simpleField "zeroing_timeout_fixed" [t| Int |] pZeroingTimeoutPerMiB :: Field pZeroingTimeoutPerMiB = withDoc "The variable part of time to wait before declaring the zeroing\ \ operation to have failed, dependent on total size of disks" . optionalField $ simpleField "zeroing_timeout_per_mib" [t| Double |] pTagsObject :: Field pTagsObject = withDoc "Tag kind" $ simpleField "kind" [t| TagKind |] pTagsName :: Field pTagsName = withDoc "Name of object" . renameField "TagsGetName" . optionalField $ simpleField "name" [t| String |] pTagsList :: Field pTagsList = withDoc "List of tag names" . renameField "TagsList" $ simpleField "tags" [t| [String] |] -- FIXME: this should be compiled at load time? pTagSearchPattern :: Field pTagSearchPattern = withDoc "Search pattern (regular expression)" . renameField "TagSearchPattern" $ simpleField "pattern" [t| NonEmptyString |] pDelayDuration :: Field pDelayDuration = withDoc "Duration parameter for 'OpTestDelay'" . renameField "DelayDuration" $ simpleField "duration" [t| Double |] pDelayOnMaster :: Field pDelayOnMaster = withDoc "on_master field for 'OpTestDelay'" . renameField "DelayOnMaster" $ defaultTrue "on_master" pDelayOnNodes :: Field pDelayOnNodes = withDoc "on_nodes field for 'OpTestDelay'" . renameField "DelayOnNodes" . defaultField [| [] |] $ simpleField "on_nodes" [t| [NonEmptyString] |] pDelayOnNodeUuids :: Field pDelayOnNodeUuids = withDoc "on_node_uuids field for 'OpTestDelay'" . renameField "DelayOnNodeUuids" . optionalField $ simpleField "on_node_uuids" [t| [NonEmptyString] |] pDelayRepeat :: Field pDelayRepeat = withDoc "Repeat parameter for OpTestDelay" . renameField "DelayRepeat" . defaultField [| forceNonNeg (0::Int) |] $ simpleField "repeat" [t| NonNegative Int |] pDelayInterruptible :: Field pDelayInterruptible = withDoc "Allows socket-based interruption of a running OpTestDelay" . renameField "DelayInterruptible" . defaultField [| False |] $ simpleField "interruptible" [t| Bool |] pDelayNoLocks :: Field pDelayNoLocks = withDoc "Don't take locks during the delay" . renameField "DelayNoLocks" $ defaultTrue "no_locks" pIAllocatorDirection :: Field pIAllocatorDirection = withDoc "IAllocator test direction" . renameField "IAllocatorDirection" $ simpleField "direction" [t| IAllocatorTestDir |] pIAllocatorMode :: Field pIAllocatorMode = withDoc "IAllocator test mode" . renameField "IAllocatorMode" $ simpleField "mode" [t| IAllocatorMode |] pIAllocatorReqName :: Field pIAllocatorReqName = withDoc "IAllocator target name (new instance, node to evac, etc.)" . renameField "IAllocatorReqName" $ simpleField "name" [t| NonEmptyString |] pIAllocatorNics :: Field pIAllocatorNics = withDoc "Custom OpTestIAllocator nics" . renameField "IAllocatorNics" . optionalField $ simpleField "nics" [t| [INicParams] |] pIAllocatorDisks :: Field pIAllocatorDisks = withDoc "Custom OpTestAllocator disks" . renameField "IAllocatorDisks" . optionalField $ simpleField "disks" [t| [JSValue] |] pIAllocatorMemory :: Field pIAllocatorMemory = withDoc "IAllocator memory field" . renameField "IAllocatorMem" . optionalField $ simpleField "memory" [t| NonNegative Int |] pIAllocatorVCpus :: Field pIAllocatorVCpus = withDoc "IAllocator vcpus field" . renameField "IAllocatorVCpus" . optionalField $ simpleField "vcpus" [t| NonNegative Int |] pIAllocatorOs :: Field pIAllocatorOs = withDoc "IAllocator os field" . renameField "IAllocatorOs" $ optionalNEStringField "os" pIAllocatorInstances :: Field pIAllocatorInstances = withDoc "IAllocator instances field" . renameField "IAllocatorInstances" . optionalField $ simpleField "instances" [t| [NonEmptyString] |] pIAllocatorEvacMode :: Field pIAllocatorEvacMode = withDoc "IAllocator evac mode" . renameField "IAllocatorEvacMode" . optionalField $ simpleField "evac_mode" [t| EvacMode |] pIAllocatorSpindleUse :: Field pIAllocatorSpindleUse = withDoc "IAllocator spindle use" . renameField "IAllocatorSpindleUse" . defaultField [| forceNonNeg (1::Int) |] $ simpleField "spindle_use" [t| NonNegative Int |] pIAllocatorCount :: Field pIAllocatorCount = withDoc "IAllocator count field" . renameField "IAllocatorCount" . defaultField [| forceNonNeg (1::Int) |] $ simpleField "count" [t| NonNegative Int |] pJQueueNotifyWaitLock :: Field pJQueueNotifyWaitLock = withDoc "'OpTestJqueue' notify_waitlock" $ defaultFalse "notify_waitlock" pJQueueNotifyExec :: Field pJQueueNotifyExec = withDoc "'OpTestJQueue' notify_exec" $ defaultFalse "notify_exec" pJQueueLogMessages :: Field pJQueueLogMessages = withDoc "'OpTestJQueue' log_messages" . defaultField [| [] |] $ simpleField "log_messages" [t| [String] |] pJQueueFail :: Field pJQueueFail = withDoc "'OpTestJQueue' fail attribute" . renameField "JQueueFail" $ defaultFalse "fail" pTestDummyResult :: Field pTestDummyResult = withDoc "'OpTestDummy' result field" . renameField "TestDummyResult" $ simpleField "result" [t| JSValue |] pTestDummyMessages :: Field pTestDummyMessages = withDoc "'OpTestDummy' messages field" . renameField "TestDummyMessages" $ simpleField "messages" [t| JSValue |] pTestDummyFail :: Field pTestDummyFail = withDoc "'OpTestDummy' fail field" . renameField "TestDummyFail" $ simpleField "fail" [t| JSValue |] pTestDummySubmitJobs :: Field pTestDummySubmitJobs = withDoc "'OpTestDummy' submit_jobs field" . renameField "TestDummySubmitJobs" $ simpleField "submit_jobs" [t| JSValue |] pNetworkName :: Field pNetworkName = withDoc "Network name" $ simpleField "network_name" [t| NonEmptyString |] pNetworkAddress4 :: Field pNetworkAddress4 = withDoc "Network address (IPv4 subnet)" . renameField "NetworkAddress4" $ simpleField "network" [t| IPv4Network |] pNetworkGateway4 :: Field pNetworkGateway4 = withDoc "Network gateway (IPv4 address)" . renameField "NetworkGateway4" . optionalField $ simpleField "gateway" [t| IPv4Address |] pNetworkAddress6 :: Field pNetworkAddress6 = withDoc "Network address (IPv6 subnet)" . renameField "NetworkAddress6" . optionalField $ simpleField "network6" [t| IPv6Network |] pNetworkGateway6 :: Field pNetworkGateway6 = withDoc "Network gateway (IPv6 address)" . renameField "NetworkGateway6" . optionalField $ simpleField "gateway6" [t| IPv6Address |] pNetworkMacPrefix :: Field pNetworkMacPrefix = withDoc "Network specific mac prefix (that overrides the cluster one)" . renameField "NetMacPrefix" $ optionalNEStringField "mac_prefix" pNetworkAddRsvdIps :: Field pNetworkAddRsvdIps = withDoc "Which IP addresses to reserve" . renameField "NetworkAddRsvdIps" . optionalField $ simpleField "add_reserved_ips" [t| [IPv4Address] |] pNetworkRemoveRsvdIps :: Field pNetworkRemoveRsvdIps = withDoc "Which external IP addresses to release" . renameField "NetworkRemoveRsvdIps" . optionalField $ simpleField "remove_reserved_ips" [t| [IPv4Address] |] pNetworkMode :: Field pNetworkMode = withDoc "Network mode when connecting to a group" $ simpleField "network_mode" [t| NICMode |] pNetworkLink :: Field pNetworkLink = withDoc "Network link when connecting to a group" $ simpleField "network_link" [t| NonEmptyString |] pAdminStateSource :: Field pAdminStateSource = withDoc "Who last changed the instance admin state" . optionalField $ simpleField "admin_state_source" [t| AdminStateSource |] pNetworkVlan :: Field pNetworkVlan = withDoc "Network vlan when connecting to a group" . defaultField [| "" |] $ stringField "network_vlan" pEnabledDataCollectors :: Field pEnabledDataCollectors = withDoc "Set the active data collectors" . optionalField $ simpleField C.dataCollectorsEnabledName [t| GenericContainer String Bool |] pDataCollectorInterval :: Field pDataCollectorInterval = withDoc "Sets the interval in that data collectors are run" . optionalField $ simpleField C.dataCollectorsIntervalName [t| GenericContainer String Int |] pNodeSslCerts :: Field pNodeSslCerts = withDoc "Whether to renew node SSL certificates" . defaultField [| False |] $ simpleField "node_certificates" [t| Bool |] pSshKeyBits :: Field pSshKeyBits = withDoc "The number of bits of the SSH key Ganeti uses" . optionalField $ simpleField "ssh_key_bits" [t| Positive Int |] pSshKeyType :: Field pSshKeyType = withDoc "The type of the SSH key Ganeti uses" . optionalField $ simpleField "ssh_key_type" [t| SshKeyType |] pRenewSshKeys :: Field pRenewSshKeys = withDoc "Whether to renew SSH keys" . defaultField [| False |] $ simpleField "renew_ssh_keys" [t| Bool |] pNodeSetup :: Field pNodeSetup = withDoc "Whether to perform a SSH setup on the new node" . defaultField [| False |] $ simpleField "node_setup" [t| Bool |] pVerifyClutter :: Field pVerifyClutter = withDoc "Whether to check for clutter in the 'authorized_keys' file." . defaultField [| False |] $ simpleField "verify_clutter" [t| Bool |] pLongSleep :: Field pLongSleep = withDoc "Whether to allow long instance shutdowns during exports" . defaultField [| False |] $ simpleField "long_sleep" [t| Bool |] pIsStrict :: Field pIsStrict = withDoc "Whether the operation is in strict mode or not." . defaultField [| True |] $ simpleField "is_strict" [t| Bool |] ganeti-3.1.0~rc2/src/Ganeti/Parsers.hs000064400000000000000000000042701476477700300175600ustar00rootroot00000000000000{-# LANGUAGE OverloadedStrings #-} {-| Utility functions for several parsers This module holds the definition for some utility functions for two parsers. The parser for the @/proc/stat@ file and the parser for the @/proc/diskstats@ file. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Parsers where import qualified Data.Attoparsec.Text as A import Data.Attoparsec.Text (Parser) import Data.Text (unpack) -- * Utility functions -- | Our own space-skipping function, because A.skipSpace also skips -- newline characters. It skips ZERO or more spaces, so it does not -- fail if there are no spaces. skipSpaces :: Parser () skipSpaces = A.skipWhile A.isHorizontalSpace -- | A parser recognizing a number preceeded by spaces. numberP :: Parser Int numberP = skipSpaces *> A.decimal -- | A parser recognizing a word preceded by spaces, and closed by a space. stringP :: Parser String stringP = skipSpaces *> fmap unpack (A.takeWhile $ not . A.isHorizontalSpace) ganeti-3.1.0~rc2/src/Ganeti/PartialParams.hs000064400000000000000000000051501476477700300206770ustar00rootroot00000000000000{-# LANGUAGE FunctionalDependencies #-} {-| Common functions for partial parameters -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.PartialParams ( PartialParams(..) , isComplete ) where import Data.Maybe (isJust) -- | Represents that data type @p@ provides partial values for -- data type @f@. -- -- Note: To avoid needless type annotations, the functional dependencies -- currently include @f -> p@. However, in theory it'd be possible for one -- filled data type to have several partially filled ones. -- -- Laws: -- -- 1. @fillParams (fillParams f p) p = fillParams f p@. -- 2. @fillParams _ (toPartial x) = x@. -- 3. @toFilled (toPartial x) = Just x@. -- -- If @p@ is also a 'Monoid' (or just 'Semigroup'), 'fillParams' is a monoid -- (semigroup) action on @f@, therefore it should additionally satisfy: -- -- - @fillParams f mempty = f@ -- - @fillParams f (p1 <> p2) = fillParams (fillParams f p1) p2@ class PartialParams f p | p -> f, f -> p where -- | Fill @f@ with any data that are set in @p@. -- Leave other parts of @f@ unchanged. fillParams :: f -> p -> f -- | Fill all fields of @p@ from @f@. toPartial :: f -> p -- | If all fields of @p@ are filled, convert it into @f@. toFilled :: p -> Maybe f -- | Returns 'True' if a given partial parameters are complete. -- See 'toFilled'. isComplete :: (PartialParams f p) => p -> Bool isComplete = isJust . toFilled ganeti-3.1.0~rc2/src/Ganeti/Path.hs000064400000000000000000000132161476477700300170350ustar00rootroot00000000000000{-| Path-related helper functions. -} {- Copyright (C) 2012, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Path ( dataDir , runDir , logDir , socketDir , luxidMessageDir , livelockDir , livelockFile , defaultQuerySocket , defaultWConfdSocket , defaultMetadSocket , confdHmacKey , clusterConfFile , lockStatusFile , tempResStatusFile , watcherPauseFile , nodedCertFile , nodedClientCertFile , queueDir , jobQueueSerialFile , jobQueueLockFile , jobQueueDrainFile , jobQueueArchiveSubDir , instanceReasonDir , getInstReasonFilename , jqueueExecutorPy ) where import System.FilePath import System.Posix.Env (getEnvDefault) import AutoConf -- | Simple helper to concat two paths. pjoin :: IO String -> String -> IO String pjoin a b = do a' <- a return $ a' b -- | Returns the root directory, which can be either the real root or -- the virtual root. getRootDir :: IO FilePath getRootDir = getEnvDefault "GANETI_ROOTDIR" "" -- | Prefixes a path with the current root directory. addNodePrefix :: FilePath -> IO FilePath addNodePrefix path = do root <- getRootDir return $ root ++ path -- | Directory for data. dataDir :: IO FilePath dataDir = addNodePrefix $ AutoConf.localstatedir "lib" "ganeti" -- | Helper for building on top of dataDir (internal). dataDirP :: FilePath -> IO FilePath dataDirP = (dataDir `pjoin`) -- | Directory for runtime files. runDir :: IO FilePath runDir = addNodePrefix $ AutoConf.localstatedir "run" "ganeti" -- | Directory for log files. logDir :: IO FilePath logDir = addNodePrefix $ AutoConf.localstatedir "log" "ganeti" -- | Directory for Unix sockets. socketDir :: IO FilePath socketDir = runDir `pjoin` "socket" -- | Directory for the jobs' livelocks. livelockDir :: IO FilePath livelockDir = runDir `pjoin` "livelocks" -- | Directory for luxid to write messages to running jobs, like -- requests to change the priority. luxidMessageDir :: IO FilePath luxidMessageDir = runDir `pjoin` "luxidmessages" -- | A helper for building a job's livelock file. It prepends -- 'livelockDir' to a given filename. livelockFile :: FilePath -> IO FilePath livelockFile = pjoin livelockDir -- | The default LUXI socket for queries. defaultQuerySocket :: IO FilePath defaultQuerySocket = socketDir `pjoin` "ganeti-query" -- | The default WConfD socket for queries. defaultWConfdSocket :: IO FilePath defaultWConfdSocket = socketDir `pjoin` "ganeti-wconfd" -- | The default MetaD socket for communication. defaultMetadSocket :: IO FilePath defaultMetadSocket = socketDir `pjoin` "ganeti-metad" -- | Path to file containing confd's HMAC key. confdHmacKey :: IO FilePath confdHmacKey = dataDirP "hmac.key" -- | Path to cluster configuration file. clusterConfFile :: IO FilePath clusterConfFile = dataDirP "config.data" -- | Path to the file representing the lock status. lockStatusFile :: IO FilePath lockStatusFile = dataDirP "locks.data" -- | Path to the file representing the lock status. tempResStatusFile :: IO FilePath tempResStatusFile = dataDirP "tempres.data" -- | Path to the watcher pause file. watcherPauseFile :: IO FilePath watcherPauseFile = dataDirP "watcher.pause" -- | Path to the noded certificate. nodedCertFile :: IO FilePath nodedCertFile = dataDirP "server.pem" -- | Path to the noded client certificate. nodedClientCertFile :: IO FilePath nodedClientCertFile = dataDirP "client.pem" -- | Job queue directory. queueDir :: IO FilePath queueDir = dataDirP "queue" -- | Job queue serial file. jobQueueSerialFile :: IO FilePath jobQueueSerialFile = queueDir `pjoin` "serial" -- | Job queue lock file jobQueueLockFile :: IO FilePath jobQueueLockFile = queueDir `pjoin` "lock" -- | Job queue drain file jobQueueDrainFile :: IO FilePath jobQueueDrainFile = queueDir `pjoin` "drain" -- | Job queue archive directory. jobQueueArchiveSubDir :: FilePath jobQueueArchiveSubDir = "archive" -- | Directory containing the reason trails for the last change of status of -- instances. instanceReasonDir :: IO FilePath instanceReasonDir = runDir `pjoin` "instance-reason" -- | The path of the file containing the reason trail for an instance, given the -- instance name. getInstReasonFilename :: String -> IO FilePath getInstReasonFilename instName = instanceReasonDir `pjoin` instName -- | The path to the Python executable for starting jobs. jqueueExecutorPy :: IO FilePath jqueueExecutorPy = return $ versionedsharedir "ganeti" "jqueue" "exec.py" ganeti-3.1.0~rc2/src/Ganeti/PyValue.hs000064400000000000000000000105171476477700300175270ustar00rootroot00000000000000{-| PyValue contains instances for the 'PyValue' typeclass. The typeclass 'PyValue' converts Haskell values to Python values. This module contains instances of this typeclass for several generic types. These instances are used in the Haskell to Python generation of opcodes and constants, for example. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} {-# LANGUAGE ExistentialQuantification #-} module Ganeti.PyValue ( PyValue(..) , PyValueEx(..) ) where import Data.Char (isAscii, isPrint, ord) import Data.List (intercalate) import Data.Map (Map) import Data.ByteString (ByteString) import Text.Printf (printf) import qualified Data.ByteString.Char8 as BC8 (foldr) import qualified Data.Map as Map import qualified Data.Set as Set (toList) import Ganeti.BasicTypes -- * PyValue represents data types convertible to Python -- | Converts Haskell values into Python values -- -- This is necessary for the default values of opcode parameters and -- return values. For example, if a default value or return type is a -- Data.Map, then it must be shown as a Python dictioanry. class PyValue a where showValue :: a -> String showValueList :: [a] -> String showValueList xs = "[" ++ intercalate "," (map showValue xs) ++ "]" instance PyValue Bool where showValue = show instance PyValue Int where showValue = show instance PyValue Integer where showValue = show instance PyValue Double where showValue = show instance PyValue Char where showValue = show showValueList = show -- (Byte)String show output does not work out of the box as a Python -- string/bytes literal, especially when special characters are involved. -- For instance, show ByteString.Char8.pack "\x03" yields "\ETX", which means -- something completely different in Python. Thus, we need to implement our own -- showValue, which does the following: -- * escapes double quotes -- * leaves all printable ASCII characters intact -- * encodes all other characters in \xYZ form instance PyValue ByteString where showValue bs = "b'" ++ (BC8.foldr (\c acc -> formatChar c ++ acc) "" bs) ++ "'" where formatChar x | x == '\\' = "\\\\" | x == '\'' = "\\\'" | isAscii x && isPrint x = [x] | otherwise = (printf "\\x%02x" $ ord x) instance (PyValue a, PyValue b) => PyValue (a, b) where showValue (x, y) = "(" ++ showValue x ++ "," ++ showValue y ++ ")" instance (PyValue a, PyValue b, PyValue c) => PyValue (a, b, c) where showValue (x, y, z) = "(" ++ showValue x ++ "," ++ showValue y ++ "," ++ showValue z ++ ")" instance PyValue a => PyValue [a] where showValue = showValueList instance (PyValue k, PyValue a) => PyValue (Map k a) where showValue mp = "{" ++ intercalate ", " (map showPair (Map.assocs mp)) ++ "}" where showPair (k, x) = showValue k ++ ":" ++ showValue x instance PyValue a => PyValue (ListSet a) where showValue = showValue . Set.toList . unListSet -- * PyValue represents an unspecified value convertible to Python -- | Encapsulates Python default values data PyValueEx = forall a. PyValue a => PyValueEx a instance PyValue PyValueEx where showValue (PyValueEx x) = showValue x ganeti-3.1.0~rc2/src/Ganeti/Query/000075500000000000000000000000001476477700300167075ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Query/Cluster.hs000064400000000000000000000050561476477700300206720ustar00rootroot00000000000000{-| Implementation of the Ganeti Query2 cluster queries. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Cluster ( clusterMasterNodeName , isWatcherPaused ) where import Control.Exception (try) import Control.Monad (liftM) import Data.Char (isSpace) import Numeric (readDec) import Ganeti.Config import Ganeti.Errors import Ganeti.Logging import Ganeti.Objects import Ganeti.Path import Ganeti.Utils.Time (getCurrentTime) -- | Get master node name. clusterMasterNodeName :: ConfigData -> ErrorResult String clusterMasterNodeName cfg = let cluster = configCluster cfg masterNodeUuid = clusterMasterNode cluster in liftM nodeName $ getNode cfg masterNodeUuid isWatcherPaused :: IO (Maybe Integer) isWatcherPaused = do logDebug "Checking if the watcher is paused" wfile <- watcherPauseFile contents <- try $ readFile wfile :: IO (Either IOError String) case contents of Left _ -> return Nothing Right s -> case readDec (dropWhile isSpace s) of [(n, rest)] | all isSpace rest -> do now <- getCurrentTime return $ if n > now then Just n else Nothing _ -> do logWarning $ "Watcher pause file contents '" ++ s ++ "' not parsable as int" return Nothing ganeti-3.1.0~rc2/src/Ganeti/Query/Common.hs000064400000000000000000000243741476477700300205050ustar00rootroot00000000000000{-| Implementation of the Ganeti Query2 common objects. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Common ( NoDataRuntime(..) , rsNoData , rsUnavail , rsNormal , rsMaybeNoData , rsMaybeUnavail , rsErrorNoData , rsErrorMaybeUnavail , rsUnknown , missingRuntime , rpcErrorToStatus , timeStampFields , uuidFields , serialFields , forthcomingFields , tagsFields , dictFieldGetter , buildNdParamField , buildBeParamField , buildHvParamField , getDefaultHypervisorSpec , getHvParamsFromCluster , aliasFields ) where import Control.Monad (guard) import qualified Data.Map as Map import Data.Maybe (fromMaybe) import Text.JSON (JSON, showJSON) import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.Config import Ganeti.Errors import Ganeti.JSON (GenericContainer(..), fromContainer, TimeAsDoubleJSON(..)) import Ganeti.Objects import Ganeti.Rpc import Ganeti.Query.Language import Ganeti.Query.Types import Ganeti.Types -- | The runtime used by queries which retrieve no live data. data NoDataRuntime = NoDataRuntime -- * Generic functions -- | Conversion from 'VType' to 'FieldType'. vTypeToQFT :: VType -> FieldType vTypeToQFT VTypeString = QFTOther vTypeToQFT VTypeMaybeString = QFTOther vTypeToQFT VTypeBool = QFTBool vTypeToQFT VTypeSize = QFTUnit vTypeToQFT VTypeInt = QFTNumber vTypeToQFT VTypeFloat = QFTNumberFloat -- * Result helpers -- | Helper for a result with no data. rsNoData :: ResultEntry rsNoData = ResultEntry RSNoData Nothing -- | Helper for result for an entity which supports no such field. rsUnavail :: ResultEntry rsUnavail = ResultEntry RSUnavail Nothing -- | Helper to declare a normal result. rsNormal :: (JSON a) => a -> ResultEntry rsNormal a = ResultEntry RSNormal $ Just (showJSON a) -- | Helper to declare a result from a 'Maybe' (the item might be -- missing, in which case we return no data). Note that there's some -- ambiguity here: in some cases, we mean 'RSNoData', but in other -- 'RSUnavail'; this is easy to solve in simple cases, but not in -- nested dicts. If you want to return 'RSUnavail' in case of 'Nothing' -- use the function 'rsMaybeUnavail'. rsMaybeNoData :: (JSON a) => Maybe a -> ResultEntry rsMaybeNoData = maybe rsNoData rsNormal -- | Helper to declare a result from a 'ErrorResult' (an error happened -- while retrieving the data from a config, or there was no data). -- This function should be used if an error signals there was no data. rsErrorNoData :: (JSON a) => ErrorResult a -> ResultEntry rsErrorNoData res = case res of Ok x -> rsNormal x Bad _ -> rsNoData -- | Helper to declare a result from a 'Maybe'. This version returns -- a 'RSUnavail' in case of 'Nothing'. It should be used for optional -- fields that are not set. For cases where 'Nothing' means that there -- was an error, consider using 'rsMaybe' instead. rsMaybeUnavail :: (JSON a) => Maybe a -> ResultEntry rsMaybeUnavail = maybe rsUnavail rsNormal -- | Helper to declare a result from 'ErrorResult Maybe'. This version -- should be used if an error signals there was no data and at the same -- time when we have optional fields that may not be setted (i.e. we -- want to return a 'RSUnavail' in case of 'Nothing'). rsErrorMaybeUnavail :: (JSON a) => ErrorResult (Maybe a) -> ResultEntry rsErrorMaybeUnavail res = case res of Ok x -> rsMaybeUnavail x Bad _ -> rsNoData -- | Helper for unknown field result. rsUnknown :: ResultEntry rsUnknown = ResultEntry RSUnknown Nothing -- | Helper for a missing runtime parameter. missingRuntime :: FieldGetter a b missingRuntime = FieldRuntime (\_ _ -> ResultEntry RSNoData Nothing) -- * Error conversion -- | Convert RpcError to ResultStatus rpcErrorToStatus :: RpcError -> ResultStatus rpcErrorToStatus OfflineNodeError = RSOffline rpcErrorToStatus _ = RSNoData -- * Common fields -- | The list of timestamp fields. timeStampFields :: (TimeStampObject a) => FieldList a b timeStampFields = [ (FieldDefinition "ctime" "CTime" QFTTimestamp "Creation timestamp", FieldSimple (rsNormal . TimeAsDoubleJSON . cTimeOf), QffNormal) , (FieldDefinition "mtime" "MTime" QFTTimestamp "Modification timestamp", FieldSimple (rsNormal . TimeAsDoubleJSON . mTimeOf), QffNormal) ] -- | The list of the field for the property of being forthcoming. forthcomingFields :: (ForthcomingObject a) => String -> FieldList a b forthcomingFields name = [ ( FieldDefinition "forthcoming" "Forthcoming" QFTBool $ "whether the " ++ name ++ " is forthcoming" , FieldSimple (rsNormal . isForthcoming), QffNormal ) ] -- | The list of UUID fields. uuidFields :: (UuidObject a) => String -> FieldList a b uuidFields name = [ (FieldDefinition "uuid" "UUID" QFTText (name ++ " UUID"), FieldSimple (rsNormal . uuidOf), QffNormal) ] -- | The list of serial number fields. serialFields :: (SerialNoObject a) => String -> FieldList a b serialFields name = [ (FieldDefinition "serial_no" "SerialNo" QFTNumber (name ++ " object serial number, incremented on each modification"), FieldSimple (rsNormal . serialOf), QffNormal) ] -- | The list of tag fields. tagsFields :: (TagsObject a) => FieldList a b tagsFields = [ (FieldDefinition "tags" "Tags" QFTOther "Tags", FieldSimple (rsNormal . tagsOf), QffNormal) ] -- * Generic parameter functions -- | Returns a field from a (possibly missing) 'DictObject'. This is -- used by parameter dictionaries, usually. Note that we have two -- levels of maybe: the top level dict might be missing, or one key in -- the dictionary might be. dictFieldGetter :: (DictObject a) => String -> Maybe a -> ResultEntry dictFieldGetter k = maybe rsNoData (rsMaybeNoData . lookup k . toDict) -- | Ndparams optimised lookup map. ndParamTypes :: Map.Map String FieldType ndParamTypes = Map.map vTypeToQFT C.ndsParameterTypes -- | Ndparams title map. ndParamTitles :: Map.Map String FieldTitle ndParamTitles = C.ndsParameterTitles -- | Ndparam getter builder: given a field, it returns a FieldConfig -- getter, that is a function that takes the config and the object and -- returns the Ndparam field specified when the getter was built. ndParamGetter :: (NdParamObject a) => String -- ^ The field we're building the getter for -> ConfigData -> a -> ResultEntry ndParamGetter field config = dictFieldGetter field . getNdParamsOf config -- | Builds the ndparam fields for an object. buildNdParamField :: (NdParamObject a) => String -> FieldData a b buildNdParamField = buildParamField "ndp" "node" ndParamTitles ndParamTypes ndParamGetter -- | Beparams optimised lookup map. beParamTypes :: Map.Map String FieldType beParamTypes = Map.map vTypeToQFT C.besParameterTypes -- | Builds the beparam fields for an object. buildBeParamField :: (String -> ConfigData -> a -> ResultEntry) -> String -> FieldData a b buildBeParamField = buildParamField "be" "backend" C.besParameterTitles beParamTypes -- | Hvparams optimised lookup map. hvParamTypes :: Map.Map String FieldType hvParamTypes = Map.map vTypeToQFT C.hvsParameterTypes -- | Builds the beparam fields for an object. buildHvParamField :: (String -> ConfigData -> a -> ResultEntry) -> String -> FieldData a b buildHvParamField = buildParamField "hv" "hypervisor" C.hvsParameterTitles hvParamTypes -- | Builds a param field for a certain getter class buildParamField :: String -- ^ Prefix -> String -- ^ Parameter group name -> Map.Map String String -- ^ Parameter title map -> Map.Map String FieldType -- ^ Parameter type map -> (String -> ConfigData -> a -> ResultEntry) -> String -- ^ The parameter name -> FieldData a b buildParamField prefix paramGroupName titleMap typeMap getter field = let full_name = prefix ++ "/" ++ field title = fromMaybe full_name $ field `Map.lookup` titleMap qft = fromMaybe QFTOther $ field `Map.lookup` typeMap desc = "The \"" ++ field ++ "\" " ++ paramGroupName ++ " parameter" in ( FieldDefinition full_name title qft desc , FieldConfig (getter field), QffNormal ) -- | Looks up the default hypervisor and its hvparams getDefaultHypervisorSpec :: ConfigData -> (Hypervisor, HvParams) getDefaultHypervisorSpec cfg = (hv, getHvParamsFromCluster cfg hv) where hv = getDefaultHypervisor cfg -- | Looks up the cluster's hvparams of the given hypervisor getHvParamsFromCluster :: ConfigData -> Hypervisor -> HvParams getHvParamsFromCluster cfg hv = fromMaybe (GenericContainer Map.empty) . Map.lookup hv . fromContainer . clusterHvparams $ configCluster cfg -- | Given an alias list and a field list, copies field definitions under a -- new field name. Aliases should be tested - see the test module -- 'Test.Ganeti.Query.Aliases'! aliasFields :: [(FieldName, FieldName)] -> FieldList a b -> FieldList a b aliasFields aliases fieldList = fieldList ++ do alias <- aliases (FieldDefinition name d1 d2 d3, v1, v2) <- fieldList guard (snd alias == name) return (FieldDefinition (fst alias) d1 d2 d3, v1, v2) ganeti-3.1.0~rc2/src/Ganeti/Query/Exec.hs000064400000000000000000000172631476477700300201400ustar00rootroot00000000000000{-| Executing jobs as processes The protocol works as follows (MP = master process, FP = forked process): * MP sets its own livelock as the livelock of the job to be executed. * FP creates its own lock file and sends its name to the MP. * MP updates the lock file name in the job file and confirms the FP it can start. * FP requests any secret parameters. * MP sends the secret parameters, if any. * Both MP and FP close the communication channel. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Exec ( forkJobProcess ) where import Control.Concurrent.Lifted (threadDelay) import Control.Monad import Control.Monad.Except (catchError, throwError) import qualified Data.Map as M import Data.Maybe (mapMaybe, fromJust) import System.Environment import System.IO.Error (annotateIOError, modifyIOError) import System.Process import System.Posix.Process import System.Posix.Signals (sigABRT, sigKILL, sigTERM, signalProcess) import System.Posix.Types (ProcessID) import Text.JSON import qualified AutoConf as AC import Ganeti.BasicTypes import Ganeti.JQueue.Objects import Ganeti.JSON (MaybeForJSON(..)) import Ganeti.Logging import Ganeti.Logging.WriterLog import Ganeti.OpCodes import qualified Ganeti.Path as P import Ganeti.Types import Ganeti.UDSServer import Ganeti.Compat (getPid') connectConfig :: ConnectConfig connectConfig = ConnectConfig { recvTmo = 30 , sendTmo = 30 } -- | Catches a potential `IOError` and sets its description via -- `annotateIOError`. This makes exceptions more informative when they -- are thrown from an unnamed `Handle`. rethrowAnnotateIOError :: String -> IO a -> IO a rethrowAnnotateIOError desc = modifyIOError (\e -> annotateIOError e desc Nothing Nothing) -- | Spawn a subprocess to execute a Job's actual code in the Python -- interpreter. The subprocess will have its standard input and output -- connected to a pair of pipes wrapped in a Client instance. Standard error -- will be inherited from the current process and can be used for early -- logging, before the executor sets up its own logging. spawnJobProcess :: JobId -> IO (ProcessID, Client) spawnJobProcess jid = withErrorLogAt CRITICAL (show jid) $ do use_debug <- isDebugMode env_ <- (M.toList . M.insert "GNT_DEBUG" (if use_debug then "1" else "0") . M.insert "PYTHONPATH" AC.versionedsharedir . M.fromList) `liftM` getEnvironment execPy <- P.jqueueExecutorPy logDebug $ "Executing " ++ AC.pythonPath ++ " " ++ execPy ++ " with PYTHONPATH=" ++ AC.versionedsharedir (master, child) <- pipeClient connectConfig let (rh, wh) = clientToHandle child let jobProc = (proc AC.pythonPath [execPy, show (fromJobId jid)]){ std_in = UseHandle rh, std_out = UseHandle wh, std_err = Inherit, env = Just env_, close_fds = True} (_, _, _, hchild) <- createProcess jobProc pid <- getPid' hchild return (fromJust pid, master) filterSecretParameters :: [QueuedOpCode] -> [MaybeForJSON (JSObject (Private JSValue))] filterSecretParameters = map (MaybeForJSON . fmap revealValInJSObject . getSecretParams) . mapMaybe (transformOpCode . qoInput) where transformOpCode :: InputOpCode -> Maybe OpCode transformOpCode inputCode = case inputCode of ValidOpCode moc -> Just (metaOpCode moc) _ -> Nothing getSecretParams :: OpCode -> Maybe (JSObject (Secret JSValue)) getSecretParams opcode = case opcode of (OpInstanceCreate {opOsparamsSecret = x}) -> x (OpInstanceReinstall {opOsparamsSecret = x}) -> x (OpTestOsParams {opOsparamsSecret = x}) -> x _ -> Nothing -- | Forks the job process and starts processing of the given job. -- Returns the livelock of the job and its process ID. forkJobProcess :: (Error e, Show e) => QueuedJob -- ^ a job to process -> FilePath -- ^ the daemons own livelock file -> (FilePath -> ResultT e IO ()) -- ^ a callback function to update the livelock file -- and process id in the job file -> ResultT e IO (FilePath, ProcessID) forkJobProcess job luxiLivelock update = do let jidStr = show . fromJobId . qjId $ job -- Retrieve secret parameters if present let secretParams = encodeStrict . filterSecretParameters . qjOps $ job logDebug $ "Setting the lockfile temporarily to " ++ luxiLivelock ++ " for job " ++ jidStr update luxiLivelock ResultT . execWriterLogT . runResultT $ do (pid, master) <- liftIO $ spawnJobProcess (qjId job) let jobLogPrefix = "[start:job-" ++ jidStr ++ ",pid=" ++ show pid ++ "] " logDebugJob = logDebug . (jobLogPrefix ++) logDebugJob "Forked a new process" let killIfAlive [] = return () killIfAlive (sig : sigs) = do logDebugJob "Getting the status of the process" status <- tryError . liftIO $ getProcessStatus False True pid case status of Left e -> logDebugJob $ "Job process already gone: " ++ show e Right (Just s) -> logDebugJob $ "Child process status: " ++ show s Right Nothing -> do logDebugJob $ "Child process running, killing by " ++ show sig liftIO $ signalProcess sig pid unless (null sigs) $ do threadDelay 100000 -- wait for 0.1s and check again killIfAlive sigs let onError = do logDebugJob "Closing the pipe to the client" withErrorLogAt WARNING "Closing the communication pipe failed" (liftIO (closeClient master)) `orElse` return () killIfAlive [sigTERM, sigABRT, sigKILL] flip catchError (\e -> onError >> throwError e) $ do let annotatedIO msg k = do logDebugJob msg liftIO $ rethrowAnnotateIOError (jobLogPrefix ++ msg) k let recv msg = annotatedIO msg (recvMsg master) send msg x = annotatedIO msg (sendMsg master x) lockfile <- recv "Getting the lockfile of the client" logDebugJob $ "Setting the lockfile to the final " ++ lockfile toErrorBase $ update lockfile send "Confirming the client it can start" "" _ <- recv "Waiting for the job to ask for secret parameters" send "Writing secret parameters to the client" secretParams liftIO $ closeClient master return (lockfile, pid) ganeti-3.1.0~rc2/src/Ganeti/Query/Export.hs000064400000000000000000000062511476477700300205300ustar00rootroot00000000000000{-| Implementation of the Ganeti Query2 export queries. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Export ( Runtime , fieldsMap , collectLiveData ) where import Control.Monad (liftM) import Ganeti.Objects import Ganeti.Rpc import Ganeti.Query.Language import Ganeti.Query.Common import Ganeti.Query.Types -- | The parsed result of the ExportList. This is a bit tricky, in -- that we already do parsing of the results in the RPC calls, so the -- runtime type is a plain 'ResultEntry', as we have just one type. type Runtime = ResultEntry -- | Small helper for rpc to rs. rpcErrToRs :: RpcError -> ResultEntry rpcErrToRs err = ResultEntry (rpcErrorToStatus err) Nothing -- | Helper for extracting fields from RPC result. rpcExtractor :: Node -> Either RpcError RpcResultExportList -> [(Node, ResultEntry)] rpcExtractor node (Right res) = [(node, rsNormal path) | path <- rpcResExportListExports res] rpcExtractor node (Left err) = [(node, rpcErrToRs err)] -- | List of all node fields. exportFields :: FieldList Node Runtime exportFields = [ (FieldDefinition "node" "Node" QFTText "Node name", FieldRuntime (\_ n -> rsNormal $ nodeName n), QffHostname) , (FieldDefinition "export" "Export" QFTText "Export name", FieldRuntime (curry fst), QffNormal) ] -- | The node fields map. fieldsMap :: FieldMap Node Runtime fieldsMap = fieldListToFieldMap exportFields -- | Collect live data from RPC query if enabled. -- -- Note that this function is \"funny\": the returned rows will not be -- 1:1 with the input, as nodes without exports will be pruned, -- whereas nodes with multiple exports will be listed multiple times. collectLiveData:: Bool -> ConfigData -> [Node] -> IO [(Node, Runtime)] collectLiveData False _ nodes = return [(n, rpcErrToRs $ RpcResultError "Live data disabled") | n <- nodes] collectLiveData True _ nodes = concatMap (uncurry rpcExtractor) `liftM` executeRpcCall nodes RpcCallExportList ganeti-3.1.0~rc2/src/Ganeti/Query/Filter.hs000064400000000000000000000304701476477700300204740ustar00rootroot00000000000000{-# LANGUAGE Rank2Types, GADTs, StandaloneDeriving #-} {-| Implementation of the Ganeti Query2 filterning. The filtering of results should be done in two phases. In the first phase, before contacting any remote nodes for runtime data, the filtering should be executed with 'Nothing' for the runtime context. This will make all non-runtime filters filter correctly, whereas all runtime filters will respond successfully. As described in the Python version too, this makes for example /Or/ filters very inefficient if they contain runtime fields. Once this first filtering phase has been done, we hopefully eliminated some remote nodes out of the list of candidates, we run the remote data gathering, and we evaluate the filter again, this time with a 'Just' runtime context. This will make all filters work correctly. Note that the second run will re-evaluate the config/simple fields, without caching; this is not perfect, but we consider config accesses very cheap (and the configuration snapshot we have won't change between the two runs, hence we will not get inconsistent results). -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Filter ( compileFilter , evaluateQueryFilter , evaluateFilterM , evaluateFilterJSON , requestedNames , makeSimpleFilter , Comparator , Comparison(..) , toCompFun , FilterOp(..) ) where import Control.Monad (liftM, mzero) import Control.Monad.Trans.Maybe (MaybeT, runMaybeT) import Control.Monad.Trans.Class (lift) import qualified Data.Map as Map import Data.Maybe (fromMaybe) import Text.JSON (JSValue(..), fromJSString) import Text.JSON.Pretty (pp_value) import qualified Ganeti.Query.RegEx as RegEx import Ganeti.BasicTypes (GenericResult (Ok, Bad), goodLookupResult, compareNameComponent) import Ganeti.Errors (ErrorResult, GanetiException(ParameterError, ProgrammerError)) import Ganeti.Objects (ConfigData) import Ganeti.Query.Language import Ganeti.Query.Types import Ganeti.Utils.Monad (anyM, allM) import Ganeti.JSON (fromJVal) -- | Compiles a filter based on field names to one based on getters. compileFilter :: FieldMap a b -> Filter FilterField -> ErrorResult (Filter (FieldGetter a b, QffMode)) compileFilter fm = traverse (\field -> maybe (Bad . ParameterError $ "Can't find field named '" ++ field ++ "'") (\(_, g, q) -> Ok (g, q)) (field `Map.lookup` fm)) -- | Processes a field value given a QffMode. qffField :: QffMode -> JSValue -> ErrorResult JSValue qffField QffNormal v = Ok v qffField QffHostname v = Ok v qffField QffTimestamp v = case v of JSArray [secs@(JSRational _ _), JSRational _ _] -> return secs _ -> Bad $ ProgrammerError "Internal error: Getter returned non-timestamp for QffTimestamp" -- | Wraps a getter, filter pair. If the getter is 'FieldRuntime' but -- we don't have a runtime context, we skip the filtering, returning -- `Nothing` in the MaybeT. Otherwise, we pass the actual value to the filter. wrapGetter :: ConfigData -> Maybe b -> a -> (FieldGetter a b, QffMode) -> (JSValue -> ErrorResult Bool) -> MaybeT ErrorResult Bool wrapGetter cfg b a (getter, qff) faction = case tryGetter cfg b a getter of Nothing -> mzero -- runtime missing, signalling that with MaybeT Nothing Just v -> lift $ case v of ResultEntry RSNormal (Just fval) -> qffField qff fval >>= faction ResultEntry RSNormal Nothing -> Bad $ ProgrammerError "Internal error: Getter returned RSNormal/Nothing" _ -> Ok True -- filter has no data to work, accepting it -- | Helper to evaluate a filter getter (and the value it generates) in -- a boolean context. trueFilter :: JSValue -> ErrorResult Bool trueFilter (JSBool x) = Ok $! x trueFilter v = Bad . ParameterError $ "Unexpected value '" ++ show (pp_value v) ++ "' in boolean context" -- | A type synonim for a rank-2 comparator function. This is used so -- that we can pass the usual '<=', '>', '==' functions to 'binOpFilter' -- and for them to be used in multiple contexts. type Comparator = forall a . (Eq a, Ord a) => a -> a -> Bool -- | Equality checker. -- -- This will handle hostnames correctly, if the mode is set to -- 'QffHostname'. eqFilter :: QffMode -> FilterValue -> JSValue -> ErrorResult Bool -- send 'QffNormal' queries to 'binOpFilter' eqFilter QffNormal flv jsv = binOpFilter (==) flv jsv -- and 'QffTimestamp' as well eqFilter QffTimestamp flv jsv = binOpFilter (==) flv jsv -- error out if we set 'QffHostname' on a non-string field eqFilter QffHostname _ (JSRational _ _) = Bad . ProgrammerError $ "QffHostname field returned a numeric value" -- test strings via 'compareNameComponent' eqFilter QffHostname (QuotedString y) (JSString x) = Ok $ goodLookupResult (fromJSString x `compareNameComponent` y) -- send all other combinations (all errors) to 'binOpFilter', which -- has good error messages eqFilter _ flv jsv = binOpFilter (==) flv jsv -- | Helper to evaluate a filder getter (and the value it generates) -- in a boolean context. Note the order of arguments is reversed from -- the filter definitions (due to the call chain), make sure to -- compare in the reverse order too!. binOpFilter :: Comparator -> FilterValue -> JSValue -> ErrorResult Bool binOpFilter comp (QuotedString y) (JSString x) = Ok $! fromJSString x `comp` y binOpFilter comp (NumericValue y) (JSRational _ x) = Ok $! x `comp` fromIntegral y binOpFilter _ expr actual = Bad . ParameterError $ "Invalid types in comparison, trying to compare " ++ show (pp_value actual) ++ " with '" ++ show expr ++ "'" -- | Implements the 'RegexpFilter' matching. regexpFilter :: FilterRegex -> JSValue -> ErrorResult Bool regexpFilter re (JSString val) = Ok $! RegEx.match (compiledRegex re) (fromJSString val) regexpFilter _ x = Bad . ParameterError $ "Invalid field value used in regexp matching,\ \ expecting string but got '" ++ show (pp_value x) ++ "'" -- | Implements the 'ContainsFilter' matching. containsFilter :: FilterValue -> JSValue -> ErrorResult Bool -- note: the next two implementations are the same, but we have to -- repeat them due to the encapsulation done by FilterValue containsFilter (QuotedString val) lst = do lst' <- fromJVal lst :: ErrorResult [String] return $! val `elem` lst' containsFilter (NumericValue val) lst = do lst' <- fromJVal lst :: ErrorResult [Integer] return $! val `elem` lst' -- | Ways we can compare things in the filter language. data Comparison = Eq | Lt | Le | Gt | Ge deriving (Eq, Ord, Show) -- | Turns a comparison into the corresponding Haskell function. toCompFun :: Comparison -> Comparator toCompFun cmp = case cmp of Eq -> (==) Lt -> (<) Le -> (<=) Gt -> (>) Ge -> (>=) -- | Operations in the leaves of the Ganeti filter language. data FilterOp field val where Truth :: FilterOp field () Comp :: Comparison -> FilterOp field FilterValue Regex :: FilterOp field FilterRegex Contains :: FilterOp field FilterValue deriving instance Eq (FilterOp field val) deriving instance Show (FilterOp field val) -- | Checks if a filter matches. -- -- The leaves of the filter are evaluated against an object using the passed -- `opFun`; that is why the object need not be passed in. -- -- The `field` type describes the "accessors" that are used to query -- values from the object; those values are to be matched against the -- `val` type in the filter leaves. -- -- Useful monads @m@ for this are `ErrorResult` and `Maybe`. evaluateFilterM :: (Monad m, Applicative m) => (forall val . FilterOp field val -> field -> val -> m Bool) -> Filter field -> m Bool evaluateFilterM opFun fil = case fil of EmptyFilter -> return True AndFilter flts -> allM recurse flts OrFilter flts -> anyM recurse flts NotFilter flt -> not <$> recurse flt TrueFilter field -> opFun Truth field () EQFilter field val -> opFun (Comp Eq) field val LTFilter field val -> opFun (Comp Lt) field val LEFilter field val -> opFun (Comp Le) field val GTFilter field val -> opFun (Comp Gt) field val GEFilter field val -> opFun (Comp Ge) field val RegexpFilter field re -> opFun Regex field re ContainsFilter field val -> opFun Contains field val where recurse = evaluateFilterM opFun -- | Verifies if a given item passes a filter. The runtime context -- might be missing, in which case most of the filters will consider -- this as passing the filter. evaluateQueryFilter :: ConfigData -> Maybe b -> a -> Filter (FieldGetter a b, QffMode) -> ErrorResult Bool evaluateQueryFilter c mb a = -- `Nothing` in the MaybeT means "missing but needed runtime context". -- Turn those cases into True (let the filter pass). fmap (fromMaybe True) . runMaybeT . evaluateFilterM (\op -> case op of Truth -> \gQff () -> wrap gQff trueFilter -- We're special casing comparison for host names. -- All other comparisons behave as usual. Comp Eq -> \gQff val -> wrap gQff $ eqFilter (snd gQff) val Comp cmp -> \gQff val -> wrap gQff $ binOpFilter (toCompFun cmp) val Regex -> \gQff re -> wrap gQff $ regexpFilter re Contains -> \gQff val -> wrap gQff $ containsFilter val ) where wrap = wrapGetter c mb a -- | Evaluates a `Filter` on a JSON object. evaluateFilterJSON :: Filter JSValue -> ErrorResult Bool evaluateFilterJSON = evaluateFilterM $ \op -> case op of Comp cmp -> let compFun = toCompFun cmp in \json fv -> pure $ json `compFun` showFilterValue fv Truth -> \field () -> trueFilter field Regex -> flip regexpFilter Contains -> flip containsFilter -- | Runs a getter with potentially missing runtime context. tryGetter :: ConfigData -> Maybe b -> a -> FieldGetter a b -> Maybe ResultEntry tryGetter _ _ item (FieldSimple getter) = Just $ getter item tryGetter cfg _ item (FieldConfig getter) = Just $ getter cfg item tryGetter _ rt item (FieldRuntime getter) = maybe Nothing (\rt' -> Just $ getter rt' item) rt tryGetter cfg rt item (FieldConfigRuntime getter) = maybe Nothing (\rt' -> Just $ getter cfg rt' item) rt tryGetter _ _ _ FieldUnknown = Just $ ResultEntry RSUnknown Nothing -- | Computes the requested names, if only names were requested (and -- with equality). Otherwise returns 'Nothing'. requestedNames :: FilterField -> Filter FilterField -> Maybe [FilterValue] requestedNames _ EmptyFilter = Just [] requestedNames namefield (OrFilter flts) = liftM concat $ mapM (requestedNames namefield) flts requestedNames namefield (EQFilter fld val) = if namefield == fld then Just [val] else Nothing requestedNames _ _ = Nothing -- | Builds a simple filter from a list of names. makeSimpleFilter :: String -> [Either String Integer] -> Filter FilterField makeSimpleFilter _ [] = EmptyFilter makeSimpleFilter namefield vals = OrFilter $ map (EQFilter namefield . either QuotedString NumericValue) vals ganeti-3.1.0~rc2/src/Ganeti/Query/FilterRules.hs000064400000000000000000000054421476477700300215100ustar00rootroot00000000000000{-| Implementation of Ganeti filter queries. -} {- Copyright (C) 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.FilterRules ( fieldsMap ) where import Ganeti.Objects import Ganeti.Query.Common import Ganeti.Query.Language import Ganeti.Query.Types -- | List of all lock fields. filterFields :: FieldList FilterRule NoDataRuntime filterFields = [ (FieldDefinition "watermark" "Watermark" QFTOther "Highest job ID used\ \ at the time when the\ \ filter was added", FieldSimple (rsNormal . frWatermark), QffNormal) , (FieldDefinition "priority" "Priority" QFTOther "Filter priority", FieldSimple (rsNormal . frPriority), QffNormal) , (FieldDefinition "predicates" "Predicates" QFTOther "List of filter\ \ predicates", FieldSimple (rsNormal . frPredicates), QffNormal) , (FieldDefinition "action" "Action" QFTOther "Filter action", FieldSimple (rsNormal . frAction), QffNormal) , (FieldDefinition "reason_trail" "ReasonTrail" QFTOther "Reason why this\ \ filter was\ \ added", FieldSimple (rsNormal . frReasonTrail), QffNormal) , (FieldDefinition "uuid" "UUID" QFTOther "Filter ID", FieldSimple (rsNormal . frUuid), QffNormal) ] -- | The lock fields map. fieldsMap :: FieldMap FilterRule NoDataRuntime fieldsMap = fieldListToFieldMap filterFields ganeti-3.1.0~rc2/src/Ganeti/Query/Group.hs000064400000000000000000000075641476477700300203530ustar00rootroot00000000000000{-| Implementation of the Ganeti Query2 node group queries. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Group (fieldsMap) where import Data.Maybe (mapMaybe) import Ganeti.Config import Ganeti.Objects import Ganeti.Query.Language import Ganeti.Query.Common import Ganeti.Query.Types import Ganeti.Utils (niceSort) groupFields :: FieldList NodeGroup NoDataRuntime groupFields = [ (FieldDefinition "alloc_policy" "AllocPolicy" QFTText "Allocation policy for group", FieldSimple (rsNormal . groupAllocPolicy), QffNormal) , (FieldDefinition "custom_diskparams" "CustomDiskParameters" QFTOther "Custom disk parameters", FieldSimple (rsNormal . groupDiskparams), QffNormal) , (FieldDefinition "custom_ipolicy" "CustomInstancePolicy" QFTOther "Custom instance policy limitations", FieldSimple (rsNormal . groupIpolicy), QffNormal) , (FieldDefinition "custom_ndparams" "CustomNDParams" QFTOther "Custom node parameters", FieldSimple (rsNormal . groupNdparams), QffNormal) , (FieldDefinition "diskparams" "DiskParameters" QFTOther "Disk parameters (merged)", FieldConfig (\cfg -> rsNormal . getGroupDiskParams cfg), QffNormal) , (FieldDefinition "ipolicy" "InstancePolicy" QFTOther "Instance policy limitations (merged)", FieldConfig (\cfg ng -> rsNormal (getGroupIpolicy cfg ng)), QffNormal) , (FieldDefinition "name" "Group" QFTText "Group name", FieldSimple (rsNormal . groupName), QffNormal) , (FieldDefinition "ndparams" "NDParams" QFTOther "Node parameters", FieldConfig (\cfg ng -> rsNormal (getGroupNdParams cfg ng)), QffNormal) , (FieldDefinition "node_cnt" "Nodes" QFTNumber "Number of nodes", FieldConfig (\cfg -> rsNormal . length . getGroupNodes cfg . uuidOf), QffNormal) , (FieldDefinition "node_list" "NodeList" QFTOther "List of nodes", FieldConfig (\cfg -> rsNormal . map nodeName . getGroupNodes cfg . uuidOf), QffNormal) , (FieldDefinition "pinst_cnt" "Instances" QFTNumber "Number of primary instances", FieldConfig (\cfg -> rsNormal . length . fst . getGroupInstances cfg . uuidOf), QffNormal) , (FieldDefinition "pinst_list" "InstanceList" QFTOther "List of primary instances", FieldConfig (\cfg -> rsNormal . niceSort . mapMaybe instName . fst . getGroupInstances cfg . uuidOf), QffNormal) ] ++ map buildNdParamField allNDParamFields ++ timeStampFields ++ uuidFields "Group" ++ serialFields "Group" ++ tagsFields -- | The group fields map. fieldsMap :: FieldMap NodeGroup NoDataRuntime fieldsMap = fieldListToFieldMap groupFields ganeti-3.1.0~rc2/src/Ganeti/Query/Instance.hs000064400000000000000000001176261476477700300210240ustar00rootroot00000000000000{-| Implementation of the Ganeti Query2 instance queries. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Instance ( Runtime , fieldsMap , collectLiveData , getInstanceInfo , instanceFields , instanceAliases ) where import Control.Applicative import Control.Monad (liftM, (>=>)) import qualified Data.ByteString.UTF8 as UTF8 import Data.Either import Data.List import Data.Maybe import Data.Monoid import qualified Data.Map as Map import Data.Ord (comparing) import qualified Text.JSON as J import Text.Printf import Ganeti.BasicTypes import Ganeti.Common import Ganeti.Config import qualified Ganeti.Constants as C import qualified Ganeti.ConstantUtils as C import Ganeti.Errors import Ganeti.JSON (MaybeForJSON(..), fromContainer) import Ganeti.Objects import Ganeti.Query.Common import Ganeti.Query.Language import Ganeti.Query.Types import Ganeti.Rpc import Ganeti.Storage.Utils import Ganeti.Types import Ganeti.Utils (formatOrdinal) -- | The LiveInfo consists of two entries whose presence is independent. -- The 'InstanceInfo' is the live instance information, accompanied by a bool -- signifying if it was found on its designated primary node or not. -- The 'InstanceConsoleInfo' describes how to connect to an instance. -- Any combination of these may or may not be present, depending on node and -- instance availability. type LiveInfo = (Maybe (InstanceInfo, Bool), Maybe InstanceConsoleInfo) -- | Runtime containing the 'LiveInfo'. See the genericQuery function in -- the Query.hs file for an explanation of the terms used. type Runtime = Either RpcError LiveInfo -- | The instance fields map. fieldsMap :: FieldMap Instance Runtime fieldsMap = fieldListToFieldMap aliasedFields -- | The instance aliases. instanceAliases :: [(FieldName, FieldName)] instanceAliases = [ ("vcpus", "be/vcpus") , ("be/memory", "be/maxmem") , ("sda_size", "disk.size/0") , ("sdb_size", "disk.size/1") , ("ip", "nic.ip/0") , ("mac", "nic.mac/0") , ("bridge", "nic.bridge/0") , ("nic_mode", "nic.mode/0") , ("nic_link", "nic.link/0") , ("nic_network", "nic.network/0") ] -- | The aliased instance fields. aliasedFields :: FieldList Instance Runtime aliasedFields = aliasFields instanceAliases instanceFields -- | The instance fields. instanceFields :: FieldList Instance Runtime instanceFields = -- Simple fields [ (FieldDefinition "admin_state" "InstanceState" QFTText "Desired state of instance", FieldSimple (rsMaybeNoData . liftM adminStateToRaw . instAdminState), QffNormal) , (FieldDefinition "admin_state_source" "InstanceStateSource" QFTText "Who last changed the desired state of the instance", FieldSimple (rsMaybeNoData . liftM adminStateSourceToRaw . instAdminStateSource), QffNormal) , (FieldDefinition "admin_up" "Autostart" QFTBool "Desired state of instance", FieldSimple (rsMaybeNoData . liftM (== AdminUp) . instAdminState), QffNormal) , (FieldDefinition "disks_active" "DisksActive" QFTBool "Desired state of instance disks", FieldSimple (rsMaybeNoData . instDisksActive), QffNormal) , (FieldDefinition "name" "Instance" QFTText "Instance name", FieldSimple (rsMaybeNoData . instName), QffHostname) , (FieldDefinition "hypervisor" "Hypervisor" QFTText "Hypervisor name", FieldSimple (rsMaybeNoData . instHypervisor), QffNormal) , (FieldDefinition "network_port" "Network_port" QFTOther "Instance network port if available (e.g. for VNC console)", FieldSimple (rsMaybeUnavail . instNetworkPort), QffNormal) , (FieldDefinition "os" "OS" QFTText "Operating system", FieldSimple (rsMaybeNoData . instOs), QffNormal) , (FieldDefinition "pnode" "Primary_node" QFTText "Primary node", FieldConfig getPrimaryNodeName, QffHostname) , (FieldDefinition "pnode.group" "PrimaryNodeGroup" QFTText "Primary node's group", FieldConfig getPrimaryNodeGroupName, QffNormal) , (FieldDefinition "pnode.group.uuid" "PrimaryNodeGroupUUID" QFTText "Primary node's group UUID", FieldConfig getPrimaryNodeGroupUuid, QffNormal) , (FieldDefinition "snodes" "Secondary_Nodes" QFTOther "Secondary nodes; usually this will just be one node", FieldConfig (getSecondaryNodeAttribute nodeName), QffNormal) , (FieldDefinition "snodes.group" "SecondaryNodesGroups" QFTOther "Node groups of secondary nodes", FieldConfig (getSecondaryNodeGroupAttribute groupName), QffNormal) , (FieldDefinition "snodes.group.uuid" "SecondaryNodesGroupsUUID" QFTOther "Node group UUIDs of secondary nodes", FieldConfig (getSecondaryNodeGroupAttribute uuidOf), QffNormal) ] ++ -- Instance parameter fields, whole [ (FieldDefinition "hvparams" "HypervisorParameters" QFTOther "Hypervisor parameters (merged)", FieldConfig ((rsNormal .) . getFilledInstHvParams (C.toList C.hvcGlobals)), QffNormal), (FieldDefinition "beparams" "BackendParameters" QFTOther "Backend parameters (merged)", FieldConfig ((rsErrorNoData .) . getFilledInstBeParams), QffNormal) , (FieldDefinition "osparams" "OpSysParameters" QFTOther "Operating system parameters (merged)", FieldConfig ((rsNormal .) . getFilledInstOsParams), QffNormal) , (FieldDefinition "custom_hvparams" "CustomHypervisorParameters" QFTOther "Custom hypervisor parameters", FieldSimple (rsNormal . instHvparams), QffNormal) , (FieldDefinition "custom_beparams" "CustomBackendParameters" QFTOther "Custom backend parameters", FieldSimple (rsNormal . instBeparams), QffNormal) , (FieldDefinition "custom_osparams" "CustomOpSysParameters" QFTOther "Custom operating system parameters", FieldSimple (rsNormal . instOsparams), QffNormal) , (FieldDefinition "custom_nicparams" "CustomNicParameters" QFTOther "Custom network interface parameters", FieldSimple (rsNormal . map nicNicparams . instNics), QffNormal) ] ++ -- Instance parameter fields, generated map (buildBeParamField beParamGetter) allBeParamFields ++ map (buildHvParamField hvParamGetter) (C.toList C.hvsParameters \\ C.toList C.hvcGlobals) ++ -- disk parameter fields [ (FieldDefinition "disk_usage" "DiskUsage" QFTUnit "Total disk space used by instance on each of its nodes; this is not the\ \ disk size visible to the instance, but the usage on the node", FieldConfig getDiskSizeRequirements, QffNormal) , (FieldDefinition "disk.count" "Disks" QFTNumber "Number of disks", FieldSimple (rsNormal . length . instDisks), QffNormal) , (FieldDefinition "disk.sizes" "Disk_sizes" QFTOther "List of disk sizes", FieldConfig getDiskSizes, QffNormal) , (FieldDefinition "disk.spindles" "Disk_spindles" QFTOther "List of disk spindles", FieldConfig getDiskSpindles, QffNormal) , (FieldDefinition "disk.names" "Disk_names" QFTOther "List of disk names", FieldConfig getDiskNames, QffNormal) , (FieldDefinition "disk.uuids" "Disk_UUIDs" QFTOther "List of disk UUIDs", FieldConfig getDiskUuids, QffNormal) -- For pre-2.14 backwards compatibility , (FieldDefinition "disk_template" "Disk_template" QFTText "Instance disk template", FieldConfig getDiskTemplate, QffNormal) ] ++ -- Per-disk parameter fields instantiateIndexedFields C.maxDisks [ (fieldDefinitionCompleter "disk.size/%d" "Disk/%d" QFTUnit "Disk size of %s disk", getIndexedOptionalConfField getInstDisksFromObj diskSize, QffNormal) , (fieldDefinitionCompleter "disk.spindles/%d" "DiskSpindles/%d" QFTNumber "Spindles of %s disk", getIndexedOptionalConfField getInstDisksFromObj diskSpindles, QffNormal) , (fieldDefinitionCompleter "disk.name/%d" "DiskName/%d" QFTText "Name of %s disk", getIndexedOptionalConfField getInstDisksFromObj diskName, QffNormal) , (fieldDefinitionCompleter "disk.uuid/%d" "DiskUUID/%d" QFTText "UUID of %s disk", getIndexedConfField getInstDisksFromObj uuidOf, QffNormal) ] ++ -- Aggregate nic parameter fields [ (FieldDefinition "nic.count" "NICs" QFTNumber "Number of network interfaces", FieldSimple (rsNormal . length . instNics), QffNormal) , (FieldDefinition "nic.macs" "NIC_MACs" QFTOther (nicAggDescPrefix ++ "MAC address"), FieldSimple (rsNormal . map nicMac . instNics), QffNormal) , (FieldDefinition "nic.ips" "NIC_IPs" QFTOther (nicAggDescPrefix ++ "IP address"), FieldSimple (rsNormal . map (MaybeForJSON . nicIp) . instNics), QffNormal) , (FieldDefinition "nic.names" "NIC_Names" QFTOther (nicAggDescPrefix ++ "name"), FieldSimple (rsNormal . map (MaybeForJSON . nicName) . instNics), QffNormal) , (FieldDefinition "nic.uuids" "NIC_UUIDs" QFTOther (nicAggDescPrefix ++ "UUID"), FieldSimple (rsNormal . map uuidOf . instNics), QffNormal) , (FieldDefinition "nic.modes" "NIC_modes" QFTOther (nicAggDescPrefix ++ "mode"), FieldConfig (\cfg -> rsNormal . map (nicpMode . fillNicParamsFromConfig cfg . nicNicparams) . instNics), QffNormal) , (FieldDefinition "nic.vlans" "NIC_VLANs" QFTOther (nicAggDescPrefix ++ "VLAN"), FieldConfig (\cfg -> rsNormal . map (MaybeForJSON . getNicVlan . fillNicParamsFromConfig cfg . nicNicparams) . instNics), QffNormal) , (FieldDefinition "nic.bridges" "NIC_bridges" QFTOther (nicAggDescPrefix ++ "bridge"), FieldConfig (\cfg -> rsNormal . map (MaybeForJSON . getNicBridge . fillNicParamsFromConfig cfg . nicNicparams) . instNics), QffNormal) , (FieldDefinition "nic.links" "NIC_links" QFTOther (nicAggDescPrefix ++ "link"), FieldConfig (\cfg -> rsNormal . map (nicpLink . fillNicParamsFromConfig cfg . nicNicparams) . instNics), QffNormal) , (FieldDefinition "nic.networks" "NIC_networks" QFTOther "List containing each interface's network", FieldSimple (rsNormal . map (MaybeForJSON . nicNetwork) . instNics), QffNormal) , (FieldDefinition "nic.networks.names" "NIC_networks_names" QFTOther "List containing the name of each interface's network", FieldConfig (\cfg -> rsNormal . map (\x -> MaybeForJSON (getNetworkName cfg <$> nicNetwork x)) . instNics), QffNormal) ] ++ -- Per-nic parameter fields instantiateIndexedFields C.maxNics [ (fieldDefinitionCompleter "nic.ip/%d" "NicIP/%d" QFTText ("IP address" ++ nicDescSuffix), getIndexedOptionalField instNics nicIp, QffNormal) , (fieldDefinitionCompleter "nic.uuid/%d" "NicUUID/%d" QFTText ("UUID address" ++ nicDescSuffix), getIndexedField instNics uuidOf, QffNormal) , (fieldDefinitionCompleter "nic.mac/%d" "NicMAC/%d" QFTText ("MAC address" ++ nicDescSuffix), getIndexedField instNics nicMac, QffNormal) , (fieldDefinitionCompleter "nic.name/%d" "NicName/%d" QFTText ("Name address" ++ nicDescSuffix), getIndexedOptionalField instNics nicName, QffNormal) , (fieldDefinitionCompleter "nic.network/%d" "NicNetwork/%d" QFTText ("Network" ++ nicDescSuffix), getIndexedOptionalField instNics nicNetwork, QffNormal) , (fieldDefinitionCompleter "nic.mode/%d" "NicMode/%d" QFTText ("Mode" ++ nicDescSuffix), getIndexedNicField nicpMode, QffNormal) , (fieldDefinitionCompleter "nic.link/%d" "NicLink/%d" QFTText ("Link" ++ nicDescSuffix), getIndexedNicField nicpLink, QffNormal) , (fieldDefinitionCompleter "nic.vlan/%d" "NicVLAN/%d" QFTText ("VLAN" ++ nicDescSuffix), getOptionalIndexedNicField getNicVlan, QffNormal) , (fieldDefinitionCompleter "nic.network.name/%d" "NicNetworkName/%d" QFTText ("Network name" ++ nicDescSuffix), getIndexedNicNetworkNameField, QffNormal) , (fieldDefinitionCompleter "nic.bridge/%d" "NicBridge/%d" QFTText ("Bridge" ++ nicDescSuffix), getOptionalIndexedNicField getNicBridge, QffNormal) ] ++ -- Live fields using special getters [ (FieldDefinition "status" "Status" QFTText statusDocText, FieldConfigRuntime statusExtract, QffNormal) , (FieldDefinition "oper_state" "Running" QFTBool "Actual state of instance", FieldRuntime operStatusExtract, QffNormal), (FieldDefinition "console" "Console" QFTOther "Instance console information", FieldRuntime consoleExtract, QffNormal) ] ++ -- Simple live fields map instanceLiveFieldBuilder instanceLiveFieldsDefs ++ -- Common fields timeStampFields ++ serialFields "Instance" ++ uuidFields "Instance" ++ forthcomingFields "Instance" ++ tagsFields -- * Helper functions for node property retrieval -- | Constant suffix of network interface field descriptions. nicDescSuffix ::String nicDescSuffix = " of %s network interface" -- | Almost-constant suffix of aggregate network interface field descriptions. nicAggDescPrefix ::String nicAggDescPrefix = "List containing each network interface's " -- | Given a network name id, returns the network's name. getNetworkName :: ConfigData -> String -> NonEmptyString getNetworkName cfg = networkName . (Map.!) (fromContainer $ configNetworks cfg) . UTF8.fromString -- | Gets the bridge of a NIC. getNicBridge :: FilledNicParams -> Maybe String getNicBridge nicParams | nicpMode nicParams == NMBridged = Just $ nicpLink nicParams | otherwise = Nothing -- | Gets the VLAN of a NIC. getNicVlan :: FilledNicParams -> Maybe String getNicVlan params | nicpMode params == NMOvs = Just $ nicpVlan params | otherwise = Nothing -- | Fill partial NIC params by using the defaults from the configuration. fillNicParamsFromConfig :: ConfigData -> PartialNicParams -> FilledNicParams fillNicParamsFromConfig cfg = fillParams (getDefaultNicParams cfg) -- | Retrieves the default network interface parameters. getDefaultNicParams :: ConfigData -> FilledNicParams getDefaultNicParams cfg = (Map.!) (fromContainer . clusterNicparams . configCluster $ cfg) $ UTF8.fromString C.ppDefault -- | Retrieves the real disk size requirements for all the disks of the -- instance. This includes the metadata etc. and is different from the values -- visible to the instance. getDiskSizeRequirements :: ConfigData -> Instance -> ResultEntry getDiskSizeRequirements cfg inst = rsErrorNoData . liftA (sum . map getSize) . getInstDisksFromObj cfg $ inst where diskType x = lidDiskType <$> diskLogicalId x getSize :: Disk -> Int getSize disk = let dt = diskType disk in case dt of Just DTDrbd8 -> fromMaybe 0 (diskSize disk) + C.drbdMetaSize Just DTDiskless -> 0 Just DTBlock -> 0 _ -> fromMaybe 0 (diskSize disk) -- | Get a list of disk sizes for an instance getDiskSizes :: ConfigData -> Instance -> ResultEntry getDiskSizes cfg = rsErrorNoData . liftA (map $ MaybeForJSON . diskSize) . getInstDisksFromObj cfg -- | Get a list of disk spindles getDiskSpindles :: ConfigData -> Instance -> ResultEntry getDiskSpindles cfg = rsErrorNoData . liftA (map (MaybeForJSON . diskSpindles)) . getInstDisksFromObj cfg -- | Get a list of disk names for an instance getDiskNames :: ConfigData -> Instance -> ResultEntry getDiskNames cfg = rsErrorNoData . liftA (map (MaybeForJSON . diskName)) . getInstDisksFromObj cfg -- | Get a list of disk UUIDs for an instance getDiskUuids :: ConfigData -> Instance -> ResultEntry getDiskUuids cfg = rsErrorNoData . liftA (map uuidOf) . getInstDisksFromObj cfg -- | Creates a functions which produces a FieldConfig 'FieldGetter' when fed -- an index. Works for fields that may not return a value, expressed through -- the Maybe monad. getIndexedOptionalConfField :: (J.JSON b) => (ConfigData -> Instance -> ErrorResult [a]) -- ^ Extracts a list of objects -> (a -> Maybe b) -- ^ Possibly gets a property -- from an object -> Int -- ^ Index in list to use -> FieldGetter Instance Runtime -- ^ Result getIndexedOptionalConfField extractor optPropertyGetter index = let getProperty x = maybeAt index x >>= optPropertyGetter in FieldConfig (\cfg -> rsErrorMaybeUnavail . liftA getProperty . extractor cfg) -- | Creates a function which produces a FieldConfig 'FieldGetter' when fed -- an index. Works only for fields that surely return a value. getIndexedConfField :: (J.JSON b) => (ConfigData -> Instance -> ErrorResult [a]) -- ^ Extracts a list of objects -> (a -> b) -- ^ Gets a property from an object -> Int -- ^ Index in list to use -> FieldGetter Instance Runtime -- ^ Result getIndexedConfField extractor propertyGetter index = let optPropertyGetter = Just . propertyGetter in getIndexedOptionalConfField extractor optPropertyGetter index -- | Returns a field that retrieves a given NIC's network name. getIndexedNicNetworkNameField :: Int -> FieldGetter Instance Runtime getIndexedNicNetworkNameField index = FieldConfig (\cfg inst -> rsMaybeUnavail $ do nicObj <- maybeAt index $ instNics inst nicNetworkId <- nicNetwork nicObj return $ getNetworkName cfg nicNetworkId) -- | Gets a fillable NIC field. getIndexedNicField :: (J.JSON a) => (FilledNicParams -> a) -> Int -> FieldGetter Instance Runtime getIndexedNicField getter = getOptionalIndexedNicField (\x -> Just . getter $ x) -- | Gets an optional fillable NIC field. getOptionalIndexedNicField :: (J.JSON a) => (FilledNicParams -> Maybe a) -> Int -> FieldGetter Instance Runtime getOptionalIndexedNicField = getIndexedFieldWithDefault (map nicNicparams . instNics) (\x _ -> getDefaultNicParams x) fillParams -- | Creates a function which produces a 'FieldGetter' when fed an index. Works -- for fields that should be filled out through the use of a default. getIndexedFieldWithDefault :: (J.JSON c) => (Instance -> [a]) -- ^ Extracts a list of incomplete objects -> (ConfigData -> Instance -> b) -- ^ Extracts the default object -> (b -> a -> b) -- ^ Fills the default object -> (b -> Maybe c) -- ^ Extracts an obj property -> Int -- ^ Index in list to use -> FieldGetter Instance Runtime -- ^ Result getIndexedFieldWithDefault listGetter defaultGetter fillFn propertyGetter index = FieldConfig (\cfg inst -> rsMaybeUnavail $ do incompleteObj <- maybeAt index $ listGetter inst let defaultObj = defaultGetter cfg inst completeObj = fillFn defaultObj incompleteObj propertyGetter completeObj) -- | Creates a function which produces a 'FieldGetter' when fed an index. Works -- for fields that may not return a value, expressed through the Maybe monad. getIndexedOptionalField :: (J.JSON b) => (Instance -> [a]) -- ^ Extracts a list of objects -> (a -> Maybe b) -- ^ Possibly gets a property -- from an object -> Int -- ^ Index in list to use -> FieldGetter Instance Runtime -- ^ Result getIndexedOptionalField extractor optPropertyGetter index = FieldSimple(\inst -> rsMaybeUnavail $ do obj <- maybeAt index $ extractor inst optPropertyGetter obj) -- | Creates a function which produces a 'FieldGetter' when fed an index. -- Works only for fields that surely return a value. getIndexedField :: (J.JSON b) => (Instance -> [a]) -- ^ Extracts a list of objects -> (a -> b) -- ^ Gets a property from an object -> Int -- ^ Index in list to use -> FieldGetter Instance Runtime -- ^ Result getIndexedField extractor propertyGetter index = let optPropertyGetter = Just . propertyGetter in getIndexedOptionalField extractor optPropertyGetter index -- | Retrieves a value from an array at an index, using the Maybe monad to -- indicate failure. maybeAt :: Int -> [a] -> Maybe a maybeAt index list | index >= length list = Nothing | otherwise = Just $ list !! index -- | Primed with format strings for everything but the type, it consumes two -- values and uses them to complete the FieldDefinition. -- Warning: a bit unsafe as it uses printf. Handle with care. fieldDefinitionCompleter :: (PrintfArg t1) => (PrintfArg t2) => FieldName -> FieldTitle -> FieldType -> FieldDoc -> t1 -> t2 -> FieldDefinition fieldDefinitionCompleter fName fTitle fType fDoc firstVal secondVal = FieldDefinition (printf fName firstVal) (printf fTitle firstVal) fType (printf fDoc secondVal) -- | Given an incomplete field definition and values that can complete it, -- return a fully functional FieldData. Cannot work for all cases, should be -- extended as necessary. fillIncompleteFields :: (t1 -> t2 -> FieldDefinition, t1 -> FieldGetter a b, QffMode) -> t1 -> t2 -> FieldData a b fillIncompleteFields (iDef, iGet, mode) firstVal secondVal = (iDef firstVal secondVal, iGet firstVal, mode) -- | Given indexed fields that describe lists, complete / instantiate them for -- a given list size. instantiateIndexedFields :: (Show t1, Integral t1) => Int -- ^ The size of the list -> [(t1 -> String -> FieldDefinition, t1 -> FieldGetter a b, QffMode)] -- ^ The indexed fields -> FieldList a b -- ^ A list of complete fields instantiateIndexedFields listSize fields = do index <- take listSize [0..] field <- fields return . fillIncompleteFields field index . formatOrdinal $ index + 1 -- * Various helper functions for property retrieval -- | Helper function for primary node retrieval getPrimaryNode :: ConfigData -> Instance -> ErrorResult Node getPrimaryNode cfg = maybe (Bad $ ParameterError "no primary node") return . instName >=> getInstPrimaryNode cfg -- | Get primary node hostname getPrimaryNodeName :: ConfigData -> Instance -> ResultEntry getPrimaryNodeName cfg inst = rsErrorNoData $ nodeName <$> getPrimaryNode cfg inst -- | Get primary node group getPrimaryNodeGroup :: ConfigData -> Instance -> ErrorResult NodeGroup getPrimaryNodeGroup cfg inst = do pNode <- getPrimaryNode cfg inst maybeToError "Configuration missing" $ getGroupOfNode cfg pNode -- | Get primary node group name getPrimaryNodeGroupName :: ConfigData -> Instance -> ResultEntry getPrimaryNodeGroupName cfg inst = rsErrorNoData $ groupName <$> getPrimaryNodeGroup cfg inst -- | Get primary node group uuid getPrimaryNodeGroupUuid :: ConfigData -> Instance -> ResultEntry getPrimaryNodeGroupUuid cfg inst = rsErrorNoData $ uuidOf <$> getPrimaryNodeGroup cfg inst -- | Get secondary nodes - the configuration objects themselves getSecondaryNodes :: ConfigData -> Instance -> ErrorResult [Node] getSecondaryNodes cfg inst = do pNode <- getPrimaryNode cfg inst iname <- maybe (Bad $ ParameterError "no name") return $ instName inst allNodes <- getInstAllNodes cfg iname return $ delete pNode allNodes -- | Get attributes of the secondary nodes getSecondaryNodeAttribute :: (J.JSON a) => (Node -> a) -> ConfigData -> Instance -> ResultEntry getSecondaryNodeAttribute getter cfg inst = rsErrorNoData $ map (J.showJSON . getter) <$> getSecondaryNodes cfg inst -- | Get secondary node groups getSecondaryNodeGroups :: ConfigData -> Instance -> ErrorResult [NodeGroup] getSecondaryNodeGroups cfg inst = do sNodes <- getSecondaryNodes cfg inst return . catMaybes $ map (getGroupOfNode cfg) sNodes -- | Get attributes of secondary node groups getSecondaryNodeGroupAttribute :: (J.JSON a) => (NodeGroup -> a) -> ConfigData -> Instance -> ResultEntry getSecondaryNodeGroupAttribute getter cfg inst = rsErrorNoData $ map (J.showJSON . getter) <$> getSecondaryNodeGroups cfg inst -- | Beparam getter builder: given a field, it returns a FieldConfig -- getter, that is a function that takes the config and the object and -- returns the Beparam field specified when the getter was built. beParamGetter :: String -- ^ The field we are building the getter for -> ConfigData -- ^ The configuration object -> Instance -- ^ The instance configuration object -> ResultEntry -- ^ The result beParamGetter field config inst = case getFilledInstBeParams config inst of Ok beParams -> dictFieldGetter field $ Just beParams Bad _ -> rsNoData -- | Hvparam getter builder: given a field, it returns a FieldConfig -- getter, that is a function that takes the config and the object and -- returns the Hvparam field specified when the getter was built. hvParamGetter :: String -- ^ The field we're building the getter for -> ConfigData -> Instance -> ResultEntry hvParamGetter field cfg inst = rsMaybeUnavail . Map.lookup (UTF8.fromString field) . fromContainer $ getFilledInstHvParams (C.toList C.hvcGlobals) cfg inst -- * Live fields functionality -- | List of node live fields. instanceLiveFieldsDefs :: [(FieldName, FieldTitle, FieldType, String, FieldDoc)] instanceLiveFieldsDefs = [ ("oper_ram", "Memory", QFTUnit, "oper_ram", "Actual memory usage as seen by hypervisor") , ("oper_vcpus", "VCPUs", QFTNumber, "oper_vcpus", "Actual number of VCPUs as seen by hypervisor") ] -- | Map each name to a function that extracts that value from the RPC result. instanceLiveFieldExtract :: FieldName -> InstanceInfo -> Instance -> J.JSValue instanceLiveFieldExtract "oper_ram" info _ = J.showJSON $ instInfoMemory info instanceLiveFieldExtract "oper_vcpus" info _ = J.showJSON $ instInfoVcpus info instanceLiveFieldExtract n _ _ = J.showJSON $ "The field " ++ n ++ " is not an expected or extractable live field!" -- | Helper for extracting an instance live field from the RPC results. instanceLiveRpcCall :: FieldName -> Runtime -> Instance -> ResultEntry instanceLiveRpcCall fname (Right (Just (res, _), _)) inst = case instanceLiveFieldExtract fname res inst of J.JSNull -> rsNoData x -> rsNormal x instanceLiveRpcCall _ (Right (Nothing, _)) _ = rsUnavail instanceLiveRpcCall _ (Left err) _ = ResultEntry (rpcErrorToStatus err) Nothing -- | Builder for node live fields. instanceLiveFieldBuilder :: (FieldName, FieldTitle, FieldType, String, FieldDoc) -> FieldData Instance Runtime instanceLiveFieldBuilder (fname, ftitle, ftype, _, fdoc) = ( FieldDefinition fname ftitle ftype fdoc , FieldRuntime $ instanceLiveRpcCall fname , QffNormal) -- * Functionality related to status and operational status extraction -- | The documentation text for the instance status field statusDocText :: String statusDocText = let si = show . instanceStatusToRaw :: InstanceStatus -> String in "Instance status; " ++ si Running ++ " if instance is set to be running and actually is, " ++ si StatusDown ++ " if instance is stopped and is not running, " ++ si WrongNode ++ " if instance running, but not on its designated primary node, " ++ si ErrorUp ++ " if instance should be stopped, but is actually running, " ++ si ErrorDown ++ " if instance should run, but doesn't, " ++ si NodeDown ++ " if instance's primary node is down, " ++ si NodeOffline ++ " if instance's primary node is marked offline, " ++ si StatusOffline ++ " if instance is offline and does not use dynamic resources" -- | Checks if the primary node of an instance is offline isPrimaryOffline :: ConfigData -> Instance -> Bool isPrimaryOffline cfg inst = let pNodeResult = maybe (Bad $ ParameterError "no primary node") return (instPrimaryNode inst) >>= getNode cfg in case pNodeResult of Ok pNode -> nodeOffline pNode Bad _ -> error "Programmer error - result assumed to be OK is Bad!" -- | Determines if user shutdown reporting is enabled userShutdownEnabled :: ConfigData -> Bool userShutdownEnabled = clusterEnabledUserShutdown . configCluster -- | Determines the status of a live instance liveInstanceStatus :: ConfigData -> (InstanceInfo, Bool) -> Instance -> InstanceStatus liveInstanceStatus cfg (instInfo, foundOnPrimary) inst | not foundOnPrimary = WrongNode | otherwise = case instanceState of InstanceStateRunning | adminState == Just AdminUp -> Running | otherwise -> ErrorUp InstanceStateShutdown | adminState == Just AdminUp && allowDown -> UserDown | adminState == Just AdminUp -> ErrorDown | otherwise -> StatusDown where adminState = instAdminState inst instanceState = instInfoState instInfo hvparams = fromContainer $ getFilledInstHvParams (C.toList C.hvcGlobals) cfg inst allowDown = userShutdownEnabled cfg && (instHypervisor inst /= Just Kvm || (Map.member (UTF8.fromString C.hvKvmUserShutdown) hvparams && hvparams Map.! UTF8.fromString C.hvKvmUserShutdown == J.JSBool True)) -- | Determines the status of a dead instance. deadInstanceStatus :: ConfigData -> Instance -> InstanceStatus deadInstanceStatus cfg inst = case instAdminState inst of Just AdminUp -> ErrorDown Just AdminDown | wasCleanedUp && userShutdownEnabled cfg -> UserDown | otherwise -> StatusDown Just AdminOffline -> StatusOffline Nothing -> StatusDown where wasCleanedUp = instAdminStateSource inst == Just UserSource -- | Determines the status of the instance, depending on whether it is possible -- to communicate with its primary node, on which node it is, and its -- configuration. determineInstanceStatus :: ConfigData -- ^ The configuration data -> Runtime -- ^ All the data from the live call -> Instance -- ^ Static instance configuration -> InstanceStatus -- ^ Result determineInstanceStatus cfg res inst | isPrimaryOffline cfg inst = NodeOffline | otherwise = case res of Left _ -> NodeDown Right (Just liveData, _) -> liveInstanceStatus cfg liveData inst Right (Nothing, _) -> deadInstanceStatus cfg inst -- | Extracts the instance status, retrieving it using the functions above and -- transforming it into a 'ResultEntry'. statusExtract :: ConfigData -> Runtime -> Instance -> ResultEntry statusExtract cfg res inst = rsNormal . J.showJSON . instanceStatusToRaw $ determineInstanceStatus cfg res inst -- | Extracts the operational status of the instance. operStatusExtract :: Runtime -> Instance -> ResultEntry operStatusExtract res _ = rsMaybeNoData $ J.showJSON <$> case res of Left _ -> Nothing Right (x, _) -> Just $ isJust x -- | Extracts the console connection information consoleExtract :: Runtime -> Instance -> ResultEntry consoleExtract (Left err) _ = ResultEntry (rpcErrorToStatus err) Nothing consoleExtract (Right (_, val)) _ = rsMaybeNoData val -- * Helper functions extracting information as necessary for the generic query -- interfaces -- | This function checks if a node with a given uuid has experienced an error -- or not. checkForNodeError :: [(String, ERpcError a)] -> String -> Maybe RpcError checkForNodeError uuidList uuid = case snd <$> pickPairUnique uuid uuidList of Just (Left err) -> Just err Just (Right _) -> Nothing Nothing -> Just . RpcResultError $ "Node response not present" -- | Finds information about the instance in the info delivered by a node findInfoInNodeResult :: Instance -> ERpcError RpcResultAllInstancesInfo -> Maybe InstanceInfo findInfoInNodeResult inst nodeResponse = case nodeResponse of Left _err -> Nothing Right allInfo -> let instances = rpcResAllInstInfoInstances allInfo maybeMatch = instName inst >>= (`pickPairUnique` instances) in snd <$> maybeMatch -- | Retrieves the instance information if it is present anywhere in the all -- instances RPC result. Notes if it originates from the primary node. -- An error is delivered if there is no result, and the primary node is down. getInstanceInfo :: [(String, ERpcError RpcResultAllInstancesInfo)] -> Instance -> ERpcError (Maybe (InstanceInfo, Bool)) getInstanceInfo uuidList inst = case instPrimaryNode inst of Nothing -> Right Nothing Just pNodeUuid -> let primarySearchResult = pickPairUnique pNodeUuid uuidList >>= findInfoInNodeResult inst . snd in case primarySearchResult of Just instInfo -> Right . Just $ (instInfo, True) Nothing -> let allSearchResult = getFirst . mconcat $ map (First . findInfoInNodeResult inst . snd) uuidList in case allSearchResult of Just instInfo -> Right . Just $ (instInfo, False) Nothing -> case checkForNodeError uuidList pNodeUuid of Just err -> Left err Nothing -> Right Nothing -- | Retrieves the console information if present anywhere in the given results getConsoleInfo :: [(String, ERpcError RpcResultInstanceConsoleInfo)] -> Instance -> Maybe InstanceConsoleInfo getConsoleInfo uuidList inst = let allValidResults = concatMap rpcResInstConsInfoInstancesInfo . rights . map snd $ uuidList in snd <$> (instName inst >>= flip pickPairUnique allValidResults) -- | Extracts all the live information that can be extracted. extractLiveInfo :: [(Node, ERpcError RpcResultAllInstancesInfo)] -> [(Node, ERpcError RpcResultInstanceConsoleInfo)] -> Instance -> Runtime extractLiveInfo nodeResultList nodeConsoleList inst = let uuidConvert = map (\(x, y) -> (uuidOf x, y)) uuidResultList = uuidConvert nodeResultList uuidConsoleList = uuidConvert nodeConsoleList in case getInstanceInfo uuidResultList inst of -- If we can't get the instance info, we can't get the console info either. -- Best to propagate the error further. Left err -> Left err Right res -> Right (res, getConsoleInfo uuidConsoleList inst) -- | Retrieves all the parameters for the console calls. getAllConsoleParams :: ConfigData -> [Instance] -> ErrorResult [InstanceConsoleInfoParams] getAllConsoleParams cfg = mapM $ \i -> InstanceConsoleInfoParams i <$> getPrimaryNode cfg i <*> getPrimaryNodeGroup cfg i <*> pure (getFilledInstHvParams [] cfg i) <*> getFilledInstBeParams cfg i -- | Compares two params according to their node, needed for grouping. compareParamsByNode :: InstanceConsoleInfoParams -> InstanceConsoleInfoParams -> Bool compareParamsByNode x y = instConsInfoParamsNode x == instConsInfoParamsNode y -- | Groups instance information calls heading out to the same nodes. consoleParamsToCalls :: [InstanceConsoleInfoParams] -> [(Node, RpcCallInstanceConsoleInfo)] consoleParamsToCalls params = let sortedParams = sortBy (comparing (instPrimaryNode . instConsInfoParamsInstance)) params groupedParams = groupBy compareParamsByNode sortedParams in map (\x -> case x of [] -> error "Programmer error: group must have one or more members" paramGroup@(y:_) -> let node = instConsInfoParamsNode y packer z = do name <- instName $ instConsInfoParamsInstance z return (name, z) in (node, RpcCallInstanceConsoleInfo . mapMaybe packer $ paramGroup) ) groupedParams -- | Retrieves a list of all the hypervisors and params used by the given -- instances. getHypervisorSpecs :: ConfigData -> [Instance] -> [(Hypervisor, HvParams)] getHypervisorSpecs cfg instances = let hvs = nub . mapMaybe instHypervisor $ instances hvParamMap = (fromContainer . clusterHvparams . configCluster $ cfg) in zip hvs . map ((Map.!) hvParamMap) $ hvs -- | Collect live data from RPC query if enabled. collectLiveData :: Bool -- ^ Live queries allowed -> ConfigData -- ^ The cluster config -> [String] -- ^ The requested fields -> [Instance] -- ^ The instance objects -> IO [(Instance, Runtime)] collectLiveData liveDataEnabled cfg fields instances | not liveDataEnabled = return . zip instances . repeat . Left . RpcResultError $ "Live data disabled" | otherwise = do let hvSpecs = getHypervisorSpecs cfg instances instanceNodes = nub . justOk $ map ( maybe (Bad $ ParameterError "no primary node") return . instPrimaryNode >=> getNode cfg) instances goodNodes = nodesWithValidConfig cfg instanceNodes instInfoRes <- executeRpcCall goodNodes (RpcCallAllInstancesInfo hvSpecs) consInfoRes <- if "console" `elem` fields then case getAllConsoleParams cfg instances of Ok p -> executeRpcCalls $ consoleParamsToCalls p Bad _ -> return . zip goodNodes . repeat . Left $ RpcResultError "Cannot construct parameters for console info call" else return [] -- The information is not necessary return . zip instances . map (extractLiveInfo instInfoRes consInfoRes) $ instances -- | An aggregate disk attribute for backward compatibility. getDiskTemplate :: ConfigData -> Instance -> ResultEntry getDiskTemplate cfg inst = let disks = getInstDisksFromObj cfg inst getDt x = lidDiskType <$> diskLogicalId x disk_types :: ErrorResult [DiskTemplate] disk_types = nub <$> catMaybes <$> map getDt <$> disks mix :: [DiskTemplate] -> J.JSValue mix [] = J.showJSON C.dtDiskless mix [t] = J.showJSON t mix _ = J.showJSON C.dtMixed in case mix <$> disk_types of Ok t -> rsNormal t Bad _ -> rsNoData ganeti-3.1.0~rc2/src/Ganeti/Query/Job.hs000064400000000000000000000135461476477700300177660ustar00rootroot00000000000000{-| Implementation of the Ganeti Query2 job queries. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Job ( RuntimeData , fieldsMap , wantArchived ) where import qualified Text.JSON as J import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.JQueue import Ganeti.Query.Common import Ganeti.Query.Language import Ganeti.Query.Types import Ganeti.Types -- | The runtime data for a job. type RuntimeData = Result (QueuedJob, Bool) -- | Job priority explanation. jobPrioDoc :: String jobPrioDoc = "Current job priority (" ++ show C.opPrioLowest ++ " to " ++ show C.opPrioHighest ++ ")" -- | Timestamp doc. tsDoc :: String -> String tsDoc = (++ " (tuple containing seconds and microseconds)") -- | Wrapper for unavailable job. maybeJob :: (J.JSON a) => (QueuedJob -> a) -> RuntimeData -> JobId -> ResultEntry maybeJob _ (Bad _) _ = rsUnavail maybeJob f (Ok (v, _)) _ = rsNormal $ f v -- | Wrapper for optional fields that should become unavailable. maybeJobOpt :: (J.JSON a) => (QueuedJob -> Maybe a) -> RuntimeData -> JobId -> ResultEntry maybeJobOpt _ (Bad _) _ = rsUnavail maybeJobOpt f (Ok (v, _)) _ = case f v of Nothing -> rsUnavail Just w -> rsNormal w -- | Simple helper for a job getter. jobGetter :: (J.JSON a) => (QueuedJob -> a) -> FieldGetter JobId RuntimeData jobGetter = FieldRuntime . maybeJob -- | Simple helper for a per-opcode getter. opsGetter :: (J.JSON a) => (QueuedOpCode -> a) -> FieldGetter JobId RuntimeData opsGetter f = FieldRuntime $ maybeJob (map f . qjOps) -- | Simple helper for a per-opcode optional field getter. opsOptGetter :: (J.JSON a) => (QueuedOpCode -> Maybe a) -> FieldGetter JobId RuntimeData opsOptGetter f = FieldRuntime $ maybeJob (map (\qo -> case f qo of Nothing -> J.JSNull Just a -> J.showJSON a) . qjOps) -- | Archived field name. archivedField :: String archivedField = "archived" -- | Check whether we should look at archived jobs as well. wantArchived :: [FilterField] -> Bool wantArchived = (archivedField `elem`) -- | List of all node fields. FIXME: QFF_JOB_ID on the id field. jobFields :: FieldList JobId RuntimeData jobFields = [ (FieldDefinition "id" "ID" QFTNumber "Job ID", FieldSimple rsNormal, QffNormal) , (FieldDefinition "status" "Status" QFTText "Job status", jobGetter calcJobStatus, QffNormal) , (FieldDefinition "priority" "Priority" QFTNumber jobPrioDoc, jobGetter calcJobPriority, QffNormal) , (FieldDefinition archivedField "Archived" QFTBool "Whether job is archived", FieldRuntime (\jinfo _ -> case jinfo of Ok (_, archive) -> rsNormal archive _ -> rsUnavail), QffNormal) , (FieldDefinition "ops" "OpCodes" QFTOther "List of all opcodes", opsGetter qoInput, QffNormal) , (FieldDefinition "opresult" "OpCode_result" QFTOther "List of opcodes results", opsGetter qoResult, QffNormal) , (FieldDefinition "opstatus" "OpCode_status" QFTOther "List of opcodes status", opsGetter qoStatus, QffNormal) , (FieldDefinition "oplog" "OpCode_log" QFTOther "List of opcode output logs", opsGetter qoLog, QffNormal) , (FieldDefinition "opstart" "OpCode_start" QFTOther "List of opcode start timestamps (before acquiring locks)", opsOptGetter qoStartTimestamp, QffNormal) , (FieldDefinition "opexec" "OpCode_exec" QFTOther "List of opcode execution start timestamps (after acquiring locks)", opsOptGetter qoExecTimestamp, QffNormal) , (FieldDefinition "opend" "OpCode_end" QFTOther "List of opcode execution end timestamps", opsOptGetter qoEndTimestamp, QffNormal) , (FieldDefinition "oppriority" "OpCode_prio" QFTOther "List of opcode priorities", opsGetter qoPriority, QffNormal) , (FieldDefinition "summary" "Summary" QFTOther "List of per-opcode summaries", opsGetter (extractOpSummary . qoInput), QffNormal) , (FieldDefinition "received_ts" "Received" QFTOther (tsDoc "Timestamp of when job was received"), FieldRuntime (maybeJobOpt qjReceivedTimestamp), QffTimestamp) , (FieldDefinition "start_ts" "Start" QFTOther (tsDoc "Timestamp of job start"), FieldRuntime (maybeJobOpt qjStartTimestamp), QffTimestamp) , (FieldDefinition "end_ts" "End" QFTOther (tsDoc "Timestamp of job end"), FieldRuntime (maybeJobOpt qjEndTimestamp), QffTimestamp) ] -- | The node fields map. fieldsMap :: FieldMap JobId RuntimeData fieldsMap = fieldListToFieldMap jobFields ganeti-3.1.0~rc2/src/Ganeti/Query/Language.hs000064400000000000000000000360551476477700300207770ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, DeriveFunctor, DeriveFoldable, DeriveTraversable #-} {-| Implementation of the Ganeti Query2 language. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Language ( Filter(..) , filterArguments , FilterField , FilterValue(..) , FilterRegex -- note: we don't export the constructor, must use helpers , mkRegex , stringRegex , compiledRegex , showFilterValue , Fields , Query(..) , QueryResult(..) , QueryFields(..) , QueryFieldsResult(..) , FieldName , FieldTitle , FieldType(..) , FieldDoc , FieldDefinition(..) , ResultEntry(..) , ResultStatus(..) , ResultValue , ItemType(..) , QueryTypeOp(..) , queryTypeOpToRaw , QueryTypeLuxi(..) , checkRS ) where import Control.DeepSeq import Control.Monad.Fail (MonadFail) import Data.Foldable import Data.Ratio (numerator, denominator) import Text.JSON.Pretty (pp_value) import Text.JSON.Types import Text.JSON import qualified Ganeti.Query.RegEx as RegEx import qualified Ganeti.Constants as C import Ganeti.THH -- * THH declarations, that require ordering. -- | Status of a query field. $(declareIADT "ResultStatus" [ ("RSNormal", 'C.rsNormal ) , ("RSUnknown", 'C.rsUnknown ) , ("RSNoData", 'C.rsNodata ) , ("RSUnavail", 'C.rsUnavail ) , ("RSOffline", 'C.rsOffline ) ]) $(makeJSONInstance ''ResultStatus) -- | No-op 'NFData' instance for 'ResultStatus', since it's a single -- constructor data-type. instance NFData ResultStatus where rnf x = seq x () -- | Check that ResultStatus is success or fail with descriptive -- message. checkRS :: (MonadFail m) => ResultStatus -> a -> m a checkRS RSNormal val = return val checkRS RSUnknown _ = fail "Unknown field" checkRS RSNoData _ = fail "No data for a field" checkRS RSUnavail _ = fail "Ganeti reports unavailable data" checkRS RSOffline _ = fail "Ganeti reports resource as offline" -- | Type of a query field. $(declareSADT "FieldType" [ ("QFTUnknown", 'C.qftUnknown ) , ("QFTText", 'C.qftText ) , ("QFTBool", 'C.qftBool ) , ("QFTNumber", 'C.qftNumber ) , ("QFTNumberFloat", 'C.qftNumberFloat ) , ("QFTUnit", 'C.qftUnit ) , ("QFTTimestamp", 'C.qftTimestamp ) , ("QFTOther", 'C.qftOther ) ]) $(makeJSONInstance ''FieldType) -- | Supported items on which Qlang works. $(declareSADT "QueryTypeOp" [ ("QRCluster", 'C.qrCluster ) , ("QRInstance", 'C.qrInstance ) , ("QRNode", 'C.qrNode ) , ("QRGroup", 'C.qrGroup ) , ("QROs", 'C.qrOs ) , ("QRExport", 'C.qrExport ) , ("QRNetwork", 'C.qrNetwork ) ]) $(makeJSONInstance ''QueryTypeOp) -- | Supported items on which Qlang works. $(declareSADT "QueryTypeLuxi" [ ("QRLock", 'C.qrLock ) , ("QRJob", 'C.qrJob ) , ("QRFilter", 'C.qrFilter ) ]) $(makeJSONInstance ''QueryTypeLuxi) -- | Overall query type. data ItemType = ItemTypeLuxi QueryTypeLuxi | ItemTypeOpCode QueryTypeOp deriving (Show, Eq) -- | Custom JSON decoder for 'ItemType'. decodeItemType :: (MonadFail m) => JSValue -> m ItemType decodeItemType (JSString s) = case queryTypeOpFromRaw s' of Just v -> return $ ItemTypeOpCode v Nothing -> case queryTypeLuxiFromRaw s' of Just v -> return $ ItemTypeLuxi v Nothing -> fail $ "Can't parse value '" ++ s' ++ "' as neither" ++ "QueryTypeLuxi nor QueryTypeOp" where s' = fromJSString s decodeItemType v = fail $ "Invalid value '" ++ show (pp_value v) ++ "for query type" -- | Custom JSON instance for 'ItemType' since its encoding is not -- consistent with the data type itself. instance JSON ItemType where showJSON (ItemTypeLuxi x) = showJSON x showJSON (ItemTypeOpCode y) = showJSON y readJSON = decodeItemType -- * Sub data types for query2 queries and responses. -- | List of requested fields. type Fields = [ String ] -- | Query2 filter expression. It's a parameteric type since we can -- filter different \"things\"; e.g. field names, or actual field -- getters, etc. data Filter a = EmptyFilter -- ^ No filter at all | AndFilter [ Filter a ] -- ^ @&@ [/expression/, ...] | OrFilter [ Filter a ] -- ^ @|@ [/expression/, ...] | NotFilter (Filter a) -- ^ @!@ /expression/ | TrueFilter a -- ^ @?@ /field/ | EQFilter a FilterValue -- ^ @(=|!=)@ /field/ /value/ | LTFilter a FilterValue -- ^ @<@ /field/ /value/ | GTFilter a FilterValue -- ^ @>@ /field/ /value/ | LEFilter a FilterValue -- ^ @<=@ /field/ /value/ | GEFilter a FilterValue -- ^ @>=@ /field/ /value/ | RegexpFilter a FilterRegex -- ^ @=~@ /field/ /regexp/ | ContainsFilter a FilterValue -- ^ @=[]@ /list-field/ /value/ deriving (Show, Eq, Ord, Functor, Foldable, Traversable) -- | Get the \"things\" a filter talks about. This is useful, e.g., -- to decide which additional fields to fetch in a query depending -- on live data. filterArguments :: Filter a -> [a] filterArguments = toList -- | Serialiser for the 'Filter' data type. showFilter :: (JSON a) => Filter a -> JSValue showFilter (EmptyFilter) = JSNull showFilter (AndFilter exprs) = JSArray $ showJSON C.qlangOpAnd : map showJSON exprs showFilter (OrFilter exprs) = JSArray $ showJSON C.qlangOpOr : map showJSON exprs showFilter (NotFilter flt) = JSArray [showJSON C.qlangOpNot, showJSON flt] showFilter (TrueFilter field) = JSArray [showJSON C.qlangOpTrue, showJSON field] showFilter (EQFilter field value) = JSArray [showJSON C.qlangOpEqual, showJSON field, showJSON value] showFilter (LTFilter field value) = JSArray [showJSON C.qlangOpLt, showJSON field, showJSON value] showFilter (GTFilter field value) = JSArray [showJSON C.qlangOpGt, showJSON field, showJSON value] showFilter (LEFilter field value) = JSArray [showJSON C.qlangOpLe, showJSON field, showJSON value] showFilter (GEFilter field value) = JSArray [showJSON C.qlangOpGe, showJSON field, showJSON value] showFilter (RegexpFilter field regexp) = JSArray [showJSON C.qlangOpRegexp, showJSON field, showJSON regexp] showFilter (ContainsFilter field value) = JSArray [showJSON C.qlangOpContains, showJSON field, showJSON value] -- | Deserializer for the 'Filter' data type. readFilter :: (JSON a) => JSValue -> Result (Filter a) readFilter JSNull = Ok EmptyFilter readFilter (JSArray (JSString op:args)) = readFilterArray (fromJSString op) args readFilter v = Error $ "Cannot deserialise filter: expected array [string, args], got " ++ show (pp_value v) -- | Helper to deserialise an array corresponding to a single filter -- and return the built filter. Note this looks generic but is (at -- least currently) only used for the NotFilter. readFilterArg :: (JSON a) => (Filter a -> Filter a) -- ^ Constructor -> [JSValue] -- ^ Single argument -> Result (Filter a) readFilterArg constr [flt] = constr <$> readJSON flt readFilterArg _ v = Error $ "Cannot deserialise field, expected [filter]" ++ " but got " ++ show (pp_value (showJSON v)) -- | Helper to deserialise an array corresponding to a single field -- and return the built filter. readFilterField :: (JSON a) => (a -> Filter a) -- ^ Constructor -> [JSValue] -- ^ Single argument -> Result (Filter a) readFilterField constr [field] = constr <$> readJSON field readFilterField _ v = Error $ "Cannot deserialise field, expected" ++ " [fieldname] but got " ++ show (pp_value (showJSON v)) -- | Helper to deserialise an array corresponding to a field and -- value, returning the built filter. readFilterFieldValue :: (JSON a, JSON b) => (a -> b -> Filter a) -- ^ Constructor -> [JSValue] -- ^ Arguments array -> Result (Filter a) readFilterFieldValue constr [field, value] = constr <$> readJSON field <*> readJSON value readFilterFieldValue _ v = Error $ "Cannot deserialise field/value pair, expected [fieldname, value]" ++ " but got " ++ show (pp_value (showJSON v)) -- | Inner deserialiser for 'Filter'. readFilterArray :: (JSON a) => String -> [JSValue] -> Result (Filter a) readFilterArray op args | op == C.qlangOpAnd = AndFilter <$> mapM readJSON args | op == C.qlangOpOr = OrFilter <$> mapM readJSON args | op == C.qlangOpNot = readFilterArg NotFilter args | op == C.qlangOpTrue = readFilterField TrueFilter args | op == C.qlangOpEqual = readFilterFieldValue EQFilter args -- Legacy filters: -- - "=" is a legacy short-cut for "==". -- - "!=" is a legacy short-cut for "not (... == ..)". | op == C.qlangOpEqualLegacy = readFilterFieldValue EQFilter args | op == C.qlangOpNotEqual = readFilterFieldValue (\f v -> NotFilter $ EQFilter f v) args | op == C.qlangOpLt = readFilterFieldValue LTFilter args | op == C.qlangOpGt = readFilterFieldValue GTFilter args | op == C.qlangOpLe = readFilterFieldValue LEFilter args | op == C.qlangOpGe = readFilterFieldValue GEFilter args | op == C.qlangOpRegexp = readFilterFieldValue RegexpFilter args | op == C.qlangOpContains = readFilterFieldValue ContainsFilter args | otherwise = Error $ "Unknown filter operand '" ++ op ++ "'" instance (JSON a) => JSON (Filter a) where showJSON = showFilter readJSON = readFilter -- | Field name to filter on. type FilterField = String -- | Value to compare the field value to, for filtering purposes. data FilterValue = QuotedString String | NumericValue Integer deriving (Show, Eq, Ord) -- | Serialiser for 'FilterValue'. The Python code just sends this to -- JSON as-is, so we'll do the same. showFilterValue :: FilterValue -> JSValue showFilterValue (QuotedString str) = showJSON str showFilterValue (NumericValue val) = showJSON val -- | Decoder for 'FilterValue'. We have to see what it contains, since -- the context doesn't give us hints on what to expect. readFilterValue :: JSValue -> Result FilterValue readFilterValue (JSString a) = Ok . QuotedString $ fromJSString a readFilterValue (JSRational _ x) = if denominator x /= 1 then Error $ "Cannot deserialise numeric filter value," ++ " expecting integral but got a fractional value: " ++ show x else Ok . NumericValue $ numerator x readFilterValue v = Error $ "Cannot deserialise filter value, expecting" ++ " string or integer, got " ++ show (pp_value v) instance JSON FilterValue where showJSON = showFilterValue readJSON = readFilterValue -- | Regexp to apply to the filter value, for filtering purposes. It -- holds both the string format, and the \"compiled\" format, so that -- we don't re-compile the regex at each match attempt. data FilterRegex = FilterRegex { stringRegex :: String -- ^ The string version of the regex , compiledRegex :: RegEx.Regex -- ^ The compiled regex } -- | Builder for 'FilterRegex'. We always attempt to compile the -- regular expression on the initialisation of the data structure; -- this might fail, if the RE is not well-formed. mkRegex :: (MonadFail m) => String -> m FilterRegex mkRegex str = FilterRegex str <$> RegEx.makeRegexM str -- | 'Show' instance: we show the constructor plus the string version -- of the regex. instance Show FilterRegex where show (FilterRegex re _) = "mkRegex " ++ show re -- | 'Eq' instance: we only compare the string versions of the regexes. instance Eq FilterRegex where (FilterRegex re1 _) == (FilterRegex re2 _) = re1 == re2 -- | 'Ord' instance: we only compare the string versions of the regexes. instance Ord FilterRegex where (FilterRegex re1 _) `compare` (FilterRegex re2 _) = re1 `compare` re2 -- | 'JSON' instance: like for show and read instances, we work only -- with the string component. instance JSON FilterRegex where showJSON (FilterRegex re _) = showJSON re readJSON s = readJSON s >>= mkRegex -- | Name of a field. type FieldName = String -- | Title of a field, when represented in tabular format. type FieldTitle = String -- | Human redable description of a field. type FieldDoc = String -- | Definition of a field. $(buildObject "FieldDefinition" "fdef" [ simpleField "name" [t| FieldName |] -- FIXME: the name has restrictions , simpleField "title" [t| FieldTitle |] , simpleField "kind" [t| FieldType |] , simpleField "doc" [t| FieldDoc |] ]) --- | Single field entry result. data ResultEntry = ResultEntry { rentryStatus :: ResultStatus -- ^ The result status , rentryValue :: Maybe ResultValue -- ^ The (optional) result value } deriving (Show, Eq) instance NFData ResultEntry where rnf (ResultEntry rs rv) = rnf rs `seq` rnf rv instance JSON ResultEntry where showJSON (ResultEntry rs rv) = showJSON (showJSON rs, maybe JSNull showJSON rv) readJSON v = do (rs, rv) <- readJSON v rv' <- case rv of JSNull -> return Nothing x -> Just <$> readJSON x return $ ResultEntry rs rv' -- | The type of one result row. type ResultRow = [ ResultEntry ] -- | Value of a field, in json encoding. -- (its type will be depending on ResultStatus and FieldType) type ResultValue = JSValue -- * Main Qlang queries and responses. -- | Query2 query. data Query = Query ItemType Fields (Filter FilterField) -- | Query2 result. $(buildObject "QueryResult" "qres" [ simpleField "fields" [t| [ FieldDefinition ] |] , simpleField "data" [t| [ ResultRow ] |] ]) -- | Query2 Fields query. -- (to get supported fields names, descriptions, and types) data QueryFields = QueryFields ItemType Fields -- | Query2 Fields result. $(buildObject "QueryFieldsResult" "qfieldres" [ simpleField "fields" [t| [FieldDefinition ] |] ]) ganeti-3.1.0~rc2/src/Ganeti/Query/Locks.hs000064400000000000000000000071401476477700300203200ustar00rootroot00000000000000{-| Implementation of Ganeti Lock field queries The actual computation of the field values is done by forwarding the request; so only have a minimal field definition here. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Locks ( fieldsMap , RuntimeData ) where import qualified Text.JSON as J import Control.Arrow (first) import Data.Tuple (swap) import Ganeti.Locking.Allocation (OwnerState(..)) import Ganeti.Locking.Locks (ClientId, ciIdentifier) import Ganeti.Query.Common import Ganeti.Query.Language import Ganeti.Query.Types -- | The runtime information for locks. As all information about locks -- is handled by WConfD, the actual information is obtained as live data. -- The type represents the information for a single lock, even though all -- locks are queried simultaneously, ahead of time. type RuntimeData = ( [(ClientId, OwnerState)] -- current state , [(ClientId, OwnerState)] -- pending requests ) -- | Obtain the owners of a lock from the runtime data. getOwners :: RuntimeData -> a -> ResultEntry getOwners (ownerinfo, _) _ = rsNormal . map (J.encode . ciIdentifier . fst) $ ownerinfo -- | Obtain the mode of a lock from the runtime data. getMode :: RuntimeData -> a -> ResultEntry getMode (ownerinfo, _) _ | null ownerinfo = rsNormal J.JSNull | any ((==) OwnExclusive . snd) ownerinfo = rsNormal "exclusive" | otherwise = rsNormal "shared" -- | Obtain the pending requests from the runtime data. getPending :: RuntimeData -> a -> ResultEntry getPending (_, pending) _ = rsNormal . map (swap . first ((:[]) . J.encode . ciIdentifier)) $ pending -- | List of all lock fields. lockFields :: FieldList String RuntimeData lockFields = [ (FieldDefinition "name" "Name" QFTOther "Lock name", FieldSimple rsNormal, QffNormal) , (FieldDefinition "mode" "Mode" QFTOther "Mode in which the lock is\ \ currently acquired\ \ (exclusive or shared)", FieldRuntime getMode, QffNormal) , (FieldDefinition "owner" "Owner" QFTOther "Current lock owner(s)", FieldRuntime getOwners, QffNormal) , (FieldDefinition "pending" "Pending" QFTOther "Jobs waiting for the lock", FieldRuntime getPending, QffNormal) ] -- | The lock fields map. fieldsMap :: FieldMap String RuntimeData fieldsMap = fieldListToFieldMap lockFields ganeti-3.1.0~rc2/src/Ganeti/Query/Network.hs000064400000000000000000000165171476477700300207060ustar00rootroot00000000000000{-| Implementation of the Ganeti Query2 node group queries. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Network ( getGroupConnection , getNetworkUuid , instIsConnected , fieldsMap ) where -- FIXME: everything except fieldsMap -- is only exported for testing. import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Map as Map import Data.Maybe (fromMaybe, mapMaybe) import Data.List (find, intercalate) import Ganeti.JSON (fromContainer) import Ganeti.Network import Ganeti.Objects import qualified Ganeti.Objects.BitArray as BA import Ganeti.Query.Language import Ganeti.Query.Common import Ganeti.Query.Types import Ganeti.Types networkFields :: FieldList Network NoDataRuntime networkFields = [ (FieldDefinition "name" "Network" QFTText "Name", FieldSimple (rsNormal . networkName), QffNormal) , (FieldDefinition "network" "Subnet" QFTText "IPv4 subnet", FieldSimple (rsNormal . networkNetwork), QffNormal) , (FieldDefinition "gateway" "Gateway" QFTOther "IPv4 gateway", FieldSimple (rsMaybeUnavail . networkGateway), QffNormal) , (FieldDefinition "network6" "IPv6Subnet" QFTOther "IPv6 subnet", FieldSimple (rsMaybeUnavail . networkNetwork6), QffNormal) , (FieldDefinition "gateway6" "IPv6Gateway" QFTOther "IPv6 gateway", FieldSimple (rsMaybeUnavail . networkGateway6), QffNormal) , (FieldDefinition "mac_prefix" "MacPrefix" QFTOther "MAC address prefix", FieldSimple (rsMaybeUnavail . networkMacPrefix), QffNormal) , (FieldDefinition "free_count" "FreeCount" QFTNumber "Number of available\ \ addresses", FieldSimple (rsNormal . getFreeCount), QffNormal) , (FieldDefinition "map" "Map" QFTText "Actual mapping", FieldSimple (rsNormal . getMap), QffNormal) , (FieldDefinition "reserved_count" "ReservedCount" QFTNumber "Number of reserved addresses", FieldSimple (rsNormal . getReservedCount), QffNormal) , (FieldDefinition "group_list" "GroupList" QFTOther "List of nodegroups (group name, NIC mode, NIC link)", FieldConfig (\cfg -> rsNormal . getGroupConnections cfg . uuidOf), QffNormal) , (FieldDefinition "group_cnt" "NodeGroups" QFTNumber "Number of nodegroups", FieldConfig (\cfg -> rsNormal . length . getGroupConnections cfg . uuidOf), QffNormal) , (FieldDefinition "inst_list" "InstanceList" QFTOther "List of instances", FieldConfig (\cfg -> rsNormal . getInstances cfg . uuidOf), QffNormal) , (FieldDefinition "inst_cnt" "Instances" QFTNumber "Number of instances", FieldConfig (\cfg -> rsNormal . length . getInstances cfg . uuidOf), QffNormal) , (FieldDefinition "external_reservations" "ExternalReservations" QFTText "External reservations", FieldSimple getExtReservationsString, QffNormal) ] ++ timeStampFields ++ uuidFields "Network" ++ serialFields "Network" ++ tagsFields -- | The group fields map. fieldsMap :: FieldMap Network NoDataRuntime fieldsMap = fieldListToFieldMap networkFields -- TODO: the following fields are not implemented yet: external_reservations -- | Given a network's UUID, this function lists all connections from -- the network to nodegroups including the respective mode and links. getGroupConnections :: ConfigData -> String -> [(String, String, String, String)] getGroupConnections cfg network_uuid = mapMaybe (getGroupConnection network_uuid) ((Map.elems . fromContainer . configNodegroups) cfg) -- | Given a network's UUID and a node group, this function assembles -- a tuple of the group's name, the mode and the link by which the -- network is connected to the group. Returns 'Nothing' if the network -- is not connected to the group. getGroupConnection :: String -> NodeGroup -> Maybe (String, String, String, String) getGroupConnection network_uuid group = let networks = fromContainer . groupNetworks $ group in case Map.lookup (UTF8.fromString network_uuid) networks of Nothing -> Nothing Just net -> Just (groupName group, getNicMode net, getNicLink net, getNicVlan net) -- | Retrieves the network's mode and formats it human-readable, -- also in case it is not available. getNicMode :: PartialNicParams -> String getNicMode nic_params = maybe "-" nICModeToRaw $ nicpModeP nic_params -- | Retrieves the network's vlan and formats it human-readable, also in -- case it it not available. getNicLink :: PartialNicParams -> String getNicLink nic_params = fromMaybe "-" (nicpLinkP nic_params) -- | Retrieves the network's link and formats it human-readable, also in -- case it it not available. getNicVlan :: PartialNicParams -> String getNicVlan nic_params = fromMaybe "-" (nicpVlanP nic_params) -- | Retrieves the network's instances' names. getInstances :: ConfigData -> String -> [String] getInstances cfg network_uuid = mapMaybe instName (filter (instIsConnected network_uuid) ((Map.elems . fromContainer . configInstances) cfg)) -- | Helper function that checks if an instance is linked to the given network. instIsConnected :: String -> Instance -> Bool instIsConnected network_uuid inst = network_uuid `elem` mapMaybe nicNetwork (instNics inst) -- | Helper function to look up a network's UUID by its name getNetworkUuid :: ConfigData -> String -> Maybe String getNetworkUuid cfg name = let net = find (\n -> name == fromNonEmpty (networkName n)) ((Map.elems . fromContainer . configNetworks) cfg) in fmap uuidOf net -- | Computes the reservations list for a network. -- -- This doesn't use the netmask for validation of the length, instead -- simply iterating over the reservations. getReservations :: Ip4Network -> Maybe AddressPool -> [Ip4Address] getReservations _ Nothing = [] getReservations net (Just pool) = map snd . filter fst $ zip (BA.toList . apReservations $ pool) (iterate nextIp4Address $ ip4BaseAddr net) -- | Computes the external reservations as string for a network. getExtReservationsString :: Network -> ResultEntry getExtReservationsString net = let addrs = getReservations (networkNetwork net) (networkExtReservations net) in rsNormal . intercalate ", " $ map show addrs ganeti-3.1.0~rc2/src/Ganeti/Query/Node.hs000064400000000000000000000320161476477700300201320ustar00rootroot00000000000000{-| Implementation of the Ganeti Query2 node queries. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Node ( Runtime , fieldsMap , collectLiveData ) where import Data.List import Data.Maybe import qualified Text.JSON as J import Ganeti.Config import Ganeti.Common import Ganeti.Objects import Ganeti.JSON (jsonHead) import Ganeti.Rpc import Ganeti.Types import Ganeti.Query.Language import Ganeti.Query.Common import Ganeti.Query.Types import Ganeti.Storage.Utils import Ganeti.Utils (niceSort) -- | Runtime is the resulting type for NodeInfo call. type Runtime = Either RpcError RpcResultNodeInfo -- | List of node live fields. nodeLiveFieldsDefs :: [(FieldName, FieldTitle, FieldType, String, FieldDoc)] nodeLiveFieldsDefs = [ ("bootid", "BootID", QFTText, "bootid", "Random UUID renewed for each system reboot, can be used\ \ for detecting reboots by tracking changes") , ("cnodes", "CNodes", QFTNumber, "cpu_nodes", "Number of NUMA domains on node (if exported by hypervisor)") , ("cnos", "CNOs", QFTNumber, "cpu_dom0", "Number of logical processors used by the node OS (dom0 for Xen)") , ("csockets", "CSockets", QFTNumber, "cpu_sockets", "Number of physical CPU sockets (if exported by hypervisor)") , ("ctotal", "CTotal", QFTNumber, "cpu_total", "Number of logical processors") , ("dfree", "DFree", QFTUnit, "storage_free", "Available storage space on storage unit") , ("dtotal", "DTotal", QFTUnit, "storage_size", "Total storage space on storage unit for instance disk allocation") , ("spfree", "SpFree", QFTNumber, "spindles_free", "Available spindles in volume group (exclusive storage only)") , ("sptotal", "SpTotal", QFTNumber, "spindles_total", "Total spindles in volume group (exclusive storage only)") , ("mfree", "MFree", QFTUnit, "memory_free", "Memory available for instance allocations") , ("mnode", "MNode", QFTUnit, "memory_dom0", "Amount of memory used by node (dom0 for Xen)") , ("mtotal", "MTotal", QFTUnit, "memory_total", "Total amount of memory of physical machine") ] -- | Helper function to extract an attribute from a maybe StorageType getAttrFromStorageInfo :: (J.JSON a) => (StorageInfo -> Maybe a) -> Maybe StorageInfo -> J.JSValue getAttrFromStorageInfo attr_fn (Just info) = case attr_fn info of Just val -> J.showJSON val Nothing -> J.JSNull getAttrFromStorageInfo _ Nothing = J.JSNull -- | Check whether the given storage info fits to the given storage type isStorageInfoOfType :: StorageType -> StorageInfo -> Bool isStorageInfoOfType stype sinfo = storageInfoType sinfo == storageTypeToRaw stype -- | Get storage info for the default storage unit getStorageInfoForDefault :: [StorageInfo] -> Maybe StorageInfo getStorageInfoForDefault sinfos = listToMaybe $ filter (not . isStorageInfoOfType StorageLvmPv) sinfos -- | Gets the storage info for a storage type -- FIXME: This needs to be extended when storage pools are implemented, -- because storage types are not necessarily unique then getStorageInfoForType :: [StorageInfo] -> StorageType -> Maybe StorageInfo getStorageInfoForType sinfos stype = listToMaybe $ filter (isStorageInfoOfType stype) sinfos -- | Map each name to a function that extracts that value from -- the RPC result. nodeLiveFieldExtract :: FieldName -> RpcResultNodeInfo -> J.JSValue nodeLiveFieldExtract "bootid" res = J.showJSON $ rpcResNodeInfoBootId res nodeLiveFieldExtract "cnodes" res = jsonHead (rpcResNodeInfoHvInfo res) hvInfoCpuNodes nodeLiveFieldExtract "cnos" res = jsonHead (rpcResNodeInfoHvInfo res) hvInfoCpuDom0 nodeLiveFieldExtract "csockets" res = jsonHead (rpcResNodeInfoHvInfo res) hvInfoCpuSockets nodeLiveFieldExtract "ctotal" res = jsonHead (rpcResNodeInfoHvInfo res) hvInfoCpuTotal nodeLiveFieldExtract "dfree" res = getAttrFromStorageInfo storageInfoStorageFree (getStorageInfoForDefault (rpcResNodeInfoStorageInfo res)) nodeLiveFieldExtract "dtotal" res = getAttrFromStorageInfo storageInfoStorageSize (getStorageInfoForDefault (rpcResNodeInfoStorageInfo res)) nodeLiveFieldExtract "spfree" res = getAttrFromStorageInfo storageInfoStorageFree (getStorageInfoForType (rpcResNodeInfoStorageInfo res) StorageLvmPv) nodeLiveFieldExtract "sptotal" res = getAttrFromStorageInfo storageInfoStorageSize (getStorageInfoForType (rpcResNodeInfoStorageInfo res) StorageLvmPv) nodeLiveFieldExtract "mfree" res = jsonHead (rpcResNodeInfoHvInfo res) hvInfoMemoryFree nodeLiveFieldExtract "mnode" res = jsonHead (rpcResNodeInfoHvInfo res) hvInfoMemoryDom0 nodeLiveFieldExtract "mtotal" res = jsonHead (rpcResNodeInfoHvInfo res) hvInfoMemoryTotal nodeLiveFieldExtract _ _ = J.JSNull -- | Helper for extracting field from RPC result. nodeLiveRpcCall :: FieldName -> Runtime -> Node -> ResultEntry nodeLiveRpcCall fname (Right res) _ = case nodeLiveFieldExtract fname res of J.JSNull -> rsNoData x -> rsNormal x nodeLiveRpcCall _ (Left err) _ = ResultEntry (rpcErrorToStatus err) Nothing -- | Builder for node live fields. nodeLiveFieldBuilder :: (FieldName, FieldTitle, FieldType, String, FieldDoc) -> FieldData Node Runtime nodeLiveFieldBuilder (fname, ftitle, ftype, _, fdoc) = ( FieldDefinition fname ftitle ftype fdoc , FieldRuntime $ nodeLiveRpcCall fname , QffNormal) -- | The docstring for the node role. Note that we use 'reverse' in -- order to keep the same order as Python. nodeRoleDoc :: String nodeRoleDoc = "Node role; " ++ intercalate ", " (map (\nrole -> "\"" ++ nodeRoleToRaw nrole ++ "\" for " ++ roleDescription nrole) (reverse [minBound..maxBound])) -- | Get node powered status. getNodePower :: ConfigData -> Node -> ResultEntry getNodePower cfg node = case getNodeNdParams cfg node of Nothing -> rsNoData Just ndp -> if null (ndpOobProgram ndp) then rsUnavail else rsNormal (nodePowered node) -- | List of all node fields. nodeFields :: FieldList Node Runtime nodeFields = [ (FieldDefinition "drained" "Drained" QFTBool "Whether node is drained", FieldSimple (rsNormal . nodeDrained), QffNormal) , (FieldDefinition "master_candidate" "MasterC" QFTBool "Whether node is a master candidate", FieldSimple (rsNormal . nodeMasterCandidate), QffNormal) , (FieldDefinition "master_capable" "MasterCapable" QFTBool "Whether node can become a master candidate", FieldSimple (rsNormal . nodeMasterCapable), QffNormal) , (FieldDefinition "name" "Node" QFTText "Node name", FieldSimple (rsNormal . nodeName), QffHostname) , (FieldDefinition "offline" "Offline" QFTBool "Whether node is marked offline", FieldSimple (rsNormal . nodeOffline), QffNormal) , (FieldDefinition "vm_capable" "VMCapable" QFTBool "Whether node can host instances", FieldSimple (rsNormal . nodeVmCapable), QffNormal) , (FieldDefinition "pip" "PrimaryIP" QFTText "Primary IP address", FieldSimple (rsNormal . nodePrimaryIp), QffNormal) , (FieldDefinition "sip" "SecondaryIP" QFTText "Secondary IP address", FieldSimple (rsNormal . nodeSecondaryIp), QffNormal) , (FieldDefinition "master" "IsMaster" QFTBool "Whether node is master", FieldConfig (\cfg node -> rsNormal (uuidOf node == clusterMasterNode (configCluster cfg))), QffNormal) , (FieldDefinition "group" "Group" QFTText "Node group", FieldConfig (\cfg node -> rsMaybeNoData (groupName <$> getGroupOfNode cfg node)), QffNormal) , (FieldDefinition "group.uuid" "GroupUUID" QFTText "UUID of node group", FieldSimple (rsNormal . nodeGroup), QffNormal) , (FieldDefinition "ndparams" "NodeParameters" QFTOther "Merged node parameters", FieldConfig ((rsMaybeNoData .) . getNodeNdParams), QffNormal) , (FieldDefinition "custom_ndparams" "CustomNodeParameters" QFTOther "Custom node parameters", FieldSimple (rsNormal . nodeNdparams), QffNormal) -- FIXME: the below could be generalised a bit, like in Python , (FieldDefinition "pinst_cnt" "Pinst" QFTNumber "Number of instances with this node as primary", FieldConfig (\cfg -> rsNormal . getNumInstances fst cfg), QffNormal) , (FieldDefinition "sinst_cnt" "Sinst" QFTNumber "Number of instances with this node as secondary", FieldConfig (\cfg -> rsNormal . getNumInstances snd cfg), QffNormal) , (FieldDefinition "pinst_list" "PriInstances" QFTOther "List of instances with this node as primary", FieldConfig (\cfg -> rsNormal . niceSort . mapMaybe instName . fst . getNodeInstances cfg . uuidOf), QffNormal) , (FieldDefinition "sinst_list" "SecInstances" QFTOther "List of instances with this node as secondary", FieldConfig (\cfg -> rsNormal . niceSort . mapMaybe instName . snd . getNodeInstances cfg . uuidOf), QffNormal) , (FieldDefinition "role" "Role" QFTText nodeRoleDoc, FieldConfig ((rsNormal .) . getNodeRole), QffNormal) , (FieldDefinition "powered" "Powered" QFTBool "Whether node is thought to be powered on", FieldConfig getNodePower, QffNormal) -- FIXME: the two fields below are incomplete in Python, part of the -- non-implemented node resource model; they are declared just for -- parity, but are not functional , (FieldDefinition "hv_state" "HypervisorState" QFTOther "Hypervisor state", FieldSimple (const rsUnavail), QffNormal) , (FieldDefinition "disk_state" "DiskState" QFTOther "Disk state", FieldSimple (const rsUnavail), QffNormal) ] ++ map nodeLiveFieldBuilder nodeLiveFieldsDefs ++ map buildNdParamField allNDParamFields ++ timeStampFields ++ uuidFields "Node" ++ serialFields "Node" ++ tagsFields -- | Helper function to retrieve the number of (primary or secondary) instances getNumInstances :: (([Instance], [Instance]) -> [Instance]) -> ConfigData -> Node -> Int getNumInstances get_fn cfg = length . get_fn . getNodeInstances cfg . uuidOf -- | The node fields map. fieldsMap :: FieldMap Node Runtime fieldsMap = fieldListToFieldMap nodeFields -- | Create an RPC result for a broken node rpcResultNodeBroken :: Node -> (Node, Runtime) rpcResultNodeBroken node = (node, Left (RpcResultError "Broken configuration")) -- | Storage-related query fields storageFields :: [String] storageFields = ["dtotal", "dfree", "spfree", "sptotal"] -- | Hypervisor-related query fields hypervisorFields :: [String] hypervisorFields = ["mnode", "mfree", "mtotal", "cnodes", "csockets", "cnos", "ctotal"] -- | Check if it is required to include domain-specific entities (for example -- storage units for storage info, hypervisor specs for hypervisor info) -- in the node_info call queryDomainRequired :: -- domain-specific fields to look for (storage, hv) [String] -- list of requested fields -> [String] -> Bool queryDomainRequired domain_fields fields = any (`elem` fields) domain_fields -- | Collect live data from RPC query if enabled. collectLiveData :: Bool -> ConfigData -> [String] -> [Node] -> IO [(Node, Runtime)] collectLiveData False _ _ nodes = return $ zip nodes (repeat $ Left (RpcResultError "Live data disabled")) collectLiveData True cfg fields nodes = do let hvs = [getDefaultHypervisorSpec cfg | queryDomainRequired hypervisorFields fields] good_nodes = nodesWithValidConfig cfg nodes storage_units n = if queryDomainRequired storageFields fields then getStorageUnitsOfNode cfg n else [] rpcres <- executeRpcCalls [(n, RpcCallNodeInfo (storage_units n) hvs) | n <- good_nodes] return $ fillUpList (fillPairFromMaybe rpcResultNodeBroken pickPairUnique) nodes rpcres ganeti-3.1.0~rc2/src/Ganeti/Query/Query.hs000064400000000000000000000454541476477700300203640ustar00rootroot00000000000000{-# LANGUAGE TupleSections #-} {-| Implementation of the Ganeti Query2 functionality. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} {- TODO: problems with the current model: 1. There's nothing preventing a result such as ResultEntry RSNormal Nothing, or ResultEntry RSNoData (Just ...); ideally, we would separate the the RSNormal and other types; we would need a new data type for this, though, with JSON encoding/decoding 2. We don't have a way to 'bind' a FieldDefinition's field type (e.q. QFTBool) with the actual value that is returned from a FieldGetter. This means that the various getter functions can return divergent types for the same field when evaluated against multiple items. This is bad; it only works today because we 'hide' everything behind JSValue, but is not nice at all. We should probably remove the separation between FieldDefinition and the FieldGetter, and introduce a new abstract data type, similar to QFT*, that contains the values too. -} module Ganeti.Query.Query ( query , queryFields , queryCompat , getRequestedNames , nameField , NoDataRuntime , uuidField ) where import Control.Arrow ((&&&)) import Control.DeepSeq import Control.Monad (filterM, foldM, liftM, unless) import Control.Monad.IO.Class import Control.Monad.Trans (lift) import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Foldable as Foldable import Data.List (intercalate, nub, find) import Data.Maybe (fromMaybe) import qualified Data.Map as Map import qualified Data.Set as Set import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Config import Ganeti.Errors import Ganeti.JQueue import Ganeti.JSON (Container, GenericContainer(..)) import Ganeti.Locking.Allocation (OwnerState, LockRequest(..), OwnerState(..)) import Ganeti.Locking.Locks (GanetiLocks, ClientId, lockName) import Ganeti.Logging import Ganeti.Objects import Ganeti.Query.Common import qualified Ganeti.Query.Export as Export import qualified Ganeti.Query.FilterRules as FilterRules import Ganeti.Query.Filter import qualified Ganeti.Query.Instance as Instance import qualified Ganeti.Query.Job as Query.Job import qualified Ganeti.Query.Group as Group import Ganeti.Query.Language import qualified Ganeti.Query.Locks as Locks import qualified Ganeti.Query.Network as Network import qualified Ganeti.Query.Node as Node import Ganeti.Query.Types import Ganeti.Path import Ganeti.THH.HsRPC (runRpcClient) import Ganeti.Types import Ganeti.Utils import Ganeti.WConfd.Client (getWConfdClient, listLocksWaitingStatus) -- | Collector type data CollectorType a b = CollectorSimple (Bool -> ConfigData -> [a] -> IO [(a, b)]) | CollectorFieldAware (Bool -> ConfigData -> [String] -> [a] -> IO [(a, b)]) -- * Helper functions -- | Builds an unknown field definition. mkUnknownFDef :: String -> FieldData a b mkUnknownFDef name = ( FieldDefinition name name QFTUnknown ("Unknown field '" ++ name ++ "'") , FieldUnknown , QffNormal ) -- | Runs a field getter on the existing contexts. execGetter :: ConfigData -> b -> a -> FieldGetter a b -> ResultEntry execGetter _ _ item (FieldSimple getter) = getter item execGetter cfg _ item (FieldConfig getter) = getter cfg item execGetter _ rt item (FieldRuntime getter) = getter rt item execGetter cfg rt item (FieldConfigRuntime getter) = getter cfg rt item execGetter _ _ _ FieldUnknown = rsUnknown -- * Main query execution -- | Helper to build the list of requested fields. This transforms the -- list of string fields to a list of field defs and getters, with -- some of them possibly being unknown fields. getSelectedFields :: FieldMap a b -- ^ Defined fields -> [String] -- ^ Requested fields -> FieldList a b -- ^ Selected fields getSelectedFields defined = map (\name -> fromMaybe (mkUnknownFDef name) $ name `Map.lookup` defined) -- | Check whether list of queried fields contains live fields. needsLiveData :: [FieldGetter a b] -> Bool needsLiveData = any isRuntimeField -- | Checks whether we have requested exactly some names. This is a -- simple wrapper over 'requestedNames' and 'nameField'. needsNames :: Query -> Maybe [FilterValue] needsNames (Query kind _ qfilter) = requestedNames (nameField kind) qfilter -- | Computes the name field for different query types. nameField :: ItemType -> FilterField nameField (ItemTypeLuxi QRJob) = "id" nameField (ItemTypeOpCode QRExport) = "node" nameField _ = "name" -- | Computes the uuid field, or the best possible substitute, for different -- query types. uuidField :: ItemType -> FilterField uuidField (ItemTypeLuxi QRJob) = nameField (ItemTypeLuxi QRJob) uuidField (ItemTypeOpCode QRExport) = nameField (ItemTypeOpCode QRExport) uuidField _ = "uuid" -- | Extracts all quoted strings from a list, ignoring the -- 'NumericValue' entries. getAllQuotedStrings :: [FilterValue] -> [String] getAllQuotedStrings = concatMap extractor where extractor (NumericValue _) = [] extractor (QuotedString val) = [val] -- | Checks that we have either requested a valid set of names, or we -- have a more complex filter. getRequestedNames :: Query -> [String] getRequestedNames qry = case needsNames qry of Just names -> getAllQuotedStrings names Nothing -> [] -- | Compute the requested job IDs. This is custom since we need to -- handle both strings and integers. getRequestedJobIDs :: Filter FilterField -> Result [JobId] getRequestedJobIDs qfilter = case requestedNames (nameField (ItemTypeLuxi QRJob)) qfilter of Nothing -> Ok [] Just [] -> Ok [] Just vals -> liftM nub $ mapM (\e -> case e of QuotedString s -> makeJobIdS s NumericValue i -> makeJobId $ fromIntegral i ) vals -- | Generic query implementation for resources that are backed by -- some configuration objects. -- -- Different query types use the same 'genericQuery' function by providing -- a collector function and a field map. The collector function retrieves -- live data, and the field map provides both the requirements and the logic -- necessary to retrieve the data needed for the field. -- -- The 'b' type in the specification is the runtime. Every query can gather -- additional live data related to the configuration object using the collector -- to perform RPC calls. -- -- The gathered data, or the failure to get it, is expressed through a runtime -- object. The type of a runtime object is determined by every query type for -- itself, and used exclusively by that query. genericQuery :: FieldMap a b -- ^ Maps field names to field definitions -> CollectorType a b -- ^ Collector of live data -> (a -> String) -- ^ Object to name function -> (ConfigData -> Container a) -- ^ Get all objects from config -> (ConfigData -> String -> ErrorResult a) -- ^ Lookup object -> ConfigData -- ^ The config to run the query against -> Bool -- ^ Whether the query should be run live -> [String] -- ^ List of requested fields -> Filter FilterField -- ^ Filter field -> [String] -- ^ List of requested names -> IO (ErrorResult QueryResult) genericQuery fieldsMap collector nameFn configFn getFn cfg live fields qfilter wanted = runResultT $ do cfilter <- toError $ compileFilter fieldsMap qfilter let allfields = (++) fields . filter (not . (`elem` fields)) . ordNub $ filterArguments qfilter count = length fields selected = getSelectedFields fieldsMap allfields (fdefs, fgetters, _) = unzip3 selected live' = live && needsLiveData fgetters objects <- toError $ case wanted of [] -> Ok . niceSortKey nameFn . Foldable.toList $ configFn cfg _ -> mapM (getFn cfg) wanted -- Run the first pass of the filter, without a runtime context; this will -- limit the objects that we'll contact for exports fobjects <- toError $ filterM (\n -> evaluateQueryFilter cfg Nothing n cfilter) objects -- Gather the runtime data and filter the results again, -- based on the gathered data runtimes <- (case collector of CollectorSimple collFn -> lift $ collFn live' cfg fobjects CollectorFieldAware collFn -> lift $ collFn live' cfg allfields fobjects) >>= (toError . filterM (\(obj, runtime) -> evaluateQueryFilter cfg (Just runtime) obj cfilter)) let fdata = map (\(obj, runtime) -> map (execGetter cfg runtime obj) fgetters) runtimes return QueryResult { qresFields = take count fdefs , qresData = map (take count) fdata } -- | Dummy recollection of the data for a lock from the prefected -- data for all locks. recollectLocksData :: ( [(GanetiLocks, [(ClientId, OwnerState)])] , [(Integer, ClientId, [LockRequest GanetiLocks])] ) -> Bool -> ConfigData -> [String] -> IO [(String, Locks.RuntimeData)] recollectLocksData (allLocks, pending) _ _ = let getPending lock = pending >>= \(_, cid, req) -> let req' = filter ((==) lock . lockName . lockAffected) req in case () of _ | any ((==) (Just OwnExclusive) . lockRequestType) req' -> [(cid, OwnExclusive)] _ | any ((==) (Just OwnShared) . lockRequestType) req' -> [(cid, OwnShared)] _ -> [] lookuplock lock = (,) lock . maybe ([], getPending lock) (\(_, c) -> (c, getPending lock)) . find ((==) lock . lockName . fst) $ allLocks in return . map lookuplock -- | Main query execution function. query :: ConfigData -- ^ The current configuration -> Bool -- ^ Whether to collect live data -> Query -- ^ The query (item, fields, filter) -> IO (ErrorResult QueryResult) -- ^ Result query cfg live (Query (ItemTypeLuxi QRJob) fields qfilter) = queryJobs cfg live fields qfilter query cfg live (Query (ItemTypeLuxi QRLock) fields qfilter) = runResultT $ do unless live (failError "Locks can only be queried live") cl <- liftIO $ do socketpath <- defaultWConfdSocket getWConfdClient socketpath livedata <- runRpcClient listLocksWaitingStatus cl logDebug $ "Live state of all locks is " ++ show livedata let allLocks = Set.toList . Set.unions $ (Set.fromList . map fst $ fst livedata) : map (\(_, _, req) -> Set.fromList $ map lockAffected req) (snd livedata) answer <- liftIO $ genericQuery Locks.fieldsMap (CollectorSimple $ recollectLocksData livedata) id (const . GenericContainer . Map.fromList . map ((UTF8.fromString &&& id) . lockName) $ allLocks) (const Ok) cfg live fields qfilter [] toError answer query cfg live qry = queryInner cfg live qry $ getRequestedNames qry -- | Dummy data collection fuction dummyCollectLiveData :: Bool -> ConfigData -> [a] -> IO [(a, NoDataRuntime)] dummyCollectLiveData _ _ = return . map (, NoDataRuntime) -- | Inner query execution function. queryInner :: ConfigData -- ^ The current configuration -> Bool -- ^ Whether to collect live data -> Query -- ^ The query (item, fields, filter) -> [String] -- ^ Requested names -> IO (ErrorResult QueryResult) -- ^ Result queryInner cfg live (Query (ItemTypeOpCode QRNode) fields qfilter) wanted = genericQuery Node.fieldsMap (CollectorFieldAware Node.collectLiveData) nodeName configNodes getNode cfg live fields qfilter wanted queryInner cfg live (Query (ItemTypeOpCode QRInstance) fields qfilter) wanted = genericQuery Instance.fieldsMap (CollectorFieldAware Instance.collectLiveData) (fromMaybe "" . instName) configInstances getInstance cfg live fields qfilter wanted queryInner cfg live (Query (ItemTypeOpCode QRGroup) fields qfilter) wanted = genericQuery Group.fieldsMap (CollectorSimple dummyCollectLiveData) groupName configNodegroups getGroup cfg live fields qfilter wanted queryInner cfg live (Query (ItemTypeOpCode QRNetwork) fields qfilter) wanted = genericQuery Network.fieldsMap (CollectorSimple dummyCollectLiveData) (fromNonEmpty . networkName) configNetworks getNetwork cfg live fields qfilter wanted queryInner cfg live (Query (ItemTypeOpCode QRExport) fields qfilter) wanted = genericQuery Export.fieldsMap (CollectorSimple Export.collectLiveData) nodeName configNodes getNode cfg live fields qfilter wanted queryInner cfg live (Query (ItemTypeLuxi QRFilter) fields qfilter) wanted = genericQuery FilterRules.fieldsMap (CollectorSimple dummyCollectLiveData) uuidOf configFilters getFilterRule cfg live fields qfilter wanted queryInner _ _ (Query qkind _ _) _ = return . Bad . GenericError $ "Query '" ++ show qkind ++ "' not supported" -- | Query jobs specific query function, needed as we need to accept -- both 'QuotedString' and 'NumericValue' as wanted names. queryJobs :: ConfigData -- ^ The current configuration -> Bool -- ^ Whether to collect live data -> [FilterField] -- ^ Item -> Filter FilterField -- ^ Filter -> IO (ErrorResult QueryResult) -- ^ Result queryJobs cfg live fields qfilter = runResultT $ do rootdir <- lift queueDir wanted_names <- toErrorStr $ getRequestedJobIDs qfilter rjids <- case wanted_names of [] | live -> do -- we can check the filesystem for actual jobs let want_arch = Query.Job.wantArchived fields jobIDs <- withErrorT (BlockDeviceError . (++) "Unable to fetch the job list: " . show) $ liftIO (determineJobDirectories rootdir want_arch) >>= ResultT . getJobIDs return $ sortJobIDs jobIDs -- else we shouldn't look at the filesystem... v -> return v cfilter <- toError $ compileFilter Query.Job.fieldsMap qfilter let selected = getSelectedFields Query.Job.fieldsMap fields (fdefs, fgetters, _) = unzip3 selected (_, filtergetters, _) = unzip3 . getSelectedFields Query.Job.fieldsMap $ Foldable.toList qfilter live' = live && needsLiveData (fgetters ++ filtergetters) disabled_data = Bad "live data disabled" -- runs first pass of the filter, without a runtime context; this -- will limit the jobs that we'll load from disk jids <- toError $ filterM (\jid -> evaluateQueryFilter cfg Nothing jid cfilter) rjids -- here we run the runtime data gathering, filtering and evaluation, -- all in the same step, so that we don't keep jobs in memory longer -- than we need; we can't be fully lazy due to the multiple monad -- wrapping across different steps qdir <- lift queueDir fdata <- foldM -- big lambda, but we use many variables from outside it... (\lst jid -> do job <- lift $ if live' then loadJobFromDisk qdir True jid else return disabled_data pass <- toError $ evaluateQueryFilter cfg (Just job) jid cfilter let nlst = if pass then let row = map (execGetter cfg job jid) fgetters in rnf row `seq` row:lst else lst -- evaluate nlst (to WHNF), otherwise we're too lazy return $! nlst ) [] jids return QueryResult { qresFields = fdefs, qresData = reverse fdata } -- | Helper for 'queryFields'. fieldsExtractor :: FieldMap a b -> [FilterField] -> QueryFieldsResult fieldsExtractor fieldsMap fields = let selected = if null fields then map snd . niceSortKey fst $ Map.toList fieldsMap else getSelectedFields fieldsMap fields in QueryFieldsResult (map (\(defs, _, _) -> defs) selected) -- | Query fields call. queryFields :: QueryFields -> ErrorResult QueryFieldsResult queryFields (QueryFields (ItemTypeOpCode QRNode) fields) = Ok $ fieldsExtractor Node.fieldsMap fields queryFields (QueryFields (ItemTypeOpCode QRGroup) fields) = Ok $ fieldsExtractor Group.fieldsMap fields queryFields (QueryFields (ItemTypeOpCode QRNetwork) fields) = Ok $ fieldsExtractor Network.fieldsMap fields queryFields (QueryFields (ItemTypeLuxi QRJob) fields) = Ok $ fieldsExtractor Query.Job.fieldsMap fields queryFields (QueryFields (ItemTypeOpCode QRExport) fields) = Ok $ fieldsExtractor Export.fieldsMap fields queryFields (QueryFields (ItemTypeOpCode QRInstance) fields) = Ok $ fieldsExtractor Instance.fieldsMap fields queryFields (QueryFields (ItemTypeLuxi QRLock) fields) = Ok $ fieldsExtractor Locks.fieldsMap fields queryFields (QueryFields (ItemTypeLuxi QRFilter) fields) = Ok $ fieldsExtractor FilterRules.fieldsMap fields queryFields (QueryFields qkind _) = Bad . GenericError $ "QueryFields '" ++ show qkind ++ "' not supported" -- | Classic query converter. It gets a standard query result on input -- and computes the classic style results. queryCompat :: QueryResult -> ErrorResult [[J.JSValue]] queryCompat (QueryResult fields qrdata) = case map fdefName $ filter ((== QFTUnknown) . fdefKind) fields of [] -> Ok $ map (map (maybe J.JSNull J.showJSON . rentryValue)) qrdata unknown -> Bad $ OpPrereqError ("Unknown output fields selected: " ++ intercalate ", " unknown) ECodeInval ganeti-3.1.0~rc2/src/Ganeti/Query/Server.hs000064400000000000000000000744001476477700300205160ustar00rootroot00000000000000{-# LANGUAGE FlexibleContexts #-} {-| Implementation of the Ganeti Query2 server. -} {- Copyright (C) 2012, 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Server ( main , checkMain , prepMain ) where import Control.Concurrent import Control.Exception import Control.Lens ((.~)) import Control.Monad (forever, when, mzero, guard, zipWithM, liftM, void) import Control.Monad.Base (MonadBase, liftBase) import Control.Monad.Except (MonadError) import Control.Monad.IO.Class import Control.Monad.Trans (lift) import Control.Monad.Trans.Maybe import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Set as Set (toList) import Data.Bits (finiteBitSize) import Data.IORef import Data.List (intersperse) import Data.Maybe (fromMaybe) import qualified Text.JSON as J import Text.JSON (encode, showJSON, JSValue(..)) import System.Info (arch) import System.Directory import System.Posix.Process (getProcessID) import System.Posix.Signals as P import qualified Ganeti.Constants as C import qualified Ganeti.ConstantUtils as ConstantUtils (unFrozenSet) import Ganeti.Errors import qualified Ganeti.Path as Path import Ganeti.Daemon import Ganeti.Daemon.Utils (handleMasterVerificationOptions) import Ganeti.Objects import Ganeti.Objects.Lens (configFiltersL) import qualified Ganeti.Config as Config import Ganeti.ConfigReader import Ganeti.BasicTypes import Ganeti.JQueue import Ganeti.JQScheduler import Ganeti.JSON (TimeAsDoubleJSON(..), alterContainerL, lookupContainer) import Ganeti.Locking.Locks (ClientId(..), ClientType(ClientOther)) import Ganeti.Logging import Ganeti.Luxi import qualified Ganeti.Query.Language as Qlang import qualified Ganeti.Query.Cluster as QCluster import Ganeti.Path ( queueDir, jobQueueLockFile, jobQueueDrainFile ) import Ganeti.Rpc import Ganeti.Query.Query import Ganeti.Query.Filter (makeSimpleFilter) import Ganeti.THH.HsRPC (runRpcClient, RpcClientMonad) import Ganeti.Types import qualified Ganeti.UDSServer as U (Handler(..), listener) import Ganeti.Utils ( lockFile, exitIfBad, watchFile , safeRenameFile, newUUID, isUUID ) import Ganeti.Utils.Monad (orM) import Ganeti.Utils.MVarLock import qualified Ganeti.Version as Version import Ganeti.WConfd.Client ( getWConfdClient, withLockedConfig, writeConfig , cleanupLocks) -- | Creates a `ClientId` that identifies the current luxi -- (process, thread). -- -- This means that this `ClientId` will be different for each request -- handled by luxid. makeLuxidClientId :: JQStatus -> IO ClientId makeLuxidClientId status = do pid <- getProcessID tid <- myThreadId return ClientId { ciIdentifier = ClientOther $ "luxid-" ++ show tid , ciLockFile = jqLivelock status , ciPid = pid } -- | Creates a connection to WConfd and locks the config, allowing -- to run some WConfd RPC commands given the locked config. -- -- This is needed when luxid wants to change the config. -- -- Example: -- -- > cid <- makeLuxidClientId ... -- > withLockedWconfdConfig cid $ \lockedCfg -> do -- > -- some (IO) action that needs to be run inside having the lock -- > writeConfig cid (updateConfig lockedCfg) withLockedWconfdConfig :: (MonadBase IO m, MonadError GanetiException m) => ClientId -> (ConfigData -> RpcClientMonad a) -> m a withLockedWconfdConfig cid f = do wconfdClient <- liftBase $ getWConfdClient =<< Path.defaultWConfdSocket runRpcClient (withLockedConfig cid False f) wconfdClient -- | Helper for classic queries. handleQuery :: [Qlang.ItemType -> Qlang.FilterField] -- ^ Fields to put into -- the query -> ConfigData -- ^ Cluster config -> Qlang.ItemType -- ^ Query type -> [Either String Integer] -- ^ Requested names -- (empty means all) -> [String] -- ^ Requested fields -> Bool -- ^ Whether to do sync queries or not -> IO (GenericResult GanetiException JSValue) handleQuery _ _ _ _ _ True = return . Bad $ OpPrereqError "Sync queries are not allowed" ECodeInval handleQuery filterFields cfg qkind names fields _ = do let simpleNameFilter field = makeSimpleFilter (field qkind) names flt = Qlang.OrFilter $ map simpleNameFilter filterFields qr <- query cfg True (Qlang.Query qkind fields flt) return $ showJSON <$> (qr >>= queryCompat) -- | Helper for classic queries. -- Queries `name` and `uuid` fields. handleClassicQuery :: ConfigData -- ^ Cluster config -> Qlang.ItemType -- ^ Query type -> [Either String Integer] -- ^ Requested names -- (empty means all) -> [String] -- ^ Requested fields -> Bool -- ^ Whether to do sync queries or not -> IO (GenericResult GanetiException JSValue) handleClassicQuery = handleQuery [nameField, uuidField] -- | Like `handleClassicQuery`, but filters only by UUID. handleUuidQuery :: ConfigData -- ^ Cluster config -> Qlang.ItemType -- ^ Query type -> [Either String Integer] -- ^ Requested names -- (empty means all) -> [String] -- ^ Requested fields -> Bool -- ^ Whether to do sync queries or not -> IO (GenericResult GanetiException JSValue) handleUuidQuery = handleQuery [uuidField] -- | Minimal wrapper to handle the missing config case. handleCallWrapper :: Lock -> JQStatus -> Result ConfigData -> LuxiOp -> IO (ErrorResult JSValue) handleCallWrapper _ _ (Bad msg) _ = return . Bad . ConfigurationError $ "I do not have access to a valid configuration, cannot\ \ process queries: " ++ msg handleCallWrapper qlock qstat (Ok config) op = handleCall qlock qstat config op -- | Actual luxi operation handler. handleCall :: Lock -> JQStatus -> ConfigData -> LuxiOp -> IO (ErrorResult JSValue) handleCall _ _ cdata QueryClusterInfo = let cluster = configCluster cdata master = QCluster.clusterMasterNodeName cdata hypervisors = clusterEnabledHypervisors cluster diskTemplates = clusterEnabledDiskTemplates cluster def_hv = case hypervisors of x:_ -> showJSON x [] -> JSNull bits = show (finiteBitSize (0::Int)) ++ "bits" arch_tuple = [bits, arch] obj = [ ("software_version", showJSON C.releaseVersion) , ("protocol_version", showJSON C.protocolVersion) , ("config_version", showJSON C.configVersion) , ("os_api_version", showJSON . maximum . Set.toList . ConstantUtils.unFrozenSet $ C.osApiVersions) , ("export_version", showJSON C.exportVersion) , ("vcs_version", showJSON Version.version) , ("architecture", showJSON arch_tuple) , ("name", showJSON $ clusterClusterName cluster) , ("master", showJSON (case master of Ok name -> name _ -> undefined)) , ("default_hypervisor", def_hv) , ("enabled_hypervisors", showJSON hypervisors) , ("hvparams", showJSON $ clusterHvparams cluster) , ("os_hvp", showJSON $ clusterOsHvp cluster) , ("beparams", showJSON $ clusterBeparams cluster) , ("osparams", showJSON $ clusterOsparams cluster) , ("ipolicy", showJSON $ clusterIpolicy cluster) , ("nicparams", showJSON $ clusterNicparams cluster) , ("ndparams", showJSON $ clusterNdparams cluster) , ("diskparams", showJSON $ clusterDiskparams cluster) , ("candidate_pool_size", showJSON $ clusterCandidatePoolSize cluster) , ("max_running_jobs", showJSON $ clusterMaxRunningJobs cluster) , ("max_tracked_jobs", showJSON $ clusterMaxTrackedJobs cluster) , ("mac_prefix", showJSON $ clusterMacPrefix cluster) , ("master_netdev", showJSON $ clusterMasterNetdev cluster) , ("master_netmask", showJSON $ clusterMasterNetmask cluster) , ("use_external_mip_script", showJSON $ clusterUseExternalMipScript cluster) , ("volume_group_name", maybe JSNull showJSON (clusterVolumeGroupName cluster)) , ("drbd_usermode_helper", maybe JSNull showJSON (clusterDrbdUsermodeHelper cluster)) , ("file_storage_dir", showJSON $ clusterFileStorageDir cluster) , ("shared_file_storage_dir", showJSON $ clusterSharedFileStorageDir cluster) , ("gluster_storage_dir", showJSON $ clusterGlusterStorageDir cluster) , ("maintain_node_health", showJSON $ clusterMaintainNodeHealth cluster) , ("ctime", showJSON . TimeAsDoubleJSON $ clusterCtime cluster) , ("mtime", showJSON . TimeAsDoubleJSON $ clusterMtime cluster) , ("uuid", showJSON $ clusterUuid cluster) , ("tags", showJSON $ clusterTags cluster) , ("uid_pool", showJSON $ clusterUidPool cluster) , ("default_iallocator", showJSON $ clusterDefaultIallocator cluster) , ("default_iallocator_params", showJSON $ clusterDefaultIallocatorParams cluster) , ("reserved_lvs", showJSON $ clusterReservedLvs cluster) , ("primary_ip_version", showJSON . ipFamilyToVersion $ clusterPrimaryIpFamily cluster) , ("prealloc_wipe_disks", showJSON $ clusterPreallocWipeDisks cluster) , ("hidden_os", showJSON $ clusterHiddenOs cluster) , ("blacklisted_os", showJSON $ clusterBlacklistedOs cluster) , ("enabled_disk_templates", showJSON diskTemplates) , ("install_image", showJSON $ clusterInstallImage cluster) , ("instance_communication_network", showJSON (clusterInstanceCommunicationNetwork cluster)) , ("zeroing_image", showJSON $ clusterZeroingImage cluster) , ("compression_tools", showJSON $ clusterCompressionTools cluster) , ("enabled_user_shutdown", showJSON $ clusterEnabledUserShutdown cluster) , ("enabled_data_collectors", showJSON . fmap dataCollectorActive $ clusterDataCollectors cluster) , ("data_collector_interval", showJSON . fmap dataCollectorInterval $ clusterDataCollectors cluster) , ("modify_ssh_setup", showJSON $ clusterModifySshSetup cluster) , ("ssh_key_type", showJSON $ clusterSshKeyType cluster) , ("ssh_key_bits", showJSON $ clusterSshKeyBits cluster) ] in case master of Ok _ -> return . Ok . J.makeObj $ obj Bad ex -> return $ Bad ex handleCall _ _ cfg (QueryTags kind name) = do let tags = case kind of TagKindCluster -> Ok . clusterTags $ configCluster cfg TagKindGroup -> groupTags <$> Config.getGroup cfg name TagKindNode -> nodeTags <$> Config.getNode cfg name TagKindInstance -> instTags <$> Config.getInstance cfg name TagKindNetwork -> networkTags <$> Config.getNetwork cfg name return (J.showJSON <$> tags) handleCall _ _ cfg (Query qkind qfields qfilter) = do result <- query cfg True (Qlang.Query qkind qfields qfilter) return $ J.showJSON <$> result handleCall _ _ _ (QueryFields qkind qfields) = do let result = queryFields (Qlang.QueryFields qkind qfields) return $ J.showJSON <$> result handleCall _ _ cfg (QueryNodes names fields lock) = handleClassicQuery cfg (Qlang.ItemTypeOpCode Qlang.QRNode) (map Left names) fields lock handleCall _ _ cfg (QueryInstances names fields lock) = handleClassicQuery cfg (Qlang.ItemTypeOpCode Qlang.QRInstance) (map Left names) fields lock handleCall _ _ cfg (QueryGroups names fields lock) = handleClassicQuery cfg (Qlang.ItemTypeOpCode Qlang.QRGroup) (map Left names) fields lock handleCall _ _ cfg (QueryJobs names fields) = handleClassicQuery cfg (Qlang.ItemTypeLuxi Qlang.QRJob) (map (Right . fromIntegral . fromJobId) names) fields False handleCall _ _ cfg (QueryFilters uuids fields) = handleUuidQuery cfg (Qlang.ItemTypeLuxi Qlang.QRFilter) (map Left uuids) fields False handleCall _ status _ (ReplaceFilter mUuid priority predicates action reason) = -- Handles both adding new filter and changing existing ones. runResultT $ do -- Check that uuid `String` is actually a UUID. uuid <- case mUuid of Nothing -> liftIO newUUID -- Request to add a new filter Just u -- Request to edit an existing filter | isUUID u -> return u | otherwise -> fail "Unable to parse UUID" timestamp <- liftIO $ reasonTrailTimestamp <$> currentTimestamp let luxidReason = ("luxid", "", timestamp) -- Ask WConfd to change the config for us. cid <- liftIO $ makeLuxidClientId status withLockedWconfdConfig cid $ \lockedCfg -> do -- Reading the latest JobID inside the Wconfd lock to really get the -- most recent one (locking may block us for some time). serial <- liftIO readSerialFromDisk case serial of Bad err -> fail $ "AddFilter: reading current JobId failed: " ++ err Ok watermark -> do let rule = FilterRule { frWatermark = watermark , frPriority = priority , frPredicates = predicates , frAction = action , frReasonTrail = reason ++ [luxidReason] , frUuid = UTF8.fromString uuid } writeConfig cid . (configFiltersL . alterContainerL (UTF8.fromString uuid) .~ Just rule) $ lockedCfg -- Return UUID of added/replaced filter. return $ showJSON uuid handleCall _ status cfg (DeleteFilter uuid) = runResultT $ do -- Check if filter exists. _ <- lookupContainer (failError $ "Filter rule with UUID " ++ uuid ++ " does not exist") (UTF8.fromString uuid) (configFilters cfg) -- Ask WConfd to change the config for us. cid <- liftIO $ makeLuxidClientId status withLockedWconfdConfig cid $ \lockedCfg -> writeConfig cid . (configFiltersL . alterContainerL (UTF8.fromString uuid) .~ Nothing) $ lockedCfg return JSNull handleCall _ _ cfg (QueryNetworks names fields lock) = handleClassicQuery cfg (Qlang.ItemTypeOpCode Qlang.QRNetwork) (map Left names) fields lock handleCall _ _ cfg (QueryConfigValues fields) = do let clusterProperty fn = showJSON . fn . configCluster $ cfg let params = [ ("cluster_name", return $ clusterProperty clusterClusterName) , ("watcher_pause", liftM (maybe JSNull showJSON) QCluster.isWatcherPaused) , ("master_node", return . genericResult (const JSNull) showJSON $ QCluster.clusterMasterNodeName cfg) , ("drain_flag", liftM (showJSON . not) isQueueOpen) , ("modify_ssh_setup", return $ clusterProperty clusterModifySshSetup) , ("ssh_key_type", return $ clusterProperty clusterSshKeyType) , ("ssh_key_bits", return $ clusterProperty clusterSshKeyBits) ] :: [(String, IO JSValue)] let answer = map (fromMaybe (return JSNull) . flip lookup params) fields answerEval <- sequence answer return . Ok . showJSON $ answerEval handleCall _ _ cfg (QueryExports nodes lock) = handleClassicQuery cfg (Qlang.ItemTypeOpCode Qlang.QRExport) (map Left nodes) ["node", "export"] lock handleCall qlock qstat cfg (SubmitJobToDrainedQueue ops) = runResultT $ do jid <- mkResultT $ allocateJobId (Config.getMasterCandidates cfg) qlock ts <- liftIO currentTimestamp job <- liftM (extendJobReasonTrail . setReceivedTimestamp ts) $ queuedJobFromOpCodes jid ops qDir <- liftIO queueDir _ <- writeAndReplicateJob cfg qDir job _ <- liftIO . forkIO $ enqueueNewJobs qstat [job] return . showJSON . fromJobId $ jid handleCall qlock qstat cfg (SubmitJob ops) = do open <- isQueueOpen if not open then return . Bad . GenericError $ "Queue drained" else handleCall qlock qstat cfg (SubmitJobToDrainedQueue ops) handleCall qlock qstat cfg (SubmitManyJobs lops) = do open <- isQueueOpen if not open then return . Bad . GenericError $ "Queue drained" else do let mcs = Config.getMasterCandidates cfg result_jobids <- allocateJobIds mcs qlock (length lops) case result_jobids of Bad s -> return . Bad . GenericError $ s Ok jids -> do ts <- currentTimestamp jobs <- liftM (map $ extendJobReasonTrail . setReceivedTimestamp ts) $ zipWithM queuedJobFromOpCodes jids lops qDir <- queueDir write_results <- mapM (writeJobToDisk qDir) jobs let annotated_results = zip write_results jobs succeeded = map snd $ filter (isOk . fst) annotated_results when (any isBad write_results) . logWarning $ "Writing some jobs failed " ++ show annotated_results replicateManyJobs qDir mcs succeeded _ <- forkIO $ enqueueNewJobs qstat succeeded return . Ok . JSArray . map (\(res, job) -> if isOk res then showJSON (True, fromJobId $ qjId job) else showJSON (False, genericResult id (const "") res)) $ annotated_results handleCall _ _ cfg (WaitForJobChange jid fields prev_job prev_log tmout) = waitForJobChange jid prev_job tmout $ computeJobUpdate cfg jid fields prev_log handleCall _ _ cfg (SetWatcherPause time) = do let mcs = Config.getMasterOrCandidates cfg _ <- executeRpcCall mcs $ RpcCallSetWatcherPause time return . Ok . maybe JSNull showJSON $ fmap TimeAsDoubleJSON time handleCall _ _ cfg (SetDrainFlag value) = do let mcs = Config.getMasterCandidates cfg fpath <- jobQueueDrainFile if value then writeFile fpath "" else removeFile fpath _ <- executeRpcCall mcs $ RpcCallSetDrainFlag value return . Ok . showJSON $ True handleCall _ qstat cfg (ChangeJobPriority jid prio) = do let jName = (++) "job " . show $ fromJobId jid maybeJob <- setJobPriority qstat jid prio case maybeJob of Bad s -> return . Ok $ showJSON (False, s) Ok (Just job) -> runResultT $ do let mcs = Config.getMasterCandidates cfg qDir <- liftIO queueDir liftIO $ replicateManyJobs qDir mcs [job] return $ showJSON (True, "Priorities of pending opcodes for " ++ jName ++ " have been changed" ++ " to " ++ show prio) Ok Nothing -> do logDebug $ jName ++ " started, will signal" fmap showJSON <$> tellJobPriority (jqLivelock qstat) jid prio handleCall _ qstat cfg (CancelJob jid kill) = do let jName = (++) "job " . show $ fromJobId jid dequeueResult <- dequeueJob qstat jid case dequeueResult of Ok True -> let jobFileFailed = (,) False . (++) ("Dequeued " ++ jName ++ ", but failed to mark as cancelled: ") jobFileSucceeded _ = (True, "Dequeued " ++ jName) in liftM (Ok . showJSON . genericResult jobFileFailed jobFileSucceeded) . runResultT $ do logDebug $ jName ++ " dequeued, marking as canceled" qDir <- liftIO queueDir (job, _) <- ResultT $ loadJobFromDisk qDir True jid now <- liftIO currentTimestamp let job' = cancelQueuedJob now job writeAndReplicateJob cfg qDir job' Ok False -> do logDebug $ jName ++ " not queued; trying to cancel directly" result <- fmap showJSON <$> cancelJob kill (jqLivelock qstat) jid when kill . void . forkIO $ do _ <- orM . intersperse (threadDelay C.luxidJobDeathDelay >> return False) . replicate C.luxidJobDeathDetectionRetries $ cleanupIfDead qstat jid wconfdsocket <- Path.defaultWConfdSocket wconfdclient <- getWConfdClient wconfdsocket void . runResultT $ runRpcClient cleanupLocks wconfdclient return result Bad s -> return . Ok . showJSON $ (False, s) handleCall qlock qstat cfg (ArchiveJob jid) = -- By adding a layer of MaybeT, we can prematurely end a computation -- using 'mzero' or other 'MonadPlus' primitive and return 'Ok False'. runResultT . liftM (showJSON . fromMaybe False) . runMaybeT $ do qDir <- liftIO queueDir let mcs = Config.getMasterCandidates cfg live = liveJobFile qDir jid archive = archivedJobFile qDir jid -- We want to wait for the Job process to actually finish executing to -- make sure its file has been properly replicated to all master -- candidates (GH #1266). We wait up to 2 * (RPC connect timeout) + 1 -- second to allow for up to two master candidates to be unreachable, -- without the archival failing. For more than that we let the job -- archival fail; it will eventually be autoarchived by the watcher. exitTimeout = (2 * C.rpcConnectTimeout + 1) * 1000 -- milliseconds (job, _) <- (lift . mkResultT $ loadJobFromDisk qDir False jid) `orElse` mzero guard $ jobFinalized job _ <- lift $ waitUntilJobExited (jqLivelock qstat) job exitTimeout withLock qlock $ do lift . withErrorT JobQueueError . annotateError "Archiving failed in an unexpected way" . mkResultT $ safeRenameFile queueDirPermissions live archive _ <- liftIO . executeRpcCall mcs $ RpcCallJobqueueRename [(live, archive)] return True handleCall qlock _ cfg (AutoArchiveJobs age timeout) = do qDir <- queueDir resultJids <- getJobIDs [qDir] case resultJids of Bad s -> return . Bad . JobQueueError $ show s Ok jids -> do result <- withLock qlock . archiveJobs cfg age timeout $ sortJobIDs jids return . Ok $ showJSON result handleCall _ _ _ (PickupJob _) = return . Bad $ GenericError "Luxi call 'PickupJob' is for internal use only" {-# ANN handleCall "HLint: ignore Too strict if" #-} -- | Special-case handler for WaitForJobChange RPC call for fields == ["status"] -- that doesn't require the use of ConfigData handleWaitForJobChangeStatus :: JobId -> JSValue -> JSValue -> Int -> IO (ErrorResult JSValue) handleWaitForJobChangeStatus jid prev_job prev_log tmout = waitForJobChange jid prev_job tmout $ computeJobUpdateStatus jid prev_log -- | Common WaitForJobChange functionality shared between handleCall and -- handleWaitForJobChangeStatus waitForJobChange :: JobId -> JSValue -> Int -> IO (JSValue, JSValue) -> IO (ErrorResult JSValue) waitForJobChange jid prev_job tmout compute_fn = do qDir <- queueDir -- verify if the job is finalized, and return immediately in this case jobresult <- loadJobFromDisk qDir False jid case jobresult of Bad s -> return . Bad $ JobLost s Ok (job, _) | not (jobFinalized job) -> do let jobfile = liveJobFile qDir jid answer <- watchFile jobfile (min tmout C.luxiWfjcTimeout) (prev_job, JSArray []) compute_fn return . Ok $ showJSON answer _ -> liftM (Ok . showJSON) compute_fn -- | Query the status of a job and return the requested fields -- and the logs newer than the given log number. computeJobUpdate :: ConfigData -> JobId -> [String] -> JSValue -> IO (JSValue, JSValue) computeJobUpdate cfg jid fields prev_log = do let sjid = show $ fromJobId jid logDebug $ "Inspecting fields " ++ show fields ++ " of job " ++ sjid let fromJSArray (JSArray xs) = xs fromJSArray _ = [] let logFilter JSNull (JSArray _) = True logFilter (JSRational _ n) (JSArray (JSRational _ m:_)) = n < m logFilter _ _ = False let filterLogs n logs = JSArray (filter (logFilter n) (logs >>= fromJSArray)) jobQuery <- handleClassicQuery cfg (Qlang.ItemTypeLuxi Qlang.QRJob) [Right . fromIntegral $ fromJobId jid] ("oplog" : fields) False let (rfields, rlogs) = case jobQuery of Ok (JSArray [JSArray (JSArray logs : answer)]) -> (answer, filterLogs prev_log logs) _ -> (map (const JSNull) fields, JSArray []) logDebug $ "Updates for job " ++ sjid ++ " are " ++ encode (rfields, rlogs) return (JSArray rfields, rlogs) -- | A version of computeJobUpdate hardcoded to only return logs and the status -- field. By hardcoding this we avoid using the luxi Query infrastructure and -- the ConfigData value it requires. computeJobUpdateStatus :: JobId -> JSValue -> IO (JSValue, JSValue) computeJobUpdateStatus jid prev_log = do qdir <- queueDir loadResult <- loadJobFromDisk qdir True jid let sjid = show $ fromJobId jid logDebug $ "Inspecting status of job " ++ sjid let (rfields, rlogs) = case loadResult of Ok (job, _) -> (J.JSArray [status], newlogs) where status = showJSON $ calcJobStatus job -- like "status" jobField oplogs = map qoLog (qjOps job) -- like "oplog" jobField newer = case J.readJSON prev_log of J.Ok n -> (\(idx, _time, _type, _msg) -> n < idx) _ -> const True newlogs = showJSON $ concatMap (filter newer) oplogs _ -> (JSArray[JSNull], JSArray []) logDebug $ "Updates for job " ++ sjid ++ " are " ++ encode (rfields, rlogs) return (rfields, rlogs) type LuxiConfig = (Lock, JQStatus, ConfigReader) luxiExec :: LuxiConfig -> LuxiOp -> IO (Bool, GenericResult GanetiException JSValue) luxiExec (qlock, qstat, creader) args = case args of -- Special case WaitForJobChange handling to avoid passing a ConfigData to -- a potentially long-lived thread. ConfigData uses lots of heap, and -- multiple handler threads retaining different versions of ConfigData -- increases luxi's memory use for concurrent jobs that modify config. WaitForJobChange jid fields prev_job prev_log tmout | fields == ["status"] -> do result <- handleWaitForJobChangeStatus jid prev_job prev_log tmout return (True, result) _ -> do cfg <- creader result <- handleCallWrapper qlock qstat cfg args return (True, result) luxiHandler :: LuxiConfig -> U.Handler LuxiOp IO JSValue luxiHandler cfg = U.Handler { U.hParse = decodeLuxiCall , U.hInputLogShort = strOfOp , U.hInputLogLong = show , U.hExec = luxiExec cfg } -- | Type alias for prepMain results type PrepResult = (Server, IORef (Result ConfigData), JQStatus) -- | Activate the master IP address. activateMasterIP :: IO (Result ()) activateMasterIP = runResultT $ do liftIO $ logDebug "Activating master IP address" conf_file <- liftIO Path.clusterConfFile config <- mkResultT $ Config.loadConfig conf_file let mnp = Config.getMasterNetworkParameters config masters = Config.getMasterNodes config ems = clusterUseExternalMipScript $ configCluster config liftIO . logDebug $ "Master IP params: " ++ show mnp res <- liftIO . executeRpcCall masters $ RpcCallNodeActivateMasterIp mnp ems _ <- liftIO $ logRpcErrors res liftIO $ logDebug "finished activating master IP address" return () -- | Check function for luxid. checkMain :: CheckFn () checkMain = handleMasterVerificationOptions -- | Prepare function for luxid. prepMain :: PrepFn () PrepResult prepMain _ _ = do socket_path <- Path.defaultQuerySocket cleanupSocket socket_path s <- describeError "binding to the Luxi socket" Nothing (Just socket_path) $ getLuxiServer True socket_path cref <- newIORef (Bad "Configuration not yet loaded") jq <- emptyJQStatus cref return (s, cref, jq) -- | Main function. main :: MainFn () PrepResult main _ _ (server, cref, jq) = do -- Subscribe to config udpates. If the config changes, write new config and -- check if the changes should trigger the scheduler. initConfigReader $ \newConfig -> do runScheduler <- atomicModifyIORef cref $ \oldConfig -> case (oldConfig, newConfig) of (Ok old, Ok new) -> (newConfig, configChangeNeedsRescheduling old new) _ -> (newConfig, True) -- no old or new config, schedule when runScheduler (updateStatusAndScheduleSomeJobs jq) let creader = readIORef cref qlockFile <- jobQueueLockFile _ <- lockFile qlockFile >>= exitIfBad "Failed to obtain the job-queue lock" qlock <- newLock _ <- P.installHandler P.sigCHLD P.Ignore Nothing _ <- forkIO . void $ activateMasterIP initJQScheduler jq finally (forever $ U.listener (luxiHandler (qlock, jq, creader)) server) (closeServer server >> removeFile qlockFile) ganeti-3.1.0~rc2/src/Ganeti/Query/Types.hs000064400000000000000000000067531476477700300203620ustar00rootroot00000000000000{-| Implementation of the Ganeti Query2 basic types. These are types internal to the library, and for example clients that use the library should not need to import it. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Query.Types ( FieldGetter(..) , QffMode(..) , FieldData , FieldList , FieldMap , isRuntimeField , fieldListToFieldMap ) where import qualified Data.Map as Map import Ganeti.Query.Language import Ganeti.Objects -- | The type of field getters. The \"a\" type represents the type -- we're querying, whereas the \"b\" type represents the \'runtime\' -- data for that type (if any). Note that we don't support multiple -- runtime sources, and we always consider the entire configuration as -- a given (so no equivalent for Python's /*_CONFIG/ and /*_GROUP/; -- configuration accesses are cheap for us). data FieldGetter a b = FieldSimple (a -> ResultEntry) | FieldRuntime (b -> a -> ResultEntry) | FieldConfig (ConfigData -> a -> ResultEntry) | FieldConfigRuntime (ConfigData -> b -> a -> ResultEntry) | FieldUnknown -- | Type defining how the value of a field is used in filtering. This -- implements the equivalent to Python's QFF_ flags, except that we -- don't use OR-able values. data QffMode = QffNormal -- ^ Value is used as-is in filters | QffTimestamp -- ^ Value is a timestamp tuple, convert to float | QffHostname -- ^ Value is a hostname, compare it smartly deriving (Show, Eq) -- | Alias for a field data (definition and getter). type FieldData a b = (FieldDefinition, FieldGetter a b, QffMode) -- | Alias for a field data list. type FieldList a b = [FieldData a b] -- | Alias for field maps. type FieldMap a b = Map.Map String (FieldData a b) -- | Helper function to check if a getter is a runtime one. isRuntimeField :: FieldGetter a b -> Bool isRuntimeField FieldRuntime {} = True isRuntimeField FieldConfigRuntime {} = True isRuntimeField _ = False -- | Helper function to obtain a FieldMap from the corresponding FieldList. fieldListToFieldMap :: FieldList a b -> FieldMap a b fieldListToFieldMap = Map.fromList . map (\v@(f, _, _) -> (fdefName f, v)) ganeti-3.1.0~rc2/src/Ganeti/Rpc.hs000064400000000000000000000723111476477700300166660ustar00rootroot00000000000000{-# LANGUAGE MultiParamTypeClasses, FunctionalDependencies, BangPatterns, TemplateHaskell #-} {-| Implementation of the RPC client. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Rpc ( RpcCall , Rpc , RpcError(..) , ERpcError , explainRpcError , executeRpcCall , executeRpcCalls , rpcErrors , logRpcErrors , rpcCallName , rpcCallTimeout , rpcCallData , rpcCallAcceptOffline , rpcResultFill , Compressed , packCompressed , toCompressed , getCompressed , RpcCallNodeActivateMasterIp(..) , RpcResultNodeActivateMasterIp(..) , RpcCallInstanceInfo(..) , InstanceState(..) , InstanceInfo(..) , RpcResultInstanceInfo(..) , RpcCallAllInstancesInfo(..) , RpcResultAllInstancesInfo(..) , InstanceConsoleInfoParams(..) , InstanceConsoleInfo(..) , RpcCallInstanceConsoleInfo(..) , RpcResultInstanceConsoleInfo(..) , RpcCallInstanceList(..) , RpcResultInstanceList(..) , HvInfo(..) , StorageInfo(..) , RpcCallNodeInfo(..) , RpcResultNodeInfo(..) , RpcCallVersion(..) , RpcResultVersion(..) , RpcCallMasterNodeName(..) , RpcResultMasterNodeName(..) , RpcCallStorageList(..) , RpcResultStorageList(..) , RpcCallTestDelay(..) , RpcResultTestDelay(..) , RpcCallExportList(..) , RpcResultExportList(..) , RpcCallJobqueueUpdate(..) , RpcCallJobqueueRename(..) , RpcCallSetWatcherPause(..) , RpcCallSetDrainFlag(..) , RpcCallUploadFile(..) , prepareRpcCallUploadFile , RpcCallWriteSsconfFiles(..) ) where import Control.Arrow (second) import Control.Monad import qualified Data.ByteString.Lazy.Char8 as BL import qualified Data.Map as Map import Data.List (zipWith4) import Data.Maybe (mapMaybe) import qualified Text.JSON as J import Text.JSON.Pretty (pp_value) import qualified Data.ByteString.Base64.Lazy as Base64 import System.Directory import System.Posix.Files ( modificationTime, accessTime, fileOwner , fileGroup, fileMode, getFileStatus) import Network.BSD (getServiceByName, servicePort) import Network.Curl hiding (content) import qualified Ganeti.Path as P import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.Codec import Ganeti.Curl.Multi import Ganeti.Errors import Ganeti.JSON (ArrayObject(..), GenericContainer(..)) import Ganeti.Logging import Ganeti.Objects import Ganeti.Runtime import Ganeti.Ssconf import Ganeti.THH import Ganeti.THH.Field import Ganeti.Types import Ganeti.Utils.Time (cTimeToClockTime) import Ganeti.Utils import Ganeti.VCluster -- * Base RPC functionality and types -- | The curl options used for RPC. curlOpts :: [CurlOption] curlOpts = [ CurlFollowLocation False , CurlSSLVerifyHost 0 , CurlSSLVerifyPeer True , CurlSSLCertType "PEM" , CurlSSLKeyType "PEM" , CurlConnectTimeout (fromIntegral C.rpcConnectTimeout) , CurlHttpHeaders ["Expect:"] ] -- | Data type for RPC error reporting. data RpcError = CurlLayerError String | JsonDecodeError String | RpcResultError String | OfflineNodeError deriving (Show, Eq) -- | Provide explanation to RPC errors. explainRpcError :: RpcError -> String explainRpcError (CurlLayerError code) = "Curl error:" ++ code explainRpcError (JsonDecodeError msg) = "Error while decoding JSON from HTTP response: " ++ msg explainRpcError (RpcResultError msg) = "Error reponse received from RPC server: " ++ msg explainRpcError OfflineNodeError = "Node is marked offline" type ERpcError = Either RpcError -- | A generic class for RPC calls. class (ArrayObject a) => RpcCall a where -- | Give the (Python) name of the procedure. rpcCallName :: a -> String -- | Calculate the timeout value for the call execution. rpcCallTimeout :: a -> Int -- | Prepare arguments of the call to be send as POST. rpcCallData :: a -> String rpcCallData = J.encode . J.JSArray . toJSArray -- | Whether we accept offline nodes when making a call. rpcCallAcceptOffline :: a -> Bool -- | Generic class that ensures matching RPC call with its respective -- result. class (RpcCall a, J.JSON b) => Rpc a b | a -> b, b -> a where -- | Create a result based on the received HTTP response. rpcResultFill :: a -> J.JSValue -> ERpcError b -- | Http Request definition. data HttpClientRequest = HttpClientRequest { requestUrl :: String -- ^ The actual URL for the node endpoint , requestData :: String -- ^ The arguments for the call , requestOpts :: [CurlOption] -- ^ The various curl options } -- | Check if a string represented address is IPv6 isIpV6 :: String -> Bool isIpV6 = (':' `elem`) -- | Prepare url for the HTTP request. prepareUrl :: (RpcCall a) => Int -> Node -> a -> String prepareUrl port node call = let node_ip = nodePrimaryIp node node_address = if isIpV6 node_ip then "[" ++ node_ip ++ "]" else node_ip path_prefix = "https://" ++ node_address ++ ":" ++ show port in path_prefix ++ "/" ++ rpcCallName call -- | Create HTTP request for a given node provided it is online, -- otherwise create empty response. prepareHttpRequest :: (RpcCall a) => Int -> [CurlOption] -> Node -> String -> a -> ERpcError HttpClientRequest prepareHttpRequest port opts node reqdata call | rpcCallAcceptOffline call || not (nodeOffline node) = Right HttpClientRequest { requestUrl = prepareUrl port node call , requestData = reqdata , requestOpts = opts ++ curlOpts } | otherwise = Left OfflineNodeError -- | Parse an HTTP reply. parseHttpReply :: (Rpc a b) => a -> ERpcError (CurlCode, String) -> ERpcError b parseHttpReply _ (Left e) = Left e parseHttpReply call (Right (CurlOK, body)) = parseHttpResponse call body parseHttpReply _ (Right (code, err)) = Left . CurlLayerError $ "code: " ++ show code ++ ", explanation: " ++ err -- | Parse a result based on the received HTTP response. parseHttpResponse :: (Rpc a b) => a -> String -> ERpcError b parseHttpResponse call res = case J.decode res of J.Error val -> Left $ JsonDecodeError val J.Ok (True, res'') -> rpcResultFill call res'' J.Ok (False, jerr) -> case jerr of J.JSString msg -> Left $ RpcResultError (J.fromJSString msg) _ -> Left . JsonDecodeError $ show (pp_value jerr) -- | Scan the list of results produced by executeRpcCall and extract -- all the RPC errors. rpcErrors :: [(a, ERpcError b)] -> [(a, RpcError)] rpcErrors = let rpcErr (node, Left err) = Just (node, err) rpcErr _ = Nothing in mapMaybe rpcErr -- | Scan the list of results produced by executeRpcCall and log all the RPC -- errors. Returns the list of errors for further processing. logRpcErrors :: (MonadLog m, Show a) => [(a, ERpcError b)] -> m [(a, RpcError)] logRpcErrors rs = let logOneRpcErr (node, err) = logError $ "Error in the RPC HTTP reply from '" ++ show node ++ "': " ++ show err errs = rpcErrors rs in mapM_ logOneRpcErr errs >> return errs -- | Get options for RPC call getOptionsForCall :: (Rpc a b) => FilePath -> FilePath -> a -> [CurlOption] getOptionsForCall cert_path client_cert_path call = [ CurlTimeout (fromIntegral $ rpcCallTimeout call) , CurlSSLCert client_cert_path , CurlSSLKey client_cert_path , CurlCAInfo cert_path ] -- | Determine to port to call noded at. getNodedPort :: IO Int getNodedPort = withDefaultOnIOError C.defaultNodedPort . liftM (fromIntegral . servicePort) $ getServiceByName C.noded "tcp" -- | Execute multiple distinct RPC calls in parallel executeRpcCalls :: (Rpc a b) => [(Node, a)] -> IO [(Node, ERpcError b)] executeRpcCalls = executeRpcCalls' . map (\(n, c) -> (n, c, rpcCallData c)) -- | Execute multiple RPC calls in parallel executeRpcCalls' :: (Rpc a b) => [(Node, a, String)] -> IO [(Node, ERpcError b)] executeRpcCalls' nodeCalls = do port <- getNodedPort cert_file <- P.nodedCertFile client_cert_file_name <- P.nodedClientCertFile client_file_exists <- doesFileExist client_cert_file_name -- This is needed to allow upgrades from 2.10 or earlier; -- note that Ganeti supports jump-upgrades. let client_cert_file = if client_file_exists then client_cert_file_name else cert_file (nodes, calls, datas) = unzip3 nodeCalls opts = map (getOptionsForCall cert_file client_cert_file) calls opts_urls = zipWith4 (\n c d o -> case prepareHttpRequest port o n d c of Left v -> Left v Right request -> Right (CurlPostFields [requestData request]: requestOpts request, requestUrl request) ) nodes calls datas opts -- split the opts_urls list; we don't want to pass the -- failed-already nodes to Curl let (lefts, rights, trail) = splitEithers opts_urls results <- execMultiCall rights results' <- case recombineEithers lefts results trail of Bad msg -> error msg Ok r -> return r -- now parse the replies let results'' = zipWith parseHttpReply calls results' pairedList = zip nodes results'' _ <- logRpcErrors pairedList return pairedList -- | Execute an RPC call for many nodes in parallel. -- NB this computes the RPC call payload string only once. executeRpcCall :: (Rpc a b) => [Node] -> a -> IO [(Node, ERpcError b)] executeRpcCall nodes call = executeRpcCalls' [(n, call, rpc_data) | n <- nodes] where rpc_data = rpcCallData call -- | Helper function that is used to read dictionaries of values. sanitizeDictResults :: [(String, J.Result a)] -> ERpcError [(String, a)] sanitizeDictResults = foldr sanitize1 (Right []) where sanitize1 _ (Left e) = Left e sanitize1 (_, J.Error e) _ = Left $ JsonDecodeError e sanitize1 (name, J.Ok v) (Right res) = Right $ (name, v) : res -- | Helper function to tranform JSON Result to Either RpcError b. -- Note: For now we really only use it for b s.t. Rpc c b for some c fromJResultToRes :: J.Result a -> (a -> b) -> ERpcError b fromJResultToRes (J.Error v) _ = Left $ JsonDecodeError v fromJResultToRes (J.Ok v) f = Right $ f v -- | Helper function transforming JSValue to Rpc result type. fromJSValueToRes :: (J.JSON a) => J.JSValue -> (a -> b) -> ERpcError b fromJSValueToRes val = fromJResultToRes (J.readJSON val) -- | An opaque data type for representing data that might be compressed -- over the wire. -- -- On Python side it is decompressed by @backend._Decompress@. newtype Compressed = Compressed { getCompressed :: BL.ByteString } deriving (Eq, Ord, Show) -- TODO Add a unit test for all octets instance J.JSON Compressed where -- zlib compress and Base64 encode the data but only if it's long enough showJSON = J.showJSON . (\x -> if BL.length (BL.take 4096 x) < 4096 then (C.rpcEncodingNone, x) else (C.rpcEncodingZlibBase64, Base64.encode . compressZlib $ x) ) . getCompressed readJSON = J.readJSON >=> decompress where decompress (enc, cont) | enc == C.rpcEncodingNone = return $ Compressed cont | enc == C.rpcEncodingZlibBase64 = liftM Compressed . either fail return . decompressZlib <=< either (fail . ("Base64: " ++)) return . Base64.decode $ cont | otherwise = fail $ "Unknown RPC encoding type: " ++ show enc packCompressed :: BL.ByteString -> Compressed packCompressed = Compressed toCompressed :: String -> Compressed toCompressed = packCompressed . BL.pack -- * RPC calls and results -- ** Instance info -- | Returns information about a single instance $(buildObject "RpcCallInstanceInfo" "rpcCallInstInfo" [ simpleField "instance" [t| String |] , simpleField "hname" [t| Hypervisor |] ]) $(declareILADT "InstanceState" [ ("InstanceStateRunning", 0) , ("InstanceStateShutdown", 1) ]) $(makeJSONInstance ''InstanceState) instance PyValue InstanceState where showValue = show . instanceStateToRaw $(buildObject "InstanceInfo" "instInfo" [ simpleField "memory" [t| Int|] , simpleField "state" [t| InstanceState |] , simpleField "vcpus" [t| Int |] , simpleField "time" [t| Int |] ]) -- This is optional here because the result may be empty if instance is -- not on a node - and this is not considered an error. $(buildObject "RpcResultInstanceInfo" "rpcResInstInfo" [ optionalField $ simpleField "inst_info" [t| InstanceInfo |]]) instance RpcCall RpcCallInstanceInfo where rpcCallName _ = "instance_info" rpcCallTimeout _ = rpcTimeoutToRaw Urgent rpcCallAcceptOffline _ = False instance Rpc RpcCallInstanceInfo RpcResultInstanceInfo where rpcResultFill _ res = case res of J.JSObject res' -> case J.fromJSObject res' of [] -> Right $ RpcResultInstanceInfo Nothing _ -> fromJSValueToRes res (RpcResultInstanceInfo . Just) _ -> Left $ JsonDecodeError ("Expected JSObject, got " ++ show (pp_value res)) -- ** AllInstancesInfo -- | Returns information about all running instances on the given nodes $(buildObject "RpcCallAllInstancesInfo" "rpcCallAllInstInfo" [ simpleField "hypervisors" [t| [(Hypervisor, HvParams)] |] ]) $(buildObject "RpcResultAllInstancesInfo" "rpcResAllInstInfo" [ simpleField "instances" [t| [(String, InstanceInfo)] |] ]) instance RpcCall RpcCallAllInstancesInfo where rpcCallName _ = "all_instances_info" rpcCallTimeout _ = rpcTimeoutToRaw Urgent rpcCallAcceptOffline _ = False rpcCallData call = J.encode ( map fst $ rpcCallAllInstInfoHypervisors call, GenericContainer . Map.fromList $ rpcCallAllInstInfoHypervisors call) instance Rpc RpcCallAllInstancesInfo RpcResultAllInstancesInfo where -- FIXME: Is there a simpler way to do it? rpcResultFill _ res = case res of J.JSObject res' -> let res'' = map (second J.readJSON) (J.fromJSObject res') :: [(String, J.Result InstanceInfo)] in case sanitizeDictResults res'' of Left err -> Left err Right insts -> Right $ RpcResultAllInstancesInfo insts _ -> Left $ JsonDecodeError ("Expected JSObject, got " ++ show (pp_value res)) -- ** InstanceConsoleInfo -- | Returns information about how to access instances on the given node $(buildObject "InstanceConsoleInfoParams" "instConsInfoParams" [ simpleField "instance" [t| Instance |] , simpleField "node" [t| Node |] , simpleField "group" [t| NodeGroup |] , simpleField "hvParams" [t| HvParams |] , simpleField "beParams" [t| FilledBeParams |] ]) $(buildObject "RpcCallInstanceConsoleInfo" "rpcCallInstConsInfo" [ simpleField "instanceInfo" [t| [(String, InstanceConsoleInfoParams)] |] ]) $(buildObject "InstanceConsoleInfo" "instConsInfo" [ simpleField "instance" [t| String |] , simpleField "kind" [t| String |] , optionalField $ simpleField "message" [t| String |] , optionalField $ simpleField "host" [t| String |] , optionalField $ simpleField "port" [t| Int |] , optionalField $ simpleField "user" [t| String |] , optionalField $ simpleField "command" [t| [String] |] , optionalField $ simpleField "display" [t| String |] ]) $(buildObject "RpcResultInstanceConsoleInfo" "rpcResInstConsInfo" [ simpleField "instancesInfo" [t| [(String, InstanceConsoleInfo)] |] ]) instance RpcCall RpcCallInstanceConsoleInfo where rpcCallName _ = "instance_console_info" rpcCallTimeout _ = rpcTimeoutToRaw Urgent rpcCallAcceptOffline _ = False rpcCallData call = J.encode . GenericContainer $ Map.fromList (rpcCallInstConsInfoInstanceInfo call) instance Rpc RpcCallInstanceConsoleInfo RpcResultInstanceConsoleInfo where rpcResultFill _ res = case res of J.JSObject res' -> let res'' = map (second J.readJSON) (J.fromJSObject res') :: [(String, J.Result InstanceConsoleInfo)] in case sanitizeDictResults res'' of Left err -> Left err Right instInfos -> Right $ RpcResultInstanceConsoleInfo instInfos _ -> Left $ JsonDecodeError ("Expected JSObject, got " ++ show (pp_value res)) -- ** InstanceList -- | Returns the list of running instances on the given nodes $(buildObject "RpcCallInstanceList" "rpcCallInstList" [ simpleField "hypervisors" [t| [Hypervisor] |] ]) $(buildObject "RpcResultInstanceList" "rpcResInstList" [ simpleField "instances" [t| [String] |] ]) instance RpcCall RpcCallInstanceList where rpcCallName _ = "instance_list" rpcCallTimeout _ = rpcTimeoutToRaw Urgent rpcCallAcceptOffline _ = False instance Rpc RpcCallInstanceList RpcResultInstanceList where rpcResultFill _ res = fromJSValueToRes res RpcResultInstanceList -- ** NodeInfo -- | Returns node information $(buildObject "RpcCallNodeInfo" "rpcCallNodeInfo" [ simpleField "storage_units" [t| [StorageUnit] |] , simpleField "hypervisors" [t| [ (Hypervisor, HvParams) ] |] ]) $(buildObject "StorageInfo" "storageInfo" [ simpleField "name" [t| String |] , simpleField "type" [t| String |] , optionalField $ simpleField "storage_free" [t| Int |] , optionalField $ simpleField "storage_size" [t| Int |] ]) -- | Common fields (as described in hv_base.py) are mandatory, -- other fields are optional. $(buildObject "HvInfo" "hvInfo" [ optionalField $ simpleField C.hvNodeinfoKeyVersion [t| [Int] |] , simpleField "memory_total" [t| Int |] , simpleField "memory_free" [t| Int |] , simpleField "memory_dom0" [t| Int |] , optionalField $ simpleField "memory_hv" [t| Int |] , simpleField "cpu_total" [t| Int |] , simpleField "cpu_nodes" [t| Int |] , simpleField "cpu_sockets" [t| Int |] , simpleField "cpu_dom0" [t| Int |] ]) $(buildObject "RpcResultNodeInfo" "rpcResNodeInfo" [ simpleField "boot_id" [t| String |] , simpleField "storage_info" [t| [StorageInfo] |] , simpleField "hv_info" [t| [HvInfo] |] ]) instance RpcCall RpcCallNodeInfo where rpcCallName _ = "node_info" rpcCallTimeout _ = rpcTimeoutToRaw Urgent rpcCallAcceptOffline _ = False rpcCallData call = J.encode ( rpcCallNodeInfoStorageUnits call , rpcCallNodeInfoHypervisors call ) instance Rpc RpcCallNodeInfo RpcResultNodeInfo where rpcResultFill _ res = fromJSValueToRes res (\(b, vg, hv) -> RpcResultNodeInfo b vg hv) -- ** Version -- | Query node version. $(buildObject "RpcCallVersion" "rpcCallVersion" []) -- | Query node reply. $(buildObject "RpcResultVersion" "rpcResultVersion" [ simpleField "version" [t| Int |] ]) instance RpcCall RpcCallVersion where rpcCallName _ = "version" rpcCallTimeout _ = rpcTimeoutToRaw Urgent rpcCallAcceptOffline _ = True rpcCallData = J.encode instance Rpc RpcCallVersion RpcResultVersion where rpcResultFill _ res = fromJSValueToRes res RpcResultVersion -- ** StorageList $(buildObject "RpcCallStorageList" "rpcCallStorageList" [ simpleField "su_name" [t| StorageType |] , simpleField "su_args" [t| [String] |] , simpleField "name" [t| String |] , simpleField "fields" [t| [StorageField] |] ]) -- FIXME: The resulting JSValues should have types appropriate for their -- StorageField value: Used -> Bool, Name -> String etc $(buildObject "RpcResultStorageList" "rpcResStorageList" [ simpleField "storage" [t| [[(StorageField, J.JSValue)]] |] ]) instance RpcCall RpcCallStorageList where rpcCallName _ = "storage_list" rpcCallTimeout _ = rpcTimeoutToRaw Normal rpcCallAcceptOffline _ = False instance Rpc RpcCallStorageList RpcResultStorageList where rpcResultFill call res = let sfields = rpcCallStorageListFields call in fromJSValueToRes res (RpcResultStorageList . map (zip sfields)) -- ** TestDelay -- | Call definition for test delay. $(buildObject "RpcCallTestDelay" "rpcCallTestDelay" [ simpleField "duration" [t| Double |] ]) -- | Result definition for test delay. data RpcResultTestDelay = RpcResultTestDelay deriving Show -- | Custom JSON instance for null result. instance J.JSON RpcResultTestDelay where showJSON _ = J.JSNull readJSON J.JSNull = return RpcResultTestDelay readJSON _ = fail "Unable to read RpcResultTestDelay" instance RpcCall RpcCallTestDelay where rpcCallName _ = "test_delay" rpcCallTimeout = ceiling . (+ 5) . rpcCallTestDelayDuration rpcCallAcceptOffline _ = False instance Rpc RpcCallTestDelay RpcResultTestDelay where rpcResultFill _ res = fromJSValueToRes res id -- ** ExportList -- | Call definition for export list. $(buildObject "RpcCallExportList" "rpcCallExportList" []) -- | Result definition for export list. $(buildObject "RpcResultExportList" "rpcResExportList" [ simpleField "exports" [t| [String] |] ]) instance RpcCall RpcCallExportList where rpcCallName _ = "export_list" rpcCallTimeout _ = rpcTimeoutToRaw Fast rpcCallAcceptOffline _ = False rpcCallData = J.encode instance Rpc RpcCallExportList RpcResultExportList where rpcResultFill _ res = fromJSValueToRes res RpcResultExportList -- ** Job Queue Replication -- | Update a job queue file $(buildObject "RpcCallJobqueueUpdate" "rpcCallJobqueueUpdate" [ simpleField "file_name" [t| String |] , simpleField "content" [t| String |] ]) $(buildObject "RpcResultJobQueueUpdate" "rpcResultJobQueueUpdate" []) instance RpcCall RpcCallJobqueueUpdate where rpcCallName _ = "jobqueue_update" rpcCallTimeout _ = rpcTimeoutToRaw Fast rpcCallAcceptOffline _ = False rpcCallData call = J.encode ( rpcCallJobqueueUpdateFileName call , toCompressed $ rpcCallJobqueueUpdateContent call ) instance Rpc RpcCallJobqueueUpdate RpcResultJobQueueUpdate where rpcResultFill _ res = case res of J.JSNull -> Right RpcResultJobQueueUpdate _ -> Left $ JsonDecodeError ("Expected JSNull, got " ++ show (pp_value res)) -- | Rename a file in the job queue $(buildObject "RpcCallJobqueueRename" "rpcCallJobqueueRename" [ simpleField "rename" [t| [(String, String)] |] ]) $(buildObject "RpcResultJobqueueRename" "rpcResultJobqueueRename" []) instance RpcCall RpcCallJobqueueRename where rpcCallName _ = "jobqueue_rename" rpcCallTimeout _ = rpcTimeoutToRaw Fast rpcCallAcceptOffline _ = False instance Rpc RpcCallJobqueueRename RpcResultJobqueueRename where rpcResultFill call res = -- Upon success, the RPC returns the list of return values of -- the rename operations, which is always None, serialized to -- null in JSON. let expected = J.showJSON . map (const J.JSNull) $ rpcCallJobqueueRenameRename call in if res == expected then Right RpcResultJobqueueRename else Left $ JsonDecodeError ("Expected JSNull, got " ++ show (pp_value res)) -- ** Watcher Status Update -- | Set the watcher status $(buildObject "RpcCallSetWatcherPause" "rpcCallSetWatcherPause" [ optionalField $ timeAsDoubleField "time" ]) instance RpcCall RpcCallSetWatcherPause where rpcCallName _ = "set_watcher_pause" rpcCallTimeout _ = rpcTimeoutToRaw Fast rpcCallAcceptOffline _ = False $(buildObject "RpcResultSetWatcherPause" "rpcResultSetWatcherPause" []) instance Rpc RpcCallSetWatcherPause RpcResultSetWatcherPause where rpcResultFill _ res = case res of J.JSNull -> Right RpcResultSetWatcherPause _ -> Left $ JsonDecodeError ("Expected JSNull, got " ++ show (pp_value res)) -- ** Queue drain status -- | Set the queu drain flag $(buildObject "RpcCallSetDrainFlag" "rpcCallSetDrainFlag" [ simpleField "value" [t| Bool |] ]) instance RpcCall RpcCallSetDrainFlag where rpcCallName _ = "jobqueue_set_drain_flag" rpcCallTimeout _ = rpcTimeoutToRaw Fast rpcCallAcceptOffline _ = False $(buildObject "RpcResultSetDrainFlag" "rpcResultSetDrainFalg" []) instance Rpc RpcCallSetDrainFlag RpcResultSetDrainFlag where rpcResultFill _ res = case res of J.JSNull -> Right RpcResultSetDrainFlag _ -> Left $ JsonDecodeError ("Expected JSNull, got " ++ show (pp_value res)) -- ** Configuration files upload to nodes -- | Upload a configuration file to nodes $(buildObject "RpcCallUploadFile" "rpcCallUploadFile" [ simpleField "file_name" [t| FilePath |] , simpleField "content" [t| Compressed |] , optionalField $ fileModeAsIntField "mode" , simpleField "uid" [t| String |] , simpleField "gid" [t| String |] , timeAsDoubleField "atime" , timeAsDoubleField "mtime" ]) instance RpcCall RpcCallUploadFile where rpcCallName _ = "upload_file_single" rpcCallTimeout _ = rpcTimeoutToRaw Normal rpcCallAcceptOffline _ = False $(buildObject "RpcResultUploadFile" "rpcResultUploadFile" []) instance Rpc RpcCallUploadFile RpcResultUploadFile where rpcResultFill _ res = case res of J.JSNull -> Right RpcResultUploadFile _ -> Left $ JsonDecodeError ("Expected JSNull, got " ++ show (pp_value res)) -- | Reads a file and constructs the corresponding 'RpcCallUploadFile' value. prepareRpcCallUploadFile :: RuntimeEnts -> FilePath -> ResultG RpcCallUploadFile prepareRpcCallUploadFile re path = do status <- liftIO $ getFileStatus path content <- liftIO $ BL.readFile path let lookupM x m = maybe (failError $ "Uid/gid " ++ show x ++ " not found, probably file " ++ show path ++ " isn't a Ganeti file") return (Map.lookup x m) uid <- lookupM (fileOwner status) (reUidToUser re) gid <- lookupM (fileGroup status) (reGidToGroup re) vpath <- liftIO $ makeVirtualPath path return $ RpcCallUploadFile vpath (packCompressed content) (Just $ fileMode status) uid gid (cTimeToClockTime $ accessTime status) (cTimeToClockTime $ modificationTime status) -- | Upload ssconf files to nodes $(buildObject "RpcCallWriteSsconfFiles" "rpcCallWriteSsconfFiles" [ simpleField "values" [t| SSConf |] ]) instance RpcCall RpcCallWriteSsconfFiles where rpcCallName _ = "write_ssconf_files" rpcCallTimeout _ = rpcTimeoutToRaw Fast rpcCallAcceptOffline _ = False $(buildObject "RpcResultWriteSsconfFiles" "rpcResultWriteSsconfFiles" []) instance Rpc RpcCallWriteSsconfFiles RpcResultWriteSsconfFiles where rpcResultFill _ res = case res of J.JSNull -> Right RpcResultWriteSsconfFiles _ -> Left $ JsonDecodeError ("Expected JSNull, got " ++ show (pp_value res)) -- | Activate the master IP address $(buildObject "RpcCallNodeActivateMasterIp" "rpcCallNodeActivateMasterIp" [ simpleField "params" [t| MasterNetworkParameters |] , simpleField "ems" [t| Bool |] ]) instance RpcCall RpcCallNodeActivateMasterIp where rpcCallName _ = "node_activate_master_ip" rpcCallTimeout _ = rpcTimeoutToRaw Fast rpcCallAcceptOffline _ = False $(buildObject "RpcResultNodeActivateMasterIp" "rpcResultNodeActivateMasterIp" []) instance Rpc RpcCallNodeActivateMasterIp RpcResultNodeActivateMasterIp where rpcResultFill _ res = case res of J.JSNull -> Right RpcResultNodeActivateMasterIp _ -> Left $ JsonDecodeError ("Expected JSNull, got " ++ show (pp_value res)) -- | Ask who the node believes is the master. $(buildObject "RpcCallMasterNodeName" "rpcCallMasterNodeName" []) instance RpcCall RpcCallMasterNodeName where rpcCallName _ = "master_node_name" rpcCallTimeout _ = rpcTimeoutToRaw Slow rpcCallAcceptOffline _ = True $(buildObject "RpcResultMasterNodeName" "rpcResultMasterNodeName" [ simpleField "master" [t| String |] ]) instance Rpc RpcCallMasterNodeName RpcResultMasterNodeName where rpcResultFill _ res = case res of J.JSString master -> Right . RpcResultMasterNodeName $ J.fromJSString master _ -> Left . JsonDecodeError . (++) "expected string, but got " . show $ pp_value res ganeti-3.1.0~rc2/src/Ganeti/Runtime.hs000064400000000000000000000200441476477700300175610ustar00rootroot00000000000000{-| Implementation of the runtime configuration details. -} {- Copyright (C) 2011, 2012, 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Runtime ( GanetiDaemon(..) , MiscGroup(..) , GanetiGroup(..) , RuntimeEnts(..) , daemonName , daemonOnlyOnMaster , daemonLogBase , daemonUser , daemonGroup , ExtraLogReason(..) , daemonLogFile , daemonsExtraLogbase , daemonsExtraLogFile , daemonPidFile , getEnts , verifyDaemonUser ) where import Control.Monad import qualified Data.Map as M import System.Exit import System.FilePath import System.IO import System.Posix.Types import System.Posix.User import Text.Printf import qualified Ganeti.ConstantUtils as ConstantUtils import qualified Ganeti.Path as Path import Ganeti.BasicTypes import AutoConf data GanetiDaemon = GanetiMasterd | GanetiMetad | GanetiNoded | GanetiRapi | GanetiConfd | GanetiWConfd | GanetiKvmd | GanetiLuxid | GanetiMond deriving (Show, Enum, Bounded, Eq, Ord) data MiscGroup = DaemonsGroup | AdminGroup deriving (Show, Enum, Bounded, Eq, Ord) data GanetiGroup = DaemonGroup GanetiDaemon | ExtraGroup MiscGroup deriving (Show, Eq, Ord) data RuntimeEnts = RuntimeEnts { reUserToUid :: M.Map GanetiDaemon UserID , reUidToUser :: M.Map UserID String , reGroupToGid :: M.Map GanetiGroup GroupID , reGidToGroup :: M.Map GroupID String } -- | Returns the daemon name for a given daemon. daemonName :: GanetiDaemon -> String daemonName GanetiMasterd = "ganeti-masterd" daemonName GanetiMetad = "ganeti-metad" daemonName GanetiNoded = "ganeti-noded" daemonName GanetiRapi = "ganeti-rapi" daemonName GanetiConfd = "ganeti-confd" daemonName GanetiWConfd = "ganeti-wconfd" daemonName GanetiKvmd = "ganeti-kvmd" daemonName GanetiLuxid = "ganeti-luxid" daemonName GanetiMond = "ganeti-mond" -- | Returns whether the daemon only runs on the master node. daemonOnlyOnMaster :: GanetiDaemon -> Bool daemonOnlyOnMaster GanetiMasterd = True daemonOnlyOnMaster GanetiMetad = False daemonOnlyOnMaster GanetiNoded = False daemonOnlyOnMaster GanetiRapi = False daemonOnlyOnMaster GanetiConfd = False daemonOnlyOnMaster GanetiWConfd = True daemonOnlyOnMaster GanetiKvmd = False daemonOnlyOnMaster GanetiLuxid = True daemonOnlyOnMaster GanetiMond = False -- | Returns the log file base for a daemon. daemonLogBase :: GanetiDaemon -> String daemonLogBase GanetiMasterd = "master-daemon" daemonLogBase GanetiMetad = "meta-daemon" daemonLogBase GanetiNoded = "node-daemon" daemonLogBase GanetiRapi = "rapi-daemon" daemonLogBase GanetiConfd = "conf-daemon" daemonLogBase GanetiWConfd = "wconf-daemon" daemonLogBase GanetiKvmd = "kvm-daemon" daemonLogBase GanetiLuxid = "luxi-daemon" daemonLogBase GanetiMond = "monitoring-daemon" -- | Returns the configured user name for a daemon. daemonUser :: GanetiDaemon -> String daemonUser GanetiMasterd = AutoConf.masterdUser daemonUser GanetiMetad = AutoConf.metadUser daemonUser GanetiNoded = AutoConf.nodedUser daemonUser GanetiRapi = AutoConf.rapiUser daemonUser GanetiConfd = AutoConf.confdUser daemonUser GanetiWConfd = AutoConf.wconfdUser daemonUser GanetiKvmd = AutoConf.kvmdUser daemonUser GanetiLuxid = AutoConf.luxidUser daemonUser GanetiMond = AutoConf.mondUser -- | Returns the configured group for a daemon. daemonGroup :: GanetiGroup -> String daemonGroup (DaemonGroup GanetiMasterd) = AutoConf.masterdGroup daemonGroup (DaemonGroup GanetiMetad) = AutoConf.metadGroup daemonGroup (DaemonGroup GanetiNoded) = AutoConf.nodedGroup daemonGroup (DaemonGroup GanetiRapi) = AutoConf.rapiGroup daemonGroup (DaemonGroup GanetiConfd) = AutoConf.confdGroup daemonGroup (DaemonGroup GanetiWConfd) = AutoConf.wconfdGroup daemonGroup (DaemonGroup GanetiLuxid) = AutoConf.luxidGroup daemonGroup (DaemonGroup GanetiKvmd) = AutoConf.kvmdGroup daemonGroup (DaemonGroup GanetiMond) = AutoConf.mondGroup daemonGroup (ExtraGroup DaemonsGroup) = AutoConf.daemonsGroup daemonGroup (ExtraGroup AdminGroup) = AutoConf.adminGroup data ExtraLogReason = AccessLog | ErrorLog -- | Some daemons might require more than one logfile. Specifically, -- right now only the Haskell http library "snap", used by the -- monitoring daemon, requires multiple log files. daemonsExtraLogbase :: GanetiDaemon -> ExtraLogReason -> String daemonsExtraLogbase daemon AccessLog = daemonLogBase daemon ++ "-access" daemonsExtraLogbase daemon ErrorLog = daemonLogBase daemon ++ "-error" -- | Returns the log file for a daemon. daemonLogFile :: GanetiDaemon -> IO FilePath daemonLogFile daemon = do logDir <- Path.logDir return $ logDir daemonLogBase daemon <.> "log" -- | Returns the extra log files for a daemon. daemonsExtraLogFile :: GanetiDaemon -> ExtraLogReason -> IO FilePath daemonsExtraLogFile daemon logreason = do logDir <- Path.logDir return $ logDir daemonsExtraLogbase daemon logreason <.> "log" -- | Returns the pid file name for a daemon. daemonPidFile :: GanetiDaemon -> IO FilePath daemonPidFile daemon = do runDir <- Path.runDir return $ runDir daemonName daemon <.> "pid" -- | All groups list. A bit hacking, as we can't enforce it's complete -- at compile time. allGroups :: [GanetiGroup] allGroups = map DaemonGroup [minBound..maxBound] ++ map ExtraGroup [minBound..maxBound] -- | Computes the group/user maps. getEnts :: (Error e) => ResultT e IO RuntimeEnts getEnts = do let userOf = liftM userID . liftIO . getUserEntryForName . daemonUser let groupOf = liftM groupID . liftIO . getGroupEntryForName . daemonGroup let allDaemons = [minBound..maxBound] :: [GanetiDaemon] users <- mapM userOf allDaemons groups <- mapM groupOf allGroups return $ RuntimeEnts (M.fromList $ zip allDaemons users) (M.fromList $ zip users (map daemonUser allDaemons)) (M.fromList $ zip allGroups groups) (M.fromList $ zip groups (map daemonGroup allGroups)) -- | Checks whether a daemon runs as the right user. verifyDaemonUser :: GanetiDaemon -> RuntimeEnts -> IO () verifyDaemonUser daemon ents = do myuid <- getEffectiveUserID -- note: we use directly ! as lookup failues shouldn't happen, due -- to the above map construction checkUidMatch (daemonName daemon) ((M.!) (reUserToUid ents) daemon) myuid -- | Check that two UIDs are matching or otherwise exit. checkUidMatch :: String -> UserID -> UserID -> IO () checkUidMatch name expected actual = when (expected /= actual) $ do hPrintf stderr "%s started using wrong user ID (%d), \ \expected %d\n" name (fromIntegral actual::Int) (fromIntegral expected::Int) :: IO () exitWith $ ExitFailure ConstantUtils.exitFailure ganeti-3.1.0~rc2/src/Ganeti/SlotMap.hs000064400000000000000000000065021476477700300175200ustar00rootroot00000000000000{-| A data structure for measuring how many of a number of available slots are taken. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.SlotMap ( Slot(..) , SlotMap , CountMap , toCountMap , isOverfull , occupySlots , hasSlotsFor ) where import Data.Map (Map) import qualified Data.Map as Map {-# ANN module "HLint: ignore Avoid lambda" #-} -- to not suggest (`Slot` 0) -- | A resource with [limit] available units and [occupied] of them taken. data Slot = Slot { slotOccupied :: Int , slotLimit :: Int } deriving (Eq, Ord, Show) -- | A set of keys of type @a@ and how many slots are available and (to be) -- occupied per key. -- -- Some keys can be overfull (more slots occupied than available). type SlotMap a = Map a Slot -- | A set of keys of type @a@ and how many there are of each. type CountMap a = Map a Int -- | Turns a `SlotMap` into a `CountMap` by throwing away the limits. toCountMap :: SlotMap a -> CountMap a toCountMap = Map.map slotOccupied -- | Whether any more slots are occupied than available. isOverfull :: SlotMap a -> Bool isOverfull m = or [ occup > limit | Slot occup limit <- Map.elems m ] -- | Fill slots of a `SlotMap`s by adding the given counts. -- Keys with counts that don't appear in the `SlotMap` get a limit of 0. occupySlots :: (Ord a) => SlotMap a -> CountMap a -> SlotMap a occupySlots sm counts = Map.unionWith (\(Slot o l) (Slot n _) -> Slot (o + n) l) sm (Map.map (\n -> Slot n 0) counts) -- | Whether the `SlotMap` has enough slots free to accomodate the given -- counts. -- -- The `SlotMap` is allowed to be overfull in some keys; this function -- still returns True as long as as adding the counts to the `SlotMap` would -- not *create or increase* overfull keys. -- -- Adding counts > 0 for a key which is not in the `SlotMap` does create -- overfull keys. hasSlotsFor :: (Ord a) => SlotMap a -> CountMap a -> Bool slotMap `hasSlotsFor` counts = let relevantSlots = slotMap `Map.intersection` counts in not $ isOverfull (relevantSlots `occupySlots` counts) ganeti-3.1.0~rc2/src/Ganeti/Ssconf.hs000064400000000000000000000201101476477700300173630ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Implementation of the Ganeti Ssconf interface. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Ssconf ( SSKey(..) , sSKeyToRaw , sSKeyFromRaw , hvparamsSSKey , getPrimaryIPFamily , parseNodesVmCapable , getNodesVmCapable , getMasterCandidatesIps , getMasterNode , parseHypervisorList , getHypervisorList , parseEnabledUserShutdown , getEnabledUserShutdown , keyToFilename , sSFilePrefix , SSConf(..) , emptySSConf ) where import Control.Arrow ((&&&)) import Control.Exception import Control.Monad (forM, liftM) import Control.Monad.Fail (MonadFail) import qualified Data.Map as M import Data.Maybe (fromMaybe) import qualified Network.Socket as Socket import System.FilePath (()) import System.IO.Error (isDoesNotExistError) import qualified Text.JSON as J import qualified AutoConf import Ganeti.BasicTypes import qualified Ganeti.Constants as C import qualified Ganeti.ConstantUtils as CU import Ganeti.JSON (GenericContainer(..), HasStringRepr(..)) import qualified Ganeti.Path as Path import Ganeti.THH import Ganeti.Types (Hypervisor) import qualified Ganeti.Types as Types import Ganeti.Utils -- * Reading individual ssconf entries -- | Maximum ssconf file size we support. maxFileSize :: Int maxFileSize = 131072 -- | ssconf file prefix, re-exported from Constants. sSFilePrefix :: FilePath sSFilePrefix = C.ssconfFileprefix $(declareLADT ''String "SSKey" ( map (ssconfConstructorName &&& id) . CU.toList $ C.validSsKeys )) instance HasStringRepr SSKey where fromStringRepr = sSKeyFromRaw toStringRepr = sSKeyToRaw -- | For a given hypervisor get the corresponding SSConf key that contains its -- parameters. -- -- The corresponding SSKeys are generated automatically by TH, but since we -- don't have convenient infrastructure for generating this function, it's just -- manual. All constructors must be given explicitly so that adding another -- hypervisor will trigger "incomplete pattern" warning and force the -- corresponding addition. hvparamsSSKey :: Types.Hypervisor -> SSKey hvparamsSSKey Types.Kvm = SSHvparamsKvm hvparamsSSKey Types.XenPvm = SSHvparamsXenPvm hvparamsSSKey Types.Chroot = SSHvparamsChroot hvparamsSSKey Types.XenHvm = SSHvparamsXenHvm hvparamsSSKey Types.Lxc = SSHvparamsLxc hvparamsSSKey Types.Fake = SSHvparamsFake -- | Convert a ssconf key into a (full) file path. keyToFilename :: FilePath -- ^ Config path root -> SSKey -- ^ Ssconf key -> FilePath -- ^ Full file name keyToFilename cfgpath key = cfgpath sSFilePrefix ++ sSKeyToRaw key -- | Runs an IO action while transforming any error into 'Bad' -- values. It also accepts an optional value to use in case the error -- is just does not exist. catchIOErrors :: Maybe a -- ^ Optional default -> IO a -- ^ Action to run -> IO (Result a) catchIOErrors def action = Control.Exception.catch (do result <- action return (Ok result) ) (\err -> let bad_result = Bad (show err) in return $ if isDoesNotExistError err then maybe bad_result Ok def else bad_result) -- | Read an ssconf file. readSSConfFile :: Maybe FilePath -- ^ Optional config path override -> Maybe String -- ^ Optional default value -> SSKey -- ^ Desired ssconf key -> IO (Result String) readSSConfFile optpath def key = do dpath <- Path.dataDir result <- catchIOErrors def . readFile . keyToFilename (fromMaybe dpath optpath) $ key return (liftM (take maxFileSize) result) -- | Parses a key-value pair of the form "key=value" from 'str', fails -- with 'desc' otherwise. parseKeyValue :: MonadFail m => String -> String -> m (String, String) parseKeyValue desc str = case sepSplit '=' str of [key, value] -> return (key, value) _ -> fail $ "Failed to parse key-value pair for " ++ desc -- | Parses a string containing an IP family parseIPFamily :: Int -> Result Socket.Family parseIPFamily fam | fam == AutoConf.pyAfInet4 = Ok Socket.AF_INET | fam == AutoConf.pyAfInet6 = Ok Socket.AF_INET6 | otherwise = Bad $ "Unknown af_family value: " ++ show fam -- | Read the primary IP family. getPrimaryIPFamily :: Maybe FilePath -> IO (Result Socket.Family) getPrimaryIPFamily optpath = do result <- readSSConfFile optpath (Just (show AutoConf.pyAfInet4)) SSPrimaryIpFamily return (liftM rStripSpace result >>= tryRead "Parsing af_family" >>= parseIPFamily) -- | Parse the nodes vm capable value from a 'String'. parseNodesVmCapable :: String -> Result [(String, Bool)] parseNodesVmCapable str = forM (lines str) $ \line -> do (key, val) <- parseKeyValue "Parsing node_vm_capable" line val' <- tryRead "Parsing value of node_vm_capable" val return (key, val') -- | Read and parse the nodes vm capable. getNodesVmCapable :: Maybe FilePath -> IO (Result [(String, Bool)]) getNodesVmCapable optPath = (parseNodesVmCapable =<<) <$> readSSConfFile optPath Nothing SSNodeVmCapable -- | Read the list of IP addresses of the master candidates of the cluster. getMasterCandidatesIps :: Maybe FilePath -> IO (Result [String]) getMasterCandidatesIps optPath = do result <- readSSConfFile optPath Nothing SSMasterCandidatesIps return $ liftM lines result -- | Read the name of the master node. getMasterNode :: Maybe FilePath -> IO (Result String) getMasterNode optPath = do result <- readSSConfFile optPath Nothing SSMasterNode return (liftM rStripSpace result) -- | Parse the list of enabled hypervisors from a 'String'. parseHypervisorList :: String -> Result [Hypervisor] parseHypervisorList str = mapM Types.hypervisorFromRaw $ lines str -- | Read and parse the list of enabled hypervisors. getHypervisorList :: Maybe FilePath -> IO (Result [Hypervisor]) getHypervisorList optPath = (parseHypervisorList =<<) <$> readSSConfFile optPath Nothing SSHypervisorList -- | Parse whether user shutdown is enabled from a 'String'. parseEnabledUserShutdown :: String -> Result Bool parseEnabledUserShutdown str = tryRead "Parsing enabled_user_shutdown" (rStripSpace str) -- | Read and parse whether user shutdown is enabled. getEnabledUserShutdown :: Maybe FilePath -> IO (Result Bool) getEnabledUserShutdown optPath = (parseEnabledUserShutdown =<<) <$> readSSConfFile optPath Nothing SSEnabledUserShutdown -- * Working with the whole ssconf map -- | The data type used for representing the ssconf. newtype SSConf = SSConf { getSSConf :: M.Map SSKey [String] } deriving (Eq, Ord, Show) instance J.JSON SSConf where showJSON = J.showJSON . GenericContainer . getSSConf readJSON = liftM (SSConf . fromContainer) . J.readJSON emptySSConf :: SSConf emptySSConf = SSConf M.empty ganeti-3.1.0~rc2/src/Ganeti/Storage/000075500000000000000000000000001476477700300172065ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Storage/Diskstats/000075500000000000000000000000001476477700300211575ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Storage/Diskstats/Parser.hs000064400000000000000000000051241476477700300227510ustar00rootroot00000000000000{-# LANGUAGE OverloadedStrings #-} {-| Diskstats proc file parser This module holds the definition of the parser that extracts status information about the disks of the system from the @/proc/diskstats@ file. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Storage.Diskstats.Parser (diskstatsParser) where import qualified Data.Attoparsec.Text as A import qualified Data.Attoparsec.Combinator as AC import Data.Attoparsec.Text (Parser) import Ganeti.Parsers import Ganeti.Storage.Diskstats.Types -- * Parser implementation -- | The parser for one line of the diskstatus file. oneDiskstatsParser :: Parser Diskstats oneDiskstatsParser = let majorP = numberP minorP = numberP nameP = stringP readsNumP = numberP mergedReadsP = numberP secReadP = numberP timeReadP = numberP writesP = numberP mergedWritesP = numberP secWrittenP = numberP timeWriteP = numberP iosP = numberP timeIOP = numberP wIOmillisP = numberP in Diskstats <$> majorP <*> minorP <*> nameP <*> readsNumP <*> mergedReadsP <*> secReadP <*> timeReadP <*> writesP <*> mergedWritesP <*> secWrittenP <*> timeWriteP <*> iosP <*> timeIOP <*> wIOmillisP <* A.endOfLine -- | The parser for a whole diskstatus file. diskstatsParser :: Parser [Diskstats] diskstatsParser = oneDiskstatsParser `AC.manyTill` A.endOfInput ganeti-3.1.0~rc2/src/Ganeti/Storage/Diskstats/Types.hs000064400000000000000000000043671476477700300226310ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Diskstats data types This module holds the definition of the data types describing the status of the disks according to the information contained in @/proc/diskstats@. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Storage.Diskstats.Types ( Diskstats(..) ) where import Ganeti.THH -- | This is the format of the report produced by each data collector. $(buildObject "Diskstats" "ds" [ simpleField "major" [t| Int |] , simpleField "minor" [t| Int |] , simpleField "name" [t| String |] , simpleField "readsNum" [t| Int |] , simpleField "mergedReads" [t| Int |] , simpleField "secRead" [t| Int |] , simpleField "timeRead" [t| Int |] , simpleField "writes" [t| Int |] , simpleField "mergedWrites" [t| Int |] , simpleField "secWritten" [t| Int |] , simpleField "timeWrite" [t| Int |] , simpleField "ios" [t| Int |] , simpleField "timeIO" [t| Int |] , simpleField "wIOmillis" [t| Int |] ]) ganeti-3.1.0~rc2/src/Ganeti/Storage/Drbd/000075500000000000000000000000001476477700300200615ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Storage/Drbd/Parser.hs000064400000000000000000000325501476477700300216560ustar00rootroot00000000000000{-# LANGUAGE OverloadedStrings #-} {-| DRBD proc file parser This module holds the definition of the parser that extracts status information from the DRBD proc file. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Storage.Drbd.Parser (drbdStatusParser, commaIntParser) where import Control.Applicative ((<|>)) import qualified Data.Attoparsec.Text as A import qualified Data.Attoparsec.Combinator as AC import Data.Attoparsec.Text (Parser) import Data.List import Data.Maybe import Data.Text (Text, unpack) import Ganeti.Storage.Drbd.Types -- | Our own space-skipping function, because A.skipSpace also skips -- newline characters. It skips ZERO or more spaces, so it does not -- fail if there are no spaces. skipSpaces :: Parser () skipSpaces = A.skipWhile A.isHorizontalSpace -- | Skips spaces and the given string, then executes a parser and -- returns its result. skipSpacesAndString :: Text -> Parser a -> Parser a skipSpacesAndString s parser = skipSpaces *> A.string s *> parser -- | Predicate verifying (potentially bad) end of lines isBadEndOfLine :: Char -> Bool isBadEndOfLine c = (c == '\0') || A.isEndOfLine c -- | Takes a parser and returns it with the content wrapped in a Maybe -- object. The resulting parser never fails, but contains Nothing if -- it couldn't properly parse the string. optional :: Parser a -> Parser (Maybe a) optional parser = (Just <$> parser) <|> pure Nothing -- | The parser for a whole DRBD status file. drbdStatusParser :: [DrbdInstMinor] -> Parser DRBDStatus drbdStatusParser instMinor = DRBDStatus <$> versionInfoParser <*> deviceParser instMinor `AC.manyTill` A.endOfInput <* A.endOfInput -- | The parser for the version information lines. versionInfoParser :: Parser VersionInfo versionInfoParser = do versionF <- optional versionP apiF <- optional apiP protoF <- optional protoP srcVersionF <- optional srcVersion ghF <- fmap unpack <$> optional gh builderF <- fmap unpack <$> optional builder if isNothing versionF && isNothing apiF && isNothing protoF && isNothing srcVersionF && isNothing ghF && isNothing builderF then fail "versionInfo" else pure $ VersionInfo versionF apiF protoF srcVersionF ghF builderF where versionP = A.string "version:" *> skipSpaces *> fmap unpack (A.takeWhile $ not . A.isHorizontalSpace) apiP = skipSpacesAndString "(api:" . fmap unpack $ A.takeWhile (/= '/') protoP = A.string "/proto:" *> fmap Data.Text.unpack (A.takeWhile (/= ')')) <* A.takeTill A.isEndOfLine <* A.endOfLine srcVersion = A.string "srcversion:" *> AC.skipMany1 A.space *> fmap unpack (A.takeTill A.isEndOfLine) <* A.endOfLine gh = A.string "GIT-hash:" *> skipSpaces *> A.takeWhile (not . A.isHorizontalSpace) builder = skipSpacesAndString "build by" $ skipSpaces *> A.takeTill A.isEndOfLine <* A.endOfLine -- | The parser for a (multi-line) string representing a device. deviceParser :: [DrbdInstMinor] -> Parser DeviceInfo deviceParser instMinor = do _ <- additionalEOL deviceNum <- skipSpaces *> A.decimal <* A.char ':' cs <- skipSpacesAndString "cs:" connStateParser if cs == Unconfigured then do _ <- additionalEOL return $ UnconfiguredDevice deviceNum else do ro <- skipSpaces *> skipRoleString *> localRemoteParser roleParser ds <- skipSpacesAndString "ds:" $ localRemoteParser diskStateParser replicProtocol <- A.space *> A.anyChar io <- skipSpaces *> ioFlagsParser <* A.skipWhile isBadEndOfLine pIndicators <- perfIndicatorsParser syncS <- conditionalSyncStatusParser cs reS <- optional resyncParser act <- optional actLogParser _ <- additionalEOL let inst = find ((deviceNum ==) . dimMinor) instMinor iName = fmap dimInstName inst return $ DeviceInfo deviceNum cs ro ds replicProtocol io pIndicators syncS reS act iName where conditionalSyncStatusParser SyncSource = Just <$> syncStatusParser conditionalSyncStatusParser SyncTarget = Just <$> syncStatusParser conditionalSyncStatusParser _ = pure Nothing skipRoleString = A.string "ro:" <|> A.string "st:" resyncParser = skipSpacesAndString "resync:" additionalInfoParser actLogParser = skipSpacesAndString "act_log:" additionalInfoParser additionalEOL = A.skipWhile A.isEndOfLine -- | The parser for the connection state. connStateParser :: Parser ConnState connStateParser = standAlone <|> disconnecting <|> unconnected <|> timeout <|> brokenPipe <|> networkFailure <|> protocolError <|> tearDown <|> wfConnection <|> wfReportParams <|> connected <|> startingSyncS <|> startingSyncT <|> wfBitMapS <|> wfBitMapT <|> wfSyncUUID <|> syncSource <|> syncTarget <|> pausedSyncS <|> pausedSyncT <|> verifyS <|> verifyT <|> unconfigured where standAlone = A.string "StandAlone" *> pure StandAlone disconnecting = A.string "Disconnectiog" *> pure Disconnecting unconnected = A.string "Unconnected" *> pure Unconnected timeout = A.string "Timeout" *> pure Timeout brokenPipe = A.string "BrokenPipe" *> pure BrokenPipe networkFailure = A.string "NetworkFailure" *> pure NetworkFailure protocolError = A.string "ProtocolError" *> pure ProtocolError tearDown = A.string "TearDown" *> pure TearDown wfConnection = A.string "WFConnection" *> pure WFConnection wfReportParams = A.string "WFReportParams" *> pure WFReportParams connected = A.string "Connected" *> pure Connected startingSyncS = A.string "StartingSyncS" *> pure StartingSyncS startingSyncT = A.string "StartingSyncT" *> pure StartingSyncT wfBitMapS = A.string "WFBitMapS" *> pure WFBitMapS wfBitMapT = A.string "WFBitMapT" *> pure WFBitMapT wfSyncUUID = A.string "WFSyncUUID" *> pure WFSyncUUID syncSource = A.string "SyncSource" *> pure SyncSource syncTarget = A.string "SyncTarget" *> pure SyncTarget pausedSyncS = A.string "PausedSyncS" *> pure PausedSyncS pausedSyncT = A.string "PausedSyncT" *> pure PausedSyncT verifyS = A.string "VerifyS" *> pure VerifyS verifyT = A.string "VerifyT" *> pure VerifyT unconfigured = A.string "Unconfigured" *> pure Unconfigured -- | Parser for recognizing strings describing two elements of the -- same type separated by a '/'. The first one is considered local, -- the second remote. localRemoteParser :: Parser a -> Parser (LocalRemote a) localRemoteParser parser = LocalRemote <$> parser <*> (A.char '/' *> parser) -- | The parser for resource roles. roleParser :: Parser Role roleParser = primary <|> secondary <|> unknown where primary = A.string "Primary" *> pure Primary secondary = A.string "Secondary" *> pure Secondary unknown = A.string "Unknown" *> pure Unknown -- | The parser for disk states. diskStateParser :: Parser DiskState diskStateParser = diskless <|> attaching <|> failed <|> negotiating <|> inconsistent <|> outdated <|> dUnknown <|> consistent <|> upToDate where diskless = A.string "Diskless" *> pure Diskless attaching = A.string "Attaching" *> pure Attaching failed = A.string "Failed" *> pure Failed negotiating = A.string "Negotiating" *> pure Negotiating inconsistent = A.string "Inconsistent" *> pure Inconsistent outdated = A.string "Outdated" *> pure Outdated dUnknown = A.string "DUnknown" *> pure DUnknown consistent = A.string "Consistent" *> pure Consistent upToDate = A.string "UpToDate" *> pure UpToDate -- | The parser for I/O flags. ioFlagsParser :: Parser String ioFlagsParser = fmap unpack . A.takeWhile $ not . isBadEndOfLine -- | The parser for performance indicators. perfIndicatorsParser :: Parser PerfIndicators perfIndicatorsParser = PerfIndicators <$> skipSpacesAndString "ns:" A.decimal <*> skipSpacesAndString "nr:" A.decimal <*> skipSpacesAndString "dw:" A.decimal <*> skipSpacesAndString "dr:" A.decimal <*> skipSpacesAndString "al:" A.decimal <*> skipSpacesAndString "bm:" A.decimal <*> skipSpacesAndString "lo:" A.decimal <*> skipSpacesAndString "pe:" A.decimal <*> skipSpacesAndString "ua:" A.decimal <*> skipSpacesAndString "ap:" A.decimal <*> optional (skipSpacesAndString "ep:" A.decimal) <*> optional (skipSpacesAndString "wo:" A.anyChar) <*> optional (skipSpacesAndString "oos:" A.decimal) <* skipSpaces <* A.endOfLine -- | The parser for the syncronization status. syncStatusParser :: Parser SyncStatus syncStatusParser = do _ <- statusBarParser percent <- skipSpacesAndString "sync'ed:" $ skipSpaces *> A.double <* A.char '%' partSyncSize <- skipSpaces *> A.char '(' *> A.decimal totSyncSize <- A.char '/' *> A.decimal <* A.char ')' sizeUnit <- sizeUnitParser <* optional A.endOfLine timeToEnd <- skipSpacesAndString "finish:" $ skipSpaces *> timeParser sp <- skipSpacesAndString "speed:" $ skipSpaces *> commaIntParser <* skipSpaces <* A.char '(' <* commaIntParser <* A.char ')' w <- skipSpacesAndString "want:" ( skipSpaces *> (Just <$> commaIntParser) ) <|> pure Nothing sSizeUnit <- skipSpaces *> sizeUnitParser sTimeUnit <- A.char '/' *> timeUnitParser _ <- A.endOfLine return $ SyncStatus percent partSyncSize totSyncSize sizeUnit timeToEnd sp w sSizeUnit sTimeUnit -- | The parser for recognizing (and discarding) the sync status bar. statusBarParser :: Parser () statusBarParser = skipSpaces *> A.char '[' *> A.skipWhile (== '=') *> A.skipWhile (== '>') *> A.skipWhile (== '.') *> A.char ']' *> pure () -- | The parser for recognizing data size units (only the ones -- actually found in DRBD files are implemented). sizeUnitParser :: Parser SizeUnit sizeUnitParser = kilobyte <|> megabyte where kilobyte = A.string "K" *> pure KiloByte megabyte = A.string "M" *> pure MegaByte -- | The parser for recognizing time (hh:mm:ss). timeParser :: Parser Time timeParser = Time <$> h <*> m <*> s where h = A.decimal :: Parser Int m = A.char ':' *> A.decimal :: Parser Int s = A.char ':' *> A.decimal :: Parser Int -- | The parser for recognizing time units (only the ones actually -- found in DRBD files are implemented). timeUnitParser :: Parser TimeUnit timeUnitParser = second where second = A.string "sec" *> pure Second -- | Haskell does not recognise ',' as the thousands separator every 3 -- digits but DRBD uses it, so we need an ah-hoc parser. -- If a number beginning with more than 3 digits without a comma is -- parsed, only the first 3 digits are considered to be valid, the rest -- is not consumed, and left for further parsing. commaIntParser :: Parser Int commaIntParser = do first <- AC.count 3 A.digit <|> AC.count 2 A.digit <|> AC.count 1 A.digit allDigits <- commaIntHelper (read first) pure allDigits -- | Helper (triplet parser) for the commaIntParser commaIntHelper :: Int -> Parser Int commaIntHelper acc = nextTriplet <|> end where nextTriplet = do _ <- A.char ',' triplet <- AC.count 3 A.digit commaIntHelper $ acc * 1000 + (read triplet :: Int) end = pure acc :: Parser Int -- | Parser for the additional information provided by DRBD <= 8.0. additionalInfoParser::Parser AdditionalInfo additionalInfoParser = AdditionalInfo <$> skipSpacesAndString "used:" A.decimal <*> (A.char '/' *> A.decimal) <*> skipSpacesAndString "hits:" A.decimal <*> skipSpacesAndString "misses:" A.decimal <*> skipSpacesAndString "starving:" A.decimal <*> skipSpacesAndString "dirty:" A.decimal <*> skipSpacesAndString "changed:" A.decimal <* A.endOfLine ganeti-3.1.0~rc2/src/Ganeti/Storage/Drbd/Types.hs000064400000000000000000000340441476477700300215260ustar00rootroot00000000000000{-| DRBD Data Types This module holds the definition of the data types describing the status of DRBD. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Storage.Drbd.Types ( DRBDStatus(..) , VersionInfo(..) , DeviceInfo(..) , ConnState(..) , LocalRemote(..) , Role(..) , DiskState(..) , PerfIndicators(..) , SyncStatus(..) , SizeUnit(..) , Time(..) , TimeUnit(..) , AdditionalInfo(..) , DrbdInstMinor(..) ) where import Control.Monad import Text.JSON import Text.Printf import Ganeti.JSON (optFieldsToObj, optionalJSField) --TODO: consider turning deviceInfos into an IntMap -- | Data type contaning all the data about the status of DRBD. data DRBDStatus = DRBDStatus { versionInfo :: VersionInfo -- ^ Version information about DRBD , deviceInfos :: [DeviceInfo] -- ^ Per-minor information } deriving (Eq, Show) -- | The DRBDStatus instance of JSON. instance JSON DRBDStatus where showJSON d = makeObj [ ("versionInfo", showJSON $ versionInfo d) , ("deviceInfos", showJSONs $ deviceInfos d) ] readJSON = error "JSON read instance not implemented for type DRBDStatus" -- | Data type describing the DRBD version. data VersionInfo = VersionInfo { version :: Maybe String -- ^ DRBD driver version , api :: Maybe String -- ^ The api version , proto :: Maybe String -- ^ The protocol version , srcversion :: Maybe String -- ^ The version of the source files , gitHash :: Maybe String -- ^ Git hash of the source files , buildBy :: Maybe String -- ^ Who built the binary (and, -- optionally, when) } deriving (Eq, Show) -- | The VersionInfo instance of JSON. instance JSON VersionInfo where showJSON (VersionInfo versionF apiF protoF srcversionF gitHashF buildByF) = optFieldsToObj [ optionalJSField "version" versionF , optionalJSField "api" apiF , optionalJSField "proto" protoF , optionalJSField "srcversion" srcversionF , optionalJSField "gitHash" gitHashF , optionalJSField "buildBy" buildByF ] readJSON = error "JSON read instance not implemented for type VersionInfo" -- | Data type describing a device. data DeviceInfo = UnconfiguredDevice Int -- ^ An DRBD minor marked as unconfigured | -- | A configured DRBD minor DeviceInfo { minorNumber :: Int -- ^ The minor index of the device , connectionState :: ConnState -- ^ State of the connection , resourceRoles :: LocalRemote Role -- ^ Roles of the resources , diskStates :: LocalRemote DiskState -- ^ Status of the disks , replicationProtocol :: Char -- ^ The replication protocol -- being used , ioFlags :: String -- ^ The input/output flags , perfIndicators :: PerfIndicators -- ^ Performance indicators , syncStatus :: Maybe SyncStatus -- ^ The status of the -- syncronization of the disk -- (only if it is happening) , resync :: Maybe AdditionalInfo -- ^ Additional info by DRBD 8.0 , actLog :: Maybe AdditionalInfo -- ^ Additional info by DRBD 8.0 , instName :: Maybe String -- ^ The name of the associated -- instance } deriving (Eq, Show) -- | The DeviceInfo instance of JSON. instance JSON DeviceInfo where showJSON (UnconfiguredDevice num) = makeObj [ ("minor", showJSON num) , ("connectionState", showJSON Unconfigured) ] showJSON (DeviceInfo minorNumberF connectionStateF (LocalRemote localRole remoteRole) (LocalRemote localState remoteState) replicProtocolF ioFlagsF perfIndicatorsF syncStatusF _ _ instNameF) = optFieldsToObj [ Just ("minor", showJSON minorNumberF) , Just ("connectionState", showJSON connectionStateF) , Just ("localRole", showJSON localRole) , Just ("remoteRole", showJSON remoteRole) , Just ("localState", showJSON localState) , Just ("remoteState", showJSON remoteState) , Just ("replicationProtocol", showJSON replicProtocolF) , Just ("ioFlags", showJSON ioFlagsF) , Just ("perfIndicators", showJSON perfIndicatorsF) , optionalJSField "syncStatus" syncStatusF , Just ("instance", maybe JSNull showJSON instNameF) ] readJSON = error "JSON read instance not implemented for type DeviceInfo" -- | Data type describing the state of the connection. data ConnState = StandAlone -- ^ No network configuration available | Disconnecting -- ^ Temporary state during disconnection | Unconnected -- ^ Prior to a connection attempt | Timeout -- ^ Following a timeout in the communication | BrokenPipe -- ^ After the connection to the peer was lost | NetworkFailure -- ^ After the connection to the partner was lost | ProtocolError -- ^ After the connection to the partner was lost | TearDown -- ^ The peer is closing the connection | WFConnection -- ^ Waiting for the peer to become visible | WFReportParams -- ^ Waiting for first packet from peer | Connected -- ^ Connected, data mirroring active | StartingSyncS -- ^ Source of a full sync started by admin | StartingSyncT -- ^ Target of a full sync started by admin | WFBitMapS -- ^ Source of a just starting partial sync | WFBitMapT -- ^ Target of a just starting partial sync | WFSyncUUID -- ^ Synchronization is about to begin | SyncSource -- ^ Source of a running synchronization | SyncTarget -- ^ Target of a running synchronization | PausedSyncS -- ^ Source of a paused synchronization | PausedSyncT -- ^ Target of a paused synchronization | VerifyS -- ^ Source of a running verification | VerifyT -- ^ Target of a running verification | Unconfigured -- ^ The device is not configured deriving (Show, Eq) -- | The ConnState instance of JSON. instance JSON ConnState where showJSON = showJSON . show readJSON = error "JSON read instance not implemented for type ConnState" -- | Algebraic data type describing something that has a local and a remote -- value. data LocalRemote a = LocalRemote { local :: a -- ^ The local value , remote :: a -- ^ The remote value } deriving (Eq, Show) -- | Data type describing. data Role = Primary -- ^ The device role is primary | Secondary -- ^ The device role is secondary | Unknown -- ^ The device role is unknown deriving (Eq, Show) -- | The Role instance of JSON. instance JSON Role where showJSON = showJSON . show readJSON = error "JSON read instance not implemented for type Role" -- | Data type describing disk states. data DiskState = Diskless -- ^ No local block device assigned to the DRBD driver | Attaching -- ^ Reading meta data | Failed -- ^ I/O failure | Negotiating -- ^ "Attach" on an already-connected device | Inconsistent -- ^ The data is inconsistent between nodes. | Outdated -- ^ Data consistent but outdated | DUnknown -- ^ No network connection available | Consistent -- ^ Consistent data, but without network connection | UpToDate -- ^ Consistent, up-to-date. This is the normal state deriving (Eq, Show) -- | The DiskState instance of JSON. instance JSON DiskState where showJSON = showJSON . show readJSON = error "JSON read instance not implemented for type DiskState" -- | Data type containing data about performance indicators. data PerfIndicators = PerfIndicators { networkSend :: Int -- ^ KiB of data sent on the network , networkReceive :: Int -- ^ KiB of data received from the network , diskWrite :: Int -- ^ KiB of data written on local disk , diskRead :: Int -- ^ KiB of data read from local disk , activityLog :: Int -- ^ Number of updates of the activity log , bitMap :: Int -- ^ Number of updates to the bitmap area of the metadata , localCount :: Int -- ^ Number of open requests to the local I/O subsystem , pending :: Int -- ^ Num of requests sent to the partner but not yet answered , unacknowledged :: Int -- ^ Num of requests received by the partner but still -- to be answered , applicationPending :: Int -- ^ Num of block I/O requests forwarded -- to DRBD but that have not yet been -- answered , epochs :: Maybe Int -- ^ Number of epoch objects , writeOrder :: Maybe Char -- ^ Currently used write ordering method , outOfSync :: Maybe Int -- ^ KiB of storage currently out of sync } deriving (Eq, Show) -- | The PerfIndicators instance of JSON. instance JSON PerfIndicators where showJSON p = optFieldsToObj [ Just ("networkSend", showJSON $ networkSend p) , Just ("networkReceive", showJSON $ networkReceive p) , Just ("diskWrite", showJSON $ diskWrite p) , Just ("diskRead", showJSON $ diskRead p) , Just ("activityLog", showJSON $ activityLog p) , Just ("bitMap", showJSON $ bitMap p) , Just ("localCount", showJSON $ localCount p) , Just ("pending", showJSON $ pending p) , Just ("unacknowledged", showJSON $ unacknowledged p) , Just ("applicationPending", showJSON $ applicationPending p) , optionalJSField "epochs" $ epochs p , optionalJSField "writeOrder" $ writeOrder p , optionalJSField "outOfSync" $ outOfSync p ] readJSON = error "JSON read instance not implemented for type PerfIndicators" -- | Data type containing data about the synchronization status of a device. data SyncStatus = SyncStatus { percentage :: Double -- ^ Percentage of syncronized data , partialSyncSize :: Int -- ^ Numerator of the fraction of synced data , totalSyncSize :: Int -- ^ Denominator of the fraction of -- synced data , syncUnit :: SizeUnit -- ^ Measurement unit of the previous -- fraction , timeToFinish :: Time -- ^ Expected time before finishing -- the syncronization , speed :: Int -- ^ Speed of the syncronization , want :: Maybe Int -- ^ Want of the syncronization , speedSizeUnit :: SizeUnit -- ^ Size unit of the speed , speedTimeUnit :: TimeUnit -- ^ Time unit of the speed } deriving (Eq, Show) -- | The SyncStatus instance of JSON. instance JSON SyncStatus where showJSON s = optFieldsToObj [ Just ("percentage", showJSON $ percentage s) , Just ("progress", showJSON $ show (partialSyncSize s) ++ "/" ++ show (totalSyncSize s)) , Just ("progressUnit", showJSON $ syncUnit s) , Just ("timeToFinish", showJSON $ timeToFinish s) , Just ("speed", showJSON $ speed s) , optionalJSField "want" $ want s , Just ("speedUnit", showJSON $ show (speedSizeUnit s) ++ "/" ++ show (speedTimeUnit s)) ] readJSON = error "JSON read instance not implemented for type SyncStatus" -- | Data type describing a size unit for memory. data SizeUnit = KiloByte | MegaByte deriving (Eq, Show) -- | The SizeUnit instance of JSON. instance JSON SizeUnit where showJSON = showJSON . show readJSON = error "JSON read instance not implemented for type SizeUnit" -- | Data type describing a time (hh:mm:ss). data Time = Time { hour :: Int , min :: Int , sec :: Int } deriving (Eq, Show) -- | The Time instance of JSON. instance JSON Time where showJSON (Time h m s) = showJSON (printf "%02d:%02d:%02d" h m s :: String) readJSON = error "JSON read instance not implemented for type Time" -- | Data type describing a time unit. data TimeUnit = Second deriving (Eq, Show) -- | The TimeUnit instance of JSON. instance JSON TimeUnit where showJSON Second = showJSON "Second" readJSON = error "JSON read instance not implemented for type TimeUnit" -- | Additional device-specific cache-like information produced by -- drbd <= 8.0. -- -- Internal debug information exported by old DRBD versions. -- Undocumented both in DRBD and here. data AdditionalInfo = AdditionalInfo { partialUsed :: Int , totalUsed :: Int , hits :: Int , misses :: Int , starving :: Int , dirty :: Int , changed :: Int } deriving (Eq, Show) -- | Data type representing the pairing of a DRBD minor with an instance. data DrbdInstMinor = DrbdInstMinor { dimNode :: String , dimMinor :: Int , dimInstName :: String , dimDiskIdx :: String , dimRole :: String , dimPeer :: String } deriving (Show) -- | The DrbdInstMinor instance of JSON. instance JSON DrbdInstMinor where showJSON (DrbdInstMinor a b c d e f) = JSArray [ showJSON a , showJSON b , showJSON c , showJSON d , showJSON e , showJSON f ] readJSON (JSArray [a, b, c, d, e, f]) = DrbdInstMinor `fmap` readJSON a `ap` readJSON b `ap` readJSON c `ap` readJSON d `ap` readJSON e `ap` readJSON f readJSON _ = fail "Unable to read a DrbdInstMinor" ganeti-3.1.0~rc2/src/Ganeti/Storage/Lvm/000075500000000000000000000000001476477700300177445ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Storage/Lvm/LVParser.hs000064400000000000000000000104221476477700300217750ustar00rootroot00000000000000{-# LANGUAGE OverloadedStrings #-} {-| Logical Volumer information parser This module holds the definition of the parser that extracts status information about the logical volumes (LVs) of the system from the output of the @lvs@ command. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Storage.Lvm.LVParser (lvParser, lvCommand, lvParams) where import qualified Data.Attoparsec.Text as A import qualified Data.Attoparsec.Combinator as AC import Data.Attoparsec.Text (Parser) import Data.Text (unpack) import Ganeti.Storage.Lvm.Types -- | The separator of the fields returned by @lvs@ lvsSeparator :: Char lvsSeparator = ';' -- * Utility functions -- | Our own space-skipping function, because A.skipSpace also skips -- newline characters. It skips ZERO or more spaces, so it does not -- fail if there are no spaces. skipSpaces :: Parser () skipSpaces = A.skipWhile A.isHorizontalSpace -- | A parser recognizing a number of bytes, represented as a number preceeded -- by a separator and followed by the "B" character. bytesP :: Parser Int bytesP = A.char lvsSeparator *> A.decimal <* A.char 'B' -- | A parser recognizing a number discarding the preceeding separator intP :: Parser Int intP = A.char lvsSeparator *> A.signed A.decimal -- | A parser recognizing a string starting with and closed by a separator (both -- are discarded) stringP :: Parser String stringP = A.char lvsSeparator *> fmap unpack (A.takeWhile (`notElem` [ lvsSeparator , '\n'] )) -- * Parser implementation -- | The command providing the data, in the format the parser expects lvCommand :: String lvCommand = "lvs" -- | The parameters for getting the data in the format the parser expects lvParams :: [String] lvParams = [ "--noheadings" , "--units", "B" , "--separator", ";" , "-o", "lv_uuid,lv_name,lv_attr,lv_major,lv_minor,lv_kernel_major\ \,lv_kernel_minor,lv_size,seg_count,lv_tags,modules,vg_uuid,vg_name,segtype\ \,seg_start,seg_start_pe,seg_size,seg_tags,seg_pe_ranges,devices" ] -- | The parser for one line of the diskstatus file. oneLvParser :: Parser LVInfo oneLvParser = let uuidP = skipSpaces *> fmap unpack (A.takeWhile (/= lvsSeparator)) nameP = stringP attrP = stringP majorP = intP minorP = intP kernelMajorP = intP kernelMinorP = intP sizeP = bytesP segCountP = intP tagsP = stringP modulesP = stringP vgUuidP = stringP vgNameP = stringP segtypeP = stringP segStartP = bytesP segStartPeP = intP segSizeP = bytesP segTagsP = stringP segPeRangesP = stringP devicesP = stringP in LVInfo <$> uuidP <*> nameP <*> attrP <*> majorP <*> minorP <*> kernelMajorP <*> kernelMinorP <*> sizeP <*> segCountP <*> tagsP <*> modulesP <*> vgUuidP <*> vgNameP <*> segtypeP <*> segStartP <*> segStartPeP <*> segSizeP <*> segTagsP <*> segPeRangesP <*> devicesP <*> return Nothing <* A.endOfLine -- | The parser for a whole diskstatus file. lvParser :: Parser [LVInfo] lvParser = oneLvParser `AC.manyTill` A.endOfInput ganeti-3.1.0~rc2/src/Ganeti/Storage/Lvm/Types.hs000064400000000000000000000052431476477700300214100ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| LVM data types This module holds the definition of the data types describing the status of the disks according to LVM (and particularly the lvs tool). -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Storage.Lvm.Types ( LVInfo(..) ) where import Ganeti.THH -- | This is the format of the report produced by each data collector. $(buildObject "LVInfo" "lvi" [ simpleField "uuid" [t| String |] , simpleField "name" [t| String |] , simpleField "attr" [t| String |] , simpleField "major" [t| Int |] , simpleField "minor" [t| Int |] , simpleField "kernel_major" [t| Int |] , simpleField "kernel_minor" [t| Int |] , simpleField "size" [t| Int |] , simpleField "seg_count" [t| Int |] , simpleField "tags" [t| String |] , simpleField "modules" [t| String |] , simpleField "vg_uuid" [t| String |] , simpleField "vg_name" [t| String |] , simpleField "segtype" [t| String |] , simpleField "seg_start" [t| Int |] , simpleField "seg_start_pe" [t| Int |] , simpleField "seg_size" [t| Int |] , simpleField "seg_tags" [t| String |] , simpleField "seg_pe_ranges" [t| String |] , simpleField "devices" [t| String |] , optionalNullSerField $ simpleField "instance" [t| String |] ]) ganeti-3.1.0~rc2/src/Ganeti/Storage/Utils.hs000064400000000000000000000073431476477700300206510ustar00rootroot00000000000000{-| Implementation of Utility functions for storage -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Storage.Utils ( getStorageUnitsOfNode , nodesWithValidConfig ) where import Ganeti.Config import Ganeti.Objects import Ganeti.Types import qualified Ganeti.Types as T import Control.Monad import Data.List (nub) import Data.Maybe -- | Get the cluster's default storage unit for a given disk template getDefaultStorageKey :: ConfigData -> DiskTemplate -> Maybe StorageKey getDefaultStorageKey cfg T.DTDrbd8 = clusterVolumeGroupName $ configCluster cfg getDefaultStorageKey cfg T.DTPlain = clusterVolumeGroupName $ configCluster cfg getDefaultStorageKey cfg T.DTFile = Just (clusterFileStorageDir $ configCluster cfg) getDefaultStorageKey _ _ = Nothing -- | Get the cluster's default spindle storage unit getDefaultSpindleSU :: ConfigData -> (StorageType, Maybe StorageKey) getDefaultSpindleSU cfg = (T.StorageLvmPv, clusterVolumeGroupName $ configCluster cfg) -- | Get the cluster's storage units from the configuration getClusterStorageUnitRaws :: ConfigData -> [StorageUnitRaw] getClusterStorageUnitRaws cfg = foldSUs (nub (maybe_units ++ [spindle_unit])) where disk_templates = clusterEnabledDiskTemplates $ configCluster cfg storage_types = map diskTemplateToStorageType disk_templates maybe_units = zip storage_types (map (getDefaultStorageKey cfg) disk_templates) spindle_unit = getDefaultSpindleSU cfg -- | fold the storage unit list by sorting out the ones without keys foldSUs :: [(StorageType, Maybe StorageKey)] -> [StorageUnitRaw] foldSUs = foldr ff [] where ff (st, Just sk) acc = SURaw st sk : acc ff (_, Nothing) acc = acc -- | Gets the value of the 'exclusive storage' flag of the node getExclusiveStorage :: ConfigData -> Node -> Maybe Bool getExclusiveStorage cfg n = liftM ndpExclusiveStorage (getNodeNdParams cfg n) -- | Determines whether a node's config contains an 'exclusive storage' flag hasExclusiveStorageFlag :: ConfigData -> Node -> Bool hasExclusiveStorageFlag cfg = isJust . getExclusiveStorage cfg -- | Filter for nodes with a valid config nodesWithValidConfig :: ConfigData -> [Node] -> [Node] nodesWithValidConfig cfg = filter (hasExclusiveStorageFlag cfg) -- | Get the storage units of the node getStorageUnitsOfNode :: ConfigData -> Node -> [StorageUnit] getStorageUnitsOfNode cfg n = let clusterSUs = getClusterStorageUnitRaws cfg es = fromJust (getExclusiveStorage cfg n) in map (addParamsToStorageUnit es) clusterSUs ganeti-3.1.0~rc2/src/Ganeti/THH.hs000064400000000000000000002012451476477700300165650ustar00rootroot00000000000000{-# LANGUAGE ParallelListComp, TemplateHaskell, Rank2Types #-} {-| TemplateHaskell helper for Ganeti Haskell code. As TemplateHaskell require that splices be defined in a separate module, we combine all the TemplateHaskell functionality that HTools needs in this module (except the one for unittests). -} {- Copyright (C) 2011, 2012, 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.THH ( declareSADT , declareLADT , declareILADT , declareIADT , makeJSONInstance , deCamelCase , genOpID , genOpLowerStrip , genAllConstr , genAllOpIDs , PyValue(..) , PyValueEx(..) , OpCodeField(..) , OpCodeDescriptor(..) , genOpCode , genStrOfOp , genStrOfKey , genLuxiOp , Field (..) , simpleField , andRestArguments , withDoc , defaultField , notSerializeDefaultField , presentInForthcoming , optionalField , optionalNullSerField , makeOptional , renameField , customField , buildObject , buildObjectWithForthcoming , buildObjectSerialisation , buildParam , genException , excErrMsg , ssconfConstructorName ) where import Control.Arrow ((&&&), second) import Control.Applicative import Control.Lens.Type (Lens, Lens') import Control.Lens (lens, set, element) import Control.Monad import Control.Monad.Base () -- Needed to prevent spurious GHC linking errors. import Control.Monad.Fail (MonadFail) import Control.Monad.Writer (tell) import qualified Control.Monad.Trans as MT import Data.Attoparsec.Text () -- Needed to prevent spurious GHC 7.4 linking errors. -- See issue #683 and https://ghc.haskell.org/trac/ghc/ticket/4899 import Data.Char import Data.Function (on) import Data.List import Data.Maybe import qualified Data.Map as M import qualified Data.Semigroup as Sem import qualified Data.Set as S import qualified Data.Text as T import Language.Haskell.TH import Language.Haskell.TH.Syntax (lift) import qualified Text.JSON as JSON import Text.JSON.Pretty (pp_value) import Ganeti.JSON (readJSONWithDesc, fromObj, DictObject(..), ArrayObject(..), maybeFromObj, mkUsedKeys, showJSONtoDict, readJSONfromDict, branchOnField, addField, allUsedKeys) import Ganeti.PartialParams import Ganeti.PyValue import Ganeti.THH.PyType import Ganeti.THH.Compat -- * Exported types -- | Optional field information. data OptionalType = NotOptional -- ^ Field is not optional | OptionalOmitNull -- ^ Field is optional, null is not serialised | OptionalSerializeNull -- ^ Field is optional, null is serialised | AndRestArguments -- ^ Special field capturing all the remaining fields -- as plain JSON values deriving (Show, Eq) -- | Serialised field data type describing how to generate code for the field. -- Each field has a type, which isn't captured in the type of the data type, -- but is saved in the 'Q' monad in 'fieldType'. -- -- Let @t@ be a type we want to parametrize the field with. There are the -- following possible types of fields: -- -- [Mandatory with no default.] Then @fieldType@ holds @t@, -- @fieldDefault = Nothing@ and @fieldIsOptional = NotOptional@. -- -- [Field with a default value.] Then @fieldType@ holds @t@ and -- @fieldDefault = Just exp@ where @exp@ is an expression of type @t@ and -- @fieldIsOptional = NotOptional@. -- -- [Optional, no default value.] Then @fieldType@ holds @Maybe t@, -- @fieldDefault = Nothing@ and @fieldIsOptional@ is either -- 'OptionalOmitNull' or 'OptionalSerializeNull'. -- -- Optional fields with a default value are prohibited, as their main -- intention is to represent the information that a request didn't contain -- the field data. -- -- /Custom (de)serialization:/ -- Field can have custom (de)serialization functions that are stored in -- 'fieldRead' and 'fieldShow'. If they aren't provided, the default is to use -- 'readJSON' and 'showJSON' for the field's type @t@. If they are provided, -- the type of the contained deserializing expression must be -- -- @ -- [(String, JSON.JSValue)] -> JSON.JSValue -> JSON.Result t -- @ -- -- where the first argument carries the whole record in the case the -- deserializing function needs to process additional information. -- -- The type of the contained serializing experssion must be -- -- @ -- t -> (JSON.JSValue, [(String, JSON.JSValue)]) -- @ -- -- where the result can provide extra JSON fields to include in the output -- record (or just return @[]@ if they're not needed). -- -- Note that for optional fields the type appearing in the custom functions -- is still @t@. Therefore making a field optional doesn't change the -- functions. -- -- There is also a special type of optional field 'AndRestArguments' which -- allows to parse any additional arguments not covered by other fields. There -- can be at most one such special field and it's type must be -- @Map String JSON.JSValue@. See also 'andRestArguments'. data Field = Field { fieldName :: T.Text , fieldType :: Q Type -- ^ the type of the field, @t@ for non-optional fields, -- @Maybe t@ for optional ones. , fieldRead :: Maybe (Q Exp) -- ^ an optional custom deserialization function of type -- @[(String, JSON.JSValue)] -> JSON.JSValue -> -- JSON.Result t@ , fieldShow :: Maybe (Q Exp) -- ^ an optional custom serialization function of type -- @t -> (JSON.JSValue, [(String, JSON.JSValue)])@ , fieldExtraKeys :: [T.Text] -- ^ a list of extra keys added by 'fieldShow' , fieldDefault :: Maybe (Q Exp) -- ^ an optional default value of type @t@ , fieldSerializeDefault :: Bool -- ^ whether not presented default value will be -- serialized , fieldConstr :: Maybe T.Text , fieldIsOptional :: OptionalType -- ^ determines if a field is optional, and if yes, -- how , fieldDoc :: T.Text , fieldPresentInForthcoming :: Bool } -- | Generates a simple field. simpleField :: String -> Q Type -> Field simpleField fname ftype = Field { fieldName = T.pack fname , fieldType = ftype , fieldRead = Nothing , fieldShow = Nothing , fieldExtraKeys = [] , fieldDefault = Nothing , fieldSerializeDefault = True , fieldConstr = Nothing , fieldIsOptional = NotOptional , fieldDoc = T.pack "" , fieldPresentInForthcoming = False } -- | Generate an AndRestArguments catch-all field. andRestArguments :: String -> Field andRestArguments fname = Field { fieldName = T.pack fname , fieldType = [t| M.Map String JSON.JSValue |] , fieldRead = Nothing , fieldShow = Nothing , fieldExtraKeys = [] , fieldDefault = Nothing , fieldSerializeDefault = True , fieldConstr = Nothing , fieldIsOptional = AndRestArguments , fieldDoc = T.pack "" , fieldPresentInForthcoming = True } withDoc :: String -> Field -> Field withDoc doc field = field { fieldDoc = T.pack doc } -- | Sets the renamed constructor field. renameField :: String -> Field -> Field renameField constrName field = field { fieldConstr = Just $ T.pack constrName } -- | Sets the default value on a field (makes it optional with a -- default value). defaultField :: Q Exp -> Field -> Field defaultField defval field = field { fieldDefault = Just defval } -- | A defaultField which will be serialized only if it's value differs from -- a default value. notSerializeDefaultField :: Q Exp -> Field -> Field notSerializeDefaultField defval field = field { fieldDefault = Just defval , fieldSerializeDefault = False } -- | Mark a field as present in the forthcoming variant. presentInForthcoming :: Field -> Field presentInForthcoming field = field { fieldPresentInForthcoming = True } -- | Marks a field optional (turning its base type into a Maybe). optionalField :: Field -> Field optionalField field = field { fieldIsOptional = OptionalOmitNull } -- | Marks a field optional (turning its base type into a Maybe), but -- with 'Nothing' serialised explicitly as /null/. optionalNullSerField :: Field -> Field optionalNullSerField field = field { fieldIsOptional = OptionalSerializeNull } -- | Make a field optional, if it isn't already. makeOptional :: Field -> Field makeOptional field = if and [ fieldIsOptional field == NotOptional , isNothing $ fieldDefault field , not $ fieldPresentInForthcoming field ] then optionalField field else field -- | Sets custom functions on a field. customField :: Name -- ^ The name of the read function -> Name -- ^ The name of the show function -> [String] -- ^ The name of extra field keys -> Field -- ^ The original field -> Field -- ^ Updated field customField readfn showfn extra field = field { fieldRead = Just (varE readfn), fieldShow = Just (varE showfn) , fieldExtraKeys = (map T.pack extra) } -- | Computes the record name for a given field, based on either the -- string value in the JSON serialisation or the custom named if any -- exists. fieldRecordName :: Field -> String fieldRecordName (Field { fieldName = name, fieldConstr = alias }) = maybe (camelCase . T.unpack $ name) T.unpack alias -- | Computes the preferred variable name to use for the value of this -- field. If the field has a specific constructor name, then we use a -- first-letter-lowercased version of that; otherwise, we simply use -- the field name. See also 'fieldRecordName'. fieldVariable :: Field -> String fieldVariable f = case (fieldConstr f) of Just name -> ensureLower . T.unpack $ name _ -> map (\c -> if c == '-' then '_' else c) . T.unpack . fieldName $ f -- | Compute the actual field type (taking into account possible -- optional status). actualFieldType :: Field -> Q Type actualFieldType f | fieldIsOptional f `elem` [NotOptional, AndRestArguments] = t | otherwise = [t| Maybe $t |] where t = fieldType f -- | Checks that a given field is not optional (for object types or -- fields which should not allow this case). checkNonOptDef :: (MonadFail m) => Field -> m () checkNonOptDef field | fieldIsOptional field == OptionalOmitNull = failWith kOpt | fieldIsOptional field == OptionalSerializeNull = failWith kOpt | isJust (fieldDefault field) = failWith kDef | otherwise = return () where failWith kind = fail $ kind ++ " field " ++ name ++ " used in parameter declaration" name = T.unpack (fieldName field) kOpt = "Optional" kDef = "Default" -- | Construct a function that parses a field value. If the field has -- a custom 'fieldRead', it's applied to @o@ and used. Otherwise -- @JSON.readJSON@ is used. parseFn :: Field -- ^ The field definition -> Q Exp -- ^ The entire object in JSON object format -> Q Exp -- ^ The resulting function that parses a JSON message parseFn field o = let fnType = [t| JSON.JSValue -> JSON.Result $(fieldType field) |] expr = maybe [| readJSONWithDesc $(stringE . T.unpack $ fieldName field) |] (`appE` o) (fieldRead field) in sigE expr fnType -- | Produces the expression that will de-serialise a given -- field. Since some custom parsing functions might need to use the -- entire object, we do take and pass the object to any custom read -- functions. loadFn :: Field -- ^ The field definition -> Q Exp -- ^ The value of the field as existing in the JSON message -> Q Exp -- ^ The entire object in JSON object format -> Q Exp -- ^ Resulting expression loadFn field expr o = [| $expr >>= $(parseFn field o) |] -- | Just as 'loadFn', but for optional fields. loadFnOpt :: Field -- ^ The field definition -> Q Exp -- ^ The value of the field as existing in the JSON message -- as Maybe -> Q Exp -- ^ The entire object in JSON object format -> Q Exp -- ^ Resulting expression loadFnOpt field@(Field { fieldDefault = Just def }) expr o = case fieldIsOptional field of NotOptional -> [| $expr >>= maybe (return $def) $(parseFn field o) |] _ -> fail $ "Field " ++ (T.unpack . fieldName $ field) ++ ":\ \ A field can't be optional and\ \ have a default value at the same time." loadFnOpt field expr o = [| $expr >>= maybe (return Nothing) (liftM Just . $(parseFn field o)) |] -- * Internal types -- | A simple field, in constrast to the customisable 'Field' type. type SimpleField = (String, Q Type) -- | A definition for a single constructor for a simple object. type SimpleConstructor = (String, [SimpleField]) -- | A definition for ADTs with simple fields. type SimpleObject = [SimpleConstructor] -- | A type alias for an opcode constructor of a regular object. type OpCodeConstructor = (String, Q Type, String, [Field], String) -- | A type alias for a Luxi constructor of a regular object. type LuxiConstructor = (String, [Field]) -- * Helper functions -- | Ensure first letter is lowercase. -- -- Used to convert type name to function prefix, e.g. in @data Aa -> -- aaToRaw@. ensureLower :: String -> String ensureLower [] = [] ensureLower (x:xs) = toLower x:xs -- | Ensure first letter is uppercase. -- -- Used to convert constructor name to component ensureUpper :: String -> String ensureUpper [] = [] ensureUpper (x:xs) = toUpper x:xs -- | fromObj (Ganeti specific) as an expression, for reuse. fromObjE :: Q Exp fromObjE = varE 'fromObj -- | ToRaw function name. toRawName :: String -> Name toRawName = mkName . (++ "ToRaw") . ensureLower -- | FromRaw function name. fromRawName :: String -> Name fromRawName = mkName . (++ "FromRaw") . ensureLower -- | Converts a name to it's varE\/litE representations. reprE :: Either String Name -> Q Exp reprE = either stringE varE -- | Apply a constructor to a list of expressions appCons :: Name -> [Exp] -> Exp appCons cname = foldl AppE (ConE cname) -- | Apply a constructor to a list of applicative expressions appConsApp :: Name -> [Exp] -> Exp appConsApp cname = foldl (\accu e -> InfixE (Just accu) (VarE '(<*>)) (Just e)) (AppE (VarE 'pure) (ConE cname)) -- | Builds a field for a normal constructor. buildConsField :: Q Type -> StrictTypeQ buildConsField ftype = do ftype' <- ftype return (myNotStrict, ftype') -- | Builds a constructor based on a simple definition (not field-based). buildSimpleCons :: Name -> SimpleObject -> Q Dec buildSimpleCons tname cons = do decl_d <- mapM (\(cname, fields) -> do fields' <- mapM (buildConsField . snd) fields return $ NormalC (mkName cname) fields') cons return $ gntDataD [] tname [] decl_d [''Show, ''Eq] -- | Generate the save function for a given type. genSaveSimpleObj :: Name -- ^ Object type -> String -- ^ Function name -> SimpleObject -- ^ Object definition -> (SimpleConstructor -> Q Clause) -- ^ Constructor save fn -> Q (Dec, Dec) genSaveSimpleObj tname sname opdefs fn = do let sigt = AppT (AppT ArrowT (ConT tname)) (ConT ''JSON.JSValue) fname = mkName sname cclauses <- mapM fn opdefs return $ (SigD fname sigt, FunD fname cclauses) -- * Template code for simple raw type-equivalent ADTs -- | Generates a data type declaration. -- -- The type will have a fixed list of instances. strADTDecl :: Name -> [String] -> Dec strADTDecl name constructors = gntDataD [] name [] (map (flip NormalC [] . mkName) constructors) [''Show, ''Eq, ''Enum, ''Bounded, ''Ord] -- | Generates a toRaw function. -- -- This generates a simple function of the form: -- -- @ -- nameToRaw :: Name -> /traw/ -- nameToRaw Cons1 = var1 -- nameToRaw Cons2 = \"value2\" -- @ genToRaw :: Name -> Name -> Name -> [(String, Either String Name)] -> Q [Dec] genToRaw traw fname tname constructors = do let sigt = AppT (AppT ArrowT (ConT tname)) (ConT traw) -- the body clauses, matching on the constructor and returning the -- raw value clauses <- mapM (\(c, v) -> clause [recP (mkName c) []] (normalB (reprE v)) []) constructors return [SigD fname sigt, FunD fname clauses] -- | Generates a fromRaw function. -- -- The function generated is monadic and can fail parsing the -- raw value. It is of the form: -- -- @ -- nameFromRaw :: (Monad m) => /traw/ -> m Name -- nameFromRaw s | s == var1 = Cons1 -- | s == \"value2\" = Cons2 -- | otherwise = fail /.../ -- @ genFromRaw :: Name -> Name -> Name -> [(String, Either String Name)] -> Q [Dec] genFromRaw traw fname tname constructors = do -- signature of form (Monad m) => String -> m $name sigt <- [t| forall m. (Monad m, MonadFail m) => $(conT traw) -> m $(conT tname) |] -- clauses for a guarded pattern let varp = mkName "s" varpe = varE varp clauses <- mapM (\(c, v) -> do -- the clause match condition g <- normalG [| $varpe == $(reprE v) |] -- the clause result r <- [| return $(conE (mkName c)) |] return (g, r)) constructors -- the otherwise clause (fallback) oth_clause <- do let err = "Invalid string value for type " ++ (nameBase tname) ++ ": " g <- normalG [| otherwise |] r <- [|fail $ $(litE (stringL err)) ++ show $varpe |] return (g, r) let fun = FunD fname [Clause [VarP varp] (GuardedB (clauses++[oth_clause])) []] return [SigD fname sigt, fun] -- | Generates a data type from a given raw format. -- -- The format is expected to multiline. The first line contains the -- type name, and the rest of the lines must contain two words: the -- constructor name and then the string representation of the -- respective constructor. -- -- The function will generate the data type declaration, and then two -- functions: -- -- * /name/ToRaw, which converts the type to a raw type -- -- * /name/FromRaw, which (monadically) converts from a raw type to the type -- -- Note that this is basically just a custom show\/read instance, -- nothing else. declareADT :: (a -> Either String Name) -> Name -> String -> [(String, a)] -> Q [Dec] declareADT fn traw sname cons = do let name = mkName sname ddecl = strADTDecl name (map fst cons) -- process cons in the format expected by genToRaw cons' = map (second fn) cons toraw <- genToRaw traw (toRawName sname) name cons' fromraw <- genFromRaw traw (fromRawName sname) name cons' return $ ddecl:toraw ++ fromraw declareLADT :: Name -> String -> [(String, String)] -> Q [Dec] declareLADT = declareADT Left declareILADT :: String -> [(String, Int)] -> Q [Dec] declareILADT sname cons = do consNames <- sequence [ newName ('_':n) | (n, _) <- cons ] consFns <- concat <$> sequence [ do sig <- sigD n [t| Int |] let expr = litE (IntegerL (toInteger i)) fn <- funD n [clause [] (normalB expr) []] return [sig, fn] | n <- consNames | (_, i) <- cons ] let cons' = [ (n, n') | (n, _) <- cons | n' <- consNames ] (consFns ++) <$> declareADT Right ''Int sname cons' declareIADT :: String -> [(String, Name)] -> Q [Dec] declareIADT = declareADT Right ''Int declareSADT :: String -> [(String, Name)] -> Q [Dec] declareSADT = declareADT Right ''String -- | Creates the showJSON member of a JSON instance declaration. -- -- This will create what is the equivalent of: -- -- @ -- showJSON = showJSON . /name/ToRaw -- @ -- -- in an instance JSON /name/ declaration genShowJSON :: String -> Q Dec genShowJSON name = do body <- [| JSON.showJSON . $(varE (toRawName name)) |] return $ FunD 'JSON.showJSON [Clause [] (NormalB body) []] -- | Creates the readJSON member of a JSON instance declaration. -- -- This will create what is the equivalent of: -- -- @ -- readJSON s = case readJSON s of -- Ok s' -> /name/FromRaw s' -- Error e -> Error /description/ -- @ -- -- in an instance JSON /name/ declaration genReadJSON :: String -> Q Dec genReadJSON name = do let s = mkName "s" body <- [| $(varE (fromRawName name)) =<< readJSONWithDesc $(stringE name) $(varE s) |] return $ FunD 'JSON.readJSON [Clause [VarP s] (NormalB body) []] -- | Generates a JSON instance for a given type. -- -- This assumes that the /name/ToRaw and /name/FromRaw functions -- have been defined as by the 'declareSADT' function. makeJSONInstance :: Name -> Q [Dec] makeJSONInstance name = do let base = nameBase name showJ <- genShowJSON base readJ <- genReadJSON base return [gntInstanceD [] (AppT (ConT ''JSON.JSON) (ConT name)) [readJ,showJ]] -- * Template code for opcodes -- | Transforms a CamelCase string into an_underscore_based_one. deCamelCase :: String -> String deCamelCase = intercalate "_" . map (map toUpper) . groupBy (\_ b -> not $ isUpper b) -- | Transform an underscore_name into a CamelCase one. camelCase :: String -> String camelCase = concatMap (ensureUpper . drop 1) . groupBy (\_ b -> b /= '_' && b /= '-') . ('_':) -- | Computes the name of a given constructor. constructorName :: Con -> Q Name constructorName (NormalC name _) = return name constructorName (RecC name _) = return name constructorName x = fail $ "Unhandled constructor " ++ show x -- | Extract all constructor names from a given type. reifyConsNames :: Name -> Q [String] reifyConsNames name = do reify_result <- reify name case extractDataDConstructors reify_result of Just cons -> mapM (liftM nameBase . constructorName) cons _ -> fail $ "Unhandled name passed to reifyConsNames, expected\ \ type constructor but got '" ++ show reify_result ++ "'" -- | Builds the generic constructor-to-string function. -- -- This generates a simple function of the following form: -- -- @ -- fname (ConStructorOne {}) = trans_fun("ConStructorOne") -- fname (ConStructorTwo {}) = trans_fun("ConStructorTwo") -- @ -- -- This builds a custom list of name\/string pairs and then uses -- 'genToRaw' to actually generate the function. genConstrToStr :: (String -> Q String) -> Name -> String -> Q [Dec] genConstrToStr trans_fun name fname = do cnames <- reifyConsNames name svalues <- mapM (liftM Left . trans_fun) cnames genToRaw ''String (mkName fname) name $ zip cnames svalues -- | Constructor-to-string for OpCode. genOpID :: Name -> String -> Q [Dec] genOpID = genConstrToStr (return . deCamelCase) -- | Strips @Op@ from the constructor name, converts to lower-case -- and adds a given prefix. genOpLowerStrip :: String -> Name -> String -> Q [Dec] genOpLowerStrip prefix = genConstrToStr (liftM ((prefix ++) . map toLower . deCamelCase) . stripPrefixM "Op") where stripPrefixM :: String -> String -> Q String stripPrefixM pfx s = maybe (fail $ s ++ " doesn't start with " ++ pfx) return $ stripPrefix pfx s -- | Builds a list with all defined constructor names for a type. -- -- @ -- vstr :: String -- vstr = [...] -- @ -- -- Where the actual values of the string are the constructor names -- mapped via @trans_fun@. genAllConstr :: (String -> String) -> Name -> String -> Q [Dec] genAllConstr trans_fun name vstr = do cnames <- reifyConsNames name let svalues = sort $ map trans_fun cnames vname = mkName vstr sig = SigD vname (AppT ListT (ConT ''String)) body = NormalB (ListE (map (LitE . StringL) svalues)) return $ [sig, ValD (VarP vname) body []] -- | Generates a list of all defined opcode IDs. genAllOpIDs :: Name -> String -> Q [Dec] genAllOpIDs = genAllConstr deCamelCase -- * Python code generation data OpCodeField = OpCodeField { ocfName :: String , ocfType :: PyType , ocfDefl :: Maybe PyValueEx , ocfDoc :: String } -- | Transfers opcode data between the opcode description (through -- @genOpCode@) and the Python code generation functions. data OpCodeDescriptor = OpCodeDescriptor { ocdName :: String , ocdType :: PyType , ocdDoc :: String , ocdFields :: [OpCodeField] , ocdDescr :: String } -- | Optionally encapsulates default values in @PyValueEx@. -- -- @maybeApp exp typ@ returns a quoted expression that encapsulates -- the default value @exp@ of an opcode parameter cast to @typ@ in a -- @PyValueEx@, if @exp@ is @Just@. Otherwise, it returns a quoted -- expression with @Nothing@. maybeApp :: Maybe (Q Exp) -> Q Type -> Q Exp maybeApp Nothing _ = [| Nothing |] maybeApp (Just expr) typ = [| Just ($(conE (mkName "PyValueEx")) ($expr :: $typ)) |] -- | Generates a Python type according to whether the field is -- optional. -- -- The type of created expression is PyType. genPyType' :: OptionalType -> Q Type -> Q PyType genPyType' opt typ = typ >>= pyOptionalType (opt /= NotOptional) -- | Generates Python types from opcode parameters. genPyType :: Field -> Q PyType genPyType f = genPyType' (fieldIsOptional f) (fieldType f) -- | Generates Python default values from opcode parameters. genPyDefault :: Field -> Q Exp genPyDefault f = maybeApp (fieldDefault f) (fieldType f) pyField :: Field -> Q Exp pyField f = genPyType f >>= \t -> [| OpCodeField $(stringE . T.unpack . fieldName $ f) t $(genPyDefault f) $(stringE . T.unpack . fieldDoc $ f) |] -- | Generates a Haskell function call to "showPyClass" with the -- necessary information on how to build the Python class string. pyClass :: OpCodeConstructor -> Q Exp pyClass (consName, consType, consDoc, consFields, consDscField) = do let consName' = stringE consName consType' <- genPyType' NotOptional consType let consDoc' = stringE consDoc [| OpCodeDescriptor $consName' consType' $consDoc' $(listE $ map pyField consFields) consDscField |] -- | Generates a function called "pyClasses" that holds the list of -- all the opcode descriptors necessary for generating the Python -- opcodes. pyClasses :: [OpCodeConstructor] -> Q [Dec] pyClasses cons = do let name = mkName "pyClasses" sig = SigD name (AppT ListT (ConT ''OpCodeDescriptor)) fn <- FunD name <$> (:[]) <$> declClause cons return [sig, fn] where declClause c = clause [] (normalB (ListE <$> mapM pyClass c)) [] -- | Converts from an opcode constructor to a Luxi constructor. opcodeConsToLuxiCons :: OpCodeConstructor -> LuxiConstructor opcodeConsToLuxiCons (x, _, _, y, _) = (x, y) -- | Generates 'DictObject' instance for an op-code. genOpCodeDictObject :: Name -- ^ Type name to use -> (LuxiConstructor -> Q Clause) -- ^ saving function -> (LuxiConstructor -> Q Exp) -- ^ loading function -> [LuxiConstructor] -- ^ Constructors -> Q [Dec] genOpCodeDictObject tname savefn loadfn cons = do tdclauses <- genSaveOpCode cons savefn fdclauses <- genLoadOpCode cons loadfn return [ gntInstanceD [] (AppT (ConT ''DictObject) (ConT tname)) [ FunD 'toDict tdclauses , FunD 'fromDictWKeys fdclauses ]] -- | Generates the OpCode data type. -- -- This takes an opcode logical definition, and builds both the -- datatype and the JSON serialisation out of it. We can't use a -- generic serialisation since we need to be compatible with Ganeti's -- own, so we have a few quirks to work around. genOpCode :: String -- ^ Type name to use -> [OpCodeConstructor] -- ^ Constructor name and parameters -> Q [Dec] genOpCode name cons = do let tname = mkName name decl_d <- mapM (\(cname, _, _, fields, _) -> do -- we only need the type of the field, without Q fields' <- mapM (fieldTypeInfo "op") fields return $ RecC (mkName cname) fields') cons let declD = gntDataD [] tname [] decl_d [''Show, ''Eq] let (allfsig, allffn) = genAllOpFields "allOpFields" cons -- DictObject let luxiCons = map opcodeConsToLuxiCons cons dictObjInst <- genOpCodeDictObject tname saveConstructor loadOpConstructor luxiCons -- rest pyDecls <- pyClasses cons return $ [declD, allfsig, allffn] ++ dictObjInst ++ pyDecls -- | Generates the function pattern returning the list of fields for a -- given constructor. genOpConsFields :: OpCodeConstructor -> Clause genOpConsFields (cname, _, _, fields, _) = let op_id = deCamelCase cname fieldnames f = map T.unpack $ fieldName f:fieldExtraKeys f fvals = map (LitE . StringL) . sort . nub $ concatMap fieldnames fields in Clause [LitP (StringL op_id)] (NormalB $ ListE fvals) [] -- | Generates a list of all fields of an opcode constructor. genAllOpFields :: String -- ^ Function name -> [OpCodeConstructor] -- ^ Object definition -> (Dec, Dec) genAllOpFields sname opdefs = let cclauses = map genOpConsFields opdefs other = Clause [WildP] (NormalB (ListE [])) [] fname = mkName sname sigt = AppT (AppT ArrowT (ConT ''String)) (AppT ListT (ConT ''String)) in (SigD fname sigt, FunD fname (cclauses++[other])) -- | Generates the \"save\" clause for an entire opcode constructor. -- -- This matches the opcode with variables named the same as the -- constructor fields (just so that the spliced in code looks nicer), -- and passes those name plus the parameter definition to 'saveObjectField'. saveConstructor :: LuxiConstructor -- ^ The constructor -> Q Clause -- ^ Resulting clause saveConstructor (sname, fields) = do let cname = mkName sname fnames <- mapM (newName . fieldVariable) fields let pat = conP cname (map varP fnames) let felems = zipWith saveObjectField fnames fields -- now build the OP_ID serialisation opid = [| [( $(stringE "OP_ID"), JSON.showJSON $(stringE . deCamelCase $ sname) )] |] flist = listE (opid:felems) -- and finally convert all this to a json object flist' = [| concat $flist |] clause [pat] (normalB flist') [] -- | Generates the main save opcode function, serializing as a dictionary. -- -- This builds a per-constructor match clause that contains the -- respective constructor-serialisation code. genSaveOpCode :: [LuxiConstructor] -- ^ Object definition -> (LuxiConstructor -> Q Clause) -- ^ Constructor save fn -> Q [Clause] genSaveOpCode opdefs fn = mapM fn opdefs -- | Generates load code for a single constructor of the opcode data type. -- The type of the resulting expression is @WriterT UsedKeys J.Result a@. loadConstructor :: Name -> (Field -> Q Exp) -> [Field] -> Q Exp loadConstructor name loadfn fields = [| MT.lift $(appConsApp name <$> mapM loadfn fields) <* tell $(fieldsUsedKeysQ fields) |] -- | Generates load code for a single constructor of the opcode data type. loadOpConstructor :: LuxiConstructor -> Q Exp loadOpConstructor (sname, fields) = loadConstructor (mkName sname) (loadObjectField fields) fields -- | Generates the loadOpCode function. genLoadOpCode :: [LuxiConstructor] -> (LuxiConstructor -> Q Exp) -- ^ Constructor load fn -> Q [Clause] genLoadOpCode opdefs fn = do let objname = objVarName opidKey = "OP_ID" opid = mkName $ map toLower opidKey st <- bindS (varP opid) [| $fromObjE $(varE objname) $(stringE opidKey) |] -- the match results (per-constructor blocks) mexps <- mapM fn opdefs fails <- [| fail $ "Unknown opcode " ++ $(varE opid) |] let mpats = map (\(me, op) -> let mp = LitP . StringL . deCamelCase . fst $ op in Match mp (NormalB me) [] ) $ zip mexps opdefs defmatch = Match WildP (NormalB fails) [] cst = NoBindS $ CaseE (VarE opid) $ mpats++[defmatch] body = mkDoE [st, cst] -- include "OP_ID" to the list of used keys bodyAndOpId <- [| $(return body) <* tell (mkUsedKeys . S.singleton . T.pack $ opidKey) |] return [Clause [VarP objname] (NormalB bodyAndOpId) []] -- * Template code for luxi -- | Constructor-to-string for LuxiOp. genStrOfOp :: Name -> String -> Q [Dec] genStrOfOp = genConstrToStr return -- | Constructor-to-string for MsgKeys. genStrOfKey :: Name -> String -> Q [Dec] genStrOfKey = genConstrToStr (return . ensureLower) -- | Generates the LuxiOp data type. -- -- This takes a Luxi operation definition and builds both the -- datatype and the function transforming the arguments to JSON. -- We can't use anything less generic, because the way different -- operations are serialized differs on both parameter- and top-level. -- -- There are two things to be defined for each parameter: -- -- * name -- -- * type -- genLuxiOp :: String -> [LuxiConstructor] -> Q [Dec] genLuxiOp name cons = do let tname = mkName name decl_d <- mapM (\(cname, fields) -> do -- we only need the type of the field, without Q fields' <- mapM actualFieldType fields let fields'' = zip (repeat myNotStrict) fields' return $ NormalC (mkName cname) fields'') cons let declD = gntDataD [] (mkName name) [] decl_d [''Show, ''Eq] -- generate DictObject instance dictObjInst <- genOpCodeDictObject tname saveLuxiConstructor loadOpConstructor cons -- .. and use it to construct 'opToArgs' of 'toDict' -- (as we know that the output of 'toDict' is always in the proper order) opToArgsType <- [t| $(conT tname) -> JSON.JSValue |] opToArgsExp <- [| JSON.showJSON . map snd . toDict |] let opToArgsName = mkName "opToArgs" opToArgsDecs = [ SigD opToArgsName opToArgsType , ValD (VarP opToArgsName) (NormalB opToArgsExp) [] ] -- rest req_defs <- declareSADT "LuxiReq" . map (\(str, _) -> ("Req" ++ str, mkName ("luxiReq" ++ str))) $ cons return $ [declD] ++ dictObjInst ++ opToArgsDecs ++ req_defs -- | Generates the \"save\" clause for entire LuxiOp constructor. saveLuxiConstructor :: LuxiConstructor -> Q Clause saveLuxiConstructor (sname, fields) = do let cname = mkName sname fnames <- mapM (newName . fieldVariable) fields let pat = conP cname (map varP fnames) let felems = zipWith saveObjectField fnames fields flist = [| concat $(listE felems) |] clause [pat] (normalB flist) [] -- * "Objects" functionality -- | Extract the field's declaration from a Field structure. fieldTypeInfo :: String -> Field -> Q (Name, Strict, Type) fieldTypeInfo field_pfx fd = do t <- actualFieldType fd let n = mkName . (field_pfx ++) . fieldRecordName $ fd return (n, myNotStrict, t) -- | Build an object declaration. buildObject :: String -> String -> [Field] -> Q [Dec] buildObject sname field_pfx fields = do when (any ((==) AndRestArguments . fieldIsOptional) . drop 1 $ reverse fields) $ fail "Objects may have only one AndRestArguments field,\ \ and it must be the last one." let name = mkName sname fields_d <- mapM (fieldTypeInfo field_pfx) fields let decl_d = RecC name fields_d let declD = gntDataD [] name [] [decl_d] [''Show, ''Eq] ser_decls <- buildObjectSerialisation sname fields return $ declD:ser_decls -- | Build an accessor function for a field of an object -- that can have a forthcoming variant. buildAccessor :: Name -- ^ name of the forthcoming constructor -> String -- ^ prefix for the forthcoming field -> Name -- ^ name of the real constructor -> String -- ^ prefix for the real field -> Name -- ^ name of the generated accessor -> String -- ^ prefix of the generated accessor -> Field -- ^ field description -> Q [Dec] buildAccessor fnm fpfx rnm rpfx nm pfx field = do let optField = makeOptional field x <- newName "x" (rpfx_name, _, _) <- fieldTypeInfo rpfx field (fpfx_name, _, ftype) <- fieldTypeInfo fpfx optField (pfx_name, _, _) <- fieldTypeInfo pfx field let r_body_core = AppE (VarE rpfx_name) $ VarE x r_body = if fieldIsOptional field == fieldIsOptional optField then r_body_core else AppE (VarE 'return) r_body_core f_body = AppE (VarE fpfx_name) $ VarE x return $ [ SigD pfx_name $ ArrowT `AppT` ConT nm `AppT` ftype , FunD pfx_name [ Clause [conP_ rnm [VarP x]] (NormalB r_body) [] , Clause [conP_ fnm [VarP x]] (NormalB f_body) [] ]] -- | Build lense declartions for a field. -- -- If the type of the field is the same in -- the forthcoming and the real variant, the lens -- will be a simple lens (Lens' s a). -- -- Otherwise, the type will be (Lens s s (Maybe a) a). -- This is because the field in forthcoming variant -- has type (Maybe a), but the real variant has type a. buildLens :: (Name, Name) -- ^ names of the forthcoming constructors -> (Name, Name) -- ^ names of the real constructors -> Name -- ^ name of the type -> String -- ^ the field prefix -> Int -- ^ arity -> (Field, Int) -- ^ the Field to generate the lens for, and its -- position -> Q [Dec] buildLens (fnm, fdnm) (rnm, rdnm) nm pfx ar (field, i) = do let optField = makeOptional field isSimple = fieldIsOptional field == fieldIsOptional optField lensnm = mkName $ pfx ++ fieldRecordName field ++ "L" (accnm, _, ftype) <- fieldTypeInfo pfx field vars <- replicateM ar (newName "x") var <- newName "val" context <- newName "val" jE <- [| Just |] let body eJ cn cdn = NormalB . (ConE cn `AppE`) . foldl (\e (j, x) -> AppE e $ if i == j then if eJ then AppE jE (VarE var) else VarE var else VarE x) (ConE cdn) $ zip [0..] vars let setterE = LamE [VarP context, VarP var] $ CaseE (VarE context) [ Match (conP_ fnm [conP_ fdnm . set (element i) WildP $ map VarP vars]) (body (not isSimple) fnm fdnm) [] , Match (conP_ rnm [conP_ rdnm . set (element i) WildP $ map VarP vars]) (body False rnm rdnm) [] ] let lensD = ValD (VarP lensnm) (NormalB $ VarE 'lens `AppE` VarE accnm `AppE` setterE) [] if isSimple then return $ (SigD lensnm $ ConT ''Lens' `AppT` ConT nm `AppT` ftype) : lensD : [] else return $ (SigD lensnm $ ConT ''Lens `AppT` ConT nm `AppT` ConT nm `AppT` (ConT ''Maybe `AppT` ftype) `AppT` ftype) : lensD : [] -- | Build an object that can have a forthcoming variant. -- This will create 3 data types: two objects, prefixed by -- "Real" and "Forthcoming", respectively, and a sum type -- of those. The JSON representation of the latter will -- be a JSON object, dispatching on the "forthcoming" key. buildObjectWithForthcoming :: String -- ^ Name of the newly defined type -> String -- ^ base prefix for field names; for the real and forthcoming -- variant, with base prefix will be prefixed with "real" -- and forthcoming, respectively. -> [Field] -- ^ List of fields in the real version -> Q [Dec] buildObjectWithForthcoming sname field_pfx fields = do let capitalPrefix = ensureUpper field_pfx forth_nm = "Forthcoming" ++ sname forth_data_nm = forth_nm ++ "Data" forth_pfx = "forthcoming" ++ capitalPrefix real_nm = "Real" ++ sname real_data_nm = real_nm ++ "Data" real_pfx = "real" ++ capitalPrefix concreteDecls <- buildObject real_data_nm real_pfx fields forthcomingDecls <- buildObject forth_data_nm forth_pfx (map makeOptional fields) let name = mkName sname real_d = NormalC (mkName real_nm) [(myNotStrict, ConT (mkName real_data_nm))] forth_d = NormalC (mkName forth_nm) [(myNotStrict, ConT (mkName forth_data_nm))] let declD = gntDataD [] name [] [real_d, forth_d] [''Show, ''Eq] read_body <- [| branchOnField "forthcoming" (liftM $(conE $ mkName forth_nm) . JSON.readJSON) (liftM $(conE $ mkName real_nm) . JSON.readJSON) |] x <- newName "x" show_real_body <- [| JSON.showJSON $(varE x) |] show_forth_body <- [| addField ("forthcoming", JSON.JSBool True) $ JSON.showJSON $(varE x) |] let rdjson = FunD 'JSON.readJSON [Clause [] (NormalB read_body) []] shjson = FunD 'JSON.showJSON [ Clause [conP_ (mkName real_nm) [VarP x]] (NormalB show_real_body) [] , Clause [conP_ (mkName forth_nm) [VarP x]] (NormalB show_forth_body) [] ] instJSONdecl = gntInstanceD [] (AppT (ConT ''JSON.JSON) (ConT name)) [rdjson, shjson] accessors <- liftM concat . flip mapM fields $ buildAccessor (mkName forth_nm) forth_pfx (mkName real_nm) real_pfx name field_pfx lenses <- liftM concat . flip mapM (zip fields [0..]) $ buildLens (mkName forth_nm, mkName forth_data_nm) (mkName real_nm, mkName real_data_nm) name field_pfx (length fields) xs <- newName "xs" fromDictWKeysbody <- [| if ("forthcoming", JSON.JSBool True) `elem` $(varE xs) then liftM $(conE $ mkName forth_nm) (fromDictWKeys $(varE xs)) else liftM $(conE $ mkName real_nm) (fromDictWKeys $(varE xs)) |] todictx_r <- [| toDict $(varE x) |] todictx_f <- [| ("forthcoming", JSON.JSBool True) : toDict $(varE x) |] let todict = FunD 'toDict [ Clause [conP_ (mkName real_nm) [VarP x]] (NormalB todictx_r) [] , Clause [conP_ (mkName forth_nm) [VarP x]] (NormalB todictx_f) [] ] fromdict = FunD 'fromDictWKeys [ Clause [VarP xs] (NormalB fromDictWKeysbody) [] ] instDict = gntInstanceD [] (AppT (ConT ''DictObject) (ConT name)) [todict, fromdict] instArray <- genArrayObjectInstance name (simpleField "forthcoming" [t| Bool |] : fields) let forthPredName = mkName $ field_pfx ++ "Forthcoming" let forthPredDecls = [ SigD forthPredName $ ArrowT `AppT` ConT name `AppT` ConT ''Bool , FunD forthPredName [ Clause [conP_ (mkName real_nm) [WildP]] (NormalB $ ConE 'False) [] , Clause [conP_ (mkName forth_nm) [WildP]] (NormalB $ ConE 'True) [] ] ] return $ concreteDecls ++ forthcomingDecls ++ [declD, instJSONdecl] ++ forthPredDecls ++ accessors ++ lenses ++ [instDict, instArray] -- | Generates an object definition: data type and its JSON instance. buildObjectSerialisation :: String -> [Field] -> Q [Dec] buildObjectSerialisation sname fields = do let name = mkName sname dictdecls <- genDictObject saveObjectField (loadObjectField fields) sname fields savedecls <- genSaveObject sname (loadsig, loadfn) <- genLoadObject sname shjson <- objectShowJSON sname rdjson <- objectReadJSON sname let instdecl = gntInstanceD [] (AppT (ConT ''JSON.JSON) (ConT name)) [rdjson, shjson] return $ dictdecls ++ savedecls ++ [loadsig, loadfn, instdecl] -- | An internal name used for naming variables that hold the entire -- object of type @[(String,JSValue)]@. objVarName :: Name objVarName = mkName "_o" -- | Provides a default 'toJSArray' for 'ArrayObject' instance using its -- existing 'DictObject' instance. The keys are serialized in the order -- they're declared. The list must contain all keys possibly generated by -- 'toDict'. defaultToJSArray :: (DictObject a) => [String] -> a -> [JSON.JSValue] defaultToJSArray keys o = let m = M.fromList $ toDict o in map (fromMaybe JSON.JSNull . flip M.lookup m) keys -- | Provides a default 'fromJSArray' for 'ArrayObject' instance using its -- existing 'DictObject' instance. The fields are deserialized in the order -- they're declared. defaultFromJSArray :: (DictObject a) => [String] -> [JSON.JSValue] -> JSON.Result a defaultFromJSArray keys xs = do let xslen = length xs explen = length keys unless (xslen == explen) (fail $ "Expected " ++ show explen ++ " arguments, got " ++ show xslen) fromDict $ zip keys xs -- | Generates an additional 'ArrayObject' instance using its -- existing 'DictObject' instance. -- -- See 'defaultToJSArray' and 'defaultFromJSArray'. genArrayObjectInstance :: Name -> [Field] -> Q Dec genArrayObjectInstance name fields = do let fnames = fieldsKeys fields instanceD (return []) (appT (conT ''ArrayObject) (conT name)) [ valD (varP 'toJSArray) (normalB [| defaultToJSArray $(lift fnames) |]) [] , valD (varP 'fromJSArray) (normalB [| defaultFromJSArray fnames |]) [] ] -- | Generates 'DictObject' instance. genDictObject :: (Name -> Field -> Q Exp) -- ^ a saving function -> (Field -> Q Exp) -- ^ a loading function -> String -- ^ an object name -> [Field] -- ^ a list of fields -> Q [Dec] genDictObject save_fn load_fn sname fields = do let name = mkName sname -- newName fails in ghc 7.10 when used on keywords newName' "data" = newName "data_ghcBug10599" newName' "instance" = newName "instance_ghcBug10599" newName' "type" = newName "type_ghcBug10599" newName' s = newName s -- toDict fnames <- mapM (newName' . fieldVariable) fields let pat = conP name (map varP fnames) tdexp = [| concat $(listE $ zipWith save_fn fnames fields) |] tdclause <- clause [pat] (normalB tdexp) [] -- fromDict fdexp <- loadConstructor name load_fn fields let fdclause = Clause [VarP objVarName] (NormalB fdexp) [] -- the ArrayObject instance generated from DictObject arrdec <- genArrayObjectInstance name fields -- the final instance return $ [gntInstanceD [] (AppT (ConT ''DictObject) (ConT name)) [ FunD 'toDict [tdclause] , FunD 'fromDictWKeys [fdclause] ]] ++ [arrdec] -- | Generates the save object functionality. genSaveObject :: String -> Q [Dec] genSaveObject sname = do let fname = mkName ("save" ++ sname) sigt <- [t| $(conT $ mkName sname) -> JSON.JSValue |] cclause <- [| showJSONtoDict |] return [SigD fname sigt, ValD (VarP fname) (NormalB cclause) []] -- | Generates the code for saving an object's field, handling the -- various types of fields that we have. saveObjectField :: Name -> Field -> Q Exp saveObjectField fvar field = do let formatFn = fromMaybe [| JSON.showJSON &&& (const []) |] $ fieldShow field formatFnTyped = sigE formatFn [t| $(fieldType field) -> (JSON.JSValue, [(String, JSON.JSValue)]) |] let formatCode v = [| let (actual, extra) = $formatFnTyped $(v) in ($nameE, actual) : extra |] case fieldIsOptional field of OptionalOmitNull -> [| case $(fvarE) of Nothing -> [] Just v -> $(formatCode [| v |]) |] OptionalSerializeNull -> [| case $(fvarE) of Nothing -> [( $nameE, JSON.JSNull )] Just v -> $(formatCode [| v |]) |] NotOptional -> case (fieldDefault field, fieldSerializeDefault field) of (Just v, False) -> [| if $v /= $fvarE then $(formatCode fvarE) else [] |] -- If a default value exists and we shouldn't serialize -- default fields - serialize only if the value differs -- from the default one. _ -> formatCode fvarE AndRestArguments -> [| M.toList $(varE fvar) |] where nameE = stringE (T.unpack . fieldName $ field) fvarE = varE fvar -- | Generates the showJSON clause for a given object name. objectShowJSON :: String -> Q Dec objectShowJSON name = do body <- [| JSON.showJSON . $(varE . mkName $ "save" ++ name) |] return $ FunD 'JSON.showJSON [Clause [] (NormalB body) []] -- | Generates the load object functionality. genLoadObject :: String -> Q (Dec, Dec) genLoadObject sname = do let fname = mkName $ "load" ++ sname sigt <- [t| JSON.JSValue -> JSON.Result $(conT $ mkName sname) |] cclause <- [| readJSONfromDict |] return $ (SigD fname sigt, FunD fname [Clause [] (NormalB cclause) []]) -- | Generates code for loading an object's field. loadObjectField :: [Field] -> Field -> Q Exp loadObjectField allFields field = do let otherNames = fieldsDictKeysQ . filter (on (/=) fieldName field) $ allFields -- these are used in all patterns below let objvar = varE objVarName objfield = stringE (T.unpack . fieldName $ field) case (fieldDefault field, fieldIsOptional field) of -- Only non-optional fields without defaults must have a value; -- we treat both optional types the same, since -- 'maybeFromObj' can deal with both missing and null values -- appropriately (the same) (Nothing, NotOptional) -> loadFn field [| fromObj $objvar $objfield |] objvar -- AndRestArguments need not to be parsed at all, -- they're just extracted from the list of other fields. (Nothing, AndRestArguments) -> [| return . M.fromList . filter (not . (`S.member` $(otherNames)) . T.pack . fst) $ $objvar |] _ -> loadFnOpt field [| maybeFromObj $objvar $objfield |] objvar fieldsKeys :: [Field] -> [String] fieldsKeys fields = map T.unpack $ concatMap (liftA2 (:) fieldName fieldExtraKeys) fields -- | Generates the set of all used JSON dictionary keys for a list of fields -- The equivalent of S.fromList (map T.pack ["f1", "f2", "f3"] ) fieldsDictKeys :: [Field] -> Exp fieldsDictKeys fields = AppE (VarE 'S.fromList) . AppE (AppE (VarE 'map) (VarE 'T.pack)) . ListE . map (LitE . StringL) $ fieldsKeys fields -- | Generates the list of all used JSON dictionary keys for a list of fields fieldsDictKeysQ :: [Field] -> Q Exp fieldsDictKeysQ = return . fieldsDictKeys -- | Generates the list of all used JSON dictionary keys for a list of fields, -- depending on if any of them has 'AndRestArguments' flag. fieldsUsedKeysQ :: [Field] -> Q Exp fieldsUsedKeysQ fields | any ((==) AndRestArguments . fieldIsOptional) fields = [| allUsedKeys |] | otherwise = [| mkUsedKeys $(fieldsDictKeysQ fields) |] -- | Builds the readJSON instance for a given object name. objectReadJSON :: String -> Q Dec objectReadJSON name = do let s = mkName "s" body <- [| $(varE . mkName $ "load" ++ name) =<< readJSONWithDesc $(stringE name) $(varE s) |] return $ FunD 'JSON.readJSON [Clause [VarP s] (NormalB body) []] -- * Inheritable parameter tables implementation -- | Compute parameter type names. paramTypeNames :: String -> (String, String) paramTypeNames root = ("Filled" ++ root ++ "Params", "Partial" ++ root ++ "Params") -- | Compute the name of a full and a partial parameter field. paramFieldNames :: String -> Field -> (Name, Name) paramFieldNames field_pfx fd = let base = field_pfx ++ fieldRecordName fd in (mkName base, mkName (base ++ "P")) -- | Compute information about the type of a parameter field. paramFieldTypeInfo :: String -> Field -> VarStrictTypeQ paramFieldTypeInfo field_pfx fd = do t <- actualFieldType fd return (snd $ paramFieldNames field_pfx fd, myNotStrict, AppT (ConT ''Maybe) t) -- | Build a parameter declaration. -- -- This function builds two different data structures: a /filled/ one, -- in which all fields are required, and a /partial/ one, in which all -- fields are optional. Due to the current record syntax issues, the -- fields need to be named differrently for the two structures, so the -- partial ones get a /P/ suffix. -- Also generate a default value for the partial parameters. buildParam :: String -> String -> [Field] -> Q [Dec] buildParam sname field_pfx fields = do let (sname_f, sname_p) = paramTypeNames sname name_f = mkName sname_f name_p = mkName sname_p fields_f <- mapM (fieldTypeInfo field_pfx) fields fields_p <- mapM (paramFieldTypeInfo field_pfx) fields let decl_f = RecC name_f fields_f decl_p = RecC name_p fields_p let declF = gntDataD [] name_f [] [decl_f] [''Show, ''Eq] let declP = gntDataD [] name_p [] [decl_p] [''Show, ''Eq] ser_decls_f <- buildObjectSerialisation sname_f fields ser_decls_p <- buildPParamSerialisation sname_p fields fill_decls <- fillParam sname field_pfx fields return $ [declF, declP] ++ ser_decls_f ++ ser_decls_p ++ fill_decls ++ buildParamAllFields sname fields -- | Builds a list of all fields of a parameter. buildParamAllFields :: String -> [Field] -> [Dec] buildParamAllFields sname fields = let vname = mkName ("all" ++ sname ++ "ParamFields") sig = SigD vname (AppT ListT (ConT ''String)) val = ListE $ map (LitE . StringL . T.unpack . fieldName) fields in [sig, ValD (VarP vname) (NormalB val) []] -- | Generates the serialisation for a partial parameter. buildPParamSerialisation :: String -> [Field] -> Q [Dec] buildPParamSerialisation sname fields = do let name = mkName sname dictdecls <- genDictObject savePParamField loadPParamField sname fields savedecls <- genSaveObject sname (loadsig, loadfn) <- genLoadObject sname shjson <- objectShowJSON sname rdjson <- objectReadJSON sname let instdecl = gntInstanceD [] (AppT (ConT ''JSON.JSON) (ConT name)) [rdjson, shjson] return $ dictdecls ++ savedecls ++ [loadsig, loadfn, instdecl] -- | Generates code to save an optional parameter field. savePParamField :: Name -> Field -> Q Exp savePParamField fvar field = do checkNonOptDef field let actualVal = mkName "v" normalexpr <- saveObjectField actualVal field -- we have to construct the block here manually, because we can't -- splice-in-splice return $ CaseE (VarE fvar) [ Match (conP_ 'Nothing []) (NormalB (ConE '[])) [] , Match (conP_ 'Just [VarP actualVal]) (NormalB normalexpr) [] ] -- | Generates code to load an optional parameter field. loadPParamField :: Field -> Q Exp loadPParamField field = do checkNonOptDef field let name = fieldName field -- these are used in all patterns below let objvar = varE objVarName objfield = stringE . T.unpack $ name loadexp = [| $(varE 'maybeFromObj) $objvar $objfield |] loadFnOpt field loadexp objvar -- | Builds a function that executes the filling of partial parameter -- from a full copy (similar to Python's fillDict). fillParam :: String -> String -> [Field] -> Q [Dec] fillParam sname field_pfx fields = do let (sname_f, sname_p) = paramTypeNames sname name_f = mkName sname_f name_p = mkName sname_p let (fnames, pnames) = unzip $ map (paramFieldNames field_pfx) fields -- due to apparent bugs in some older GHC versions, we need to add these -- prefixes to avoid "binding shadows ..." errors fbinds <- mapM (newName . ("f_" ++) . nameBase) fnames let fConP = conP_ name_f (map VarP fbinds) pbinds <- mapM (newName . ("p_" ++) . nameBase) pnames let pConP = conP_ name_p (map VarP pbinds) -- PartialParams instance -------- -- fillParams let fromMaybeExp fn pn = AppE (AppE (VarE 'fromMaybe) (VarE fn)) (VarE pn) fupdates = appCons name_f $ zipWith fromMaybeExp fbinds pbinds fclause = Clause [fConP, pConP] (NormalB fupdates) [] -- toPartial let tpupdates = appCons name_p $ map (AppE (ConE 'Just) . VarE) fbinds tpclause = Clause [fConP] (NormalB tpupdates) [] -- toFilled let tfupdates = appConsApp name_f $ map VarE pbinds tfclause = Clause [pConP] (NormalB tfupdates) [] -- the instance let instType = AppT (AppT (ConT ''PartialParams) (ConT name_f)) (ConT name_p) -- Monoid instance for the partial part ---- -- mempty let memptyExp = appCons name_p $ map (const $ VarE 'empty) fields memptyClause = Clause [] (NormalB memptyExp) [] -- mappend pbinds2 <- mapM (newName . ("p2_" ++) . nameBase) pnames let pConP2 = conP_ name_p (map VarP pbinds2) -- note the reversal of 'l' and 'r' in the call to <|> -- as we want the result to be the rightmost value let altExp = zipWith (\l r -> AppE (AppE (VarE '(<|>)) (VarE r)) (VarE l)) mappendExp = appCons name_p $ altExp pbinds pbinds2 mappendClause = Clause [pConP, pConP2] (NormalB mappendExp) [] mappendAlias = Clause [] (NormalB $ VarE '(Sem.<>)) [] let monoidType = AppT (ConT ''Monoid) (ConT name_p) let semigroupType = AppT (ConT ''Sem.Semigroup) (ConT name_p) -- the instances combined return [ gntInstanceD [] instType [ FunD 'fillParams [fclause] , FunD 'toPartial [tpclause] , FunD 'toFilled [tfclause] ] , gntInstanceD [] semigroupType [ FunD '(Sem.<>) [mappendClause] ] , gntInstanceD [] monoidType [ FunD 'mempty [memptyClause] , FunD 'mappend [mappendAlias] ]] -- * Template code for exceptions -- | Exception simple error message field. excErrMsg :: (String, Q Type) excErrMsg = ("errMsg", [t| String |]) -- | Builds an exception type definition. genException :: String -- ^ Name of new type -> SimpleObject -- ^ Constructor name and parameters -> Q [Dec] genException name cons = do let tname = mkName name declD <- buildSimpleCons tname cons (savesig, savefn) <- genSaveSimpleObj tname ("save" ++ name) cons $ uncurry saveExcCons (loadsig, loadfn) <- genLoadExc tname ("load" ++ name) cons return [declD, loadsig, loadfn, savesig, savefn] -- | Generates the \"save\" clause for an entire exception constructor. -- -- This matches the exception with variables named the same as the -- constructor fields (just so that the spliced in code looks nicer), -- and calls showJSON on it. saveExcCons :: String -- ^ The constructor name -> [SimpleField] -- ^ The parameter definitions for this -- constructor -> Q Clause -- ^ Resulting clause saveExcCons sname fields = do let cname = mkName sname fnames <- mapM (newName . fst) fields let pat = conP cname (map varP fnames) felems = if null fnames then conE '() -- otherwise, empty list has no type else listE $ map (\f -> [| JSON.showJSON $(varE f) |]) fnames let tup = tupE [ litE (stringL sname), felems ] clause [pat] (normalB [| JSON.showJSON $tup |]) [] -- | Generates load code for a single constructor of an exception. -- -- Generates the code (if there's only one argument, we will use a -- list, not a tuple: -- -- @ -- do -- (x1, x2, ...) <- readJSON args -- return $ Cons x1 x2 ... -- @ loadExcConstructor :: Name -> String -> [SimpleField] -> Q Exp loadExcConstructor inname sname fields = do let name = mkName sname f_names <- mapM (newName . fst) fields let read_args = AppE (VarE 'JSON.readJSON) (VarE inname) let binds = case f_names of [x] -> BindS (ListP [VarP x]) _ -> BindS (TupP (map VarP f_names)) cval = appCons name $ map VarE f_names return $ mkDoE [binds read_args, NoBindS (AppE (VarE 'return) cval)] {-| Generates the loadException function. This generates a quite complicated function, along the lines of: @ loadFn (JSArray [JSString name, args]) = case name of "A1" -> do (x1, x2, ...) <- readJSON args return $ A1 x1 x2 ... "a2" -> ... s -> fail $ "Unknown exception" ++ s loadFn v = fail $ "Expected array but got " ++ show v @ -} genLoadExc :: Name -> String -> SimpleObject -> Q (Dec, Dec) genLoadExc tname sname opdefs = do let fname = mkName sname exc_name <- newName "name" exc_args <- newName "args" exc_else <- newName "s" arg_else <- newName "v" fails <- [| fail $ "Unknown exception '" ++ $(varE exc_else) ++ "'" |] -- default match for unknown exception name let defmatch = Match (VarP exc_else) (NormalB fails) [] -- the match results (per-constructor blocks) str_matches <- mapM (\(s, params) -> do body_exp <- loadExcConstructor exc_args s params return $ Match (LitP (StringL s)) (NormalB body_exp) []) opdefs -- the first function clause; we can't use [| |] due to TH -- limitations, so we have to build the AST by hand let clause1 = Clause [conP_ 'JSON.JSArray [ListP [conP_ 'JSON.JSString [VarP exc_name], VarP exc_args]]] (NormalB (CaseE (AppE (VarE 'JSON.fromJSString) (VarE exc_name)) (str_matches ++ [defmatch]))) [] -- the fail expression for the second function clause let err = "Invalid exception: expected '(string, [args])' " ++ " but got " fail_type <- [| fail $ err ++ show (pp_value $(varE arg_else)) ++ "'" |] -- the second function clause let clause2 = Clause [VarP arg_else] (NormalB fail_type) [] sigt <- [t| JSON.JSValue -> JSON.Result $(conT tname) |] return $ (SigD fname sigt, FunD fname [clause1, clause2]) -- | Compute the ssconf constructor name from its file name. ssconfConstructorName :: String -> String ssconfConstructorName = camelCase . ("s_s_" ++) ganeti-3.1.0~rc2/src/Ganeti/THH/000075500000000000000000000000001476477700300162255ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/THH/Compat.hs000064400000000000000000000101711476477700300200040ustar00rootroot00000000000000{-# LANGUAGE CPP, TemplateHaskell #-} {-| Shim library for supporting various Template Haskell versions -} {- Copyright (C) 2018, 2021 Ganeti Project Contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.THH.Compat ( gntInstanceD , gntDataD , extractDataDConstructors , myNotStrict , nonUnaryTupE , mkDoE , conP_ ) where import Language.Haskell.TH import Language.Haskell.TH.Syntax -- | Convert Names to DerivClauses -- -- template-haskell 2.12 (GHC 8.2) has changed the DataD class of -- constructors to expect [DerivClause] instead of [Names]. Handle this in a -- backwards-compatible way. #if MIN_VERSION_template_haskell(2,12,0) derivesFromNames :: [Name] -> [DerivClause] derivesFromNames names = [DerivClause Nothing $ map ConT names] #else derivesFromNames :: [Name] -> Cxt derivesFromNames names = map ConT names #endif -- | DataD "constructor" function -- -- Handle TH 2.11 and 2.12 changes in a transparent manner using the pre-2.11 -- API. #if MIN_VERSION_template_haskell(2,21,0) gntDataD :: Cxt -> Name -> [TyVarBndr BndrVis] -> [Con] -> [Name] -> Dec #elif MIN_VERSION_template_haskell(2,17,0) gntDataD :: Cxt -> Name -> [TyVarBndr ()] -> [Con] -> [Name] -> Dec #else gntDataD :: Cxt -> Name -> [TyVarBndr] -> [Con] -> [Name] -> Dec #endif gntDataD x y z a b = #if MIN_VERSION_template_haskell(2,12,0) DataD x y z Nothing a $ derivesFromNames b #elif MIN_VERSION_template_haskell(2,11,0) DataD x y z Nothing a $ map ConT b #else DataD x y z a b #endif -- | InstanceD "constructor" function -- -- Handle TH 2.11 and 2.12 changes in a transparent manner using the pre-2.11 -- API. gntInstanceD :: Cxt -> Type -> [Dec] -> Dec gntInstanceD x y = #if MIN_VERSION_template_haskell(2,11,0) InstanceD Nothing x y #else InstanceD x y #endif -- | Extract constructors from a DataD instance -- -- Handle TH 2.11 changes by abstracting pattern matching against DataD. extractDataDConstructors :: Info -> Maybe [Con] extractDataDConstructors info = case info of #if MIN_VERSION_template_haskell(2,11,0) TyConI (DataD _ _ _ Nothing cons _) -> Just cons #else TyConI (DataD _ _ _ cons _) -> Just cons #endif _ -> Nothing -- | Strict has been replaced by Bang, so redefine NotStrict in terms of the -- latter. #if MIN_VERSION_template_haskell(2,11,0) myNotStrict :: Bang myNotStrict = Bang NoSourceUnpackedness NoSourceStrictness #else myNotStrict = NotStrict #endif -- | TupE changed from '[Exp] -> Exp' to '[Maybe Exp] -> Exp'. -- Provide the old signature for compatibility. nonUnaryTupE :: [Exp] -> Exp #if MIN_VERSION_template_haskell(2,16,0) nonUnaryTupE es = TupE $ map Just es #else nonUnaryTupE es = TupE $ es #endif -- | DoE is now qualified with an optional ModName mkDoE :: [Stmt] -> Exp mkDoE s = #if MIN_VERSION_template_haskell(2,17,0) DoE Nothing s #else DoE s #endif conP_ :: Name -> [Pat] -> Pat #if MIN_VERSION_template_haskell(2,18,0) conP_ name = ConP name [] #else conP_ = ConP #endif ganeti-3.1.0~rc2/src/Ganeti/THH/Field.hs000064400000000000000000000122661476477700300176130ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Helpers for creating various kinds of 'Field's. They aren't directly needed for the Template Haskell code in Ganeti.THH, so better keep them in a separate module. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.THH.Field ( specialNumericalField , timeAsDoubleField , timeStampFields , uuidFields , serialFields , TagSet(..) , emptyTagSet , tagsFields , fileModeAsIntField , processIdField ) where import Control.Applicative ((<$>)) import Control.Monad import qualified Data.ByteString as BS import qualified Data.Set as Set import Language.Haskell.TH import qualified Text.JSON as JSON import System.Posix.Types (FileMode, ProcessID) import System.Time (ClockTime(..)) import Ganeti.JSON (TimeAsDoubleJSON(..)) import Ganeti.THH -- * Internal functions -- | Wrapper around a special parse function, suitable as field-parsing -- function. numericalReadFn :: JSON.JSON a => (String -> JSON.Result a) -> [(String, JSON.JSValue)] -> JSON.JSValue -> JSON.Result a numericalReadFn _ _ v@(JSON.JSRational _ _) = JSON.readJSON v numericalReadFn f _ (JSON.JSString x) = f $ JSON.fromJSString x numericalReadFn _ _ _ = JSON.Error "A numerical field has to be a number or\ \ a string." -- | Sets the read function to also accept string parsable by the given -- function. specialNumericalField :: Name -> Field -> Field specialNumericalField f field = field { fieldRead = Just (appE (varE 'numericalReadFn) (varE f)) } -- | Creates a new mandatory field that reads time as the (floating point) -- number of seconds since the standard UNIX epoch, and represents it in -- Haskell as 'ClockTime'. timeAsDoubleField :: String -> Field timeAsDoubleField fname = (simpleField fname [t| ClockTime |]) { fieldRead = Just $ [| \_ -> liftM unTimeAsDoubleJSON . JSON.readJSON |] , fieldShow = Just $ [| \c -> (JSON.showJSON $ TimeAsDoubleJSON c, []) |] } -- | A helper function for creating fields whose Haskell representation is -- 'Integral' and which are serialized as numbers. integralField :: Q Type -> String -> Field integralField typq fname = let (~->) = appT . appT arrowT -- constructs an arrow type (~::) = sigE . varE -- (f ~:: t) constructs (f :: t) in (simpleField fname typq) { fieldRead = Just $ [| \_ -> liftM $('fromInteger ~:: (conT ''Integer ~-> typq)) . JSON.readJSON |] , fieldShow = Just $ [| \c -> (JSON.showJSON . $('toInteger ~:: (typq ~-> conT ''Integer)) $ c, []) |] } -- * External functions and data types -- | Timestamp fields description. timeStampFields :: [Field] timeStampFields = map (defaultField [| TOD 0 0 |] . timeAsDoubleField) ["ctime", "mtime"] -- | Serial number fields description. serialFields :: [Field] serialFields = [ presentInForthcoming . renameField "Serial" $ simpleField "serial_no" [t| Int |] ] -- | UUID fields description. uuidFields :: [Field] uuidFields = [ presentInForthcoming $ simpleField "uuid" [t| BS.ByteString |] ] -- | Tag set type. newtype TagSet = TagSet { unTagSet :: Set.Set String } deriving (Eq, Show) instance JSON.JSON TagSet where showJSON = JSON.showJSON . unTagSet readJSON = (TagSet <$>) . JSON.readJSON -- | Empty tag set value. emptyTagSet :: TagSet emptyTagSet = TagSet Set.empty -- | Tag field description. tagsFields :: [Field] tagsFields = [ defaultField [| emptyTagSet |] $ simpleField "tags" [t| TagSet |] ] -- ** Fields related to POSIX data types -- | Creates a new mandatory field that reads a file mode in the standard -- POSIX file mode representation. The Haskell type of the field is 'FileMode'. fileModeAsIntField :: String -> Field fileModeAsIntField = integralField [t| FileMode |] -- | Creates a new mandatory field that contains a POSIX process ID. processIdField :: String -> Field processIdField = integralField [t| ProcessID |] ganeti-3.1.0~rc2/src/Ganeti/THH/HsRPC.hs000064400000000000000000000111121476477700300174740ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, FunctionalDependencies, FlexibleContexts, CPP, GeneralizedNewtypeDeriving, TypeFamilies, UndecidableInstances #-} -- {-# OPTIONS_GHC -fno-warn-warnings-deprecations #-} {-| Creates a client out of list of RPC server components. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.THH.HsRPC ( RpcClientMonad , runRpcClient , mkRpcCall , mkRpcCalls ) where import Control.Monad import Control.Monad.Base import Control.Monad.Except import Control.Monad.Fail (MonadFail) import Control.Monad.Reader import Control.Monad.Trans.Control import Language.Haskell.TH import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Errors import Ganeti.JSON (fromJResultE) import Ganeti.THH.Types import Ganeti.UDSServer -- * The monad for RPC clients -- | The monad for all client RPC functions. -- Given a client value, it runs the RPC call in IO and either retrieves the -- result or the error. newtype RpcClientMonad a = RpcClientMonad { runRpcClientMonad :: ReaderT Client ResultG a } deriving (Functor, Applicative, Monad, MonadFail, MonadIO, MonadBase IO, MonadError GanetiException) instance MonadBaseControl IO RpcClientMonad where #if MIN_VERSION_monad_control(1,0,0) -- Needs Undecidable instances type StM RpcClientMonad b = StM (ReaderT Client ResultG) b liftBaseWith f = RpcClientMonad $ liftBaseWith $ \r -> f (r . runRpcClientMonad) restoreM = RpcClientMonad . restoreM #else newtype StM RpcClientMonad b = StMRpcClientMonad { runStMRpcClientMonad :: StM (ReaderT Client ResultG) b } liftBaseWith f = RpcClientMonad . liftBaseWith $ \r -> f (liftM StMRpcClientMonad . r . runRpcClientMonad) restoreM = RpcClientMonad . restoreM . runStMRpcClientMonad #endif -- * The TH functions to construct RPC client functions from RPC server ones -- | Given a client run a given client RPC action. runRpcClient :: (MonadBase IO m, MonadError GanetiException m) => RpcClientMonad a -> Client -> m a runRpcClient = (toErrorBase .) . runReaderT . runRpcClientMonad callMethod :: (J.JSON r, J.JSON args) => String -> args -> RpcClientMonad r callMethod method args = do client <- RpcClientMonad ask let request = buildCall method (J.showJSON args) liftIO $ sendMsg client request response <- liftIO $ recvMsg client toError $ parseResponse response >>= fromJResultE "Parsing RPC JSON response" . J.readJSON -- | Given a server RPC function (such as from WConfd.Core), creates -- the corresponding client function. The monad of the result type of the -- given function is replaced by 'RpcClientMonad' and the new function -- is implemented to issue a RPC call to the server. mkRpcCall :: Name -> Q [Dec] mkRpcCall name = do let bname = nameBase name fname = mkName bname -- the name of the generated function (args, rtype) <- funArgs <$> typeOfFun name rarg <- argumentType rtype let ftype = foldr (\a t -> AppT (AppT ArrowT a) t) (AppT (ConT ''RpcClientMonad) rarg) args body <- [| $(curryN $ length args) (callMethod $(stringE bname)) |] return [ SigD fname ftype , ValD (VarP fname) (NormalB body) [] ] -- Given a list of server RPC functions creates the corresponding client -- RPC functions. -- -- See 'mkRpcCall' mkRpcCalls :: [Name] -> Q [Dec] mkRpcCalls = liftM concat . mapM mkRpcCall ganeti-3.1.0~rc2/src/Ganeti/THH/PyRPC.hs000064400000000000000000000170741476477700300175270ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-warnings-deprecations #-} {-| Combines the construction of RPC server components and their Python stubs. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.THH.PyRPC ( genPyUDSRpcStub , genPyUDSRpcStubStr ) where import Prelude hiding ((<>)) import Control.Monad import Data.Char (toLower, toUpper) import Data.Maybe (fromMaybe) import Language.Haskell.TH import Language.Haskell.TH.Syntax (liftString) import Text.PrettyPrint import Ganeti.THH.Types -- | The indentation step in generated Python files. pythonIndentStep :: Int pythonIndentStep = 2 -- | A helper function that nests a block of generated output by the default -- step (see 'pythonIndentStep'). nest' :: Doc -> Doc nest' = nest pythonIndentStep -- | The name of an abstract function to which all method in a Python stub -- are forwarded to. genericInvokeName :: String genericInvokeName = "_GenericInvoke" -- | The name of a function that returns the socket path for reaching the -- appropriate RPC client. socketPathName :: String socketPathName = "_GetSocketPath" -- | Create a Python expression that applies a given function to a list of -- given expressions apply :: String -> [Doc] -> Doc apply name as = text name <> parens (hcat $ punctuate (text ", ") as) -- | An empty line block. emptyLine :: Doc emptyLine = text "" -- apparently using 'empty' doesn't work lowerFirst :: String -> String lowerFirst (x:xs) = toLower x : xs lowerFirst [] = [] upperFirst :: String -> String upperFirst (x:xs) = toUpper x : xs upperFirst [] = [] -- | Creates a method declaration given a function name and a list of -- Haskell types corresponding to its arguments. toFunc :: String -> [Type] -> Q Doc toFunc fname as = do args <- zipWithM varName [1..] as let args' = text "self" : args callName = lowerFirst fname return $ (text "def" <+> apply fname args') <> colon $+$ nest' (text "return" <+> text "self." <> apply genericInvokeName (text (show callName) : args) ) where -- | Create a name for a method argument, given its index position -- and Haskell type. varName :: Int -> Type -> Q Doc varName _ (VarT n) = lowerFirstNameQ n varName _ (ConT n) = lowerFirstNameQ n varName idx (AppT ListT t) = listOf idx t varName idx (AppT (ConT n) t) | n == ''[] = listOf idx t | otherwise = kind1Of idx n t varName idx (AppT (AppT (TupleT 2) t) t') = pairOf idx t t' varName idx (AppT (AppT (ConT n) t) t') | n == ''(,) = pairOf idx t t' varName idx t = do report False $ "Don't know how to make a Python variable name from " ++ show t ++ "; using a numbered one." return $ text ('_' : show idx) -- | Create a name for a method argument, knowing that its a list of -- a given type. listOf :: Int -> Type -> Q Doc listOf idx t = (<> text "List") <$> varName idx t -- | Create a name for a method argument, knowing that its wrapped in -- a type of kind @* -> *@. kind1Of :: Int -> Name -> Type -> Q Doc kind1Of idx name t = (<> text (nameBase name)) <$> varName idx t -- | Create a name for a method argument, knowing that its a pair of -- the given types. pairOf :: Int -> Type -> Type -> Q Doc pairOf idx t t' = do tn <- varName idx t tn' <- varName idx t' return $ tn <> text "_" <> tn' <> text "_Pair" lowerFirstNameQ :: Name -> Q Doc lowerFirstNameQ = return . text . lowerFirst . nameBase -- | Creates a method declaration by inspecting (reifying) Haskell's function -- name. nameToFunc :: Name -> Q Doc nameToFunc name = do (as, _) <- funArgs `liftM` typeOfFun name -- If the function has just one argument, try if it isn't a tuple; -- if not, use the arguments as they are. let as' = fromMaybe as $ case as of [t] -> tupleArgs t -- TODO CHECK! _ -> Nothing toFunc (upperFirst $ nameBase name) as' -- | Generates a Python class stub, given a class name, the list of Haskell -- functions to expose as methods, and a optionally a piece of code to -- include. namesToClass :: String -- ^ the class name -> Doc -- ^ Python code to include in the class -> [Name] -- ^ the list of functions to include -> Q Doc namesToClass cname pycode fns = do fnsCode <- mapM (liftM ($+$ emptyLine) . nameToFunc) fns return $ vcat [ text "class" <+> apply cname [text "object"] <> colon , nest' ( pycode $+$ vcat fnsCode ) ] -- | Takes a list of function names and creates a RPC handler that delegates -- calls to them, as well as writes out the corresponding Python stub. -- -- See 'mkRpcM' for the requirements on the passed functions and the returned -- expression. genPyUDSRpcStub :: String -- ^ the name of the class to be generated -> String -- ^ the name of the constant from @constants.py@ holding -- the path to a UDS socket -> [Name] -- ^ names of functions to include -> Q Doc genPyUDSRpcStub className constName = liftM (header $+$) . namesToClass className stubCode where header = text "# This file is automatically generated, do not edit!" $+$ text "# pylint: skip-file" stubCode = abstrMethod genericInvokeName [ text "method", text "*args"] $+$ method socketPathName [] ( text "from ganeti import pathutils" $+$ text "return" <+> text "pathutils." <> text constName) method name args body = text "def" <+> apply name (text "self" : args) <> colon $+$ nest' body $+$ emptyLine abstrMethod name args = method name args $ text "raise" <+> apply "NotImplementedError" [] -- The same as 'genPyUDSRpcStub', but returns the result as a @String@ -- expression. genPyUDSRpcStubStr :: String -- ^ the name of the class to be generated -> String -- ^ the constant in @pathutils.py@ holding the socket path -> [Name] -- ^ functions to include -> Q Exp genPyUDSRpcStubStr className constName names = liftString . render =<< genPyUDSRpcStub className constName names ganeti-3.1.0~rc2/src/Ganeti/THH/PyType.hs000064400000000000000000000113611476477700300200150ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| PyType helper for Ganeti Haskell code. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.THH.PyType ( PyType(..) , pyType , pyOptionalType ) where import Control.Monad import Data.List (intercalate) import Language.Haskell.TH import Language.Haskell.TH.Syntax (Lift(..)) import Ganeti.PyValue -- | Represents a Python encoding of types. data PyType = PTMaybe PyType | PTApp PyType [PyType] | PTOther String | PTAny | PTDictOf | PTListOf | PTNone | PTObject | PTOr | PTSetOf | PTTupleOf deriving (Show, Eq, Ord) -- TODO: We could use th-lift to generate this instance automatically. instance Lift PyType where lift (PTMaybe x) = [| PTMaybe x |] lift (PTApp tf as) = [| PTApp tf as |] lift (PTOther i) = [| PTOther i |] lift PTAny = [| PTAny |] lift PTDictOf = [| PTDictOf |] lift PTListOf = [| PTListOf |] lift PTNone = [| PTNone |] lift PTObject = [| PTObject |] lift PTOr = [| PTOr |] lift PTSetOf = [| PTSetOf |] lift PTTupleOf = [| PTTupleOf |] instance PyValue PyType where -- Use lib/ht.py type aliases to avoid Python creating redundant -- new match functions for commonly used OpCode param types. showValue (PTMaybe (PTOther "NonEmptyString")) = ht "MaybeString" showValue (PTMaybe (PTOther "Bool")) = ht "MaybeBool" showValue (PTMaybe PTDictOf) = ht "MaybeDict" showValue (PTMaybe PTListOf) = ht "MaybeList" showValue (PTMaybe x) = ptApp (ht "Maybe") [x] showValue (PTApp tf as) = ptApp (showValue tf) as showValue (PTOther i) = ht i showValue PTAny = ht "Any" showValue PTDictOf = ht "DictOf" showValue PTListOf = ht "ListOf" showValue PTNone = ht "None" showValue PTObject = ht "Object" showValue PTOr = ht "Or" showValue PTSetOf = ht "SetOf" showValue PTTupleOf = ht "TupleOf" ht :: String -> String ht = ("ht.T" ++) ptApp :: String -> [PyType] -> String ptApp name ts = name ++ "(" ++ intercalate ", " (map showValue ts) ++ ")" -- | Converts a Haskell type name into a Python type name. pyTypeName :: Name -> PyType pyTypeName name = case nameBase name of "()" -> PTNone "Map" -> PTDictOf "Set" -> PTSetOf "ListSet" -> PTSetOf "Either" -> PTOr "GenericContainer" -> PTDictOf "JSValue" -> PTAny "JSObject" -> PTObject str -> PTOther str -- | Converts a Haskell type into a Python type. pyType :: Type -> Q PyType pyType t | not (null args) = PTApp `liftM` pyType fn `ap` mapM pyType args where (fn, args) = pyAppType t pyType (ConT name) = return $ pyTypeName name pyType ListT = return PTListOf pyType (TupleT 0) = return PTNone pyType (TupleT _) = return PTTupleOf pyType typ = fail $ "unhandled case for type " ++ show typ -- | Returns a type and its type arguments. pyAppType :: Type -> (Type, [Type]) pyAppType = g [] where g as (AppT typ1 typ2) = g (typ2 : as) typ1 g as typ = (typ, as) -- | @pyType opt typ@ converts Haskell type @typ@ into a Python type, -- where @opt@ determines if the converted type is optional (i.e., -- Maybe). pyOptionalType :: Bool -> Type -> Q PyType pyOptionalType True typ = PTMaybe <$> pyType typ pyOptionalType False typ = pyType typ ganeti-3.1.0~rc2/src/Ganeti/THH/RPC.hs000064400000000000000000000100111476477700300171760ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, ExistentialQuantification #-} {-| Implements Template Haskell generation of RPC server components from Haskell functions. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.THH.RPC ( Request(..) , RpcServer , dispatch , mkRpcM ) where import Control.Arrow ((&&&)) import Control.Monad import Control.Monad.Except (MonadError, throwError) import Data.Map (Map) import qualified Data.Map as Map import Language.Haskell.TH import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Errors import Ganeti.JSON (fromJResultE, fromJVal) import Ganeti.THH.Types import qualified Ganeti.UDSServer as US data RpcFn m = forall i o . (J.JSON i, J.JSON o) => RpcFn (i -> m o) type RpcServer m = US.Handler Request m J.JSValue -- | A RPC request consiting of a method and its argument(s). data Request = Request { rMethod :: String, rArgs :: J.JSValue } deriving (Eq, Ord, Show) decodeRequest :: J.JSValue -> J.JSValue -> Result Request decodeRequest method args = Request <$> fromJVal method <*> pure args dispatch :: (Monad m) => Map String (RpcFn (ResultT GanetiException m)) -> RpcServer m dispatch fs = US.Handler { US.hParse = decodeRequest , US.hInputLogShort = rMethod , US.hInputLogLong = rMethod , US.hExec = liftToHandler . exec } where orError :: (MonadError e m, Error e) => Maybe a -> e -> m a orError m e = maybe (throwError e) return m exec (Request m as) = do (RpcFn f) <- orError (Map.lookup m fs) (strMsg $ "No such method: " ++ m) i <- fromJResultE "RPC input" . J.readJSON $ as o <- f i -- lift $ f i return $ J.showJSON o liftToHandler :: (Monad m) => ResultT GanetiException m J.JSValue -> US.HandlerResult m J.JSValue liftToHandler = liftM ((,) True) . runResultT -- | Converts a function into the appropriate @RpcFn m@ expression. -- The function's result must be monadic. toRpcFn :: Name -> Q Exp toRpcFn name = [| RpcFn $( uncurryVar name ) |] -- | Convert a list of named expressions into an expression containing a list -- of name/expression pairs. rpcFnsList :: [(String, Q Exp)] -> Q Exp rpcFnsList = listE . map (\(name, expr) -> tupE [stringE name, expr]) -- | Takes a list of function names and creates a RPC handler that delegates -- calls to them. -- -- The functions must conform to -- @(J.JSON i, J.JSON o) => i -> ResultT GanetiException m o@. The @m@ -- monads types of all the functions must unify. -- -- The result expression is of type @RpcServer m@. mkRpcM :: [Name] -- ^ the names of functions to include -> Q Exp mkRpcM names = [| dispatch . Map.fromList $ $( rpcFnsList . map (nameBase &&& toRpcFn) $ names ) |] ganeti-3.1.0~rc2/src/Ganeti/THH/Types.hs000064400000000000000000000110211476477700300176600ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, DeriveFunctor, CPP #-} {-| Utility Template Haskell functions for working with types. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.THH.Types ( typeOfFun , funArgs , tupleArgs , argumentType , uncurryVarType , uncurryVar , curryN , OneTuple(..) ) where import Control.Arrow (first) import Control.Monad (liftM, replicateM) import Language.Haskell.TH import qualified Text.JSON as J import Ganeti.THH.Compat (nonUnaryTupE) -- | This fills the gap between @()@ and @(,)@, providing a wrapper for -- 1-element tuples. It's needed for RPC, where arguments for a function are -- sent as a list of values, and therefore for 1-argument functions we need -- this wrapper, which packs/unpacks 1-element lists. newtype OneTuple a = OneTuple { getOneTuple :: a } deriving (Eq, Ord, Show, Functor) -- The value is stored in @JSON@ as a 1-element list. instance J.JSON a => J.JSON (OneTuple a) where showJSON (OneTuple a) = J.JSArray [J.showJSON a] readJSON (J.JSArray [x]) = liftM OneTuple (J.readJSON x) readJSON _ = J.Error "Unable to read 1 tuple" -- | Returns the type of a function. If the given name doesn't correspond to a -- function, fails. typeOfFun :: Name -> Q Type typeOfFun name = reify name >>= args where args :: Info -> Q Type args (VarI _ tp _) = return tp args _ = fail $ "Not a function: " ++ show name -- | Splits a function type into the types of its arguments and the result. funArgs :: Type -> ([Type], Type) funArgs = first reverse . f [] where f ts (ForallT _ _ x) = f ts x f ts (AppT (AppT ArrowT t) x) = f (t:ts) x f ts x = (ts, x) tupleArgs :: Type -> Maybe [Type] tupleArgs = fmap reverse . f [] where f ts (TupleT _) = Just ts f ts (AppT (AppT ArrowT x) t) = f (t:ts) x f _ _ = Nothing -- | Given a type of the form @m a@, this function extracts @a@. -- If the given type is of another form, it fails with an error message. argumentType :: Type -> Q Type argumentType (AppT _ t) = return t argumentType t = fail $ "Not a type of the form 'm a': " ++ show t -- | Generic 'uncurry' that counts the number of function arguments in a type -- and constructs the appropriate uncurry function into @i -> o@. -- It the type has no arguments, it's converted into @() -> o@. uncurryVarType :: Type -> Q Exp uncurryVarType = uncurryN . length . fst . funArgs where uncurryN 0 = do f <- newName "f" return $ LamE [VarP f, TupP []] (VarE f) uncurryN 1 = [| (. getOneTuple) |] uncurryN n = do f <- newName "f" ps <- replicateM n (newName "x") return $ LamE [VarP f, TupP $ map VarP ps] (foldl AppE (VarE f) $ map VarE ps) -- | Creates an uncurried version of a function. -- If the function has no arguments, it's converted into @() -> o@. uncurryVar :: Name -> Q Exp uncurryVar name = do t <- typeOfFun name appE (uncurryVarType t) (varE name) -- | Generic 'curry' that constructs a curring function of a given arity. curryN :: Int -> Q Exp curryN 0 = [| ($ ()) |] curryN 1 = [| (. OneTuple) |] curryN n = do f <- newName "f" ps <- replicateM n (newName "x") return $ LamE (VarP f : map VarP ps) (AppE (VarE f) (nonUnaryTupE $ map VarE ps)) ganeti-3.1.0~rc2/src/Ganeti/Types.hs000064400000000000000000001047651476477700300172570ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, DeriveFunctor #-} {-| Some common Ganeti types. This holds types common to both core work, and to htools. Types that are very core specific (e.g. configuration objects) should go in 'Ganeti.Objects', while types that are specific to htools in-memory representation should go into 'Ganeti.HTools.Types'. -} {- Copyright (C) 2012, 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Types ( AllocPolicy(..) , allocPolicyFromRaw , allocPolicyToRaw , InstanceStatus(..) , instanceStatusFromRaw , instanceStatusToRaw , DiskTemplate(..) , diskTemplateToRaw , diskTemplateFromRaw , diskTemplateMovable , TagKind(..) , tagKindToRaw , tagKindFromRaw , NonNegative , fromNonNegative , mkNonNegative , Positive , fromPositive , mkPositive , Negative , fromNegative , mkNegative , NonEmpty , fromNonEmpty , mkNonEmpty , NonEmptyString , QueryResultCode , IPv4Address , mkIPv4Address , IPv4Network , mkIPv4Network , IPv6Address , mkIPv6Address , IPv6Network , mkIPv6Network , MigrationMode(..) , migrationModeToRaw , VerifyOptionalChecks(..) , verifyOptionalChecksToRaw , DdmSimple(..) , DdmFull(..) , ddmFullToRaw , CVErrorCode(..) , cVErrorCodeToRaw , Hypervisor(..) , hypervisorFromRaw , hypervisorToRaw , OobCommand(..) , oobCommandToRaw , OobStatus(..) , oobStatusToRaw , StorageType(..) , storageTypeToRaw , EvacMode(..) , evacModeToRaw , FileDriver(..) , fileDriverToRaw , InstCreateMode(..) , instCreateModeToRaw , RebootType(..) , rebootTypeToRaw , ExportMode(..) , exportModeToRaw , IAllocatorTestDir(..) , iAllocatorTestDirToRaw , IAllocatorMode(..) , iAllocatorModeToRaw , NICMode(..) , nICModeToRaw , JobStatus(..) , jobStatusToRaw , jobStatusFromRaw , FinalizedJobStatus(..) , finalizedJobStatusToRaw , JobId , fromJobId , makeJobId , makeJobIdS , RelativeJobId , JobIdDep(..) , JobDependency(..) , absoluteJobDependency , getJobIdFromDependency , OpSubmitPriority(..) , opSubmitPriorityToRaw , parseSubmitPriority , fmtSubmitPriority , OpStatus(..) , opStatusToRaw , opStatusFromRaw , ELogType(..) , eLogTypeToRaw , ReasonElem , ReasonTrail , StorageUnit(..) , StorageUnitRaw(..) , StorageKey , addParamsToStorageUnit , diskTemplateToStorageType , VType(..) , vTypeFromRaw , vTypeToRaw , NodeRole(..) , nodeRoleToRaw , roleDescription , DiskMode(..) , diskModeToRaw , BlockDriver(..) , blockDriverToRaw , AdminState(..) , adminStateFromRaw , adminStateToRaw , AdminStateSource(..) , adminStateSourceFromRaw , adminStateSourceToRaw , StorageField(..) , storageFieldToRaw , DiskAccessMode(..) , diskAccessModeToRaw , LocalDiskStatus(..) , localDiskStatusFromRaw , localDiskStatusToRaw , localDiskStatusName , ReplaceDisksMode(..) , replaceDisksModeToRaw , RpcTimeout(..) , rpcTimeoutFromRaw -- FIXME: no used anywhere , rpcTimeoutToRaw , HotplugTarget(..) , hotplugTargetToRaw , HotplugAction(..) , hotplugActionToRaw , SshKeyType(..) , sshKeyTypeToRaw , Private(..) , showPrivateJSObject , Secret(..) , showSecretJSObject , revealValInJSObject , redacted , HvParams , OsParams , OsParamsPrivate , TimeStampObject(..) , UuidObject(..) , ForthcomingObject(..) , SerialNoObject(..) , TagsObject(..) ) where import Control.Monad (liftM) import Control.Monad.Fail (MonadFail) import qualified Text.JSON as JSON import Text.JSON (JSON, readJSON, showJSON) import Data.Ratio (numerator, denominator) import System.Time (ClockTime) import qualified Ganeti.ConstantUtils as ConstantUtils import Ganeti.JSON (Container, HasStringRepr(..)) import qualified Ganeti.THH as THH import Ganeti.THH.Field (TagSet) import Ganeti.Utils -- * Generic types -- | Type that holds a non-negative value. newtype NonNegative a = NonNegative { fromNonNegative :: a } deriving (Show, Eq, Ord) -- | Smart constructor for 'NonNegative'. mkNonNegative :: (MonadFail m, Num a, Ord a, Show a) => a -> m (NonNegative a) mkNonNegative i | i >= 0 = return (NonNegative i) | otherwise = fail $ "Invalid value for non-negative type '" ++ show i ++ "'" instance (JSON.JSON a, Num a, Ord a, Show a) => JSON.JSON (NonNegative a) where showJSON = JSON.showJSON . fromNonNegative readJSON v = JSON.readJSON v >>= mkNonNegative -- | Type that holds a positive value. newtype Positive a = Positive { fromPositive :: a } deriving (Show, Eq, Ord) -- | Smart constructor for 'Positive'. mkPositive :: (MonadFail m, Num a, Ord a, Show a) => a -> m (Positive a) mkPositive i | i > 0 = return (Positive i) | otherwise = fail $ "Invalid value for positive type '" ++ show i ++ "'" instance (JSON.JSON a, Num a, Ord a, Show a) => JSON.JSON (Positive a) where showJSON = JSON.showJSON . fromPositive readJSON v = JSON.readJSON v >>= mkPositive -- | Type that holds a negative value. newtype Negative a = Negative { fromNegative :: a } deriving (Show, Eq, Ord) -- | Smart constructor for 'Negative'. mkNegative :: (MonadFail m, Num a, Ord a, Show a) => a -> m (Negative a) mkNegative i | i < 0 = return (Negative i) | otherwise = fail $ "Invalid value for negative type '" ++ show i ++ "'" instance (JSON.JSON a, Num a, Ord a, Show a) => JSON.JSON (Negative a) where showJSON = JSON.showJSON . fromNegative readJSON v = JSON.readJSON v >>= mkNegative -- | Type that holds a non-null list. newtype NonEmpty a = NonEmpty { fromNonEmpty :: [a] } deriving (Show, Eq, Ord) -- | Smart constructor for 'NonEmpty'. mkNonEmpty :: (MonadFail m) => [a] -> m (NonEmpty a) mkNonEmpty [] = fail "Received empty value for non-empty list" mkNonEmpty xs = return (NonEmpty xs) instance (JSON.JSON a) => JSON.JSON (NonEmpty a) where showJSON = JSON.showJSON . fromNonEmpty readJSON v = JSON.readJSON v >>= mkNonEmpty -- | A simple type alias for non-empty strings. type NonEmptyString = NonEmpty Char type QueryResultCode = Int newtype IPv4Address = IPv4Address { fromIPv4Address :: String } deriving (Show, Eq, Ord) -- FIXME: this should check that 'address' is a valid ip mkIPv4Address :: Monad m => String -> m IPv4Address mkIPv4Address address = return IPv4Address { fromIPv4Address = address } instance JSON.JSON IPv4Address where showJSON = JSON.showJSON . fromIPv4Address readJSON v = JSON.readJSON v >>= mkIPv4Address newtype IPv4Network = IPv4Network { fromIPv4Network :: String } deriving (Show, Eq, Ord) -- FIXME: this should check that 'address' is a valid ip mkIPv4Network :: Monad m => String -> m IPv4Network mkIPv4Network address = return IPv4Network { fromIPv4Network = address } instance JSON.JSON IPv4Network where showJSON = JSON.showJSON . fromIPv4Network readJSON v = JSON.readJSON v >>= mkIPv4Network newtype IPv6Address = IPv6Address { fromIPv6Address :: String } deriving (Show, Eq, Ord) -- FIXME: this should check that 'address' is a valid ip mkIPv6Address :: Monad m => String -> m IPv6Address mkIPv6Address address = return IPv6Address { fromIPv6Address = address } instance JSON.JSON IPv6Address where showJSON = JSON.showJSON . fromIPv6Address readJSON v = JSON.readJSON v >>= mkIPv6Address newtype IPv6Network = IPv6Network { fromIPv6Network :: String } deriving (Show, Eq, Ord) -- FIXME: this should check that 'address' is a valid ip mkIPv6Network :: Monad m => String -> m IPv6Network mkIPv6Network address = return IPv6Network { fromIPv6Network = address } instance JSON.JSON IPv6Network where showJSON = JSON.showJSON . fromIPv6Network readJSON v = JSON.readJSON v >>= mkIPv6Network -- * Ganeti types -- | Instance disk template type. The disk template is a name for the -- constructor of the disk configuration 'DiskLogicalId' used for -- serialization, configuration values, etc. $(THH.declareLADT ''String "DiskTemplate" [ ("DTDiskless", "diskless") , ("DTFile", "file") , ("DTSharedFile", "sharedfile") , ("DTPlain", "plain") , ("DTBlock", "blockdev") , ("DTDrbd8", "drbd") , ("DTRbd", "rbd") , ("DTExt", "ext") , ("DTGluster", "gluster") ]) $(THH.makeJSONInstance ''DiskTemplate) instance THH.PyValue DiskTemplate where showValue = show . diskTemplateToRaw instance HasStringRepr DiskTemplate where fromStringRepr = diskTemplateFromRaw toStringRepr = diskTemplateToRaw -- | Predicate on disk templates indicating if instances based on this -- disk template can freely be moved (to any node in the node group). diskTemplateMovable :: DiskTemplate -> Bool -- Note: we deliberately do not use wildcard pattern to force an -- update of this function whenever a new disk template is added. diskTemplateMovable DTDiskless = True diskTemplateMovable DTFile = False diskTemplateMovable DTSharedFile = True diskTemplateMovable DTPlain = False diskTemplateMovable DTBlock = False diskTemplateMovable DTDrbd8 = False diskTemplateMovable DTRbd = True diskTemplateMovable DTExt = True diskTemplateMovable DTGluster = True -- | Data type representing what items the tag operations apply to. $(THH.declareLADT ''String "TagKind" [ ("TagKindInstance", "instance") , ("TagKindNode", "node") , ("TagKindGroup", "nodegroup") , ("TagKindCluster", "cluster") , ("TagKindNetwork", "network") ]) $(THH.makeJSONInstance ''TagKind) -- | The Group allocation policy type. -- -- Note that the order of constructors is important as the automatic -- Ord instance will order them in the order they are defined, so when -- changing this data type be careful about the interaction with the -- desired sorting order. $(THH.declareLADT ''String "AllocPolicy" [ ("AllocPreferred", "preferred") , ("AllocLastResort", "last_resort") , ("AllocUnallocable", "unallocable") ]) $(THH.makeJSONInstance ''AllocPolicy) -- | The Instance real state type. $(THH.declareLADT ''String "InstanceStatus" [ ("StatusDown", "ADMIN_down") , ("StatusOffline", "ADMIN_offline") , ("ErrorDown", "ERROR_down") , ("ErrorUp", "ERROR_up") , ("NodeDown", "ERROR_nodedown") , ("NodeOffline", "ERROR_nodeoffline") , ("Running", "running") , ("UserDown", "USER_down") , ("WrongNode", "ERROR_wrongnode") ]) $(THH.makeJSONInstance ''InstanceStatus) -- | Migration mode. $(THH.declareLADT ''String "MigrationMode" [ ("MigrationLive", "live") , ("MigrationNonLive", "non-live") ]) $(THH.makeJSONInstance ''MigrationMode) -- | Verify optional checks. $(THH.declareLADT ''String "VerifyOptionalChecks" [ ("VerifyNPlusOneMem", "nplusone_mem") , ("VerifyHVParamAssessment", "hvparam_assessment") ]) $(THH.makeJSONInstance ''VerifyOptionalChecks) -- | Cluster verify error codes. $(THH.declareLADT ''String "CVErrorCode" [ ("CvECLUSTERCFG", "ECLUSTERCFG") , ("CvECLUSTERCERT", "ECLUSTERCERT") , ("CvECLUSTERCLIENTCERT", "ECLUSTERCLIENTCERT") , ("CvECLUSTERFILECHECK", "ECLUSTERFILECHECK") , ("CvECLUSTERDANGLINGNODES", "ECLUSTERDANGLINGNODES") , ("CvECLUSTERDANGLINGINST", "ECLUSTERDANGLINGINST") , ("CvEINSTANCEBADNODE", "EINSTANCEBADNODE") , ("CvEINSTANCEDOWN", "EINSTANCEDOWN") , ("CvEINSTANCELAYOUT", "EINSTANCELAYOUT") , ("CvEINSTANCEMISSINGDISK", "EINSTANCEMISSINGDISK") , ("CvEINSTANCEFAULTYDISK", "EINSTANCEFAULTYDISK") , ("CvEINSTANCEWRONGNODE", "EINSTANCEWRONGNODE") , ("CvEINSTANCESPLITGROUPS", "EINSTANCESPLITGROUPS") , ("CvEINSTANCEPOLICY", "EINSTANCEPOLICY") , ("CvEINSTANCEUNSUITABLENODE", "EINSTANCEUNSUITABLENODE") , ("CvEINSTANCEMISSINGCFGPARAMETER", "EINSTANCEMISSINGCFGPARAMETER") , ("CvENODEDRBD", "ENODEDRBD") , ("CvENODEDRBDVERSION", "ENODEDRBDVERSION") , ("CvENODEDRBDHELPER", "ENODEDRBDHELPER") , ("CvENODEFILECHECK", "ENODEFILECHECK") , ("CvENODEHOOKS", "ENODEHOOKS") , ("CvENODEHV", "ENODEHV") , ("CvENODELVM", "ENODELVM") , ("CvENODEN1", "ENODEN1") , ("CvENODENET", "ENODENET") , ("CvENODEOS", "ENODEOS") , ("CvENODEORPHANINSTANCE", "ENODEORPHANINSTANCE") , ("CvENODEORPHANLV", "ENODEORPHANLV") , ("CvENODERPC", "ENODERPC") , ("CvENODESSH", "ENODESSH") , ("CvENODEVERSION", "ENODEVERSION") , ("CvENODESETUP", "ENODESETUP") , ("CvENODETIME", "ENODETIME") , ("CvENODEOOBPATH", "ENODEOOBPATH") , ("CvENODEUSERSCRIPTS", "ENODEUSERSCRIPTS") , ("CvENODEFILESTORAGEPATHS", "ENODEFILESTORAGEPATHS") , ("CvENODEFILESTORAGEPATHUNUSABLE", "ENODEFILESTORAGEPATHUNUSABLE") , ("CvENODESHAREDFILESTORAGEPATHUNUSABLE", "ENODESHAREDFILESTORAGEPATHUNUSABLE") , ("CvENODEGLUSTERSTORAGEPATHUNUSABLE", "ENODEGLUSTERSTORAGEPATHUNUSABLE") , ("CvEGROUPDIFFERENTPVSIZE", "EGROUPDIFFERENTPVSIZE") , ("CvEEXTAGS", "EEXTAGS") ]) $(THH.makeJSONInstance ''CVErrorCode) -- | Dynamic device modification, just add/remove version. $(THH.declareLADT ''String "DdmSimple" [ ("DdmSimpleAdd", "add") , ("DdmSimpleAttach", "attach") , ("DdmSimpleRemove", "remove") , ("DdmSimpleDetach", "detach") ]) $(THH.makeJSONInstance ''DdmSimple) -- | Dynamic device modification, all operations version. -- -- TODO: DDM_SWAP, DDM_MOVE? $(THH.declareLADT ''String "DdmFull" [ ("DdmFullAdd", "add") , ("DdmFullAttach", "attach") , ("DdmFullRemove", "remove") , ("DdmFullDetach", "detach") , ("DdmFullModify", "modify") ]) $(THH.makeJSONInstance ''DdmFull) -- | Hypervisor type definitions. $(THH.declareLADT ''String "Hypervisor" [ ("Kvm", "kvm") , ("XenPvm", "xen-pvm") , ("Chroot", "chroot") , ("XenHvm", "xen-hvm") , ("Lxc", "lxc") , ("Fake", "fake") ]) $(THH.makeJSONInstance ''Hypervisor) instance THH.PyValue Hypervisor where showValue = show . hypervisorToRaw instance HasStringRepr Hypervisor where fromStringRepr = hypervisorFromRaw toStringRepr = hypervisorToRaw -- | Oob command type. $(THH.declareLADT ''String "OobCommand" [ ("OobHealth", "health") , ("OobPowerCycle", "power-cycle") , ("OobPowerOff", "power-off") , ("OobPowerOn", "power-on") , ("OobPowerStatus", "power-status") ]) $(THH.makeJSONInstance ''OobCommand) -- | Oob command status $(THH.declareLADT ''String "OobStatus" [ ("OobStatusCritical", "CRITICAL") , ("OobStatusOk", "OK") , ("OobStatusUnknown", "UNKNOWN") , ("OobStatusWarning", "WARNING") ]) $(THH.makeJSONInstance ''OobStatus) -- | Storage type. $(THH.declareLADT ''String "StorageType" [ ("StorageFile", "file") , ("StorageSharedFile", "sharedfile") , ("StorageGluster", "gluster") , ("StorageLvmPv", "lvm-pv") , ("StorageLvmVg", "lvm-vg") , ("StorageDiskless", "diskless") , ("StorageBlock", "blockdev") , ("StorageRados", "rados") , ("StorageExt", "ext") ]) $(THH.makeJSONInstance ''StorageType) -- | Storage keys are identifiers for storage units. Their content varies -- depending on the storage type, for example a storage key for LVM storage -- is the volume group name. type StorageKey = String -- | Storage parameters type SPExclusiveStorage = Bool -- | Storage units without storage-type-specific parameters data StorageUnitRaw = SURaw StorageType StorageKey -- | Full storage unit with storage-type-specific parameters data StorageUnit = SUFile StorageKey | SUSharedFile StorageKey | SUGluster StorageKey | SULvmPv StorageKey SPExclusiveStorage | SULvmVg StorageKey SPExclusiveStorage | SUDiskless StorageKey | SUBlock StorageKey | SURados StorageKey | SUExt StorageKey deriving (Eq) instance Show StorageUnit where show (SUFile key) = showSUSimple StorageFile key show (SUSharedFile key) = showSUSimple StorageSharedFile key show (SUGluster key) = showSUSimple StorageGluster key show (SULvmPv key es) = showSULvm StorageLvmPv key es show (SULvmVg key es) = showSULvm StorageLvmVg key es show (SUDiskless key) = showSUSimple StorageDiskless key show (SUBlock key) = showSUSimple StorageBlock key show (SURados key) = showSUSimple StorageRados key show (SUExt key) = showSUSimple StorageExt key instance JSON StorageUnit where showJSON (SUFile key) = showJSON (StorageFile, key, []::[String]) showJSON (SUSharedFile key) = showJSON (StorageSharedFile, key, []::[String]) showJSON (SUGluster key) = showJSON (StorageGluster, key, []::[String]) showJSON (SULvmPv key es) = showJSON (StorageLvmPv, key, [es]) showJSON (SULvmVg key es) = showJSON (StorageLvmVg, key, [es]) showJSON (SUDiskless key) = showJSON (StorageDiskless, key, []::[String]) showJSON (SUBlock key) = showJSON (StorageBlock, key, []::[String]) showJSON (SURados key) = showJSON (StorageRados, key, []::[String]) showJSON (SUExt key) = showJSON (StorageExt, key, []::[String]) -- FIXME: add readJSON implementation readJSON _ = fail "Not implemented" -- | Composes a string representation of storage types without -- storage parameters showSUSimple :: StorageType -> StorageKey -> String showSUSimple st sk = show (storageTypeToRaw st, sk, []::[String]) -- | Composes a string representation of the LVM storage types showSULvm :: StorageType -> StorageKey -> SPExclusiveStorage -> String showSULvm st sk es = show (storageTypeToRaw st, sk, [es]) -- | Mapping from disk templates to storage types. diskTemplateToStorageType :: DiskTemplate -> StorageType diskTemplateToStorageType DTExt = StorageExt diskTemplateToStorageType DTFile = StorageFile diskTemplateToStorageType DTSharedFile = StorageSharedFile diskTemplateToStorageType DTDrbd8 = StorageLvmVg diskTemplateToStorageType DTPlain = StorageLvmVg diskTemplateToStorageType DTRbd = StorageRados diskTemplateToStorageType DTDiskless = StorageDiskless diskTemplateToStorageType DTBlock = StorageBlock diskTemplateToStorageType DTGluster = StorageGluster -- | Equips a raw storage unit with its parameters addParamsToStorageUnit :: SPExclusiveStorage -> StorageUnitRaw -> StorageUnit addParamsToStorageUnit _ (SURaw StorageBlock key) = SUBlock key addParamsToStorageUnit _ (SURaw StorageDiskless key) = SUDiskless key addParamsToStorageUnit _ (SURaw StorageExt key) = SUExt key addParamsToStorageUnit _ (SURaw StorageFile key) = SUFile key addParamsToStorageUnit _ (SURaw StorageSharedFile key) = SUSharedFile key addParamsToStorageUnit _ (SURaw StorageGluster key) = SUGluster key addParamsToStorageUnit es (SURaw StorageLvmPv key) = SULvmPv key es addParamsToStorageUnit es (SURaw StorageLvmVg key) = SULvmVg key es addParamsToStorageUnit _ (SURaw StorageRados key) = SURados key -- | Node evac modes. -- -- This is part of the 'IAllocator' interface and it is used, for -- example, in 'Ganeti.HTools.Loader.RqType'. However, it must reside -- in this module, and not in 'Ganeti.HTools.Types', because it is -- also used by 'Ganeti.Constants'. $(THH.declareLADT ''String "EvacMode" [ ("ChangePrimary", "primary-only") , ("ChangeSecondary", "secondary-only") , ("ChangeAll", "all") ]) $(THH.makeJSONInstance ''EvacMode) -- | The file driver type. $(THH.declareLADT ''String "FileDriver" [ ("FileLoop", "loop") , ("FileBlktap", "blktap") , ("FileBlktap2", "blktap2") ]) $(THH.makeJSONInstance ''FileDriver) -- | The instance create mode. $(THH.declareLADT ''String "InstCreateMode" [ ("InstCreate", "create") , ("InstImport", "import") , ("InstRemoteImport", "remote-import") ]) $(THH.makeJSONInstance ''InstCreateMode) -- | Reboot type. $(THH.declareLADT ''String "RebootType" [ ("RebootSoft", "soft") , ("RebootHard", "hard") , ("RebootFull", "full") ]) $(THH.makeJSONInstance ''RebootType) -- | Export modes. $(THH.declareLADT ''String "ExportMode" [ ("ExportModeLocal", "local") , ("ExportModeRemote", "remote") ]) $(THH.makeJSONInstance ''ExportMode) -- | IAllocator run types (OpTestIAllocator). $(THH.declareLADT ''String "IAllocatorTestDir" [ ("IAllocatorDirIn", "in") , ("IAllocatorDirOut", "out") ]) $(THH.makeJSONInstance ''IAllocatorTestDir) -- | IAllocator mode. FIXME: use this in "HTools.Backend.IAlloc". $(THH.declareLADT ''String "IAllocatorMode" [ ("IAllocatorAlloc", "allocate") , ("IAllocatorAllocateSecondary", "allocate-secondary") , ("IAllocatorMultiAlloc", "multi-allocate") , ("IAllocatorReloc", "relocate") , ("IAllocatorNodeEvac", "node-evacuate") , ("IAllocatorChangeGroup", "change-group") ]) $(THH.makeJSONInstance ''IAllocatorMode) -- | Network mode. $(THH.declareLADT ''String "NICMode" [ ("NMBridged", "bridged") , ("NMRouted", "routed") , ("NMOvs", "openvswitch") , ("NMPool", "pool") ]) $(THH.makeJSONInstance ''NICMode) -- | The JobStatus data type. Note that this is ordered especially -- such that greater\/lesser comparison on values of this type makes -- sense. $(THH.declareLADT ''String "JobStatus" [ ("JOB_STATUS_QUEUED", "queued") , ("JOB_STATUS_WAITING", "waiting") , ("JOB_STATUS_CANCELING", "canceling") , ("JOB_STATUS_RUNNING", "running") , ("JOB_STATUS_CANCELED", "canceled") , ("JOB_STATUS_SUCCESS", "success") , ("JOB_STATUS_ERROR", "error") ]) $(THH.makeJSONInstance ''JobStatus) -- | Finalized job status. $(THH.declareLADT ''String "FinalizedJobStatus" [ ("JobStatusCanceled", "canceled") , ("JobStatusSuccessful", "success") , ("JobStatusFailed", "error") ]) $(THH.makeJSONInstance ''FinalizedJobStatus) -- | The Ganeti job type. newtype JobId = JobId { fromJobId :: Int } deriving (Show, Eq, Ord) -- | Builds a job ID. makeJobId :: (MonadFail m) => Int -> m JobId makeJobId i | i >= 0 = return $ JobId i | otherwise = fail $ "Invalid value for job ID ' " ++ show i ++ "'" -- | Builds a job ID from a string. makeJobIdS :: (MonadFail m) => String -> m JobId makeJobIdS s = tryRead "parsing job id" s >>= makeJobId -- | Parses a job ID. parseJobId :: (MonadFail m) => JSON.JSValue -> m JobId parseJobId (JSON.JSString x) = makeJobIdS $ JSON.fromJSString x parseJobId (JSON.JSRational _ x) = if denominator x /= 1 then fail $ "Got fractional job ID from master daemon?! Value:" ++ show x -- FIXME: potential integer overflow here on 32-bit platforms else makeJobId . fromIntegral . numerator $ x parseJobId x = fail $ "Wrong type/value for job id: " ++ show x instance JSON.JSON JobId where showJSON = JSON.showJSON . fromJobId readJSON = parseJobId -- | Relative job ID type alias. type RelativeJobId = Negative Int -- | Job ID dependency. data JobIdDep = JobDepRelative RelativeJobId | JobDepAbsolute JobId deriving (Show, Eq, Ord) instance JSON.JSON JobIdDep where showJSON (JobDepRelative i) = showJSON i showJSON (JobDepAbsolute i) = showJSON i readJSON v = case JSON.readJSON v::JSON.Result (Negative Int) of -- first try relative dependency, usually most common JSON.Ok r -> return $ JobDepRelative r JSON.Error _ -> liftM JobDepAbsolute (parseJobId v) -- | From job ID dependency and job ID, compute the absolute dependency. absoluteJobIdDep :: (MonadFail m) => JobIdDep -> JobId -> m JobIdDep absoluteJobIdDep (JobDepAbsolute jid) _ = return $ JobDepAbsolute jid absoluteJobIdDep (JobDepRelative rjid) jid = liftM JobDepAbsolute . makeJobId $ fromJobId jid + fromNegative rjid -- | Job Dependency type. data JobDependency = JobDependency JobIdDep [FinalizedJobStatus] deriving (Show, Eq, Ord) instance JSON JobDependency where showJSON (JobDependency dep status) = showJSON (dep, status) readJSON = liftM (uncurry JobDependency) . readJSON -- | From job dependency and job id compute an absolute job dependency. absoluteJobDependency :: (MonadFail m) => JobDependency -> JobId -> m JobDependency absoluteJobDependency (JobDependency jdep fstats) jid = liftM (flip JobDependency fstats) $ absoluteJobIdDep jdep jid -- | From a job dependency get the absolute job id it depends on, -- if given absolutely. getJobIdFromDependency :: JobDependency -> [JobId] getJobIdFromDependency (JobDependency (JobDepAbsolute jid) _) = [jid] getJobIdFromDependency _ = [] -- | Valid opcode priorities for submit. $(THH.declareIADT "OpSubmitPriority" [ ("OpPrioLow", 'ConstantUtils.priorityLow) , ("OpPrioNormal", 'ConstantUtils.priorityNormal) , ("OpPrioHigh", 'ConstantUtils.priorityHigh) ]) $(THH.makeJSONInstance ''OpSubmitPriority) -- | Parse submit priorities from a string. parseSubmitPriority :: (MonadFail m) => String -> m OpSubmitPriority parseSubmitPriority "low" = return OpPrioLow parseSubmitPriority "normal" = return OpPrioNormal parseSubmitPriority "high" = return OpPrioHigh parseSubmitPriority str = fail $ "Unknown priority '" ++ str ++ "'" -- | Format a submit priority as string. fmtSubmitPriority :: OpSubmitPriority -> String fmtSubmitPriority OpPrioLow = "low" fmtSubmitPriority OpPrioNormal = "normal" fmtSubmitPriority OpPrioHigh = "high" -- | Our ADT for the OpCode status at runtime (while in a job). $(THH.declareLADT ''String "OpStatus" [ ("OP_STATUS_QUEUED", "queued") , ("OP_STATUS_WAITING", "waiting") , ("OP_STATUS_CANCELING", "canceling") , ("OP_STATUS_RUNNING", "running") , ("OP_STATUS_CANCELED", "canceled") , ("OP_STATUS_SUCCESS", "success") , ("OP_STATUS_ERROR", "error") ]) $(THH.makeJSONInstance ''OpStatus) -- | Type for the job message type. $(THH.declareLADT ''String "ELogType" [ ("ELogMessage", "message") , ("ELogMessageList", "message-list") , ("ELogRemoteImport", "remote-import") , ("ELogJqueueTest", "jqueue-test") , ("ELogDelayTest", "delay-test") ]) $(THH.makeJSONInstance ''ELogType) -- | Type of one element of a reason trail, of form -- @(source, reason, timestamp)@. type ReasonElem = (String, String, Integer) -- | Type representing a reason trail. type ReasonTrail = [ReasonElem] -- | The VTYPES, a mini-type system in Python. $(THH.declareLADT ''String "VType" [ ("VTypeString", "string") , ("VTypeMaybeString", "maybe-string") , ("VTypeBool", "bool") , ("VTypeSize", "size") , ("VTypeInt", "int") , ("VTypeFloat", "float") ]) $(THH.makeJSONInstance ''VType) instance THH.PyValue VType where showValue = THH.showValue . vTypeToRaw -- * Node role type $(THH.declareLADT ''String "NodeRole" [ ("NROffline", "O") , ("NRDrained", "D") , ("NRRegular", "R") , ("NRCandidate", "C") , ("NRMaster", "M") ]) $(THH.makeJSONInstance ''NodeRole) -- | The description of the node role. roleDescription :: NodeRole -> String roleDescription NROffline = "offline" roleDescription NRDrained = "drained" roleDescription NRRegular = "regular" roleDescription NRCandidate = "master candidate" roleDescription NRMaster = "master" -- * Disk types $(THH.declareLADT ''String "DiskMode" [ ("DiskRdOnly", "ro") , ("DiskRdWr", "rw") ]) $(THH.makeJSONInstance ''DiskMode) -- | The persistent block driver type. Currently only one type is allowed. $(THH.declareLADT ''String "BlockDriver" [ ("BlockDrvManual", "manual") ]) $(THH.makeJSONInstance ''BlockDriver) -- * Instance types $(THH.declareLADT ''String "AdminState" [ ("AdminOffline", "offline") , ("AdminDown", "down") , ("AdminUp", "up") ]) $(THH.makeJSONInstance ''AdminState) $(THH.declareLADT ''String "AdminStateSource" [ ("AdminSource", "admin") , ("UserSource", "user") ]) $(THH.makeJSONInstance ''AdminStateSource) instance THH.PyValue AdminStateSource where showValue = THH.showValue . adminStateSourceToRaw -- * Storage field type $(THH.declareLADT ''String "StorageField" [ ( "SFUsed", "used") , ( "SFName", "name") , ( "SFAllocatable", "allocatable") , ( "SFFree", "free") , ( "SFSize", "size") ]) $(THH.makeJSONInstance ''StorageField) -- * Disk access protocol $(THH.declareLADT ''String "DiskAccessMode" [ ( "DiskUserspace", "userspace") , ( "DiskKernelspace", "kernelspace") ]) $(THH.makeJSONInstance ''DiskAccessMode) -- | Local disk status -- -- Python code depends on: -- DiskStatusOk < DiskStatusUnknown < DiskStatusFaulty $(THH.declareILADT "LocalDiskStatus" [ ("DiskStatusOk", 1) , ("DiskStatusSync", 2) , ("DiskStatusUnknown", 3) , ("DiskStatusFaulty", 4) ]) localDiskStatusName :: LocalDiskStatus -> String localDiskStatusName DiskStatusFaulty = "faulty" localDiskStatusName DiskStatusOk = "ok" localDiskStatusName DiskStatusSync = "syncing" localDiskStatusName DiskStatusUnknown = "unknown" -- | Replace disks type. $(THH.declareLADT ''String "ReplaceDisksMode" [ -- Replace disks on primary ("ReplaceOnPrimary", "replace_on_primary") -- Replace disks on secondary , ("ReplaceOnSecondary", "replace_on_secondary") -- Change secondary node , ("ReplaceNewSecondary", "replace_new_secondary") , ("ReplaceAuto", "replace_auto") ]) $(THH.makeJSONInstance ''ReplaceDisksMode) -- | Basic timeouts for RPC calls. $(THH.declareILADT "RpcTimeout" [ ("Urgent", 60) -- 1 minute , ("Fast", 5 * 60) -- 5 minutes , ("Normal", 15 * 60) -- 15 minutes , ("Slow", 3600) -- 1 hour , ("FourHours", 4 * 3600) -- 4 hours , ("OneDay", 86400) -- 1 day ]) -- | Hotplug action. $(THH.declareLADT ''String "HotplugAction" [ ("HAAdd", "hotadd") , ("HARemove", "hotremove") , ("HAMod", "hotmod") ]) $(THH.makeJSONInstance ''HotplugAction) -- | Hotplug Device Target. $(THH.declareLADT ''String "HotplugTarget" [ ("HTDisk", "disk") , ("HTNic", "nic") ]) $(THH.makeJSONInstance ''HotplugTarget) -- | SSH key type. $(THH.declareLADT ''String "SshKeyType" [ ("RSA", "rsa") , ("DSA", "dsa") , ("ECDSA", "ecdsa") ]) $(THH.makeJSONInstance ''SshKeyType) -- * Private type and instances redacted :: String redacted = "" -- | A container for values that should be happy to be manipulated yet -- refuses to be shown unless explicitly requested. newtype Private a = Private { getPrivate :: a } deriving (Eq, Ord, Functor) instance (Show a, JSON.JSON a) => JSON.JSON (Private a) where readJSON = liftM Private . JSON.readJSON showJSON (Private x) = JSON.showJSON x -- | "Show" the value of the field. -- -- It would be better not to implement this at all. -- Alas, Show OpCode requires Show Private. instance Show a => Show (Private a) where show _ = redacted instance THH.PyValue a => THH.PyValue (Private a) where showValue (Private x) = "Private(" ++ THH.showValue x ++ ")" instance Applicative Private where pure = Private Private f <*> Private x = Private (f x) instance Monad Private where (Private x) >>= f = f x return = pure showPrivateJSObject :: (JSON.JSON a) => [(String, a)] -> JSON.JSObject (Private JSON.JSValue) showPrivateJSObject value = JSON.toJSObject $ map f value where f (k, v) = (k, Private $ JSON.showJSON v) -- * Secret type and instances -- | A container for values that behaves like Private, but doesn't leak the -- value through showJSON newtype Secret a = Secret { getSecret :: a } deriving (Eq, Ord, Functor) instance (Show a, JSON.JSON a) => JSON.JSON (Secret a) where readJSON = liftM Secret . JSON.readJSON showJSON = const . JSON.JSString $ JSON.toJSString redacted instance Show a => Show (Secret a) where show _ = redacted instance THH.PyValue a => THH.PyValue (Secret a) where showValue (Secret x) = "Secret(" ++ THH.showValue x ++ ")" instance Applicative Secret where pure = Secret Secret f <*> Secret x = Secret (f x) instance Monad Secret where (Secret x) >>= f = f x return = pure -- | We return "\" here to satisfy the idempotence of serialization -- and deserialization, although this will impact the meaningfulness of secret -- parameters within configuration tests. showSecretJSObject :: (JSON.JSON a) => [(String, a)] -> JSON.JSObject (Secret JSON.JSValue) showSecretJSObject value = JSON.toJSObject $ map f value where f (k, _) = (k, Secret $ JSON.showJSON redacted) revealValInJSObject :: JSON.JSObject (Secret JSON.JSValue) -> JSON.JSObject (Private JSON.JSValue) revealValInJSObject object = JSON.toJSObject . map f $ JSON.fromJSObject object where f (k, v) = (k, Private $ getSecret v) -- | The hypervisor parameter type. This is currently a simple map, -- without type checking on key/value pairs. type HvParams = Container JSON.JSValue -- | The OS parameters type. This is, and will remain, a string -- container, since the keys are dynamically declared by the OSes, and -- the values are always strings. type OsParams = Container String type OsParamsPrivate = Container (Private String) -- | Class of objects that have timestamps. class TimeStampObject a where cTimeOf :: a -> ClockTime mTimeOf :: a -> ClockTime -- | Class of objects that have an UUID. class UuidObject a where uuidOf :: a -> String -- | Class of objects that can be forthcoming. class ForthcomingObject a where isForthcoming :: a -> Bool -- | Class of object that have a serial number. class SerialNoObject a where serialOf :: a -> Int -- | Class of objects that have tags. class TagsObject a where tagsOf :: a -> TagSet ganeti-3.1.0~rc2/src/Ganeti/UDSServer.hs000064400000000000000000000441131476477700300177630ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} {-| Implementation of the Ganeti Unix Domain Socket JSON server interface. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.UDSServer ( ConnectConfig(..) , ServerConfig(..) , Client , Server , RecvResult(..) , MsgKeys(..) , strOfKey -- * Unix sockets , openClientSocket , closeClientSocket , openServerSocket , closeServerSocket , acceptSocket -- * Client and server , connectClient , connectServer , pipeClient , acceptClient , closeClient , clientToFd , clientToHandle , closeServer , buildResponse , parseResponse , buildCall , parseCall , recvMsg , recvMsgExt , sendMsg -- * Client handler , Handler(..) , HandlerResult , listener ) where import Control.Concurrent.Lifted (fork, yield) import Control.Monad.Base import Control.Monad.Trans.Control import Control.Exception (catch) import Control.Monad import qualified Data.ByteString as B import qualified Data.ByteString.UTF8 as UTF8 import Data.IORef import Data.List import Data.Word (Word8) import qualified Network.Socket as S import System.Directory (removeFile) import System.IO ( hClose, hFlush, hPutStr, hWaitForInput, Handle, IOMode(..) , hSetBuffering, BufferMode(..)) import System.IO.Error (isEOFError) import System.Posix.Types (Fd) import System.Posix.IO (createPipe, fdToHandle, handleToFd) import System.Timeout import Text.JSON (encodeStrict, decodeStrict) import qualified Text.JSON as J import Text.JSON.Types import Ganeti.BasicTypes import Ganeti.Errors (GanetiException(..), ErrorResult) import Ganeti.JSON (fromJResult, fromJVal, fromJResultE, fromObj) import Ganeti.Logging import Ganeti.THH import Ganeti.Utils import Ganeti.Utils.Time (getCurrentTimeUSec) import Ganeti.Constants (privateParametersBlacklist) -- * Utility functions -- | Wrapper over System.Timeout.timeout that fails in the IO monad. withTimeout :: Int -> String -> IO a -> IO a withTimeout secs descr action = do result <- timeout (secs * 1000000) action case result of Nothing -> fail $ "Timeout in " ++ descr Just v -> return v -- * Generic protocol functionality -- | Result of receiving a message from the socket. data RecvResult = RecvConnClosed -- ^ Connection closed | RecvError String -- ^ Any other error | RecvOk String -- ^ Successfull receive deriving (Show, Eq) -- | The end-of-message separator. eOM :: Word8 eOM = 3 -- | The end-of-message encoded as a ByteString. bEOM :: B.ByteString bEOM = B.singleton eOM -- | Valid keys in the requests and responses. data MsgKeys = Method | Args | Success | Result -- | The serialisation of MsgKeys into strings in messages. $(genStrOfKey ''MsgKeys "strOfKey") -- Information required for creating a server connection. data ServerConfig = ServerConfig { connPermissions :: FilePermissions , connConfig :: ConnectConfig } -- Information required for creating a client or server connection. data ConnectConfig = ConnectConfig { recvTmo :: Int , sendTmo :: Int } -- | A client encapsulation. Note that it has separate read and write handle. -- For sockets it is the same handle. It is required for bi-directional -- inter-process pipes though. data Client = Client { rsocket :: Handle -- ^ The read part of -- the client socket , wsocket :: Handle -- ^ The write part of -- the client socket , rbuf :: IORef B.ByteString -- ^ Already received buffer , clientConfig :: ConnectConfig } -- | A server encapsulation. data Server = Server { sSocket :: S.Socket -- ^ The bound server socket , sPath :: FilePath -- ^ The scoket's path , serverConfig :: ConnectConfig } -- * Unix sockets -- | Creates a Unix socket and connects it to the specified @path@, -- where @timeout@ specifies the connection timeout. openClientSocket :: Int -- ^ connection timeout -> FilePath -- ^ socket path -> IO Handle openClientSocket tmo path = do sock <- S.socket S.AF_UNIX S.Stream S.defaultProtocol withTimeout tmo "creating a connection" $ S.connect sock (S.SockAddrUnix path) S.socketToHandle sock ReadWriteMode -- | Closes the handle. -- Performing the operation on a handle that has already been closed has no -- effect; doing so is not an error. -- All other operations on a closed handle will fail. closeClientSocket :: Handle -> IO () closeClientSocket = hClose -- | Creates a Unix socket and binds it to the specified @path@. openServerSocket :: FilePath -> IO S.Socket openServerSocket path = do sock <- S.socket S.AF_UNIX S.Stream S.defaultProtocol S.bind sock (S.SockAddrUnix path) return sock closeServerSocket :: S.Socket -> FilePath -> IO () closeServerSocket sock path = do S.close sock removeFile path acceptSocket :: S.Socket -> IO Handle acceptSocket sock = do -- ignore client socket address (clientSock, _) <- S.accept sock S.socketToHandle clientSock ReadWriteMode -- * Client and server -- | Connects to the master daemon and returns a Client. connectClient :: ConnectConfig -- ^ configuration for the client -> Int -- ^ connection timeout -> FilePath -- ^ socket path -> IO Client connectClient conf tmo path = do h <- openClientSocket tmo path rf <- newIORef B.empty return Client { rsocket=h, wsocket=h, rbuf=rf, clientConfig=conf } -- | Creates and returns a server endpoint. connectServer :: ServerConfig -> Bool -> FilePath -> IO Server connectServer sconf setOwner path = do s <- openServerSocket path when setOwner $ do res <- ensurePermissions path (connPermissions sconf) exitIfBad "Error - could not set socket properties" res S.listen s 5 -- 5 is the max backlog return Server { sSocket = s, sPath = path, serverConfig = connConfig sconf } -- | Creates a new bi-directional client pipe. The two returned clients -- talk to each other through the pipe. pipeClient :: ConnectConfig -> IO (Client, Client) pipeClient conf = let newClient r w = do rf <- newIORef B.empty rh <- fdToHandle r wh <- fdToHandle w return Client { rsocket = rh, wsocket = wh , rbuf = rf, clientConfig = conf } in do (r1, w1) <- createPipe (r2, w2) <- createPipe (,) <$> newClient r1 w2 <*> newClient r2 w1 -- | Closes a server endpoint. closeServer :: (MonadBase IO m) => Server -> m () closeServer server = liftBase $ closeServerSocket (sSocket server) (sPath server) -- | Accepts a client acceptClient :: Server -> IO Client acceptClient s = do handle <- acceptSocket (sSocket s) new_buffer <- newIORef B.empty return Client { rsocket=handle , wsocket=handle , rbuf=new_buffer , clientConfig=serverConfig s } -- | Closes the client socket. -- Performing the operation on a client that has already been closed has no -- effect; doing so is not an error. -- All other operations on a closed client will fail with an exception. closeClient :: Client -> IO () closeClient client = do closeClientSocket . wsocket $ client closeClientSocket . rsocket $ client -- | Extracts the read (the first) and the write (the second) file descriptor -- of a client. This closes the underlying 'Handle's, therefore the original -- client is closed and unusable after the call. -- -- The purpose of this function is to keep the communication channel open, -- while replacing a 'Client' with some other means. clientToFd :: Client -> IO (Fd, Fd) clientToFd client | rh == wh = join (,) <$> handleToFd rh | otherwise = (,) <$> handleToFd rh <*> handleToFd wh where rh = rsocket client wh = wsocket client -- | Extracts the read (first) and the write (second) handles of a client. -- The purpose of this function is to allow using a client's handles as -- input/output streams elsewhere. clientToHandle :: Client -> (Handle, Handle) clientToHandle client = (rsocket client, wsocket client) -- | Sends a message over a transport. sendMsg :: Client -> String -> IO () sendMsg s buf = withTimeout (sendTmo $ clientConfig s) "sending a message" $ do t1 <- getCurrentTimeUSec let handle = wsocket s -- Allow buffering (up to 1MiB) when writing to the socket. Note that -- otherwise we get the default of sending each byte in a separate -- system call, resulting in very poor performance. hSetBuffering handle (BlockBuffering . Just $ 1024 * 1024) hPutStr handle buf B.hPut handle bEOM hFlush handle t2 <- getCurrentTimeUSec logDebug $ "sendMsg: " ++ show ((t2 - t1) `div` 1000) ++ "ms" -- | Given a current buffer and the handle, it will read from the -- network until we get a full message, and it will return that -- message and the leftover buffer contents. recvUpdate :: ConnectConfig -> Handle -> B.ByteString -> IO (B.ByteString, B.ByteString) recvUpdate conf handle obuf = do nbuf <- withTimeout (recvTmo conf) "reading a response" $ do _ <- hWaitForInput handle (-1) B.hGetNonBlocking handle 4096 let (msg, remaining) = B.break (eOM ==) nbuf newbuf = B.append obuf msg if B.null remaining then recvUpdate conf handle newbuf else return (newbuf, B.copy (B.tail remaining)) -- | Waits for a message over a transport. recvMsg :: Client -> IO String recvMsg s = do cbuf <- readIORef $ rbuf s let (imsg, ibuf) = B.break (eOM ==) cbuf (msg, nbuf) <- if B.null ibuf -- if old buffer didn't contain a full message -- then we read from network: then recvUpdate (clientConfig s) (rsocket s) cbuf -- else we return data from our buffer, copying it so that the whole -- message isn't retained and can be garbage collected else return (imsg, B.copy (B.tail ibuf)) writeIORef (rbuf s) nbuf return $ UTF8.toString msg -- | Extended wrapper over recvMsg. recvMsgExt :: Client -> IO RecvResult recvMsgExt s = Control.Exception.catch (liftM RecvOk (recvMsg s)) $ \e -> return $ if isEOFError e then RecvConnClosed else RecvError (show e) -- | Serialize a request to String. buildCall :: (J.JSON mth, J.JSON args) => mth -- ^ The method -> args -- ^ The arguments -> String -- ^ The serialized form buildCall mth args = let keyToObj :: (J.JSON a) => MsgKeys -> a -> (String, J.JSValue) keyToObj k v = (strOfKey k, J.showJSON v) in encodeStrict $ toJSObject [ keyToObj Method mth, keyToObj Args args ] -- | Parse the required keys out of a call. parseCall :: (J.JSON mth, J.JSON args) => String -> Result (mth, args) parseCall s = do arr <- fromJResult "parsing top-level JSON message" $ decodeStrict s :: Result (JSObject JSValue) let keyFromObj :: (J.JSON a) => MsgKeys -> Result a keyFromObj = fromObj (fromJSObject arr) . strOfKey (,) <$> keyFromObj Method <*> keyFromObj Args -- | Serialize the response to String. buildResponse :: Bool -- ^ Success -> JSValue -- ^ The arguments -> String -- ^ The serialized form buildResponse success args = let ja = [ (strOfKey Success, JSBool success) , (strOfKey Result, args)] jo = toJSObject ja in encodeStrict jo -- | Try to decode an error from the server response. This function -- will always fail, since it's called only on the error path (when -- status is False). decodeError :: JSValue -> ErrorResult JSValue decodeError val = case fromJVal val of Ok e -> Bad e Bad msg -> Bad $ GenericError msg -- | Check that luxi responses contain the required keys and that the -- call was successful. parseResponse :: String -> ErrorResult JSValue parseResponse s = do when (UTF8.replacement_char `elem` s) $ failError "Failed to decode UTF-8,\ \ detected replacement char after decoding" oarr <- fromJResultE "Parsing LUXI response" (decodeStrict s) let arr = J.fromJSObject oarr status <- fromObj arr (strOfKey Success) result <- fromObj arr (strOfKey Result) if status then return result else decodeError result -- | Logs an outgoing message. logMsg :: (Show e, J.JSON e, MonadLog m) => Handler i m o -> i -- ^ the received request (used for logging) -> GenericResult e J.JSValue -- ^ A message to be sent -> m () logMsg handler req (Bad err) = logWarning $ "Failed to execute request " ++ hInputLogLong handler req ++ ": " ++ show err logMsg handler req (Ok result) = do -- only log the first 2,000 chars of the result logDebug $ "Result (truncated): " ++ take 2000 (J.encode result) logDebug $ "Successfully handled " ++ hInputLogShort handler req -- | Prepares an outgoing message. prepareMsg :: (J.JSON e) => GenericResult e J.JSValue -- ^ A message to be sent -> (Bool, J.JSValue) prepareMsg (Bad err) = (False, J.showJSON err) prepareMsg (Ok result) = (True, result) -- * Processing client requests type HandlerResult m o = m (Bool, GenericResult GanetiException o) data Handler i m o = Handler { hParse :: J.JSValue -> J.JSValue -> Result i -- ^ parses method and its arguments into the input type , hInputLogShort :: i -> String -- ^ short description of an input, for the INFO logging level , hInputLogLong :: i -> String -- ^ long description of an input, for the DEBUG logging level , hExec :: i -> HandlerResult m o -- ^ executes the handler on an input } handleJsonMessage :: (J.JSON o, Monad m) => Handler i m o -- ^ handler -> i -- ^ parsed input -> HandlerResult m J.JSValue handleJsonMessage handler req = do (close, call_result) <- hExec handler req return (close, fmap J.showJSON call_result) -- | Takes a request as a 'String', parses it, passes it to a handler and -- formats its response. handleRawMessage :: (J.JSON o, MonadLog m) => Handler i m o -- ^ handler -> String -- ^ raw unparsed input -> m (Bool, String) handleRawMessage handler payload = case parseCall payload >>= uncurry (hParse handler) of Bad err -> do let errmsg = "Failed to parse request: " ++ err logWarning errmsg return (False, buildResponse False (J.showJSON errmsg)) Ok req -> do logDebug $ "Request: " ++ hInputLogLong handler req (close, call_result_json) <- handleJsonMessage handler req logMsg handler req call_result_json let (status, response) = prepareMsg call_result_json return (close, buildResponse status response) isRisky :: RecvResult -> Bool isRisky msg = case msg of RecvOk payload -> any (`isInfixOf` payload) privateParametersBlacklist _ -> False -- | Reads a request, passes it to a handler and sends a response back to the -- client. handleClient :: (J.JSON o, MonadBase IO m, MonadLog m) => Handler i m o -> Client -> m Bool handleClient handler client = do msg <- liftBase $ recvMsgExt client debugMode <- liftBase isDebugMode when (debugMode && isRisky msg) $ logAlert "POSSIBLE LEAKING OF CONFIDENTIAL PARAMETERS. \ \Daemon is running in debug mode. \ \The text of the request has been logged." logDebug $ "Received message (truncated): " ++ take 500 (show msg) case msg of RecvConnClosed -> logDebug "Connection closed" >> return False RecvError err -> logWarning ("Error during message receiving: " ++ err) >> return False RecvOk payload -> do (close, outMsg) <- handleRawMessage handler payload liftBase $ sendMsg client outMsg return close -- | Main client loop: runs one loop of 'handleClient', and if that -- doesn't report a finished (closed) connection, restarts itself. clientLoop :: (J.JSON o, MonadBase IO m, MonadLog m) => Handler i m o -> Client -> m () clientLoop handler client = do result <- handleClient handler client {- It's been observed sometimes that reading immediately after sending a response leads to worse performance, as there is nothing to read and the system calls are just wasted. Thus yielding before reading gives other threads a chance to proceed and provides a natural pause, leading to a bit more efficient communication. -} if result then yield >> clientLoop handler client else liftBase $ closeClient client -- | Main listener loop: accepts clients, forks an I/O thread to handle -- that client. listener :: (J.JSON o, MonadBaseControl IO m, MonadLog m) => Handler i m o -> Server -> m () listener handler server = do client <- liftBase $ acceptClient server _ <- fork $ clientLoop handler client return () ganeti-3.1.0~rc2/src/Ganeti/Utils.hs000064400000000000000000000674411476477700300172520ustar00rootroot00000000000000{-# LANGUAGE FlexibleContexts, ScopedTypeVariables, CPP #-} {-| Utility functions. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils ( debug , debugFn , debugXy , sepSplit , findFirst , stdDev , if' , select , applyIf , commaJoin , ensureQuoted , tryRead , readMaybe , formatTable , printTable , parseUnit , parseUnitAssumeBinary , plural , niceSort , niceSortKey , exitIfBad , exitErr , exitWhen , exitUnless , logWarningIfBad , rStripSpace , newUUID , isUUID , chompPrefix , warn , wrap , trim , defaultHead , exitIfEmpty , splitEithers , recombineEithers , resolveAddr , monadicThe , setOwnerAndGroupFromNames , setOwnerWGroupR , formatOrdinal , tryAndLogIOError , withDefaultOnIOError , lockFile , FStat , nullFStat , getFStat , getFStatSafe , needsReload , watchFile , watchFileBy , safeRenameFile , FilePermissions(..) , ensurePermissions , ordNub , isSubsequenceOf , frequency ) where import Control.Concurrent import Control.Exception (try, bracket) import Control.Monad import Control.Monad.Fail (MonadFail) import qualified Data.Attoparsec.ByteString as A import qualified Data.ByteString.UTF8 as UTF8 import Data.Char (toUpper, isAlphaNum, isDigit, isSpace) import qualified Data.Either as E import Data.Function (on) import Data.IORef import Data.List import qualified Data.List.NonEmpty as NonEmpty import qualified Data.Map as M import Data.Maybe (fromMaybe) import qualified Data.Set as S import Numeric (showOct) import System.Directory (renameFile, createDirectoryIfMissing) import System.FilePath.Posix (takeDirectory) import System.INotify import System.Posix.Types import Debug.Trace import Network.Socket import Ganeti.Utils.Time (getCurrentTimeUSec) import Ganeti.BasicTypes import Ganeti.Compat import qualified Ganeti.ConstantUtils as ConstantUtils import Ganeti.Logging import Ganeti.Runtime import System.IO import System.Exit import System.Posix.Files import System.Posix.IO -- * Debug functions -- | To be used only for debugging, breaks referential integrity. debug :: Show a => a -> a debug x = trace (show x) x -- | Displays a modified form of the second parameter before returning -- it. debugFn :: Show b => (a -> b) -> a -> a debugFn fn x = debug (fn x) `seq` x -- | Show the first parameter before returning the second one. debugXy :: Show a => a -> b -> b debugXy = seq . debug -- * Miscellaneous -- | Apply the function if condition holds, otherwise use default value. applyIf :: Bool -> (a -> a) -> a -> a applyIf b f x = if b then f x else x -- | Comma-join a string list. commaJoin :: [String] -> String commaJoin = intercalate "," -- | Split a list on a separator and return a list of lists. sepSplit :: Eq a => a -> [a] -> [[a]] sepSplit sep s | null s = [] | null xs = [x] | null ys = [x,[]] | otherwise = x:sepSplit sep ys where (x, xs) = break (== sep) s ys = drop 1 xs -- | Finds the first unused element in a set starting from a given base. findFirst :: (Ord a, Enum a) => a -> S.Set a -> a findFirst base xs = case S.splitMember base xs of (_, False, _) -> base (_, True, ys) -> fromMaybe (succ base) $ (fmap fst . find (uncurry (<)) . zip [succ base..] . S.toAscList $ ys) `mplus` fmap (succ . fst) (S.maxView ys) -- | Simple pluralize helper plural :: Int -> String -> String -> String plural 1 s _ = s plural _ _ p = p -- | Ensure a value is quoted if needed. ensureQuoted :: String -> String ensureQuoted v = if not (all (\c -> isAlphaNum c || c == '.') v) then '\'':v ++ "'" else v -- * Mathematical functions -- Simple and slow statistical functions, please replace with better -- versions -- | Standard deviation function. stdDev :: [Double] -> Double stdDev lst = -- first, calculate the list length and sum lst in a single step, -- for performance reasons let (ll', sx) = foldl' (\(rl, rs) e -> let rl' = rl + 1 rs' = rs + e in rl' `seq` rs' `seq` (rl', rs')) (0::Int, 0) lst ll = fromIntegral ll'::Double mv = sx / ll av = foldl' (\accu em -> let d = em - mv in accu + d * d) 0.0 lst in sqrt (av / ll) -- stddev -- * Logical functions -- Avoid syntactic sugar and enhance readability. These functions are proposed -- by some for inclusion in the Prelude, and at the moment they are present -- (with various definitions) in the utility-ht package. Some rationale and -- discussion is available at -- | \"if\" as a function, rather than as syntactic sugar. if' :: Bool -- ^ condition -> a -- ^ \"then\" result -> a -- ^ \"else\" result -> a -- ^ \"then\" or "else" result depending on the condition if' True x _ = x if' _ _ y = y -- * Parsing utility functions -- | Parse results from readsPrec. parseChoices :: MonadFail m => String -> String -> [(a, String)] -> m a parseChoices _ _ [(v, "")] = return v parseChoices name s [(_, e)] = fail $ name ++ ": leftover characters when parsing '" ++ s ++ "': '" ++ e ++ "'" parseChoices name s _ = fail $ name ++ ": cannot parse string '" ++ s ++ "'" -- | Safe 'read' function returning data encapsulated in a Result. tryRead :: (MonadFail m, Read a) => String -> String -> m a tryRead name s = parseChoices name s $ reads s -- | Parse a string using the 'Read' instance. -- Succeeds if there is exactly one valid result. -- -- /Backport from Text.Read introduced in base-4.6.0.0/ readMaybe :: Read a => String -> Maybe a readMaybe s = case reads s of [(a, "")] -> Just a _ -> Nothing -- | Format a table of strings to maintain consistent length. formatTable :: [[String]] -> [Bool] -> [[String]] formatTable vals numpos = let vtrans = transpose vals -- transpose, so that we work on rows -- rather than columns mlens = map (maximum . map length) vtrans expnd = map (\(flds, isnum, ml) -> map (\val -> let delta = ml - length val filler = replicate delta ' ' in if delta > 0 then if isnum then filler ++ val else val ++ filler else val ) flds ) (zip3 vtrans numpos mlens) in transpose expnd -- | Constructs a printable table from given header and rows printTable :: String -> [String] -> [[String]] -> [Bool] -> String printTable lp header rows isnum = unlines . map ((++) lp . (:) ' ' . unwords) $ formatTable (header:rows) isnum -- | Converts a unit (e.g. m or GB) into a scaling factor. parseUnitValue :: (MonadFail m) => Bool -> String -> m Rational parseUnitValue noDecimal unit -- binary conversions first | null unit = return 1 | unit == "m" || upper == "MIB" = return 1 | unit == "g" || upper == "GIB" = return kbBinary | unit == "t" || upper == "TIB" = return $ kbBinary * kbBinary -- SI conversions | unit == "M" || upper == "MB" = return mbFactor | unit == "G" || upper == "GB" = return $ mbFactor * kbDecimal | unit == "T" || upper == "TB" = return $ mbFactor * kbDecimal * kbDecimal | otherwise = fail $ "Unknown unit '" ++ unit ++ "'" where upper = map toUpper unit kbBinary = 1024 :: Rational kbDecimal = if noDecimal then kbBinary else 1000 decToBin = kbDecimal / kbBinary -- factor for 1K conversion mbFactor = decToBin * decToBin -- twice the factor for just 1K -- | Tries to extract number and scale from the given string. -- -- Input must be in the format NUMBER+ SPACE* [UNIT]. If no unit is -- specified, it defaults to MiB. Return value is always an integral -- value in MiB; if the first argument is True, all kilos are binary. parseUnitEx :: (MonadFail m, Integral a, Read a) => Bool -> String -> m a parseUnitEx noDecimal str = -- TODO: enhance this by splitting the unit parsing code out and -- accepting floating-point numbers case (reads str::[(Int, String)]) of [(v, suffix)] -> let unit = dropWhile (== ' ') suffix in do scaling <- parseUnitValue noDecimal unit return $ truncate (fromIntegral v * scaling) _ -> fail $ "Can't parse string '" ++ str ++ "'" -- | Tries to extract number and scale from the given string. -- -- Input must be in the format NUMBER+ SPACE* [UNIT]. If no unit is -- specified, it defaults to MiB. Return value is always an integral -- value in MiB. parseUnit :: (MonadFail m, Integral a, Read a) => String -> m a parseUnit = parseUnitEx False -- | Tries to extract a number and scale from a given string, taking -- all kilos to be binary. parseUnitAssumeBinary :: (MonadFail m, Integral a, Read a) => String -> m a parseUnitAssumeBinary = parseUnitEx True -- | Unwraps a 'Result', exiting the program if it is a 'Bad' value, -- otherwise returning the actual contained value. exitIfBad :: String -> Result a -> IO a exitIfBad msg (Bad s) = exitErr (msg ++ ": " ++ s) exitIfBad _ (Ok v) = return v -- | Exits immediately with an error message. exitErr :: String -> IO a exitErr errmsg = do hPutStrLn stderr $ "Error: " ++ errmsg exitWith (ExitFailure 1) -- | Exits with an error message if the given boolean condition if true. exitWhen :: Bool -> String -> IO () exitWhen True msg = exitErr msg exitWhen False _ = return () -- | Exits with an error message /unless/ the given boolean condition -- if true, the opposite of 'exitWhen'. exitUnless :: Bool -> String -> IO () exitUnless cond = exitWhen (not cond) -- | Unwraps a 'Result', logging a warning message and then returning a default -- value if it is a 'Bad' value, otherwise returning the actual contained value. logWarningIfBad :: String -> a -> Result a -> IO a logWarningIfBad msg defVal (Bad s) = do logWarning $ msg ++ ": " ++ s return defVal logWarningIfBad _ _ (Ok v) = return v -- | Try an IO interaction, log errors and unfold as a 'Result'. tryAndLogIOError :: IO a -> String -> (a -> Result b) -> IO (Result b) tryAndLogIOError io msg okfn = try io >>= either (\ e -> do let combinedmsg = msg ++ ": " ++ show (e :: IOError) logError combinedmsg return . Bad $ combinedmsg) (return . okfn) -- | Try an IO interaction and return a default value if the interaction -- throws an IOError. withDefaultOnIOError :: a -> IO a -> IO a withDefaultOnIOError a io = try io >>= either (\ (_ :: IOError) -> return a) return -- | Print a warning, but do not exit. warn :: String -> IO () warn = hPutStrLn stderr . (++) "Warning: " -- | Helper for 'niceSort'. Computes the key element for a given string. extractKey :: [Either Integer String] -- ^ Current (partial) key, reversed -> String -- ^ Remaining string -> ([Either Integer String], String) extractKey ek [] = (reverse ek, []) extractKey ek xs@(x:_) = let (span_fn, conv_fn) = if isDigit x then (isDigit, Left . read) else (not . isDigit, Right) (k, rest) = span span_fn xs in extractKey (conv_fn k:ek) rest {-| Sort a list of strings based on digit and non-digit groupings. Given a list of names @['a1', 'a10', 'a11', 'a2']@ this function will sort the list in the logical order @['a1', 'a2', 'a10', 'a11']@. The sort algorithm breaks each name in groups of either only-digits or no-digits, and sorts based on each group. Internally, this is not implemented via regexes (like the Python version), but via actual splitting of the string in sequences of either digits or everything else, and converting the digit sequences in /Left Integer/ and the non-digit ones in /Right String/, at which point sorting becomes trivial due to the built-in 'Either' ordering; we only need one extra step of dropping the key at the end. -} niceSort :: [String] -> [String] niceSort = niceSortKey id -- | Key-version of 'niceSort'. We use 'sortBy' and @compare `on` fst@ -- since we don't want to add an ordering constraint on the /a/ type, -- hence the need to only compare the first element of the /(key, a)/ -- tuple. niceSortKey :: (a -> String) -> [a] -> [a] niceSortKey keyfn = map snd . sortBy (compare `on` fst) . map (\s -> (fst . extractKey [] $ keyfn s, s)) -- | Strip space characthers (including newline). As this is -- expensive, should only be run on small strings. rStripSpace :: String -> String rStripSpace = reverse . dropWhile isSpace . reverse -- | Returns a random UUID. -- This is a Linux-specific method as it uses the /proc filesystem. newUUID :: IO String newUUID = do contents <- readFile ConstantUtils.randomUuidFile return $! rStripSpace $ take 128 contents -- | Parser that doesn't fail on a valid UUIDs (same as -- "Ganeti.Constants.uuidRegex"). uuidCheckParser :: A.Parser () uuidCheckParser = do -- Not using Attoparsec.Char8 because "all attempts to use characters -- above code point U+00FF will give wrong answers" and we don't -- want such things to be accepted as UUIDs. let lowerHex = A.satisfy (\c -> (48 <= c && c <= 57) || -- 0-9 (97 <= c && c <= 102)) -- a-f hx n = A.count n lowerHex d = A.word8 45 -- '-' void $ hx 8 >> d >> hx 4 >> d >> hx 4 >> d >> hx 4 >> d >> hx 12 -- | Checks if the string is a valid UUID as in "Ganeti.Constants.uuidRegex". isUUID :: String -> Bool isUUID = isRight . A.parseOnly (uuidCheckParser <* A.endOfInput) . UTF8.fromString {-| Strip a prefix from a string, allowing the last character of the prefix (which is assumed to be a separator) to be absent from the string if the string terminates there. \>>> chompPrefix \"foo:bar:\" \"a:b:c\" Nothing \>>> chompPrefix \"foo:bar:\" \"foo:bar:baz\" Just \"baz\" \>>> chompPrefix \"foo:bar:\" \"foo:bar:\" Just \"\" \>>> chompPrefix \"foo:bar:\" \"foo:bar\" Just \"\" \>>> chompPrefix \"foo:bar:\" \"foo:barbaz\" Nothing -} chompPrefix :: String -> String -> Maybe String chompPrefix pfx str = if pfx `isPrefixOf` str || str == init pfx then Just $ drop (length pfx) str else Nothing -- | Breaks a string in lines with length \<= maxWidth. -- -- NOTE: The split is OK if: -- -- * It doesn't break a word, i.e. the next line begins with space -- (@isSpace . head $ rest@) or the current line ends with space -- (@null revExtra@); -- -- * It breaks a very big word that doesn't fit anyway (@null revLine@). wrap :: Int -- ^ maxWidth -> String -- ^ string that needs wrapping -> [String] -- ^ string \"broken\" in lines wrap maxWidth = filter (not . null) . map trim . wrap0 where wrap0 :: String -> [String] wrap0 text | length text <= maxWidth = [text] | isSplitOK = line : wrap0 rest | otherwise = line' : wrap0 rest' where (line, rest) = splitAt maxWidth text (revExtra, revLine) = break isSpace . reverse $ line (line', rest') = (reverse revLine, reverse revExtra ++ rest) isSplitOK = null revLine || null revExtra || startsWithSpace rest startsWithSpace (x:_) = isSpace x startsWithSpace _ = False -- | Removes surrounding whitespace. Should only be used in small -- strings. trim :: String -> String trim = reverse . dropWhile isSpace . reverse . dropWhile isSpace -- | A safer head version, with a default value. defaultHead :: a -> [a] -> a defaultHead def [] = def defaultHead _ (x:_) = x -- | A 'head' version in the I/O monad, for validating parameters -- without which we cannot continue. exitIfEmpty :: String -> [a] -> IO a exitIfEmpty _ (x:_) = return x exitIfEmpty s [] = exitErr s -- | Obtain the unique element of a list in an arbitrary monad. monadicThe :: (Eq a, MonadFail m) => String -> [a] -> m a monadicThe s [] = fail s monadicThe s (x:xs) | all (x ==) xs = return x | otherwise = fail s -- | Split an 'Either' list into two separate lists (containing the -- 'Left' and 'Right' elements, plus a \"trail\" list that allows -- recombination later. -- -- This is splitter; for recombination, look at 'recombineEithers'. -- The sum of \"left\" and \"right\" lists should be equal to the -- original list length, and the trail list should be the same length -- as well. The entries in the resulting lists are reversed in -- comparison with the original list. splitEithers :: [Either a b] -> ([a], [b], [Bool]) splitEithers = foldl' splitter ([], [], []) where splitter (l, r, t) e = case e of Left v -> (v:l, r, False:t) Right v -> (l, v:r, True:t) -- | Recombines two \"left\" and \"right\" lists using a \"trail\" -- list into a single 'Either' list. -- -- This is the counterpart to 'splitEithers'. It does the opposite -- transformation, and the output list will be the reverse of the -- input lists. Since 'splitEithers' also reverses the lists, calling -- these together will result in the original list. -- -- Mismatches in the structure of the lists (e.g. inconsistent -- lengths) are represented via 'Bad'; normally this function should -- not fail, if lists are passed as generated by 'splitEithers'. recombineEithers :: (Show a, Show b) => [a] -> [b] -> [Bool] -> Result [Either a b] recombineEithers lefts rights trail = foldM recombiner ([], lefts, rights) trail >>= checker where checker (eithers, [], []) = Ok eithers checker (_, lefts', rights') = Bad $ "Inconsistent results after recombination, l'=" ++ show lefts' ++ ", r'=" ++ show rights' recombiner (es, l:ls, rs) False = Ok (Left l:es, ls, rs) recombiner (es, ls, r:rs) True = Ok (Right r:es, ls, rs) recombiner (_, ls, rs) t = Bad $ "Inconsistent trail log: l=" ++ show ls ++ ", r=" ++ show rs ++ ",t=" ++ show t -- | Default hints for the resolver resolveAddrHints :: Maybe AddrInfo resolveAddrHints = Just defaultHints { addrFlags = [AI_NUMERICHOST, AI_NUMERICSERV] } -- | Resolves a numeric address. resolveAddr :: Int -> String -> IO (Result (Family, SockAddr)) resolveAddr port str = do resolved <- getAddrInfo resolveAddrHints (Just str) (Just (show port)) return $ case resolved of [] -> Bad "Invalid results from lookup?" best:_ -> Ok (addrFamily best, addrAddress best) -- | Set the owner and the group of a file (given as names, not numeric id). setOwnerAndGroupFromNames :: FilePath -> GanetiDaemon -> GanetiGroup -> IO () setOwnerAndGroupFromNames filename daemon dGroup = do -- TODO: it would be nice to rework this (or getEnts) so that runtimeEnts -- is read only once per daemon startup, and then cached for further usage. runtimeEnts <- runResultT getEnts ents <- exitIfBad "Can't find required user/groups" runtimeEnts -- note: we use directly ! as lookup failures shouldn't happen, due -- to the map construction let uid = reUserToUid ents M.! daemon let gid = reGroupToGid ents M.! dGroup setOwnerAndGroup filename uid gid -- | Resets permissions so that the owner can read/write and the group only -- read. All other permissions are cleared. setOwnerWGroupR :: FilePath -> IO () setOwnerWGroupR path = setFileMode path mode where mode = foldl unionFileModes nullFileMode [ownerReadMode, ownerWriteMode, groupReadMode] -- | Formats an integral number, appending a suffix. formatOrdinal :: (Integral a, Show a) => a -> String formatOrdinal num | num > 10 && num < 20 = suffix "th" | tens == 1 = suffix "st" | tens == 2 = suffix "nd" | tens == 3 = suffix "rd" | otherwise = suffix "th" where tens = num `mod` 10 suffix s = show num ++ s -- | Attempt, in a non-blocking way, to obtain a lock on a given file; report -- back success. -- Returns the file descriptor so that the lock can be released by closing lockFile :: FilePath -> IO (Result Fd) lockFile path = runResultT . liftIO $ do handle <- openFile path WriteMode fd <- handleToFd handle setLock fd (WriteLock, AbsoluteSeek, 0, 0) return fd -- | File stat identifier. type FStat = (EpochTime, FileID, FileOffset) -- | Null 'FStat' value. nullFStat :: FStat nullFStat = (-1, -1, -1) -- | Computes the file cache data from a FileStatus structure. buildFileStatus :: FileStatus -> FStat buildFileStatus ofs = let modt = modificationTime ofs inum = fileID ofs fsize = fileSize ofs in (modt, inum, fsize) -- | Wrapper over 'buildFileStatus'. This reads the data from the -- filesystem and then builds our cache structure. getFStat :: FilePath -> IO FStat getFStat p = liftM buildFileStatus (getFileStatus p) -- | Safe version of 'getFStat', that ignores IOErrors. getFStatSafe :: FilePath -> IO FStat getFStatSafe fpath = liftM (either (const nullFStat) id) ((try $ getFStat fpath) :: IO (Either IOError FStat)) -- | Check if the file needs reloading needsReload :: FStat -> FilePath -> IO (Maybe FStat) needsReload oldstat path = do newstat <- getFStat path return $ if newstat /= oldstat then Just newstat else Nothing -- | Until the given point in time (useconds since the epoch), wait -- for the output of a given method to change and return the new value; -- make use of the promise that the output only changes if the reference -- has a value different than the given one. watchFileEx :: (Eq b) => Integer -> b -> IORef b -> (a -> Bool) -> IO a -> IO a watchFileEx endtime base ref check read_fn = do current <- getCurrentTimeUSec if current > endtime then read_fn else do val <- readIORef ref if val /= base then do new <- read_fn if check new then return new else do logDebug "Observed change not relevant" threadDelay 100000 watchFileEx endtime val ref check read_fn else do threadDelay 100000 watchFileEx endtime base ref check read_fn -- | Within the given timeout (in seconds), wait for for the output -- of the given method to satisfy a given predicate and return the new value; -- make use of the promise that the method will only change its value, if -- the given file changes on disk. If the file does not exist on disk, return -- immediately. watchFileBy :: FilePath -> Int -> (a -> Bool) -> IO a -> IO a watchFileBy fpath timeout check read_fn = do current <- getCurrentTimeUSec let endtime = current + fromIntegral timeout * 1000000 fstat <- getFStatSafe fpath ref <- newIORef fstat bracket initINotify killINotify $ \inotify -> do let do_watch e = do logDebug $ "Notified of change in " ++ fpath ++ "; event: " ++ show e when (e == Ignored) (addWatch inotify [Modify, Delete] (toInotifyPath fpath) do_watch >> return ()) fstat' <- getFStatSafe fpath writeIORef ref fstat' _ <- addWatch inotify [Modify, Delete] (toInotifyPath fpath) do_watch newval <- read_fn if check newval then do logDebug $ "File " ++ fpath ++ " changed during setup of inotify" return newval else watchFileEx endtime fstat ref check read_fn -- | Within the given timeout (in seconds), wait for for the output -- of the given method to change and return the new value; make use of -- the promise that the method will only change its value, if -- the given file changes on disk. If the file does not exist on disk, return -- immediately. watchFile :: Eq a => FilePath -> Int -> a -> IO a -> IO a watchFile fpath timeout old = watchFileBy fpath timeout (/= old) -- | Type describing ownership and permissions of newly generated -- directories and files. All parameters are optional, with nothing -- meaning that the default value should be left untouched. data FilePermissions = FilePermissions { fpOwner :: Maybe GanetiDaemon , fpGroup :: Maybe GanetiGroup , fpPermissions :: FileMode } -- | Ensure that a given file or directory has the permissions, and -- possibly ownerships, as required. ensurePermissions :: FilePath -> FilePermissions -> IO (Result ()) ensurePermissions fpath perms = do -- Fetch the list of entities runtimeEnts <- runResultT getEnts ents <- exitIfBad "Can't determine user/group ids" runtimeEnts -- Get the existing file properties eitherFileStatus <- try $ getFileStatus fpath :: IO (Either IOError FileStatus) -- And see if any modifications are needed (flip $ either (return . Bad . show)) eitherFileStatus $ \fstat -> do ownertry <- case fpOwner perms of Nothing -> return $ Right () Just owner -> try $ do let ownerid = reUserToUid ents M.! owner unless (ownerid == fileOwner fstat) $ do logDebug $ "Changing owner of " ++ fpath ++ " to " ++ show owner setOwnerAndGroup fpath ownerid (-1) grouptry <- case fpGroup perms of Nothing -> return $ Right () Just grp -> try $ do let groupid = reGroupToGid ents M.! grp unless (groupid == fileGroup fstat) $ do logDebug $ "Changing group of " ++ fpath ++ " to " ++ show grp setOwnerAndGroup fpath (-1) groupid let fp = fpPermissions perms permtry <- if fileMode fstat == fp then return $ Right () else try $ do logInfo $ "Changing permissions of " ++ fpath ++ " to " ++ showOct fp "" setFileMode fpath fp let errors = E.lefts ([ownertry, grouptry, permtry] :: [Either IOError ()]) if null errors then return $ Ok () else return . Bad $ show errors -- | Safely rename a file, creating the target directory, if needed. safeRenameFile :: FilePermissions -> FilePath -> FilePath -> IO (Result ()) safeRenameFile perms from to = do directtry <- try $ renameFile from to case (directtry :: Either IOError ()) of Right () -> return $ Ok () Left _ -> do result <- try $ do let dir = takeDirectory to createDirectoryIfMissing True dir _ <- ensurePermissions dir perms renameFile from to return $ either (Bad . show) Ok (result :: Either IOError ()) -- | Removes duplicates, preserving order. ordNub :: (Ord a) => [a] -> [a] ordNub = let go _ [] = [] go s (x:xs) = if x `S.member` s then go s xs else x : go (S.insert x s) xs in go S.empty -- | Returns a list of tuples of elements and the number of times they occur -- in a list frequency :: Ord t => [t] -> [(Int, t)] frequency = map (\x -> (length x, NonEmpty.head x)) . NonEmpty.group . sort ganeti-3.1.0~rc2/src/Ganeti/Utils/000075500000000000000000000000001476477700300167025ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/Utils/AsyncWorker.hs000064400000000000000000000221371476477700300215120ustar00rootroot00000000000000{-# LANGUAGE FlexibleContexts #-} {-| Provides a general functionality for workers that run on the background and perform some task when triggered. Each task can process multiple triggers, if they're coming faster than the tasks are being processed. Properties: - If a worked is triggered, it will perform its action eventually. (i.e. it won't miss a trigger). - If the worker is busy, the new action will start immediately when it finishes the current one. - If the worker is idle, it'll start the action immediately. - If the caller uses 'triggerAndWait', the call will return just after the earliest action following the trigger is finished. - If the caller uses 'triggerWithResult', it will recive an 'Async' value that can be used to wait for the result (which will be available once the earliest action following the trigger finishes). - If the worker finishes an action and there are no pending triggers since the start of the last action, it becomes idle and waits for a new trigger. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.AsyncWorker ( AsyncWorker , mkAsyncWorker , mkAsyncWorker_ , trigger , trigger_ , triggerWithResult , triggerWithResult_ , triggerWithResultMany , triggerWithResultMany_ , triggerAndWait , triggerAndWait_ , triggerAndWaitMany , triggerAndWaitMany_ , Async , wait , waitMany ) where import Control.Monad import Control.Monad.Base import Control.Monad.Trans.Control import Control.Concurrent (ThreadId) import Control.Concurrent.Lifted (fork, yield) import Control.Concurrent.MVar.Lifted import Data.Monoid import qualified Data.Traversable as T import Data.IORef.Lifted -- * The definition and construction of asynchronous workers -- Represents the state of the requests to the worker. The worker is either -- 'Idle', or has 'Pending' triggers to process. After the corresponding -- action is run, all the 'MVar's in the list are notified with the result. -- Note that the action needs to be run even if the list is empty, as it -- means that there are pending requests, only nobody needs to be notified of -- their results. data TriggerState i a = Idle | Pending i [MVar a] -- | Adds a new trigger to the current state (therefore the result is always -- 'Pending'), optionally adding a 'MVar' that will receive the output. addTrigger :: (Monoid i) => i -> Maybe (MVar a) -> TriggerState i a -> TriggerState i a addTrigger i mmvar state = let rs = recipients state in Pending (input state <> i) (maybe rs (: rs) mmvar) where recipients Idle = [] recipients (Pending _ rs) = rs input Idle = mempty input (Pending j _) = j -- | Represent an asynchronous worker whose single action execution returns a -- value of type @a@. data AsyncWorker i a = AsyncWorker ThreadId (IORef (TriggerState i a)) (MVar ()) -- | Given an action, construct an 'AsyncWorker'. mkAsyncWorker :: (Monoid i, MonadBaseControl IO m) => (i -> m a) -> m (AsyncWorker i a) mkAsyncWorker act = do trig <- newMVar () ref <- newIORef Idle thId <- fork . forever $ do takeMVar trig -- wait for a trigger state <- swap ref Idle -- check the state of pending requests -- if there are pending requests, run the action and send them results case state of Idle -> return () -- all trigers have been processed, we've -- been woken up by a trigger that has been -- already included in the last run Pending i rs -> act i >>= forM_ rs . flip tryPutMVar -- Give other threads a chance to do work while we're waiting for -- something to happen. yield return $ AsyncWorker thId ref trig where swap :: (MonadBase IO m) => IORef a -> a -> m a swap ref x = atomicModifyIORef ref ((,) x) -- | Given an action, construct an 'AsyncWorker' with no input. mkAsyncWorker_ :: (MonadBaseControl IO m) => m a -> m (AsyncWorker () a) mkAsyncWorker_ = mkAsyncWorker . const -- * Triggering workers and obtaining their results -- | An asynchronous result that will eventually yield a value. newtype Async a = Async { asyncResult :: MVar a } -- | Waits for an asynchronous result to finish and yield a value. wait :: (MonadBase IO m) => Async a -> m a wait = readMVar . asyncResult -- | Waits for all asynchronous results in a collection to finish and yield a -- value. waitMany :: (MonadBase IO m, T.Traversable t) => t (Async a) -> m (t a) waitMany = T.mapM wait -- An internal function for triggering a worker, optionally registering -- a callback 'MVar' triggerInternal :: (MonadBase IO m, Monoid i) => i -> Maybe (MVar a) -> AsyncWorker i a -> m () triggerInternal i mmvar (AsyncWorker _ ref trig) = do atomicModifyIORef ref (\ts -> (addTrigger i mmvar ts, ())) _ <- tryPutMVar trig () return () -- | Trigger a worker, letting it run its action asynchronously, but do not -- wait for the result. trigger :: (MonadBase IO m, Monoid i) => i -> AsyncWorker i a -> m () trigger = flip triggerInternal Nothing -- | Trigger a worker with no input, letting it run its action asynchronously, -- but do not wait for the result. trigger_ :: (MonadBase IO m) => AsyncWorker () a -> m () trigger_ = trigger () -- | Trigger a worker and wait until the action following this trigger -- finishes. The returned `Async` value can be used to wait for the result of -- the action. triggerWithResult :: (MonadBase IO m, Monoid i) => i -> AsyncWorker i a -> m (Async a) triggerWithResult i worker = do result <- newEmptyMVar triggerInternal i (Just result) worker return $ Async result -- | Trigger a worker and wait until the action following this trigger -- finishes. -- -- See 'triggerWithResult'. triggerWithResult_ :: (MonadBase IO m) => AsyncWorker () a -> m (Async a) triggerWithResult_ = triggerWithResult () -- | Trigger a list of workers and wait until all the actions following these -- triggers finish. The returned collection of `Async` values can be used to -- wait for the results of the actions. triggerWithResultMany :: (T.Traversable t, MonadBase IO m, Monoid i) => i -> t (AsyncWorker i a) -> m (t (Async a)) triggerWithResultMany i = T.mapM (triggerWithResult i) -- -- | Trigger a list of workers with no inputs and wait until all the actions -- following these triggers finish. -- -- See 'triggerWithResultMany'. triggerWithResultMany_ :: (T.Traversable t, MonadBase IO m) => t (AsyncWorker () a) -> m (t (Async a)) triggerWithResultMany_ = triggerWithResultMany () -- * Helper functions for waiting for results just after triggering workers -- | Trigger a list of workers and wait until all the actions following these -- triggers finish. Returns the results of the actions. -- -- Note that there is a significant difference between 'triggerAndWaitMany' -- and @mapM triggerAndWait@. The latter runs all the actions of the workers -- sequentially, while the former runs them in parallel. triggerAndWaitMany :: (T.Traversable t, MonadBase IO m, Monoid i) => i -> t (AsyncWorker i a) -> m (t a) triggerAndWaitMany i = waitMany <=< triggerWithResultMany i -- See 'triggetAndWaitMany'. triggerAndWaitMany_ :: (T.Traversable t, MonadBase IO m) => t (AsyncWorker () a) -> m (t a) triggerAndWaitMany_ = triggerAndWaitMany () -- | Trigger a worker and wait until the action following this trigger -- finishes. Return the result of the action. triggerAndWait :: (MonadBase IO m, Monoid i) => i -> AsyncWorker i a -> m a triggerAndWait i = wait <=< triggerWithResult i -- | Trigger a worker with no input and wait until the action following this -- trigger finishes. Return the result of the action. triggerAndWait_ :: (MonadBase IO m) => AsyncWorker () a -> m a triggerAndWait_ = triggerAndWait () ganeti-3.1.0~rc2/src/Ganeti/Utils/Atomic.hs000064400000000000000000000140071476477700300204540ustar00rootroot00000000000000{-# LANGUAGE FlexibleContexts #-} {-| Utility functions for atomic file access. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.Atomic ( atomicWriteFile , atomicUpdateFile , withLockedFile , atomicUpdateLockedFile , atomicUpdateLockedFile_ ) where import qualified Control.Exception.Lifted as L import Control.Monad import Control.Monad.Base (MonadBase(..)) import Control.Monad.Except import Control.Monad.Trans.Control import System.FilePath.Posix (takeDirectory, takeBaseName) import System.IO import System.Directory (renameFile) import qualified System.Posix.IO as Posix import System.Posix.Types import Ganeti.BasicTypes import Ganeti.Errors import Ganeti.Logging (logAlert) import Ganeti.Compat (openFd, closeFd) import Ganeti.Utils import Ganeti.Utils.UniStd (fsyncFile) -- | Atomically write a file, by first writing the contents into a temporary -- file and then renaming it to the old position. atomicWriteFile :: FilePath -> String -> IO () atomicWriteFile path contents = atomicUpdateFile path (\_ fh -> hPutStr fh contents) -- | Calls fsync(2) on a given file. -- If the operation fails, issue an alert log message and continue. -- Doesn't throw an exception. fsyncFileChecked :: FilePath -> IO () fsyncFileChecked path = runResultT (fsyncFile path) >>= genericResult logMsg return where logMsg e = logAlert $ "Can't fsync file '" ++ path ++ "': " ++ e -- | Atomically update a file, by first creating a temporary file, running the -- given action on it, and then renaming it to the old position. -- Usually the action will write to the file and update its permissions. -- The action is allowed to close the file descriptor, but isn't required to do -- so. atomicUpdateFile :: (MonadBaseControl IO m) => FilePath -> (FilePath -> Handle -> m a) -> m a atomicUpdateFile path action = do -- Put a separator on the filename pattern to produce temporary filenames -- such as job-1234-NNNNNN.tmp instead of job-1234NNNNNN. The latter can cause -- problems (as well as user confusion) because temporary filenames have the -- same format as real filenames, and anything that scans a directory won't be -- able to tell them apart. let filenameTemplate = takeBaseName path ++ "-.tmp" (tmppath, tmphandle) <- liftBase $ openBinaryTempFile (takeDirectory path) filenameTemplate r <- L.finally (action tmppath tmphandle) (liftBase (hClose tmphandle >> fsyncFileChecked tmppath)) -- if all went well, rename the file liftBase $ renameFile tmppath path return r -- | Opens a file in a R/W mode, locks it (blocking if needed) and runs -- a given action while the file is locked. Releases the lock and -- closes the file afterwards. withLockedFile :: (MonadError e m, Error e, MonadBaseControl IO m) => FilePath -> (Fd -> m a) -> m a withLockedFile path = L.bracket (openAndLock path) (liftBase . closeFd) where openAndLock :: (MonadError e m, Error e, MonadBaseControl IO m) => FilePath -> m Fd openAndLock p = liftBase $ do fd <- openFd p Posix.ReadWrite Nothing Posix.defaultFileFlags Posix.waitToSetLock fd (Posix.WriteLock, AbsoluteSeek, 0, 0) return fd -- | Just as 'atomicUpdateFile', but in addition locks the file during the -- operation using 'withLockedFile' and checks if the file has been modified. -- The action is only run if it hasn't, otherwise an error is thrown. -- The file must exist. -- Returns the new file status after the operation is finished. atomicUpdateLockedFile :: FilePath -> FStat -> (FilePath -> Handle -> IO a) -> ResultG (FStat, a) atomicUpdateLockedFile path fstat action = toErrorBase . withErrorT (LockError . (show :: IOError -> String)) $ withLockedFile path checkStatAndRun where checkStatAndRun _ = do newstat <- liftIO $ getFStat path unless (fstat == newstat) (failError $ "Cannot overwrite file " ++ path ++ ": it has been modified since last written" ++ " (" ++ show fstat ++ " != " ++ show newstat ++ ")") liftIO $ atomicUpdateFile path actionAndStat actionAndStat tmppath tmphandle = do r <- action tmppath tmphandle hClose tmphandle -- close the handle so that we get meaningful stats finalstat <- liftIO $ getFStat tmppath return (finalstat, r) -- | Just as 'atomicUpdateLockedFile', but discards the action result. atomicUpdateLockedFile_ :: FilePath -> FStat -> (FilePath -> Handle -> IO a) -> ResultG FStat atomicUpdateLockedFile_ path oldstat = liftM fst . atomicUpdateLockedFile path oldstat ganeti-3.1.0~rc2/src/Ganeti/Utils/IORef.hs000064400000000000000000000057251476477700300202130ustar00rootroot00000000000000{-# LANGUAGE FlexibleContexts, Rank2Types #-} {-| Utility functions for working with IORefs. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.IORef ( atomicModifyWithLens , atomicModifyIORefErr , atomicModifyIORefErrLog ) where import Control.Monad import Control.Monad.Base import Data.IORef.Lifted import Data.Tuple (swap) import Ganeti.BasicTypes import Ganeti.Lens import Ganeti.Logging import Ganeti.Logging.WriterLog -- | Atomically modifies an 'IORef' using a lens atomicModifyWithLens :: (MonadBase IO m) => IORef a -> Lens a a b c -> (b -> (r, c)) -> m r atomicModifyWithLens ref l f = atomicModifyIORef ref (swap . traverseOf l f) -- | Atomically modifies an 'IORef' using a function that can possibly fail. -- If it fails, the value of the 'IORef' is preserved. atomicModifyIORefErr :: (MonadBase IO m) => IORef a -> (a -> GenericResult e (a, b)) -> ResultT e m b atomicModifyIORefErr ref f = let f' x = genericResult ((,) x . Bad) (fmap Ok) (f x) in ResultT $ atomicModifyIORef ref f' -- | Atomically modifies an 'IORef' using a function that can possibly fail -- and log errors. -- If it fails, the value of the 'IORef' is preserved. -- Any log messages are passed to the outer monad. atomicModifyIORefErrLog :: (MonadBase IO m, MonadLog m) => IORef a -> (a -> ResultT e WriterLog (a, b)) -> ResultT e m b atomicModifyIORefErrLog ref f = ResultT $ do let f' x = let ((a, b), w) = runWriterLog . liftM (genericResult ((,) x . Bad) (fmap Ok)) . runResultT $ f x in (a, (b, w)) (b, w) <- atomicModifyIORef ref f' dumpLogSeq w return b ganeti-3.1.0~rc2/src/Ganeti/Utils/Livelock.hs000064400000000000000000000072461476477700300210170ustar00rootroot00000000000000{-| Utilities related to livelocks and death detection -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.Livelock ( Livelock , mkLivelockFile , listLiveLocks , isDead ) where import qualified Control.Exception as E import Control.Monad import Control.Monad.Except import System.Directory (doesFileExist, getDirectoryContents) import System.FilePath.Posix (()) import System.IO import qualified System.Posix.IO as Posix import System.Posix.Types (Fd) import System.Time (ClockTime(..), getClockTime) import Ganeti.BasicTypes import Ganeti.Logging import Ganeti.Compat (openFd, closeFd) import Ganeti.Path (livelockFile, livelockDir) import Ganeti.Utils (lockFile) type Livelock = FilePath -- | Appends the current time to the given prefix, creates -- the lockfile in the appropriate directory, and locks it. -- Returns its full path and the file's file descriptor. mkLivelockFile :: (Error e, MonadError e m, MonadIO m) => FilePath -> m (Fd, Livelock) mkLivelockFile prefix = do (TOD secs _) <- liftIO getClockTime lockfile <- liftIO . livelockFile $ prefix ++ "_" ++ show secs fd <- liftIO (lockFile lockfile) >>= \r -> case r of Bad msg -> failError $ "Locking the livelock file " ++ lockfile ++ ": " ++ msg Ok fd -> return fd return (fd, lockfile) -- | List currently existing livelocks. Underapproximate if -- some error occurs. listLiveLocks :: IO [FilePath] listLiveLocks = fmap (genericResult (const [] :: IOError -> [FilePath]) id) . runResultT . liftIO $ do dir <- livelockDir entries <- getDirectoryContents dir filterM doesFileExist $ map (dir ) entries -- | Detect whether a the process identified by the given path -- does not exist any more. This function never fails and only -- returns True if it has positive knowledge that the process -- does not exist any more (i.e., if it managed successfully -- obtain a shared lock on the file). isDead :: Livelock -> IO Bool isDead fpath = fmap (isOk :: Result () -> Bool) . runResultT . liftIO $ do filepresent <- doesFileExist fpath when filepresent . E.bracket (openFd fpath Posix.ReadOnly Nothing Posix.defaultFileFlags) closeFd $ \fd -> do logDebug $ "Attempting to get a lock of " ++ fpath Posix.setLock fd (Posix.ReadLock, AbsoluteSeek, 0, 0) logDebug "Got the lock, the process is dead" ganeti-3.1.0~rc2/src/Ganeti/Utils/MVarLock.hs000064400000000000000000000035431476477700300207210ustar00rootroot00000000000000{-# LANGUAGE FlexibleContexts #-} {-| Utility functions for using MVars as simple locks. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.MVarLock ( Lock() , newLock , withLock ) where import Control.Exception.Lifted import Control.Concurrent.MVar.Lifted import Control.Monad import Control.Monad.Base (MonadBase(..)) import Control.Monad.Trans.Control (MonadBaseControl(..)) newtype Lock = MVarLock (MVar ()) newLock :: (MonadBase IO m) => m Lock newLock = MVarLock `liftM` newMVar () withLock :: (MonadBaseControl IO m) => Lock -> m a -> m a withLock (MVarLock l) = bracket_ (takeMVar l) (putMVar l ()) ganeti-3.1.0~rc2/src/Ganeti/Utils/Monad.hs000064400000000000000000000066361476477700300203070ustar00rootroot00000000000000{-| Utility functions for MonadPlus operations -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.Monad ( mretryN , retryMaybeN , anyM , allM , orM , unfoldrM , unfoldrM' , retryErrorN ) where import Control.Monad import Control.Monad.Except import Control.Monad.Trans.Maybe -- | Retries the given action up to @n@ times. -- The action signals failure by 'mzero'. mretryN :: (MonadPlus m) => Int -> (Int -> m a) -> m a mretryN n = msum . flip map [1..n] -- | Retries the given action up to @n@ times. -- The action signals failure by 'mzero'. retryMaybeN :: (Monad m) => Int -> (Int -> MaybeT m a) -> m (Maybe a) retryMaybeN = (runMaybeT .) . mretryN -- | Retries the given action up to @n@ times until it succeeds. -- If all actions fail, the error of the last one is returned. -- The action is always run at least once, even if @n@ is less than 1. retryErrorN :: (MonadError e m) => Int -> (Int -> m a) -> m a retryErrorN n f = loop 1 where loop i | i < n = catchError (f i) (const $ loop (i + 1)) | otherwise = f i -- * From monad-loops (until we can / want to depend on it): -- | Short-circuit 'any' with a monadic predicate. anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool anyM p = foldM (\v x -> if v then return True else p x) False -- | Short-circuit 'all' with a monadic predicate. allM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool allM p = foldM (\v x -> if v then p x else return False) True -- | Short-circuit 'or' for values of type Monad m => m Bool orM :: (Monad m) => [m Bool] -> m Bool orM = anyM id -- |See 'Data.List.unfoldr'. This is a monad-friendly version of that. unfoldrM :: (Monad m) => (a -> m (Maybe (b,a))) -> a -> m [b] unfoldrM = unfoldrM' -- | See 'Data.List.unfoldr'. This is a monad-friendly version of that, with a -- twist. Rather than returning a list, it returns any MonadPlus type of your -- choice. unfoldrM' :: (Monad m, MonadPlus f) => (a -> m (Maybe (b,a))) -> a -> m (f b) unfoldrM' f z = do x <- f z case x of Nothing -> return mzero Just (x', z') -> do xs <- unfoldrM' f z' return (return x' `mplus` xs) ganeti-3.1.0~rc2/src/Ganeti/Utils/MultiMap.hs000064400000000000000000000113141476477700300207660ustar00rootroot00000000000000{-# LANGUAGE Rank2Types, TypeFamilies #-} {-| Implements multi-maps - maps that map keys to sets of values This module uses the standard naming convention as other collection-like libraries and is meant to be imported qualified. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.MultiMap ( MultiMap() , multiMap , multiMapL , multiMapValueL , null , findValue , elem , lookup , member , insert , fromList , delete , deleteAll , values ) where import Prelude hiding (lookup, null, elem) import Control.Monad import qualified Data.Foldable as F import qualified Data.Map as M import qualified Data.Semigroup as Sem import Data.Maybe (fromMaybe, isJust) import qualified Data.Set as S import qualified Text.JSON as J import Ganeti.Lens -- | A multi-map that contains multiple values for a single key. -- It doesn't distinguish non-existent keys and keys with the empty -- set as the value. newtype MultiMap k v = MultiMap { getMultiMap :: M.Map k (S.Set v) } deriving (Eq, Ord, Show) instance (Ord v, Ord k) => Sem.Semigroup (MultiMap k v) where (MultiMap x) <> (MultiMap y) = MultiMap $ M.unionWith S.union x y instance (Ord v, Ord k) => Monoid (MultiMap k v) where mempty = MultiMap M.empty mappend = (Sem.<>) instance F.Foldable (MultiMap k) where foldMap f = F.foldMap (F.foldMap f) . getMultiMap instance (J.JSON k, Ord k, J.JSON v, Ord v) => J.JSON (MultiMap k v) where showJSON = J.showJSON . getMultiMap readJSON = liftM MultiMap . J.readJSON -- | Creates a multi-map from a map of sets. multiMap :: (Ord k, Ord v) => M.Map k (S.Set v) -> MultiMap k v multiMap = MultiMap . M.filter (not . S.null) -- | A 'Lens' that allows to access a set under a given key in a multi-map. multiMapL :: (Ord k, Ord v) => k -> Lens' (MultiMap k v) (S.Set v) multiMapL k f = fmap MultiMap . at k (fmap (mfilter (not . S.null) . Just) . f . fromMaybe S.empty) . getMultiMap {-# INLINE multiMapL #-} -- | Return the set corresponding to a given key. lookup :: (Ord k, Ord v) => k -> MultiMap k v -> S.Set v lookup k = view (multiMapL k) -- | Tests if the given key has a non-empty set of values. member :: (Ord k, Ord v) => k -> MultiMap k v -> Bool member = (S.null .) . lookup -- | Tries to find a key corresponding to a given value. findValue :: (Ord k, Ord v) => v -> MultiMap k v -> Maybe k findValue v = fmap fst . F.find (S.member v . snd) . M.toList . getMultiMap -- | Returns 'True' iff a given value is present in a set of some key. elem :: (Ord k, Ord v) => v -> MultiMap k v -> Bool elem = (isJust .) . findValue null :: MultiMap k v -> Bool null = M.null . getMultiMap insert :: (Ord k, Ord v) => k -> v -> MultiMap k v -> MultiMap k v insert k v = set (multiMapValueL k v) True fromList :: (Ord k, Ord v) => [(k, v)] -> MultiMap k v fromList = foldr (uncurry insert) mempty delete :: (Ord k, Ord v) => k -> v -> MultiMap k v -> MultiMap k v delete k v = set (multiMapValueL k v) False deleteAll :: (Ord k, Ord v) => k -> MultiMap k v -> MultiMap k v deleteAll k = set (multiMapL k) S.empty values :: (Ord k, Ord v) => MultiMap k v -> S.Set v values = F.fold . getMultiMap -- | A 'Lens' that allows to access a given value/key pair in a multi-map. -- -- It is similar to the 'At' instance, but uses more convenient 'Bool' -- instead of 'Maybe ()' and a pair key/value. multiMapValueL :: (Ord k, Ord v) => k -> v -> Lens' (MultiMap k v) Bool multiMapValueL k v = multiMapL k . atSet v {-# INLINE multiMapValueL #-} ganeti-3.1.0~rc2/src/Ganeti/Utils/Random.hs000064400000000000000000000045571476477700300204710ustar00rootroot00000000000000{-| Utilities related to randomized computations. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.Random ( generateSecret , generateOneMAC , delayRandom ) where import Control.Concurrent (threadDelay) import Control.Monad import Control.Monad.State import System.Random import Text.Printf -- | Generates a random secret of a given length. -- The type is chosen so that it can be easily wrapped into a state monad. generateSecret :: (RandomGen g) => Int -> g -> (String, g) generateSecret n = runState . liftM (concatMap $ printf "%02x") $ replicateM n (state $ randomR (0 :: Int, 255)) -- | Given a prefix, randomly generates a full MAC address. -- -- See 'generateMAC' for discussion about how this function uses -- the random generator. generateOneMAC :: (RandomGen g) => String -> g -> (String, g) generateOneMAC prefix = runState $ let randByte = state (randomR (0, 255 :: Int)) in printf "%s:%02x:%02x:%02x" prefix <$> randByte <*> randByte <*> randByte -- | Wait a time period randomly chosen within the given bounds -- (in microseconds). delayRandom :: (Int, Int) -> IO () delayRandom = threadDelay <=< randomRIO ganeti-3.1.0~rc2/src/Ganeti/Utils/Statistics.hs000064400000000000000000000121361476477700300213730ustar00rootroot00000000000000{-# LANGUAGE BangPatterns #-} {-| Utility functions for statistical accumulation. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.Statistics ( Statistics , TagTagMap , AggregateComponent(..) , getSumStatistics , getStdDevStatistics , getMapStatistics , getStatisticValue , updateStatistics ) where import qualified Data.Foldable as Foldable import Data.List (foldl') import qualified Data.Map as Map -- | Type to store the number of instances for each exclusion and location -- pair. This is necessary to calculate second component of location score. type TagTagMap = Map.Map (String, String) Int -- | Abstract type of statistical accumulations. They behave as if the given -- statistics were computed on the list of values, but they allow a potentially -- more efficient update of a given value. data Statistics = SumStatistics Double | StdDevStatistics Double Double Double -- count, sum, and not the sum of squares---instead the -- computed variance for better precission. | MapStatistics TagTagMap deriving Show -- | Abstract type of per-node statistics measures. The SimpleNumber is used -- to construct SumStatistics and StdDevStatistics while SpreadValues is used -- to construct MapStatistics. data AggregateComponent = SimpleNumber Double | SpreadValues TagTagMap -- Each function below depends on the contents of AggregateComponent but it's -- necessary to define each function as a function processing both -- SimpleNumber and SpreadValues instances (see Metrics.hs). That's why -- pattern matches for invalid type defined as functions which change nothing. -- | Get a statistics that sums up the values. getSumStatistics :: [AggregateComponent] -> Statistics getSumStatistics xs = let addComponent s (SimpleNumber x) = let !s' = s + x in s' addComponent s _ = s st = foldl' addComponent 0 xs in SumStatistics st -- | Get a statistics for the standard deviation. getStdDevStatistics :: [AggregateComponent] -> Statistics getStdDevStatistics xs = let addComponent (n, s) (SimpleNumber x) = let !n' = n + 1 !s' = s + x in (n', s') addComponent (n, s) _ = (n, s) (nt, st) = foldl' addComponent (0, 0) xs mean = st / nt center (SimpleNumber x) = x - mean center _ = 0 nvar = foldl' (\v x -> let d = center x in v + d * d) 0 xs in StdDevStatistics nt st (nvar / nt) -- | Get a statistics for the standard deviation. getMapStatistics :: [AggregateComponent] -> Statistics getMapStatistics xs = let addComponent m (SpreadValues x) = let !m' = Map.unionWith (+) m x in m' addComponent m _ = m mt = foldl' addComponent Map.empty xs in MapStatistics mt -- | Obtain the value of a statistics. getStatisticValue :: Statistics -> Double getStatisticValue (SumStatistics s) = s getStatisticValue (StdDevStatistics _ _ var) = sqrt var getStatisticValue (MapStatistics m) = fromIntegral $ Foldable.sum m - Map.size m -- Function above calculates sum (N_i - 1) over each map entry. -- | In a given statistics replace on value by another. This -- will only give meaningful results, if the original value -- was actually part of the statistics. updateStatistics :: Statistics -> (AggregateComponent, AggregateComponent) -> Statistics updateStatistics (SumStatistics s) (SimpleNumber x, SimpleNumber y) = SumStatistics $ s + (y - x) updateStatistics (StdDevStatistics n s var) (SimpleNumber x, SimpleNumber y) = let !ds = y - x !dss = y * y - x * x !dnnvar = (n * dss - 2 * s * ds) - ds * ds !s' = s + ds !var' = max 0 $ var + dnnvar / (n * n) in StdDevStatistics n s' var' updateStatistics (MapStatistics m) (SpreadValues x, SpreadValues y) = let nm = Map.unionWith (+) (Map.unionWith (-) m x) y in MapStatistics nm updateStatistics s _ = s ganeti-3.1.0~rc2/src/Ganeti/Utils/Time.hs000064400000000000000000000074411476477700300201420ustar00rootroot00000000000000{-| Time utility functions. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.Time ( getCurrentTime , getCurrentTimeUSec , clockTimeToString , clockTimeToCTime , clockTimeToUSec , cTimeToClockTime , diffClockTimes , addToClockTime , noTimeDiff , TimeDiff(..) ) where import qualified System.Time as STime import System.Time (ClockTime(..), getClockTime) import Foreign.C.Types (CTime(CTime)) import System.Posix.Types (EpochTime) -- | Returns the current time as an 'Integer' representing the number -- of seconds from the Unix epoch. getCurrentTime :: IO Integer getCurrentTime = do TOD ctime _ <- getClockTime return ctime -- | Returns the current time as an 'Integer' representing the number -- of microseconds from the Unix epoch (hence the need for 'Integer'). getCurrentTimeUSec :: IO Integer getCurrentTimeUSec = fmap clockTimeToUSec getClockTime -- | Convert a ClockTime into a (seconds-only) timestamp. clockTimeToString :: ClockTime -> String clockTimeToString (TOD t _) = show t -- | Convert a ClockTime into a (seconds-only) 'EpochTime' (AKA @time_t@). clockTimeToCTime :: ClockTime -> EpochTime clockTimeToCTime (TOD secs _) = fromInteger secs -- | Convert a ClockTime the number of microseconds since the epoch. clockTimeToUSec :: ClockTime -> Integer clockTimeToUSec (TOD ctime pico) = -- pico: 10^-12, micro: 10^-6, so we have to shift seconds left and -- picoseconds right ctime * 1000000 + pico `div` 1000000 -- | Convert a ClockTime into a (seconds-only) 'EpochTime' (AKA @time_t@). cTimeToClockTime :: EpochTime -> ClockTime cTimeToClockTime (CTime timet) = TOD (toInteger timet) 0 {- | Like 'System.Time.TimeDiff' but misses 'tdYear', 'tdMonth'. Their meaning depends on the start date and causes bugs like this one: This type is much simpler and less error-prone. Also, our 'tdSec' has type 'Integer', which cannot overflow. Like in original 'System.Time.TimeDiff' picoseconds can be negative, that is, TimeDiff's are not normalized by default. -} data TimeDiff = TimeDiff {tdSec :: Integer, tdPicosec :: Integer} deriving (Show) noTimeDiff :: TimeDiff noTimeDiff = TimeDiff 0 0 diffClockTimes :: ClockTime -> ClockTime -> TimeDiff diffClockTimes (STime.TOD secA picoA) (STime.TOD secB picoB) = TimeDiff (secA-secB) (picoA-picoB) addToClockTime :: TimeDiff -> ClockTime -> ClockTime addToClockTime (TimeDiff secDiff picoDiff) (TOD secA picoA) = case divMod (picoA+picoDiff) (1000*1000*1000*1000) of (secD, picoB) -> TOD (secA+secDiff+secD) picoB ganeti-3.1.0~rc2/src/Ganeti/Utils/UniStd.hs000064400000000000000000000042431476477700300204470ustar00rootroot00000000000000{-# LANGUAGE ForeignFunctionInterface #-} {-| Necessary foreign function calls ...with foreign functions declared in unistd.h -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.UniStd ( fsyncFile ) where import Control.Exception (bracket) import Foreign.C import qualified System.Posix as Posix import System.Posix.Types import Ganeti.Compat (openFd, closeFd) import Ganeti.BasicTypes foreign import ccall "fsync" fsync :: CInt -> IO CInt -- Opens a file and calls fsync(2) on the file descriptor. -- -- Because of a bug in GHC 7.6.3 (at least), calling 'hIsClosed' on a handle -- to get the file descriptor leaks memory. Therefore we open a given file -- just to sync it and close it again. fsyncFile :: (Error e) => FilePath -> ResultT e IO () fsyncFile path = liftIO $ bracket (openFd path Posix.ReadOnly Nothing Posix.defaultFileFlags) closeFd $ \(Fd fd) -> throwErrnoPathIfMinus1_ "fsyncFile" path $ fsync fd ganeti-3.1.0~rc2/src/Ganeti/Utils/Validate.hs000064400000000000000000000107251476477700300207740ustar00rootroot00000000000000{-# LANGUAGE FlexibleInstances, FlexibleContexts, GeneralizedNewtypeDeriving #-} {-| A validation monad and corresponding utilities The monad allows code to emit errors during checking. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.Utils.Validate ( ValidationMonadT() , ValidationMonad , report , reportIf , runValidate , runValidateT , execValidateT , execValidate , evalValidateT , evalValidate , Validatable(..) , validate' ) where import Control.Arrow import Control.Monad import Control.Monad.Except import Control.Monad.Writer import qualified Data.Foldable as F import Data.Functor.Identity import Data.List (intercalate) import Data.Sequence -- | Monad for running validation checks. newtype ValidationMonadT m a = ValidationMonad { runValidationMonad :: WriterT (Seq String) m a } deriving (Functor, Applicative, Monad) type ValidationMonad = ValidationMonadT Identity -- | An utility function that emits a single message into a validation monad. report :: (Monad m) => String -> ValidationMonadT m () report = ValidationMonad . tell . singleton -- | An utility function that conditionally emits a message into -- a validation monad. -- It's a combination of 'when' and 'report'. reportIf :: (Monad m) => Bool -> String -> ValidationMonadT m () reportIf b = when b . report -- | An utility function that runs a monadic validation action -- and returns the list of errors. runValidateT :: (Monad m) => ValidationMonadT m a -> m (a, [String]) runValidateT = liftM (second F.toList) . runWriterT . runValidationMonad -- | An utility function that runs a monadic validation action -- and returns the list of errors. runValidate :: ValidationMonad a -> (a, [String]) runValidate = runIdentity . runValidateT -- | An utility function that runs a monadic validation action -- and returns the list of errors. execValidateT :: (Monad m) => ValidationMonadT m () -> m [String] execValidateT = liftM F.toList . execWriterT . runValidationMonad -- | An utility function that runs a validation action -- and returns the list of errors. execValidate :: ValidationMonad () -> [String] execValidate = runIdentity . execValidateT -- | A helper function for throwing an exception if a list of errors -- is non-empty. throwIfErrors :: (MonadError String m) => (a, [String]) -> m a throwIfErrors (x, []) = return x throwIfErrors (_, es) = throwError $ "Validation errors: " ++ intercalate "; " es -- | Runs a validation action and if there are errors, combine them -- into an exception. evalValidate :: (MonadError String m) => ValidationMonad a -> m a evalValidate = throwIfErrors . runValidate -- | Runs a validation action and if there are errors, combine them -- into an exception. evalValidateT :: (MonadError String m) => ValidationMonadT m a -> m a evalValidateT k = runValidateT k >>= throwIfErrors -- | A typeclass for objects that can be validated. -- That is, they can perform an internal check and emit any -- errors encountered. -- Emiting no errors means the object is valid. class Validatable a where validate :: a -> ValidationMonad () -- | Run validation and return the original value as well. -- the original value. validate' :: (Validatable a) => a -> ValidationMonad a validate' x = x <$ validate x ganeti-3.1.0~rc2/src/Ganeti/VCluster.hs000064400000000000000000000043051476477700300177070ustar00rootroot00000000000000{-| Utilities for virtual clusters. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.VCluster ( makeVirtualPath ) where import Control.Monad (liftM) import Data.Set (member) import System.Posix.Env (getEnv) import System.FilePath.Posix import Ganeti.ConstantUtils (unFrozenSet) import Ganeti.Constants getRootDirectory :: IO (Maybe FilePath) getRootDirectory = fmap normalise `liftM` getEnv vClusterRootdirEnvname -- | Pure computation of the virtual path from the original path -- and the vcluster root virtualPath :: FilePath -> FilePath -> FilePath virtualPath fpath root = let relpath = makeRelative root fpath in if member fpath (unFrozenSet vClusterVpathWhitelist) then fpath else vClusterVirtPathPrefix relpath -- | Given a path, make it a virtual one, if in a vcluster environment. -- Otherwise, return unchanged. makeVirtualPath :: FilePath -> IO FilePath makeVirtualPath fpath = maybe fpath (virtualPath fpath) `liftM` getRootDirectory ganeti-3.1.0~rc2/src/Ganeti/Version.hs.in000064400000000000000000000003661476477700300201750ustar00rootroot00000000000000-- Hey Emacs, this is a -*- haskell -*- file {- | Auto-generated module holding version information. -} module Ganeti.Version ( version ) where -- | The version of the sources. version :: String version = "(ganeti) version %ver%" ganeti-3.1.0~rc2/src/Ganeti/WConfd/000075500000000000000000000000001476477700300167625ustar00rootroot00000000000000ganeti-3.1.0~rc2/src/Ganeti/WConfd/Client.hs000064400000000000000000000063261476477700300205430ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| The Ganeti WConfd client functions. The client functions are automatically generated from Ganeti.WConfd.Core -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.Client where import Control.Exception.Lifted (bracket) import Ganeti.THH.HsRPC import Ganeti.Constants import Ganeti.JSON (unMaybeForJSON) import Ganeti.Locking.Locks (ClientId) import Ganeti.Objects (ConfigData) import Ganeti.UDSServer (ConnectConfig(..), Client, connectClient) import Ganeti.WConfd.Core (exportedFunctions) -- * Generated client functions $(mkRpcCalls exportedFunctions) -- * Helper functions for creating the client -- | The default WConfd client configuration wconfdConnectConfig :: ConnectConfig wconfdConnectConfig = ConnectConfig { recvTmo = wconfdDefRwto , sendTmo = wconfdDefRwto } -- | Given a socket path, creates a WConfd client with the default -- configuration and timeout. getWConfdClient :: FilePath -> IO Client getWConfdClient = connectClient wconfdConnectConfig wconfdDefCtmo -- * Helper functions for getting a remote lock -- | Calls the `lockConfig` RPC until the lock is obtained. waitLockConfig :: ClientId -> Bool -- ^ whether the lock shall be in shared mode -> RpcClientMonad ConfigData waitLockConfig c shared = do mConfigData <- lockConfig c shared case unMaybeForJSON mConfigData of Just configData -> return configData Nothing -> waitLockConfig c shared -- | Calls the `lockConfig` RPC until the lock is obtained, -- runs a function on the obtained config, and calls `unlockConfig`. withLockedConfig :: ClientId -> Bool -- ^ whether the lock shall be in shared mode -> (ConfigData -> RpcClientMonad a) -- ^ action to run -> RpcClientMonad a withLockedConfig c shared = -- Unlock config even if something throws. bracket (waitLockConfig c shared) (const $ unlockConfig c) ganeti-3.1.0~rc2/src/Ganeti/WConfd/ConfigModifications.hs000064400000000000000000000664221476477700300232460ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, NoMonomorphismRestriction, FlexibleContexts #-} {-| The WConfd functions for direct configuration manipulation This module contains the client functions exported by WConfD for specific configuration manipulation. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.ConfigModifications where import Control.Lens (_2) import Control.Lens.Getter ((^.)) import Control.Lens.Setter ((.~), (%~)) import qualified Data.ByteString.UTF8 as UTF8 import Control.Lens.Traversal (mapMOf) import Control.Monad (unless, when, forM_, foldM, liftM2) import Control.Monad.Except (MonadError, throwError) import Control.Monad.Fail (MonadFail) import Control.Monad.IO.Class (liftIO) import Control.Monad.Trans.State (StateT, get, put, modify, runStateT, execStateT) import Data.Foldable (fold) import Data.List (elemIndex) import Data.Maybe (isJust, maybeToList, fromMaybe, fromJust) import Language.Haskell.TH (Name) import System.Time (getClockTime, ClockTime) import Text.Printf (printf) import qualified Data.Map as M import qualified Data.Set as S import Ganeti.BasicTypes (GenericResult(..), genericResult, toError) import Ganeti.Constants (lastDrbdPort) import Ganeti.Errors (GanetiException(..)) import Ganeti.JSON (Container, GenericContainer(..), alterContainerL , lookupContainer, MaybeForJSON(..), TimeAsDoubleJSON(..)) import Ganeti.Locking.Locks (ClientId, ciIdentifier) import Ganeti.Logging.Lifted (logDebug, logInfo) import Ganeti.Objects import Ganeti.Objects.Lens import Ganeti.Types (AdminState, AdminStateSource) import Ganeti.WConfd.ConfigState (ConfigState, csConfigData, csConfigDataL) import Ganeti.WConfd.Monad (WConfdMonad, modifyConfigWithLock , modifyConfigAndReturnWithLock) import qualified Ganeti.WConfd.TempRes as T type DiskUUID = String type InstanceUUID = String type NodeUUID = String -- * accessor functions getInstanceByUUID :: ConfigState -> InstanceUUID -> GenericResult GanetiException Instance getInstanceByUUID cs uuid = lookupContainer (Bad . ConfigurationError $ printf "Could not find instance with UUID %s" uuid) (UTF8.fromString uuid) (configInstances . csConfigData $ cs) -- * getters -- | Gets all logical volumes in the cluster getAllLVs :: ConfigState -> S.Set String getAllLVs = S.fromList . concatMap getLVsOfDisk . M.elems . fromContainer . configDisks . csConfigData where convert (LogicalVolume lvG lvV) = lvG ++ "/" ++ lvV getDiskLV :: Disk -> Maybe String getDiskLV disk = case diskLogicalId disk of Just (LIDPlain lv) -> Just (convert lv) _ -> Nothing getLVsOfDisk :: Disk -> [String] getLVsOfDisk disk = maybeToList (getDiskLV disk) ++ concatMap getLVsOfDisk (diskChildren disk) -- | Gets the ids of nodes, instances, node groups, -- networks, disks, nics, and the cluster itself. getAllIDs :: ConfigState -> S.Set String getAllIDs cs = let lvs = getAllLVs cs keysFromC :: GenericContainer a b -> [a] keysFromC = M.keys . fromContainer valuesFromC :: GenericContainer a b -> [b] valuesFromC = M.elems . fromContainer instKeys = keysFromC . configInstances . csConfigData $ cs nodeKeys = keysFromC . configNodes . csConfigData $ cs instValues = map uuidOf . valuesFromC . configInstances . csConfigData $ cs nodeValues = map uuidOf . valuesFromC . configNodes . csConfigData $ cs nodeGroupValues = map uuidOf . valuesFromC . configNodegroups . csConfigData $ cs networkValues = map uuidOf . valuesFromC . configNetworks . csConfigData $ cs disksValues = map uuidOf . valuesFromC . configDisks . csConfigData $ cs nics = map nicUuid . concatMap instNics . valuesFromC . configInstances . csConfigData $ cs cluster = uuidOf . configCluster . csConfigData $ cs in S.union lvs . S.fromList $ map UTF8.toString instKeys ++ map UTF8.toString nodeKeys ++ instValues ++ nodeValues ++ nodeGroupValues ++ networkValues ++ disksValues ++ map UTF8.toString nics ++ [cluster] getAllMACs :: ConfigState -> S.Set String getAllMACs = S.fromList . map nicMac . concatMap instNics . M.elems . fromContainer . configInstances . csConfigData -- | Checks if the two objects are equal, -- excluding timestamps. The serial number of -- current must be one greater than that of target. -- -- If this is true, it implies that the update RPC -- updated the config, but did not successfully return. isIdentical :: (Eq a, SerialNoObjectL a, TimeStampObjectL a) => ClockTime -> a -> a -> Bool isIdentical now target current = (mTimeL .~ now $ current) == ((serialL %~ (+1)) . (mTimeL .~ now) $ target) -- | Checks if the two objects given have the same serial number checkSerial :: SerialNoObject a => a -> a -> GenericResult GanetiException () checkSerial target current = if serialOf target == serialOf current then Ok () else Bad . ConfigurationError $ printf "Configuration object updated since it has been read: %d != %d" (serialOf current) (serialOf target) -- | Updates an object present in a container. -- The presence of the object in the container -- is determined by the uuid of the object. -- -- A check that serial number of the -- object is consistent with the serial number -- of the object in the container is performed. -- -- If the check passes, the object's serial number -- is incremented, and modification time is updated, -- and then is inserted into the container. replaceIn :: (UuidObject a, TimeStampObjectL a, SerialNoObjectL a) => ClockTime -> a -> Container a -> GenericResult GanetiException (Container a) replaceIn now target = alterContainerL (UTF8.fromString (uuidOf target)) extract where extract Nothing = Bad $ ConfigurationError "Configuration object unknown" extract (Just current) = do checkSerial target current return . Just . (serialL %~ (+1)) . (mTimeL .~ now) $ target -- | Utility fuction that combines the two -- possible actions that could be taken when -- given a target. -- -- If the target is identical to the current -- value, we return the modification time of -- the current value, and not change the config. -- -- If not, we update the config. updateConfigIfNecessary :: (Monad m, MonadError GanetiException m, Eq a, UuidObject a, SerialNoObjectL a, TimeStampObjectL a) => ClockTime -> a -> (ConfigState -> Container a) -> (ConfigState -> m ((Int, ClockTime), ConfigState)) -> ConfigState -> m ((Int, ClockTime), ConfigState) updateConfigIfNecessary now target getContainer f cs = do let container = getContainer cs current <- lookupContainer (toError . Bad . ConfigurationError $ "Configuraton object unknown") (UTF8.fromString (uuidOf target)) container if isIdentical now target current then return ((serialOf current, mTimeOf current), cs) else f cs -- * UUID config checks -- | Checks if the config has the given UUID checkUUIDpresent :: UuidObject a => ConfigState -> a -> Bool checkUUIDpresent cs a = uuidOf a `S.member` getAllIDs cs -- | Checks if the given UUID is new (i.e., no in the config) checkUniqueUUID :: UuidObject a => ConfigState -> a -> Bool checkUniqueUUID cs a = not $ checkUUIDpresent cs a -- * RPC checks -- | Verifications done before adding an instance. -- Currently confirms that the instance's macs are not -- in use, and that the instance's UUID being -- present (or not present) in the config based on -- weather the instance is being replaced (or not). -- -- TODO: add more verifications to this call; -- the client should have a lock on the name of the instance. addInstanceChecks :: Instance -> Bool -> ConfigState -> GenericResult GanetiException () addInstanceChecks inst replace cs = do let macsInUse = S.fromList (map nicMac (instNics inst)) `S.intersection` getAllMACs cs unless (S.null macsInUse) . Bad . ConfigurationError $ printf "Cannot add instance %s; MAC addresses %s already in use" (show $ instName inst) (show macsInUse) if replace then do let check = checkUUIDpresent cs inst unless check . Bad . ConfigurationError $ printf "Cannot add %s: UUID %s already in use" (show $ instName inst) (UTF8.toString (instUuid inst)) else do let check = checkUniqueUUID cs inst unless check . Bad . ConfigurationError $ printf "Cannot replace %s: UUID %s not present" (show $ instName inst) (UTF8.toString (instUuid inst)) addDiskChecks :: Disk -> Bool -> ConfigState -> GenericResult GanetiException () addDiskChecks disk replace cs = if replace then unless (checkUUIDpresent cs disk) . Bad . ConfigurationError $ printf "Cannot add %s: UUID %s already in use" (show $ diskName disk) (UTF8.toString (diskUuid disk)) else unless (checkUniqueUUID cs disk) . Bad . ConfigurationError $ printf "Cannot replace %s: UUID %s not present" (show $ diskName disk) (UTF8.toString (diskUuid disk)) attachInstanceDiskChecks :: InstanceUUID -> DiskUUID -> MaybeForJSON Int -> ConfigState -> GenericResult GanetiException () attachInstanceDiskChecks uuidInst uuidDisk idx' cs = do let diskPresent = elem uuidDisk . map (UTF8.toString . diskUuid) . M.elems . fromContainer . configDisks . csConfigData $ cs unless diskPresent . Bad . ConfigurationError $ printf "Disk %s doesn't exist" uuidDisk inst <- getInstanceByUUID cs uuidInst let numDisks = length $ instDisks inst idx = fromMaybe numDisks (unMaybeForJSON idx') when (idx < 0) . Bad . GenericError $ "Not accepting negative indices" when (idx > numDisks) . Bad . GenericError $ printf "Got disk index %d, but there are only %d" idx numDisks let insts = M.elems . fromContainer . configInstances . csConfigData $ cs forM_ insts (\inst' -> when (uuidDisk `elem` instDisks inst') . Bad . ReservationError $ printf "Disk %s already attached to instance %s" uuidDisk (show . fromMaybe "" $ instName inst')) -- * Pure config modifications functions attachInstanceDisk' :: InstanceUUID -> DiskUUID -> MaybeForJSON Int -> ClockTime -> ConfigState -> ConfigState attachInstanceDisk' iUuid dUuid idx' ct cs = let inst = genericResult (error "impossible") id (getInstanceByUUID cs iUuid) numDisks = length $ instDisks inst idx = fromMaybe numDisks (unMaybeForJSON idx') insert = instDisksL %~ (\ds -> take idx ds ++ [dUuid] ++ drop idx ds) incr = instSerialL %~ (+ 1) time = instMtimeL .~ ct inst' = time . incr . insert $ inst disks = updateIvNames idx inst' (configDisks . csConfigData $ cs) ri = csConfigDataL . configInstancesL . alterContainerL (UTF8.fromString iUuid) .~ Just inst' rds = csConfigDataL . configDisksL .~ disks in rds . ri $ cs where updateIvNames :: Int -> Instance -> Container Disk -> Container Disk updateIvNames idx inst (GenericContainer m) = let dUuids = drop idx (instDisks inst) upgradeIv m' (idx'', dUuid') = M.adjust (diskIvNameL .~ "disk/" ++ show idx'') dUuid' m' in GenericContainer $ foldl upgradeIv m (zip [idx..] (fmap UTF8.fromString dUuids)) -- * Monadic config modification functions which can return errors detachInstanceDisk' :: MonadError GanetiException m => InstanceUUID -> DiskUUID -> ClockTime -> ConfigState -> m ConfigState detachInstanceDisk' iUuid dUuid ct cs = let resetIv :: MonadError GanetiException m => Int -> [DiskUUID] -> ConfigState -> m ConfigState resetIv startIdx disks = mapMOf (csConfigDataL . configDisksL) (\cd -> foldM (\c (idx, dUuid') -> mapMOf (alterContainerL dUuid') (\md -> case md of Nothing -> throwError . ConfigurationError $ printf "Could not find disk with UUID %s" (UTF8.toString dUuid') Just disk -> return . Just . (diskIvNameL .~ ("disk/" ++ show idx)) $ disk) c) cd (zip [startIdx..] (fmap UTF8.fromString disks))) iL = csConfigDataL . configInstancesL . alterContainerL (UTF8.fromString iUuid) in case cs ^. iL of Nothing -> throwError . ConfigurationError $ printf "Could not find instance with UUID %s" iUuid Just ist -> case elemIndex dUuid (instDisks ist) of Nothing -> return cs Just idx -> let ist' = (instDisksL %~ filter (/= dUuid)) . (instSerialL %~ (+1)) . (instMtimeL .~ ct) $ ist cs' = iL .~ Just ist' $ cs dks = drop (idx + 1) (instDisks ist) in resetIv idx dks cs' removeInstanceDisk' :: MonadError GanetiException m => InstanceUUID -> DiskUUID -> ClockTime -> ConfigState -> m ConfigState removeInstanceDisk' iUuid dUuid ct = let f cs | elem dUuid . fold . fmap instDisks . configInstances . csConfigData $ cs = throwError . ProgrammerError $ printf "Cannot remove disk %s. Disk is attached to an instance" dUuid | elem dUuid . foldMap (:[]) . fmap (UTF8.toString . diskUuid) . configDisks . csConfigData $ cs = return . ((csConfigDataL . configDisksL . alterContainerL (UTF8.fromString dUuid)) .~ Nothing) . ((csConfigDataL . configClusterL . clusterSerialL) %~ (+1)) . ((csConfigDataL . configClusterL . clusterMtimeL) .~ ct) $ cs | otherwise = return cs in (f =<<) . detachInstanceDisk' iUuid dUuid ct -- * RPCs -- | Add a new instance to the configuration, release DRBD minors, -- and commit temporary IPs, all while temporarily holding the config -- lock. Return True upon success and False if the config lock was not -- available and the client should retry. addInstance :: Instance -> ClientId -> Bool -> WConfdMonad Bool addInstance inst cid replace = do ct <- liftIO getClockTime logDebug $ "AddInstance: client " ++ show (ciIdentifier cid) ++ " adding instance " ++ uuidOf inst ++ " with name " ++ show (instName inst) let setCtime = instCtimeL .~ ct setMtime = instMtimeL .~ ct addInst i = csConfigDataL . configInstancesL . alterContainerL (UTF8.fromString $ uuidOf i) .~ Just i commitRes tr = mapMOf csConfigDataL $ T.commitReservedIps cid tr r <- modifyConfigWithLock (\tr cs -> do toError $ addInstanceChecks inst replace cs commitRes tr $ addInst (setMtime . setCtime $ inst) cs) . T.releaseDRBDMinors . UTF8.fromString $ uuidOf inst logDebug $ "AddInstance: result of config modification is " ++ show r return $ isJust r addInstanceDisk :: InstanceUUID -> Disk -> MaybeForJSON Int -> Bool -> WConfdMonad Bool addInstanceDisk iUuid disk idx replace = do logInfo $ printf "Adding disk %s to configuration" (UTF8.toString (diskUuid disk)) ct <- liftIO getClockTime let addD = csConfigDataL . configDisksL . alterContainerL (UTF8.fromString (uuidOf disk)) .~ Just disk incrSerialNo = csConfigDataL . configSerialL %~ (+1) r <- modifyConfigWithLock (\_ cs -> do toError $ addDiskChecks disk replace cs let cs' = incrSerialNo . addD $ cs toError $ attachInstanceDiskChecks iUuid (UTF8.toString (diskUuid disk)) idx cs' return $ attachInstanceDisk' iUuid (UTF8.toString (diskUuid disk)) idx ct cs') . T.releaseDRBDMinors $ UTF8.fromString (uuidOf disk) return $ isJust r attachInstanceDisk :: InstanceUUID -> DiskUUID -> MaybeForJSON Int -> WConfdMonad Bool attachInstanceDisk iUuid dUuid idx = do ct <- liftIO getClockTime r <- modifyConfigWithLock (\_ cs -> do toError $ attachInstanceDiskChecks iUuid dUuid idx cs return $ attachInstanceDisk' iUuid dUuid idx ct cs) (return ()) return $ isJust r -- | Detach a disk from an instance. detachInstanceDisk :: InstanceUUID -> DiskUUID -> WConfdMonad Bool detachInstanceDisk iUuid dUuid = do ct <- liftIO getClockTime isJust <$> modifyConfigWithLock (const $ detachInstanceDisk' iUuid dUuid ct) (return ()) -- | Detach a disk from an instance and -- remove it from the config. removeInstanceDisk :: InstanceUUID -> DiskUUID -> WConfdMonad Bool removeInstanceDisk iUuid dUuid = do ct <- liftIO getClockTime isJust <$> modifyConfigWithLock (const $ removeInstanceDisk' iUuid dUuid ct) (return ()) -- | Remove the instance from the configuration. removeInstance :: InstanceUUID -> WConfdMonad Bool removeInstance iUuid = do ct <- liftIO getClockTime let iL = csConfigDataL . configInstancesL . alterContainerL (UTF8.fromString iUuid) pL = csConfigDataL . configClusterL . clusterTcpudpPortPoolL sL = csConfigDataL . configClusterL . clusterSerialL mL = csConfigDataL . configClusterL . clusterMtimeL -- Add the instances' network port to the cluster pool f :: Monad m => StateT ConfigState m () f = get >>= (maybe (return ()) (maybe (return ()) (modify . (pL %~) . (:)) . instNetworkPort) . (^. iL)) -- Release all IP addresses to the pool g :: (MonadError GanetiException m, Functor m, MonadFail m) => StateT ConfigState m () g = get >>= (maybe (return ()) (mapM_ (\nic -> when ((isJust . nicNetwork $ nic) && (isJust . nicIp $ nic)) $ do let network = fromJust . nicNetwork $ nic ip <- readIp4Address (fromJust . nicIp $ nic) get >>= mapMOf csConfigDataL (T.commitReleaseIp (UTF8.fromString network) ip) >>= put) . instNics) . (^. iL)) -- Remove the instance and update cluster serial num, and mtime h :: Monad m => StateT ConfigState m () h = modify $ (iL .~ Nothing) . (sL %~ (+1)) . (mL .~ ct) isJust <$> modifyConfigWithLock (const $ execStateT (f >> g >> h)) (return ()) -- | Allocate a port. -- The port will be taken from the available port pool or from the -- default port range (and in this case we increase -- highest_used_port). allocatePort :: WConfdMonad (MaybeForJSON Int) allocatePort = do maybePort <- modifyConfigAndReturnWithLock (\_ cs -> let portPoolL = csConfigDataL . configClusterL . clusterTcpudpPortPoolL hupL = csConfigDataL . configClusterL . clusterHighestUsedPortL in case cs ^. portPoolL of [] -> if cs ^. hupL >= lastDrbdPort then throwError . ConfigurationError $ printf "The highest used port is greater than %s. Aborting." lastDrbdPort else return (cs ^. hupL + 1, hupL %~ (+1) $ cs) (p:ps) -> return (p, portPoolL .~ ps $ cs)) (return ()) return . MaybeForJSON $ maybePort -- | Adds a new port to the available port pool. addTcpUdpPort :: Int -> WConfdMonad Bool addTcpUdpPort port = let pL = csConfigDataL . configClusterL . clusterTcpudpPortPoolL f :: Monad m => ConfigState -> m ConfigState f = mapMOf pL (return . (port:) . filter (/= port)) in isJust <$> modifyConfigWithLock (const f) (return ()) -- | Set the instances' status to a given value. setInstanceStatus :: InstanceUUID -> MaybeForJSON AdminState -> MaybeForJSON Bool -> MaybeForJSON AdminStateSource -> WConfdMonad (MaybeForJSON Instance) setInstanceStatus iUuid m1 m2 m3 = do ct <- liftIO getClockTime let modifyInstance = maybe id (instAdminStateL .~) (unMaybeForJSON m1) . maybe id (instDisksActiveL .~) (unMaybeForJSON m2) . maybe id (instAdminStateSourceL .~) (unMaybeForJSON m3) reviseInstance = (instSerialL %~ (+1)) . (instMtimeL .~ ct) g :: Instance -> Instance g i = if modifyInstance i == i then i else reviseInstance . modifyInstance $ i iL = csConfigDataL . configInstancesL . alterContainerL (UTF8.fromString iUuid) f :: MonadError GanetiException m => StateT ConfigState m Instance f = get >>= (maybe (throwError . ConfigurationError $ printf "Could not find instance with UUID %s" iUuid) (liftM2 (>>) (modify . (iL .~) . Just) return . g) . (^. iL)) MaybeForJSON <$> modifyConfigAndReturnWithLock (const $ runStateT f) (return ()) -- | Sets the primary node of an existing instance setInstancePrimaryNode :: InstanceUUID -> NodeUUID -> WConfdMonad Bool setInstancePrimaryNode iUuid nUuid = isJust <$> modifyConfigWithLock (\_ -> mapMOf (csConfigDataL . configInstancesL . alterContainerL (UTF8.fromString iUuid)) (\mi -> case mi of Nothing -> throwError . ConfigurationError $ printf "Could not find instance with UUID %s" iUuid Just ist -> return . Just $ (instPrimaryNodeL .~ nUuid) ist)) (return ()) -- | The configuration is updated by the provided cluster updateCluster :: Cluster -> WConfdMonad (MaybeForJSON (Int, TimeAsDoubleJSON)) updateCluster cluster = do ct <- liftIO getClockTime r <- modifyConfigAndReturnWithLock (\_ cs -> do let currentCluster = configCluster . csConfigData $ cs if isIdentical ct cluster currentCluster then return ((serialOf currentCluster, mTimeOf currentCluster), cs) else do toError $ checkSerial cluster currentCluster let updateC = (clusterSerialL %~ (+1)) . (clusterMtimeL .~ ct) return ((serialOf cluster + 1, ct) , csConfigDataL . configClusterL .~ updateC cluster $ cs)) (return ()) return . MaybeForJSON $ fmap (_2 %~ TimeAsDoubleJSON) r -- | The configuration is updated by the provided node updateNode :: Node -> WConfdMonad (MaybeForJSON (Int, TimeAsDoubleJSON)) updateNode node = do ct <- liftIO getClockTime let nL = csConfigDataL . configNodesL updateC = (clusterSerialL %~ (+1)) . (clusterMtimeL .~ ct) r <- modifyConfigAndReturnWithLock (\_ -> updateConfigIfNecessary ct node (^. nL) (\cs -> do nC <- toError $ replaceIn ct node (cs ^. nL) return ((serialOf node + 1, ct), (nL .~ nC) . (csConfigDataL . configClusterL %~ updateC) $ cs))) (return ()) return . MaybeForJSON $ fmap (_2 %~ TimeAsDoubleJSON) r -- | The configuration is updated by the provided instance updateInstance :: Instance -> WConfdMonad (MaybeForJSON (Int, TimeAsDoubleJSON)) updateInstance inst = do ct <- liftIO getClockTime let iL = csConfigDataL . configInstancesL r <- modifyConfigAndReturnWithLock (\_ -> updateConfigIfNecessary ct inst (^. iL) (\cs -> do iC <- toError $ replaceIn ct inst (cs ^. iL) return ((serialOf inst + 1, ct), (iL .~ iC) cs))) (return ()) return . MaybeForJSON $ fmap (_2 %~ TimeAsDoubleJSON) r -- | The configuration is updated by the provided nodegroup updateNodeGroup :: NodeGroup -> WConfdMonad (MaybeForJSON (Int, TimeAsDoubleJSON)) updateNodeGroup ng = do ct <- liftIO getClockTime let ngL = csConfigDataL . configNodegroupsL r <- modifyConfigAndReturnWithLock (\_ -> updateConfigIfNecessary ct ng (^. ngL) (\cs -> do ngC <- toError $ replaceIn ct ng (cs ^. ngL) return ((serialOf ng + 1, ct), (ngL .~ ngC) cs))) (return ()) return . MaybeForJSON $ fmap (_2 %~ TimeAsDoubleJSON) r -- | The configuration is updated by the provided network updateNetwork :: Network -> WConfdMonad (MaybeForJSON (Int, TimeAsDoubleJSON)) updateNetwork net = do ct <- liftIO getClockTime let nL = csConfigDataL . configNetworksL r <- modifyConfigAndReturnWithLock (\_ -> updateConfigIfNecessary ct net (^. nL) (\cs -> do nC <- toError $ replaceIn ct net (cs ^. nL) return ((serialOf net + 1, ct), (nL .~ nC) cs))) (return ()) return . MaybeForJSON $ fmap (_2 %~ TimeAsDoubleJSON) r -- | The configuration is updated by the provided disk updateDisk :: Disk -> WConfdMonad (MaybeForJSON (Int, TimeAsDoubleJSON)) updateDisk disk = do ct <- liftIO getClockTime let dL = csConfigDataL . configDisksL r <- modifyConfigAndReturnWithLock (\_ -> updateConfigIfNecessary ct disk (^. dL) (\cs -> do dC <- toError $ replaceIn ct disk (cs ^. dL) return ((serialOf disk + 1, ct), (dL .~ dC) cs))) . T.releaseDRBDMinors . UTF8.fromString $ uuidOf disk return . MaybeForJSON $ fmap (_2 %~ TimeAsDoubleJSON) r -- * The list of functions exported to RPC. exportedFunctions :: [Name] exportedFunctions = [ 'addInstance , 'addInstanceDisk , 'addTcpUdpPort , 'allocatePort , 'attachInstanceDisk , 'detachInstanceDisk , 'removeInstance , 'removeInstanceDisk , 'setInstancePrimaryNode , 'setInstanceStatus , 'updateCluster , 'updateDisk , 'updateInstance , 'updateNetwork , 'updateNode , 'updateNodeGroup ] ganeti-3.1.0~rc2/src/Ganeti/WConfd/ConfigState.hs000064400000000000000000000056161476477700300215340ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Pure functions for manipulating the configuration state. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.ConfigState ( ConfigState , csConfigData , csConfigDataL , mkConfigState , bumpSerial , needsFullDist ) where import Data.Function (on) import System.Time (ClockTime(..)) import Ganeti.Config import Ganeti.Lens import Ganeti.Objects import Ganeti.Objects.Lens -- | In future this data type will include the current configuration -- ('ConfigData') and the last 'FStat' of its file. data ConfigState = ConfigState { csConfigData :: ConfigData } deriving (Eq, Show) $(makeCustomLenses ''ConfigState) -- | Creates a new configuration state. -- This method will expand as more fields are added to 'ConfigState'. mkConfigState :: ConfigData -> ConfigState mkConfigState = ConfigState bumpSerial :: (SerialNoObjectL a, TimeStampObjectL a) => ClockTime -> a -> a bumpSerial now = set mTimeL now . over serialL succ -- | Given two versions of the configuration, determine if its distribution -- needs to be fully committed before returning the corresponding call to -- WConfD. needsFullDist :: ConfigState -> ConfigState -> Bool needsFullDist = on (/=) (watched . csConfigData) where watched = (,,,,,,) <$> clusterCandidateCerts . configCluster <*> clusterMasterNode . configCluster <*> getMasterNodes <*> getMasterCandidates -- kvmd is running depending on the following: <*> clusterEnabledUserShutdown . configCluster <*> clusterEnabledHypervisors . configCluster <*> fmap nodeVmCapable . configNodes ganeti-3.1.0~rc2/src/Ganeti/WConfd/ConfigVerify.hs000064400000000000000000000124751476477700300217210ustar00rootroot00000000000000{-# LANGUAGE FlexibleContexts #-} {-| Implementation of functions specific to configuration management. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.ConfigVerify ( verifyConfig , verifyConfigErr ) where import Control.Monad.Except import Control.Monad (forM_) import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Foldable as F import qualified Data.Map as M import qualified Data.Set as S import Ganeti.Errors import Ganeti.JSON (GenericContainer(..), Container) import Ganeti.Objects import Ganeti.Types import Ganeti.Utils import Ganeti.Utils.Validate -- * Configuration checks -- | A helper function that returns the key set of a container. keysSet :: (Ord k) => GenericContainer k v -> S.Set k keysSet = M.keysSet . fromContainer -- | Checks that all objects are indexed by their proper UUID. checkUUIDKeys :: (UuidObject a, Show a) => String -> Container a -> ValidationMonad () checkUUIDKeys what = mapM_ check . M.toList . fromContainer where check (uuid, x) = reportIf (uuid /= UTF8.fromString (uuidOf x)) $ what ++ " '" ++ show x ++ "' is indexed by wrong UUID '" ++ UTF8.toString uuid ++ "'" -- | Checks that all linked UUID of given objects exist. checkUUIDRefs :: (UuidObject a, Show a, F.Foldable f) => String -> String -> (a -> [String]) -> f a -> Container b -> ValidationMonad () checkUUIDRefs whatObj whatTarget linkf xs targets = F.mapM_ check xs where uuids = keysSet targets check x = forM_ (linkf x) $ \uuid -> reportIf (not $ S.member (UTF8.fromString uuid) uuids) $ whatObj ++ " '" ++ show x ++ "' references a non-existing " ++ whatTarget ++ " UUID '" ++ uuid ++ "'" -- | Checks consistency of a given configuration. -- -- TODO: Currently this implements only some very basic checks. -- Evenually all checks from Python ConfigWriter need to be moved here -- (see issue #759). verifyConfig :: ConfigData -> ValidationMonad () verifyConfig cd = do let cluster = configCluster cd nodes = configNodes cd nodegroups = configNodegroups cd instances = configInstances cd networks = configNetworks cd disks = configDisks cd -- global cluster checks let enabledHvs = clusterEnabledHypervisors cluster hvParams = clusterHvparams cluster reportIf (null enabledHvs) "enabled hypervisors list doesn't have any entries" -- we don't need to check for invalid HVS as they would fail to parse let missingHvp = S.fromList enabledHvs S.\\ keysSet hvParams reportIf (not $ S.null missingHvp) $ "hypervisor parameters missing for the enabled hypervisor(s) " ++ (commaJoin . map hypervisorToRaw . S.toList $ missingHvp) let enabledDiskTemplates = clusterEnabledDiskTemplates cluster reportIf (null enabledDiskTemplates) "enabled disk templates list doesn't have any entries" -- we don't need to check for invalid templates as they wouldn't parse let masterNodeName = clusterMasterNode cluster reportIf (not $ UTF8.fromString masterNodeName `S.member` keysSet (configNodes cd)) $ "cluster has invalid primary node " ++ masterNodeName -- UUIDs checkUUIDKeys "node" nodes checkUUIDKeys "nodegroup" nodegroups checkUUIDKeys "instances" instances checkUUIDKeys "network" networks checkUUIDKeys "disk" disks -- UUID references checkUUIDRefs "node" "nodegroup" (return . nodeGroup) nodes nodegroups checkUUIDRefs "instance" "primary node" (maybe [] return . instPrimaryNode) instances nodes checkUUIDRefs "instance" "disks" instDisks instances disks -- | Checks consistency of a given configuration. -- If there is an error, throw 'ConfigVerifyError'. verifyConfigErr :: (MonadError GanetiException m) => ConfigData -> m () verifyConfigErr cd = case runValidate $ verifyConfig cd of (_, []) -> return () (_, es) -> throwError $ ConfigVerifyError "Validation failed" es ganeti-3.1.0~rc2/src/Ganeti/WConfd/ConfigWriter.hs000064400000000000000000000224151476477700300217240ustar00rootroot00000000000000{-# LANGUAGE Rank2Types, FlexibleContexts #-} {-| Implementation of functions specific to configuration management. -} {- Copyright (C) 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.ConfigWriter ( loadConfigFromFile , readConfig , writeConfig , saveConfigAsyncTask , distMCsAsyncTask , distSSConfAsyncTask ) where import Control.Monad.Base import Control.Monad.Except import qualified Control.Monad.State.Strict as S import Control.Monad.Trans.Control import Control.Monad.Trans (lift) import Control.Monad (liftM, unless, (>=>)) import Data.Monoid import qualified Data.Set as Set import Ganeti.BasicTypes import Ganeti.Errors import Ganeti.Config import Ganeti.Logging import Ganeti.Objects import Ganeti.Rpc import Ganeti.Runtime import Ganeti.Utils import Ganeti.Utils.Atomic import Ganeti.Utils.AsyncWorker import Ganeti.WConfd.ConfigState import Ganeti.WConfd.Monad import Ganeti.WConfd.Ssconf -- | From a distribution target get a predicate on nodes whether it -- should be distributed to this node. targetToPredicate :: DistributionTarget -> Node -> Bool targetToPredicate Everywhere = const True targetToPredicate (ToGroups gs) = (`Set.member` gs) . nodeGroup -- | Loads the configuration from the file, if it hasn't been loaded yet. -- The function is internal and isn't thread safe. loadConfigFromFile :: FilePath -> ResultG (ConfigData, FStat) loadConfigFromFile path = withLockedFile path $ \_ -> do stat <- liftBase $ getFStat path cd <- mkResultT (loadConfig path) return (cd, stat) -- | Writes the current configuration to the file. The function isn't thread -- safe. -- Neither distributes the configuration (to nodes and ssconf) nor -- updates the serial number. writeConfigToFile :: (MonadBase IO m, MonadError GanetiException m, MonadLog m) => ConfigData -> FilePath -> FStat -> m FStat writeConfigToFile cfg path oldstat = do logDebug $ "Async. config. writer: Commencing write\ \ serial no " ++ show (serialOf cfg) r <- toErrorBase $ atomicUpdateLockedFile_ path oldstat doWrite logDebug "Async. config. writer: written" return r where doWrite fname fh = do setOwnerAndGroupFromNames fname GanetiWConfd (DaemonGroup GanetiConfd) setOwnerWGroupR fname saveConfig fh cfg -- Reads the current configuration state in the 'WConfdMonad'. readConfig :: WConfdMonad ConfigData readConfig = csConfigData <$> readConfigState -- Replaces the current configuration state within the 'WConfdMonad'. writeConfig :: ConfigData -> WConfdMonad () writeConfig cd = modifyConfigState $ const ((), mkConfigState cd) -- * Asynchronous tasks -- | Runs the given action on success, or logs an error on failure. finishOrLog :: (Show e, MonadLog m) => Priority -> String -> (a -> m ()) -> GenericResult e a -> m () finishOrLog logPrio logPrefix = genericResult (logAt logPrio . (++) (logPrefix ++ ": ") . show) -- | Creates a stateless asynchronous task that handles errors in its actions. mkStatelessAsyncTask :: (MonadBaseControl IO m, MonadLog m, Show e, Monoid i) => Priority -> String -> (i -> ResultT e m ()) -> m (AsyncWorker i ()) mkStatelessAsyncTask logPrio logPrefix action = mkAsyncWorker $ runResultT . action >=> finishOrLog logPrio logPrefix return -- | Creates an asynchronous task that handles errors in its actions. -- If an error occurs, it's logged and the internal state remains unchanged. mkStatefulAsyncTask :: (MonadBaseControl IO m, MonadLog m, Show e, Monoid i) => Priority -> String -> s -> (s -> i -> ResultT e m s) -> m (AsyncWorker i ()) mkStatefulAsyncTask logPrio logPrefix start action = flip S.evalStateT start . mkAsyncWorker $ \i -> S.get >>= lift . runResultT . flip action i >>= finishOrLog logPrio logPrefix S.put -- put on success -- | Construct an asynchronous worker whose action is to save the -- configuration to the master file. -- The worker's action reads the configuration using the given @IO@ action -- and uses 'FStat' to check if the configuration hasn't been modified by -- another process. -- -- If 'Any' of the input requests is true, given additional worker -- will be executed synchronously after sucessfully writing the configuration -- file. Otherwise, they'll be just triggered asynchronously. saveConfigAsyncTask :: FilePath -- ^ Path to the config file -> FStat -- ^ The initial state of the config. file -> IO ConfigState -- ^ An action to read the current config -> [AsyncWorker DistributionTarget ()] -- ^ Workers to be triggered afterwards -> ResultG (AsyncWorker (Any, DistributionTarget) ()) saveConfigAsyncTask fpath fstat cdRef workers = lift . mkStatefulAsyncTask EMERGENCY "Can't write the master configuration file" fstat $ \oldstat (Any flush, target) -> do cd <- liftBase (csConfigData `liftM` cdRef) writeConfigToFile cd fpath oldstat <* if flush then logDebug "Running distribution synchronously" >> triggerAndWaitMany target workers else logDebug "Running distribution asynchronously" >> mapM (trigger target) workers -- | Performs a RPC call on the given list of nodes and logs any failures. -- If any of the calls fails, fail the computation with 'failError'. execRpcCallAndLog :: (Rpc a b) => [Node] -> a -> ResultG () execRpcCallAndLog nodes req = do rs <- liftIO $ executeRpcCall nodes req es <- logRpcErrors rs unless (null es) $ failError "At least one of the RPC calls failed" -- | Construct an asynchronous worker whose action is to distribute the -- configuration to master candidates. distMCsAsyncTask :: RuntimeEnts -> FilePath -- ^ Path to the config file -> IO ConfigState -- ^ An action to read the current config -> ResultG (AsyncWorker DistributionTarget ()) distMCsAsyncTask ents cpath cdRef = lift . mkStatelessAsyncTask ERROR "Can't distribute the configuration\ \ to master candidates" $ \target -> do cd <- liftBase (csConfigData <$> cdRef) :: ResultG ConfigData logDebug $ "Distributing the configuration to master candidates,\ \ serial no " ++ show (serialOf cd) ++ ", " ++ show target fupload <- prepareRpcCallUploadFile ents cpath execRpcCallAndLog (filter (targetToPredicate target) $ getMasterCandidates cd) fupload logDebug "Successfully finished distributing the configuration" -- | Construct an asynchronous worker whose action is to construct SSConf -- and distribute it to master candidates. -- The worker's action reads the configuration using the given @IO@ action, -- computes the current SSConf, compares it to the previous version, and -- if different, distributes it. distSSConfAsyncTask :: IO ConfigState -- ^ An action to read the current config -> ResultG (AsyncWorker DistributionTarget ()) distSSConfAsyncTask cdRef = lift . mkStatefulAsyncTask ERROR "Can't distribute Ssconf" emptySSConf $ \oldssc target -> do cd <- liftBase (csConfigData <$> cdRef) :: ResultG ConfigData let ssc = mkSSConf cd if oldssc == ssc then logDebug "SSConf unchanged, not distributing" else do logDebug $ "Starting the distribution of SSConf\ \ serial no " ++ show (serialOf cd) ++ ", " ++ show target execRpcCallAndLog (filter (targetToPredicate target) $ getOnlineNodes cd) (RpcCallWriteSsconfFiles ssc) logDebug "Successfully finished distributing SSConf" return ssc ganeti-3.1.0~rc2/src/Ganeti/WConfd/Core.hs000064400000000000000000000413671476477700300202210ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| The Ganeti WConfd core functions. This module defines all the functions that WConfD exports for RPC calls. They are in a separate module so that in a later stage, TemplateHaskell can generate, e.g., the python interface for those. -} {- Copyright (C) 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.Core where import Control.Arrow ((&&&)) import Control.Concurrent (myThreadId) import Control.Lens.Setter (set) import Control.Monad (liftM, unless) import qualified Data.Map as M import qualified Data.Set as S import Language.Haskell.TH (Name) import System.Posix.Process (getProcessID) import qualified System.Random as Rand import Ganeti.BasicTypes import qualified Ganeti.Constants as C import qualified Ganeti.JSON as J import qualified Ganeti.Locking.Allocation as L import Ganeti.Logging (logDebug, logWarning) import Ganeti.Locking.Locks ( GanetiLocks(ConfigLock, BGL) , LockLevel(LevelConfig) , lockLevel, LockLevel , ClientType(ClientOther), ClientId(..) ) import qualified Ganeti.Locking.Waiting as LW import Ganeti.Objects (ConfigData, DRBDSecret, LogicalVolume, Ip4Address) import Ganeti.Objects.Lens (configClusterL, clusterMasterNodeL) import Ganeti.WConfd.ConfigState (csConfigDataL) import qualified Ganeti.WConfd.ConfigVerify as V import Ganeti.WConfd.DeathDetection (cleanupLocks) import Ganeti.WConfd.Language import Ganeti.WConfd.Monad import qualified Ganeti.WConfd.TempRes as T import qualified Ganeti.WConfd.ConfigModifications as CM import qualified Ganeti.WConfd.ConfigWriter as CW -- * Functions available to the RPC module -- Just a test function echo :: String -> WConfdMonad String echo = return -- ** Configuration related functions checkConfigLock :: ClientId -> L.OwnerState -> WConfdMonad () checkConfigLock cid state = do la <- readLockAllocation unless (L.holdsLock cid ConfigLock state la) . failError $ "Requested lock " ++ show state ++ " on the configuration missing" -- | Read the configuration. readConfig :: WConfdMonad ConfigData readConfig = CW.readConfig -- | Write the configuration, checking that an exclusive lock is held. -- If not, the call fails. writeConfig :: ClientId -> ConfigData -> WConfdMonad () writeConfig ident cdata = do checkConfigLock ident L.OwnExclusive -- V.verifyConfigErr cdata CW.writeConfig cdata -- | Explicitly run verification of the configuration. -- The caller doesn't need to hold the configuration lock. verifyConfig :: WConfdMonad () verifyConfig = CW.readConfig >>= V.verifyConfigErr -- *** Locks on the configuration (only transitional, will be removed later) -- | Tries to acquire 'ConfigLock' for the client. -- If the second parameter is set to 'True', the lock is acquired in -- shared mode. -- -- If the lock was successfully acquired, returns the current configuration -- state. lockConfig :: ClientId -> Bool -- ^ set to 'True' if the lock should be shared -> WConfdMonad (J.MaybeForJSON ConfigData) lockConfig cid shared = do let (reqtype, owntype) = if shared then (ReqShared, L.OwnShared) else (ReqExclusive, L.OwnExclusive) la <- readLockAllocation if L.holdsLock cid ConfigLock owntype la then do -- warn if we already have the lock, but continue (with no-op) -- on the locks logWarning $ "Client " ++ show cid ++ " asked to lock the config" ++ " while owning the lock" liftM (J.MaybeForJSON . Just) CW.readConfig else do waiting <- tryUpdateLocks cid [(ConfigLock, reqtype)] liftM J.MaybeForJSON $ case waiting of [] -> liftM Just CW.readConfig _ -> return Nothing -- | Release the config lock, if the client currently holds it. unlockConfig :: ClientId -> WConfdMonad () unlockConfig cid = freeLocksLevel cid LevelConfig -- | Write the configuration, if the config lock is held exclusively, -- and release the config lock. It the caller does not have the config -- lock, return False. writeConfigAndUnlock :: ClientId -> ConfigData -> WConfdMonad Bool writeConfigAndUnlock cid cdata = do la <- readLockAllocation if L.holdsLock cid ConfigLock L.OwnExclusive la then do CW.writeConfig cdata unlockConfig cid return True else do logWarning $ show cid ++ " tried writeConfigAndUnlock without owning" ++ " the config lock" return False -- | Force the distribution of configuration without actually modifying it. -- It is not necessary to hold a lock for this operation. flushConfig :: WConfdMonad () flushConfig = forceConfigStateDistribution Everywhere -- | Force the distribution of configuration to a given group without actually -- modifying it. It is not necessary to hold a lock for this operation. flushConfigGroup :: String -> WConfdMonad () flushConfigGroup = forceConfigStateDistribution . ToGroups . S.singleton -- ** Temporary reservations related functions dropAllReservations :: ClientId -> WConfdMonad () dropAllReservations cid = modifyTempResState (const $ T.dropAllReservations cid) -- *** DRBD computeDRBDMap :: WConfdMonad T.DRBDMap computeDRBDMap = uncurry T.computeDRBDMap =<< readTempResState -- Allocate a drbd minor. -- -- The free minor will be automatically computed from the existing devices. -- A node can not be given multiple times. -- The result is the list of minors, in the same order as the passed nodes. allocateDRBDMinor :: T.DiskUUID -> [T.NodeUUID] -> WConfdMonad [T.DRBDMinor] allocateDRBDMinor disk nodes = modifyTempResStateErr (\cfg -> T.allocateDRBDMinor cfg disk nodes) -- Release temporary drbd minors allocated for a given disk using -- 'allocateDRBDMinor'. -- -- This should be called on the error paths, on the success paths -- it's automatically called by the ConfigWriter add and update -- functions. releaseDRBDMinors :: T.DiskUUID -> WConfdMonad () releaseDRBDMinors disk = modifyTempResState (const $ T.releaseDRBDMinors disk) -- *** MACs -- Randomly generate a MAC for an instance and reserve it for -- a given client. generateMAC :: ClientId -> J.MaybeForJSON T.NetworkUUID -> WConfdMonad T.MAC generateMAC cid (J.MaybeForJSON netId) = do g <- liftIO Rand.newStdGen modifyTempResStateErr $ T.generateMAC g cid netId -- Reserves a MAC for an instance in the list of temporary reservations. reserveMAC :: ClientId -> T.MAC -> WConfdMonad () reserveMAC = (modifyTempResStateErr .) . T.reserveMAC -- *** DRBDSecrets -- Randomly generate a DRBDSecret for an instance and reserves it for -- a given client. generateDRBDSecret :: ClientId -> WConfdMonad DRBDSecret generateDRBDSecret cid = do g <- liftIO Rand.newStdGen modifyTempResStateErr $ T.generateDRBDSecret g cid -- *** LVs reserveLV :: ClientId -> LogicalVolume -> WConfdMonad () reserveLV jobId lv = modifyTempResStateErr $ T.reserveLV jobId lv -- *** IPv4s -- | Reserve a given IPv4 address for use by an instance. reserveIp :: ClientId -> T.NetworkUUID -> Ip4Address -> Bool -> WConfdMonad () reserveIp = (((modifyTempResStateErr .) .) .) . T.reserveIp -- | Give a specific IP address back to an IP pool. -- The IP address is returned to the IP pool designated by network id -- and marked as reserved. releaseIp :: ClientId -> T.NetworkUUID -> Ip4Address -> WConfdMonad () releaseIp = (((modifyTempResStateErr .) const .) .) . T.releaseIp -- Find a free IPv4 address for an instance and reserve it. generateIp :: ClientId -> T.NetworkUUID -> WConfdMonad Ip4Address generateIp = (modifyTempResStateErr .) . T.generateIp -- | Commit all reserved/released IP address to an IP pool. -- The IP addresses are taken from the network's IP pool and marked as -- reserved/free for instances. -- -- Note that the reservations are kept, they are supposed to be cleaned -- when a job finishes. commitTemporaryIps :: ClientId -> WConfdMonad () commitTemporaryIps = modifyConfigDataErr_ . T.commitReservedIps -- | Immediately release an IP address, without using the reservations pool. commitReleaseTemporaryIp :: T.NetworkUUID -> Ip4Address -> WConfdMonad () commitReleaseTemporaryIp net_uuid addr = modifyConfigDataErr_ (const $ T.commitReleaseIp net_uuid addr) -- | List all IP reservations for the current client. -- -- This function won't be needed once the corresponding calls are moved to -- WConfd. listReservedIps :: ClientId -> WConfdMonad [T.IPv4Reservation] listReservedIps jobId = liftM (S.toList . T.listReservedIps jobId . snd) readTempResState -- ** Locking related functions -- | List the locks of a given owner (i.e., a job-id lockfile pair). listLocks :: ClientId -> WConfdMonad [(GanetiLocks, L.OwnerState)] listLocks cid = liftM (M.toList . L.listLocks cid) readLockAllocation -- | List all active locks. listAllLocks :: WConfdMonad [GanetiLocks] listAllLocks = liftM L.listAllLocks readLockAllocation -- | List all active locks with their owners. listAllLocksOwners :: WConfdMonad [(GanetiLocks, [(ClientId, L.OwnerState)])] listAllLocksOwners = liftM L.listAllLocksOwners readLockAllocation -- | Get full information of the lock waiting status, i.e., provide -- the information about all locks owners and all pending requests. listLocksWaitingStatus :: WConfdMonad ( [(GanetiLocks, [(ClientId, L.OwnerState)])] , [(Integer, ClientId, [L.LockRequest GanetiLocks])] ) listLocksWaitingStatus = liftM ( (L.listAllLocksOwners . LW.getAllocation) &&& (S.toList . LW.getPendingRequests) ) readLockWaiting -- | Try to update the locks of a given owner (i.e., a job-id lockfile pair). -- This function always returns immediately. If the lock update was possible, -- the empty list is returned; otherwise, the lock status is left completly -- unchanged, and the return value is the list of jobs which need to release -- some locks before this request can succeed. tryUpdateLocks :: ClientId -> GanetiLockRequest -> WConfdMonad [ClientId] tryUpdateLocks cid req = liftM S.toList . (>>= toErrorStr) $ modifyLockWaiting (LW.updateLocks cid (fromGanetiLockRequest req)) -- | Try to update the locks of a given owner and make that a pending -- request if not immediately possible. updateLocksWaiting :: ClientId -> Integer -> GanetiLockRequest -> WConfdMonad [ClientId] updateLocksWaiting cid prio req = liftM S.toList . (>>= toErrorStr) . modifyLockWaiting $ LW.safeUpdateLocksWaiting prio cid (fromGanetiLockRequest req) -- | Tell whether a given owner has pending requests. hasPendingRequest :: ClientId -> WConfdMonad Bool hasPendingRequest cid = liftM (LW.hasPendingRequest cid) readLockWaiting -- | Free all locks of a given owner (i.e., a job-id lockfile pair). freeLocks :: ClientId -> WConfdMonad () freeLocks cid = modifyLockWaiting_ $ LW.releaseResources cid -- | Free all locks of a given owner (i.e., a job-id lockfile pair) -- of a given level in the Ganeti sense (e.g., "cluster", "node"). freeLocksLevel :: ClientId -> LockLevel -> WConfdMonad () freeLocksLevel cid level = modifyLockWaiting_ $ LW.freeLocksPredicate ((==) level . lockLevel) cid -- | Downgrade all locks of the given level to shared. downGradeLocksLevel :: ClientId -> LockLevel -> WConfdMonad () downGradeLocksLevel cid level = modifyLockWaiting_ $ LW.downGradeLocksPredicate ((==) level . lockLevel) cid -- | Intersect the possesed locks of an owner with a given set. intersectLocks :: ClientId -> [GanetiLocks] -> WConfdMonad () intersectLocks cid locks = modifyLockWaiting_ $ LW.intersectLocks locks cid -- | Opportunistically allocate locks for a given owner. opportunisticLockUnion :: ClientId -> [(GanetiLocks, L.OwnerState)] -> WConfdMonad [GanetiLocks] opportunisticLockUnion cid req = modifyLockWaiting $ LW.opportunisticLockUnion cid req -- | Opprtunistially allocate locks for a given owner, requesting a -- certain minimum of success. guardedOpportunisticLockUnion :: Int -> ClientId -> [(GanetiLocks, L.OwnerState)] -> WConfdMonad [GanetiLocks] guardedOpportunisticLockUnion count cid req = modifyLockWaiting $ LW.guardedOpportunisticLockUnion count cid req -- * Prepareation for cluster destruction -- | Prepare daemon for cluster destruction. This consists of -- verifying that the requester owns the BGL exclusively, transfering the BGL -- to WConfD itself, and modifying the configuration so that no -- node is the master any more. Note that, since we own the BGL exclusively, -- we can safely modify the configuration, as no other process can request -- changes. prepareClusterDestruction :: ClientId -> WConfdMonad () prepareClusterDestruction cid = do la <- readLockAllocation unless (L.holdsLock cid BGL L.OwnExclusive la) . failError $ "Cluster destruction requested without owning BGL exclusively" logDebug $ "preparing cluster destruction as requested by " ++ show cid -- transfer BGL to ourselfs. The do this, by adding a super-priority waiting -- request and then releasing the BGL of the requestor. dh <- daemonHandle pid <- liftIO getProcessID tid <- liftIO myThreadId let mycid = ClientId { ciIdentifier = ClientOther $ "wconfd-" ++ show tid , ciLockFile = dhLivelock dh , ciPid = pid } _ <- modifyLockWaiting $ LW.updateLocksWaiting (fromIntegral C.opPrioHighest - 1) mycid [L.requestExclusive BGL] _ <- modifyLockWaiting $ LW.updateLocks cid [L.requestRelease BGL] -- To avoid beeing restarted we change the configuration to a no-master -- state. modifyConfigState $ (,) () . set (csConfigDataL . configClusterL . clusterMasterNodeL) "" -- * The list of all functions exported to RPC. exportedFunctions :: [Name] exportedFunctions = [ 'echo , 'cleanupLocks , 'prepareClusterDestruction -- config , 'readConfig , 'writeConfig , 'verifyConfig , 'lockConfig , 'unlockConfig , 'writeConfigAndUnlock , 'flushConfig , 'flushConfigGroup -- temporary reservations (common) , 'dropAllReservations -- DRBD , 'computeDRBDMap , 'allocateDRBDMinor , 'releaseDRBDMinors -- MACs , 'reserveMAC , 'generateMAC -- DRBD secrets , 'generateDRBDSecret -- LVs , 'reserveLV -- IPv4s , 'reserveIp , 'releaseIp , 'generateIp , 'commitTemporaryIps , 'commitReleaseTemporaryIp , 'listReservedIps -- locking , 'listLocks , 'listAllLocks , 'listAllLocksOwners , 'listLocksWaitingStatus , 'tryUpdateLocks , 'updateLocksWaiting , 'freeLocks , 'freeLocksLevel , 'downGradeLocksLevel , 'intersectLocks , 'opportunisticLockUnion , 'guardedOpportunisticLockUnion , 'hasPendingRequest ] ++ CM.exportedFunctions ganeti-3.1.0~rc2/src/Ganeti/WConfd/DeathDetection.hs000064400000000000000000000076471476477700300222200ustar00rootroot00000000000000{-| Utility function for detecting the death of a job holding resources To clean up resources owned by jobs that die for some reason, we need to detect whether a job is still alive. As we have no control over PID reuse, our approach is that each requester for a resource has to provide a file where it owns an exclusive lock on. The kernel will make sure the lock is removed if the process dies. We can probe for such a lock by requesting a shared lock on the file. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.DeathDetection ( cleanupLocksTask , cleanupLocks ) where import Control.Concurrent (threadDelay) import qualified Control.Exception as E import Control.Monad import System.Directory (removeFile) import Ganeti.BasicTypes import qualified Ganeti.Constants as C import qualified Ganeti.Locking.Allocation as L import Ganeti.Locking.Locks (ClientId(..)) import Ganeti.Logging.Lifted (logDebug, logInfo) import Ganeti.Utils.Livelock import Ganeti.WConfd.Monad import Ganeti.WConfd.Persistent -- | Interval to run clean-up tasks in microseconds cleanupInterval :: Int cleanupInterval = C.wconfdDeathdetectionIntervall * 1000000 -- | Go through all owners once and clean them up, if they're dead. cleanupLocks :: WConfdMonad () cleanupLocks = do owners <- liftM L.lockOwners readLockAllocation mylivelock <- liftM dhLivelock daemonHandle logDebug $ "Current lock owners: " ++ show owners let cleanupIfDead owner = do let fpath = ciLockFile owner died <- if fpath == mylivelock then return False else liftIO (isDead fpath) when died $ do logInfo $ show owner ++ " died, releasing locks and reservations" persCleanup persistentTempRes owner persCleanup persistentLocks owner _ <- liftIO . E.try $ removeFile fpath :: WConfdMonad (Either IOError ()) return () mapM_ cleanupIfDead owners -- | Thread periodically cleaning up locks of lock owners that died. cleanupLocksTask :: WConfdMonadInt () cleanupLocksTask = forever . runResultT $ do logDebug "Death detection timer fired" cleanupLocks remainingFiles <- liftIO listLiveLocks mylivelock <- liftM dhLivelock daemonHandle logDebug $ "Livelockfiles remaining: " ++ show remainingFiles let cleanupStaleIfDead fpath = do died <- if fpath == mylivelock then return False else liftIO (isDead fpath) when died $ do logInfo $ "Cleaning up stale file " ++ fpath _ <- liftIO . E.try $ removeFile fpath :: WConfdMonad (Either IOError ()) return () mapM_ cleanupStaleIfDead remainingFiles liftIO $ threadDelay cleanupInterval ganeti-3.1.0~rc2/src/Ganeti/WConfd/Language.hs000064400000000000000000000057351476477700300210530ustar00rootroot00000000000000{-| Function related to serialisation of WConfD requests -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.Language ( LockRequestType(..) , GanetiLockRequest , fromGanetiLockRequest ) where import qualified Text.JSON as J import Ganeti.Locking.Allocation import Ganeti.Locking.Locks (GanetiLocks) -- * Serialisation related to locking -- | Operation to be carried out on a lock (request exclusive/shared ownership, -- or release). data LockRequestType = ReqExclusive | ReqShared | ReqRelease deriving (Eq, Show) instance J.JSON LockRequestType where showJSON ReqExclusive = J.showJSON "exclusive" showJSON ReqShared = J.showJSON "shared" showJSON ReqRelease = J.showJSON "release" readJSON (J.JSString x) = let s = J.fromJSString x in case s of "exclusive" -> J.Ok ReqExclusive "shared" -> J.Ok ReqShared "release" -> J.Ok ReqRelease _ -> J.Error $ "Unknown lock update request " ++ s readJSON _ = J.Error "Update requests need to be strings" -- | The type describing how lock update requests are passed over the wire. type GanetiLockRequest = [(GanetiLocks, LockRequestType)] -- | Transform a Lock LockReqeustType pair into a LockRequest. toLockRequest :: (GanetiLocks, LockRequestType) -> LockRequest GanetiLocks toLockRequest (a, ReqExclusive) = requestExclusive a toLockRequest (a, ReqShared) = requestShared a toLockRequest (a, ReqRelease) = requestRelease a -- | From a GanetiLockRequest obtain a list of -- Ganeti.Lock.Allocation.LockRequest, suitable to updateLocks. fromGanetiLockRequest :: GanetiLockRequest -> [LockRequest GanetiLocks] fromGanetiLockRequest = map toLockRequest ganeti-3.1.0~rc2/src/Ganeti/WConfd/Monad.hs000064400000000000000000000437061476477700300203660ustar00rootroot00000000000000{-# LANGUAGE MultiParamTypeClasses, TypeFamilies, GeneralizedNewtypeDeriving, TemplateHaskell, CPP, UndecidableInstances #-} {-| All RPC calls are run within this monad. It encapsulates: * IO operations, * failures, * working with the daemon state. Code that is specific either to the configuration or the lock management, should go into their corresponding dedicated modules. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.Monad ( DaemonHandle , dhConfigPath , dhLivelock , mkDaemonHandle , WConfdMonadInt , runWConfdMonadInt , WConfdMonad , daemonHandle , modifyConfigState , modifyConfigStateWithImmediate , forceConfigStateDistribution , readConfigState , modifyConfigDataErr_ , modifyConfigAndReturnWithLock , modifyConfigWithLock , modifyLockWaiting , modifyLockWaiting_ , readLockWaiting , readLockAllocation , modifyTempResState , modifyTempResStateErr , readTempResState , DistributionTarget(..) ) where import Control.Arrow ((&&&), second) import Control.Concurrent (forkIO, myThreadId) import Control.Exception.Lifted (bracket) import Control.Monad import Control.Monad.Base import Control.Monad.Reader import Control.Monad.State import Control.Monad.Trans.Control import Data.Functor.Identity import Data.IORef.Lifted import Data.Monoid (Any(..)) import qualified Data.Semigroup as Sem import qualified Data.Set as S import Data.Tuple (swap) import System.Posix.Process (getProcessID) import System.Time (getClockTime, ClockTime) import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Errors import Ganeti.JQueue (notifyJob) import Ganeti.Lens import qualified Ganeti.Locking.Allocation as LA import Ganeti.Locking.Locks import qualified Ganeti.Locking.Waiting as LW import Ganeti.Logging import Ganeti.Logging.WriterLog import Ganeti.Objects (ConfigData) import Ganeti.Utils.AsyncWorker import Ganeti.Utils.IORef import Ganeti.Utils.Livelock (Livelock) import Ganeti.WConfd.ConfigState import Ganeti.WConfd.TempRes -- * Monoid of the locations to flush the configuration to -- | Data type describing where the configuration has to be distributed to. data DistributionTarget = Everywhere | ToGroups (S.Set String) deriving Show instance Sem.Semigroup DistributionTarget where Everywhere <> _ = Everywhere _ <> Everywhere = Everywhere (ToGroups a) <> (ToGroups b) = ToGroups (a `S.union` b) instance Monoid DistributionTarget where mempty = ToGroups S.empty mappend = (Sem.<>) -- * Pure data types used in the monad -- | The state of the daemon, capturing both the configuration state and the -- locking state. data DaemonState = DaemonState { dsConfigState :: ConfigState , dsLockWaiting :: GanetiLockWaiting , dsTempRes :: TempResState } $(makeCustomLenses ''DaemonState) data DaemonHandle = DaemonHandle { dhDaemonState :: IORef DaemonState -- ^ The current state of the daemon , dhConfigPath :: FilePath -- ^ The configuration file path -- all static information that doesn't change during the life-time of the -- daemon should go here; -- all IDs of threads that do asynchronous work should probably also go here , dhSaveConfigWorker :: AsyncWorker (Any, DistributionTarget) () , dhSaveLocksWorker :: AsyncWorker () () , dhSaveTempResWorker :: AsyncWorker () () , dhLivelock :: Livelock } mkDaemonHandle :: FilePath -> ConfigState -> GanetiLockWaiting -> TempResState -> (IO ConfigState -> [AsyncWorker DistributionTarget ()] -> ResultG (AsyncWorker (Any, DistributionTarget) ())) -- ^ A function that creates a worker that asynchronously -- saves the configuration to the master file. -> (IO ConfigState -> ResultG (AsyncWorker DistributionTarget ())) -- ^ A function that creates a worker that asynchronously -- distributes the configuration to master candidates -> (IO ConfigState -> ResultG (AsyncWorker DistributionTarget ())) -- ^ A function that creates a worker that asynchronously -- distributes SSConf to nodes -> (IO GanetiLockWaiting -> ResultG (AsyncWorker () ())) -- ^ A function that creates a worker that asynchronously -- saves the lock allocation state. -> (IO TempResState -> ResultG (AsyncWorker () ())) -- ^ A function that creates a worker that asynchronously -- saves the temporary reservations state. -> Livelock -> ResultG DaemonHandle mkDaemonHandle cpath cstat lstat trstat saveWorkerFn distMCsWorkerFn distSSConfWorkerFn saveLockWorkerFn saveTempResWorkerFn livelock = do ds <- newIORef $ DaemonState cstat lstat trstat let readConfigIO = dsConfigState `liftM` readIORef ds :: IO ConfigState ssconfWorker <- distSSConfWorkerFn readConfigIO distMCsWorker <- distMCsWorkerFn readConfigIO saveWorker <- saveWorkerFn readConfigIO [ distMCsWorker , ssconfWorker ] saveLockWorker <- saveLockWorkerFn $ dsLockWaiting `liftM` readIORef ds saveTempResWorker <- saveTempResWorkerFn $ dsTempRes `liftM` readIORef ds return $ DaemonHandle ds cpath saveWorker saveLockWorker saveTempResWorker livelock -- * The monad and its instances -- | A type alias for easier referring to the actual content of the monad -- when implementing its instances. type WConfdMonadIntType = ReaderT DaemonHandle IO -- | The internal part of the monad without error handling. newtype WConfdMonadInt a = WConfdMonadInt { getWConfdMonadInt :: WConfdMonadIntType a } deriving (Functor, Applicative, Monad, MonadIO, MonadBase IO, MonadLog) instance MonadBaseControl IO WConfdMonadInt where #if MIN_VERSION_monad_control(1,0,0) -- Needs Undecidable instances type StM WConfdMonadInt b = StM WConfdMonadIntType b liftBaseWith f = WConfdMonadInt $ liftBaseWith $ \r -> f (r . getWConfdMonadInt) restoreM = WConfdMonadInt . restoreM #else newtype StM WConfdMonadInt b = StMWConfdMonadInt { runStMWConfdMonadInt :: StM WConfdMonadIntType b } liftBaseWith f = WConfdMonadInt . liftBaseWith $ \r -> f (liftM StMWConfdMonadInt . r . getWConfdMonadInt) restoreM = WConfdMonadInt . restoreM . runStMWConfdMonadInt #endif -- | Runs the internal part of the WConfdMonad monad on a given daemon -- handle. runWConfdMonadInt :: WConfdMonadInt a -> DaemonHandle -> IO a runWConfdMonadInt (WConfdMonadInt k) = runReaderT k -- | The complete monad with error handling. type WConfdMonad = ResultT GanetiException WConfdMonadInt -- | A pure monad that logs and reports errors used for atomic modifications. type AtomicModifyMonad a = ResultT GanetiException WriterLog a -- * Basic functions in the monad -- | Returns the daemon handle. daemonHandle :: WConfdMonad DaemonHandle daemonHandle = lift . WConfdMonadInt $ ask -- | Returns the current configuration, given a handle readConfigState :: WConfdMonad ConfigState readConfigState = liftM dsConfigState . readIORef . dhDaemonState =<< daemonHandle -- | From a result of a configuration change, determine if the -- configuration was changed and if full distribution is needed. -- If so, also bump the serial number. unpackConfigResult :: ClockTime -> ConfigState -> (a, ConfigState) -> ((a, Bool, Bool), ConfigState) unpackConfigResult now cs (r, cs') | cs /= cs' = ( (r, True, needsFullDist cs cs') , over csConfigDataL (bumpSerial now) cs' ) | otherwise = ((r, False, False), cs') -- | Atomically modifies the configuration state in the WConfdMonad -- with a computation that can possibly fail; immediately afterwards, -- while config write is still going on, do the followup action. Return -- only after replication is finished. modifyConfigStateErrWithImmediate :: (TempResState -> ConfigState -> AtomicModifyMonad (a, ConfigState)) -> WConfdMonad () -> WConfdMonad a modifyConfigStateErrWithImmediate f immediateFollowup = do dh <- daemonHandle now <- liftIO getClockTime let modCS ds@(DaemonState { dsTempRes = tr }) = mapMOf2 dsConfigStateL (\cs -> liftM (unpackConfigResult now cs) (f tr cs)) ds (r, modified, distSync) <- atomicModifyIORefErrLog (dhDaemonState dh) (liftM swap . modCS) if modified then if distSync then do logDebug $ "Triggering config write" ++ " together with full synchronous distribution" res <- liftBase . triggerWithResult (Any True, Everywhere) $ dhSaveConfigWorker dh immediateFollowup wait res logDebug "Config write and distribution finished" else do -- trigger the config. saving worker and wait for it logDebug $ "Triggering config write" ++ " and asynchronous distribution" res <- liftBase . triggerWithResult (Any False, Everywhere) $ dhSaveConfigWorker dh immediateFollowup wait res logDebug "Config writer finished with local task" else immediateFollowup return r -- | Atomically modifies the configuration state in the WConfdMonad -- with a computation that can possibly fail. modifyConfigStateErr :: (TempResState -> ConfigState -> AtomicModifyMonad (a, ConfigState)) -> WConfdMonad a modifyConfigStateErr = flip modifyConfigStateErrWithImmediate (return ()) -- | Atomically modifies the configuration state in the WConfdMonad -- with a computation that can possibly fail. modifyConfigStateErr_ :: (TempResState -> ConfigState -> AtomicModifyMonad ConfigState) -> WConfdMonad () modifyConfigStateErr_ f = modifyConfigStateErr ((liftM ((,) ()) .) . f) -- | Atomically modifies the configuration state in the WConfdMonad. modifyConfigState :: (ConfigState -> (a, ConfigState)) -> WConfdMonad a modifyConfigState f = modifyConfigStateErr ((return .) . const f) -- | Atomically modifies the configuration state in WConfdMonad; immediately -- afterwards (while the config write-out is not necessarily finished) do -- another acation. modifyConfigStateWithImmediate :: (ConfigState -> (a, ConfigState)) -> WConfdMonad () -> WConfdMonad a modifyConfigStateWithImmediate f = modifyConfigStateErrWithImmediate ((return .) . const f) -- | Force the distribution of configuration without actually modifying it. -- -- We need a separate call for this operation, because 'modifyConfigState' only -- triggers the distribution when the configuration changes. forceConfigStateDistribution :: DistributionTarget -> WConfdMonad () forceConfigStateDistribution target = do logDebug "Forcing synchronous config write together with full distribution" dh <- daemonHandle liftBase . triggerAndWait (Any True, target) . dhSaveConfigWorker $ dh logDebug "Forced config write and distribution finished" -- | Atomically modifies the configuration data in the WConfdMonad -- with a computation that can possibly fail. modifyConfigDataErr_ :: (TempResState -> ConfigData -> AtomicModifyMonad ConfigData) -> WConfdMonad () modifyConfigDataErr_ f = modifyConfigStateErr_ (traverseOf csConfigDataL . f) -- | Atomically modifies the state of temporary reservations in -- WConfdMonad in the presence of possible errors. modifyTempResStateErr :: (ConfigData -> StateT TempResState ErrorResult a) -> WConfdMonad a modifyTempResStateErr f = do -- we use Compose to traverse the composition of applicative functors -- @ErrorResult@ and @(,) a@ let f' ds = traverseOf2 dsTempResL (runStateT (f (csConfigData . dsConfigState $ ds))) ds dh <- daemonHandle r <- toErrorBase $ atomicModifyIORefErr (dhDaemonState dh) (liftM swap . f') -- logDebug $ "Current temporary reservations: " ++ J.encode tr logDebug "Triggering temporary reservations write" liftBase . triggerAndWait_ . dhSaveTempResWorker $ dh logDebug "Temporary reservations write finished" return r -- | Atomically modifies the state of temporary reservations in -- WConfdMonad. modifyTempResState :: (ConfigData -> State TempResState a) -> WConfdMonad a modifyTempResState f = modifyTempResStateErr (mapStateT (return . runIdentity) . f) -- | Reads the state of of the configuration and temporary reservations -- in WConfdMonad. readTempResState :: WConfdMonad (ConfigData, TempResState) readTempResState = liftM (csConfigData . dsConfigState &&& dsTempRes) . readIORef . dhDaemonState =<< daemonHandle -- | Atomically modifies the lock waiting state in WConfdMonad. modifyLockWaiting :: (GanetiLockWaiting -> ( GanetiLockWaiting , (a, S.Set ClientId) )) -> WConfdMonad a modifyLockWaiting f = do dh <- lift . WConfdMonadInt $ ask let f' = (id &&& fst) . f (lockAlloc, (r, nfy)) <- atomicModifyWithLens (dhDaemonState dh) dsLockWaitingL f' logDebug $ "Current lock status: " ++ J.encode lockAlloc logDebug "Triggering lock state write" liftBase . triggerAndWait_ . dhSaveLocksWorker $ dh logDebug "Lock write finished" unless (S.null nfy) $ do logDebug . (++) "Locks became available for " . show $ S.toList nfy liftIO . mapM_ (notifyJob . ciPid) $ S.toList nfy logDebug "Finished notifying processes" return r -- | Atomically modifies the lock allocation state in WConfdMonad, not -- producing any result modifyLockWaiting_ :: (GanetiLockWaiting -> (GanetiLockWaiting, S.Set ClientId)) -> WConfdMonad () modifyLockWaiting_ = modifyLockWaiting . ((second $ (,) ()) .) -- | Read the lock waiting state in WConfdMonad. readLockWaiting :: WConfdMonad GanetiLockWaiting readLockWaiting = liftM dsLockWaiting . readIORef . dhDaemonState =<< daemonHandle -- | Read the underlying lock allocation. readLockAllocation :: WConfdMonad (LA.LockAllocation GanetiLocks ClientId) readLockAllocation = liftM LW.getAllocation readLockWaiting -- | Modify the configuration while temporarily acquiring -- the configuration lock. If the configuration lock is held by -- someone else, nothing is changed and Nothing is returned. modifyConfigAndReturnWithLock :: (TempResState -> ConfigState -> AtomicModifyMonad (a, ConfigState)) -> State TempResState () -> WConfdMonad (Maybe a) modifyConfigAndReturnWithLock f tempres = do now <- liftIO getClockTime dh <- lift . WConfdMonadInt $ ask pid <- liftIO getProcessID tid <- liftIO myThreadId let cid = ClientId { ciIdentifier = ClientOther $ "wconfd-" ++ show tid , ciLockFile = dhLivelock dh , ciPid = pid } let modCS ds@(DaemonState { dsTempRes = tr }) = mapMOf2 dsConfigStateL (\cs -> liftM (unpackConfigResult now cs) (f tr cs)) ds maybeDist <- bracket (atomicModifyWithLens (dhDaemonState dh) dsLockWaitingL $ swap . LW.updateLocks cid [LA.requestExclusive ConfigLock]) (\(res, _) -> case res of Ok s | S.null s -> do (_, nfy) <- atomicModifyWithLens (dhDaemonState dh) dsLockWaitingL $ swap . LW.updateLocks cid [LA.requestRelease ConfigLock] unless (S.null nfy) . liftIO . void . forkIO $ do logDebug . (++) "Locks became available for " . show $ S.toList nfy mapM_ (notifyJob . ciPid) $ S.toList nfy logDebug "Finished notifying processes" _ -> return ()) (\(res, _) -> case res of Ok s | S.null s ->do ret <- atomicModifyIORefErrLog (dhDaemonState dh) (liftM swap . modCS) atomicModifyWithLens (dhDaemonState dh) dsTempResL $ runState tempres return $ Just ret _ -> return Nothing) flip (maybe $ return Nothing) maybeDist $ \(val, modified, dist) -> do when modified $ do logDebug . (++) "Triggering config write; distribution " $ if dist then "synchronously" else "asynchronously" liftBase . triggerAndWait (Any dist, Everywhere) $ dhSaveConfigWorker dh logDebug "Config write finished" logDebug "Triggering temporary reservations write" liftBase . triggerAndWait_ . dhSaveTempResWorker $ dh logDebug "Temporary reservations write finished" return $ Just val modifyConfigWithLock :: (TempResState -> ConfigState -> AtomicModifyMonad ConfigState) -> State TempResState () -> WConfdMonad (Maybe ()) modifyConfigWithLock f = modifyConfigAndReturnWithLock f' where f' tr cs = fmap ((,) ()) (f tr cs) ganeti-3.1.0~rc2/src/Ganeti/WConfd/Persistent.hs000064400000000000000000000107561476477700300214670ustar00rootroot00000000000000{-# LANGUAGE MultiParamTypeClasses, TypeFamilies #-} {-| Common types and functions for persistent resources In particular: - locks - temporary reservations -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.Persistent ( Persistent(..) , writePersistentAsyncTask , readPersistent , persistentLocks , persistentTempRes ) where import Control.Monad.Except import System.Directory (doesFileExist) import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Errors import qualified Ganeti.JSON as J import Ganeti.Locking.Waiting (emptyWaiting, releaseResources) import Ganeti.Locking.Locks (ClientId(..), GanetiLockWaiting) import Ganeti.Logging import qualified Ganeti.Path as Path import Ganeti.WConfd.Monad import Ganeti.WConfd.TempRes ( TempResState, emptyTempResState , dropAllReservations) import Ganeti.Utils.Atomic import Ganeti.Utils.AsyncWorker -- * Common definitions -- ** The data type that collects all required operations -- | A collection of operations needed for persisting a resource. data Persistent a = Persistent { persName :: String , persPath :: IO FilePath , persEmpty :: a , persCleanup :: ClientId -> WConfdMonad () -- ^ The clean-up action needs to be a full 'WConfdMonad' action as it -- might need to do some complex processing, such as notifying -- clients that some locks are available. } -- ** Common functions -- | Construct an asynchronous worker whose action is to save the -- current state of the persistent state. -- The worker's action reads the state using the given @IO@ -- action. Any inbetween changes to the file are tacitly ignored. writePersistentAsyncTask :: (J.JSON a) => Persistent a -> IO a -> ResultG (AsyncWorker () ()) writePersistentAsyncTask pers readAction = mkAsyncWorker_ $ catchError (do let prefix = "Async. " ++ persName pers ++ " writer: " fpath <- liftIO $ persPath pers logDebug $ prefix ++ "Starting write to " ++ fpath state <- liftIO readAction toErrorBase . liftIO . atomicWriteFile fpath . J.encode $ state logDebug $ prefix ++ "written" ) (logEmergency . (++) ("Can't write " ++ persName pers ++ " state: ") . show) -- | Load a persistent data structure from disk. readPersistent :: (J.JSON a) => Persistent a -> ResultG a readPersistent pers = do logDebug $ "Reading " ++ persName pers file <- liftIO $ persPath pers file_present <- liftIO $ doesFileExist file if file_present then liftIO (persPath pers >>= readFile) >>= J.fromJResultE ("parsing " ++ persName pers) . J.decodeStrict else do logInfo $ "Note: No saved data for " ++ persName pers ++ ", tacitly assuming empty." return (persEmpty pers) -- * Implementations -- ** Locks persistentLocks :: Persistent GanetiLockWaiting persistentLocks = Persistent { persName = "lock allocation state" , persPath = Path.lockStatusFile , persEmpty = emptyWaiting , persCleanup = modifyLockWaiting_ . releaseResources } -- ** Temporary reservations persistentTempRes :: Persistent TempResState persistentTempRes = Persistent { persName = "temporary reservations" , persPath = Path.tempResStatusFile , persEmpty = emptyTempResState , persCleanup = modifyTempResState . const . dropAllReservations } ganeti-3.1.0~rc2/src/Ganeti/WConfd/Server.hs000064400000000000000000000113731476477700300205710ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| The implementation of Ganeti WConfd daemon server. As TemplateHaskell require that splices be defined in a separate module, we combine all the TemplateHaskell functionality that HTools needs in this module (except the one for unittests). -} {- Copyright (C) 2013, 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.Server where import Control.Concurrent (forkIO) import Control.Exception import Control.Monad import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.Daemon import Ganeti.Daemon.Utils (handleMasterVerificationOptions) import Ganeti.Logging (logDebug) import qualified Ganeti.Path as Path import Ganeti.THH.RPC import Ganeti.UDSServer import Ganeti.Errors (formatError) import Ganeti.Runtime import Ganeti.Utils import Ganeti.Utils.Livelock (mkLivelockFile) import Ganeti.WConfd.ConfigState import Ganeti.WConfd.ConfigVerify import Ganeti.WConfd.ConfigWriter import Ganeti.WConfd.Core import Ganeti.WConfd.DeathDetection (cleanupLocksTask) import Ganeti.WConfd.Monad import Ganeti.WConfd.Persistent handler :: DaemonHandle -> RpcServer WConfdMonadInt handler _ = $( mkRpcM exportedFunctions ) -- | Type alias for prepMain results type PrepResult = (Server, DaemonHandle) -- | Check function for luxid. checkMain :: CheckFn () checkMain = handleMasterVerificationOptions -- | Prepare function for luxid. prepMain :: PrepFn () PrepResult prepMain _ _ = do socket_path <- Path.defaultWConfdSocket cleanupSocket socket_path s <- describeError "binding to the socket" Nothing (Just socket_path) $ connectServer serverConfig True socket_path -- TODO: Lock the configuration file so that running the daemon twice fails? conf_file <- Path.clusterConfFile dh <- toErrorBase . withErrorT (strMsg . ("Initialization of the daemon failed" ++) . formatError) $ do ents <- getEnts (cdata, cstat) <- loadConfigFromFile conf_file verifyConfigErr cdata lock <- readPersistent persistentLocks tempres <- readPersistent persistentTempRes (_, livelock) <- mkLivelockFile C.wconfLivelockPrefix mkDaemonHandle conf_file (mkConfigState cdata) lock tempres (saveConfigAsyncTask conf_file cstat) (distMCsAsyncTask ents conf_file) distSSConfAsyncTask (writePersistentAsyncTask persistentLocks) (writePersistentAsyncTask persistentTempRes) livelock return (s, dh) serverConfig :: ServerConfig serverConfig = ServerConfig -- All the daemons that need to talk to WConfd should be -- running as the same user - the former master daemon user. FilePermissions { fpOwner = Just GanetiWConfd , fpGroup = Just $ ExtraGroup DaemonsGroup , fpPermissions = 0o0600 } ConnectConfig { recvTmo = 60 , sendTmo = 60 } -- | Main function. main :: MainFn () PrepResult main _ _ (server, dh) = do logDebug "Starting the cleanup task" _ <- forkIO $ runWConfdMonadInt cleanupLocksTask dh finally (forever $ runWConfdMonadInt (listener (handler dh) server) dh) (liftIO $ closeServer server) -- | Options list and functions. options :: [OptType] options = [ oNoDaemonize , oNoUserChecks , oDebug , oSyslogUsage , oForceNode , oNoVoting , oYesDoIt ] ganeti-3.1.0~rc2/src/Ganeti/WConfd/Ssconf.hs000064400000000000000000000146561476477700300205650ustar00rootroot00000000000000{-| Converts a configuration state into a Ssconf map. As TemplateHaskell require that splices be defined in a separate module, we combine all the TemplateHaskell functionality that HTools needs in this module (except the one for unittests). -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.Ssconf ( SSConf(..) , emptySSConf , mkSSConf ) where import Control.Arrow ((&&&), (***), first) import qualified Data.ByteString.UTF8 as UTF8 import Data.Foldable (Foldable(..), toList) import Data.List (partition) import Data.Maybe (mapMaybe) import qualified Data.Map as M import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Config import Ganeti.Constants import Ganeti.JSON (fromContainer, lookupContainer) import Ganeti.Objects import Ganeti.Ssconf import Ganeti.Utils import Ganeti.Types eqPair :: (String, String) -> String eqPair (x, y) = x ++ "=" ++ y mkSSConfHvparams :: Cluster -> [(Hypervisor, [String])] mkSSConfHvparams cluster = map (id &&& hvparams) [minBound..maxBound] where hvparams :: Hypervisor -> [String] hvparams h = maybe [] hvparamsStrings $ lookupContainer Nothing h (clusterHvparams cluster) -- | Convert a collection of hypervisor parameters to strings in the form -- @key=value@. hvparamsStrings :: HvParams -> [String] hvparamsStrings = map (eqPair . (UTF8.toString *** hvparamShow)) . M.toList . fromContainer -- | Convert a hypervisor parameter in its JSON representation to a String. -- Strings, numbers and booleans are just printed (without quotes), booleans -- printed as @True@/@False@ and other JSON values (should they exist) as -- their JSON representations. hvparamShow :: J.JSValue -> String hvparamShow (J.JSString s) = J.fromJSString s hvparamShow (J.JSRational _ r) = J.showJSRational r [] hvparamShow (J.JSBool b) = show b hvparamShow x = J.encode x mkSSConf :: ConfigData -> SSConf mkSSConf cdata = SSConf . M.fromList $ [ (SSClusterName, return $ clusterClusterName cluster) , (SSClusterTags, toList . unTagSet $ tagsOf cluster) , (SSFileStorageDir, return $ clusterFileStorageDir cluster) , (SSSharedFileStorageDir, return $ clusterSharedFileStorageDir cluster) , (SSGlusterStorageDir, return $ clusterGlusterStorageDir cluster) , (SSMasterCandidates, mapLines nodeName mcs) , (SSMasterCandidatesIps, mapLines nodePrimaryIp mcs) , (SSMasterCandidatesCerts, mapLines eqPair . toPairs . clusterCandidateCerts $ cluster) , (SSMasterIp, return $ clusterMasterIp cluster) , (SSMasterNetdev, return $ clusterMasterNetdev cluster) , (SSMasterNetmask, return . show $ clusterMasterNetmask cluster) , (SSMasterNode, return . genericResult (const "NO MASTER") nodeName . getNode cdata $ clusterMasterNode cluster) , (SSNodeList, mapLines nodeName nodes) , (SSNodePrimaryIps, mapLines (spcPair . (nodeName &&& nodePrimaryIp)) nodes ) , (SSNodeSecondaryIps, mapLines (spcPair . (nodeName &&& nodeSecondaryIp)) nodes ) , (SSNodeVmCapable, mapLines (eqPair . (nodeName &&& show . nodeVmCapable)) nodes) , (SSOfflineNodes, mapLines nodeName offline ) , (SSOnlineNodes, mapLines nodeName online ) , (SSPrimaryIpFamily, return . show . ipFamilyToRaw . clusterPrimaryIpFamily $ cluster) , (SSInstanceList, niceSort . mapMaybe instName . toList . configInstances $ cdata) , (SSReleaseVersion, return releaseVersion) , (SSHypervisorList, mapLines hypervisorToRaw . clusterEnabledHypervisors $ cluster) , (SSMaintainNodeHealth, return . show . clusterMaintainNodeHealth $ cluster) , (SSUidPool, mapLines formatUidRange . clusterUidPool $ cluster) , (SSNodegroups, mapLines (spcPair . (uuidOf &&& groupName)) nodeGroups) , (SSNetworks, mapLines (spcPair . (uuidOf &&& (fromNonEmpty . networkName))) . configNetworks $ cdata) , (SSEnabledUserShutdown, return . show . clusterEnabledUserShutdown $ cluster) , (SSSshPorts, mapLines (eqPair . (nodeName &&& getSshPort cdata)) nodes) ] ++ map (first hvparamsSSKey) (mkSSConfHvparams cluster) where mapLines :: (Foldable f) => (a -> String) -> f a -> [String] mapLines f = map f . toList spcPair (x, y) = x ++ " " ++ y toPairs = M.assocs . M.mapKeys UTF8.toString . fromContainer cluster = configCluster cdata mcs = getMasterOrCandidates cdata nodes = niceSortKey nodeName . toList $ configNodes cdata (offline, online) = partition nodeOffline nodes nodeGroups = niceSortKey groupName . toList $ configNodegroups cdata -- This will return the empty string only for the situation where the -- configuration is corrupted and no nodegroup can be found for that node. getSshPort :: ConfigData -> Node -> String getSshPort cfg node = maybe "" (show . ndpSshPort) $ getNodeNdParams cfg node ganeti-3.1.0~rc2/src/Ganeti/WConfd/TempRes.hs000064400000000000000000000452461476477700300207100ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, Rank2Types, FlexibleContexts #-} {-| Pure functions for manipulating reservations of temporary objects NOTE: Reservations aren't released specifically, they're just all released at the end of a job. This could be improved in the future. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Ganeti.WConfd.TempRes ( TempRes , mkTempRes , TempResState(..) , emptyTempResState , NodeUUID , InstanceUUID , DiskUUID , NetworkUUID , DRBDMinor , DRBDMap , trsDRBDL , computeDRBDMap , computeDRBDMap' , allocateDRBDMinor , releaseDRBDMinors , MAC , generateMAC , reserveMAC , generateDRBDSecret , reserveLV , IPv4ResAction(..) , IPv4Reservation(..) , reserveIp , releaseIp , generateIp , commitReleaseIp , commitReservedIps , listReservedIps , dropAllReservations , isReserved , reserve , dropReservationsFor , reserved ) where import Control.Lens.At import Control.Monad.Except import Control.Monad.State import Control.Monad.Trans.Maybe import Control.Monad (liftM, when, unless, mfilter, forM, forM_, (>=>)) import qualified Data.ByteString as BS import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Foldable as F import Data.Maybe import Data.Map (Map) import qualified Data.Map as M import Data.Monoid import qualified Data.Semigroup as Sem import qualified Data.Set as S import System.Random import qualified Text.JSON as J import Ganeti.BasicTypes import Ganeti.Config import qualified Ganeti.Constants as C import Ganeti.Errors import qualified Ganeti.JSON as J import Ganeti.Lens import qualified Ganeti.Network as N import Ganeti.Locking.Locks (ClientId) import Ganeti.Logging import Ganeti.Objects import Ganeti.THH import Ganeti.Objects.Lens (configNetworksL) import Ganeti.Utils import Ganeti.Utils.Monad import Ganeti.Utils.Random import qualified Ganeti.Utils.MultiMap as MM -- * The main reservation state -- ** Aliases to make types more meaningful: type NodeUUID = BS.ByteString type InstanceUUID = BS.ByteString type DiskUUID = BS.ByteString type NetworkUUID = BS.ByteString type DRBDMinor = Int -- | A map of the usage of DRBD minors type DRBDMap = Map NodeUUID (Map DRBDMinor DiskUUID) -- | A map of the usage of DRBD minors with possible duplicates type DRBDMap' = Map NodeUUID (Map DRBDMinor [DiskUUID]) -- * The state data structure -- | Types of IPv4 reservation actions. data IPv4ResAction = IPv4Reserve | IPv4Release deriving (Eq, Ord, Show, Bounded, Enum) instance J.JSON IPv4ResAction where showJSON IPv4Reserve = J.JSString . J.toJSString $ C.reserveAction showJSON IPv4Release = J.JSString . J.toJSString $ C.releaseAction readJSON = J.readEitherString >=> \s -> case () of _ | s == C.reserveAction -> return IPv4Reserve | s == C.releaseAction -> return IPv4Release | otherwise -> fail $ "Invalid IP reservation action: " ++ s -- | The values stored in the IPv4 reservation table. data IPv4Reservation = IPv4Res { ipv4ResAction :: IPv4ResAction , ipv4ResNetwork :: NetworkUUID , ipv4ResAddr :: Ip4Address } deriving (Eq, Ord, Show) instance J.JSON IPv4Reservation where -- Notice that addr and net are in a different order, to be compatible -- with the original Python representation (while it's used). showJSON (IPv4Res a net addr) = J.showJSON (a, addr, net) readJSON = fmap (\(a, addr, net) -> IPv4Res a net addr) . J.readJSON -- | A polymorphic data structure for managing temporary resources assigned -- to jobs. newtype TempRes j a = TempRes { getTempRes :: MM.MultiMap j a } deriving (Eq, Ord, Show) instance (Ord j, Ord a) => Sem.Semigroup (TempRes j a) where (TempRes x) <> (TempRes y) = TempRes $ x <> y instance (Ord j, Ord a) => Monoid (TempRes j a) where mempty = TempRes mempty mappend = (Sem.<>) instance (J.JSON j, Ord j, J.JSON a, Ord a) => J.JSON (TempRes j a) where showJSON = J.showJSON . getTempRes readJSON = liftM TempRes . J.readJSON -- | Create a temporary reservations from a given multi-map. mkTempRes :: MM.MultiMap j a -> TempRes j a mkTempRes = TempRes -- | The state of the temporary reservations $(buildObject "TempResState" "trs" [ simpleField "dRBD" [t| DRBDMap |] , simpleField "mACs" [t| TempRes ClientId MAC |] , simpleField "dRBDSecrets" [t| TempRes ClientId DRBDSecret |] , simpleField "lVs" [t| TempRes ClientId LogicalVolume |] , simpleField "iPv4s" [t| TempRes ClientId IPv4Reservation |] ]) emptyTempResState :: TempResState emptyTempResState = TempResState M.empty mempty mempty mempty mempty $(makeCustomLenses ''TempResState) -- ** Utility functions -- | Issues a reservation error. resError :: (MonadError GanetiException m) => String -> m a resError = throwError . ReservationError -- | Converts 'GenericError' into a 'ReservationError'. toResError :: (MonadError GanetiException m) => m a -> m a toResError = flip catchError (throwError . f) where f (GenericError msg) = ReservationError msg f e = e -- | Filter values from the nested map and remove any nested maps -- that become empty. filterNested :: (Ord a, Ord b) => (c -> Bool) -> Map a (Map b c) -> Map a (Map b c) filterNested p = M.filter (not . M.null) . fmap (M.filter p) -- | Converts a lens that works on maybe values into a lens that works -- on regular ones. A missing value on the input is replaced by -- 'mempty'. -- The output is is @Just something@ iff @something /= mempty@. maybeLens :: (Monoid a, Monoid b, Eq b) => Lens s t (Maybe a) (Maybe b) -> Lens s t a b maybeLens l f = l (fmap (mfilter (/= mempty) . Just) . f . fromMaybe mempty) -- * DRBD functions -- | Compute the map of used DRBD minor/nodes, including possible -- duplicates. -- An error is returned if the configuration isn't consistent -- (for example if a referenced disk is missing etc.). computeDRBDMap' :: (MonadError GanetiException m) => ConfigData -> TempResState -> m DRBDMap' computeDRBDMap' cfg trs = flip execStateT (fmap (fmap (: [])) (trsDRBD trs)) $ F.forM_ (configDisks cfg) addMinors where -- | Creates a lens for modifying the list of instances nodeMinor :: NodeUUID -> DRBDMinor -> Lens' DRBDMap' [DiskUUID] nodeMinor node minor = maybeLens (at node) . maybeLens (at minor) -- | Adds minors of a disk within the state monad addMinors disk = do let minors = getDrbdMinorsForDisk disk forM_ minors $ \(minor, node) -> nodeMinor (UTF8.fromString node) minor %= (UTF8.fromString (uuidOf disk) :) -- | Compute the map of used DRBD minor/nodes. -- Report any duplicate entries as an error. -- -- Unlike 'computeDRBDMap'', includes entries for all nodes, even if empty. computeDRBDMap :: (MonadError GanetiException m) => ConfigData -> TempResState -> m DRBDMap computeDRBDMap cfg trs = do m <- computeDRBDMap' cfg trs let dups = filterNested ((>= 2) . length) m unless (M.null dups) . resError $ "Duplicate DRBD ports detected: " ++ show (M.toList $ fmap M.toList dups) return $ fmap (M.mapMaybe listToMaybe) m `M.union` (fmap (const mempty) . J.fromContainer . configNodes $ cfg) -- Allocate a drbd minor. -- -- The free minor will be automatically computed from the existing devices. -- A node can not be given multiple times. -- The result is the list of minors, in the same order as the passed nodes. allocateDRBDMinor :: (MonadError GanetiException m, MonadState TempResState m) => ConfigData -> DiskUUID -> [NodeUUID] -> m [DRBDMinor] allocateDRBDMinor cfg disk nodes = do unless (nodes == ordNub nodes) . resError $ "Duplicate nodes detected in list '" ++ show nodes ++ "'" dMap <- computeDRBDMap' cfg =<< get let usedMap = fmap M.keysSet dMap let alloc :: S.Set DRBDMinor -> Map DRBDMinor DiskUUID -> (DRBDMinor, Map DRBDMinor DiskUUID) alloc used m = let k = findFirst 0 (M.keysSet m `S.union` used) in (k, M.insert k disk m) forM nodes $ \node -> trsDRBDL . maybeLens (at node) %%= alloc (M.findWithDefault mempty node usedMap) -- Release temporary drbd minors allocated for a given disk using -- 'allocateDRBDMinor'. releaseDRBDMinors :: (MonadState TempResState m) => DiskUUID -> m () releaseDRBDMinors disk = trsDRBDL %= filterNested (/= disk) -- * Other temporary resources -- | Tests if a given value is reserved for a given job. isReserved :: (Ord a, Ord j) => a -> TempRes j a -> Bool isReserved x = MM.elem x . getTempRes -- | Tries to reserve a given value for a given job. reserve :: (MonadError GanetiException m, Show a, Ord a, Ord j) => j -> a -> TempRes j a -> m (TempRes j a) reserve jobid x tr = do when (isReserved x tr) . resError $ "Duplicate reservation for resource '" ++ show x ++ "'" return . TempRes . MM.insert jobid x $ getTempRes tr dropReservationsFor :: (Ord a, Ord j) => j -> TempRes j a -> TempRes j a dropReservationsFor jobid = TempRes . MM.deleteAll jobid . getTempRes reservedFor :: (Ord a, Ord j) => j -> TempRes j a -> S.Set a reservedFor jobid = MM.lookup jobid . getTempRes reserved :: (Ord a, Ord j) => TempRes j a -> S.Set a reserved = MM.values . getTempRes -- | Computes the set of all reserved resources and passes it to -- the given function. -- This allows it to avoid resources that are already in use. withReserved :: (MonadError GanetiException m, Show a, Ord a, Ord j) => j -> (S.Set a -> m a) -> TempRes j a -> m (a, TempRes j a) withReserved jobid genfn tr = do x <- genfn (reserved tr) (,) x `liftM` reserve jobid x tr -- | Repeatedly tries to run a given monadic function until it succeeds -- and the returned value is free to reserve. -- If such a value is found, it's reserved and returned. -- Otherwise fails with an error. generate :: (MonadError GanetiException m, Show a, Ord a, Ord j) => j -> S.Set a -> m (Maybe a) -> TempRes j a -> m (a, TempRes j a) generate jobid existing genfn = withReserved jobid f where retries = 64 :: Int f res = do let vals = res `S.union` existing xOpt <- retryMaybeN retries (\_ -> mfilter (`S.notMember` vals) (MaybeT genfn)) maybe (resError "Not able generate new resource") -- TODO: (last tried: " ++ %s)" % new_resource return xOpt -- | A variant of 'generate' for randomized computations. generateRand :: (MonadError GanetiException m, Show a, Ord a, Ord j, RandomGen g) => g -> j -> S.Set a -> (g -> (Maybe a, g)) -> TempRes j a -> m (a, TempRes j a) generateRand rgen jobid existing genfn tr = evalStateT (generate jobid existing (state genfn) tr) rgen -- | Embeds a stateful computation in a stateful monad. stateM :: (MonadState s m) => (s -> m (a, s)) -> m a stateM f = get >>= f >>= \(x, s) -> liftM (const x) (put s) -- | Embeds a state-modifying computation in a stateful monad. modifyM :: (MonadState s m) => (s -> m s) -> m () modifyM f = get >>= f >>= put -- ** Functions common to all reservations -- | Removes all resources reserved by a given job. -- -- If a new reservation resource type is added, it must be added here as well. dropAllReservations :: ClientId -> State TempResState () dropAllReservations jobId = modify $ (trsMACsL %~ dropReservationsFor jobId) . (trsDRBDSecretsL %~ dropReservationsFor jobId) . (trsLVsL %~ dropReservationsFor jobId) . (trsIPv4sL %~ dropReservationsFor jobId) -- | Looks up a network by its UUID. lookupNetwork :: (MonadError GanetiException m) => ConfigData -> NetworkUUID -> m Network lookupNetwork cd netId = J.lookupContainer (resError $ "Network '" ++ show netId ++ "' not found") netId (configNetworks cd) -- ** IDs -- ** MAC addresses -- Randomly generate a MAC for an instance. -- Checks that the generated MAC isn't used by another instance. -- -- Note that we only consume, but not return the state of a random number -- generator. This is because the whole operation needs to be pure (for atomic -- 'IORef' updates) and therefore we can't use 'getStdRandom'. Therefore the -- approach we take is to instead use 'newStdGen' and discard the split -- generator afterwards. generateMAC :: (RandomGen g, MonadError GanetiException m, Functor m) => g -> ClientId -> Maybe NetworkUUID -> ConfigData -> StateT TempResState m MAC generateMAC rgen jobId netId cd = do net <- case netId of Just n -> Just <$> lookupNetwork cd n Nothing -> return Nothing let prefix = fromMaybe (clusterMacPrefix . configCluster $ cd) (networkMacPrefix =<< net) let existing = S.fromList $ getAllMACs cd StateT $ traverseOf2 trsMACsL (generateRand rgen jobId existing (over _1 Just . generateOneMAC prefix)) -- Reserves a MAC for an instance in the list of temporary reservations. reserveMAC :: (MonadError GanetiException m, MonadState TempResState m, Functor m) => ClientId -> MAC -> ConfigData -> m () reserveMAC jobId mac cd = do let existing = S.fromList $ getAllMACs cd when (S.member mac existing) $ resError "MAC already in use" modifyM $ traverseOf trsMACsL (reserve jobId mac) -- ** DRBD secrets generateDRBDSecret :: (RandomGen g, MonadError GanetiException m, Functor m) => g -> ClientId -> ConfigData -> StateT TempResState m DRBDSecret generateDRBDSecret rgen jobId cd = do let existing = S.fromList $ getAllDrbdSecrets cd StateT $ traverseOf2 trsDRBDSecretsL (generateRand rgen jobId existing (over _1 Just . generateSecret C.drbdSecretLength)) -- ** LVs reserveLV :: (MonadError GanetiException m, MonadState TempResState m, Functor m) => ClientId -> LogicalVolume -> ConfigData -> m () reserveLV jobId lv cd = do existing <- toError $ getAllLVs cd when (S.member lv existing) $ resError "MAC already in use" modifyM $ traverseOf trsLVsL (reserve jobId lv) -- ** IPv4 addresses -- | Lists all IPv4 addresses reserved for a given network. usedIPv4Addrs :: NetworkUUID -> S.Set IPv4Reservation -> S.Set Ip4Address usedIPv4Addrs netuuid = S.map ipv4ResAddr . S.filter ((== netuuid) . ipv4ResNetwork) -- | Reserve a given IPv4 address for use by an instance. reserveIp :: (MonadError GanetiException m, MonadState TempResState m, Functor m) => ClientId -> NetworkUUID -> Ip4Address -> Bool -- ^ whether to check externally reserved IPs -> ConfigData -> m () reserveIp jobId netuuid addr checkExt cd = toResError $ do net <- lookupNetwork cd netuuid isres <- N.isReserved N.PoolInstances addr net when isres . resError $ "IP address already in use" when checkExt $ do isextres <- N.isReserved N.PoolExt addr net when isextres . resError $ "IP is externally reserved" let action = IPv4Res IPv4Reserve netuuid addr modifyM $ traverseOf trsIPv4sL (reserve jobId action) -- | Give a specific IP address back to an IP pool. -- The IP address is returned to the IP pool designated by network id -- and marked as reserved. releaseIp :: (MonadError GanetiException m, MonadState TempResState m, Functor m) => ClientId -> NetworkUUID -> Ip4Address -> m () releaseIp jobId netuuid addr = let action = IPv4Res { ipv4ResAction = IPv4Release , ipv4ResNetwork = netuuid , ipv4ResAddr = addr } in modifyM $ traverseOf trsIPv4sL (reserve jobId action) -- Find a free IPv4 address for an instance and reserve it. generateIp :: (MonadError GanetiException m, MonadState TempResState m, Functor m) => ClientId -> NetworkUUID -> ConfigData -> m Ip4Address generateIp jobId netuuid cd = toResError $ do net <- lookupNetwork cd netuuid let f res = do let ips = usedIPv4Addrs netuuid res addr <- N.findFree (`S.notMember` ips) net maybe (resError "Cannot generate IP. Network is full") (return . IPv4Res IPv4Reserve netuuid) addr liftM ipv4ResAddr . stateM $ traverseOf2 trsIPv4sL (withReserved jobId f) -- | Commit a reserved/released IP address to an IP pool. -- The IP address is taken from the network's IP pool and marked as -- reserved/free for instances. commitIp :: (MonadError GanetiException m, Functor m) => IPv4Reservation -> ConfigData -> m ConfigData commitIp (IPv4Res actType netuuid addr) cd = toResError $ do let call = case actType of IPv4Reserve -> N.reserve IPv4Release -> N.release f Nothing = resError $ "Network '" ++ show netuuid ++ "' not found" f (Just net) = Just `liftM` call N.PoolInstances addr net traverseOf (configNetworksL . J.alterContainerL netuuid) f cd -- | Immediately release an IP address, without using the reservations pool. commitReleaseIp :: (MonadError GanetiException m, Functor m) => NetworkUUID -> Ip4Address -> ConfigData -> m ConfigData commitReleaseIp netuuid addr = commitIp (IPv4Res IPv4Release netuuid addr) -- | Commit all reserved/released IP address to an IP pool. -- The IP addresses are taken from the network's IP pool and marked as -- reserved/free for instances. -- -- Note that the reservations are kept, they are supposed to be cleaned -- when a job finishes. commitReservedIps :: (MonadError GanetiException m, Functor m, MonadLog m) => ClientId -> TempResState -> ConfigData -> m ConfigData commitReservedIps jobId tr cd = do let res = reservedFor jobId (trsIPv4s tr) logDebug $ "Commiting reservations: " ++ show res F.foldrM commitIp cd res listReservedIps :: ClientId -> TempResState -> S.Set IPv4Reservation listReservedIps jobid = reservedFor jobid . trsIPv4s ganeti-3.1.0~rc2/src/OLD-NEWS000064400000000000000000000460501476477700300155130ustar00rootroot00000000000000Ganeti-htools release notes =========================== **Note**: After version 0.3.1, the htools sources have been integrated into the ganeti core repository, and released together with the ganeti releases. Thus this NEWS file is obsolete. Version 0.3.1 (Fri, 11 Mar 2011) -------------------------------- Minor bugfix release: - Fixed source archive generation: the hscolour.css file was an invalid symlink, and the man pages were not correctly timestamped (leading to unneeded build-time rebuilds) - Improved the Luxi backend to show which attribute fails parsing - Small improvements to the man pages, and also ship the HTML version of man pages in the source archive Version 0.3.0 (Fri, 04 Feb 2011) -------------------------------- A significant release that breaks compatibility with Ganeti versions below 2.4 due to the node group changes. Only the RAPI backend can talk to older clusters, but it is recommended to use this version only with Ganeti 2.4. All commands are now multi-group aware (but to various degrees), so allocation, balancing and capacity calculation respects the group layout and will not create “broken” instances by using nodes from different groups. For a regular, single-group cluster, no changes should be directly visible to the users. A multi-group cluster however will change some things slightly: - hbal will require a target group to operate on (no cluster-wide balancing yet) - evacuation of (DRBD) instances from a node will be restricted to nodes in the same group, as inter-group moves are not implemented yet - capacity, while showing correct data, will not give per-group details yet There are other changes in this release: - fixed a long-standing bug in hscan related to node memory data - changed the text backend format, which unfortunately invalidates old files - error handling improvements, so that invalid input data reports better where the error is - the simulation backend changes its syntax, now it takes the allocation policy too, and can generate multiple groups - (internal) man page generation moved to pandoc from hand-written, which is helpful as it can also generate HTML versions - the balancing algorithm has been changed to work in parallel, if the code is linked against the multi-threaded runtime; this gives a very good speedup (~80% on 4 cores, ~60-70% of 12 cores) Version 0.2.8 (Thu, 23 Dec 2010) -------------------------------- A bug fix release: - fixed balancing function for big clusters, which will improve corner cases where hbal didn't see any solution even though the cluster was obviously not well balanced - fixed exit code of hbal in case of (Luxi) job errors - changed the signal handling in hbal in order to make hbal control easier: instead of synchronising on the count of signals, make SIGINT cause graceful termination, and SIGTERM an immediate one - increased the tag exclusion weight so that it has greater importance during the balancing - slight improvement to the speed of balancing via algorithm tweaks Version 0.2.7 (Thu, 07 Oct 2010) -------------------------------- Bug fixes: - fixed the error message for hail multi-evacuation mode - improve evacuation mode for offline secondary nodes (ignore available memory) New features: - add a new option ``-S`` to hbal and hspace that saves the cluster state at the end of the processing in the text format used by the ``-t`` option, for later re-processing - a two new options to hbal, -g and --min-gain-limit, that should help in limiting the number of balances steps with a low gain in the final stages - hbal, when executing jobs, will now wait for the current jobs to finish at the first stop (e.g. ^C); if the user wants immediate exit, another signal should be sent - added “normalized” physical CPU units in hspace output (NPU), which represents units of physical CPUs free/used, based on the max-cpu ratio Version 0.2.6 (Mon, 26 Jul 2010) -------------------------------- Exactly three months since the last release. Many internal changes, plus a couple of important changes in the balancing algorithm. First, the balancing may now introduce N+1 errors, if this solves other, more critical problems. For the moment, this means that moving instances away from offline nodes is allowed even if it creates N+1 errors, and that means evacuation can be done in more cases. Second, the scoring for N+1 has changed. In previous versions, it simply counted the number of failing N+1 nodes, which means moving an instance away from a N+1 failed node (but without the node 'clearing' the N+1 status) was not reflected in the cluster score. As such, the balancing algorithm managed to clear N+1 errors only sometimes, since usually it takes more than one move for this, and the first prerequisite move was not 'rewarded' appropriately and thus it was not selected. Now, it is possible to fix many more error cases than before: on a simulated 40 node cluster full with instances (symmetrically allocated on all nodes), around five nodes can be evacuated before N+1 errors can be solved, whereas 0.2.5 could evacuate at best one node. There were some other internal changes to the scoring algorithm, such that now the metrics have associated weights, and they are not all of the same importance anymore. As of now, the only change is that offline instances have a higher weight, which should favour proper node evacuations. Among the other changes: - fixed the hspace KM_POOL_* metrics, which were returned as the final state and not as the delta between the initial and final states - fixed hspace handling of N+1 failing clusters: before, it used to generate a 'fake' response, and the structure of this response was not always in sync with the real responses, leading to missing items; currently it proceeds correctly through the code (skipping the computation), and uses the same display mechanisms as the normal case - fixed hscan exit code for RAPI failures: previously it finished with success even if all the clusters failed, which was creating issues with the live-test script; now it exits with exit code 2 for RAPI failures (unfortunately this is still not optimal as LUXI failures will use exit code 1, the same as the command line) - changed the limit values for CPU/disk, which previously were used optionally, whereas now they are always used; the default cpu ratio limit is now 64 VCPUs per PCPU - changed the internal handling of the short name vs. original (Ganeti-provided) name; now internally we always use the full name, and only in display routines we show the shortened (called 'alias') name; as a result, the -O and --excluded-instances options now accept both the full name and the shortened name - changed internal handling of JSON conversions and errors, such that now we show a better context for failure messages, which should help with diagnosing the malformed message - changed the names for a few node fields, and added some more nodes; this is most likely to help with debugging, and not with regular operation though - changed the node fields option to allow the '+' prefix to mean 'extend the default fields list' rather than start from fresh (similar to Ganeti's implementation) - a few internal changes related to the LUXI protocol implementation, which should make it more safe against potential bugs, one optiomization that should help with large messages, and some patches in preparation for potential expansion of the LUXI backend functionality And finally, many improvements on unittests and the live-test script. Test coverage is much enhanced, and the test infrastructure has better error reporting; this should lead down-the-road to better code and fewer bugsâ€Ļ Version 0.2.5 (Mon, 26 Apr 2010) -------------------------------- Some internal cleanup plus a few user-visible changes: - new option for marking instances as 'do-not-move' during rebalancing - allow ``hscan`` to scan the local cluster via Luxi - add more metrics to ``hspace`` which show the delta between original state and final state better (only valid for tiered allocation) Version 0.2.4 (Mon, 22 Feb 2010) -------------------------------- Two improvements for node evacuation: - hbal takes a new parameter ``--evac-mode`` that restricts the instances to be moved to the ones on offline/drained nodes, which should reduce the work done - hail supports the new ``multi-evacuate`` mode of the IAllocator protocol, that will be released in a minor release on the Ganeti 2.1 branch Version 0.2.3 (Thu, 4 Feb 2010) -------------------------------- A small release: - Fixes selection of secondary node: previously, if the cluster had many N+1 failures, a N+1 failed node could be selected as secondary even if it did not have enough memory to allow the instance to be migrated/failed over to it; this is bad for automated tools, since we can get the cluster in an unhealthy state - Switch the text backend to a single input file, that is generated now by hscan and shouldn't be generated manually via gnt-node/instance list anymore; this allows richer information to be kept in the file, and simplifies a little the internals of the text backend Version 0.2.2 (Tue, 29 Dec 2009) -------------------------------- Small release, 0.2.1 was broken and thus this was released earlier: - Release 0.2.1 broke the LUXI backend due to a typo, fixed - Added a live-test script that should catch errors like the above one in the future (needs a working, non-empty cluster) - Changed RAPI and LUXI backends to treat drained nodes as offline, similar to the IAllocator backend change in 0.2.0 (which was wrongly marked as affecting all backends) - Changed the metrics for offline instances and N1 score from percent to count, in order to increase the priority of evacuations - Added a new metric (offline primary instances) which should fix the evacuation of a offline node in a 2-node cluster Version 0.2.1 (Wed, 2 Dec 2009) -------------------------------- - Added instance exclusion defined via instance tags - Fixed the output of hspace to be again parseable from the shell Version 0.2.0 (Tue, 10 Nov 2009) -------------------------------- A significant release, with a few new major features: - Added direct execution of the hbal solution when using the Luxi backend; the steps for each instance moves are submitted as a single jobs, and the different jobs are submitted as groups in order to parallelise the execution of moves - Added support for balancing based on dynamic utilisation data for instances, fed in via a text file; by default, all instances are considered equal and this change also improves the equalisation of secondary instances per node - Added support for tiered capacity calculation in hspace, where we start from a maximum instance spec and decrease the spec when we run out of resources; this should give a better measure of available capacity on 'fragmented' clusters; this is done separately from the current fixed-mode computation Also there have been many minor improvements: - Added option for showing instances (“--print-instances”), similar to the print nodes option - Added support for customising the node list via an argument to the print nodes option in the form of a comma-separated list of field names; currently the field names are not documented, expecting further changes in a next release - Enhanced the error reporting in the Luxi and Rapi backends - Changed the handling of drained nodes, now being treated the same as offline nodes, for Ganeti 2.0.4+ compatibility - A number of internal changes, simplifying code and merging some disparate functions - Simplify the build system in relation to creation of archives Version 0.1.8 (Tue, 29 Sep 2009) -------------------------------- - Brown-paper-bag release fixing haddock issues Version 0.1.7 (Mon, 28 Sep 2009) -------------------------------- - Fixed a bug in the Luxi backend for big responses - Fixed test suite exit code in presence of test failures - Changed the migrate operation to run instead failover for instances which were marked as not running in the input data (this could have been changed since then, but it's better than today's always migrate) - Added support for 'cheap' moves only (only migrate/failover) in balancing - Added support for building without curl (thus no RAPI backend) Version 0.1.6 (Wed, 19 Aug 2009) -------------------------------- - Added support for Luxi (the native Ganeti protocol) - Added support for simulated clusters (for hspace only) - Added timeouts for the RAPI backend - Fixed a few inconsistencies in the command line handling - Fixed handling of errors while loading data - The 'network' is a new dependency due to the Luxi addition Version 0.1.5 (Thu, 09 Jul 2009) -------------------------------- - Removed obsolete hn1 program; this allowed removal of a lot of supporting code - Lots of changes in hspace: the output now is a shell fragment in order for script to source it or parse it easier; added failure reasons; optimised to use less memory for large clusters - Optimized the scoring algorithm (used by all tools) so that now computations should be faster Version 0.1.4 (Tue, 16 Jun 2009) -------------------------------- - Added CPU count/ratio of virtual-to-physical CPUs to the cluster scoring methods; this means that now the balancer, the iallocator plugin and so on will try to keep the VCPU-to-PCPU ratio equal across the cluster - Fixed some hscan bugs - Fixed the way iallocator reads the total disk size (was broken and it was always falling back to summing the disk sizes) - Internals: fixed most compile-time warnings Version 0.1.3 (Fri, 05 Jun 2009) -------------------------------- - Fix a bug in the ReplacePrimary instance moves, affecting most of the tools Version 0.1.2 (Tue, 02 Jun 2009) -------------------------------- - Add a new program, “hspace”, which computes the free space on a cluster (based on a given instance spec) - Improvements in API docs and partially in the user docs - Started adding unittests Version 0.1.1 (Tue, 26 May 2009) -------------------------------- - Add a new program, “hail”, which is an iallocator plugin and can allocate/relocate instances - Experimental support for non-mirrored instances (hail supports them, hbal should no longer abort when it finds such instances and simply ignore them) - The RAPI port and/or scheme can be overriden now, and even “file://” schemes can be used if the message body has been saved under the appropriate name - Lots of code reorganization, esp. rewritten loading pipeline - Better data checking and better error messages in case validation fails; tools now consider nodes with error in input data (‘?’ returned by ganeti) as offline - Small enhancement to the makefile for simpler packaging Version 0.1.0 (Tue, 19 May 2009) -------------------------------- - Drop compatibility with Ganeti 1.2 - Add a new minimum score option (with a very low default), should help with very good clusters (but is still not optimal) - Add a --quiet option to hbal - Add support for reading offline nodes directly from the cluster Version 0.0.8 (Tue, 21 Apr 2009) -------------------------------- - hbal: prevent mismatches in wrong node names being passed to -O, by aborting in this case - add the ability to write the commands (-C) to a script via (-C), so that it can be later executed directly; this has also changed the commands to include the ncessary -f flags to skip confirmations - add checks for extra argument in hbal and hn1, so that unintended errors are catched - raise the accepted “missing” memory limit to 512MB, to cover usual Xen reservations Version 0.0.7 (Mon, 23 Mar 2009) -------------------------------- - added support for offline nodes, which are not used as targets for instance relocation and if they hold instances the hbal algorithm will attempt to relocate these away - added support for offline instances, which now will no longer skew the free memory estimation of nodes; the algorithm will no longer create conditions for N+1 failures when such instances are later started - implemented a complete model of node resources, in order to prevent an unintended re-occurrence of cases like the offline instance were we miscalculate some node resource; this gives warning now in case the node reported free disk or free memory deviates by more than a set amount from the expected value - a new tool *hscan* that can generate the input text-file for the other tools by collection via RAPI - some small changes to the build system to make it more friendly; also included the generated documentation in the source archive Version 0.0.6 (Mon, 16 Mar 2009) -------------------------------- - re-factored the hbal algorithm to make it stable in the sense that it gives the same solution when restarted from the middle; barring rounding of disk/memory and incomplete reporting from Ganeti (for 1.2), it should be now feasible to rely on its output without generating moves ad infinitum - the hbal algorithm now uses two more variables: the node N+1 failures and the amount of reserved memory; the first of which tries to ‘fix’ the N+1 status, the latter tries to distribute secondaries more equally - the hbal algorithm now uses two more moves at each step: replace+failover and failover+replace (besides the original failover, replace, and failover+replace+failover) - slightly changed the build system to embed GIT version/tags into the binaries so that we know for a binary from which tree it was done, either via ‘--version’ or via “strings hbal|grep version” - changed the solution list and in general the hbal output to be more clear by default, and changed “gnt-instance failover” to “gnt-instance migrate” - added man pages for the two binaries Version 0.0.5 (Mon, 09 Mar 2009) -------------------------------- - a few small improvements for hbal (possibly undone by later changes), hbal is now quite faster - fix documentation building - allow hbal to work on non N+1 compliant clusters, but without guarantees that the end cluster will be compliant; in any case, this should give a smaller number of nodes that are not compliant if the cluster state permits it - strip common domain suffix from nodes and instances, so that output is shorter and hopefully clearer Version 0.0.4 (Sun, 15 Feb 2009) -------------------------------- - better balancing algorithm in hbal - implemented an RAPI collector, now the cluster data can be gathered automatically via RAPI and doesn't need manual export of node and instance list Version 0.0.3 (Wed, 28 Jan 2009) -------------------------------- - initial release of the hbal, a cluster rebalancing tool - input data format changed due to hbal requirements Version 0.0.2 (Tue, 06 Jan 2009) -------------------------------- - fix handling of some common cases (cluster N+1 compliant from the start, too big depth given, failure to compute solution) - add option to print the needed command list for reaching the proposed solution Version 0.0.1 (Tue, 06 Jan 2009) -------------------------------- - initial release of hn1 tool .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/src/README000064400000000000000000000110421476477700300152510ustar00rootroot00000000000000Ganeti Cluster tools (ganeti-htools) ==================================== **Note:** This document is obsolete and mostly documents the situation before the htools sources were integrated into the ganeti-core codebase. Information about the current situation is available in the ``INSTALL`` file in the top-level directory. These are some simple cluster tools for fixing common allocation problems on Ganeti 2.0 clusters. Note that these tools are most useful for bigger cluster sizes (e.g. more than five or ten machines); at lower sizes, the computations they do can also be done manually. Most of the tools revolve around the concept of keeping the cluster N+1 compliant: this means that in case of failure of any node, the instances affected can be failed over (via ``gnt-node failover`` or ``gnt-instance failover``) to their secondary node, and there is enough memory reserved for this operation without needing to shutdown other instances or rebalance the cluster. **Quick start** (see the installation section for more details): - (have the ghc compiler and the prerequisite libraries installed) - ``make`` - ``./hbal -m $cluster -C -p`` - look at the original and final cluster layout, and if acceptable, execute the given commands Available tools --------------- Cluster rebalancer ~~~~~~~~~~~~~~~~~~ The rebalancer uses a simple algorithm to try to get the nodes of the cluster as equal as possible in their resource usage. It tries to repeatedly move each instance one step, so that the cluster score becomes better. We stop when no further move can improve the score. For algorithm details and usage, see the man page ``hbal(1)``. IAllocator plugin ~~~~~~~~~~~~~~~~~ The ``hail`` iallocator plugin can be used for allocations of mirrored and non-mirrored instances and for relocations of mirrored instances. It needs to be installed in Ganeti's iallocator search path—usually ``/usr/lib/ganeti/iallocators`` or ``/usr/local/lib/ganeti/iallocators``, and after that it can be used via ganeti's ``--iallocator`` option (in various gnt-node/gnt-instance commands). See the man page ``hail(1)`` for more details. Cluster capacity estimator ~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``hspace`` program will, given an input instance specification, estimate how many instances of those type can be place on the cluster before it will become full (as in any new allocation would fail N+1 checks). For more details, see the man page hspace(1). Integration with Ganeti ----------------------- The ``hbal`` and ``hspace`` programs can either get their input from text files, locally from the master daemon (when run on the master node of a cluster), or remote from a cluster via RAPI. The "-L" argument enables local collection (with an optional path to the unix socket). For online collection via RAPI, the "-m" argument should specify the cluster or master node name. Only ``hbal`` and ``hspace`` use these arguments, ``hail`` uses the standard iallocator API and thus doesn't need any special setup (just needs to be installed in the right directory). For generating the text files, a separate tool (``hscan``) is provided to automate their gathering if RAPI is available, which is better since it can extract more precise information. In case RAPI is not usable for whatever reason, ``gnt-node list`` and ``gnt-instance list`` could be used, and their output concatenated in a single file, separated by one blank line. If you need to do this manually, you'll need to check the sources to see which fields are needed exactly. The ``hail`` program gets its data automatically from Ganeti when used as described in its section. Installation ------------ If installing from source, you need a working ghc compiler (6.8 at least) and some extra Haskell libraries which usually need to be installed manually: - `json `_ - `curl `_ - `network `_ - `parallel `_, version 1.x Once these are installed, just typing *make* in the top-level directory should be enough. If you edit the documentation sources, you will need the ``pandoc`` program to rebuilt it. Only the ``hail`` program needs to be installed in a specific place, the other tools are not location-dependent. For running the (admittedly small) unittest suite (via *make check*), the QuickCheck version 2 library is needed. Internal (implementation) documentation is available in the ``apidoc`` directory. .. vim: set textwidth=72 : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/src/haddock-prologue000064400000000000000000000003731476477700300175500ustar00rootroot00000000000000This is the internal documentation for the Haskell components of Ganeti. The @Ganeti.HTools@ subtree contain the htools-specific component (rebalancing, allocation, capacity), while the @Ganeti@ tree contains basic functionality and core components. ganeti-3.1.0~rc2/src/lint-hints.hs000064400000000000000000000025511476477700300170230ustar00rootroot00000000000000{-| Custom hint lints for Ganeti. Since passing --hint to hlint will override, not extend the built-in hints, we need to import the existing hints so that we get full coverage. -} import "hint" HLint.HLint import "hint" HLint.Dollar -- The following two hints warn to simplify e.g. "map (\v -> (v, -- True)) lst" to "zip lst (repeat True)", which is more abstract warn = map (\v -> (v, x)) y ==> zip y (repeat x) where _ = notIn v x warn = map (\v -> (x, v)) ==> zip (repeat x) where _ = notIn v x -- The following warn on use of length instead of null warn = length x > 0 ==> not (null x) warn = length x /= 0 ==> not (null x) warn = length x == 0 ==> null x -- Never use head, use 'case' which covers all possibilities warn = head x ==> case x of { y:_ -> y } where note = "Head is unsafe, please use case and handle the empty list as well" -- Never use tail, use 'case' which covers all possibilities warn = tail x ==> case x of { _:y -> y } where note = "Tail is unsafe, please use case and handle the empty list as well" ignore "Use first" ignore "Use &&&" ignore "Use &&" ignore "Reduce duplication" ignore "Use import/export shortcut" -- FIXME: remove ignore "Use void" when GHC 6.x is deprecated ignore "Use void" -- Configure newer HLint 1.9.26 to behave like older versions -- FIXME: Cleanup code and remove this ignore "Redundant bracket" ignore "Functor law" ganeti-3.1.0~rc2/test/000075500000000000000000000000001476477700300145635ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/autotools/000075500000000000000000000000001476477700300166145ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/autotools/autotools-check-news.test000064400000000000000000000012131476477700300235700ustar00rootroot00000000000000# Test a correct NEWS file against a stable release RELEASE=2.6.2 ./autotools/check-news < $TESTDATA_DIR/NEWS_OK.txt >>>= 0 # Test a correct NEWS file against an alpha release RELEASE=2.8.0~alpha ./autotools/check-news < $TESTDATA_DIR/NEWS_OK.txt >>>= 0 # Test a NEWS file with previous unreleased versions against a stable release RELEASE=2.6.2 ./autotools/check-news < $TESTDATA_DIR/NEWS_previous_unreleased.txt >>>2/Unreleased version after current release 2.6.2/ >>>= !0 # Test a NEWS file with previous unreleased versions against an alpha release RELEASE=2.8.0~alpha ./autotools/check-news < $TESTDATA_DIR/NEWS_previous_unreleased.txt >>>= 0 ganeti-3.1.0~rc2/test/data/000075500000000000000000000000001476477700300154745ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/NEWS_OK.txt000064400000000000000000000017511476477700300174060ustar00rootroot00000000000000News ==== Version 2.8.0 beta1 ------------------- *(unreleased)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Stuff New features ~~~~~~~~~~~~ - More stuff Version 2.7.0 rc2 ----------------- *(Released Fri, 24 May 2013)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Other stuff Since rc1: - Many bugfixes Version 2.7.0 rc1 ----------------- *(Released Fri, 3 May 2013)* - Things Version 2.7.0 beta1 ------------------- *(Released Wed, 6 Feb 2013)* This was the first beta release of the 2.7 series. All important changes are listed in the latest 2.7 entry. Version 2.6.2 ------------- *(Released Fri, 21 Dec 2012)* Hic sunt pink bunnies. Version 2.6.1 ------------- *(Released Fri, 12 Oct 2012)* Team members come, team members go. Version 2.6.0 ------------- *(Released Fri, 27 Jul 2012)* Many things happened before this point. .. vim: set textwidth=72 syntax=rst : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/test/data/NEWS_previous_unreleased.txt000064400000000000000000000017321476477700300231570ustar00rootroot00000000000000News ==== Version 2.8.0 beta1 ------------------- *(unreleased)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Stuff New features ~~~~~~~~~~~~ - More stuff Version 2.7.0 rc2 ----------------- *(Released Fri, 24 May 2013)* Incompatible/important changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Other stuff Since rc1: - Many bugfixes Version 2.7.0 rc1 ----------------- *(Released Fri, 3 May 2013)* - Things Version 2.7.0 beta1 ------------------- *(Released Wed, 6 Feb 2013)* This was the first beta release of the 2.7 series. All important changes are listed in the latest 2.7 entry. Version 2.6.2 ------------- *(Released Fri, 21 Dec 2012)* Hic sunt pink bunnies. Version 2.6.1 ------------- *(unreleased)* Team members come, team members go. Version 2.6.0 ------------- *(Released Fri, 27 Jul 2012)* Many things happened before this point. .. vim: set textwidth=72 syntax=rst : .. Local Variables: .. mode: rst .. fill-column: 72 .. End: ganeti-3.1.0~rc2/test/data/bdev-drbd-8.0.txt000064400000000000000000000016541476477700300203770ustar00rootroot00000000000000disk { size 0s _is_default; # bytes on-io-error detach; fencing dont-care _is_default; } net { timeout 60 _is_default; # 1/10 seconds max-epoch-size 16384; max-buffers 16384; unplug-watermark 128 _is_default; connect-int 10 _is_default; # seconds ping-int 10 _is_default; # seconds sndbuf-size 8388608; # bytes ko-count 0 _is_default; after-sb-0pri disconnect _is_default; after-sb-1pri disconnect _is_default; after-sb-2pri disconnect _is_default; rr-conflict disconnect _is_default; ping-timeout 5 _is_default; # 1/10 seconds } syncer { rate 30720k; # bytes/second after -1 _is_default; al-extents 257; } protocol A; _this_host { device "/dev/drbd63"; disk "/dev/xenvg/test.data"; meta-disk "/dev/xenvg/test.meta" [ 0 ]; address 192.0.2.1:11000; } _remote_host { address 192.0.2.2:11000; } ganeti-3.1.0~rc2/test/data/bdev-drbd-8.3.txt000064400000000000000000000017431476477700300204010ustar00rootroot00000000000000disk { size 0s _is_default; # bytes on-io-error detach; fencing dont-care _is_default; max-bio-bvecs 0 _is_default; } net { timeout 60 _is_default; # 1/10 seconds max-epoch-size 2048 _is_default; max-buffers 2048 _is_default; unplug-watermark 128 _is_default; connect-int 10 _is_default; # seconds ping-int 10 _is_default; # seconds sndbuf-size 131070 _is_default; # bytes ko-count 0 _is_default; after-sb-0pri discard-zero-changes; after-sb-1pri consensus; after-sb-2pri disconnect _is_default; rr-conflict disconnect _is_default; ping-timeout 5 _is_default; # 1/10 seconds } syncer { rate 61440k; # bytes/second after -1 _is_default; al-extents 257; } protocol C; _this_host { device minor 0; disk "/dev/xenvg/test.data"; meta-disk "/dev/xenvg/test.meta" [ 0 ]; address ipv4 192.0.2.1:11000; } _remote_host { address ipv4 192.0.2.2:11000; } ganeti-3.1.0~rc2/test/data/bdev-drbd-8.4-no-disk-params.txt000064400000000000000000000012031476477700300232140ustar00rootroot00000000000000resource resource0 { options { } net { cram-hmac-alg "md5"; shared-secret "shared_secret_123"; after-sb-0pri discard-zero-changes; after-sb-1pri consensus; } _remote_host { address ipv4 192.0.2.2:11000; } _this_host { address ipv4 192.0.2.1:11000; volume 0 { device minor 0; disk "/dev/xenvg/test.data"; meta-disk "/dev/xenvg/test.meta"; disk { } } } } ganeti-3.1.0~rc2/test/data/bdev-drbd-8.4.txt000064400000000000000000000014021476477700300203720ustar00rootroot00000000000000resource resource0 { options { } net { cram-hmac-alg "md5"; shared-secret "shared_secret_123"; after-sb-0pri discard-zero-changes; after-sb-1pri consensus; } _remote_host { address ipv4 192.0.2.2:11000; } _this_host { address ipv4 192.0.2.1:11000; volume 0 { device minor 0; disk "/dev/xenvg/test.data"; meta-disk "/dev/xenvg/test.meta" [ 0 ]; disk { size 2097152s; # bytes resync-rate 61440k; # bytes/second } } } } ganeti-3.1.0~rc2/test/data/bdev-drbd-disk.txt000064400000000000000000000005371476477700300210230ustar00rootroot00000000000000disk { size 0s _is_default; # bytes on-io-error detach; fencing dont-care _is_default; } syncer { rate 250k _is_default; # bytes/second after -1 _is_default; al-extents 257; } _this_host { device "/dev/drbd58"; disk "/dev/xenvg/test.data"; meta-disk "/dev/xenvg/test.meta" [ 0 ]; } ganeti-3.1.0~rc2/test/data/bdev-drbd-net-ip4.txt000064400000000000000000000014431476477700300213460ustar00rootroot00000000000000net { timeout 60 _is_default; # 1/10 seconds max-epoch-size 2048 _is_default; max-buffers 2048 _is_default; unplug-watermark 128 _is_default; connect-int 10 _is_default; # seconds ping-int 10 _is_default; # seconds sndbuf-size 131070 _is_default; # bytes ko-count 0 _is_default; after-sb-0pri disconnect _is_default; after-sb-1pri disconnect _is_default; after-sb-2pri disconnect _is_default; rr-conflict disconnect _is_default; ping-timeout 5 _is_default; # 1/10 seconds } syncer { rate 250k _is_default; # bytes/second after -1 _is_default; al-extents 127 _is_default; } protocol C; _this_host { device "/dev/drbd59"; address 192.0.2.1:11002; } _remote_host { address 192.0.2.2:11002; } ganeti-3.1.0~rc2/test/data/bdev-drbd-net-ip6.txt000064400000000000000000000016131476477700300213470ustar00rootroot00000000000000net { timeout 60 _is_default; # 1/10 seconds max-epoch-size 2048 _is_default; max-buffers 2048 _is_default; unplug-watermark 128 _is_default; connect-int 10 _is_default; # seconds ping-int 10 _is_default; # seconds sndbuf-size 0 _is_default; # bytes rcvbuf-size 0 _is_default; # bytes ko-count 0 _is_default; cram-hmac-alg "md5"; shared-secret "a6526cb6118297c9c82c7003924e236ceac0d867"; after-sb-0pri discard-zero-changes; after-sb-1pri consensus; after-sb-2pri disconnect _is_default; rr-conflict disconnect _is_default; ping-timeout 5 _is_default; # 1/10 seconds } syncer { rate 61440k; # bytes/second after -1 _is_default; al-extents 257; } protocol C; _this_host { device minor 0; address ipv6 [2001:db8:65::1]:11048; } _remote_host { address ipv6 [2001:db8:66::1]:11048; } ganeti-3.1.0~rc2/test/data/bdev-rbd/000075500000000000000000000000001476477700300171615ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/bdev-rbd/json_output_empty.txt000064400000000000000000000000031476477700300235220ustar00rootroot00000000000000{} ganeti-3.1.0~rc2/test/data/bdev-rbd/json_output_extra_matches.txt000064400000000000000000000006661476477700300252320ustar00rootroot00000000000000{"4":{"pool":"rbd","name":"d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0","snap":"-","device":"\/dev\/rbd4"},"1":{"pool":"rbd","name":"b9e31bb3-4d4f-4a2c-bc63-207a0bc4b287.rbd.disk0","snap":"-","device":"\/dev\/rbd1"},"2":{"pool":"rbd","name":"abe7957a-ec96-490f-9c08-53b1c51cecf0.rbd.disk0","snap":"-","device":"\/dev\/rbd2"},"3":{"pool":"rbd","name":"d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0","snap":"-","device":"\/dev\/rbd3"}} ganeti-3.1.0~rc2/test/data/bdev-rbd/json_output_no_matches.txt000064400000000000000000000003341476477700300245130ustar00rootroot00000000000000{"1":{"pool":"rbd","name":"b9e31bb3-4d4f-4a2c-bc63-207a0bc4b287.rbd.disk0","snap":"-","device":"\/dev\/rbd1"},"2":{"pool":"rbd","name":"abe7957a-ec96-490f-9c08-53b1c51cecf0.rbd.disk0","snap":"-","device":"\/dev\/rbd2"}} ganeti-3.1.0~rc2/test/data/bdev-rbd/json_output_ok.txt000064400000000000000000000005111476477700300230010ustar00rootroot00000000000000{"1":{"pool":"rbd","name":"b9e31bb3-4d4f-4a2c-bc63-207a0bc4b287.rbd.disk0","snap":"-","device":"\/dev\/rbd1"},"2":{"pool":"rbd","name":"abe7957a-ec96-490f-9c08-53b1c51cecf0.rbd.disk0","snap":"-","device":"\/dev\/rbd2"},"3":{"pool":"rbd","name":"d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0","snap":"-","device":"\/dev\/rbd3"}} ganeti-3.1.0~rc2/test/data/bdev-rbd/output_invalid.txt000064400000000000000000000000241476477700300227640ustar00rootroot00000000000000invalid rbd output ganeti-3.1.0~rc2/test/data/bdev-rbd/plain_output_new_empty.txt000064400000000000000000000000001476477700300245220ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/bdev-rbd/plain_output_new_extra_matches.txt000064400000000000000000000005431476477700300262270ustar00rootroot00000000000000id pool image snap device 4 rbd d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0 - /dev/rbd4 1 rbd b9e31bb3-4d4f-4a2c-bc63-207a0bc4b287.rbd.disk0 - /dev/rbd1 2 rbd abe7957a-ec96-490f-9c08-53b1c51cecf0.rbd.disk0 - /dev/rbd2 3 rbd d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0 - /dev/rbd3 ganeti-3.1.0~rc2/test/data/bdev-rbd/plain_output_new_no_matches.txt000064400000000000000000000003251476477700300255160ustar00rootroot00000000000000id pool image snap device 1 rbd b9e31bb3-4d4f-4a2c-bc63-207a0bc4b287.rbd.disk0 - /dev/rbd1 2 rbd abe7957a-ec96-490f-9c08-53b1c51cecf0.rbd.disk0 - /dev/rbd2 ganeti-3.1.0~rc2/test/data/bdev-rbd/plain_output_new_ok.txt000064400000000000000000000003251476477700300240070ustar00rootroot000000000000001 rbd b9e31bb3-4d4f-4a2c-bc63-207a0bc4b287.rbd.disk0 - /dev/rbd1 2 rbd abe7957a-ec96-490f-9c08-53b1c51cecf0.rbd.disk0 - /dev/rbd2 3 rbd d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0 - /dev/rbd3 ganeti-3.1.0~rc2/test/data/bdev-rbd/plain_output_old_empty.txt000064400000000000000000000000321476477700300245140ustar00rootroot00000000000000id pool image snap device ganeti-3.1.0~rc2/test/data/bdev-rbd/plain_output_old_extra_matches.txt000064400000000000000000000004361476477700300262150ustar00rootroot00000000000000id pool image snap device 4 rbd d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0 - /dev/rbd4 1 rbd b9e31bb3-4d4f-4a2c-bc63-207a0bc4b287.rbd.disk0 - /dev/rbd1 2 rbd abe7957a-ec96-490f-9c08-53b1c51cecf0.rbd.disk0 - /dev/rbd2 3 rbd d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0 - /dev/rbd3 ganeti-3.1.0~rc2/test/data/bdev-rbd/plain_output_old_no_matches.txt000064400000000000000000000002341476477700300255020ustar00rootroot00000000000000id pool image snap device 1 rbd b9e31bb3-4d4f-4a2c-bc63-207a0bc4b287.rbd.disk0 - /dev/rbd1 2 rbd abe7957a-ec96-490f-9c08-53b1c51cecf0.rbd.disk0 - /dev/rbd2 ganeti-3.1.0~rc2/test/data/bdev-rbd/plain_output_old_ok.txt000064400000000000000000000003351476477700300237750ustar00rootroot00000000000000id pool image snap device 1 rbd b9e31bb3-4d4f-4a2c-bc63-207a0bc4b287.rbd.disk0 - /dev/rbd1 2 rbd abe7957a-ec96-490f-9c08-53b1c51cecf0.rbd.disk0 - /dev/rbd2 3 rbd d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0 - /dev/rbd3 ganeti-3.1.0~rc2/test/data/cert1.pem000064400000000000000000000023151476477700300172160ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIDXTCCAkWgAwIBAgIJAKDpJJob+ltEMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX aWRnaXRzIFB0eSBMdGQwHhcNMTgwMjI4MTExODIwWhcNMTgwMzAxMTExODIwWjBF MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB CgKCAQEA004ggWrYNvXFO2hj4J2bAw3O/FNIpmiV6PJ/AaixYPXqV8YW2zv5dq3M RTDAFYtnRqVLgNMkY2Cr8k1p7tZfIKvj+x0F1jBEqrKRj1Rg15tRVnNcpskwnDPG THSrwsjw3EacEo/QNxNlPud1+2JsfrOEimPbQQpe+6BaPSvYOjF9rgoLUsJ6yBdY ATOlz4sRaJ3ZDpOGp4MgmpsWl4AE19idw2mzpb4pVciptpNthImP+W4vu6GUJwNa nCIecZLHbXmxBRVOnX2TFjXoCY6tegEyluiYdCXlT/9M2XP0pzLn+Fh2e2Qngufg JP0rVtx8xdtna9DH5S922BJcQIvpEwIDAQABo1AwTjAdBgNVHQ4EFgQUq4qIt/j5 KjQYhsibqecmmqx+Zu4wHwYDVR0jBBgwFoAUq4qIt/j5KjQYhsibqecmmqx+Zu4w DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAzZJwKDbPNSZMh0DFAJQu ZVoVZ/JLw2LdDcSNa3Gh+il/Z6HaSnEQy/aRhZ4e85e9YngVu+PeEDTY30p4lr+b sETSWdHppjyblJil7KhUMcLZhkwEyqnzf2CZZTMDh1A96IeFzbIDiclGwUQzR/Op pRyF45eh9E9+byqnpyS1xJ+1ng3VXPyJsKTJUCh2M6ObOOXhPrtNzM+4iSHZD3B+ yShCnlofClVz3vmvpWM4ZGxxQ60m07439EhuIvJ9t1Gjo1UmvhaJjZcHcA8+yiDG FYbmgbUx4u+hmwJFRbKWwzyDnx1C7b/UGRDw/NYPbe0vbjUb+YlLkz7qduGoXvMm EA== -----END CERTIFICATE----- ganeti-3.1.0~rc2/test/data/cert2.pem000064400000000000000000000055651476477700300172310ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCq8AvqgIioEckG M24Jb4I5K8zHnjBSMIGgdleMHo+Z+PO61bFJCq5cqBL7qdiVqtveSjJ5BLvB59jm NQMfW8UcfnfBfdb2Mt5qPnS9agDlqMFfdnNdgMiop8/Qam/R878IqeNwQE84LUzk GSdBqDqNc9k/PKwDeuJRyuIc9n4+0aRZf8zf1Aidxa5Ilo+S0paAoaMgrNaNKRNK uwHNrFkchcOLy5Catu1s4VwTePHk5q7rQTvzXK4i8qbmMMiHpUH1W9mxjtHNhoxR oyHps5xwBpYpEk3h5XaVEl8CqG0G7rmtK4ViSA4dQ9hs//V2xTOBszjuD+wMHdrN ugBqr3JZAgMBAAECggEAHsxhWT8PYDjUH2nkyY5tyB88Jjs6OZTDxkWMQJLBDNzu DRuzdZWXFNqzkORpQY4nT0XPEj7oFWfQjKnAhkXind+rdBFYScOgj0PxhK80uixN qwWMg6xQexLBPqvuucVRBh6V/AOaQmTnFbHygMHuys20ttAXrgjV/iav1sgStv6D WTbgzXS1QEwrMu47RIiu3z4IoZfjbej+Zj0I836V8NNz81mZ2moqZ7RA8ETA245k nlqmsX9F//V2PdgV/RShJTT28x3NYp6QIYPs9VOcJ6momNOzBn7tg6wa+EgPIYHh tHzvf0AHPwBllvrimNGD+T8PG7m2nsXZVmsO5VBiAQKBgQDVCkIQZDjov17VxsxM FzSNAg9zCwOKrMYXhZYzrpaIpbX0MDMH2bME+Yp76l1M/MCOHS9EV62TjBcegS4b hxAEyiA6X+PA4uAED4Acp2MqRRBuv8RQZMkbHs/Lj/CtjegTiRfBooyptxE1rANa xbib5RNJPO1CbSMZGjx0hhNe9QKBgQDNaFeJDNSmAQ8ujbn/x9iWKEOfGGzUbEIc DAHJmbfzgxiNTW08AXLOZMyg7gOvTn7tMJnWC0BJFpvrL0efXJ+vFI2R9lW4zI87 P57kGsHYQQk6FtOhV/OvWfQ1scjucQaUT9BiArDYR4GgKm3T2NoETMabKlp1wFtc ffmdk3RfVQKBgEYsKdSiXohzuLYr1FFf92RXAGXBg/oirOFElFQTtuvtwYBcfAKi 96+0zqPAb9kTDA4DmPUm+Dq4k0jt/hT9KQ6a0YDI4wk+8dEElgtaK3TZ6O7B5dUh TYjMXl/L2tgf/QiqSJP0iebBMT7/mN9Gb2eSTgb6taACuOPk23L6UtkdAoGAfzMG QhB3/vTY+fM6I3MWZKY6eeMeQc1ogwXMdZODnoCoS5iO9IHRHo69SUsbbQwm/asD GNGO1bPyigmVSNKK8FjB8omhO/cxG3eiZY9MSya7GAXauCdG+Ge0GywlScMkV+O1 H3ybFtPxKcYcjPvUxqTkuGHZ8uFTskswsKwHfKECgYEAq/OWtWpV19FpKCSrYTeT AIavcm3yhdo1w4UAsgPvmA232ENdEmW/6acZpluPUykcjH5IDnd3yEF4txWLgWqe lRif8McmFAWy1KGT+QQ1THG3BPNieUemEByaz6AqNJkKXt13sU7IyESY7ZuPhDWF T+g1GUL97SNZy9rNtk188cI= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIDXTCCAkWgAwIBAgIJANX+BCMHrXdfMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX aWRnaXRzIFB0eSBMdGQwHhcNMTgwMjI4MTExODExWhcNMTgwMzAxMTExODExWjBF MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB CgKCAQEAqvAL6oCIqBHJBjNuCW+COSvMx54wUjCBoHZXjB6PmfjzutWxSQquXKgS +6nYlarb3koyeQS7wefY5jUDH1vFHH53wX3W9jLeaj50vWoA5ajBX3ZzXYDIqKfP 0Gpv0fO/CKnjcEBPOC1M5BknQag6jXPZPzysA3riUcriHPZ+PtGkWX/M39QIncWu SJaPktKWgKGjIKzWjSkTSrsBzaxZHIXDi8uQmrbtbOFcE3jx5Oau60E781yuIvKm 5jDIh6VB9VvZsY7RzYaMUaMh6bOccAaWKRJN4eV2lRJfAqhtBu65rSuFYkgOHUPY bP/1dsUzgbM47g/sDB3azboAaq9yWQIDAQABo1AwTjAdBgNVHQ4EFgQUBOHZqk3W R1MOkLPs6ylGiuYwJC4wHwYDVR0jBBgwFoAUBOHZqk3WR1MOkLPs6ylGiuYwJC4w DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAgtbWLQfe60IJzTVbt8Tr dN+FGrRbIwrc07Dx86UKC3nWRg11XwMSQa4E6PXQcKcBmkkO0sLcUjRpyRnWEXRo OuQyyv5aEtPsUz2QgTFCRKaaaVZlSthGbuxBVdLydxGr39Yk9G7isX6UebJxNLJG 2hdhvIJewgd81BXsVQFtqq33BYzyI9/4lF7e4u+0HJm/fbjsOMDXkrV7+afYAhQ+ f/y0o94G+w/7WaHOwpvahD4IgfE6fbNEzoX/irA2Y4uWmXOfbClPIh3pdNUW9PGh XnfjPYMf9J2M9ffzcrHCjEqJYnCN2DysB0fjimEh2lSvLlsWFgqGQGxMEN/u6a5z xQ== -----END CERTIFICATE----- ganeti-3.1.0~rc2/test/data/cgroup_root/000075500000000000000000000000001476477700300200365ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/cpuset/000075500000000000000000000000001476477700300213415ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/cpuset/some_group/000075500000000000000000000000001476477700300235205ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/cpuset/some_group/lxc/000075500000000000000000000000001476477700300243065ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/cpuset/some_group/lxc/instance1/000075500000000000000000000000001476477700300261735ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/cpuset/some_group/lxc/instance1/cpuset.cpus000064400000000000000000000000041476477700300303640ustar00rootroot000000000000000-1 ganeti-3.1.0~rc2/test/data/cgroup_root/devices/000075500000000000000000000000001476477700300214605ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/devices/some_group/000075500000000000000000000000001476477700300236375ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/devices/some_group/lxc/000075500000000000000000000000001476477700300244255ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/devices/some_group/lxc/instance1/000075500000000000000000000000001476477700300263125ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/devices/some_group/lxc/instance1/devices.list000064400000000000000000000000121476477700300306220ustar00rootroot00000000000000a *:* rwm ganeti-3.1.0~rc2/test/data/cgroup_root/memory/000075500000000000000000000000001476477700300213465ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/memory/lxc/000075500000000000000000000000001476477700300221345ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/memory/lxc/instance1/000075500000000000000000000000001476477700300240215ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/cgroup_root/memory/lxc/instance1/memory.limit_in_bytes000064400000000000000000000000041476477700300302570ustar00rootroot00000000000000128 ganeti-3.1.0~rc2/test/data/cluster_config_2.10.json000064400000000000000000000437111476477700300220430ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "ctime": 1343869045.604884, "default_iallocator": "hail", "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "resync-rate": 1024 }, "ext": {}, "file": {}, "plain": { "stripes": 2 }, "rbd": { "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "file_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": {}, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "vga": "", "vhost_net": false, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false }, "lxc": { "cpu_mask": "" }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "use_localtime": false, "vif_script": "", "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xm" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "use_bootloader": false, "vif_script": "", "xen_cmd": "xm" } }, "ipolicy": { "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.79471, "ndparams": { "exclusive_storage": false, "oob_program": "", "spindle_count": 1 }, "nicparams": { "default": { "link": "br974", "mode": "bridged" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32101, 32102, 32103, 32104, 32105 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg" }, "ctime": 1343869045.605523, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "beparams": {}, "ctime": 1354038435.343601, "disk_template": "plain", "disks": [ { "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "params": {}, "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "beparams": {}, "ctime": 1363620258.608976, "disk_template": "drbd", "disks": [ { "children": [ { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "params": {}, "size": 1024 }, { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "params": {}, "size": 128 } ], "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "params": {}, "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.874901, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "beparams": {}, "ctime": 1355186880.451181, "disk_template": "plain", "disks": [ { "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "params": {}, "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1367352404.758083, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.575009, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2, "vcpu-ratio": 3.14 }, "mtime": 1361963775.575009, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.604884, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.934807, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.885368, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.353329, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "serial_no": 7625, "version": 2100000 } ganeti-3.1.0~rc2/test/data/cluster_config_2.11.json000064400000000000000000000444511476477700300220460ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_certs": {}, "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "ctime": 1343869045.604884, "default_iallocator": "hail", "default_iallocator_params": {}, "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "resync-rate": 1024 }, "ext": {}, "file": {}, "plain": { "stripes": 2 }, "rbd": { "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "enabled_user_shutdown": false, "file_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": {}, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "vga": "", "vhost_net": false, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false }, "lxc": { "cpu_mask": "" }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "use_localtime": false, "vif_script": "", "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xm" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "use_bootloader": false, "vif_script": "", "xen_cmd": "xm" } }, "ipolicy": { "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.79471, "ndparams": { "exclusive_storage": false, "oob_program": "", "spindle_count": 1 }, "nicparams": { "default": { "link": "br974", "mode": "bridged" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32101, 32102, 32103, 32104, 32105 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg" }, "ctime": 1343869045.605523, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1354038435.343601, "disk_template": "plain", "disks": [ { "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "params": {}, "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "admin_state_source": "user", "beparams": {}, "ctime": 1363620258.608976, "disk_template": "drbd", "disks": [ { "children": [ { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "params": {}, "size": 1024, "uuid": "55b3fa41-2bfe-4aef-ac32-fbf3be06a242" }, { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "params": {}, "size": 128, "uuid": "33eff786-0152-4653-8fc8-ea280fea9297" } ], "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "params": {}, "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.874901, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "disk_template": "plain", "disks": [ { "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "params": {}, "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1367352404.758083, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.575009, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2, "vcpu-ratio": 3.14 }, "mtime": 1361963775.575009, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.604884, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.934807, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.885368, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.353329, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "serial_no": 7625, "version": 2110000 } ganeti-3.1.0~rc2/test/data/cluster_config_2.12.json000064400000000000000000000356671476477700300220600ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_certs": {}, "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "compression_tools": ["gzip", "gzip-fast", "gzip-slow"], "ctime": 1343869045.604884, "default_iallocator": "hail", "default_iallocator_params": {}, "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "resync-rate": 1024 }, "ext": {}, "file": {}, "plain": { "stripes": 2 }, "rbd": { "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "enabled_user_shutdown": false, "file_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": {}, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "vga": "", "vhost_net": false, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false }, "lxc": { "cpu_mask": "" }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "use_localtime": false, "vif_script": "", "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xm" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "use_bootloader": false, "vif_script": "", "xen_cmd": "xm" } }, "install_image": "", "instance_communication_network": "", "ipolicy": { "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.79471, "ndparams": { "exclusive_storage": false, "oob_program": "", "spindle_count": 1 }, "nicparams": { "default": { "link": "br974", "mode": "bridged" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32101, 32102, 32103, 32104, 32105 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg", "zeroing_image": "" }, "ctime": 1343869045.605523, "disks": { "150bd154-8e23-44d1-b762-5065ae5a507b": { "ctime": 1354038435.343601, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "mtime": 1354038435.343601, "params": {}, "serial_no": 1, "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" }, "77ced3a5-6756-49ae-8d1f-274e27664c05": { "children": [ { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "params": {}, "size": 1024, "uuid": "55b3fa41-2bfe-4aef-ac32-fbf3be06a242" }, { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "params": {}, "size": 128, "uuid": "33eff786-0152-4653-8fc8-ea280fea9297" } ], "ctime": 1363620258.608976, "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "mtime": 1363620258.608976, "params": {}, "serial_no": 1, "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" }, "79acf611-be58-4334-9fe4-4f2b73ae8abb": { "ctime": 1355186880.451181, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "mtime": 1355186880.451181, "params": {}, "serial_no": 1, "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } }, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1354038435.343601, "disk_template": "plain", "disks": [ "150bd154-8e23-44d1-b762-5065ae5a507b" ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1363620258.608976, "disk_template": "drbd", "disks": [ "77ced3a5-6756-49ae-8d1f-274e27664c05" ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.874901, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1355186880.451181, "disk_template": "plain", "disks": [ "79acf611-be58-4334-9fe4-4f2b73ae8abb" ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1367352404.758083, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.575009, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2, "vcpu-ratio": 3.14 }, "mtime": 1361963775.575009, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.604884, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.934807, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.885368, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.353329, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "serial_no": 7625, "version": 2120000 } ganeti-3.1.0~rc2/test/data/cluster_config_2.13.json000064400000000000000000000373141476477700300220500ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_certs": {}, "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "compression_tools": ["gzip", "gzip-fast", "gzip-slow"], "ctime": 1343869045.604884, "data_collectors": {"cpu-avg-load": {"active": true, "interval": 5000000.0}, "diskstats": {"active": true, "interval": 5000000.0}, "drbd": {"active": true, "interval": 5000000.0}, "inst-status-xen": {"active": true, "interval": 5000000.0}, "lv": {"active": true, "interval": 5000000.0}}, "default_iallocator": "hail", "default_iallocator_params": {}, "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "resync-rate": 1024 }, "ext": {}, "file": {}, "plain": { "stripes": 2 }, "rbd": { "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "enabled_user_shutdown": false, "file_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": {}, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "vga": "", "vhost_net": false, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false }, "lxc": { "cpu_mask": "", "lxc_cgroup_use": "", "lxc_devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "lxc_drop_capabilities": "mac_override,sys_boot,sys_module,sys_time", "lxc_extra_config": "", "lxc_tty": 6, "lxc_startup_wait": 30 }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "use_localtime": false, "vif_script": "", "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xm" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "use_bootloader": false, "vif_script": "", "xen_cmd": "xm" } }, "install_image": "", "instance_communication_network": "", "ipolicy": { "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.79471, "ndparams": { "exclusive_storage": false, "oob_program": "", "spindle_count": 1 }, "nicparams": { "default": { "link": "br974", "mode": "bridged" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32101, 32102, 32103, 32104, 32105 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg", "zeroing_image": "" }, "ctime": 1343869045.605523, "disks": { "150bd154-8e23-44d1-b762-5065ae5a507b": { "ctime": 1354038435.343601, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "mtime": 1354038435.343601, "params": {}, "serial_no": 1, "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" }, "77ced3a5-6756-49ae-8d1f-274e27664c05": { "children": [ { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "params": {}, "size": 1024 }, { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "params": {}, "size": 128 } ], "ctime": 1363620258.608976, "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "mtime": 1363620258.608976, "params": {}, "serial_no": 1, "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" }, "79acf611-be58-4334-9fe4-4f2b73ae8abb": { "ctime": 1355186880.451181, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "mtime": 1355186880.451181, "params": {}, "serial_no": 1, "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } }, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1354038435.343601, "disk_template": "plain", "disks": [ "150bd154-8e23-44d1-b762-5065ae5a507b" ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1363620258.608976, "disk_template": "drbd", "disks": [ "77ced3a5-6756-49ae-8d1f-274e27664c05" ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.874901, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1355186880.451181, "disk_template": "plain", "disks": [ "79acf611-be58-4334-9fe4-4f2b73ae8abb" ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1367352404.758083, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.575009, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2, "vcpu-ratio": 3.14 }, "mtime": 1361963775.575009, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.604884, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.934807, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.885368, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.353329, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "filters": {}, "serial_no": 7625, "version": 2130000 } ganeti-3.1.0~rc2/test/data/cluster_config_2.14.json000064400000000000000000000430121476477700300220410ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_certs": {}, "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "compression_tools": [ "gzip", "gzip-fast", "gzip-slow" ], "ctime": 1343869045.6048839, "data_collectors": { "cpu-avg-load": { "active": true, "interval": 5000000.0 }, "diskstats": { "active": true, "interval": 5000000.0 }, "drbd": { "active": true, "interval": 5000000.0 }, "inst-status-xen": { "active": true, "interval": 5000000.0 }, "lv": { "active": true, "interval": 5000000.0 } }, "default_iallocator": "hail", "default_iallocator_params": {}, "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "protocol": "C", "resync-rate": 1024 }, "ext": { "access": "kernelspace" }, "file": {}, "gluster": { "access": "kernelspace", "host": "127.0.0.1", "port": 24007, "volume": "gv0" }, "plain": { "stripes": 2 }, "rbd": { "access": "kernelspace", "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "enabled_user_shutdown": false, "file_storage_dir": "", "gluster_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": { "migration_mode": "live" }, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_aio": "threads", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_caps": "", "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "user_shutdown": false, "vga": "", "vhost_net": false, "virtio_net_queues": 1, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false, "vnet_hdr": true }, "lxc": { "cpu_mask": "", "devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "drop_capabilities": "mac_override,sys_boot,sys_module,sys_time,sys_admin", "extra_cgroups": "", "extra_config": "", "lxc_cgroup_use": "", "lxc_devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "lxc_drop_capabilities": "mac_override,sys_boot,sys_module,sys_time", "lxc_extra_config": "", "lxc_startup_wait": 30, "lxc_tty": 6, "num_ttys": 6, "startup_timeout": 30 }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "soundhw": "", "use_localtime": false, "vif_script": "", "vif_type": "ioemu", "viridian": false, "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xm" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "soundhw": "", "use_bootloader": false, "vif_script": "", "xen_cmd": "xm" } }, "install_image": "", "instance_communication_network": "", "ipolicy": { "disk-templates": [ "drbd", "plain", "sharedfile", "file" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "max_running_jobs": 20, "max_tracked_jobs": 25, "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.7947099, "ndparams": { "cpu_speed": 1.0, "exclusive_storage": false, "oob_program": "", "ovs": false, "ovs_link": "", "ovs_name": "switch1", "spindle_count": 1, "ssh_port": 22 }, "nicparams": { "default": { "link": "br974", "mode": "bridged", "vlan": "" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "osparams_private_cluster": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32104, 32105, 32101, 32102, 32103 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg", "zeroing_image": "" }, "ctime": 1343869045.6055231, "disks": { "150bd154-8e23-44d1-b762-5065ae5a507b": { "ctime": 1354038435.343601, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "mtime": 1354038435.343601, "nodes": [ "2ae3d962-2dad-44f2-bdb1-85f77107f907" ], "params": {}, "serial_no": 1, "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" }, "77ced3a5-6756-49ae-8d1f-274e27664c05": { "children": [ { "ctime": 1421677173.7280669, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "mtime": 1421677173.7280591, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024 }, { "ctime": 1421677173.728096, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "mtime": 1421677173.7280879, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 128 } ], "ctime": 1363620258.6089759, "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "mtime": 1363620258.6089759, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" }, "79acf611-be58-4334-9fe4-4f2b73ae8abb": { "ctime": 1355186880.4511809, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "mtime": 1355186880.4511809, "nodes": [ "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } }, "filters": {}, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1354038435.343601, "disks": [ "150bd154-8e23-44d1-b762-5065ae5a507b" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1363620258.6089759, "disks": [ "77ced3a5-6756-49ae-8d1f-274e27664c05" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.8749011, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "osparams_private": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1355186880.4511809, "disks": [ "79acf611-be58-4334-9fe4-4f2b73ae8abb" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1421677173.729104, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.5750091, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2000000000000002, "vcpu-ratio": 3.1400000000000001 }, "mtime": 1361963775.5750091, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.6048839, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.9348071, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.8853681, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.3533289, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "serial_no": 7626, "version": 2140000 } ganeti-3.1.0~rc2/test/data/cluster_config_2.15.json000064400000000000000000000431451476477700300220510ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_certs": {}, "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "compression_tools": [ "gzip", "gzip-fast", "gzip-slow" ], "ctime": 1343869045.6048839, "data_collectors": { "cpu-avg-load": { "active": true, "interval": 5000000.0 }, "diskstats": { "active": true, "interval": 5000000.0 }, "drbd": { "active": true, "interval": 5000000.0 }, "inst-status-xen": { "active": true, "interval": 5000000.0 }, "lv": { "active": true, "interval": 5000000.0 }, "xen-cpu-avg-load": { "active": true, "interval": 5000000.0 } }, "default_iallocator": "hail", "default_iallocator_params": {}, "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "protocol": "C", "resync-rate": 1024 }, "ext": { "access": "kernelspace" }, "file": {}, "gluster": { "access": "kernelspace", "host": "127.0.0.1", "port": 24007, "volume": "gv0" }, "plain": { "stripes": 2 }, "rbd": { "access": "kernelspace", "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "enabled_user_shutdown": false, "file_storage_dir": "", "gluster_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": { "migration_mode": "live" }, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_aio": "threads", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_caps": "", "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "user_shutdown": false, "vga": "", "vhost_net": false, "virtio_net_queues": 1, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false, "vnet_hdr": true }, "lxc": { "cpu_mask": "", "devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "drop_capabilities": "mac_override,sys_boot,sys_module,sys_time,sys_admin", "extra_cgroups": "", "extra_config": "", "lxc_cgroup_use": "", "lxc_devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "lxc_drop_capabilities": "mac_override,sys_boot,sys_module,sys_time", "lxc_extra_config": "", "lxc_startup_wait": 30, "lxc_tty": 6, "num_ttys": 6, "startup_timeout": 30 }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "soundhw": "", "use_localtime": false, "vif_script": "", "vif_type": "ioemu", "viridian": false, "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xm" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "soundhw": "", "use_bootloader": false, "vif_script": "", "xen_cmd": "xm" } }, "install_image": "", "instance_communication_network": "", "ipolicy": { "disk-templates": [ "drbd", "plain", "sharedfile", "file" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "max_running_jobs": 20, "max_tracked_jobs": 25, "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.7947099, "ndparams": { "cpu_speed": 1.0, "exclusive_storage": false, "oob_program": "", "ovs": false, "ovs_link": "", "ovs_name": "switch1", "spindle_count": 1, "ssh_port": 22 }, "nicparams": { "default": { "link": "br974", "mode": "bridged", "vlan": "" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "osparams_private_cluster": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32104, 32105, 32101, 32102, 32103 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg", "zeroing_image": "" }, "ctime": 1343869045.6055231, "disks": { "150bd154-8e23-44d1-b762-5065ae5a507b": { "ctime": 1354038435.343601, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "mtime": 1354038435.343601, "nodes": [ "2ae3d962-2dad-44f2-bdb1-85f77107f907" ], "params": {}, "serial_no": 1, "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" }, "77ced3a5-6756-49ae-8d1f-274e27664c05": { "children": [ { "ctime": 1421677173.7280669, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "mtime": 1421677173.7280591, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024 }, { "ctime": 1421677173.728096, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "mtime": 1421677173.7280879, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 128 } ], "ctime": 1363620258.6089759, "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "mtime": 1363620258.6089759, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" }, "79acf611-be58-4334-9fe4-4f2b73ae8abb": { "ctime": 1355186880.4511809, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "mtime": 1355186880.4511809, "nodes": [ "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } }, "filters": {}, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1354038435.343601, "disks": [ "150bd154-8e23-44d1-b762-5065ae5a507b" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1363620258.6089759, "disks": [ "77ced3a5-6756-49ae-8d1f-274e27664c05" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.8749011, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "osparams_private": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1355186880.4511809, "disks": [ "79acf611-be58-4334-9fe4-4f2b73ae8abb" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1421677173.729104, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.5750091, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2000000000000002, "vcpu-ratio": 3.1400000000000001 }, "mtime": 1361963775.5750091, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.6048839, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.9348071, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.8853681, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.3533289, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "serial_no": 7627, "version": 2150000 } ganeti-3.1.0~rc2/test/data/cluster_config_2.16.json000064400000000000000000000432071476477700300220510ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_certs": {}, "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "compression_tools": [ "gzip", "gzip-fast", "gzip-slow" ], "ctime": 1343869045.6048839, "data_collectors": { "cpu-avg-load": { "active": true, "interval": 5000000.0 }, "diskstats": { "active": true, "interval": 5000000.0 }, "drbd": { "active": true, "interval": 5000000.0 }, "inst-status-xen": { "active": true, "interval": 5000000.0 }, "lv": { "active": true, "interval": 5000000.0 }, "xen-cpu-avg-load": { "active": true, "interval": 5000000.0 } }, "default_iallocator": "hail", "default_iallocator_params": {}, "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "protocol": "C", "resync-rate": 1024 }, "ext": { "access": "kernelspace" }, "file": {}, "gluster": { "access": "kernelspace", "host": "127.0.0.1", "port": 24007, "volume": "gv0" }, "plain": { "stripes": 2 }, "rbd": { "access": "kernelspace", "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "enabled_user_shutdown": false, "file_storage_dir": "", "gluster_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": { "migration_mode": "live" }, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_aio": "threads", "disk_cache": "default", "disk_discard": "ignore", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_caps": "", "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "user_shutdown": false, "vga": "", "vhost_net": false, "virtio_net_queues": 1, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false, "vnet_hdr": true }, "lxc": { "cpu_mask": "", "devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "drop_capabilities": "mac_override,sys_boot,sys_module,sys_time,sys_admin", "extra_cgroups": "", "extra_config": "", "lxc_cgroup_use": "", "lxc_devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "lxc_drop_capabilities": "mac_override,sys_boot,sys_module,sys_time", "lxc_extra_config": "", "lxc_startup_wait": 30, "lxc_tty": 6, "num_ttys": 6, "startup_timeout": 30 }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "soundhw": "", "use_localtime": false, "vif_script": "", "vif_type": "ioemu", "viridian": false, "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xm" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "soundhw": "", "use_bootloader": false, "vif_script": "", "xen_cmd": "xm" } }, "install_image": "", "instance_communication_network": "", "ipolicy": { "disk-templates": [ "drbd", "plain", "sharedfile", "file" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "max_running_jobs": 20, "max_tracked_jobs": 25, "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.7947099, "ndparams": { "cpu_speed": 1.0, "exclusive_storage": false, "oob_program": "", "ovs": false, "ovs_link": "", "ovs_name": "switch1", "spindle_count": 1, "ssh_port": 22 }, "nicparams": { "default": { "link": "br974", "mode": "bridged", "vlan": "" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "osparams_private_cluster": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32104, 32105, 32101, 32102, 32103 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg", "zeroing_image": "" }, "ctime": 1343869045.6055231, "disks": { "150bd154-8e23-44d1-b762-5065ae5a507b": { "ctime": 1354038435.343601, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "mtime": 1354038435.343601, "nodes": [ "2ae3d962-2dad-44f2-bdb1-85f77107f907" ], "params": {}, "serial_no": 1, "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" }, "77ced3a5-6756-49ae-8d1f-274e27664c05": { "children": [ { "ctime": 1421677173.7280669, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "mtime": 1421677173.7280591, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024 }, { "ctime": 1421677173.728096, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "mtime": 1421677173.7280879, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 128 } ], "ctime": 1363620258.6089759, "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "mtime": 1363620258.6089759, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" }, "79acf611-be58-4334-9fe4-4f2b73ae8abb": { "ctime": 1355186880.4511809, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "mtime": 1355186880.4511809, "nodes": [ "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } }, "filters": {}, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1354038435.343601, "disks": [ "150bd154-8e23-44d1-b762-5065ae5a507b" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1363620258.6089759, "disks": [ "77ced3a5-6756-49ae-8d1f-274e27664c05" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.8749011, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "osparams_private": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1355186880.4511809, "disks": [ "79acf611-be58-4334-9fe4-4f2b73ae8abb" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1421677173.729104, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.5750091, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2000000000000002, "vcpu-ratio": 3.1400000000000001 }, "mtime": 1361963775.5750091, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.6048839, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.9348071, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.8853681, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.3533289, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "serial_no": 7627, "version": 2160000 } ganeti-3.1.0~rc2/test/data/cluster_config_2.7.json000064400000000000000000000332171476477700300217710ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "ctime": 1343869045.604884, "default_iallocator": "hail", "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "resync-rate": 1024 }, "ext": {}, "file": {}, "plain": { "stripes": 2 }, "rbd": { "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_hypervisors": [ "xen-pvm" ], "file_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": {}, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "vga": "", "vhost_net": false, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false }, "lxc": { "cpu_mask": "" }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "use_localtime": false, "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "use_bootloader": false } }, "ipolicy": { "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "node1.example.com", "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.79471, "ndparams": { "exclusive_storage": false, "oob_program": "", "spindle_count": 1 }, "nicparams": { "default": { "link": "br974", "mode": "bridged" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32101, 32102, 32103, 32104, 32105 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg" }, "ctime": 1343869045.605523, "instances": { "instance1.example.com": { "admin_state": "up", "beparams": {}, "ctime": 1363620258.608976, "disk_template": "drbd", "disks": [ { "children": [ { "dev_type": "lvm", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "params": {}, "physical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "size": 1024 }, { "dev_type": "lvm", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "params": {}, "physical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "size": 128 } ], "dev_type": "drbd8", "iv_name": "disk/0", "logical_id": [ "node1.example.com", "node3.example.com", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "params": {}, "physical_id": [ "198.51.100.82", 32100, "198.51.100.84", 32100, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "size": 1024 } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.874901, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {} } ], "os": "busybox", "osparams": {}, "primary_node": "node1.example.com", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "instance2.example.com": { "admin_state": "up", "beparams": {}, "ctime": 1355186880.451181, "disk_template": "plain", "disks": [ { "dev_type": "lvm", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "params": {}, "physical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "size": 102400 } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {} } ], "os": "debian-image", "osparams": {}, "primary_node": "node3.example.com", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" }, "instance3.example.com": { "admin_state": "up", "beparams": {}, "ctime": 1354038435.343601, "disk_template": "plain", "disks": [ { "dev_type": "lvm", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "params": {}, "physical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "size": 1280 } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {} } ], "os": "debian-image", "osparams": {}, "primary_node": "node2.example.com", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" } }, "mtime": 1361984633.373014, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "max": {}, "min": {}, "std": {} }, "mtime": 1361963775.575009, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "max": { "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2 }, "spindle-ratio": 5.2, "std": {}, "vcpu-ratio": 3.14 }, "mtime": 1361963775.575009, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "node1.example.com": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.353329, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true }, "node2.example.com": { "ctime": 1343869045.604884, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "node3.example.com": { "ctime": 1343869205.934807, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.885368, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true } }, "serial_no": 7624, "version": 2070000 } ganeti-3.1.0~rc2/test/data/cluster_config_2.8.json000064400000000000000000000350531476477700300217720ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "ctime": 1343869045.604884, "default_iallocator": "hail", "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "resync-rate": 1024 }, "ext": {}, "file": {}, "plain": { "stripes": 2 }, "rbd": { "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "file_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": {}, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "vga": "", "vhost_net": false, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false }, "lxc": { "cpu_mask": "" }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "use_localtime": false, "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "use_bootloader": false } }, "ipolicy": { "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "node1.example.com", "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.79471, "ndparams": { "exclusive_storage": false, "oob_program": "", "spindle_count": 1 }, "nicparams": { "default": { "link": "br974", "mode": "bridged" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32101, 32102, 32103, 32104, 32105 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg" }, "ctime": 1343869045.605523, "instances": { "instance1.example.com": { "admin_state": "up", "beparams": {}, "ctime": 1363620258.608976, "disk_template": "drbd", "disks": [ { "children": [ { "dev_type": "lvm", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "params": {}, "physical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "size": 1024 }, { "dev_type": "lvm", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "params": {}, "physical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "size": 128 } ], "dev_type": "drbd8", "iv_name": "disk/0", "logical_id": [ "node1.example.com", "node3.example.com", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "params": {}, "physical_id": [ "198.51.100.82", 32100, "198.51.100.84", 32100, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.874901, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "primary_node": "node1.example.com", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "instance2.example.com": { "admin_state": "up", "beparams": {}, "ctime": 1355186880.451181, "disk_template": "plain", "disks": [ { "dev_type": "lvm", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "params": {}, "physical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "primary_node": "node3.example.com", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" }, "instance3.example.com": { "admin_state": "up", "beparams": {}, "ctime": 1354038435.343601, "disk_template": "plain", "disks": [ { "dev_type": "lvm", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "params": {}, "physical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "primary_node": "node2.example.com", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" } }, "mtime": 1367352404.758083, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.575009, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2, "vcpu-ratio": 3.14 }, "mtime": 1361963775.575009, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "node1.example.com": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.353329, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true }, "node2.example.com": { "ctime": 1343869045.604884, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "node3.example.com": { "ctime": 1343869205.934807, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.885368, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true } }, "serial_no": 7625, "version": 2080000 } ganeti-3.1.0~rc2/test/data/cluster_config_2.9.json000064400000000000000000000465211476477700300217750ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "ctime": 1343869045.604884, "default_iallocator": "hail", "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "resync-rate": 1024 }, "ext": {}, "file": {}, "plain": { "stripes": 2 }, "rbd": { "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "file_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": {}, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "vga": "", "vhost_net": false, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false }, "lxc": { "cpu_mask": "" }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "use_localtime": false, "vif_script": "", "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xm" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "use_bootloader": false, "vif_script": "", "xen_cmd": "xm" } }, "ipolicy": { "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.79471, "ndparams": { "exclusive_storage": false, "oob_program": "", "spindle_count": 1 }, "nicparams": { "default": { "link": "br974", "mode": "bridged" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32101, 32102, 32103, 32104, 32105 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg" }, "ctime": 1343869045.605523, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "beparams": {}, "ctime": 1354038435.343601, "disk_template": "plain", "disks": [ { "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "params": {}, "physical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "beparams": {}, "ctime": 1363620258.608976, "disk_template": "drbd", "disks": [ { "children": [ { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "params": {}, "physical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "size": 1024 }, { "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "params": {}, "physical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "size": 128 } ], "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "params": {}, "physical_id": [ "198.51.100.82", 32100, "198.51.100.84", 32100, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.874901, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "beparams": {}, "ctime": 1355186880.451181, "disk_template": "plain", "disks": [ { "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "params": {}, "physical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } ], "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1367352404.758083, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.575009, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2, "vcpu-ratio": 3.14 }, "mtime": 1361963775.575009, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.604884, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.934807, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.885368, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.353329, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "serial_no": 7625, "version": 2090000 } ganeti-3.1.0~rc2/test/data/cluster_config_3.0.json000064400000000000000000000432741476477700300217670ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_certs": {}, "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "compression_tools": [ "gzip", "gzip-fast", "gzip-slow" ], "ctime": 1343869045.6048839, "data_collectors": { "cpu-avg-load": { "active": true, "interval": 5000000.0 }, "diskstats": { "active": true, "interval": 5000000.0 }, "drbd": { "active": true, "interval": 5000000.0 }, "inst-status-xen": { "active": true, "interval": 5000000.0 }, "lv": { "active": true, "interval": 5000000.0 }, "xen-cpu-avg-load": { "active": true, "interval": 5000000.0 } }, "default_iallocator": "hail", "default_iallocator_params": {}, "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "protocol": "C", "resync-rate": 1024 }, "ext": { "access": "kernelspace" }, "file": {}, "gluster": { "access": "kernelspace", "host": "127.0.0.1", "port": 24007, "volume": "gv0" }, "plain": { "stripes": 2 }, "rbd": { "access": "kernelspace", "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "enabled_user_shutdown": false, "file_storage_dir": "", "gluster_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": { "migration_mode": "live" }, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_aio": "threads", "disk_cache": "default", "disk_discard": "ignore", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_caps": "", "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "user_shutdown": false, "vga": "", "vhost_net": false, "virtio_net_queues": 1, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false, "vnet_hdr": true }, "lxc": { "cpu_mask": "", "devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "drop_capabilities": "mac_override,sys_boot,sys_module,sys_time,sys_admin", "extra_cgroups": "", "extra_config": "", "lxc_cgroup_use": "", "lxc_devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "lxc_drop_capabilities": "mac_override,sys_boot,sys_module,sys_time", "lxc_extra_config": "", "lxc_startup_wait": 30, "lxc_tty": 6, "num_ttys": 6, "startup_timeout": 30 }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "soundhw": "", "use_localtime": false, "vif_script": "", "vif_type": "ioemu", "viridian": false, "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xl" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "soundhw": "", "use_bootloader": false, "vif_script": "", "xen_cmd": "xl" } }, "install_image": "", "instance_communication_network": "", "ipolicy": { "disk-templates": [ "drbd", "plain", "sharedfile", "file" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "max_running_jobs": 20, "max_tracked_jobs": 25, "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.7947099, "ndparams": { "cpu_speed": 1.0, "exclusive_storage": false, "oob_program": "", "ovs": false, "ovs_link": "", "ovs_name": "switch1", "spindle_count": 1, "ssh_port": 22 }, "nicparams": { "default": { "link": "br974", "mode": "bridged", "vlan": "" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "osparams_private_cluster": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32104, 32105, 32101, 32102, 32103 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg", "zeroing_image": "", "ssh_key_type": "rsa", "ssh_key_bits": 2048 }, "ctime": 1343869045.6055231, "disks": { "150bd154-8e23-44d1-b762-5065ae5a507b": { "ctime": 1354038435.343601, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "mtime": 1354038435.343601, "nodes": [ "2ae3d962-2dad-44f2-bdb1-85f77107f907" ], "params": {}, "serial_no": 1, "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" }, "77ced3a5-6756-49ae-8d1f-274e27664c05": { "children": [ { "ctime": 1421677173.7280669, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "mtime": 1421677173.7280591, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024 }, { "ctime": 1421677173.728096, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "mtime": 1421677173.7280879, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 128 } ], "ctime": 1363620258.6089759, "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "mtime": 1363620258.6089759, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" }, "79acf611-be58-4334-9fe4-4f2b73ae8abb": { "ctime": 1355186880.4511809, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "mtime": 1355186880.4511809, "nodes": [ "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } }, "filters": {}, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1354038435.343601, "disks": [ "150bd154-8e23-44d1-b762-5065ae5a507b" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1363620258.6089759, "disks": [ "77ced3a5-6756-49ae-8d1f-274e27664c05" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.8749011, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "osparams_private": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1355186880.4511809, "disks": [ "79acf611-be58-4334-9fe4-4f2b73ae8abb" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1421677173.729104, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.5750091, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2000000000000002, "vcpu-ratio": 3.1400000000000001 }, "mtime": 1361963775.5750091, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.6048839, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.9348071, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.8853681, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.3533289, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "serial_no": 7627, "version": 3000000 } ganeti-3.1.0~rc2/test/data/cluster_config_3.1.json000064400000000000000000000432741476477700300217700ustar00rootroot00000000000000{ "cluster": { "beparams": { "default": { "always_failover": false, "auto_balance": true, "maxmem": 128, "minmem": 128, "spindle_use": 1, "vcpus": 1 } }, "blacklisted_os": [], "candidate_certs": {}, "candidate_pool_size": 10, "cluster_name": "cluster.name.example.com", "compression_tools": [ "gzip", "gzip-fast", "gzip-slow" ], "ctime": 1343869045.6048839, "data_collectors": { "cpu-avg-load": { "active": true, "interval": 5000000.0 }, "diskstats": { "active": true, "interval": 5000000.0 }, "drbd": { "active": true, "interval": 5000000.0 }, "inst-status-xen": { "active": true, "interval": 5000000.0 }, "lv": { "active": true, "interval": 5000000.0 }, "xen-cpu-avg-load": { "active": true, "interval": 5000000.0 } }, "default_iallocator": "hail", "default_iallocator_params": {}, "disk_state_static": {}, "diskparams": { "blockdev": {}, "diskless": {}, "drbd": { "c-delay-target": 1, "c-fill-target": 200, "c-max-rate": 2048, "c-min-rate": 1024, "c-plan-ahead": 1, "data-stripes": 2, "disk-barriers": "bf", "disk-custom": "", "dynamic-resync": false, "meta-barriers": true, "meta-stripes": 2, "metavg": "xenvg", "net-custom": "", "protocol": "C", "resync-rate": 1024 }, "ext": { "access": "kernelspace" }, "file": {}, "gluster": { "access": "kernelspace", "host": "127.0.0.1", "port": 24007, "volume": "gv0" }, "plain": { "stripes": 2 }, "rbd": { "access": "kernelspace", "pool": "rbd" }, "sharedfile": {} }, "drbd_usermode_helper": "/bin/true", "enabled_disk_templates": [ "drbd", "plain", "file", "sharedfile" ], "enabled_hypervisors": [ "xen-pvm" ], "enabled_user_shutdown": false, "file_storage_dir": "", "gluster_storage_dir": "", "hidden_os": [], "highest_used_port": 32105, "hv_state_static": { "xen-pvm": { "cpu_node": 1, "cpu_total": 1, "mem_hv": 0, "mem_node": 0, "mem_total": 0 } }, "hvparams": { "chroot": { "init_script": "/ganeti-chroot" }, "fake": { "migration_mode": "live" }, "kvm": { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_aio": "threads", "disk_cache": "default", "disk_discard": "ignore", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-kvmU", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 4, "migration_caps": "", "migration_downtime": 30, "migration_mode": "live", "migration_port": 4041, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "user_shutdown": false, "vga": "", "vhost_net": false, "virtio_net_queues": 1, "vnc_bind_address": "", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false, "vnet_hdr": true }, "lxc": { "cpu_mask": "", "devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "drop_capabilities": "mac_override,sys_boot,sys_module,sys_time,sys_admin", "extra_cgroups": "", "extra_config": "", "lxc_cgroup_use": "", "lxc_devices": "c 1:3 rw,c 1:5 rw,c 1:7 rw,c 1:8 rw,c 1:9 rw,c 1:10 rw,c 5:0 rw,c 5:1 rw,c 5:2 rw,c 136:* rw", "lxc_drop_capabilities": "mac_override,sys_boot,sys_module,sys_time", "lxc_extra_config": "", "lxc_startup_wait": 30, "lxc_tty": 6, "num_ttys": 6, "startup_timeout": 30 }, "xen-hvm": { "acpi": true, "blockdev_prefix": "hd", "boot_order": "cd", "cdrom_image_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "device_model": "/usr/lib/xen/bin/qemu-dm", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "migration_mode": "non-live", "migration_port": 8082, "nic_type": "rtl8139", "pae": true, "pci_pass": "", "reboot_behavior": "reboot", "soundhw": "", "use_localtime": false, "vif_script": "", "vif_type": "ioemu", "viridian": false, "vnc_bind_address": "0.0.0.0", "vnc_password_file": "/your/vnc-cluster-password", "xen_cmd": "xl" }, "xen-pvm": { "blockdev_prefix": "sd", "bootloader_args": "", "bootloader_path": "", "cpu_cap": 0, "cpu_mask": "all", "cpu_weight": 256, "cpuid": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "/boot/vmlinuz-xenU", "migration_mode": "live", "migration_port": 8082, "reboot_behavior": "reboot", "root_path": "/dev/xvda1", "soundhw": "", "use_bootloader": false, "vif_script": "", "xen_cmd": "xl" } }, "install_image": "", "instance_communication_network": "", "ipolicy": { "disk-templates": [ "drbd", "plain", "sharedfile", "file" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 12 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "mac_prefix": "aa:bb:cc", "maintain_node_health": false, "master_ip": "192.0.2.87", "master_netdev": "eth0", "master_netmask": 32, "master_node": "9a12d554-75c0-4cb1-8064-103365145db0", "max_running_jobs": 20, "max_tracked_jobs": 25, "modify_etc_hosts": true, "modify_ssh_setup": true, "mtime": 1361964122.7947099, "ndparams": { "cpu_speed": 1.0, "exclusive_storage": false, "oob_program": "", "ovs": false, "ovs_link": "", "ovs_name": "switch1", "spindle_count": 1, "ssh_port": 22 }, "nicparams": { "default": { "link": "br974", "mode": "bridged", "vlan": "" } }, "os_hvp": { "TEMP-Ganeti-QA-OS": { "xen-hvm": { "acpi": false, "pae": true }, "xen-pvm": { "root_path": "/dev/sda5" } } }, "osparams": {}, "osparams_private_cluster": {}, "prealloc_wipe_disks": false, "primary_ip_family": 2, "reserved_lvs": [], "rsahostkeypub": "YOURKEY", "serial_no": 3189, "shared_file_storage_dir": "/srv/ganeti/shared-file-storage", "tags": [ "mytag" ], "tcpudp_port_pool": [ 32104, 32105, 32101, 32102, 32103 ], "uid_pool": [], "use_external_mip_script": false, "uuid": "dddf8c12-f2d8-4718-a35b-7804daf12a3f", "volume_group_name": "xenvg", "zeroing_image": "", "ssh_key_type": "rsa", "ssh_key_bits": 2048 }, "ctime": 1343869045.6055231, "disks": { "150bd154-8e23-44d1-b762-5065ae5a507b": { "ctime": 1354038435.343601, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "b27a576a-13f7-4f07-885c-63fcad4fdfcc.disk0" ], "mode": "rw", "mtime": 1354038435.343601, "nodes": [ "2ae3d962-2dad-44f2-bdb1-85f77107f907" ], "params": {}, "serial_no": 1, "size": 1280, "uuid": "150bd154-8e23-44d1-b762-5065ae5a507b" }, "77ced3a5-6756-49ae-8d1f-274e27664c05": { "children": [ { "ctime": 1421677173.7280669, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_data" ], "mtime": 1421677173.7280591, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024 }, { "ctime": 1421677173.728096, "dev_type": "plain", "logical_id": [ "xenvg", "5c390722-6a7a-4bb4-9cef-98d896a8e6b1.disk0_meta" ], "mtime": 1421677173.7280879, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 128 } ], "ctime": 1363620258.6089759, "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a", 32100, 0, 0, "d3c3fd475fcbaf5fd177fb245ac43b71247ada38" ], "mode": "rw", "mtime": 1363620258.6089759, "nodes": [ "9a12d554-75c0-4cb1-8064-103365145db0", "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 1024, "uuid": "77ced3a5-6756-49ae-8d1f-274e27664c05" }, "79acf611-be58-4334-9fe4-4f2b73ae8abb": { "ctime": 1355186880.4511809, "dev_type": "plain", "iv_name": "disk/0", "logical_id": [ "xenvg", "3e559cd7-1024-4294-a923-a9fd13182b2f.disk0" ], "mode": "rw", "mtime": 1355186880.4511809, "nodes": [ "41f9c238-173c-4120-9e41-04ad379b647a" ], "params": {}, "serial_no": 1, "size": 102400, "uuid": "79acf611-be58-4334-9fe4-4f2b73ae8abb" } }, "filters": {}, "instances": { "4e091bdc-e205-4ed7-8a47-0c9130a6619f": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1354038435.343601, "disks": [ "150bd154-8e23-44d1-b762-5065ae5a507b" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1354224585.700732, "name": "instance3.example.com", "nics": [ { "mac": "aa:bb:cc:5e:5c:75", "nicparams": {}, "uuid": "1ab090c1-e017-406c-afb4-fc285cb43e31" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "serial_no": 4, "tags": [], "uuid": "4e091bdc-e205-4ed7-8a47-0c9130a6619f" }, "6c078d22-3eb6-4780-857d-81772e09eef1": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1363620258.6089759, "disks": [ "77ced3a5-6756-49ae-8d1f-274e27664c05" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1363620320.8749011, "name": "instance1.example.com", "nics": [ { "mac": "aa:bb:cc:b2:6e:0b", "nicparams": {}, "uuid": "2c953d72-fac4-4aa9-a225-4131bb271791" } ], "os": "busybox", "osparams": {}, "osparams_private": {}, "primary_node": "9a12d554-75c0-4cb1-8064-103365145db0", "serial_no": 2, "uuid": "6c078d22-3eb6-4780-857d-81772e09eef1" }, "8fde9f6d-e1f1-4850-9e9c-154966f622f5": { "admin_state": "up", "admin_state_source": "admin", "beparams": {}, "ctime": 1355186880.4511809, "disks": [ "79acf611-be58-4334-9fe4-4f2b73ae8abb" ], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1355186898.307642, "name": "instance2.example.com", "nics": [ { "mac": "aa:bb:cc:56:83:fb", "nicparams": {}, "uuid": "1cf95562-e676-4fd0-8214-e8b84a2f7bd1" } ], "os": "debian-image", "osparams": {}, "osparams_private": {}, "primary_node": "41f9c238-173c-4120-9e41-04ad379b647a", "serial_no": 2, "tags": [], "uuid": "8fde9f6d-e1f1-4850-9e9c-154966f622f5" } }, "mtime": 1421677173.729104, "networks": { "99f0128a-1c84-44da-90b9-9581ea00c075": { "ext_reservations": "1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", "name": "a network", "network": "203.0.113.0/24", "reservations": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "serial_no": 1, "uuid": "99f0128a-1c84-44da-90b9-9581ea00c075" } }, "nodegroups": { "5244a46d-7506-4e14-922d-02b58153dde1": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": {}, "mtime": 1361963775.5750091, "name": "default", "ndparams": {}, "networks": {}, "serial_no": 125, "tags": [], "uuid": "5244a46d-7506-4e14-922d-02b58153dde1" }, "6c0a8916-b719-45ad-95dd-82192b1e473f": { "alloc_policy": "preferred", "diskparams": {}, "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 18, "spindle-use": 14 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 5.2000000000000002, "vcpu-ratio": 3.1400000000000001 }, "mtime": 1361963775.5750091, "name": "another", "ndparams": { "exclusive_storage": true }, "networks": {}, "serial_no": 125, "tags": [], "uuid": "6c0a8916-b719-45ad-95dd-82192b1e473f" } }, "nodes": { "2ae3d962-2dad-44f2-bdb1-85f77107f907": { "ctime": 1343869045.6048839, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1358348755.779906, "name": "node2.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.83", "secondary_ip": "198.51.100.83", "serial_no": 6, "tags": [], "uuid": "2ae3d962-2dad-44f2-bdb1-85f77107f907", "vm_capable": true }, "41f9c238-173c-4120-9e41-04ad379b647a": { "ctime": 1343869205.9348071, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1353019704.8853681, "name": "node3.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.84", "secondary_ip": "198.51.100.84", "serial_no": 2, "tags": [], "uuid": "41f9c238-173c-4120-9e41-04ad379b647a", "vm_capable": true }, "9a12d554-75c0-4cb1-8064-103365145db0": { "ctime": 1349722460.022264, "drained": false, "group": "5244a46d-7506-4e14-922d-02b58153dde1", "master_candidate": true, "master_capable": true, "mtime": 1359986533.3533289, "name": "node1.example.com", "ndparams": {}, "offline": false, "powered": true, "primary_ip": "192.0.2.82", "secondary_ip": "198.51.100.82", "serial_no": 197, "tags": [], "uuid": "9a12d554-75c0-4cb1-8064-103365145db0", "vm_capable": true } }, "serial_no": 7627, "version": 3010000 } ganeti-3.1.0~rc2/test/data/htools/000075500000000000000000000000001476477700300170045ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/htools/clean-nonzero-score.data000064400000000000000000000013301476477700300235170ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|91552|0|91424|953674|953674|16|N|fake-uuid-01|1 node-01-002|91552|0|91296|953674|953674|16|N|fake-uuid-01|1 node-01-003|91552|0|91296|953674|953674|16|N|fake-uuid-01|1 new-0|128|1024|1|running|Y|node-01-003||diskless||1 new-1|128|1024|1|running|Y|node-01-002||diskless||1 new-2|128|1024|1|running|Y|node-01-001||diskless||1 new-3|128|1024|1|running|Y|node-01-003||diskless||1 new-4|128|1024|1|running|Y|node-01-002||diskless||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/common-suffix.data000064400000000000000000000007221476477700300224320ustar00rootroot00000000000000default|fake-uuid-01|preferred|| node1.example.com|1024|0|1024|95367|95367|4|N|fake-uuid-01|1 node2.example.com|1024|0|896|95367|94343|4|N|fake-uuid-01|1 instance1.example.com|128|1024|1|running|Y|node2.example.com||plain| |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,1|diskless,file,sharedfile,plain,blockdev,drbd,rbd|4.0|32.0 default|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,1|diskless,file,sharedfile,plain,blockdev,drbd,rbd|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/empty-cluster.data000064400000000000000000000004401476477700300224520ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hail-alloc-dedicated-1.json000064400000000000000000000135641476477700300237570ustar00rootroot00000000000000{ "cluster_tags": [], "instances": { "instance2-quarter": { "disk_space_total": 10240, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 10240, "spindles": 1 } ], "memory": 1024, "nics": [], "nodes": [ "node2-quarter" ], "spindle_use": 1, "tags": [], "vcpus": 1 }, "instance3-half": { "disk_space_total": 20480, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 20480, "spindles": 2 } ], "memory": 2048, "nics": [], "nodes": [ "node3-half" ], "spindle_use": 2, "tags": [], "vcpus": 2 }, "instance4-full": { "disk_space_total": 40960, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 40960, "spindles": 4 } ], "memory": 4096, "nics": [], "nodes": [ "node4-full" ], "spindle_use": 4, "tags": [], "vcpus": 4 } }, "nodegroups": { "uuid-group-1": { "alloc_policy": "preferred", "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 1, "disk-count": 1, "disk-size": 10240, "memory-size": 1024, "nic-count": 1, "spindle-use": 1 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 10220, "memory-size": 1022, "nic-count": 1, "spindle-use": 1 } }, { "max": { "cpu-count": 2, "disk-count": 2, "disk-size": 20480, "memory-size": 2048, "nic-count": 1, "spindle-use": 2 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 20460, "memory-size": 2046, "nic-count": 1, "spindle-use": 2 } }, { "max": { "cpu-count": 4, "disk-count": 4, "disk-size": 40960, "memory-size": 4096, "nic-count": 1, "spindle-use": 4 }, "min": { "cpu-count": 4, "disk-count": 4, "disk-size": 40940, "memory-size": 4094, "nic-count": 1, "spindle-use": 4 } } ], "spindle-ratio": 1.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 10240, "memory-size": 1024, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "name": "default", "networks": [], "tags": [] } }, "nodes": { "node1-empty": { "drained": false, "free_disk": 40960, "free_memory": 4096, "free_spindles": 4, "group": "uuid-group-1", "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": true, "spindle_count": 1 }, "offline": false, "primary_ip": "192.0.2.1", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "198.51.100.1", "tags": [], "total_cpus": 5, "total_disk": 40960, "total_memory": 4096, "total_spindles": 5, "vm_capable": true }, "node2-quarter": { "drained": false, "free_disk": 30720, "free_memory": 3072, "free_spindles": 3, "group": "uuid-group-1", "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": true, "spindle_count": 1 }, "offline": false, "primary_ip": "192.0.2.2", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "198.51.100.2", "tags": [], "total_cpus": 5, "total_disk": 40960, "total_memory": 4096, "total_spindles": 5, "vm_capable": true }, "node3-half": { "drained": false, "free_disk": 20480, "free_memory": 2048, "free_spindles": 2, "group": "uuid-group-1", "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": true, "spindle_count": 1 }, "offline": false, "primary_ip": "192.0.2.3", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "198.51.100.3", "tags": [], "total_cpus": 5, "total_disk": 40960, "total_memory": 4096, "total_spindles": 5, "vm_capable": true }, "node4-full": { "drained": false, "free_disk": 0, "free_memory": 0, "free_spindles": 0, "group": "uuid-group-1", "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": true, "spindle_count": 1 }, "offline": false, "primary_ip": "192.0.2.4", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "198.51.100.4", "tags": [], "total_cpus": 5, "total_disk": 40960, "total_memory": 4096, "total_spindles": 5, "vm_capable": true } }, "request": { "disk_space_total": 10230, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 10230, "spindles": 1 } ], "hypervisor": "xen-pvm", "memory": 1023, "name": "instance-new-quarter", "nics": [], "os": "instance-debootstrap", "required_nodes": 1, "spindle_use": 1, "tags": [], "type": "allocate", "vcpus": 1 }, "enabled_hypervisors": ["xen-pvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-desired-location.json000064400000000000000000000056561476477700300251430ustar00rootroot00000000000000{ "cluster_tags": [ "htools:nlocation:power", "htools:desiredlocation:power" ], "nodegroups": { "uuid-group-1": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "networks": [], "alloc_policy": "preferred", "tags": [], "name": "default" } }, "cluster_name": "cluster", "instances": {}, "nodes": { "node1": { "total_disk": 307200, "total_cpus": 4, "group": "uuid-group-1", "i_pri_up_memory": 0, "tags": [ "power:a" ], "master_candidate": true, "free_memory": 3072, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_cpus": 1, "master_capable": true, "free_disk": 307200, "drained": false, "total_memory": 3072, "i_pri_memory": 0, "reserved_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node2": { "total_disk": 307200, "total_cpus": 4, "group": "uuid-group-1", "i_pri_up_memory": 0, "tags": [ "power:b" ], "master_candidate": true, "free_memory": 4096, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_cpus": 1, "master_capable": true, "free_disk": 307200, "drained": false, "total_memory": 4096, "i_pri_memory": 0, "reserved_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false } }, "request": { "disk_space_total": 0, "disk_template": "plain", "disks": [ { "size": 1024 } ], "hypervisor": "xen-pvm", "memory": 256, "name": "instance-new", "nics": [], "os": "instance-debootstrap", "required_nodes": 1, "spindle_use": 1, "tags": [ "power:a" ], "type": "allocate", "vcpus": 1 }, "enabled_hypervisors": ["xen-pvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-drbd-restricted.json000064400000000000000000000013061476477700300247630ustar00rootroot00000000000000{ "cluster_tags": [], "instances": {}, "nodegroups": {}, "nodes": {}, "request": { "disk_space_total": 1024, "disk_template": "drbd", "disks": [ { "mode": "rw", "size": 1024, "spindles": 1 } ], "hypervisor": "xen-pvm", "memory": 1024, "name": "instance1", "nics": [ { "bridge": null, "ip": null, "mac": "00:11:22:33:44:55" } ], "os": "instance-debootstrap", "required_nodes": 2, "restrict-to-nodes": [ "node-01", "node-02", "node-03" ], "spindle_use": 1, "tags": [], "type": "allocate", "vcpus": 1 }, "enabled_hypervisors": ["xen-pvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-drbd.json000064400000000000000000000276601476477700300226300ustar00rootroot00000000000000{ "cluster_tags": [ "htools:iextags:test", "htools:iextags:service-group" ], "nodegroups": { "uuid-group-1": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "networks": [], "alloc_policy": "preferred", "tags": [], "name": "default" } }, "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "memory-size": 32768, "cpu-count": 8, "disk-count": 16, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "enabled_hypervisors": [ "xen-pvm", "xen-hvm" ], "cluster_name": "cluster", "instances": { "instance14": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:eb:0b:a5", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "spindle_use": 1, "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance13": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:9c", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance18": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 128, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:55:94:93", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 8192, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance19": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:15:92:6f", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance2": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:73:20:3e", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance3": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 }, { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 384, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:ec:e8:a2", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance4": { "disks": [ { "spindles": 2, "mode": "rw", "size": 2048 } ], "disk_space_total": 2176, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:62:b0:76", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node4", "node3" ], "os": "instance-debootstrap" }, "instance8": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 } ], "disk_space_total": 256, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:3f:6d:e3", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance9": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [ "test:test" ], "nics": [ { "ip": null, "mac": "aa:00:00:10:d2:01", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "instance-debootstrap" }, "instance20": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:db:2a:6d", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" } }, "version": 2, "nodes": { "node1": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.1", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 31389, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1377280, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.1", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node2": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.2", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 31746, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1376640, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.2", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node3": { "total_disk": 1377304, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.3", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 31234, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1373336, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.3", "i_pri_memory": 2432, "free_spindles": 6, "total_spindles": 12, "vm_capable": true, "offline": false }, "node4": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.4", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 22914, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1371520, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.4", "i_pri_memory": 23552, "free_spindles": 0, "total_spindles": 12, "vm_capable": true, "offline": false } }, "request": { "disks": [ { "spindles": 1, "mode": "rw", "size": 1024 } ], "required_nodes": 2, "name": "instance1", "tags": [], "hypervisor": "xen-pvm", "disk_space_total": 1024, "nics": [ { "ip": null, "mac": "00:11:22:33:44:55", "bridge": null } ], "vcpus": 1, "spindle_use": 1, "os": "instance-debootstrap", "disk_template": "drbd", "memory": 1024, "type": "allocate" }, "enabled_hypervisors": ["xen-pvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-ext.json000064400000000000000000000007401476477700300225030ustar00rootroot00000000000000{ "cluster_tags": [], "instances": {}, "nodegroups": {}, "nodes": {}, "request": { "disk_space_total": 0, "disk_template": "ext", "disks": [ { "size": 1024 } ], "hypervisor": "xen-pvm", "memory": 1000, "name": "instance-new", "nics": [], "os": "instance-debootstrap", "required_nodes": 1, "spindle_use": 1, "tags": [], "type": "allocate", "vcpus": 1 }, "enabled_hypervisors": ["xen-pvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-invalid-network.json000064400000000000000000000073501476477700300250240ustar00rootroot00000000000000{ "cluster_tags": [], "instances": {}, "ipolicy": { "max": { "disk-size": 2048 }, "min": { "disk-size": 1024 } }, "nodegroups": { "uuid-group-1": { "alloc_policy": "preferred", "ipolicy": { "disk-templates": [ "drbd" ], "minmax": [ { "max": { "cpu-count": 2, "disk-count": 8, "disk-size": 2048, "memory-size": 12800, "nic-count": 8, "spindle-use": 8 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 4.0 }, "name": "Group 1", "networks": ["uuid-net-1-1", "uuid-net-1-2"], "tags": [] }, "uuid-group-2": { "alloc_policy": "preferred", "ipolicy": { "disk-templates": [ "file" ], "minmax": [ { "max": { "cpu-count": 2, "disk-count": 8, "disk-size": 2048, "memory-size": 12800, "nic-count": 8, "spindle-use": 8 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 4.0 }, "name": "Group 2", "networks": ["uuid-net-2-1", "uuid-net-2-2", "uuid-net-2-3"], "tags": [] } }, "nodes": { "node1_1": { "drained": false, "free_disk": 7168, "free_memory": 4096, "free_spindles": 0, "group": "uuid-group-1", "ndparams": { "spindle_count": 1, "exclusive_storage": false }, "offline": false, "reserved_memory": 1017, "reserved_cpus": 1, "total_cpus": 4, "total_disk": 7168, "total_memory": 4096, "total_spindles": 0 }, "node2_1": { "drained": false, "free_disk": 7168, "free_memory": 4096, "free_spindles": 0, "group": "uuid-group-2", "ndparams": { "spindle_count": 1, "exclusive_storage": false }, "offline": false, "reserved_memory": 1017, "reserved_cpus": 1, "total_cpus": 4, "total_disk": 7168, "total_memory": 4096, "total_spindles": 0 } }, "request": { "disk_space_total": 1536, "disk_template": "file", "disks": [ { "size": 1536 } ], "memory": 1024, "name": "instance1", "required_nodes": 1, "spindle_use": 2, "nics":[ { "mac":"aa:00:00:85:f3:a7", "network":"uuid-net-1-1", "nicparams":{} }, { "mac":"aa:00:00:85:f3:a8", "network":"uuid-net-1-2", "nicparams":{} } ], "tags": [], "type": "allocate", "vcpus": 1 }, "enabled_hypervisors": ["xen-pvm"], "version": 2 } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-invalid-twodisks.json000064400000000000000000000037651476477700300252100ustar00rootroot00000000000000{ "cluster_tags": [], "instances": {}, "ipolicy": { "max": { "disk-size": 2048 }, "min": { "disk-size": 1024 } }, "nodegroups": { "uuid-group-1": { "alloc_policy": "preferred", "ipolicy": { "disk-templates": [ "file" ], "minmax" : [ { "max": { "cpu-count": 2, "disk-count": 8, "disk-size": 2048, "memory-size": 12800, "nic-count": 8, "spindle-use": 8 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 4.0 }, "name": "default", "networks": [], "tags": [] } }, "nodes": { "node1": { "drained": false, "free_disk": 1377280, "free_memory": 31389, "free_spindles": 12, "group": "uuid-group-1", "ndparams": { "spindle_count": 1, "exclusive_storage": false }, "offline": false, "reserved_memory": 1017, "reserved_cpus": 1, "total_cpus": 4, "total_disk": 1377280, "total_memory": 32763, "total_spindles": 12 } }, "request": { "disk_space_total": 1536, "disk_template": "file", "disks": [ { "spindles": 1, "size": 768 }, { "spindles": 1, "size": 768 } ], "memory": 1024, "name": "instance1", "required_nodes": 1, "spindle_use": 2, "tags": [], "type": "allocate", "vcpus": 1, "nics": [] }, "enabled_hypervisors": ["xen-pvm"], "version": 2 } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-nlocation.json000064400000000000000000000070001476477700300236650ustar00rootroot00000000000000{ "cluster_tags": [ "htools:nlocation:power" ], "nodegroups": { "uuid-group-1": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "networks": [], "alloc_policy": "preferred", "tags": [], "name": "default" } }, "cluster_name": "cluster", "instances": {}, "nodes": { "node1": { "total_disk": 307200, "total_cpus": 4, "group": "uuid-group-1", "i_pri_up_memory": 0, "tags": [ "power:a" ], "master_candidate": true, "free_memory": 4096, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_cpus": 1, "master_capable": true, "free_disk": 307200, "drained": false, "total_memory": 4096, "i_pri_memory": 0, "reserved_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node2": { "total_disk": 307200, "total_cpus": 4, "group": "uuid-group-1", "i_pri_up_memory": 0, "tags": [ "power:a" ], "master_candidate": true, "free_memory": 4096, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_cpus": 1, "master_capable": true, "free_disk": 307200, "drained": false, "total_memory": 4096, "i_pri_memory": 0, "reserved_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node3": { "total_disk": 107200, "total_cpus": 4, "group": "uuid-group-1", "i_pri_up_memory": 0, "tags": [ "power:b" ], "master_candidate": true, "free_memory": 1024, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_cpus": 1, "master_capable": true, "free_disk": 107200, "drained": false, "total_memory": 1024, "i_pri_memory": 0, "reserved_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false } }, "request": { "disk_space_total": 0, "disk_template": "drbd", "disks": [ { "size": 1024 } ], "hypervisor": "xen-pvm", "memory": 256, "name": "instance-new", "nics": [], "os": "instance-debootstrap", "required_nodes": 2, "spindle_use": 1, "tags": [ ], "type": "allocate", "vcpus": 1 }, "enabled_hypervisors": ["xen-pvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-plain-tags.json000064400000000000000000000135231476477700300237450ustar00rootroot00000000000000{ "cluster_tags": [ "htools:iextags:service" ], "instances": { "instance-1": { "admin_state": "up", "admin_state_source": "admin", "disk_space_total": 256, "disk_template": "drbd", "disks": [ { "mode": "rw", "size": 128, "spindles": 1 } ], "hypervisor": "xen-pvm", "memory": 128, "nics": [ { "bridge": "xen-br0", "ip": null, "link": "xen-br0", "mac": "aa:00:00:15:92:6f", "mode": "bridged" } ], "nodes": [ "node1", "node2" ], "os": "debian-image", "spindle_use": 1, "tags": [ "service:foo" ], "vcpus": 1 }, "instance-2": { "admin_state": "up", "admin_state_source": "admin", "disk_space_total": 256, "disk_template": "drbd", "disks": [ { "mode": "rw", "size": 128, "spindles": 1 } ], "hypervisor": "xen-pvm", "memory": 128, "nics": [ { "bridge": "xen-br0", "ip": null, "link": "xen-br0", "mac": "aa:00:00:15:92:6f", "mode": "bridged" } ], "nodes": [ "node2", "node3" ], "os": "debian-image", "spindle_use": 1, "tags": [ "service:foo" ], "vcpus": 1 }, "instance-3": { "admin_state": "up", "admin_state_source": "admin", "disk_space_total": 256, "disk_template": "drbd", "disks": [ { "mode": "rw", "size": 128, "spindles": 1 } ], "hypervisor": "xen-pvm", "memory": 128, "nics": [ { "bridge": "xen-br0", "ip": null, "link": "xen-br0", "mac": "aa:00:00:15:92:6f", "mode": "bridged" } ], "nodes": [ "node3", "node1" ], "os": "debian-image", "spindle_use": 1, "tags": [ "service:foo" ], "vcpus": 1 } }, "nodegroups": { "uuid-group-1": { "alloc_policy": "preferred", "ipolicy": { "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "minmax": [ { "max": { "cpu-count": 8, "disk-count": 16, "disk-size": 1048576, "memory-size": 32768, "nic-count": 8, "spindle-use": 8 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 128, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 4.0 }, "name": "default", "networks": [], "tags": [] } }, "nodes": { "node1": { "drained": false, "free_disk": 1377024, "free_memory": 32635, "free_spindles": 12, "group": "uuid-group-1", "i_pri_memory": 0, "i_pri_up_memory": 0, "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": false, "oob_program": null, "spindle_count": 1 }, "offline": false, "primary_ip": "192.168.1.1", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "192.168.2.1", "tags": [], "total_cpus": 4, "total_disk": 1377280, "total_memory": 32763, "total_spindles": 12, "vm_capable": true }, "node2": { "drained": false, "free_disk": 1377024, "free_memory": 32635, "free_spindles": 12, "group": "uuid-group-1", "i_pri_memory": 0, "i_pri_up_memory": 0, "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": false, "oob_program": null, "spindle_count": 1 }, "offline": false, "primary_ip": "192.168.1.2", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "192.168.2.2", "tags": [], "total_cpus": 4, "total_disk": 1377280, "total_memory": 32763, "total_spindles": 12, "vm_capable": true }, "node3": { "drained": false, "free_disk": 1377024, "free_memory": 32635, "free_spindles": 12, "group": "uuid-group-1", "i_pri_memory": 0, "i_pri_up_memory": 0, "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": false, "oob_program": null, "spindle_count": 1 }, "offline": false, "primary_ip": "192.168.1.3", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "192.168.2.3", "tags": [], "total_cpus": 4, "total_disk": 1377280, "total_memory": 32763, "total_spindles": 12, "vm_capable": true } }, "request": { "disk_space_total": 1024, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 1024, "spindles": 1 } ], "hypervisor": "xen-pvm", "memory": 1024, "name": "instance-new", "nics": [ { "bridge": null, "ip": null, "mac": "00:11:22:33:44:55" } ], "os": "instance-debootstrap", "required_nodes": 1, "spindle_use": 1, "tags": [ "service:foo", "service:bar" ], "type": "allocate", "vcpus": 1 }, "enabled_hypervisors": ["xen-pvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-restricted-network.json000064400000000000000000000135731476477700300255520ustar00rootroot00000000000000{ "cluster_tags": [], "instances": { "instance1": { "disks": [ { "mode": "rw", "size": 1024 } ], "disk_space_total": 1024, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:eb:0b:a5", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node1_1", "node1_2" ], "os": "debian-image" }, "instance2": { "disks": [ { "mode": "rw", "size": 1024 } ], "disk_space_total": 1024, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:eb:0b:a5", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node1_2", "node1_1" ], "os": "debian-image" } }, "ipolicy": { "max": { "disk-size": 2048 }, "min": { "disk-size": 1024 } }, "nodegroups": { "uuid-group-1": { "alloc_policy": "last_resort", "ipolicy": { "disk-templates": [ "drbd" ], "minmax": [ { "max": { "cpu-count": 2, "disk-count": 8, "disk-size": 2048, "memory-size": 12800, "nic-count": 8, "spindle-use": 8 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 4.0 }, "name": "Group 1", "networks": ["uuid-net-1-1", "uuid-net-1-2"], "tags": [] }, "uuid-group-2": { "alloc_policy": "preferred", "ipolicy": { "disk-templates": [ "drbd" ], "minmax": [ { "max": { "cpu-count": 2, "disk-count": 8, "disk-size": 2048, "memory-size": 12800, "nic-count": 8, "spindle-use": 8 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 4.0 }, "name": "Group 2", "networks": ["uuid-net-2-1", "uuid-net-2-2", "uuid-net-2-3"], "tags": [] } }, "nodes": { "node1_1": { "drained": false, "free_disk": 4096, "free_memory": 3840, "free_spindles": 0, "group": "uuid-group-1", "ndparams": { "spindle_count": 1, "exclusive_storage": false }, "offline": false, "reserved_memory": 1017, "reserved_cpus": 1, "total_cpus": 4, "total_disk": 7168, "total_memory": 4096, "total_spindles": 0 }, "node1_2": { "drained": false, "free_disk": 4096, "free_memory": 3968, "free_spindles": 0, "group": "uuid-group-1", "ndparams": { "spindle_count": 1, "exclusive_storage": false }, "offline": false, "reserved_memory": 1017, "reserved_cpus": 1, "total_cpus": 4, "total_disk": 7168, "total_memory": 32763, "total_spindles": 0 }, "node2_1": { "drained": false, "free_disk": 7168, "free_memory": 4096, "free_spindles": 0, "group": "uuid-group-2", "ndparams": { "spindle_count": 1, "exclusive_storage": false }, "offline": false, "reserved_memory": 1017, "reserved_cpus": 1, "total_cpus": 4, "total_disk": 7168, "total_memory": 4096, "total_spindles": 0 }, "node2_2": { "drained": false, "free_disk": 7168, "free_memory": 4096, "free_spindles": 0, "group": "uuid-group-2", "ndparams": { "spindle_count": 1, "exclusive_storage": false }, "offline": false, "reserved_memory": 1017, "reserved_cpus": 1, "total_cpus": 4, "total_disk": 7168, "total_memory": 4096, "total_spindles": 0 } }, "request": { "disk_space_total": 3072, "disk_template": "drbd", "disks": [ { "size": 1536 }, { "size": 1536 } ], "memory": 1024, "name": "instance1", "required_nodes": 2, "spindle_use": 2, "nics":[ { "mac":"aa:00:00:85:f3:a7", "network":"uuid-net-1-1", "nicparams":{} }, { "mac":"aa:00:00:85:f3:a8", "network":"uuid-net-1-2", "nicparams":{} } ], "tags": [], "type": "allocate", "vcpus": 1 }, "enabled_hypervisors": ["xen-pvm"], "version": 2 } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-secondary.json000064400000000000000000000146101476477700300236730ustar00rootroot00000000000000{ "cluster_tags": [], "instances": { "instance-2-3":{ "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 1024, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:9c", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node-2-3" ], "os": "instance-debootstrap" }, "instance-to-be-converted-to-drbd":{ "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 1024, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:9c", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node-2-1" ], "os": "instance-debootstrap" } }, "nodegroups": { "uuid-group-1": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "networks": [], "alloc_policy": "preferred", "tags": [], "name": "default" }, "uuid-group-2": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "networks": [], "alloc_policy": "preferred", "tags": [], "name": "default" } }, "nodes": { "node-1-1" : { "total_disk": 91552, "total_cpus": 16, "group": "uuid-group-1", "secondary_ip": "192.168.2.1", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 3100, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 0, "reserved_cpus": 1, "master_capable": true, "free_disk": 91552, "drained": false, "total_memory": 3100, "primary_ip": "192.168.1.1", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node-2-1" : { "total_disk": 91552, "total_cpus": 16, "group": "uuid-group-2", "secondary_ip": "192.168.2.101", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 3100, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 0, "reserved_cpus": 1, "master_capable": true, "free_disk": 91552, "drained": false, "total_memory": 3100, "primary_ip": "192.168.1.101", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node-2-2" : { "total_disk": 91552, "total_cpus": 16, "group": "uuid-group-2", "secondary_ip": "192.168.2.102", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 3100, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 0, "reserved_cpus": 1, "master_capable": true, "free_disk": 91552, "drained": false, "total_memory": 3100, "primary_ip": "192.168.1.102", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node-2-3" : { "total_disk": 91552, "total_cpus": 16, "group": "uuid-group-2", "secondary_ip": "192.168.2.103", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 3100, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 0, "reserved_cpus": 1, "master_capable": true, "free_disk": 91552, "drained": false, "total_memory": 3100, "primary_ip": "192.168.1.103", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false } }, "request": { "name": "instance-to-be-converted-to-drbd", "type": "allocate-secondary" }, "enabled_hypervisors": ["xen-pvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-spindles.json000064400000000000000000000217531476477700300235330ustar00rootroot00000000000000{ "cluster_tags": [ "htools:iextags:test", "htools:iextags:service-group" ], "nodegroups": { "uuid-group-1": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 2 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "alloc_policy": "preferred", "networks": [], "tags": [], "name": "group1" }, "uuid-group-2": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 2 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 2 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 3 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "alloc_policy": "preferred", "networks": [], "tags": [], "name": "group2" } }, "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "memory-size": 32768, "cpu-count": 8, "disk-count": 16, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "enabled_hypervisors": [ "xen-pvm", "xen-hvm" ], "cluster_name": "cluster", "instances": { "instance1": { "disks": [ { "spindles": 1, "mode": "rw", "size": 650000 } ], "disk_space_total": 650000, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:91", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "plain", "memory": 1024, "nodes": [ "node1" ], "os": "instance-debootstrap" }, "instance2": { "disks": [ { "spindles": 2, "mode": "rw", "size": 256 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:92", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "plain", "memory": 1024, "nodes": [ "node2" ], "os": "instance-debootstrap" }, "instance3": { "disks": [ { "spindles": 1, "mode": "rw", "size": 650000 } ], "disk_space_total": 650000, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:93", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "plain", "memory": 1024, "nodes": [ "node3" ], "os": "instance-debootstrap" }, "instance4": { "disks": [ { "spindles": 2, "mode": "rw", "size": 256 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:94", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "plain", "memory": 1024, "nodes": [ "node4" ], "os": "instance-debootstrap" } }, "version": 2, "nodes": { "node1": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.1", "i_pri_up_memory": 1024, "tags": [], "master_candidate": true, "free_memory": 30722, "ndparams": { "spindle_count": 2, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 687280, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.1", "i_pri_memory": 1024, "free_spindles": 1, "total_spindles": 2, "vm_capable": true, "offline": false }, "node2": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.2", "i_pri_up_memory": 1024, "tags": [], "master_candidate": true, "free_memory": 30722, "ndparams": { "spindle_count": 2, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1377024, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.2", "i_pri_memory": 1024, "free_spindles": 0, "total_spindles": 2, "vm_capable": true, "offline": false }, "node3": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-2", "secondary_ip": "192.168.2.3", "i_pri_up_memory": 1024, "tags": [], "master_candidate": true, "free_memory": 30722, "ndparams": { "spindle_count": 2, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 687280, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.3", "i_pri_memory": 1204, "free_spindles": 1, "total_spindles": 2, "vm_capable": true, "offline": false }, "node4": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-2", "secondary_ip": "192.168.2.4", "i_pri_up_memory": 1024, "tags": [], "master_candidate": true, "free_memory": 30722, "ndparams": { "spindle_count": 2, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1377024, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.4", "i_pri_memory": 1024, "free_spindles": 0, "total_spindles": 2, "vm_capable": true, "offline": false } }, "request": { "disks": [ { "spindles": 1, "mode": "rw", "size": 1024 } ], "required_nodes": 1, "name": "instance10", "tags": [], "hypervisor": "xen-pvm", "disk_space_total": 1024, "nics": [ { "ip": null, "mac": "00:11:22:33:44:55", "bridge": null } ], "vcpus": 1, "spindle_use": 3, "os": "instance-debootstrap", "disk_template": "plain", "memory": 1024, "type": "allocate" } } ganeti-3.1.0~rc2/test/data/htools/hail-alloc-twodisks.json000064400000000000000000000037661476477700300235650ustar00rootroot00000000000000{ "cluster_tags": [], "instances": {}, "ipolicy": { "max": { "disk-size": 2048 }, "min": { "disk-size": 1024 } }, "nodegroups": { "uuid-group-1": { "alloc_policy": "preferred", "ipolicy": { "disk-templates": [ "file" ], "minmax": [ { "max": { "cpu-count": 2, "disk-count": 8, "disk-size": 2048, "memory-size": 12800, "nic-count": 8, "spindle-use": 8 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 } } ], "spindle-ratio": 32.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 1024, "memory-size": 128, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 4.0 }, "name": "default", "networks": [], "tags": [] } }, "nodes": { "node1": { "drained": false, "free_disk": 1377280, "free_memory": 31389, "free_spindles": 12, "group": "uuid-group-1", "ndparams": { "spindle_count": 1, "exclusive_storage": false }, "offline": false, "reserved_memory": 1017, "reserved_cpus": 1, "total_cpus": 4, "total_disk": 1377280, "total_memory": 32763, "total_spindles": 12 } }, "request": { "disk_space_total": 3072, "disk_template": "file", "disks": [ { "spindles": 1, "size": 1536 }, { "spindles": 1, "size": 1536 } ], "memory": 1024, "name": "instance1", "required_nodes": 1, "spindle_use": 2, "tags": [], "type": "allocate", "vcpus": 1, "nics": [] }, "enabled_hypervisors": ["xen-pvm"], "version": 2 } ganeti-3.1.0~rc2/test/data/htools/hail-change-group.json000064400000000000000000000324151476477700300231760ustar00rootroot00000000000000{ "cluster_tags": [ "htools:iextags:test", "htools:iextags:service-group" ], "nodegroups": { "uuid-group-1": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "alloc_policy": "preferred", "networks": [], "tags": [], "name": "default" }, "uuid-group-2": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "alloc_policy": "preferred", "networks": [], "tags": [], "name": "empty" } }, "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "memory-size": 32768, "cpu-count": 8, "disk-count": 16, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "enabled_hypervisors": [ "xen-pvm", "xen-hvm" ], "cluster_name": "cluster", "instances": { "instance14": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:eb:0b:a5", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance13": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:9c", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance18": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 128, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:55:94:93", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 8192, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance19": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:15:92:6f", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance2": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:73:20:3e", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance3": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 }, { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 384, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:ec:e8:a2", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance4": { "disks": [ { "spindles": 2, "mode": "rw", "size": 2048 } ], "disk_space_total": 2176, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:62:b0:76", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node4", "node3" ], "os": "instance-debootstrap" }, "instance8": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 } ], "disk_space_total": 256, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:3f:6d:e3", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance9": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [ "test:test" ], "nics": [ { "ip": null, "mac": "aa:00:00:10:d2:01", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "instance-debootstrap" }, "instance20": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:db:2a:6d", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" } }, "version": 2, "nodes": { "node1": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.1", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 31389, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1377280, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.1", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node3": { "total_disk": 1377304, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.3", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 31234, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1373336, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.3", "i_pri_memory": 2432, "free_spindles": 6, "total_spindles": 12, "vm_capable": true, "offline": false }, "node4": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.4", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 22914, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1371520, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.4", "i_pri_memory": 23552, "free_spindles": 0, "total_spindles": 12, "vm_capable": true, "offline": false }, "node10": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-2", "secondary_ip": "192.168.2.10", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 31746, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1376640, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.10", "i_pri_memory": 23552, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node11": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-2", "secondary_ip": "192.168.2.11", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 31746, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1376640, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.11", "i_pri_memory": 23552, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false } }, "request": { "instances": [ "instance14" ], "target_groups": [], "type": "change-group" } } ganeti-3.1.0~rc2/test/data/htools/hail-invalid-reloc.json000064400000000000000000000004231476477700300233410ustar00rootroot00000000000000{ "cluster_tags": [], "nodegroups": {}, "nodes": {}, "instances": {}, "request": { "relocate_from": [ "node4" ], "required_nodes": "aaa", "type": "relocate", "name": 0, "disk_space_total": "aaa" }, "enabled_hypervisors": ["kvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-multialloc-dedicated.json000064400000000000000000000147031476477700300246700ustar00rootroot00000000000000{ "cluster_tags": [], "instances": { "instance2-quarter": { "disk_space_total": 10240, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 10240, "spindles": 1 } ], "memory": 1024, "nics": [], "nodes": [ "node2-quarter" ], "spindle_use": 1, "tags": [], "vcpus": 1 }, "instance3-half": { "disk_space_total": 20480, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 20480, "spindles": 2 } ], "memory": 2048, "nics": [], "nodes": [ "node3-half" ], "spindle_use": 2, "tags": [], "vcpus": 2 }, "instance4-full": { "disk_space_total": 40960, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 40960, "spindles": 4 } ], "memory": 4096, "nics": [], "nodes": [ "node4-full" ], "spindle_use": 4, "tags": [], "vcpus": 4 } }, "nodegroups": { "uuid-group-1": { "alloc_policy": "preferred", "ipolicy": { "disk-templates": [ "plain" ], "minmax": [ { "max": { "cpu-count": 1, "disk-count": 1, "disk-size": 10240, "memory-size": 1024, "nic-count": 1, "spindle-use": 1 }, "min": { "cpu-count": 1, "disk-count": 1, "disk-size": 10220, "memory-size": 1022, "nic-count": 1, "spindle-use": 1 } }, { "max": { "cpu-count": 2, "disk-count": 2, "disk-size": 20480, "memory-size": 2048, "nic-count": 1, "spindle-use": 2 }, "min": { "cpu-count": 2, "disk-count": 2, "disk-size": 20460, "memory-size": 2046, "nic-count": 1, "spindle-use": 2 } }, { "max": { "cpu-count": 4, "disk-count": 4, "disk-size": 40960, "memory-size": 4096, "nic-count": 1, "spindle-use": 4 }, "min": { "cpu-count": 4, "disk-count": 4, "disk-size": 40940, "memory-size": 4094, "nic-count": 1, "spindle-use": 4 } } ], "spindle-ratio": 1.0, "std": { "cpu-count": 1, "disk-count": 1, "disk-size": 10240, "memory-size": 1024, "nic-count": 1, "spindle-use": 1 }, "vcpu-ratio": 1.0 }, "name": "default", "networks": [], "tags": [] } }, "nodes": { "node1-empty": { "drained": false, "free_disk": 40960, "free_memory": 4096, "free_spindles": 4, "group": "uuid-group-1", "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": true, "spindle_count": 1 }, "offline": false, "primary_ip": "192.0.2.1", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "198.51.100.1", "tags": [], "total_cpus": 5, "total_disk": 40960, "total_memory": 4096, "total_spindles": 5, "vm_capable": true }, "node2-quarter": { "drained": false, "free_disk": 30720, "free_memory": 3072, "free_spindles": 3, "group": "uuid-group-1", "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": true, "spindle_count": 1 }, "offline": false, "primary_ip": "192.0.2.2", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "198.51.100.2", "tags": [], "total_cpus": 5, "total_disk": 40960, "total_memory": 4096, "total_spindles": 5, "vm_capable": true }, "node3-half": { "drained": false, "free_disk": 20480, "free_memory": 2048, "free_spindles": 2, "group": "uuid-group-1", "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": true, "spindle_count": 1 }, "offline": false, "primary_ip": "192.0.2.3", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "198.51.100.3", "tags": [], "total_cpus": 5, "total_disk": 40960, "total_memory": 4096, "total_spindles": 5, "vm_capable": true }, "node4-full": { "drained": false, "free_disk": 0, "free_memory": 0, "free_spindles": 0, "group": "uuid-group-1", "master_candidate": true, "master_capable": true, "ndparams": { "exclusive_storage": true, "spindle_count": 1 }, "offline": false, "primary_ip": "192.0.2.4", "reserved_cpus": 0, "reserved_memory": 0, "secondary_ip": "198.51.100.4", "tags": [], "total_cpus": 5, "total_disk": 40960, "total_memory": 4096, "total_spindles": 5, "vm_capable": true } }, "request": { "instances": [ { "disk_space_total": 10230, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 10230, "spindles": 1 } ], "hypervisor": "xen-pvm", "memory": 1023, "name": "instance-new-quarter-1", "nics": [], "os": "instance-debootstrap", "required_nodes": 1, "spindle_use": 1, "tags": [], "vcpus": 1 }, { "disk_space_total": 10230, "disk_template": "plain", "disks": [ { "mode": "rw", "size": 10230, "spindles": 1 } ], "hypervisor": "xen-pvm", "memory": 1023, "name": "instance-new-quarter-2", "nics": [], "os": "instance-debootstrap", "required_nodes": 1, "spindle_use": 1, "tags": [], "vcpus": 1 } ], "type": "multi-allocate" }, "enabled_hypervisors": ["xen-pvm"] } ganeti-3.1.0~rc2/test/data/htools/hail-node-evac.json000064400000000000000000000266241476477700300224650ustar00rootroot00000000000000{ "cluster_tags": [ "htools:iextags:test", "htools:iextags:service-group" ], "nodegroups": { "uuid-group-1": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "alloc_policy": "preferred", "networks": [], "tags": [], "name": "default" } }, "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "min": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "memory-size": 32768, "cpu-count": 8, "disk-count": 16, "spindle-use": 8 }, "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "enabled_hypervisors": [ "xen-pvm", "xen-hvm" ], "cluster_name": "cluster", "instances": { "instance14": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:eb:0b:a5", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance13": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:9c", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance18": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 128, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:55:94:93", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 8192, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance19": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:15:92:6f", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance2": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:73:20:3e", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance3": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 }, { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 384, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:ec:e8:a2", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance4": { "disks": [ { "spindles": 2, "mode": "rw", "size": 2048 } ], "disk_space_total": 2176, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:62:b0:76", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node4", "node3" ], "os": "instance-debootstrap" }, "instance8": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 } ], "disk_space_total": 256, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:3f:6d:e3", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance9": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [ "test:test" ], "nics": [ { "ip": null, "mac": "aa:00:00:10:d2:01", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "instance-debootstrap" }, "instance20": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:db:2a:6d", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" } }, "version": 2, "nodes": { "node1": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.1", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 31389, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1377280, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.1", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node2": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.2", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 31746, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1376640, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.2", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node3": { "total_disk": 1377304, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.3", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 31234, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1373336, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.3", "i_pri_memory": 2432, "free_spindles": 6, "total_spindles": 12, "vm_capable": true, "offline": false }, "node4": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.4", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 22914, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1371520, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.4", "i_pri_memory": 23552, "free_spindles": 0, "total_spindles": 12, "vm_capable": true, "offline": false } }, "request": { "evac_mode": "all", "instances": [ "instance2" ], "type": "node-evacuate" } } ganeti-3.1.0~rc2/test/data/htools/hail-reloc-drbd-crowded.json000064400000000000000000000266621476477700300242700ustar00rootroot00000000000000{ "cluster_tags": [ "htools:iextags:test", "htools:iextags:service-group" ], "nodegroups": { "uuid-group-1": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "alloc_policy": "preferred", "networks": [], "tags": [], "name": "default" } }, "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "memory-size": 32768, "cpu-count": 8, "disk-count": 16, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "enabled_hypervisors": [ "xen-pvm", "xen-hvm" ], "cluster_name": "cluster", "instances": { "instance14": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:eb:0b:a5", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance13": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:9c", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance18": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 128, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:55:94:93", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 8192, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance19": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:15:92:6f", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance2": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:73:20:3e", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance3": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 }, { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 384, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:ec:e8:a2", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance4": { "disks": [ { "spindles": 2, "mode": "rw", "size": 2048 } ], "disk_space_total": 2176, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:62:b0:76", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node4", "node3" ], "os": "instance-debootstrap" }, "instance8": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 } ], "disk_space_total": 256, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:3f:6d:e3", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance9": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [ "test:test" ], "nics": [ { "ip": null, "mac": "aa:00:00:10:d2:01", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "instance-debootstrap" }, "instance20": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:db:2a:6d", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" } }, "version": 2, "nodes": { "node1": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.1", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 89, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 0, "reserved_cpus": 1, "master_capable": true, "free_disk": 1377280, "drained": false, "total_memory": 89, "primary_ip": "192.168.1.1", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node2": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.2", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 46, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 0, "reserved_cpus": 1, "master_capable": true, "free_disk": 1376640, "drained": false, "total_memory": 46, "primary_ip": "192.168.1.2", "i_pri_memory": 0, "free_spindles": 11, "total_spindles": 12, "vm_capable": true, "offline": false }, "node3": { "total_disk": 1377304, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.3", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 31234, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1373336, "drained": false, "total_memory": 31618, "primary_ip": "192.168.1.3", "i_pri_memory": 2432, "free_spindles": 6, "total_spindles": 12, "vm_capable": true, "offline": false }, "node4": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.4", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 22914, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1371520, "drained": false, "total_memory": 31234, "primary_ip": "192.168.1.4", "i_pri_memory": 23552, "free_spindles": 0, "total_spindles": 12, "vm_capable": true, "offline": false } }, "request": { "relocate_from": [ "node4" ], "required_nodes": 1, "type": "relocate", "name": "instance14", "disk_space_total": 256 } } ganeti-3.1.0~rc2/test/data/htools/hail-reloc-drbd.json000064400000000000000000000267041476477700300226400ustar00rootroot00000000000000{ "cluster_tags": [ "htools:iextags:test", "htools:iextags:service-group" ], "nodegroups": { "uuid-group-1": { "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "alloc_policy": "preferred", "networks": [], "tags": [], "name": "default" } }, "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 1024, "memory-size": 128, "cpu-count": 1, "disk-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "memory-size": 32768, "cpu-count": 8, "disk-count": 16, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "enabled_hypervisors": [ "xen-pvm", "xen-hvm" ], "cluster_name": "cluster", "instances": { "instance14": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:eb:0b:a5", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance13": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:7f:8c:9c", "link": "xen-br1", "mode": "bridged", "bridge": "xen-br1" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance18": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 128, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:55:94:93", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 8192, "nodes": [ "node4" ], "os": "instance-debootstrap" }, "instance19": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:15:92:6f", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance2": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:73:20:3e", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "up", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "debian-image" }, "instance3": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 }, { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 384, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:ec:e8:a2", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance4": { "disks": [ { "spindles": 2, "mode": "rw", "size": 2048 } ], "disk_space_total": 2176, "hypervisor": "xen-pvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:62:b0:76", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node4", "node3" ], "os": "instance-debootstrap" }, "instance8": { "disks": [ { "spindles": 1, "mode": "rw", "size": 256 } ], "disk_space_total": 256, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:3f:6d:e3", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "debian-image" }, "instance9": { "disks": [ { "spindles": 1, "mode": "rw", "size": 128 } ], "disk_space_total": 256, "hypervisor": "xen-pvm", "tags": [ "test:test" ], "nics": [ { "ip": null, "mac": "aa:00:00:10:d2:01", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "drbd", "memory": 128, "nodes": [ "node3", "node4" ], "os": "instance-debootstrap" }, "instance20": { "disks": [ { "spindles": 1, "mode": "rw", "size": 512 } ], "disk_space_total": 512, "hypervisor": "kvm", "tags": [], "nics": [ { "ip": null, "mac": "aa:00:00:db:2a:6d", "link": "xen-br0", "mode": "bridged", "bridge": "xen-br0" } ], "vcpus": 1, "spindle_use": 1, "admin_state": "down", "admin_state_source": "admin", "disk_template": "plain", "memory": 128, "nodes": [ "node4" ], "os": "instance-debootstrap" } }, "version": 2, "nodes": { "node1": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.1", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 31389, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1377280, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.1", "i_pri_memory": 0, "free_spindles": 12, "total_spindles": 12, "vm_capable": true, "offline": false }, "node2": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.2", "i_pri_up_memory": 0, "tags": [], "master_candidate": true, "free_memory": 31746, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1376640, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.2", "i_pri_memory": 0, "free_spindles": 11, "total_spindles": 12, "vm_capable": true, "offline": false }, "node3": { "total_disk": 1377304, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.3", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 31234, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1373336, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.3", "i_pri_memory": 2432, "free_spindles": 6, "total_spindles": 12, "vm_capable": true, "offline": false }, "node4": { "total_disk": 1377280, "total_cpus": 4, "group": "uuid-group-1", "secondary_ip": "192.168.2.4", "i_pri_up_memory": 128, "tags": [], "master_candidate": true, "free_memory": 22914, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false }, "reserved_memory": 1017, "reserved_cpus": 1, "master_capable": true, "free_disk": 1371520, "drained": false, "total_memory": 32763, "primary_ip": "192.168.1.4", "i_pri_memory": 23552, "free_spindles": 0, "total_spindles": 12, "vm_capable": true, "offline": false } }, "request": { "relocate_from": [ "node4" ], "required_nodes": 1, "type": "relocate", "name": "instance14", "disk_space_total": 256 } } ganeti-3.1.0~rc2/test/data/htools/hbal-cpu-speed.data000064400000000000000000000005741476477700300224360ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-slow|1600|0|1400|100000|96000|8|M|fake-uuid-01|8||N|8|1|1.0 node-fast|1600|0|1400|100000|96000|8|N|fake-uuid-01|8||N|8|1|3.0 inst1|100|1000|8|running|Y|node-slow|node-fast|drbd| inst2|100|1000|8|running|Y|node-slow|node-fast|drbd| inst3|100|1000|8|running|Y|node-fast|node-slow|drbd| inst4|100|1000|8|running|Y|node-fast|node-slow|drbd| ganeti-3.1.0~rc2/test/data/htools/hbal-desiredlocation-1.data000064400000000000000000000004251476477700300240520ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|15360|409600|357800|16|N|fake-uuid-01|1|power:a node-02|16384|0|16384|409600|357800|16|N|fake-uuid-01|1|power:b inst1|1024|51200|1|running|Y|node-01|node-02|drbd|power:b|1 htools:desiredlocation:power htools:nlocation:power ganeti-3.1.0~rc2/test/data/htools/hbal-desiredlocation-2.data000064400000000000000000000006121476477700300240510ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|14236|409600|306600|16|N|fake-uuid-01|1|power:b node-02|16384|0|16384|409600|306600|16|N|fake-uuid-01|1|power:a node-03|16384|0|16384|409600|409600|16|N|fake-uuid-01|1| inst1|1024|51200|1|running|Y|node-01|node-02|drbd|power:a|1 inst2|1024|51200|1|running|Y|node-01|node-02|drbd|power:b|1 htools:desiredlocation:power htools:nlocation:power ganeti-3.1.0~rc2/test/data/htools/hbal-desiredlocation-3.data000064400000000000000000000006611476477700300240560ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|15360|409600|307200|16|N|fake-uuid-01|1|power:c,computer:l node-02|16384|0|16384|409600|307200|16|N|fake-uuid-01|1|power:b node-03|16384|0|16384|409600|409600|16|N|fake-uuid-01|1|power:b,computer:s inst1|1024|102400|1|running|Y|node-01|node-02|drbd|power:b,computer:s|1 htools:nlocation:power htools:nlocation:computer htools:desiredlocation:power htools:desiredlocation:computer ganeti-3.1.0~rc2/test/data/htools/hbal-desiredlocation-4.data000064400000000000000000000005731476477700300240610ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|15360|409600|357800|16|N|fake-uuid-01|1|power:a,computer:l node-02|16384|0|16384|409600|357800|16|N|fake-uuid-01|1|power:b,computer:s inst1|1024|51200|1|running|Y|node-01|node-02|drbd|power:b,computer:s,computer:l|1 htools:nlocation:power htools:nlocation:computer htools:desiredlocation:power htools:desiredlocation:computer ganeti-3.1.0~rc2/test/data/htools/hbal-dyn.data000064400000000000000000000012431476477700300213350ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-000|91552|0|91040|3100|3100|32|M|fake-uuid-01|1 node-01-001|91552|0|91040|3100|3100|32|N|fake-uuid-01|1 inst-00|128|0|1|running|Y|node-01-000||ext||1 inst-01|128|0|1|running|Y|node-01-000||ext||1 inst-02|128|0|1|running|Y|node-01-000||ext||1 inst-03|128|0|1|running|Y|node-01-000||ext||1 inst-10|256|0|2|running|Y|node-01-001||ext||1 inst-11|256|0|2|running|Y|node-01-001||ext||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hbal-evac.data000064400000000000000000000014351476477700300214640ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-E|4000|0|2000|6000|3000|32|Y|fake-uuid-01|4 node-1|4000|0|3000|6000|4000|32|N|fake-uuid-01|4 node-2|4000|0|3000|6000|1000|32|N|fake-uuid-01|4 node-3|4000|0|2000|6000|2000|32|N|fake-uuid-01|4 inst-p1|1000|1000|1|running|Y|node-E|node-1|drbd||1 inst-p2|1000|1000|1|running|Y|node-E|node-3|drbd||1 inst-s1|1000|1000|1|running|Y|node-2|node-E|drbd||1 inst-12|1000|1000|1|running|Y|node-1|node-2|drbd||1 inst-32a|1000|1000|1|running|Y|node-3|node-2|drbd||1 inst-32b|1000|1000|1|running|Y|node-3|node-2|drbd||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hbal-excl-tags.data000064400000000000000000000007741476477700300224420ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|14336|409600|153600|16|N|fake-uuid-01|1 node-02|16384|0|13312|409600|153600|16|N|fake-uuid-01|1 dns1|1024|51200|1|running|Y|node-01|node-02|drbd|service-group:dns,foo|1 dns2|1024|51200|1|running|Y|node-01|node-02|drbd|service-group:dns,foo|1 ftp1|1024|51200|1|running|Y|node-02|node-01|drbd|test:ftp,bar|1 ftp2|1024|51200|1|running|Y|node-02|node-01|drbd|test:ftp,bar|1 admin|1024|51200|1|running|Y|node-02|node-01|drbd|foo|1 htools:iextags:service-group ganeti-3.1.0~rc2/test/data/htools/hbal-forth.data000064400000000000000000000011441476477700300216650ustar00rootroot00000000000000default|fake-uuid|preferred|| node1|65523|1023|65523|3405248|3405248|24|M|fake-uuid|1||N|0|1|1.0 node2|65523|1023|65523|3405248|3405248|24|N|fake-uuid|1||N|0|1|1.0 node3|65523|1023|65523|3405248|3405248|24|N|fake-uuid|1||N|0|1|1.0 forthcoming-inst1|128|2176|1|ADMIN_down|Y|node1|node2|drbd||1|-|Y forthcoming-inst2|128|2176|1|ADMIN_down|Y|node1|node2|drbd||1|-|Y forthcoming-inst3|128|2176|1|ADMIN_down|Y|node1|node2|drbd||1|-|Y |128,1,1024,1,1,1|128,1,512,1,1,1;1024,8,1048576,16,8,12|drbd,plain,diskless|4.0|32.0 default|128,1,1024,1,1,1|128,1,512,1,1,1;1024,8,1048576,16,8,12|drbd,plain,diskless|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hbal-location-1.data000064400000000000000000000010241476477700300225060ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|15360|409600|306600|16|N|fake-uuid-01|1|power:a node-02|16384|0|15360|409600|306600|16|N|fake-uuid-01|1|power:a node-11|16384|0|15360|409600|306600|16|N|fake-uuid-01|1|power:b node-12|16384|0|15360|409600|306600|16|N|fake-uuid-01|1|power:b inst01|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst02|1024|51200|1|running|Y|node-02|node-01|drbd||1 inst11|1024|51200|1|running|Y|node-11|node-12|drbd||1 inst12|1024|51200|1|running|Y|node-12|node-11|drbd||1 htools:nlocation:power ganeti-3.1.0~rc2/test/data/htools/hbal-location-2.data000064400000000000000000000005601476477700300225130ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|15360|409600|306600|16|N|fake-uuid-01|1|power:a,power:c node-02|2048|0|1024|109600|6600|16|N|fake-uuid-01|1|power:b node-03|2048|0|2048|409600|409600|16|N|fake-uuid-01|1|power:a,power:c inst1|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst2|1024|51200|1|running|Y|node-02|node-01|drbd||1 htools:nlocation:power ganeti-3.1.0~rc2/test/data/htools/hbal-location-exclusion.data000064400000000000000000000007361476477700300243700ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-1|16384|0|15360|409600|358400|16|N|fake-uuid-01|1|power:a node-2|16384|0|15360|409600|358400|16|N|fake-uuid-01|1|power:a node-3|16384|0|16384|409600|358400|16|N|fake-uuid-01|1|power:b node-4|16384|0|16384|409600|358400|1|N|fake-uuid-01|1|power:b inst01|1024|51200|1|running|Y|node-1|node-3|drbd|service-group:dns|1 inst02|1024|51200|1|running|Y|node-2|node-4|drbd|service-group:dns|1 htools:nlocation:power htools:iextags:service-group ganeti-3.1.0~rc2/test/data/htools/hbal-migration-1.data000064400000000000000000000012141476477700300226700ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|12288|409600|204800|16|N|fake-uuid-01|1|hv:new,hvthisisnotamigrationtag node-02|16384|0|16384|409600|306600|16|N|fake-uuid-01|1| node-03|16384|0|16384|409600|306600|16|N|fake-uuid-01|1|hv:new node-04|16384|0|16384|409600|306600|16|N|fake-uuid-01|1| inst121|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst122|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst131|1024|51200|1|running|Y|node-01|node-03|drbd||1 inst132|1024|51200|1|running|Y|node-01|node-03|drbd||1 inst141|1024|51200|1|running|Y|node-01|node-04|drbd||1 inst142|1024|51200|1|running|Y|node-01|node-04|drbd||1 htools:migration:hv ganeti-3.1.0~rc2/test/data/htools/hbal-migration-2.data000064400000000000000000000011711476477700300226730ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|12288|409600|204800|16|N|fake-uuid-01|1|hv:new node-02|16384|0|16384|409600|306600|16|N|fake-uuid-01|1|hv:new node-03|16384|0|16384|409600|306600|16|N|fake-uuid-01|1|hv:new node-04|16384|0|16384|409600|306600|16|N|fake-uuid-01|1| inst121|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst122|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst131|1024|51200|1|running|Y|node-01|node-03|drbd||1 inst132|1024|51200|1|running|Y|node-01|node-03|drbd||1 inst141|1024|51200|1|running|Y|node-01|node-04|drbd||1 inst142|1024|51200|1|running|Y|node-01|node-04|drbd||1 htools:migration:hv ganeti-3.1.0~rc2/test/data/htools/hbal-migration-3.data000064400000000000000000000012361476477700300226760ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|12288|409600|204800|16|N|fake-uuid-01|1|hv:old node-02|16384|0|16384|409600|306600|16|N|fake-uuid-01|1|hv:old node-03|16384|0|16384|409600|306600|16|N|fake-uuid-01|1|hv:new node-04|16384|0|16384|409600|306600|16|N|fake-uuid-01|1| inst121|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst122|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst131|1024|51200|1|running|Y|node-01|node-03|drbd||1 inst132|1024|51200|1|running|Y|node-01|node-03|drbd||1 inst141|1024|51200|1|running|Y|node-01|node-04|drbd||1 inst142|1024|51200|1|running|Y|node-01|node-04|drbd||1 htools:migration:hv htools:allowmigration:hv:old::hv:new ganeti-3.1.0~rc2/test/data/htools/hbal-soft-errors.data000064400000000000000000000062771476477700300230440ustar00rootroot00000000000000default|876e0957-5186-4eae-a95d-f45ee343857d|preferred|| dv-02|64465|2789|61272|1667020|1293772|24|M|876e0957-5186-4eae-a95d-f45ee343857d|1||N|0|24 dv-03|64465|2666|61284|1667020|1293772|24|N|876e0957-5186-4eae-a95d-f45ee343857d|1||N|0|24 dv-04|64465|320|63723|1667020|1293772|24|N|876e0957-5186-4eae-a95d-f45ee343857d|1||N|0|24 seq1|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- seq2|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- seq3|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- seq4|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- seq5|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- seq6|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- seq7|1024|10368|1|running|Y|dv-02|dv-03|drbd||1|- seq8|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- seq9|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- seq10|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- seq11|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- seq12|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- seq13|1024|10368|1|running|Y|dv-02|dv-03|drbd||1|- seq14|1024|10368|1|running|Y|dv-02|dv-03|drbd||1|- seq15|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- seq16|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- seq17|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- seq18|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- seq19|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- seq20|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- seq21|1024|10368|1|running|Y|dv-02|dv-03|drbd||1|- seq22|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- seq23|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- seq24|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- test|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- test2|1024|10368|1|running|Y|dv-02|dv-03|drbd||1|- test3|1024|10368|1|running|Y|dv-02|dv-03|drbd||1|- test4|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test5|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test6|1024|10368|1|running|Y|dv-02|dv-03|drbd||1|- test7|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- test8|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- test9|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- test10|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test11|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test12|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- test13|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- test14|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- test15|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- test16|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test17|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test18|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- test19|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- test20|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- test21|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- test22|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test23|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test24|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test25|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test26|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test27|1024|10368|1|running|Y|dv-03|dv-04|drbd||1|- test28|1024|10368|1|running|Y|dv-02|dv-04|drbd||1|- test29|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- test30|1024|10368|1|running|Y|dv-03|dv-02|drbd||1|- |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|drbd,plain|4.0|32.0 default|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|drbd,plain|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hbal-soft-errors2.data000064400000000000000000000006501476477700300231130ustar00rootroot00000000000000default|fake-uuid-01|preferred|| node-01|16384|0|14336|409600|306600|16|N|fake-uuid-01|1| node-02|16384|0|16384|409600|306600|16|N|fake-uuid-01|1| inst121|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst122|1024|51200|1|running|Y|node-01|node-02|drbd||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|drbd,plain|4.0|1.0 default|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|drbd,plain|4.0|1.0 ganeti-3.1.0~rc2/test/data/htools/hbal-split-insts.data000064400000000000000000000206711476477700300230420ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| group-02|fake-uuid-02|preferred|| node-01-001|98304|0|96256|8388608|8355840|16|N|fake-uuid-01|1 node-01-002|98304|0|96256|8388608|8355840|16|N|fake-uuid-01|1 node-01-003|98304|0|96256|8388608|8355840|16|N|fake-uuid-01|1 node-01-004|98304|0|96256|8388608|8355840|16|N|fake-uuid-01|1 node-01-005|98304|0|96256|8388608|8355840|16|N|fake-uuid-01|1 node-01-006|98304|0|96256|8388608|8355840|16|N|fake-uuid-01|1 node-01-007|98304|0|96256|8388608|8355840|16|N|fake-uuid-02|1 node-01-008|98304|0|96256|8388608|8355840|16|N|fake-uuid-02|1 new-0|128|1024|1|running|Y|node-01-008|node-01-007|drbd||1 new-1|128|1024|1|running|Y|node-01-006|node-01-005|drbd||1 new-2|128|1024|1|running|Y|node-01-004|node-01-003|drbd||1 new-3|128|1024|1|running|Y|node-01-002|node-01-001|drbd||1 new-4|128|1024|1|running|Y|node-01-007|node-01-008|drbd||1 new-5|128|1024|1|running|Y|node-01-005|node-01-006|drbd||1 new-6|128|1024|1|running|Y|node-01-003|node-01-004|drbd||1 new-7|128|1024|1|running|Y|node-01-001|node-01-002|drbd||1 new-8|128|1024|1|running|Y|node-01-008|node-01-006|drbd||1 new-9|128|1024|1|running|Y|node-01-007|node-01-005|drbd||1 new-10|128|1024|1|running|Y|node-01-004|node-01-002|drbd||1 new-11|128|1024|1|running|Y|node-01-003|node-01-001|drbd||1 new-12|128|1024|1|running|Y|node-01-006|node-01-008|drbd||1 new-13|128|1024|1|running|Y|node-01-005|node-01-007|drbd||1 new-14|128|1024|1|running|Y|node-01-002|node-01-004|drbd||1 new-15|128|1024|1|running|Y|node-01-001|node-01-003|drbd||1 new-16|128|1024|1|running|Y|node-01-008|node-01-005|drbd||1 new-17|128|1024|1|running|Y|node-01-007|node-01-006|drbd||1 new-18|128|1024|1|running|Y|node-01-004|node-01-001|drbd||1 new-19|128|1024|1|running|Y|node-01-003|node-01-002|drbd||1 new-20|128|1024|1|running|Y|node-01-006|node-01-007|drbd||1 new-21|128|1024|1|running|Y|node-01-005|node-01-008|drbd||1 new-22|128|1024|1|running|Y|node-01-002|node-01-003|drbd||1 new-23|128|1024|1|running|Y|node-01-001|node-01-004|drbd||1 new-24|128|1024|1|running|Y|node-01-008|node-01-004|drbd||1 new-25|128|1024|1|running|Y|node-01-007|node-01-003|drbd||1 new-26|128|1024|1|running|Y|node-01-006|node-01-002|drbd||1 new-27|128|1024|1|running|Y|node-01-005|node-01-001|drbd||1 new-28|128|1024|1|running|Y|node-01-004|node-01-008|drbd||1 new-29|128|1024|1|running|Y|node-01-003|node-01-007|drbd||1 new-30|128|1024|1|running|Y|node-01-002|node-01-006|drbd||1 new-31|128|1024|1|running|Y|node-01-001|node-01-005|drbd||1 new-32|128|1024|1|running|Y|node-01-008|node-01-003|drbd||1 new-33|128|1024|1|running|Y|node-01-007|node-01-004|drbd||1 new-34|128|1024|1|running|Y|node-01-006|node-01-001|drbd||1 new-35|128|1024|1|running|Y|node-01-005|node-01-002|drbd||1 new-36|128|1024|1|running|Y|node-01-004|node-01-007|drbd||1 new-37|128|1024|1|running|Y|node-01-003|node-01-008|drbd||1 new-38|128|1024|1|running|Y|node-01-002|node-01-005|drbd||1 new-39|128|1024|1|running|Y|node-01-001|node-01-006|drbd||1 new-40|128|1024|1|running|Y|node-01-008|node-01-002|drbd||1 new-41|128|1024|1|running|Y|node-01-007|node-01-001|drbd||1 new-42|128|1024|1|running|Y|node-01-006|node-01-004|drbd||1 new-43|128|1024|1|running|Y|node-01-005|node-01-003|drbd||1 new-44|128|1024|1|running|Y|node-01-004|node-01-006|drbd||1 new-45|128|1024|1|running|Y|node-01-003|node-01-005|drbd||1 new-46|128|1024|1|running|Y|node-01-002|node-01-008|drbd||1 new-47|128|1024|1|running|Y|node-01-001|node-01-007|drbd||1 new-48|128|1024|1|running|Y|node-01-008|node-01-001|drbd||1 new-49|128|1024|1|running|Y|node-01-007|node-01-002|drbd||1 new-50|128|1024|1|running|Y|node-01-006|node-01-003|drbd||1 new-51|128|1024|1|running|Y|node-01-005|node-01-004|drbd||1 new-52|128|1024|1|running|Y|node-01-004|node-01-005|drbd||1 new-53|128|1024|1|running|Y|node-01-003|node-01-006|drbd||1 new-54|128|1024|1|running|Y|node-01-002|node-01-007|drbd||1 new-55|128|1024|1|running|Y|node-01-001|node-01-008|drbd||1 new-56|128|1024|1|running|Y|node-01-008|node-01-007|drbd||1 new-57|128|1024|1|running|Y|node-01-006|node-01-005|drbd||1 new-58|128|1024|1|running|Y|node-01-004|node-01-003|drbd||1 new-59|128|1024|1|running|Y|node-01-002|node-01-001|drbd||1 new-60|128|1024|1|running|Y|node-01-007|node-01-008|drbd||1 new-61|128|1024|1|running|Y|node-01-005|node-01-006|drbd||1 new-62|128|1024|1|running|Y|node-01-003|node-01-004|drbd||1 new-63|128|1024|1|running|Y|node-01-001|node-01-002|drbd||1 new-64|128|1024|1|running|Y|node-01-008|node-01-006|drbd||1 new-65|128|1024|1|running|Y|node-01-007|node-01-005|drbd||1 new-66|128|1024|1|running|Y|node-01-004|node-01-002|drbd||1 new-67|128|1024|1|running|Y|node-01-003|node-01-001|drbd||1 new-68|128|1024|1|running|Y|node-01-006|node-01-008|drbd||1 new-69|128|1024|1|running|Y|node-01-005|node-01-007|drbd||1 new-70|128|1024|1|running|Y|node-01-002|node-01-004|drbd||1 new-71|128|1024|1|running|Y|node-01-001|node-01-003|drbd||1 new-72|128|1024|1|running|Y|node-01-008|node-01-005|drbd||1 new-73|128|1024|1|running|Y|node-01-007|node-01-006|drbd||1 new-74|128|1024|1|running|Y|node-01-004|node-01-001|drbd||1 new-75|128|1024|1|running|Y|node-01-003|node-01-002|drbd||1 new-76|128|1024|1|running|Y|node-01-006|node-01-007|drbd||1 new-77|128|1024|1|running|Y|node-01-005|node-01-008|drbd||1 new-78|128|1024|1|running|Y|node-01-002|node-01-003|drbd||1 new-79|128|1024|1|running|Y|node-01-001|node-01-004|drbd||1 new-80|128|1024|1|running|Y|node-01-008|node-01-004|drbd||1 new-81|128|1024|1|running|Y|node-01-007|node-01-003|drbd||1 new-82|128|1024|1|running|Y|node-01-006|node-01-002|drbd||1 new-83|128|1024|1|running|Y|node-01-005|node-01-001|drbd||1 new-84|128|1024|1|running|Y|node-01-004|node-01-008|drbd||1 new-85|128|1024|1|running|Y|node-01-003|node-01-007|drbd||1 new-86|128|1024|1|running|Y|node-01-002|node-01-006|drbd||1 new-87|128|1024|1|running|Y|node-01-001|node-01-005|drbd||1 new-88|128|1024|1|running|Y|node-01-008|node-01-003|drbd||1 new-89|128|1024|1|running|Y|node-01-007|node-01-004|drbd||1 new-90|128|1024|1|running|Y|node-01-006|node-01-001|drbd||1 new-91|128|1024|1|running|Y|node-01-005|node-01-002|drbd||1 new-92|128|1024|1|running|Y|node-01-004|node-01-007|drbd||1 new-93|128|1024|1|running|Y|node-01-003|node-01-008|drbd||1 new-94|128|1024|1|running|Y|node-01-002|node-01-005|drbd||1 new-95|128|1024|1|running|Y|node-01-001|node-01-006|drbd||1 new-96|128|1024|1|running|Y|node-01-008|node-01-002|drbd||1 new-97|128|1024|1|running|Y|node-01-007|node-01-001|drbd||1 new-98|128|1024|1|running|Y|node-01-006|node-01-004|drbd||1 new-99|128|1024|1|running|Y|node-01-005|node-01-003|drbd||1 new-100|128|1024|1|running|Y|node-01-004|node-01-006|drbd||1 new-101|128|1024|1|running|Y|node-01-003|node-01-005|drbd||1 new-102|128|1024|1|running|Y|node-01-002|node-01-008|drbd||1 new-103|128|1024|1|running|Y|node-01-001|node-01-007|drbd||1 new-104|128|1024|1|running|Y|node-01-008|node-01-001|drbd||1 new-105|128|1024|1|running|Y|node-01-007|node-01-002|drbd||1 new-106|128|1024|1|running|Y|node-01-006|node-01-003|drbd||1 new-107|128|1024|1|running|Y|node-01-005|node-01-004|drbd||1 new-108|128|1024|1|running|Y|node-01-004|node-01-005|drbd||1 new-109|128|1024|1|running|Y|node-01-003|node-01-006|drbd||1 new-110|128|1024|1|running|Y|node-01-002|node-01-007|drbd||1 new-111|128|1024|1|running|Y|node-01-001|node-01-008|drbd||1 new-112|128|1024|1|running|Y|node-01-008|node-01-007|drbd||1 new-113|128|1024|1|running|Y|node-01-006|node-01-005|drbd||1 new-114|128|1024|1|running|Y|node-01-004|node-01-003|drbd||1 new-115|128|1024|1|running|Y|node-01-002|node-01-001|drbd||1 new-116|128|1024|1|running|Y|node-01-007|node-01-008|drbd||1 new-117|128|1024|1|running|Y|node-01-005|node-01-006|drbd||1 new-118|128|1024|1|running|Y|node-01-003|node-01-004|drbd||1 new-119|128|1024|1|running|Y|node-01-001|node-01-002|drbd||1 new-120|128|1024|1|running|Y|node-01-008|node-01-006|drbd||1 new-121|128|1024|1|running|Y|node-01-007|node-01-005|drbd||1 new-122|128|1024|1|running|Y|node-01-004|node-01-002|drbd||1 new-123|128|1024|1|running|Y|node-01-003|node-01-001|drbd||1 new-124|128|1024|1|running|Y|node-01-006|node-01-008|drbd||1 new-125|128|1024|1|running|Y|node-01-005|node-01-007|drbd||1 new-126|128|1024|1|running|Y|node-01-002|node-01-004|drbd||1 new-127|128|1024|1|running|Y|node-01-001|node-01-003|drbd||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd|4.0|32.0 group-02|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hroller-full.data000064400000000000000000000021461476477700300222510ustar00rootroot00000000000000group-01|fake-uuid-01|preferred| node-00|91552|0|91424|3102|1052|16|N|fake-uuid-01|1 node-01|91552|0|91424|3102|1052|16|N|fake-uuid-01|1 node-10|91552|0|91424|3102|1052|16|N|fake-uuid-01|1 node-11|91552|0|91424|3102|1052|16|N|fake-uuid-01|1 node-20|91552|0|91424|3102|1052|16|N|fake-uuid-01|1 node-21|91552|0|91424|3102|1052|16|N|fake-uuid-01|1 node-30|91553|0|91424|3102|1053|16|N|fake-uuid-01|1 node-31|91553|0|91424|3102|1053|16|M|fake-uuid-01|1 inst-00|128|1024|1|running|Y|node-00|node-01|drbd||1 inst-00|128|1024|1|running|Y|node-01|node-00|drbd||1 inst-10|128|1024|1|running|Y|node-10|node-11|drbd||1 inst-11|128|1024|1|running|Y|node-11|node-10|drbd||1 inst-20|128|1024|1|running|Y|node-20|node-21|drbd||1 inst-21|128|1024|1|running|Y|node-21|node-20|drbd||1 inst-30|128|1024|1|running|Y|node-30|node-31|drbd||1 inst-31|128|1024|1|running|Y|node-31|node-30|drbd||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hroller-nodegroups.data000064400000000000000000000015221476477700300234710ustar00rootroot00000000000000group-01|fake-uuid-01|preferred| group-02|fake-uuid-02|preferred| node-01-000|91552|0|91424|3100|1052|16|N|fake-uuid-01|1 node-01-001|91552|0|91424|3100|1052|16|N|fake-uuid-01|1 node-01-002|91552|0|91424|3100|1052|16|N|fake-uuid-01|1 node-02-000|91552|0|91552|3100|3100|16|M|fake-uuid-02|1 inst-00|128|1024|1|running|Y|node-01-000||plain||1 inst-01|128|1024|1|running|Y|node-01-000||plain||1 inst-10|128|1024|1|running|Y|node-01-001||plain||1 inst-11|128|1024|1|running|Y|node-01-001||plain||1 inst-20|128|1024|1|running|Y|node-01-002||plain||1 inst-21|128|1024|1|running|Y|node-01-002||plain||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hroller-nonredundant.data000064400000000000000000000023231476477700300240030ustar00rootroot00000000000000group-01|fake-uuid-01|preferred| node-01-000|91552|0|91424|3100|1052|16|M|fake-uuid-01|1 node-01-001|91552|0|91424|3100|1052|16|N|fake-uuid-01|1 node-01-002|91552|0|91424|3100|1052|16|N|fake-uuid-01|1 node-01-003|91552|0|91424|3100|1052|16|N|fake-uuid-01|1 node-01-004|91552|0|91424|3100|1052|16|N|fake-uuid-01|1 node-01-005|91552|0|91424|3100|1052|16|N|fake-uuid-01|1 inst-00|128|1024|1|running|Y|node-01-000||plain||1 inst-01|128|1024|1|running|Y|node-01-000||plain||1 inst-10|128|1024|1|running|Y|node-01-001||plain||1 inst-11|128|1024|1|running|Y|node-01-001||plain||1 inst-20|128|1024|1|running|Y|node-01-002||plain||1 inst-21|128|1024|1|running|Y|node-01-002||plain||1 inst-30|128|1024|1|running|Y|node-01-003||plain||1 inst-31|128|1024|1|running|Y|node-01-003||plain||1 inst-40|128|1024|1|running|Y|node-01-004||plain||1 inst-41|128|1024|1|running|Y|node-01-004||plain||1 inst-50|128|1024|1|running|Y|node-01-005||plain||1 inst-51|128|1024|1|running|Y|node-01-005||plain||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hroller-online.data000064400000000000000000000013001476477700300225620ustar00rootroot00000000000000group-01|fake-uuid-01|preferred| node-01-001|91552|0|91424|953674|953674|16|N|fake-uuid-01|1 node-01-002|91552|0|91296|953674|953674|16|N|fake-uuid-01|1 node-01-003|91552|0|91296|953674|953674|16|M|fake-uuid-01|1 node-01-004|91552|0|91296|953674|953674|16|N|fake-uuid-01|1 new-0|128|1152|1|running|Y|node-01-001|node-01-002|drbd||1 new-1|128|1152|1|running|Y|node-01-003|node-01-002|drbd||1 new-2|128|1152|1|running|Y|node-01-004|node-01-003|drbd||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-bad-group.data000064400000000000000000000014251476477700300227620ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| group-bad|fake-uuid-bad|preferred|| node-01|81920|0|71920|200|200|16|M|fake-uuid-01 node-02|81920|0|71920|200|200|16|N|fake-uuid-01 node-remain|81920|0|7920|200|200|16|N|fake-uuid-bad node-drained|81920|0|8920|200|200|16|N|fake-uuid-bad inst-11|10000|0|1|running|Y|node-01||ext||1 inst-12|10000|0|1|running|Y|node-02||ext||1 inst-2|10000|0|1|running|Y|node-remain||ext||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-bad|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-existing.data000064400000000000000000000010661476477700300227350ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|81920|0|71920|200|200|16|M|fake-uuid-01 node-02|81920|0|71920|200|200|16|N|fake-uuid-01 node-bad|81920|0|1920|200|200|16|Y|fake-uuid-01 inst-11|10000|0|1|running|Y|node-01||ext||1 inst-12|10000|0|1|running|Y|node-02||ext||1 inst-bad|80000|0|1|running|Y|node-bad||ext||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-groups-one.data000064400000000000000000000012231476477700300231740ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|2049|0|1|3073|1|16|N|fake-uuid-01|1||N|0|1 node-01-002|2049|0|1025|3073|1|16|N|fake-uuid-01|1||N|0|1 node-02-001|2049|0|2049|2049|2049|16|N|fake-uuid-01|1||N|0|1 node-02-002|2049|0|2049|2049|2049|16|N|fake-uuid-01|1||N|0|1 old-0|1024|1024|1|running|Y|node-01-001|node-01-002|drbd||1|1 old-1|1024|1024|1|running|Y|node-01-002|node-01-001|drbd||1|1 old-2|1024|1024|1|running|Y|node-01-001|node-01-002|drbd||1|1 |1024,1,1024,1,1,1|1024,1,1024,1,1,1;2048,8,2048,16,8,12|drbd|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-groups-two.data000064400000000000000000000014661476477700300232350ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| group-02|fake-uuid-02|preferred|| node-01-001|2049|0|1|3073|1|16|N|fake-uuid-01|1||N|0|1 node-01-002|2049|0|1025|3073|1|16|N|fake-uuid-01|1||N|0|1 node-02-001|2049|0|2049|2049|2049|16|N|fake-uuid-02|1||N|0|1 node-02-002|2049|0|2049|2049|2049|16|N|fake-uuid-02|1||N|0|1 old-0|1024|1024|1|running|Y|node-01-001|node-01-002|drbd||1|1 old-1|1024|1024|1|running|Y|node-01-002|node-01-001|drbd||1|1 old-2|1024|1024|1|running|Y|node-01-001|node-01-002|drbd||1|1 |1024,1,1024,1,1,1|1024,1,1024,1,1,1;2048,8,2048,16,8,12|drbd|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-02|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-tiered-dualspec-exclusive.data000064400000000000000000000012531476477700300261600ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|10||Y|10 node-01-002|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|10||Y|9 node-01-003|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|8||Y|8 node-01-004|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|8||Y|8 |63488,2,522240,1,1,2|129024,4,1047552,1,1,4;131072,4,1048576,16,8,4;63488,2,522240,1,1,2;65536,2,524288,16,8,2|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 group-01|63488,2,522240,1,1,2|129024,4,1047552,1,1,4;131072,4,1048576,16,8,4;63488,2,522240,1,1,2;65536,2,524288,16,8,2|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-tiered-dualspec.data000064400000000000000000000012301476477700300241460ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|1 node-01-002|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|1 node-01-003|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|1 node-01-004|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|1 |63488,2,522240,1,1,1|129024,4,1047552,1,1,1;131072,4,1048576,16,8,12;63488,2,522240,1,1,1;65536,2,524288,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 group-01|63488,2,522240,1,1,1|129024,4,1047552,1,1,1;131072,4,1048576,16,8,12;63488,2,522240,1,1,1;65536,2,524288,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-tiered-exclusive.data000064400000000000000000000011331476477700300243570ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|10||Y|10 node-01-002|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|10||Y|9 node-01-003|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|8||Y|8 node-01-004|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|8||Y|8 |129024,4,1047552,1,1,1|129024,4,1047552,1,1,1;131072,4,1048576,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 group-01|129024,4,1047552,1,1,1|129024,4,1047552,1,1,1;131072,4,1048576,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-tiered-ipolicy.data000064400000000000000000000007451476477700300240300ustar00rootroot00000000000000group-01|fake-uuid-01|preferred| node-01-001|2000|200|1800|4300|4300|8|N|fake-uuid-01 node-01-002|2000|4|1996|3900|3900|8|N|fake-uuid-01 node-01-003|2000|4|1996|3900|3900|8|N|fake-uuid-01 node-01-004|2000|4|1996|3900|3900|8|N|fake-uuid-01 |936,4,1064,1,1,1|900,4,2200,1,1,1;1000,4,2600,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 group-01|900,4,2200,1,1,1|900,4,2200,1,1,1;1000,4,2600,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-tiered-mixed.data000064400000000000000000000014141476477700300234600ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| group-02|fake-uuid-02|preferred|| node-01-001|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|10||Y|10 node-01-002|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|10||Y|10 node-01-003|262144|1024|261120|2097152|2097152|8|N|fake-uuid-02|8||N|8 node-01-004|262144|1024|261120|2097152|2097152|8|N|fake-uuid-02|8||N|8 |129024,4,1047552,1,1,1|129024,4,1047552,1,1,1;131072,4,1048576,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 group-01|129024,4,1047552,1,1,1|129024,4,1047552,1,1,1;131072,4,1048576,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 group-02|129024,4,1047552,1,1,1|129024,4,1047552,1,1,1;131072,4,1048576,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-tiered-resourcetypes.data000064400000000000000000000007461476477700300252750ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|2000|200|1700|5000|5000|8|N|fake-uuid-01 node-01-002|2000|4|1996|5000|4900|8|N|fake-uuid-01 node-01-003|2000|4|1996|5000|5000|8|N|fake-uuid-01 node-01-004|2000|4|1996|5000|5000|8|N|fake-uuid-01 |900,4,2200,1,1,1|900,4,2000,1,1,1;1000,4,2600,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 group-01|900,4,2200,1,1,1|900,4,2000,1,1,1;1000,4,2600,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-tiered-vcpu.data000064400000000000000000000011161476477700300233260ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|1||N|1|2 node-01-002|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|1||N|1|2 node-01-003|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|1||N|1|3 node-01-004|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|1||N|1|4 |30720,2,64512,1,1,1|30720,2,64512,1,1,1;32768,4,65536,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|1.0|32.0 group-01|30720,2,64512,1,1,1|30720,2,64512,1,1,1;32768,4,65536,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|1.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hspace-tiered.data000064400000000000000000000011041476477700300223500ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|1 node-01-002|262144|65536|196608|2097152|2097152|8|N|fake-uuid-01|1 node-01-003|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|1 node-01-004|262144|1024|261120|2097152|2097152|8|N|fake-uuid-01|1 |129024,4,1047552,1,1,1|129024,4,1047552,1,1,1;131072,4,1048576,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 group-01|129024,4,1047552,1,1,1|129024,4,1047552,1,1,1;131072,4,1048576,16,8,12|plain,diskless,file,sharedfile,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hsqueeze-mixed-instances.data000064400000000000000000000011371476477700300245630ustar00rootroot00000000000000group-01|fake-uuid-01|preferred| node-01-000|639|0|512|1052|1052|9|M|fake-uuid-01|1 node-01-001|639|0|512|1052|1052|9|N|fake-uuid-01|1 node-01-002|639|0|385|1052|28|9|N|fake-uuid-01|1 node-01-003|639|0|512|1052|28|9|N|fake-uuid-01|1 node-01-004|639|0|385|1052|28|9|N|fake-uuid-01|1 node-01-005|639|0|512|1052|28|9|N|fake-uuid-01|1 inst-00|127|0|1|running|Y|node-01-000||ext||1 inst-10|127|0|1|running|Y|node-01-001||ext||1 inst-2-3|127|1024|1|running|Y|node-01-002|node-01-003|drbd||1 inst-4-5|127|1024|1|running|Y|node-01-004|node-01-005|drbd||1 |127,1,0,0,1,1|127,1,0,1,1,1;256,1,0,0,2,2|ext|1.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hsqueeze-overutilized.data000064400000000000000000000024021476477700300242110ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-000|512|0|4|1052|1052|9|M|fake-uuid-01|1||N|0|1 node-01-001|512|0|4|1052|1052|9|N|fake-uuid-01|1||N|0|1 node-01-002|512|0|512|1052|1052|9|Y|fake-uuid-01|1|htools:standby:manual|N|0|1 node-01-003|512|0|512|1052|1052|9|Y|fake-uuid-01|1||N|0|1 node-01-004|512|0|512|1052|1052|9|Y|fake-uuid-01|1|htools:standby:manual|N|0|1 node-01-005|512|0|512|1052|1052|9|Y|fake-uuid-01|1|htools:standby:manual|N|0|1 node-01-006|512|0|512|1052|1052|9|Y|fake-uuid-01|1|htools:standby:manual|N|0|1 node-01-007|512|0|512|1052|1052|9|Y|fake-uuid-01|1|pink:bunny|N|0|1 inst-00|127|0|1|running|Y|node-01-000||ext||1|- inst-01|127|0|1|running|Y|node-01-000||ext||1|- inst-02|127|0|1|running|Y|node-01-000||ext||1|- inst-03|127|0|1|running|Y|node-01-000||ext||1|- inst-10|127|0|1|running|Y|node-01-001||ext||1|- inst-11|127|0|1|running|Y|node-01-001||ext||1|- inst-12|127|0|1|running|Y|node-01-001||ext||1|- inst-13|127|0|1|running|Y|node-01-001||ext||1|- |127,1,0,0,1,1|127,1,0,1,1,1;256,1,0,0,2,2|ext|1.0|32.0 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/hsqueeze-underutilized.data000064400000000000000000000013771476477700300243650ustar00rootroot00000000000000group-01|fake-uuid-01|preferred| node-01-000|512|0|258|1052|1052|9|M|fake-uuid-01|1 node-01-001|512|0|258|1052|1052|9|N|fake-uuid-01|1 node-01-002|512|0|385|1052|1052|9|N|fake-uuid-01|1 node-01-003|512|0|385|1052|1052|9|N|fake-uuid-01|1 node-01-004|512|0|385|1052|1052|9|N|fake-uuid-01|1 node-01-005|512|0|385|1052|1052|9|N|fake-uuid-01|1 inst-00|127|0|1|running|Y|node-01-000||ext||1 inst-01|127|0|1|running|Y|node-01-000||ext||1 inst-10|127|0|1|running|Y|node-01-001||ext||1 inst-11|127|0|1|running|Y|node-01-001||ext||1 inst-20|127|0|1|running|Y|node-01-002||ext||1 inst-30|127|0|1|running|Y|node-01-003||ext||1 inst-40|127|0|1|running|Y|node-01-004||ext||1 inst-50|127|0|1|running|Y|node-01-005||ext||1 |127,1,0,0,1,1|127,1,0,1,1,1;256,1,0,0,2,2|ext|1.0|32.0 ganeti-3.1.0~rc2/test/data/htools/invalid-node.data000064400000000000000000000006631476477700300222150ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|1024|0|1024|95367|95367|4|N|fake-uuid-01|1 node-01-002|1024|0|896|95367|94343|4|N|fake-uuid-01|1 new-0|128|1024|1|running|Y|no-such-node||plain| |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,8|diskless,file,sharedfile,plain,blockdev,drbd,rbd|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,8|diskless,file,sharedfile,plain,blockdev,drbd,rbd|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/missing-resources.data000064400000000000000000000005571476477700300233270ustar00rootroot00000000000000default|fake-uuid-01|preferred|| node1|1024|0|1024|95367|95367|4|N|fake-uuid-01|1 node2|1024|0|0|95367|0|4|N|fake-uuid-01|1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,8|diskless,file,sharedfile,plain,blockdev,drbd,rbd|4.0|32.0 default|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,8|diskless,file,sharedfile,plain,blockdev,drbd,rbd|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/multiple-master.data000064400000000000000000000007241476477700300227660ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|91552|0|91424|953674|953674|16|M|fake-uuid-01|1 node-01-002|91552|0|91296|953674|953674|16|N|fake-uuid-01|1 node-01-003|91552|0|91296|953674|953674|16|M|fake-uuid-01|1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/multiple-tags.data000064400000000000000000000015241476477700300224300ustar00rootroot00000000000000group-01|fake-uuid-01|preferred| node-01-001|91552|0|91424|953674|953674|16|M|fake-uuid-01|1|red node-01-002|91552|0|91296|953674|953674|16|N|fake-uuid-01|1|blue node-01-003|91552|0|91296|953674|953674|16|N|fake-uuid-01|1| node-01-004|91552|0|91296|953674|953674|16|N|fake-uuid-01|1|blue,red node-01-005|91552|0|91296|953674|953674|16|N|fake-uuid-01|1|red node-01-006|91552|0|91296|953674|953674|16|N|fake-uuid-01|1|blue new-0|128|1152|1|running|Y|node-01-001|node-01-002|drbd||1 new-1|128|1152|1|running|Y|node-01-003|node-01-004|drbd||1 new-1|128|1152|1|running|Y|node-01-005|node-01-006|drbd||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/n1-failure.data000064400000000000000000000025651476477700300216120ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| group-02|fake-uuid-02|unallocable|| node-01-001|256|0|0|7629394|7625298|16|N|fake-uuid-01|1 node-01-002|256|0|0|7629394|7625298|16|N|fake-uuid-01|1 node-01-003|256|0|0|7629394|7625298|16|N|fake-uuid-01|1 node-01-004|256|0|0|7629394|7625298|16|N|fake-uuid-01|1 node-02-001|65536|0|65536|7629394|7629394|16|N|fake-uuid-01|1 node-02-002|65536|0|65536|7629394|7629394|16|N|fake-uuid-01|1 node-02-003|65536|0|65536|7629394|7629394|16|N|fake-uuid-01|1 node-02-004|65536|0|65536|7629394|7629394|16|N|fake-uuid-01|1 new-0|128|1024|1|running|Y|node-01-004|node-01-003|drbd||1 new-1|128|1024|1|running|Y|node-01-002|node-01-001|drbd||1 new-2|128|1024|1|running|Y|node-01-003|node-01-001|drbd||1 new-3|128|1024|1|running|Y|node-01-001|node-01-004|drbd||1 new-4|128|1024|1|running|Y|node-01-002|node-01-004|drbd||1 new-5|128|1024|1|running|Y|node-01-003|node-01-002|drbd||1 new-6|128|1024|1|running|Y|node-01-004|node-01-002|drbd||1 new-7|128|1024|1|running|Y|node-01-001|node-01-003|drbd||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-02|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/partly-used.data000064400000000000000000000007441476477700300221150ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01|16384|0|15360|409600|204200|16|N|fake-uuid-01|1| node-02|16384|0|15360|409600|204200|16|N|fake-uuid-01|1| node-03|16384|0|15360|409600|204200|16|N|fake-uuid-01|1| node-04|16384|0|16384|409600|306600|16|N|fake-uuid-01|1| node-05|16384|0|16384|409600|306600|16|N|fake-uuid-01|1| inst12|1024|51200|1|running|Y|node-01|node-02|drbd||1 inst23|1024|51200|1|running|Y|node-02|node-03|drbd||1 inst31|1024|51200|1|running|Y|node-03|node-01|drbd||1 ganeti-3.1.0~rc2/test/data/htools/plain-n1-restriction.data000064400000000000000000000015431476477700300236240ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| nodeA|4096|0|3096|409600|309600|4|M|fake-uuid-01|4 nodeB|4096|0|1096|409600|109600|4|N|fake-uuid-01|4 nodeC|4096|0|1096|409600|109600|4|N|fake-uuid-01|4 nodeD|4096|0|1096|409600|109600|4|N|fake-uuid-01|4 instA1|1000|100000|1|running|Y|nodeA||plain||1 instB1|1000|100000|1|running|Y|nodeB||plain||1 instB2|1000|100000|1|running|Y|nodeB||plain||1 instB3|1000|100000|1|running|Y|nodeB||plain||1 instC1|3000|300000|3|running|Y|nodeC||plain||3 instD1|1000|100000|1|running|Y|nodeD||plain||1 instD2|1000|100000|1|running|Y|nodeD||plain||1 instD3|1000|100000|1|running|Y|nodeD||plain||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/rapi/000075500000000000000000000000001476477700300177375ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/htools/rapi/groups.json000064400000000000000000000022231476477700300221500ustar00rootroot00000000000000[ { "uuid": "uuid-group-1", "tags": [], "ipolicy": { "std": { "cpu-count": 1, "nic-count": 1, "disk-size": 1024, "memory-size": 128, "disk-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "cpu-count": 1, "nic-count": 1, "disk-size": 1024, "memory-size": 128, "disk-count": 1, "spindle-use": 1 }, "max": { "cpu-count": 8, "nic-count": 8, "disk-size": 1048576, "memory-size": 32768, "disk-count": 16, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "node_cnt": 4, "serial_no": 15, "node_list": [ "node1", "node2", "node3", "node4" ], "ctime": null, "mtime": 1325251614.671967, "alloc_policy": "preferred", "name": "default" } ] ganeti-3.1.0~rc2/test/data/htools/rapi/info.json000064400000000000000000000065561476477700300216010ustar00rootroot00000000000000{ "maintain_node_health": true, "hvparams": { "xen-pvm": { "use_bootloader": false, "migration_mode": "live", "kernel_args": "ro", "migration_port": 8002, "bootloader_args": "", "root_path": "/dev/sda1", "blockdev_prefix": "sd", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-2.6-xenU", "initrd_path": "", "reboot_behavior": "reboot" }, "xen-hvm": { "nic_type": "rtl8139", "use_localtime": false, "migration_mode": "non-live", "boot_order": "cd", "migration_port": 8002, "cpu_mask": "all", "vnc_bind_address": "0.0.0.0", "reboot_behavior": "reboot", "blockdev_prefix": "hd", "cdrom_image_path": "", "device_model": "/usr/lib/xen/bin/qemu-dm", "pae": true, "vnc_password_file": "/etc/ganeti/vnc-cluster-password", "disk_type": "paravirtual", "kernel_path": "/usr/lib/xen/boot/hvmloader", "acpi": true } }, "default_hypervisor": "xen-pvm", "uid_pool": [], "prealloc_wipe_disks": false, "primary_ip_version": 4, "mtime": 1331075221.432734, "os_hvp": { "instance-debootstrap": { "xen-pvm": { "root_path": "/dev/xvda1", "kernel_path": "/boot/vmlinuz-2.6.38" } } }, "osparams": { "debootstrap": { "dhcp": "no", "partition_style": "none", "packages": "ssh" } }, "shared_file_storage_dir": "", "master_netmask": 32, "uuid": "1616c1cc-f793-499c-b1c5-48264c2d2976", "use_external_mip_script": false, "export_version": 0, "hidden_os": [ "lenny" ], "os_api_version": 20, "master": "node4", "nicparams": { "default": { "link": "xen-br0", "mode": "bridged" } }, "protocol_version": 2050000, "config_version": 2050000, "software_version": "2.5.0~rc5", "tags": [ "htools:iextags:test", "htools:iextags:service-group" ], "ipolicy": { "std": { "nic-count": 1, "disk-size": 1024, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "minmax": [ { "min": { "nic-count": 1, "disk-size": 128, "disk-count": 1, "memory-size": 128, "cpu-count": 1, "spindle-use": 1 }, "max": { "nic-count": 8, "disk-size": 1048576, "disk-count": 16, "memory-size": 32768, "cpu-count": 8, "spindle-use": 8 } } ], "vcpu-ratio": 4.0, "disk-templates": [ "sharedfile", "diskless", "plain", "blockdev", "drbd", "file", "rbd" ], "spindle-ratio": 32.0 }, "candidate_pool_size": 3, "file_storage_dir": "/srv/ganeti/file-storage", "blacklisted_os": [], "enabled_hypervisors": [ "xen-pvm", "xen-hvm" ], "reserved_lvs": [ "xenvg/test" ], "drbd_usermode_helper": "/bin/true", "default_iallocator": "hail", "ctime": 1271079848.3199999, "name": "cluster", "master_netdev": "xen-br0", "ndparams": { "spindle_count": 1, "oob_program": null }, "architecture": [ "64bit", "x86_64" ], "volume_group_name": "xenvg", "beparams": { "default": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128 } } } ganeti-3.1.0~rc2/test/data/htools/rapi/instances.json000064400000000000000000000437731476477700300226370ustar00rootroot00000000000000[ { "disk_usage": 256, "oper_vcpus": 1, "serial_no": 7, "hvparams": { "root_path": "/dev/xvda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-ganetixenu-2.6.38", "initrd_path": "", "reboot_behavior": "reboot" }, "oper_state": true, "disk_template": "drbd", "mtime": 1330349951.511833, "nic.modes": [ "bridged" ], "oper_ram": 128, "pnode": "node3", "nic.bridges": [ "xen-br0" ], "status": "running", "custom_hvparams": {}, "tags": [], "nic.ips": [ null ], "snodes": [ "node4" ], "nic.macs": [ "aa:00:00:73:20:3e" ], "name": "instance2", "network_port": null, "ctime": 1327334413.084552, "custom_beparams": {}, "custom_nicparams": [ {} ], "uuid": "4b9ff2a2-3399-4141-b4e1-cde418b1dfec", "disk.sizes": [ 128 ], "disk.spindles": [ null ], "admin_state": "up", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "debian-image", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } }, { "disk_usage": 384, "oper_vcpus": null, "serial_no": 6, "hvparams": { "root_path": "/dev/xvda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-ganetixenu-2.6.38", "initrd_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "plain", "mtime": 1325681489.4059889, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node4", "nic.bridges": [ "xen-br0" ], "status": "ADMIN_down", "custom_hvparams": { "root_path": "/dev/xvda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "kernel_path": "/boot/vmlinuz-ganetixenu-2.6.38", "initrd_path": "" }, "tags": [], "nic.ips": [ null ], "snodes": [], "nic.macs": [ "aa:00:00:ec:e8:a2" ], "name": "instance3", "network_port": null, "ctime": 1312272250.96, "custom_beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "maxmem": 128, "spindle_use": 1 }, "custom_nicparams": [ { "link": "xen-br0", "mode": "bridged" } ], "uuid": "3cecca87-eae7-476c-847c-818a28764989", "disk.sizes": [ 256, 128 ], "disk.spindles": [ null, null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "debian-image", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } }, { "disk_usage": 2176, "oper_vcpus": null, "serial_no": 23, "hvparams": { "root_path": "/dev/xvda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-ganetixenu-2.6.38", "initrd_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "drbd", "mtime": 1325681487.384176, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node4", "nic.bridges": [ "xen-br0" ], "status": "ADMIN_down", "custom_hvparams": {}, "tags": [ "service-group:dns" ], "nic.ips": [ null ], "snodes": [ "node3" ], "nic.macs": [ "aa:00:00:62:b0:76" ], "name": "instance4", "network_port": null, "ctime": 1274885795.4000001, "custom_beparams": {}, "custom_nicparams": [ {} ], "uuid": "33f4c063-bb65-41b2-af29-d8a631201bd7", "disk.sizes": [ 2048 ], "disk.spindles": [ null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "lenny-image", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } }, { "disk_usage": 256, "oper_vcpus": null, "serial_no": 9, "hvparams": { "spice_password_file": "", "spice_use_tls": false, "spice_use_vdagent": true, "nic_type": "paravirtual", "vnc_bind_address": "0.0.0.0", "cdrom2_image_path": "", "usb_mouse": "", "spice_streaming_video": "", "use_chroot": false, "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "migration_downtime": 30, "floppy_image_path": "", "security_model": "none", "cdrom_image_path": "", "spice_ip_version": 0, "vhost_net": false, "cpu_mask": "all", "disk_cache": "default", "kernel_path": "/boot/vmlinuz-2.6.38-gg426-generic", "initrd_path": "/boot/initrd.img-2.6.38-gg426-generic", "spice_jpeg_wan_compression": "", "vnc_tls": false, "cdrom_disk_type": "", "use_localtime": false, "security_domain": "", "serial_console": false, "spice_bind": "", "spice_zlib_glz_wan_compression": "", "kvm_flag": "", "vnc_password_file": "", "disk_type": "paravirtual", "vnc_x509_verify": false, "spice_image_compression": "", "spice_playback_compression": true, "kernel_args": "ro", "root_path": "/dev/vda1", "vnc_x509_path": "", "acpi": true, "keymap": "", "boot_order": "disk", "mem_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "plain", "mtime": 1325681492.191576, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node4", "nic.bridges": [ "xen-br0" ], "status": "ADMIN_down", "custom_hvparams": {}, "tags": [], "nic.ips": [ null ], "snodes": [], "nic.macs": [ "aa:00:00:3f:6d:e3" ], "name": "instance8", "network_port": 12111, "ctime": 1311771325.6600001, "custom_beparams": {}, "custom_nicparams": [ {} ], "uuid": "1ea53cc3-cc69-43da-b261-f22ac47896ea", "disk.sizes": [ 256 ], "disk.spindles": [ null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "debian-image", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } }, { "disk_usage": 256, "oper_vcpus": null, "serial_no": 31, "hvparams": { "root_path": "/dev/sda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-2.6-xenU", "initrd_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "drbd", "mtime": 1325681490.685926, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node3", "nic.bridges": [ "xen-br0" ], "status": "ADMIN_down", "custom_hvparams": { "root_path": "/dev/sda1", "kernel_args": "ro", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "kernel_path": "/boot/vmlinuz-2.6-xenU", "initrd_path": "" }, "tags": [ "gogu:test" ], "nic.ips": [ null ], "snodes": [ "node4" ], "nic.macs": [ "aa:00:00:10:d2:01" ], "name": "instance9", "network_port": null, "ctime": 1271937489.76, "custom_beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "maxmem": 128, "spindle_use": 1 }, "custom_nicparams": [ {} ], "uuid": "4927ac66-a3c5-45c6-be39-97e2b119557e", "disk.sizes": [ 128 ], "disk.spindles": [ null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "lenny-image", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } }, { "disk_usage": 512, "oper_vcpus": null, "serial_no": 11, "hvparams": { "root_path": "/dev/sda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-2.6-xenU", "initrd_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "plain", "mtime": 1325681493.0002201, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node4", "nic.bridges": [ "xen-br1" ], "status": "ADMIN_down", "custom_hvparams": {}, "tags": [], "nic.ips": [ null ], "snodes": [], "nic.macs": [ "aa:00:00:7f:8c:9c" ], "name": "instance13", "network_port": null, "ctime": 1305129727.7, "custom_beparams": {}, "custom_nicparams": [ { "link": "xen-br1" } ], "uuid": "b864e453-f072-41fe-9973-7673c2161e34", "disk.sizes": [ 512 ], "disk.spindles": [ null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br1" ], "os": "busybox", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } }, { "disk_usage": 256, "oper_vcpus": null, "serial_no": 11, "hvparams": { "root_path": "/dev/xvda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-ganetixenu-2.6.38", "initrd_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "drbd", "mtime": 1325681493.8268771, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node3", "nic.bridges": [ "xen-br0" ], "status": "ADMIN_down", "custom_hvparams": {}, "tags": [], "nic.ips": [ null ], "snodes": [ "node4" ], "nic.macs": [ "aa:00:00:eb:0b:a5" ], "name": "instance14", "network_port": null, "ctime": 1312285580.27, "custom_beparams": {}, "custom_nicparams": [ {} ], "uuid": "e9dae1c9-b4cb-4f11-b0e9-65931a6b3524", "disk.sizes": [ 128 ], "disk.spindles": [ null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "debian-image", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } }, { "disk_usage": 128, "oper_vcpus": null, "serial_no": 9, "hvparams": { "root_path": "/dev/sda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-2.6-xenU", "initrd_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "plain", "mtime": 1325681491.0986331, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node4", "nic.bridges": [ "xen-br0" ], "status": "ADMIN_down", "custom_hvparams": {}, "tags": [], "nic.ips": [ null ], "snodes": [], "nic.macs": [ "aa:00:00:55:94:93" ], "name": "instance18", "network_port": null, "ctime": 1297176343.1700001, "custom_beparams": { "minmem": 8192, "maxmem": 8192 }, "custom_nicparams": [ {} ], "uuid": "2f14bc3b-8448-4b2f-a592-d7a216244b22", "disk.sizes": [ 128 ], "disk.spindles": [ null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "busybox", "beparams": { "auto_balance": true, "minmem": 8192, "vcpus": 1, "always_failover": false, "maxmem": 8192, "spindle_use": 1 } }, { "disk_usage": 256, "oper_vcpus": null, "serial_no": 10, "hvparams": { "root_path": "/dev/xvda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-ganetixenu-2.6.38", "initrd_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "drbd", "mtime": 1325681491.5785329, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node3", "nic.bridges": [ "xen-br0" ], "status": "ADMIN_down", "custom_hvparams": {}, "tags": [], "nic.ips": [ null ], "snodes": [ "node4" ], "nic.macs": [ "aa:00:00:15:92:6f" ], "name": "instance19", "network_port": null, "ctime": 1312464490.7, "custom_beparams": {}, "custom_nicparams": [ {} ], "uuid": "624c1844-82a2-474e-bdaf-1bafa820fdcf", "disk.sizes": [ 128 ], "disk.spindles": [ null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "debian-image", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } }, { "disk_usage": 512, "oper_vcpus": null, "serial_no": 14, "hvparams": { "spice_password_file": "", "spice_use_tls": false, "spice_use_vdagent": true, "nic_type": "paravirtual", "vnc_bind_address": "0.0.0.0", "cdrom2_image_path": "", "usb_mouse": "", "spice_streaming_video": "", "use_chroot": false, "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "migration_downtime": 30, "floppy_image_path": "", "security_model": "none", "cdrom_image_path": "", "spice_ip_version": 0, "vhost_net": false, "cpu_mask": "all", "disk_cache": "default", "kernel_path": "/boot/vmlinuz-2.6.38-gg426-generic", "initrd_path": "/boot/initrd.img-2.6.38-gg426-generic", "spice_jpeg_wan_compression": "", "vnc_tls": false, "cdrom_disk_type": "", "use_localtime": false, "security_domain": "", "serial_console": false, "spice_bind": "", "spice_zlib_glz_wan_compression": "", "kvm_flag": "", "vnc_password_file": "", "disk_type": "paravirtual", "vnc_x509_verify": false, "spice_image_compression": "", "spice_playback_compression": true, "kernel_args": "ro", "root_path": "/dev/vda1", "vnc_x509_path": "", "acpi": true, "keymap": "", "boot_order": "disk", "mem_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "plain", "mtime": 1325681494.699162, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node4", "nic.bridges": [ "xen-br0" ], "status": "ADMIN_down", "custom_hvparams": {}, "tags": [], "nic.ips": [ null ], "snodes": [], "nic.macs": [ "aa:00:00:db:2a:6d" ], "name": "instance20", "network_port": 12107, "ctime": 1305208955.75, "custom_beparams": {}, "custom_nicparams": [ { "link": "xen-br0" } ], "uuid": "4f65c14d-be87-4303-a8dc-ba1b86e2a3b3", "disk.sizes": [ 512 ], "disk.spindles": [ null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "lenny-image+default", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } }, { "disk_usage": 256, "oper_vcpus": null, "serial_no": 10, "hvparams": { "root_path": "/dev/xvda1", "kernel_args": "ro", "blockdev_prefix": "sd", "use_bootloader": false, "bootloader_args": "", "bootloader_path": "", "cpu_mask": "all", "kernel_path": "/boot/vmlinuz-ganetixenu-2.6.38", "initrd_path": "", "reboot_behavior": "reboot" }, "oper_state": false, "disk_template": "drbd", "mtime": 1325681489.0591741, "nic.modes": [ "bridged" ], "oper_ram": null, "pnode": "node3", "nic.bridges": [ "xen-br0" ], "status": "ADMIN_down", "custom_hvparams": {}, "tags": [], "nic.ips": [ null ], "snodes": [ "node4" ], "nic.macs": [ "aa:00:00:cb:96:c1" ], "name": "instance21", "network_port": null, "ctime": 1312552008.1199999, "custom_beparams": {}, "custom_nicparams": [ {} ], "uuid": "6f2f7824-8392-408e-ac54-c938f4fb0638", "disk.sizes": [ 128 ], "disk.spindles": [ null ], "admin_state": "down", "admin_state_source": "admin", "nic.links": [ "xen-br0" ], "os": "debian-image", "beparams": { "auto_balance": true, "minmem": 128, "vcpus": 1, "always_failover": false, "maxmem": 128, "spindle_use": 1 } } ] ganeti-3.1.0~rc2/test/data/htools/rapi/nodes.json000064400000000000000000000070371476477700300217510ustar00rootroot00000000000000[ { "cnodes": 2, "cnos": 1, "csockets": 2, "ctime": 1324472016.2968869, "ctotal": 4, "dfree": 1377280, "drained": false, "dtotal": 1377280, "group.uuid": "uuid-group-1", "master_candidate": true, "master_capable": true, "mfree": 31389, "mnode": 1017, "mtime": 1331075221.432734, "mtotal": 32763, "name": "node1", "offline": false, "pinst_cnt": 0, "pinst_list": [], "pip": "192.168.1.1", "role": "C", "serial_no": 3, "sinst_cnt": 0, "sinst_list": [], "sip": "192.168.1.2", "spfree": 0, "sptotal": 0, "tags": [], "uuid": "7750ef3d-450f-4724-9d3d-8726d6335417", "vm_capable": true, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false } }, { "cnodes": 2, "cnos": 1, "csockets": 2, "ctime": 1324472016.2968869, "ctotal": 4, "dfree": 1376640, "drained": false, "dtotal": 1377280, "group.uuid": "uuid-group-1", "master_candidate": true, "master_capable": true, "mfree": 31746, "mnode": 1017, "mtime": 1331075221.432734, "mtotal": 32763, "name": "node2", "offline": false, "pinst_cnt": 0, "pinst_list": [], "pip": "192.168.1.2", "role": "C", "serial_no": 3, "sinst_cnt": 0, "sinst_list": [], "sip": "192.168.2.2", "spfree": 0, "sptotal": 0, "tags": [], "uuid": "7750ef3d-450f-4724-9d3d-8726d6335417", "vm_capable": true, "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false } }, { "cnodes": 2, "cnos": 1, "dfree": 1373336, "drained": false, "dtotal": 1377304, "mfree": 31234, "mtime": 1331075172.0123219, "pip": "192.168.1.3", "serial_no": 129, "sinst_cnt": 1, "sip": "192.168.2.3", "uuid": "2c7acf04-599d-4707-aba4-bf07a2685f63", "sinst_list": [ "instance4" ], "csockets": 2, "role": "C", "ctotal": 4, "offline": false, "vm_capable": true, "pinst_cnt": 5, "mtotal": 32763, "tags": [], "group.uuid": "uuid-group-1", "master_capable": true, "name": "node3", "master_candidate": true, "ctime": 1271425438.5, "mnode": 1017, "spfree": 0, "sptotal": 0, "pinst_list": [ "instance14", "instance19", "instance2", "instance21", "instance9" ], "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false } }, { "cnodes": 2, "cnos": 1, "dfree": 1371520, "drained": false, "dtotal": 1377280, "mfree": 31746, "mtime": 1318339824.54, "pip": "192.168.1.4", "serial_no": 8, "sinst_cnt": 5, "sip": "192.168.2.4", "uuid": "f25357c1-7fee-4471-b8a9-c7f28669e439", "sinst_list": [ "instance2", "instance21", "instance14", "instance9", "instance19" ], "csockets": 2, "role": "M", "ctotal": 4, "offline": false, "vm_capable": true, "pinst_cnt": 7, "mtotal": 32763, "tags": [], "group.uuid": "uuid-group-1", "master_capable": true, "name": "node4", "master_candidate": true, "ctime": 1309185898.51, "mnode": 1017, "spfree": 0, "sptotal": 0, "pinst_list": [ "instance20", "instance3", "instance15", "instance4", "instance13", "instance8", "instance18" ], "ndparams": { "spindle_count": 1, "oob_program": null, "exclusive_storage": false } } ] ganeti-3.1.0~rc2/test/data/htools/shared-n1-failure.data000064400000000000000000000015431476477700300230510ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| nodeA|4096|0|1096|200|200|4|M|fake-uuid-01|4 nodeB|4096|0|1096|200|200|4|N|fake-uuid-01|4 nodeC|4096|0|1096|200|200|4|N|fake-uuid-01|4 nodeD|4096|0|1096|200|200|4|N|fake-uuid-01|4 instA1|1000|0|1|running|Y|nodeA||ext||1 instA2|1000|0|1|running|Y|nodeA||ext||1 instA3|1000|0|1|running|Y|nodeA||ext||1 instB1|1000|0|1|running|Y|nodeB||ext||1 instB2|1000|0|1|running|Y|nodeB||ext||1 instB3|1000|0|1|running|Y|nodeB||ext||1 instC1|3000|0|3|running|Y|nodeC||ext||3 instD1|1000|0|1|running|Y|nodeD||ext||1 instD2|1000|0|1|running|Y|nodeD||ext||1 instD3|1000|0|1|running|Y|nodeD||ext||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/shared-n1-restriction.data000064400000000000000000000014231476477700300237640ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| nodeA|4096|0|3096|200|200|4|M|fake-uuid-01|4 nodeB|4096|0|1096|200|200|4|N|fake-uuid-01|4 nodeC|4096|0|1096|200|200|4|N|fake-uuid-01|4 nodeD|4096|0|1096|200|200|4|N|fake-uuid-01|4 instA1|1000|0|1|running|Y|nodeA||ext||1 instB1|1000|0|1|running|Y|nodeB||ext||1 instB2|1000|0|1|running|Y|nodeB||ext||1 instB3|1000|0|1|running|Y|nodeB||ext||1 instC1|3000|0|3|running|Y|nodeC||ext||3 instD1|1000|0|1|running|Y|nodeD||ext||1 instD2|1000|0|1|running|Y|nodeD||ext||1 instD3|1000|0|1|running|Y|nodeD||ext||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/htools/unique-reboot-order.data000064400000000000000000000012461476477700300235510ustar00rootroot00000000000000group-01|fake-uuid-01|preferred|| node-01-001|91552|0|91424|3500|1196|16|M|fake-uuid-01|1 node-01-002|91552|0|91296|3500|1196|16|N|fake-uuid-01|1 node-01-003|91552|0|91296|3500|1196|16|N|fake-uuid-01|1 new-0|128|1152|1|running|Y|node-01-001|node-01-002|drbd||1 new-1|128|1152|1|running|Y|node-01-002|node-01-003|drbd||1 nonred-0|128|1152|1|running|Y|node-01-001||plain||1 nonred-1|128|1152|1|running|Y|node-01-003||plain||1 |128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 group-01|128,1,1024,1,1,1|128,1,1024,1,1,1;32768,8,1048576,16,8,12|diskless,file,sharedfile,plain,blockdev,drbd,rbd,ext|4.0|32.0 ganeti-3.1.0~rc2/test/data/instance-disks.txt000064400000000000000000000034111476477700300211530ustar00rootroot00000000000000[[{"admin_state": "up", "admin_state_source": "user", "beparams": {}, "ctime": 1372838883.9710441, "disks": ["5d61e205-bf89-4ba8-a319-589b7bb7419e"], "disks_active": true, "hvparams": {}, "hypervisor": "xen-pvm", "mtime": 1372838946.2599809, "name": "instance1.example.com", "nics": [ { "mac": "aa:00:00:1d:ba:63", "nicparams": {}, "uuid": "7b7f4249-fab8-4b3f-b446-d7a2aff37644" } ], "os": "busybox", "osparams": {}, "osparams_private": {}, "primary_node": "60e687a0-21fc-4577-997f-ccd08925fa65", "serial_no": 2, "uuid": "aec390cb-5eae-44e6-bcc2-ec14d31347f0" }, [{ "children": [ { "dev_type": "plain", "logical_id": [ "xenvg", "df9ff3f6-a833-48ff-8bd5-bff2eaeab759.disk0_data" ], "params": {}, "size": 1024, "uuid": "eaff6322-1bfb-4d59-b306-4535730917cc", "serial_no": 1, "ctime": 1372838946.2599809, "mtime": 1372838946.2599809 }, { "dev_type": "plain", "logical_id": [ "xenvg", "df9ff3f6-a833-48ff-8bd5-bff2eaeab759.disk0_meta" ], "params": {}, "size": 128, "uuid": "bf512e95-2a49-4cb3-8d1f-30a503f6bf1b", "serial_no": 1, "mtime": 1372838946.2599809, "ctime": 1372838946.2599809 } ], "dev_type": "drbd", "iv_name": "disk/0", "logical_id": [ "60e687a0-21fc-4577-997f-ccd08925fa65", "c739c7f3-79d8-4e20-ac68-662e16577d2e", 11000, 0, 0, "9bdb15fb7ab6bb4610a313d654ed4d0d2433713e" ], "mode": "rw", "params": {}, "size": 1024, "uuid": "5d61e205-bf89-4ba8-a319-589b7bb7419e", "serial_no": 1, "ctime": 1372838946.2599809, "mtime": 1372838946.2599809 }]] ] ganeti-3.1.0~rc2/test/data/instance-minor-pairing.txt000064400000000000000000000001421476477700300226070ustar00rootroot00000000000000[["machine1.example.com",0,"instance1.example.com","disk/0","secondary", "machine2.example.com"]] ganeti-3.1.0~rc2/test/data/ip-addr-show-dummy0.txt000064400000000000000000000004041476477700300217420ustar00rootroot000000000000007: dummy0: mtu 1500 qdisc noop state DOWN link/ether 06:d2:06:24:99:dc brd ff:ff:ff:ff:ff:ff inet 192.0.2.1/32 scope global dummy0 inet6 2001:db8:85a3::8a2e:370:7334/128 scope global valid_lft forever preferred_lft forever ganeti-3.1.0~rc2/test/data/ip-addr-show-lo-ipv4.txt000064400000000000000000000001471476477700300220250ustar00rootroot000000000000001: lo: mtu 16436 qdisc noqueue state UNKNOWN inet 127.0.0.1/8 scope host lo ganeti-3.1.0~rc2/test/data/ip-addr-show-lo-ipv6.txt000064400000000000000000000001641476477700300220260ustar00rootroot000000000000001: lo: mtu 16436 inet6 ::1/128 scope host valid_lft forever preferred_lft forever ganeti-3.1.0~rc2/test/data/ip-addr-show-lo-oneline-ipv4.txt000064400000000000000000000000501476477700300234450ustar00rootroot000000000000001: lo inet 127.0.0.1/8 scope host lo ganeti-3.1.0~rc2/test/data/ip-addr-show-lo-oneline-ipv6.txt000064400000000000000000000001221476477700300234470ustar00rootroot000000000000001: lo inet6 ::1/128 scope host \ valid_lft forever preferred_lft forever ganeti-3.1.0~rc2/test/data/ip-addr-show-lo-oneline.txt000064400000000000000000000003711476477700300225730ustar00rootroot000000000000001: lo: mtu 16436 qdisc noqueue state UNKNOWN \ link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 1: lo inet 127.0.0.1/8 scope host lo 1: lo inet6 ::1/128 scope host \ valid_lft forever preferred_lft forever ganeti-3.1.0~rc2/test/data/ip-addr-show-lo.txt000064400000000000000000000003551476477700300211460ustar00rootroot000000000000001: lo: mtu 16436 qdisc noqueue state UNKNOWN link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo inet6 ::1/128 scope host valid_lft forever preferred_lft forever ganeti-3.1.0~rc2/test/data/kvm_0.12.5_help.txt000064400000000000000000000260751476477700300206570ustar00rootroot00000000000000QEMU PC emulator version 0.12.5 (qemu-kvm-0.12.5), Copyright (c) 2003-2008 Fabrice Bellard usage: qemu [options] [disk_image] 'disk_image' is a raw hard image image for IDE hard disk 0 Standard options: -h or -help display this help and exit -version display version information and exit -M machine select emulated machine (-M ? for list) -cpu cpu select CPU (-cpu ? for list) -smp n[,maxcpus=cpus][,cores=cores][,threads=threads][,sockets=sockets] set the number of CPUs to 'n' [default=1] maxcpus= maximum number of total cpus, including offline CPUs for hotplug etc. cores= number of CPU cores on one socket threads= number of threads on one CPU core sockets= number of discrete sockets in the system -numa node[,mem=size][,cpus=cpu[-cpu]][,nodeid=node] -fda/-fdb file use 'file' as floppy disk 0/1 image -hda/-hdb file use 'file' as IDE hard disk 0/1 image -hdc/-hdd file use 'file' as IDE hard disk 2/3 image -cdrom file use 'file' as IDE cdrom image (cdrom is ide1 master) -drive [file=file][,if=type][,bus=n][,unit=m][,media=d][,index=i] [,cyls=c,heads=h,secs=s[,trans=t]][,snapshot=on|off] [,cache=writethrough|writeback|none][,format=f][,serial=s] [,addr=A][,id=name][,aio=threads|native] [,boot=on|off] use 'file' as a drive image -set group.id.arg=value set parameter for item of type i.e. -set drive.$id.file=/path/to/image -global driver.property=value set a global default for a driver property -mtdblock file use 'file' as on-board Flash memory image -sd file use 'file' as SecureDigital card image -pflash file use 'file' as a parallel flash image -boot [order=drives][,once=drives][,menu=on|off] 'drives': floppy (a), hard disk (c), CD-ROM (d), network (n) -snapshot write to temporary files instead of disk image files -m megs set virtual RAM size to megs MB [default=128] -k language use keyboard layout (for example 'fr' for French) -audio-help print list of audio drivers and their options -soundhw c1,... enable audio support and only specified sound cards (comma separated list) use -soundhw ? to get the list of supported cards use -soundhw all to enable all of them -usb enable the USB driver (will be the default soon) -usbdevice name add the host or guest USB device 'name' -device driver[,options] add device -name string1[,process=string2] set the name of the guest string1 sets the window title and string2 the process name (on Linux) -uuid %08x-%04x-%04x-%04x-%012x specify machine UUID Display options: -nographic disable graphical output and redirect serial I/Os to console -curses use a curses/ncurses interface instead of SDL -no-frame open SDL window without a frame and window decorations -alt-grab use Ctrl-Alt-Shift to grab mouse (instead of Ctrl-Alt) -ctrl-grab use Right-Ctrl to grab mouse (instead of Ctrl-Alt) -no-quit disable SDL window close capability -sdl enable SDL -portrait rotate graphical output 90 deg left (only PXA LCD) -vga [std|cirrus|vmware|xenfb|none] select video card type -full-screen start in full screen -vnc display start a VNC server on display i386 target only: -win2k-hack use it when installing Windows 2000 to avoid a disk full bug -no-fd-bootchk disable boot signature checking for floppy disks -no-acpi disable ACPI -no-hpet disable HPET -balloon none disable balloon device -balloon virtio[,addr=str] enable virtio balloon device (default) -acpitable [sig=str][,rev=n][,oem_id=str][,oem_table_id=str][,oem_rev=n][,asl_compiler_id=str][,asl_compiler_rev=n][,data=file1[:file2]...] ACPI table description -smbios file=binary Load SMBIOS entry from binary file -smbios type=0[,vendor=str][,version=str][,date=str][,release=%d.%d] Specify SMBIOS type 0 fields -smbios type=1[,manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str] Specify SMBIOS type 1 fields Network options: -net nic[,vlan=n][,macaddr=mac][,model=type][,name=str][,addr=str][,vectors=v] create a new Network Interface Card and connect it to VLAN 'n' -net user[,vlan=n][,name=str][,net=addr[/mask]][,host=addr][,restrict=y|n] [,hostname=host][,dhcpstart=addr][,dns=addr][,tftp=dir][,bootfile=f] [,hostfwd=rule][,guestfwd=rule][,smb=dir[,smbserver=addr]] connect the user mode network stack to VLAN 'n', configure its DHCP server and enabled optional services -net tap[,vlan=n][,name=str][,fd=h][,ifname=name][,script=file][,downscript=dfile][,sndbuf=nbytes][,vnet_hdr=on|off] connect the host TAP network interface to VLAN 'n' and use the network scripts 'file' (default=/etc/kvm/kvm-ifup) and 'dfile' (default=/etc/kvm/kvm-ifdown); use '[down]script=no' to disable script execution; use 'fd=h' to connect to an already opened TAP interface use 'sndbuf=nbytes' to limit the size of the send buffer; the default of 'sndbuf=1048576' can be disabled using 'sndbuf=0' use vnet_hdr=off to avoid enabling the IFF_VNET_HDR tap flag; use vnet_hdr=on to make the lack of IFF_VNET_HDR support an error condition -net socket[,vlan=n][,name=str][,fd=h][,listen=[host]:port][,connect=host:port] connect the vlan 'n' to another VLAN using a socket connection -net socket[,vlan=n][,name=str][,fd=h][,mcast=maddr:port] connect the vlan 'n' to multicast maddr and port -net vde[,vlan=n][,name=str][,sock=socketpath][,port=n][,group=groupname][,mode=octalmode] connect the vlan 'n' to port 'n' of a vde switch running on host and listening for incoming connections on 'socketpath'. Use group 'groupname' and mode 'octalmode' to change default ownership and permissions for communication port. -net dump[,vlan=n][,file=f][,len=n] dump traffic on vlan 'n' to file 'f' (max n bytes per packet) -net none use it alone to have zero network devices; if no -net option is provided, the default is '-net nic -net user' -netdev [user|tap|vde|socket],id=str[,option][,option][,...] Character device options: -chardev null,id=id -chardev socket,id=id[,host=host],port=host[,to=to][,ipv4][,ipv6][,nodelay] [,server][,nowait][,telnet] (tcp) -chardev socket,id=id,path=path[,server][,nowait][,telnet] (unix) -chardev udp,id=id[,host=host],port=port[,localaddr=localaddr] [,localport=localport][,ipv4][,ipv6] -chardev msmouse,id=id -chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]] -chardev file,id=id,path=path -chardev pipe,id=id,path=path -chardev pty,id=id -chardev stdio,id=id,[,signal=on|off] -chardev braille,id=id -chardev tty,id=id,path=path -chardev parport,id=id,path=path Bluetooth(R) options: -bt hci,null dumb bluetooth HCI - doesn't respond to commands -bt hci,host[:id] use host's HCI with the given name -bt hci[,vlan=n] emulate a standard HCI in virtual scatternet 'n' -bt vhci[,vlan=n] add host computer to virtual scatternet 'n' using VHCI -bt device:dev[,vlan=n] emulate a bluetooth device 'dev' in scatternet 'n' Linux/Multiboot boot specific: -kernel bzImage use 'bzImage' as kernel image -append cmdline use 'cmdline' as kernel command line -initrd file use 'file' as initial ram disk Debug/Expert options: -serial dev redirect the serial port to char device 'dev' -parallel dev redirect the parallel port to char device 'dev' -monitor dev redirect the monitor to char device 'dev' -qmp dev like -monitor but opens in 'control' mode. -mon chardev=[name][,mode=readline|control][,default] -pidfile file write PID to 'file' -singlestep always run in singlestep mode -S freeze CPU at startup (use 'c' to start execution) -gdb dev wait for gdb connection on 'dev' -s shorthand for -gdb tcp::1234 -d item1,... output log to /tmp/qemu.log (use -d ? for a list of log items) -hdachs c,h,s[,t] force hard disk 0 physical geometry and the optional BIOS translation (t=none or lba) (usually qemu can guess them) -L path set the directory for the BIOS, VGA BIOS and keymaps -bios file set the filename for the BIOS -enable-kvm enable KVM full virtualization support -no-reboot exit instead of rebooting -no-shutdown stop before shutdown -loadvm [tag|id] start right away with a saved state (loadvm in monitor) -daemonize daemonize QEMU after initializing -option-rom rom load a file, rom, into the option ROM space -clock force the use of the given methods for timer alarm. To see what timers are available use -clock ? -rtc [base=utc|localtime|date][,clock=host|vm][,driftfix=none|slew] set the RTC base and clock, enable drift fix for clock ticks -icount [N|auto] enable virtual instruction counter with 2^N clock ticks per instruction -watchdog i6300esb|ib700 enable virtual hardware watchdog [default=none] -watchdog-action reset|shutdown|poweroff|pause|debug|none action when watchdog fires [default=reset] -echr chr set terminal escape character instead of ctrl-a -virtioconsole c set virtio console -show-cursor show cursor -tb-size n set TB size -incoming uri wait on uri for incoming migration -nodefaults don't create default devices. -chroot dir Chroot to dir just before starting the VM. -runas user Change to user id user just before starting the VM. -readconfig -writeconfig read/write config file -no-kvm disable KVM hardware virtualization -no-kvm-irqchip disable KVM kernel mode PIC/IOAPIC/LAPIC -no-kvm-pit disable KVM kernel mode PIT -no-kvm-pit-reinjection disable KVM kernel mode PIT interrupt reinjection -pcidevice host=bus:dev.func[,dma=none][,name=string] expose a PCI device to the guest OS. dma=none: don't perform any dma translations (default is to use an iommu) 'string' is used in log output. -enable-nesting enable support for running a VM inside the VM (AMD only) -nvram FILE provide ia64 nvram contents -tdf enable guest time drift compensation -kvm-shadow-memory MEGABYTES allocate MEGABYTES for kvm mmu shadowing -mem-path FILE provide backing storage for guest RAM -mem-prealloc preallocate guest memory (use with -mempath) During emulation, the following keys are useful: ctrl-alt-f toggle full screen ctrl-alt-n switch to virtual console 'n' ctrl-alt toggle mouse and keyboard grab When using -nographic, press 'ctrl-a h' to get some help. ganeti-3.1.0~rc2/test/data/kvm_0.15.90_help.txt000064400000000000000000000320641476477700300207410ustar00rootroot00000000000000QEMU emulator version 0.15.90, Copyright (c) 2003-2008 Fabrice Bellard usage: qemu [options] [disk_image] 'disk_image' is a raw hard disk image for IDE hard disk 0 Standard options: -h or -help display this help and exit -version display version information and exit -machine [type=]name[,prop[=value][,...]] selects emulated machine (-machine ? for list) property accel=accel1[:accel2[:...]] selects accelerator supported accelerators are kvm, xen, tcg (default: tcg) -cpu cpu select CPU (-cpu ? for list) -smp n[,maxcpus=cpus][,cores=cores][,threads=threads][,sockets=sockets] set the number of CPUs to 'n' [default=1] maxcpus= maximum number of total cpus, including offline CPUs for hotplug, etc cores= number of CPU cores on one socket threads= number of threads on one CPU core sockets= number of discrete sockets in the system -numa node[,mem=size][,cpus=cpu[-cpu]][,nodeid=node] -fda/-fdb file use 'file' as floppy disk 0/1 image -hda/-hdb file use 'file' as IDE hard disk 0/1 image -hdc/-hdd file use 'file' as IDE hard disk 2/3 image -cdrom file use 'file' as IDE cdrom image (cdrom is ide1 master) -drive [file=file][,if=type][,bus=n][,unit=m][,media=d][,index=i] [,cyls=c,heads=h,secs=s[,trans=t]][,snapshot=on|off] [,cache=writethrough|writeback|none|directsync|unsafe][,format=f] [,serial=s][,addr=A][,id=name][,aio=threads|native] [,readonly=on|off] use 'file' as a drive image -set group.id.arg=value set parameter for item of type i.e. -set drive.$id.file=/path/to/image -global driver.property=value set a global default for a driver property -mtdblock file use 'file' as on-board Flash memory image -sd file use 'file' as SecureDigital card image -pflash file use 'file' as a parallel flash image -boot [order=drives][,once=drives][,menu=on|off] [,splash=sp_name][,splash-time=sp_time] 'drives': floppy (a), hard disk (c), CD-ROM (d), network (n) 'sp_name': the file's name that would be passed to bios as logo picture, if menu=on 'sp_time': the period that splash picture last if menu=on, unit is ms -snapshot write to temporary files instead of disk image files -m megs set virtual RAM size to megs MB [default=128] -mem-path FILE provide backing storage for guest RAM -mem-prealloc preallocate guest memory (use with -mem-path) -k language use keyboard layout (for example 'fr' for French) -audio-help print list of audio drivers and their options -soundhw c1,... enable audio support and only specified sound cards (comma separated list) use -soundhw ? to get the list of supported cards use -soundhw all to enable all of them -usb enable the USB driver (will be the default soon) -usbdevice name add the host or guest USB device 'name' -device driver[,prop[=value][,...]] add device (based on driver) prop=value,... sets driver properties use -device ? to print all possible drivers use -device driver,? to print all possible properties File system options: -fsdev fsdriver,id=id,path=path,[security_model={mapped|passthrough|none}] [,writeout=immediate][,readonly] Virtual File system pass-through options: -virtfs local,path=path,mount_tag=tag,security_model=[mapped|passthrough|none] [,writeout=immediate][,readonly] -virtfs_synth Create synthetic file system image -name string1[,process=string2] set the name of the guest string1 sets the window title and string2 the process name (on Linux) -uuid %08x-%04x-%04x-%04x-%012x specify machine UUID Display options: -display sdl[,frame=on|off][,alt_grab=on|off][,ctrl_grab=on|off] [,window_close=on|off]|curses|none| vnc=[,] select display type -nographic disable graphical output and redirect serial I/Os to console -curses use a curses/ncurses interface instead of SDL -no-frame open SDL window without a frame and window decorations -alt-grab use Ctrl-Alt-Shift to grab mouse (instead of Ctrl-Alt) -ctrl-grab use Right-Ctrl to grab mouse (instead of Ctrl-Alt) -no-quit disable SDL window close capability -sdl enable SDL -spice enable spice -portrait rotate graphical output 90 deg left (only PXA LCD) -rotate rotate graphical output some deg left (only PXA LCD) -vga [std|cirrus|vmware|qxl|xenfb|none] select video card type -full-screen start in full screen -g WxH[xDEPTH] Set the initial graphical resolution and depth -vnc display start a VNC server on display i386 target only: -win2k-hack use it when installing Windows 2000 to avoid a disk full bug -no-fd-bootchk disable boot signature checking for floppy disks -no-acpi disable ACPI -no-hpet disable HPET -balloon none disable balloon device -balloon virtio[,addr=str] enable virtio balloon device (default) -acpitable [sig=str][,rev=n][,oem_id=str][,oem_table_id=str][,oem_rev=n][,asl_compiler_id=str][,asl_compiler_rev=n][,{data|file}=file1[:file2]...] ACPI table description -smbios file=binary load SMBIOS entry from binary file -smbios type=0[,vendor=str][,version=str][,date=str][,release=%d.%d] specify SMBIOS type 0 fields -smbios type=1[,manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str] specify SMBIOS type 1 fields Network options: -net nic[,vlan=n][,macaddr=mac][,model=type][,name=str][,addr=str][,vectors=v] create a new Network Interface Card and connect it to VLAN 'n' -net user[,vlan=n][,name=str][,net=addr[/mask]][,host=addr][,restrict=on|off] [,hostname=host][,dhcpstart=addr][,dns=addr][,tftp=dir][,bootfile=f] [,hostfwd=rule][,guestfwd=rule][,smb=dir[,smbserver=addr]] connect the user mode network stack to VLAN 'n', configure its DHCP server and enabled optional services -net tap[,vlan=n][,name=str][,fd=h][,ifname=name][,script=file][,downscript=dfile][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off][,vhostfd=h][,vhostforce=on|off] connect the host TAP network interface to VLAN 'n' and use the network scripts 'file' (default=/etc/qemu-ifup) and 'dfile' (default=/etc/qemu-ifdown) use '[down]script=no' to disable script execution use 'fd=h' to connect to an already opened TAP interface use 'sndbuf=nbytes' to limit the size of the send buffer (the default is disabled 'sndbuf=0' to enable flow control set 'sndbuf=1048576') use vnet_hdr=off to avoid enabling the IFF_VNET_HDR tap flag use vnet_hdr=on to make the lack of IFF_VNET_HDR support an error condition use vhost=on to enable experimental in kernel accelerator (only has effect for virtio guests which use MSIX) use vhostforce=on to force vhost on for non-MSIX virtio guests use 'vhostfd=h' to connect to an already opened vhost net device -net socket[,vlan=n][,name=str][,fd=h][,listen=[host]:port][,connect=host:port] connect the vlan 'n' to another VLAN using a socket connection -net socket[,vlan=n][,name=str][,fd=h][,mcast=maddr:port[,localaddr=addr]] connect the vlan 'n' to multicast maddr and port use 'localaddr=addr' to specify the host address to send packets from -net vde[,vlan=n][,name=str][,sock=socketpath][,port=n][,group=groupname][,mode=octalmode] connect the vlan 'n' to port 'n' of a vde switch running on host and listening for incoming connections on 'socketpath'. Use group 'groupname' and mode 'octalmode' to change default ownership and permissions for communication port. -net dump[,vlan=n][,file=f][,len=n] dump traffic on vlan 'n' to file 'f' (max n bytes per packet) -net none use it alone to have zero network devices. If no -net option is provided, the default is '-net nic -net user' -netdev [user|tap|vde|socket],id=str[,option][,option][,...] Character device options: -chardev null,id=id[,mux=on|off] -chardev socket,id=id[,host=host],port=host[,to=to][,ipv4][,ipv6][,nodelay] [,server][,nowait][,telnet][,mux=on|off] (tcp) -chardev socket,id=id,path=path[,server][,nowait][,telnet],[mux=on|off] (unix) -chardev udp,id=id[,host=host],port=port[,localaddr=localaddr] [,localport=localport][,ipv4][,ipv6][,mux=on|off] -chardev msmouse,id=id[,mux=on|off] -chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]] [,mux=on|off] -chardev file,id=id,path=path[,mux=on|off] -chardev pipe,id=id,path=path[,mux=on|off] -chardev pty,id=id[,mux=on|off] -chardev stdio,id=id[,mux=on|off][,signal=on|off] -chardev braille,id=id[,mux=on|off] -chardev tty,id=id,path=path[,mux=on|off] -chardev parport,id=id,path=path[,mux=on|off] -chardev spicevmc,id=id,name=name[,debug=debug] Bluetooth(R) options: -bt hci,null dumb bluetooth HCI - doesn't respond to commands -bt hci,host[:id] use host's HCI with the given name -bt hci[,vlan=n] emulate a standard HCI in virtual scatternet 'n' -bt vhci[,vlan=n] add host computer to virtual scatternet 'n' using VHCI -bt device:dev[,vlan=n] emulate a bluetooth device 'dev' in scatternet 'n' Linux/Multiboot boot specific: -kernel bzImage use 'bzImage' as kernel image -append cmdline use 'cmdline' as kernel command line -initrd file use 'file' as initial ram disk Debug/Expert options: -serial dev redirect the serial port to char device 'dev' -parallel dev redirect the parallel port to char device 'dev' -monitor dev redirect the monitor to char device 'dev' -qmp dev like -monitor but opens in 'control' mode -mon chardev=[name][,mode=readline|control][,default] -debugcon dev redirect the debug console to char device 'dev' -pidfile file write PID to 'file' -singlestep always run in singlestep mode -S freeze CPU at startup (use 'c' to start execution) -gdb dev wait for gdb connection on 'dev' -s shorthand for -gdb tcp::1234 -d item1,... output log to /tmp/qemu.log (use -d ? for a list of log items) -D logfile output log to logfile (instead of the default /tmp/qemu.log) -hdachs c,h,s[,t] force hard disk 0 physical geometry and the optional BIOS translation (t=none or lba) (usually qemu can guess them) -L path set the directory for the BIOS, VGA BIOS and keymaps -bios file set the filename for the BIOS -enable-kvm enable KVM full virtualization support -xen-domid id specify xen guest domain id -xen-create create domain using xen hypercalls, bypassing xend warning: should not be used when xend is in use -xen-attach attach to existing xen domain xend will use this when starting qemu -no-reboot exit instead of rebooting -no-shutdown stop before shutdown -loadvm [tag|id] start right away with a saved state (loadvm in monitor) -daemonize daemonize QEMU after initializing -option-rom rom load a file, rom, into the option ROM space -clock force the use of the given methods for timer alarm. To see what timers are available use -clock ? -rtc [base=utc|localtime|date][,clock=host|vm][,driftfix=none|slew] set the RTC base and clock, enable drift fix for clock ticks (x86 only) -icount [N|auto] enable virtual instruction counter with 2^N clock ticks per instruction -watchdog i6300esb|ib700 enable virtual hardware watchdog [default=none] -watchdog-action reset|shutdown|poweroff|pause|debug|none action when watchdog fires [default=reset] -echr chr set terminal escape character instead of ctrl-a -virtioconsole c set virtio console -show-cursor show cursor -tb-size n set TB size -incoming p prepare for incoming migration, listen on port p -nodefaults don't create default devices -chroot dir chroot to dir just before starting the VM -runas user change to user id user just before starting the VM -prom-env variable=value set OpenBIOS nvram variables -semihosting semihosting mode -old-param old param mode -readconfig -writeconfig read/write config file -nodefconfig do not load default config files at startup -trace [events=][,file=] specify tracing options During emulation, the following keys are useful: ctrl-alt-f toggle full screen ctrl-alt-n switch to virtual console 'n' ctrl-alt toggle mouse and keyboard grab When using -nographic, press 'ctrl-a h' to get some help. ganeti-3.1.0~rc2/test/data/kvm_0.9.1_help.txt000064400000000000000000000136031476477700300205720ustar00rootroot00000000000000QEMU PC emulator version 0.9.1 (kvm-72), Copyright (c) 2003-2008 Fabrice Bellard usage: qemu [options] [disk_image] 'disk_image' is a raw hard image image for IDE hard disk 0 Standard options: -M machine select emulated machine (-M ? for list) -cpu cpu select CPU (-cpu ? for list) -fda/-fdb file use 'file' as floppy disk 0/1 image -hda/-hdb file use 'file' as IDE hard disk 0/1 image -hdc/-hdd file use 'file' as IDE hard disk 2/3 image -cdrom file use 'file' as IDE cdrom image (cdrom is ide1 master) -drive [file=file][,if=type][,bus=n][,unit=m][,media=d][,index=i] [,cyls=c,heads=h,secs=s[,trans=t]][,snapshot=on|off] [,cache=on|off][,format=f][,boot=on|off] use 'file' as a drive image -mtdblock file use 'file' as on-board Flash memory image -sd file use 'file' as SecureDigital card image -pflash file use 'file' as a parallel flash image -boot [a|c|d|n] boot on floppy (a), hard disk (c), CD-ROM (d), or network (n) -snapshot write to temporary files instead of disk image files -no-frame open SDL window without a frame and window decorations -alt-grab use Ctrl-Alt-Shift to grab mouse (instead of Ctrl-Alt) -no-quit disable SDL window close capability -no-fd-bootchk disable boot signature checking for floppy disks -m megs set virtual RAM size to megs MB [default=128] -smp n set the number of CPUs to 'n' [default=1] -nographic disable graphical output and redirect serial I/Os to console -portrait rotate graphical output 90 deg left (only PXA LCD) -k language use keyboard layout (for example "fr" for French) -audio-help print list of audio drivers and their options -soundhw c1,... enable audio support and only specified sound cards (comma separated list) use -soundhw ? to get the list of supported cards use -soundhw all to enable all of them -localtime set the real time clock to local time [default=utc] -full-screen start in full screen -win2k-hack use it when installing Windows 2000 to avoid a disk full bug -usb enable the USB driver (will be the default soon) -usbdevice name add the host or guest USB device 'name' -name string set the name of the guest Network options: -net nic[,vlan=n][,macaddr=addr][,model=type] create a new Network Interface Card and connect it to VLAN 'n' -net user[,vlan=n][,hostname=host] connect the user mode network stack to VLAN 'n' and send hostname 'host' to DHCP clients -net tap[,vlan=n][,fd=h][,ifname=name][,script=file][,downscript=dfile] connect the host TAP network interface to VLAN 'n' and use the network scripts 'file' (default=/etc/kvm/kvm-ifup) and 'dfile' (default=/etc/kvm/kvm-ifdown); use '[down]script=no' to disable script execution; use 'fd=h' to connect to an already opened TAP interface -net socket[,vlan=n][,fd=h][,listen=[host]:port][,connect=host:port] connect the vlan 'n' to another VLAN using a socket connection -net socket[,vlan=n][,fd=h][,mcast=maddr:port] connect the vlan 'n' to multicast maddr and port -net none use it alone to have zero network devices; if no -net option is provided, the default is '-net nic -net user' -tftp dir allow tftp access to files in dir [-net user] -bootp file advertise file in BOOTP replies -smb dir allow SMB access to files in 'dir' [-net user] -redir [tcp|udp]:host-port:[guest-host]:guest-port redirect TCP or UDP connections from host to guest [-net user] Linux boot specific: -kernel bzImage use 'bzImage' as kernel image -append cmdline use 'cmdline' as kernel command line -initrd file use 'file' as initial ram disk Debug/Expert options: -monitor dev redirect the monitor to char device 'dev' -serial dev redirect the serial port to char device 'dev' -parallel dev redirect the parallel port to char device 'dev' -pidfile file Write PID to 'file' -S freeze CPU at startup (use 'c' to start execution) -s wait gdb connection to port -p port set gdb connection port [default=1234] -d item1,... output log to /tmp/qemu.log (use -d ? for a list of log items) -hdachs c,h,s[,t] force hard disk 0 physical geometry and the optional BIOS translation (t=none or lba) (usually qemu can guess them) -L path set the directory for the BIOS, VGA BIOS and keymaps -no-kvm disable KVM hardware virtualization -no-kvm-irqchip disable KVM kernel mode PIC/IOAPIC/LAPIC -no-kvm-pit disable KVM kernel mode PIT -std-vga simulate a standard VGA card with VESA Bochs Extensions (default is CL-GD5446 PCI VGA) -no-acpi disable ACPI -curses use a curses/ncurses interface instead of SDL -no-reboot exit instead of rebooting -no-shutdown stop before shutdown -loadvm [tag|id] start right away with a saved state (loadvm in monitor) -vnc display start a VNC server on display -daemonize daemonize QEMU after initializing -tdf inject timer interrupts that got lost -kvm-shadow-memory megs set the amount of shadow pages to be allocated -mem-path set the path to hugetlbfs/tmpfs mounted directory, also enables allocation of guest memory with huge pages -option-rom rom load a file, rom, into the option ROM space -clock force the use of the given methods for timer alarm. To see what timers are available use -clock ? -startdate select initial date of the clock -icount [N|auto] Enable virtual instruction counter with 2^N clock ticks per instruction During emulation, the following keys are useful: ctrl-alt-f toggle full screen ctrl-alt-n switch to virtual console 'n' ctrl-alt toggle mouse and keyboard grab When using -nographic, press 'ctrl-a h' to get some help. ganeti-3.1.0~rc2/test/data/kvm_0.9.1_help_boot_test.txt000064400000000000000000000136141476477700300226560ustar00rootroot00000000000000QEMU PC emulator version 0.9.1 (kvm-72), Copyright (c) 2003-2008 Fabrice Bellard usage: qemu [options] [disk_image] 'disk_image' is a raw hard image image for IDE hard disk 0 Standard options: -M machine select emulated machine (-M ? for list) -cpu cpu select CPU (-cpu ? for list) -fda/-fdb file use 'file' as floppy disk 0/1 image -hda/-hdb file use 'file' as IDE hard disk 0/1 image -hdc/-hdd file use 'file' as IDE hard disk 2/3 image -cdrom file use 'file' as IDE cdrom image (cdrom is ide1 master) -drive [file=file][,if=type][,bus=n][,unit=m][,media=d][,index=i] [,cyls=c,heads=h,secs=s[,trans=t]][,snapshot=on|off] -fakeopt [,cache=on|off][,format=f][,boot=on|off] use 'file' as a drive image -mtdblock file use 'file' as on-board Flash memory image -sd file use 'file' as SecureDigital card image -pflash file use 'file' as a parallel flash image -boot [a|c|d|n] boot on floppy (a), hard disk (c), CD-ROM (d), or network (n) -snapshot write to temporary files instead of disk image files -no-frame open SDL window without a frame and window decorations -alt-grab use Ctrl-Alt-Shift to grab mouse (instead of Ctrl-Alt) -no-quit disable SDL window close capability -no-fd-bootchk disable boot signature checking for floppy disks -m megs set virtual RAM size to megs MB [default=128] -smp n set the number of CPUs to 'n' [default=1] -nographic disable graphical output and redirect serial I/Os to console -portrait rotate graphical output 90 deg left (only PXA LCD) -k language use keyboard layout (for example "fr" for French) -audio-help print list of audio drivers and their options -soundhw c1,... enable audio support and only specified sound cards (comma separated list) use -soundhw ? to get the list of supported cards use -soundhw all to enable all of them -localtime set the real time clock to local time [default=utc] -full-screen start in full screen -win2k-hack use it when installing Windows 2000 to avoid a disk full bug -usb enable the USB driver (will be the default soon) -usbdevice name add the host or guest USB device 'name' -name string set the name of the guest Network options: -net nic[,vlan=n][,macaddr=addr][,model=type] create a new Network Interface Card and connect it to VLAN 'n' -net user[,vlan=n][,hostname=host] connect the user mode network stack to VLAN 'n' and send hostname 'host' to DHCP clients -net tap[,vlan=n][,fd=h][,ifname=name][,script=file][,downscript=dfile] connect the host TAP network interface to VLAN 'n' and use the network scripts 'file' (default=/etc/kvm/kvm-ifup) and 'dfile' (default=/etc/kvm/kvm-ifdown); use '[down]script=no' to disable script execution; use 'fd=h' to connect to an already opened TAP interface -net socket[,vlan=n][,fd=h][,listen=[host]:port][,connect=host:port] connect the vlan 'n' to another VLAN using a socket connection -net socket[,vlan=n][,fd=h][,mcast=maddr:port] connect the vlan 'n' to multicast maddr and port -net none use it alone to have zero network devices; if no -net option is provided, the default is '-net nic -net user' -tftp dir allow tftp access to files in dir [-net user] -bootp file advertise file in BOOTP replies -smb dir allow SMB access to files in 'dir' [-net user] -redir [tcp|udp]:host-port:[guest-host]:guest-port redirect TCP or UDP connections from host to guest [-net user] Linux boot specific: -kernel bzImage use 'bzImage' as kernel image -append cmdline use 'cmdline' as kernel command line -initrd file use 'file' as initial ram disk Debug/Expert options: -monitor dev redirect the monitor to char device 'dev' -serial dev redirect the serial port to char device 'dev' -parallel dev redirect the parallel port to char device 'dev' -pidfile file Write PID to 'file' -S freeze CPU at startup (use 'c' to start execution) -s wait gdb connection to port -p port set gdb connection port [default=1234] -d item1,... output log to /tmp/qemu.log (use -d ? for a list of log items) -hdachs c,h,s[,t] force hard disk 0 physical geometry and the optional BIOS translation (t=none or lba) (usually qemu can guess them) -L path set the directory for the BIOS, VGA BIOS and keymaps -no-kvm disable KVM hardware virtualization -no-kvm-irqchip disable KVM kernel mode PIC/IOAPIC/LAPIC -no-kvm-pit disable KVM kernel mode PIT -std-vga simulate a standard VGA card with VESA Bochs Extensions (default is CL-GD5446 PCI VGA) -no-acpi disable ACPI -curses use a curses/ncurses interface instead of SDL -no-reboot exit instead of rebooting -no-shutdown stop before shutdown -loadvm [tag|id] start right away with a saved state (loadvm in monitor) -vnc display start a VNC server on display -daemonize daemonize QEMU after initializing -tdf inject timer interrupts that got lost -kvm-shadow-memory megs set the amount of shadow pages to be allocated -mem-path set the path to hugetlbfs/tmpfs mounted directory, also enables allocation of guest memory with huge pages -option-rom rom load a file, rom, into the option ROM space -clock force the use of the given methods for timer alarm. To see what timers are available use -clock ? -startdate select initial date of the clock -icount [N|auto] Enable virtual instruction counter with 2^N clock ticks per instruction During emulation, the following keys are useful: ctrl-alt-f toggle full screen ctrl-alt-n switch to virtual console 'n' ctrl-alt toggle mouse and keyboard grab When using -nographic, press 'ctrl-a h' to get some help. ganeti-3.1.0~rc2/test/data/kvm_1.0_help.txt000064400000000000000000000327251476477700300204310ustar00rootroot00000000000000QEMU emulator version 1.0 (qemu-kvm-1.0 Debian 1.0+dfsg-2), Copyright (c) 2003-2008 Fabrice Bellard usage: qemu [options] [disk_image] 'disk_image' is a raw hard disk image for IDE hard disk 0 Standard options: -h or -help display this help and exit -version display version information and exit -machine [type=]name[,prop[=value][,...]] selects emulated machine (-machine ? for list) property accel=accel1[:accel2[:...]] selects accelerator supported accelerators are kvm, xen, tcg (default: tcg) -cpu cpu select CPU (-cpu ? for list) -smp n[,maxcpus=cpus][,cores=cores][,threads=threads][,sockets=sockets] set the number of CPUs to 'n' [default=1] maxcpus= maximum number of total cpus, including offline CPUs for hotplug, etc cores= number of CPU cores on one socket threads= number of threads on one CPU core sockets= number of discrete sockets in the system -numa node[,mem=size][,cpus=cpu[-cpu]][,nodeid=node] -fda/-fdb file use 'file' as floppy disk 0/1 image -hda/-hdb file use 'file' as IDE hard disk 0/1 image -hdc/-hdd file use 'file' as IDE hard disk 2/3 image -cdrom file use 'file' as IDE cdrom image (cdrom is ide1 master) -drive [file=file][,if=type][,bus=n][,unit=m][,media=d][,index=i] [,cyls=c,heads=h,secs=s[,trans=t]][,snapshot=on|off] [,cache=writethrough|writeback|none|directsync|unsafe][,format=f] [,serial=s][,addr=A][,id=name][,aio=threads|native] [,readonly=on|off] use 'file' as a drive image -set group.id.arg=value set parameter for item of type i.e. -set drive.$id.file=/path/to/image -global driver.property=value set a global default for a driver property -mtdblock file use 'file' as on-board Flash memory image -sd file use 'file' as SecureDigital card image -pflash file use 'file' as a parallel flash image -boot [order=drives][,once=drives][,menu=on|off] [,splash=sp_name][,splash-time=sp_time] 'drives': floppy (a), hard disk (c), CD-ROM (d), network (n) 'sp_name': the file's name that would be passed to bios as logo picture, if menu=on 'sp_time': the period that splash picture last if menu=on, unit is ms -snapshot write to temporary files instead of disk image files -m megs set virtual RAM size to megs MB [default=128] -mem-path FILE provide backing storage for guest RAM -mem-prealloc preallocate guest memory (use with -mem-path) -k language use keyboard layout (for example 'fr' for French) -audio-help print list of audio drivers and their options -soundhw c1,... enable audio support and only specified sound cards (comma separated list) use -soundhw ? to get the list of supported cards use -soundhw all to enable all of them -usb enable the USB driver (will be the default soon) -usbdevice name add the host or guest USB device 'name' -device driver[,prop[=value][,...]] add device (based on driver) prop=value,... sets driver properties use -device ? to print all possible drivers use -device driver,? to print all possible properties File system options: -fsdev fsdriver,id=id,path=path,[security_model={mapped|passthrough|none}] [,writeout=immediate][,readonly] Virtual File system pass-through options: -virtfs local,path=path,mount_tag=tag,security_model=[mapped|passthrough|none] [,writeout=immediate][,readonly] -virtfs_synth Create synthetic file system image -name string1[,process=string2] set the name of the guest string1 sets the window title and string2 the process name (on Linux) -uuid %08x-%04x-%04x-%04x-%012x specify machine UUID Display options: -display sdl[,frame=on|off][,alt_grab=on|off][,ctrl_grab=on|off] [,window_close=on|off]|curses|none| vnc=[,] select display type -nographic disable graphical output and redirect serial I/Os to console -curses use a curses/ncurses interface instead of SDL -no-frame open SDL window without a frame and window decorations -alt-grab use Ctrl-Alt-Shift to grab mouse (instead of Ctrl-Alt) -ctrl-grab use Right-Ctrl to grab mouse (instead of Ctrl-Alt) -no-quit disable SDL window close capability -sdl enable SDL -spice enable spice -portrait rotate graphical output 90 deg left (only PXA LCD) -rotate rotate graphical output some deg left (only PXA LCD) -vga [std|cirrus|vmware|qxl|xenfb|none] select video card type -full-screen start in full screen -g WxH[xDEPTH] Set the initial graphical resolution and depth -vnc display start a VNC server on display i386 target only: -win2k-hack use it when installing Windows 2000 to avoid a disk full bug -no-fd-bootchk disable boot signature checking for floppy disks -no-acpi disable ACPI -no-hpet disable HPET -balloon none disable balloon device -balloon virtio[,addr=str] enable virtio balloon device (default) -acpitable [sig=str][,rev=n][,oem_id=str][,oem_table_id=str][,oem_rev=n][,asl_compiler_id=str][,asl_compiler_rev=n][,{data|file}=file1[:file2]...] ACPI table description -smbios file=binary load SMBIOS entry from binary file -smbios type=0[,vendor=str][,version=str][,date=str][,release=%d.%d] specify SMBIOS type 0 fields -smbios type=1[,manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str] specify SMBIOS type 1 fields Network options: -net nic[,vlan=n][,macaddr=mac][,model=type][,name=str][,addr=str][,vectors=v] create a new Network Interface Card and connect it to VLAN 'n' -net user[,vlan=n][,name=str][,net=addr[/mask]][,host=addr][,restrict=on|off] [,hostname=host][,dhcpstart=addr][,dns=addr][,tftp=dir][,bootfile=f] [,hostfwd=rule][,guestfwd=rule][,smb=dir[,smbserver=addr]] connect the user mode network stack to VLAN 'n', configure its DHCP server and enabled optional services -net tap[,vlan=n][,name=str][,fd=h][,ifname=name][,script=file][,downscript=dfile][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off][,vhostfd=h][,vhostforce=on|off] connect the host TAP network interface to VLAN 'n' and use the network scripts 'file' (default=/etc/kvm/kvm-ifup) and 'dfile' (default=/etc/kvm/kvm-ifdown) use '[down]script=no' to disable script execution use 'fd=h' to connect to an already opened TAP interface use 'sndbuf=nbytes' to limit the size of the send buffer (the default is disabled 'sndbuf=0' to enable flow control set 'sndbuf=1048576') use vnet_hdr=off to avoid enabling the IFF_VNET_HDR tap flag use vnet_hdr=on to make the lack of IFF_VNET_HDR support an error condition use vhost=on to enable experimental in kernel accelerator (only has effect for virtio guests which use MSIX) use vhostforce=on to force vhost on for non-MSIX virtio guests use 'vhostfd=h' to connect to an already opened vhost net device -net socket[,vlan=n][,name=str][,fd=h][,listen=[host]:port][,connect=host:port] connect the vlan 'n' to another VLAN using a socket connection -net socket[,vlan=n][,name=str][,fd=h][,mcast=maddr:port[,localaddr=addr]] connect the vlan 'n' to multicast maddr and port use 'localaddr=addr' to specify the host address to send packets from -net vde[,vlan=n][,name=str][,sock=socketpath][,port=n][,group=groupname][,mode=octalmode] connect the vlan 'n' to port 'n' of a vde switch running on host and listening for incoming connections on 'socketpath'. Use group 'groupname' and mode 'octalmode' to change default ownership and permissions for communication port. -net dump[,vlan=n][,file=f][,len=n] dump traffic on vlan 'n' to file 'f' (max n bytes per packet) -net none use it alone to have zero network devices. If no -net option is provided, the default is '-net nic -net user' -netdev [user|tap|vde|socket],id=str[,option][,option][,...] Character device options: -chardev null,id=id[,mux=on|off] -chardev socket,id=id[,host=host],port=host[,to=to][,ipv4][,ipv6][,nodelay] [,server][,nowait][,telnet][,mux=on|off] (tcp) -chardev socket,id=id,path=path[,server][,nowait][,telnet],[mux=on|off] (unix) -chardev udp,id=id[,host=host],port=port[,localaddr=localaddr] [,localport=localport][,ipv4][,ipv6][,mux=on|off] -chardev msmouse,id=id[,mux=on|off] -chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]] [,mux=on|off] -chardev file,id=id,path=path[,mux=on|off] -chardev pipe,id=id,path=path[,mux=on|off] -chardev pty,id=id[,mux=on|off] -chardev stdio,id=id[,mux=on|off][,signal=on|off] -chardev braille,id=id[,mux=on|off] -chardev tty,id=id,path=path[,mux=on|off] -chardev parport,id=id,path=path[,mux=on|off] -chardev spicevmc,id=id,name=name[,debug=debug] Bluetooth(R) options: -bt hci,null dumb bluetooth HCI - doesn't respond to commands -bt hci,host[:id] use host's HCI with the given name -bt hci[,vlan=n] emulate a standard HCI in virtual scatternet 'n' -bt vhci[,vlan=n] add host computer to virtual scatternet 'n' using VHCI -bt device:dev[,vlan=n] emulate a bluetooth device 'dev' in scatternet 'n' Linux/Multiboot boot specific: -kernel bzImage use 'bzImage' as kernel image -append cmdline use 'cmdline' as kernel command line -initrd file use 'file' as initial ram disk Debug/Expert options: -serial dev redirect the serial port to char device 'dev' -parallel dev redirect the parallel port to char device 'dev' -monitor dev redirect the monitor to char device 'dev' -qmp dev like -monitor but opens in 'control' mode -mon chardev=[name][,mode=readline|control][,default] -debugcon dev redirect the debug console to char device 'dev' -pidfile file write PID to 'file' -singlestep always run in singlestep mode -S freeze CPU at startup (use 'c' to start execution) -gdb dev wait for gdb connection on 'dev' -s shorthand for -gdb tcp::1234 -d item1,... output log to /tmp/qemu.log (use -d ? for a list of log items) -D logfile output log to logfile (instead of the default /tmp/qemu.log) -hdachs c,h,s[,t] force hard disk 0 physical geometry and the optional BIOS translation (t=none or lba) (usually qemu can guess them) -L path set the directory for the BIOS, VGA BIOS and keymaps -bios file set the filename for the BIOS -enable-kvm enable KVM full virtualization support -xen-domid id specify xen guest domain id -xen-create create domain using xen hypercalls, bypassing xend warning: should not be used when xend is in use -xen-attach attach to existing xen domain xend will use this when starting qemu -no-reboot exit instead of rebooting -no-shutdown stop before shutdown -loadvm [tag|id] start right away with a saved state (loadvm in monitor) -daemonize daemonize QEMU after initializing -option-rom rom load a file, rom, into the option ROM space -clock force the use of the given methods for timer alarm. To see what timers are available use -clock ? -rtc [base=utc|localtime|date][,clock=host|vm][,driftfix=none|slew] set the RTC base and clock, enable drift fix for clock ticks (x86 only) -icount [N|auto] enable virtual instruction counter with 2^N clock ticks per instruction -watchdog i6300esb|ib700 enable virtual hardware watchdog [default=none] -watchdog-action reset|shutdown|poweroff|pause|debug|none action when watchdog fires [default=reset] -echr chr set terminal escape character instead of ctrl-a -virtioconsole c set virtio console -show-cursor show cursor -tb-size n set TB size -incoming p prepare for incoming migration, listen on port p -nodefaults don't create default devices -chroot dir chroot to dir just before starting the VM -runas user change to user id user just before starting the VM -prom-env variable=value set OpenBIOS nvram variables -semihosting semihosting mode -old-param old param mode -readconfig -writeconfig read/write config file -nodefconfig do not load default config files at startup -trace [events=][,file=] specify tracing options -no-kvm disable KVM hardware virtualization -no-kvm-irqchip disable KVM kernel mode PIC/IOAPIC/LAPIC -no-kvm-pit disable KVM kernel mode PIT -no-kvm-pit-reinjection disable KVM kernel mode PIT interrupt reinjection -tdf enable guest time drift compensation -kvm-shadow-memory MEGABYTES allocate MEGABYTES for kvm mmu shadowing During emulation, the following keys are useful: ctrl-alt-f toggle full screen ctrl-alt-n switch to virtual console 'n' ctrl-alt toggle mouse and keyboard grab When using -nographic, press 'ctrl-a h' to get some help. ganeti-3.1.0~rc2/test/data/kvm_1.1.2_help.txt000064400000000000000000000346241476477700300205720ustar00rootroot00000000000000QEMU emulator version 1.1.2 (qemu-kvm-1.1.2+dfsg-2~bpo60+1, Debian), Copyright (c) 2003-2008 Fabrice Bellard usage: kvm [options] [disk_image] 'disk_image' is a raw hard disk image for IDE hard disk 0 Standard options: -h or -help display this help and exit -version display version information and exit -machine [type=]name[,prop[=value][,...]] selects emulated machine (-machine ? for list) property accel=accel1[:accel2[:...]] selects accelerator supported accelerators are kvm, xen, tcg (default: tcg) kernel_irqchip=on|off controls accelerated irqchip support kvm_shadow_mem=size of KVM shadow MMU -cpu cpu select CPU (-cpu ? for list) -smp n[,maxcpus=cpus][,cores=cores][,threads=threads][,sockets=sockets] set the number of CPUs to 'n' [default=1] maxcpus= maximum number of total cpus, including offline CPUs for hotplug, etc cores= number of CPU cores on one socket threads= number of threads on one CPU core sockets= number of discrete sockets in the system -numa node[,mem=size][,cpus=cpu[-cpu]][,nodeid=node] -fda/-fdb file use 'file' as floppy disk 0/1 image -hda/-hdb file use 'file' as IDE hard disk 0/1 image -hdc/-hdd file use 'file' as IDE hard disk 2/3 image -cdrom file use 'file' as IDE cdrom image (cdrom is ide1 master) -drive [file=file][,if=type][,bus=n][,unit=m][,media=d][,index=i] [,cyls=c,heads=h,secs=s[,trans=t]][,snapshot=on|off] [,cache=writethrough|writeback|none|directsync|unsafe][,format=f] [,serial=s][,addr=A][,id=name][,aio=threads|native] [,readonly=on|off][,copy-on-read=on|off] [[,bps=b]|[[,bps_rd=r][,bps_wr=w]]][[,iops=i]|[[,iops_rd=r][,iops_wr=w]] use 'file' as a drive image -set group.id.arg=value set parameter for item of type i.e. -set drive.$id.file=/path/to/image -global driver.prop=value set a global default for a driver property -mtdblock file use 'file' as on-board Flash memory image -sd file use 'file' as SecureDigital card image -pflash file use 'file' as a parallel flash image -boot [order=drives][,once=drives][,menu=on|off] [,splash=sp_name][,splash-time=sp_time] 'drives': floppy (a), hard disk (c), CD-ROM (d), network (n) 'sp_name': the file's name that would be passed to bios as logo picture, if menu=on 'sp_time': the period that splash picture last if menu=on, unit is ms -snapshot write to temporary files instead of disk image files -m megs set virtual RAM size to megs MB [default=128] -mem-path FILE provide backing storage for guest RAM -mem-prealloc preallocate guest memory (use with -mem-path) -k language use keyboard layout (for example 'fr' for French) -audio-help print list of audio drivers and their options -soundhw c1,... enable audio support and only specified sound cards (comma separated list) use -soundhw ? to get the list of supported cards use -soundhw all to enable all of them -balloon none disable balloon device -balloon virtio[,addr=str] enable virtio balloon device (default) -usb enable the USB driver (will be the default soon) -usbdevice name add the host or guest USB device 'name' -device driver[,prop[=value][,...]] add device (based on driver) prop=value,... sets driver properties use -device ? to print all possible drivers use -device driver,? to print all possible properties File system options: -fsdev fsdriver,id=id[,path=path,][security_model={mapped-xattr|mapped-file|passthrough|none}] [,writeout=immediate][,readonly][,socket=socket|sock_fd=sock_fd] Virtual File system pass-through options: -virtfs local,path=path,mount_tag=tag,security_model=[mapped-xattr|mapped-file|passthrough|none] [,writeout=immediate][,readonly][,socket=socket|sock_fd=sock_fd] -virtfs_synth Create synthetic file system image -name string1[,process=string2] set the name of the guest string1 sets the window title and string2 the process name (on Linux) -uuid %08x-%04x-%04x-%04x-%012x specify machine UUID Display options: -display sdl[,frame=on|off][,alt_grab=on|off][,ctrl_grab=on|off] [,window_close=on|off]|curses|none| vnc=[,] select display type -nographic disable graphical output and redirect serial I/Os to console -curses use a curses/ncurses interface instead of SDL -no-frame open SDL window without a frame and window decorations -alt-grab use Ctrl-Alt-Shift to grab mouse (instead of Ctrl-Alt) -ctrl-grab use Right-Ctrl to grab mouse (instead of Ctrl-Alt) -no-quit disable SDL window close capability -sdl enable SDL -spice enable spice -portrait rotate graphical output 90 deg left (only PXA LCD) -rotate rotate graphical output some deg left (only PXA LCD) -vga [std|cirrus|vmware|qxl|xenfb|none] select video card type -full-screen start in full screen -vnc display start a VNC server on display i386 target only: -win2k-hack use it when installing Windows 2000 to avoid a disk full bug -no-fd-bootchk disable boot signature checking for floppy disks -no-acpi disable ACPI -no-hpet disable HPET -acpitable [sig=str][,rev=n][,oem_id=str][,oem_table_id=str][,oem_rev=n][,asl_compiler_id=str][,asl_compiler_rev=n][,{data|file}=file1[:file2]...] ACPI table description -smbios file=binary load SMBIOS entry from binary file -smbios type=0[,vendor=str][,version=str][,date=str][,release=%d.%d] specify SMBIOS type 0 fields -smbios type=1[,manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str] specify SMBIOS type 1 fields Network options: -net nic[,vlan=n][,macaddr=mac][,model=type][,name=str][,addr=str][,vectors=v] create a new Network Interface Card and connect it to VLAN 'n' -net user[,vlan=n][,name=str][,net=addr[/mask]][,host=addr][,restrict=on|off] [,hostname=host][,dhcpstart=addr][,dns=addr][,tftp=dir][,bootfile=f] [,hostfwd=rule][,guestfwd=rule][,smb=dir[,smbserver=addr]] connect the user mode network stack to VLAN 'n', configure its DHCP server and enabled optional services -net tap[,vlan=n][,name=str][,fd=h][,ifname=name][,script=file][,downscript=dfile][,helper=helper][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off][,vhostfd=h][,vhostforce=on|off] connect the host TAP network interface to VLAN 'n' use network scripts 'file' (default=/etc/kvm/kvm-ifup) to configure it and 'dfile' (default=/etc/kvm/kvm-ifdown) to deconfigure it use '[down]script=no' to disable script execution use network helper 'helper' (default=/usr/lib/qemu-bridge-helper) to configure it use 'fd=h' to connect to an already opened TAP interface use 'sndbuf=nbytes' to limit the size of the send buffer (the default is disabled 'sndbuf=0' to enable flow control set 'sndbuf=1048576') use vnet_hdr=off to avoid enabling the IFF_VNET_HDR tap flag use vnet_hdr=on to make the lack of IFF_VNET_HDR support an error condition use vhost=on to enable experimental in kernel accelerator (only has effect for virtio guests which use MSIX) use vhostforce=on to force vhost on for non-MSIX virtio guests use 'vhostfd=h' to connect to an already opened vhost net device -net bridge[,vlan=n][,name=str][,br=bridge][,helper=helper] connects a host TAP network interface to a host bridge device 'br' (default=br0) using the program 'helper' (default=/usr/lib/qemu-bridge-helper) -net socket[,vlan=n][,name=str][,fd=h][,listen=[host]:port][,connect=host:port] connect the vlan 'n' to another VLAN using a socket connection -net socket[,vlan=n][,name=str][,fd=h][,mcast=maddr:port[,localaddr=addr]] connect the vlan 'n' to multicast maddr and port use 'localaddr=addr' to specify the host address to send packets from -net socket[,vlan=n][,name=str][,fd=h][,udp=host:port][,localaddr=host:port] connect the vlan 'n' to another VLAN using an UDP tunnel -net vde[,vlan=n][,name=str][,sock=socketpath][,port=n][,group=groupname][,mode=octalmode] connect the vlan 'n' to port 'n' of a vde switch running on host and listening for incoming connections on 'socketpath'. Use group 'groupname' and mode 'octalmode' to change default ownership and permissions for communication port. -net dump[,vlan=n][,file=f][,len=n] dump traffic on vlan 'n' to file 'f' (max n bytes per packet) -net none use it alone to have zero network devices. If no -net option is provided, the default is '-net nic -net user' -netdev [user|tap|bridge|vde|socket],id=str[,option][,option][,...] Character device options: -chardev null,id=id[,mux=on|off] -chardev socket,id=id[,host=host],port=host[,to=to][,ipv4][,ipv6][,nodelay] [,server][,nowait][,telnet][,mux=on|off] (tcp) -chardev socket,id=id,path=path[,server][,nowait][,telnet],[mux=on|off] (unix) -chardev udp,id=id[,host=host],port=port[,localaddr=localaddr] [,localport=localport][,ipv4][,ipv6][,mux=on|off] -chardev msmouse,id=id[,mux=on|off] -chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]] [,mux=on|off] -chardev file,id=id,path=path[,mux=on|off] -chardev pipe,id=id,path=path[,mux=on|off] -chardev pty,id=id[,mux=on|off] -chardev stdio,id=id[,mux=on|off][,signal=on|off] -chardev braille,id=id[,mux=on|off] -chardev tty,id=id,path=path[,mux=on|off] -chardev parport,id=id,path=path[,mux=on|off] -chardev spicevmc,id=id,name=name[,debug=debug] -iscsi [user=user][,password=password] [,header-digest=CRC32C|CR32C-NONE|NONE-CRC32C|NONE [,initiator-name=iqn] iSCSI session parameters Bluetooth(R) options: -bt hci,null dumb bluetooth HCI - doesn't respond to commands -bt hci,host[:id] use host's HCI with the given name -bt hci[,vlan=n] emulate a standard HCI in virtual scatternet 'n' -bt vhci[,vlan=n] add host computer to virtual scatternet 'n' using VHCI -bt device:dev[,vlan=n] emulate a bluetooth device 'dev' in scatternet 'n' Linux/Multiboot boot specific: -kernel bzImage use 'bzImage' as kernel image -append cmdline use 'cmdline' as kernel command line -initrd file use 'file' as initial ram disk -dtb file use 'file' as device tree image Debug/Expert options: -serial dev redirect the serial port to char device 'dev' -parallel dev redirect the parallel port to char device 'dev' -monitor dev redirect the monitor to char device 'dev' -qmp dev like -monitor but opens in 'control' mode -mon chardev=[name][,mode=readline|control][,default] -debugcon dev redirect the debug console to char device 'dev' -pidfile file write PID to 'file' -singlestep always run in singlestep mode -S freeze CPU at startup (use 'c' to start execution) -gdb dev wait for gdb connection on 'dev' -s shorthand for -gdb tcp::1234 -d item1,... output log to /tmp/qemu.log (use -d ? for a list of log items) -D logfile output log to logfile (instead of the default /tmp/qemu.log) -hdachs c,h,s[,t] force hard disk 0 physical geometry and the optional BIOS translation (t=none or lba) (usually QEMU can guess them) -L path set the directory for the BIOS, VGA BIOS and keymaps -bios file set the filename for the BIOS -enable-kvm enable KVM full virtualization support -xen-domid id specify xen guest domain id -xen-create create domain using xen hypercalls, bypassing xend warning: should not be used when xend is in use -xen-attach attach to existing xen domain xend will use this when starting QEMU -no-reboot exit instead of rebooting -no-shutdown stop before shutdown -loadvm [tag|id] start right away with a saved state (loadvm in monitor) -daemonize daemonize QEMU after initializing -option-rom rom load a file, rom, into the option ROM space -clock force the use of the given methods for timer alarm. To see what timers are available use -clock ? -rtc [base=utc|localtime|date][,clock=host|rt|vm][,driftfix=none|slew] set the RTC base and clock, enable drift fix for clock ticks (x86 only) -icount [N|auto] enable virtual instruction counter with 2^N clock ticks per instruction -watchdog i6300esb|ib700 enable virtual hardware watchdog [default=none] -watchdog-action reset|shutdown|poweroff|pause|debug|none action when watchdog fires [default=reset] -echr chr set terminal escape character instead of ctrl-a -virtioconsole c set virtio console -show-cursor show cursor -tb-size n set TB size -incoming p prepare for incoming migration, listen on port p -nodefaults don't create default devices -chroot dir chroot to dir just before starting the VM -runas user change to user id user just before starting the VM -readconfig -writeconfig read/write config file -nodefconfig do not load default config files at startup -no-user-config do not load user-provided config files at startup -trace [events=][,file=] specify tracing options -qtest CHR specify tracing options -qtest-log LOG specify tracing options -no-kvm disable KVM hardware virtualization -no-kvm-irqchip disable KVM kernel mode PIC/IOAPIC/LAPIC -no-kvm-pit disable KVM kernel mode PIT -no-kvm-pit-reinjection disable KVM kernel mode PIT interrupt reinjection During emulation, the following keys are useful: ctrl-alt-f toggle full screen ctrl-alt-n switch to virtual console 'n' ctrl-alt toggle mouse and keyboard grab When using -nographic, press 'ctrl-a h' to get some help. ganeti-3.1.0~rc2/test/data/kvm_5.2.0_help.txt000064400000000000000000000715431476477700300205760ustar00rootroot00000000000000QEMU emulator version 5.2.0 (Debian 1:5.2+dfsg-11+deb11u1) Copyright (c) 2003-2020 Fabrice Bellard and the QEMU Project developers usage: qemu-system-x86_64 [options] [disk_image] 'disk_image' is a raw hard disk image for IDE hard disk 0 Standard options: -h or -help display this help and exit -version display version information and exit -machine [type=]name[,prop[=value][,...]] selects emulated machine ('-machine help' for list) property accel=accel1[:accel2[:...]] selects accelerator supported accelerators are kvm, xen, hax, hvf, whpx or tcg (default: tcg) vmport=on|off|auto controls emulation of vmport (default: auto) dump-guest-core=on|off include guest memory in a core dump (default=on) mem-merge=on|off controls memory merge support (default: on) aes-key-wrap=on|off controls support for AES key wrapping (default=on) dea-key-wrap=on|off controls support for DEA key wrapping (default=on) suppress-vmdesc=on|off disables self-describing migration (default=off) nvdimm=on|off controls NVDIMM support (default=off) memory-encryption=@var{} memory encryption object to use (default=none) hmat=on|off controls ACPI HMAT support (default=off) -cpu cpu select CPU ('-cpu help' for list) -accel [accel=]accelerator[,prop[=value][,...]] select accelerator (kvm, xen, hax, hvf, whpx or tcg; use 'help' for a list) igd-passthru=on|off (enable Xen integrated Intel graphics passthrough, default=off) kernel-irqchip=on|off|split controls accelerated irqchip support (default=on) kvm-shadow-mem=size of KVM shadow MMU in bytes tb-size=n (TCG translation block cache size) thread=single|multi (enable multi-threaded TCG) -smp [cpus=]n[,maxcpus=cpus][,cores=cores][,threads=threads][,dies=dies][,sockets=sockets] set the number of CPUs to 'n' [default=1] maxcpus= maximum number of total cpus, including offline CPUs for hotplug, etc cores= number of CPU cores on one socket (for PC, it's on one die) threads= number of threads on one CPU core dies= number of CPU dies on one socket (for PC only) sockets= number of discrete sockets in the system -numa node[,mem=size][,cpus=firstcpu[-lastcpu]][,nodeid=node][,initiator=node] -numa node[,memdev=id][,cpus=firstcpu[-lastcpu]][,nodeid=node][,initiator=node] -numa dist,src=source,dst=destination,val=distance -numa cpu,node-id=node[,socket-id=x][,core-id=y][,thread-id=z] -numa hmat-lb,initiator=node,target=node,hierarchy=memory|first-level|second-level|third-level,data-type=access-latency|read-latency|write-latency[,latency=lat][,bandwidth=bw] -numa hmat-cache,node-id=node,size=size,level=level[,associativity=none|direct|complex][,policy=none|write-back|write-through][,line=size] -add-fd fd=fd,set=set[,opaque=opaque] Add 'fd' to fd 'set' -set group.id.arg=value set parameter for item of type i.e. -set drive.$id.file=/path/to/image -global driver.property=value -global driver=driver,property=property,value=value set a global default for a driver property -boot [order=drives][,once=drives][,menu=on|off] [,splash=sp_name][,splash-time=sp_time][,reboot-timeout=rb_time][,strict=on|off] 'drives': floppy (a), hard disk (c), CD-ROM (d), network (n) 'sp_name': the file's name that would be passed to bios as logo picture, if menu=on 'sp_time': the period that splash picture last if menu=on, unit is ms 'rb_timeout': the timeout before guest reboot when boot failed, unit is ms -m [size=]megs[,slots=n,maxmem=size] configure guest RAM size: initial amount of guest memory slots: number of hotplug slots (default: none) maxmem: maximum amount of guest memory (default: none) NOTE: Some architectures might enforce a specific granularity -mem-path FILE provide backing storage for guest RAM -mem-prealloc preallocate guest memory (use with -mem-path) -k language use keyboard layout (for example 'fr' for French) -audio-help show -audiodev equivalent of the currently specified audio settings -audiodev [driver=]driver,id=id[,prop[=value][,...]] specifies the audio backend to use id= identifier of the backend timer-period= timer period in microseconds in|out.mixing-engine= use mixing engine to mix streams inside QEMU in|out.fixed-settings= use fixed settings for host audio in|out.frequency= frequency to use with fixed settings in|out.channels= number of channels to use with fixed settings in|out.format= sample format to use with fixed settings valid values: s8, s16, s32, u8, u16, u32, f32 in|out.voices= number of voices to use in|out.buffer-length= length of buffer in microseconds -audiodev none,id=id,[,prop[=value][,...]] dummy driver that discards all output -audiodev alsa,id=id[,prop[=value][,...]] in|out.dev= name of the audio device to use in|out.period-length= length of period in microseconds in|out.try-poll= attempt to use poll mode threshold= threshold (in microseconds) when playback starts -audiodev oss,id=id[,prop[=value][,...]] in|out.dev= path of the audio device to use in|out.buffer-count= number of buffers in|out.try-poll= attempt to use poll mode try-mmap= try using memory mapped access exclusive= open device in exclusive mode dsp-policy= set timing policy (0..10), -1 to use fragment mode -audiodev pa,id=id[,prop[=value][,...]] server= PulseAudio server address in|out.name= source/sink device name in|out.latency= desired latency in microseconds -audiodev spice,id=id[,prop[=value][,...]] -audiodev wav,id=id[,prop[=value][,...]] path= path of wav file to record -soundhw c1,... enable audio support and only specified sound cards (comma separated list) use '-soundhw help' to get the list of supported cards use '-soundhw all' to enable all of them -device driver[,prop[=value][,...]] add device (based on driver) prop=value,... sets driver properties use '-device help' to print all possible drivers use '-device driver,help' to print all possible properties -name string1[,process=string2][,debug-threads=on|off] set the name of the guest string1 sets the window title and string2 the process name When debug-threads is enabled, individual threads are given a separate name NOTE: The thread names are for debugging and not a stable API. -uuid %08x-%04x-%04x-%04x-%012x specify machine UUID Block device options: -fda/-fdb file use 'file' as floppy disk 0/1 image -hda/-hdb file use 'file' as IDE hard disk 0/1 image -hdc/-hdd file use 'file' as IDE hard disk 2/3 image -cdrom file use 'file' as IDE cdrom image (cdrom is ide1 master) -blockdev [driver=]driver[,node-name=N][,discard=ignore|unmap] [,cache.direct=on|off][,cache.no-flush=on|off] [,read-only=on|off][,auto-read-only=on|off] [,force-share=on|off][,detect-zeroes=on|off|unmap] [,driver specific parameters...] configure a block backend -drive [file=file][,if=type][,bus=n][,unit=m][,media=d][,index=i] [,cache=writethrough|writeback|none|directsync|unsafe][,format=f] [,snapshot=on|off][,rerror=ignore|stop|report] [,werror=ignore|stop|report|enospc][,id=name] [,aio=threads|native|io_uring] [,readonly=on|off][,copy-on-read=on|off] [,discard=ignore|unmap][,detect-zeroes=on|off|unmap] [[,bps=b]|[[,bps_rd=r][,bps_wr=w]]] [[,iops=i]|[[,iops_rd=r][,iops_wr=w]]] [[,bps_max=bm]|[[,bps_rd_max=rm][,bps_wr_max=wm]]] [[,iops_max=im]|[[,iops_rd_max=irm][,iops_wr_max=iwm]]] [[,iops_size=is]] [[,group=g]] use 'file' as a drive image -mtdblock file use 'file' as on-board Flash memory image -sd file use 'file' as SecureDigital card image -pflash file use 'file' as a parallel flash image -snapshot write to temporary files instead of disk image files -fsdev local,id=id,path=path,security_model=mapped-xattr|mapped-file|passthrough|none [,writeout=immediate][,readonly][,fmode=fmode][,dmode=dmode] [[,throttling.bps-total=b]|[[,throttling.bps-read=r][,throttling.bps-write=w]]] [[,throttling.iops-total=i]|[[,throttling.iops-read=r][,throttling.iops-write=w]]] [[,throttling.bps-total-max=bm]|[[,throttling.bps-read-max=rm][,throttling.bps-write-max=wm]]] [[,throttling.iops-total-max=im]|[[,throttling.iops-read-max=irm][,throttling.iops-write-max=iwm]]] [[,throttling.iops-size=is]] -fsdev proxy,id=id,socket=socket[,writeout=immediate][,readonly] -fsdev proxy,id=id,sock_fd=sock_fd[,writeout=immediate][,readonly] -fsdev synth,id=id -virtfs local,path=path,mount_tag=tag,security_model=mapped-xattr|mapped-file|passthrough|none [,id=id][,writeout=immediate][,readonly][,fmode=fmode][,dmode=dmode][,multidevs=remap|forbid|warn] -virtfs proxy,mount_tag=tag,socket=socket[,id=id][,writeout=immediate][,readonly] -virtfs proxy,mount_tag=tag,sock_fd=sock_fd[,id=id][,writeout=immediate][,readonly] -virtfs synth,mount_tag=tag[,id=id][,readonly] -iscsi [user=user][,password=password] [,header-digest=CRC32C|CR32C-NONE|NONE-CRC32C|NONE [,initiator-name=initiator-iqn][,id=target-iqn] [,timeout=timeout] iSCSI session parameters USB options: -usb enable on-board USB host controller (if not enabled by default) -usbdevice name add the host or guest USB device 'name' Display options: -display spice-app[,gl=on|off] -display gtk[,grab_on_hover=on|off][,gl=on|off]| -display vnc=[,] -display curses[,charset=] -display egl-headless[,rendernode=] -display none select display backend type The default display is equivalent to "-display gtk" -nographic disable graphical output and redirect serial I/Os to console -curses shorthand for -display curses -alt-grab use Ctrl-Alt-Shift to grab mouse (instead of Ctrl-Alt) -ctrl-grab use Right-Ctrl to grab mouse (instead of Ctrl-Alt) -no-quit disable SDL window close capability -sdl shorthand for -display sdl -spice [port=port][,tls-port=secured-port][,x509-dir=] [,x509-key-file=][,x509-key-password=] [,x509-cert-file=][,x509-cacert-file=] [,x509-dh-key-file=][,addr=addr][,ipv4|ipv6|unix] [,tls-ciphers=] [,tls-channel=[main|display|cursor|inputs|record|playback]] [,plaintext-channel=[main|display|cursor|inputs|record|playback]] [,sasl][,password=][,disable-ticketing] [,image-compression=[auto_glz|auto_lz|quic|glz|lz|off]] [,jpeg-wan-compression=[auto|never|always]] [,zlib-glz-wan-compression=[auto|never|always]] [,streaming-video=[off|all|filter]][,disable-copy-paste] [,disable-agent-file-xfer][,agent-mouse=[on|off]] [,playback-compression=[on|off]][,seamless-migration=[on|off]] [,gl=[on|off]][,rendernode=] enable spice at least one of {port, tls-port} is mandatory -portrait rotate graphical output 90 deg left (only PXA LCD) -rotate rotate graphical output some deg left (only PXA LCD) -vga [std|cirrus|vmware|qxl|xenfb|tcx|cg3|virtio|none] select video card type -full-screen start in full screen -vnc shorthand for -display vnc= i386 target only: -win2k-hack use it when installing Windows 2000 to avoid a disk full bug -no-fd-bootchk disable boot signature checking for floppy disks -no-acpi disable ACPI -no-hpet disable HPET -acpitable [sig=str][,rev=n][,oem_id=str][,oem_table_id=str][,oem_rev=n][,asl_compiler_id=str][,asl_compiler_rev=n][,{data|file}=file1[:file2]...] ACPI table description -smbios file=binary load SMBIOS entry from binary file -smbios type=0[,vendor=str][,version=str][,date=str][,release=%d.%d] [,uefi=on|off] specify SMBIOS type 0 fields -smbios type=1[,manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str] specify SMBIOS type 1 fields -smbios type=2[,manufacturer=str][,product=str][,version=str][,serial=str] [,asset=str][,location=str] specify SMBIOS type 2 fields -smbios type=3[,manufacturer=str][,version=str][,serial=str][,asset=str] [,sku=str] specify SMBIOS type 3 fields -smbios type=4[,sock_pfx=str][,manufacturer=str][,version=str][,serial=str] [,asset=str][,part=str][,max-speed=%d][,current-speed=%d] specify SMBIOS type 4 fields -smbios type=11[,value=str][,path=filename] specify SMBIOS type 11 fields -smbios type=17[,loc_pfx=str][,bank=str][,manufacturer=str][,serial=str] [,asset=str][,part=str][,speed=%d] specify SMBIOS type 17 fields Network options: -netdev user,id=str[,ipv4[=on|off]][,net=addr[/mask]][,host=addr] [,ipv6[=on|off]][,ipv6-net=addr[/int]][,ipv6-host=addr] [,restrict=on|off][,hostname=host][,dhcpstart=addr] [,dns=addr][,ipv6-dns=addr][,dnssearch=domain][,domainname=domain] [,tftp=dir][,tftp-server-name=name][,bootfile=f][,hostfwd=rule][,guestfwd=rule][,smb=dir[,smbserver=addr]] configure a user mode network backend with ID 'str', its DHCP server and optional services -netdev tap,id=str[,fd=h][,fds=x:y:...:z][,ifname=name][,script=file][,downscript=dfile] [,br=bridge][,helper=helper][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off] [,vhostfd=h][,vhostfds=x:y:...:z][,vhostforce=on|off][,queues=n] [,poll-us=n] configure a host TAP network backend with ID 'str' connected to a bridge (default=br0) use network scripts 'file' (default=/etc/qemu-ifup) to configure it and 'dfile' (default=/etc/qemu-ifdown) to deconfigure it use '[down]script=no' to disable script execution use network helper 'helper' (default=/usr/lib/qemu/qemu-bridge-helper) to configure it use 'fd=h' to connect to an already opened TAP interface use 'fds=x:y:...:z' to connect to already opened multiqueue capable TAP interfaces use 'sndbuf=nbytes' to limit the size of the send buffer (the default is disabled 'sndbuf=0' to enable flow control set 'sndbuf=1048576') use vnet_hdr=off to avoid enabling the IFF_VNET_HDR tap flag use vnet_hdr=on to make the lack of IFF_VNET_HDR support an error condition use vhost=on to enable experimental in kernel accelerator (only has effect for virtio guests which use MSIX) use vhostforce=on to force vhost on for non-MSIX virtio guests use 'vhostfd=h' to connect to an already opened vhost net device use 'vhostfds=x:y:...:z to connect to multiple already opened vhost net devices use 'queues=n' to specify the number of queues to be created for multiqueue TAP use 'poll-us=n' to specify the maximum number of microseconds that could be spent on busy polling for vhost net -netdev bridge,id=str[,br=bridge][,helper=helper] configure a host TAP network backend with ID 'str' that is connected to a bridge (default=br0) using the program 'helper (default=/usr/lib/qemu/qemu-bridge-helper) -netdev l2tpv3,id=str,src=srcaddr,dst=dstaddr[,srcport=srcport][,dstport=dstport] [,rxsession=rxsession],txsession=txsession[,ipv6=on/off][,udp=on/off] [,cookie64=on/off][,counter][,pincounter][,txcookie=txcookie] [,rxcookie=rxcookie][,offset=offset] configure a network backend with ID 'str' connected to an Ethernet over L2TPv3 pseudowire. Linux kernel 3.3+ as well as most routers can talk L2TPv3. This transport allows connecting a VM to a VM, VM to a router and even VM to Host. It is a nearly-universal standard (RFC3931). Note - this implementation uses static pre-configured tunnels (same as the Linux kernel). use 'src=' to specify source address use 'dst=' to specify destination address use 'udp=on' to specify udp encapsulation use 'srcport=' to specify source udp port use 'dstport=' to specify destination udp port use 'ipv6=on' to force v6 L2TPv3 uses cookies to prevent misconfiguration as well as a weak security measure use 'rxcookie=0x012345678' to specify a rxcookie use 'txcookie=0x012345678' to specify a txcookie use 'cookie64=on' to set cookie size to 64 bit, otherwise 32 use 'counter=off' to force a 'cut-down' L2TPv3 with no counter use 'pincounter=on' to work around broken counter handling in peer use 'offset=X' to add an extra offset between header and data -netdev socket,id=str[,fd=h][,listen=[host]:port][,connect=host:port] configure a network backend to connect to another network using a socket connection -netdev socket,id=str[,fd=h][,mcast=maddr:port[,localaddr=addr]] configure a network backend to connect to a multicast maddr and port use 'localaddr=addr' to specify the host address to send packets from -netdev socket,id=str[,fd=h][,udp=host:port][,localaddr=host:port] configure a network backend to connect to another network using an UDP tunnel -netdev vde,id=str[,sock=socketpath][,port=n][,group=groupname][,mode=octalmode] configure a network backend to connect to port 'n' of a vde switch running on host and listening for incoming connections on 'socketpath'. Use group 'groupname' and mode 'octalmode' to change default ownership and permissions for communication port. -netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off] configure a vhost-user network, backed by a chardev 'dev' -netdev vhost-vdpa,id=str,vhostdev=/path/to/dev configure a vhost-vdpa network,Establish a vhost-vdpa netdev -netdev hubport,id=str,hubid=n[,netdev=nd] configure a hub port on the hub with ID 'n' -nic [tap|bridge|user|l2tpv3|vde|vhost-user|socket][,option][,...][mac=macaddr] initialize an on-board / default host NIC (using MAC address macaddr) and connect it to the given host network backend -nic none use it alone to have zero network devices (the default is to provided a 'user' network connection) -net nic[,macaddr=mac][,model=type][,name=str][,addr=str][,vectors=v] configure or create an on-board (or machine default) NIC and connect it to hub 0 (please use -nic unless you need a hub) -net [user|tap|bridge|vde|socket][,option][,option][,...] old way to initialize a host network interface (use the -netdev option if possible instead) Character device options: -chardev help -chardev null,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev socket,id=id[,host=host],port=port[,to=to][,ipv4][,ipv6][,nodelay][,reconnect=seconds] [,server][,nowait][,telnet][,websocket][,reconnect=seconds][,mux=on|off] [,logfile=PATH][,logappend=on|off][,tls-creds=ID][,tls-authz=ID] (tcp) -chardev socket,id=id,path=path[,server][,nowait][,telnet][,websocket][,reconnect=seconds] [,mux=on|off][,logfile=PATH][,logappend=on|off][,abstract=on|off][,tight=on|off] (unix) -chardev udp,id=id[,host=host],port=port[,localaddr=localaddr] [,localport=localport][,ipv4][,ipv6][,mux=on|off] [,logfile=PATH][,logappend=on|off] -chardev msmouse,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]] [,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev ringbuf,id=id[,size=size][,logfile=PATH][,logappend=on|off] -chardev file,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev pipe,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev pty,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev stdio,id=id[,mux=on|off][,signal=on|off][,logfile=PATH][,logappend=on|off] -chardev braille,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev serial,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev tty,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev parallel,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev parport,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off] -chardev spicevmc,id=id,name=name[,debug=debug][,logfile=PATH][,logappend=on|off] -chardev spiceport,id=id,name=name[,debug=debug][,logfile=PATH][,logappend=on|off] TPM device options: -tpmdev passthrough,id=id[,path=path][,cancel-path=path] use path to provide path to a character device; default is /dev/tpm0 use cancel-path to provide path to TPM's cancel sysfs entry; if not provided it will be searched for in /sys/class/misc/tpm?/device -tpmdev emulator,id=id,chardev=dev configure the TPM device using chardev backend Linux/Multiboot boot specific: -kernel bzImage use 'bzImage' as kernel image -append cmdline use 'cmdline' as kernel command line -initrd file use 'file' as initial ram disk -dtb file use 'file' as device tree image Debug/Expert options: -fw_cfg [name=],file= add named fw_cfg entry with contents from file -fw_cfg [name=],string= add named fw_cfg entry with contents from string -serial dev redirect the serial port to char device 'dev' -parallel dev redirect the parallel port to char device 'dev' -monitor dev redirect the monitor to char device 'dev' -qmp dev like -monitor but opens in 'control' mode -qmp-pretty dev like -qmp but uses pretty JSON formatting -mon [chardev=]name[,mode=readline|control][,pretty[=on|off]] -debugcon dev redirect the debug console to char device 'dev' -pidfile file write PID to 'file' -singlestep always run in singlestep mode --preconfig pause QEMU before machine is initialized (experimental) -S freeze CPU at startup (use 'c' to start execution) -realtime [mlock=on|off] run qemu with realtime features mlock=on|off controls mlock support (default: on) -overcommit [mem-lock=on|off][cpu-pm=on|off] run qemu with overcommit hints mem-lock=on|off controls memory lock support (default: off) cpu-pm=on|off controls cpu power management (default: off) -gdb dev accept gdb connection on 'dev'. (QEMU defaults to starting the guest without waiting for gdb to connect; use -S too if you want it to not start execution.) -s shorthand for -gdb tcp::1234 -d item1,... enable logging of specified items (use '-d help' for a list of log items) -D logfile output log to logfile (default stderr) -dfilter range,.. filter debug output to range of addresses (useful for -d cpu,exec,etc..) -seed number seed the pseudo-random number generator -L path set the directory for the BIOS, VGA BIOS and keymaps -bios file set the filename for the BIOS -enable-kvm enable KVM full virtualization support -xen-domid id specify xen guest domain id -xen-attach attach to existing xen domain libxl will use this when starting QEMU -xen-domid-restrict restrict set of available xen operations to specified domain id. (Does not affect xenpv machine type). -no-reboot exit instead of rebooting -no-shutdown stop before shutdown -loadvm [tag|id] start right away with a saved state (loadvm in monitor) -daemonize daemonize QEMU after initializing -option-rom rom load a file, rom, into the option ROM space -rtc [base=utc|localtime|][,clock=host|rt|vm][,driftfix=none|slew] set the RTC base and clock, enable drift fix for clock ticks (x86 only) -icount [shift=N|auto][,align=on|off][,sleep=on|off,rr=record|replay,rrfile=,rrsnapshot=] enable virtual instruction counter with 2^N clock ticks per instruction, enable aligning the host and virtual clocks or disable real time cpu sleeping -watchdog model enable virtual hardware watchdog [default=none] -watchdog-action reset|shutdown|poweroff|inject-nmi|pause|debug|none action when watchdog fires [default=reset] -echr chr set terminal escape character instead of ctrl-a -show-cursor show cursor -tb-size n set TB size -incoming tcp:[host]:port[,to=maxport][,ipv4][,ipv6] -incoming rdma:host:port[,ipv4][,ipv6] -incoming unix:socketpath prepare for incoming migration, listen on specified protocol and socket address -incoming fd:fd -incoming exec:cmdline accept incoming migration on given file descriptor or from given external command -incoming defer wait for the URI to be specified via migrate_incoming -only-migratable allow only migratable devices -nodefaults don't create default devices -chroot dir chroot to dir just before starting the VM -runas user change to user id user just before starting the VM user can be numeric uid:gid instead -sandbox on[,obsolete=allow|deny][,elevateprivileges=allow|deny|children] [,spawn=allow|deny][,resourcecontrol=allow|deny] Enable seccomp mode 2 system call filter (default 'off'). use 'obsolete' to allow obsolete system calls that are provided by the kernel, but typically no longer used by modern C library implementations. use 'elevateprivileges' to allow or deny QEMU process to elevate its privileges by blacklisting all set*uid|gid system calls. The value 'children' will deny set*uid|gid system calls for main QEMU process but will allow forks and execves to run unprivileged use 'spawn' to avoid QEMU to spawn new threads or processes by blacklisting *fork and execve use 'resourcecontrol' to disable process affinity and schedular priority -readconfig -writeconfig read/write config file -no-user-config do not load default user-provided config files at startup -trace [[enable=]][,events=][,file=] specify tracing options -plugin [file=][,arg=] load a plugin -enable-fips enable FIPS 140-2 compliance -msg [timestamp[=on|off]][,guest-name=[on|off]] control error message format timestamp=on enables timestamps (default: off) guest-name=on enables guest name prefix but only if -name guest option is set (default: off) -dump-vmstate Output vmstate information in JSON format to file. Use the scripts/vmstate-static-checker.py file to check for possible regressions in migration code by comparing two such vmstate dumps. -enable-sync-profile enable synchronization profiling Generic object creation: -object TYPENAME[,PROP1=VALUE1,...] create a new object of type TYPENAME setting properties in the order they are specified. Note that the 'id' property must be set. These objects are placed in the '/objects' path. During emulation, the following keys are useful: ctrl-alt-f toggle full screen ctrl-alt-n switch to virtual console 'n' ctrl-alt toggle mouse and keyboard grab When using -nographic, press 'ctrl-a h' to get some help. See for how to report bugs. More information on the QEMU project at . ganeti-3.1.0~rc2/test/data/kvm_6.0.0_machine.txt000064400000000000000000000130721476477700300212420ustar00rootroot00000000000000Supported machines are: microvm microvm (i386) pc-i440fx-zesty Ubuntu 17.04 PC (i440FX + PIIX, 1996) pc-i440fx-yakkety Ubuntu 16.10 PC (i440FX + PIIX, 1996) pc-i440fx-xenial Ubuntu 16.04 PC (i440FX + PIIX, 1996) pc-i440fx-wily Ubuntu 15.04 PC (i440FX + PIIX, 1996) pc-i440fx-trusty Ubuntu 14.04 PC (i440FX + PIIX, 1996) ubuntu Ubuntu 21.10 PC (i440FX + PIIX, 1996) (alias of pc-i440fx-impish) pc-i440fx-impish Ubuntu 21.10 PC (i440FX + PIIX, 1996) (default) pc-i440fx-impish-hpb Ubuntu 21.10 PC (i440FX + PIIX +host-phys-bits=true, 1996) pc-i440fx-hirsute Ubuntu 21.04 PC (i440FX + PIIX, 1996) pc-i440fx-hirsute-hpb Ubuntu 21.04 PC (i440FX + PIIX +host-phys-bits=true, 1996) pc-i440fx-groovy Ubuntu 20.10 PC (i440FX + PIIX, 1996) pc-i440fx-groovy-hpb Ubuntu 20.10 PC (i440FX + PIIX +host-phys-bits=true, 1996) pc-i440fx-focal Ubuntu 20.04 PC (i440FX + PIIX, 1996) pc-i440fx-focal-hpb Ubuntu 20.04 PC (i440FX + PIIX +host-phys-bits=true, 1996) pc-i440fx-eoan Ubuntu 19.10 PC (i440FX + PIIX, 1996) pc-i440fx-eoan-hpb Ubuntu 19.10 PC (i440FX + PIIX +host-phys-bits=true, 1996) pc-i440fx-disco Ubuntu 19.04 PC (i440FX + PIIX, 1996) pc-i440fx-disco-hpb Ubuntu 19.04 PC (i440FX + PIIX +host-phys-bits=true, 1996) pc-i440fx-cosmic Ubuntu 18.10 PC (i440FX + PIIX, 1996) pc-i440fx-cosmic-hpb Ubuntu 18.10 PC (i440FX + PIIX +host-phys-bits=true, 1996) pc-i440fx-bionic Ubuntu 18.04 PC (i440FX + PIIX, 1996) pc-i440fx-bionic-hpb Ubuntu 18.04 PC (i440FX + PIIX, +host-phys-bits=true, 1996) pc-i440fx-artful Ubuntu 17.10 PC (i440FX + PIIX, 1996) pc Standard PC (i440FX + PIIX, 1996) (alias of pc-i440fx-6.0) pc-i440fx-6.0 Standard PC (i440FX + PIIX, 1996) pc-i440fx-5.2 Standard PC (i440FX + PIIX, 1996) pc-i440fx-5.1 Standard PC (i440FX + PIIX, 1996) pc-i440fx-5.0 Standard PC (i440FX + PIIX, 1996) pc-i440fx-4.2 Standard PC (i440FX + PIIX, 1996) pc-i440fx-4.1 Standard PC (i440FX + PIIX, 1996) pc-i440fx-4.0 Standard PC (i440FX + PIIX, 1996) pc-i440fx-3.1 Standard PC (i440FX + PIIX, 1996) pc-i440fx-3.0 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.9 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.8 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.7 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.6 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.5 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.4 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.3 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.2 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.12 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.11 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.10 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.1 Standard PC (i440FX + PIIX, 1996) pc-i440fx-2.0 Standard PC (i440FX + PIIX, 1996) pc-i440fx-1.7 Standard PC (i440FX + PIIX, 1996) pc-i440fx-1.6 Standard PC (i440FX + PIIX, 1996) pc-i440fx-1.5 Standard PC (i440FX + PIIX, 1996) pc-i440fx-1.4 Standard PC (i440FX + PIIX, 1996) pc-q35-zesty Ubuntu 17.04 PC (Q35 + ICH9, 2009) pc-q35-yakkety Ubuntu 16.10 PC (Q35 + ICH9, 2009) pc-q35-xenial Ubuntu 16.04 PC (Q35 + ICH9, 2009) ubuntu-q35 Ubuntu 21.10 PC (Q35 + ICH9, 2009) (alias of pc-q35-impish) pc-q35-impish Ubuntu 21.10 PC (Q35 + ICH9, 2009) pc-q35-impish-hpb Ubuntu 21.10 PC (Q35 + ICH9, +host-phys-bits=true, 2009) pc-q35-hirsute Ubuntu 21.04 PC (Q35 + ICH9, 2009) pc-q35-hirsute-hpb Ubuntu 21.04 PC (Q35 + ICH9, +host-phys-bits=true, 2009) pc-q35-groovy Ubuntu 20.10 PC (Q35 + ICH9, 2009) pc-q35-groovy-hpb Ubuntu 20.10 PC (Q35 + ICH9, +host-phys-bits=true, 2009) pc-q35-focal Ubuntu 20.04 PC (Q35 + ICH9, 2009) pc-q35-focal-hpb Ubuntu 20.04 PC (Q35 + ICH9, +host-phys-bits=true, 2009) pc-q35-eoan Ubuntu 19.10 PC (Q35 + ICH9, 2009) pc-q35-eoan-hpb Ubuntu 19.10 PC (Q35 + ICH9, +host-phys-bits=true, 2009) pc-q35-disco Ubuntu 19.04 PC (Q35 + ICH9, 2009) pc-q35-disco-hpb Ubuntu 19.04 PC (Q35 + ICH9, +host-phys-bits=true, 2009) pc-q35-cosmic Ubuntu 18.10 PC (Q35 + ICH9, 2009) pc-q35-cosmic-hpb Ubuntu 18.10 PC (Q35 + ICH9, +host-phys-bits=true, 2009) pc-q35-bionic Ubuntu 18.04 PC (Q35 + ICH9, 2009) pc-q35-bionic-hpb Ubuntu 18.04 PC (Q35 + ICH9, +host-phys-bits=true, 2009) pc-q35-artful Ubuntu 17.10 PC (Q35 + ICH9, 2009) q35 Standard PC (Q35 + ICH9, 2009) (alias of pc-q35-6.0) pc-q35-6.0 Standard PC (Q35 + ICH9, 2009) pc-q35-5.2 Standard PC (Q35 + ICH9, 2009) pc-q35-5.1 Standard PC (Q35 + ICH9, 2009) pc-q35-5.0 Standard PC (Q35 + ICH9, 2009) pc-q35-4.2 Standard PC (Q35 + ICH9, 2009) pc-q35-4.1 Standard PC (Q35 + ICH9, 2009) pc-q35-4.0.1 Standard PC (Q35 + ICH9, 2009) pc-q35-4.0 Standard PC (Q35 + ICH9, 2009) pc-q35-3.1 Standard PC (Q35 + ICH9, 2009) pc-q35-3.0 Standard PC (Q35 + ICH9, 2009) pc-q35-2.9 Standard PC (Q35 + ICH9, 2009) pc-q35-2.8 Standard PC (Q35 + ICH9, 2009) pc-q35-2.7 Standard PC (Q35 + ICH9, 2009) pc-q35-2.6 Standard PC (Q35 + ICH9, 2009) pc-q35-2.5 Standard PC (Q35 + ICH9, 2009) pc-q35-2.4 Standard PC (Q35 + ICH9, 2009) pc-q35-2.12 Standard PC (Q35 + ICH9, 2009) pc-q35-2.11 Standard PC (Q35 + ICH9, 2009) pc-q35-2.10 Standard PC (Q35 + ICH9, 2009) isapc ISA-only PC none empty machine x-remote Experimental remote machineganeti-3.1.0~rc2/test/data/kvm_current_help.txt000077700000000000000000000000001476477700300246712kvm_5.2.0_help.txtustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/kvm_runtime.json000064400000000000000000000071201476477700300207270ustar00rootroot00000000000000[ [ "/usr/bin/kvm", "-name", "xen-test-inst2", "-m", 1024, "-smp", "1", "-pidfile", "/var/run/ganeti/kvm-hypervisor/pid/xen-test-inst2", "-balloon", "virtio", "-daemonize", "-machine", "pc-1.1", "-monitor", "unix:/var/run/ganeti/kvm-hypervisor/ctrl/xen-test-inst2.monitor,server,nowait", "-serial", "unix:/var/run/ganeti/kvm-hypervisor/ctrl/xen-test-inst2.serial,server,nowait", "-usb", "-usbdevice", "tablet", "-vnc", ":5100" ], [ { "mac": "aa:00:00:bf:2f:16", "nicparams": { "link": "br0", "mode": "bridged" }, "hvinfo": { "driver": "virtio-net-pci", "id": "nic-003fc157-66a8-4e6d", "bus": "pci.0", "addr": "0x8" }, "uuid": "003fc157-66a8-4e6d-8b7e-ec4f69751396" } ], { "acpi": true, "boot_order": "disk", "cdrom2_image_path": "", "cdrom_disk_type": "", "cdrom_image_path": "", "cpu_cores": 0, "cpu_mask": "all", "cpu_sockets": 0, "cpu_threads": 0, "cpu_type": "", "disk_cache": "default", "disk_type": "paravirtual", "floppy_image_path": "", "initrd_path": "", "kernel_args": "ro", "kernel_path": "", "keymap": "", "kvm_extra": "", "kvm_flag": "", "kvm_path": "/usr/bin/kvm", "machine_version": "", "mem_path": "", "migration_bandwidth": 32, "migration_downtime": 30, "migration_mode": "live", "migration_port": 8102, "nic_type": "paravirtual", "reboot_behavior": "reboot", "root_path": "/dev/vda1", "security_domain": "", "security_model": "none", "serial_console": true, "serial_speed": 38400, "soundhw": "", "spice_bind": "", "spice_image_compression": "", "spice_ip_version": 0, "spice_jpeg_wan_compression": "", "spice_password_file": "", "spice_playback_compression": true, "spice_streaming_video": "", "spice_tls_ciphers": "HIGH:-DES:-3DES:-EXPORT:-ADH", "spice_use_tls": false, "spice_use_vdagent": true, "spice_zlib_glz_wan_compression": "", "usb_devices": "", "usb_mouse": "", "use_chroot": false, "use_localtime": false, "vga": "", "vhost_net": false, "vnc_bind_address": "0.0.0.0", "vnc_password_file": "", "vnc_tls": false, "vnc_x509_path": "", "vnc_x509_verify": false, "vnet_hdr": true }, [ [ { "dev_type": "lvm", "iv_name": "disk/0", "logical_id": [ "autovg", "b9d4ee8e-c81b-42eb-9899-60481886c7ac.disk0" ], "mode": "rw", "name": "disk0", "params": { "stripes": 1 }, "hvinfo": { "driver": "virtio-blk-pci", "id": "disk-7c079136-2573-4112", "bus": "pci.0", "addr": "0x9" }, "size": 1024, "uuid": "7c079136-2573-4112-82d0-0d3d2aa90d8f" }, "/var/run/ganeti/instance-disks/xen-test-inst2:0", "rbd://123451214123/" ], [ { "dev_type": "lvm", "iv_name": "disk/1", "logical_id": [ "autovg", "602c0a3b-d09b-4ebe-9774-5f12ef654a1f.disk1" ], "mode": "rw", "name": "disk1", "params": { "stripes": 1 }, "hvinfo": { "driver": "virtio-blk-pci", "id": "disk-9f5c5bd4-6f60-480b", "bus": "pci.0", "addr": "0xa" }, "size": 512, "uuid": "9f5c5bd4-6f60-480b-acdc-9bb1a4b7df79" }, "/var/run/ganeti/instance-disks/xen-test-inst2:1", null ] ] ] ganeti-3.1.0~rc2/test/data/lvs_lv.txt000064400000000000000000000010001476477700300175310ustar00rootroot00000000000000 nhasjL-cnZi-uqLS-WRLj-tkXI-nvCB-n0o2lj;df9ff3f6-a833-48ff-8bd5-bff2eaeab759.disk0_data;-wi-ao;-1;-1;253;0;1073741824B;1;originstname+instance1.example.com;;uZgXit-eiRr-vRqe-xpEo-e9nU-mTuR-9nfVIU;xenvg;linear;0B;0;1073741824B;;/dev/sda5:0-15;/dev/sda5(0) 5fW5mE-SBSs-GSU0-KZDg-hnwb-sZOC-zZt736;df9ff3f6-a833-48ff-8bd5-bff2eaeab759.disk0_meta;-wi-ao;-1;-1;253;1;134217728B;1;originstname+instance1.example.com;;uZgXit-eiRr-vRqe-xpEo-e9nU-mTuR-9nfVIU;xenvg;linear;0B;0;134217728B;;/dev/sda5:16-17;/dev/sda5(16) ganeti-3.1.0~rc2/test/data/mond-data.txt000064400000000000000000000010701476477700300200770ustar00rootroot00000000000000[{"node":"node1.example.com","reports":[{"name":"cpu-avg-load","version":"B","format_version":1,"timestamp":1379507272000000000,"category":null,"kind":0,"data":{"cpu_number":4,"cpus":[4.108859597350646e-2,4.456554528165781e-2,6.203619909502262e-2,5.595448881893895e-2],"cpu_total":0.203643517607712}}]},{"node":"node2.example.com","reports":[{"name":"cpu-avg-load","version":"B","format_version":1,"timestamp":1379507280000000000,"category":null,"kind":0,"data":{"cpu_number":2,"cpus":[4.155409618511363e-3,3.4586452012150787e-3],"cpu_total":7.614031289927129e-3}}]}] ganeti-3.1.0~rc2/test/data/ovfdata/000075500000000000000000000000001476477700300171205ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/ovfdata/compr_disk.vmdk.gz000064400000000000000000000006261476477700300225600ustar00rootroot00000000000000‹¤ŽdNcompr_disk.vmdkíŅ?oĶ@`6KlĀË)ŲĢD„n(.!$ÔTíˆL|%VķOļ dãŖã$žōˆPŸG:ŊÖũî|īŲŸŗ/ˇgI’”ķ˜nbU—Ģåx˜ž˙”Gņíč~4ĨëŧŠËf7ußI§UĖ›xŗ]ĮqoąZŽæe3+§“ve{iÚWŋ›vO(ēÚˇĻ×wa8„É×wדĢĐ[Æ_ߊļ‰ķÍĸxØīš™ÅŽ­ŧÉÃe^Įöŗė2M‹âûųĻŦšŸųüãŨíĄË0ŊQoũˆĢElĒíųt;/—E›īÂÁQ8‹yą†GI§í'ØgoY^äë&VģîĻËĸŊÖŠŪËS7Ā?õúÔ đ¨ü^Xt(ganeti-3.1.0~rc2/test/data/ovfdata/config.ini000064400000000000000000000006151476477700300210700ustar00rootroot00000000000000[instance] disk0_dump = rawdisk.raw nic0_mode = routed name = ganeti-test-xen hypervisor = xen-pvm disk_count = 1 nic0_mac = aa:00:00:d8:2c:1e nic_count = 1 nic0_link = br0 nic0_ip = None nic0_network = test disk0_ivname = disk/0 disk0_size = 0 [hypervisor] root-path = /dev/sda kernel_args = ro [export] version = 0 os = lenny-image [os] [backend] auto_balance = False vcpus = 1 memory = 512 ganeti-3.1.0~rc2/test/data/ovfdata/corrupted_resources.ovf000064400000000000000000000100351476477700300237340ustar00rootroot00000000000000 Virtual disk information The list of logical networks The bridged network A virtual machine AyertiennaSUSE.x86_64-0.0.2 The kind of installed guest operating system Virtual hardware requirements Virtual Hardware Family 0 AyertiennaSUSE.x86_64-0.0.2 vmx-04 hertz * 10^6 Number of Virtual CPUs 1 virtual CPU(s) 1 3 1 byte * 2^20 Memory Size 512MB of memory 2 4 512 0 SCSI Controller scsiController0 4 lsilogic 6 0 IDE Controller ideController0 5 5 0 disk1 ovf:/disk/vmdisk1 8 4 17 2 true bridged E1000 ethernet adapter on "bridged" ethernet0 9 E1000 10 ganeti-3.1.0~rc2/test/data/ovfdata/empty.ini000064400000000000000000000000571476477700300207610ustar00rootroot00000000000000[instance] [hypervisor] [export] [os] [backend]ganeti-3.1.0~rc2/test/data/ovfdata/empty.ovf000064400000000000000000000013121476477700300207670ustar00rootroot00000000000000 A virtual machine ganeti-3.1.0~rc2/test/data/ovfdata/ganeti.mf000064400000000000000000000001711476477700300207120ustar00rootroot00000000000000SHA1(ganeti.ovf)= d298200d9044c54b0fde13efaa90e564badc5961 SHA1(new_disk.vmdk)= 711c48f14c934228b8e117d036c913cdb9d63305 ganeti-3.1.0~rc2/test/data/ovfdata/ganeti.ovf000064400000000000000000000100051476477700300210770ustar00rootroot00000000000000 List of the virtual disks used in the package 0 False plain lenny-image bridged aa:00:00:d8:2c:1e none xen-br0 xen-pvm /dev/sda ro Logical networks used in the package Logical network used by this appliance. A virtual machine ganeti-test-xen The kind of installed guest operating system Ubuntu Virtual hardware requirements for a virtual machine Virtual Hardware Family 0 Ubuntu-freshly-created virtualbox-2.2 1 virtual CPU 1 virtual CPU Number of virtual CPUs 1 3 1 2048 MB of memory 2048 MB of memory Memory Size 2 4 MegaBytes 2048 Ethernet adapter on 'NAT' Ethernet adapter on 'NAT' 5 10 PCNet32 true disk1 disk1 Disk Image 7 17 /disk/vmdisk1 3 0 ganeti-3.1.0~rc2/test/data/ovfdata/gzip_disk.ovf000064400000000000000000000100421476477700300216140ustar00rootroot00000000000000 List of the virtual disks used in the package 0 False plain lenny-image bridged aa:00:00:d8:2c:1e none xen-br0 xen-pvm /dev/sda ro Logical networks used in the package Logical network used by this appliance. A virtual machine ganeti-test-xen The kind of installed guest operating system Ubuntu Virtual hardware requirements for a virtual machine Virtual Hardware Family 0 Ubuntu-freshly-created virtualbox-2.2 1 virtual CPU 1 virtual CPU Number of virtual CPUs 1 3 1 2048 MB of memory 2048 MB of memory Memory Size 2 4 MegaBytes 2048 Ethernet adapter on 'NAT' Ethernet adapter on 'NAT' 5 10 PCNet32 true disk1 disk1 Disk Image 7 17 /disk/vmdisk1 3 0 ganeti-3.1.0~rc2/test/data/ovfdata/new_disk.vmdk000064400000000000000000002000001476477700300215760ustar00rootroot00000000000000KDMVd€€ # Disk DescriptorFile version=1 CID=4e54f404 parentCID=ffffffff createType="monolithicSparse" # Extent description RW 100 SPARSE "new_disk.vmdk" # The Disk Data Base #DDB ddb.virtualHWVersion = "4" ddb.geometry.cylinders = "0" ddb.geometry.heads = "16" ddb.geometry.sectors = "63" ddb.adapterType = "ide" ganeti-3.1.0~rc2/test/data/ovfdata/no_disk.ini000064400000000000000000000005001476477700300212420ustar00rootroot00000000000000[instance] disk0_dump = iamnothere.raw nic0_mode = nic name = ganeti-test-xen disk_count = 1 nic0_mac = aa:00:00:d8:2c:1e nic_count = 1 nic0_link = xen-br0 nic0_ip = None disk0_ivname = disk/0 disk0_size = 0 [hypervisor] root-path = /dev/sda kernel_args = ro [export] version = 0 [os] [backend] auto_balance = False ganeti-3.1.0~rc2/test/data/ovfdata/no_disk_in_ref.ovf000064400000000000000000000101651476477700300226070ustar00rootroot00000000000000 List of the virtual disks used in the package Logical networks used in the package Logical network used by this appliance. A virtual machine The kind of installed guest operating system Ubuntu Virtual hardware requirements for a virtual machine Virtual Hardware Family 0 Ubuntu-freshly-created virtualbox-2.2 1 virtual CPU 1 virtual CPU Number of virtual CPUs 1 3 1 2048 MB of memory 2048 MB of memory Memory Size 2 4 MegaBytes 2048 ideController0 ideController0 IDE Controller 3 5 PIIX4 1 Ethernet adapter on 'NAT' Ethernet adapter on 'NAT' 5 10 PCNet32 true NAT disk1 disk1 Disk Image 7 17 /disk/vmdisk1 3 0 disk1 disk1 Disk Image 9 17 /disk/vmdisk1 3 0 ganeti-3.1.0~rc2/test/data/ovfdata/no_os.ini000064400000000000000000000005561476477700300207440ustar00rootroot00000000000000[instance] disk0_dump = rawdisk.raw nic0_mode = bridged name = ganeti-test-xen hypervisor = xen-pvm disk_count = 1 nic0_mac = aa:00:00:d8:2c:1e nic_count = 1 nic0_link = xen-br0 nic0_ip = None disk0_ivname = disk/0 disk0_size = 0 [hypervisor] root-path = /dev/sda kernel_args = ro [export] version = 0 [os] [backend] auto_balance = False vcpus = 1 memory = 2048 ganeti-3.1.0~rc2/test/data/ovfdata/no_ovf.ova000064400000000000000000000026011476477700300211140ustar00rootroot00000000000000‹ŽMNí˜moÛ6€ķšŋ‚Ķ€ĄfK”mŲõm^ZKÛÕI1ėC Yĸc"钔÷×īHQŽå8nd´č€ĸyo|îx˛ēœ]⋰ Žžx AÛ;ōÚ~Điuáˇ|Ī7ëđoß ŧ#Œŋ…´`ˇƒvįyß)Ÿ’dR…ĄŖđ:Taš‰p)ov`œEœMDJ?RÜĻ Z!)gĮnz"pŌ˜˛ëcįęōŧŅsū>üÔhŧ"ŒˆP‘MVčÃÅ2ņÅTqž ŋé51z:Éh7üĀÃŊŪŗßЕ$ĸāīˤhJúČ÷0nxŊî^âNû}Œ›]Üéļũ sÆ$ás‚é˛oŧâcgĶ­ƒ a&™RķžëĘhFŌP6ãTM›\\ģ’KŦ[õ~DĶ‡M–’ēK :.v#žĻœvā­j(ĘøąąhÚČ5\ß=]|zO$ĪDD^$ B5Ĩ §ĐE`ŗöŋ\.›đ ĩhBâ6œNl­,Īæ* “ņJ*’îHäVŌR"˖qęÃĨv˙žøsl|5(ƒ+Æ"⠟ 4xOĻD@‡Šaáœ&Ļú3A€õ‹ŠÆÂņÕøŦyÛ >í†ũå7b*o07žqŒ…“MÁį’~!ĮN†ˆßÅ­ĀqM@ˇqp NÆ$Ōlm#6åC{V¤c +"5üŽŲÎ5ĩ­ …ķ0ĸj7&čāļSZŧ+ŪŖ Zu˛RũŠüž—kę ēąá(úHųĸ>É{`ãHyU›2EÄ4„“šrN":ĨyXéjF͙J“ŸĨ$LßÎáūž8w;įķ,ŅWylų0™ŸÃdö{ŗ-Hƒ7D-šØÅírFPBĨB|Š~ $ˆåÚ˛ÄÎē0ņY˜B؉ ņ5¤”+hŧDF‚ÎMí×*ū ­ …ÜĢkŨæißOsPęáu×,Rgķ/Đ–? Ŗe¤œ9d;ÜĶ—WkäĘoįzHÂE)nIe¸×rĖlãōr52“$hŋʈTwÖ`o(‹5Xs…’P\kMċ Hš(ĨtŨŨ)Ø]Ëãu(bŨGĨÍíģ0ŗJHĪ$%L•Š yŒâôĀéŸ%FŲp)ŧ!Ņy˜Ōd5pīin9ŲÁ1:zV{ciKšTįQ .á:ąŋn{-÷ĐÅ.Ōۆ×ŪåÅl„Ü2ĸÁ¨ĖKŋ.ú[#c8ƒ¤ŋĀČĀŪĮ`āîTŲō°y5Ūdé„Ũ8û“wWŌúšw‡î|lÖ¯¯Ø>•ĪŦõîr™­Ú`ĢŊŗ\f§xŅR-Ģ^ZÜ2°'ų+ Ą>jĩąŊžĻ>ĒĘ|cJWG~AR.VHĪĶjœ;ØŋxŠK••0ûÕ0ˇĀ éUm†ú &’8÷čĮą R×ŦíķÆWã—č„3%8ŒBQ s&'•ĐļĒĄõ÷ĩđczą:ņÉxt0IzgëUBĶ~4šq61IÍOƒ-FÅö>°Áį::=;+ÉĄT;ÕŽSK•+ų–Ŋƒˇ4SÛĐÖëÛf™âúˇrt7.‡&@ažc˙søšš¯ĐŠ ‹ŠŖtj,ĢaĒaĮûfč˙û&Ŗ(<ŕāuŋ Īf^të WėėosķĀæ›ėë ^sŠŠ$‡ēÎŽ6tí7ĩ/)=L˛÷X’í $ģߖ¤x[*‘UčJĄĖ~–Ø/@kģąąg€œaĪķQđ>Q‡sĨ 3ôËįŒĢß­ĪüĄÚ„)œV›1Ī+ŋ0Íx[âmŖ‡J>pū\īß-ˇøošá“˙ú˙+kŠĨ–ZjŠĨ–ZjŠĨ–ZjŠĨ–ZjŠ&˙^(ĐA(ganeti-3.1.0~rc2/test/data/ovfdata/other/000075500000000000000000000000001476477700300202415ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/ovfdata/other/rawdisk.raw000064400000000000000000000120001476477700300224110ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/data/ovfdata/ova.ova000064400000000000000000000102621476477700300204150ustar00rootroot00000000000000‹œsSNíÚˆĨķâĮņqšŋ{­$ųI’#Iš93;;víŽÖîbr×b-W’ÎÎ93{ڙsÖ9gví•\I’$I’$I’$I’$I’$I’$I’$I’ä÷9fwf,wÆŊ~ˇßũŊ^yĪî>įyžsžī÷<Ī9sŽŠR­ŌĒŽŦīėûŨ ÅꥥžĄUÃĢGGÆōīâČđĐpgyûīÅŅÕ}ÅâęáU16ēĒo¨¸Ē82ÚWúũŌAŗÍVŠQ(ô•ĻJ­ŌĖlŖ´¯šģtˆõjõÚDŊö;ŽŌŋĮēs¯Ÿ™.ė­4šÕzm}qåP˙šVŦÛRÛ[™ŽīŠōÄXģāÖBÖ_;]ĒM­ī¯Ôvlī,¨5×÷ījĩöŦlNėĒĖ”š+Ë3­É•õÆÔ`ö0Xéín°Ø[}m–.w“FŠYūåmöíŦĖ îkNTg‹ƒų9Đ]cpxpĶøÖk/Ģ4ëŗ‰ĘÆééúDŠ•ŖŲ^iĩĒĩŠÍ™õš;ØÛüíwpEĩŅš-MoßßlUfąīë›ÕģŪˇoßĘ}#įų?ø×­ŲŪŲ×@ĩ–gcmĸ2ˇÕT­u`ĢŠÎ™ÚŋaEĄ°î˛ĘdĨQɊÍö?ŗāüętwŽv5*ØZeßĩåjs÷ĘŊ3åŨũĒ9´ÉŦUėėėbpá>ÖmÎęÛ+íĄéísŧ6Yßđ—jŗU¨OZģ*…ŊŨc,´÷Ü,Ė6+åBĩÖšeOibwiĒ˛n°ŗMwķö;÷Ü^}<÷žĮ’ŋģ§ũP.k?Ôîcę.Ģ7fJ­ôwf_ŠQY9QŸlîŠLT'ĢŨŲkļlåŽÖĖôiÍ=ĨFŗ2wX‹Žc]†p흡[xtíåWtŸØãå Cë.8¸ŌÆŲVũŧŌt{Z6œ_šnVēĢÎ_|påËKSÍ Ũ:=xËļ=•FŠũŦč>Eēˇônģ¸4SŲ0]ŠÕöTg:ãx`iwƒŋ¸‡îö•Öžzc÷Â}V':ƒZË^Ö÷7ęŗ­JšnŪ:[ëåʆjyĒRîŪGgÉĸ•6nÚX.7*Íæ†RiíĐPûŋōŲk‡'Ö{tŪ 7ŋdnųÅõZoåƒËŽû—jm÷†ësIŲŲčMFgÉÜ! öŽiū€,8ėÎN.ܟ1Ú[mÖ?ßöž÷ėY<ļŊU.)5˛¨•ų_ô¸õzk`OŠĩkÃ`š˛w°Y.u÷ppųÂõwWĩĘôĩĨFfŋQīŽ;Ų‚Z|¯ŨĨ ĸģėgOáuŊŖ?Ô9[ŸĘi2]¨u×øĮ§joWŋü|YˇšŌœhT÷tîiŅîģ{ßš?{¯6 Ĩ={ĻĢí“beNÅyõoŪ”øĮüƒZp=pŨē¸˛o šĢÚ>=æ.ķwã+ĶLi"Ģ-:¸öTw/ž­Jŗ5'BîûāÉĩčŧę=œwžfäā0tv{yFqwĩVn_;gz:05[i_)įöUhvv6˙‘,Æ;gk­ŲCŽŌĄQīÖŪ]Xj”Û×Æ7Î=ÄŪ:…]Ŋ• ĘuŗÕFeĻRk5 šÎJŋ6fŲÍÂĢT´_×n™îėĸ3xs÷1÷@ į—fĒĶû× ūlÍE;īÍáøæöewņĸE+/xBŒ—ŗËŧTŊŅ˜Ė•d×ôū‰FĨÔj_Ä~uŖ_Û÷å9é6ôÆdgũúá•Ã‡Ú[gĩįđĸkųøÂ!kŋaYģŠÔÚâßtɎuƒ n[´Éüą;Ôf‡ÛÎMķŸLĪÎėŦ4ÚĪŌyģhööņŗgŨÁ}Ė›ŒboíCÎOį–š7Vqé­ž`áĸ zŖyél)ŗŌÚā./?0ÆãKááĄUgļž×>ā™ĘLŊąéŖüK›.q¤ˇvļ)l¯ū­˛Ŧá^ŪđŽZÂđ|ƒģŖVm5ķĐĻJįíĪu¯ˇíâÛ˙Áė´æ_6AÕreSŊÖjÔsÁlŋČ/uvšŨ§f|ķ–ÂÁ—5;#˛Ņ%ĖÎÜmÛgwvnžd|ü¯‹guîļÅÛ{×4wž,zĩŒi˜Ė¯U{ö/cün°Ä?ŋŗQasŖēwy'ÅĒå {qIgEŪŦįW‹ęÄÁ§˙†Éî{ų_ŧũĐãŋ­–÷l9ü C įáĀōåĪĮ–ŧ-ËûÃVĄTΒöģV8ã⍗ŸąôúGģø•9›7ĖŖËųĄßōŒß”w}#ÃK|ÎbZZŲeĖZNũZī ŌÜh\˛üŠĘŖ­•—>-ķW_âiŗŊŊIaSŪS-ë¤YŊĖęßrąĒԚõZõēâČXņˇOß?y֍üËÎēÎgKŸĘųĢ/q*;€ŒwŸ_ÆTŽ-ķ,[ÂT^XoļænĪīąy`ƒŊOcz/XaŅƋ†~áˆ˙ū—Į‰rŖ>ŗŒ™Z°ū§jĶæËļmũ /Vg/s˛–rŪũķWŊß{ÆÖ ūō/Ąn›ûŨhŨā܇ÉVüģ?ífąŸ˙N÷ņkß˙Ŧ*æÎ÷?ÃcŖŖ#ŖcíīV}˙ķŋáĸÍ[¯8<Ū×ūŲwXûĮß{ˇÖûķŋįVūC_ßņŊŋžpD÷Īŋ˙šˇ`EáčsđÆi…Î۔šWģzŖũMŌŠšoüŠ+6o^ŋjÕXqõhixŞÎ+B{ŅdĪŠîo헱õũÍVū1ŗ-¯™3ÕŋUĘũ+VœVØr}+›Ę_MW\veĄ¸zlll¸¸ē°ũ’—mßRčī}œˇíŠķģ_Yĩ7mÎZî<¸œ™…ķJÍJaÅi›7ŸˇbEšŧseī3­ ¯ė}gSX_č_ÕßšŠ÷KP÷AU˕îŌŲŲjyeįë•õũ#;‹å‰ryõ@idįØĀĒōŲ#k&‹“åÕc“Ģ'Ī.Œ­šˇU÷¸×÷õ âĮœy›ÍÔËž°Zß?:<ŧŗ˛srįĀPš42°jõš‘RqÍØĀäĐČŲő‘ŗ+cŖşŨįÂ],ũū'ę33‡Üī5ā?Ō é¤tJ:-‘ÎJƒi$ĨsŌšiSē ]”ļĨíéĘtuē6M¤ŠÔū˙ˆëŠ™öĨŌMé–t[ē#Ũ•îI÷ĨŌCé‘ôXz"=•žIĪĨŌKé•ôZz#Ŋ•ŪIīĨŌGé“ôYú"}•žIßĨŌOéđÃúúū”ŽJĮ¤ãŌ é¤tJ:-‘ÎJƒi$ĨsŌšiSē ]”ļĨíéĘtuē6M¤Š´;ÕS3íK7¤›Ō-éļtGē+Ũ“îK¤‡Ō#éąôDz*=“žK/¤—Ō+éĩôFz+Ŋ“ŪK¤Ō'éŗôEú*}“žK?¤ŸŌáČņ§ŖŌ1é¸tB:)’NKg¤ŗŌ`Icéœtnڔ.HĨmi{ē2]ŽMi*íNõÔLûŌ éĻtKē-Ũ‘îJ÷¤ûŌéĄôHz,=‘žJĪ¤įŌ éĨôJz-Ŋ‘ŪJī¤÷ŌéŖôIú,}‘žJߤīŌé§tøá9ūtT:&—NH'ĨSŌiéŒtVL#i,“ÎM›Ōéĸ´-mOWĻĢĶĩi"MĨŨŠžši_ē!Ũ”nIˇĨ;Ō]éžt_z =”IĨ'ŌSé™ô\z!Ŋ”^I¯Ĩ7Ō[éô^ú }”>IŸĨ/ŌWé›ô]ú!ũ”?"ĮŸŽJĮ¤ãŽø]D€˙h'Ņ×wr:5žÎLŠ˜FĶš´>mL[Ōxښ.M;ŌUéšTJ•TM3éē4›ö§ĶÍéÖt{ē3ŨîM÷§ĶÃéŅôxz2=žMĪ§ĶËéÕôzz3ŊŪMī§ĶĮéĶôyú2}žMߧĶaėëûc:2ŽMĮ§ĶÉéÔtz:3 ¤bMkŌú´1mIãikē4íHWĨkR)UR5ͤëŌlڟnL7§[ĶíéÎtwē7ŨŸL§GĶãéÉôtz6=Ÿ^L/§WĶëéÍôvz7ŊŸ>L§OĶįéËôuú6}Ÿ~L‡ũWŽ?™ŽNĮĻãĶ‰éätj:=™R1Ļ5i}ژļ¤ņ´5]šv¤ĢŌ5Š”*ŠšfŌui6íO7Ļ›Ķ­éötgē;Ũ›îOĻ‡ĶŖéņôdz:=›žO/Ļ—ĶĢéõôfz;Ŋ›ŪOĻĶ§éķôeú:}›žO?ĻÃū”ãOGĻŖĶąéøtb:9šNOgĻTLŖiMZŸ6Ļ-i List of the virtual disks used in the package Logical networks used in the package Logical network used by this appliance. A virtual machine The kind of installed guest operating system Ubuntu Virtual hardware requirements for a virtual machine Virtual Hardware Family 0 Ubuntu-freshly-created virtualbox-2.2 1 virtual CPU 1 virtual CPU Number of virtual CPUs 1 3 1 2048 MB of memory 2048 MB of memory Memory Size 2 4 MegaBytes 2048 ideController0 ideController0 IDE Controller 3 5 PIIX4 1 Ethernet adapter on 'NAT' Ethernet adapter on 'NAT' 5 10 PCNet32 true bridged disk1 disk1 Disk Image 7 17 /disk/vmdisk1 3 0 disk1 disk1 Disk Image 9 17 /disk/vmdisk1 3 0 ganeti-3.1.0~rc2/test/data/ovfdata/wrong_config.ini000064400000000000000000000000201476477700300222720ustar00rootroot00000000000000It's just wrong ganeti-3.1.0~rc2/test/data/ovfdata/wrong_extension.ovd000064400000000000000000000120601476477700300230610ustar00rootroot00000000000000 Virtual disk information The list of logical networks The bridged network A virtual machine AyertiennaSUSE.x86_64-0.0.2 The kind of installed guest operating system Virtual hardware requirements Virtual Hardware Family 0 AyertiennaSUSE.x86_64-0.0.2 vmx-04 hertz * 10^6 Number of Virtual CPUs 1 virtual CPU(s) 1 3 1 byte * 2^20 Memory Size 512MB of memory 2 4 512 0 USB Controller usb 3 23 0 SCSI Controller scsiController0 4 lsilogic 6 0 IDE Controller ideController0 5 5 0 false Floppy Drive floppy0 6 14 0 false cdrom1 7 5 15 0 disk1 ovf:/disk/vmdisk1 8 4 17 2 true bridged E1000 ethernet adapter on "bridged" ethernet0 9 E1000 10 ganeti-3.1.0~rc2/test/data/ovfdata/wrong_manifest.mf000064400000000000000000000002011476477700300224570ustar00rootroot00000000000000SHA1(new_disk.vmdk)= 0500304662fb8a6a7925b5a43bc0e05d6a03720d SHA1(wrong_manifest.ovf)= 0500304662fb8a6a7965b5a43bc0e05d6a03720d ganeti-3.1.0~rc2/test/data/ovfdata/wrong_manifest.ovf000064400000000000000000000104031476477700300226540ustar00rootroot00000000000000 List of the virtual disks used in the package 0 False lenny-image bridged aa:00:00:d8:2c:1e None xen-br0 xen-pvm /dev/sda ro Logical networks used in the package Logical network used by this appliance. A virtual machine The kind of installed guest operating system Ubuntu Virtual hardware requirements for a virtual machine Virtual Hardware Family 0 Ubuntu-freshly-created virtualbox-2.2 1 virtual CPU 1 virtual CPU Number of virtual CPUs 1 3 1 2048 MB of memory 2048 MB of memory Memory Size 2 4 MegaBytes 2048 ideController0 ideController0 IDE Controller 3 5 PIIX4 1 Ethernet adapter on 'NAT' Ethernet adapter on 'NAT' 5 10 PCNet32 true bridged network disk1 disk1 Disk Image 7 17 /disk/vmdisk1 3 0 ganeti-3.1.0~rc2/test/data/ovfdata/wrong_ova.ova000064400000000000000000000120601476477700300216270ustar00rootroot00000000000000 Virtual disk information The list of logical networks The bridged network A virtual machine AyertiennaSUSE.x86_64-0.0.2 The kind of installed guest operating system Virtual hardware requirements Virtual Hardware Family 0 AyertiennaSUSE.x86_64-0.0.2 vmx-04 hertz * 10^6 Number of Virtual CPUs 1 virtual CPU(s) 1 3 1 byte * 2^20 Memory Size 512MB of memory 2 4 512 0 USB Controller usb 3 23 0 SCSI Controller scsiController0 4 lsilogic 6 0 IDE Controller ideController0 5 5 0 false Floppy Drive floppy0 6 14 0 false cdrom1 7 5 15 0 disk1 ovf:/disk/vmdisk1 8 4 17 2 true bridged E1000 ethernet adapter on "bridged" ethernet0 9 E1000 10 ganeti-3.1.0~rc2/test/data/ovfdata/wrong_xml.ovf000064400000000000000000000071331476477700300216540ustar00rootroot00000000000000 Virtual disk information The list of logical networks The bridged network A virtual machine AyertiennaSUSE.x86_64-0.0.2 The kind of installed guest operating system Virtual hardware requirements Virtual Hardware Family 0 AyertiennaSUSE.x86_64-0.0.2 vmx-04 hertz * 10^6 Number of Virtual CPUs 1 virtual CPU(s) 1 3 1 byte * 2^20 Memory Size 512MB of memory 2 4 512 0 SCSI Controller scsiController0 4 lsilogic 6 0 IDE Controller ideController0 5 5 0 disk1 ovf:/disk/vmdisk1 8 4 17 ganeti-3.1.0~rc2/test/data/proc_cgroup.txt000064400000000000000000000000671476477700300205620ustar00rootroot000000000000003:devices:/some_group 2:memory:/ 1:cpuset:/some_group ganeti-3.1.0~rc2/test/data/proc_cpuinfo.txt000064400000000000000000000073201476477700300207250ustar00rootroot00000000000000processor : 0 vendor_id : GenuineIntel cpu family : 6 model : 58 model name : Intel(R) Core(TM) i5-3320M CPU @ 2.60GHz stepping : 9 microcode : 0x13 cpu MHz : 1200.000 cache size : 3072 KB physical id : 0 siblings : 4 core id : 0 cpu cores : 2 apicid : 0 initial apicid : 0 fpu : yes fpu_exception : yes cpuid level : 13 wp : yes flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm ida arat epb xsaveopt pln pts dtherm tpr_shadow vnmi flexpriority ept vpid fsgsbase smep erms bogomips : 5188.22 clflush size : 64 cache_alignment : 64 address sizes : 36 bits physical, 48 bits virtual power management: processor : 1 vendor_id : GenuineIntel cpu family : 6 model : 58 model name : Intel(R) Core(TM) i5-3320M CPU @ 2.60GHz stepping : 9 microcode : 0x13 cpu MHz : 1200.000 cache size : 3072 KB physical id : 0 siblings : 4 core id : 0 cpu cores : 2 apicid : 1 initial apicid : 1 fpu : yes fpu_exception : yes cpuid level : 13 wp : yes flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm ida arat epb xsaveopt pln pts dtherm tpr_shadow vnmi flexpriority ept vpid fsgsbase smep erms bogomips : 5188.22 clflush size : 64 cache_alignment : 64 address sizes : 36 bits physical, 48 bits virtual power management: processor : 2 vendor_id : GenuineIntel cpu family : 6 model : 58 model name : Intel(R) Core(TM) i5-3320M CPU @ 2.60GHz stepping : 9 microcode : 0x13 cpu MHz : 1200.000 cache size : 3072 KB physical id : 0 siblings : 4 core id : 1 cpu cores : 2 apicid : 2 initial apicid : 2 fpu : yes fpu_exception : yes cpuid level : 13 wp : yes flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm ida arat epb xsaveopt pln pts dtherm tpr_shadow vnmi flexpriority ept vpid fsgsbase smep erms bogomips : 5188.22 clflush size : 64 cache_alignment : 64 address sizes : 36 bits physical, 48 bits virtual power management: processor : 3 vendor_id : GenuineIntel cpu family : 6 model : 58 model name : Intel(R) Core(TM) i5-3320M CPU @ 2.60GHz stepping : 9 microcode : 0x13 cpu MHz : 1200.000 cache size : 3072 KB physical id : 0 siblings : 4 core id : 1 cpu cores : 2 apicid : 3 initial apicid : 3 fpu : yes fpu_exception : yes cpuid level : 13 wp : yes flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm ida arat epb xsaveopt pln pts dtherm tpr_shadow vnmi flexpriority ept vpid fsgsbase smep erms bogomips : 5188.22 clflush size : 64 cache_alignment : 64 address sizes : 36 bits physical, 48 bits virtual power management: ganeti-3.1.0~rc2/test/data/proc_diskstats.txt000064400000000000000000000027761476477700300213050ustar00rootroot00000000000000 1 0 ram0 0 0 0 0 0 0 0 0 0 0 0 1 1 ram1 0 0 0 0 0 0 0 0 0 0 0 1 2 ram2 0 0 0 0 0 0 0 0 0 0 0 1 3 ram3 0 0 0 0 0 0 0 0 0 0 0 1 4 ram4 0 0 0 0 0 0 0 0 0 0 0 1 5 ram5 0 0 0 0 0 0 0 0 0 0 0 1 6 ram6 0 0 0 0 0 0 0 0 0 0 0 1 7 ram7 0 0 0 0 0 0 0 0 0 0 0 1 8 ram8 0 0 0 0 0 0 0 0 0 0 0 1 9 ram9 0 0 0 0 0 0 0 0 0 0 0 1 10 ram10 0 0 0 0 0 0 0 0 0 0 0 1 11 ram11 0 0 0 0 0 0 0 0 0 0 0 1 12 ram12 0 0 0 0 0 0 0 0 0 0 0 1 13 ram13 0 0 0 0 0 0 0 0 0 0 0 1 14 ram14 0 0 0 0 0 0 0 0 0 0 0 1 15 ram15 0 0 0 0 0 0 0 0 0 0 0 7 0 loop0 0 0 0 0 0 0 0 0 0 0 0 7 1 loop1 0 0 0 0 0 0 0 0 0 0 0 7 2 loop2 0 0 0 0 0 0 0 0 0 0 0 7 3 loop3 0 0 0 0 0 0 0 0 0 0 0 7 4 loop4 0 0 0 0 0 0 0 0 0 0 0 7 5 loop5 0 0 0 0 0 0 0 0 0 0 0 7 6 loop6 0 0 0 0 0 0 0 0 0 0 0 7 7 loop7 0 0 0 0 0 0 0 0 0 0 0 8 0 sda 89502 4833 4433387 89244 519115 62738 16059726 465120 0 149148 554564 8 1 sda1 505 2431 8526 132 478 174 124358 8500 0 340 8632 8 2 sda2 2 0 4 4 0 0 0 0 0 4 4 8 5 sda5 88802 2269 4422249 89032 453703 62564 15935368 396244 0 90064 485500 252 0 dm-0 90978 0 4420002 158632 582226 0 15935368 5592012 0 167688 5750652 252 1 dm-1 88775 0 4402378 157204 469594 0 15136008 4910424 0 164556 5067640 252 2 dm-2 1956 0 15648 1052 99920 0 799360 682492 0 4516 683552 8 16 sdb 0 0 0 0 0 0 0 0 0 0 0 ganeti-3.1.0~rc2/test/data/proc_drbd8.txt000064400000000000000000000035761476477700300202760ustar00rootroot00000000000000version: 8.0.12 (api:86/proto:86) GIT-hash: 5c9f89594553e32adb87d9638dce591782f947e3 build by XXX 0: cs:Connected st:Primary/Secondary ds:UpToDate/UpToDate C r--- ns:4375577 nr:0 dw:4446279 dr:674 al:1067 bm:69 lo:0 pe:0 ua:0 ap:0 resync: used:0/61 hits:0 misses:0 starving:0 dirty:0 changed:0 act_log: used:0/257 hits:793749 misses:1067 starving:0 dirty:0 changed:1067 1: cs:Connected st:Secondary/Primary ds:UpToDate/UpToDate C r--- ns:738320 nr:0 dw:738320 dr:554400 al:67 bm:0 lo:0 pe:0 ua:0 ap:0 resync: used:0/61 hits:0 misses:0 starving:0 dirty:0 changed:0 act_log: used:0/257 hits:92464 misses:67 starving:0 dirty:0 changed:67 2: cs:Unconfigured 4: cs:WFConnection st:Primary/Unknown ds:UpToDate/DUnknown C r--- ns:738320 nr:0 dw:738320 dr:554400 al:67 bm:0 lo:0 pe:0 ua:0 ap:0 resync: used:0/61 hits:0 misses:0 starving:0 dirty:0 changed:0 act_log: used:0/257 hits:92464 misses:67 starving:0 dirty:0 changed:67 5: cs:Connected st:Primary/Secondary ds:UpToDate/Diskless C r--- ns:4375581 nr:0 dw:4446283 dr:674 al:1069 bm:69 lo:0 pe:0 ua:0 ap:0 resync: used:0/61 hits:0 misses:0 starving:0 dirty:0 changed:0 act_log: used:0/257 hits:793750 misses:1069 starving:0 dirty:0 changed:1069 6: cs:Connected st:Secondary/Primary ds:Diskless/UpToDate C r--- ns:0 nr:4375581 dw:5186925 dr:327 al:75 bm:214 lo:0 pe:0 ua:0 ap:0 7: cs:WFConnection st:Secondary/Unknown ds:UpToDate/DUnknown C r--- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 resync: used:0/61 hits:0 misses:0 starving:0 dirty:0 changed:0 act_log: used:0/257 hits:0 misses:0 starving:0 dirty:0 changed:0 8: cs:StandAlone st:Secondary/Unknown ds:UpToDate/DUnknown r--- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 resync: used:0/61 hits:0 misses:0 starving:0 dirty:0 changed:0 act_log: used:0/257 hits:0 misses:0 starving:0 dirty:0 changed:0 ganeti-3.1.0~rc2/test/data/proc_drbd80-emptyline.txt000064400000000000000000000010161476477700300223450ustar00rootroot00000000000000version: 8.0.12 (api:86/proto:86) GIT-hash: 5c9f89594553e32adb87d9638dce591782f947e3 build by root@node1.example.com, 2009-05-22 12:47:52 0: cs:Connected st:Primary/Secondary ds:UpToDate/UpToDate C r--- ns:78728316 nr:0 dw:77675644 dr:1277039 al:254 bm:270 lo:0 pe:0 ua:0 ap:0 resync: used:0/61 hits:65657 misses:135 starving:0 dirty:0 changed:135 act_log: used:0/257 hits:11378843 misses:254 starving:0 dirty:0 changed:254 1: cs:Unconfigured 2: cs:Unconfigured 5: cs:Unconfigured 6: cs:Unconfigured ganeti-3.1.0~rc2/test/data/proc_drbd80-emptyversion.txt000064400000000000000000000007531476477700300231120ustar00rootroot00000000000000GIT-hash: 5c9f89594553e32adb87d9638dce591782f947e3 build by root@node1.example.com, 2009-05-22 12:47:52 0: cs:Connected st:Primary/Secondary ds:UpToDate/UpToDate C r--- ns:78728316 nr:0 dw:77675644 dr:1277039 al:254 bm:270 lo:0 pe:0 ua:0 ap:0 resync: used:0/61 hits:65657 misses:135 starving:0 dirty:0 changed:135 act_log: used:0/257 hits:11378843 misses:254 starving:0 dirty:0 changed:254 1: cs:Unconfigured 2: cs:Unconfigured 5: cs:Unconfigured 6: cs:Unconfigured ganeti-3.1.0~rc2/test/data/proc_drbd83.txt000064400000000000000000000022551476477700300203520ustar00rootroot00000000000000version: 8.3.1 (api:88/proto:86-89) GIT-hash: fd40f4a8f9104941537d1afc8521e584a6d3003c build by phil@fat-tyre, 2009-03-27 12:19:49 0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r---- ns:140978 nr:0 dw:9906 dr:131533 al:27 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 1: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r--- ns:0 nr:140980 dw:140980 dr:0 al:0 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 2: cs:Unconfigured 4: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C r---- ns:140978 nr:0 dw:9906 dr:131534 al:27 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 5: cs:Connected ro:Primary/Secondary ds:UpToDate/Diskless C r---- ns:140978 nr:0 dw:9906 dr:131533 al:19 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 6: cs:Connected ro:Secondary/Primary ds:Diskless/UpToDate C r--- ns:0 nr:140978 dw:140978 dr:0 al:0 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 7: cs:WFConnection ro:Secondary/Unknown ds:UpToDate/DUnknown C r--- ns:0 nr:140978 dw:140978 dr:0 al:0 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 8: cs:StandAlone ro:Secondary/Unknown ds:UpToDate/DUnknown r--- ns:0 nr:140978 dw:140978 dr:0 al:0 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 ganeti-3.1.0~rc2/test/data/proc_drbd83_sync.txt000064400000000000000000000015501476477700300214030ustar00rootroot00000000000000version: 8.3.1 (api:88/proto:86-89) GIT-hash: fd40f4a8f9104941537d1afc8521e584a6d3003c build by phil@fat-tyre, 2009-03-27 12:19:49 0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r---- ns:140978 nr:0 dw:9906 dr:131533 al:27 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 1: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r--- ns:0 nr:140980 dw:140980 dr:0 al:0 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 2: cs:Unconfigured 3: cs:SyncTarget ro:Primary/Secondary ds:Inconsistent/UpToDate C r---- ns:0 nr:178176 dw:178176 dr:0 al:104 bm:42 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:346112 [=====>..............] sync'ed: 34.9% (346112/524288)M finish: 0:00:05 speed: 59,392 (59,392) K/sec 4: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C r---- ns:140978 nr:0 dw:9906 dr:131534 al:27 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 ganeti-3.1.0~rc2/test/data/proc_drbd83_sync_krnl2.6.39.txt000064400000000000000000000015441476477700300231140ustar00rootroot00000000000000version: 8.3.1 (api:88/proto:86-89) GIT-hash: fd40f4a8f9104941537d1afc8521e584a6d3003c build by phil@fat-tyre, 2009-03-27 12:19:49 0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r---- ns:140978 nr:0 dw:9906 dr:131533 al:27 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 1: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r--- ns:0 nr:140980 dw:140980 dr:0 al:0 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 2: cs:Unconfigured 3: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent A r----- ns:373888 nr:0 dw:0 dr:374088 al:0 bm:22 lo:7 pe:27 ua:7 ap:0 ep:1 wo:f oos:15358208 [>....................] sync'ed: 2.4% (14996/15360)Mfinish: 0:04:08 speed: 61,736 (61,736) K/sec 4: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C r---- ns:140978 nr:0 dw:9906 dr:131534 al:27 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 ganeti-3.1.0~rc2/test/data/proc_drbd83_sync_want.txt000064400000000000000000000006471476477700300224420ustar00rootroot00000000000000version: 8.3.11 (api:88/proto:86-96) srcversion: 2D876214BAAD53B31ADC1D6 0: cs:SyncTarget ro:Secondary/Primary ds:Inconsistent/UpToDate C r----- ns:0 nr:460288 dw:460160 dr:0 al:0 bm:28 lo:2 pe:4 ua:1 ap:0 ep:1 wo:f oos:588416 [=======>............] sync'ed: 44.4% (588416/1048576)K finish: 0:00:08 speed: 65,736 (65,736) want: 61,440 K/sec 1: cs:Unconfigured 2: cs:Unconfigured 3: cs:Unconfigured ganeti-3.1.0~rc2/test/data/proc_drbd84.txt000064400000000000000000000015731476477700300203550ustar00rootroot00000000000000version: 8.4.2 (api:1/proto:86-101) GIT-hash: 7ad5f850d711223713d6dcadc3dd48860321070c build by root@example.com, 2013-04-10 07:45:25 0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r----- ns:1048576 nr:0 dw:0 dr:1048776 al:0 bm:64 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 1: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r----- ns:0 nr:1048576 dw:1048576 dr:0 al:0 bm:64 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 2: cs:Unconfigured 4: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C r----- ns:0 nr:0 dw:0 dr:200 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1048320 6: cs:Connected ro:Secondary/Primary ds:Diskless/UpToDate C r----- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 8: cs:StandAlone ro:Secondary/Unknown ds:UpToDate/DUnknown r----- ns:0 nr:0 dw:0 dr:200 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1048320 ganeti-3.1.0~rc2/test/data/proc_drbd84_emptyfirst.txt000064400000000000000000000013451476477700300226400ustar00rootroot00000000000000version: 8.4.2 (api:1/proto:86-101) GIT-hash: 7ad5f850d711223713d6dcadc3dd48860321070c build by root@example.com, 2013-04-10 07:45:25 1: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r----- ns:0 nr:1048576 dw:1048576 dr:0 al:0 bm:64 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 2: cs:Unconfigured 4: cs:WFConnection ro:Primary/Unknown ds:UpToDate/DUnknown C r----- ns:0 nr:0 dw:0 dr:200 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1048320 6: cs:Connected ro:Secondary/Primary ds:Diskless/UpToDate C r----- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 8: cs:StandAlone ro:Secondary/Unknown ds:UpToDate/DUnknown r----- ns:0 nr:0 dw:0 dr:200 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:1048320 ganeti-3.1.0~rc2/test/data/proc_drbd84_sync.txt000064400000000000000000000011121476477700300213760ustar00rootroot00000000000000version: 8.4.2 (api:1/proto:86-101) GIT-hash: 7ad5f850d711223713d6dcadc3dd48860321070c build by root@example.com, 2013-04-10 07:45:25 0: cs:StandAlone ro:Primary/Unknown ds:UpToDate/DUnknown r----- ns:0 nr:0 dw:33318 dr:730 al:15 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:d oos:1048320 3: cs:Unconfigured 5: cs:SyncSource ro:Secondary/Secondary ds:UpToDate/Inconsistent C r---n- ns:716992 nr:0 dw:0 dr:719432 al:0 bm:43 lo:0 pe:33 ua:18 ap:0 ep:1 wo:f oos:335744 [============>.......] sync'ed: 68.5% (335744/1048576)K finish: 0:00:05 speed: 64,800 (64,800) K/sec ganeti-3.1.0~rc2/test/data/proc_meminfo.txt000064400000000000000000000026131476477700300207140ustar00rootroot00000000000000MemTotal: 32263108 kB MemFree: 257212 kB MemAvailable: 21604764 kB Buffers: 1100556 kB Cached: 20549156 kB SwapCached: 25536 kB Active: 14881888 kB Inactive: 15349404 kB Active(anon): 737500 kB Inactive(anon): 8720328 kB Active(file): 14144388 kB Inactive(file): 6629076 kB Unevictable: 33204 kB Mlocked: 33204 kB SwapTotal: 4194300 kB SwapFree: 4111972 kB Dirty: 26092 kB Writeback: 0 kB AnonPages: 8589380 kB Mapped: 642380 kB Shmem: 845856 kB KReclaimable: 1033824 kB Slab: 1231812 kB SReclaimable: 1033824 kB SUnreclaim: 197988 kB KernelStack: 15120 kB PageTables: 42612 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB CommitLimit: 20325852 kB Committed_AS: 42052296 kB VmallocTotal: 34359738367 kB VmallocUsed: 223368 kB VmallocChunk: 0 kB Percpu: 24512 kB HardwareCorrupted: 0 kB AnonHugePages: 4270080 kB ShmemHugePages: 0 kB ShmemPmdMapped: 0 kB FileHugePages: 0 kB FilePmdMapped: 0 kB HugePages_Total: 0 HugePages_Free: 0 HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 0 kB DirectMap4k: 2666876 kB DirectMap2M: 14553088 kB DirectMap1G: 16777216 kB ganeti-3.1.0~rc2/test/data/qa-minimal-nodes-instances-only.json000064400000000000000000000016221476477700300244670ustar00rootroot00000000000000{ "name": "xen-test-qa-minimal-nodes-instances-only", "# Lists of disks": null, "disks": [ { "size": "1G", "growth": "2G" }, { "size": "512M", "growth": "768M" } ], "enabled-disk-templates": [ "plain", "drbd", "diskless" ], "nodes": [ { "# Master node": null, "primary": "xen-test-0", "secondary": "192.0.2.1" }, { "primary": "xen-test-1", "secondary": "192.0.2.2" }, { "primary": "xen-test-2", "secondary": "192.0.2.3" }, { "primary": "xen-test-3", "secondary": "192.0.2.4" } ], "instances": [ { "name": "xen-test-inst1", "nic.mac/0": "AA:00:00:11:11:11" }, { "name": "xen-test-inst2", "nic.mac/0": "AA:00:00:22:22:22" } ], "tests": { "default": false }, "# vim: set syntax=javascript :": null } ganeti-3.1.0~rc2/test/data/sys_drbd_usermode_helper.txt000064400000000000000000000000121476477700300233010ustar00rootroot00000000000000/bin/true ganeti-3.1.0~rc2/test/data/vgreduce-removemissing-2.02.02.txt000064400000000000000000000007201476477700300235240ustar00rootroot00000000000000 Couldn't find device with uuid 'gg4cmC-4lrT-EN1v-39OA-6S2b-6eEI-wWlJJJ'. Couldn't find all physical volumes for volume group xenvg. Couldn't find device with uuid 'gg4cmC-4lrT-EN1v-39OA-6S2b-6eEI-wWlJJJ'. Couldn't find all physical volumes for volume group xenvg. Couldn't find device with uuid 'gg4cmC-4lrT-EN1v-39OA-6S2b-6eEI-wWlJJJ'. Couldn't find device with uuid 'gg4cmC-4lrT-EN1v-39OA-6S2b-6eEI-wWlJJJ'. Wrote out consistent volume group xenvg ganeti-3.1.0~rc2/test/data/vgreduce-removemissing-2.02.66-fail.txt000064400000000000000000000064061476477700300244560ustar00rootroot00000000000000 Couldn't find device with uuid bHRa26-svpL-ihJX-e0S4-2HNz-wAAi-AlBFtl. WARNING: Partial LV 4ba7abfa-8459-43b6-b00f-c016244980f0.disk0 needs to be repaired or removed. WARNING: Partial LV e972960d-4e35-46b2-9cda-7029916b28c1.disk0_data needs to be repaired or removed. WARNING: Partial LV e972960d-4e35-46b2-9cda-7029916b28c1.disk0_meta needs to be repaired or removed. WARNING: Partial LV 4fa40b51-dd4d-4fd9-aef1-35cc3a0f1f11.disk0_data needs to be repaired or removed. WARNING: Partial LV 4fa40b51-dd4d-4fd9-aef1-35cc3a0f1f11.disk0_meta needs to be repaired or removed. WARNING: Partial LV 0a184b34-1270-4f1a-94df-86da2167cfee.disk0_data needs to be repaired or removed. WARNING: Partial LV 0a184b34-1270-4f1a-94df-86da2167cfee.disk0_meta needs to be repaired or removed. WARNING: Partial LV 7e49c8a9-9c65-4e76-810e-bd3d7a1d97a9.disk0_data needs to be repaired or removed. WARNING: Partial LV 7e49c8a9-9c65-4e76-810e-bd3d7a1d97a9.disk0_meta needs to be repaired or removed. WARNING: Partial LV 290a3fd4-c035-4fbe-9a18-f5a0889bd45d.disk0_data needs to be repaired or removed. WARNING: Partial LV 290a3fd4-c035-4fbe-9a18-f5a0889bd45d.disk0_meta needs to be repaired or removed. WARNING: Partial LV c579be32-c041-4f1b-ae3e-c58aac9c2593.disk0_data needs to be repaired or removed. WARNING: Partial LV c579be32-c041-4f1b-ae3e-c58aac9c2593.disk0_meta needs to be repaired or removed. WARNING: Partial LV 47524563-3788-4a89-a61f-4274134dea73.disk0_data needs to be repaired or removed. WARNING: Partial LV 47524563-3788-4a89-a61f-4274134dea73.disk0_meta needs to be repaired or removed. WARNING: Partial LV ede9f706-a0dc-4202-96f2-1728240bbf05.disk0_data needs to be repaired or removed. WARNING: Partial LV ede9f706-a0dc-4202-96f2-1728240bbf05.disk0_meta needs to be repaired or removed. WARNING: Partial LV 731d9f1b-3f2f-4860-85b3-217a36b9c48e.disk1_data needs to be repaired or removed. WARNING: Partial LV 731d9f1b-3f2f-4860-85b3-217a36b9c48e.disk1_meta needs to be repaired or removed. WARNING: Partial LV f449ccfd-4e6b-42d6-9a52-838371988ab5.disk0_data needs to be repaired or removed. WARNING: Partial LV f449ccfd-4e6b-42d6-9a52-838371988ab5.disk0_meta needs to be repaired or removed. WARNING: Partial LV 69bb4f61-fd0c-4c89-a57f-5285ae99b3bd.disk0_data needs to be repaired or removed. WARNING: Partial LV 9c29c24a-97ed-4fc7-b479-7a3385365a71.disk0 needs to be repaired or removed. WARNING: Partial LV a919d93e-0f51-4e4d-9018-e25ee7d5b36b.disk0 needs to be repaired or removed. WARNING: Partial LV d2501e6b-56a4-43b6-8856-471e5d49e892.disk0_data needs to be repaired or removed. WARNING: Partial LV d2501e6b-56a4-43b6-8856-471e5d49e892.disk0_meta needs to be repaired or removed. WARNING: Partial LV 31a1f85a-ecc8-40c0-88aa-e694626906a3.disk0 needs to be repaired or removed. WARNING: Partial LV d124d70a-4776-4e00-bf0d-43511c29c534.disk0_data needs to be repaired or removed. WARNING: Partial LV d124d70a-4776-4e00-bf0d-43511c29c534.disk0_meta needs to be repaired or removed. WARNING: Partial LV f73b4499-34ec-4f70-a543-e43152a8644a.disk0 needs to be repaired or removed. WARNING: There are still partial LVs in VG xenvg. To remove them unconditionally use: vgreduce --removemissing --force. Proceeding to remove empty missing PVs. ganeti-3.1.0~rc2/test/data/vgreduce-removemissing-2.02.66-ok.txt000064400000000000000000000001631476477700300241460ustar00rootroot00000000000000 Couldn't find device with uuid NzfYON-F7ky-1Szf-aGf1-v8Xa-Bt1W-8V3bou. Wrote out consistent volume group xenvg ganeti-3.1.0~rc2/test/data/vgs-missing-pvs-2.02.02.txt000064400000000000000000000004571476477700300221160ustar00rootroot00000000000000 Couldn't find device with uuid 'gg4cmC-4lrT-EN1v-39OA-6S2b-6eEI-wWlJJJ'. Couldn't find all physical volumes for volume group xenvg. Couldn't find device with uuid 'gg4cmC-4lrT-EN1v-39OA-6S2b-6eEI-wWlJJJ'. Couldn't find all physical volumes for volume group xenvg. Volume group xenvg not found ganeti-3.1.0~rc2/test/data/vgs-missing-pvs-2.02.66.txt000064400000000000000000000001601476477700300221170ustar00rootroot00000000000000 Couldn't find device with uuid bHRa26-svpL-ihJX-e0S4-2HNz-wAAi-AlBFtl. xenvg 2 52 0 wz-pn- 1.31t 1.07t ganeti-3.1.0~rc2/test/data/xen-xl-info-4.0.1.txt000064400000000000000000000022651476477700300210440ustar00rootroot00000000000000host : host.example.com release : 3.2.0 version : #1 SMP Tue Jan 1 00:00:00 UTC 2013 machine : x86_64 nr_cpus : 4 nr_nodes : 1 cores_per_socket : 2 threads_per_core : 1 cpu_mhz : 2800 hw_caps : bfebfbff:20100800:00000000:00000940:0004e3bd:00000000:00000001:00000000 virt_caps : total_memory : 16378 free_memory : 8004 node_to_cpu : node0:0-3 node_to_memory : node0:8004 node_to_dma32_mem : node0:2985 max_node_id : 0 xen_major : 4 xen_minor : 0 xen_extra : .1 xen_caps : xen-3.0-x86_64 xen-3.0-x86_32p xen_scheduler : credit xen_pagesize : 4096 platform_params : virt_start=0xffff800000000000 xen_changeset : unavailable xen_commandline : placeholder dom0_mem=1024M com1=115200,8n1 console=com1 cc_compiler : gcc version 4.4.5 (Debian 4.4.5-8) cc_compile_by : user cc_compile_domain : example.com cc_compile_date : Tue Jan 1 00:00:00 UTC 2013 xend_config_format : 4 ganeti-3.1.0~rc2/test/data/xen-xl-list-4.0.1-dom0-only.txt000064400000000000000000000002371476477700300226750ustar00rootroot00000000000000Name ID Mem VCPUs State Time(s) Domain-0 0 1023 1 r----- 121152.6 ganeti-3.1.0~rc2/test/data/xen-xl-list-4.0.1-four-instances.txt000064400000000000000000000006141476477700300240160ustar00rootroot00000000000000Name ID Mem VCPUs State Time(s) Domain-0 0 1023 1 r----- 154706.1 server01.example.com 1 1024 1 -b---- 167643.2 web3106215069.example.com 3 4096 1 -b---- 466690.9 testinstance.example.com 2 2048 2 r----- 244443.0 ganeti-3.1.0~rc2/test/data/xen-xl-list-4.4-crashed-instances.txt000064400000000000000000000006141476477700300243210ustar00rootroot00000000000000Name ID Mem VCPUs State Time(s) Domain-0 0 1023 1 r----- 154706.1 server01.example.com 1 1024 1 -b---- 167643.2 (null) 28441 789 1 --psc- 1.2 alsodying.example.com 28448 1024 1 --psc- 1.4 ganeti-3.1.0~rc2/test/data/xen-xl-list-long-4.0.1.txt000064400000000000000000000106601476477700300220170ustar00rootroot00000000000000(domain (domid 0) (cpu_weight 2048) (cpu_cap 0) (bootloader ) (on_crash restart) (uuid 00000000-0000-0000-0000-000000000000) (bootloader_args ) (vcpus 24) (name Domain-0) (cpus ((0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) ) ) (on_reboot restart) (on_poweroff destroy) (maxmem 16777215) (memory 1023) (shadow_memory 0) (features ) (on_xend_start ignore) (on_xend_stop ignore) (cpu_time 184000.41332) (online_vcpus 1) (image (linux (kernel ) (superpages 0) (nomigrate 0) (tsc_mode 0))) (status 2) (state r-----) ) (domain (domid 119) (cpu_weight 256) (cpu_cap 0) (bootloader ) (on_crash restart) (uuid e430b4b8-dc91-9390-dfe0-b83c138ea0aa) (bootloader_args ) (vcpus 1) (description ) (name instance1.example.com) (cpus (())) (on_reboot restart) (on_poweroff destroy) (maxmem 128) (memory 128) (shadow_memory 0) (features ) (on_xend_start ignore) (on_xend_stop ignore) (start_time 1357749308.05) (cpu_time 24.116146647) (online_vcpus 1) (image (linux (kernel /boot/vmlinuz-ganetixenu) (args 'root=/dev/xvda1 ro') (superpages 0) (videoram 4) (pci ()) (nomigrate 0) (tsc_mode 0) (notes (HV_START_LOW 18446603336221196288) (FEATURES '!writable_page_tables|pae_pgdir_above_4gb') (VIRT_BASE 18446744071562067968) (GUEST_VERSION 2.6) (PADDR_OFFSET 0) (GUEST_OS linux) (HYPERCALL_PAGE 18446744071578849280) (LOADER generic) (SUSPEND_CANCEL 1) (PAE_MODE yes) (ENTRY 18446744071592116736) (XEN_VERSION xen-3.0) ) ) ) (status 2) (state -b----) (store_mfn 8836555) (console_mfn 8735251) (device (vif (bridge xen-br0) (mac aa:00:00:30:8d:9d) (script /etc/xen/scripts/vif-bridge) (uuid f57c4758-cf0a-8227-6d13-fe26ece82d75) (backend 0) ) ) (device (console (protocol vt100) (location 2) (uuid 7695737a-ffc2-4e0d-7f6d-734143b8afc4) ) ) (device (vbd (protocol x86_64-abi) (uuid 409e1ff8-435a-4704-80bb-4bfe800d932e) (bootable 1) (dev sda:disk) (uname phy:/var/run/ganeti/instance-disks/instance1.example.com:0 ) (mode w) (backend 0) (VDI ) ) ) ) ganeti-3.1.0~rc2/test/data/xen-xl-uptime-4.0.1.txt000064400000000000000000000002411476477700300214040ustar00rootroot00000000000000Name ID Uptime Domain-0 0 98 days, 2:27:44 instance1.example.com 119 15 days, 20:57:07 ganeti-3.1.0~rc2/test/hs/000075500000000000000000000000001476477700300151755ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/000075500000000000000000000000001476477700300161145ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/AutoConf.hs000064400000000000000000000223031476477700300201660ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittests for 'AutoConf' -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.AutoConf where import qualified Data.Char as Char (isAlpha) import Test.HUnit as HUnit import qualified AutoConf import qualified Test.Ganeti.TestHelper as TestHelper {-# ANN module "HLint: ignore Use camelCase" #-} -- | 'isFilePath x' tests whether @x@ is a valid filepath -- -- A valid filepath must be absolute and must not contain commas. isFilePath :: String -> Bool isFilePath ('/':str) = ',' `notElem` str isFilePath _ = False -- | 'isGntScript x' tests whether @x@ is a valid Ganeti script -- -- A valid Ganeti script is prefixed by "gnt-" and the rest of the -- 'String' contains only alphabetic 'Char's. isGntScript :: String -> Bool isGntScript str = case span (/= '-') str of (x, '-':y) -> x == "gnt" && all Char.isAlpha y _ -> False -- | 'isGroup x' tests whether @x@ is a valid group name -- -- A valid group name name is an alphabetic 'String' possibly -- containing '-'. isGroup :: String -> Bool isGroup = all (\c -> Char.isAlpha c || c == '-') -- | 'isProgram x' tests whether @x@ is a valid program name -- -- A valid program name is an alphabetic 'String'. isProgram :: String -> Bool isProgram = all Char.isAlpha -- | 'isUser x' tests whether @x@ is a valid username -- -- See 'isGroup'. isUser :: String -> Bool isUser = isGroup case_versionSuffix :: Assertion case_versionSuffix = HUnit.assertBool "'versionSuffix' is invalid" (case AutoConf.versionSuffix of "" -> True '~':x -> not (null x) _ -> False) case_localstatedir :: Assertion case_localstatedir = HUnit.assertBool "'localstatedir' is invalid" (isFilePath AutoConf.localstatedir) case_sysconfdir :: Assertion case_sysconfdir = HUnit.assertBool "'sysconfdir' is invalid" (isFilePath AutoConf.sysconfdir) case_sshConfigDir :: Assertion case_sshConfigDir = HUnit.assertBool "'sshConfigDir' is invalid" (isFilePath AutoConf.sshConfigDir) case_sshLoginUser :: Assertion case_sshLoginUser = HUnit.assertBool "'sshLoginUser' is invalid" (isUser AutoConf.sshLoginUser) case_sshConsoleUser :: Assertion case_sshConsoleUser = HUnit.assertBool "'sshConsoleUser' is invalid" (isUser AutoConf.sshConsoleUser) case_exportDir :: Assertion case_exportDir = HUnit.assertBool "'exportDir' is invalid" (isFilePath AutoConf.exportDir) case_osSearchPath :: Assertion case_osSearchPath = HUnit.assertBool "'osSearchPath' is invalid" (all isFilePath AutoConf.osSearchPath) case_esSearchPath :: Assertion case_esSearchPath = HUnit.assertBool "'esSearchPath' is invalid" (all isFilePath AutoConf.esSearchPath) case_xenBootloader :: Assertion case_xenBootloader = HUnit.assertBool "'xenBootloader' is invalid" (null AutoConf.xenBootloader || isFilePath AutoConf.xenBootloader) case_xenConfigDir :: Assertion case_xenConfigDir = HUnit.assertBool "'xenConfigDir' is invalid" (isFilePath AutoConf.xenConfigDir) case_xenKernel :: Assertion case_xenKernel = HUnit.assertBool "'xenKernel' is invalid" (isFilePath AutoConf.xenKernel) case_xenInitrd :: Assertion case_xenInitrd = HUnit.assertBool "'xenInitrd' is invalid" (isFilePath AutoConf.xenInitrd) case_kvmKernel :: Assertion case_kvmKernel = HUnit.assertBool "'kvmKernel' is invalid" (isFilePath AutoConf.kvmKernel) case_iallocatorSearchPath :: Assertion case_iallocatorSearchPath = HUnit.assertBool "'iallocatorSearchPath' is invalid" (all isFilePath AutoConf.iallocatorSearchPath) case_kvmPath :: Assertion case_kvmPath = HUnit.assertBool "'kvmPath' is invalid" (isFilePath AutoConf.kvmPath) case_ipPath :: Assertion case_ipPath = HUnit.assertBool "'ipPath' is invalid" (isFilePath AutoConf.ipPath) case_socatPath :: Assertion case_socatPath = HUnit.assertBool "'socatPath' is invalid" (isFilePath AutoConf.socatPath) case_toolsdir :: Assertion case_toolsdir = HUnit.assertBool "'toolsdir' is invalid" (isFilePath AutoConf.toolsdir) case_gntScripts :: Assertion case_gntScripts = HUnit.assertBool "'gntScripts' is invalid" (all isGntScript AutoConf.gntScripts) case_htoolsProgs :: Assertion case_htoolsProgs = HUnit.assertBool "'htoolsProgs' is invalid" (all isProgram AutoConf.htoolsProgs) case_pkglibdir :: Assertion case_pkglibdir = HUnit.assertBool "'pkglibdir' is invalid" (isFilePath AutoConf.pkglibdir) case_sharedir :: Assertion case_sharedir = HUnit.assertBool "'sharedir' is invalid" (isFilePath AutoConf.sharedir) case_versionedsharedir :: Assertion case_versionedsharedir = HUnit.assertBool "'versionedsharedir' is invalid" (isFilePath AutoConf.versionedsharedir) case_drbdBarriers :: Assertion case_drbdBarriers = HUnit.assertBool "'drbdBarriers' is invalid" (AutoConf.drbdBarriers `elem` ["n", "bf"]) case_syslogUsage :: Assertion case_syslogUsage = HUnit.assertBool "'syslogUsage' is invalid" (AutoConf.syslogUsage `elem` ["no", "yes", "only"]) case_daemonsGroup :: Assertion case_daemonsGroup = HUnit.assertBool "'daemonsGroup' is invalid" (isGroup AutoConf.daemonsGroup) case_adminGroup :: Assertion case_adminGroup = HUnit.assertBool "'adminGroup' is invalid" (isGroup AutoConf.adminGroup) case_masterdUser :: Assertion case_masterdUser = HUnit.assertBool "'masterdUser' is invalid" (isUser AutoConf.masterdUser) case_masterdGroup :: Assertion case_masterdGroup = HUnit.assertBool "'masterdGroup' is invalid" (isGroup AutoConf.masterdGroup) case_rapiUser :: Assertion case_rapiUser = HUnit.assertBool "'rapiUser' is invalid" (isUser AutoConf.rapiUser) case_rapiGroup :: Assertion case_rapiGroup = HUnit.assertBool "'rapiGroup' is invalid" (isGroup AutoConf.rapiGroup) case_confdUser :: Assertion case_confdUser = HUnit.assertBool "'confdUser' is invalid" (isUser AutoConf.confdUser) case_confdGroup :: Assertion case_confdGroup = HUnit.assertBool "'confdGroup' is invalid" (isGroup AutoConf.confdGroup) case_luxidUser :: Assertion case_luxidUser = HUnit.assertBool "'luxidUser' is invalid" (isUser AutoConf.luxidUser) case_luxidGroup :: Assertion case_luxidGroup = HUnit.assertBool "'luxidGroup' is invalid" (isGroup AutoConf.luxidGroup) case_nodedUser :: Assertion case_nodedUser = HUnit.assertBool "'nodedUser' is invalid" (isUser AutoConf.nodedUser) case_nodedGroup :: Assertion case_nodedGroup = HUnit.assertBool "'nodedGroup' is invalid" (isGroup AutoConf.nodedGroup) case_mondUser :: Assertion case_mondUser = HUnit.assertBool "'mondUser' is invalid" (isUser AutoConf.mondUser) case_mondGroup :: Assertion case_mondGroup = HUnit.assertBool "'mondGroup' is invalid" (isUser AutoConf.mondGroup) case_diskSeparator :: Assertion case_diskSeparator = HUnit.assertBool "'diskSeparator' is invalid" (not (null AutoConf.diskSeparator)) case_qemuimgPath :: Assertion case_qemuimgPath = HUnit.assertBool "'qemuimgPath' is invalid" (isFilePath AutoConf.qemuimgPath) TestHelper.testSuite "AutoConf" [ 'case_versionSuffix , 'case_localstatedir , 'case_sysconfdir , 'case_sshConfigDir , 'case_sshLoginUser , 'case_sshConsoleUser , 'case_exportDir , 'case_osSearchPath , 'case_esSearchPath , 'case_xenBootloader , 'case_xenConfigDir , 'case_xenKernel , 'case_xenInitrd , 'case_kvmKernel , 'case_iallocatorSearchPath , 'case_kvmPath , 'case_ipPath , 'case_socatPath , 'case_toolsdir , 'case_gntScripts , 'case_htoolsProgs , 'case_pkglibdir , 'case_sharedir , 'case_versionedsharedir , 'case_drbdBarriers , 'case_syslogUsage , 'case_daemonsGroup , 'case_adminGroup , 'case_masterdUser , 'case_masterdGroup , 'case_rapiUser , 'case_rapiGroup , 'case_confdUser , 'case_confdGroup , 'case_luxidUser , 'case_luxidGroup , 'case_nodedUser , 'case_nodedGroup , 'case_mondUser , 'case_mondGroup , 'case_diskSeparator , 'case_qemuimgPath ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/000075500000000000000000000000001476477700300173235ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Attoparsec.hs000064400000000000000000000051301476477700300217630ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittests for Attoparsec support for unicode -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Attoparsec (testAttoparsec) where import Test.HUnit import Test.Ganeti.TestHelper import qualified Data.Attoparsec.Text as A import Data.Attoparsec.Text (Parser) import Data.Text (pack, unpack) -- | Unicode test string, first part. part1 :: String part1 = "äßĉ" -- | Unicode test string, second part. part2 :: String part2 = "ðèŲ‚" -- | Simple parser able to split a string in two parts, name and -- value, separated by a '=' sign. simpleParser :: Parser (String, String) simpleParser = do n <- A.takeTill (\c -> A.isHorizontalSpace c || c == '=') A.skipWhile A.isHorizontalSpace _ <- A.char '=' A.skipWhile A.isHorizontalSpace v <- A.takeTill A.isEndOfLine return (unpack n, unpack v) {-# ANN case_unicodeParsing "HLint: ignore Use camelCase" #-} -- | Tests whether a Unicode string is still Unicode after being -- parsed. case_unicodeParsing :: Assertion case_unicodeParsing = case A.parseOnly simpleParser text of Right (name, value) -> do assertEqual "name part" part1 name assertEqual "value part" part2 value Left msg -> assertFailure $ "Failed to parse: " ++ msg where text = Data.Text.pack $ part1 ++ " = \t" ++ part2 testSuite "Attoparsec" [ 'case_unicodeParsing ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/BasicTypes.hs000064400000000000000000000132671476477700300217360ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, FlexibleInstances, TypeSynonymInstances #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.BasicTypes (testBasicTypes) where import Test.QuickCheck hiding (Result) import Test.QuickCheck.Function import Control.Monad import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.BasicTypes -- Since we actually want to test these, don't tell us not to use them :) {-# ANN module "HLint: ignore Functor law" #-} {-# ANN module "HLint: ignore Monad law, left identity" #-} {-# ANN module "HLint: ignore Monad law, right identity" #-} {-# ANN module "HLint: ignore Use >=>" #-} {-# ANN module "HLint: ignore Use ." #-} -- * Arbitrary instances instance (Arbitrary a) => Arbitrary (Result a) where arbitrary = oneof [ Bad <$> arbitrary , Ok <$> arbitrary ] -- * Test cases -- | Tests the functor identity law: -- -- > fmap id == id prop_functor_id :: Result Int -> Property prop_functor_id ri = fmap id ri ==? ri -- | Tests the functor composition law: -- -- > fmap (f . g) == fmap f . fmap g prop_functor_composition :: Result Int -> Fun Int Int -> Fun Int Int -> Property prop_functor_composition ri (Fun _ f) (Fun _ g) = fmap (f . g) ri ==? (fmap f . fmap g) ri -- | Tests the applicative identity law: -- -- > pure id <*> v = v prop_applicative_identity :: Result Int -> Property prop_applicative_identity v = pure id <*> v ==? v -- | Tests the applicative composition law: -- -- > pure (.) <*> u <*> v <*> w = u <*> (v <*> w) prop_applicative_composition :: Result (Fun Int Int) -> Result (Fun Int Int) -> Result Int -> Property prop_applicative_composition u v w = let u' = fmap apply u v' = fmap apply v in pure (.) <*> u' <*> v' <*> w ==? u' <*> (v' <*> w) -- | Tests the applicative homomorphism law: -- -- > pure f <*> pure x = pure (f x) prop_applicative_homomorphism :: Fun Int Int -> Int -> Property prop_applicative_homomorphism (Fun _ f) x = ((pure f <*> pure x)::Result Int) ==? pure (f x) -- | Tests the applicative interchange law: -- -- > u <*> pure y = pure ($ y) <*> u prop_applicative_interchange :: Result (Fun Int Int) -> Int -> Property prop_applicative_interchange f y = let u = fmap apply f -- need to extract the actual function from Fun in u <*> pure y ==? pure ($ y) <*> u -- | Tests the applicative\/functor correspondence: -- -- > fmap f x = pure f <*> x prop_applicative_functor :: Fun Int Int -> Result Int -> Property prop_applicative_functor (Fun _ f) x = fmap f x ==? pure f <*> x -- | Tests the applicative\/monad correspondence: -- -- > pure = return -- -- > (<*>) = ap prop_applicative_monad :: Int -> Result (Fun Int Int) -> Property prop_applicative_monad v f = let v' = pure v :: Result Int f' = fmap apply f -- need to extract the actual function from Fun in v' ==? return v .&&. (f' <*> v') ==? f' `ap` v' -- | Tests the monad laws: -- -- > return a >>= k == k a -- -- > m >>= return == m -- -- > m >>= (\x -> k x >>= h) == (m >>= k) >>= h prop_monad_laws :: Int -> Result Int -> Fun Int (Result Int) -> Fun Int (Result Int) -> Property prop_monad_laws a m (Fun _ k) (Fun _ h) = conjoin [ counterexample "return a >>= k == k a" ((return a >>= k) ==? k a) , counterexample "m >>= return == m" ((m >>= return) ==? m) , counterexample "m >>= (\\x -> k x >>= h) == (m >>= k) >>= h)" ((m >>= (\x -> k x >>= h)) ==? ((m >>= k) >>= h)) ] -- | Tests the monad plus laws: -- -- > mzero >>= f = mzero -- -- > v >> mzero = mzero prop_monadplus_mzero :: Result Int -> Fun Int (Result Int) -> Property prop_monadplus_mzero v (Fun _ f) = counterexample "mzero >>= f = mzero" ((mzero >>= f) ==? mzero) .&&. -- FIXME: since we have "many" mzeros, we can't test for equality, -- just that we got back a 'Bad' value; I'm not sure if this means -- our MonadPlus instance is not sound or not... counterexample "v >> mzero = mzero" (isBad (v >> mzero)) testSuite "BasicTypes" [ 'prop_functor_id , 'prop_functor_composition , 'prop_applicative_identity , 'prop_applicative_composition , 'prop_applicative_homomorphism , 'prop_applicative_interchange , 'prop_applicative_functor , 'prop_applicative_monad , 'prop_monad_laws , 'prop_monadplus_mzero ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Common.hs000064400000000000000000000171251476477700300211150ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for the 'Ganeti.Common' module. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Common ( testCommon , checkOpt , passFailOpt , checkEarlyExit ) where import Test.QuickCheck hiding (Result) import Test.HUnit import qualified System.Console.GetOpt as GetOpt import System.Exit import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.BasicTypes import Ganeti.Common import Ganeti.HTools.Program.Main (personalities) {-# ANN module "HLint: ignore Use camelCase" #-} -- | Helper to check for correct parsing of an option. checkOpt :: (StandardOptions b) => (a -> Maybe String) -- ^ Converts the value into a cmdline form -> b -- ^ The default options -> (String -> c) -- ^ Fail test function -> (String -> d -> d -> c) -- ^ Check for equality function -> (a -> d) -- ^ Transforms the value to a compare val -> (a, GenericOptType b, b -> d) -- ^ Triple of value, the -- option, function to -- extract the set value -- from the options -> c checkOpt repr defaults failfn eqcheck valfn (val, opt@(GetOpt.Option _ longs _ _, _), fn) = case longs of [] -> failfn "no long options?" cmdarg:_ -> case parseOptsInner defaults ["--" ++ cmdarg ++ maybe "" ("=" ++) (repr val)] "prog" [opt] [] of Left e -> failfn $ "Failed to parse option '" ++ cmdarg ++ ": " ++ show e Right (options, _) -> eqcheck ("Wrong value in option " ++ cmdarg ++ "?") (valfn val) (fn options) -- | Helper to check for correct and incorrect parsing of an option. passFailOpt :: (StandardOptions b) => b -- ^ The default options -> (String -> c) -- ^ Fail test function -> c -- ^ Pass function -> (GenericOptType b, String, String) -- ^ The list of enabled options, fail value and pass value -> c passFailOpt defaults failfn passfn (opt@(GetOpt.Option _ longs _ _, _), bad, good) = let first_opt = case longs of [] -> error "no long options?" x:_ -> x prefix = "--" ++ first_opt ++ "=" good_cmd = prefix ++ good bad_cmd = prefix ++ bad in case (parseOptsInner defaults [bad_cmd] "prog" [opt] [], parseOptsInner defaults [good_cmd] "prog" [opt] []) of (Left _, Right _) -> passfn (Right _, Right _) -> failfn $ "Command line '" ++ bad_cmd ++ "' succeeded when it shouldn't" (Left _, Left _) -> failfn $ "Command line '" ++ good_cmd ++ "' failed when it shouldn't" (Right _, Left _) -> failfn $ "Command line '" ++ bad_cmd ++ "' succeeded when it shouldn't, while command line '" ++ good_cmd ++ "' failed when it shouldn't" -- | Helper to test that a given option is accepted OK with quick exit. checkEarlyExit :: (StandardOptions a) => a -> String -> [GenericOptType a] -> [ArgCompletion] -> Assertion checkEarlyExit defaults name options arguments = mapM_ (\param -> case parseOptsInner defaults [param] name options arguments of Left (code, _) -> assertEqual ("Program " ++ name ++ " returns invalid code " ++ show code ++ " for option " ++ param) ExitSuccess code _ -> assertFailure $ "Program " ++ name ++ " doesn't consider option " ++ param ++ " as early exit one" ) ["-h", "--help", "-V", "--version"] -- | Test parseYesNo. prop_parse_yes_no :: Bool -> Bool -> String -> Property prop_parse_yes_no def testval val = forAll (elements [val, "yes", "no"]) $ \actual_val -> if testval then parseYesNo def Nothing ==? Ok def else let result = parseYesNo def (Just actual_val) in if actual_val `elem` ["yes", "no"] then result ==? Ok (actual_val == "yes") else property $ isBad result -- | Check that formatCmdUsage works similar to Python _FormatUsage. case_formatCommands :: Assertion case_formatCommands = assertEqual "proper wrap for HTools Main" resCmdTest (formatCommands personalities) where resCmdTest :: [String] resCmdTest = [ " hail - Ganeti IAllocator plugin that implements the instance\ \ placement and" , " movement using the same algorithm as hbal(1)" , " harep - auto-repair tool that detects certain kind of problems\ \ with" , " instances and applies the allowed set of solutions" , " hbal - cluster balancer that looks at the current state of\ \ the cluster and" , " computes a series of steps designed to bring the\ \ cluster into a" , " better state" , " hcheck - cluster checker; prints information about cluster's\ \ health and" , " checks whether a rebalance done using hbal would help" , " hinfo - cluster information printer; it prints information\ \ about the current" , " cluster state and its residing nodes/instances" , " hroller - cluster rolling maintenance helper; it helps\ \ scheduling node reboots" , " in a manner that doesn't conflict with the instances'\ \ topology" , " hscan - tool for scanning clusters via RAPI and saving their\ \ data in the" , " input format used by hbal(1) and hspace(1)" , " hspace - computes how many additional instances can be fit on a\ \ cluster," , " while maintaining N+1 status." , " hsqueeze - cluster dynamic power management; it powers up and\ \ down nodes to" , " keep the amount of free online resources in a given\ \ range" ] testSuite "Common" [ 'prop_parse_yes_no , 'case_formatCommands ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Confd/000075500000000000000000000000001476477700300203545ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Confd/Types.hs000064400000000000000000000074371476477700300220270ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Confd.Types ( testConfd_Types , ConfdRequestType(..) , ConfdReqField(..) , ConfdReqQ(..) ) where import Test.QuickCheck import Test.HUnit import qualified Text.JSON as J import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.Confd.Types as Confd {-# ANN module "HLint: ignore Use camelCase" #-} -- * Arbitrary instances $(genArbitrary ''ConfdRequestType) $(genArbitrary ''ConfdReqField) $(genArbitrary ''ConfdReqQ) instance Arbitrary ConfdQuery where arbitrary = oneof [ pure EmptyQuery , PlainQuery <$> genName , DictQuery <$> arbitrary ] $(genArbitrary ''ConfdRequest) $(genArbitrary ''ConfdReplyStatus) instance Arbitrary ConfdReply where arbitrary = ConfdReply <$> arbitrary <*> arbitrary <*> pure J.JSNull <*> arbitrary $(genArbitrary ''ConfdErrorType) $(genArbitrary ''ConfdNodeRole) -- * Test cases -- | Test 'ConfdQuery' serialisation. prop_ConfdQuery_serialisation :: ConfdQuery -> Property prop_ConfdQuery_serialisation = testSerialisation -- | Test bad types deserialisation for 'ConfdQuery'. case_ConfdQuery_BadTypes :: Assertion case_ConfdQuery_BadTypes = do let helper jsval = case J.readJSON jsval of J.Error _ -> return () J.Ok cq -> assertFailure $ "Parsed " ++ show jsval ++ " as query " ++ show (cq::ConfdQuery) helper $ J.showJSON (1::Int) helper $ J.JSBool True helper $ J.JSBool False helper $ J.JSArray [] -- | Test 'ConfdReplyStatus' serialisation. prop_ConfdReplyStatus_serialisation :: ConfdReplyStatus -> Property prop_ConfdReplyStatus_serialisation = testSerialisation -- | Test 'ConfdReply' serialisation. prop_ConfdReply_serialisation :: ConfdReply -> Property prop_ConfdReply_serialisation = testSerialisation -- | Test 'ConfdErrorType' serialisation. prop_ConfdErrorType_serialisation :: ConfdErrorType -> Property prop_ConfdErrorType_serialisation = testSerialisation -- | Test 'ConfdNodeRole' serialisation. prop_ConfdNodeRole_serialisation :: ConfdNodeRole -> Property prop_ConfdNodeRole_serialisation = testSerialisation testSuite "Confd/Types" [ 'prop_ConfdQuery_serialisation , 'case_ConfdQuery_BadTypes , 'prop_ConfdReplyStatus_serialisation , 'prop_ConfdReply_serialisation , 'prop_ConfdErrorType_serialisation , 'prop_ConfdNodeRole_serialisation ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Confd/Utils.hs000064400000000000000000000115331476477700300220130ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Confd.Utils (testConfd_Utils) where import Test.QuickCheck import qualified Text.JSON as J import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Confd.Types () import qualified Ganeti.BasicTypes as BasicTypes import qualified Ganeti.Confd.Types as Confd import qualified Ganeti.Confd.Utils as Confd.Utils import qualified Ganeti.Constants as C import qualified Ganeti.Hash as Hash -- | Test that signing messages and checking signatures is correct. It -- also tests, indirectly the serialisation of messages so we don't -- need a separate test for that. prop_req_sign :: Hash.HashKey -- ^ The hash key -> NonNegative Integer -- ^ The base timestamp -> Positive Integer -- ^ Delta for out of window -> Bool -- ^ Whether delta should be + or - -> Confd.ConfdRequest -> Property prop_req_sign key (NonNegative timestamp) (Positive bad_delta) pm crq = forAll (choose (0, fromIntegral C.confdMaxClockSkew)) $ \ good_delta -> let encoded = J.encode crq salt = show timestamp signed = J.encode $ Confd.Utils.signMessage key salt encoded good_timestamp = timestamp + if pm then good_delta else (-good_delta) bad_delta' = fromIntegral C.confdMaxClockSkew + bad_delta bad_timestamp = timestamp + if pm then bad_delta' else (-bad_delta') ts_ok = Confd.Utils.parseRequest key signed good_timestamp ts_bad = Confd.Utils.parseRequest key signed bad_timestamp in counterexample "Failed to parse good message" (ts_ok ==? BasicTypes.Ok (encoded, crq)) .&&. counterexample ("Managed to deserialise message with bad\ \ timestamp, got " ++ show ts_bad) (ts_bad ==? BasicTypes.Bad "Too old/too new timestamp or clock skew") -- | Tests that a ConfdReply can be properly encoded, signed and parsed using -- the proper salt, but fails parsing with the wrong salt. prop_rep_salt :: Hash.HashKey -- ^ The hash key -> Confd.ConfdReply -- ^ A Confd reply -> Property prop_rep_salt hmac reply = forAll arbitrary $ \salt1 -> forAll (arbitrary `suchThat` (/= salt1)) $ \salt2 -> let innerMsg = J.encode reply msg = J.encode $ Confd.Utils.signMessage hmac salt1 innerMsg in Confd.Utils.parseReply hmac msg salt1 ==? BasicTypes.Ok (innerMsg, reply) .&&. Confd.Utils.parseReply hmac msg salt2 ==? BasicTypes.Bad "The received salt differs from the expected salt" -- | Tests that signing with a different key fails detects failure -- correctly. prop_bad_key :: String -- ^ Salt -> Confd.ConfdRequest -- ^ Request -> Property prop_bad_key salt crq = -- fixme: we hardcode here the expected length of a sha1 key, as -- otherwise we could have two short keys that differ only in the -- final zero elements count, and those will be expanded to be the -- same forAll (vector 20) $ \key_sign -> forAll (vector 20 `suchThat` (/= key_sign)) $ \key_verify -> let signed = Confd.Utils.signMessage key_sign salt (J.encode crq) encoded = J.encode signed in counterexample ("Accepted message signed with different key" ++ encoded) $ (Confd.Utils.parseSignedMessage key_verify encoded :: BasicTypes.Result (String, String, Confd.ConfdRequest)) ==? BasicTypes.Bad "HMAC verification failed" testSuite "Confd/Utils" [ 'prop_req_sign , 'prop_rep_salt , 'prop_bad_key ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Constants.hs000064400000000000000000000067521476477700300216450ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittests for constants -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Constants (testConstants) where import Test.HUnit (Assertion) import qualified Test.HUnit as HUnit import qualified Ganeti.Constants as Constants import qualified Ganeti.ConstantUtils as ConstantUtils import qualified Test.Ganeti.TestHelper as TestHelper {-# ANN module "HLint: ignore Use camelCase" #-} case_buildVersion :: Assertion case_buildVersion = do HUnit.assertBool "Config major lower-bound violation" (Constants.configMajor >= 0) HUnit.assertBool "Config major upper-bound violation" (Constants.configMajor <= 99) HUnit.assertBool "Config minor lower-bound violation" (Constants.configMinor >= 0) HUnit.assertBool "Config minor upper-bound violation" (Constants.configMinor <= 99) HUnit.assertBool "Config revision lower-bound violation" (Constants.configRevision >= 0) HUnit.assertBool "Config revision upper-bound violation" (Constants.configRevision <= 9999) HUnit.assertBool "Config version lower-bound violation" (Constants.configVersion >= 0) HUnit.assertBool "Config version upper-bound violation" (Constants.configVersion <= 99999999) HUnit.assertEqual "Build version" (ConstantUtils.buildVersion 0 0 0) 0 HUnit.assertEqual "Build version" (ConstantUtils.buildVersion 10 10 1010) 10101010 HUnit.assertEqual "Build version" (ConstantUtils.buildVersion 12 34 5678) 12345678 HUnit.assertEqual "Build version" (ConstantUtils.buildVersion 99 99 9999) 99999999 HUnit.assertEqual "Build version" (ConstantUtils.buildVersion Constants.configMajor Constants.configMinor Constants.configRevision) Constants.configVersion HUnit.assertEqual "Build version" (ConstantUtils.buildVersion Constants.configMajor Constants.configMinor Constants.configRevision) Constants.protocolVersion TestHelper.testSuite "Constants" [ 'case_buildVersion ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Daemon.hs000064400000000000000000000060541476477700300210670ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Daemon (testDaemon) where import Test.QuickCheck hiding (Result) import Test.HUnit import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Common import Ganeti.Common import Ganeti.Daemon as Daemon {-# ANN module "HLint: ignore Use camelCase" #-} -- | Test a few string arguments. prop_string_arg :: String -> Property prop_string_arg argument = let args = [ (argument, oBindAddress, optBindAddress) ] in conjoin $ map (checkOpt Just defaultOptions failTest (const (==?)) Just) args -- | Test a few integer arguments (only one for now). prop_numeric_arg :: Int -> Property prop_numeric_arg argument = checkOpt (Just . show) defaultOptions failTest (const (==?)) (Just . fromIntegral) (argument, oPort 0, optPort) -- | Test a few boolean arguments. case_bool_arg :: Assertion case_bool_arg = mapM_ (checkOpt (const Nothing) defaultOptions assertFailure assertEqual id) [ (False, oNoDaemonize, optDaemonize) , (True, oDebug, optDebug) , (True, oNoUserChecks, optNoUserChecks) ] -- | Tests a few invalid arguments. case_wrong_arg :: Assertion case_wrong_arg = mapM_ (passFailOpt defaultOptions assertFailure (return ())) [ (oSyslogUsage, "foo", "yes") , (oPort 0, "x", "10") ] -- | Test that the option list supports some common options. case_stdopts :: Assertion case_stdopts = checkEarlyExit defaultOptions "prog" [oShowHelp, oShowVer] [] testSuite "Daemon" [ 'prop_string_arg , 'prop_numeric_arg , 'case_bool_arg , 'case_wrong_arg , 'case_stdopts ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Errors.hs000064400000000000000000000035441476477700300211410ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for "Ganeti.Errors". -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Errors (testErrors) where import Test.QuickCheck import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import qualified Ganeti.Errors as Errors $(genArbitrary ''Errors.ErrorCode) $(genArbitrary ''Errors.GanetiException) -- | Tests error serialisation. prop_GenericError_serialisation :: Errors.GanetiException -> Property prop_GenericError_serialisation = testSerialisation testSuite "Errors" [ 'prop_GenericError_serialisation ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/000075500000000000000000000000001476477700300205335ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Backend/000075500000000000000000000000001476477700300220625ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Backend/MonD.hs000064400000000000000000000114301476477700300232520ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittests for htools' ganeti-mond backend -} {- Copyright (C) 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Backend.MonD (testHTools_Backend_MonD ) where import qualified Test.HUnit as HUnit import qualified Text.JSON as J import qualified Ganeti.BasicTypes as BT import qualified Ganeti.DataCollectors.CPUload as CPUload import Ganeti.Cpu.Types (CPUavgload(..)) import Ganeti.DataCollectors.Types (DCReport(..)) import Ganeti.HTools.Backend.MonD import Ganeti.JSON import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper {-# ANN module "HLint: ignore Use camelCase" #-} -- | Test a MonD data file. case_parseMonDData :: HUnit.Assertion case_parseMonDData = do let mond_data_file = "mond-data.txt" n1 = "node1.example.com" n2 = "node2.example.com" t1 = 1379507272000000000 t2 = 1379507280000000000 cpu_number1 = 4 cpu_number2 = 2 cpus1 = [ 0.04108859597350646,0.04456554528165781 , 0.06203619909502262,0.05595448881893895] cpus2 = [0.004155409618511363,0.0034586452012150787] cpu_total1 = 0.203643517607712 cpu_total2 = 0.007614031289927129 dcr1 = DCReport CPUload.dcName CPUload.dcVersion CPUload.dcFormatVersion t1 CPUload.dcCategory CPUload.dcKind (J.showJSON (CPUavgload cpu_number1 cpus1 cpu_total1)) dcr2 = DCReport CPUload.dcName CPUload.dcVersion CPUload.dcFormatVersion t2 CPUload.dcCategory CPUload.dcKind (J.showJSON (CPUavgload cpu_number2 cpus2 cpu_total2)) expected_list = [(n1,[dcr1]),(n2,[dcr2])] ans <- readTestData mond_data_file case pMonDData ans of BT.Ok l -> HUnit.assertBool ("Parsing " ++ mond_data_file ++ " failed") (isAlEqual expected_list l) BT.Bad s -> HUnit.assertFailure $ "Parsing failed: " ++ s -- | Check for quality two list of tuples. isAlEqual :: [(String, [DCReport])] -> [(String, [DCReport])] -> Bool isAlEqual a b = and (zipWith tupleIsAlEqual a b) -- | Check a tuple for quality. tupleIsAlEqual :: (String, [DCReport]) -> (String, [DCReport]) -> Bool tupleIsAlEqual (na, a) (nb, b) = na == nb && and (zipWith dcReportIsAlmostEqual a b) -- | Check if two DCReports are equal. Only reports from CPUload Data -- Collectors are supported. dcReportIsAlmostEqual :: DCReport -> DCReport -> Bool dcReportIsAlmostEqual a b = dcReportName a == dcReportName b && dcReportVersion a == dcReportVersion b && dcReportFormatVersion a == dcReportFormatVersion b && dcReportTimestamp a == dcReportTimestamp b && dcReportCategory a == dcReportCategory b && dcReportKind a == dcReportKind b && case () of _ | CPUload.dcName == dcReportName a -> cpuavgloadDataIsAlmostEq (dcReportData a) (dcReportData b) | otherwise -> False -- | Converts two JSValue objects and compares them. cpuavgloadDataIsAlmostEq :: J.JSValue -> J.JSValue -> Bool cpuavgloadDataIsAlmostEq a b = case fromJVal a :: BT.Result CPUavgload of BT.Bad _ -> False BT.Ok cavA -> case fromJVal b :: BT.Result CPUavgload of BT.Bad _ -> False BT.Ok cavB -> compareCPUavgload cavA cavB -- | Compares two CPuavgload objects. compareCPUavgload :: CPUavgload -> CPUavgload -> Bool compareCPUavgload a b = let relError x y = relativeError x y <= 1e-9 in cavCpuNumber a == cavCpuNumber b && relError (cavCpuTotal a) (cavCpuTotal b) && length (cavCpus a) == length (cavCpus b) && and (zipWith relError (cavCpus a) (cavCpus b)) testSuite "HTools/Backend/MonD" [ 'case_parseMonDData ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Backend/Simu.hs000064400000000000000000000103241476477700300233330ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Backend.Simu (testHTools_Backend_Simu) where import Test.QuickCheck hiding (Result) import Control.Monad import qualified Data.IntMap as IntMap import Text.Printf (printf) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.BasicTypes import qualified Ganeti.Constants as C import qualified Ganeti.HTools.Backend.Simu as Simu import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Loader as Loader import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Types as Types -- | Generates a tuple of specs for simulation. genSimuSpec :: Gen (String, Int, Int, Int, Int) genSimuSpec = do pol <- elements [C.allocPolicyPreferred, C.allocPolicyLastResort, C.allocPolicyUnallocable, "p", "a", "u"] -- should be reasonable (nodes/group), bigger values only complicate -- the display of failed tests, and we don't care (in this particular -- test) about big node groups nodes <- choose (0, 20) dsk <- choose (0, maxDsk) mem <- choose (0, maxMem) cpu <- choose (0, maxCpu) return (pol, nodes, dsk, mem, cpu) -- | Checks that given a set of corrects specs, we can load them -- successfully, and that at high-level the values look right. prop_Load :: Property prop_Load = forAll (choose (0, 10)) $ \ngroups -> forAll (replicateM ngroups genSimuSpec) $ \specs -> let strspecs = map (\(p, n, d, m, c) -> printf "%s,%d,%d,%d,%d" p n d m c::String) specs totnodes = sum $ map (\(_, n, _, _, _) -> n) specs mdc_in = concatMap (\(_, n, d, m, c) -> replicate n (fromIntegral m, fromIntegral d, fromIntegral c, fromIntegral m, fromIntegral d)) specs :: [(Double, Double, Double, Int, Int)] in case Simu.parseData strspecs of Bad msg -> failTest $ "Failed to load specs: " ++ msg Ok (Loader.ClusterData gl nl il tags ipol) -> let nodes = map snd $ IntMap.toAscList nl nidx = map Node.idx nodes mdc_out = map (\n -> (Node.tMem n, Node.tDsk n, Node.tCpu n, Node.fMem n, Node.fDsk n)) nodes in conjoin [ Container.size gl ==? ngroups , Container.size nl ==? totnodes , Container.size il ==? 0 , length tags ==? 0 , ipol ==? Types.defIPolicy , nidx ==? [1..totnodes] , mdc_in ==? mdc_out , map Group.iPolicy (Container.elems gl) ==? replicate ngroups Types.defIPolicy ] testSuite "HTools/Backend/Simu" [ 'prop_Load ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Backend/Text.hs000064400000000000000000000321001476477700300233360ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Backend.Text (testHTools_Backend_Text) where import Test.QuickCheck import qualified Data.Map as Map import Data.List import Data.Maybe import System.Time (ClockTime(..)) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.TestHTools import Test.Ganeti.HTools.Instance (genInstanceSmallerThanNode, genInstanceOnNodeList) import Test.Ganeti.HTools.Node (genNode, genOnlineNode, genEmptyOnlineNode , genUniqueNodeList) import Ganeti.BasicTypes import Ganeti.Types (InstanceStatus(..)) import qualified Ganeti.HTools.AlgorithmParams as Alg import qualified Ganeti.HTools.Backend.Text as Text import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Loader as Loader import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Types as Types import qualified Ganeti.Utils as Utils -- * Instance text loader tests toYN :: Bool -> String toYN True = "Y" toYN False = "N" prop_Load_Instance :: String -> Int -> Int -> Int -> Types.InstanceStatus -> NonEmptyList Char -> String -> NonNegative Int -> NonNegative Int -> Bool -> Types.DiskTemplate -> Int -> NonNegative Int -> Bool -> [String] -> Property prop_Load_Instance name mem dsk vcpus status (NonEmpty pnode) snode (NonNegative pdx) (NonNegative sdx) autobal dt su (NonNegative spindles) forth ignoredFields = pnode /= snode && pdx /= sdx ==> let vcpus_s = show vcpus dsk_s = show dsk mem_s = show mem su_s = show su status_s = Types.instanceStatusToRaw status ndx = if null snode then [(pnode, pdx)] else [(pnode, pdx), (snode, sdx)] nl = Map.fromList ndx tags = "" sbal = toYN autobal sdt = Types.diskTemplateToRaw dt inst = Text.loadInst nl $ [name, mem_s, dsk_s, vcpus_s, status_s, sbal, pnode, snode, sdt, tags, su_s, show spindles, toYN forth] ++ ignoredFields fail1 = Text.loadInst nl [name, mem_s, dsk_s, vcpus_s, status_s, sbal, pnode, pnode, tags] in case inst of Bad msg -> failTest $ "Failed to load instance: " ++ msg Ok (_, i) -> counterexample "Mismatch in some field while\ \ loading the instance" $ Instance.name i == name && Instance.vcpus i == vcpus && Instance.mem i == mem && Instance.pNode i == pdx && Instance.sNode i == (if null snode then Node.noSecondary else sdx) && Instance.autoBalance i == autobal && Instance.spindleUse i == su && Instance.getTotalSpindles i == Just spindles && Instance.forthcoming i == forth && isBad fail1 prop_Load_InstanceFail :: [(String, Int)] -> [String] -> Property prop_Load_InstanceFail ktn fields = length fields < 10 ==> case Text.loadInst nl fields of Ok _ -> failTest "Managed to load instance from invalid data" Bad msg -> counterexample ("Unrecognised error message: " ++ msg) $ "Invalid/incomplete instance data: '" `isPrefixOf` msg where nl = Map.fromList ktn genInstanceNodes :: Gen (Instance.Instance, Node.List, Types.NameAssoc) genInstanceNodes = do (nl, na) <- genUniqueNodeList genOnlineNode inst <- genInstanceOnNodeList nl return (inst, nl, na) prop_InstanceLSIdempotent :: Property prop_InstanceLSIdempotent = forAll genInstanceNodes $ \(inst, nl, assoc) -> (Text.loadInst assoc . Utils.sepSplit '|' . Text.serializeInstance nl) inst ==? Ok (Instance.name inst, inst) prop_Load_Node :: String -> Int -> Int -> Int -> Int -> Int -> Int -> Bool -> Bool prop_Load_Node name tm nm fm td fd tc fo = let conv v = if v < 0 then "?" else show v tm_s = conv tm nm_s = conv nm fm_s = conv fm td_s = conv td fd_s = conv fd tc_s = conv tc fo_s = toYN fo any_broken = any (< 0) [tm, nm, fm, td, fd, tc] gid = Group.uuid defGroup in case Text.loadNode defGroupAssoc [name, tm_s, nm_s, fm_s, td_s, fd_s, tc_s, fo_s, gid] of Nothing -> False Just (name', node) -> if fo || any_broken then Node.offline node else Node.name node == name' && name' == name && Node.alias node == name && Node.tMem node == fromIntegral tm && Node.nMem node == nm && Node.fMem node == fm && Node.tDsk node == fromIntegral td && Node.fDsk node == fd && Node.tCpu node == fromIntegral tc prop_Load_NodeFail :: [String] -> Property prop_Load_NodeFail fields = length fields < 8 ==> isNothing $ Text.loadNode Map.empty fields prop_Load_NodeSuccess :: String -> NonNegative Int -> NonNegative Int -> NonNegative Int -> NonNegative Int -> NonNegative Int -> NonNegative Int -> Bool -> NonNegative Int -> Bool -> NonNegative Int -> NonNegative Int -> NonNegative Int -> [String] -> Property prop_Load_NodeSuccess name (NonNegative tm) (NonNegative nm) (NonNegative fm) (NonNegative td) (NonNegative fd) (NonNegative tc) fo (NonNegative spindles) excl_stor (NonNegative free_spindles) (NonNegative nos_cpu) (NonNegative cpu_speed) ignoredFields = forAll genTags $ \tags -> let node' = Text.loadNode defGroupAssoc $ [ name, show tm, show nm, show fm , show td, show fd, show tc , toYN fo, Group.uuid defGroup , show spindles , intercalate "," tags , toYN excl_stor , show free_spindles , show nos_cpu , show cpu_speed ] ++ ignoredFields in case node' of Bad msg -> failTest $ "Failed to load node: " ++ msg Ok (_, node) -> conjoin [ Node.name node ==? name , Node.tMem node ==? fromIntegral tm , Node.nMem node ==? nm , Node.fMem node ==? fm , Node.tDsk node ==? fromIntegral td , Node.fDsk node ==? fd , Node.tCpu node ==? fromIntegral tc , Node.nTags node ==? tags , Node.fSpindles node ==? free_spindles , Node.nCpu node ==? nos_cpu , Node.tCpuSpeed node ==? fromIntegral cpu_speed ] prop_NodeLSIdempotent :: Property prop_NodeLSIdempotent = forAll (genNode (Just 1) Nothing) $ \node -> -- override failN1 to what loadNode returns by default -- override pMem, xMem as they are updated after loading [in updateMemStat] let n = Node.setPolicy Types.defIPolicy $ node { Node.failN1 = True, Node.offline = False, Node.pMem = 0, Node.xMem = 0 } in (Text.loadNode defGroupAssoc. Utils.sepSplit '|' . Text.serializeNode defGroupList) n ==? Just (Node.name n, n) prop_ISpecIdempotent :: Types.ISpec -> Property prop_ISpecIdempotent ispec = case Text.loadISpec "dummy" . Utils.sepSplit ',' . Text.serializeISpec $ ispec of Bad msg -> failTest $ "Failed to load ispec: " ++ msg Ok ispec' -> ispec' ==? ispec prop_MultipleMinMaxISpecsIdempotent :: [Types.MinMaxISpecs] -> Property prop_MultipleMinMaxISpecsIdempotent minmaxes = case Text.loadMultipleMinMaxISpecs "dummy" . Utils.sepSplit ';' . Text.serializeMultipleMinMaxISpecs $ minmaxes of Bad msg -> failTest $ "Failed to load min/max ispecs: " ++ msg Ok minmaxes' -> minmaxes' ==? minmaxes prop_IPolicyIdempotent :: Types.IPolicy -> Property prop_IPolicyIdempotent ipol = case Text.loadIPolicy . Utils.sepSplit '|' $ Text.serializeIPolicy owner ipol of Bad msg -> failTest $ "Failed to load ispec: " ++ msg Ok res -> res ==? (owner, ipol) where owner = "dummy" -- | This property, while being in the text tests, does more than just -- test end-to-end the serialisation and loading back workflow; it -- also tests the Loader.mergeData and the actual -- Cluster.iterateAlloc (for well-behaving w.r.t. instance -- allocations, not for the business logic). As such, it's a quite -- complex and slow test, and that's the reason we restrict it to -- small cluster sizes. prop_CreateSerialise :: Property prop_CreateSerialise = forAll genTags $ \ctags -> forAll (choose (1, 20)) $ \maxiter -> forAll (choose (2, 10)) $ \count -> forAll genEmptyOnlineNode $ \node -> forAll (genInstanceSmallerThanNode node `suchThat` -- We want to test with a working node, so don't generate a -- status that indicates a problem with the node. (\i -> Instance.runSt i `elem` [ StatusDown , StatusOffline , ErrorDown , ErrorUp , Running , UserDown ])) $ \inst -> let nl = makeSmallCluster node count reqnodes = Instance.requiredNodes $ Instance.diskTemplate inst opts = Alg.defaultOptions in case Cluster.genAllocNodes opts defGroupList nl reqnodes True >>= \allocn -> Cluster.iterateAlloc opts nl Container.empty (Just maxiter) inst allocn [] [] of Bad msg -> failTest $ "Failed to allocate: " ++ msg Ok (_, _, _, [], _) -> counterexample "Failed to allocate: no allocations" False Ok (_, nl', il, _, _) -> let -- makeSmallCluster created an empty cluster, that had some -- instances allocated, so we need to simulate that the hyperwisor -- now reports less fMem, otherwise Loader.checkData will detect -- missing memory after deserialization. nl1 = Container.map (\n -> n { Node.fMem = Node.recordedFreeMem n }) nl' cdata = Loader.ClusterData defGroupList nl1 il ctags Types.defIPolicy saved = Text.serializeCluster cdata in case Text.parseData saved >>= Loader.mergeData [] [] [] [] (TOD 0 0) of Bad msg -> failTest $ "Failed to load/merge: " ++ msg Ok (Loader.ClusterData gl2 nl2 il2 ctags2 cpol2) -> let (_, nl3) = Loader.updateMissing nl2 il2 0 in conjoin [ ctags2 ==? ctags , cpol2 ==? Types.defIPolicy , il2 ==? il , gl2 ==? defGroupList , nl3 ==? nl1 ] testSuite "HTools/Backend/Text" [ 'prop_Load_Instance , 'prop_Load_InstanceFail , 'prop_InstanceLSIdempotent , 'prop_Load_Node , 'prop_Load_NodeFail , 'prop_Load_NodeSuccess , 'prop_NodeLSIdempotent , 'prop_ISpecIdempotent , 'prop_MultipleMinMaxISpecsIdempotent , 'prop_IPolicyIdempotent , 'prop_CreateSerialise ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/CLI.hs000064400000000000000000000122101476477700300214720ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.CLI (testHTools_CLI) where import Test.HUnit import Test.QuickCheck import Control.Monad import Data.List import Text.Printf (printf) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Common import Ganeti.BasicTypes import Ganeti.HTools.CLI as CLI import qualified Ganeti.HTools.Program.Main as Program import qualified Ganeti.HTools.Types as Types {-# ANN module "HLint: ignore Use camelCase" #-} -- | Test correct parsing. prop_parseISpec :: String -> Int -> Int -> Int -> Maybe Int -> Property prop_parseISpec descr dsk mem cpu spn = let (str, spn') = case spn of Nothing -> (printf "%d,%d,%d" dsk mem cpu::String, 1) Just spn'' -> (printf "%d,%d,%d,%d" dsk mem cpu spn''::String, spn'') in parseISpecString descr str ==? Ok (Types.RSpec cpu mem dsk spn') -- | Test parsing failure due to wrong section count. prop_parseISpecFail :: String -> Property prop_parseISpecFail descr = forAll (choose (0,100) `suchThat` (not . flip elem [3, 4])) $ \nelems -> forAll (replicateM nelems arbitrary) $ \values -> let str = intercalate "," $ map show (values::[Int]) in case parseISpecString descr str of Ok v -> failTest $ "Expected failure, got " ++ show v _ -> passTest -- | Test a few string arguments. prop_string_arg :: String -> Property prop_string_arg argument = let args = [ (oDataFile, optDataFile) , (oDynuFile, optDynuFile) , (oSaveCluster, optSaveCluster) , (oPrintCommands, optShowCmds) , (genOLuxiSocket "", optLuxi) , (oIAllocSrc, optIAllocSrc) ] in conjoin $ map (\(o, opt) -> checkOpt Just defaultOptions failTest (const (==?)) Just (argument, o, opt)) args -- | Test a few positive arguments. prop_numeric_arg :: Positive Double -> Property prop_numeric_arg (Positive argument) = let args = [ (oMaxCpu, optMcpu) , (oMinDisk, Just . optMdsk) , (oMinGain, Just . optMinGain) , (oMinGainLim, Just . optMinGainLim) , (oMinScore, Just . optMinScore) ] in conjoin $ map (\(x, y) -> checkOpt (Just . show) defaultOptions failTest (const (==?)) Just (argument, x, y)) args -- | Test a few boolean arguments. case_bool_arg :: Assertion case_bool_arg = mapM_ (checkOpt (const Nothing) defaultOptions assertFailure assertEqual id) [ (False, oDiskMoves, optDiskMoves) , (False, oInstMoves, optInstMoves) , (True, oEvacMode, optEvacMode) , (True, oExecJobs, optExecJobs) , (True, oNoHeaders, optNoHeaders) , (True, oNoSimulation, optNoSimulation) ] -- | Tests a few invalid arguments. case_wrong_arg :: Assertion case_wrong_arg = mapM_ (passFailOpt defaultOptions assertFailure (return ())) [ (oSpindleUse, "-1", "1") , (oSpindleUse, "a", "1") , (oMaxCpu, "-1", "1") , (oMinDisk, "a", "1") , (oMinGainLim, "a", "1") , (oMaxSolLength, "x", "10") , (oStdSpec, "no-such-spec", "1,1,1") , (oTieredSpec, "no-such-spec", "1,1,1") ] -- | Test that all binaries support some common options. case_stdopts :: Assertion case_stdopts = mapM_ (\(name, (_, o, a, _)) -> do o' <- o checkEarlyExit defaultOptions name (o' ++ genericOpts) a) Program.personalities testSuite "HTools/CLI" [ 'prop_parseISpec , 'prop_parseISpecFail , 'prop_string_arg , 'prop_numeric_arg , 'case_bool_arg , 'case_wrong_arg , 'case_stdopts ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Cluster.hs000064400000000000000000000442201476477700300225120ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Cluster (testHTools_Cluster) where import Test.QuickCheck hiding (Result) import Control.Monad (liftM) import qualified Data.IntMap as IntMap import Data.Maybe import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.TestHTools import Test.Ganeti.HTools.Instance ( genInstanceSmallerThanNode , genInstanceMaybeBiggerThanNode ) import Test.Ganeti.HTools.Node (genOnlineNode, genNode) import Ganeti.BasicTypes import qualified Ganeti.HTools.AlgorithmParams as Alg import qualified Ganeti.HTools.Backend.IAlloc as IAlloc import qualified Ganeti.HTools.Cluster as Cluster import qualified Ganeti.HTools.Cluster.AllocationSolution as AllocSol import qualified Ganeti.HTools.Cluster.Evacuate as Evacuate import qualified Ganeti.HTools.Cluster.Metrics as Metrics import qualified Ganeti.HTools.Cluster.Utils as ClusterUtils import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Types as Types import qualified Ganeti.Types as Types (EvacMode(..)) {-# ANN module "HLint: ignore Use camelCase" #-} -- * Helpers -- | Make a small cluster, both nodes and instances. makeSmallEmptyCluster :: Node.Node -> Int -> Instance.Instance -> (Node.List, Instance.List, Instance.Instance) makeSmallEmptyCluster node count inst = (makeSmallCluster node count, Container.empty, setInstanceSmallerThanNode node inst) -- | Checks if a node is "big" enough. isNodeBig :: Int -> Node.Node -> Bool isNodeBig size node = Node.availDisk node > size * Types.unitDsk && Node.availMem node > size * Types.unitMem && Node.availCpu node > size * Types.unitCpu canBalance :: Cluster.Table -> Bool -> Bool -> Bool -> Bool canBalance tbl@(Cluster.Table _ _ ini_cv _) dm im evac = maybe False (\(Cluster.Table _ _ fin_cv _) -> ini_cv - fin_cv > 1e-12) $ Cluster.tryBalance (Alg.defaultOptions { Alg.algMinGain = 0.0 , Alg.algMinGainLimit = 0.0 , Alg.algDiskMoves = dm , Alg.algInstanceMoves = im , Alg.algEvacMode = evac}) tbl -- | Assigns a new fresh instance to a cluster; this is not -- allocation, so no resource checks are done. assignInstance :: Node.List -> Instance.List -> Instance.Instance -> Types.Idx -> Types.Idx -> (Node.List, Instance.List) assignInstance nl il inst pdx sdx = let pnode = Container.find pdx nl snode = Container.find sdx nl maxiidx = if Container.null il then 0 else fst (Container.findMax il) + 1 inst' = inst { Instance.idx = maxiidx, Instance.pNode = pdx, Instance.sNode = sdx } pnode' = Node.setPri pnode inst' snode' = Node.setSec snode inst' nl' = Container.addTwo pdx pnode' sdx snode' nl il' = Container.add maxiidx inst' il in (nl', il') -- | Checks if an instance is mirrored. isMirrored :: Instance.Instance -> Bool isMirrored = (/= Types.MirrorNone) . Instance.mirrorType -- | Returns the possible change node types for a disk template. evacModeOptions :: Types.MirrorType -> [Types.EvacMode] evacModeOptions Types.MirrorNone = [] evacModeOptions Types.MirrorInternal = [minBound..maxBound] -- DRBD can do all evacModeOptions Types.MirrorExternal = [Types.ChangePrimary, Types.ChangeAll] -- * Test cases -- | Check that the cluster score is close to zero for a homogeneous -- cluster. prop_Score_Zero :: Node.Node -> Property prop_Score_Zero node = forAll (choose (1, 1024)) $ \count -> (not (Node.offline node) && not (Node.failN1 node) && (count > 0) && (Node.tDsk node > 0) && (Node.tMem node > 0) && (Node.tSpindles node > 0) && (Node.tCpu node > 0)) ==> let fn = Node.buildPeers node Container.empty nlst = replicate count fn score = Metrics.compCVNodes nlst -- we can't say == 0 here as the floating point errors accumulate; -- this should be much lower than the default score in CLI.hs in score <= 1e-12 -- | Check that cluster stats are sane. prop_CStats_sane :: Property prop_CStats_sane = forAll (choose (1, 1024)) $ \count -> forAll genOnlineNode $ \node -> let fn = Node.buildPeers node Container.empty nlst = zip [1..] $ replicate count fn::[(Types.Ndx, Node.Node)] nl = Container.fromList nlst cstats = Cluster.totalResources nl in Cluster.csAdsk cstats >= 0 && Cluster.csAdsk cstats <= Cluster.csFdsk cstats -- | Check that one instance is allocated correctly on an empty cluster, -- without rebalances needed. prop_Alloc_sane :: Instance.Instance -> Property prop_Alloc_sane inst = forAll (choose (5, 20)) $ \count -> forAll genOnlineNode $ \node -> let (nl, il, inst') = makeSmallEmptyCluster node count inst reqnodes = Instance.requiredNodes $ Instance.diskTemplate inst opts = Alg.defaultOptions in case Cluster.genAllocNodes Alg.defaultOptions defGroupList nl reqnodes True >>= Cluster.tryAlloc opts nl il inst' of Bad msg -> failTest msg Ok as -> case AllocSol.asSolution as of Nothing -> failTest "Failed to allocate, empty solution" Just (xnl, xi, _, cv) -> let il' = Container.add (Instance.idx xi) xi il tbl = Cluster.Table xnl il' cv [] in counterexample "Cluster can be balanced after allocation" (not (canBalance tbl True True False)) .&&. counterexample "Solution score differs from actual node list" (abs (Metrics.compCV xnl - cv) < 1e-12) -- | Checks that on a 2-5 node cluster, we can allocate a random -- instance spec via tiered allocation (whatever the original instance -- spec), on either one or two nodes. Furthermore, we test that -- computed allocation statistics are correct. prop_CanTieredAlloc :: Property prop_CanTieredAlloc = forAll (choose (2, 5)) $ \count -> forAll (liftM (Node.setPolicy Types.defIPolicy) (genOnlineNode `suchThat` isNodeBig 5)) $ \node -> forAll (genInstanceMaybeBiggerThanNode node) $ \inst -> let nl = makeSmallCluster node count il = Container.empty rqnodes = Instance.requiredNodes $ Instance.diskTemplate inst allocnodes = Cluster.genAllocNodes Alg.defaultOptions defGroupList nl rqnodes True opts = Alg.defaultOptions in case allocnodes >>= \allocnodes' -> Cluster.tieredAlloc opts nl il (Just 5) inst allocnodes' [] [] of Bad msg -> failTest $ "Failed to tiered alloc: " ++ msg Ok (_, nl', il', ixes, cstats) -> let (ai_alloc, ai_pool, ai_unav) = Cluster.computeAllocationDelta (Cluster.totalResources nl) (Cluster.totalResources nl') all_nodes fn = sum $ map fn (Container.elems nl) all_res fn = sum $ map fn [ai_alloc, ai_pool, ai_unav] in conjoin [ counterexample "No instances allocated" $ not (null ixes) , IntMap.size il' ==? length ixes , length ixes ==? length cstats , all_res Types.allocInfoVCpus ==? all_nodes Node.hiCpu , all_res Types.allocInfoNCpus ==? all_nodes Node.tCpu , all_res Types.allocInfoMem ==? truncate (all_nodes Node.tMem) , all_res Types.allocInfoDisk ==? truncate (all_nodes Node.tDsk) ] -- | Helper function to create a cluster with the given range of nodes -- and allocate an instance on it. genClusterAlloc :: Int -> Node.Node -> Instance.Instance -> Result (Node.List, Instance.List, Instance.Instance) genClusterAlloc count node inst = let nl = makeSmallCluster node count reqnodes = Instance.requiredNodes $ Instance.diskTemplate inst opts = Alg.defaultOptions in case Cluster.genAllocNodes Alg.defaultOptions defGroupList nl reqnodes True >>= Cluster.tryAlloc opts nl Container.empty inst of Bad msg -> Bad $ "Can't allocate: " ++ msg Ok as -> case AllocSol.asSolution as of Nothing -> Bad "Empty solution?" Just (xnl, xi, _, _) -> let xil = Container.add (Instance.idx xi) xi Container.empty in Ok (xnl, xil, xi) -- | Checks that on a 4-8 node cluster, once we allocate an instance, -- we can also relocate it. prop_AllocRelocate :: Property prop_AllocRelocate = forAll (choose (4, 8)) $ \count -> forAll (genOnlineNode `suchThat` isNodeBig 4) $ \node -> forAll (genInstanceSmallerThanNode node `suchThat` isMirrored) $ \inst -> case genClusterAlloc count node inst of Bad msg -> failTest msg Ok (nl, il, inst') -> case IAlloc.processRelocate Alg.defaultOptions defGroupList nl il (Instance.idx inst) 1 [(if Instance.diskTemplate inst' == Types.DTDrbd8 then Instance.sNode else Instance.pNode) inst'] of Ok _ -> passTest Bad msg -> failTest $ "Failed to relocate: " ++ msg -- | Helper property checker for the result of a nodeEvac or -- changeGroup operation. check_EvacMode :: Group.Group -> Instance.Instance -> Result (Node.List, Instance.List, Evacuate.EvacSolution) -> Property check_EvacMode grp inst result = case result of Bad msg -> failTest $ "Couldn't evacuate/change group:" ++ msg Ok (_, _, es) -> let moved = Evacuate.esMoved es failed = Evacuate.esFailed es opcodes = not . null $ Evacuate.esOpCodes es in conjoin [ failmsg ("'failed' not empty: " ++ show failed) (null failed) , failmsg "'opcodes' is null" opcodes , case moved of [(idx', gdx, _)] -> failmsg "invalid instance moved" (idx == idx') .&&. failmsg "wrong target group" (gdx == Group.idx grp) v -> failmsg ("invalid solution: " ++ show v) False ] where failmsg :: String -> Bool -> Property failmsg msg = counterexample ("Failed to evacuate: " ++ msg) idx = Instance.idx inst -- | Checks that on a 4-8 node cluster, once we allocate an instance, -- we can also node-evacuate it. prop_AllocEvacuate :: Property prop_AllocEvacuate = forAll (choose (4, 8)) $ \count -> forAll (genOnlineNode `suchThat` isNodeBig 4) $ \node -> forAll (genInstanceSmallerThanNode node `suchThat` isMirrored) $ \inst -> case genClusterAlloc count node inst of Bad msg -> failTest msg Ok (nl, il, inst') -> conjoin . map (\mode -> check_EvacMode defGroup inst' $ Evacuate.tryNodeEvac Alg.defaultOptions defGroupList nl il mode [Instance.idx inst']) . evacModeOptions . Instance.mirrorType $ inst' -- | Checks that on a 4-8 node cluster with two node groups, once we -- allocate an instance on the first node group, we can also change -- its group. prop_AllocChangeGroup :: Property prop_AllocChangeGroup = forAll (choose (4, 8)) $ \count -> forAll (genOnlineNode `suchThat` isNodeBig 4) $ \node -> forAll (genInstanceSmallerThanNode node `suchThat` isMirrored) $ \inst -> case genClusterAlloc count node inst of Bad msg -> failTest msg Ok (nl, il, inst') -> -- we need to add a second node group and nodes to the cluster let nl2 = Container.elems $ makeSmallCluster node count grp2 = Group.setIdx defGroup (Group.idx defGroup + 1) maxndx = maximum . map Node.idx $ nl2 nl3 = map (\n -> n { Node.group = Group.idx grp2 , Node.idx = Node.idx n + maxndx }) nl2 nl4 = Container.fromList . map (\n -> (Node.idx n, n)) $ nl3 gl' = Container.add (Group.idx grp2) grp2 defGroupList nl' = IntMap.union nl nl4 opts = Alg.defaultOptions in check_EvacMode grp2 inst' $ Cluster.tryChangeGroup opts gl' nl' il [] [Instance.idx inst'] -- | Check that allocating multiple instances on a cluster, then -- adding an empty node, results in a valid rebalance. prop_AllocBalance :: Property prop_AllocBalance = forAll (genNode (Just 5) (Just 128)) $ \node -> forAll (choose (3, 5)) $ \count -> not (Node.offline node) && not (Node.failN1 node) ==> let nl = makeSmallCluster node count hnode = snd $ IntMap.findMax nl nl' = IntMap.deleteMax nl il = Container.empty allocnodes = Cluster.genAllocNodes Alg.defaultOptions defGroupList nl' 2 True i_templ = createInstance Types.unitMem Types.unitDsk Types.unitCpu opts = Alg.defaultOptions in case allocnodes >>= \allocnodes' -> Cluster.iterateAlloc opts nl' il (Just 5) i_templ allocnodes' [] [] of Bad msg -> failTest $ "Failed to allocate: " ++ msg Ok (_, _, _, [], _) -> failTest "Failed to allocate: no instances" Ok (_, xnl, il', _, _) -> let ynl = Container.add (Node.idx hnode) hnode xnl cv = Metrics.compCV ynl tbl = Cluster.Table ynl il' cv [] in counterexample "Failed to rebalance" $ canBalance tbl True True False -- | Checks consistency. prop_CheckConsistency :: Node.Node -> Instance.Instance -> Bool prop_CheckConsistency node inst = let nl = makeSmallCluster node 3 (node1, node2, node3) = case Container.elems nl of [a, b, c] -> (a, b, c) l -> error $ "Invalid node list out of makeSmallCluster/3: " ++ show l node3' = node3 { Node.group = 1 } nl' = Container.add (Node.idx node3') node3' nl inst1 = Instance.setBoth inst (Node.idx node1) (Node.idx node2) inst2 = Instance.setBoth inst (Node.idx node1) Node.noSecondary inst3 = Instance.setBoth inst (Node.idx node1) (Node.idx node3) ccheck = Cluster.findSplitInstances nl' . Container.fromList in null (ccheck [(0, inst1)]) && null (ccheck [(0, inst2)]) && (not . null $ ccheck [(0, inst3)]) -- | For now, we only test that we don't lose instances during the split. prop_SplitCluster :: Node.Node -> Instance.Instance -> Property prop_SplitCluster node inst = forAll (choose (0, 100)) $ \icnt -> let nl = makeSmallCluster node 2 (nl', il') = foldl (\(ns, is) _ -> assignInstance ns is inst 0 1) (nl, Container.empty) [1..icnt] gni = ClusterUtils.splitCluster nl' il' in sum (map (Container.size . snd . snd) gni) == icnt && all (\(guuid, (nl'', _)) -> all ((== guuid) . Node.group) (Container.elems nl'')) gni -- | Helper function to check if we can allocate an instance on a -- given node list. Successful allocation is denoted by 'Nothing', -- otherwise the 'Just' value will contain the error message. canAllocOn :: Node.List -> Int -> Instance.Instance -> Maybe String canAllocOn nl reqnodes inst = case Cluster.genAllocNodes Alg.defaultOptions defGroupList nl reqnodes True >>= Cluster.tryAlloc Alg.defaultOptions nl Container.empty inst of Bad msg -> Just $ "Can't allocate: " ++ msg Ok as -> case AllocSol.asSolution as of Nothing -> Just $ "No allocation solution; failures: " ++ show (AllocSol.collapseFailures $ AllocSol.asFailures as) Just _ -> Nothing -- | Checks that allocation obeys minimum and maximum instance -- policies. The unittest generates a random node, duplicates it /count/ -- times, and generates a random instance that can be allocated on -- this mini-cluster; it then checks that after applying a policy that -- the instance doesn't fits, the allocation fails. prop_AllocPolicy :: Property prop_AllocPolicy = forAll genOnlineNode $ \node -> forAll (choose (5, 20)) $ \count -> forAll (genInstanceSmallerThanNode node) $ \inst -> forAll (arbitrary `suchThat` (isBad . flip (Instance.instMatchesPolicy inst) (Node.exclStorage node))) $ \ipol -> let rqn = Instance.requiredNodes $ Instance.diskTemplate inst node' = Node.setPolicy ipol node nl = makeSmallCluster node' count in counterexample "Allocation check:" (isNothing (canAllocOn (makeSmallCluster node count) rqn inst)) .&&. counterexample "Policy failure check:" (isJust $ canAllocOn nl rqn inst) testSuite "HTools/Cluster" [ 'prop_Score_Zero , 'prop_CStats_sane , 'prop_Alloc_sane , 'prop_CanTieredAlloc , 'prop_AllocRelocate , 'prop_AllocEvacuate , 'prop_AllocChangeGroup , 'prop_AllocBalance , 'prop_CheckConsistency , 'prop_SplitCluster , 'prop_AllocPolicy ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Container.hs000064400000000000000000000073721476477700300230220ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Container (testHTools_Container) where import Test.QuickCheck import Data.Maybe import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.TestHTools import Test.Ganeti.HTools.Node (genNode) import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Node as Node -- we silence the following due to hlint bug fixed in later versions {-# ANN prop_addTwo "HLint: ignore Avoid lambda" #-} prop_addTwo :: [Container.Key] -> Int -> Int -> Bool prop_addTwo cdata i1 i2 = fn i1 i2 cont == fn i2 i1 cont && fn i1 i2 cont == fn i1 i2 (fn i1 i2 cont) where cont = foldl (\c x -> Container.add x x c) Container.empty cdata fn x1 x2 = Container.addTwo x1 x1 x2 x2 prop_nameOf :: Node.Node -> Property prop_nameOf node = let nl = makeSmallCluster node 1 in case Container.elems nl of [] -> failTest "makeSmallCluster 1 returned empty cluster?" _:_:_ -> failTest "makeSmallCluster 1 returned >1 node?" fnode:_ -> Container.nameOf nl (Node.idx fnode) ==? Node.name fnode -- | We test that in a cluster, given a random node, we can find it by -- its name and alias, as long as all names and aliases are unique, -- and that we fail to find a non-existing name. prop_findByName :: Property prop_findByName = forAll (genNode (Just 1) Nothing) $ \node -> forAll (choose (1, 20)) $ \ cnt -> forAll (choose (0, cnt - 1)) $ \ fidx -> forAll (genUniquesList (cnt * 2) arbitrary) $ \ allnames -> forAll (arbitrary `suchThat` (`notElem` allnames)) $ \ othername -> let names = zip (take cnt allnames) (drop cnt allnames) nl = makeSmallCluster node cnt nodes = Container.elems nl nodes' = map (\((name, alias), nn) -> (Node.idx nn, nn { Node.name = name, Node.alias = alias })) $ zip names nodes nl' = Container.fromList nodes' target = snd (nodes' !! fidx) in conjoin [ Container.findByName nl' (Node.name target) ==? Just target , Container.findByName nl' (Node.alias target) ==? Just target , counterexample "Found non-existing name" (isNothing (Container.findByName nl' othername)) ] testSuite "HTools/Container" [ 'prop_addTwo , 'prop_nameOf , 'prop_findByName ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Graph.hs000064400000000000000000000170201476477700300221300ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for Ganeti.Htools.Graph -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Graph (testHTools_Graph) where import Test.QuickCheck import Test.HUnit import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.HTools.Graph import qualified Data.Graph as Graph import qualified Data.IntMap as IntMap {-# ANN module "HLint: ignore Use camelCase" #-} data TestableGraph = TestableGraph Graph.Graph deriving (Show) data TestableClique = TestableClique Graph.Graph deriving (Show) -- | Generate node bounds and edges for an undirected graph. -- A graph is undirected if for every (a, b) edge there is a -- corresponding (b, a) one. undirEdges :: Gen (Graph.Bounds, [Graph.Edge]) undirEdges = sized undirEdges' where undirEdges' 0 = return ((0, 0), []) undirEdges' n = do maxv <- choose (1, n) edges <- listOf1 $ do i <- choose (0, maxv) j <- choose (0, maxv) `suchThat` (/= i) return [(i, j), (j, i)] return ((0, maxv), concat edges) -- | Generate node bounds and edges for a clique. -- In a clique all nodes are directly connected to each other. cliqueEdges :: Gen (Graph.Bounds, [Graph.Edge]) cliqueEdges = sized cliqueEdges' where cliqueEdges' 0 = return ((0, 0), []) cliqueEdges' n = do maxv <- choose (0, n) let edges = [(x, y) | x <- [0..maxv], y <- [0..maxv], x /= y] return ((0, maxv), edges) instance Arbitrary TestableGraph where arbitrary = do (mybounds, myedges) <- undirEdges return . TestableGraph $ Graph.buildG mybounds myedges instance Arbitrary TestableClique where arbitrary = do (mybounds, myedges) <- cliqueEdges return . TestableClique $ Graph.buildG mybounds myedges -- | Check that the empty vertex color map is empty. case_emptyVertColorMapNull :: Assertion case_emptyVertColorMapNull = assertBool "" $ IntMap.null emptyVertColorMap -- | Check that the empty vertex color map is zero in size. case_emptyVertColorMapEmpty :: Assertion case_emptyVertColorMapEmpty = assertEqual "" 0 $ IntMap.size emptyVertColorMap -- | Check if each two consecutive elements on a list -- respect a given condition. anyTwo :: (a -> a -> Bool) -> [a] -> Bool anyTwo _ [] = True anyTwo _ [_] = True anyTwo op (x:y:xs) = (x `op` y) && anyTwo op (y:xs) -- | Check order of vertices returned by verticesByDegreeAsc. prop_verticesByDegreeAscAsc :: TestableGraph -> Bool prop_verticesByDegreeAscAsc (TestableGraph g) = anyTwo (<=) (degrees asc) where degrees = map (length . neighbors g) asc = verticesByDegreeAsc g -- | Check order of vertices returned by verticesByDegreeDesc. prop_verticesByDegreeDescDesc :: TestableGraph -> Bool prop_verticesByDegreeDescDesc (TestableGraph g) = anyTwo (>=) (degrees desc) where degrees = map (length . neighbors g) desc = verticesByDegreeDesc g -- | Check that our generated graphs are colorable prop_isColorableTestableGraph :: TestableGraph -> Bool prop_isColorableTestableGraph (TestableGraph g) = isColorable g -- | Check that our generated graphs are colorable prop_isColorableTestableClique :: TestableClique -> Bool prop_isColorableTestableClique (TestableClique g) = isColorable g -- | Check that the given algorithm colors a clique with the same number of -- colors as the vertices number. prop_colorClique :: (Graph.Graph -> VertColorMap) -> TestableClique -> Property prop_colorClique alg (TestableClique g) = numvertices ==? numcolors where numcolors = (IntMap.size . colorVertMap) $ alg g numvertices = length (Graph.vertices g) -- | Specific check for the LF algorithm. prop_colorLFClique :: TestableClique -> Property prop_colorLFClique = prop_colorClique colorLF -- | Specific check for the Dsatur algorithm. prop_colorDsaturClique :: TestableClique -> Property prop_colorDsaturClique = prop_colorClique colorDsatur -- | Specific check for the Dcolor algorithm. prop_colorDcolorClique :: TestableClique -> Property prop_colorDcolorClique = prop_colorClique colorDcolor -- Check that all nodes are colored. prop_colorAllNodes :: (Graph.Graph -> VertColorMap) -> TestableGraph -> Property prop_colorAllNodes alg (TestableGraph g) = numvertices ==? numcolored where numcolored = IntMap.foldr ((+) . length) 0 vcMap vcMap = colorVertMap $ alg g numvertices = length (Graph.vertices g) -- | Specific check for the LF algorithm. prop_colorLFAllNodes :: TestableGraph -> Property prop_colorLFAllNodes = prop_colorAllNodes colorLF -- | Specific check for the Dsatur algorithm. prop_colorDsaturAllNodes :: TestableGraph -> Property prop_colorDsaturAllNodes = prop_colorAllNodes colorDsatur -- | Specific check for the Dcolor algorithm. prop_colorDcolorAllNodes :: TestableGraph -> Property prop_colorDcolorAllNodes = prop_colorAllNodes colorDcolor -- | Check that no two vertices sharing the same edge have the same color. prop_colorProper :: (Graph.Graph -> VertColorMap) -> TestableGraph -> Bool prop_colorProper alg (TestableGraph g) = all isEdgeOk $ Graph.edges g where isEdgeOk :: Graph.Edge -> Bool isEdgeOk (v1, v2) = color v1 /= color v2 color v = cMap IntMap.! v cMap = alg g -- | Specific check for the LF algorithm. prop_colorLFProper :: TestableGraph -> Bool prop_colorLFProper = prop_colorProper colorLF -- | Specific check for the Dsatur algorithm. prop_colorDsaturProper :: TestableGraph -> Bool prop_colorDsaturProper = prop_colorProper colorDsatur -- | Specific check for the Dcolor algorithm. prop_colorDcolorProper :: TestableGraph -> Bool prop_colorDcolorProper = prop_colorProper colorDcolor -- | List of tests for the Graph module. testSuite "HTools/Graph" [ 'case_emptyVertColorMapNull , 'case_emptyVertColorMapEmpty , 'prop_verticesByDegreeAscAsc , 'prop_verticesByDegreeDescDesc , 'prop_colorLFClique , 'prop_colorDsaturClique , 'prop_colorDcolorClique , 'prop_colorLFAllNodes , 'prop_colorDsaturAllNodes , 'prop_colorDcolorAllNodes , 'prop_colorLFProper , 'prop_colorDsaturProper , 'prop_colorDcolorProper , 'prop_isColorableTestableGraph , 'prop_isColorableTestableClique ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Instance.hs000064400000000000000000000220421476477700300226330ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Instance ( testHTools_Instance , genInstanceSmallerThanNode , genInstanceMaybeBiggerThanNode , genInstanceOnNodeList , genInstanceList , Instance.Instance(..) ) where import Control.Arrow ((&&&)) import Control.Monad (liftM) import Test.QuickCheck hiding (Result) import Test.Ganeti.TestHTools (nullISpec) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.HTools.Types () import Ganeti.BasicTypes import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Loader as Loader import qualified Ganeti.HTools.Types as Types -- * Arbitrary instances -- | Generates a random instance with maximum and minimum disk/mem/cpu values. genInstanceWithin :: Int -> Int -> Int -> Int -> Int -> Int -> Int -> Maybe Int -> Gen Instance.Instance genInstanceWithin min_mem min_dsk min_cpu min_spin max_mem max_dsk max_cpu max_spin = do name <- genFQDN mem <- choose (min_mem, max_mem) dsk <- choose (min_dsk, max_dsk) run_st <- arbitrary pn <- arbitrary sn <- arbitrary vcpus <- choose (min_cpu, max_cpu) dt <- arbitrary spindles <- case max_spin of Nothing -> genMaybe $ choose (min_spin, maxSpindles) Just ls -> liftM Just $ choose (min_spin, ls) forthcoming <- arbitrary let disk = Instance.Disk dsk spindles return $ Instance.create name mem dsk [disk] vcpus run_st [] True pn sn dt 1 [] forthcoming -- | Generate an instance with maximum disk/mem/cpu values. genInstanceSmallerThan :: Int -> Int -> Int -> Maybe Int -> Gen Instance.Instance genInstanceSmallerThan = genInstanceWithin 1 0 1 0 -- | Generates an instance smaller than a node. genInstanceSmallerThanNode :: Node.Node -> Gen Instance.Instance genInstanceSmallerThanNode node = genInstanceSmallerThan (Node.availMem node `div` 2) (Node.availDisk node `div` 2) (Node.availCpu node `div` 2) (if Node.exclStorage node then Just $ Node.fSpindlesForth node `div` 2 else Nothing) -- | Generates an instance possibly bigger than a node. -- In any case, that instance will be bigger than the node's ipolicy's lower -- bound. genInstanceMaybeBiggerThanNode :: Node.Node -> Gen Instance.Instance genInstanceMaybeBiggerThanNode node = let minISpec = runListHead nullISpec Types.minMaxISpecsMinSpec . Types.iPolicyMinMaxISpecs $ Node.iPolicy node in genInstanceWithin (Types.iSpecMemorySize minISpec) (Types.iSpecDiskSize minISpec) (Types.iSpecCpuCount minISpec) (Types.iSpecSpindleUse minISpec) (Node.availMem node + Types.unitMem * 2) (Node.availDisk node + Types.unitDsk * 3) (Node.availCpu node + Types.unitCpu * 4) (if Node.exclStorage node then Just $ Node.fSpindles node + Types.unitSpindle * 5 else Nothing) -- | Generates an instance with nodes on a node list. -- The following rules are respected: -- 1. The instance is never bigger than its primary node -- 2. If possible the instance has different pnode and snode -- 3. Else disk templates which require secondary nodes are disabled genInstanceOnNodeList :: Node.List -> Gen Instance.Instance genInstanceOnNodeList nl = do let nsize = Container.size nl pnode <- choose (0, nsize-1) let (snodefilter, dtfilter) = if nsize >= 2 then ((/= pnode), const True) else (const True, not . Instance.hasSecondary) snode <- choose (0, nsize-1) `suchThat` snodefilter i <- genInstanceSmallerThanNode (Container.find pnode nl) `suchThat` dtfilter return $ i { Instance.pNode = pnode, Instance.sNode = snode } -- | Generates an instance list given an instance generator. genInstanceList :: Gen Instance.Instance -> Gen Instance.List genInstanceList igen = fmap (snd . Loader.assignIndices) names_instances where names_instances = map (Instance.name &&& id) <$> listOf igen -- let's generate a random instance instance Arbitrary Instance.Instance where arbitrary = genInstanceSmallerThan maxMem maxDsk maxCpu Nothing -- * Test cases -- Simple instance tests, we only have setter/getters prop_creat :: Instance.Instance -> Property prop_creat inst = Instance.name inst ==? Instance.alias inst prop_setIdx :: Instance.Instance -> Types.Idx -> Property prop_setIdx inst idx = Instance.idx (Instance.setIdx inst idx) ==? idx prop_setName :: Instance.Instance -> String -> Bool prop_setName inst name = Instance.name newinst == name && Instance.alias newinst == name where newinst = Instance.setName inst name prop_setAlias :: Instance.Instance -> String -> Bool prop_setAlias inst name = Instance.name newinst == Instance.name inst && Instance.alias newinst == name where newinst = Instance.setAlias inst name prop_setPri :: Instance.Instance -> Types.Ndx -> Property prop_setPri inst pdx = Instance.pNode (Instance.setPri inst pdx) ==? pdx prop_setSec :: Instance.Instance -> Types.Ndx -> Property prop_setSec inst sdx = Instance.sNode (Instance.setSec inst sdx) ==? sdx prop_setBoth :: Instance.Instance -> Types.Ndx -> Types.Ndx -> Bool prop_setBoth inst pdx sdx = Instance.pNode si == pdx && Instance.sNode si == sdx where si = Instance.setBoth inst pdx sdx prop_shrinkMG :: Instance.Instance -> Property prop_shrinkMG inst = Instance.mem inst >= 2 * Types.unitMem ==> case Instance.shrinkByType inst Types.FailMem of Ok inst' -> Instance.mem inst' ==? Instance.mem inst - Types.unitMem Bad msg -> failTest msg prop_shrinkMF :: Instance.Instance -> Property prop_shrinkMF inst = forAll (choose (0, 2 * Types.unitMem - 1)) $ \mem -> let inst' = inst { Instance.mem = mem} in isBad $ Instance.shrinkByType inst' Types.FailMem prop_shrinkCG :: Instance.Instance -> Property prop_shrinkCG inst = Instance.vcpus inst >= 2 * Types.unitCpu ==> case Instance.shrinkByType inst Types.FailCPU of Ok inst' -> Instance.vcpus inst' ==? Instance.vcpus inst - Types.unitCpu Bad msg -> failTest msg prop_shrinkCF :: Instance.Instance -> Property prop_shrinkCF inst = forAll (choose (0, 2 * Types.unitCpu - 1)) $ \vcpus -> let inst' = inst { Instance.vcpus = vcpus } in isBad $ Instance.shrinkByType inst' Types.FailCPU prop_shrinkDG :: Instance.Instance -> Property prop_shrinkDG inst = Instance.dsk inst >= 2 * Types.unitDsk ==> case Instance.shrinkByType inst Types.FailDisk of Ok inst' -> Instance.dsk inst' ==? Instance.dsk inst - Types.unitDsk Bad msg -> failTest msg prop_shrinkDF :: Instance.Instance -> Property prop_shrinkDF inst = forAll (choose (0, 2 * Types.unitDsk - 1)) $ \dsk -> let inst' = inst { Instance.dsk = dsk , Instance.disks = [Instance.Disk dsk Nothing] } in isBad $ Instance.shrinkByType inst' Types.FailDisk prop_setMovable :: Instance.Instance -> Bool -> Property prop_setMovable inst m = Instance.movable inst' ==? m where inst' = Instance.setMovable inst m testSuite "HTools/Instance" [ 'prop_creat , 'prop_setIdx , 'prop_setName , 'prop_setAlias , 'prop_setPri , 'prop_setSec , 'prop_setBoth , 'prop_shrinkMG , 'prop_shrinkMF , 'prop_shrinkCG , 'prop_shrinkCF , 'prop_shrinkDG , 'prop_shrinkDF , 'prop_setMovable ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Loader.hs000064400000000000000000000100551476477700300222760ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Loader (testHTools_Loader) where import Test.QuickCheck import qualified Data.IntMap as IntMap import qualified Data.Map as Map import Data.List import System.Time (ClockTime(..)) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.HTools.Node () import qualified Ganeti.BasicTypes as BasicTypes import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Loader as Loader import qualified Ganeti.HTools.Node as Node prop_lookupNode :: [(String, Int)] -> String -> String -> Property prop_lookupNode ktn inst node = Loader.lookupNode nl inst node ==? Map.lookup node nl where nl = Map.fromList ktn prop_lookupInstance :: [(String, Int)] -> String -> Property prop_lookupInstance kti inst = Loader.lookupInstance il inst ==? Map.lookup inst il where il = Map.fromList kti prop_assignIndices :: Property prop_assignIndices = -- generate nodes with unique names forAll (arbitrary `suchThat` (\nodes -> let names = map Node.name nodes in length names == length (nub names))) $ \nodes -> let (nassoc, kt) = Loader.assignIndices (map (\n -> (Node.name n, n)) nodes) in Map.size nassoc == length nodes && Container.size kt == length nodes && (null nodes || maximum (IntMap.keys kt) == length nodes - 1) -- | Checks that the number of primary instances recorded on the nodes -- is zero. prop_mergeData :: [Node.Node] -> Bool prop_mergeData ns = let na = Container.fromList $ map (\n -> (Node.idx n, n)) ns in case Loader.mergeData [] [] [] [] (TOD 0 0) (Loader.emptyCluster {Loader.cdNodes = na}) of BasicTypes.Bad _ -> False BasicTypes.Ok (Loader.ClusterData _ nl il _ _) -> let nodes = Container.elems nl instances = Container.elems il in (sum . map (length . Node.pList)) nodes == 0 && null instances -- | Check that compareNameComponent on equal strings works. prop_compareNameComponent_equal :: String -> Bool prop_compareNameComponent_equal s = BasicTypes.compareNameComponent s s == BasicTypes.LookupResult BasicTypes.ExactMatch s -- | Check that compareNameComponent on prefix strings works. prop_compareNameComponent_prefix :: NonEmptyList Char -> String -> Bool prop_compareNameComponent_prefix (NonEmpty s1) s2 = BasicTypes.compareNameComponent (s1 ++ "." ++ s2) s1 == BasicTypes.LookupResult BasicTypes.PartialMatch s1 testSuite "HTools/Loader" [ 'prop_lookupNode , 'prop_lookupInstance , 'prop_assignIndices , 'prop_mergeData , 'prop_compareNameComponent_equal , 'prop_compareNameComponent_prefix ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Node.hs000064400000000000000000000461641476477700300217670ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Node ( testHTools_Node , Node.Node(..) , setInstanceSmallerThanNode , genNode , genOnlineNode , genEmptyOnlineNode , genNodeList , genUniqueNodeList ) where import Test.QuickCheck import Test.HUnit import Control.Monad import qualified Data.Map as Map import qualified Data.Graph as Graph import Data.List import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.TestHTools import Test.Ganeti.HTools.Instance ( genInstanceSmallerThanNode , genInstanceList , genInstanceOnNodeList) import Ganeti.BasicTypes import qualified Ganeti.HTools.Loader as Loader import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Types as Types import qualified Ganeti.HTools.Graph as HGraph {-# ANN module "HLint: ignore Use camelCase" #-} -- * Arbitrary instances -- | Generates an arbitrary node based on sizing information. genNode :: Maybe Int -- ^ Minimum node size in terms of units -> Maybe Int -- ^ Maximum node size (when Nothing, bounded -- just by the max... constants) -> Gen Node.Node genNode min_multiplier max_multiplier = do let (base_mem, base_dsk, base_cpu, base_spindles) = case min_multiplier of Just mm -> (mm * Types.unitMem, mm * Types.unitDsk, mm * Types.unitCpu, mm) Nothing -> (0, 0, 0, 0) (top_mem, top_dsk, top_cpu, top_spindles) = case max_multiplier of Just mm -> (mm * Types.unitMem, mm * Types.unitDsk, mm * Types.unitCpu, mm) Nothing -> (maxMem, maxDsk, maxCpu, maxSpindles) name <- genFQDN mem_t <- choose (base_mem, top_mem) mem_f <- choose (base_mem, mem_t) mem_n <- choose (0, mem_t - mem_f) dsk_t <- choose (base_dsk, top_dsk) dsk_f <- choose (base_dsk, dsk_t) cpu_t <- choose (base_cpu, top_cpu) cpu_n <- choose (base_cpu, cpu_t) offl <- arbitrary spindles <- choose (base_spindles, top_spindles) let n = Node.create name (fromIntegral mem_t) mem_n mem_f (fromIntegral dsk_t) dsk_f (fromIntegral cpu_t) cpu_n offl spindles 0 0 False n1 = Node.setPolicy nullIPolicy n n2 = Loader.updateMemStat n1 Container.empty return $ Node.buildPeers n2 Container.empty -- | Helper function to generate a sane node. genOnlineNode :: Gen Node.Node genOnlineNode = arbitrary `suchThat` (\n -> not (Node.offline n) && not (Node.failN1 n) && Node.availDisk n > 2 * Types.unitDsk && Node.availMem n > 2 * Types.unitMem && Node.availCpu n > 2 && Node.tSpindles n > 2) -- | Helper function to generate a sane empty node with consistent -- internal data. genEmptyOnlineNode :: Gen Node.Node genEmptyOnlineNode = (do node <- arbitrary let fmem = truncate (Node.tMem node) - Node.nMem node let node' = node { Node.offline = False , Node.fMem = fmem , Node.fMemForth = fmem , Node.pMem = fromIntegral fmem / Node.tMem node , Node.pMemForth = fromIntegral fmem / Node.tMem node , Node.rMem = 0 , Node.rMemForth = 0 , Node.pRem = 0 , Node.pRemForth = 0 , Node.xMem = 0 } return node') `suchThat` (\ n -> not (Node.failN1 n) && Node.availDisk n > 0 && Node.availMem n > 0 && Node.availCpu n > 0 && Node.tSpindles n > 0) -- | Generate a node with exclusive storage enabled. genExclStorNode :: Gen Node.Node genExclStorNode = do n <- genOnlineNode fs <- choose (Types.unitSpindle, Node.tSpindles n) fsForth <- choose (Types.unitSpindle, fs) let pd = fromIntegral fs / fromIntegral (Node.tSpindles n)::Double let pdForth = fromIntegral fsForth / fromIntegral (Node.tSpindles n)::Double return n { Node.exclStorage = True , Node.fSpindles = fs , Node.fSpindlesForth = fsForth , Node.pDsk = pd , Node.pDskForth = pdForth } -- | Generate a node with exclusive storage possibly enabled. genMaybeExclStorNode :: Gen Node.Node genMaybeExclStorNode = oneof [genOnlineNode, genExclStorNode] -- and a random node instance Arbitrary Node.Node where arbitrary = genNode Nothing Nothing -- | Node list generator. -- Given a node generator, create a random length node list. Note that "real" -- clusters always have at least one node, so we don't generate empty node -- lists here. genNodeList :: Gen Node.Node -> Gen Node.List genNodeList ngen = fmap (snd . Loader.assignIndices) names_nodes where names_nodes = (fmap . map) (\n -> (Node.name n, n)) nodes nodes = listOf1 ngen `suchThat` ((\ns -> ns == nub ns) . map Node.name) -- | Node list generator where node names are unique genUniqueNodeList :: Gen Node.Node -> Gen (Node.List, Types.NameAssoc) genUniqueNodeList ngen = (do nl <- genNodeList ngen let na = (fst . Loader.assignIndices) $ map (\n -> (Node.name n, n)) (Container.elems nl) return (nl, na)) `suchThat` (\(nl, na) -> Container.size nl == Map.size na) -- | Generate a node list, an instance list, and a node graph. -- We choose instances with nodes contained in the node list. genNodeGraph :: Gen (Maybe Graph.Graph, Node.List, Instance.List) genNodeGraph = do nl <- genNodeList genOnlineNode `suchThat` ((2<=).Container.size) il <- genInstanceList (genInstanceOnNodeList nl) return (Node.mkNodeGraph nl il, nl, il) -- * Test cases prop_setAlias :: Node.Node -> String -> Bool prop_setAlias node name = Node.name newnode == Node.name node && Node.alias newnode == name where newnode = Node.setAlias node name prop_setOffline :: Node.Node -> Bool -> Property prop_setOffline node status = Node.offline newnode ==? status where newnode = Node.setOffline node status prop_setMcpu :: Node.Node -> Double -> Property prop_setMcpu node mc = Types.iPolicyVcpuRatio (Node.iPolicy newnode) ==? mc where newnode = Node.setMcpu node mc -- Check if adding an instance that consumes exactly all reserved -- memory does not raise an N+1 error prop_addPri_NoN1Fail :: Property prop_addPri_NoN1Fail = forAll genMaybeExclStorNode $ \node -> forAll (genInstanceSmallerThanNode node) $ \inst -> let inst' = inst { Instance.mem = Node.fMem node - Node.rMem node } in (Node.addPri node inst' /=? Bad Types.FailN1) -- | Check that an instance add with too high memory or disk will be -- rejected. prop_addPriFM :: Node.Node -> Instance.Instance -> Property prop_addPriFM node inst = Instance.mem inst >= Node.fMem node && not (Node.failN1 node) && Instance.usesMemory inst ==> (Node.addPri node inst'' ==? Bad Types.FailMem) where inst' = setInstanceSmallerThanNode node inst inst'' = inst' { Instance.mem = Instance.mem inst } -- | Check that adding a primary instance with too much disk fails -- with type FailDisk. prop_addPriFD :: Instance.Instance -> Property prop_addPriFD inst = forAll (genNode (Just 1) Nothing) $ \node -> forAll (elements Instance.localStorageTemplates) $ \dt -> Instance.dsk inst >= Node.fDsk node && not (Node.failN1 node) ==> let inst' = setInstanceSmallerThanNode node inst inst'' = inst' { Instance.dsk = Instance.dsk inst , Instance.diskTemplate = dt } in (Node.addPri node inst'' ==? Bad Types.FailDisk) -- | Check if an instance exceeds a spindles limit or has no spindles set. hasInstTooManySpindles :: Instance.Instance -> Int -> Bool hasInstTooManySpindles inst sp_lim = case Instance.getTotalSpindles inst of Just s -> s > sp_lim Nothing -> True -- | Check that adding a primary instance with too many spindles fails -- with type FailSpindles (when exclusive storage is enabled). prop_addPriFS :: Instance.Instance -> Property prop_addPriFS inst = forAll genExclStorNode $ \node -> forAll (elements Instance.localStorageTemplates) $ \dt -> hasInstTooManySpindles inst (Node.fSpindles node) && not (Node.failN1 node) ==> let inst' = setInstanceSmallerThanNode node inst inst'' = inst' { Instance.disks = Instance.disks inst , Instance.diskTemplate = dt } in (Node.addPri node inst'' ==? Bad Types.FailSpindles) -- | Check that adding a primary instance with too many VCPUs fails -- with type FailCPU. prop_addPriFC :: Property prop_addPriFC = forAll (choose (1, maxCpu)) $ \extra -> forAll genMaybeExclStorNode $ \node -> forAll (arbitrary `suchThat` Instance.notOffline `suchThat` (not . Instance.forthcoming)) $ \inst -> let inst' = setInstanceSmallerThanNode node inst inst'' = inst' { Instance.vcpus = Node.availCpu node + extra } in case Node.addPri node inst'' of Bad Types.FailCPU -> passTest v -> failTest $ "Expected OpFail FailCPU, but got " ++ show v -- | Check that an instance add with too high memory or disk will be -- rejected. prop_addSec :: Node.Node -> Instance.Instance -> Int -> Property prop_addSec node inst pdx = ((Instance.mem inst >= (Node.fMem node - Node.rMem node) && not (Instance.isOffline inst)) || Instance.dsk inst >= Node.fDsk node || (Node.exclStorage node && hasInstTooManySpindles inst (Node.fSpindles node))) && not (Node.failN1 node) ==> isBad (Node.addSec node inst pdx) -- | Check that an offline instance with reasonable disk size but -- extra mem/cpu can always be added. prop_addOfflinePri :: NonNegative Int -> NonNegative Int -> Property prop_addOfflinePri (NonNegative extra_mem) (NonNegative extra_cpu) = forAll genMaybeExclStorNode $ \node -> forAll (genInstanceSmallerThanNode node) $ \inst -> let inst' = inst { Instance.runSt = Types.StatusOffline , Instance.mem = Node.availMem node + extra_mem , Instance.vcpus = Node.availCpu node + extra_cpu } in case Node.addPriEx True node inst' of Ok _ -> passTest v -> failTest $ "Expected OpGood, but got: " ++ show v -- | Check that an offline instance with reasonable disk size but -- extra mem/cpu can always be added. prop_addOfflineSec :: NonNegative Int -> NonNegative Int -> Types.Ndx -> Property prop_addOfflineSec (NonNegative extra_mem) (NonNegative extra_cpu) pdx = forAll genMaybeExclStorNode $ \node -> forAll (genInstanceSmallerThanNode node) $ \inst -> let inst' = inst { Instance.runSt = Types.StatusOffline , Instance.mem = Node.availMem node + extra_mem , Instance.vcpus = Node.availCpu node + extra_cpu , Instance.diskTemplate = Types.DTDrbd8 } in case Node.addSec node inst' pdx of Ok _ -> passTest v -> failTest $ "Expected OpGood/OpGood, but got: " ++ show v -- | Checks for memory reservation changes. prop_rMem :: Instance.Instance -> Property prop_rMem inst = not (Instance.isOffline inst) && not (Instance.forthcoming inst) ==> -- TODO Should we also require ((> Types.unitMem) . Node.fMemForth) ? forAll (genMaybeExclStorNode `suchThat` ((> Types.unitMem) . Node.fMem)) $ \node -> -- ab = auto_balance, nb = non-auto_balance -- we use -1 as the primary node of the instance let inst' = inst { Instance.pNode = -1, Instance.autoBalance = True , Instance.diskTemplate = Types.DTDrbd8 } inst_ab = setInstanceSmallerThanNode node inst' inst_nb = inst_ab { Instance.autoBalance = False } -- now we have the two instances, identical except the -- autoBalance attribute orig_rmem = Node.rMem node inst_idx = Instance.idx inst_ab node_add_ab = Node.addSec node inst_ab (-1) node_add_nb = Node.addSec node inst_nb (-1) node_del_ab = liftM (`Node.removeSec` inst_ab) node_add_ab node_del_nb = liftM (`Node.removeSec` inst_nb) node_add_nb in case (node_add_ab, node_add_nb, node_del_ab, node_del_nb) of (Ok a_ab, Ok a_nb, Ok d_ab, Ok d_nb) -> counterexample "Consistency checks failed" $ Node.rMem a_ab > orig_rmem && Node.rMem a_ab - orig_rmem == Instance.mem inst_ab && Node.rMem a_nb == orig_rmem && Node.rMem d_ab == orig_rmem && Node.rMem d_nb == orig_rmem && -- this is not related to rMem, but as good a place to -- test as any inst_idx `elem` Node.sList a_ab && inst_idx `notElem` Node.sList d_ab x -> failTest $ "Failed to add/remove instances: " ++ show x -- | Check mdsk setting. prop_setMdsk :: Node.Node -> SmallRatio -> Bool prop_setMdsk node mx = Node.loDsk node' >= 0 && fromIntegral (Node.loDsk node') <= Node.tDsk node && Node.availDisk node' >= 0 && Node.availDisk node' <= Node.fDsk node' && fromIntegral (Node.availDisk node') <= Node.tDsk node' && Node.mDsk node' == mx' where node' = Node.setMdsk node mx' SmallRatio mx' = mx -- Check tag maps prop_tagMaps_idempotent :: Property prop_tagMaps_idempotent = forAll genTags $ \tags -> Node.delTags (Node.addTags m tags) tags ==? m where m = Map.empty prop_tagMaps_reject :: Property prop_tagMaps_reject = forAll (genTags `suchThat` (not . null)) $ \tags -> let m = Node.addTags Map.empty tags in all (\t -> Node.rejectAddTags m [t]) tags prop_showField :: Node.Node -> Property prop_showField node = forAll (elements Node.defaultFields) $ \ field -> fst (Node.showHeader field) /= Types.unknownField && Node.showField node field /= Types.unknownField prop_computeGroups :: [Node.Node] -> Bool prop_computeGroups nodes = let ng = Node.computeGroups nodes onlyuuid = map fst ng in length nodes == sum (map (length . snd) ng) && all (\(guuid, ns) -> all ((== guuid) . Node.group) ns) ng && length (nub onlyuuid) == length onlyuuid && (null nodes || not (null ng)) -- Check idempotence of add/remove operations prop_addPri_idempotent :: Property prop_addPri_idempotent = forAll genMaybeExclStorNode $ \node -> forAll (genInstanceSmallerThanNode node) $ \inst -> case Node.addPri node inst of Ok node' -> Node.removePri node' inst ==? node _ -> failTest "Can't add instance" prop_addSec_idempotent :: Property prop_addSec_idempotent = forAll genMaybeExclStorNode $ \node -> forAll (genInstanceSmallerThanNode node) $ \inst -> let pdx = Node.idx node + 1 inst' = Instance.setPri inst pdx inst'' = inst' { Instance.diskTemplate = Types.DTDrbd8 } in case Node.addSec node inst'' pdx of Ok node' -> Node.removeSec node' inst'' ==? node _ -> failTest "Can't add instance" -- | Check that no graph is created on an empty node list. case_emptyNodeList :: Assertion case_emptyNodeList = assertEqual "" Nothing $ Node.mkNodeGraph emptynodes emptyinstances where emptynodes = Container.empty :: Node.List emptyinstances = Container.empty :: Instance.List -- | Check that the number of vertices of a nodegraph is equal to the number of -- nodes in the original node list. prop_numVertices :: Property prop_numVertices = forAll genNodeGraph $ \(graph, nl, _) -> (fmap numvertices graph ==? Just (Container.size nl)) where numvertices = length . Graph.vertices -- | Check that the number of edges of a nodegraph is equal to twice the number -- of instances with secondary nodes in the original instance list. prop_numEdges :: Property prop_numEdges = forAll genNodeGraph $ \(graph, _, il) -> (fmap numedges graph ==? Just (numwithsec il * 2)) where numedges = length . Graph.edges numwithsec = length . filter Instance.hasSecondary . Container.elems -- | Check that a node graph is colorable. prop_nodeGraphIsColorable :: Property prop_nodeGraphIsColorable = forAll genNodeGraph $ \(graph, _, _) -> fmap HGraph.isColorable graph ==? Just True -- | Check that each edge in a nodegraph is an instance. prop_instanceIsEdge :: Property prop_instanceIsEdge = forAll genNodeGraph $ \(graph, _, il) -> fmap (\g -> all (`isEdgeOn` g) (iwithsec il)) graph ==? Just True where i `isEdgeOn` g = iEdges i `intersect` Graph.edges g == iEdges i iEdges i = [ (Instance.pNode i, Instance.sNode i) , (Instance.sNode i, Instance.pNode i)] iwithsec = filter Instance.hasSecondary . Container.elems -- | Check that each instance in an edge in the resulting nodegraph. prop_edgeIsInstance :: Property prop_edgeIsInstance = forAll genNodeGraph $ \(graph, _, il) -> fmap (all (`isInstanceIn` il).Graph.edges) graph ==? Just True where e `isInstanceIn` il = any (`hasNodes` e) (Container.elems il) i `hasNodes` (v1,v2) = Instance.allNodes i `elem` permutations [v1,v2] -- | List of tests for the Node module. testSuite "HTools/Node" [ 'prop_setAlias , 'prop_setOffline , 'prop_setMcpu , 'prop_addPriFM , 'prop_addPriFD , 'prop_addPriFS , 'prop_addPriFC , 'prop_addPri_NoN1Fail , 'prop_addSec , 'prop_addOfflinePri , 'prop_addOfflineSec , 'prop_rMem , 'prop_setMdsk , 'prop_tagMaps_idempotent , 'prop_tagMaps_reject , 'prop_showField , 'prop_computeGroups , 'prop_addPri_idempotent , 'prop_addSec_idempotent , 'case_emptyNodeList , 'prop_numVertices , 'prop_numEdges , 'prop_nodeGraphIsColorable , 'prop_edgeIsInstance , 'prop_instanceIsEdge ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/PeerMap.hs000064400000000000000000000061721476477700300224260ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.PeerMap (testHTools_PeerMap) where import Test.QuickCheck import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import qualified Ganeti.HTools.PeerMap as PeerMap -- | Make sure add is idempotent. prop_addIdempotent :: PeerMap.PeerMap -> PeerMap.Key -> PeerMap.Elem -> Property prop_addIdempotent pmap key em = fn (fn puniq) ==? fn puniq where fn = PeerMap.add key em puniq = PeerMap.accumArray const pmap -- | Make sure remove is idempotent. prop_removeIdempotent :: PeerMap.PeerMap -> PeerMap.Key -> Property prop_removeIdempotent pmap key = fn (fn puniq) ==? fn puniq where fn = PeerMap.remove key puniq = PeerMap.accumArray const pmap -- | Make sure a missing item returns 0. prop_findMissing :: PeerMap.PeerMap -> PeerMap.Key -> Property prop_findMissing pmap key = PeerMap.find key (PeerMap.remove key puniq) ==? 0 where puniq = PeerMap.accumArray const pmap -- | Make sure an added item is found. prop_addFind :: PeerMap.PeerMap -> PeerMap.Key -> PeerMap.Elem -> Property prop_addFind pmap key em = PeerMap.find key (PeerMap.add key em puniq) ==? em where puniq = PeerMap.accumArray const pmap -- | Manual check that maxElem returns the maximum indeed, or 0 for null. prop_maxElem :: PeerMap.PeerMap -> Property prop_maxElem pmap = PeerMap.maxElem puniq ==? if null puniq then 0 else (maximum . snd . unzip) puniq where puniq = PeerMap.accumArray const pmap -- | List of tests for the PeerMap module. testSuite "HTools/PeerMap" [ 'prop_addIdempotent , 'prop_removeIdempotent , 'prop_maxElem , 'prop_addFind , 'prop_findMissing ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/HTools/Types.hs000064400000000000000000000166731476477700300222100ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, FlexibleInstances, TypeSynonymInstances #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.HTools.Types ( testHTools_Types , Types.AllocPolicy(..) , Types.DiskTemplate(..) , Types.FailMode(..) , Types.ISpec(..) , Types.IPolicy(..) , nullIPolicy ) where import Test.QuickCheck hiding (Result) import Test.HUnit import Control.Monad (replicateM) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.TestHTools import Test.Ganeti.Types (allDiskTemplates) import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.ConstantUtils import qualified Ganeti.HTools.Types as Types {-# ANN module "HLint: ignore Use camelCase" #-} -- * Helpers -- * Arbitrary instance $(genArbitrary ''Types.FailMode) instance Arbitrary a => Arbitrary (Types.OpResult a) where arbitrary = arbitrary >>= \c -> if c then Ok <$> arbitrary else Bad <$> arbitrary instance Arbitrary Types.ISpec where arbitrary = do mem_s <- arbitrary::Gen (NonNegative Int) dsk_c <- arbitrary::Gen (NonNegative Int) dsk_s <- arbitrary::Gen (NonNegative Int) cpu_c <- arbitrary::Gen (NonNegative Int) nic_c <- arbitrary::Gen (NonNegative Int) su <- arbitrary::Gen (NonNegative Int) return Types.ISpec { Types.iSpecMemorySize = fromEnum mem_s , Types.iSpecCpuCount = fromEnum cpu_c , Types.iSpecDiskSize = fromEnum dsk_s , Types.iSpecDiskCount = fromEnum dsk_c , Types.iSpecNicCount = fromEnum nic_c , Types.iSpecSpindleUse = fromEnum su } -- | Generates an ispec bigger than the given one. genBiggerISpec :: Types.ISpec -> Gen Types.ISpec genBiggerISpec imin = do mem_s <- choose (Types.iSpecMemorySize imin, maxBound) dsk_c <- choose (Types.iSpecDiskCount imin, maxBound) dsk_s <- choose (Types.iSpecDiskSize imin, maxBound) cpu_c <- choose (Types.iSpecCpuCount imin, maxBound) nic_c <- choose (Types.iSpecNicCount imin, maxBound) su <- choose (Types.iSpecSpindleUse imin, maxBound) return Types.ISpec { Types.iSpecMemorySize = fromEnum mem_s , Types.iSpecCpuCount = fromEnum cpu_c , Types.iSpecDiskSize = fromEnum dsk_s , Types.iSpecDiskCount = fromEnum dsk_c , Types.iSpecNicCount = fromEnum nic_c , Types.iSpecSpindleUse = fromEnum su } genMinMaxISpecs :: Gen Types.MinMaxISpecs genMinMaxISpecs = do imin <- arbitrary imax <- genBiggerISpec imin return Types.MinMaxISpecs { Types.minMaxISpecsMinSpec = imin , Types.minMaxISpecsMaxSpec = imax } instance Arbitrary Types.MinMaxISpecs where arbitrary = genMinMaxISpecs genMinMaxStdISpecs :: Gen (Types.MinMaxISpecs, Types.ISpec) genMinMaxStdISpecs = do imin <- arbitrary istd <- genBiggerISpec imin imax <- genBiggerISpec istd return (Types.MinMaxISpecs { Types.minMaxISpecsMinSpec = imin , Types.minMaxISpecsMaxSpec = imax }, istd) genIPolicySpecs :: Gen ([Types.MinMaxISpecs], Types.ISpec) genIPolicySpecs = do num_mm <- choose (1, 6) -- 6 is just an arbitrary limit std_compl <- choose (1, num_mm) mm_head <- replicateM (std_compl - 1) genMinMaxISpecs (mm_middle, istd) <- genMinMaxStdISpecs mm_tail <- replicateM (num_mm - std_compl) genMinMaxISpecs return (mm_head ++ (mm_middle : mm_tail), istd) instance Arbitrary Types.IPolicy where arbitrary = do (iminmax, istd) <- genIPolicySpecs num_tmpl <- choose (0, length allDiskTemplates) dts <- genUniquesList num_tmpl arbitrary vcpu_ratio <- choose (1.0, maxVcpuRatio) spindle_ratio <- choose (1.0, maxSpindleRatio) return Types.IPolicy { Types.iPolicyMinMaxISpecs = iminmax , Types.iPolicyStdSpec = istd , Types.iPolicyDiskTemplates = dts , Types.iPolicyVcpuRatio = vcpu_ratio , Types.iPolicySpindleRatio = spindle_ratio } -- * Test cases prop_ISpec_serialisation :: Types.ISpec -> Property prop_ISpec_serialisation = testSerialisation prop_IPolicy_serialisation :: Types.IPolicy -> Property prop_IPolicy_serialisation = testSerialisation prop_opToResult :: Types.OpResult Int -> Property prop_opToResult op = case op of Bad _ -> counterexample ("expected bad but got " ++ show r) $ isBad r Ok v -> case r of Bad msg -> failTest ("expected Ok but got Bad " ++ msg) Ok v' -> v ==? v' where r = Types.opToResult op prop_eitherToResult :: Either String Int -> Bool prop_eitherToResult ei = case ei of Left _ -> isBad r Right v -> case r of Bad _ -> False Ok v' -> v == v' where r = eitherToResult ei :: Result Int -- | Test 'AutoRepairType' ordering is as expected and consistent with Python -- codebase. case_AutoRepairType_sort :: Assertion case_AutoRepairType_sort = do let expected = [ Types.ArFixStorage , Types.ArMigrate , Types.ArFailover , Types.ArReinstall ] all_hs_raw = mkSet $ map Types.autoRepairTypeToRaw [minBound..maxBound] assertEqual "Haskell order" expected [minBound..maxBound] assertEqual "consistent with Python" C.autoRepairAllTypes all_hs_raw -- | Test 'AutoRepairResult' type is equivalent with Python codebase. case_AutoRepairResult_pyequiv :: Assertion case_AutoRepairResult_pyequiv = do let all_py_results = C.autoRepairAllResults all_hs_results = mkSet $ map Types.autoRepairResultToRaw [minBound..maxBound] assertEqual "for AutoRepairResult equivalence" all_py_results all_hs_results testSuite "HTools/Types" [ 'prop_ISpec_serialisation , 'prop_IPolicy_serialisation , 'prop_opToResult , 'prop_eitherToResult , 'case_AutoRepairType_sort , 'case_AutoRepairResult_pyequiv ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Hypervisor/000075500000000000000000000000001476477700300214755ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Hypervisor/Xen/000075500000000000000000000000001476477700300222275ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Hypervisor/Xen/XlParser.hs000064400000000000000000000171371476477700300243340ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for @xl list --long@ parser -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Hypervisor.Xen.XlParser ( testHypervisor_Xen_XlParser ) where import Test.HUnit import Test.QuickCheck as QuickCheck hiding (Result) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Control.Monad (liftM) import qualified Data.Attoparsec.Text as A import Data.Text (pack) import Data.Char import qualified Data.Map as Map import Text.Printf import Ganeti.Hypervisor.Xen.Types import Ganeti.Hypervisor.Xen.XlParser {-# ANN module "HLint: ignore Use camelCase" #-} -- * Arbitraries -- | Generator for 'ListConfig'. -- -- A completely arbitrary configuration would contain too many lists -- and its size would be to big to be actually parsable in reasonable -- time. This generator builds a random Config that is still of a -- reasonable size, and it also Avoids generating strings that might -- be interpreted as numbers. genConfig :: Int -> Gen LispConfig genConfig 0 = -- only terminal values for size 0 frequency [ (5, liftM LCString (genName `suchThat` (not . canBeNumber))) , (5, liftM LCDouble arbitrary) ] genConfig n = -- for size greater than 0, allow "some" lists frequency [ (5, liftM LCString (resize n genName `suchThat` (not . canBeNumber))) , (5, liftM LCDouble arbitrary) , (1, liftM LCList (choose (1, n) >>= (\n' -> vectorOf n' (genConfig $ n `div` n')))) ] -- | Arbitrary instance for 'LispConfig' using 'genConfig'. instance Arbitrary LispConfig where arbitrary = sized genConfig -- | Determines conservatively whether a string could be a number. canBeNumber :: String -> Bool canBeNumber [] = False canBeNumber [c] = canBeNumberChar c canBeNumber (c:xs) = canBeNumberChar c && canBeNumber xs -- | Determines whether a char can be part of the string representation of a -- number (even in scientific notation). canBeNumberChar :: Char -> Bool canBeNumberChar c = isDigit c || (c `elem` "eE-") -- | Generates an arbitrary @xl uptime@ output line. instance Arbitrary UptimeInfo where arbitrary = do name <- genFQDN NonNegative idNum <- arbitrary :: Gen (NonNegative Int) NonNegative days <- arbitrary :: Gen (NonNegative Int) hours <- choose (0, 23) :: Gen Int mins <- choose (0, 59) :: Gen Int secs <- choose (0, 59) :: Gen Int let uptime :: String uptime = if days /= 0 then printf "%d days, %d:%d:%d" days hours mins secs else printf "%d:%d:%d" hours mins secs return $ UptimeInfo name idNum uptime -- * Helper functions for tests -- | Function for testing whether a domain configuration is parsed correctly. testDomain :: String -> Map.Map String Domain -> Assertion testDomain fileName expectedContent = do fileContent <- readTestData fileName case A.parseOnly xlListParser $ pack fileContent of Left msg -> assertFailure $ "Parsing failed: " ++ msg Right obtained -> assertEqual fileName expectedContent obtained -- | Function for testing whether a @xl uptime@ output (stored in a file) -- is parsed correctly. testUptimeInfo :: String -> Map.Map Int UptimeInfo -> Assertion testUptimeInfo fileName expectedContent = do fileContent <- readTestData fileName case A.parseOnly xlUptimeParser $ pack fileContent of Left msg -> assertFailure $ "Parsing failed: " ++ msg Right obtained -> assertEqual fileName expectedContent obtained -- | Determines whether two LispConfig are equal, with the exception of Double -- values, that just need to be \"almost equal\". -- -- Meant mainly for testing purposes, given that Double values may be slightly -- rounded during parsing. isAlmostEqual :: LispConfig -> LispConfig -> Property isAlmostEqual (LCList c1) (LCList c2) = (length c1 ==? length c2) .&&. conjoin (zipWith isAlmostEqual c1 c2) isAlmostEqual (LCString s1) (LCString s2) = s1 ==? s2 isAlmostEqual (LCDouble d1) (LCDouble d2) = counterexample msg $ rel <= 1e-12 where rel = relativeError d1 d2 msg = "Relative error " ++ show rel ++ " not smaller than 1e-12\n" ++ "expected: " ++ show d2 ++ "\n but got: " ++ show d1 isAlmostEqual a b = failTest $ "Comparing different types: '" ++ show a ++ "' with '" ++ show b ++ "'" -- | Function to serialize LispConfigs in such a way that they can be rebuilt -- again by the lispConfigParser. serializeConf :: LispConfig -> String serializeConf (LCList c) = "(" ++ unwords (map serializeConf c) ++ ")" serializeConf (LCString s) = s serializeConf (LCDouble d) = show d -- | Function to serialize UptimeInfos in such a way that they can be rebuilt -- againg by the uptimeLineParser. serializeUptime :: UptimeInfo -> String serializeUptime (UptimeInfo name idNum uptime) = printf "%s\t%d\t%s" name idNum uptime -- | Test whether a randomly generated config can be parsed. -- Implicitly, this also tests that the Show instance of Config is correct. prop_config :: LispConfig -> Property prop_config conf = case A.parseOnly lispConfigParser . pack . serializeConf $ conf of Left msg -> failTest $ "Parsing failed: " ++ msg Right obtained -> counterexample "Failing almost equal check" $ isAlmostEqual obtained conf -- | Test whether a randomly generated UptimeInfo text line can be parsed. prop_uptimeInfo :: UptimeInfo -> Property prop_uptimeInfo uInfo = case A.parseOnly uptimeLineParser . pack . serializeUptime $ uInfo of Left msg -> failTest $ "Parsing failed: " ++ msg Right obtained -> obtained ==? uInfo -- | Test a Xen 4.0.1 @xl list --long@ output. case_xen401list :: Assertion case_xen401list = testDomain "xen-xl-list-long-4.0.1.txt" $ Map.fromList [ ("Domain-0", Domain 0 "Domain-0" 184000.41332 ActualRunning Nothing) , ("instance1.example.com", Domain 119 "instance1.example.com" 24.116146647 ActualBlocked Nothing) ] -- | Test a Xen 4.0.1 @xl uptime@ output. case_xen401uptime :: Assertion case_xen401uptime = testUptimeInfo "xen-xl-uptime-4.0.1.txt" $ Map.fromList [ (0, UptimeInfo "Domain-0" 0 "98 days, 2:27:44") , (119, UptimeInfo "instance1.example.com" 119 "15 days, 20:57:07") ] testSuite "Hypervisor/Xen/XlParser" [ 'prop_config , 'prop_uptimeInfo , 'case_xen401list , 'case_xen401uptime ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/JQScheduler.hs000064400000000000000000000524411476477700300220360ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, ScopedTypeVariables, NamedFieldPuns #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for the job scheduler. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.JQScheduler (testJQScheduler) where import Control.Lens ((&), (.~), _2) import qualified Data.ByteString.UTF8 as UTF8 import Data.List (inits) import Data.Maybe import qualified Data.Map as Map import Data.Set (Set, difference) import qualified Data.Set as Set import Text.JSON (JSValue(..)) import Test.HUnit import Test.QuickCheck import Test.Ganeti.JQueue.Objects (genQueuedOpCode, genJobId, justNoTs) import Test.Ganeti.SlotMap (genTestKey, overfullKeys) import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Test.Ganeti.Types () import Ganeti.JQScheduler.Filtering import Ganeti.JQScheduler.ReasonRateLimiting import Ganeti.JQScheduler.Types import Ganeti.JQueue.Lens import Ganeti.JQueue.Objects import Ganeti.Objects (FilterRule(..), FilterPredicate(..), FilterAction(..), filterRuleOrder) import Ganeti.OpCodes import Ganeti.OpCodes.Lens import Ganeti.Query.Language (Filter(..), FilterValue(..)) import Ganeti.SlotMap import Ganeti.Types import Ganeti.Utils (isSubsequenceOf, newUUID) {-# ANN module "HLint: ignore Use camelCase" #-} genRateLimitReason :: Gen String genRateLimitReason = do Slot{ slotLimit = n } <- arbitrary l <- genTestKey return $ "rate-limit:" ++ show n ++ ":" ++ l instance Arbitrary QueuedJob where arbitrary = do -- For our scheduler testing purposes here, we only care about -- opcodes, job ID and reason rate limits. jid <- genJobId ops <- resize 5 . listOf1 $ do o <- genQueuedOpCode -- Put some rate limits into the OpCode. limitString <- genRateLimitReason return $ o & qoInputL . validOpCodeL . metaParamsL . opReasonL . traverse . _2 .~ limitString return $ QueuedJob jid ops justNoTs justNoTs justNoTs Nothing Nothing instance Arbitrary JobWithStat where arbitrary = nullJobWithStat <$> arbitrary shrink job = [ job { jJob = x } | x <- shrink (jJob job) ] instance Arbitrary Queue where arbitrary = do let genJobsUniqueJIDs :: [JobWithStat] -> Gen [JobWithStat] genJobsUniqueJIDs = listOfUniqueBy arbitrary (qjId . jJob) queued <- genJobsUniqueJIDs [] running <- genJobsUniqueJIDs queued manip <- genJobsUniqueJIDs (queued ++ running) return $ Queue queued running manip shrink q = [ q { qEnqueued = x } | x <- shrink (qEnqueued q) ] ++ [ q { qRunning = x } | x <- shrink (qRunning q) ] ++ [ q { qManipulated = x } | x <- shrink (qManipulated q) ] -- * Test cases -- | Tests rate limit reason trail parsing. case_parseReasonRateLimit :: Assertion case_parseReasonRateLimit = do assertBool "default case" $ let a = parseReasonRateLimit "rate-limit:20:my label" b = parseReasonRateLimit "rate-limit:21:my label" in and [ a == Just ("20:my label", 20) , b == Just ("21:my label", 21) ] assertEqual "be picky about whitespace" Nothing (parseReasonRateLimit " rate-limit:20:my label") -- | Tests that "rateLimit:n:..." and "rateLimit:m:..." become different -- rate limiting buckets. prop_slotMapFromJob_conflicting_buckets :: Property prop_slotMapFromJob_conflicting_buckets = do let sameBucketReasonStringGen :: Gen (String, String) sameBucketReasonStringGen = do Positive (n :: Int) <- arbitrary Positive (m :: Int) <- arbitrary `suchThat` (/= Positive n) l <- genPrintableAsciiString return ( "rate-limit:" ++ show n ++ ":" ++ l , "rate-limit:" ++ show m ++ ":" ++ l ) forAll sameBucketReasonStringGen $ \(s1, s2) -> do (lab1, lim1) <- parseReasonRateLimit s1 (lab2, _ ) <- parseReasonRateLimit s2 let sm = Map.fromList [(lab1, Slot 1 lim1)] cm = Map.fromList [(lab2, 1)] in return $ (sm `occupySlots` cm) ==? Map.fromList [ (lab1, Slot 1 lim1) , (lab2, Slot 1 0) ] :: Gen Property -- | Tests some basic cases for reason rate limiting. case_reasonRateLimit :: Assertion case_reasonRateLimit = do let mkJobWithReason jobNum reasonTrail = do opc <- genSample genQueuedOpCode jid <- makeJobId jobNum let opc' = opc & (qoInputL . validOpCodeL . metaParamsL . opReasonL) .~ reasonTrail return . nullJobWithStat $ QueuedJob { qjId = jid , qjOps = [opc'] , qjReceivedTimestamp = Nothing , qjStartTimestamp = Nothing , qjEndTimestamp = Nothing , qjLivelock = Nothing , qjProcessId = Nothing } -- 3 jobs, limited to 2 of them running. j1 <- mkJobWithReason 1 [("source1", "rate-limit:2:hello", 0)] j2 <- mkJobWithReason 2 [("source1", "rate-limit:2:hello", 0)] j3 <- mkJobWithReason 3 [("source1", "rate-limit:2:hello", 0)] assertEqual "[j1] should not be rate-limited" [j1] (reasonRateLimit (Queue [j1] [] []) [j1]) assertEqual "[j1, j2] should not be rate-limited" [j1, j2] (reasonRateLimit (Queue [j1, j2] [] []) [j1, j2]) assertEqual "j3 should be rate-limited 1" [j1, j2] (reasonRateLimit (Queue [j1, j2, j3] [] []) [j1, j2, j3]) assertEqual "j3 should be rate-limited 2" [j2] (reasonRateLimit (Queue [j2, j3] [j1] []) [j2, j3]) assertEqual "j3 should be rate-limited 3" [] (reasonRateLimit (Queue [j3] [j1] [j2]) [j3]) -- | Tests the specified properties of `reasonRateLimit`, as defined in -- `doc/design-optables.rst`. prop_reasonRateLimit :: Property prop_reasonRateLimit = forAllShrink arbitrary shrink $ \q -> let slotMapFromJobWithStat = slotMapFromJobs . map jJob enqueued = qEnqueued q toRun = reasonRateLimit q enqueued oldSlots = slotMapFromJobWithStat (qRunning q) newSlots = slotMapFromJobWithStat (qRunning q ++ toRun) -- What would happen without rate limiting. newSlotsNoLimits = slotMapFromJobWithStat (qRunning q ++ enqueued) in -- Ensure it's unlikely that jobs are all in different buckets. cover' 50 (any ((> 1) . slotOccupied) . Map.elems $ newSlotsNoLimits) "some jobs have the same rate-limit bucket" -- Ensure it's likely that rate limiting has any effect. . cover' 50 (overfullKeys newSlotsNoLimits `difference` overfullKeys oldSlots /= Set.empty) "queued jobs cannot be started because of rate limiting" $ conjoin [ counterexample "scheduled jobs must be subsequence" $ toRun `isSubsequenceOf` enqueued -- This is the key property: , counterexample "no job may exceed its bucket limits, except from\ \ jobs that were already running with exceeded\ \ limits; those must not increase" $ conjoin [ if occup <= limit -- Within limits, all fine. then passTest -- Bucket exceeds limits - it must have exceeded them -- in the initial running list already, with the same -- slot count. else Map.lookup k oldSlots ==? Just slot | (k, slot@(Slot occup limit)) <- Map.toList newSlots ] ] -- | Tests that filter rule ordering is determined (solely) by priority, -- watermark and UUID, as defined in `doc/design-optables.rst`. prop_filterRuleOrder :: Property prop_filterRuleOrder = property $ do a <- arbitrary b <- arbitrary `suchThat` ((frUuid a /=) . frUuid) return $ filterRuleOrder a b ==? (frPriority a, frWatermark a, frUuid a) `compare` (frPriority b, frWatermark b, frUuid b) -- | Tests common inputs for `matchPredicate`, especially the predicates -- and fields available to them as defined in the spec. case_matchPredicate :: Assertion case_matchPredicate = do jid1 <- makeJobId 1 clusterName <- mkNonEmpty "cluster1" let job = QueuedJob { qjId = jid1 , qjOps = [ QueuedOpCode { qoInput = ValidOpCode MetaOpCode { metaParams = CommonOpParams { opDryRun = Nothing , opDebugLevel = Nothing , opPriority = OpPrioHigh , opDepends = Just [] , opComment = Nothing , opReason = [("source1", "reason1", 1234)] } , metaOpCode = OpClusterRename { opName = clusterName } } , qoStatus = OP_STATUS_QUEUED , qoResult = JSNull , qoLog = [] , qoPriority = -1 , qoStartTimestamp = Nothing , qoExecTimestamp = Nothing , qoEndTimestamp = Nothing } ] , qjReceivedTimestamp = Nothing , qjStartTimestamp = Nothing , qjEndTimestamp = Nothing , qjLivelock = Nothing , qjProcessId = Nothing } let watermark = jid1 check = matchPredicate job watermark -- jobid filters assertEqual "matching jobid filter" True . check $ FPJobId (EQFilter "id" (NumericValue 1)) assertEqual "non-matching jobid filter" False . check $ FPJobId (EQFilter "id" (NumericValue 2)) assertEqual "non-matching jobid filter (string passed)" False . check $ FPJobId (EQFilter "id" (QuotedString "1")) -- jobid filters: watermarks assertEqual "matching jobid watermark filter" True . check $ FPJobId (EQFilter "id" (QuotedString "watermark")) -- opcode filters assertEqual "matching opcode filter (type of opcode)" True . check $ FPOpCode (EQFilter "OP_ID" (QuotedString "OP_CLUSTER_RENAME")) assertEqual "non-matching opcode filter (type of opcode)" False . check $ FPOpCode (EQFilter "OP_ID" (QuotedString "OP_INSTANCE_CREATE")) assertEqual "matching opcode filter (nested access)" True . check $ FPOpCode (EQFilter "name" (QuotedString "cluster1")) assertEqual "non-matching opcode filter (nonexistent nested access)" False . check $ FPOpCode (EQFilter "something" (QuotedString "cluster1")) -- reason filters assertEqual "matching reason filter (reason field)" True . check $ FPReason (EQFilter "reason" (QuotedString "reason1")) assertEqual "non-matching reason filter (reason field)" False . check $ FPReason (EQFilter "reason" (QuotedString "reasonGarbage")) assertEqual "matching reason filter (source field)" True . check $ FPReason (EQFilter "source" (QuotedString "source1")) assertEqual "matching reason filter (timestamp field)" True . check $ FPReason (EQFilter "timestamp" (NumericValue 1234)) assertEqual "non-matching reason filter (nonexistent field)" False . check $ FPReason (EQFilter "something" (QuotedString "")) -- | Tests that jobs selected by `applyingFilter` actually match -- and have an effect (are not CONTINUE filters). prop_applyingFilter :: Property prop_applyingFilter = forAllShrink arbitrary shrink $ \job -> forAllShrink (arbitrary `suchThat` (isJust . flip applyingFilter job . Set.fromList)) shrink $ \filters -> let applying = applyingFilter (Set.fromList filters) job in case applying of Just f -> job `matches` f && frAction f /= Continue Nothing -> error "Should not happen" case_jobFiltering :: Assertion case_jobFiltering = do clusterName <- mkNonEmpty "cluster1" jid1 <- makeJobId 1 jid2 <- makeJobId 2 jid3 <- makeJobId 3 jid4 <- makeJobId 4 unsetPrio <- mkNonNegative 1234 uuid1 <- fmap UTF8.fromString newUUID let j1 = nullJobWithStat QueuedJob { qjId = jid1 , qjOps = [ QueuedOpCode { qoInput = ValidOpCode MetaOpCode { metaParams = CommonOpParams { opDryRun = Nothing , opDebugLevel = Nothing , opPriority = OpPrioHigh , opDepends = Just [] , opComment = Nothing , opReason = [("source1", "reason1", 1234)]} , metaOpCode = OpClusterRename { opName = clusterName } } , qoStatus = OP_STATUS_QUEUED , qoResult = JSNull , qoLog = [] , qoPriority = -1 , qoStartTimestamp = Nothing , qoExecTimestamp = Nothing , qoEndTimestamp = Nothing } ] , qjReceivedTimestamp = Nothing , qjStartTimestamp = Nothing , qjEndTimestamp = Nothing , qjLivelock = Nothing , qjProcessId = Nothing } j2 = j1 & jJobL . qjIdL .~ jid2 j3 = j1 & jJobL . qjIdL .~ jid3 j4 = j1 & jJobL . qjIdL .~ jid4 fr1 = FilterRule { frWatermark = jid1 , frPriority = unsetPrio , frPredicates = [FPJobId (EQFilter "id" (NumericValue 1))] , frAction = Reject , frReasonTrail = [] , frUuid = uuid1 } -- Gives the rule a new UUID. rule fr = do uuid <- fmap UTF8.fromString newUUID return fr{ frUuid = uuid } -- Helper to create filter chains: assigns the filters in the list -- increasing priorities, so that filters listed first are processed -- first. chain :: [FilterRule] -> Set FilterRule chain frs | any ((/= unsetPrio) . frPriority) frs = error "Filter was passed to `chain` that already had a priority." | otherwise = Set.fromList [ fr{ frPriority = prio } | (fr, Just prio) <- zip frs (map mkNonNegative [1..]) ] fr2 <- rule fr1{ frAction = Accept } fr3 <- rule fr1{ frAction = Pause } fr4 <- rule fr1{ frPredicates = [FPJobId (GTFilter "id" (QuotedString "watermark"))] } fr5 <- rule fr1{ frPredicates = [] } fr6 <- rule fr5{ frAction = Continue } fr7 <- rule fr6{ frAction = RateLimit 2 } fr8 <- rule fr4{ frAction = Continue, frWatermark = jid1 } fr9 <- rule fr8{ frAction = RateLimit 2 } assertEqual "j1 should be rejected (by fr1)" [] (jobFiltering (Queue [j1] [] []) (chain [fr1]) [j1]) assertEqual "j1 should be rejected (by fr1, it has priority)" [] (jobFiltering (Queue [j1] [] []) (chain [fr1, fr2]) [j1]) assertEqual "j1 should be accepted (by fr2, it has priority)" [j1] (jobFiltering (Queue [j1] [] []) (chain [fr2, fr1]) [j1]) assertEqual "j1 should be paused (by fr3)" [] (jobFiltering (Queue [j1] [] []) (chain [fr3]) [j1]) assertEqual "j2 should be rejected (over watermark1)" [j1] (jobFiltering (Queue [j1, j2] [] []) (chain [fr4]) [j1, j2]) assertEqual "all jobs should be rejected (since no predicates)" [] (jobFiltering (Queue [j1, j2] [] []) (chain [fr5]) [j1, j2]) assertEqual "j3 should be rate-limited" [j1, j2] (jobFiltering (Queue [j1, j2, j3] [] []) (chain [fr6, fr7]) [j1, j2, j3]) assertEqual "j4 should be rate-limited" -- j1 doesn't apply to fr8/fr9 (since they match only watermark > jid1) -- so j1 gets scheduled [j1, j2, j3] (jobFiltering (Queue [j1, j2, j3, j4] [] []) (chain [fr8, fr9]) [j1, j2, j3, j4]) -- | Tests the specified properties of `jobFiltering`, as defined in -- `doc/design-optables.rst`. prop_jobFiltering :: Property prop_jobFiltering = forAllShrink (arbitrary `suchThat` (not . null . qEnqueued)) shrink $ \q -> forAllShrink (resize 4 arbitrary) shrink $ \(NonEmpty filterList) -> let running = qRunning q ++ qManipulated q enqueued = qEnqueued q filters = Set.fromList filterList toRun = jobFiltering q filters enqueued -- do the filtering -- Helpers -- Whether `fr` applies to more than `n` of the `jobs` -- (that is, more than allowed). exceeds :: Int -> FilterRule -> [JobWithStat] -> Bool exceeds n fr jobs = n < (length . filter ((frUuid fr ==) . frUuid) . mapMaybe (applyingFilter filters) $ map jJob jobs) {- TODO(#1318): restore coverage checks after a way to do it nicely has been found. -- Helpers for ensuring sensible coverage. -- Makes sure that each action appears with some probability. actionName = head . words . show allActions = map actionName [ Accept, Continue, Pause, Reject , RateLimit 0 ] applyingActions = map (actionName . frAction) . mapMaybe (applyingFilter filters) $ map jJob enqueued perc = 4 -- percent; low because it's per action actionCovers = foldr (.) id [ stableCover (a `elem` applyingActions) perc ("is " ++ a) | a <- allActions ] -} -- Note: if using `covers`, it should be before `conjoin` (see -- QuickCheck bugs 25 and 27). in conjoin [ counterexample "scheduled jobs must be subsequence" $ toRun `isSubsequenceOf` enqueued , counterexample "a reason for each job (not) being scheduled" . -- All enqueued jobs must have a reason why they were (not) -- scheduled, determined by the filter that applies. flip all enqueued $ \job -> case applyingFilter filters (jJob job) of -- If no filter matches, the job must run. Nothing -> job `elem` toRun Just fr@FilterRule{ frAction } -> case frAction of -- ACCEPT filter permit the job immediately, -- PAUSE/REJECT forbid running, CONTINUE filters cannot -- be the output of `applyingFilter`, and -- RATE_LIMIT filters have a more more complex property. Accept -> job `elem` toRun Continue -> error "must not happen" Pause -> job `notElem` toRun Reject -> job `notElem` toRun RateLimit n -> let -- Jobs in queue before our job. jobsBefore = takeWhile (/= job) enqueued in if job `elem` toRun -- If it got scheduled, the job and any job -- before it doesn't overfill the rate limit. then not . exceeds n fr $ running ++ jobsBefore ++ [job] -- If didn't get scheduled, then the rate limit -- was already full before scheduling or the job -- or one of the jobs before made it full. else any (exceeds n fr . (running ++)) (inits $ jobsBefore ++ [job]) -- The `inits` bit includes the [] and [...job] -- cases. ] testSuite "JQScheduler" [ 'case_parseReasonRateLimit , 'prop_slotMapFromJob_conflicting_buckets , 'case_reasonRateLimit , 'prop_reasonRateLimit , 'prop_filterRuleOrder , 'case_matchPredicate , 'prop_applyingFilter , 'case_jobFiltering -- Temporarily disabled until we fix the coverage (#1318) --, 'prop_jobFiltering ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/JQueue.hs000064400000000000000000000274321476477700300210650ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittests for the job queue functionality. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.JQueue (testJQueue) where import Control.Monad (when) import Control.Monad.Fail (MonadFail) import Data.Char (isAscii) import Data.List (nub, sort) import System.Directory import System.FilePath import System.IO.Temp import System.Posix.Files import Test.HUnit import Test.QuickCheck as QuickCheck import Test.QuickCheck.Monadic import Text.JSON import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Test.Ganeti.Types () import Ganeti.BasicTypes import qualified Ganeti.Constants as C import Ganeti.JQueue import Ganeti.OpCodes import Ganeti.Path import Ganeti.Types as Types import Test.Ganeti.JQueue.Objects (justNoTs, genQueuedOpCode, emptyJob, genJobId) {-# ANN module "HLint: ignore Use camelCase" #-} -- * Test cases -- | Tests default priority value. case_JobPriorityDef :: Assertion case_JobPriorityDef = do ej <- emptyJob assertEqual "for default priority" C.opPrioDefault $ calcJobPriority ej -- | Test arbitrary priorities. prop_JobPriority :: Property prop_JobPriority = forAll (listOf1 (genQueuedOpCode `suchThat` (not . opStatusFinalized . qoStatus))) $ \ops -> property $ do jid0 <- makeJobId 0 let job = QueuedJob jid0 ops justNoTs justNoTs justNoTs Nothing Nothing return $ calcJobPriority job ==? minimum (map qoPriority ops) :: Gen Property -- | Tests default job status. case_JobStatusDef :: Assertion case_JobStatusDef = do ej <- emptyJob assertEqual "for job status" JOB_STATUS_SUCCESS $ calcJobStatus ej -- | Test some job status properties. prop_JobStatus :: Property prop_JobStatus = forAll genJobId $ \jid -> forAll genQueuedOpCode $ \op -> let job1 = QueuedJob jid [op] justNoTs justNoTs justNoTs Nothing Nothing st1 = calcJobStatus job1 op_succ = op { qoStatus = OP_STATUS_SUCCESS } op_err = op { qoStatus = OP_STATUS_ERROR } op_cnl = op { qoStatus = OP_STATUS_CANCELING } op_cnd = op { qoStatus = OP_STATUS_CANCELED } -- computes status for a job with an added opcode before st_pre_op pop = calcJobStatus (job1 { qjOps = pop:qjOps job1 }) -- computes status for a job with an added opcode after st_post_op pop = calcJobStatus (job1 { qjOps = qjOps job1 ++ [pop] }) in conjoin [ counterexample "pre-success doesn't change status" (st_pre_op op_succ ==? st1) , counterexample "post-success doesn't change status" (st_post_op op_succ ==? st1) , counterexample "pre-error is error" (st_pre_op op_err ==? JOB_STATUS_ERROR) , counterexample "pre-canceling is canceling" (st_pre_op op_cnl ==? JOB_STATUS_CANCELING) , counterexample "pre-canceled is canceled" (st_pre_op op_cnd ==? JOB_STATUS_CANCELED) ] -- | Tests job status equivalence with Python. Very similar to OpCodes test. case_JobStatusPri_py_equiv :: Assertion case_JobStatusPri_py_equiv = do let num_jobs = 2000::Int jobs <- genSample (vectorOf num_jobs $ do num_ops <- choose (1, 5) ops <- vectorOf num_ops genQueuedOpCode jid <- genJobId return $ QueuedJob jid ops justNoTs justNoTs justNoTs Nothing Nothing) let serialized = encode jobs -- check for non-ASCII fields, usually due to 'arbitrary :: String' mapM_ (\job -> when (any (not . isAscii) (encode job)) . assertFailure $ "Job has non-ASCII fields: " ++ show job ) jobs py_stdout <- runPython "from ganeti import jqueue\n\ \from ganeti import serializer\n\ \import sys\n\ \job_data = serializer.Load(sys.stdin.read())\n\ \decoded = [jqueue._QueuedJob.Restore(None, o, False, False)\n\ \ for o in job_data]\n\ \encoded = [(job.CalcStatus(), job.CalcPriority())\n\ \ for job in decoded]\n\ \sys.stdout.buffer.write(serializer.Dump(encoded))" serialized >>= checkPythonResult let deserialised = decode py_stdout::Text.JSON.Result [(String, Int)] decoded <- case deserialised of Text.JSON.Ok jobs' -> return jobs' Error msg -> assertFailure ("Unable to decode jobs: " ++ msg) -- this already raised an exception, but we need it -- for proper types >> fail "Unable to decode jobs" assertEqual "Mismatch in number of returned jobs" (length decoded) (length jobs) mapM_ (\(py_sp, job) -> let hs_sp = (jobStatusToRaw $ calcJobStatus job, calcJobPriority job) in assertEqual ("Different result after encoding/decoding for " ++ show job) hs_sp py_sp ) $ zip decoded jobs -- | Tests listing of Job ids. prop_ListJobIDs :: Property prop_ListJobIDs = monadicIO $ do let extractJobIDs :: (Show e, MonadFail m) => m (GenericResult e a) -> m a extractJobIDs = (>>= genericResult (fail . show) return) jobs <- pick $ resize 10 (listOf1 genJobId `suchThat` (\l -> l == nub l)) (e, f, g) <- run . withSystemTempDirectory "jqueue-test-ListJobIDs." $ \tempdir -> do empty_dir <- extractJobIDs $ getJobIDs [tempdir] mapM_ (\jid -> writeFile (tempdir jobFileName jid) "") jobs full_dir <- extractJobIDs $ getJobIDs [tempdir] invalid_dir <- getJobIDs [tempdir "no-such-dir"] return (empty_dir, sortJobIDs full_dir, invalid_dir) _ <- stop $ conjoin [ counterexample "empty directory" $ e ==? [] , counterexample "directory with valid names" $ f ==? sortJobIDs jobs , counterexample "invalid directory" $ isBad g ] return () -- | Tests loading jobs from disk. prop_LoadJobs :: Property prop_LoadJobs = monadicIO $ do ops <- pick $ resize 5 (listOf1 genQueuedOpCode) jid <- pick genJobId let job = QueuedJob jid ops justNoTs justNoTs justNoTs Nothing Nothing job_s = encode job -- check that jobs in the right directories are parsed correctly (missing, current, archived, missing_current, broken) <- run . withSystemTempDirectory "jqueue-test-LoadJobs." $ \tempdir -> do let load a = loadJobFromDisk tempdir a jid live_path = liveJobFile tempdir jid arch_path = archivedJobFile tempdir jid createDirectory $ tempdir jobQueueArchiveSubDir createDirectory $ dropFileName arch_path -- missing job missing <- load True writeFile live_path job_s -- this should exist current <- load False removeFile live_path writeFile arch_path job_s -- this should exist (archived) archived <- load True -- this should be missing missing_current <- load False removeFile arch_path writeFile live_path "invalid job" broken <- load True return (missing, current, archived, missing_current, broken) _ <- stop $ conjoin [ missing ==? noSuchJob , current ==? Ganeti.BasicTypes.Ok (job, False) , archived ==? Ganeti.BasicTypes.Ok (job, True) , missing_current ==? noSuchJob , counterexample "broken job" (isBad broken) ] return () -- | Tests computing job directories. Creates random directories, -- files and stale symlinks in a directory, and checks that we return -- \"the right thing\". prop_DetermineDirs :: Property prop_DetermineDirs = monadicIO $ do count <- pick $ choose (2, 10) nums <- pick $ genUniquesList count (arbitrary::Gen (QuickCheck.Positive Int)) let (valid, invalid) = splitAt (count `div` 2) $ map (\(QuickCheck.Positive i) -> show i) nums (tempdir, non_arch, with_arch, invalid_root) <- run . withSystemTempDirectory "jqueue-test-DetermineDirs." $ \tempdir -> do let arch_dir = tempdir jobQueueArchiveSubDir createDirectory arch_dir mapM_ (createDirectory . (arch_dir )) valid mapM_ (\p -> writeFile (arch_dir p) "") invalid mapM_ (\p -> createSymbolicLink "/dev/null/no/such/file" (arch_dir p <.> "missing")) invalid non_arch <- determineJobDirectories tempdir False with_arch <- determineJobDirectories tempdir True invalid_root <- determineJobDirectories (tempdir "no-such-subdir") True return (tempdir, non_arch, with_arch, invalid_root) let arch_dir = tempdir jobQueueArchiveSubDir _ <- stop $ conjoin [ non_arch ==? [tempdir] , sort with_arch ==? sort (tempdir:map (arch_dir ) valid) , invalid_root ==? [tempdir "no-such-subdir"] ] return () -- | Tests the JSON serialisation for 'InputOpCode'. prop_InputOpCode :: MetaOpCode -> Int -> Property prop_InputOpCode meta i = conjoin [ readJSON (showJSON valid) ==? Text.JSON.Ok valid , readJSON (showJSON invalid) ==? Text.JSON.Ok invalid ] where valid = ValidOpCode meta invalid = InvalidOpCode (showJSON i) -- | Tests 'extractOpSummary'. prop_extractOpSummary :: MetaOpCode -> Int -> Property prop_extractOpSummary meta i = conjoin [ counterexample "valid opcode" $ extractOpSummary (ValidOpCode meta) ==? summary , counterexample "invalid opcode, correct object" $ extractOpSummary (InvalidOpCode jsobj) ==? summary , counterexample "invalid opcode, empty object" $ extractOpSummary (InvalidOpCode emptyo) ==? invalid , counterexample "invalid opcode, object with invalid OP_ID" $ extractOpSummary (InvalidOpCode invobj) ==? invalid , counterexample "invalid opcode, not jsobject" $ extractOpSummary (InvalidOpCode jsinval) ==? invalid ] where summary = opSummary (metaOpCode meta) jsobj = showJSON $ toJSObject [("OP_ID", showJSON ("OP_" ++ summary))] emptyo = showJSON $ toJSObject ([]::[(String, JSValue)]) invobj = showJSON $ toJSObject [("OP_ID", showJSON False)] jsinval = showJSON i invalid = "INVALID_OP" testSuite "JQueue" [ 'case_JobPriorityDef , 'prop_JobPriority , 'case_JobStatusDef , 'prop_JobStatus , 'case_JobStatusPri_py_equiv , 'prop_ListJobIDs , 'prop_LoadJobs , 'prop_DetermineDirs , 'prop_InputOpCode , 'prop_extractOpSummary ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/JQueue/000075500000000000000000000000001476477700300205215ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/JQueue/Objects.hs000064400000000000000000000045461476477700300224570ustar00rootroot00000000000000{-| Unittests for 'Ganeti.JQueue.Objects'. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.JQueue.Objects ( justNoTs , genQueuedOpCode , emptyJob , genJobId ) where import Control.Monad.Fail (MonadFail) import Test.QuickCheck as QuickCheck import Text.JSON import Test.Ganeti.Types () import Test.Ganeti.OpCodes () import qualified Ganeti.Constants as C import Ganeti.JQueue import Ganeti.Types as Types -- | noTimestamp in Just form. justNoTs :: Maybe Timestamp justNoTs = Just noTimestamp -- | Generates a simple queued opcode. genQueuedOpCode :: Gen QueuedOpCode genQueuedOpCode = QueuedOpCode <$> (ValidOpCode <$> arbitrary) <*> arbitrary <*> pure JSNull <*> pure [] <*> choose (C.opPrioLowest, C.opPrioHighest) <*> pure justNoTs <*> pure justNoTs <*> pure justNoTs -- | Generates an static, empty job. emptyJob :: (MonadFail m) => m QueuedJob emptyJob = do jid0 <- makeJobId 0 return $ QueuedJob jid0 [] justNoTs justNoTs justNoTs Nothing Nothing -- | Generates a job ID. genJobId :: Gen JobId genJobId = do p <- arbitrary::Gen (Types.NonNegative Int) makeJobId $ fromNonNegative p ganeti-3.1.0~rc2/test/hs/Test/Ganeti/JSON.hs000064400000000000000000000117401476477700300204330ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.JSON (testJSON) where import Control.Monad import Data.List import Test.HUnit import Test.QuickCheck import qualified Text.JSON as J import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Types () import qualified Ganeti.BasicTypes as BasicTypes import Ganeti.JSON (nestedAccessByKey, nestedAccessByKeyDotted) import qualified Ganeti.JSON as JSON {-# ANN module "HLint: ignore Use camelCase" #-} instance (Arbitrary a) => Arbitrary (JSON.MaybeForJSON a) where arbitrary = liftM JSON.MaybeForJSON arbitrary instance Arbitrary JSON.TimeAsDoubleJSON where arbitrary = liftM JSON.TimeAsDoubleJSON arbitrary prop_toArray :: [Int] -> Property prop_toArray intarr = let arr = map J.showJSON intarr in case JSON.toArray (J.JSArray arr) of BasicTypes.Ok arr' -> arr ==? arr' BasicTypes.Bad err -> failTest $ "Failed to parse array: " ++ err prop_toArrayFail :: Int -> String -> Bool -> Property prop_toArrayFail i s b = -- poor man's instance Arbitrary JSValue forAll (elements [J.showJSON i, J.showJSON s, J.showJSON b]) $ \item -> case JSON.toArray item::BasicTypes.Result [J.JSValue] of BasicTypes.Bad _ -> passTest BasicTypes.Ok result -> failTest $ "Unexpected parse, got " ++ show result arrayMaybeToJson :: (J.JSON a) => [Maybe a] -> String -> JSON.JSRecord arrayMaybeToJson xs k = [(k, J.JSArray $ map sh xs)] where sh x = case x of Just v -> J.showJSON v Nothing -> J.JSNull prop_arrayMaybeFromObj :: String -> [Maybe Int] -> String -> Property prop_arrayMaybeFromObj t xs k = case JSON.tryArrayMaybeFromObj t (arrayMaybeToJson xs k) k of BasicTypes.Ok xs' -> xs' ==? xs BasicTypes.Bad e -> failTest $ "Parsing failing, got: " ++ show e prop_arrayMaybeFromObjFail :: String -> String -> Property prop_arrayMaybeFromObjFail t k = case JSON.tryArrayMaybeFromObj t [] k of BasicTypes.Ok r -> property (fail $ "Unexpected result, got: " ++ show (r::[Maybe Int]) :: Gen Property) BasicTypes.Bad e -> conjoin [ Data.List.isInfixOf t e ==? True , Data.List.isInfixOf k e ==? True ] prop_MaybeForJSON_serialisation :: JSON.MaybeForJSON String -> Property prop_MaybeForJSON_serialisation = testSerialisation prop_TimeAsDoubleJSON_serialisation :: JSON.TimeAsDoubleJSON -> Property prop_TimeAsDoubleJSON_serialisation = testSerialisation isJError :: J.Result a -> Bool isJError (J.Error _) = True isJError _ = False case_nestedAccessByKey :: Assertion case_nestedAccessByKey = do J.Ok v <- return $ J.decode "{\"key1\": {\"key2\": \"val\"}}" nestedAccessByKey [] v @?= J.Ok v nestedAccessByKey ["key1", "key2"] v @?= J.Ok (J.JSString $ J.toJSString "val") assertBool "access to nonexistent key should fail" . isJError $ nestedAccessByKey ["key1", "nonexistent"] v case_nestedAccessByKeyDotted :: Assertion case_nestedAccessByKeyDotted = do J.Ok v <- return $ J.decode "{\"key1\": {\"key2\": \"val\"}}" assertBool "access to empty key should fail" . isJError $ nestedAccessByKeyDotted "" v nestedAccessByKeyDotted "key1.key2" v @?= J.Ok (J.JSString $ J.toJSString "val") assertBool "access to nonexistent key should fail" . isJError $ nestedAccessByKeyDotted "key1.nonexistent" v testSuite "JSON" [ 'prop_toArray , 'prop_toArrayFail , 'prop_arrayMaybeFromObj , 'prop_arrayMaybeFromObjFail , 'prop_MaybeForJSON_serialisation , 'prop_TimeAsDoubleJSON_serialisation , 'case_nestedAccessByKey , 'case_nestedAccessByKeyDotted ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Jobs.hs000064400000000000000000000030351476477700300205550ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Jobs (testJobs) where import Test.Ganeti.TestHelper {-# ANN module "HLint: ignore Unused LANGUAGE pragma" #-} testSuite "Jobs" [ ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Kvmd.hs000064400000000000000000000102421476477700300205570ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittests for the KVM daemon. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Kvmd (testKvmd) where import Control.Concurrent import Control.Exception (try) import qualified Network.Socket as Socket import System.Directory import System.FilePath import System.IO import qualified Ganeti.Kvmd as Kvmd import qualified Ganeti.UDSServer as UDSServer import Test.HUnit as HUnit import qualified Test.Ganeti.TestHelper as TestHelper (testSuite) import qualified Test.Ganeti.TestCommon as TestCommon (getTempFileName) import qualified Ganeti.Logging as Logging {-# ANN module "HLint: ignore Use camelCase" #-} startKvmd :: FilePath -> IO ThreadId startKvmd dir = forkIO (do Logging.setupLogging Nothing "ganeti-kvmd" False False False Logging.SyslogNo Kvmd.startWith dir) stopKvmd :: ThreadId -> IO () stopKvmd = killThread delayKvmd :: IO () delayKvmd = threadDelay 1000000 detectShutdown :: (Handle -> IO ()) -> IO Bool detectShutdown putFn = do monitorDir <- TestCommon.getTempFileName "ganeti" let monitor = "instance" <.> Kvmd.monitorExtension monitorFile = monitorDir monitor shutdownFile = Kvmd.shutdownPath monitorFile -- ensure the KVM directory exists createDirectoryIfMissing True monitorDir -- ensure the shutdown file does not exist (try (removeFile shutdownFile) :: IO (Either IOError ())) >> return () -- start KVM daemon threadId <- startKvmd monitorDir threadDelay 1000 -- create a Unix socket sock <- UDSServer.openServerSocket monitorFile Socket.listen sock 1 handle <- UDSServer.acceptSocket sock -- read 'qmp_capabilities' message res <- try . hGetLine $ handle :: IO (Either IOError String) case res of Left err -> assertFailure $ "Expecting " ++ show Kvmd.monitorGreeting ++ ", received " ++ show err Right str -> Kvmd.monitorGreeting @=? str -- send Qmp messages putFn handle hFlush handle -- close the Unix socket UDSServer.closeClientSocket handle UDSServer.closeServerSocket sock monitorFile -- KVM needs time to create the shutdown file delayKvmd -- stop the KVM daemon stopKvmd threadId -- check for shutdown file doesFileExist shutdownFile case_DetectAdminShutdown :: Assertion case_DetectAdminShutdown = do res <- detectShutdown putMessage assertBool "Detected user shutdown instead of administrator shutdown" $ not res where putMessage handle = do hPrint handle "POWERDOWN" hPrint handle "SHUTDOWN" case_DetectUserShutdown :: Assertion case_DetectUserShutdown = do res <- detectShutdown putMessage assertBool "Detected administrator shutdown instead of user shutdown" res where putMessage handle = hPrint handle "SHUTDOWN" TestHelper.testSuite "Kvmd" [ 'case_DetectAdminShutdown , 'case_DetectUserShutdown ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Locking/000075500000000000000000000000001476477700300207115ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Locking/Allocation.hs000064400000000000000000000351151476477700300233370ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Tests for lock allocation. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Locking.Allocation ( testLocking_Allocation , TestLock , TestOwner , requestSucceeded ) where import qualified Data.Foldable as F import qualified Data.Map as M import Data.Maybe (fromMaybe) import qualified Data.Set as S import qualified Text.JSON as J import Test.QuickCheck import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Ganeti.BasicTypes import Ganeti.Locking.Allocation import Ganeti.Locking.Types {- Ganeti.Locking.Allocation is polymorphic in the types of locks and lock owners. So we can use much simpler types here than Ganeti's real locks and lock owners, knowning that polymorphic functions cannot exploit the simplicity of the types they're deling with. -} data TestOwner = TestOwner Int deriving (Ord, Eq, Show) instance Arbitrary TestOwner where arbitrary = TestOwner <$> choose (0, 2) data TestLock = TestBigLock | TestCollectionLockA | TestLockA Int | TestCollectionLockB | TestLockB Int deriving (Ord, Eq, Show, Read) instance Arbitrary TestLock where arbitrary = frequency [ (1, elements [ TestBigLock , TestCollectionLockA , TestCollectionLockB ]) , (2, TestLockA <$> choose (0, 2)) , (2, TestLockB <$> choose (0, 2)) ] instance Lock TestLock where lockImplications (TestLockA _) = [TestCollectionLockA, TestBigLock] lockImplications (TestLockB _) = [TestCollectionLockB, TestBigLock] lockImplications TestBigLock = [] lockImplications _ = [TestBigLock] {- All states of a LockAllocation ever available outside the Ganeti.Locking.Allocation module must be constructed by starting with emptyAllocation and applying the exported functions. -} instance Arbitrary OwnerState where arbitrary = elements [OwnShared, OwnExclusive] instance Arbitrary a => Arbitrary (LockRequest a) where arbitrary = LockRequest <$> arbitrary <*> genMaybe arbitrary data UpdateRequest b a = UpdateRequest b [LockRequest a] | FreeLockRequest b deriving Show instance (Arbitrary a, Arbitrary b) => Arbitrary (UpdateRequest a b) where arbitrary = frequency [ (4, UpdateRequest <$> arbitrary <*> (choose (1, 4) >>= vector)) , (1, FreeLockRequest <$> arbitrary) ] -- | Transform an UpdateRequest into the corresponding state transformer. asAllocTrans :: (Lock a, Ord b, Show b) => LockAllocation a b -> UpdateRequest b a -> LockAllocation a b asAllocTrans state (UpdateRequest owner updates) = fst $ updateLocks owner updates state asAllocTrans state (FreeLockRequest owner) = freeLocks state owner -- | Fold a sequence of requests to transform a lock allocation onto the empty -- allocation. As we consider all exported LockAllocation transformers, any -- LockAllocation definable is obtained in this way. foldUpdates :: (Lock a, Ord b, Show b) => [UpdateRequest b a] -> LockAllocation a b foldUpdates = foldl asAllocTrans emptyAllocation instance (Arbitrary a, Lock a, Arbitrary b, Ord b, Show b) => Arbitrary (LockAllocation a b) where arbitrary = foldUpdates <$> (choose (0, 8) >>= vector) -- | Basic property of locking: the exclusive locks of one user -- are disjoint from any locks of any other user. prop_LocksDisjoint :: Property prop_LocksDisjoint = forAll (arbitrary :: Gen (LockAllocation TestLock TestOwner)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \a -> forAll (arbitrary `suchThat` (/= a)) $ \b -> let aExclusive = M.keysSet . M.filter (== OwnExclusive) $ listLocks a state bAll = M.keysSet $ listLocks b state in counterexample (show a ++ "'s exclusive lock" ++ " is not respected by " ++ show b) (S.null $ S.intersection aExclusive bAll) -- | Verify that the list of active locks indeed contains all locks that -- are owned by someone. prop_LockslistComplete :: Property prop_LockslistComplete = forAll (arbitrary :: Gen TestOwner) $ \a -> forAll ((arbitrary :: Gen (LockAllocation TestLock TestOwner)) `suchThat` (not . M.null . listLocks a)) $ \state -> counterexample "All owned locks must be mentioned in the all-locks list" $ let allLocks = listAllLocks state in all (`elem` allLocks) (M.keys $ listLocks a state) -- | Verify that the list of all locks with states is contained in the list -- of all locks. prop_LocksAllOwnersSubsetLockslist :: Property prop_LocksAllOwnersSubsetLockslist = forAll (arbitrary :: Gen (LockAllocation TestLock TestOwner)) $ \state -> counterexample "The list of all active locks must contain all locks mentioned\ \ in the locks state" $ S.isSubsetOf (S.fromList . map fst $ listAllLocksOwners state) (S.fromList $ listAllLocks state) -- | Verify that all locks of all owners are mentioned in the list of all locks' -- owner's state. prop_LocksAllOwnersComplete :: Property prop_LocksAllOwnersComplete = forAll (arbitrary :: Gen TestOwner) $ \a -> forAll ((arbitrary :: Gen (LockAllocation TestLock TestOwner)) `suchThat` (not . M.null . listLocks a)) $ \state -> counterexample "Owned locks must be mentioned in list of all locks' state" $ let allLocksState = listAllLocksOwners state in flip all (M.toList $ listLocks a state) $ \(lock, ownership) -> elem (a, ownership) . fromMaybe [] $ lookup lock allLocksState -- | Verify that all lock owners mentioned in the list of all locks' owner's -- state actually own their lock. prop_LocksAllOwnersSound :: Property prop_LocksAllOwnersSound = forAll ((arbitrary :: Gen (LockAllocation TestLock TestOwner)) `suchThat` (not . null . listAllLocksOwners)) $ \state -> counterexample "All locks mentioned in listAllLocksOwners must be owned by\ \ the mentioned owner" . flip all (listAllLocksOwners state) $ \(lock, owners) -> flip all owners $ \(owner, ownership) -> holdsLock owner lock ownership state -- | Verify that exclusive group locks are honored, i.e., verify that if someone -- holds a lock, then no one else can hold a lock on an exclusive lock on an -- implied lock. prop_LockImplicationX :: Property prop_LockImplicationX = forAll (arbitrary :: Gen (LockAllocation TestLock TestOwner)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \a -> forAll (arbitrary `suchThat` (/= a)) $ \b -> let bExclusive = M.keysSet . M.filter (== OwnExclusive) $ listLocks b state in counterexample "Others cannot have an exclusive lock on an implied lock" . flip all (M.keys $ listLocks a state) $ \lock -> flip all (lockImplications lock) $ \impliedlock -> not $ S.member impliedlock bExclusive -- | Verify that shared group locks are honored, i.e., verify that if someone -- holds an exclusive lock, then no one else can hold any form on lock on an -- implied lock. prop_LockImplicationS :: Property prop_LockImplicationS = forAll (arbitrary :: Gen (LockAllocation TestLock TestOwner)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \a -> forAll (arbitrary `suchThat` (/= a)) $ \b -> let aExclusive = M.keys . M.filter (== OwnExclusive) $ listLocks a state bAll = M.keysSet $ listLocks b state in counterexample "Others cannot hold locks implied by an exclusive lock" . flip all aExclusive $ \lock -> flip all (lockImplications lock) $ \impliedlock -> not $ S.member impliedlock bAll -- | Verify that locks can only be modified by updates of the owner. prop_LocksStable :: Property prop_LocksStable = forAll (arbitrary :: Gen (LockAllocation TestLock TestOwner)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \a -> forAll (arbitrary `suchThat` (/= a)) $ \b -> forAll (arbitrary :: Gen [LockRequest TestLock]) $ \request -> let (state', _) = updateLocks b request state in (listLocks a state ==? listLocks a state') -- | Verify that a given request is statisfied in list of owned locks requestSucceeded :: Ord a => M.Map a OwnerState -> LockRequest a -> Bool requestSucceeded owned (LockRequest lock status) = M.lookup lock owned == status -- | Verify that lock updates are atomic, i.e., either we get all the required -- locks, or the state is completely unchanged. prop_LockupdateAtomic :: Property prop_LockupdateAtomic = forAll (arbitrary :: Gen (LockAllocation TestLock TestOwner)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \a -> forAll (arbitrary :: Gen [LockRequest TestLock]) $ \request -> let (state', result) = updateLocks a request state in if result == Ok S.empty then counterexample ("Update succeeded, but in final state " ++ show state' ++ "not all locks are as requested") $ let owned = listLocks a state' in all (requestSucceeded owned) request else counterexample ("Update failed, but state changed to " ++ show state') (state == state') -- | Verify that releasing a lock always succeeds. prop_LockReleaseSucceeds :: Property prop_LockReleaseSucceeds = forAll (arbitrary :: Gen (LockAllocation TestLock TestOwner)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \a -> forAll (arbitrary :: Gen TestLock) $ \lock -> let (_, result) = updateLocks a [requestRelease lock] state in counterexample ("Releasing a lock has to suceed uncondiationally, but got " ++ show result) (isOk result) -- | Verify the property that only the blocking owners prevent -- lock allocation. We deliberatly go for the expensive variant -- restraining by suchThat, as otherwise the number of cases actually -- covered is too small. prop_BlockSufficient :: Property prop_BlockSufficient = forAll (arbitrary :: Gen TestOwner) $ \a -> forAll (arbitrary :: Gen TestLock) $ \lock -> forAll (elements [ [requestShared lock] , [requestExclusive lock]]) $ \request -> forAll ((arbitrary :: Gen (LockAllocation TestLock TestOwner)) `suchThat` (genericResult (const False) (not . S.null) . snd . updateLocks a request)) $ \state -> let (_, result) = updateLocks a request state blockedOn = genericResult (const S.empty) id result in counterexample "After all blockers release, a request must succeed" . isOk . snd . updateLocks a request $ F.foldl freeLocks state blockedOn -- | Verify the property that every blocking owner is necessary, i.e., even -- if we only keep the locks of one of the blocking owners, the request still -- will be blocked. We deliberatly use the expensive variant of restraining -- to ensure good coverage. To make sure the request can always be blocked -- by two owners, for a shared request we request two different locks. prop_BlockNecessary :: Property prop_BlockNecessary = forAll (arbitrary :: Gen TestOwner) $ \a -> forAll (arbitrary :: Gen TestLock) $ \lock -> forAll (arbitrary `suchThat` (/= lock)) $ \lock' -> forAll (elements [ [requestShared lock, requestShared lock'] , [requestExclusive lock]]) $ \request -> forAll ((arbitrary :: Gen (LockAllocation TestLock TestOwner)) `suchThat` (genericResult (const False) ((>= 2) . S.size) . snd . updateLocks a request)) $ \state -> let (_, result) = updateLocks a request state blockers = genericResult (const S.empty) id result in counterexample "Each blocker alone must block the request" . flip all (S.elems blockers) $ \blocker -> (==) (Ok $ S.singleton blocker) . snd . updateLocks a request . F.foldl freeLocks state $ S.filter (/= blocker) blockers instance J.JSON TestOwner where showJSON (TestOwner x) = J.showJSON x readJSON = (>>= return . TestOwner) . J.readJSON instance J.JSON TestLock where showJSON = J.showJSON . show readJSON = (>>= return . read) . J.readJSON -- | Verify that for LockAllocation we have readJSON . showJSON = Ok. prop_ReadShow :: Property prop_ReadShow = forAll (arbitrary :: Gen (LockAllocation TestLock TestOwner)) $ \state -> J.readJSON (J.showJSON state) ==? J.Ok state -- | Verify that the list of lock owners is complete. prop_OwnerComplete :: Property prop_OwnerComplete = forAll (arbitrary :: Gen (LockAllocation TestLock TestOwner)) $ \state -> foldl freeLocks state (lockOwners state) ==? emptyAllocation -- | Verify that each owner actually owns a lock. prop_OwnerSound :: Property prop_OwnerSound = forAll ((arbitrary :: Gen (LockAllocation TestLock TestOwner)) `suchThat` (not . null . lockOwners)) $ \state -> counterexample "All subjects listed as owners must own at least one lock" . flip all (lockOwners state) $ \owner -> not . M.null $ listLocks owner state -- | Verify that for LockRequest we have readJSON . showJSON = Ok. prop_ReadShowRequest :: Property prop_ReadShowRequest = forAll (arbitrary :: Gen (LockRequest TestLock)) $ \state -> J.readJSON (J.showJSON state) ==? J.Ok state testSuite "Locking/Allocation" [ 'prop_LocksDisjoint , 'prop_LockslistComplete , 'prop_LocksAllOwnersSubsetLockslist , 'prop_LocksAllOwnersComplete , 'prop_LocksAllOwnersSound , 'prop_LockImplicationX , 'prop_LockImplicationS , 'prop_LocksStable , 'prop_LockupdateAtomic , 'prop_LockReleaseSucceeds , 'prop_BlockSufficient , 'prop_BlockNecessary , 'prop_ReadShow , 'prop_OwnerComplete , 'prop_OwnerSound , 'prop_ReadShowRequest ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Locking/Locks.hs000064400000000000000000000105741476477700300223270ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Tests for the lock data structure -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Locking.Locks (testLocking_Locks) where import Control.Applicative (liftA2) import Control.Monad (liftM) import System.Posix.Types (CPid) import Test.QuickCheck import Text.JSON import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Types () import Ganeti.Locking.Locks import Ganeti.Locking.Types instance Arbitrary GanetiLocks where arbitrary = oneof [ return BGL , return ClusterLockSet , return InstanceLockSet , Instance <$> genFQDN , return NodeGroupLockSet , NodeGroup <$> genUUID , return NodeResLockSet , NodeRes <$> genUUID , return NodeLockSet , Node <$> genUUID , return NetworkLockSet , Network <$> genUUID ] -- | Verify that readJSON . showJSON = Ok prop_ReadShow :: Property prop_ReadShow = forAll (arbitrary :: Gen GanetiLocks) $ \a -> readJSON (showJSON a) ==? Ok a -- | Verify the implied locks are earlier in the lock order. prop_ImpliedOrder :: Property prop_ImpliedOrder = forAll ((arbitrary :: Gen GanetiLocks) `suchThat` (not . null . lockImplications)) $ \b -> counterexample "Implied locks must be earlier in the lock order" . flip all (lockImplications b) $ \a -> a < b -- | Verify the intervall property of the locks. prop_ImpliedIntervall :: Property prop_ImpliedIntervall = forAll ((arbitrary :: Gen GanetiLocks) `suchThat` (not . null . lockImplications)) $ \b -> forAll (elements $ lockImplications b) $ \a -> forAll (arbitrary `suchThat` liftA2 (&&) (a <) (<= b)) $ \x -> counterexample ("Locks between a group and a member of the group" ++ " must also belong to the group") $ a `elem` lockImplications x instance Arbitrary LockLevel where arbitrary = elements [LevelCluster ..] -- | Verify that readJSON . showJSON = Ok for lock levels prop_ReadShowLevel :: Property prop_ReadShowLevel = forAll (arbitrary :: Gen LockLevel) $ \a -> readJSON (showJSON a) ==? Ok a instance Arbitrary ClientType where arbitrary = oneof [ ClientOther <$> arbitrary , ClientJob <$> arbitrary ] -- | Verify that readJSON . showJSON = Ok for ClientType prop_ReadShow_ClientType :: Property prop_ReadShow_ClientType = forAll (arbitrary :: Gen ClientType) $ \a -> readJSON (showJSON a) ==? Ok a instance Arbitrary CPid where arbitrary = liftM fromIntegral (arbitrary :: Gen Integer) instance Arbitrary ClientId where arbitrary = ClientId <$> arbitrary <*> arbitrary <*> arbitrary -- | Verify that readJSON . showJSON = Ok for ClientId prop_ReadShow_ClientId :: Property prop_ReadShow_ClientId = forAll (arbitrary :: Gen ClientId) $ \a -> readJSON (showJSON a) ==? Ok a testSuite "Locking/Locks" [ 'prop_ReadShow , 'prop_ImpliedOrder , 'prop_ImpliedIntervall , 'prop_ReadShowLevel , 'prop_ReadShow_ClientType , 'prop_ReadShow_ClientId ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Locking/Waiting.hs000064400000000000000000000463201476477700300226540ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Tests for lock waiting structure. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Locking.Waiting (testLocking_Waiting) where import Control.Applicative (liftA2) import Control.Monad (liftM) import qualified Data.Map as M import qualified Data.Set as S import qualified Text.JSON as J import Test.QuickCheck import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Test.Ganeti.Locking.Allocation (TestLock, TestOwner, requestSucceeded) import Ganeti.BasicTypes (isBad, genericResult, runListHead) import Ganeti.Locking.Allocation (LockRequest, listLocks) import qualified Ganeti.Locking.Allocation as L import Ganeti.Locking.Types (Lock) import Ganeti.Locking.Waiting {- Ganeti.Locking.Waiting is polymorphic in the types of locks, lock owners, and priorities. So we can use much simpler types here than Ganeti's real locks and lock owners, knowning that polymorphic functions cannot exploit the simplicity of the types they're deling with. To avoid code duplication, we use the test structure from Test.Ganeti.Locking.Allocation. -} {- All states of a LockWaiting ever available outside the module can be obtained from @emptyWaiting@ applying one of the update operations. -} data UpdateRequest a b c = Update b [LockRequest a] | UpdateWaiting c b [LockRequest a] | RemovePending b | IntersectRequest b [a] | OpportunisticUnion b [(a, L.OwnerState)] deriving Show instance (Arbitrary a, Arbitrary b, Arbitrary c) => Arbitrary (UpdateRequest a b c) where arbitrary = frequency [ (2, Update <$> arbitrary <*> (choose (1, 4) >>= vector)) , (4, UpdateWaiting <$> arbitrary <*> arbitrary <*> (choose (1, 4) >>= vector)) , (1, RemovePending <$> arbitrary) , (1, IntersectRequest <$> arbitrary <*> (choose (1, 4) >>= vector)) , (1, OpportunisticUnion <$> arbitrary <*> (choose (1, 4) >>= vector)) ] -- | Transform an UpdateRequest into the corresponding state transformer. asWaitingTrans :: (Lock a, Ord b, Ord c) => LockWaiting a b c -> UpdateRequest a b c -> LockWaiting a b c asWaitingTrans state (Update owner req) = fst $ updateLocks owner req state asWaitingTrans state (UpdateWaiting prio owner req) = fst $ updateLocksWaiting prio owner req state asWaitingTrans state (RemovePending owner) = removePendingRequest owner state asWaitingTrans state (IntersectRequest owner locks) = fst $ intersectLocks locks owner state asWaitingTrans state (OpportunisticUnion owner locks) = fst $ opportunisticLockUnion owner locks state -- | Fold a sequence of requests to transform a waiting strucutre onto the -- empty waiting. As we consider all exported transformations, any waiting -- structure can be obtained this way. foldUpdates :: (Lock a, Ord b, Ord c) => [UpdateRequest a b c] -> LockWaiting a b c foldUpdates = foldl asWaitingTrans emptyWaiting instance (Arbitrary a, Lock a, Arbitrary b, Ord b, Arbitrary c, Ord c) => Arbitrary (LockWaiting a b c) where arbitrary = foldUpdates <$> (choose (0, 8) >>= vector) -- | Verify that an owner with a pending request cannot make any -- changes to the lock structure. prop_NoActionWithPendingRequests :: Property prop_NoActionWithPendingRequests = forAll (arbitrary :: Gen TestOwner) $ \a -> forAll ((arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) `suchThat` (S.member a . getPendingOwners)) $ \state -> forAll (arbitrary :: Gen [LockRequest TestLock]) $ \req -> forAll arbitrary $ \prio -> counterexample "Owners with pending requests may not update locks" . all (isBad . fst . snd) $ [updateLocks, updateLocksWaiting prio] <*> [a] <*> [req] <*> [state] -- | Quantifier for blocked requests. Quantifies over the generic situation -- that there is a state, an owner, and a request that is blocked for that -- owner. To obtain such a situation, we use the fact that there must be a -- different owner having at least one lock. forAllBlocked :: (Testable prop) => (LockWaiting TestLock TestOwner Integer -- State -> TestOwner -- The owner of the blocked request -> Integer -- The priority -> [LockRequest TestLock] -- Request -> prop) -> Property forAllBlocked predicate = forAll (arbitrary :: Gen TestOwner) $ \a -> forAll (arbitrary :: Gen Integer) $ \prio -> forAll (arbitrary `suchThat` (/=) a) $ \b -> forAll ((arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) `suchThat` foldl (liftA2 (&&)) (const True) [ not . S.member a . getPendingOwners , M.null . listLocks a . getAllocation , not . M.null . listLocks b . getAllocation]) $ \state -> forAll ((arbitrary :: Gen [LockRequest TestLock]) `suchThat` (genericResult (const False) (not . S.null) . fst . snd . flip (updateLocksWaiting prio a) state)) $ \req -> predicate state a prio req -- | Verify that an owner has a pending request after a waiting request -- not fullfilled immediately. prop_WaitingRequestsGetPending :: Property prop_WaitingRequestsGetPending = forAllBlocked $ \state owner prio req -> counterexample "After a not immediately fulfilled waiting request, owner\ \ must have a pending request" . S.member owner . getPendingOwners . fst $ updateLocksWaiting prio owner req state -- | Verify that pending requests get fullfilled once all blockers release -- their resources. prop_PendingGetFulfilledEventually :: Property prop_PendingGetFulfilledEventually = forAllBlocked $ \state owner prio req -> let oldpending = getPendingOwners state (state', (resultBlockers, _)) = updateLocksWaiting prio owner req state blockers = genericResult (const S.empty) id resultBlockers state'' = S.foldl (\s a -> fst $ releaseResources a s) state' $ S.union oldpending blockers finallyOwned = listLocks owner $ getAllocation state'' in counterexample "After all blockers and old pending owners give up their\ \ resources, a pending request must be granted\ \ automatically" $ all (requestSucceeded finallyOwned) req -- | Verify that the owner of a pending request gets notified once all blockers -- release their resources. prop_PendingGetNotifiedEventually :: Property prop_PendingGetNotifiedEventually = forAllBlocked $ \state owner prio req -> let oldpending = getPendingOwners state (state', (resultBlockers, _)) = updateLocksWaiting prio owner req state blockers = genericResult (const S.empty) id resultBlockers releaseOneOwner (s, tonotify) o = let (s', newnotify) = releaseResources o s in (s', newnotify `S.union` tonotify) (_, notified) = S.foldl releaseOneOwner (state', S.empty) $ S.union oldpending blockers in counterexample "After all blockers and old pending owners give up their\ \ resources, a pending owner must be notified" $ S.member owner notified -- | Verify that some progress is made after the direct blockers give up their -- locks. Note that we cannot guarantee that the original requester gets its -- request granted, as someone else might have a more important priority. prop_Progress :: Property prop_Progress = forAllBlocked $ \state owner prio req -> let (state', (resultBlockers, _)) = updateLocksWaiting prio owner req state blockers = genericResult (const S.empty) id resultBlockers releaseOneOwner (s, tonotify) o = let (s', newnotify) = releaseResources o s in (s', newnotify `S.union` tonotify) (_, notified) = S.foldl releaseOneOwner (state', S.empty) blockers in counterexample "Some progress must be made after all blockers release\ \ their locks" . not . S.null $ notified S.\\ blockers -- | Verify that the notifications send out are sound, i.e., upon notification -- the requests actually are fulfilled. To be sure to have at least one -- notification we, again, use the scenario that a request is blocked and then -- all the blockers release their resources. prop_ProgressSound :: Property prop_ProgressSound = forAllBlocked $ \state owner prio req -> let (state', (resultBlockers, _)) = updateLocksWaiting prio owner req state blockers = genericResult (const S.empty) id resultBlockers releaseOneOwner (s, tonotify) o = let (s', newnotify) = releaseResources o s in (s', newnotify `S.union` tonotify) (state'', notified) = S.foldl releaseOneOwner (state', S.empty) blockers requestFulfilled o = runListHead False (\(_, _, r) -> all (requestSucceeded . listLocks o $ getAllocation state'') r) . S.toList . S.filter (\(_, b, _) -> b == o) . getPendingRequests $ state' in counterexample "If an owner gets notified, his request must be satisfied" . all requestFulfilled . S.toList $ notified S.\\ blockers -- | Verify that all pending requests are valid and cannot be fulfilled in -- the underlying lock allocation. prop_PendingJustified :: Property prop_PendingJustified = forAll ((arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) `suchThat` (not . S.null . getPendingRequests)) $ \state -> let isJustified (_, b, req) = genericResult (const False) (not . S.null) . snd . L.updateLocks b req $ getAllocation state in counterexample "Pending requests must be good and not fulfillable" . all isJustified . S.toList $ getPendingRequests state -- | Verify that `updateLocks` is idempotent, except that in the repetition, -- no waiters are notified. prop_UpdateIdempotent :: Property prop_UpdateIdempotent = forAll (arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \owner -> forAll (arbitrary :: Gen [LockRequest TestLock]) $ \req -> let (state', (answer', _)) = updateLocks owner req state (state'', (answer'', nfy)) = updateLocks owner req state' in conjoin [ counterexample ("repeated updateLocks waiting gave different\ \ answers: " ++ show answer' ++ " /= " ++ show answer'') $ answer' == answer'' , counterexample "updateLocks not idempotent" $ extRepr state' == extRepr state'' , counterexample ("notifications (" ++ show nfy ++ ") on replay") $ S.null nfy ] -- | Verify that extRepr . fromExtRepr = id for all valid extensional -- representations. prop_extReprPreserved :: Property prop_extReprPreserved = forAll (arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) $ \state -> let rep = extRepr state rep' = extRepr $ fromExtRepr rep in counterexample "a lock waiting obtained from an extensional representation\ \ must have the same extensional representation" $ rep' == rep -- | Verify that any state is indistinguishable from its canonical version -- (i.e., the one obtained from the extensional representation) with respect -- to updateLocks. prop_SimulateUpdateLocks :: Property prop_SimulateUpdateLocks = forAll (arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \owner -> forAll (arbitrary :: Gen [LockRequest TestLock]) $ \req -> let state' = fromExtRepr $ extRepr state (finState, (result, notify)) = updateLocks owner req state (finState', (result', notify')) = updateLocks owner req state' in counterexample "extRepr-equal states must behave equal on updateLocks" $ and [ result == result' , notify == notify' , extRepr finState == extRepr finState' ] -- | Verify that any state is indistinguishable from its canonical version -- (i.e., the one obtained from the extensional representation) with respect -- to updateLocksWaiting. prop_SimulateUpdateLocksWaiting :: Property prop_SimulateUpdateLocksWaiting = forAll (arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \owner -> forAll (arbitrary :: Gen Integer) $ \prio -> forAll (arbitrary :: Gen [LockRequest TestLock]) $ \req -> let state' = fromExtRepr $ extRepr state (finState, (result, notify)) = updateLocksWaiting prio owner req state (finState', (result', notify')) = updateLocksWaiting prio owner req state' in counterexample "extRepr-equal states must behave equal on updateLocks" $ and [ result == result' , notify == notify' , extRepr finState == extRepr finState' ] -- | Verify that if a requestor has no pending requests, `safeUpdateWaiting` -- conincides with `updateLocksWaiting`. prop_SafeUpdateWaitingCorrect :: Property prop_SafeUpdateWaitingCorrect = forAll (arbitrary :: Gen TestOwner) $ \owner -> forAll ((arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) `suchThat` (not . hasPendingRequest owner)) $ \state -> forAll (arbitrary :: Gen Integer) $ \prio -> forAll (arbitrary :: Gen [LockRequest TestLock]) $ \req -> let (state', answer') = updateLocksWaiting prio owner req state (state'', answer'') = safeUpdateLocksWaiting prio owner req state in conjoin [ counterexample ("safeUpdateLocksWaiting gave different answer: " ++ show answer' ++ " /= " ++ show answer'') $ answer' == answer'' , counterexample ("safeUpdateLocksWaiting gave different states\ \ after answer " ++ show answer' ++ ": " ++ show (extRepr state') ++ " /= " ++ show (extRepr state'')) $ extRepr state' == extRepr state'' ] -- | Verify that `safeUpdateLocksWaiting` is idempotent, that in the repetition -- no notifications are done. prop_SafeUpdateWaitingIdempotent :: Property prop_SafeUpdateWaitingIdempotent = forAll (arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \owner -> forAll (arbitrary :: Gen Integer) $ \prio -> forAll (arbitrary :: Gen [LockRequest TestLock]) $ \req -> let (state', (answer', _)) = safeUpdateLocksWaiting prio owner req state (state'', (answer'', nfy)) = safeUpdateLocksWaiting prio owner req state' in conjoin [ counterexample ("repeated safeUpdateLocks waiting gave different\ \ answers: " ++ show answer' ++ " /= " ++ show answer'') $ answer' == answer'' , counterexample "safeUpdateLocksWaiting not idempotent" $ extRepr state' == extRepr state'' , counterexample ("notifications (" ++ show nfy ++ ") on replay") $ S.null nfy ] -- | Verify that for LockWaiting we have readJSON . showJSON is extensionally -- equivalent to Ok. prop_ReadShow :: Property prop_ReadShow = forAll (arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) $ \state -> (liftM extRepr . J.readJSON $ J.showJSON state) ==? (J.Ok $ extRepr state) -- | Verify that opportunistic union only increases the locks held. prop_OpportunisticMonotone :: Property prop_OpportunisticMonotone = forAll (arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \a -> forAll ((choose (1,3) >>= vector) :: Gen [(TestLock, L.OwnerState)]) $ \req -> let (state', _) = opportunisticLockUnion a req state oldOwned = listLocks a $ getAllocation state oldLocks = M.keys oldOwned newOwned = listLocks a $ getAllocation state' in counterexample "Opportunistic union may only increase the set of locks\ \ held" . flip all oldLocks $ \lock -> M.lookup lock newOwned >= M.lookup lock oldOwned -- | Verify the result list of the opportunistic union: if a lock is not in -- the result that, than its state has not changed, and if it is, it is as -- requested. The latter property is tested in that liberal way, so that we -- really can take arbitrary requests, including those that require both, shared -- and exlusive state for the same lock. prop_OpportunisticAnswer :: Property prop_OpportunisticAnswer = forAll (arbitrary :: Gen (LockWaiting TestLock TestOwner Integer)) $ \state -> forAll (arbitrary :: Gen TestOwner) $ \a -> forAll ((choose (1,3) >>= vector) :: Gen [(TestLock, L.OwnerState)]) $ \req -> let (state', (result, _)) = opportunisticLockUnion a req state oldOwned = listLocks a $ getAllocation state newOwned = listLocks a $ getAllocation state' involvedLocks = M.keys oldOwned ++ map fst req in conjoin [ counterexample ("Locks not in the answer set " ++ show result ++ " may not be changed, but found " ++ show state') . flip all involvedLocks $ \lock -> (lock `elem` result) || (M.lookup lock oldOwned == M.lookup lock newOwned) , counterexample ("Locks not in the answer set " ++ show result ++ " must be as requested, but found " ++ show state') . flip all involvedLocks $ \lock -> notElem lock result || maybe False (flip elem req . (,) lock) (M.lookup lock newOwned) ] testSuite "Locking/Waiting" [ 'prop_NoActionWithPendingRequests , 'prop_WaitingRequestsGetPending , 'prop_PendingGetFulfilledEventually , 'prop_PendingGetNotifiedEventually , 'prop_Progress , 'prop_ProgressSound , 'prop_PendingJustified , 'prop_extReprPreserved , 'prop_UpdateIdempotent , 'prop_SimulateUpdateLocks , 'prop_SimulateUpdateLocksWaiting , 'prop_ReadShow , 'prop_SafeUpdateWaitingCorrect , 'prop_SafeUpdateWaitingIdempotent , 'prop_OpportunisticMonotone , 'prop_OpportunisticAnswer ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Luxi.hs000064400000000000000000000153321476477700300206040ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Luxi (testLuxi) where import Test.HUnit import Test.QuickCheck import Test.QuickCheck.Monadic (monadicIO, run, stop) import Data.List import Control.Concurrent (forkIO) import Control.Exception (bracket) import qualified Text.JSON as J import Test.Ganeti.OpCodes () import Test.Ganeti.Query.Language (genFilter) import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Test.Ganeti.Types (genReasonTrail) import Ganeti.BasicTypes import qualified Ganeti.Luxi as Luxi import qualified Ganeti.UDSServer as US {-# ANN module "HLint: ignore Use camelCase" #-} -- * Luxi tests $(genArbitrary ''Luxi.LuxiReq) instance Arbitrary Luxi.LuxiOp where arbitrary = do lreq <- arbitrary case lreq of Luxi.ReqQuery -> Luxi.Query <$> arbitrary <*> genFields <*> genFilter Luxi.ReqQueryFields -> Luxi.QueryFields <$> arbitrary <*> genFields Luxi.ReqQueryNodes -> Luxi.QueryNodes <$> listOf genFQDN <*> genFields <*> arbitrary Luxi.ReqQueryGroups -> Luxi.QueryGroups <$> arbitrary <*> arbitrary <*> arbitrary Luxi.ReqQueryNetworks -> Luxi.QueryNetworks <$> arbitrary <*> arbitrary <*> arbitrary Luxi.ReqQueryInstances -> Luxi.QueryInstances <$> listOf genFQDN <*> genFields <*> arbitrary Luxi.ReqQueryFilters -> Luxi.QueryFilters <$> arbitrary <*> genFields Luxi.ReqReplaceFilter -> Luxi.ReplaceFilter <$> genMaybe genUUID <*> arbitrary <*> arbitrary <*> arbitrary <*> genReasonTrail Luxi.ReqDeleteFilter -> Luxi.DeleteFilter <$> genUUID Luxi.ReqQueryJobs -> Luxi.QueryJobs <$> arbitrary <*> genFields Luxi.ReqQueryExports -> Luxi.QueryExports <$> listOf genFQDN <*> arbitrary Luxi.ReqQueryConfigValues -> Luxi.QueryConfigValues <$> genFields Luxi.ReqQueryClusterInfo -> pure Luxi.QueryClusterInfo Luxi.ReqQueryTags -> do kind <- arbitrary Luxi.QueryTags kind <$> genLuxiTagName kind Luxi.ReqSubmitJob -> Luxi.SubmitJob <$> resize maxOpCodes arbitrary Luxi.ReqSubmitJobToDrainedQueue -> Luxi.SubmitJobToDrainedQueue <$> resize maxOpCodes arbitrary Luxi.ReqSubmitManyJobs -> Luxi.SubmitManyJobs <$> resize maxOpCodes arbitrary Luxi.ReqWaitForJobChange -> Luxi.WaitForJobChange <$> arbitrary <*> genFields <*> pure J.JSNull <*> pure J.JSNull <*> arbitrary Luxi.ReqPickupJob -> Luxi.PickupJob <$> arbitrary Luxi.ReqArchiveJob -> Luxi.ArchiveJob <$> arbitrary Luxi.ReqAutoArchiveJobs -> Luxi.AutoArchiveJobs <$> arbitrary <*> arbitrary Luxi.ReqCancelJob -> Luxi.CancelJob <$> arbitrary <*> arbitrary Luxi.ReqChangeJobPriority -> Luxi.ChangeJobPriority <$> arbitrary <*> arbitrary Luxi.ReqSetDrainFlag -> Luxi.SetDrainFlag <$> arbitrary Luxi.ReqSetWatcherPause -> Luxi.SetWatcherPause <$> arbitrary -- | Simple check that encoding/decoding of LuxiOp works. prop_CallEncoding :: Luxi.LuxiOp -> Property prop_CallEncoding op = (US.parseCall (US.buildCall (Luxi.strOfOp op) (Luxi.opToArgs op)) >>= uncurry Luxi.decodeLuxiCall) ==? Ok op -- | Server ping-pong helper. luxiServerPong :: Luxi.Client -> IO () luxiServerPong c = do msg <- Luxi.recvMsgExt c case msg of Luxi.RecvOk m -> Luxi.sendMsg c m >> luxiServerPong c _ -> return () -- | Client ping-pong helper. luxiClientPong :: Luxi.Client -> [String] -> IO [String] luxiClientPong c = mapM (\m -> Luxi.sendMsg c m >> Luxi.recvMsg c) -- | Monadic check that, given a server socket, we can connect via a -- client to it, and that we can send a list of arbitrary messages and -- get back what we sent. prop_ClientServer :: [[DNSChar]] -> Property prop_ClientServer dnschars = monadicIO $ do let msgs = map (map dnsGetChar) dnschars fpath <- run $ getTempFileName "luxitest" -- we need to create the server first, otherwise (if we do it in the -- forked thread) the client could try to connect to it before it's -- ready server <- run $ Luxi.getLuxiServer False fpath -- fork the server responder _ <- run . forkIO $ bracket (Luxi.acceptClient server) (\c -> Luxi.closeClient c >> Luxi.closeServer server) luxiServerPong replies <- run $ bracket (Luxi.getLuxiClient fpath) Luxi.closeClient (`luxiClientPong` msgs) _ <- stop $ replies ==? msgs return () -- | Check that Python and Haskell define the same Luxi requests list. case_AllDefined :: Assertion case_AllDefined = do py_stdout <- runPython "from ganeti import luxi\n\ \print('\\n'.join(luxi.REQ_ALL))" "" >>= checkPythonResult let py_ops = sort $ lines py_stdout hs_ops = Luxi.allLuxiCalls extra_py = py_ops \\ hs_ops extra_hs = hs_ops \\ py_ops assertBool ("Luxi calls missing from Haskell code:\n" ++ unlines extra_py) (null extra_py) assertBool ("Extra Luxi calls in the Haskell code:\n" ++ unlines extra_hs) (null extra_hs) testSuite "Luxi" [ 'prop_CallEncoding , 'prop_ClientServer , 'case_AllDefined ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Network.hs000064400000000000000000000052411476477700300213120ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, TypeSynonymInstances, FlexibleInstances #-} {-# OPTIONS_GHC -fno-warn-orphans #-} module Test.Ganeti.Network ( testNetwork , genBitStringMaxLen ) where import Data.Maybe (fromMaybe) import Test.QuickCheck import Ganeti.Network as Network import Ganeti.Objects as Objects import Ganeti.Objects.BitArray as BA import Test.Ganeti.Objects ( genBitStringMaxLen ) import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper -- * Test cases -- | Check that the address pool's properties are calculated correctly. prop_addressPoolProperties :: Network -> Property prop_addressPoolProperties a = conjoin [ counterexample ("Not all reservations are included in 'allReservations' of " ++ "address pool:" ++ show a) (allReservationsSubsumesInternal a) , counterexample ("Not all external reservations are covered by 'allReservations' " ++ "of address pool: " ++ show a) (allReservationsSubsumesExternal a) , counterexample ("The counts of free and reserved addresses do not add up for " ++ "address pool: " ++ show a) (checkCounts a) , counterexample ("'isFull' wrongly classified the status of the address pool: " ++ show a) (checkIsFull a) , counterexample ("Network map is inconsistent with reservations of address pool: " ++ show a) (checkGetMap a) ] -- | Checks for the subset relation on 'Maybe' values. subsetMaybe :: Maybe BitArray -> Maybe BitArray -> Bool subsetMaybe (Just x) (Just y) = subset x y subsetMaybe x y = x == y -- only if they're both Nothing -- | Check that all internally reserved ips are included in 'allReservations'. allReservationsSubsumesInternal :: Network -> Bool allReservationsSubsumesInternal a = reservations a `subsetMaybe` allReservations a -- | Check that all externally reserved ips are included in 'allReservations'. allReservationsSubsumesExternal :: Network -> Bool allReservationsSubsumesExternal a = extReservations a `subsetMaybe` allReservations a -- | Check that the counts of free and reserved ips add up. checkCounts :: Network -> Property checkCounts a = netIpv4NumHosts a ==? toInteger (getFreeCount a + getReservedCount a) -- | Check that the detection of a full network works correctly. checkIsFull :: Network -> Property checkIsFull a = isFull a ==? maybe True (and . toList) (allReservations a) -- | Check that the map representation of the network corresponds to the -- network's reservations. checkGetMap :: Network -> Property checkGetMap a = fromMaybe BA.empty (allReservations a) ==? fromList (Prelude.map (== 'X') (getMap a)) testSuite "Network" [ 'prop_addressPoolProperties ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Objects.hs000064400000000000000000000726011476477700300212560ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, TypeSynonymInstances, FlexibleInstances, OverloadedStrings #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Objects ( testObjects , Node(..) , genConfigDataWithNetworks , genDisk , genDiskWithChildren , genEmptyCluster , genInst , genInstWithNets , genValidNetwork , genBitStringMaxLen ) where import Test.QuickCheck import qualified Test.HUnit as HUnit import Control.Monad import qualified Data.ByteString as BS import qualified Data.ByteString.UTF8 as UTF8 import Data.Char import qualified Data.List as List import qualified Data.Map as Map import Data.Maybe (fromMaybe) import Data.Word (Word32) import GHC.Exts (IsString(..)) import System.Time (ClockTime(..)) import qualified Text.JSON as J import Test.Ganeti.Query.Language () import Test.Ganeti.SlotMap (genSlotLimit) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Types () import qualified Ganeti.Constants as C import qualified Ganeti.ConstantUtils as CU import Ganeti.Network import Ganeti.Objects as Objects import qualified Ganeti.Objects.BitArray as BA import Ganeti.JSON import Ganeti.Types -- * Arbitrary instances instance Arbitrary (Container DataCollectorConfig) where arbitrary = do let names = map UTF8.fromString $ CU.toList C.dataCollectorNames activations <- vector $ length names timeouts <- vector $ length names let configs = zipWith DataCollectorConfig activations timeouts return GenericContainer { fromContainer = Map.fromList $ zip names configs } instance Arbitrary BS.ByteString where arbitrary = genPrintableByteString instance Arbitrary a => Arbitrary (Private a) where arbitrary = Private <$> arbitrary $(genArbitrary ''PartialNDParams) instance Arbitrary (Container J.JSValue) where arbitrary = return $ GenericContainer Map.empty instance Arbitrary Node where arbitrary = Node <$> genFQDN <*> genFQDN <*> genFQDN <*> arbitrary <*> arbitrary <*> arbitrary <*> genFQDN <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> fmap UTF8.fromString genUUID <*> arbitrary <*> arbitrary $(genArbitrary ''BlockDriver) $(genArbitrary ''DiskMode) instance Arbitrary LogicalVolume where arbitrary = LogicalVolume <$> validName <*> validName where validName = -- we intentionally omit '.' and '-' to avoid forbidden names listOf1 $ elements (['a'..'z'] ++ ['A'..'Z'] ++ ['0'..'9'] ++ "+_") instance Arbitrary DiskLogicalId where arbitrary = oneof [ LIDPlain <$> arbitrary , LIDDrbd8 <$> genFQDN <*> genFQDN <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary , LIDFile <$> arbitrary <*> arbitrary , LIDBlockDev <$> arbitrary <*> arbitrary , LIDRados <$> arbitrary <*> arbitrary ] -- | 'Disk' 'arbitrary' instance. Since we don't test disk hierarchy -- properties, we only generate disks with no children (FIXME), as -- generating recursive datastructures is a bit more work. instance Arbitrary Disk where arbitrary = frequency [ (2, liftM RealDisk $ RealDiskData <$> arbitrary <*> pure [] <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary) , (1, liftM ForthcomingDisk $ ForthcomingDiskData <$> arbitrary <*> pure [] <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary) ] -- FIXME: we should generate proper values, >=0, etc., but this is -- hard for partial ones, where all must be wrapped in a 'Maybe' $(genArbitrary ''PartialBeParams) $(genArbitrary ''AdminState) $(genArbitrary ''AdminStateSource) $(genArbitrary ''PartialNicParams) $(genArbitrary ''PartialNic) instance Arbitrary ForthcomingInstanceData where arbitrary = ForthcomingInstanceData -- name <$> genMaybe genFQDN -- primary node <*> genMaybe genFQDN -- OS <*> genMaybe genFQDN -- hypervisor <*> arbitrary -- hvparams -- FIXME: add non-empty hvparams when they're a proper type <*> pure (GenericContainer Map.empty) -- beparams <*> arbitrary -- osparams <*> pure (GenericContainer Map.empty) -- osparams_private <*> pure (GenericContainer Map.empty) -- admin_state <*> genMaybe arbitrary -- admin_state_source <*> genMaybe arbitrary -- nics <*> arbitrary -- disks <*> vectorOf 5 arbitrary -- disks active <*> genMaybe arbitrary -- network port <*> arbitrary -- ts <*> arbitrary <*> arbitrary -- uuid <*> arbitrary -- serial <*> arbitrary -- tags <*> arbitrary instance Arbitrary RealInstanceData where arbitrary = RealInstanceData -- name <$> genFQDN -- primary node <*> genFQDN -- OS <*> genFQDN -- hypervisor <*> arbitrary -- hvparams -- FIXME: add non-empty hvparams when they're a proper type <*> pure (GenericContainer Map.empty) -- beparams <*> arbitrary -- osparams <*> pure (GenericContainer Map.empty) -- osparams_private <*> pure (GenericContainer Map.empty) -- admin_state <*> arbitrary -- admin_state_source <*> arbitrary -- nics <*> arbitrary -- disks <*> vectorOf 5 arbitrary -- disks active <*> arbitrary -- network port <*> arbitrary -- ts <*> arbitrary <*> arbitrary -- uuid <*> arbitrary -- serial <*> arbitrary -- tags <*> arbitrary instance Arbitrary Instance where arbitrary = frequency [ (1, ForthcomingInstance <$> arbitrary) , (3, RealInstance <$> arbitrary) ] -- | Generates an instance that is connected to the given networks -- and possibly some other networks genInstWithNets :: [String] -> Gen Instance genInstWithNets nets = do plain_inst <- RealInstance <$> arbitrary enhanceInstWithNets plain_inst nets -- | Generates an instance that is connected to some networks genInst :: Gen Instance genInst = genInstWithNets [] -- | Enhances a given instance with network information, by connecting it to the -- given networks and possibly some other networks enhanceInstWithNets :: Instance -> [String] -> Gen Instance enhanceInstWithNets inst nets = do mac <- arbitrary ip <- arbitrary nicparams <- arbitrary name <- arbitrary uuid <- arbitrary -- generate some more networks than the given ones num_more_nets <- choose (0,3) more_nets <- vectorOf num_more_nets genUUID let genNic net = PartialNic mac ip nicparams net name uuid partial_nics = map (genNic . Just) (List.nub (nets ++ more_nets)) new_inst = case inst of RealInstance rinst -> RealInstance rinst { realInstNics = partial_nics } ForthcomingInstance _ -> inst return new_inst genDiskWithChildren :: Int -> Gen Disk genDiskWithChildren num_children = do logicalid <- arbitrary children <- vectorOf num_children (genDiskWithChildren 0) nodes <- arbitrary ivname <- genName size <- arbitrary mode <- arbitrary name <- genMaybe genName spindles <- arbitrary params <- arbitrary uuid <- fmap UTF8.fromString genUUID serial <- arbitrary time <- arbitrary return . RealDisk $ RealDiskData logicalid children nodes ivname size mode name spindles params uuid serial time time genDisk :: Gen Disk genDisk = genDiskWithChildren 3 -- | FIXME: This generates completely random data, without normal -- validation rules. $(genArbitrary ''PartialISpecParams) $(genArbitrary ''FilledISpecParams) $(genArbitrary ''MinMaxISpecs) $(genArbitrary ''FilledIPolicy) $(genArbitrary ''IpFamily) $(genArbitrary ''FilledNDParams) $(genArbitrary ''FilledNicParams) $(genArbitrary ''FilledBeParams) -- | FIXME: This generates completely random data, without normal -- validation rules. $(genArbitrary ''PartialIPolicy) -- | No real arbitrary instance for 'ClusterHvParams' yet. instance Arbitrary ClusterHvParams where arbitrary = return $ GenericContainer Map.empty -- | No real arbitrary instance for 'OsHvParams' yet. instance Arbitrary OsHvParams where arbitrary = return $ GenericContainer Map.empty -- | No real arbitrary instance for 'GroupDiskParams' yet. instance Arbitrary GroupDiskParams where arbitrary = return $ GenericContainer Map.empty instance Arbitrary ClusterNicParams where arbitrary = (GenericContainer . Map.singleton (UTF8.fromString C.ppDefault)) <$> arbitrary instance Arbitrary OsParams where arbitrary = (GenericContainer . Map.fromList) <$> arbitrary instance Arbitrary Objects.ClusterOsParamsPrivate where arbitrary = (GenericContainer . Map.fromList) <$> arbitrary instance Arbitrary ClusterOsParams where arbitrary = (GenericContainer . Map.fromList) <$> arbitrary instance Arbitrary ClusterBeParams where arbitrary = (GenericContainer . Map.fromList) <$> arbitrary $(genArbitrary ''Cluster) instance Arbitrary ConfigData where arbitrary = genEmptyCluster 0 >>= genConfigDataWithNetworks instance Arbitrary AddressPool where arbitrary = AddressPool . BA.fromList <$> arbitrary instance Arbitrary Network where arbitrary = genValidNetwork instance Arbitrary FilterAction where arbitrary = oneof [ pure Accept , pure Pause , pure Reject , pure Continue , RateLimit <$> genSlotLimit ] instance Arbitrary FilterPredicate where arbitrary = oneof [ FPJobId <$> arbitrary , FPOpCode <$> arbitrary , FPReason <$> arbitrary ] instance Arbitrary FilterRule where arbitrary = FilterRule <$> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> fmap UTF8.fromString genUUID instance Arbitrary SshKeyType where arbitrary = oneof [ pure RSA , pure DSA , pure ECDSA ] -- | Generates a network instance with minimum netmasks of /24. Generating -- bigger networks slows down the tests, because long bit strings are generated -- for the reservations. genValidNetwork :: Gen Objects.Network genValidNetwork = do -- generate netmask for the IPv4 network netmask <- fromIntegral <$> choose (24::Int, 30) let hostLen = 32-netmask name <- genName >>= mkNonEmpty mac_prefix <- genMaybe genName net <- Objects.ip4AddressFromNumber . (* 2^hostLen) . fromIntegral <$> choose (1::Int, 2^netmask-1) net6 <- genMaybe genIp6Net gateway <- genMaybe arbitrary gateway6 <- genMaybe genIp6Addr res <- liftM Just (genBitString $ netmask2NumHosts netmask) ext_res <- liftM Just (genBitString $ netmask2NumHosts netmask) uuid <- arbitrary ctime <- arbitrary mtime <- arbitrary let n = Network name mac_prefix (mkIp4Network net netmask) net6 gateway gateway6 res ext_res uuid ctime mtime 0 emptyTagSet return n -- | Generate an arbitrary string consisting of '0' and '1' of the given length. genBitString :: Int -> Gen AddressPool genBitString len = (AddressPool . BA.fromList) `liftM` vectorOf len (elements [False, True]) -- | Generate an arbitrary string consisting of '0' and '1' of the maximum given -- length. genBitStringMaxLen :: Int -> Gen AddressPool genBitStringMaxLen maxLen = choose (0, maxLen) >>= genBitString -- | Generator for config data with an empty cluster (no instances), -- with N defined nodes. genEmptyCluster :: Int -> Gen ConfigData genEmptyCluster ncount = do nodes <- vector ncount version <- arbitrary grp <- arbitrary let guuid = uuidOf grp nodes' = zipWith (\n idx -> let newname = takeWhile (/= '.') (nodeName n) ++ "-" ++ show idx in ( UTF8.fromString newname , n { nodeGroup = guuid, nodeName = newname})) nodes [(1::Int)..] nodemap = Map.fromList nodes' contnodes = if Map.size nodemap /= ncount then error ("Inconsistent node map, duplicates in" ++ " node name list? Names: " ++ show (map fst nodes')) else GenericContainer nodemap continsts = GenericContainer Map.empty networks = GenericContainer Map.empty disks = GenericContainer Map.empty filters = GenericContainer Map.empty let contgroups = GenericContainer $ Map.singleton (UTF8.fromString guuid) grp serial <- arbitrary -- timestamp fields ctime <- arbitrary mtime <- arbitrary cluster <- resize 8 arbitrary let c = ConfigData version cluster contnodes contgroups continsts networks disks filters ctime mtime serial return c -- | FIXME: make an even simpler base version of creating a cluster. -- | Generates config data with a couple of networks. genConfigDataWithNetworks :: ConfigData -> Gen ConfigData genConfigDataWithNetworks old_cfg = do num_nets <- choose (0, 3) -- generate a list of network names (no duplicates) net_names <- genUniquesList num_nets genName >>= mapM mkNonEmpty -- generate a random list of networks (possibly with duplicate names) nets <- vectorOf num_nets genValidNetwork -- use unique names for the networks let nets_unique = map ( \(name, net) -> net { networkName = name } ) (zip net_names nets) net_map = GenericContainer $ Map.fromList (map (\n -> (UTF8.fromString $ uuidOf n, n)) nets_unique) new_cfg = old_cfg { configNetworks = net_map } return new_cfg -- * Test properties -- | Tests that fillDict behaves correctly prop_fillDict :: [(Int, Int)] -> [(Int, Int)] -> Property prop_fillDict defaults custom = let d_map = Map.fromList defaults d_keys = map fst defaults c_map = Map.fromList custom c_keys = map fst custom in conjoin [ counterexample "Empty custom filling" (fillDict d_map Map.empty [] == d_map) , counterexample "Empty defaults filling" (fillDict Map.empty c_map [] == c_map) , counterexample "Delete all keys" (fillDict d_map c_map (d_keys++c_keys) == Map.empty) ] prop_LogicalVolume_serialisation :: LogicalVolume -> Property prop_LogicalVolume_serialisation = testSerialisation prop_LogicalVolume_deserialisationFail :: Property prop_LogicalVolume_deserialisationFail = conjoin . map (testDeserialisationFail (LogicalVolume "" "")) $ [ J.JSArray [] , J.JSString $ J.toJSString "/abc" , J.JSString $ J.toJSString "abc/" , J.JSString $ J.toJSString "../." , J.JSString $ J.toJSString "g/snapshot" , J.JSString $ J.toJSString "g/a_mimagex" , J.JSString $ J.toJSString "g/r;3" ] -- | Test that the serialisation of 'DiskLogicalId', which is -- implemented manually, is idempotent. Since we don't have a -- standalone JSON instance for DiskLogicalId (it's a data type that -- expands over two fields in a JSObject), we test this by actially -- testing entire Disk serialisations. So this tests two things at -- once, basically. prop_Disk_serialisation :: Disk -> Property prop_Disk_serialisation = testSerialisation prop_Disk_array_serialisation :: Disk -> Property prop_Disk_array_serialisation = testArraySerialisation -- | Check that node serialisation is idempotent. prop_Node_serialisation :: Node -> Property prop_Node_serialisation = testSerialisation -- | Check that instance serialisation is idempotent. prop_Inst_serialisation :: Instance -> Property prop_Inst_serialisation = testSerialisation -- | Check that address pool serialisation is idempotent. prop_AddressPool_serialisation :: AddressPool -> Property prop_AddressPool_serialisation = testSerialisation -- | Check that network serialisation is idempotent. prop_Network_serialisation :: Network -> Property prop_Network_serialisation = testSerialisation -- | Check that filter action serialisation is idempotent. prop_FilterAction_serialisation :: FilterAction -> Property prop_FilterAction_serialisation = testSerialisation -- | Check that filter predicate serialisation is idempotent. prop_FilterPredicate_serialisation :: FilterPredicate -> Property prop_FilterPredicate_serialisation = testSerialisation -- | Check config serialisation. prop_Config_serialisation :: Property prop_Config_serialisation = forAll (choose (0, maxNodes `div` 4) >>= genEmptyCluster) testSerialisation -- | Custom HUnit test to check the correspondence between Haskell-generated -- networks and their Python decoded, validated and re-encoded version. -- For the technical background of this unit test, check the documentation -- of "case_py_compat_types" of test/hs/Test/Ganeti/Opcodes.hs casePyCompatNetworks :: HUnit.Assertion casePyCompatNetworks = do let num_networks = 500::Int networks <- genSample (vectorOf num_networks genValidNetwork) let networks_with_properties = map getNetworkProperties networks serialized = J.encode networks -- check for non-ASCII fields, usually due to 'arbitrary :: String' mapM_ (\net -> when (any (not . isAscii) (J.encode net)) . HUnit.assertFailure $ "Network has non-ASCII fields: " ++ show net ) networks py_stdout <- runPython "from ganeti import network\n\ \from ganeti import objects\n\ \from ganeti import serializer\n\ \import sys\n\ \net_data = serializer.Load(sys.stdin.read())\n\ \decoded = [objects.Network.FromDict(n) for n in net_data]\n\ \encoded = []\n\ \for net in decoded:\n\ \ a = network.AddressPool(net)\n\ \ encoded.append((a.GetFreeCount(), a.GetReservedCount(), \\\n\ \ net.ToDict()))\n\ \sys.stdout.buffer.write(serializer.Dump(encoded))" serialized >>= checkPythonResult let deserialised = J.decode py_stdout::J.Result [(Int, Int, Network)] decoded <- case deserialised of J.Ok ops -> return ops J.Error msg -> HUnit.assertFailure ("Unable to decode networks: " ++ msg) -- this already raised an expection, but we need it -- for proper types >> fail "Unable to decode networks" HUnit.assertEqual "Mismatch in number of returned networks" (length decoded) (length networks_with_properties) mapM_ (uncurry (HUnit.assertEqual "Different result after encoding/decoding") ) $ zip networks_with_properties decoded -- | Creates a tuple of the given network combined with some of its properties -- to be compared against the same properties generated by the python code. getNetworkProperties :: Network -> (Int, Int, Network) getNetworkProperties net = (getFreeCount net, getReservedCount net, net) -- | Tests the compatibility between Haskell-serialized node groups and their -- python-decoded and encoded version. casePyCompatNodegroups :: HUnit.Assertion casePyCompatNodegroups = do let num_groups = 500::Int groups <- genSample (vectorOf num_groups genNodeGroup) let serialized = J.encode groups -- check for non-ASCII fields, usually due to 'arbitrary :: String' mapM_ (\group -> when (any (not . isAscii) (J.encode group)) . HUnit.assertFailure $ "Node group has non-ASCII fields: " ++ show group ) groups py_stdout <- runPython "from ganeti import objects\n\ \from ganeti import serializer\n\ \import sys\n\ \group_data = serializer.Load(sys.stdin.read())\n\ \decoded = [objects.NodeGroup.FromDict(g) for g in group_data]\n\ \encoded = [g.ToDict() for g in decoded]\n\ \sys.stdout.buffer.write(serializer.Dump(encoded))" serialized >>= checkPythonResult let deserialised = J.decode py_stdout::J.Result [NodeGroup] decoded <- case deserialised of J.Ok ops -> return ops J.Error msg -> HUnit.assertFailure ("Unable to decode node groups: " ++ msg) -- this already raised an expection, but we need it -- for proper types >> fail "Unable to decode node groups" HUnit.assertEqual "Mismatch in number of returned node groups" (length decoded) (length groups) mapM_ (uncurry (HUnit.assertEqual "Different result after encoding/decoding") ) $ zip groups decoded -- | Generates a node group with up to 3 networks. -- | FIXME: This generates still somewhat completely random data, without normal -- validation rules. genNodeGroup :: Gen NodeGroup genNodeGroup = do name <- genFQDN members <- pure [] ndparams <- arbitrary alloc_policy <- arbitrary ipolicy <- arbitrary diskparams <- pure (GenericContainer Map.empty) num_networks <- choose (0, 3) net_uuid_list <- vectorOf num_networks (arbitrary::Gen BS.ByteString) nic_param_list <- vectorOf num_networks (arbitrary::Gen PartialNicParams) net_map <- pure (GenericContainer . Map.fromList $ zip net_uuid_list nic_param_list) hv_state <- arbitrary disk_state <- arbitrary -- timestamp fields ctime <- arbitrary mtime <- arbitrary uuid <- genFQDN `suchThat` (/= name) serial <- arbitrary tags <- arbitrary let group = NodeGroup name members ndparams alloc_policy ipolicy diskparams net_map hv_state disk_state ctime mtime (UTF8.fromString uuid) serial tags return group instance Arbitrary NodeGroup where arbitrary = genNodeGroup instance Arbitrary Ip4Address where arbitrary = liftM mkIp4Address $ (,,,) <$> choose (0, 255) <*> choose (0, 255) <*> choose (0, 255) <*> choose (0, 255) $(genArbitrary ''Ip4Network) -- | Tests conversions of ip addresses from/to numbers. prop_ip4AddressAsNum :: Ip4Address -> Property prop_ip4AddressAsNum ip4 = ip4AddressFromNumber (ip4AddressToNumber ip4) ==? ip4 -- | Tests that the number produced by 'ip4AddressToNumber' has the correct -- order of bytes. prop_ip4AddressToNumber :: Word32 -> Property prop_ip4AddressToNumber w = let byte :: Int -> Word32 byte i = (w `div` (256^i)) `mod` 256 ipaddr = List.intercalate "." $ map (show . byte) [3,2..0] in ip4AddressToNumber <$> readIp4Address ipaddr ==? (return (toInteger w) :: Either String Integer) -- | IsString instance for 'Ip4Address', to help write the tests. instance IsString Ip4Address where fromString s = fromMaybe (error $ "Failed to parse address from " ++ s) (readIp4Address s) -- | Tests a few simple cases of IPv4 next address. caseNextIp4Address :: HUnit.Assertion caseNextIp4Address = do HUnit.assertEqual "" "0.0.0.1" $ nextIp4Address "0.0.0.0" HUnit.assertEqual "" "0.0.0.0" $ nextIp4Address "255.255.255.255" HUnit.assertEqual "" "1.2.3.5" $ nextIp4Address "1.2.3.4" HUnit.assertEqual "" "1.3.0.0" $ nextIp4Address "1.2.255.255" HUnit.assertEqual "" "1.2.255.63" $ nextIp4Address "1.2.255.62" -- | Tests the compatibility between Haskell-serialized instances and their -- python-decoded and encoded version. -- Note: this can be enhanced with logical validations on the decoded objects casePyCompatInstances :: HUnit.Assertion casePyCompatInstances = do let num_inst = 500::Int instances <- genSample (vectorOf num_inst genInst) let serialized = J.encode instances -- check for non-ASCII fields, usually due to 'arbitrary :: String' mapM_ (\inst -> when (any (not . isAscii) (J.encode inst)) . HUnit.assertFailure $ "Instance has non-ASCII fields: " ++ show inst ) instances py_stdout <- runPython "from ganeti import objects\n\ \from ganeti import serializer\n\ \import sys\n\ \inst_data = serializer.Load(sys.stdin.read())\n\ \decoded = [objects.Instance.FromDict(i) for i in inst_data]\n\ \encoded = [i.ToDict() for i in decoded]\n\ \sys.stdout.buffer.write(serializer.Dump(encoded))" serialized >>= checkPythonResult let deserialised = J.decode py_stdout::J.Result [Instance] decoded <- case deserialised of J.Ok ops -> return ops J.Error msg -> HUnit.assertFailure ("Unable to decode instance: " ++ msg) -- this already raised an expection, but we need it -- for proper types >> fail "Unable to decode instances" HUnit.assertEqual "Mismatch in number of returned instances" (length decoded) (length instances) mapM_ (uncurry (HUnit.assertEqual "Different result after encoding/decoding") ) $ zip instances decoded -- | A helper function for creating 'LIDPlain' values. mkLIDPlain :: String -> String -> DiskLogicalId mkLIDPlain = (LIDPlain .) . LogicalVolume -- | Tests that the logical ID is correctly found in a plain disk caseIncludeLogicalIdPlain :: HUnit.Assertion caseIncludeLogicalIdPlain = let vg_name = "xenvg" :: String lv_name = "1234sdf-qwef-2134-asff-asd2-23145d.data" :: String lv = LogicalVolume vg_name lv_name time = TOD 0 0 d = RealDisk $ RealDiskData (LIDPlain lv) [] ["node1.example.com"] "diskname" 1000 DiskRdWr Nothing Nothing Nothing "asdfgr-1234-5123-daf3-sdfw-134f43" 0 time time in HUnit.assertBool "Unable to detect that plain Disk includes logical ID" $ includesLogicalId lv d -- | Tests that the logical ID is correctly found in a DRBD disk caseIncludeLogicalIdDrbd :: HUnit.Assertion caseIncludeLogicalIdDrbd = let vg_name = "xenvg" :: String lv_name = "1234sdf-qwef-2134-asff-asd2-23145d.data" :: String time = TOD 0 0 d = RealDisk $ RealDiskData (LIDDrbd8 "node1.example.com" "node2.example.com" 2000 1 5 (Private "secret")) [ RealDisk $ RealDiskData (mkLIDPlain "onevg" "onelv") [] ["node1.example.com", "node2.example.com"] "disk1" 1000 DiskRdWr Nothing Nothing Nothing "145145-asdf-sdf2-2134-asfd-534g2x" 0 time time , RealDisk $ RealDiskData (mkLIDPlain vg_name lv_name) [] ["node1.example.com", "node2.example.com"] "disk2" 1000 DiskRdWr Nothing Nothing Nothing "6gd3sd-423f-ag2j-563b-dg34-gj3fse" 0 time time ] ["node1.example.com", "node2.example.com"] "diskname" 1000 DiskRdWr Nothing Nothing Nothing "asdfgr-1234-5123-daf3-sdfw-134f43" 0 time time in HUnit.assertBool "Unable to detect that plain Disk includes logical ID" $ includesLogicalId (LogicalVolume vg_name lv_name) d -- | Tests that the logical ID is correctly NOT found in a plain disk caseNotIncludeLogicalIdPlain :: HUnit.Assertion caseNotIncludeLogicalIdPlain = let vg_name = "xenvg" :: String lv_name = "1234sdf-qwef-2134-asff-asd2-23145d.data" :: String time = TOD 0 0 d = RealDisk $ RealDiskData (mkLIDPlain "othervg" "otherlv") [] ["node1.example.com"] "diskname" 1000 DiskRdWr Nothing Nothing Nothing "asdfgr-1234-5123-daf3-sdfw-134f43" 0 time time in HUnit.assertBool "Unable to detect that plain Disk includes logical ID" $ not (includesLogicalId (LogicalVolume vg_name lv_name) d) testSuite "Objects" [ 'prop_fillDict , 'prop_LogicalVolume_serialisation , 'prop_LogicalVolume_deserialisationFail , 'prop_Disk_serialisation , 'prop_Disk_array_serialisation , 'prop_Inst_serialisation , 'prop_AddressPool_serialisation , 'prop_Network_serialisation , 'prop_Node_serialisation , 'prop_Config_serialisation , 'prop_FilterAction_serialisation , 'prop_FilterPredicate_serialisation , 'casePyCompatNetworks , 'casePyCompatNodegroups , 'casePyCompatInstances , 'prop_ip4AddressAsNum , 'prop_ip4AddressToNumber , 'caseNextIp4Address , 'caseIncludeLogicalIdPlain , 'caseIncludeLogicalIdDrbd , 'caseNotIncludeLogicalIdPlain ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Objects/000075500000000000000000000000001476477700300207145ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Objects/BitArray.hs000064400000000000000000000066601476477700300227750ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for bit arrays -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Objects.BitArray ( testObjects_BitArray , genBitArray ) where import Test.QuickCheck import Control.Applicative import Control.Monad import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.Objects.BitArray as BA -- * Arbitrary instances instance Arbitrary BitArray where arbitrary = fromList <$> arbitrary genBitArray :: Int -> Gen BitArray genBitArray = liftA fromList . vector prop_BitArray_serialisation :: BitArray -> Property prop_BitArray_serialisation = testSerialisation prop_BitArray_foldr :: [Bool] -> Property prop_BitArray_foldr bs = BA.foldr (((:) .) . (,)) [] (fromList bs) ==? zip bs [0..] prop_BitArray_fromToList :: BitArray -> Property prop_BitArray_fromToList bs = BA.fromList (BA.toList bs) ==? bs prop_BitArray_and :: [Bool] -> [Bool] -> Property prop_BitArray_and xs ys = (BA.fromList xs -&- BA.fromList ys) ==? BA.fromList (zipWith (&&) xs ys) prop_BitArray_or :: [Bool] -> [Bool] -> Property prop_BitArray_or xs ys = let xsl = length xs ysl = length ys l = max xsl ysl comb = zipWith (||) (xs ++ replicate (l - xsl) False) (ys ++ replicate (l - ysl) False) in (BA.fromList xs -|- BA.fromList ys) ==? BA.fromList comb -- | Check that the counts of 1 bits holds. prop_BitArray_counts :: Property prop_BitArray_counts = property $ do n <- choose (0, 3) ones <- replicateM n (lst True) zrs <- replicateM n (lst False) start <- lst False let count = sum . map length $ ones bs = start ++ concat (zipWith (++) ones zrs) return $ count1 (BA.fromList bs) ==? count where lst x = (`replicate` x) `liftM` choose (0, 2) -- | Check that the counts of free and occupied bits add up. prop_BitArray_countsSum :: BitArray -> Property prop_BitArray_countsSum a = count0 a + count1 a ==? size a testSuite "Objects_BitArray" [ 'prop_BitArray_serialisation , 'prop_BitArray_foldr , 'prop_BitArray_fromToList , 'prop_BitArray_and , 'prop_BitArray_or , 'prop_BitArray_counts , 'prop_BitArray_countsSum ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/OpCodes.hs000064400000000000000000001062641476477700300212240ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.OpCodes ( testOpCodes , OpCodes.OpCode(..) ) where import Test.HUnit as HUnit import Test.QuickCheck as QuickCheck import Control.Monad import Data.Char import Data.List import Data.Ratio ((%)) import qualified Data.Map as Map import qualified Text.JSON as J import Text.Printf (printf) import Test.Ganeti.Objects () import Test.Ganeti.Query.Language () import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Types (genReasonTrail) import Ganeti.BasicTypes import qualified Ganeti.Constants as C import qualified Ganeti.ConstantUtils as CU import qualified Ganeti.OpCodes as OpCodes import Ganeti.Types import Ganeti.OpParams import Ganeti.JSON {-# ANN module "HLint: ignore Use camelCase" #-} -- * Arbitrary instances arbitraryOpTagsGet :: Gen OpCodes.OpCode arbitraryOpTagsGet = do kind <- arbitrary OpCodes.OpTagsSet kind <$> genTags <*> genOpCodesTagName kind arbitraryOpTagsSet :: Gen OpCodes.OpCode arbitraryOpTagsSet = do kind <- arbitrary OpCodes.OpTagsSet kind <$> genTags <*> genOpCodesTagName kind arbitraryOpTagsDel :: Gen OpCodes.OpCode arbitraryOpTagsDel = do kind <- arbitrary OpCodes.OpTagsDel kind <$> genTags <*> genOpCodesTagName kind $(genArbitrary ''OpCodes.ReplaceDisksMode) $(genArbitrary ''DiskAccess) instance Arbitrary OpCodes.DiskIndex where arbitrary = choose (0, C.maxDisks - 1) >>= OpCodes.mkDiskIndex instance Arbitrary INicParams where arbitrary = INicParams <$> genMaybe genNameNE <*> genMaybe genName <*> genMaybe genNameNE <*> genMaybe genNameNE <*> genMaybe genNameNE <*> genMaybe genName <*> genMaybe genNameNE <*> genMaybe genNameNE instance Arbitrary IDiskParams where arbitrary = IDiskParams <$> arbitrary <*> arbitrary <*> genMaybe genNameNE <*> genMaybe genNameNE <*> genMaybe genNameNE <*> genMaybe genNameNE <*> genMaybe genNameNE <*> arbitrary <*> genMaybe genNameNE <*> genAndRestArguments instance Arbitrary RecreateDisksInfo where arbitrary = oneof [ pure RecreateDisksAll , RecreateDisksIndices <$> arbitrary , RecreateDisksParams <$> arbitrary ] instance Arbitrary DdmOldChanges where arbitrary = oneof [ DdmOldIndex <$> arbitrary , DdmOldMod <$> arbitrary ] instance (Arbitrary a) => Arbitrary (SetParamsMods a) where arbitrary = oneof [ pure SetParamsEmpty , SetParamsDeprecated <$> arbitrary , SetParamsNew <$> arbitrary ] instance Arbitrary ExportTarget where arbitrary = oneof [ ExportTargetLocal <$> genNodeNameNE , ExportTargetRemote <$> pure [] ] arbitraryDataCollector :: Gen (GenericContainer String Bool) arbitraryDataCollector = do els <- listOf . elements $ CU.toList C.dataCollectorNames activation <- vector $ length els return . GenericContainer . Map.fromList $ zip els activation arbitraryDataCollectorInterval :: Gen (Maybe (GenericContainer String Int)) arbitraryDataCollectorInterval = do els <- listOf . elements $ CU.toList C.dataCollectorNames intervals <- vector $ length els genMaybe . return . containerFromList $ zip els intervals {- | Choose numbers across orders of magnitudes in order to detect inconsistencies in the choice between fix point and scientific formatting. Also choose equally between finite and periodic decimal fractions. -} arbitraryDuration :: Gen Double arbitraryDuration = oneof $ (do mantissa <- choose (0, 1000) expon <- choose (0,12::Int) return $ fromRational $ mantissa % 10^expon) : (do numerator <- choose (0, 1000) denominator <- choose (1, 1000) expon <- choose (-40,4::Int) return $ fromRational (numerator % denominator) * 2^^expon) : [] instance Arbitrary OpCodes.OpCode where arbitrary = do op_id <- elements OpCodes.allOpIDs case op_id of "OP_TEST_DELAY" -> OpCodes.OpTestDelay <$> arbitraryDuration <*> arbitrary <*> genNodeNamesNE <*> return Nothing <*> arbitrary <*> arbitrary <*> arbitrary "OP_INSTANCE_REPLACE_DISKS" -> OpCodes.OpInstanceReplaceDisks <$> genFQDN <*> return Nothing <*> arbitrary <*> arbitrary <*> arbitrary <*> genDiskIndices <*> genMaybe genNodeNameNE <*> return Nothing <*> genMaybe genNameNE "OP_INSTANCE_FAILOVER" -> OpCodes.OpInstanceFailover <$> genFQDN <*> return Nothing <*> arbitrary <*> arbitrary <*> genMaybe genNodeNameNE <*> return Nothing <*> arbitrary <*> arbitrary <*> genMaybe genNameNE "OP_INSTANCE_MIGRATE" -> OpCodes.OpInstanceMigrate <$> genFQDN <*> return Nothing <*> arbitrary <*> arbitrary <*> genMaybe genNodeNameNE <*> return Nothing <*> arbitrary <*> arbitrary <*> arbitrary <*> genMaybe genNameNE <*> arbitrary <*> arbitrary "OP_TAGS_GET" -> arbitraryOpTagsGet "OP_TAGS_SEARCH" -> OpCodes.OpTagsSearch <$> genNameNE "OP_TAGS_SET" -> arbitraryOpTagsSet "OP_TAGS_DEL" -> arbitraryOpTagsDel "OP_CLUSTER_POST_INIT" -> pure OpCodes.OpClusterPostInit "OP_CLUSTER_RENEW_CRYPTO" -> OpCodes.OpClusterRenewCrypto <$> arbitrary -- Node SSL certificates <*> arbitrary -- renew_ssh_keys <*> arbitrary -- ssh_key_type <*> arbitrary -- ssh_key_bits <*> arbitrary -- verbose <*> arbitrary -- debug "OP_CLUSTER_DESTROY" -> pure OpCodes.OpClusterDestroy "OP_CLUSTER_QUERY" -> pure OpCodes.OpClusterQuery "OP_CLUSTER_VERIFY" -> OpCodes.OpClusterVerify <$> arbitrary <*> arbitrary <*> genListSet Nothing <*> genListSet Nothing <*> arbitrary <*> genMaybe genNameNE <*> arbitrary "OP_CLUSTER_VERIFY_CONFIG" -> OpCodes.OpClusterVerifyConfig <$> arbitrary <*> arbitrary <*> genListSet Nothing <*> arbitrary "OP_CLUSTER_VERIFY_GROUP" -> OpCodes.OpClusterVerifyGroup <$> genNameNE <*> arbitrary <*> arbitrary <*> genListSet Nothing <*> genListSet Nothing <*> arbitrary <*> arbitrary "OP_CLUSTER_VERIFY_DISKS" -> OpCodes.OpClusterVerifyDisks <$> genMaybe genNameNE <*> arbitrary "OP_GROUP_VERIFY_DISKS" -> OpCodes.OpGroupVerifyDisks <$> genNameNE <*> arbitrary "OP_CLUSTER_REPAIR_DISK_SIZES" -> OpCodes.OpClusterRepairDiskSizes <$> genNodeNamesNE "OP_CLUSTER_CONFIG_QUERY" -> OpCodes.OpClusterConfigQuery <$> genFieldsNE "OP_CLUSTER_RENAME" -> OpCodes.OpClusterRename <$> genNameNE "OP_CLUSTER_SET_PARAMS" -> OpCodes.OpClusterSetParams <$> arbitrary -- force <*> emptyMUD -- hv_state <*> emptyMUD -- disk_state <*> genMaybe genName -- vg_name <*> genMaybe arbitrary -- enabled_hypervisors <*> genMaybe genEmptyContainer -- hvparams <*> emptyMUD -- beparams <*> genMaybe genEmptyContainer -- os_hvp <*> genMaybe genEmptyContainer -- osparams <*> genMaybe genEmptyContainer -- osparams_private_cluster <*> genMaybe genEmptyContainer -- diskparams <*> genMaybe arbitrary -- candidate_pool_size <*> genMaybe arbitrary -- max_running_jobs <*> genMaybe arbitrary -- max_tracked_jobs <*> arbitrary -- uid_pool <*> arbitrary -- add_uids <*> arbitrary -- remove_uids <*> arbitrary -- maintain_node_health <*> arbitrary -- prealloc_wipe_disks <*> arbitrary -- nicparams <*> emptyMUD -- ndparams <*> emptyMUD -- ipolicy <*> genMaybe genPrintableAsciiString -- drbd_helper <*> genMaybe genPrintableAsciiString -- default_iallocator <*> emptyMUD -- default_iallocator_params <*> genMaybe genMacPrefix -- mac_prefix <*> genMaybe genPrintableAsciiString -- master_netdev <*> arbitrary -- master_netmask <*> genMaybe (listOf genPrintableAsciiStringNE) -- reserved_lvs <*> genMaybe (listOf ((,) <$> arbitrary <*> genPrintableAsciiStringNE)) -- hidden_os <*> genMaybe (listOf ((,) <$> arbitrary <*> genPrintableAsciiStringNE)) -- blacklisted_os <*> arbitrary -- use_external_mip_script <*> arbitrary -- enabled_disk_templates <*> arbitrary -- modify_etc_hosts <*> genMaybe genName -- file_storage_dir <*> genMaybe genName -- shared_file_storage_dir <*> genMaybe genName -- gluster_file_storage_dir <*> genMaybe genPrintableAsciiString -- install_image <*> genMaybe genPrintableAsciiString -- instance_communication_network <*> genMaybe genPrintableAsciiString -- zeroing_image <*> genMaybe (listOf genPrintableAsciiStringNE) -- compression_tools <*> arbitrary -- enabled_user_shutdown <*> genMaybe arbitraryDataCollector -- enabled_data_collectors <*> arbitraryDataCollectorInterval -- data_collector_interval "OP_CLUSTER_REDIST_CONF" -> pure OpCodes.OpClusterRedistConf "OP_CLUSTER_ACTIVATE_MASTER_IP" -> pure OpCodes.OpClusterActivateMasterIp "OP_CLUSTER_DEACTIVATE_MASTER_IP" -> pure OpCodes.OpClusterDeactivateMasterIp "OP_QUERY" -> OpCodes.OpQuery <$> arbitrary <*> arbitrary <*> genNamesNE <*> pure Nothing "OP_QUERY_FIELDS" -> OpCodes.OpQueryFields <$> arbitrary <*> genMaybe genNamesNE "OP_OOB_COMMAND" -> OpCodes.OpOobCommand <$> genNodeNamesNE <*> return Nothing <*> arbitrary <*> arbitrary <*> arbitrary <*> (arbitrary `suchThat` (>0)) "OP_NODE_REMOVE" -> OpCodes.OpNodeRemove <$> genNodeNameNE <*> return Nothing "OP_NODE_ADD" -> OpCodes.OpNodeAdd <$> genNodeNameNE <*> emptyMUD <*> emptyMUD <*> genMaybe genNameNE <*> genMaybe genNameNE <*> arbitrary <*> genMaybe genNameNE <*> arbitrary <*> arbitrary <*> emptyMUD <*> arbitrary "OP_NODE_QUERYVOLS" -> OpCodes.OpNodeQueryvols <$> genNamesNE <*> genNodeNamesNE "OP_NODE_QUERY_STORAGE" -> OpCodes.OpNodeQueryStorage <$> genNamesNE <*> arbitrary <*> genNodeNamesNE <*> genMaybe genNameNE "OP_NODE_MODIFY_STORAGE" -> OpCodes.OpNodeModifyStorage <$> genNodeNameNE <*> return Nothing <*> arbitrary <*> genMaybe genNameNE <*> pure emptyJSObject "OP_REPAIR_NODE_STORAGE" -> OpCodes.OpRepairNodeStorage <$> genNodeNameNE <*> return Nothing <*> arbitrary <*> genMaybe genNameNE <*> arbitrary "OP_NODE_SET_PARAMS" -> OpCodes.OpNodeSetParams <$> genNodeNameNE <*> return Nothing <*> arbitrary <*> emptyMUD <*> emptyMUD <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> genMaybe genNameNE <*> emptyMUD <*> arbitrary "OP_NODE_POWERCYCLE" -> OpCodes.OpNodePowercycle <$> genNodeNameNE <*> return Nothing <*> arbitrary "OP_NODE_MIGRATE" -> OpCodes.OpNodeMigrate <$> genNodeNameNE <*> return Nothing <*> arbitrary <*> arbitrary <*> genMaybe genNodeNameNE <*> return Nothing <*> arbitrary <*> arbitrary <*> genMaybe genNameNE "OP_NODE_EVACUATE" -> OpCodes.OpNodeEvacuate <$> arbitrary <*> genNodeNameNE <*> return Nothing <*> genMaybe genNodeNameNE <*> return Nothing <*> genMaybe genNameNE <*> arbitrary <*> arbitrary "OP_INSTANCE_CREATE" -> OpCodes.OpInstanceCreate <$> genFQDN -- instance_name <*> arbitrary -- force_variant <*> arbitrary -- wait_for_sync <*> arbitrary -- name_check <*> arbitrary -- ignore_ipolicy <*> arbitrary -- opportunistic_locking <*> pure emptyJSObject -- beparams <*> arbitrary -- disks <*> arbitrary -- disk_template <*> genMaybe genNameNE -- group_name <*> arbitrary -- file_driver <*> genMaybe genNameNE -- file_storage_dir <*> pure emptyJSObject -- hvparams <*> arbitrary -- hypervisor <*> genMaybe genNameNE -- iallocator <*> arbitrary -- identify_defaults <*> arbitrary -- ip_check <*> arbitrary -- conflicts_check <*> arbitrary -- mode <*> arbitrary -- nics <*> arbitrary -- no_install <*> pure emptyJSObject -- osparams <*> genMaybe arbitraryPrivateJSObj -- osparams_private <*> genMaybe arbitrarySecretJSObj -- osparams_secret <*> genMaybe genNameNE -- os_type <*> genMaybe genNodeNameNE -- pnode <*> return Nothing -- pnode_uuid <*> genMaybe genNodeNameNE -- snode <*> return Nothing -- snode_uuid <*> genMaybe (pure []) -- source_handshake <*> genMaybe genNodeNameNE -- source_instance_name <*> arbitrary -- source_shutdown_timeout <*> genMaybe genNodeNameNE -- source_x509_ca <*> return Nothing -- src_node <*> genMaybe genNodeNameNE -- src_node_uuid <*> genMaybe genNameNE -- src_path <*> genPrintableAsciiString -- compress <*> arbitrary -- start <*> arbitrary -- forthcoming <*> arbitrary -- commit <*> (genTags >>= mapM mkNonEmpty) -- tags <*> arbitrary -- instance_communication <*> arbitrary -- helper_startup_timeout <*> arbitrary -- helper_shutdown_timeout "OP_INSTANCE_MULTI_ALLOC" -> OpCodes.OpInstanceMultiAlloc <$> arbitrary <*> genMaybe genNameNE <*> pure [] "OP_INSTANCE_REINSTALL" -> OpCodes.OpInstanceReinstall <$> genFQDN <*> return Nothing <*> arbitrary <*> genMaybe genNameNE <*> genMaybe (pure emptyJSObject) <*> genMaybe arbitraryPrivateJSObj <*> genMaybe arbitrarySecretJSObj "OP_INSTANCE_REMOVE" -> OpCodes.OpInstanceRemove <$> genFQDN <*> return Nothing <*> arbitrary <*> arbitrary "OP_INSTANCE_RENAME" -> OpCodes.OpInstanceRename <$> genFQDN <*> return Nothing <*> genNodeNameNE <*> arbitrary <*> arbitrary "OP_INSTANCE_STARTUP" -> OpCodes.OpInstanceStartup <$> genFQDN <*> -- instance_name return Nothing <*> -- instance_uuid arbitrary <*> -- force arbitrary <*> -- ignore_offline_nodes pure emptyJSObject <*> -- hvparams pure emptyJSObject <*> -- beparams arbitrary <*> -- no_remember arbitrary <*> -- startup_paused arbitrary -- shutdown_timeout "OP_INSTANCE_SHUTDOWN" -> OpCodes.OpInstanceShutdown <$> genFQDN <*> return Nothing <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary "OP_INSTANCE_REBOOT" -> OpCodes.OpInstanceReboot <$> genFQDN <*> return Nothing <*> arbitrary <*> arbitrary <*> arbitrary "OP_INSTANCE_MOVE" -> OpCodes.OpInstanceMove <$> genFQDN <*> return Nothing <*> arbitrary <*> arbitrary <*> genNodeNameNE <*> return Nothing <*> genPrintableAsciiString <*> arbitrary "OP_INSTANCE_CONSOLE" -> OpCodes.OpInstanceConsole <$> genFQDN <*> return Nothing "OP_INSTANCE_ACTIVATE_DISKS" -> OpCodes.OpInstanceActivateDisks <$> genFQDN <*> return Nothing <*> arbitrary <*> arbitrary "OP_INSTANCE_DEACTIVATE_DISKS" -> OpCodes.OpInstanceDeactivateDisks <$> genFQDN <*> return Nothing <*> arbitrary "OP_INSTANCE_RECREATE_DISKS" -> OpCodes.OpInstanceRecreateDisks <$> genFQDN <*> return Nothing <*> arbitrary <*> genNodeNamesNE <*> return Nothing <*> genMaybe genNameNE "OP_INSTANCE_QUERY_DATA" -> OpCodes.OpInstanceQueryData <$> arbitrary <*> genNodeNamesNE <*> arbitrary "OP_INSTANCE_SET_PARAMS" -> OpCodes.OpInstanceSetParams <$> genFQDN -- instance_name <*> return Nothing -- instance_uuid <*> arbitrary -- force <*> arbitrary -- force_variant <*> arbitrary -- ignore_ipolicy <*> arbitrary -- nics <*> arbitrary -- disks <*> pure emptyJSObject -- beparams <*> arbitrary -- runtime_mem <*> pure emptyJSObject -- hvparams <*> arbitrary -- disk_template <*> pure emptyJSObject -- ext_params <*> arbitrary -- file_driver <*> genMaybe genNameNE -- file_storage_dir <*> genMaybe genNodeNameNE -- pnode <*> return Nothing -- pnode_uuid <*> genMaybe genNodeNameNE -- remote_node <*> return Nothing -- remote_node_uuid <*> genMaybe genNameNE -- iallocator <*> genMaybe genNameNE -- os_name <*> pure emptyJSObject -- osparams <*> genMaybe arbitraryPrivateJSObj -- osparams_private <*> arbitrary -- wait_for_sync <*> arbitrary -- offline <*> arbitrary -- conflicts_check <*> arbitrary -- hotplug <*> arbitrary -- instance_communication "OP_INSTANCE_GROW_DISK" -> OpCodes.OpInstanceGrowDisk <$> genFQDN <*> return Nothing <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary "OP_INSTANCE_CHANGE_GROUP" -> OpCodes.OpInstanceChangeGroup <$> genFQDN <*> return Nothing <*> arbitrary <*> genMaybe genNameNE <*> genMaybe (resize maxNodes (listOf genNameNE)) "OP_GROUP_ADD" -> OpCodes.OpGroupAdd <$> genNameNE <*> arbitrary <*> emptyMUD <*> genMaybe genEmptyContainer <*> emptyMUD <*> emptyMUD <*> emptyMUD "OP_GROUP_ASSIGN_NODES" -> OpCodes.OpGroupAssignNodes <$> genNameNE <*> arbitrary <*> genNodeNamesNE <*> return Nothing "OP_GROUP_SET_PARAMS" -> OpCodes.OpGroupSetParams <$> genNameNE <*> arbitrary <*> emptyMUD <*> genMaybe genEmptyContainer <*> emptyMUD <*> emptyMUD <*> emptyMUD "OP_GROUP_REMOVE" -> OpCodes.OpGroupRemove <$> genNameNE "OP_GROUP_RENAME" -> OpCodes.OpGroupRename <$> genNameNE <*> genNameNE "OP_GROUP_EVACUATE" -> OpCodes.OpGroupEvacuate <$> genNameNE <*> arbitrary <*> genMaybe genNameNE <*> genMaybe genNamesNE <*> arbitrary <*> arbitrary "OP_OS_DIAGNOSE" -> OpCodes.OpOsDiagnose <$> genFieldsNE <*> genNamesNE "OP_EXT_STORAGE_DIAGNOSE" -> OpCodes.OpOsDiagnose <$> genFieldsNE <*> genNamesNE "OP_BACKUP_PREPARE" -> OpCodes.OpBackupPrepare <$> genFQDN <*> return Nothing <*> arbitrary "OP_BACKUP_EXPORT" -> OpCodes.OpBackupExport <$> genFQDN -- instance_name <*> return Nothing -- instance_uuid <*> genPrintableAsciiString -- compress <*> arbitrary -- shutdown_timeout <*> arbitrary -- target_node <*> return Nothing -- target_node_uuid <*> arbitrary -- shutdown <*> arbitrary -- remove_instance <*> arbitrary -- ignore_remove_failures <*> arbitrary -- mode <*> genMaybe (pure []) -- x509_key_name <*> genMaybe genNameNE -- destination_x509_ca <*> arbitrary -- zero_free_space <*> arbitrary -- zeroing_timeout_fixed <*> arbitrary -- zeroing_timeout_per_mib <*> arbitrary -- long_sleep "OP_BACKUP_REMOVE" -> OpCodes.OpBackupRemove <$> genFQDN <*> return Nothing "OP_TEST_ALLOCATOR" -> OpCodes.OpTestAllocator <$> arbitrary <*> arbitrary <*> genNameNE <*> genMaybe (pure []) <*> genMaybe (pure []) <*> arbitrary <*> genMaybe genNameNE <*> (genTags >>= mapM mkNonEmpty) <*> arbitrary <*> arbitrary <*> genMaybe genNameNE <*> arbitrary <*> genMaybe genNodeNamesNE <*> arbitrary <*> genMaybe genNamesNE <*> arbitrary <*> arbitrary <*> genMaybe genNameNE "OP_TEST_JQUEUE" -> OpCodes.OpTestJqueue <$> arbitrary <*> arbitrary <*> resize 20 (listOf genFQDN) <*> arbitrary "OP_TEST_OS_PARAMS" -> OpCodes.OpTestOsParams <$> genMaybe arbitrarySecretJSObj "OP_TEST_DUMMY" -> OpCodes.OpTestDummy <$> pure J.JSNull <*> pure J.JSNull <*> pure J.JSNull <*> pure J.JSNull "OP_NETWORK_ADD" -> OpCodes.OpNetworkAdd <$> genNameNE <*> genIPv4Network <*> genMaybe genIPv4Address <*> pure Nothing <*> pure Nothing <*> genMaybe genMacPrefix <*> genMaybe (listOf genIPv4Address) <*> arbitrary <*> (genTags >>= mapM mkNonEmpty) "OP_NETWORK_REMOVE" -> OpCodes.OpNetworkRemove <$> genNameNE <*> arbitrary "OP_NETWORK_RENAME" -> OpCodes.OpNetworkRename <$> genNameNE <*> genNameNE "OP_NETWORK_SET_PARAMS" -> OpCodes.OpNetworkSetParams <$> genNameNE <*> genMaybe genIPv4Address <*> pure Nothing <*> pure Nothing <*> genMaybe genMacPrefix <*> genMaybe (listOf genIPv4Address) <*> genMaybe (listOf genIPv4Address) "OP_NETWORK_CONNECT" -> OpCodes.OpNetworkConnect <$> genNameNE <*> genNameNE <*> arbitrary <*> genNameNE <*> genPrintableAsciiString <*> arbitrary "OP_NETWORK_DISCONNECT" -> OpCodes.OpNetworkDisconnect <$> genNameNE <*> genNameNE "OP_RESTRICTED_COMMAND" -> OpCodes.OpRestrictedCommand <$> arbitrary <*> genNodeNamesNE <*> return Nothing <*> genNameNE _ -> fail $ "Undefined arbitrary for opcode " ++ op_id instance Arbitrary OpCodes.CommonOpParams where arbitrary = OpCodes.CommonOpParams <$> arbitrary <*> arbitrary <*> arbitrary <*> resize 5 arbitrary <*> genMaybe genName <*> genReasonTrail -- * Helper functions -- | Empty JSObject. emptyJSObject :: J.JSObject J.JSValue emptyJSObject = J.toJSObject [] -- | Empty maybe unchecked dictionary. emptyMUD :: Gen (Maybe (J.JSObject J.JSValue)) emptyMUD = genMaybe $ pure emptyJSObject -- | Generates an empty container. genEmptyContainer :: (Ord a) => Gen (GenericContainer a b) genEmptyContainer = pure . GenericContainer $ Map.fromList [] -- | Generates list of disk indices. genDiskIndices :: Gen [DiskIndex] genDiskIndices = do cnt <- choose (0, C.maxDisks) genUniquesList cnt arbitrary -- | Generates a list of node names. genNodeNames :: Gen [String] genNodeNames = resize maxNodes (listOf genFQDN) -- | Generates a list of node names in non-empty string type. genNodeNamesNE :: Gen [NonEmptyString] genNodeNamesNE = genNodeNames >>= mapM mkNonEmpty -- | Gets a node name in non-empty type. genNodeNameNE :: Gen NonEmptyString genNodeNameNE = genFQDN >>= mkNonEmpty -- | Gets a name (non-fqdn) in non-empty type. genNameNE :: Gen NonEmptyString genNameNE = genName >>= mkNonEmpty -- | Gets a list of names (non-fqdn) in non-empty type. genNamesNE :: Gen [NonEmptyString] genNamesNE = resize maxNodes (listOf genNameNE) -- | Returns a list of non-empty fields. genFieldsNE :: Gen [NonEmptyString] genFieldsNE = genFields >>= mapM mkNonEmpty -- | Generate a 3-byte MAC prefix. genMacPrefix :: Gen NonEmptyString genMacPrefix = do octets <- vectorOf 3 $ choose (0::Int, 255) mkNonEmpty . intercalate ":" $ map (printf "%02x") octets -- | JSObject of arbitrary data. -- -- Since JSValue does not implement Arbitrary, I'll simply generate -- (String, String) objects. arbitraryPrivateJSObj :: Gen (J.JSObject (Private J.JSValue)) arbitraryPrivateJSObj = constructor <$> (fromNonEmpty <$> genNameNE) <*> (fromNonEmpty <$> genNameNE) where constructor k v = showPrivateJSObject [(k, v)] -- | JSObject of arbitrary secret data. arbitrarySecretJSObj :: Gen (J.JSObject (Secret J.JSValue)) arbitrarySecretJSObj = constructor <$> (fromNonEmpty <$> genNameNE) <*> (fromNonEmpty <$> genNameNE) where constructor k v = showSecretJSObject [(k, v)] -- | Arbitrary instance for MetaOpCode, defined here due to TH ordering. $(genArbitrary ''OpCodes.MetaOpCode) -- | Small helper to check for a failed JSON deserialisation isJsonError :: J.Result a -> Bool isJsonError (J.Error _) = True isJsonError _ = False -- * Test cases -- | Check that opcode serialization is idempotent. prop_serialization :: OpCodes.OpCode -> Property prop_serialization = testSerialisation -- | Check that Python and Haskell defined the same opcode list. case_AllDefined :: HUnit.Assertion case_AllDefined = do py_stdout <- runPython "from ganeti import opcodes\n\ \from ganeti import serializer\n\ \import sys\n\ \sys.stdout.buffer.write(\ \ serializer.Dump([opid for opid in opcodes.OP_MAPPING]))" "" >>= checkPythonResult py_ops <- case J.decode py_stdout::J.Result [String] of J.Ok ops -> return ops J.Error msg -> HUnit.assertFailure ("Unable to decode opcode names: " ++ msg) -- this already raised an expection, but we need it -- for proper types >> fail "Unable to decode opcode names" let hs_ops = sort OpCodes.allOpIDs extra_py = py_ops \\ hs_ops extra_hs = hs_ops \\ py_ops HUnit.assertBool ("Missing OpCodes from the Haskell code:\n" ++ unlines extra_py) (null extra_py) HUnit.assertBool ("Extra OpCodes in the Haskell code:\n" ++ unlines extra_hs) (null extra_hs) -- | Custom HUnit test case that forks a Python process and checks -- correspondence between Haskell-generated OpCodes and their Python -- decoded, validated and re-encoded version. -- -- Note that we have a strange beast here: since launching Python is -- expensive, we don't do this via a usual QuickProperty, since that's -- slow (I've tested it, and it's indeed quite slow). Rather, we use a -- single HUnit assertion, and in it we manually use QuickCheck to -- generate 500 opcodes times the number of defined opcodes, which -- then we pass in bulk to Python. The drawbacks to this method are -- two fold: we cannot control the number of generated opcodes, since -- HUnit assertions don't get access to the test options, and for the -- same reason we can't run a repeatable seed. We should probably find -- a better way to do this, for example by having a -- separately-launched Python process (if not running the tests would -- be skipped). case_py_compat_types :: HUnit.Assertion case_py_compat_types = do let num_opcodes = length OpCodes.allOpIDs * 100 opcodes <- genSample (vectorOf num_opcodes (arbitrary::Gen OpCodes.MetaOpCode)) let with_sum = map (\o -> (OpCodes.opSummary $ OpCodes.metaOpCode o, o)) opcodes serialized = J.encode opcodes -- check for non-ASCII fields, usually due to 'arbitrary :: String' mapM_ (\op -> when (any (not . isAscii) (J.encode op)) . HUnit.assertFailure $ "OpCode has non-ASCII fields: " ++ show op ) opcodes py_stdout <- runPython "from ganeti import opcodes\n\ \from ganeti import serializer\n\ \import sys\n\ \op_data = serializer.Load(sys.stdin.read())\n\ \decoded = [opcodes.OpCode.LoadOpCode(o) for o in op_data]\n\ \for op in decoded:\n\ \ op.Validate(True)\n\ \encoded = [(op.Summary(), op.__getstate__())\n\ \ for op in decoded]\n\ \sys.stdout.buffer.write(serializer.Dump(\ \ encoded,\ \ private_encoder=serializer.EncodeWithPrivateFields))" serialized >>= checkPythonResult let deserialised = J.decode py_stdout::J.Result [(String, OpCodes.MetaOpCode)] decoded <- case deserialised of J.Ok ops -> return ops J.Error msg -> HUnit.assertFailure ("Unable to decode opcodes: " ++ msg) -- this already raised an expection, but we need it -- for proper types >> fail "Unable to decode opcodes" HUnit.assertEqual "Mismatch in number of returned opcodes" (length decoded) (length with_sum) mapM_ (uncurry (HUnit.assertEqual "Different result after encoding/decoding") ) $ zip with_sum decoded -- | Custom HUnit test case that forks a Python process and checks -- correspondence between Haskell OpCodes fields and their Python -- equivalent. case_py_compat_fields :: HUnit.Assertion case_py_compat_fields = do let hs_fields = sort $ map (\op_id -> (op_id, OpCodes.allOpFields op_id)) OpCodes.allOpIDs py_stdout <- runPython "from ganeti import opcodes\n\ \import sys\n\ \from ganeti import serializer\n\ \fields = [(k, sorted([p[0] for p in v.OP_PARAMS]))\n\ \ for k, v in opcodes.OP_MAPPING.items()]\n\ \sys.stdout.buffer.write(serializer.Dump(fields))" "" >>= checkPythonResult let deserialised = J.decode py_stdout::J.Result [(String, [String])] py_fields <- case deserialised of J.Ok v -> return $ sort v J.Error msg -> HUnit.assertFailure ("Unable to decode op fields: " ++ msg) -- this already raised an expection, but we need it -- for proper types >> fail "Unable to decode op fields" HUnit.assertEqual "Mismatch in number of returned opcodes" (length hs_fields) (length py_fields) HUnit.assertEqual "Mismatch in defined OP_IDs" (map fst hs_fields) (map fst py_fields) mapM_ (\((py_id, py_flds), (hs_id, hs_flds)) -> do HUnit.assertEqual "Mismatch in OP_ID" py_id hs_id HUnit.assertEqual ("Mismatch in fields for " ++ hs_id) py_flds hs_flds ) $ zip hs_fields py_fields -- | Checks that setOpComment works correctly. prop_setOpComment :: OpCodes.MetaOpCode -> String -> Property prop_setOpComment op comment = let (OpCodes.MetaOpCode common _) = OpCodes.setOpComment comment op in OpCodes.opComment common ==? Just comment -- | Tests wrong (negative) disk index. prop_mkDiskIndex_fail :: QuickCheck.Positive Int -> Property prop_mkDiskIndex_fail (Positive i) = case mkDiskIndex (negate i) of Bad msg -> counterexample "error message " $ "Invalid value" `isPrefixOf` msg Ok v -> failTest $ "Succeeded to build disk index '" ++ show v ++ "' from negative value " ++ show (negate i) -- | Tests a few invalid 'readRecreateDisks' cases. case_readRecreateDisks_fail :: Assertion case_readRecreateDisks_fail = do assertBool "null" $ isJsonError (J.readJSON J.JSNull::J.Result RecreateDisksInfo) assertBool "string" $ isJsonError (J.readJSON (J.showJSON "abc")::J.Result RecreateDisksInfo) -- | Tests a few invalid 'readDdmOldChanges' cases. case_readDdmOldChanges_fail :: Assertion case_readDdmOldChanges_fail = do assertBool "null" $ isJsonError (J.readJSON J.JSNull::J.Result DdmOldChanges) assertBool "string" $ isJsonError (J.readJSON (J.showJSON "abc")::J.Result DdmOldChanges) -- | Tests a few invalid 'readExportTarget' cases. case_readExportTarget_fail :: Assertion case_readExportTarget_fail = do assertBool "null" $ isJsonError (J.readJSON J.JSNull::J.Result ExportTarget) assertBool "int" $ isJsonError (J.readJSON (J.showJSON (5::Int))::J.Result ExportTarget) testSuite "OpCodes" [ 'prop_serialization , 'case_AllDefined , 'case_py_compat_types , 'case_py_compat_fields , 'prop_setOpComment , 'prop_mkDiskIndex_fail , 'case_readRecreateDisks_fail , 'case_readDdmOldChanges_fail , 'case_readExportTarget_fail ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/PartialParams.hs000064400000000000000000000053101476477700300224160ustar00rootroot00000000000000{-| Common tests for PartialParams instances -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.PartialParams ( testFillParamsLaw1 , testToParamsLaw2 , testToFilledLaw3 , testToFilledMonoidLaw1 , testToFilledMonoidLaw2 ) where import Data.Semigroup ((<>)) import Test.QuickCheck import Ganeti.PartialParams import Test.Ganeti.TestCommon -- | Checks for serialisation idempotence. testFillParamsLaw1 :: (PartialParams f p, Show f, Eq f) => f -> p -> Property testFillParamsLaw1 f p = fillParams (fillParams f p) p ==? fillParams f p -- | Tests that filling partial parameters satisfies the law. testToParamsLaw2 :: (PartialParams f p, Show f, Eq f) => f -> f -> Property testToParamsLaw2 x f = fillParams x (toPartial f) ==? f -- | Tests that converting partial to filled parameters satisfies the law. testToFilledLaw3 :: (PartialParams f p, Show f, Eq f) => f -> Property testToFilledLaw3 f = toFilled (toPartial f) ==? Just f -- | Tests that the partial params behave correctly as a monoid action. testToFilledMonoidLaw1 :: (PartialParams f p, Show f, Eq f, Monoid p) => f -> Property testToFilledMonoidLaw1 f = fillParams f mempty ==? f -- | Tests that the partial params behave correctly as a monoid action. testToFilledMonoidLaw2 :: (PartialParams f p, Show f, Eq f, Monoid p) => f -> p -> p -> Property testToFilledMonoidLaw2 f p1 p2 = fillParams f (p1 <> p2) ==? fillParams (fillParams f p1) p2 ganeti-3.1.0~rc2/test/hs/Test/Ganeti/PyValue.hs000064400000000000000000000071621476477700300212520ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for Ganeti.Pyvalue -} {- Copyright (C) 2019 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.PyValue ( testPyValue ) where import Test.QuickCheck import qualified Test.HUnit as HUnit import Data.List import qualified Data.ByteString as BS import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.ByteString.Base64 as B64 import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Types () import Ganeti.PyValue -- * Arbitrary instances instance Arbitrary BS.ByteString where arbitrary = UTF8.fromString <$> arbitrary -- | Custom HUnit test to check the correspondence between ByteStrings and -- Python bytes. We use ast.literal_eval to evaluate the byte literals and then -- write the resulting bytestrings back to Haskell for comparison. -- -- For the technical background of this unit test, check the documentation -- of "case_py_compat_types" of test/hs/Test/Ganeti/Opcodes.hs -- -- Note that System.Process.readProcessWithExitCode (used by runPython) returns -- Python's stdout as a String, by calling hGetContents on the standard output -- handle. This means that data is decoded using the system locale, making the -- channel not 8-bit-clean when run with a non-UTF-8 locale (such as POSIX). We -- could use System.Process.Typed to get past this problem, but that's an extra -- dependency not used elsewhere. Instead we work around this issue by -- base64-encoding the UTF-8 output on the Python side. caseByteStringsToBytes :: HUnit.Assertion caseByteStringsToBytes = do let num_bs = 500::Int bytestrings <- generate $ vectorOf num_bs (arbitrary::Gen BS.ByteString) let input = intercalate "\n" $ map showValue bytestrings py_stdout <- runPython "from ast import literal_eval\n\ \import sys\n\ \import base64\n\ \for item in sys.stdin:\n\ \ data = literal_eval(item)\n\ \ sys.stdout.buffer.write(base64.b64encode(data) + b'\\n')" input >>= checkPythonResult let decoded = map (B64.decodeLenient . UTF8.fromString) $ lines py_stdout HUnit.assertEqual "Mismatch in number of returned bytestrings" (length decoded) (length bytestrings) mapM_ (uncurry (HUnit.assertEqual "Different result after encoding/decoding") ) $ zip bytestrings decoded testSuite "PyValue" [ 'caseByteStringsToBytes ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Query/000075500000000000000000000000001476477700300204305ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Query/Aliases.hs000064400000000000000000000062561476477700300223560ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for query aliases. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Query.Aliases ( testQuery_Aliases ) where import Data.List import Test.Ganeti.TestHelper import Test.HUnit import Ganeti.Query.Common () import qualified Ganeti.Query.Instance as I import Ganeti.Query.Language import Ganeti.Query.Types {-# ANN module "HLint: ignore Use camelCase" #-} -- | Converts field list to field name list toFieldNameList :: FieldList a b -> [FieldName] toFieldNameList = map (\(x,_,_) -> fdefName x) -- | Converts alias list to alias name list toAliasNameList :: [(FieldName, FieldName)] -> [FieldName] toAliasNameList = map fst -- | Converts alias list to alias target list toAliasTargetList :: [(FieldName, FieldName)] -> [FieldName] toAliasTargetList = map snd -- | Checks for shadowing checkShadowing :: String -> FieldList a b -> [(FieldName, FieldName)] -> Assertion checkShadowing name fields aliases = assertBool (name ++ " aliases do not shadow fields") . null $ toFieldNameList fields `intersect` toAliasNameList aliases -- | Checks for target existence checkTargets :: String -> FieldList a b -> [(FieldName, FieldName)] -> Assertion checkTargets name fields aliases = assertBool (name ++ " alias targets exist") . null $ toAliasTargetList aliases \\ toFieldNameList fields -- | Check that instance aliases do not shadow existing fields case_instanceAliasesNoShadowing :: Assertion case_instanceAliasesNoShadowing = checkShadowing "Instance" I.instanceFields I.instanceAliases -- | Check that instance alias targets exist case_instanceAliasesTargetsExist :: Assertion case_instanceAliasesTargetsExist = checkTargets "Instance" I.instanceFields I.instanceAliases testSuite "Query/Aliases" [ 'case_instanceAliasesNoShadowing, 'case_instanceAliasesTargetsExist ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Query/Filter.hs000064400000000000000000000206411476477700300222140ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Query.Filter (testQuery_Filter) where import Test.QuickCheck hiding (Result) import Test.QuickCheck.Monadic import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Map as Map import Data.List import Text.JSON (showJSON) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Objects (genEmptyCluster) import Ganeti.BasicTypes import Ganeti.JSON import Ganeti.Objects import Ganeti.Query.Filter import Ganeti.Query.Language import Ganeti.Query.Query import Ganeti.Utils (niceSort) -- * Helpers -- | Run a query and check that we got a specific response. checkQueryResults :: ConfigData -> Query -> String -> [[ResultEntry]] -> Property checkQueryResults cfg qr descr expected = monadicIO $ do result <- run (query cfg False qr) >>= resultProp _ <- stop $ counterexample ("Inconsistent results in " ++ descr) (qresData result ==? expected) return () -- | Makes a node name query, given a filter. makeNodeQuery :: Filter FilterField -> Query makeNodeQuery = Query (ItemTypeOpCode QRNode) ["name"] -- | Checks if a given operation failed. expectBadQuery :: ConfigData -> Query -> String -> Property expectBadQuery cfg qr descr = monadicIO $ do result <- run (query cfg False qr) case result of Bad _ -> return () Ok a -> stop . failTest $ "Expected failure in " ++ descr ++ " but got " ++ show a -- | A helper to construct a list of results from an expected names list. namesToResult :: [String] -> [[ResultEntry]] namesToResult = map ((:[]) . ResultEntry RSNormal . Just . showJSON) -- | Generates a cluster and returns its node names too. genClusterNames :: Int -> Int -> Gen (ConfigData, [String]) genClusterNames min_nodes max_nodes = do numnodes <- choose (min_nodes, max_nodes) cfg <- genEmptyCluster numnodes return (cfg , niceSort . map UTF8.toString . Map.keys . fromContainer $ configNodes cfg) -- * Test cases -- | Tests single node filtering: eq should return it, and (lt and gt) -- should fail. prop_node_single_filter :: Property prop_node_single_filter = forAll (genClusterNames 1 maxNodes) $ \(cfg, allnodes) -> forAll (elements allnodes) $ \nname -> let fvalue = QuotedString nname buildflt n = n "name" fvalue expsingle = namesToResult [nname] othernodes = nname `delete` allnodes expnot = namesToResult othernodes test_query = checkQueryResults cfg . makeNodeQuery in conjoin [ test_query (buildflt EQFilter) "single-name 'EQ' filter" expsingle , test_query (NotFilter (buildflt EQFilter)) "single-name 'NOT EQ' filter" expnot , test_query (AndFilter [buildflt LTFilter, buildflt GTFilter]) "single-name 'AND [LT,GT]' filter" [] , test_query (AndFilter [buildflt LEFilter, buildflt GEFilter]) "single-name 'And [LE,GE]' filter" expsingle ] -- | Tests node filtering based on name equality: many 'OrFilter' -- should return all results combined, many 'AndFilter' together -- should return nothing. Note that we need at least 2 nodes so that -- the 'AndFilter' case breaks. prop_node_many_filter :: Property prop_node_many_filter = forAll (genClusterNames 2 maxNodes) $ \(cfg, nnames) -> let eqfilter = map (EQFilter "name" . QuotedString) nnames alln = namesToResult nnames test_query = checkQueryResults cfg . makeNodeQuery num_zero = NumericValue 0 in conjoin [ test_query (OrFilter eqfilter) "all nodes 'Or' name filter" alln , test_query (AndFilter eqfilter) "all nodes 'And' name filter" [] -- this next test works only because genEmptyCluster generates a -- cluster with no instances , test_query (EQFilter "pinst_cnt" num_zero) "pinst_cnt 'Eq' 0" alln , test_query (GTFilter "sinst_cnt" num_zero) "sinst_cnt 'GT' 0" [] ] -- | Tests name ordering consistency: requesting a 'simple filter' -- results in identical name ordering as the wanted names, requesting -- a more complex filter results in a niceSort-ed order. prop_node_name_ordering :: Property prop_node_name_ordering = forAll (genClusterNames 2 6) $ \(cfg, nnames) -> forAll (elements (subsequences nnames)) $ \sorted_nodes -> forAll (elements (permutations sorted_nodes)) $ \chosen_nodes -> let orfilter = OrFilter $ map (EQFilter "name" . QuotedString) chosen_nodes alln = namesToResult chosen_nodes all_sorted = namesToResult $ niceSort chosen_nodes test_query = checkQueryResults cfg . makeNodeQuery in conjoin [ test_query orfilter "simple filter/requested" alln , test_query (AndFilter [orfilter]) "complex filter/sorted" all_sorted ] -- | Tests node regex filtering. This is a very basic test :( prop_node_regex_filter :: Property prop_node_regex_filter = forAll (genClusterNames 0 maxNodes) $ \(cfg, nnames) -> case mkRegex ".*"::Result FilterRegex of Bad msg -> failTest $ "Can't build regex?! Error: " ++ msg Ok rx -> checkQueryResults cfg (makeNodeQuery (RegexpFilter "name" rx)) "rows for all nodes regexp filter" $ namesToResult nnames -- | Tests node regex filtering. This is a very basic test :( prop_node_bad_filter :: String -> Int -> Property prop_node_bad_filter rndname rndint = forAll (genClusterNames 1 maxNodes) $ \(cfg, _) -> let test_query = expectBadQuery cfg . makeNodeQuery string_value = QuotedString rndname numeric_value = NumericValue $ fromIntegral rndint in case mkRegex ".*"::Result FilterRegex of Bad msg -> failTest $ "Can't build regex?! Error: " ++ msg Ok rx -> conjoin [ test_query (RegexpFilter "offline" rx) "regex filter against boolean field" , test_query (EQFilter "name" numeric_value) "numeric value eq against string field" , test_query (TrueFilter "name") "true filter against string field" , test_query (EQFilter "offline" string_value) "quoted string eq against boolean field" , test_query (ContainsFilter "name" string_value) "quoted string in non-list field" , test_query (ContainsFilter "name" numeric_value) "numeric value in non-list field" ] -- | Tests make simple filter. prop_makeSimpleFilter :: Property prop_makeSimpleFilter = forAll (resize 10 $ listOf1 genName) $ \names -> forAll (resize 10 $ listOf1 arbitrary) $ \ids -> forAll genName $ \namefield -> conjoin [ counterexample "test expected names" $ makeSimpleFilter namefield (map Left names) ==? OrFilter (map (EQFilter namefield . QuotedString) names) , counterexample "test expected IDs" $ makeSimpleFilter namefield (map Right ids) ==? OrFilter (map (EQFilter namefield . NumericValue) ids) , counterexample "test empty names" $ makeSimpleFilter namefield [] ==? EmptyFilter ] testSuite "Query/Filter" [ 'prop_node_single_filter , 'prop_node_many_filter , 'prop_node_name_ordering , 'prop_node_regex_filter , 'prop_node_bad_filter , 'prop_makeSimpleFilter ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Query/Instance.hs000064400000000000000000000133101476477700300225260ustar00rootroot00000000000000{-# LANGUAGE TupleSections, TemplateHaskell #-} {-| Unittests for Instance Queries. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Query.Instance ( testQuery_Instance ) where import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Map as Map import System.Time (ClockTime(..)) import Ganeti.JSON import Ganeti.Objects import Ganeti.Query.Instance import Ganeti.Rpc import Ganeti.Types import Test.Ganeti.TestHelper import Test.HUnit {-# ANN module "HLint: ignore Use camelCase" #-} -- | Creates an instance with the desired name, pnode uuid, -- 'AdminState', and 'AdminStateSource'. All other fields are -- placeholders. createInstance :: String -> String -> AdminState -> AdminStateSource -> Instance createInstance name pnodeUuid adminState adminStateSource = RealInstance $ RealInstanceData name pnodeUuid "" Kvm (GenericContainer Map.empty) (PartialBeParams Nothing Nothing Nothing Nothing Nothing Nothing) (GenericContainer Map.empty) (GenericContainer Map.empty) adminState adminStateSource [] [] False Nothing epochTime epochTime (UTF8.fromString "") 0 emptyTagSet where epochTime = TOD 0 0 -- | A fake InstanceInfo to be used to check values. fakeInstanceInfo :: InstanceInfo fakeInstanceInfo = InstanceInfo 0 InstanceStateRunning 0 0 -- | Erroneous node response - the exact error does not matter. responseError :: String -> (String, ERpcError a) responseError name = (name, Left . RpcResultError $ "Insignificant error") -- | Successful response - the error does not really matter. responseSuccess :: String -> [String] -> (String, ERpcError RpcResultAllInstancesInfo) responseSuccess name instNames = (name, Right . RpcResultAllInstancesInfo . map (, fakeInstanceInfo) $ instNames) -- | The instance used for testing. Called Waldo as test cases involve trouble -- finding information related to it. waldoInstance :: Instance waldoInstance = createInstance "Waldo" "prim" AdminUp AdminSource -- | Check that an error is thrown when the node is offline case_nodeOffline :: Assertion case_nodeOffline = let responses = [ responseError "prim" , responseError "second" , responseSuccess "node" ["NotWaldo", "DefinitelyNotWaldo"] ] in case getInstanceInfo responses waldoInstance of Left _ -> return () Right _ -> assertFailure "Error occurred when instance info is missing and node is offline" -- | Check that a Right Nothing is returned when the node is online, yet no info -- is present anywhere in the system. case_nodeOnlineNoInfo :: Assertion case_nodeOnlineNoInfo = let responses = [ responseSuccess "prim" ["NotWaldo1"] , responseSuccess "second" ["NotWaldo2"] , responseError "node" ] in case getInstanceInfo responses waldoInstance of Left _ -> assertFailure "Error occurred when instance info could be found on primary" Right Nothing -> return () Right _ -> assertFailure "Some instance info found when none should be" -- | Check the case when the info is on the primary node case_infoOnPrimary :: Assertion case_infoOnPrimary = let responses = [ responseSuccess "prim" ["NotWaldo1", "Waldo"] , responseSuccess "second" ["NotWaldo2"] , responseSuccess "node" ["NotWaldo3"] ] in case getInstanceInfo responses waldoInstance of Left _ -> assertFailure "Cannot retrieve instance info when present on primary node" Right (Just (_, True)) -> return () Right _ -> assertFailure "Instance info not found on primary node, despite being there" -- | Check the case when the info is on the primary node case_infoOnSecondary :: Assertion case_infoOnSecondary = let responses = [ responseSuccess "prim" ["NotWaldo1"] , responseSuccess "second" ["Waldo", "NotWaldo2"] , responseError "node" ] in case getInstanceInfo responses waldoInstance of Left _ -> assertFailure "Cannot retrieve instance info when present on secondary node" Right (Just (_, False)) -> return () Right _ -> assertFailure "Instance info not found on secondary node, despite being there" testSuite "Query_Instance" [ 'case_nodeOffline , 'case_nodeOnlineNoInfo , 'case_infoOnPrimary , 'case_infoOnSecondary ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Query/Language.hs000064400000000000000000000170341476477700300225140ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, FlexibleInstances #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Query.Language ( testQuery_Language , genFilter , genJSValue ) where import Test.HUnit (Assertion, assertEqual) import Test.QuickCheck import Control.Arrow (second) import Text.JSON import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.JSON import Ganeti.Query.Language {-# ANN module "HLint: ignore Use camelCase" #-} instance Arbitrary (Filter FilterField) where arbitrary = genFilter instance Arbitrary FilterRegex where arbitrary = genName >>= mkRegex -- a name should be a good regex -- | Custom 'Filter' generator (top-level), which enforces a -- (sane) limit on the depth of the generated filters. genFilter :: Gen (Filter FilterField) genFilter = choose (0, 10) >>= genFilter' -- | Custom generator for filters that correctly halves the state of -- the generators at each recursive step, per the QuickCheck -- documentation, in order not to run out of memory. genFilter' :: Int -> Gen (Filter FilterField) genFilter' 0 = oneof [ pure EmptyFilter , TrueFilter <$> genName , EQFilter <$> genName <*> value , LTFilter <$> genName <*> value , GTFilter <$> genName <*> value , LEFilter <$> genName <*> value , GEFilter <$> genName <*> value , RegexpFilter <$> genName <*> arbitrary , ContainsFilter <$> genName <*> value ] where value = oneof [ QuotedString <$> genName , NumericValue <$> arbitrary ] genFilter' n = oneof [ AndFilter <$> vectorOf n'' (genFilter' n') , OrFilter <$> vectorOf n'' (genFilter' n') , NotFilter <$> genFilter' n' ] where n' = n `div` 2 -- sub-filter generator size n'' = max n' 2 -- but we don't want empty or 1-element lists, -- so use this for and/or filter list length $(genArbitrary ''QueryTypeOp) $(genArbitrary ''QueryTypeLuxi) $(genArbitrary ''ItemType) $(genArbitrary ''ResultStatus) $(genArbitrary ''FieldType) $(genArbitrary ''FieldDefinition) -- | Generates an arbitrary JSValue. We do this via a function a not -- via arbitrary instance since that would require us to define an -- arbitrary for JSValue, which can be recursive, entering the usual -- problems with that; so we only generate the base types, not the -- recursive ones, and not 'JSNull', which we can't use in a -- 'RSNormal' 'ResultEntry'. genJSValue :: Gen JSValue genJSValue = oneof [ JSBool <$> arbitrary , JSRational <$> pure False <*> arbitrary , JSString <$> (toJSString <$> arbitrary) , (JSArray . map showJSON) <$> (arbitrary::Gen [Int]) , JSObject . toJSObject . map (second showJSON) <$> (arbitrary::Gen [(String, Int)]) ] -- | Generates a 'ResultEntry' value. genResultEntry :: Gen ResultEntry genResultEntry = do rs <- arbitrary rv <- case rs of RSNormal -> Just <$> genJSValue _ -> pure Nothing return $ ResultEntry rs rv $(genArbitrary ''QueryFieldsResult) -- | Tests that serialisation/deserialisation of filters is -- idempotent. prop_filter_serialisation :: Property prop_filter_serialisation = forAll genFilter testSerialisation -- | Tests that filter regexes are serialised correctly. prop_filterregex_instances :: FilterRegex -> Property prop_filterregex_instances rex = counterexample "failed JSON encoding" (testSerialisation rex) -- | Tests 'ResultStatus' serialisation. prop_resultstatus_serialisation :: ResultStatus -> Property prop_resultstatus_serialisation = testSerialisation -- | Tests 'FieldType' serialisation. prop_fieldtype_serialisation :: FieldType -> Property prop_fieldtype_serialisation = testSerialisation -- | Tests 'FieldDef' serialisation. prop_fielddef_serialisation :: FieldDefinition -> Property prop_fielddef_serialisation = testSerialisation -- | Tests 'ResultEntry' serialisation. Needed especially as this is -- done manually, and not via buildObject (different serialisation -- format). prop_resultentry_serialisation :: Property prop_resultentry_serialisation = forAll genResultEntry testSerialisation -- | Tests 'FieldDef' serialisation. We use a made-up maximum limit of -- 20 for the generator, since otherwise the lists become too long and -- we don't care so much about list length but rather structure. prop_fieldsresult_serialisation :: Property prop_fieldsresult_serialisation = forAll (resize 20 arbitrary::Gen QueryFieldsResult) testSerialisation -- | Tests 'ItemType' serialisation. prop_itemtype_serialisation :: ItemType -> Property prop_itemtype_serialisation = testSerialisation -- | Tests basic cases of filter parsing, including legacy ones. case_filterParsing :: Assertion case_filterParsing = do let check :: String -> Filter String -> Assertion check str expected = do jsval <- fromJResult "could not parse filter" $ decode str assertEqual str expected jsval val = QuotedString "val" valRegex <- mkRegex "val" check "null" EmptyFilter check "[\"&\", null, null]" $ AndFilter [EmptyFilter, EmptyFilter] check "[\"|\", null, null]" $ OrFilter [EmptyFilter, EmptyFilter] check "[\"!\", null]" $ NotFilter EmptyFilter check "[\"?\", \"field\"]" $ TrueFilter "field" check "[\"==\", \"field\", \"val\"]" $ EQFilter "field" val check "[\"<\", \"field\", \"val\"]" $ LTFilter "field" val check "[\">\", \"field\", \"val\"]" $ GTFilter "field" val check "[\"<=\", \"field\", \"val\"]" $ LEFilter "field" val check "[\">=\", \"field\", \"val\"]" $ GEFilter "field" val check "[\"=~\", \"field\", \"val\"]" $ RegexpFilter "field" valRegex check "[\"=[]\", \"field\", \"val\"]" $ ContainsFilter "field" val -- Legacy filters check "[\"=\", \"field\", \"val\"]" $ EQFilter "field" val check "[\"!=\", \"field\", \"val\"]" $ NotFilter (EQFilter "field" val) testSuite "Query/Language" [ 'prop_filter_serialisation , 'prop_filterregex_instances , 'prop_resultstatus_serialisation , 'prop_fieldtype_serialisation , 'prop_fielddef_serialisation , 'prop_resultentry_serialisation , 'prop_fieldsresult_serialisation , 'prop_itemtype_serialisation , 'case_filterParsing ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Query/Network.hs000064400000000000000000000065701476477700300224250ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for Network Queries. -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Query.Network ( testQuery_Network ) where import Ganeti.JSON import Ganeti.Objects import Ganeti.Query.Network import Test.Ganeti.Objects import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Test.QuickCheck import qualified Data.ByteString.UTF8 as UTF8 import qualified Data.Map as Map import Data.Maybe -- | Check if looking up a valid network ID of a nodegroup yields -- a non-Nothing result. prop_getGroupConnection :: NodeGroup -> Property prop_getGroupConnection group = let net_keys = map UTF8.toString . Map.keys . fromContainer . groupNetworks $ group in True ==? all (\nk -> isJust (getGroupConnection nk group)) net_keys -- | Checks if looking up an ID of a non-existing network in a node group -- yields 'Nothing'. prop_getGroupConnection_notFound :: NodeGroup -> String -> Property prop_getGroupConnection_notFound group uuid = let net_map = fromContainer . groupNetworks $ group in not (UTF8.fromString uuid `Map.member` net_map) ==> isNothing (getGroupConnection uuid group) -- | Checks whether actually connected instances are identified as such. prop_instIsConnected :: ConfigData -> Property prop_instIsConnected cfg = let nets = (fromContainer . configNetworks) cfg net_keys = map UTF8.toString $ Map.keys nets in forAll (genInstWithNets net_keys) $ \inst -> True ==? all (`instIsConnected` inst) net_keys -- | Tests whether instances that are not connected to a network are -- correctly classified as such. prop_instIsConnected_notFound :: ConfigData -> String -> Property prop_instIsConnected_notFound cfg network_uuid = let nets = (fromContainer . configNetworks) cfg net_keys = map UTF8.toString $ Map.keys nets in notElem network_uuid net_keys ==> forAll (genInstWithNets net_keys) $ \inst -> not (instIsConnected network_uuid inst) testSuite "Query_Network" [ 'prop_getGroupConnection , 'prop_getGroupConnection_notFound , 'prop_instIsConnected , 'prop_instIsConnected_notFound ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Query/Query.hs000064400000000000000000000403451476477700300220770ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, BangPatterns #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Query.Query (testQuery_Query) where import Test.HUnit (Assertion, assertEqual) import Test.QuickCheck hiding (Result) import Test.QuickCheck.Monadic import Data.Function (on) import Data.List import qualified Data.Map as Map import Data.Maybe import qualified Data.Set as Set import Text.JSON (JSValue(..), showJSON) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Objects (genEmptyCluster) import Ganeti.BasicTypes import Ganeti.Errors import Ganeti.JSON import Ganeti.Objects import Ganeti.Query.Filter import qualified Ganeti.Query.Group as Group import Ganeti.Query.Language import qualified Ganeti.Query.Node as Node import Ganeti.Query.Query import qualified Ganeti.Query.Job as Job import Ganeti.Utils (sepSplit) {-# ANN module "HLint: ignore Use camelCase" #-} -- * Helpers -- | Checks if a list of field definitions contains unknown fields. hasUnknownFields :: [FieldDefinition] -> Bool hasUnknownFields = (QFTUnknown `notElem`) . map fdefKind -- * Test cases -- ** Node queries -- | Tests that querying any existing fields, via either query or -- queryFields, will not return unknown fields. prop_queryNode_noUnknown :: Property prop_queryNode_noUnknown = forAll (choose (0, maxNodes) >>= genEmptyCluster) $ \cluster -> forAll (elements (Map.keys Node.fieldsMap)) $ \field -> monadicIO $ do QueryResult fdefs fdata <- run (query cluster False (Query (ItemTypeOpCode QRNode) [field] EmptyFilter)) >>= resultProp QueryFieldsResult fdefs' <- resultProp $ queryFields (QueryFields (ItemTypeOpCode QRNode) [field]) _ <- stop $ conjoin [ counterexample ("Got unknown fields via query (" ++ show fdefs ++ ")") (hasUnknownFields fdefs) , counterexample ("Got unknown result status via query (" ++ show fdata ++ ")") (all (all ((/= RSUnknown) . rentryStatus)) fdata) , counterexample ("Got unknown fields via query fields (" ++ show fdefs'++ ")") (hasUnknownFields fdefs') ] return () -- | Tests that an unknown field is returned as such. prop_queryNode_Unknown :: Property prop_queryNode_Unknown = forAll (choose (0, maxNodes) >>= genEmptyCluster) $ \cluster -> forAll (arbitrary `suchThat` (`notElem` Map.keys Node.fieldsMap)) $ \field -> monadicIO $ do QueryResult fdefs fdata <- run (query cluster False (Query (ItemTypeOpCode QRNode) [field] EmptyFilter)) >>= resultProp QueryFieldsResult fdefs' <- resultProp $ queryFields (QueryFields (ItemTypeOpCode QRNode) [field]) _ <- stop $ conjoin [ counterexample ("Got known fields via query (" ++ show fdefs ++ ")") (not $ hasUnknownFields fdefs) , counterexample ("Got /= ResultUnknown result status via query (" ++ show fdata ++ ")") (all (all ((== RSUnknown) . rentryStatus)) fdata) , counterexample ("Got a Just in a result value (" ++ show fdata ++ ")") (all (all (isNothing . rentryValue)) fdata) , counterexample ("Got known fields via query fields (" ++ show fdefs' ++ ")") (not $ hasUnknownFields fdefs') ] return () -- | Checks that a result type is conforming to a field definition. checkResultType :: FieldDefinition -> ResultEntry -> Property checkResultType _ (ResultEntry RSNormal Nothing) = failTest "Nothing result in RSNormal field" checkResultType _ (ResultEntry _ Nothing) = passTest checkResultType fdef (ResultEntry RSNormal (Just v)) = case (fdefKind fdef, v) of (QFTText , JSString {}) -> passTest (QFTBool , JSBool {}) -> passTest (QFTNumber , JSRational {}) -> passTest (QFTNumberFloat , JSRational {}) -> passTest (QFTTimestamp , JSRational {}) -> passTest (QFTUnit , JSRational {}) -> passTest (QFTOther , _) -> passTest -- meh, QFT not precise... (kind, _) -> failTest $ "Type mismatch, field definition says " ++ show kind ++ " but returned value is " ++ show v ++ " for field '" ++ fdefName fdef ++ "'" checkResultType _ (ResultEntry r (Just _)) = failTest $ "Just result in " ++ show r ++ " field" -- | Tests that querying any existing fields, the following three -- properties hold: RSNormal corresponds to a Just value, any other -- value corresponds to Nothing, and for a RSNormal and value field, -- the type of the value corresponds to the type of the field as -- declared in the FieldDefinition. prop_queryNode_types :: Property prop_queryNode_types = forAll (choose (0, maxNodes)) $ \numnodes -> forAll (genEmptyCluster numnodes) $ \cfg -> forAll (elements (Map.keys Node.fieldsMap)) $ \field -> monadicIO $ do QueryResult fdefs fdata <- run (query cfg False (Query (ItemTypeOpCode QRNode) [field] EmptyFilter)) >>= resultProp _ <- stop $ conjoin [ counterexample ("Inconsistent result entries (" ++ show fdata ++ ")") (conjoin $ map (conjoin . zipWith checkResultType fdefs) fdata) , counterexample "Wrong field definitions length" (length fdefs ==? 1) , counterexample "Wrong field result rows length" (all ((== 1) . length) fdata) , counterexample "Wrong number of result rows" (length fdata ==? numnodes) ] return () -- | Test that queryFields with empty fields list returns all node fields. case_queryNode_allfields :: Assertion case_queryNode_allfields = do fdefs <- case queryFields (QueryFields (ItemTypeOpCode QRNode) []) of Bad msg -> fail $ "Error in query all fields: " ++ formatError msg Ok (QueryFieldsResult v) -> return v let field_sort = compare `on` fdefName assertEqual "Mismatch in all fields list" (sortBy field_sort . map (\(f, _, _) -> f) $ Map.elems Node.fieldsMap) (sortBy field_sort fdefs) -- | Check if cluster node names are unique (first elems). areNodeNamesSane :: ConfigData -> Bool areNodeNamesSane cfg = let fqdns = map nodeName . Map.elems . fromContainer $ configNodes cfg names = map (head . sepSplit '.') fqdns in length names == length (nub names) -- | Check that the nodes reported by a name filter are sane. prop_queryNode_filter :: Property prop_queryNode_filter = forAll (choose (1, maxNodes)) $ \nodes -> forAll (genEmptyCluster nodes `suchThat` areNodeNamesSane) $ \cluster -> monadicIO $ do let node_list = map nodeName . Map.elems . fromContainer $ configNodes cluster count <- pick $ choose (1, nodes) fqdn_set <- pick . genSetHelper node_list $ Just count let fqdns = Set.elems fqdn_set names = map (head . sepSplit '.') fqdns flt = makeSimpleFilter "name" $ map Left names QueryResult _ fdata <- run (query cluster False (Query (ItemTypeOpCode QRNode) ["name"] flt)) >>= resultProp _ <- stop $ conjoin [ counterexample "Invalid node names" $ map (map rentryValue) fdata ==? map (\f -> [Just (showJSON f)]) fqdns ] return () -- ** Group queries prop_queryGroup_noUnknown :: Property prop_queryGroup_noUnknown = forAll (choose (0, maxNodes) >>= genEmptyCluster) $ \cluster -> forAll (elements (Map.keys Group.fieldsMap)) $ \field -> monadicIO $ do QueryResult fdefs fdata <- run (query cluster False (Query (ItemTypeOpCode QRGroup) [field] EmptyFilter)) >>= resultProp QueryFieldsResult fdefs' <- resultProp $ queryFields (QueryFields (ItemTypeOpCode QRGroup) [field]) _ <- stop $ conjoin [ counterexample ("Got unknown fields via query (" ++ show fdefs ++ ")") (hasUnknownFields fdefs) , counterexample ("Got unknown result status via query (" ++ show fdata ++ ")") (all (all ((/= RSUnknown) . rentryStatus)) fdata) , counterexample ("Got unknown fields via query fields (" ++ show fdefs' ++ ")") (hasUnknownFields fdefs') ] return () prop_queryGroup_Unknown :: Property prop_queryGroup_Unknown = forAll (choose (0, maxNodes) >>= genEmptyCluster) $ \cluster -> forAll (arbitrary `suchThat` (`notElem` Map.keys Group.fieldsMap)) $ \field -> monadicIO $ do QueryResult fdefs fdata <- run (query cluster False (Query (ItemTypeOpCode QRGroup) [field] EmptyFilter)) >>= resultProp QueryFieldsResult fdefs' <- resultProp $ queryFields (QueryFields (ItemTypeOpCode QRGroup) [field]) _ <- stop $ conjoin [ counterexample ("Got known fields via query (" ++ show fdefs ++ ")") (not $ hasUnknownFields fdefs) , counterexample ("Got /= ResultUnknown result status via query (" ++ show fdata ++ ")") (all (all ((== RSUnknown) . rentryStatus)) fdata) , counterexample ("Got a Just in a result value (" ++ show fdata ++ ")") (all (all (isNothing . rentryValue)) fdata) , counterexample ("Got known fields via query fields (" ++ show fdefs' ++ ")") (not $ hasUnknownFields fdefs') ] return () prop_queryGroup_types :: Property prop_queryGroup_types = forAll (choose (0, maxNodes)) $ \numnodes -> forAll (genEmptyCluster numnodes) $ \cfg -> forAll (elements (Map.keys Group.fieldsMap)) $ \field -> monadicIO $ do QueryResult fdefs fdata <- run (query cfg False (Query (ItemTypeOpCode QRGroup) [field] EmptyFilter)) >>= resultProp _ <- stop $ conjoin [ counterexample ("Inconsistent result entries (" ++ show fdata ++ ")") (conjoin $ map (conjoin . zipWith checkResultType fdefs) fdata) , counterexample "Wrong field definitions length" (length fdefs ==? 1) , counterexample "Wrong field result rows length" (all ((== 1) . length) fdata) ] return () case_queryGroup_allfields :: Assertion case_queryGroup_allfields = do fdefs <- case queryFields (QueryFields (ItemTypeOpCode QRGroup) []) of Bad msg -> fail $ "Error in query all fields: " ++ formatError msg Ok (QueryFieldsResult v) -> return v let field_sort = compare `on` fdefName assertEqual "Mismatch in all fields list" (sortBy field_sort . map (\(f, _, _) -> f) $ Map.elems Group.fieldsMap) (sortBy field_sort fdefs) -- | Check that the node count reported by a group list is sane. -- -- FIXME: also verify the node list, etc. prop_queryGroup_nodeCount :: Property prop_queryGroup_nodeCount = forAll (choose (0, maxNodes)) $ \nodes -> forAll (genEmptyCluster nodes) $ \cluster -> monadicIO $ do QueryResult _ fdata <- run (query cluster False (Query (ItemTypeOpCode QRGroup) ["node_cnt"] EmptyFilter)) >>= resultProp _ <- stop $ conjoin [ counterexample "Invalid node count" $ map (map rentryValue) fdata ==? [[Just (showJSON nodes)]] ] return () -- ** Job queries -- | Tests that querying any existing fields, via either query or -- queryFields, will not return unknown fields. This uses 'undefined' -- for config, as job queries shouldn't use the configuration, and an -- explicit filter as otherwise non-live queries wouldn't return any -- result rows. prop_queryJob_noUnknown :: Property prop_queryJob_noUnknown = forAll (listOf (arbitrary::Gen (Positive Integer))) $ \ids -> forAll (elements (Map.keys Job.fieldsMap)) $ \field -> monadicIO $ do let qtype = ItemTypeLuxi QRJob flt = makeSimpleFilter (nameField qtype) $ map (\(Positive i) -> Right i) ids QueryResult fdefs fdata <- run (query undefined False (Query qtype [field] flt)) >>= resultProp QueryFieldsResult fdefs' <- resultProp $ queryFields (QueryFields qtype [field]) _ <- stop $ conjoin [ counterexample ("Got unknown fields via query (" ++ show fdefs ++ ")") (hasUnknownFields fdefs) , counterexample ("Got unknown result status via query (" ++ show fdata ++ ")") (all (all ((/= RSUnknown) . rentryStatus)) fdata) , counterexample ("Got unknown fields via query fields (" ++ show fdefs'++ ")") (hasUnknownFields fdefs') ] return () -- | Tests that an unknown field is returned as such. prop_queryJob_Unknown :: Property prop_queryJob_Unknown = forAll (listOf (arbitrary::Gen (Positive Integer))) $ \ids -> forAll (arbitrary `suchThat` (`notElem` Map.keys Job.fieldsMap)) $ \field -> monadicIO $ do let qtype = ItemTypeLuxi QRJob flt = makeSimpleFilter (nameField qtype) $ map (\(Positive i) -> Right i) ids QueryResult fdefs fdata <- run (query undefined False (Query qtype [field] flt)) >>= resultProp QueryFieldsResult fdefs' <- resultProp $ queryFields (QueryFields qtype [field]) _ <- stop $ conjoin [ counterexample ("Got known fields via query (" ++ show fdefs ++ ")") (not $ hasUnknownFields fdefs) , counterexample ("Got /= ResultUnknown result status via query (" ++ show fdata ++ ")") (all (all ((== RSUnknown) . rentryStatus)) fdata) , counterexample ("Got a Just in a result value (" ++ show fdata ++ ")") (all (all (isNothing . rentryValue)) fdata) , counterexample ("Got known fields via query fields (" ++ show fdefs' ++ ")") (not $ hasUnknownFields fdefs') ] return () -- ** Misc other tests -- | Tests that requested names checking behaves as expected. prop_getRequestedNames :: Property prop_getRequestedNames = forAll genName $ \node1 -> let chk = getRequestedNames . Query (ItemTypeOpCode QRNode) [] q_node1 = QuotedString node1 eq_name = EQFilter "name" eq_node1 = eq_name q_node1 in conjoin [ counterexample "empty filter" $ chk EmptyFilter ==? [] , counterexample "and filter" $ chk (AndFilter [eq_node1]) ==? [] , counterexample "simple equality" $ chk eq_node1 ==? [node1] , counterexample "non-name field" $ chk (EQFilter "foo" q_node1) ==? [] , counterexample "non-simple filter" $ chk (OrFilter [ eq_node1 , LTFilter "foo" q_node1]) ==? [] ] testSuite "Query/Query" [ 'prop_queryNode_noUnknown , 'prop_queryNode_Unknown , 'prop_queryNode_types , 'prop_queryNode_filter , 'case_queryNode_allfields , 'prop_queryGroup_noUnknown , 'prop_queryGroup_Unknown , 'prop_queryGroup_types , 'case_queryGroup_allfields , 'prop_queryGroup_nodeCount , 'prop_queryJob_noUnknown , 'prop_queryJob_Unknown , 'prop_getRequestedNames ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Rpc.hs000064400000000000000000000120601476477700300204020ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Rpc (testRpc) where import Test.QuickCheck import Test.QuickCheck.Monadic (monadicIO, run, stop) import qualified Data.Map as Map import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Test.Ganeti.Objects (genInst) import qualified Ganeti.Rpc as Rpc import qualified Ganeti.Objects as Objects import qualified Ganeti.Types as Types import qualified Ganeti.JSON as JSON import Ganeti.Types instance Arbitrary Rpc.RpcCallInstanceConsoleInfo where arbitrary = Rpc.RpcCallInstanceConsoleInfo <$> genConsoleInfoCallParams instance Arbitrary Rpc.Compressed where arbitrary = Rpc.toCompressed <$> arbitrary genStorageUnit :: Gen StorageUnit genStorageUnit = do storage_type <- arbitrary storage_key <- genName storage_es <- arbitrary return $ addParamsToStorageUnit storage_es (SURaw storage_type storage_key) genStorageUnits :: Gen [StorageUnit] genStorageUnits = do num_storage_units <- choose (0, 5) vectorOf num_storage_units genStorageUnit -- FIXME: Generate more interesting hvparams -- | Generate Hvparams genHvParams :: Gen Objects.HvParams genHvParams = return $ JSON.GenericContainer Map.empty -- | Generate hypervisor specifications to be used for the NodeInfo call genHvSpecs :: Gen [(Types.Hypervisor, Objects.HvParams)] genHvSpecs = do numhv <- choose (0, 5) hvs <- vectorOf numhv arbitrary hvparams <- vectorOf numhv genHvParams let specs = zip hvs hvparams return specs instance Arbitrary Rpc.RpcCallAllInstancesInfo where arbitrary = Rpc.RpcCallAllInstancesInfo <$> genHvSpecs instance Arbitrary Rpc.RpcCallInstanceList where arbitrary = Rpc.RpcCallInstanceList <$> arbitrary instance Arbitrary Rpc.RpcCallNodeInfo where arbitrary = Rpc.RpcCallNodeInfo <$> genStorageUnits <*> genHvSpecs -- | Generates per-instance console info params for the 'InstanceConsoleInfo' -- call. genConsoleInfoCallParams :: Gen [(String, Rpc.InstanceConsoleInfoParams)] genConsoleInfoCallParams = do numInstances <- choose (0, 3) names <- vectorOf numInstances arbitrary params <- vectorOf numInstances genInstanceConsoleInfoParams return $ zip names params -- | Generates parameters for the console info call, consisting of an instance -- object, node object, 'HvParams', and 'FilledBeParams'. genInstanceConsoleInfoParams :: Gen Rpc.InstanceConsoleInfoParams genInstanceConsoleInfoParams = Rpc.InstanceConsoleInfoParams <$> genInst <*> arbitrary <*> arbitrary <*> genHvParams <*> arbitrary -- | Monadic check that, for an offline node and a call that does not support -- offline nodes, we get a OfflineNodeError response. runOfflineTest :: (Rpc.Rpc a b, Eq b, Show b) => a -> Property runOfflineTest call = forAll (arbitrary `suchThat` Objects.nodeOffline) $ \node -> monadicIO $ do res <- run $ Rpc.executeRpcCall [node] call _ <- stop $ res ==? [(node, Left Rpc.OfflineNodeError)] return () prop_noffl_request_allinstinfo :: Rpc.RpcCallAllInstancesInfo -> Property prop_noffl_request_allinstinfo = runOfflineTest prop_noffl_request_instconsinfo :: Rpc.RpcCallInstanceConsoleInfo -> Property prop_noffl_request_instconsinfo = runOfflineTest prop_noffl_request_instlist :: Rpc.RpcCallInstanceList -> Property prop_noffl_request_instlist = runOfflineTest prop_noffl_request_nodeinfo :: Rpc.RpcCallNodeInfo -> Property prop_noffl_request_nodeinfo = runOfflineTest -- | Test that the serialisation of 'Compressed' is idempotent. prop_Compressed_serialisation :: Rpc.Compressed -> Property prop_Compressed_serialisation = testSerialisation testSuite "Rpc" [ 'prop_noffl_request_allinstinfo , 'prop_noffl_request_instconsinfo , 'prop_noffl_request_instlist , 'prop_noffl_request_nodeinfo , 'prop_Compressed_serialisation ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Runtime.hs000064400000000000000000000126031476477700300213040ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for "Ganeti.Runtime". -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Runtime (testRuntime) where import Test.HUnit import qualified Text.JSON as J import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.Runtime {-# ANN module "HLint: ignore Use camelCase" #-} -- | Tests the compatibility between Haskell and Python log files. case_LogFiles :: Assertion case_LogFiles = do let daemons = [minBound..maxBound]::[GanetiDaemon] dnames = map daemonName daemons dfiles <- mapM daemonLogFile daemons let serialized = J.encode dnames py_stdout <- runPython "from ganeti import constants\n\ \from ganeti import serializer\n\ \import sys\n\ \daemons = serializer.Load(sys.stdin.read())\n\ \logfiles = [constants.DAEMONS_LOGFILES[d] for d in daemons]\n\ \sys.stdout.buffer.write(serializer.Dump(logfiles))" serialized >>= checkPythonResult let deserialised = J.decode py_stdout::J.Result [String] decoded <- case deserialised of J.Ok ops -> return ops J.Error msg -> assertFailure ("Unable to decode log files: " ++ msg) -- this already raised an expection, but we need it -- for proper types >> fail "Unable to decode log files" assertEqual "Mismatch in number of returned log files" (length decoded) (length daemons) mapM_ (uncurry (assertEqual "Different result after encoding/decoding") ) $ zip dfiles decoded -- | Tests the compatibility between Haskell and Python users. case_UsersGroups :: Assertion case_UsersGroups = do -- note: we don't have here a programatic way to list all users, so -- we harcode some parts of the two (hs/py) lists let daemons = [minBound..maxBound]::[GanetiDaemon] users = map daemonUser daemons groups = map daemonGroup $ map DaemonGroup daemons ++ map ExtraGroup [minBound..maxBound] py_stdout <- runPython "from ganeti import constants\n\ \from ganeti import serializer\n\ \import sys\n\ \users = [constants.MASTERD_USER,\n\ \ constants.METAD_USER,\n\ \ constants.NODED_USER,\n\ \ constants.RAPI_USER,\n\ \ constants.CONFD_USER,\n\ \ constants.WCONFD_USER,\n\ \ constants.KVMD_USER,\n\ \ constants.LUXID_USER,\n\ \ constants.MOND_USER,\n\ \ ]\n\ \groups = [constants.MASTERD_GROUP,\n\ \ constants.METAD_GROUP,\n\ \ constants.NODED_GROUP,\n\ \ constants.RAPI_GROUP,\n\ \ constants.CONFD_GROUP,\n\ \ constants.WCONFD_GROUP,\n\ \ constants.KVMD_GROUP,\n\ \ constants.LUXID_GROUP,\n\ \ constants.MOND_GROUP,\n\ \ constants.DAEMONS_GROUP,\n\ \ constants.ADMIN_GROUP,\n\ \ ]\n\ \encoded = (users, groups)\n\ \sys.stdout.buffer.write(serializer.Dump(encoded))" "" >>= checkPythonResult let deserialised = J.decode py_stdout::J.Result ([String], [String]) (py_users, py_groups) <- case deserialised of J.Ok ops -> return ops J.Error msg -> assertFailure ("Unable to decode users/groups: " ++ msg) -- this already raised an expection, but we need it for proper -- types >> fail "Unable to decode users/groups" assertEqual "Mismatch in number of returned users" (length py_users) (length users) assertEqual "Mismatch in number of returned users" (length py_groups) (length groups) mapM_ (uncurry (assertEqual "Different result for users") ) $ zip users py_users mapM_ (uncurry (assertEqual "Different result for groups") ) $ zip groups py_groups testSuite "Runtime" [ 'case_LogFiles , 'case_UsersGroups ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/SlotMap.hs000064400000000000000000000212671476477700300212460ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, ScopedTypeVariables #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for the SlotMap. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.SlotMap ( testSlotMap , genSlotLimit , genTestKey , overfullKeys ) where import Prelude hiding (all) import Control.Monad import Data.Foldable (all) import qualified Data.Map as Map import Data.Map (Map, member, keys, keysSet) import Data.Set (Set, size, union) import qualified Data.Set as Set import Test.HUnit import Test.QuickCheck import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Test.Ganeti.Types () import Ganeti.SlotMap {-# ANN module "HLint: ignore Use camelCase" #-} -- | Generates a number typical for the limit of a `Slot`. -- Useful for constructing resource bounds when not directly constructing -- the relevant `Slot`s. genSlotLimit :: Gen Int genSlotLimit = frequency [ (9, choose (1, 5)) , (1, choose (1, 100)) ] -- Don't create huge slot limits. instance Arbitrary Slot where arbitrary = do limit <- genSlotLimit occ <- choose (0, limit * 2) return $ Slot occ limit -- | Generates a number typical for the occupied count of a `Slot`. -- Useful for constructing `CountMap`s. genSlotCount :: Gen Int genSlotCount = slotOccupied <$> arbitrary -- | Takes a slot and resamples its `slotOccupied` count to fit the limit. resampleFittingSlot :: Slot -> Gen Slot resampleFittingSlot (Slot _ limit) = do occ <- choose (0, limit) return $ Slot occ limit -- | What we use as key for testing `SlotMap`s. type TestKey = String -- | Generates short strings used as `SlotMap` keys. -- -- We limit ourselves to a small set of key strings with high probability to -- increase the chance that `SlotMap`s actually have more than one slot taken. genTestKey :: Gen TestKey genTestKey = frequency [ (9, elements ["a", "b", "c", "d", "e"]) , (1, genPrintableAsciiString) ] -- | Generates small lists. listSizeGen :: Gen Int listSizeGen = frequency [ (9, choose (1, 5)) , (1, choose (1, 100)) ] -- | Generates a `SlotMap` given a generator for the keys (see `genTestKey`). genSlotMap :: (Ord a) => Gen a -> Gen (SlotMap a) genSlotMap keyGen = do n <- listSizeGen -- don't create huge `SlotMap`s Map.fromList <$> vectorOf n ((,) <$> keyGen <*> arbitrary) -- | Generates a `CountMap` given a generator for the keys (see `genTestKey`). genCountMap :: (Ord a) => Gen a -> Gen (CountMap a) genCountMap keyGen = do n <- listSizeGen -- don't create huge `CountMap`s Map.fromList <$> vectorOf n ((,) <$> keyGen <*> genSlotCount) -- | Tells which keys of a `SlotMap` are overfull. overfullKeys :: (Ord a) => SlotMap a -> Set a overfullKeys sm = Set.fromList [ a | (a, Slot occ limit) <- Map.toList sm, occ > limit ] -- | Generates a `SlotMap` for which all slots are within their limits. genFittingSlotMap :: (Ord a) => Gen a -> Gen (SlotMap a) genFittingSlotMap keyGen = do -- Generate a SlotMap, then resample all slots to be fitting. slotMap <- traverse resampleFittingSlot =<< genSlotMap keyGen when (isOverfull slotMap) $ error "BUG: FittingSlotMap Gen is wrong" return slotMap -- * Test cases case_isOverfull :: Assertion case_isOverfull = do assertBool "overfull" . isOverfull $ Map.fromList [("buck", Slot 3 2)] assertBool "not overfull" . not . isOverfull $ Map.fromList [("buck", Slot 2 2)] assertBool "empty" . not . isOverfull $ (Map.fromList [] :: SlotMap TestKey) case_occupySlots_examples :: Assertion case_occupySlots_examples = do let a n = ("a", Slot n 2) let b n = ("b", Slot n 4) let sm = Map.fromList [a 1, b 2] cm = Map.fromList [("a", 1), ("b", 1), ("c", 5)] assertEqual "fitting occupySlots" (sm `occupySlots` cm) (Map.fromList [a 2, b 3, ("c", Slot 5 0)]) -- | Union of the keys of two maps. keyUnion :: (Ord a) => Map a b -> Map a c -> Set a keyUnion a b = keysSet a `union` keysSet b -- | Tests properties of `SlotMap`s being filled up. prop_occupySlots :: Property prop_occupySlots = forAll arbitrary $ \(sm :: SlotMap Int, cm :: CountMap Int) -> let smOcc = sm `occupySlots` cm in conjoin [ counterexample "input keys are preserved" $ all (`member` smOcc) (keyUnion sm cm) , counterexample "all keys must come from the input keys" $ all (`Set.member` keyUnion sm cm) (keys smOcc) ] -- | Tests for whether there's still space for a job given its rate -- limits. case_hasSlotsFor_examples :: Assertion case_hasSlotsFor_examples = do let a n = ("a", Slot n 2) let b n = ("b", Slot n 4) let c n = ("c", Slot n 8) let sm = Map.fromList [a 1, b 2] assertBool "fits" $ sm `hasSlotsFor` Map.fromList [("a", 1), ("b", 1)] assertBool "doesn't fit" . not $ sm `hasSlotsFor` Map.fromList [("a", 1), ("b", 3)] let smOverfull = Map.fromList [a 1, b 2, c 10] assertBool "fits (untouched keys overfull)" $ isOverfull smOverfull && smOverfull `hasSlotsFor` Map.fromList [("a", 1), ("b", 1)] assertBool "empty fitting" $ Map.empty `hasSlotsFor` (Map.empty :: CountMap TestKey) assertBool "empty not fitting" . not $ Map.empty `hasSlotsFor` Map.fromList [("a", 1), ("b", 100)] assertBool "empty not fitting" . not $ Map.empty `hasSlotsFor` Map.fromList [("a", 1)] -- | Tests properties of `hasSlotsFor` on `SlotMap`s that are known to -- respect their limits. prop_hasSlotsFor_fitting :: Property prop_hasSlotsFor_fitting = forAll (genFittingSlotMap genTestKey) $ \sm -> forAll (genCountMap genTestKey) $ \cm -> sm `hasSlotsFor` cm ==? not (isOverfull $ sm `occupySlots` cm) -- | Tests properties of `hasSlotsFor`, irrespective of whether the -- input `SlotMap`s respect their limits or not. prop_hasSlotsFor :: Property prop_hasSlotsFor = let -- Generates `SlotMap`s for combining. genMaps = resize 10 $ do -- We don't need very large SlotMaps. sm1 <- genSlotMap genTestKey -- We need to make sm2 smaller to make `hasSlots` below more -- likely (otherwise the LHS of ==> is always false). sm2 <- sized $ \n -> resize (n `div` 3) (genSlotMap genTestKey) -- We also want to test (sm1, sm1); we have to make it more -- likely for it to ever happen. frequency [ (1, return (sm1, sm1)) , (9, return (sm1, sm2)) ] in forAll genMaps $ \(sm1, sm2) -> let fits = sm1 `hasSlotsFor` toCountMap sm2 smOcc = sm1 `occupySlots` toCountMap sm2 oldOverfullBucks = overfullKeys sm1 newOverfullBucks = overfullKeys smOcc in conjoin [ counterexample "if there's enough extra space, then the new\ \ overfull keys must be as before" $ fits ==> (newOverfullBucks ==? oldOverfullBucks) -- Note that the other way around does not hold: -- (newOverfullBucks == oldOverfullBucks) ==> fits , counterexample "joining SlotMaps must not change the number of\ \ overfull keys (but may change their slot\ \ counts" . property $ size newOverfullBucks >= size oldOverfullBucks ] testSuite "SlotMap" [ 'case_isOverfull , 'case_occupySlots_examples , 'prop_occupySlots , 'case_hasSlotsFor_examples , 'prop_hasSlotsFor_fitting , 'prop_hasSlotsFor ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Ssconf.hs000064400000000000000000000065421476477700300211210ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Ssconf (testSsconf) where import Test.QuickCheck import qualified Test.HUnit as HUnit import Data.List import qualified Data.Map as M import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import qualified Ganeti.Ssconf as Ssconf import qualified Ganeti.Types as Types -- * Ssconf tests $(genArbitrary ''Ssconf.SSKey) instance Arbitrary Ssconf.SSConf where arbitrary = fmap (Ssconf.SSConf . M.fromList) arbitrary -- * Reading SSConf prop_filename :: Ssconf.SSKey -> Property prop_filename key = counterexample "Key doesn't start with correct prefix" $ Ssconf.sSFilePrefix `isPrefixOf` Ssconf.keyToFilename "" key caseParseNodesVmCapable :: HUnit.Assertion caseParseNodesVmCapable = do let str = "node1.example.com=True\nnode2.example.com=False" result = Ssconf.parseNodesVmCapable str expected = return [ ("node1.example.com", True) , ("node2.example.com", False) ] HUnit.assertEqual "Mismatch in parsed and expected result" expected result caseParseHypervisorList :: HUnit.Assertion caseParseHypervisorList = do let result = Ssconf.parseHypervisorList "kvm\nxen-pvm\nxen-hvm" expected = return [Types.Kvm, Types.XenPvm, Types.XenHvm] HUnit.assertEqual "Mismatch in parsed and expected result" expected result caseParseEnabledUserShutdown :: HUnit.Assertion caseParseEnabledUserShutdown = do let result1 = Ssconf.parseEnabledUserShutdown "True" result2 = Ssconf.parseEnabledUserShutdown "False" HUnit.assertEqual "Mismatch in parsed and expected result" (return True) result1 HUnit.assertEqual "Mismatch in parsed and expected result" (return False) result2 -- * Creating and writing SSConf -- | Verify that for SSConf we have readJSON . showJSON = Ok. prop_ReadShow :: Ssconf.SSConf -> Property prop_ReadShow = testSerialisation testSuite "Ssconf" [ 'prop_filename , 'caseParseNodesVmCapable , 'caseParseHypervisorList , 'caseParseEnabledUserShutdown , 'prop_ReadShow ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Storage/000075500000000000000000000000001476477700300207275ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Storage/Diskstats/000075500000000000000000000000001476477700300227005ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Storage/Diskstats/Parser.hs000064400000000000000000000120571476477700300244750ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for the @/proc/diskstats@ parser -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Storage.Diskstats.Parser (testBlock_Diskstats_Parser) where import Test.QuickCheck as QuickCheck hiding (Result) import Test.HUnit import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import qualified Data.Attoparsec.Text as A import Data.Text (pack) import Text.Printf import Ganeti.Storage.Diskstats.Parser (diskstatsParser) import Ganeti.Storage.Diskstats.Types {-# ANN module "HLint: ignore Use camelCase" #-} -- | Test a diskstats. case_diskstats :: Assertion case_diskstats = testParser diskstatsParser "proc_diskstats.txt" [ Diskstats 1 0 "ram0" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 1 "ram1" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 2 "ram2" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 3 "ram3" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 4 "ram4" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 5 "ram5" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 6 "ram6" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 7 "ram7" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 8 "ram8" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 9 "ram9" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 10 "ram10" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 11 "ram11" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 12 "ram12" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 13 "ram13" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 14 "ram14" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 1 15 "ram15" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 7 0 "loop0" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 7 1 "loop1" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 7 2 "loop2" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 7 3 "loop3" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 7 4 "loop4" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 7 5 "loop5" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 7 6 "loop6" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 7 7 "loop7" 0 0 0 0 0 0 0 0 0 0 0 , Diskstats 8 0 "sda" 89502 4833 4433387 89244 519115 62738 16059726 465120 0 149148 554564 , Diskstats 8 1 "sda1" 505 2431 8526 132 478 174 124358 8500 0 340 8632 , Diskstats 8 2 "sda2" 2 0 4 4 0 0 0 0 0 4 4 , Diskstats 8 5 "sda5" 88802 2269 4422249 89032 453703 62564 15935368 396244 0 90064 485500 , Diskstats 252 0 "dm-0" 90978 0 4420002 158632 582226 0 15935368 5592012 0 167688 5750652 , Diskstats 252 1 "dm-1" 88775 0 4402378 157204 469594 0 15136008 4910424 0 164556 5067640 , Diskstats 252 2 "dm-2" 1956 0 15648 1052 99920 0 799360 682492 0 4516 683552 , Diskstats 8 16 "sdb" 0 0 0 0 0 0 0 0 0 0 0 ] -- | The instance for generating arbitrary Diskstats instance Arbitrary Diskstats where arbitrary = Diskstats <$> genNonNegative <*> genNonNegative <*> genName <*> genNonNegative <*> genNonNegative <*> genNonNegative <*> genNonNegative <*> genNonNegative <*> genNonNegative <*> genNonNegative <*> genNonNegative <*> genNonNegative <*> genNonNegative <*> genNonNegative -- | Serialize a list of Diskstats in a parsable way serializeDiskstatsList :: [Diskstats] -> String serializeDiskstatsList = concatMap serializeDiskstats -- | Serialize a Diskstats in a parsable way serializeDiskstats :: Diskstats -> String serializeDiskstats ds = printf "\t%d\t%d %s %d %d %d %d %d %d %d %d %d %d %d\n" (dsMajor ds) (dsMinor ds) (dsName ds) (dsReadsNum ds) (dsMergedReads ds) (dsSecRead ds) (dsTimeRead ds) (dsWrites ds) (dsMergedWrites ds) (dsSecWritten ds) (dsTimeWrite ds) (dsIos ds) (dsTimeIO ds) (dsWIOmillis ds) -- | Test whether an arbitrary Diskstats is parsed correctly prop_diskstats :: [Diskstats] -> Property prop_diskstats dsList = case A.parseOnly diskstatsParser $ pack (serializeDiskstatsList dsList) of Left msg -> failTest $ "Parsing failed: " ++ msg Right obtained -> dsList ==? obtained testSuite "Block/Diskstats/Parser" [ 'case_diskstats, 'prop_diskstats ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Storage/Drbd/000075500000000000000000000000001476477700300216025ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Storage/Drbd/Parser.hs000064400000000000000000000470621476477700300234030ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittests for the DRBD Parser -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Storage.Drbd.Parser (testBlock_Drbd_Parser) where import Test.QuickCheck as QuickCheck hiding (Result) import Test.HUnit import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import qualified Data.Attoparsec.Text as A import Data.List (intercalate) import Data.Text (pack) import Ganeti.Storage.Drbd.Parser (drbdStatusParser, commaIntParser) import Ganeti.Storage.Drbd.Types {-# ANN module "HLint: ignore Use camelCase" #-} -- | Test a DRBD 8.0 file with an empty line inside. case_drbd80_emptyline :: Assertion case_drbd80_emptyline = testParser (drbdStatusParser []) "proc_drbd80-emptyline.txt" $ DRBDStatus ( VersionInfo (Just "8.0.12") (Just "86") (Just "86") Nothing (Just "5c9f89594553e32adb87d9638dce591782f947e3") (Just "root@node1.example.com, 2009-05-22 12:47:52") ) [ DeviceInfo 0 Connected (LocalRemote Primary Secondary) (LocalRemote UpToDate UpToDate) 'C' "r---" (PerfIndicators 78728316 0 77675644 1277039 254 270 0 0 0 0 Nothing Nothing Nothing) Nothing (Just $ AdditionalInfo 0 61 65657 135 0 0 135) (Just $ AdditionalInfo 0 257 11378843 254 0 0 254) Nothing, UnconfiguredDevice 1, UnconfiguredDevice 2, UnconfiguredDevice 5, UnconfiguredDevice 6 ] -- | Test a DRBD 8.0 file with an empty version. case_drbd80_emptyversion :: Assertion case_drbd80_emptyversion = testParser (drbdStatusParser []) "proc_drbd80-emptyversion.txt" $ DRBDStatus ( VersionInfo Nothing Nothing Nothing Nothing (Just "5c9f89594553e32adb87d9638dce591782f947e3") (Just "root@node1.example.com, 2009-05-22 12:47:52") ) [ DeviceInfo 0 Connected (LocalRemote Primary Secondary) (LocalRemote UpToDate UpToDate) 'C' "r---" (PerfIndicators 78728316 0 77675644 1277039 254 270 0 0 0 0 Nothing Nothing Nothing) Nothing (Just $ AdditionalInfo 0 61 65657 135 0 0 135) (Just $ AdditionalInfo 0 257 11378843 254 0 0 254) Nothing, UnconfiguredDevice 1, UnconfiguredDevice 2, UnconfiguredDevice 5, UnconfiguredDevice 6 ] -- | Test a DRBD 8.4 file with an ongoing synchronization. case_drbd84_sync :: Assertion case_drbd84_sync = testParser (drbdStatusParser []) "proc_drbd84_sync.txt" $ DRBDStatus ( VersionInfo (Just "8.4.2") (Just "1") (Just "86-101") Nothing (Just "7ad5f850d711223713d6dcadc3dd48860321070c") (Just "root@example.com, 2013-04-10 07:45:25") ) [ DeviceInfo 0 StandAlone (LocalRemote Primary Unknown) (LocalRemote UpToDate DUnknown) ' ' "r-----" (PerfIndicators 0 0 33318 730 15 0 0 0 0 0 (Just 1) (Just 'd') (Just 1048320)) Nothing Nothing Nothing Nothing, UnconfiguredDevice 3, DeviceInfo 5 SyncSource (LocalRemote Secondary Secondary) (LocalRemote UpToDate Inconsistent) 'C' "r---n-" (PerfIndicators 716992 0 0 719432 0 43 0 33 18 0 (Just 1) (Just 'f') (Just 335744)) (Just $ SyncStatus 68.5 335744 1048576 KiloByte (Time 0 0 5) 64800 Nothing KiloByte Second) Nothing Nothing Nothing ] -- | Test a DRBD 8.4 file. case_drbd84 :: Assertion case_drbd84 = testParser (drbdStatusParser []) "proc_drbd84.txt" $ DRBDStatus ( VersionInfo (Just "8.4.2") (Just "1") (Just "86-101") Nothing (Just "7ad5f850d711223713d6dcadc3dd48860321070c") (Just "root@example.com, 2013-04-10 07:45:25") ) [ DeviceInfo 0 Connected (LocalRemote Primary Secondary) (LocalRemote UpToDate UpToDate) 'C' "r-----" (PerfIndicators 1048576 0 0 1048776 0 64 0 0 0 0 (Just 1) (Just 'f') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 1 Connected (LocalRemote Secondary Primary) (LocalRemote UpToDate UpToDate) 'C' "r-----" (PerfIndicators 0 1048576 1048576 0 0 64 0 0 0 0 (Just 1) (Just 'f') (Just 0)) Nothing Nothing Nothing Nothing, UnconfiguredDevice 2, DeviceInfo 4 WFConnection (LocalRemote Primary Unknown) (LocalRemote UpToDate DUnknown) 'C' "r-----" (PerfIndicators 0 0 0 200 0 0 0 0 0 0 (Just 1) (Just 'f') (Just 1048320)) Nothing Nothing Nothing Nothing, DeviceInfo 6 Connected (LocalRemote Secondary Primary) (LocalRemote Diskless UpToDate) 'C' "r-----" (PerfIndicators 0 0 0 0 0 0 0 0 0 0 (Just 1) (Just 'b') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 8 StandAlone (LocalRemote Secondary Unknown) (LocalRemote UpToDate DUnknown) ' ' "r-----" (PerfIndicators 0 0 0 200 0 0 0 0 0 0 (Just 1) (Just 'f') (Just 1048320)) Nothing Nothing Nothing Nothing ] -- | Test a DRBD 8.4 file with the first resource empty. case_drbd84_emptyfirst :: Assertion case_drbd84_emptyfirst = testParser (drbdStatusParser []) "proc_drbd84_emptyfirst.txt" $ DRBDStatus ( VersionInfo (Just "8.4.2") (Just "1") (Just "86-101") Nothing (Just "7ad5f850d711223713d6dcadc3dd48860321070c") (Just "root@example.com, 2013-04-10 07:45:25") ) [ DeviceInfo 1 Connected (LocalRemote Secondary Primary) (LocalRemote UpToDate UpToDate) 'C' "r-----" (PerfIndicators 0 1048576 1048576 0 0 64 0 0 0 0 (Just 1) (Just 'f') (Just 0)) Nothing Nothing Nothing Nothing, UnconfiguredDevice 2, DeviceInfo 4 WFConnection (LocalRemote Primary Unknown) (LocalRemote UpToDate DUnknown) 'C' "r-----" (PerfIndicators 0 0 0 200 0 0 0 0 0 0 (Just 1) (Just 'f') (Just 1048320)) Nothing Nothing Nothing Nothing, DeviceInfo 6 Connected (LocalRemote Secondary Primary) (LocalRemote Diskless UpToDate) 'C' "r-----" (PerfIndicators 0 0 0 0 0 0 0 0 0 0 (Just 1) (Just 'b') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 8 StandAlone (LocalRemote Secondary Unknown) (LocalRemote UpToDate DUnknown) ' ' "r-----" (PerfIndicators 0 0 0 200 0 0 0 0 0 0 (Just 1) (Just 'f') (Just 1048320)) Nothing Nothing Nothing Nothing ] -- | Test a DRBD 8.3 file with a NULL caracter inside. case_drbd83_sync_krnl2_6_39 :: Assertion case_drbd83_sync_krnl2_6_39 = testParser (drbdStatusParser []) "proc_drbd83_sync_krnl2.6.39.txt" $ DRBDStatus ( VersionInfo (Just "8.3.1") (Just "88") (Just "86-89") Nothing (Just "fd40f4a8f9104941537d1afc8521e584a6d3003c") (Just "phil@fat-tyre, 2009-03-27 12:19:49") ) [ DeviceInfo 0 Connected (LocalRemote Primary Secondary) (LocalRemote UpToDate UpToDate) 'C' "r----" (PerfIndicators 140978 0 9906 131533 27 8 0 0 0 0 (Just 1) (Just 'b') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 1 Connected (LocalRemote Secondary Primary) (LocalRemote UpToDate UpToDate) 'C' "r---" (PerfIndicators 0 140980 140980 0 0 8 0 0 0 0 (Just 1) (Just 'f') (Just 0)) Nothing Nothing Nothing Nothing, UnconfiguredDevice 2, DeviceInfo 3 SyncSource (LocalRemote Primary Secondary) (LocalRemote UpToDate Inconsistent) 'A' "r-----" (PerfIndicators 373888 0 0 374088 0 22 7 27 7 0 (Just 1) (Just 'f') (Just 15358208)) (Just $ SyncStatus 2.4 14996 15360 MegaByte (Time 0 4 8) 61736 Nothing KiloByte Second) Nothing Nothing Nothing, DeviceInfo 4 WFConnection (LocalRemote Primary Unknown) (LocalRemote UpToDate DUnknown) 'C' "r----" (PerfIndicators 140978 0 9906 131534 27 8 0 0 0 0 (Just 1) (Just 'b') (Just 0)) Nothing Nothing Nothing Nothing ] -- | Test a DRBD 8.3 file with an ongoing synchronization. case_drbd83_sync :: Assertion case_drbd83_sync = testParser (drbdStatusParser []) "proc_drbd83_sync.txt" $ DRBDStatus ( VersionInfo (Just "8.3.1") (Just "88") (Just "86-89") Nothing (Just "fd40f4a8f9104941537d1afc8521e584a6d3003c") (Just "phil@fat-tyre, 2009-03-27 12:19:49") ) [ DeviceInfo 0 Connected (LocalRemote Primary Secondary) (LocalRemote UpToDate UpToDate) 'C' "r----" (PerfIndicators 140978 0 9906 131533 27 8 0 0 0 0 (Just 1) (Just 'b') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 1 Connected (LocalRemote Secondary Primary) (LocalRemote UpToDate UpToDate) 'C' "r---" (PerfIndicators 0 140980 140980 0 0 8 0 0 0 0 (Just 1) (Just 'f') (Just 0)) Nothing Nothing Nothing Nothing, UnconfiguredDevice 2, DeviceInfo 3 SyncTarget (LocalRemote Primary Secondary) (LocalRemote Inconsistent UpToDate) 'C' "r----" (PerfIndicators 0 178176 178176 0 104 42 0 0 0 0 (Just 1) (Just 'b') (Just 346112)) (Just $ SyncStatus 34.9 346112 524288 MegaByte (Time 0 0 5) 59392 Nothing KiloByte Second) Nothing Nothing Nothing, DeviceInfo 4 WFConnection (LocalRemote Primary Unknown) (LocalRemote UpToDate DUnknown) 'C' "r----" (PerfIndicators 140978 0 9906 131534 27 8 0 0 0 0 (Just 1) (Just 'b') (Just 0)) Nothing Nothing Nothing Nothing ] -- | Test a DRBD 8.3 file not from git sources, with an ongoing synchronization -- and the "want" field case_drbd83_sync_want :: Assertion case_drbd83_sync_want = testParser (drbdStatusParser []) "proc_drbd83_sync_want.txt" $ DRBDStatus ( VersionInfo (Just "8.3.11") (Just "88") (Just "86-96") (Just "2D876214BAAD53B31ADC1D6") Nothing Nothing ) [ DeviceInfo 0 SyncTarget (LocalRemote Secondary Primary) (LocalRemote Inconsistent UpToDate) 'C' "r-----" (PerfIndicators 0 460288 460160 0 0 28 2 4 1 0 (Just 1) (Just 'f') (Just 588416)) (Just $ SyncStatus 44.4 588416 1048576 KiloByte (Time 0 0 8) 65736 (Just 61440) KiloByte Second) Nothing Nothing Nothing, UnconfiguredDevice 1, UnconfiguredDevice 2, UnconfiguredDevice 3 ] -- | Test a DRBD 8.3 file. case_drbd83 :: Assertion case_drbd83 = testParser (drbdStatusParser []) "proc_drbd83.txt" $ DRBDStatus ( VersionInfo (Just "8.3.1") (Just "88") (Just "86-89") Nothing (Just "fd40f4a8f9104941537d1afc8521e584a6d3003c") (Just "phil@fat-tyre, 2009-03-27 12:19:49") ) [ DeviceInfo 0 Connected (LocalRemote Primary Secondary) (LocalRemote UpToDate UpToDate) 'C' "r----" (PerfIndicators 140978 0 9906 131533 27 8 0 0 0 0 (Just 1) (Just 'b') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 1 Connected (LocalRemote Secondary Primary) (LocalRemote UpToDate UpToDate) 'C' "r---" (PerfIndicators 0 140980 140980 0 0 8 0 0 0 0 (Just 1) (Just 'f') (Just 0)) Nothing Nothing Nothing Nothing, UnconfiguredDevice 2, DeviceInfo 4 WFConnection (LocalRemote Primary Unknown) (LocalRemote UpToDate DUnknown) 'C' "r----" (PerfIndicators 140978 0 9906 131534 27 8 0 0 0 0 (Just 1) (Just 'b') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 5 Connected (LocalRemote Primary Secondary) (LocalRemote UpToDate Diskless) 'C' "r----" (PerfIndicators 140978 0 9906 131533 19 8 0 0 0 0 (Just 1) (Just 'b') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 6 Connected (LocalRemote Secondary Primary) (LocalRemote Diskless UpToDate) 'C' "r---" (PerfIndicators 0 140978 140978 0 0 8 0 0 0 0 (Just 1) (Just 'f') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 7 WFConnection (LocalRemote Secondary Unknown) (LocalRemote UpToDate DUnknown) 'C' "r---" (PerfIndicators 0 140978 140978 0 0 8 0 0 0 0 (Just 1) (Just 'f') (Just 0)) Nothing Nothing Nothing Nothing, DeviceInfo 8 StandAlone (LocalRemote Secondary Unknown) (LocalRemote UpToDate DUnknown) ' ' "r---" (PerfIndicators 0 140978 140978 0 0 8 0 0 0 0 (Just 1) (Just 'f') (Just 0)) Nothing Nothing Nothing Nothing ] -- | Test a DRBD 8.0 file with a missing device. case_drbd8 :: Assertion case_drbd8 = testParser (drbdStatusParser []) "proc_drbd8.txt" $ DRBDStatus ( VersionInfo (Just "8.0.12") (Just "86") (Just "86") Nothing (Just "5c9f89594553e32adb87d9638dce591782f947e3") (Just "XXX") ) [ DeviceInfo 0 Connected (LocalRemote Primary Secondary) (LocalRemote UpToDate UpToDate) 'C' "r---" (PerfIndicators 4375577 0 4446279 674 1067 69 0 0 0 0 Nothing Nothing Nothing) Nothing (Just $ AdditionalInfo 0 61 0 0 0 0 0) (Just $ AdditionalInfo 0 257 793749 1067 0 0 1067) Nothing, DeviceInfo 1 Connected (LocalRemote Secondary Primary) (LocalRemote UpToDate UpToDate) 'C' "r---" (PerfIndicators 738320 0 738320 554400 67 0 0 0 0 0 Nothing Nothing Nothing) Nothing (Just $ AdditionalInfo 0 61 0 0 0 0 0) (Just $ AdditionalInfo 0 257 92464 67 0 0 67) Nothing, UnconfiguredDevice 2, DeviceInfo 4 WFConnection (LocalRemote Primary Unknown) (LocalRemote UpToDate DUnknown) 'C' "r---" (PerfIndicators 738320 0 738320 554400 67 0 0 0 0 0 Nothing Nothing Nothing) Nothing (Just $ AdditionalInfo 0 61 0 0 0 0 0) (Just $ AdditionalInfo 0 257 92464 67 0 0 67) Nothing, DeviceInfo 5 Connected (LocalRemote Primary Secondary) (LocalRemote UpToDate Diskless) 'C' "r---" (PerfIndicators 4375581 0 4446283 674 1069 69 0 0 0 0 Nothing Nothing Nothing) Nothing (Just $ AdditionalInfo 0 61 0 0 0 0 0) (Just $ AdditionalInfo 0 257 793750 1069 0 0 1069) Nothing, DeviceInfo 6 Connected (LocalRemote Secondary Primary) (LocalRemote Diskless UpToDate) 'C' "r---" (PerfIndicators 0 4375581 5186925 327 75 214 0 0 0 0 Nothing Nothing Nothing) Nothing Nothing Nothing Nothing, DeviceInfo 7 WFConnection (LocalRemote Secondary Unknown) (LocalRemote UpToDate DUnknown) 'C' "r---" (PerfIndicators 0 0 0 0 0 0 0 0 0 0 Nothing Nothing Nothing) Nothing (Just $ AdditionalInfo 0 61 0 0 0 0 0) (Just $ AdditionalInfo 0 257 0 0 0 0 0) Nothing, DeviceInfo 8 StandAlone (LocalRemote Secondary Unknown) (LocalRemote UpToDate DUnknown) ' ' "r---" (PerfIndicators 0 0 0 0 0 0 0 0 0 0 Nothing Nothing Nothing) Nothing (Just $ AdditionalInfo 0 61 0 0 0 0 0) (Just $ AdditionalInfo 0 257 0 0 0 0 0) Nothing ] -- | Function for splitting a list in chunks of a given size. -- FIXME: an equivalent function exists in Data.List.Split, but it seems -- pointless to add this package as a dependence just for this single -- use. In case it is ever added, just remove this function definition -- and use the one from the package. splitEvery :: Int -> [e] -> [[e]] splitEvery i l = map (take i) (splitter l (:) []) where splitter [] _ n = n splitter li c n = li `c` splitter (drop i li) c n -- | Function for testing whether a single comma-separated integer is -- parsed correctly. testCommaInt :: String -> Int -> Assertion testCommaInt numString expectedResult = case A.parseOnly commaIntParser $ pack numString of Left msg -> assertFailure $ "Parsing failed: " ++ msg Right obtained -> assertEqual numString expectedResult obtained -- | Generate a property test for CommaInt numbers in a given range. gen_prop_CommaInt :: Int -> Int -> Property gen_prop_CommaInt minVal maxVal = forAll (choose (minVal, maxVal)) $ \i -> case A.parseOnly commaIntParser $ pack (generateCommaInt i) of Left msg -> failTest $ "Parsing failed: " ++ msg Right obtained -> i ==? obtained where generateCommaInt x = ((reverse . intercalate ",") . splitEvery 3) . reverse $ show x -- | Test if <4 digit integers are recognized correctly. prop_commaInt_noCommas :: Property prop_commaInt_noCommas = gen_prop_CommaInt 0 999 -- | Test if integers with 1 comma are recognized correctly. prop_commaInt_1Comma :: Property prop_commaInt_1Comma = gen_prop_CommaInt 1000 999999 -- | Test if integers with multiple commas are recognized correctly. prop_commaInt_multipleCommas :: Property prop_commaInt_multipleCommas = gen_prop_CommaInt 1000000 (maxBound :: Int) -- | Test whether the parser is actually able to behave as intended with -- numbers without commas. That is, if a number with more than 3 digits -- is parsed, only up to the first 3 digits are considered (because they -- are a valid commaInt), and the rest is ignored. -- e.g.: parse "1234" = 123 prop_commaInt_max3WithoutComma :: Property prop_commaInt_max3WithoutComma = forAll (choose (0, maxBound :: Int)) $ \i -> case A.parseOnly commaIntParser $ pack (show i) of Left msg -> failTest $ "Parsing failed: " ++ msg Right obtained -> obtained < 1000 .&&. getFirst3Digits i ==? obtained where getFirst3Digits x = if x >= 1000 then getFirst3Digits $ x `div` 10 else x -- | Test if non-triplets are handled correctly (they are assumed NOT being part -- of the number). case_commaInt_non_triplet :: Assertion case_commaInt_non_triplet = testCommaInt "61,736,12" 61736 testSuite "Block/Drbd/Parser" [ 'case_drbd80_emptyline, 'case_drbd80_emptyversion, 'case_drbd84_sync, 'case_drbd84, 'case_drbd84_emptyfirst, 'case_drbd83_sync_krnl2_6_39, 'case_drbd83_sync, 'case_drbd83_sync_want, 'case_drbd83, 'case_drbd8, 'case_commaInt_non_triplet, 'prop_commaInt_noCommas, 'prop_commaInt_1Comma, 'prop_commaInt_multipleCommas, 'prop_commaInt_max3WithoutComma ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Storage/Drbd/Types.hs000064400000000000000000000133721476477700300232500ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for the types representing DRBD status -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Storage.Drbd.Types (testBlock_Drbd_Types) where import Test.QuickCheck import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Text.JSON import Text.Printf import Ganeti.JSON import Ganeti.Storage.Drbd.Types {-# ANN module "HLint: ignore Use camelCase" #-} {-# ANN module "HLint: ignore Use string literal" #-} -- * Arbitrary instances $(genArbitrary ''ConnState) $(genArbitrary ''Role) $(genArbitrary ''DiskState) $(genArbitrary ''SizeUnit) $(genArbitrary ''TimeUnit) -- | Natural numbers generator. natural :: Gen Int natural = choose (0, maxBound :: Int) -- | Generator of percentages. percent :: Gen Double percent = choose (0 :: Double, 100 :: Double) -- | Generator of write order flags. wOrderFlag :: Gen Char wOrderFlag = elements ['b', 'f', 'd', 'n'] -- | Property for testing the JSON serialization of a DeviceInfo. prop_DeviceInfo :: Property prop_DeviceInfo = property $ do minor <- natural state <- arbitrary locRole <- arbitrary remRole <- arbitrary locState <- arbitrary remState <- arbitrary alg <- choose ('A','C') ns <- natural nr <- natural dw <- natural dr <- natural al <- natural bm <- natural lc <- natural pe <- natural ua <- natural ap <- natural ep <- genMaybe natural wo <- genMaybe wOrderFlag oos <- genMaybe natural inst <- genMaybe arbitrary let obtained = showJSON $ DeviceInfo minor state (LocalRemote locRole remRole) (LocalRemote locState remState) alg "r----" perfInd Nothing Nothing Nothing inst perfInd = PerfIndicators ns nr dw dr al bm lc pe ua ap ep wo oos expected = makeObj [ ("minor", showJSON minor) , ("connectionState", showJSON state) , ("localRole", showJSON locRole) , ("remoteRole", showJSON remRole) , ("localState", showJSON locState) , ("remoteState", showJSON remState) , ("replicationProtocol", showJSON alg) , ("ioFlags", showJSON "r----") , ("perfIndicators", showJSON perfInd) , ("instance", maybe JSNull showJSON inst) ] return $ obtained ==? expected -- | Property for testing the JSON serialization of a PerfIndicators. prop_PerfIndicators :: Property prop_PerfIndicators = property $ do ns <- natural nr <- natural dw <- natural dr <- natural al <- natural bm <- natural lc <- natural pe <- natural ua <- natural ap <- natural ep <- genMaybe natural wo <- genMaybe wOrderFlag oos <- genMaybe natural let expected = showJSON $ PerfIndicators ns nr dw dr al bm lc pe ua ap ep wo oos obtained = optFieldsToObj [ Just ("networkSend", showJSON ns) , Just ("networkReceive", showJSON nr) , Just ("diskWrite", showJSON dw) , Just ("diskRead", showJSON dr) , Just ("activityLog", showJSON al) , Just ("bitMap", showJSON bm) , Just ("localCount", showJSON lc) , Just ("pending", showJSON pe) , Just ("unacknowledged", showJSON ua) , Just ("applicationPending", showJSON ap) , optionalJSField "epochs" ep , optionalJSField "writeOrder" wo , optionalJSField "outOfSync" oos ] return $ obtained ==? expected -- | Function for testing the JSON serialization of a SyncStatus. prop_SyncStatus :: Property prop_SyncStatus = property $ do perc <- percent numer <- natural denom <- natural sizeU1 <- arbitrary h <- choose (0, 23) m <- choose (0, 59) s <- choose (0, 59) sp <- natural wa <- genMaybe natural sizeU2 <- arbitrary timeU <- arbitrary let obtained = showJSON $ SyncStatus perc numer denom sizeU1 (Time h m s) sp wa sizeU2 timeU expected = optFieldsToObj [ Just ("percentage", showJSON perc) , Just ("progress", showJSON $ show numer ++ "/" ++ show denom) , Just ("progressUnit", showJSON sizeU1) , Just ("timeToFinish", showJSON (printf "%02d:%02d:%02d" h m s :: String)) , Just ("speed", showJSON sp) , optionalJSField "want" wa , Just ("speedUnit", showJSON $ show sizeU2 ++ "/" ++ show timeU) ] return $ obtained ==? expected testSuite "Block/Drbd/Types" [ 'prop_DeviceInfo , 'prop_PerfIndicators , 'prop_SyncStatus ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Storage/Lvm/000075500000000000000000000000001476477700300214655ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Storage/Lvm/LVParser.hs000064400000000000000000000111221476477700300235140ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for the LV Parser -} {- Copyright (C) 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Storage.Lvm.LVParser (testStorage_Lvm_LVParser) where import Test.QuickCheck as QuickCheck hiding (Result) import Test.HUnit import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Data.List (intercalate) import Ganeti.Storage.Lvm.LVParser import Ganeti.Storage.Lvm.Types {-# ANN module "HLint: ignore Use camelCase" #-} -- | Test parsing a LV @lvs@ output. case_lvs_lv :: Assertion case_lvs_lv = testParser lvParser "lvs_lv.txt" [ LVInfo "nhasjL-cnZi-uqLS-WRLj-tkXI-nvCB-n0o2lj" "df9ff3f6-a833-48ff-8bd5-bff2eaeab759.disk0_data" "-wi-ao" (negate 1) (negate 1) 253 0 1073741824 1 "originstname+instance1.example.com" "" "uZgXit-eiRr-vRqe-xpEo-e9nU-mTuR-9nfVIU" "xenvg" "linear" 0 0 1073741824 "" "/dev/sda5:0-15" "/dev/sda5(0)" Nothing , LVInfo "5fW5mE-SBSs-GSU0-KZDg-hnwb-sZOC-zZt736" "df9ff3f6-a833-48ff-8bd5-bff2eaeab759.disk0_meta" "-wi-ao" (negate 1) (negate 1) 253 1 134217728 1 "originstname+instance1.example.com" "" "uZgXit-eiRr-vRqe-xpEo-e9nU-mTuR-9nfVIU" "xenvg" "linear" 0 0 134217728 "" "/dev/sda5:16-17" "/dev/sda5(16)" Nothing ] -- | Serialize a LVInfo in the same format that is output by @lvs@. -- The "instance" field is not serialized because it's not provided by @lvs@ -- so it is not part of this test. serializeLVInfo :: LVInfo -> String serializeLVInfo l = intercalate ";" [ lviUuid l , lviName l , lviAttr l , show $ lviMajor l , show $ lviMinor l , show $ lviKernelMajor l , show $ lviKernelMinor l , show (lviSize l) ++ "B" , show $ lviSegCount l , lviTags l , lviModules l , lviVgUuid l , lviVgName l , lviSegtype l , show (lviSegStart l) ++ "B" , show $ lviSegStartPe l , show (lviSegSize l) ++ "B" , lviSegTags l , lviSegPeRanges l , lviDevices l ] ++ "\n" -- | Serialize a list of LVInfo in the same format that is output by @lvs@. serializeLVInfos :: [LVInfo] -> String serializeLVInfos = concatMap serializeLVInfo -- | Arbitrary instance for LVInfo. -- The instance is always Nothing because it is not part of the parsed data: -- it is added afterwards from a different source. instance Arbitrary LVInfo where arbitrary = LVInfo <$> genUUID -- uuid <*> genName -- name <*> genName -- attr <*> arbitrary -- major <*> arbitrary -- minor <*> arbitrary -- kernel_major <*> arbitrary -- kernel_minor <*> genNonNegative -- size <*> arbitrary -- seg_cont <*> genName -- tags <*> genName -- modules <*> genUUID -- vg_uuid <*> genName -- vg_name <*> genName -- segtype <*> genNonNegative -- seg_start <*> arbitrary -- seg_start_pe <*> genNonNegative -- seg_size <*> genName -- seg_tags <*> genName -- seg_pe_ranges <*> genName -- devices <*> return Nothing -- instance -- | Test if a randomly generated LV lvs output is properly parsed. prop_parse_lvs_lv :: [LVInfo] -> Property prop_parse_lvs_lv expected = genPropParser lvParser (serializeLVInfos expected) expected testSuite "Storage/Lvm/LVParser" [ 'case_lvs_lv, 'prop_parse_lvs_lv ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/THH.hs000064400000000000000000000132421476477700300203040ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, FunctionalDependencies #-} {-# OPTIONS -fno-warn-unused-binds #-} {-| Unittests for our template-haskell generated code. -} {- Copyright (C) 2012 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.THH ( testTHH ) where import Test.QuickCheck import Text.JSON import Ganeti.THH import Ganeti.PartialParams import Test.Ganeti.PartialParams import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon {-# ANN module "HLint: ignore Use camelCase" #-} -- * Custom types -- | Type used to test optional field implementation. Equivalent to -- @data TestObj = TestObj { tobjA :: Maybe Int, tobjB :: Maybe Int -- }@. $(buildObject "TestObj" "tobj" [ optionalField $ simpleField "a" [t| Int |] , optionalNullSerField $ simpleField "b" [t| Int |] ]) -- | Arbitrary instance for 'TestObj'. $(genArbitrary ''TestObj) -- | Tests that serialising an (arbitrary) 'TestObj' instance is -- correct: fully optional fields are represented in the resulting -- dictionary only when non-null, optional-but-required fields are -- always represented (with either null or an actual value). prop_OptFields :: TestObj -> Property prop_OptFields to = let a_member = case tobjA to of Nothing -> [] Just x -> [("a", showJSON x)] b_member = [("b", case tobjB to of Nothing -> JSNull Just x -> showJSON x)] in showJSON to ==? makeObj (a_member ++ b_member) -- | Test serialization of TestObj. prop_TestObj_serialization :: TestObj -> Property prop_TestObj_serialization = testArraySerialisation -- | Test that all superfluous keys will fail to parse. prop_TestObj_deserialisationFail :: Property prop_TestObj_deserialisationFail = forAll ((arbitrary :: Gen [(String, Int)]) `suchThat` any (flip notElem ["a", "b"] . fst)) $ testDeserialisationFail (TestObj Nothing Nothing) . encJSDict -- | A unit-like data type. $(buildObject "UnitObj" "uobj" []) $(genArbitrary ''UnitObj) -- | Test serialization of UnitObj. prop_UnitObj_serialization :: UnitObj -> Property prop_UnitObj_serialization = testArraySerialisation -- | Test that all superfluous keys will fail to parse. prop_UnitObj_deserialisationFail :: Property prop_UnitObj_deserialisationFail = forAll ((arbitrary :: Gen [(String, Int)]) `suchThat` (not . null)) $ testDeserialisationFail UnitObj . encJSDict $(buildParam "Test" "tparam" [ simpleField "c" [t| Int |] , simpleField "d" [t| String |] ]) $(genArbitrary ''FilledTestParams) $(genArbitrary ''PartialTestParams) -- | Tests that filling partial parameters works as expected. prop_fillWithPartialParams :: Property prop_fillWithPartialParams = let partial = PartialTestParams (Just 4) Nothing filled = FilledTestParams 2 "42" expected = FilledTestParams 4 "42" in fillParams filled partial ==? expected -- | Tests that filling partial parameters satisfies the law. prop_fillPartialLaw1 :: FilledTestParams -> PartialTestParams -> Property prop_fillPartialLaw1 = testFillParamsLaw1 -- | Tests that filling partial parameters works as expected. prop_toParams :: Property prop_toParams = let filled = FilledTestParams 2 "42" expected = FilledTestParams 4 "42" in toPartial (FilledTestParams 2 "42") ==? PartialTestParams (Just 2) (Just "42") -- | Tests that filling partial parameters satisfies the law. prop_fillPartialLaw2 :: FilledTestParams -> FilledTestParams -> Property prop_fillPartialLaw2 = testToParamsLaw2 -- | Tests that filling partial parameters satisfies the law. prop_fillPartialLaw3 :: FilledTestParams -> Property prop_fillPartialLaw3 = testToFilledLaw3 -- | Tests that the monoid action laws are satisfied. prop_fillPartialMonoidLaw1 :: FilledTestParams -> Property prop_fillPartialMonoidLaw1 = testToFilledMonoidLaw1 -- | Tests that the monoid action laws are satisfied. prop_fillPartialMonoidLaw2 :: FilledTestParams -> PartialTestParams -> PartialTestParams -> Property prop_fillPartialMonoidLaw2 = testToFilledMonoidLaw2 testSuite "THH" [ 'prop_OptFields , 'prop_TestObj_serialization , 'prop_TestObj_deserialisationFail , 'prop_UnitObj_serialization , 'prop_UnitObj_deserialisationFail , 'prop_fillWithPartialParams , 'prop_fillPartialLaw1 , 'prop_toParams , 'prop_fillPartialLaw2 , 'prop_fillPartialLaw3 , 'prop_fillPartialMonoidLaw1 , 'prop_fillPartialMonoidLaw2 ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/THH/000075500000000000000000000000001476477700300177465ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/THH/Types.hs000064400000000000000000000045211476477700300214100ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for 'Ganeti.THH.Types'. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.THH.Types ( testTHH_Types ) where import Test.QuickCheck as QuickCheck hiding (Result) import qualified Text.JSON as J import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Ganeti.THH.Types {-# ANN module "HLint: ignore Use camelCase" #-} -- * Instances instance Arbitrary a => Arbitrary (OneTuple a) where arbitrary = fmap OneTuple arbitrary -- * Properties -- | Tests OneTuple serialisation. prop_OneTuple_serialisation :: OneTuple String -> Property prop_OneTuple_serialisation = testSerialisation -- | Tests OneTuple doesn't deserialize wrong input. prop_OneTuple_deserialisationFail :: Property prop_OneTuple_deserialisationFail = conjoin . map (testDeserialisationFail (OneTuple "")) $ [ J.JSArray [] , J.JSArray [J.showJSON "a", J.showJSON "b"] , J.JSArray [J.showJSON (1 :: Int)] ] -- * Test suite testSuite "THH_Types" [ 'prop_OneTuple_serialisation , 'prop_OneTuple_deserialisationFail ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/TestCommon.hs000064400000000000000000000521401476477700300217510ustar00rootroot00000000000000{-# LANGUAGE CPP, FlexibleInstances #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Common helper functions and instances for all Ganeti tests. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.TestCommon ( maxMem , maxDsk , maxCpu , maxSpindles , maxVcpuRatio , maxSpindleRatio , maxNodes , maxOpCodes , (==?) , (/=?) , failTest , passTest , stableCover , pythonCmd , runPython , checkPythonResult , DNSChar(..) , genPrintableAsciiChar , genPrintableAsciiString , genPrintableAsciiStringNE , genPrintableByteString , genName , genFQDN , genUUID , genMaybe , genSublist , genMap , genTags , genFields , genUniquesList , SmallRatio(..) , genSetHelper , genSet , genListSet , genAndRestArguments , genIPv4Address , genIPv4Network , genIp6Addr , genIp6Net , genOpCodesTagName , genLuxiTagName , netmask2NumHosts , testSerialisation , testArraySerialisation , testDeserialisationFail , resultProp , readTestData , genSample , testParser , genPropParser , genNonNegative , relativeError , getTempFileName , listOfUniqueBy , counterexample , cover' ) where import Control.Exception (catchJust) import Control.Monad import Control.Monad.Fail (MonadFail, fail) import Data.Attoparsec.Text (Parser, parseOnly) import Data.List import qualified Data.Map as M import Data.Text (pack) import Data.Word import qualified Data.ByteString as BS import qualified Data.ByteString.UTF8 as UTF8 #if !MIN_VERSION_QuickCheck(2,10,0) import Data.Char (isPrint) #endif import qualified Data.Set as Set import System.Directory (getTemporaryDirectory, removeFile) import System.Environment (getEnv) import System.Exit (ExitCode(..)) import System.IO (hClose, openTempFile) import System.IO.Error (isDoesNotExistError) import System.Process (readProcessWithExitCode) import qualified Test.HUnit as HUnit import Test.QuickCheck import Test.QuickCheck.Monadic import qualified Text.JSON as J import Numeric import qualified Ganeti.BasicTypes as BasicTypes import Ganeti.JSON (ArrayObject(..)) import Ganeti.Objects (TagSet(..)) import Ganeti.Types import Ganeti.Utils.Monad (unfoldrM) -- * Compatibility instances instance MonadFail Gen where fail = error "No monadfail instance" instance MonadFail (Either String) where fail x = Left x -- * Arbitrary orphan instances instance Arbitrary TagSet where arbitrary = (TagSet . Set.fromList) <$> genTags -- * Constants -- | Maximum memory (1TiB, somewhat random value). maxMem :: Int maxMem = 1024 * 1024 -- | Maximum disk (8TiB, somewhat random value). maxDsk :: Int maxDsk = 1024 * 1024 * 8 -- | Max CPUs (1024, somewhat random value). maxCpu :: Int maxCpu = 1024 -- | Max spindles (1024, somewhat random value). maxSpindles :: Int maxSpindles = 1024 -- | Max vcpu ratio (random value). maxVcpuRatio :: Double maxVcpuRatio = 1024.0 -- | Max spindle ratio (random value). maxSpindleRatio :: Double maxSpindleRatio = 1024.0 -- | Max nodes, used just to limit arbitrary instances for smaller -- opcode definitions (e.g. list of nodes in OpTestDelay). maxNodes :: Int maxNodes = 32 -- | Max opcodes or jobs in a submit job and submit many jobs. maxOpCodes :: Int maxOpCodes = 16 -- * Helper functions -- | Checks for equality with proper annotation. The first argument is -- the computed value, the second one the expected value. (==?) :: (Show a, Eq a) => a -> a -> Property (==?) x y = counterexample ("Expected equality, but got mismatch\nexpected: " ++ show y ++ "\n but got: " ++ show x) (x == y) infix 3 ==? -- | Checks for inequality with proper annotation. The first argument -- is the computed value, the second one the expected (not equal) -- value. (/=?) :: (Show a, Eq a) => a -> a -> Property (/=?) x y = counterexample ("Expected inequality, but got equality: '" ++ show x ++ "'.") (x /= y) infix 3 /=? -- | Show a message and fail the test. failTest :: String -> Property failTest msg = counterexample msg False -- | A 'True' property. passTest :: Property passTest = property True -- | QuickCheck 2.12 swapped the order of the first two arguments, so provide a -- compatibility function here cover' :: Testable prop => Double -> Bool -> String -> prop -> Property #if MIN_VERSION_QuickCheck(2, 12, 0) cover' = cover #else cover' p x = cover x (round p) #endif -- | A stable version of QuickCheck's `cover`. In its current implementation, -- cover will not detect insufficient coverage if the actual coverage in the -- sample is 0. Work around this by lifting the probability to at least -- 10 percent. -- The underlying issue is tracked at -- https://github.com/nick8325/quickcheck/issues/26 stableCover :: Testable prop => Bool -> Double -> String -> prop -> Property stableCover p percent s prop = let newlabel = "(stabilized to at least 10%) " ++ s in forAll (frequency [(1, return True), (9, return False)]) $ \ basechance -> cover' (10 + (percent * 9 / 10)) (basechance || p) newlabel prop -- | Return the python binary to use. If the PYTHON environment -- variable is defined, use its value, otherwise use just \"python3\". pythonCmd :: IO String pythonCmd = catchJust (guard . isDoesNotExistError) (getEnv "PYTHON") (const (return "python3")) -- | Run Python with an expression, returning the exit code, standard -- output and error. runPython :: String -> String -> IO (ExitCode, String, String) runPython expr stdin = do py_binary <- pythonCmd readProcessWithExitCode py_binary ["-c", expr] stdin -- | Check python exit code, and fail via HUnit assertions if -- non-zero. Otherwise, return the standard output. checkPythonResult :: (ExitCode, String, String) -> IO String checkPythonResult (py_code, py_stdout, py_stderr) = do HUnit.assertEqual ("python exited with error: " ++ py_stderr) ExitSuccess py_code return py_stdout -- * Arbitrary instances -- | Defines a DNS name. newtype DNSChar = DNSChar { dnsGetChar::Char } instance Arbitrary DNSChar where arbitrary = liftM DNSChar $ elements (['a'..'z'] ++ ['0'..'9'] ++ "_-") instance Show DNSChar where show = show . dnsGetChar -- * Generators -- | Generates printable ASCII characters (from ' ' to '~'). genPrintableAsciiChar :: Gen Char genPrintableAsciiChar = choose ('\x20', '\x7e') -- | Generates a short string (0 <= n <= 40 chars) from printable ASCII. genPrintableAsciiString :: Gen String genPrintableAsciiString = do n <- choose (0, 40) vectorOf n genPrintableAsciiChar -- | Generates a short string (1 <= n <= 40 chars) from printable ASCII. genPrintableAsciiStringNE :: Gen NonEmptyString genPrintableAsciiStringNE = do n <- choose (1, 40) vectorOf n genPrintableAsciiChar >>= mkNonEmpty -- | Generates a printable Unicode ByteString genPrintableByteString :: Gen BS.ByteString #if MIN_VERSION_QuickCheck(2, 10, 0) genPrintableByteString = fmap (UTF8.fromString . getPrintableString) arbitrary #else genPrintableByteString = fmap UTF8.fromString $ listOf (arbitrary `suchThat` isPrint) #endif -- | Generates a single name component. genName :: Gen String genName = do n <- choose (1, 16) dn <- vector n return (map dnsGetChar dn) -- | Generates an entire FQDN. genFQDN :: Gen String genFQDN = do ncomps <- choose (1, 4) names <- vectorOf ncomps genName return $ intercalate "." names -- | Generates a UUID-like string. -- -- Only to be used for QuickCheck testing. For obtaining actual UUIDs use -- the newUUID function in Ganeti.Utils genUUID :: Gen String genUUID = do c1 <- vector 6 c2 <- vector 4 c3 <- vector 4 c4 <- vector 4 c5 <- vector 4 c6 <- vector 4 c7 <- vector 6 return $ map dnsGetChar c1 ++ "-" ++ map dnsGetChar c2 ++ "-" ++ map dnsGetChar c3 ++ "-" ++ map dnsGetChar c4 ++ "-" ++ map dnsGetChar c5 ++ "-" ++ map dnsGetChar c6 ++ "-" ++ map dnsGetChar c7 -- | Combinator that generates a 'Maybe' using a sub-combinator. genMaybe :: Gen a -> Gen (Maybe a) genMaybe subgen = frequency [ (1, pure Nothing), (3, Just <$> subgen) ] -- | Generates a sublist of a given list, keeping the ordering. -- The generated elements are always a subset of the list. -- -- In order to better support corner cases, the size of the sublist is -- chosen to have the uniform distribution. genSublist :: [a] -> Gen [a] genSublist xs = choose (0, l) >>= g xs l where l = length xs g _ _ 0 = return [] g [] _ _ = return [] g ys n k | k == n = return ys g (y:ys) n k = frequency [ (k, liftM (y :) (g ys (n - 1) (k - 1))) , (n - k, g ys (n - 1) k) ] -- | Generates a map given generators for keys and values. genMap :: (Ord k, Ord v) => Gen k -> Gen v -> Gen (M.Map k v) genMap kg vg = M.fromList <$> listOf ((,) <$> kg <*> vg) -- | Defines a tag type. newtype TagChar = TagChar { tagGetChar :: Char } -- | All valid tag chars. This doesn't need to match _exactly_ -- Ganeti's own tag regex, just enough for it to be close. tagChar :: String tagChar = ['a'..'z'] ++ ['A'..'Z'] ++ ['0'..'9'] ++ ".+*/:@-" instance Arbitrary TagChar where arbitrary = liftM TagChar $ elements tagChar -- | Generates a tag genTag :: Gen [TagChar] genTag = do -- the correct value would be C.maxTagLen, but that's way too -- verbose in unittests, and at the moment I don't see any possible -- bugs with longer tags and the way we use tags in htools n <- choose (1, 10) vector n -- | Generates a list of tags (correctly upper bounded). genTags :: Gen [String] genTags = do -- the correct value would be C.maxTagsPerObj, but per the comment -- in genTag, we don't use tags enough in htools to warrant testing -- such big values n <- choose (0, 10::Int) tags <- mapM (const genTag) [1..n] return $ map (map tagGetChar) tags -- | Generates a fields list. This uses the same character set as a -- DNS name (just for simplicity). genFields :: Gen [String] genFields = do n <- choose (1, 32) vectorOf n genName -- | Generates a list of a given size with non-duplicate elements. genUniquesList :: (Eq a, Arbitrary a, Ord a) => Int -> Gen a -> Gen [a] genUniquesList cnt generator = do set <- foldM (\set _ -> do newelem <- generator `suchThat` (`Set.notMember` set) return (Set.insert newelem set)) Set.empty [1..cnt] return $ Set.toList set newtype SmallRatio = SmallRatio Double deriving Show instance Arbitrary SmallRatio where arbitrary = liftM SmallRatio $ choose (0, 1) -- | Helper for 'genSet', declared separately due to type constraints. genSetHelper :: (Ord a) => [a] -> Maybe Int -> Gen (Set.Set a) genSetHelper candidates size = do size' <- case size of Nothing -> choose (0, length candidates) Just s | s > length candidates -> error $ "Invalid size " ++ show s ++ ", maximum is " ++ show (length candidates) | otherwise -> return s foldM (\set _ -> do newelem <- elements candidates `suchThat` (`Set.notMember` set) return (Set.insert newelem set)) Set.empty [1..size'] -- | Generates a 'Set' of arbitrary elements. genSet :: (Ord a, Bounded a, Enum a) => Maybe Int -> Gen (Set.Set a) genSet = genSetHelper [minBound..maxBound] -- | Generates a 'Set' of arbitrary elements wrapped in a 'ListSet' genListSet :: (Ord a, Bounded a, Enum a) => Maybe Int -> Gen (BasicTypes.ListSet a) genListSet is = BasicTypes.ListSet <$> genSet is -- | Generate an arbitrary element of and AndRestArguments field. genAndRestArguments :: Gen (M.Map String J.JSValue) genAndRestArguments = do n <- choose (0::Int, 10) let oneParam _ = do name <- choose (15 ::Int, 25) >>= flip vectorOf (elements tagChar) intvalue <- arbitrary value <- oneof [ J.JSString . J.toJSString <$> genName , return $ J.showJSON (intvalue :: Int) ] return (name, value) M.fromList `liftM` mapM oneParam [1..n] -- | Generate an arbitrary IPv4 address in textual form. genIPv4 :: Gen String genIPv4 = do a <- choose (1::Int, 255) b <- choose (0::Int, 255) c <- choose (0::Int, 255) d <- choose (0::Int, 255) return . intercalate "." $ map show [a, b, c, d] genIPv4Address :: Gen IPv4Address genIPv4Address = mkIPv4Address =<< genIPv4 -- | Generate an arbitrary IPv4 network in textual form. genIPv4AddrRange :: Gen String genIPv4AddrRange = do pfxLen <- choose (8::Int, 30) -- Generate a number that fits in pfxLen bits addr <- choose(1::Int, 2^pfxLen-1) let hostLen = 32 - pfxLen -- ...and shift it left until it becomes a 32-bit number -- with the low hostLen bits unset net = addr * 2^hostLen netBytes = [(net `div` 2^x) `mod` 256 | x <- [24::Int, 16, 8, 0]] return $ intercalate "." (map show netBytes) ++ "/" ++ show pfxLen genIPv4Network :: Gen IPv4Network genIPv4Network = mkIPv4Network =<< genIPv4AddrRange -- | Helper function to compute the number of hosts in a network -- given the netmask. (For IPv4 only.) netmask2NumHosts :: Word8 -> Int netmask2NumHosts n = 2^(32-n) -- | Generates an arbitrary IPv6 network address in textual form. -- The generated address is not simpflified, e. g. an address like -- "2607:f0d0:1002:0051:0000:0000:0000:0004" does not become -- "2607:f0d0:1002:51::4" genIp6Addr :: Gen String genIp6Addr = do rawIp <- vectorOf 8 $ choose (0::Integer, 65535) return $ intercalate ":" (map (`showHex` "") rawIp) -- | Helper function to convert an integer in to an IPv6 address in textual -- form ip6AddressFromNumber :: Integer -> [Char] ip6AddressFromNumber ipInt = -- chunksOf splits a sequence in chunks of n elements length each -- chunksOf 4 "20010db80000" = ["2001", "0db8", "0000"] let chunksOf :: Int -> [a] -> [[a]] chunksOf _ [] = [] chunksOf n lst = take n lst : (chunksOf n $ drop n lst) -- Left-pad a sequence with a fixed element if it's shorter than n -- elements, or trim it to the first n elements otherwise -- e.g. lPadTrim 6 '0' "abcd" = "00abcd" lPadTrim :: Int -> a -> [a] -> [a] lPadTrim n p lst | length lst < n = lPadTrim n p $ p : lst | otherwise = take n lst rawIp = lPadTrim 32 '0' $ (`showHex` "") ipInt in intercalate ":" $ chunksOf 4 rawIp -- | Generates an arbitrary IPv6 network in textual form. genIp6Net :: Gen String genIp6Net = do netmask <- choose (8::Int, 126) raw_ip <- (* 2^(128-netmask)) <$> choose(1::Integer, 2^netmask-1) return $ ip6AddressFromNumber raw_ip ++ "/" ++ show netmask -- | Generates a valid, arbitrary tag name with respect to the given -- 'TagKind' for opcodes. genOpCodesTagName :: TagKind -> Gen (Maybe String) genOpCodesTagName TagKindCluster = return Nothing genOpCodesTagName _ = Just <$> genFQDN -- | Generates a valid, arbitrary tag name with respect to the given -- 'TagKind' for Luxi. genLuxiTagName :: TagKind -> Gen String genLuxiTagName TagKindCluster = return "" genLuxiTagName _ = genFQDN -- * Helper functions -- | Checks for serialisation idempotence. testSerialisation :: (Eq a, Show a, J.JSON a) => a -> Property testSerialisation a = case J.readJSON (J.showJSON a) of J.Error msg -> failTest $ "Failed to deserialise: " ++ msg J.Ok a' -> a ==? a' -- | Checks for array serialisation idempotence. testArraySerialisation :: (Eq a, Show a, ArrayObject a) => a -> Property testArraySerialisation a = case fromJSArray (toJSArray a) of J.Error msg -> failTest $ "Failed to deserialise: " ++ msg J.Ok a' -> a ==? a' -- | Checks if the deserializer doesn't accept forbidden values. -- The first argument is ignored, it just enforces the correct type. testDeserialisationFail :: (Eq a, Show a, J.JSON a) => a -> J.JSValue -> Property testDeserialisationFail a val = case liftM (`asTypeOf` a) $ J.readJSON val of J.Error _ -> passTest J.Ok x -> failTest $ "Parsed invalid value " ++ show val ++ " to: " ++ show x -- | Result to PropertyM IO. resultProp :: (Show a) => BasicTypes.GenericResult a b -> PropertyM IO b resultProp (BasicTypes.Bad err) = stop . failTest $ show err resultProp (BasicTypes.Ok val) = return val -- | Return the source directory of Ganeti. getSourceDir :: IO FilePath getSourceDir = catchJust (guard . isDoesNotExistError) (getEnv "TOP_SRCDIR") (const (return ".")) -- | Returns the path of a file in the test data directory, given its name. testDataFilename :: String -> String -> IO FilePath testDataFilename datadir name = do src <- getSourceDir return $ src ++ datadir ++ name -- | Returns the content of the specified haskell test data file. readTestData :: String -> IO String readTestData filename = do name <- testDataFilename "/test/data/" filename readFile name -- | Generate arbitrary values in the IO monad. This is a simple -- wrapper over 'sample''. genSample :: Gen a -> IO a genSample gen = do values <- sample' gen case values of [] -> error "sample' returned an empty list of values??" x:_ -> return x -- | Function for testing whether a file is parsed correctly. testParser :: (Show a, Eq a) => Parser a -> String -> a -> HUnit.Assertion testParser parser fileName expectedContent = do fileContent <- readTestData fileName case parseOnly parser $ pack fileContent of Left msg -> HUnit.assertFailure $ "Parsing failed: " ++ msg Right obtained -> HUnit.assertEqual fileName expectedContent obtained -- | Generate a property test for parsers. genPropParser :: (Show a, Eq a) => Parser a -> String -> a -> Property genPropParser parser s expected = case parseOnly parser $ pack s of Left msg -> failTest $ "Parsing failed: " ++ msg Right obtained -> expected ==? obtained -- | Generate an arbitrary non negative integer number genNonNegative :: Gen Int genNonNegative = fmap fromEnum (arbitrary::Gen (Test.QuickCheck.NonNegative Int)) -- | Computes the relative error of two 'Double' numbers. -- -- This is the \"relative error\" algorithm in -- http:\/\/randomascii.wordpress.com\/2012\/02\/25\/ -- comparing-floating-point-numbers-2012-edition (URL split due to too -- long line). relativeError :: Double -> Double -> Double relativeError d1 d2 = let delta = abs $ d1 - d2 a1 = abs d1 a2 = abs d2 greatest = max a1 a2 in if delta == 0 then 0 else delta / greatest -- | Helper to a get a temporary file name. getTempFileName :: String -> IO FilePath getTempFileName filename = do tempdir <- getTemporaryDirectory (fpath, handle) <- openTempFile tempdir filename _ <- hClose handle removeFile fpath return fpath -- | @listOfUniqueBy gen keyFun forbidden@: Generates a list of random length, -- where all generated elements will be unique by the keying function -- @keyFun@. They will also be distinct from all elements in @forbidden@ by -- the keying function. -- -- As for 'listOf', the maximum output length depends on the size parameter. -- -- Example: -- -- > listOfUniqueBy (arbitrary :: Gen String) (length) ["hey"] -- > -- Generates a list of strings of different length, but not of length 3. -- -- The passed @gen@ should not make key collisions too likely, since the -- implementation uses `suchThat`, looping until enough unique elements -- have been generated. If the @gen@ makes collisions likely, this function -- will consequently be slow, or not terminate if it is not possible to -- generate enough elements, like in: -- -- > listOfUniqueBy (arbitrary :: Gen Int) (`mod` 2) [] -- > -- May not terminate depending on the size parameter of the Gen, -- > -- since there are only 2 unique keys (0 and 1). listOfUniqueBy :: (Ord b) => Gen a -> (a -> b) -> [a] -> Gen [a] listOfUniqueBy gen keyFun forbidden = do let keysOf = Set.fromList . map keyFun k <- sized $ \n -> choose (0, n) flip unfoldrM (0, keysOf forbidden) $ \(i, usedKeys) -> if i == k then return Nothing else do x <- gen `suchThat` ((`Set.notMember` usedKeys) . keyFun) return $ Just (x, (i + 1, Set.insert (keyFun x) usedKeys)) ganeti-3.1.0~rc2/test/hs/Test/Ganeti/TestHTools.hs000064400000000000000000000127331476477700300217350ustar00rootroot00000000000000{-# OPTIONS_GHC -fno-warn-orphans #-} {-| Common functionality for htools-related unittests. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.TestHTools ( nullIPolicy , nullISpec , defGroup , defGroupList , defGroupAssoc , createInstance , makeSmallCluster , setInstanceSmallerThanNode ) where import qualified Data.Map as Map import Test.Ganeti.TestCommon import qualified Ganeti.Constants as C import qualified Ganeti.HTools.Container as Container import qualified Ganeti.HTools.Group as Group import qualified Ganeti.HTools.Instance as Instance import qualified Ganeti.HTools.Loader as Loader import qualified Ganeti.HTools.Node as Node import qualified Ganeti.HTools.Types as Types -- * Helpers -- | An ISpec with 0 resources. nullISpec :: Types.ISpec nullISpec = Types.ISpec { Types.iSpecMemorySize = 0 , Types.iSpecCpuCount = 0 , Types.iSpecDiskSize = 0 , Types.iSpecDiskCount = 0 , Types.iSpecNicCount = 0 , Types.iSpecSpindleUse = 0 } -- | Null iPolicy, and by null we mean very liberal. nullIPolicy :: Types.IPolicy nullIPolicy = Types.IPolicy { Types.iPolicyMinMaxISpecs = [Types.MinMaxISpecs { Types.minMaxISpecsMinSpec = nullISpec , Types.minMaxISpecsMaxSpec = Types.ISpec { Types.iSpecMemorySize = maxBound , Types.iSpecCpuCount = maxBound , Types.iSpecDiskSize = maxBound , Types.iSpecDiskCount = C.maxDisks , Types.iSpecNicCount = C.maxNics , Types.iSpecSpindleUse = maxBound } }] , Types.iPolicyStdSpec = Types.ISpec { Types.iSpecMemorySize = Types.unitMem , Types.iSpecCpuCount = Types.unitCpu , Types.iSpecDiskSize = Types.unitDsk , Types.iSpecDiskCount = 1 , Types.iSpecNicCount = 1 , Types.iSpecSpindleUse = 1 } , Types.iPolicyDiskTemplates = [minBound..maxBound] , Types.iPolicyVcpuRatio = maxVcpuRatio -- somewhat random value, high -- enough to not impact us , Types.iPolicySpindleRatio = maxSpindleRatio } -- | Default group definition. defGroup :: Group.Group defGroup = flip Group.setIdx 0 $ Group.create "default" Types.defaultGroupID Types.AllocPreferred [] nullIPolicy [] -- | Default group, as a (singleton) 'Group.List'. defGroupList :: Group.List defGroupList = Container.fromList [(Group.idx defGroup, defGroup)] -- | Default group, as a string map. defGroupAssoc :: Map.Map String Types.Gdx defGroupAssoc = Map.singleton (Group.uuid defGroup) (Group.idx defGroup) -- | Create an instance given its spec. createInstance :: Int -> Int -> Int -> Instance.Instance createInstance mem dsk vcpus = Instance.create "inst-unnamed" mem dsk [Instance.Disk dsk Nothing] vcpus Types.Running [] True (-1) (-1) Types.DTDrbd8 1 [] False -- | Create a small cluster by repeating a node spec. makeSmallCluster :: Node.Node -> Int -> Node.List makeSmallCluster node count = let origname = Node.name node origalias = Node.alias node nodes = map (\idx -> node { Node.name = origname ++ "-" ++ show idx , Node.alias = origalias ++ "-" ++ show idx }) [1..count] fn = flip Node.buildPeers Container.empty namelst = map (\n -> (Node.name n, fn n)) nodes (_, nlst) = Loader.assignIndices namelst in nlst -- | Update an instance to be smaller than a node. setInstanceSmallerThanNode :: Node.Node -> Instance.Instance -> Instance.Instance setInstanceSmallerThanNode node inst = let new_dsk = Node.availDisk node `div` 2 in inst { Instance.mem = Node.availMem node `div` 2 , Instance.dsk = new_dsk , Instance.vcpus = Node.availCpu node `div` 2 , Instance.disks = [Instance.Disk new_dsk (if Node.exclStorage node then Just $ Node.fSpindlesForth node `div` 2 else Nothing)] } ganeti-3.1.0~rc2/test/hs/Test/Ganeti/TestHelper.hs000064400000000000000000000126511476477700300217430ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittest helpers for TemplateHaskell components. -} {- Copyright (C) 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.TestHelper ( testSuite , genArbitrary ) where import Data.List (stripPrefix, isPrefixOf) import Data.Maybe (fromMaybe) import Test.Framework import Test.Framework.Providers.HUnit import Test.Framework.Providers.QuickCheck2 import Test.HUnit (Assertion) import Test.QuickCheck import Language.Haskell.TH import Ganeti.THH.Compat -- | Test property prefix. propPrefix :: String propPrefix = "prop_" -- | Test case prefix. casePrefix :: String casePrefix = "case_" -- | Test case prefix without underscore. case2Pfx :: String case2Pfx = "case" -- | Tries to drop a prefix from a string. simplifyName :: String -> String -> String simplifyName pfx string = fromMaybe string (stripPrefix pfx string) -- | Builds a test from a QuickCheck property. runProp :: Testable prop => String -> prop -> Test runProp = testProperty . simplifyName propPrefix -- | Builds a test for a HUnit test case. runCase :: String -> Assertion -> Test runCase = testCase . simplifyName casePrefix -- | Runs the correct test provider for a given test, based on its -- name (not very nice, but...). run :: Name -> Q Exp run name = let str = nameBase name nameE = varE name strE = litE (StringL str) in case () of _ | propPrefix `isPrefixOf` str -> [| runProp $strE $nameE |] | casePrefix `isPrefixOf` str -> [| runCase $strE $nameE |] | case2Pfx `isPrefixOf` str -> [| (testCase . simplifyName case2Pfx) $strE $nameE |] | otherwise -> fail $ "Unsupported test function name '" ++ str ++ "'" -- | Convert slashes in a name to underscores. mapSlashes :: String -> String mapSlashes = map (\c -> if c == '/' then '_' else c) -- | Builds a test suite. testSuite :: String -> [Name] -> Q [Dec] testSuite tsname tdef = do let fullname = mkName $ "test" ++ mapSlashes tsname tests <- mapM run tdef sigtype <- [t| Test |] body <- [| testGroup $(litE $ stringL tsname) $(return $ ListE tests) |] return [ SigD fullname sigtype , ValD (VarP fullname) (NormalB body) [] ] -- | Builds an arbitrary value for a given constructor. This doesn't -- use the actual types of the fields, since we expect arbitrary -- instances for all of the types anyway, we only care about the -- number of fields. mkConsArbitrary :: (Name, [a]) -> Exp mkConsArbitrary (name, types) = let infix_arb a = InfixE (Just a) (VarE '(<*>)) (Just (VarE 'arbitrary)) constr = AppE (VarE 'pure) (ConE name) in foldl (\a _ -> infix_arb a) constr types -- | Extracts the name and the types from a constructor. conInfo :: Con -> (Name, [Type]) conInfo (NormalC name t) = (name, map snd t) conInfo (RecC name t) = (name, map (\(_, _, x) -> x) t) conInfo (InfixC t1 name t2) = (name, [snd t1, snd t2]) conInfo (ForallC _ _ subcon) = conInfo subcon -- | Builds an arbitrary instance for a regular data type (i.e. not Bounded). mkRegularArbitrary :: Name -> [Con] -> Q [Dec] mkRegularArbitrary name cons = do expr <- case cons of [] -> fail "Can't make Arbitrary instance for an empty data type" [x] -> return $ mkConsArbitrary (conInfo x) xs -> appE (varE 'oneof) $ listE (map (return . mkConsArbitrary . conInfo) xs) return [gntInstanceD [] (AppT (ConT ''Arbitrary) (ConT name)) [ValD (VarP 'arbitrary) (NormalB expr) []]] -- | Builds a default Arbitrary instance for a type. This requires -- that all members are of types that already have Arbitrary -- instances, and that the arbitrary instances are well behaved -- (w.r.t. recursive data structures, or similar concerns). In that -- sense, this is not appropriate for all data types, just those that -- are simple but very repetitive or have many simple fields. genArbitrary :: Name -> Q [Dec] genArbitrary name = do r <- reify name case r of TyConI (DataD _ _ _ _ cons _) -> mkRegularArbitrary name cons TyConI (NewtypeD _ _ _ _ con _) -> mkRegularArbitrary name [con] TyConI (TySynD _ _ (ConT tn)) -> genArbitrary tn _ -> fail $ "Invalid type in call to genArbitrary for " ++ show name ++ ", type " ++ show r ganeti-3.1.0~rc2/test/hs/Test/Ganeti/TestImports.hs.in000064400000000000000000000002251476477700300225600ustar00rootroot00000000000000-- Hey Emacs, this is a -*- haskell -*- file {-| Auto-generated file importing all production modules. -} module Test.Ganeti.TestImports () where ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Types.hs000064400000000000000000000372071476477700300207740ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for 'Ganeti.Types'. -} {- Copyright (C) 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Types ( testTypes , AllocPolicy(..) , DiskTemplate(..) , allDiskTemplates , InstanceStatus(..) , NonEmpty(..) , Hypervisor(..) , JobId(..) , genReasonTrail ) where import System.Time (ClockTime(..)) import Test.QuickCheck as QuickCheck hiding (Result) import Test.HUnit import qualified Text.JSON as J import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.BasicTypes import qualified Ganeti.Constants as C import qualified Ganeti.ConstantUtils as ConstantUtils import Ganeti.Types as Types import Ganeti.JSON {-# ANN module "HLint: ignore Use camelCase" #-} -- * Arbitrary instance instance Arbitrary ClockTime where arbitrary = TOD <$> arbitrary <*> fmap (`mod` (10^(12::Int))) arbitrary instance (Arbitrary a, Ord a, Num a, Show a) => Arbitrary (Types.Positive a) where arbitrary = do (QuickCheck.Positive i) <- arbitrary Types.mkPositive i instance (Arbitrary a, Ord a, Num a, Show a) => Arbitrary (Types.NonNegative a) where arbitrary = do (QuickCheck.NonNegative i) <- arbitrary Types.mkNonNegative i instance (Arbitrary a, Ord a, Num a, Show a) => Arbitrary (Types.Negative a) where arbitrary = do (QuickCheck.Positive i) <- arbitrary Types.mkNegative $ negate i instance (Arbitrary a) => Arbitrary (Types.NonEmpty a) where arbitrary = do QuickCheck.NonEmpty lst <- arbitrary Types.mkNonEmpty lst $(genArbitrary ''AllocPolicy) -- | Valid disk templates (depending on configure options). allDiskTemplates :: [DiskTemplate] allDiskTemplates = [minBound..maxBound]::[DiskTemplate] -- | Custom 'Arbitrary' instance for 'DiskTemplate', which needs to -- handle the case of file storage being disabled at configure time. instance Arbitrary DiskTemplate where arbitrary = elements allDiskTemplates $(genArbitrary ''InstanceStatus) $(genArbitrary ''MigrationMode) $(genArbitrary ''VerifyOptionalChecks) $(genArbitrary ''DdmSimple) $(genArbitrary ''DdmFull) $(genArbitrary ''CVErrorCode) $(genArbitrary ''Hypervisor) $(genArbitrary ''TagKind) $(genArbitrary ''OobCommand) -- | Valid storage types. allStorageTypes :: [StorageType] allStorageTypes = [minBound..maxBound]::[StorageType] -- | Custom 'Arbitrary' instance for 'StorageType', which needs to -- handle the case of file storage being disabled at configure time. instance Arbitrary StorageType where arbitrary = elements allStorageTypes $(genArbitrary ''EvacMode) $(genArbitrary ''FileDriver) $(genArbitrary ''InstCreateMode) $(genArbitrary ''RebootType) $(genArbitrary ''ExportMode) $(genArbitrary ''IAllocatorTestDir) $(genArbitrary ''IAllocatorMode) $(genArbitrary ''NICMode) $(genArbitrary ''JobStatus) $(genArbitrary ''FinalizedJobStatus) instance Arbitrary JobId where arbitrary = do (Positive i) <- arbitrary makeJobId i $(genArbitrary ''JobIdDep) $(genArbitrary ''JobDependency) $(genArbitrary ''OpSubmitPriority) $(genArbitrary ''OpStatus) $(genArbitrary ''ELogType) -- | Generates one element of a reason trail genReasonElem :: Gen ReasonElem genReasonElem = (,,) <$> genFQDN <*> genFQDN <*> arbitrary -- | Generates a reason trail genReasonTrail :: Gen ReasonTrail genReasonTrail = do size <- choose (0, 10) vectorOf size genReasonElem -- * Properties prop_AllocPolicy_serialisation :: AllocPolicy -> Property prop_AllocPolicy_serialisation = testSerialisation -- | Test 'AllocPolicy' ordering is as expected. case_AllocPolicy_order :: Assertion case_AllocPolicy_order = assertEqual "sort order" [ Types.AllocPreferred , Types.AllocLastResort , Types.AllocUnallocable ] [minBound..maxBound] prop_DiskTemplate_serialisation :: DiskTemplate -> Property prop_DiskTemplate_serialisation = testSerialisation prop_InstanceStatus_serialisation :: InstanceStatus -> Property prop_InstanceStatus_serialisation = testSerialisation -- | Tests building non-negative numbers. prop_NonNeg_pass :: QuickCheck.NonNegative Int -> Property prop_NonNeg_pass (QuickCheck.NonNegative i) = case mkNonNegative i of Bad msg -> failTest $ "Fail to build non-negative: " ++ msg Ok nn -> fromNonNegative nn ==? i -- | Tests building non-negative numbers. prop_NonNeg_fail :: QuickCheck.Positive Int -> Property prop_NonNeg_fail (QuickCheck.Positive i) = case mkNonNegative (negate i)::Result (Types.NonNegative Int) of Bad _ -> passTest Ok nn -> failTest $ "Built non-negative number '" ++ show nn ++ "' from negative value " ++ show i -- | Tests building positive numbers. prop_Positive_pass :: QuickCheck.Positive Int -> Property prop_Positive_pass (QuickCheck.Positive i) = case mkPositive i of Bad msg -> failTest $ "Fail to build positive: " ++ msg Ok nn -> fromPositive nn ==? i -- | Tests building positive numbers. prop_Positive_fail :: QuickCheck.NonNegative Int -> Property prop_Positive_fail (QuickCheck.NonNegative i) = case mkPositive (negate i)::Result (Types.Positive Int) of Bad _ -> passTest Ok nn -> failTest $ "Built positive number '" ++ show nn ++ "' from negative or zero value " ++ show i -- | Tests building negative numbers. prop_Neg_pass :: QuickCheck.Positive Int -> Property prop_Neg_pass (QuickCheck.Positive i) = case mkNegative i' of Bad msg -> failTest $ "Fail to build negative: " ++ msg Ok nn -> fromNegative nn ==? i' where i' = negate i -- | Tests building negative numbers. prop_Neg_fail :: QuickCheck.NonNegative Int -> Property prop_Neg_fail (QuickCheck.NonNegative i) = case mkNegative i::Result (Types.Negative Int) of Bad _ -> passTest Ok nn -> failTest $ "Built negative number '" ++ show nn ++ "' from non-negative value " ++ show i -- | Tests building non-empty lists. prop_NonEmpty_pass :: QuickCheck.NonEmptyList String -> Property prop_NonEmpty_pass (QuickCheck.NonEmpty xs) = case mkNonEmpty xs of Bad msg -> failTest $ "Fail to build non-empty list: " ++ msg Ok nn -> fromNonEmpty nn ==? xs -- | Tests building positive numbers. case_NonEmpty_fail :: Assertion case_NonEmpty_fail = assertEqual "building non-empty list from an empty list" (Bad "Received empty value for non-empty list") (mkNonEmpty ([]::[Int])) -- | Tests migration mode serialisation. prop_MigrationMode_serialisation :: MigrationMode -> Property prop_MigrationMode_serialisation = testSerialisation -- | Tests verify optional checks serialisation. prop_VerifyOptionalChecks_serialisation :: VerifyOptionalChecks -> Property prop_VerifyOptionalChecks_serialisation = testSerialisation -- | Tests 'DdmSimple' serialisation. prop_DdmSimple_serialisation :: DdmSimple -> Property prop_DdmSimple_serialisation = testSerialisation -- | Tests 'DdmFull' serialisation. prop_DdmFull_serialisation :: DdmFull -> Property prop_DdmFull_serialisation = testSerialisation -- | Tests 'CVErrorCode' serialisation. prop_CVErrorCode_serialisation :: CVErrorCode -> Property prop_CVErrorCode_serialisation = testSerialisation -- | Tests equivalence with Python, based on Constants.hs code. case_CVErrorCode_pyequiv :: Assertion case_CVErrorCode_pyequiv = do let all_py_codes = C.cvAllEcodesStrings all_hs_codes = ConstantUtils.mkSet $ map Types.cVErrorCodeToRaw [minBound..maxBound] assertEqual "for CVErrorCode equivalence" all_py_codes all_hs_codes -- | Test 'Hypervisor' serialisation. prop_Hypervisor_serialisation :: Hypervisor -> Property prop_Hypervisor_serialisation = testSerialisation -- | Test 'OobCommand' serialisation. prop_OobCommand_serialisation :: OobCommand -> Property prop_OobCommand_serialisation = testSerialisation -- | Test 'StorageType' serialisation. prop_StorageType_serialisation :: StorageType -> Property prop_StorageType_serialisation = testSerialisation -- | Test 'NodeEvacMode' serialisation. prop_NodeEvacMode_serialisation :: EvacMode -> Property prop_NodeEvacMode_serialisation = testSerialisation -- | Test 'FileDriver' serialisation. prop_FileDriver_serialisation :: FileDriver -> Property prop_FileDriver_serialisation = testSerialisation -- | Test 'InstCreate' serialisation. prop_InstCreateMode_serialisation :: InstCreateMode -> Property prop_InstCreateMode_serialisation = testSerialisation -- | Test 'RebootType' serialisation. prop_RebootType_serialisation :: RebootType -> Property prop_RebootType_serialisation = testSerialisation -- | Test 'ExportMode' serialisation. prop_ExportMode_serialisation :: ExportMode -> Property prop_ExportMode_serialisation = testSerialisation -- | Test 'IAllocatorTestDir' serialisation. prop_IAllocatorTestDir_serialisation :: IAllocatorTestDir -> Property prop_IAllocatorTestDir_serialisation = testSerialisation -- | Test 'IAllocatorMode' serialisation. prop_IAllocatorMode_serialisation :: IAllocatorMode -> Property prop_IAllocatorMode_serialisation = testSerialisation -- | Tests equivalence with Python, based on Constants.hs code. case_IAllocatorMode_pyequiv :: Assertion case_IAllocatorMode_pyequiv = do let all_py_codes = C.validIallocatorModes all_hs_codes = ConstantUtils.mkSet $ map Types.iAllocatorModeToRaw [minBound..maxBound] assertEqual "for IAllocatorMode equivalence" all_py_codes all_hs_codes -- | Test 'NICMode' serialisation. prop_NICMode_serialisation :: NICMode -> Property prop_NICMode_serialisation = testSerialisation -- | Test 'OpStatus' serialisation. prop_OpStatus_serialization :: OpStatus -> Property prop_OpStatus_serialization = testSerialisation -- | Test 'JobStatus' serialisation. prop_JobStatus_serialization :: JobStatus -> Property prop_JobStatus_serialization = testSerialisation -- | Test 'JobStatus' ordering is as expected. case_JobStatus_order :: Assertion case_JobStatus_order = assertEqual "sort order" [ Types.JOB_STATUS_QUEUED , Types.JOB_STATUS_WAITING , Types.JOB_STATUS_CANCELING , Types.JOB_STATUS_RUNNING , Types.JOB_STATUS_CANCELED , Types.JOB_STATUS_SUCCESS , Types.JOB_STATUS_ERROR ] [minBound..maxBound] -- | Tests equivalence with Python, based on Constants.hs code. case_NICMode_pyequiv :: Assertion case_NICMode_pyequiv = do let all_py_codes = C.nicValidModes all_hs_codes = ConstantUtils.mkSet $ map Types.nICModeToRaw [minBound..maxBound] assertEqual "for NICMode equivalence" all_py_codes all_hs_codes -- | Test 'FinalizedJobStatus' serialisation. prop_FinalizedJobStatus_serialisation :: FinalizedJobStatus -> Property prop_FinalizedJobStatus_serialisation = testSerialisation -- | Tests equivalence with Python, based on Constants.hs code. case_FinalizedJobStatus_pyequiv :: Assertion case_FinalizedJobStatus_pyequiv = do let all_py_codes = C.jobsFinalized all_hs_codes = ConstantUtils.mkSet $ map Types.finalizedJobStatusToRaw [minBound..maxBound] assertEqual "for FinalizedJobStatus equivalence" all_py_codes all_hs_codes -- | Tests JobId serialisation (both from string and ints). prop_JobId_serialisation :: JobId -> Property prop_JobId_serialisation jid = conjoin [ testSerialisation jid , (J.readJSON . J.showJSON . show $ fromJobId jid) ==? J.Ok jid , case (fromJVal . J.showJSON . negate $ fromJobId jid)::Result JobId of Bad _ -> passTest Ok jid' -> failTest $ "Parsed negative job id as id " ++ show (fromJobId jid') ] -- | Tests that fractional job IDs are not accepted. prop_JobId_fractional :: Property prop_JobId_fractional = forAll (arbitrary `suchThat` (\d -> fromIntegral (truncate d::Int) /= d)) $ \d -> case J.readJSON (J.showJSON (d::Double)) of J.Error _ -> passTest J.Ok jid -> failTest $ "Parsed fractional value " ++ show d ++ " as job id " ++ show (fromJobId jid) -- | Tests that a job ID is not parseable from \"bad\" JSON values. case_JobId_BadTypes :: Assertion case_JobId_BadTypes = do let helper jsval = case J.readJSON jsval of J.Error _ -> return () J.Ok jid -> assertFailure $ "Parsed " ++ show jsval ++ " as job id " ++ show (fromJobId jid) helper J.JSNull helper (J.JSBool True) helper (J.JSBool False) helper (J.JSArray []) -- | Test 'JobDependency' serialisation. prop_JobDependency_serialisation :: JobDependency -> Property prop_JobDependency_serialisation = testSerialisation -- | Test 'OpSubmitPriority' serialisation. prop_OpSubmitPriority_serialisation :: OpSubmitPriority -> Property prop_OpSubmitPriority_serialisation = testSerialisation -- | Tests string formatting for 'OpSubmitPriority'. prop_OpSubmitPriority_string :: OpSubmitPriority -> Property prop_OpSubmitPriority_string prio = parseSubmitPriority (fmtSubmitPriority prio) ==? Just prio -- | Test 'ELogType' serialisation. prop_ELogType_serialisation :: ELogType -> Property prop_ELogType_serialisation = testSerialisation testSuite "Types" [ 'prop_AllocPolicy_serialisation , 'case_AllocPolicy_order , 'prop_DiskTemplate_serialisation , 'prop_InstanceStatus_serialisation , 'prop_NonNeg_pass , 'prop_NonNeg_fail , 'prop_Positive_pass , 'prop_Positive_fail , 'prop_Neg_pass , 'prop_Neg_fail , 'prop_NonEmpty_pass , 'case_NonEmpty_fail , 'prop_MigrationMode_serialisation , 'prop_VerifyOptionalChecks_serialisation , 'prop_DdmSimple_serialisation , 'prop_DdmFull_serialisation , 'prop_CVErrorCode_serialisation , 'case_CVErrorCode_pyequiv , 'prop_Hypervisor_serialisation , 'prop_OobCommand_serialisation , 'prop_StorageType_serialisation , 'prop_NodeEvacMode_serialisation , 'prop_FileDriver_serialisation , 'prop_InstCreateMode_serialisation , 'prop_RebootType_serialisation , 'prop_ExportMode_serialisation , 'prop_IAllocatorTestDir_serialisation , 'prop_IAllocatorMode_serialisation , 'case_IAllocatorMode_pyequiv , 'prop_NICMode_serialisation , 'prop_OpStatus_serialization , 'prop_JobStatus_serialization , 'case_JobStatus_order , 'case_NICMode_pyequiv , 'prop_FinalizedJobStatus_serialisation , 'case_FinalizedJobStatus_pyequiv , 'prop_JobId_serialisation , 'prop_JobId_fractional , 'case_JobId_BadTypes , 'prop_JobDependency_serialisation , 'prop_OpSubmitPriority_serialisation , 'prop_OpSubmitPriority_string , 'prop_ELogType_serialisation ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Utils.hs000064400000000000000000000357331476477700300207720ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell, CPP, ScopedTypeVariables #-} {-| Unittests for ganeti-htools. -} {- Copyright (C) 2009, 2010, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Utils (testUtils) where import Test.QuickCheck hiding (Result) import Test.HUnit import Data.Char (isSpace) import qualified Data.Either as Either import qualified Data.List.NonEmpty as NonEmpty import Data.List.NonEmpty (NonEmpty((:|))) import Data.List import Data.Maybe (listToMaybe) import qualified Data.Set as S import qualified Text.JSON as J import Ganeti.Query.RegEx ((=~)) import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.BasicTypes import qualified Ganeti.Constants as C import qualified Ganeti.JSON as JSON import Ganeti.Utils as Utils {-# ANN module "HLint: ignore Use camelCase" #-} -- | Helper to generate a small string that doesn't contain commas. genNonCommaString :: Gen String genNonCommaString = do size <- choose (0, 20) -- arbitrary max size vectorOf size (arbitrary `suchThat` (/=) ',') genNonEmpty :: (Arbitrary a) => Gen (NonEmpty a) genNonEmpty = fmap (uncurry (:|)) arbitrary -- | If the list is not just an empty element, and if the elements do -- not contain commas, then join+split should be idempotent. prop_commaJoinSplit :: Property prop_commaJoinSplit = forAll (choose (0, 20)) $ \llen -> forAll (vectorOf llen genNonCommaString `suchThat` (/=) [""]) $ \lst -> sepSplit ',' (commaJoin lst) ==? lst -- | Split and join should always be idempotent. prop_commaSplitJoin :: String -> Property prop_commaSplitJoin s = commaJoin (sepSplit ',' s) ==? s -- | Test 'findFirst' on several possible inputs. prop_findFirst :: Property prop_findFirst = forAll (genSublist [0..5 :: Int]) $ \xs -> forAll (choose (-2, 7)) $ \base -> counterexample "findFirst utility function" $ let r = findFirst base (S.fromList xs) (ss, es) = partition (< r) $ dropWhile (< base) xs -- the prefix must be a range of numbers -- and the suffix must not start with 'r' in conjoin [ and $ zipWith ((==) . (+ 1)) ss (drop 1 ss) , maybe True (> r) (listToMaybe es) ] -- | fromObjWithDefault, we test using the Maybe monad and an integer -- value. prop_fromObjWithDefault :: Integer -> String -> Bool prop_fromObjWithDefault def_value random_key = -- a missing key will be returned with the default JSON.fromObjWithDefault [] random_key def_value == Just def_value && -- a found key will be returned as is, not with default JSON.fromObjWithDefault [(random_key, J.showJSON def_value)] random_key (def_value+1) == Just def_value -- | Test that functional if' behaves like the syntactic sugar if. prop_if'if :: Bool -> Int -> Int -> Property prop_if'if cnd a b = if' cnd a b ==? if cnd then a else b -- | Test basic select functionality prop_select :: Int -- ^ Default result -> [Int] -- ^ List of False values -> [Int] -- ^ List of True values -> Property -- ^ Test result prop_select def lst1 lst2 = select def (flist ++ tlist) ==? expectedresult where expectedresult = defaultHead def lst2 flist = zip (repeat False) lst1 tlist = zip (repeat True) lst2 -- | Test basic select functionality with undefined default prop_select_undefd_plain :: [Int] -- ^ List of False values -> NonEmpty Int -- ^ List of True values -> Property -- ^ Test result prop_select_undefd_plain lst1 lst2 = select undefined (flist ++ tlist) ==? NonEmpty.head lst2 where flist = zip (repeat False) lst1 tlist = zip (repeat True) (NonEmpty.toList lst2) prop_select_undefd :: [Int] -> Property prop_select_undefd lst1 = forAll genNonEmpty $ prop_select_undefd_plain lst1 -- | Test basic select functionality with undefined list values prop_select_undefv_plain :: [Int] -- ^ List of False values -> NonEmpty Int -- ^ List of True values -> Property -- ^ Test result prop_select_undefv_plain lst1 lst2 = select undefined cndlist ==? NonEmpty.head lst2 where flist = zip (repeat False) lst1 tlist = zip (repeat True) (NonEmpty.toList lst2) cndlist = flist ++ tlist ++ [undefined] prop_select_undefv :: [Int] -> Property prop_select_undefv lst1 = forAll genNonEmpty $ prop_select_undefv_plain lst1 prop_parseUnit :: NonNegative Int -> Property prop_parseUnit (NonNegative n) = conjoin [ parseUnit (show n) ==? (Ok n::Result Int) , parseUnit (show n ++ "m") ==? (Ok n::Result Int) , parseUnit (show n ++ "M") ==? (Ok (truncate n_mb)::Result Int) , parseUnit (show n ++ "g") ==? (Ok (n*1024)::Result Int) , parseUnit (show n ++ "G") ==? (Ok (truncate n_gb)::Result Int) , parseUnit (show n ++ "t") ==? (Ok (n*1048576)::Result Int) , parseUnit (show n ++ "T") ==? (Ok (truncate n_tb)::Result Int) , counterexample "Internal error/overflow?" (n_mb >=0 && n_gb >= 0 && n_tb >= 0) , property (isBad (parseUnit (show n ++ "x")::Result Int)) ] where n_mb = (fromIntegral n::Rational) * 1000 * 1000 / 1024 / 1024 n_gb = n_mb * 1000 n_tb = n_gb * 1000 {-# ANN case_niceSort_static "HLint: ignore Use camelCase" #-} case_niceSort_static :: Assertion case_niceSort_static = do assertEqual "empty list" [] $ niceSort [] assertEqual "punctuation" [",", "."] $ niceSort [",", "."] assertEqual "decimal numbers" ["0.1", "0.2"] $ niceSort ["0.1", "0.2"] assertEqual "various numbers" ["0,099", "0.1", "0.2", "0;099"] $ niceSort ["0;099", "0,099", "0.1", "0.2"] assertEqual "simple concat" ["0000", "a0", "a1", "a2", "a20", "a99", "b00", "b10", "b70"] $ niceSort ["a0", "a1", "a99", "a20", "a2", "b10", "b70", "b00", "0000"] assertEqual "ranges" ["A", "Z", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2", "a20-3", "a99-3", "a99-10", "b"] $ niceSort ["a0-0", "a1-0", "a99-10", "a20-3", "a0-4", "a99-3", "a09-2", "Z", "a9-1", "A", "b"] assertEqual "large" ["3jTwJPtrXOY22bwL2YoW", "Eegah9ei", "KOt7vn1dWXi", "KVQqLPDjcPjf8T3oyzjcOsfkb", "WvNJd91OoXvLzdEiEXa6", "Z8Ljf1Pf5eBfNg171wJR", "a07h8feON165N67PIE", "bH4Q7aCu3PUPjK3JtH", "cPRi0lM7HLnSuWA2G9", "guKJkXnkULealVC8CyF1xefym", "pqF8dkU5B1cMnyZuREaSOADYx", "uHXAyYYftCSG1o7qcCqe", "xij88brTulHYAv8IEOyU", "xpIUJeVT1Rp"] $ niceSort ["Eegah9ei", "xij88brTulHYAv8IEOyU", "3jTwJPtrXOY22bwL2YoW", "Z8Ljf1Pf5eBfNg171wJR", "WvNJd91OoXvLzdEiEXa6", "uHXAyYYftCSG1o7qcCqe", "xpIUJeVT1Rp", "KOt7vn1dWXi", "a07h8feON165N67PIE", "bH4Q7aCu3PUPjK3JtH", "cPRi0lM7HLnSuWA2G9", "KVQqLPDjcPjf8T3oyzjcOsfkb", "guKJkXnkULealVC8CyF1xefym", "pqF8dkU5B1cMnyZuREaSOADYx"] assertEqual "hostnames" ["host1.example.com", "host2.example.com", "host03.example.com", "host11.example.com", "host255.example.com"] $ niceSort ["host2.example.com", "host11.example.com", "host03.example.com", "host1.example.com", "host255.example.com"] -- | Tests single-string behaviour of 'niceSort'. prop_niceSort_single :: Property prop_niceSort_single = forAll genName $ \name -> conjoin [ counterexample "single string" $ [name] ==? niceSort [name] , counterexample "single plus empty" $ ["", name] ==? niceSort [name, ""] ] -- | Tests some generic 'niceSort' properties. Note that the last test -- must add a non-digit prefix; a digit one might change ordering. prop_niceSort_generic :: Property prop_niceSort_generic = forAll (resize 20 arbitrary) $ \names -> let n_sorted = niceSort names in conjoin [ counterexample "length" $ length names ==? length n_sorted , counterexample "same strings" $ sort names ==? sort n_sorted , counterexample "idempotence" $ n_sorted ==? niceSort n_sorted , counterexample "static prefix" $ n_sorted ==? map tail (niceSort $ map (" "++) names) ] -- | Tests that niceSorting numbers is identical to actual sorting -- them (in numeric form). prop_niceSort_numbers :: Property prop_niceSort_numbers = forAll (listOf (arbitrary::Gen (NonNegative Int))) $ \numbers -> map show (sort numbers) ==? niceSort (map show numbers) -- | Tests that 'niceSort' and 'niceSortKey' are equivalent. prop_niceSortKey_equiv :: Property prop_niceSortKey_equiv = forAll (resize 20 arbitrary) $ \names -> forAll (vectorOf (length names) (arbitrary::Gen Int)) $ \numbers -> let n_sorted = niceSort names in conjoin [ counterexample "key id" $ n_sorted ==? niceSortKey id names , counterexample "key rev" $ niceSort (map reverse names) ==? map reverse (niceSortKey reverse names) , counterexample "key snd" $ n_sorted ==? map snd (niceSortKey snd $ zip numbers names) ] -- | Tests 'rStripSpace'. prop_rStripSpace :: NonEmptyList Char -> Property prop_rStripSpace (NonEmpty str) = forAll (resize 50 $ listOf1 (arbitrary `suchThat` isSpace)) $ \whitespace -> conjoin [ counterexample "arb. string last char is not space" $ case rStripSpace str of [] -> True xs -> not . isSpace $ last xs , counterexample "whitespace suffix is stripped" $ rStripSpace str ==? rStripSpace (str ++ whitespace) , counterexample "whitespace reduced to null" $ rStripSpace whitespace ==? "" , counterexample "idempotent on empty strings" $ rStripSpace "" ==? "" ] -- | Tests that the newUUID function produces valid UUIDs. case_new_uuid :: Assertion case_new_uuid = do uuid <- newUUID assertBool "newUUID" $ isUUID uuid {-# ANN case_new_uuid_regex "HLint: ignore Use camelCase" #-} -- | Tests that the newUUID function produces valid UUIDs. case_new_uuid_regex :: Assertion case_new_uuid_regex = do uuid <- newUUID assertBool "newUUID" $ uuid =~ C.uuidRegex -- | Test normal operation for 'chompPrefix'. -- -- Any random prefix of a string must be stripped correctly, including the empty -- prefix, and the whole string. prop_chompPrefix_normal :: String -> Property prop_chompPrefix_normal str = forAll (choose (0, length str)) $ \size -> chompPrefix (take size str) str ==? (Just $ drop size str) -- | Test that 'chompPrefix' correctly allows the last char (the separator) to -- be absent if the string terminates there. prop_chompPrefix_last :: Property prop_chompPrefix_last = forAll (choose (1, 20)) $ \len -> forAll (vectorOf len arbitrary) $ \pfx -> chompPrefix pfx pfx ==? Just "" .&&. chompPrefix pfx (init pfx) ==? Just "" -- | Test that chompPrefix on the empty string always returns Nothing for -- prefixes of length 2 or more. prop_chompPrefix_empty_string :: Property prop_chompPrefix_empty_string = forAll (choose (2, 20)) $ \len -> forAll (vectorOf len arbitrary) $ \pfx -> chompPrefix pfx "" ==? Nothing -- | Test 'chompPrefix' returns Nothing when the prefix doesn't match. prop_chompPrefix_nothing :: Property prop_chompPrefix_nothing = forAll (choose (1, 20)) $ \len -> forAll (vectorOf len arbitrary) $ \pfx -> forAll (arbitrary `suchThat` (\s -> not (pfx `isPrefixOf` s) && s /= init pfx)) $ \str -> chompPrefix pfx str ==? Nothing -- | Tests 'trim'. prop_trim :: NonEmptyList Char -> Property prop_trim (NonEmpty str) = forAll (listOf1 $ elements " \t\n\r\f") $ \whitespace -> forAll (choose (0, length whitespace)) $ \n -> let (preWS, postWS) = splitAt n whitespace in conjoin [ counterexample "arb. string first and last char are not space" $ case trim str of [] -> True xs -> (not . isSpace . head) xs && (not . isSpace . last) xs , counterexample "whitespace is striped" $ trim str ==? trim (preWS ++ str ++ postWS) , counterexample "whitespace reduced to null" $ trim whitespace ==? "" , counterexample "idempotent on empty strings" $ trim "" ==? "" ] -- | Tests 'splitEithers' and 'recombineEithers'. prop_splitRecombineEithers :: [Either Int Int] -> Property prop_splitRecombineEithers es = conjoin [ counterexample "only lefts are mapped correctly" $ splitEithers (map Left lefts) ==? (reverse lefts, emptylist, falses) , counterexample "only rights are mapped correctly" $ splitEithers (map Right rights) ==? (emptylist, reverse rights, trues) , counterexample "recombination is no-op" $ recombineEithers splitleft splitright trail ==? Ok es , counterexample "fail on too long lefts" $ isBad (recombineEithers (0:splitleft) splitright trail) , counterexample "fail on too long rights" $ isBad (recombineEithers splitleft (0:splitright) trail) , counterexample "fail on too long trail" $ isBad (recombineEithers splitleft splitright (True:trail)) ] where (lefts, rights) = Either.partitionEithers es falses = map (const False) lefts trues = map (const True) rights (splitleft, splitright, trail) = splitEithers es emptylist = []::[Int] testSuite "Utils" [ 'prop_commaJoinSplit , 'prop_commaSplitJoin , 'prop_findFirst , 'prop_fromObjWithDefault , 'prop_if'if , 'prop_select , 'prop_select_undefd , 'prop_select_undefv , 'prop_parseUnit , 'case_niceSort_static , 'prop_niceSort_single , 'prop_niceSort_generic , 'prop_niceSort_numbers , 'prop_niceSortKey_equiv , 'prop_rStripSpace , 'prop_trim , 'case_new_uuid , 'case_new_uuid_regex , 'prop_chompPrefix_normal , 'prop_chompPrefix_last , 'prop_chompPrefix_empty_string , 'prop_chompPrefix_nothing , 'prop_splitRecombineEithers ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Utils/000075500000000000000000000000001476477700300204235ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Utils/MultiMap.hs000064400000000000000000000056411476477700300225150ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for mutli-maps -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Utils.MultiMap ( testUtils_MultiMap ) where import qualified Data.Set as S import qualified Data.Map as M import Test.QuickCheck import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.Utils.MultiMap as MM instance (Arbitrary k, Ord k, Arbitrary v, Ord v) => Arbitrary (MultiMap k v) where arbitrary = frequency [ (1, (multiMap . M.fromList) <$> listOf ((,) <$> arbitrary <*> (S.fromList <$> listOf arbitrary))) , (4, MM.insert <$> arbitrary <*> arbitrary <*> arbitrary) , (1, MM.fromList <$> listOf ((,) <$> arbitrary <*> arbitrary)) , (3, MM.delete <$> arbitrary <*> arbitrary <*> arbitrary) , (1, MM.deleteAll <$> arbitrary <*> arbitrary) ] -- | A data type for testing extensional equality. data Three = One | Two | Three deriving (Eq, Ord, Show, Enum, Bounded) instance Arbitrary Three where arbitrary = elements [minBound..maxBound] -- | Tests the extensional equality of multi-maps. prop_MultiMap_equality :: MultiMap Three Three -> MultiMap Three Three -> Property prop_MultiMap_equality m1 m2 = let testKey k = MM.lookup k m1 == MM.lookup k m2 in counterexample ("Extensional equality of '" ++ show m1 ++ "' and '" ++ show m2 ++ " doesn't match '=='.") $ all testKey [minBound..maxBound] ==? (m1 == m2) prop_MultiMap_serialisation :: MultiMap Int Int -> Property prop_MultiMap_serialisation = testSerialisation testSuite "Utils/MultiMap" [ 'prop_MultiMap_equality , 'prop_MultiMap_serialisation ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Utils/Statistics.hs000064400000000000000000000047271476477700300231230ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unit tests for Ganeti statistics utils. -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Utils.Statistics (testUtils_Statistics) where import Test.QuickCheck (Property, forAll, choose, vectorOf) import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Ganeti.Utils (stdDev) import Ganeti.Utils.Statistics -- | Test the update function for standard deviations against the naive -- implementation. prop_stddev_update :: Property prop_stddev_update = forAll (choose (0, 6) >>= flip vectorOf (choose (0, 1))) $ \xs -> forAll (choose (0, 1)) $ \a -> forAll (choose (0, 1)) $ \b -> forAll (choose (1, 6) >>= flip vectorOf (choose (0, 1))) $ \ys -> let original = xs ++ [a] ++ ys modified = xs ++ [b] ++ ys with_update = getStatisticValue $ updateStatistics (getStdDevStatistics $ map SimpleNumber original) (SimpleNumber a, SimpleNumber b) direct = stdDev modified in counterexample ("Value computed by update " ++ show with_update ++ " differs too much from correct value " ++ show direct) (abs (with_update - direct) < 1e-10) testSuite "Utils/Statistics" [ 'prop_stddev_update ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/Utils/Time.hs000064400000000000000000000063671476477700300216710ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-| Unittests for time utilities -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.Utils.Time ( testUtils_Time ) where import Ganeti.Utils.Time (TimeDiff(TimeDiff), noTimeDiff, addToClockTime, diffClockTimes, clockTimeToString) import System.Time (ClockTime(TOD)) import Test.QuickCheck import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon prop_clockTimeToString :: Integer -> Integer -> Property prop_clockTimeToString ts pico = clockTimeToString (TOD ts pico) ==? show ts genPicoseconds :: Gen Integer genPicoseconds = choose (0, 999999999999) genTimeDiff :: Gen TimeDiff genTimeDiff = TimeDiff <$> arbitrary <*> genPicoseconds genClockTime :: Gen ClockTime genClockTime = TOD <$> choose (946681200, 2082754800) <*> genPicoseconds prop_addToClockTime_identity :: Property prop_addToClockTime_identity = forAll genClockTime addToClockTime_identity addToClockTime_identity :: ClockTime -> Property addToClockTime_identity a = addToClockTime noTimeDiff a ==? a {- | Verify our work-around for ghc bug #2519. Taking `diffClockTimes` form `System.Time`, this test fails with an exception. -} prop_timediffAdd :: Property prop_timediffAdd = forAll genClockTime $ \a -> forAll genClockTime $ \b -> forAll genClockTime $ \c -> timediffAdd a b c timediffAdd :: ClockTime -> ClockTime -> ClockTime -> Property timediffAdd a b c = let fwd = diffClockTimes a b back = diffClockTimes b a in addToClockTime fwd (addToClockTime back c) ==? c prop_timediffAddCommutative :: Property prop_timediffAddCommutative = forAll genTimeDiff $ \a -> forAll genTimeDiff $ \b -> forAll genClockTime $ \c -> timediffAddCommutative a b c timediffAddCommutative :: TimeDiff -> TimeDiff -> ClockTime -> Property timediffAddCommutative a b c = addToClockTime a (addToClockTime b c) ==? addToClockTime b (addToClockTime a c) testSuite "Utils/Time" [ 'prop_clockTimeToString , 'prop_addToClockTime_identity , 'prop_timediffAdd , 'prop_timediffAddCommutative ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/WConfd/000075500000000000000000000000001476477700300205035ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/Test/Ganeti/WConfd/Ssconf.hs000064400000000000000000000041461476477700300222770ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Unittests for Ssconf writing -} {- Copyright (C) 2015 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.WConfd.Ssconf (testWConfd_Ssconf) where import Test.QuickCheck import qualified Data.Map as M import qualified Data.Set as S import Test.Ganeti.Objects () import Test.Ganeti.TestHelper import Test.Ganeti.TestCommon import Ganeti.Objects (ConfigData) import qualified Ganeti.Ssconf as Ssconf import qualified Ganeti.WConfd.Ssconf as Ssconf -- * Ssconf construction tests hasAllKeys :: Ssconf.SSConf -> Property hasAllKeys ssc = counterexample "Missing SSConf key in the output" $ M.keysSet (Ssconf.getSSConf ssc) ==? S.fromList [minBound..maxBound] prop_mkSSConf_all_keys :: ConfigData -> Property prop_mkSSConf_all_keys = hasAllKeys . Ssconf.mkSSConf testSuite "WConfd/Ssconf" [ 'prop_mkSSConf_all_keys ] ganeti-3.1.0~rc2/test/hs/Test/Ganeti/WConfd/TempRes.hs000064400000000000000000000052471476477700300224260ustar00rootroot00000000000000{-# LANGUAGE TemplateHaskell #-} {-# OPTIONS_GHC -fno-warn-orphans #-} {-| Tests for temporary configuration resources allocation -} {- Copyright (C) 2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Test.Ganeti.WConfd.TempRes (testWConfd_TempRes) where import Test.QuickCheck import Test.Ganeti.Objects () import Test.Ganeti.TestCommon import Test.Ganeti.TestHelper import Test.Ganeti.Locking.Locks () -- the JSON ClientId instance import Test.Ganeti.Utils.MultiMap () import Ganeti.WConfd.TempRes -- * Instances instance Arbitrary IPv4ResAction where arbitrary = elements [minBound..maxBound] instance Arbitrary IPv4Reservation where arbitrary = IPv4Res <$> arbitrary <*> arbitrary <*> arbitrary instance (Arbitrary k, Ord k, Arbitrary v, Ord v) => Arbitrary (TempRes k v) where arbitrary = mkTempRes <$> arbitrary instance Arbitrary TempResState where arbitrary = TempResState <$> genMap arbitrary (genMap arbitrary arbitrary) <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary -- * Tests prop_IPv4Reservation_serialisation :: IPv4Reservation -> Property prop_IPv4Reservation_serialisation = testSerialisation prop_TempRes_serialisation :: TempRes Int Int -> Property prop_TempRes_serialisation = testSerialisation -- * The tests combined testSuite "WConfd/TempRes" [ 'prop_IPv4Reservation_serialisation , 'prop_TempRes_serialisation ] ganeti-3.1.0~rc2/test/hs/cli-tests-defs.sh000064400000000000000000000035421476477700300203630ustar00rootroot00000000000000# # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # This is an shell testing configuration fragment. HBINARY=${HBINARY:-./test/hs/hpc-htools} export TESTDATA_DIR=${TOP_SRCDIR:-.}/test/data/htools export PYTESTDATA_DIR=${TOP_SRCDIR:-.}/test/data hbal() { HTOOLS=hbal $HBINARY "$@" } hscan() { HTOOLS=hscan $HBINARY "$@" } hail() { HTOOLS=hail $HBINARY "$@" } hspace() { HTOOLS=hspace $HBINARY "$@" } hinfo() { HTOOLS=hinfo $HBINARY "$@" } hcheck() { HTOOLS=hinfo $HBINARY "$@" } hroller() { HTOOLS=hroller $HBINARY "$@" } ALL_ROLES="hbal hscan hail hspace hinfo hcheck hroller" ganeti-3.1.0~rc2/test/hs/hpc-htools.hs000077700000000000000000000000001476477700300226672../../src/htools.hsustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/hpc-mon-collector.hs000077700000000000000000000000001476477700300254012../../src/mon-collector.hsustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/htest.hs000064400000000000000000000131711476477700300166630ustar00rootroot00000000000000{-| Unittest runner for ganeti-htools. -} {- Copyright (C) 2009, 2011, 2012, 2013 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} module Main(main) where import Data.Monoid (mappend) import Test.Framework import System.Environment (getArgs) import System.Log.Logger import Test.AutoConf import Test.Ganeti.TestImports () import Test.Ganeti.Attoparsec import Test.Ganeti.BasicTypes import Test.Ganeti.Common import Test.Ganeti.Constants import Test.Ganeti.Confd.Utils import Test.Ganeti.Confd.Types import Test.Ganeti.Daemon import Test.Ganeti.Errors import Test.Ganeti.HTools.Backend.MonD import Test.Ganeti.HTools.Backend.Simu import Test.Ganeti.HTools.Backend.Text import Test.Ganeti.HTools.CLI import Test.Ganeti.HTools.Cluster import Test.Ganeti.HTools.Container import Test.Ganeti.HTools.Graph import Test.Ganeti.HTools.Instance import Test.Ganeti.HTools.Loader import Test.Ganeti.HTools.Node import Test.Ganeti.HTools.PeerMap import Test.Ganeti.HTools.Types import Test.Ganeti.Hypervisor.Xen.XlParser import Test.Ganeti.JSON import Test.Ganeti.Jobs import Test.Ganeti.JQueue import Test.Ganeti.JQScheduler import Test.Ganeti.Kvmd import Test.Ganeti.Locking.Allocation import Test.Ganeti.Locking.Locks import Test.Ganeti.Locking.Waiting import Test.Ganeti.Luxi import Test.Ganeti.Network import Test.Ganeti.Objects import Test.Ganeti.Objects.BitArray import Test.Ganeti.OpCodes import Test.Ganeti.PyValue import Test.Ganeti.Query.Aliases import Test.Ganeti.Query.Filter import Test.Ganeti.Query.Instance import Test.Ganeti.Query.Language import Test.Ganeti.Query.Network import Test.Ganeti.Query.Query import Test.Ganeti.Rpc import Test.Ganeti.Runtime import Test.Ganeti.SlotMap import Test.Ganeti.Ssconf import Test.Ganeti.Storage.Diskstats.Parser import Test.Ganeti.Storage.Drbd.Parser import Test.Ganeti.Storage.Drbd.Types import Test.Ganeti.Storage.Lvm.LVParser import Test.Ganeti.THH import Test.Ganeti.THH.Types import Test.Ganeti.Types import Test.Ganeti.Utils import Test.Ganeti.Utils.Time import Test.Ganeti.Utils.MultiMap import Test.Ganeti.Utils.Statistics import Test.Ganeti.WConfd.Ssconf import Test.Ganeti.WConfd.TempRes -- | Our default test options, overring the built-in test-framework -- ones (but not the supplied command line parameters). defOpts :: TestOptions defOpts = TestOptions { topt_seed = Nothing , topt_maximum_generated_tests = Just 500 , topt_maximum_unsuitable_generated_tests = Just 5000 , topt_maximum_test_size = Nothing , topt_maximum_test_depth = Nothing , topt_timeout = Nothing } -- | All our defined tests. allTests :: [Test] allTests = [ testAutoConf , testBasicTypes , testAttoparsec , testCommon , testConstants , testConfd_Types , testConfd_Utils , testDaemon , testBlock_Diskstats_Parser , testBlock_Drbd_Parser , testBlock_Drbd_Types , testErrors , testHTools_Backend_MonD , testHTools_Backend_Simu , testHTools_Backend_Text , testHTools_CLI , testHTools_Cluster , testHTools_Container , testHTools_Graph , testHTools_Instance , testHTools_Loader , testHTools_Node , testHTools_PeerMap , testHTools_Types , testHypervisor_Xen_XlParser , testJSON , testJobs , testJQueue , testJQScheduler , testKvmd , testLocking_Allocation , testLocking_Locks , testLocking_Waiting , testLuxi , testNetwork , testObjects , testObjects_BitArray , testOpCodes , testPyValue , testQuery_Aliases , testQuery_Filter , testQuery_Instance , testQuery_Language , testQuery_Network , testQuery_Query , testRpc , testRuntime , testSlotMap , testSsconf , testStorage_Lvm_LVParser , testTHH , testTHH_Types , testTypes , testUtils , testUtils_Time , testUtils_MultiMap , testUtils_Statistics , testWConfd_Ssconf , testWConfd_TempRes ] -- | Main function. Note we don't use defaultMain since we want to -- control explicitly our test sizes (and override the default). main :: IO () main = do ropts <- getArgs >>= interpretArgsOrExit let opts = maybe defOpts (defOpts `mappend`) $ ropt_test_options ropts -- silence the logging system, so that tests can execute I/O actions -- which create logs without polluting stderr -- FIXME: improve this by allowing tests to use logging if needed updateGlobalLogger rootLoggerName (setLevel EMERGENCY) defaultMainWithOpts allTests (ropts { ropt_test_options = Just opts }) ganeti-3.1.0~rc2/test/hs/live-test.sh000075500000000000000000000130721476477700300174530ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2009, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # This is a live-testing script for most/all of the htools # programs. It needs either to run on a live cluster or access to a # cluster via ssh and an exported LUXI interface (via socat, for # example). The cluster must not be empty (otherwise the hail relocate # test will fail). # Use: if running on a cluster master, just running it should be # enough. If running remotely, set env vars as follows: LUXI to the # export unix socket, RAPI to the cluster IP, CLUSTER to the command # used to login on the cluster (e.g. CLUSTER="ssh root@cluster"). Note # that when run against a multi-group cluster, the GROUP variable # should be set to one of the groups (some operations work only on one # group) set -e : ${RAPI:=localhost} GROUP=${GROUP:+-G $GROUP} . $(dirname $0)/cli-tests-defs.sh T=`mktemp -d` trap 'rm -rf $T' EXIT echo Using $T as temporary dir echo Checking command line for prog in $ALL_ROLES; do $prog --version $prog --help >/dev/null ! $prog --no-such-option 2>/dev/null done echo Testing hscan/rapi hscan -d$T $RAPI -p echo Testing hscan/luxi hscan -d$T -L$LUXI -p echo Comparing hscan results... diff -u $T/$RAPI.data $T/LOCAL.data FN=$($CLUSTER head -n1 /var/lib/ganeti/ssconf_node_list) FI=$($CLUSTER head -n1 /var/lib/ganeti/ssconf_instance_list) echo Testing hbal/luxi hbal -L$LUXI $GROUP -p --print-instances -C$T/hbal-luxi-cmds.sh bash -n $T/hbal-luxi-cmds.sh echo Testing hbal/rapi hbal -m$RAPI $GROUP -p --print-instances -C$T/hbal-rapi-cmds.sh bash -n $T/hbal-rapi-cmds.sh echo Testing hbal/text hbal -t$T/$RAPI.data $GROUP -p --print-instances -C$T/hbal-text-cmds.sh bash -n $T/hbal-text-cmds.sh echo Comparing hbal results diff -u $T/hbal-luxi-cmds.sh $T/hbal-rapi-cmds.sh diff -u $T/hbal-luxi-cmds.sh $T/hbal-text-cmds.sh echo Testing hbal/text with evacuation mode hbal -t$T/$RAPI.data $GROUP -E echo Testing hbal/text with no disk moves hbal -t$T/$RAPI.data $GROUP --no-disk-moves echo Testing hbal/text with no instance moves hbal -t$T/$RAPI.data $GROUP --no-instance-moves echo Testing hbal/text with offline node mode hbal -t$T/$RAPI.data $GROUP -O$FN echo Testing hbal/text with utilization data echo "$FI 2 2 2 2" > $T/util.data hbal -t$T/$RAPI.data $GROUP -U $T/util.data echo Testing hbal/text with bad utilization data echo "$FI 2 a 3b" > $T/util.data ! hbal -t$T/$RAPI.data $GROUP -U $T/util.data echo Testing hbal/text with instance exclusion/selection hbal -t$T/$RAPI.data $GROUP --exclude-instances=$FI hbal -t$T/$RAPI.data $GROUP --select-instances=$FI ! hbal -t$T/$RAPI.data --exclude-instances=no_such_instance ! hbal -t$T/$RAPI.data --select-instances=no_such_instance echo Testing hbal/text with tag exclusion hbal -t $T/$RAPI.data $GROUP --exclusion-tags=no_such_tag echo Testing hbal multiple backend failure ! hbal -t $T/$RAPI.data -L$LUXI echo Testing hbal no backend failure ! hbal echo Getting data files for hail for dtemplate in plain drbd; do $CLUSTER gnt-debug allocator --dir in --mode allocate --mem 128m \ --disks 128m -t $dtemplate -o no_such_os no_such_instance \ > $T/h-alloc-$dtemplate.json done $CLUSTER gnt-debug allocator --dir in --mode relocate \ -o no_such_os $FI > $T/h-reloc.json $CLUSTER gnt-debug allocator --dir in --mode multi-evacuate \ $FN > $T/h-evacuate.json for dtemplate in plain drbd; do echo Testing hail/allocate-$dtemplate hail $T/h-alloc-$dtemplate.json done echo Testing hail/relocate for instance $FI hail $T/h-reloc.json echo Testing hail/evacuate for node $FN hail $T/h-evacuate.json HOUT="$T/hspace.out" check_hspace_out() { set -u set -e source "$HOUT" echo ALLOC_INSTANCES=$HTS_ALLOC_INSTANCES echo TSPEC=$HTS_TSPEC echo OK=$HTS_OK } TIER="--tiered 102400,8192,2" SIMU="--simu=preferred,10,6835937,32768,4" echo Testing hspace/luxi hspace -L$LUXI $TIER -v > $HOUT ( check_hspace_out ) || exit 1 echo Testing hspace/rapi hspace -m$RAPI $TIER -v > $HOUT ( check_hspace_out ) || exit 1 echo Testing hspace/text hspace -t$T/$RAPI.data $TIER -v > $HOUT ( check_hspace_out ) || exit 1 echo Testing hspace/simu # ~6T disk space, 32G ram, 4 VCPUs hspace $SIMU $TIER -v > $HOUT ( check_hspace_out ) || exit 1 # Wrong tiered spec input ! hspace $SIMU --tiered 1,2,3x ! hspace $SIMU --tiered 1,2,x ! hspace $SIMU --tiered 1,2 # Wrong simu spec ! hspace --simu=1,2,x echo All OK ganeti-3.1.0~rc2/test/hs/offline-test.sh000075500000000000000000000107331476477700300201370ustar00rootroot00000000000000#!/bin/bash # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # This is an offline testing script for most/all of the htools # programs, checking basic command line functionality. # Optional argument that specifies the test files to run. If not # specified, then all tests are run. # # For example, a value of 'balancing' runs the file # 'shelltests/htools-balancing.test'. Multiple files can be specified # using shell notation, for example, '{balancing,basic}'. TESTS=${1:-*} set -e set -o pipefail . $(dirname $0)/cli-tests-defs.sh echo Running offline htools tests export T=`mktemp -d` trap 'rm -rf $T' EXIT trap 'echo FAIL to build test files' ERR echo Using $T as temporary dir echo -n Generating hspace simulation data for hinfo and hbal... # this cluster spec should be fine ./test/hs/hspace --simu p,4,8T,64g,16 -S $T/simu-onegroup \ --disk-template drbd -l 8 -v -v -v >/dev/null 2>&1 echo OK echo -n Generating hinfo and hbal test files for multi-group... ./test/hs/hspace --simu p,4,8T,64g,16 --simu p,4,8T,64g,16 \ -S $T/simu-twogroups --disk-template drbd -l 8 >/dev/null 2>&1 echo OK echo -n Generating test files for rebalancing... # we generate a cluster with two node groups, one with unallocable # policy, then we change all nodes from this group to the allocable # one, and we check for rebalancing FROOT="$T/simu-rebal-orig" ./test/hs/hspace --simu u,4,8T,64g,16 --simu p,4,8T,64g,16 \ -S $FROOT --disk-template drbd -l 8 >/dev/null 2>&1 for suffix in standard tiered; do RELOC="$T/simu-rebal-merged.$suffix" # this relocates the nodes sed -re 's/^(node-.*|fake-uuid-)-02(|.*)/\1-01\2/' \ < $FROOT.$suffix > $RELOC done export BACKEND_BAL_STD="-t$T/simu-rebal-merged.standard" export BACKEND_BAL_TIER="-t$T/simu-rebal-merged.tiered" echo OK # For various tests export BACKEND_DYNU="-t $T/simu-onegroup.standard" export BACKEND_EXCL="-t $T/simu-onegroup.standard" echo -n Generating data files for IAllocator checks... for evac_mode in primary-only secondary-only all; do sed -e 's/"evac_mode": "all"/"evac_mode": "'${evac_mode}'"/' \ -e 's/"spindles": [0-9]\+,//' \ < $TESTDATA_DIR/hail-node-evac.json \ > $T/hail-node-evac.json.$evac_mode done for bf in hail-alloc-drbd hail-alloc-invalid-twodisks hail-alloc-twodisks \ hail-change-group hail-node-evac hail-reloc-drbd hail-alloc-spindles; do f=$bf.json sed -e 's/"exclusive_storage": false/"exclusive_storage": true/' \ < $TESTDATA_DIR/$f > $T/$f.excl-stor sed -e 's/"exclusive_storage": false/"exclusive_storage": true/' \ -e 's/"spindles": [0-9]\+,//' \ < $TESTDATA_DIR/$f > $T/$f.fail-excl-stor done echo OK echo -n Checking file-based RAPI... mkdir -p $T/hscan export RAPI_URL="file://$TESTDATA_DIR/rapi" ./test/hs/hscan -d $T/hscan/ -p -v -v $RAPI_URL >/dev/null 2>&1 # check that we file parsing is correct, i.e. hscan saves correct text # files, and is idempotent (rapi+text == rapi); more is tested in # shelltest later RAPI_TXT="$(ls $T/hscan/*.data|head -n1)" ./test/hs/hinfo -p --print-instances -m $RAPI_URL > $T/hscan/direct.hinfo 2>&1 ./test/hs/hinfo -p --print-instances -t $RAPI_TXT > $T/hscan/fromtext.hinfo 2>&1 echo OK echo Running shelltest... shelltest $SHELLTESTARGS \ ${TOP_SRCDIR:-.}/test/hs/shelltests/htools-$TESTS.test ganeti-3.1.0~rc2/test/hs/shelltests/000075500000000000000000000000001476477700300173675ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/hs/shelltests/htools-balancing.test000064400000000000000000000124271476477700300235220ustar00rootroot00000000000000### std tests # test basic parsing ./test/hs/hinfo -v -v -p --print-instances $BACKEND_BAL_STD >>>= 0 ./test/hs/hbal -v -v -v -p --print-instances $BACKEND_BAL_STD -G group-01 >>> !/(Nothing to do, exiting|No solution found)/ >>>2 !/(Nothing to do, exiting|No solution found)/ >>>= 0 # test command output ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C -S $T/simu-rebal.standard >>> /gnt-instance (failover|migrate|replace-disks)/ >>>= 0 # test that correct priorities are accepted ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C -S $T/simu-rebal.standard --prio low >>> /gnt-instance (failover|migrate|replace-disks)/ >>>= 0 # test that hbal won't execute rebalances when using the text backend ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -X >>>2 Error: hbal: Execution of commands possible only on LUXI >>>= !0 # test that hbal won't execute any moves if we request an absurdly-high # minimum-improvement ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C --min-gain 10000 --min-gain-limit 10000 >>>/No solution found/ >>>= 0 # test saving commands ./test/hs/hbal $BACKEND_BAL_STD -G group-01 -C$T/rebal-cmds.standard >>>= 0 # and now check the file (depends on previous test) cat $T/rebal-cmds.standard >>> /gnt-instance (failover|migrate|replace-disks)/ >>>= 0 # state saved before rebalancing should be identical; depends on the # previous test diff -u $T/simu-rebal-merged.standard $T/simu-rebal.standard.original >>> >>>= 0 # no double rebalance; depends on previous test ./test/hs/hbal -t $T/simu-rebal.standard.balanced -G group-01 >>> /(Nothing to do, exiting|No solution found)/ >>>= 0 # hcheck sees no reason to rebalance after rebalancing was already done ./test/hs/hcheck -t$T/simu-rebal.standard.balanced --machine-readable >>> /HCHECK_INIT_CLUSTER_NEED_REBALANCE=0/ >>>= 0 ### now tiered tests # test basic parsing ./test/hs/hinfo -v -v -p --print-instances $BACKEND_BAL_TIER >>>= 0 ./test/hs/hbal -v -v -v -p --print-instances $BACKEND_BAL_TIER -G group-01 >>> !/(Nothing to do, exiting|No solution found)/ >>>2 !/(Nothing to do, exiting|No solution found)/ >>>= 0 # test command output ./test/hs/hbal $BACKEND_BAL_TIER -G group-01 -C -S $T/simu-rebal.tiered >>> /gnt-instance (failover|migrate|replace-disks)/ >>>= 0 # test saving commands ./test/hs/hbal $BACKEND_BAL_TIER -G group-01 -C$T/rebal-cmds.tiered >>>= 0 # and now check the file (depends on previous test) cat $T/rebal-cmds.tiered >>> /gnt-instance (failover|migrate|replace-disks)/ >>>= 0 # state saved before rebalancing should be identical; depends on the # previous test diff -u $T/simu-rebal-merged.tiered $T/simu-rebal.tiered.original >>> >>>= 0 # no double rebalance; depends on previous test ./test/hs/hbal -t $T/simu-rebal.tiered.balanced -G group-01 >>> /(Nothing to do, exiting|No solution found)/ >>>= 0 ### now some other custom tests # n+1 bad instances are reported as such ./test/hs/hbal -t$TESTDATA_DIR/n1-failure.data -G group-01 >>>/Initial check done: 4 bad nodes, 8 bad instances./ >>>=0 # same test again, different message check (shelltest can't test multiple # messages via regexp ./test/hs/hbal -t$TESTDATA_DIR/n1-failure.data -G group-01 >>>/Cluster is not N\+1 happy, continuing but no guarantee that the cluster will end N\+1 happy./ >>>2 >>>=0 # and hcheck should report this as needs rebalancing ./test/hs/hcheck -t$TESTDATA_DIR/n1-failure.data >>>/Cluster needs rebalancing./ >>>= 1 # ... unless we request no-simulation mode ./test/hs/hcheck -t$TESTDATA_DIR/n1-failure.data --no-simulation >>>/Running in no-simulation mode./ >>>= 0 # and a clean cluster should be reported as such ./test/hs/hcheck $BACKEND_BAL_STD >>>/No need to rebalance cluster, no problems found./ >>>= 0 # ... and even one with non-zero score ./test/hs/hcheck -t $TESTDATA_DIR/clean-nonzero-score.data >>>/No need to rebalance cluster, no problems found./ >>>= 0 # hbal should work on empty groups as well ./test/hs/hbal -t$TESTDATA_DIR/n1-failure.data -G group-02 >>>/Group size 0 nodes, 0 instances/ >>>= 0 # By default, hbal should assume equal, non-zero utilisation ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data >>>/Solution length=1/ >>>=0 # ...but the --ignore-dynu option should be honored ./test/hs/hbal -t$TESTDATA_DIR/hbal-dyn.data --ignore-dynu >>>/Cluster is already well balanced/ >>>=0 # Test CPU speed is taken into account ./test/hs/hbal -t$TESTDATA_DIR/hbal-cpu-speed.data --ignore-dynu >>>/inst[12] node-slow:node-fast => node-fast:node-slow/ >>>=0 # By default, hbal should not take any action in an overloaded cluster ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors.data >>>/Solution length=0/ >>>=0 ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors2.data >>>/Solution length=0/ >>>=0 # ...and suggest to the user to weaken movement restrictions; ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors.data >>>/consider using the --ignore-soft-errors option/ >>>=0 ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors2.data >>>/consider using the --ignore-soft-errors option/ >>>=0 # allowing soft errors, an improvement should be found. ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors.data --ignore-soft-errors >>>/Solution length=[1-9]/ >>>=0 ./test/hs/hbal -t$TESTDATA_DIR/hbal-soft-errors2.data --ignore-soft-errors >>>/Solution length=[1-9]/ >>>=0 # forthcoming instances can be balanced as well ./test/hs/hbal -t$TESTDATA_DIR/hbal-forth.data >>>/Solution length=[1-9]/ >>>=0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-basic.test000064400000000000000000000013631476477700300226620ustar00rootroot00000000000000# help/version tests ./test/hs/hail --version >>>= 0 ./test/hs/hail --help >>>= 0 ./test/hs/hail --help-completion >>>= 0 ./test/hs/hbal --version >>>= 0 ./test/hs/hbal --help >>>= 0 ./test/hs/hbal --help-completion >>>= 0 ./test/hs/hspace --version >>>= 0 ./test/hs/hspace --help >>>= 0 ./test/hs/hspace --help-completion >>>= 0 ./test/hs/hscan --version >>>= 0 ./test/hs/hscan --help >>>= 0 ./test/hs/hscan --help-completion >>>= 0 ./test/hs/hinfo --version >>>= 0 ./test/hs/hinfo --help >>>= 0 ./test/hs/hinfo --help-completion >>>= 0 ./test/hs/hcheck --version >>>= 0 ./test/hs/hcheck --help >>>= 0 ./test/hs/hcheck --help-completion >>>= 0 ./test/hs/hroller --version >>>= 0 ./test/hs/hroller --help >>>= 0 ./test/hs/hroller --help-completion >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-dynutil.test000064400000000000000000000011311476477700300232620ustar00rootroot00000000000000echo a > $T/dynu; ./test/hs/hbal -U $T/dynu $BACKEND_DYNU >>>2 /Cannot parse line/ >>>= !0 echo a b c d e f g h > $T/dynu; ./test/hs/hbal -U $T/dynu $BACKEND_DYNU >>>2 /Cannot parse line/ >>>= !0 echo inst cpu mem dsk net >$T/dynu; ./test/hs/hbal -U $T/dynu $BACKEND_DYNU >>>2 /cannot parse string '(cpu|mem|dsk|net)'/ >>>= !0 # unknown instances are currently just ignored echo no-such-inst 2 2 2 2 > $T/dynu; ./test/hs/hbal -U $T/dynu $BACKEND_DYNU >>>= 0 # new-0 is the name of the first instance allocated by hspace echo new-0 2 2 2 2 > $T/dynu; ./test/hs/hbal -U $T/dynu $BACKEND_DYNU >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-excl.test000064400000000000000000000007251476477700300225350ustar00rootroot00000000000000./test/hs/hbal $BACKEND_EXCL --exclude-instances no-such-instance >>>2 /Unknown instance/ >>>= !0 ./test/hs/hbal $BACKEND_EXCL --select-instances no-such-instances >>>2 /Unknown instance/ >>>= !0 ./test/hs/hbal $BACKEND_EXCL --exclude-instances new-0 --select-instances new-1 >>>= 0 # Test exclusion tags too (both from the command line and cluster tags). ./test/hs/hbal -t $TESTDATA_DIR/hbal-excl-tags.data --exclusion-tags test >>> /Cluster score improved/ >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-hail.test000064400000000000000000000205101476477700300225110ustar00rootroot00000000000000# test that on invalid files it can't parse the request ./test/hs/hail /dev/null >>>2 /Invalid JSON/ >>>= !0 # another invalid example echo '[]' | ./test/hs/hail - >>>2 /Unable to read JSObject/ >>>= !0 # empty dict echo '{}' | ./test/hs/hail - >>>2 /key 'request' not found/ >>>= !0 echo '{"request": 0}' | ./test/hs/hail - >>>2 /key 'request'/ >>>= !0 ./test/hs/hail $TESTDATA_DIR/hail-invalid-reloc.json >>>2 /key 'name': Unable to read String/ >>>= !0 # and now start the real tests ./test/hs/hail $TESTDATA_DIR/hail-alloc-drbd.json >>> /"success":true,.*,"result":\["node2","node1"\]/ >>>= 0 ./test/hs/hail $TESTDATA_DIR/hail-reloc-drbd.json >>> /"success":true,.*,"result":\["node1"\]/ >>>= 0 ./test/hs/hail $TESTDATA_DIR/hail-node-evac.json >>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/ >>>= 0 ./test/hs/hail $TESTDATA_DIR/hail-change-group.json >>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/ >>>= 0 # check that hail correctly applies the disk policy on a per-disk basis ./test/hs/hail $TESTDATA_DIR/hail-alloc-twodisks.json --no-capacity-checks >>> /"success":true,.*,"result":\["node1"\]/ >>>= 0 ./test/hs/hail $TESTDATA_DIR/hail-alloc-invalid-twodisks.json >>> /"success":false,.*FailDisk: 1/ >>>= 0 # check that hail honors network requirements ./test/hs/hail $TESTDATA_DIR/hail-alloc-restricted-network.json >>> /"success":true,"info":"Request successful: Selected group: Group 1.*/ >>>= 0 # check that hail fails if no nodegroup can meet network and disk template requirements ./test/hs/hail $TESTDATA_DIR/hail-alloc-invalid-network.json >>> /"success":false,/ >>>= 0 # check that hail succeeds with the same test data, but with the network restrictions removed cat $TESTDATA_DIR/hail-alloc-invalid-network.json | grep -v -e '"network":"uuid-net-1-."' | ./test/hs/hail - --no-capacity-checks >>> /"success":true,"info":"Request successful: Selected group: Group 2.*/ >>>= 0 # Run some of the tests above, with exclusive storage enabled ./test/hs/hail $T/hail-alloc-drbd.json.excl-stor >>> /"success":true,.*,"result":\["node.","node."\]/ >>>= 0 ./test/hs/hail $T/hail-reloc-drbd.json.excl-stor >>> /"success":true,.*,"result":\["node1"\]/ >>>= 0 ./test/hs/hail $T/hail-node-evac.json.excl-stor >>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/ >>>= 0 ./test/hs/hail $T/hail-change-group.json.excl-stor >>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/ >>>= 0 ./test/hs/hail $T/hail-alloc-twodisks.json.excl-stor >>> /"success":true,.*,"result":\["node1"\]/ >>>= 0 ./test/hs/hail $T/hail-alloc-invalid-twodisks.json.excl-stor >>> /"success":false,.*FailDisk: 1"/ >>>= 0 # Same tests with exclusive storage enabled, but no spindles info in instances ./test/hs/hail $T/hail-alloc-drbd.json.fail-excl-stor >>> /"success":false,.*FailSpindles: 12"/ >>>= 0 ./test/hs/hail $T/hail-reloc-drbd.json.fail-excl-stor >>> /"success":false,.*FailSpindles/ >>>= 0 ./test/hs/hail $T/hail-node-evac.json.fail-excl-stor >>> /"success":true,"info":"Request successful: 1 instances failed to move and 0 were moved successfully",.*FailSpindles/ >>>= 0 ./test/hs/hail $T/hail-change-group.json.fail-excl-stor >>> /"success":true,"info":"Request successful: 1 instances failed to move and 0 were moved successfully",.*FailSpindles: 2"/ >>>= 0 ./test/hs/hail $T/hail-alloc-twodisks.json.fail-excl-stor >>> /"success":false,.*FailSpindles: 1"/ >>>= 0 # check that hail correctly parses admin state ./test/hs/hail -v -v $TESTDATA_DIR/hail-alloc-drbd.json >>>2 /runSt = StatusDown/ >>>=0 # check that hail can use the simu backend ./test/hs/hail --simu p,8,8T,16g,16 $TESTDATA_DIR/hail-alloc-drbd.json >>> /"success":true,/ >>>= 0 # check that hail can use the text backend ./test/hs/hail -t $T/simu-rebal-merged.standard $TESTDATA_DIR/hail-alloc-drbd.json >>> /"success":true,/ >>>= 0 # check that hail can use the simu backend ./test/hs/hail -t $T/simu-rebal-merged.standard $TESTDATA_DIR/hail-alloc-drbd.json >>> /"success":true,/ >>>= 0 # check that hail pre/post saved state differs after allocation ./test/hs/hail -v -v -v -p $TESTDATA_DIR/hail-alloc-drbd.json -S $T/hail-alloc >/dev/null 2>&1 && ! diff -q $T/hail-alloc.pre-ialloc $T/hail-alloc.post-ialloc >>> /Files .* and .* differ/ >>>= 0 # check that hail pre/post saved state differs after relocation ./test/hs/hail -v -v -v -p $TESTDATA_DIR/hail-reloc-drbd.json -S $T/hail-reloc >/dev/null 2>&1 && ! diff -q $T/hail-reloc.pre-ialloc $T/hail-reloc.post-ialloc >>> /Files .* and .* differ/ >>>= 0 ./test/hs/hail $TESTDATA_DIR/hail-reloc-drbd-crowded.json >>> /"success":false,/ >>>= 0 ./test/hs/hail --ignore-soft-errors $TESTDATA_DIR/hail-reloc-drbd-crowded.json >>> /"success":true,/ >>>= 0 # evac tests ./test/hs/hail $T/hail-node-evac.json.primary-only >>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/ >>>= 0 ./test/hs/hail $T/hail-node-evac.json.secondary-only >>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/ >>>= 0 ./test/hs/hail $T/hail-node-evac.json.all >>> /"success":true,"info":"Request successful: 0 instances failed to move and 1 were moved successfully"/ >>>= 0 # Check interaction between policies and spindles ./test/hs/hail $TESTDATA_DIR/hail-alloc-spindles.json --no-capacity-checks >>> /"success":true,"info":"Request successful: Selected group: group2,.*FailSpindles: 2,.*"result":\["node4"\]/ >>>= 0 ./test/hs/hail $T/hail-alloc-spindles.json.excl-stor >>> /"success":true,"info":"Request successful: Selected group: group1,.*FailSpindles: 2",.*"result":\["node1"\]/ >>>= 0 # Check that --ignore-soft-errors works and ignores tag errors ./test/hs/hail $TESTDATA_DIR/hail-alloc-plain-tags.json >>> /"success":false,.*FailTags: 3/ >>>= 0 ./test/hs/hail --ignore-soft-errors $TESTDATA_DIR/hail-alloc-plain-tags.json >>> /"success":true/ >>>= 0 # Verify dedicated allocation ./test/hs/hail $TESTDATA_DIR/hail-alloc-dedicated-1.json >>> /"success":true.*"result":\["node2-quarter"\]/ >>>= 0 # Verify dedicated multi-allocation ./test/hs/hail $TESTDATA_DIR/hail-multialloc-dedicated.json >>> /"success":true.*"result":\[\[\[.*\["node2-quarter"\]/ >>>= 0 # Verify that global N+1 redundancy is honored, unless requested not to ./test/hs/hail -t $TESTDATA_DIR/shared-n1-restriction.data $TESTDATA_DIR/hail-alloc-ext.json >>> /"success":true.*"result":\["node[BCD]"\]/ >>>= 0 ./test/hs/hail -t $TESTDATA_DIR/shared-n1-restriction.data $TESTDATA_DIR/hail-alloc-ext.json --no-capacity-checks >>> /"success":true.*"result":\["nodeA"\]/ >>>= 0 ./test/hs/hail -t $TESTDATA_DIR/plain-n1-restriction.data $TESTDATA_DIR/hail-alloc-ext.json >>> /"success":true.*"result":\["node[BCD]"\]/ >>>= 0 ./test/hs/hail -t $TESTDATA_DIR/plain-n1-restriction.data $TESTDATA_DIR/hail-alloc-ext.json --no-capacity-checks >>> /"success":true.*"result":\["nodeA"\]/ >>>= 0 # Verify that allocation restrications are honored ./test/hs/hail -t $TESTDATA_DIR/partly-used.data $TESTDATA_DIR/hail-alloc-drbd.json >>> /successes 20,.*"result":\["node-0[45]","node-0[45]"\]/ >>>= 0 ./test/hs/hail -t $TESTDATA_DIR/partly-used.data --restrict-allocation-to node-01,node-02,node-03 $TESTDATA_DIR/hail-alloc-drbd.json >>> /successes 6,.*"result":\["node-0[123]","node-0[123]"\]/ >>>= 0 ./test/hs/hail -t $TESTDATA_DIR/partly-used.data $TESTDATA_DIR/hail-alloc-drbd-restricted.json >>> /successes 6,.*"result":\["node-0[123]","node-0[123]"\]/ >>>= 0 ./test/hs/hail -t $TESTDATA_DIR/partly-used.data --restrict-allocation-to node-01,node-02 $TESTDATA_DIR/hail-alloc-drbd-restricted.json >>> /successes 2,.*"result":\["node-0[12]","node-0[12]"\]/ >>>= 0 ./test/hs/hail -t $TESTDATA_DIR/partly-used.data --restrict-allocation-to node-03,node-04 $TESTDATA_DIR/hail-alloc-drbd-restricted.json >>> /successes 2,.*"result":\["node-0[34]","node-0[34]"\]/ >>>= 0 # Verify allocate-secondary ./test/hs/hail $TESTDATA_DIR/hail-alloc-secondary.json >>> /successes 2, failures 0.*"result":"node-2-2"/ >>>= 0 # Check that hail account location tags ./test/hs/hail $TESTDATA_DIR/hail-alloc-nlocation.json >>> /"success":true,.*,"result":\["node3","node2"\]/ >>>= 0 # Desired location test ./test/hs/hail $TESTDATA_DIR/hail-alloc-desired-location.json >>> /"success":true,.*,"result":\["node1"\]/ >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-hbal-evac.test000064400000000000000000000005601476477700300234210ustar00rootroot00000000000000./test/hs/hbal -t $TESTDATA_DIR/hbal-evac.data >>>/inst-32. node-3:node-2 => node-2:node-1.* (.| )*Solution length=4/ >>>= 0 ./test/hs/hbal --evac-mode -t $TESTDATA_DIR/hbal-evac.data >>>/a=f r:node-1 f (.| )*Solution length=3/ >>>= 0 ./test/hs/hbal --evac-mode --restricted-migration -t $TESTDATA_DIR/hbal-evac.data >>>/a=f r:node-1 (.| )*Solution length=3/ >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-hbal.test000064400000000000000000000040401476477700300225020ustar00rootroot00000000000000./test/hs/hbal --print-nodes=name,pcnt -t $TESTDATA_DIR/hbal-migration-1.data >>>2/Final cluster status: Name pcnt node-01 3 node-02 0 node-03 3 node-04 0/ >>>= 0 ./test/hs/hbal --print-nodes=name,pcnt -t $TESTDATA_DIR/hbal-migration-2.data >>>2/Final cluster status: Name pcnt node-01 2 node-02 2 node-03 2 node-04 0/ >>>= 0 ./test/hs/hbal --print-nodes=name,pcnt -t $TESTDATA_DIR/hbal-migration-3.data >>>2/Final cluster status: Name pcnt node-01 2 node-02 2 node-03 2 node-04 0/ >>>= 0 ./test/hs/hbal --print-nodes=name,pcnt -t $TESTDATA_DIR/hbal-desiredlocation-1.data >>>2/Final cluster status: Name pcnt node-01 0 node-02 1/ >>>= 0 ./test/hs/hbal --print-nodes=name,pcnt -t $TESTDATA_DIR/hbal-desiredlocation-2.data >>>2/Final cluster status: Name pcnt node-01 1 node-02 1 node-03 0/ >>>= 0 ./test/hs/hbal --print-nodes=name,pcnt -t $TESTDATA_DIR/hbal-desiredlocation-3.data >>>2/Final cluster status: Name pcnt node-01 0 node-02 0 node-03 1/ >>>= 0 ./test/hs/hbal --print-nodes=name,pcnt -t $TESTDATA_DIR/hbal-desiredlocation-4.data >>>2/Final cluster status: Name pcnt node-01 0 node-02 1/ >>>= 0 ./test/hs/hbal -t $TESTDATA_DIR/hbal-location-1.data >>>/Solution length=[1-9]/ >>>= 0 ./test/hs/hbal --print-nodes=name,pcnt -t $TESTDATA_DIR/hbal-location-exclusion.data >>>2/Final cluster status: Name pcnt node-1 0 node-2 1 node-3 1 node-4 0/ >>>= 0 ./test/hs/hbal -t $TESTDATA_DIR/shared-n1-failure.data --ignore-dynu >>>/Cluster is already well balanced/ >>>= 0 ./test/hs/hbal -t $TESTDATA_DIR/shared-n1-failure.data --ignore-dynu -pname,pcnt -O nodeA >>>2/Final cluster status: Name pcnt nodeA 0 nodeB 4 nodeC 2 nodeD 4/ >>>= 0 ./test/hs/hbal -t $TESTDATA_DIR/shared-n1-failure.data --ignore-dynu -O nodeC >>>/No solution found/ >>>= 0 ./test/hs/hbal --print-nodes=name,pcnt -t $TESTDATA_DIR/hbal-location-2.data >>>2/Final cluster status: Name pcnt node-01 1 node-02 0 node-03 1/ >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-hcheck.test000064400000000000000000000013161476477700300230240ustar00rootroot00000000000000./test/hs/hcheck -t $TESTDATA_DIR/shared-n1-failure.data >>>/Nodes not directly evacuateable: 1/ >>>= 1 ./test/hs/hcheck -t $TESTDATA_DIR/shared-n1-failure.data --machine-readable >>>/HCHECK_INIT_CLUSTER_GN1_FAIL=1/ >>>= 1 ./test/hs/hcheck -t $TESTDATA_DIR/shared-n1-failure.data --machine-readable >>>/HCHECK_INIT_CLUSTER_NEED_REBALANCE=1/ >>>= 1 ./test/hs/hcheck -t $TESTDATA_DIR/shared-n1-failure.data --machine-readable --no-capacity-checks >>>/HCHECK_INIT_CLUSTER_NEED_REBALANCE=0/ >>>= 0 ./test/hs/hcheck -t $TESTDATA_DIR/shared-n1-failure.data >>>/Cluster is not healthy: True/ >>>= 1 ./test/hs/hcheck -t $TESTDATA_DIR/shared-n1-failure.data --no-capacity-checks >>>/Cluster is not healthy: False/ >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-hroller.test000064400000000000000000000047631476477700300232570ustar00rootroot00000000000000./test/hs/hroller --no-headers --ignore-non-redundant -t $TESTDATA_DIR/unique-reboot-order.data >>> node-01-002 node-01-003,node-01-001 >>>= 0 ./test/hs/hroller --no-headers --skip-non-redundant -t $TESTDATA_DIR/unique-reboot-order.data >>> node-01-002 >>>= 0 ./test/hs/hroller --no-headers -t $TESTDATA_DIR/unique-reboot-order.data >>>/^node-01-00. node-01-00. node-01-001$/ >>>= 0 ./test/hs/hroller --ignore-non-redundant -O node-01-002 --no-headers -t $TESTDATA_DIR/unique-reboot-order.data >>> node-01-003,node-01-001 >>>= 0 ./test/hs/hroller --ignore-non-redundant -O node-01-003 --no-headers -t $TESTDATA_DIR/unique-reboot-order.data >>> node-01-002 node-01-001 >>>= 0 ./test/hs/hroller --node-tags=red --no-headers -t $TESTDATA_DIR/multiple-tags.data >>>/^node-01-00[45],node-01-00[45],node-01-001$/ >>>= 0 ./test/hs/hroller --node-tags=blue --no-headers -t $TESTDATA_DIR/multiple-tags.data >>>/^node-01-00[246],node-01-00[246],node-01-00[246]$/ >>>= 0 ./test/hs/hroller --no-headers --offline-maintenance -t $TESTDATA_DIR/hroller-online.data >>>/node-01-00.,node-01-00. node-01-001,node-01-003/ >>>= 0 ./test/hs/hroller --no-headers -t $TESTDATA_DIR/hroller-online.data >>>/node-01-00.,node-01-00. node-01-002 node-01-003/ >>>= 0 ./test/hs/hroller --no-headers -t $TESTDATA_DIR/hroller-nonredundant.data >>>/^node-01-00.,node-01-00. node-01-00.,node-01-00. node-01-00.,node-01-000$/ >>>= 0 ./test/hs/hroller --skip-non-redundant -t $TESTDATA_DIR/hroller-nonredundant.data >>>2 Error: Cannot create node graph >>>=1 ./test/hs/hroller --no-headers --ignore-non-redundant -t $TESTDATA_DIR/hroller-nonredundant.data >>>/^node-01-00.,node-01-00.,node-01-00.,node-01-00.,node-01-00.,node-01-000$/ >>>= 0 ./test/hs/hroller --no-headers -t $TESTDATA_DIR/hroller-nodegroups.data >>>/^node-01-00. node-01-00. node-01-00.,node-02-000$/ >>>= 0 ./test/hs/hroller --no-headers -t $TESTDATA_DIR/hroller-full.data >>>/^node-..,node-..,node-..,node-.. node-..,node-..,node-..,node-31$/ >>>= 0 ./test/hs/hroller --no-headers --full-evacuation -t $TESTDATA_DIR/hroller-full.data >>>/^node-..,node-.. node-..,node-.. node-..,node-.. node-..,node-31$/ >>>= 0 ./test/hs/hroller --no-headers --full-evacuation --one-step-only --print-moves -t $TESTDATA_DIR/hroller-full.data >>>/^node-.. node-.. inst-.. node-.. node-.. inst-.. node-.. node-.. inst-.. node-.. node-.. inst-.. node-.. node-..$/ >>>= 0 ./test/hs/hroller --full-evacuation -t $TESTDATA_DIR/unique-reboot-order.data >>>2 Error: Not enough capacity to move secondaries >>>=1 ganeti-3.1.0~rc2/test/hs/shelltests/htools-hspace.test000064400000000000000000000106271476477700300230470ustar00rootroot00000000000000# test that hspace machine readable output looks correct ./test/hs/hspace --simu p,4,8T,64g,16 --machine-readable --disk-template drbd -l 8 >>> /^HTS_OK=1/ >>>= 0 # test again via a file and shell parsing ./test/hs/hspace --simu p,4,8T,64g,16 --machine-readable --disk-template drbd -l 8 > $T/capacity && sh -c ". $T/capacity && test x\$HTS_OK = x1" >>>= 0 # standard & tiered allocation, using shell parsing to do multiple checks ./test/hs/hspace --machine-readable -t $TESTDATA_DIR/hspace-tiered.data --no-capacity-checks > $T/capacity && sh -c ". $T/capacity && test \"\${HTS_TSPEC}\" = '131072,1048576,4,12=4 129984,1048320,4,12=2' && test \"\${HTS_ALLOC_INSTANCES}\" = 6" >>>=0 # again, but with a policy containing two min/max specs pairs ./test/hs/hspace --machine-readable -t $TESTDATA_DIR/hspace-tiered-dualspec.data --no-capacity-checks > $T/capacity && sh -c ". $T/capacity && test \"\${HTS_TSPEC}\" = '131072,1048576,4,12=4 129984,1048320,4,12=2 65472,524288,2,12=2' && test \"\${HTS_ALLOC_INSTANCES}\" = 14" >>>2 >>>=0 # With exclusive storage ./test/hs/hspace --machine-readable -t $TESTDATA_DIR/hspace-tiered-exclusive.data --no-capacity-checks > $T/capacity && sh -c ". $T/capacity && test \"\${HTS_TSPEC}\" = '131072,1048576,4,10=1 131072,1048576,4,9=1 131072,1048576,4,8=2' && test \"\${HTS_ALLOC_INSTANCES}\" = 6 && test \"\${HTS_TRL_SPN_FREE}\" = 0 && test \"\${HTS_FIN_SPN_FREE}\" = 29" >>>=0 # With exclusive storage and a policy containing two min/max specs pairs ./test/hs/hspace --machine-readable -t $TESTDATA_DIR/hspace-tiered-dualspec-exclusive.data --no-capacity-checks > $T/capacity && sh -c ". $T/capacity && test \"\${HTS_TSPEC}\" = '131072,1048576,4,4=4 129984,1048320,4,4=2 65472,524288,2,2=2' && test \"\${HTS_ALLOC_INSTANCES}\" = 14 && test \"\${HTS_TRL_SPN_FREE}\" = 7 && test \"\${HTS_FIN_SPN_FREE}\" = 7" >>>2 >>>=0 # Mixed cluster, half with exclusive storage ./test/hs/hspace --machine-readable -t $TESTDATA_DIR/hspace-tiered-mixed.data --no-capacity-checks > $T/capacity && sh -c ". $T/capacity && test \"\${HTS_TSPEC}\" = '131072,1048576,4,12=2 131072,1048576,4,10=2 129984,1048320,4,10=2' && test \"\${HTS_ALLOC_INSTANCES}\" = 6 && test \"\${HTS_TRL_SPN_FREE}\" = 0 && test \"\${HTS_FIN_SPN_FREE}\" = 18" >>>=0 # Verify that instance policy for disks is adhered to ./test/hs/hspace --machine-readable -t $TESTDATA_DIR/hspace-tiered-ipolicy.data --no-capacity-checks >>>/HTS_TRL_INST_CNT=4/ >>>=0 # ...and instance positioning in human-readable form ./test/hs/hspace -pname,pcnt -t $TESTDATA_DIR/hspace-tiered-ipolicy.data --no-capacity-checks >>>2/Tiered allocation status: Name pcnt node-01-001 1 node-01-002 1 node-01-003 1 node-01-004 1/ >>>=0 ./test/hs/hspace -pname,pcnt -t $TESTDATA_DIR/hspace-tiered-resourcetypes.data --no-capacity-checks >>>2/Tiered allocation status: Name pcnt node-01-001 1 node-01-002 2 node-01-003 2 node-01-004 2/ >>>=0 # VCPU-dominated allocation ./test/hs/hspace --machine-readable -t $TESTDATA_DIR/hspace-tiered-vcpu.data > $T/capacity && sh -c ". $T/capacity && test \"\${HTS_TSPEC}\" = '32768,65536,4,12=4 32768,65536,2,12=2' && test \"\${HTS_ALLOC_INSTANCES}\" = 10" >>>=0 # Presence of overfull group ./test/hs/hspace -t $TESTDATA_DIR/hspace-groups-one.data >>>/0 instances allocated/ >>>=0 ./test/hs/hspace --independent-groups -t $TESTDATA_DIR/hspace-groups-one.data >>>/0 instances allocated/ >>>=0 ./test/hs/hspace --accept-existing -t $TESTDATA_DIR/hspace-groups-one.data >>>/2 instances allocated/ >>>=0 ./test/hs/hspace -t $TESTDATA_DIR/hspace-groups-two.data >>>/0 instances allocated/ >>>=0 ./test/hs/hspace --independent-groups -t $TESTDATA_DIR/hspace-groups-two.data >>>/2 instances allocated/ >>>=0 ./test/hs/hspace --accept-existing -t $TESTDATA_DIR/hspace-groups-two.data >>>/2 instances allocated/ >>>=0 ./test/hs/hspace -t $TESTDATA_DIR/hspace-existing.data >>>/ 0 instances allocated/ >>>=0 ./test/hs/hspace -t $TESTDATA_DIR/hspace-existing.data --accept-existing >>>/ [1-9][0-9]* instances allocated/ >>>=0 ./test/hs/hspace -t $TESTDATA_DIR/hspace-bad-group.data >>>/ 0 instances allocated/ >>>=0 ./test/hs/hspace -t $TESTDATA_DIR/hspace-bad-group.data -v -v -v >>>2/Bad groups: \["group-bad"\]/ >>>=0 ./test/hs/hspace -t $TESTDATA_DIR/hspace-bad-group.data --accept-existing >>>/ 0 instances allocated/ >>>=0 ./test/hs/hspace -t $TESTDATA_DIR/hspace-bad-group.data --independent-groups >>>/ [1-9][0-9]* instances allocated/ >>>=0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-hsqueeze.test000064400000000000000000000041001476477700300234220ustar00rootroot00000000000000./test/hs/hsqueeze -v -t $TESTDATA_DIR/hsqueeze-underutilized.data >>>/Offline candidates: node-01-00[1-5],node-01-00[1-5],node-01-00[1-5],node-01-00[1-5],node-01-00[1-5]/ >>>= 0 ./test/hs/hsqueeze --no-headers --target-resources=1.0 -t $TESTDATA_DIR/hsqueeze-underutilized.data >>>/^node-01-00[2345] node-01-00[2345] node-01-00[2345]$/ >>>= 0 ./test/hs/hsqueeze --no-headers -t $TESTDATA_DIR/hsqueeze-underutilized.data >>>/^node-01-00[2345] node-01-00[2345]$/ >>>= 0 ./test/hs/hsqueeze --no-headers --target-resources=0 -t $TESTDATA_DIR/hsqueeze-underutilized.data >>>/^node-01-00[2345] node-01-00[2345] node-01-00[2345] node-01-00[2345]$/ >>>= 0 ./test/hs/hsqueeze --no-headers --target-resources=0 -t $TESTDATA_DIR/hsqueeze-underutilized.data -C >>>/gnt-instance migrate -f -n node-01-00[01] inst-50(.| )*gnt-node add-tags node-01-005 htools:standby:auto(.| )*gnt-node power -f off node-01-005/ >>>=0 ./test/hs/hsqueeze -v -t $TESTDATA_DIR/hsqueeze-overutilized.data >>>/Online candidates: node-01-00[2456],node-01-00[2456],node-01-00[2456],node-01-00[2456]/ >>>= 0 ./test/hs/hsqueeze --no-headers -t $TESTDATA_DIR/hsqueeze-overutilized.data >>>/^node-01-00[2456] node-01-00[2456]$/ >>>= 0 ./test/hs/hsqueeze -t $TESTDATA_DIR/hsqueeze-overutilized.data --minimal-resources=0 >>>/No action/ >>>= 0 ./test/hs/hsqueeze --no-headers -t $TESTDATA_DIR/hsqueeze-overutilized.data --minimal-resources=4.0 --target-resources=5.0 >>>/^node-01-00[2456] node-01-00[2456] node-01-00[2456] node-01-00[2456]$/ >>>= 0 ./test/hs/hsqueeze --no-headers -t $TESTDATA_DIR/hsqueeze-overutilized.data --minimal-resources=4.0 --target-resources=5.0 -C >>>/gnt-node power -f on node-01-006(.| )*gnt-instance migrate -f -n node-01-006 inst-/ >>>= 0 ./test/hs/hsqueeze -v -t $TESTDATA_DIR/hsqueeze-overutilized.data --minimal-resources=4.0 --target-resources=5.0 >>>/will not yield enough capacity/ >>>= 0 ./test/hs/hsqueeze -t $TESTDATA_DIR/hsqueeze-mixed-instances.data --no-headers >>> node-01-001 >>>= 0 ./test/hs/hsqueeze -t $TESTDATA_DIR/hsqueeze-mixed-instances.data -v >>>/Offline candidates: node-01-001 / >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-invalid.test000064400000000000000000000030001476477700300232150ustar00rootroot00000000000000# invalid option test ./test/hs/hail --no-such-option >>>= 2 # invalid option test ./test/hs/hbal --no-such-option >>>= 2 # invalid option test ./test/hs/hspace --no-such-option >>>= 2 # invalid option test ./test/hs/hscan --no-such-option >>>= 2 # invalid option test ./test/hs/hinfo --no-such-option >>>= 2 # invalid option test ./test/hs/hcheck --no-such-option >>>= 2 # invalid option test ./test/hs/hroller --no-such-option >>>= 2 # extra arguments ./test/hs/hspace unexpected-argument >>>2 Error: This program doesn't take any arguments. >>>=1 ./test/hs/hbal unexpected-argument >>>2 Error: This program doesn't take any arguments. >>>=1 ./test/hs/hinfo unexpected-argument >>>2 Error: This program doesn't take any arguments. >>>=1 ./test/hs/hcheck unexpected-argument >>>2 Error: This program doesn't take any arguments. >>>=1 ./test/hs/hroller unexpected-argument >>>2 Error: This program doesn't take any arguments. >>>=1 # hroller should notice the absence of a master node ./test/hs/hroller -t$TESTDATA_DIR/empty-cluster.data >>>2/Error: No master node found/ >>>=1 # hroller fails to build a graph for an empty cluster ./test/hs/hroller -f -t$TESTDATA_DIR/empty-cluster.data >>>2/Error: Cannot create node graph/ >>>=1 # hroller should reject a configuration with more than one master, # even with -f ./test/hs/hroller -f -t$TESTDATA_DIR/multiple-master.data >>>2/Error: Found more than one master node/ >>>=1 # hbal doesn't accept invalid priority ./test/hs/hbal --priority=abc >>>2/Unknown priority/ >>>=1 ganeti-3.1.0~rc2/test/hs/shelltests/htools-mon-collector.test000064400000000000000000000223351476477700300243600ustar00rootroot00000000000000# Test that mon-collector won't run without specifying a personality ./test/hs/hpc-mon-collector >>>= !0 # Test that standard options are accepted, both at top level # and subcommands level ./test/hs/hpc-mon-collector --help >>>= 0 ./test/hs/hpc-mon-collector --help-completion >>>= 0 ./test/hs/hpc-mon-collector --version >>>= 0 ./test/hs/hpc-mon-collector drbd --help >>>= 0 ./test/hs/hpc-mon-collector drbd --help-completion >>>= 0 ./test/hs/hpc-mon-collector drbd --version >>>= 0 # Test that the drbd collector fails parsing /dev/null ./test/hs/hpc-mon-collector drbd --drbd-status=/dev/null --drbd-pairing=/dev/null >>>2/Malformed JSON/ >>>= !0 # Test that a non-existent file is correctly reported ./test/hs/hpc-mon-collector drbd --drbd-status=/dev/no-such-file --drbd-pairing=/dev/no-such-file >>>2/Error: reading from file: .* does not exist/ >>>= !0 # Test that arguments are rejected ./test/hs/hpc-mon-collector drbd /dev/null >>>2/takes exactly zero arguments/ >>>= !0 # Test that a standard test file is parsed correctly ./test/hs/hpc-mon-collector drbd --drbd-status=$PYTESTDATA_DIR/proc_drbd83.txt --drbd-pairing=$PYTESTDATA_DIR/instance-minor-pairing.txt >>>=0 # Test that the drbd collector fails parsing /dev/zero, but is not # stuck forever printing \NUL chars ./test/hs/hpc-mon-collector drbd --drbd-status=/dev/zero --drbd-pairing=$PYTESTDATA_DIR/instance-minor-pairing.txt >>>2 Error: "\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL\NUL" [] Failed reading: versionInfo >>>= !0 # Tests for diskstats ./test/hs/hpc-mon-collector diskstats --help >>>= 0 ./test/hs/hpc-mon-collector diskstats --help-completion >>>= 0 ./test/hs/hpc-mon-collector diskstats --version >>>= 0 # Test that the diskstats collector fails parsing a non-diskstats file ./test/hs/hpc-mon-collector diskstats -f /dev/zero >>>2/Failed reading/ >>>= !0 # Test that a non-existent file is correctly reported ./test/hs/hpc-mon-collector diskstats --file=/proc/no-such-file >>>2/Error: reading from file: .* does not exist/ >>>= !0 # Test that arguments are rejected ./test/hs/hpc-mon-collector diskstats /dev/null >>>2/takes exactly zero arguments/ >>>= !0 # Test that a standard test file is parsed correctly ./test/hs/hpc-mon-collector diskstats -f $PYTESTDATA_DIR/proc_diskstats.txt >>>=0 # Tests for lv ./test/hs/hpc-mon-collector lv --help >>>= 0 ./test/hs/hpc-mon-collector lv --help-completion >>>= 0 ./test/hs/hpc-mon-collector lv --version >>>= 0 # Test that the lv collector fails parsing a non-lv data ./test/hs/hpc-mon-collector lv -f $PYTESTDATA_DIR/proc_diskstats.txt >>>= !0 # Test that lv correctly reports a non-existent file ./test/hs/hpc-mon-collector lv --file=/proc/no-such-file >>>2/Error: reading from file: .* does not exist/ >>>= !0 # Test that lv rejects arguments ./test/hs/hpc-mon-collector lv /dev/null >>>2/takes exactly zero arguments/ >>>= !0 # Test that lv parses correctly a standard test file ./test/hs/hpc-mon-collector lv -f $PYTESTDATA_DIR/lvs_lv.txt -i $PYTESTDATA_DIR/instance-disks.txt >>>/"instance":"instance1.example.com"/ >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-multi-group.test000064400000000000000000000026461476477700300240720ustar00rootroot00000000000000# standard multi-group tests ./test/hs/hinfo -v -v -p --print-instances -t$T/simu-twogroups.standard >>>= 0 ./test/hs/hbal -t$T/simu-twogroups.standard >>>= !0 # hbal should not be able to balance ./test/hs/hbal -t$T/simu-twogroups.standard >>>2 /Found multiple node groups/ >>>= !0 # but hbal should be able to balance one node group ./test/hs/hbal -t$T/simu-twogroups.standard -G group-01 >>>= 0 # and it should not find an invalid group ./test/hs/hbal -t$T/simu-twogroups.standard -G no-such-group >>>= !0 # tiered allocs multi-group tests ./test/hs/hinfo -v -v -p --print-instances -t$T/simu-twogroups.tiered >>>= 0 ./test/hs/hbal -t$T/simu-twogroups.tiered >>>= !0 # hbal should not be able to balance ./test/hs/hbal -t$T/simu-twogroups.tiered >>>2 /Found multiple node groups/ >>>= !0 # but hbal should be able to balance one node group ./test/hs/hbal -t$T/simu-twogroups.tiered -G group-01 >>>= 0 # and it should not find an invalid group ./test/hs/hbal -t$T/simu-twogroups.tiered -G no-such-group >>>= !0 # hcheck should be able to run with multiple groups ./test/hs/hcheck -t$T/simu-twogroups.tiered --machine-readable >>> /HCHECK_OK=1/ >>>= 0 # hcheck should be able to improve a group with split instances, and also # warn us about them ./test/hs/hbal -t $TESTDATA_DIR/hbal-split-insts.data -G group-01 -O node-01-001 -v >>> /Cluster score improved from .* to .*/ >>>2/Found instances belonging to multiple node groups:/ >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-no-backend.test000064400000000000000000000005651476477700300236050ustar00rootroot00000000000000# hail no input file ./test/hs/hail >>>= 1 # hbal no backend ./test/hs/hbal >>>= 1 # hspace no backend ./test/hs/hspace >>>= 1 # hinfo no backend ./test/hs/hinfo >>>= 1 # hroller no backend ./test/hs/hroller >>>= 1 # hbal multiple backends ./test/hs/hbal -t /dev/null -m localhost >>>2 Error: Only one of the rapi, luxi, and data files options should be given. >>>= 1 ganeti-3.1.0~rc2/test/hs/shelltests/htools-rapi.test000064400000000000000000000004101476477700300225240ustar00rootroot00000000000000# test loading data via RAPI ./test/hs/hinfo -v -v -p --print-instances -m $RAPI_URL >>>= 0 ./test/hs/hbal -v -v -p --print-instances -m $RAPI_URL >>>= 0 # this compares generated files from hscan diff -u $T/hscan/direct.hinfo $T/hscan/fromtext.hinfo >>> >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-single-group.test000064400000000000000000000013641476477700300242150ustar00rootroot00000000000000# standard single-group tests ./test/hs/hinfo -v -v -p --print-instances -t$T/simu-onegroup.standard >>>= 0 ./test/hs/hbal -v -v -p --print-instances -t$T/simu-onegroup.standard >>>= 0 # tiered single-group tests ./test/hs/hinfo -v -v -p --print-instances -t$T/simu-onegroup.tiered >>>= 0 ./test/hs/hbal -v -v -p --print-instances -t$T/simu-onegroup.tiered >>>= 0 # hcheck should not find reason to rebalance ./test/hs/hcheck -t$T/simu-onegroup.tiered --machine-readable >>> /HCHECK_INIT_CLUSTER_NEED_REBALANCE=0/ >>>= 0 # hroller should be able to print the solution ./test/hs/hroller -t$T/simu-onegroup.tiered >>>= 0 # hroller should be able to print the solution, in verbose mode as well ./test/hs/hroller -t$T/simu-onegroup.tiered -v -v >>>= 0 ganeti-3.1.0~rc2/test/hs/shelltests/htools-text-backend.test000064400000000000000000000015231476477700300241500ustar00rootroot00000000000000# missing resources test ./test/hs/hbal -t $TESTDATA_DIR/missing-resources.data >>>2 /node node2 is missing .* ram and .* disk/ >>>= 0 ./test/hs/hinfo -t $TESTDATA_DIR/missing-resources.data >>>2 /node node2 is missing .* ram and .* disk/ >>>= 0 # common suffix test ./test/hs/hbal -t $TESTDATA_DIR/common-suffix.data -v -v >>>/Stripping common suffix of '\.example\.com' from names/ >>>= 0 ./test/hs/hinfo -t $TESTDATA_DIR/common-suffix.data -v -v >>>/Stripping common suffix of '\.example\.com' from names/ >>>= 0 # invalid node test ./test/hs/hbal -t $TESTDATA_DIR/invalid-node.data >>>2 /Unknown node '.*' for instance new-0/ >>>= !0 ./test/hs/hspace -t $TESTDATA_DIR/invalid-node.data >>>2 /Unknown node '.*' for instance new-0/ >>>= !0 ./test/hs/hinfo -t $TESTDATA_DIR/invalid-node.data >>>2 /Unknown node '.*' for instance new-0/ >>>= !0 ganeti-3.1.0~rc2/test/py/000075500000000000000000000000001476477700300152135ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/integration/000075500000000000000000000000001476477700300175365ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/integration/test_dummy.py000064400000000000000000000026011476477700300223010ustar00rootroot00000000000000# # # Copyright (C) 2022 the Ganeti project # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from ganeti import errors def test_dummy(): assert True ganeti-3.1.0~rc2/test/py/legacy/000075500000000000000000000000001476477700300164575ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/legacy/__init__.py000064400000000000000000000025551476477700300205770ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """This module contains all python test code""" ganeti-3.1.0~rc2/test/py/legacy/bash_completion.bash000075500000000000000000000134751476477700300225010ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e -u -o pipefail # Disable any locales export LC_ALL=C readonly bash_completion=${TOP_BUILDDIR:-.}/doc/examples/bash_completion-debug readonly default_wordbreaks=$' \t\n"'\''@><=;|&(:' err() { echo "$@" echo 'Aborting' exit 1 } contains() { local -r needle="$1"; shift for value; do if [[ "$value" = "$needle" ]]; then return 0 fi done return 1 } # Prepares a subshell for testing bash completion functions setup() { local -r unused=UNUSED set +e +u -o pipefail . $bash_completion COMP_KEY=$unused COMP_LINE=$unused COMP_POINT=$unused COMP_TYPE=$unused COMP_WORDBREAKS="$default_wordbreaks" GANETI_COMPL_LOG= unset COMP_CWORD unset COMP_WORDS } # Check if default wordbreaks are still valid (this detects cases where Bash # were to change its built-in default value) # TODO: May need an update for older Bash versions; some didn't include the # colon character (':') in COMP_WORDBREAKS ( bashdef=$(env - bash --noprofile --norc -c 'echo -n "$COMP_WORDBREAKS"') case "$bashdef" in $default_wordbreaks) ;; *) err 'Bash uses unknown value for COMP_WORDBREAKS' ;; esac ) # Check for --help for cmd in gnt-{instance,node,group,job}; do ( setup COMP_CWORD=2 COMP_WORDS=( $cmd list - ) _${cmd/-/_} contains --help "${COMPREPLY[@]}" || \ err "'$cmd list' did not list --help as an option" ) done # Completing a yes/no option ( setup COMP_CWORD=3 COMP_WORDS=( gnt-node modify --drained ) _gnt_node if [[ "${COMPREPLY[*]}" != 'no yes' ]]; then err "Completing '${COMP_WORDS[@]}' did not give correct result" fi ) # Completing a multiple-choice option ( setup COMP_CWORD=2 COMP_WORDS=( gnt-debug allocator --disk-template=sh foo ) _gnt_debug if [[ "${COMPREPLY[*]}" != sharedfile ]]; then err "Completing '${COMP_WORDS[*]}' did not give correct result" fi ) # Completing a node name ( setup # Override built-in function _ganeti_nodes() { echo aanode1 bbnode2 aanode3 } COMP_CWORD=4 COMP_WORDS=( gnt-node modify --drained yes aa ) _gnt_node if [[ "${COMPREPLY[*]}" != 'aanode1 aanode3' ]]; then err 'Completing node names failed' fi ) # Completing an option when it's not at the end ( setup # Override built-in function _ganeti_instances() { echo inst{1..5} } # Completing word in the middle COMP_CWORD=2 COMP_WORDS=( gnt-instance list --o inst3 inst inst5 ) _gnt_node contains --output "${COMPREPLY[@]}" || err 'Did not complete parameter' ) # Completing an instance name ( setup # Override built-in function _ganeti_instances() { echo inst{1..5} } # Completing word in the middle COMP_CWORD=5 COMP_WORDS=( gnt-instance list -o foobar inst1 inst inst5 ) _gnt_instance if [[ "${COMPREPLY[*]}" != "$(echo inst{1..5})" ]]; then err "Completing '${COMP_WORDS[*]}' did not give correct result" fi ) # Need to test node expansion with different wordbreak settings [[ "$default_wordbreaks" == *:* ]] || \ err 'No colon in default wordbreak characters' for wb in "$default_wordbreaks" "${default_wordbreaks/:/}"; do ( setup # Override built-in function _ganeti_nodes() { echo node{A..C} } COMP_WORDBREAKS="$wb" # Completing nodes COMP_CWORD=3 COMP_WORDS=( gnt-instance add -n ) _gnt_instance if [[ "${COMPREPLY[*]}" != 'nodeA nodeA: nodeB nodeB: nodeC nodeC:' ]]; then err 'Got wrong node list' fi COMP_CWORD=3 COMP_WORDS=( gnt-instance add -n nodeB ) _gnt_instance if [[ "${COMPREPLY[*]}" != 'nodeB nodeB:' ]]; then err 'Got wrong node list' fi COMP_CWORD=3 COMP_WORDS=( gnt-instance add -n nodeC: ) _gnt_instance if [[ "$COMP_WORDBREAKS" == *:* ]]; then expected='nodeA nodeB' else expected='nodeC:nodeA nodeC:nodeB' fi if [[ "${COMPREPLY[*]}" != "$expected" ]]; then err 'Got wrong node list' fi ) done # Need to test different settings for the extglob shell option for opt in -u -s; do verify_extglob() { if [[ "$(shopt -p extglob)" != "shopt $opt extglob" ]]; then err 'The "extglob" shell option has an unexpected value' fi } ( shopt $opt extglob verify_extglob setup verify_extglob # Completing nodes COMP_CWORD=4 COMP_WORDS=( gnt-instance add --os-type busybox --no-n ) _gnt_instance if [[ "${COMPREPLY[*]}" != '--no-name-check --no-nics' ]]; then err "Completing '${COMP_WORDS[*]}' did not give correct result" fi verify_extglob ) done exit 0 ganeti-3.1.0~rc2/test/py/legacy/cfgupgrade_unittest.py000075500000000000000000000467061476477700300231170ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2012, 2013, 2014, 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing tools/cfgupgrade""" import os import sys import unittest import shutil import tempfile import operator import json from unittest import mock from ganeti import cli from ganeti import constants from ganeti import utils from ganeti import serializer from ganeti import netutils from ganeti.utils import version from ganeti.tools.cfgupgrade import CfgUpgrade, ParseOptions, Error import testutils def GetMinimalConfig(): return { "version": constants.CONFIG_VERSION, "cluster": { "master_node": "node1-uuid", "ipolicy": None, "default_iallocator_params": {}, "diskparams": {}, "ndparams": {}, "candidate_certs": {}, "install_image": "", "instance_communication_network": "", "zeroing_image": "", "compression_tools": constants.IEC_DEFAULT_TOOLS, "enabled_user_shutdown": False, "data_collectors": { "diskstats": { "active": True, "interval": 5000000 }, "drbd": { "active": True, "interval": 5000000 }, "lv": { "active": True, "interval": 5000000 }, "inst-status-xen": { "active": True, "interval": 5000000 }, "cpu-avg-load": { "active": True, "interval": 5000000 }, "xen-cpu-avg-load": { "active": True, "interval": 5000000 }, }, "ssh_key_type": "dsa", "ssh_key_bits": 1024, }, "instances": {}, "disks": {}, "networks": {}, "filters": {}, "nodegroups": {}, "nodes": { "node1-uuid": { "name": "node1", "uuid": "node1-uuid" } }, } def _RunUpgrade(path, dry_run, no_verify, ignore_hostname=True, downgrade=False, fail=False): args = ["--debug", "--force", "--path=%s" % path, "--confdir=%s" % path] if ignore_hostname: args.append("--ignore-hostname") if dry_run: args.append("--dry-run") if no_verify: args.append("--no-verify") if downgrade: args.append("--downgrade") opts, args = ParseOptions(args=args) upgrade = CfgUpgrade(opts, args) with mock.patch('sys.exit'): with mock.patch.object(upgrade, 'SetupLogging'): with mock.patch.object(cli, 'ToStderr'): upgrade.Run() if sys.exit.called: raise Error("upgrade failed") class TestCfgupgrade(unittest.TestCase): def setUp(self): # Since we are comparing large dictionaries here, this is vital to getting # useful feedback about differences in config content using assertEquals. self.maxDiff = None self.tmpdir = tempfile.mkdtemp() self.config_path = utils.PathJoin(self.tmpdir, "config.data") self.noded_cert_path = utils.PathJoin(self.tmpdir, "server.pem") self.rapi_cert_path = utils.PathJoin(self.tmpdir, "rapi.pem") self.rapi_users_path = utils.PathJoin(self.tmpdir, "rapi", "users") self.rapi_users_path_pre24 = utils.PathJoin(self.tmpdir, "rapi_users") self.known_hosts_path = utils.PathJoin(self.tmpdir, "known_hosts") self.confd_hmac_path = utils.PathJoin(self.tmpdir, "hmac.key") self.cds_path = utils.PathJoin(self.tmpdir, "cluster-domain-secret") self.ss_master_node_path = utils.PathJoin(self.tmpdir, "ssconf_master_node") self.file_storage_paths = utils.PathJoin(self.tmpdir, "file-storage-paths") def tearDown(self): shutil.rmtree(self.tmpdir) def _LoadConfig(self): return serializer.LoadJson(utils.ReadFile(self.config_path)) def _LoadTestDataConfig(self, filename): return serializer.LoadJson(testutils.ReadTestData(filename)) def _CreateValidConfigDir(self): utils.WriteFile(self.noded_cert_path, data="") utils.WriteFile(self.known_hosts_path, data="") utils.WriteFile(self.ss_master_node_path, data="node.has.another.name.example.net") def testNoConfigDir(self): self.assertFalse(utils.ListVisibleFiles(self.tmpdir)) self.assertRaises(Exception, _RunUpgrade, self.tmpdir, False, True) self.assertRaises(Exception, _RunUpgrade, self.tmpdir, True, True) def testWrongHostname(self): self._CreateValidConfigDir() utils.WriteFile(self.config_path, data=serializer.DumpJson(GetMinimalConfig())) hostname = netutils.GetHostname().name assert hostname != utils.ReadOneLineFile(self.ss_master_node_path) self.assertRaises(Exception, _RunUpgrade, self.tmpdir, False, True, ignore_hostname=False) def testCorrectHostname(self): self._CreateValidConfigDir() utils.WriteFile(self.config_path, data=serializer.DumpJson(GetMinimalConfig())) utils.WriteFile(self.ss_master_node_path, data="%s\n" % netutils.GetHostname().name) _RunUpgrade(self.tmpdir, False, True, ignore_hostname=False) def testInconsistentConfig(self): self._CreateValidConfigDir() # There should be no "config_version" cfg = GetMinimalConfig() cfg["version"] = 0 cfg["cluster"]["config_version"] = 0 utils.WriteFile(self.config_path, data=serializer.DumpJson(cfg)) self.assertRaises(Exception, _RunUpgrade, self.tmpdir, False, True) def testInvalidConfig(self): self._CreateValidConfigDir() # Missing version from config utils.WriteFile(self.config_path, data=serializer.DumpJson({})) self.assertRaises(Exception, _RunUpgrade, self.tmpdir, False, True) def _TestUpgradeFromFile(self, filename, dry_run): cfg = self._LoadTestDataConfig(filename) self._TestUpgradeFromData(cfg, dry_run) def _TestSimpleUpgrade(self, from_version, dry_run, file_storage_dir=None, shared_file_storage_dir=None): cfg = GetMinimalConfig() cfg["version"] = from_version cluster = cfg["cluster"] if file_storage_dir: cluster["file_storage_dir"] = file_storage_dir if shared_file_storage_dir: cluster["shared_file_storage_dir"] = shared_file_storage_dir self._TestUpgradeFromData(cfg, dry_run) def _TestUpgradeFromData(self, cfg, dry_run): assert "version" in cfg from_version = cfg["version"] self._CreateValidConfigDir() utils.WriteFile(self.config_path, data=serializer.DumpJson(cfg)) self.assertFalse(os.path.isfile(self.rapi_cert_path)) self.assertFalse(os.path.isfile(self.confd_hmac_path)) self.assertFalse(os.path.isfile(self.cds_path)) _RunUpgrade(self.tmpdir, dry_run, True) if dry_run: expversion = from_version checkfn = operator.not_ else: expversion = constants.CONFIG_VERSION checkfn = operator.truth self.assertTrue(checkfn(os.path.isfile(self.rapi_cert_path))) self.assertTrue(checkfn(os.path.isfile(self.confd_hmac_path))) self.assertTrue(checkfn(os.path.isfile(self.cds_path))) newcfg = self._LoadConfig() self.assertEqual(newcfg["version"], expversion) def testRapiUsers(self): self.assertFalse(os.path.exists(self.rapi_users_path)) self.assertFalse(os.path.exists(self.rapi_users_path_pre24)) self.assertFalse(os.path.exists(os.path.dirname(self.rapi_users_path))) utils.WriteFile(self.rapi_users_path_pre24, data="some user\n") self._TestSimpleUpgrade(version.BuildVersion(2, 3, 0), False) self.assertTrue(os.path.isdir(os.path.dirname(self.rapi_users_path))) self.assertTrue(os.path.islink(self.rapi_users_path_pre24)) self.assertTrue(os.path.isfile(self.rapi_users_path)) self.assertEqual(os.readlink(self.rapi_users_path_pre24), self.rapi_users_path) for path in [self.rapi_users_path, self.rapi_users_path_pre24]: self.assertEqual(utils.ReadFile(path), "some user\n") def testRapiUsers24AndAbove(self): self.assertFalse(os.path.exists(self.rapi_users_path)) self.assertFalse(os.path.exists(self.rapi_users_path_pre24)) os.mkdir(os.path.dirname(self.rapi_users_path)) utils.WriteFile(self.rapi_users_path, data="other user\n") self._TestSimpleUpgrade(version.BuildVersion(2, 3, 0), False) self.assertTrue(os.path.islink(self.rapi_users_path_pre24)) self.assertTrue(os.path.isfile(self.rapi_users_path)) self.assertEqual(os.readlink(self.rapi_users_path_pre24), self.rapi_users_path) for path in [self.rapi_users_path, self.rapi_users_path_pre24]: self.assertEqual(utils.ReadFile(path), "other user\n") def testRapiUsersExistingSymlink(self): self.assertFalse(os.path.exists(self.rapi_users_path)) self.assertFalse(os.path.exists(self.rapi_users_path_pre24)) os.mkdir(os.path.dirname(self.rapi_users_path)) os.symlink(self.rapi_users_path, self.rapi_users_path_pre24) utils.WriteFile(self.rapi_users_path, data="hello world\n") self._TestSimpleUpgrade(version.BuildVersion(2, 2, 0), False) self.assertTrue(os.path.isfile(self.rapi_users_path) and not os.path.islink(self.rapi_users_path)) self.assertTrue(os.path.islink(self.rapi_users_path_pre24)) self.assertEqual(os.readlink(self.rapi_users_path_pre24), self.rapi_users_path) for path in [self.rapi_users_path, self.rapi_users_path_pre24]: self.assertEqual(utils.ReadFile(path), "hello world\n") def testRapiUsersExistingTarget(self): self.assertFalse(os.path.exists(self.rapi_users_path)) self.assertFalse(os.path.exists(self.rapi_users_path_pre24)) os.mkdir(os.path.dirname(self.rapi_users_path)) utils.WriteFile(self.rapi_users_path, data="other user\n") utils.WriteFile(self.rapi_users_path_pre24, data="hello world\n") self.assertRaises(Exception, self._TestSimpleUpgrade, version.BuildVersion(2, 2, 0), False) for path in [self.rapi_users_path, self.rapi_users_path_pre24]: self.assertTrue(os.path.isfile(path) and not os.path.islink(path)) self.assertEqual(utils.ReadFile(self.rapi_users_path), "other user\n") self.assertEqual(utils.ReadFile(self.rapi_users_path_pre24), "hello world\n") def testRapiUsersDryRun(self): self.assertFalse(os.path.exists(self.rapi_users_path)) self.assertFalse(os.path.exists(self.rapi_users_path_pre24)) utils.WriteFile(self.rapi_users_path_pre24, data="some user\n") self._TestSimpleUpgrade(version.BuildVersion(2, 3, 0), True) self.assertFalse(os.path.isdir(os.path.dirname(self.rapi_users_path))) self.assertTrue(os.path.isfile(self.rapi_users_path_pre24) and not os.path.islink(self.rapi_users_path_pre24)) self.assertFalse(os.path.exists(self.rapi_users_path)) def testRapiUsers24AndAboveDryRun(self): self.assertFalse(os.path.exists(self.rapi_users_path)) self.assertFalse(os.path.exists(self.rapi_users_path_pre24)) os.mkdir(os.path.dirname(self.rapi_users_path)) utils.WriteFile(self.rapi_users_path, data="other user\n") self._TestSimpleUpgrade(version.BuildVersion(2, 3, 0), True) self.assertTrue(os.path.isfile(self.rapi_users_path) and not os.path.islink(self.rapi_users_path)) self.assertFalse(os.path.exists(self.rapi_users_path_pre24)) self.assertEqual(utils.ReadFile(self.rapi_users_path), "other user\n") def testRapiUsersExistingSymlinkDryRun(self): self.assertFalse(os.path.exists(self.rapi_users_path)) self.assertFalse(os.path.exists(self.rapi_users_path_pre24)) os.mkdir(os.path.dirname(self.rapi_users_path)) os.symlink(self.rapi_users_path, self.rapi_users_path_pre24) utils.WriteFile(self.rapi_users_path, data="hello world\n") self._TestSimpleUpgrade(version.BuildVersion(2, 2, 0), True) self.assertTrue(os.path.islink(self.rapi_users_path_pre24)) self.assertTrue(os.path.isfile(self.rapi_users_path) and not os.path.islink(self.rapi_users_path)) self.assertEqual(os.readlink(self.rapi_users_path_pre24), self.rapi_users_path) for path in [self.rapi_users_path, self.rapi_users_path_pre24]: self.assertEqual(utils.ReadFile(path), "hello world\n") def testFileStoragePathsDryRun(self): self.assertFalse(os.path.exists(self.file_storage_paths)) self._TestSimpleUpgrade(version.BuildVersion(2, 6, 0), True, file_storage_dir=self.tmpdir, shared_file_storage_dir="/tmp") self.assertFalse(os.path.exists(self.file_storage_paths)) def testFileStoragePathsBoth(self): self.assertFalse(os.path.exists(self.file_storage_paths)) self._TestSimpleUpgrade(version.BuildVersion(2, 6, 0), False, file_storage_dir=self.tmpdir, shared_file_storage_dir="/tmp") lines = utils.ReadFile(self.file_storage_paths).splitlines() self.assertTrue(lines.pop(0).startswith("# ")) self.assertTrue(lines.pop(0).startswith("# cfgupgrade")) self.assertEqual(lines.pop(0), self.tmpdir) self.assertEqual(lines.pop(0), "/tmp") self.assertFalse(lines) self.assertEqual(os.stat(self.file_storage_paths).st_mode & 0o777, 0o600, msg="Wrong permissions") def testFileStoragePathsSharedOnly(self): self.assertFalse(os.path.exists(self.file_storage_paths)) self._TestSimpleUpgrade(version.BuildVersion(2, 5, 0), False, file_storage_dir=None, shared_file_storage_dir=self.tmpdir) lines = utils.ReadFile(self.file_storage_paths).splitlines() self.assertTrue(lines.pop(0).startswith("# ")) self.assertTrue(lines.pop(0).startswith("# cfgupgrade")) self.assertEqual(lines.pop(0), self.tmpdir) self.assertFalse(lines) def testUpgradeFrom_2_0(self): self._TestSimpleUpgrade(version.BuildVersion(2, 0, 0), False) def testUpgradeFrom_2_1(self): self._TestSimpleUpgrade(version.BuildVersion(2, 1, 0), False) def testUpgradeFrom_2_2(self): self._TestSimpleUpgrade(version.BuildVersion(2, 2, 0), False) def testUpgradeFrom_2_3(self): self._TestSimpleUpgrade(version.BuildVersion(2, 3, 0), False) def testUpgradeFrom_2_4(self): self._TestSimpleUpgrade(version.BuildVersion(2, 4, 0), False) def testUpgradeFrom_2_5(self): self._TestSimpleUpgrade(version.BuildVersion(2, 5, 0), False) def testUpgradeFrom_2_6(self): self._TestSimpleUpgrade(version.BuildVersion(2, 6, 0), False) def testUpgradeFrom_2_7(self): self._TestSimpleUpgrade(version.BuildVersion(2, 7, 0), False) def testUpgradeFullConfigFrom_2_7(self): self._TestUpgradeFromFile("cluster_config_2.7.json", False) def testUpgradeFullConfigFrom_2_8(self): self._TestUpgradeFromFile("cluster_config_2.8.json", False) def testUpgradeFullConfigFrom_2_9(self): self._TestUpgradeFromFile("cluster_config_2.9.json", False) def testUpgradeFullConfigFrom_2_10(self): self._TestUpgradeFromFile("cluster_config_2.10.json", False) def testUpgradeFullConfigFrom_2_11(self): self._TestUpgradeFromFile("cluster_config_2.11.json", False) def testUpgradeFullConfigFrom_2_12(self): self._TestUpgradeFromFile("cluster_config_2.12.json", False) def testUpgradeFullConfigFrom_2_13(self): self._TestUpgradeFromFile("cluster_config_2.13.json", False) def testUpgradeFullConfigFrom_2_14(self): self._TestUpgradeFromFile("cluster_config_2.14.json", False) def testUpgradeFullConfigFrom_2_15(self): self._TestUpgradeFromFile("cluster_config_2.15.json", False) def testUpgradeFullConfigFrom_2_16(self): self._TestUpgradeFromFile("cluster_config_2.16.json", False) def testUpgradeFullConfigFrom_3_0(self): self._TestUpgradeFromFile("cluster_config_3.0.json", False) def testUpgradeFullConfigFrom_3_1(self): self._TestUpgradeFromFile("cluster_config_3.1.json", False) def testUpgradeCurrent(self): self._TestSimpleUpgrade(constants.CONFIG_VERSION, False) def _RunDowngradeUpgrade(self): oldconf = self._LoadConfig() _RunUpgrade(self.tmpdir, False, True, downgrade=True) _RunUpgrade(self.tmpdir, False, True) newconf = self._LoadConfig() self.assertEqual(oldconf, newconf) def testDowngrade(self): self._TestSimpleUpgrade(constants.CONFIG_VERSION, False) self._RunDowngradeUpgrade() def testDowngradeFullConfig(self): """Test for upgrade + downgrade combination.""" # This test can work only with the previous version of a configuration! oldconfname = "cluster_config_3.0.json" self._TestUpgradeFromFile(oldconfname, False) _RunUpgrade(self.tmpdir, False, True, downgrade=True) oldconf = self._LoadTestDataConfig(oldconfname) newconf = self._LoadConfig() self.maxDiff = None self.assertEqual(oldconf, newconf) def testDowngradeFullConfigBackwardFrom_2_7(self): """Test for upgrade + downgrade + upgrade combination.""" self._TestUpgradeFromFile("cluster_config_2.7.json", False) self._RunDowngradeUpgrade() def _RunDowngradeTwice(self): """Make sure that downgrade is idempotent.""" _RunUpgrade(self.tmpdir, False, True, downgrade=True) oldconf = self._LoadConfig() _RunUpgrade(self.tmpdir, False, True, downgrade=True) newconf = self._LoadConfig() self.assertEqual(oldconf, newconf) def testDowngradeTwice(self): self._TestSimpleUpgrade(constants.CONFIG_VERSION, False) self._RunDowngradeTwice() def testDowngradeTwiceFullConfigFrom_2_7(self): self._TestUpgradeFromFile("cluster_config_2.7.json", False) self._RunDowngradeTwice() def testUpgradeDryRunFrom_2_0(self): self._TestSimpleUpgrade(version.BuildVersion(2, 0, 0), True) def testUpgradeDryRunFrom_2_1(self): self._TestSimpleUpgrade(version.BuildVersion(2, 1, 0), True) def testUpgradeDryRunFrom_2_2(self): self._TestSimpleUpgrade(version.BuildVersion(2, 2, 0), True) def testUpgradeDryRunFrom_2_3(self): self._TestSimpleUpgrade(version.BuildVersion(2, 3, 0), True) def testUpgradeDryRunFrom_2_4(self): self._TestSimpleUpgrade(version.BuildVersion(2, 4, 0), True) def testUpgradeDryRunFrom_2_5(self): self._TestSimpleUpgrade(version.BuildVersion(2, 5, 0), True) def testUpgradeDryRunFrom_2_6(self): self._TestSimpleUpgrade(version.BuildVersion(2, 6, 0), True) def testUpgradeCurrentDryRun(self): self._TestSimpleUpgrade(constants.CONFIG_VERSION, True) def testDowngradeDryRun(self): self._TestSimpleUpgrade(constants.CONFIG_VERSION, False) oldconf = self._LoadConfig() _RunUpgrade(self.tmpdir, True, True, downgrade=True) newconf = self._LoadConfig() self.assertEqual(oldconf["version"], newconf["version"]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/check-cert-expired_unittest.bash000075500000000000000000000047371476477700300247410ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e set -o pipefail export PYTHON=${PYTHON:=python3} CCE=tools/check-cert-expired err() { echo "$@" echo 'Aborting' exit 1 } impexpd_helper() { $PYTHON "${TOP_SRCDIR:-.}/test/py/legacy/import-export_unittest-helper" "$@" } $CCE 2>/dev/null && err 'Accepted empty argument list' $CCE foo bar 2>/dev/null && err 'Accepted more than one argument' $CCE foo bar baz 2>/dev/null && err 'Accepted more than one argument' tmpdir=$(mktemp -d) trap "rm -rf $tmpdir" EXIT [[ -f "$tmpdir/cert-not" ]] && err 'File existed when it should not' $CCE $tmpdir/cert-not 2>/dev/null && err 'Accepted non-existent file' VALIDITY=1 impexpd_helper $tmpdir/cert-valid gencert $CCE $tmpdir/cert-valid 2>/dev/null && \ err 'Reported valid certificate as expired' VALIDITY=-50 impexpd_helper $tmpdir/cert-expired gencert $CCE $tmpdir/cert-expired 2>/dev/null || \ err 'Reported expired certificate as valid' echo > $tmpdir/cert-invalid $CCE $tmpdir/cert-invalid 2>/dev/null && \ err 'Reported invalid certificate as expired' echo 'Hello World' > $tmpdir/cert-invalid2 $CCE $tmpdir/cert-invalid2 2>/dev/null && \ err 'Reported invalid certificate as expired' exit 0 ganeti-3.1.0~rc2/test/py/legacy/cli-test.bash000075500000000000000000000003131476477700300210420ustar00rootroot00000000000000#!/bin/bash export SCRIPTS=${TOP_BUILDDIR:-.}/scripts export DAEMONS=${TOP_BUILDDIR:-.}/daemons shelltest $SHELLTESTARGS \ ${TOP_SRCDIR:-.}/test/py/legacy/{gnt,ganeti}-*.test \ -- --hide-successes ganeti-3.1.0~rc2/test/py/legacy/cmdlib/000075500000000000000000000000001476477700300177115ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/legacy/cmdlib/__init__.py000064400000000000000000000025561476477700300220320ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """This module contains all cmdlib unit tests""" ganeti-3.1.0~rc2/test/py/legacy/cmdlib/backup_unittest.py000064400000000000000000000246511476477700300234770ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tests for LUBackup*""" from ganeti import constants from ganeti import objects from ganeti import opcodes from ganeti import query from testsupport import * import testutils class TestLUBackupPrepare(CmdlibTestCase): @patchUtils("instance_utils") def testPrepareLocalExport(self, utils): utils.ReadOneLineFile.return_value = "cluster_secret" inst = self.cfg.AddNewInstance() op = opcodes.OpBackupPrepare(instance_name=inst.name, mode=constants.EXPORT_MODE_LOCAL) self.ExecOpCode(op) @patchUtils("instance_utils") def testPrepareRemoteExport(self, utils): utils.ReadOneLineFile.return_value = "cluster_secret" inst = self.cfg.AddNewInstance() self.rpc.call_x509_cert_create.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(inst.primary_node, ("key_name", testutils.ReadTestData("cert1.pem"))) op = opcodes.OpBackupPrepare(instance_name=inst.name, mode=constants.EXPORT_MODE_REMOTE) self.ExecOpCode(op) def InstanceRemoved(remove_instance): """Checks whether the instance was removed during a test of opcode execution. """ def WrappingFunction(fn): def CheckingFunction(self, *args, **kwargs): fn(self, *args, **kwargs) instance_removed = (self.rpc.call_blockdev_remove.called - self.rpc.call_blockdev_snapshot.called) > 0 if remove_instance and not instance_removed: raise self.fail(msg="Instance not removed when it should have been") if not remove_instance and instance_removed: raise self.fail(msg="Instance removed when it should not have been") return CheckingFunction return WrappingFunction def TrySnapshots(try_snapshot): """Checks whether an attempt to snapshot disks should have been attempted. """ def WrappingFunction(fn): def CheckingFunction(self, *args, **kwargs): fn(self, *args, **kwargs) snapshots_tried = self.rpc.call_blockdev_snapshot.called > 0 if try_snapshot and not snapshots_tried: raise self.fail(msg="Disks should have been snapshotted but weren't") if not try_snapshot and snapshots_tried: raise self.fail(msg="Disks snapshotted without a need to do so") return CheckingFunction return WrappingFunction class TestLUBackupExportBase(CmdlibTestCase): def setUp(self): super(TestLUBackupExportBase, self).setUp() self.rpc.call_instance_start.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) self.rpc.call_blockdev_assemble.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, ("/dev/mock_path", "/dev/mock_link_name", None)) self.rpc.call_blockdev_shutdown.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, None) self.rpc.call_blockdev_snapshot.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, ("mock_vg", "mock_id")) self.rpc.call_blockdev_remove.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, None) self.rpc.call_export_start.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, "export_daemon") def ImpExpStatus(node_uuid, name): return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node_uuid, [objects.ImportExportStatus( exit_status=0 )]) self.rpc.call_impexp_status.side_effect = ImpExpStatus def ImpExpCleanup(node_uuid, name): return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node_uuid) self.rpc.call_impexp_cleanup.side_effect = ImpExpCleanup self.rpc.call_finalize_export.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, None) def testRemoveRunningInstanceWithoutShutdown(self): inst = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP) op = opcodes.OpBackupExport(instance_name=inst.name, target_node=self.master.name, shutdown=False, remove_instance=True) self.ExecOpCodeExpectOpPrereqError( op, "Can not remove instance without shutting it down before") class TestLUBackupExportLocalExport(TestLUBackupExportBase): def setUp(self): # The initial instance prep super(TestLUBackupExportLocalExport, self).setUp() self.target_node = self.cfg.AddNewNode() self.op = opcodes.OpBackupExport(mode=constants.EXPORT_MODE_LOCAL, target_node=self.target_node.name) self._PrepareInstance() self.rpc.call_import_start.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.target_node, "import_daemon") def _PrepareInstance(self, online=False, snapshottable=True): """Produces an instance for export tests, and updates the opcode. """ if online: admin_state = constants.ADMINST_UP else: admin_state = constants.ADMINST_DOWN if snapshottable: disk_template = constants.DT_PLAIN else: disk_template = constants.DT_FILE inst = self.cfg.AddNewInstance(admin_state=admin_state, disk_template=disk_template) self.op = self.CopyOpCode(self.op, instance_name=inst.name) @TrySnapshots(True) @InstanceRemoved(False) def testPlainExportWithShutdown(self): self._PrepareInstance(online=True) self.ExecOpCode(self.op) @TrySnapshots(False) @InstanceRemoved(False) def testFileExportWithShutdown(self): self._PrepareInstance(online=True, snapshottable=False) self.ExecOpCodeExpectOpExecError(self.op, ".*--long-sleep option.*") @TrySnapshots(False) @InstanceRemoved(False) def testFileLongSleepExport(self): self._PrepareInstance(online=True, snapshottable=False) op = self.CopyOpCode(self.op, long_sleep=True) self.ExecOpCode(op) @TrySnapshots(True) @InstanceRemoved(False) def testPlainLiveExport(self): self._PrepareInstance(online=True) op = self.CopyOpCode(self.op, shutdown=False) self.ExecOpCode(op) @TrySnapshots(False) @InstanceRemoved(False) def testFileLiveExport(self): self._PrepareInstance(online=True, snapshottable=False) op = self.CopyOpCode(self.op, shutdown=False) self.ExecOpCodeExpectOpExecError(op, ".*live export.*") @TrySnapshots(False) @InstanceRemoved(False) def testPlainOfflineExport(self): self._PrepareInstance(online=False) self.ExecOpCode(self.op) @TrySnapshots(False) @InstanceRemoved(False) def testFileOfflineExport(self): self._PrepareInstance(online=False, snapshottable=False) self.ExecOpCode(self.op) @TrySnapshots(False) @InstanceRemoved(True) def testExportRemoveOfflineInstance(self): self._PrepareInstance(online=False) op = self.CopyOpCode(self.op, remove_instance=True) self.ExecOpCode(op) @TrySnapshots(False) @InstanceRemoved(True) def testExportRemoveOnlineInstance(self): self._PrepareInstance(online=True) op = self.CopyOpCode(self.op, remove_instance=True) self.ExecOpCode(op) @TrySnapshots(False) @InstanceRemoved(False) def testValidCompressionTool(self): op = self.CopyOpCode(self.op, compress="lzop") self.cfg.SetCompressionTools(["gzip", "lzop"]) self.ExecOpCode(op) @InstanceRemoved(False) def testInvalidCompressionTool(self): op = self.CopyOpCode(self.op, compress="invalid") self.cfg.SetCompressionTools(["gzip", "lzop"]) self.ExecOpCodeExpectOpPrereqError(op, "Compression tool not allowed") def testLiveLongSleep(self): op = self.CopyOpCode(self.op, shutdown=False, long_sleep=True) self.ExecOpCodeExpectOpPrereqError(op, ".*long sleep.*") class TestLUBackupExportRemoteExport(TestLUBackupExportBase): def setUp(self): super(TestLUBackupExportRemoteExport, self).setUp() self.inst = self.cfg.AddNewInstance() self.op = opcodes.OpBackupExport(mode=constants.EXPORT_MODE_REMOTE, instance_name=self.inst.name, target_node=[], x509_key_name=["mock_key_name"], destination_x509_ca="mock_dest_ca") @InstanceRemoved(False) def testRemoteExportWithoutX509KeyName(self): op = self.CopyOpCode(self.op, x509_key_name=self.REMOVE) self.ExecOpCodeExpectOpPrereqError(op, "Missing X509 key name for encryption") @InstanceRemoved(False) def testRemoteExportWithoutX509DestCa(self): op = self.CopyOpCode(self.op, destination_x509_ca=self.REMOVE) self.ExecOpCodeExpectOpPrereqError(op, "Missing destination X509 CA") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/cluster_unittest.py000064400000000000000000002716651476477700300237240ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2008, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tests for LUCluster* """ import OpenSSL import copy import unittest import re import shutil import os from ganeti.cmdlib import cluster from ganeti.cmdlib.cluster import verify from ganeti import constants from ganeti import errors from ganeti import netutils from ganeti import objects from ganeti import opcodes from ganeti import utils from ganeti import pathutils from ganeti import query from ganeti.hypervisor import hv_xen from testsupport import * import testutils class TestClusterVerifySsh(unittest.TestCase): def testMultipleGroups(self): fn = verify.LUClusterVerifyGroup._SelectSshCheckNodes mygroupnodes = [ objects.Node(name="node20", group="my", offline=False, master_candidate=True), objects.Node(name="node21", group="my", offline=False, master_candidate=True), objects.Node(name="node22", group="my", offline=False, master_candidate=False), objects.Node(name="node23", group="my", offline=False, master_candidate=True), objects.Node(name="node24", group="my", offline=False, master_candidate=True), objects.Node(name="node25", group="my", offline=False, master_candidate=False), objects.Node(name="node26", group="my", offline=True, master_candidate=True), ] nodes = [ objects.Node(name="node1", group="g1", offline=True, master_candidate=True), objects.Node(name="node2", group="g1", offline=False, master_candidate=False), objects.Node(name="node3", group="g1", offline=False, master_candidate=True), objects.Node(name="node4", group="g1", offline=True, master_candidate=True), objects.Node(name="node5", group="g1", offline=False, master_candidate=True), objects.Node(name="node10", group="xyz", offline=False, master_candidate=True), objects.Node(name="node11", group="xyz", offline=False, master_candidate=True), objects.Node(name="node40", group="alloff", offline=True, master_candidate=True), objects.Node(name="node41", group="alloff", offline=True, master_candidate=True), objects.Node(name="node50", group="aaa", offline=False, master_candidate=True), ] + mygroupnodes assert not utils.FindDuplicates(n.name for n in nodes) (online, perhost, _) = fn(mygroupnodes, "my", nodes) self.assertEqual(online, ["node%s" % i for i in range(20, 26)]) self.assertEqual(set(perhost.keys()), set(online)) self.assertEqual(perhost, { "node20": ["node10", "node2", "node50"], "node21": ["node11", "node3", "node50"], "node22": ["node10", "node5", "node50"], "node23": ["node11", "node2", "node50"], "node24": ["node10", "node3", "node50"], "node25": ["node11", "node5", "node50"], }) def testSingleGroup(self): fn = verify.LUClusterVerifyGroup._SelectSshCheckNodes nodes = [ objects.Node(name="node1", group="default", offline=True, master_candidate=True), objects.Node(name="node2", group="default", offline=False, master_candidate=True), objects.Node(name="node3", group="default", offline=False, master_candidate=True), objects.Node(name="node4", group="default", offline=True, master_candidate=True), ] assert not utils.FindDuplicates(n.name for n in nodes) (online, perhost, _) = fn(nodes, "default", nodes) self.assertEqual(online, ["node2", "node3"]) self.assertEqual(set(perhost.keys()), set(online)) self.assertEqual(perhost, { "node2": [], "node3": [], }) class TestLUClusterActivateMasterIp(CmdlibTestCase): def testSuccess(self): op = opcodes.OpClusterActivateMasterIp() self.rpc.call_node_activate_master_ip.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.rpc.call_node_activate_master_ip.assert_called_once_with( self.master_uuid, self._MatchMasterParams(), False) def testFailure(self): op = opcodes.OpClusterActivateMasterIp() self.rpc.call_node_activate_master_ip.return_value = \ self.RpcResultsBuilder() \ .CreateFailedNodeResult(self.master) \ self.ExecOpCodeExpectOpExecError(op) class TestLUClusterDeactivateMasterIp(CmdlibTestCase): def testSuccess(self): op = opcodes.OpClusterDeactivateMasterIp() self.rpc.call_node_deactivate_master_ip.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.rpc.call_node_deactivate_master_ip.assert_called_once_with( self.master_uuid, self._MatchMasterParams(), False) def testFailure(self): op = opcodes.OpClusterDeactivateMasterIp() self.rpc.call_node_deactivate_master_ip.return_value = \ self.RpcResultsBuilder() \ .CreateFailedNodeResult(self.master) \ self.ExecOpCodeExpectOpExecError(op) class TestLUClusterConfigQuery(CmdlibTestCase): def testInvalidField(self): op = opcodes.OpClusterConfigQuery(output_fields=["pinky_bunny"]) self.ExecOpCodeExpectOpPrereqError(op, "pinky_bunny") def testAllFields(self): op = opcodes.OpClusterConfigQuery(output_fields=list(query.CLUSTER_FIELDS)) self.rpc.call_get_watcher_pause.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, -1) ret = self.ExecOpCode(op) self.assertEqual(1, self.rpc.call_get_watcher_pause.call_count) self.assertEqual(len(ret), len(query.CLUSTER_FIELDS)) def testEmpytFields(self): op = opcodes.OpClusterConfigQuery(output_fields=[]) self.ExecOpCode(op) self.assertFalse(self.rpc.call_get_watcher_pause.called) class TestLUClusterDestroy(CmdlibTestCase): def testExistingNodes(self): op = opcodes.OpClusterDestroy() self.cfg.AddNewNode() self.cfg.AddNewNode() self.ExecOpCodeExpectOpPrereqError(op, "still 2 node\(s\)") def testExistingInstances(self): op = opcodes.OpClusterDestroy() self.cfg.AddNewInstance() self.cfg.AddNewInstance() self.ExecOpCodeExpectOpPrereqError(op, "still 2 instance\(s\)") def testEmptyCluster(self): op = opcodes.OpClusterDestroy() self.ExecOpCode(op) self.assertSingleHooksCall([self.master.name], "cluster-destroy", constants.HOOKS_PHASE_POST) class TestLUClusterPostInit(CmdlibTestCase): def testExecution(self): op = opcodes.OpClusterPostInit() self.ExecOpCode(op) self.assertSingleHooksCall([self.master.uuid], "cluster-init", constants.HOOKS_PHASE_POST) class TestLUClusterQuery(CmdlibTestCase): def testSimpleInvocation(self): op = opcodes.OpClusterQuery() self.ExecOpCode(op) def testIPv6Cluster(self): op = opcodes.OpClusterQuery() self.cluster.primary_ip_family = netutils.IP6Address.family self.ExecOpCode(op) class TestLUClusterRedistConf(CmdlibTestCase): def testSimpleInvocation(self): op = opcodes.OpClusterRedistConf() self.ExecOpCode(op) class TestLUClusterRename(CmdlibTestCase): NEW_NAME = "new-name.example.com" NEW_IP = "203.0.113.100" def testNoChanges(self): op = opcodes.OpClusterRename(name=self.cfg.GetClusterName()) self.ExecOpCodeExpectOpPrereqError(op, "name nor the IP address") def testReachableIp(self): op = opcodes.OpClusterRename(name=self.NEW_NAME) self.netutils_mod.GetHostname.return_value = \ HostnameMock(self.NEW_NAME, self.NEW_IP) self.netutils_mod.TcpPing.return_value = True self.ExecOpCodeExpectOpPrereqError(op, "is reachable on the network") def testValidRename(self): op = opcodes.OpClusterRename(name=self.NEW_NAME) self.netutils_mod.GetHostname.return_value = \ HostnameMock(self.NEW_NAME, self.NEW_IP) self.ExecOpCode(op) self.assertEqual(1, self.ssh_mod.WriteKnownHostsFile.call_count) self.rpc.call_node_deactivate_master_ip.assert_called_once_with( self.master_uuid, self._MatchMasterParams(), False) self.rpc.call_node_activate_master_ip.assert_called_once_with( self.master_uuid, self._MatchMasterParams(), False) def testRenameOfflineMaster(self): op = opcodes.OpClusterRename(name=self.NEW_NAME) self.master.offline = True self.netutils_mod.GetHostname.return_value = \ HostnameMock(self.NEW_NAME, self.NEW_IP) self.ExecOpCode(op) class TestLUClusterRepairDiskSizes(CmdlibTestCase): def testNoInstances(self): op = opcodes.OpClusterRepairDiskSizes() self.ExecOpCode(op) def _SetUpInstanceSingleDisk(self, dev_type=constants.DT_PLAIN): pnode = self.master snode = self.cfg.AddNewNode() disk = self.cfg.CreateDisk(dev_type=dev_type, primary_node=pnode, secondary_node=snode) inst = self.cfg.AddNewInstance(disks=[disk]) return (inst, disk) def testSingleInstanceOnFailingNode(self): (inst, _) = self._SetUpInstanceSingleDisk() op = opcodes.OpClusterRepairDiskSizes(instances=[inst.name]) self.rpc.call_blockdev_getdimensions.return_value = \ self.RpcResultsBuilder() \ .CreateFailedNodeResult(self.master) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("Failure in blockdev_getdimensions") def _ExecOpClusterRepairDiskSizes(self, node_data): # not specifying instances repairs all op = opcodes.OpClusterRepairDiskSizes() self.rpc.call_blockdev_getdimensions.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, node_data) return self.ExecOpCode(op) def testInvalidResultData(self): for data in [[], [None], ["invalid"], [("still", "invalid")]]: self.ResetMocks() self._SetUpInstanceSingleDisk() self._ExecOpClusterRepairDiskSizes(data) self.mcpu.assertLogContainsRegex("ignoring") def testCorrectSize(self): self._SetUpInstanceSingleDisk() changed = self._ExecOpClusterRepairDiskSizes([(1024 * 1024 * 1024, None)]) self.mcpu.assertLogIsEmpty() self.assertEqual(0, len(changed)) def testWrongSize(self): self._SetUpInstanceSingleDisk() changed = self._ExecOpClusterRepairDiskSizes([(512 * 1024 * 1024, None)]) self.assertEqual(1, len(changed)) def testCorrectDRBD(self): self._SetUpInstanceSingleDisk(dev_type=constants.DT_DRBD8) changed = self._ExecOpClusterRepairDiskSizes([(1024 * 1024 * 1024, None)]) self.mcpu.assertLogIsEmpty() self.assertEqual(0, len(changed)) def testWrongDRBDChild(self): (_, disk) = self._SetUpInstanceSingleDisk(dev_type=constants.DT_DRBD8) disk.children[0].size = 512 changed = self._ExecOpClusterRepairDiskSizes([(1024 * 1024 * 1024, None)]) self.assertEqual(1, len(changed)) def testExclusiveStorageInvalidResultData(self): self._SetUpInstanceSingleDisk() self.master.ndparams[constants.ND_EXCLUSIVE_STORAGE] = True self._ExecOpClusterRepairDiskSizes([(1024 * 1024 * 1024, None)]) self.mcpu.assertLogContainsRegex( "did not return valid spindles information") def testExclusiveStorageCorrectSpindles(self): (_, disk) = self._SetUpInstanceSingleDisk() disk.spindles = 1 self.master.ndparams[constants.ND_EXCLUSIVE_STORAGE] = True changed = self._ExecOpClusterRepairDiskSizes([(1024 * 1024 * 1024, 1)]) self.assertEqual(0, len(changed)) def testExclusiveStorageWrongSpindles(self): self._SetUpInstanceSingleDisk() self.master.ndparams[constants.ND_EXCLUSIVE_STORAGE] = True changed = self._ExecOpClusterRepairDiskSizes([(1024 * 1024 * 1024, 1)]) self.assertEqual(1, len(changed)) class TestLUClusterSetParams(CmdlibTestCase): UID_POOL = [(10, 1000)] def testUidPool(self): op = opcodes.OpClusterSetParams(uid_pool=self.UID_POOL) self.ExecOpCode(op) self.assertEqual(self.UID_POOL, self.cluster.uid_pool) def testAddUids(self): old_pool = [(1, 9)] self.cluster.uid_pool = list(old_pool) op = opcodes.OpClusterSetParams(add_uids=self.UID_POOL) self.ExecOpCode(op) self.assertEqual(set(self.UID_POOL + old_pool), set(self.cluster.uid_pool)) def testRemoveUids(self): additional_pool = [(1, 9)] self.cluster.uid_pool = self.UID_POOL + additional_pool op = opcodes.OpClusterSetParams(remove_uids=self.UID_POOL) self.ExecOpCode(op) self.assertEqual(additional_pool, self.cluster.uid_pool) def testMacPrefix(self): mac_prefix = "aa:01:02" op = opcodes.OpClusterSetParams(mac_prefix=mac_prefix) self.ExecOpCode(op) self.assertEqual(mac_prefix, self.cluster.mac_prefix) def testEmptyMacPrefix(self): mac_prefix = "" op = opcodes.OpClusterSetParams(mac_prefix=mac_prefix) self.ExecOpCodeExpectOpPrereqError( op, "Parameter 'OP_CLUSTER_SET_PARAMS.mac_prefix' fails validation") def testInvalidMacPrefix(self): mac_prefix = "az:00:00" op = opcodes.OpClusterSetParams(mac_prefix=mac_prefix) self.ExecOpCodeExpectOpPrereqError(op, "Invalid MAC address prefix") def testMasterNetmask(self): op = opcodes.OpClusterSetParams(master_netmask=26) self.ExecOpCode(op) self.assertEqual(26, self.cluster.master_netmask) def testInvalidDiskparams(self): for diskparams in [{constants.DT_DISKLESS: {constants.LV_STRIPES: 0}}, {constants.DT_DRBD8: {constants.RBD_POOL: "pool"}}, {constants.DT_DRBD8: {constants.RBD_ACCESS: "bunny"}}]: self.ResetMocks() op = opcodes.OpClusterSetParams(diskparams=diskparams) self.ExecOpCodeExpectOpPrereqError(op, "verify diskparams") def testValidDiskparams(self): diskparams = {constants.DT_RBD: {constants.RBD_POOL: "mock_pool", constants.RBD_ACCESS: "kernelspace"}} op = opcodes.OpClusterSetParams(diskparams=diskparams) self.ExecOpCode(op) self.assertEqual(diskparams[constants.DT_RBD], self.cluster.diskparams[constants.DT_RBD]) def testMinimalDiskparams(self): diskparams = {constants.DT_RBD: {constants.RBD_POOL: "mock_pool"}} self.cluster.diskparams = {} op = opcodes.OpClusterSetParams(diskparams=diskparams) self.ExecOpCode(op) self.assertEqual(diskparams, self.cluster.diskparams) def testValidDiskparamsAccess(self): for value in constants.DISK_VALID_ACCESS_MODES: self.ResetMocks() op = opcodes.OpClusterSetParams(diskparams={ constants.DT_RBD: {constants.RBD_ACCESS: value} }) self.ExecOpCode(op) got = self.cluster.diskparams[constants.DT_RBD][constants.RBD_ACCESS] self.assertEqual(value, got) def testInvalidDiskparamsAccess(self): for value in ["default", "pinky_bunny"]: self.ResetMocks() op = opcodes.OpClusterSetParams(diskparams={ constants.DT_RBD: {constants.RBD_ACCESS: value} }) self.ExecOpCodeExpectOpPrereqError(op, "Invalid value of 'rbd:access'") def testUnsetDrbdHelperWithDrbdDisks(self): self.cfg.AddNewInstance(disks=[ self.cfg.CreateDisk(dev_type=constants.DT_DRBD8, create_nodes=True)]) op = opcodes.OpClusterSetParams(drbd_helper="") self.ExecOpCodeExpectOpPrereqError(op, "Cannot disable drbd helper") def testFileStorageDir(self): op = opcodes.OpClusterSetParams(file_storage_dir="/random/path") self.ExecOpCode(op) self.assertEqual("/random/path", self.cluster.file_storage_dir) def testSetFileStorageDirToCurrentValue(self): op = opcodes.OpClusterSetParams( file_storage_dir=self.cluster.file_storage_dir) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("file storage dir already set to value") def testUnsetFileStorageDirFileStorageEnabled(self): self.cfg.SetEnabledDiskTemplates([constants.DT_FILE]) op = opcodes.OpClusterSetParams(file_storage_dir='') self.ExecOpCodeExpectOpPrereqError(op, "Unsetting the 'file' storage") def testUnsetFileStorageDirFileStorageDisabled(self): self.cfg.SetEnabledDiskTemplates([constants.DT_PLAIN]) op = opcodes.OpClusterSetParams(file_storage_dir='') self.ExecOpCode(op) def testSetFileStorageDirFileStorageDisabled(self): self.cfg.SetEnabledDiskTemplates([constants.DT_PLAIN]) op = opcodes.OpClusterSetParams(file_storage_dir='/some/path/') self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("although file storage is not enabled") def testSharedFileStorageDir(self): op = opcodes.OpClusterSetParams(shared_file_storage_dir="/random/path") self.ExecOpCode(op) self.assertEqual("/random/path", self.cluster.shared_file_storage_dir) def testSetSharedFileStorageDirToCurrentValue(self): op = opcodes.OpClusterSetParams(shared_file_storage_dir="/random/path") self.ExecOpCode(op) op = opcodes.OpClusterSetParams(shared_file_storage_dir="/random/path") self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("shared file storage dir already set to" " value") def testUnsetSharedFileStorageDirSharedFileStorageEnabled(self): self.cfg.SetEnabledDiskTemplates([constants.DT_SHARED_FILE]) op = opcodes.OpClusterSetParams(shared_file_storage_dir='') self.ExecOpCodeExpectOpPrereqError(op, "Unsetting the 'sharedfile' storage") def testUnsetSharedFileStorageDirSharedFileStorageDisabled(self): self.cfg.SetEnabledDiskTemplates([constants.DT_PLAIN]) op = opcodes.OpClusterSetParams(shared_file_storage_dir='') self.ExecOpCode(op) def testSetSharedFileStorageDirSharedFileStorageDisabled(self): self.cfg.SetEnabledDiskTemplates([constants.DT_PLAIN]) op = opcodes.OpClusterSetParams(shared_file_storage_dir='/some/path/') self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("although sharedfile storage is not" " enabled") def testValidDrbdHelper(self): node1 = self.cfg.AddNewNode() node1.offline = True self.rpc.call_drbd_helper.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, "/bin/true") \ .AddOfflineNode(node1) \ .Build() op = opcodes.OpClusterSetParams(drbd_helper="/bin/true") self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("Not checking drbd helper on offline node") def testDrbdHelperFailingNode(self): self.rpc.call_drbd_helper.return_value = \ self.RpcResultsBuilder() \ .AddFailedNode(self.master) \ .Build() op = opcodes.OpClusterSetParams(drbd_helper="/bin/true") self.ExecOpCodeExpectOpPrereqError(op, "Error checking drbd helper") def testInvalidDrbdHelper(self): self.rpc.call_drbd_helper.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, "/bin/false") \ .Build() op = opcodes.OpClusterSetParams(drbd_helper="/bin/true") self.ExecOpCodeExpectOpPrereqError(op, "drbd helper is /bin/false") def testDrbdHelperWithoutDrbdDiskTemplate(self): drbd_helper = "/bin/random_helper" self.cfg.SetEnabledDiskTemplates([constants.DT_DISKLESS]) self.rpc.call_drbd_helper.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, drbd_helper) \ .Build() op = opcodes.OpClusterSetParams(drbd_helper=drbd_helper) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("but did not enable") def testResetDrbdHelperDrbdDisabled(self): drbd_helper = "" self.cfg.SetEnabledDiskTemplates([constants.DT_DISKLESS]) op = opcodes.OpClusterSetParams(drbd_helper=drbd_helper) self.ExecOpCode(op) self.assertEqual(None, self.cluster.drbd_usermode_helper) def testResetDrbdHelperDrbdEnabled(self): drbd_helper = "" self.cluster.enabled_disk_templates = [constants.DT_DRBD8] op = opcodes.OpClusterSetParams(drbd_helper=drbd_helper) self.ExecOpCodeExpectOpPrereqError( op, "Cannot disable drbd helper while DRBD is enabled.") def testEnableDrbdNoHelper(self): self.cluster.enabled_disk_templates = [constants.DT_DISKLESS] self.cluster.drbd_usermode_helper = None enabled_disk_templates = [constants.DT_DRBD8] op = opcodes.OpClusterSetParams( enabled_disk_templates=enabled_disk_templates) self.ExecOpCodeExpectOpPrereqError( op, "Cannot enable DRBD without a DRBD usermode helper set") def testEnableDrbdHelperSet(self): drbd_helper = "/bin/random_helper" self.rpc.call_drbd_helper.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, drbd_helper) \ .Build() self.cfg.SetEnabledDiskTemplates([constants.DT_DISKLESS]) self.cluster.drbd_usermode_helper = drbd_helper enabled_disk_templates = [constants.DT_DRBD8] op = opcodes.OpClusterSetParams( enabled_disk_templates=enabled_disk_templates, ipolicy={constants.IPOLICY_DTS: enabled_disk_templates}) self.ExecOpCode(op) self.assertEqual(drbd_helper, self.cluster.drbd_usermode_helper) def testDrbdHelperAlreadySet(self): drbd_helper = "/bin/true" self.rpc.call_drbd_helper.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, "/bin/true") \ .Build() self.cfg.SetEnabledDiskTemplates([constants.DT_DISKLESS]) op = opcodes.OpClusterSetParams(drbd_helper=drbd_helper) self.ExecOpCode(op) self.assertEqual(drbd_helper, self.cluster.drbd_usermode_helper) self.mcpu.assertLogContainsRegex("DRBD helper already in desired state") def testSetDrbdHelper(self): drbd_helper = "/bin/true" self.rpc.call_drbd_helper.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, "/bin/true") \ .Build() self.cluster.drbd_usermode_helper = "/bin/false" self.cfg.SetEnabledDiskTemplates([constants.DT_DRBD8]) op = opcodes.OpClusterSetParams(drbd_helper=drbd_helper) self.ExecOpCode(op) self.assertEqual(drbd_helper, self.cluster.drbd_usermode_helper) def testBeparams(self): beparams = {constants.BE_VCPUS: 32} op = opcodes.OpClusterSetParams(beparams=beparams) self.ExecOpCode(op) self.assertEqual(32, self.cluster .beparams[constants.PP_DEFAULT][constants.BE_VCPUS]) def testNdparams(self): ndparams = {constants.ND_EXCLUSIVE_STORAGE: True} op = opcodes.OpClusterSetParams(ndparams=ndparams) self.ExecOpCode(op) self.assertEqual(True, self.cluster .ndparams[constants.ND_EXCLUSIVE_STORAGE]) def testNdparamsResetOobProgram(self): ndparams = {constants.ND_OOB_PROGRAM: ""} op = opcodes.OpClusterSetParams(ndparams=ndparams) self.ExecOpCode(op) self.assertEqual(constants.NDC_DEFAULTS[constants.ND_OOB_PROGRAM], self.cluster.ndparams[constants.ND_OOB_PROGRAM]) def testHvState(self): hv_state = {constants.HT_FAKE: {constants.HVST_CPU_TOTAL: 8}} op = opcodes.OpClusterSetParams(hv_state=hv_state) self.ExecOpCode(op) self.assertEqual(8, self.cluster.hv_state_static [constants.HT_FAKE][constants.HVST_CPU_TOTAL]) def testDiskState(self): disk_state = { constants.DT_PLAIN: { "mock_vg": {constants.DS_DISK_TOTAL: 10} } } op = opcodes.OpClusterSetParams(disk_state=disk_state) self.ExecOpCode(op) self.assertEqual(10, self.cluster .disk_state_static[constants.DT_PLAIN]["mock_vg"] [constants.DS_DISK_TOTAL]) def testDefaultIPolicy(self): ipolicy = constants.IPOLICY_DEFAULTS op = opcodes.OpClusterSetParams(ipolicy=ipolicy) self.ExecOpCode(op) def testIPolicyNewViolation(self): import ganeti.constants as C ipolicy = C.IPOLICY_DEFAULTS ipolicy[C.ISPECS_MINMAX][0][C.ISPECS_MIN][C.ISPEC_MEM_SIZE] = 128 ipolicy[C.ISPECS_MINMAX][0][C.ISPECS_MAX][C.ISPEC_MEM_SIZE] = 128 self.cfg.AddNewInstance(beparams={C.BE_MINMEM: 512, C.BE_MAXMEM: 512}) op = opcodes.OpClusterSetParams(ipolicy=ipolicy) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("instances violate them") def testNicparamsNoInstance(self): nicparams = { constants.NIC_LINK: "mock_bridge" } op = opcodes.OpClusterSetParams(nicparams=nicparams) self.ExecOpCode(op) self.assertEqual("mock_bridge", self.cluster.nicparams [constants.PP_DEFAULT][constants.NIC_LINK]) def testNicparamsInvalidConf(self): nicparams = { constants.NIC_MODE: constants.NIC_MODE_BRIDGED, constants.NIC_LINK: "" } op = opcodes.OpClusterSetParams(nicparams=nicparams) self.ExecOpCodeExpectException(op, errors.ConfigurationError, "NIC link") def testNicparamsInvalidInstanceConf(self): nicparams = { constants.NIC_MODE: constants.NIC_MODE_BRIDGED, constants.NIC_LINK: "mock_bridge" } self.cfg.AddNewInstance(nics=[ self.cfg.CreateNic(nicparams={constants.NIC_LINK: None})]) op = opcodes.OpClusterSetParams(nicparams=nicparams) self.ExecOpCodeExpectOpPrereqError(op, "Missing bridged NIC link") def testNicparamsMissingIp(self): nicparams = { constants.NIC_MODE: constants.NIC_MODE_ROUTED } self.cfg.AddNewInstance() op = opcodes.OpClusterSetParams(nicparams=nicparams) self.ExecOpCodeExpectOpPrereqError(op, "routed NIC with no ip address") def testNicparamsWithInstance(self): nicparams = { constants.NIC_LINK: "mock_bridge" } self.cfg.AddNewInstance() op = opcodes.OpClusterSetParams(nicparams=nicparams) self.ExecOpCode(op) def testDefaultHvparams(self): hvparams = constants.HVC_DEFAULTS op = opcodes.OpClusterSetParams(hvparams=hvparams) self.ExecOpCode(op) self.assertEqual(hvparams, self.cluster.hvparams) def testMinimalHvparams(self): hvparams = { constants.HT_FAKE: { constants.HV_MIGRATION_MODE: constants.HT_MIGRATION_NONLIVE } } self.cluster.hvparams = {} op = opcodes.OpClusterSetParams(hvparams=hvparams) self.ExecOpCode(op) self.assertEqual(hvparams, self.cluster.hvparams) def testOsHvp(self): os_hvp = { "mocked_os": { constants.HT_FAKE: { constants.HV_MIGRATION_MODE: constants.HT_MIGRATION_NONLIVE } }, "other_os": constants.HVC_DEFAULTS } op = opcodes.OpClusterSetParams(os_hvp=os_hvp) self.ExecOpCode(op) self.assertEqual(constants.HT_MIGRATION_NONLIVE, self.cluster.os_hvp["mocked_os"][constants.HT_FAKE] [constants.HV_MIGRATION_MODE]) self.assertEqual(constants.HVC_DEFAULTS, self.cluster.os_hvp["other_os"]) def testRemoveOsHvp(self): os_hvp = {"mocked_os": {constants.HT_FAKE: None}} op = opcodes.OpClusterSetParams(os_hvp=os_hvp) self.ExecOpCode(op) assert constants.HT_FAKE not in self.cluster.os_hvp["mocked_os"] def testRemoveOsFromOsHvpList(self): os_hvp = { "mocked_os_1": { constants.HT_FAKE: { constants.HV_MIGRATION_MODE: constants.HT_MIGRATION_NONLIVE } }, "mocked_os_2": {} # This is the one that needs to be removed. } op = opcodes.OpClusterSetParams(os_hvp=os_hvp) self.ExecOpCode(op) assert (constants.HT_FAKE in self.cluster.os_hvp["mocked_os_1"] and "mocked_os_2" not in self.cluster.os_hvp) def testDefaultOsHvp(self): os_hvp = {"mocked_os": constants.HVC_DEFAULTS.copy()} self.cluster.os_hvp = {"mocked_os": {}} op = opcodes.OpClusterSetParams(os_hvp=os_hvp) self.ExecOpCode(op) self.assertEqual(os_hvp, self.cluster.os_hvp) def testOsparams(self): osparams = { "mocked_os": { "param1": "value1", "param2": None }, "other_os": { "param1": None } } self.cluster.osparams = {"other_os": {"param1": "value1"}} self.cluster.osparams_private_cluster = {} op = opcodes.OpClusterSetParams(osparams=osparams) self.ExecOpCode(op) self.assertEqual({"mocked_os": {"param1": "value1"}}, self.cluster.osparams) def testEnabledHypervisors(self): enabled_hypervisors = [constants.HT_XEN_HVM, constants.HT_XEN_PVM] op = opcodes.OpClusterSetParams(enabled_hypervisors=enabled_hypervisors) self.ExecOpCode(op) self.assertEqual(enabled_hypervisors, self.cluster.enabled_hypervisors) def testEnabledHypervisorsWithoutHypervisorParams(self): enabled_hypervisors = [constants.HT_FAKE] self.cluster.hvparams = {} op = opcodes.OpClusterSetParams(enabled_hypervisors=enabled_hypervisors) self.ExecOpCode(op) self.assertEqual(enabled_hypervisors, self.cluster.enabled_hypervisors) self.assertEqual(constants.HVC_DEFAULTS[constants.HT_FAKE], self.cluster.hvparams[constants.HT_FAKE]) @testutils.patch_object(utils, "FindFile") def testValidDefaultIallocator(self, find_file_mock): find_file_mock.return_value = "/random/path" default_iallocator = "/random/path" op = opcodes.OpClusterSetParams(default_iallocator=default_iallocator) self.ExecOpCode(op) self.assertEqual(default_iallocator, self.cluster.default_iallocator) @testutils.patch_object(utils, "FindFile") def testInvalidDefaultIallocator(self, find_file_mock): find_file_mock.return_value = None default_iallocator = "/random/path" op = opcodes.OpClusterSetParams(default_iallocator=default_iallocator) self.ExecOpCodeExpectOpPrereqError(op, "Invalid default iallocator script") def testEnabledDiskTemplates(self): enabled_disk_templates = [constants.DT_DISKLESS, constants.DT_PLAIN] op = opcodes.OpClusterSetParams( enabled_disk_templates=enabled_disk_templates, ipolicy={constants.IPOLICY_DTS: enabled_disk_templates}) self.ExecOpCode(op) self.assertEqual(enabled_disk_templates, self.cluster.enabled_disk_templates) def testEnabledDiskTemplatesVsIpolicy(self): enabled_disk_templates = [constants.DT_DISKLESS, constants.DT_PLAIN] op = opcodes.OpClusterSetParams( enabled_disk_templates=enabled_disk_templates, ipolicy={constants.IPOLICY_DTS: [constants.DT_FILE]}) self.ExecOpCodeExpectOpPrereqError(op, "but not enabled on the cluster") def testDisablingDiskTemplatesOfInstances(self): old_disk_templates = [constants.DT_DISKLESS, constants.DT_PLAIN] self.cfg.SetEnabledDiskTemplates(old_disk_templates) self.cfg.AddNewInstance( disks=[self.cfg.CreateDisk(dev_type=constants.DT_PLAIN)]) new_disk_templates = [constants.DT_DISKLESS, constants.DT_DRBD8] op = opcodes.OpClusterSetParams( enabled_disk_templates=new_disk_templates, ipolicy={constants.IPOLICY_DTS: new_disk_templates}) self.ExecOpCodeExpectOpPrereqError(op, "least one disk using it") def testEnabledDiskTemplatesWithoutVgName(self): enabled_disk_templates = [constants.DT_PLAIN] self.cluster.volume_group_name = None op = opcodes.OpClusterSetParams( enabled_disk_templates=enabled_disk_templates) self.ExecOpCodeExpectOpPrereqError(op, "specify a volume group") def testDisableDiskTemplateWithExistingInstance(self): enabled_disk_templates = [constants.DT_DISKLESS] self.cfg.AddNewInstance( disks=[self.cfg.CreateDisk(dev_type=constants.DT_PLAIN)]) op = opcodes.OpClusterSetParams( enabled_disk_templates=enabled_disk_templates, ipolicy={constants.IPOLICY_DTS: enabled_disk_templates}) self.ExecOpCodeExpectOpPrereqError(op, "Cannot disable disk template") def testDisableDiskTemplateWithExistingInstanceDiskless(self): self.cfg.AddNewInstance(disks=[]) enabled_disk_templates = [constants.DT_PLAIN] op = opcodes.OpClusterSetParams( enabled_disk_templates=enabled_disk_templates, ipolicy={constants.IPOLICY_DTS: enabled_disk_templates}) self.ExecOpCodeExpectOpPrereqError(op, "Cannot disable disk template") def testVgNameNoLvmDiskTemplateEnabled(self): vg_name = "test_vg" self.cfg.SetEnabledDiskTemplates([constants.DT_DISKLESS]) op = opcodes.OpClusterSetParams(vg_name=vg_name) self.ExecOpCode(op) self.assertEqual(vg_name, self.cluster.volume_group_name) self.mcpu.assertLogIsEmpty() def testUnsetVgNameWithLvmDiskTemplateEnabled(self): vg_name = "" self.cluster.enabled_disk_templates = [constants.DT_PLAIN] op = opcodes.OpClusterSetParams(vg_name=vg_name) self.ExecOpCodeExpectOpPrereqError(op, "Cannot unset volume group") def testUnsetVgNameWithLvmInstance(self): vg_name = "" self.cfg.AddNewInstance( disks=[self.cfg.CreateDisk(dev_type=constants.DT_PLAIN)]) op = opcodes.OpClusterSetParams(vg_name=vg_name) self.ExecOpCodeExpectOpPrereqError(op, "Cannot unset volume group") def testUnsetVgNameWithNoLvmDiskTemplateEnabled(self): vg_name = "" self.cfg.SetEnabledDiskTemplates([constants.DT_DISKLESS]) op = opcodes.OpClusterSetParams(vg_name=vg_name) self.ExecOpCode(op) self.assertEqual(None, self.cluster.volume_group_name) def testVgNameToOldName(self): vg_name = self.cluster.volume_group_name op = opcodes.OpClusterSetParams(vg_name=vg_name) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("already in desired state") def testVgNameWithFailingNode(self): vg_name = "test_vg" op = opcodes.OpClusterSetParams(vg_name=vg_name) self.rpc.call_vg_list.return_value = \ self.RpcResultsBuilder() \ .AddFailedNode(self.master) \ .Build() self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("Error while gathering data on node") def testVgNameWithValidNode(self): vg_name = "test_vg" op = opcodes.OpClusterSetParams(vg_name=vg_name) self.rpc.call_vg_list.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {vg_name: 1024 * 1024}) \ .Build() self.ExecOpCode(op) def testVgNameWithTooSmallNode(self): vg_name = "test_vg" op = opcodes.OpClusterSetParams(vg_name=vg_name) self.rpc.call_vg_list.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {vg_name: 1}) \ .Build() self.ExecOpCodeExpectOpPrereqError(op, "too small") def testMiscParameters(self): op = opcodes.OpClusterSetParams(candidate_pool_size=123, maintain_node_health=True, modify_etc_hosts=True, prealloc_wipe_disks=True, reserved_lvs=["/dev/mock_lv"], use_external_mip_script=True) self.ExecOpCode(op) self.mcpu.assertLogIsEmpty() self.assertEqual(123, self.cluster.candidate_pool_size) self.assertEqual(True, self.cluster.maintain_node_health) self.assertEqual(True, self.cluster.modify_etc_hosts) self.assertEqual(True, self.cluster.prealloc_wipe_disks) self.assertEqual(["/dev/mock_lv"], self.cluster.reserved_lvs) self.assertEqual(True, self.cluster.use_external_mip_script) def testAddHiddenOs(self): self.cluster.hidden_os = ["hidden1", "hidden2"] op = opcodes.OpClusterSetParams(hidden_os=[(constants.DDM_ADD, "hidden2"), (constants.DDM_ADD, "hidden3")]) self.ExecOpCode(op) self.assertEqual(["hidden1", "hidden2", "hidden3"], self.cluster.hidden_os) self.mcpu.assertLogContainsRegex("OS hidden2 already") def testRemoveBlacklistedOs(self): self.cluster.blacklisted_os = ["blisted1", "blisted2"] op = opcodes.OpClusterSetParams(blacklisted_os=[ (constants.DDM_REMOVE, "blisted2"), (constants.DDM_REMOVE, "blisted3")]) self.ExecOpCode(op) self.assertEqual(["blisted1"], self.cluster.blacklisted_os) self.mcpu.assertLogContainsRegex("OS blisted3 not found") def testMasterNetdev(self): master_netdev = "test_dev" op = opcodes.OpClusterSetParams(master_netdev=master_netdev) self.ExecOpCode(op) self.assertEqual(master_netdev, self.cluster.master_netdev) def testMasterNetdevFailNoForce(self): master_netdev = "test_dev" op = opcodes.OpClusterSetParams(master_netdev=master_netdev) self.rpc.call_node_deactivate_master_ip.return_value = \ self.RpcResultsBuilder() \ .CreateFailedNodeResult(self.master) self.ExecOpCodeExpectOpExecError(op, "Could not disable the master ip") def testMasterNetdevFailForce(self): master_netdev = "test_dev" op = opcodes.OpClusterSetParams(master_netdev=master_netdev, force=True) self.rpc.call_node_deactivate_master_ip.return_value = \ self.RpcResultsBuilder() \ .CreateFailedNodeResult(self.master) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("Could not disable the master ip") def testCompressionToolSuccess(self): compression_tools = ["certainly_not_a_default", "gzip"] op = opcodes.OpClusterSetParams(compression_tools=compression_tools) self.ExecOpCode(op) self.assertEqual(compression_tools, self.cluster.compression_tools) def testCompressionToolCompatibility(self): compression_tools = ["not_gzip", "not_not_not_gzip"] op = opcodes.OpClusterSetParams(compression_tools=compression_tools) self.ExecOpCodeExpectOpPrereqError(op, ".*the gzip utility must be.*") def testCompressionToolForbiddenValues(self): for value in ["none", "\"rm -rf all.all\"", "ls$IFS-la"]: compression_tools = [value, "gzip"] op = opcodes.OpClusterSetParams(compression_tools=compression_tools) self.ExecOpCodeExpectOpPrereqError(op, re.escape(value)) class TestLUClusterVerify(CmdlibTestCase): def testVerifyAllGroups(self): op = opcodes.OpClusterVerify() result = self.ExecOpCode(op) self.assertEqual(2, len(result["jobs"])) def testVerifyDefaultGroups(self): op = opcodes.OpClusterVerify(group_name="default") result = self.ExecOpCode(op) self.assertEqual(1, len(result["jobs"])) class TestLUClusterVerifyConfig(CmdlibTestCase): def setUp(self): super(TestLUClusterVerifyConfig, self).setUp() self._load_cert_patcher = testutils \ .patch_object(OpenSSL.crypto, "load_certificate") self._load_cert_mock = self._load_cert_patcher.start() self._verify_cert_patcher = testutils \ .patch_object(utils, "VerifyCertificate") self._verify_cert_mock = self._verify_cert_patcher.start() self._read_file_patcher = testutils.patch_object(utils, "ReadFile") self._read_file_mock = self._read_file_patcher.start() self._can_read_patcher = testutils.patch_object(utils, "CanRead") self._can_read_mock = self._can_read_patcher.start() self._can_read_mock.return_value = True self._read_file_mock.return_value = True self._verify_cert_mock.return_value = (None, "") self._load_cert_mock.return_value = True def tearDown(self): super(TestLUClusterVerifyConfig, self).tearDown() self._can_read_patcher.stop() self._read_file_patcher.stop() self._verify_cert_patcher.stop() self._load_cert_patcher.stop() def testSuccessfulRun(self): self.cfg.AddNewInstance() op = opcodes.OpClusterVerifyConfig() result = self.ExecOpCode(op) self.assertTrue(result) def testDanglingNode(self): node = self.cfg.AddNewNode() self.cfg.AddNewInstance(primary_node=node) node.group = "invalid" op = opcodes.OpClusterVerifyConfig() result = self.ExecOpCode(op) self.mcpu.assertLogContainsRegex( "following nodes \(and their instances\) belong to a non existing group") self.assertFalse(result) def testDanglingInstance(self): inst = self.cfg.AddNewInstance() inst.primary_node = "invalid" op = opcodes.OpClusterVerifyConfig() result = self.ExecOpCode(op) self.mcpu.assertLogContainsRegex( "following instances have a non-existing primary-node") self.assertFalse(result) def testDanglingDisk(self): self.cfg.AddOrphanDisk() op = opcodes.OpClusterVerifyConfig() result = self.ExecOpCode(op) self.assertTrue(result) class TestLUClusterVerifyGroup(CmdlibTestCase): def testEmptyNodeGroup(self): group = self.cfg.AddNewNodeGroup() op = opcodes.OpClusterVerifyGroup(group_name=group.name, verbose=True) result = self.ExecOpCode(op) self.assertTrue(result) self.mcpu.assertLogContainsRegex("Empty node group, skipping verification") def testSimpleInvocation(self): op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) def testSimpleInvocationWithInstance(self): self.cfg.AddNewInstance(disks=[]) op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) def testGhostNode(self): group = self.cfg.AddNewNodeGroup() node = self.cfg.AddNewNode(group=group.uuid, offline=True) self.master.offline = True self.cfg.AddNewInstance(disk_template=constants.DT_DRBD8, primary_node=self.master, secondary_node=node) self.rpc.call_blockdev_getmirrorstatus_multi.return_value = \ RpcResultsBuilder() \ .AddOfflineNode(self.master) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) def testValidRpcResult(self): self.cfg.AddNewInstance(disks=[]) self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) def testVerifyNodeDrbdSuccess(self): ninfo = self.cfg.AddNewNode() disk = self.cfg.CreateDisk(dev_type=constants.DT_DRBD8, primary_node=self.master, secondary_node=ninfo) instance = self.cfg.AddNewInstance(disks=[disk]) instanceinfo = self.cfg.GetAllInstancesInfo() disks_info = self.cfg.GetAllDisksInfo() drbd_map = {ninfo.uuid: {0: disk.uuid}} minors = verify.LUClusterVerifyGroup._ComputeDrbdMinors( ninfo, instanceinfo, disks_info, drbd_map, lambda *args: None) self.assertEqual(minors, {0: (disk.uuid, instance.uuid, False)}) class TestLUClusterVerifyClientCerts(CmdlibTestCase): def _AddNormalNode(self): self.normalnode = copy.deepcopy(self.master) self.normalnode.master_candidate = False self.normalnode.uuid = "normal-node-uuid" self.cfg.AddNode(self.normalnode, None) def testVerifyMasterCandidate(self): client_cert = "client-cert-digest" self.cluster.candidate_certs = {self.master.uuid: client_cert} self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {constants.NV_CLIENT_CERT: (None, client_cert)}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) def testVerifyMasterCandidateInvalid(self): client_cert = "client-cert-digest" self.cluster.candidate_certs = {self.master.uuid: client_cert} self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {constants.NV_CLIENT_CERT: (666, "Invalid Certificate")}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) regexps = ( "Client certificate", "failed validation", "gnt-cluster renew-crypto --new-node-certificates", ) for r in regexps: self.mcpu.assertLogContainsRegex(r) def testVerifyNoMasterCandidateMap(self): client_cert = "client-cert-digest" self.cluster.candidate_certs = {} self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {constants.NV_CLIENT_CERT: (None, client_cert)}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex( "list of master candidate certificates is empty") self.mcpu.assertLogContainsRegex( "gnt-cluster renew-crypto --new-node-certificates") def testVerifyNoSharingMasterCandidates(self): client_cert = "client-cert-digest" self.cluster.candidate_certs = { self.master.uuid: client_cert, "some-other-master-candidate-uuid": client_cert} self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {constants.NV_CLIENT_CERT: (None, client_cert)}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex( "two master candidates configured to use the same") self.mcpu.assertLogContainsRegex( "gnt-cluster renew-crypto --new-node-certificates") def testVerifyMasterCandidateCertMismatch(self): client_cert = "client-cert-digest" self.cluster.candidate_certs = {self.master.uuid: "different-cert-digest"} self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {constants.NV_CLIENT_CERT: (None, client_cert)}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("does not match its entry") self.mcpu.assertLogContainsRegex( "gnt-cluster renew-crypto --new-node-certificates") def testVerifyMasterCandidateUnregistered(self): client_cert = "client-cert-digest" self.cluster.candidate_certs = {"other-node-uuid": "different-cert-digest"} self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {constants.NV_CLIENT_CERT: (None, client_cert)}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("does not have an entry") self.mcpu.assertLogContainsRegex( "gnt-cluster renew-crypto --new-node-certificates") def testVerifyMasterCandidateOtherNodesCert(self): client_cert = "client-cert-digest" self.cluster.candidate_certs = {"other-node-uuid": client_cert} self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {constants.NV_CLIENT_CERT: (None, client_cert)}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("using a certificate of another node") self.mcpu.assertLogContainsRegex( "gnt-cluster renew-crypto --new-node-certificates") def testNormalNodeStillInList(self): self._AddNormalNode() client_cert_master = "client-cert-digest-master" client_cert_normal = "client-cert-digest-normal" self.cluster.candidate_certs = { self.normalnode.uuid: client_cert_normal, self.master.uuid: client_cert_master} self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.normalnode, {constants.NV_CLIENT_CERT: (None, client_cert_normal)}) \ .AddSuccessfulNode(self.master, {constants.NV_CLIENT_CERT: (None, client_cert_master)}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) regexps = ( "not a master candidate", "still listed", "gnt-cluster renew-crypto --new-node-certificates", ) for r in regexps: self.mcpu.assertLogContainsRegex(r) def testNormalNodeStealingMasterCandidateCert(self): self._AddNormalNode() client_cert_master = "client-cert-digest-master" self.cluster.candidate_certs = { self.master.uuid: client_cert_master} self.rpc.call_node_verify.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.normalnode, {constants.NV_CLIENT_CERT: (None, client_cert_master)}) \ .AddSuccessfulNode(self.master, {constants.NV_CLIENT_CERT: (None, client_cert_master)}) \ .Build() op = opcodes.OpClusterVerifyGroup(group_name="default", verbose=True) self.ExecOpCode(op) regexps = ( "not a master candidate", "certificate of another node which is master candidate", "gnt-cluster renew-crypto --new-node-certificates", ) for r in regexps: self.mcpu.assertLogContainsRegex(r) class TestLUClusterVerifyGroupMethods(CmdlibTestCase): """Base class for testing individual methods in LUClusterVerifyGroup. """ def setUp(self): super(TestLUClusterVerifyGroupMethods, self).setUp() self.op = opcodes.OpClusterVerifyGroup(group_name="default") def PrepareLU(self, lu): lu._exclusive_storage = False lu.master_node = self.master_uuid lu.group_info = self.group verify.LUClusterVerifyGroup.all_node_info = \ property(fget=lambda _: self.cfg.GetAllNodesInfo()) class TestLUClusterVerifyGroupVerifyNode(TestLUClusterVerifyGroupMethods): @withLockedLU def testInvalidNodeResult(self, lu): self.assertFalse(lu._VerifyNode(self.master, None)) self.assertFalse(lu._VerifyNode(self.master, "")) @withLockedLU def testInvalidVersion(self, lu): self.assertFalse(lu._VerifyNode(self.master, {"version": None})) self.assertFalse(lu._VerifyNode(self.master, {"version": ""})) self.assertFalse(lu._VerifyNode(self.master, { "version": (constants.PROTOCOL_VERSION - 1, constants.RELEASE_VERSION) })) self.mcpu.ClearLogMessages() self.assertTrue(lu._VerifyNode(self.master, { "version": (constants.PROTOCOL_VERSION, constants.RELEASE_VERSION + "x") })) self.mcpu.assertLogContainsRegex("software version mismatch") def _GetValidNodeResult(self, additional_fields): ret = { "version": (constants.PROTOCOL_VERSION, constants.RELEASE_VERSION), constants.NV_NODESETUP: [] } ret.update(additional_fields) return ret @withLockedLU def testHypervisor(self, lu): lu._VerifyNode(self.master, self._GetValidNodeResult({ constants.NV_HYPERVISOR: { constants.HT_XEN_PVM: None, constants.HT_XEN_HVM: "mock error" } })) self.mcpu.assertLogContainsRegex(constants.HT_XEN_HVM) self.mcpu.assertLogContainsRegex("mock error") @withLockedLU def testHvParams(self, lu): lu._VerifyNode(self.master, self._GetValidNodeResult({ constants.NV_HVPARAMS: [("mock item", constants.HT_XEN_HVM, "mock error")] })) self.mcpu.assertLogContainsRegex(constants.HT_XEN_HVM) self.mcpu.assertLogContainsRegex("mock item") self.mcpu.assertLogContainsRegex("mock error") @withLockedLU def testSuccessfulResult(self, lu): self.assertTrue(lu._VerifyNode(self.master, self._GetValidNodeResult({}))) self.mcpu.assertLogIsEmpty() class TestLUClusterVerifyGroupVerifyNodeTime(TestLUClusterVerifyGroupMethods): @withLockedLU def testInvalidNodeResult(self, lu): for ndata in [{}, {constants.NV_TIME: "invalid"}]: self.mcpu.ClearLogMessages() lu._VerifyNodeTime(self.master, ndata, None, None) self.mcpu.assertLogContainsRegex("Node returned invalid time") @withLockedLU def testNodeDiverges(self, lu): for ntime in [(0, 0), (2000, 0)]: self.mcpu.ClearLogMessages() lu._VerifyNodeTime(self.master, {constants.NV_TIME: ntime}, 1000, 1005) self.mcpu.assertLogContainsRegex("Node time diverges") @withLockedLU def testSuccessfulResult(self, lu): lu._VerifyNodeTime(self.master, {constants.NV_TIME: (0, 0)}, 0, 5) self.mcpu.assertLogIsEmpty() class TestLUClusterVerifyGroupUpdateVerifyNodeLVM( TestLUClusterVerifyGroupMethods): def setUp(self): super(TestLUClusterVerifyGroupUpdateVerifyNodeLVM, self).setUp() self.VALID_NRESULT = { constants.NV_VGLIST: {"mock_vg": 30000}, constants.NV_PVLIST: [ { "name": "mock_pv", "vg_name": "mock_vg", "size": 5000, "free": 2500, "attributes": [], "lv_list": [] } ] } @withLockedLU def testNoVgName(self, lu): lu._UpdateVerifyNodeLVM(self.master, {}, None, None) self.mcpu.assertLogIsEmpty() @withLockedLU def testEmptyNodeResult(self, lu): lu._UpdateVerifyNodeLVM(self.master, {}, "mock_vg", None) self.mcpu.assertLogContainsRegex("unable to check volume groups") self.mcpu.assertLogContainsRegex("Can't get PV list from node") @withLockedLU def testValidNodeResult(self, lu): lu._UpdateVerifyNodeLVM(self.master, self.VALID_NRESULT, "mock_vg", None) self.mcpu.assertLogIsEmpty() @withLockedLU def testValidNodeResultExclusiveStorage(self, lu): lu._exclusive_storage = True lu._UpdateVerifyNodeLVM(self.master, self.VALID_NRESULT, "mock_vg", verify.LUClusterVerifyGroup.NodeImage()) self.mcpu.assertLogIsEmpty() class TestLUClusterVerifyGroupVerifyGroupDRBDVersion( TestLUClusterVerifyGroupMethods): @withLockedLU def testEmptyNodeResult(self, lu): lu._VerifyGroupDRBDVersion({}) self.mcpu.assertLogIsEmpty() @withLockedLU def testValidNodeResult(self, lu): lu._VerifyGroupDRBDVersion( RpcResultsBuilder() .AddSuccessfulNode(self.master, { constants.NV_DRBDVERSION: "8.3.0" }) .Build()) self.mcpu.assertLogIsEmpty() @withLockedLU def testDifferentVersions(self, lu): node1 = self.cfg.AddNewNode() lu._VerifyGroupDRBDVersion( RpcResultsBuilder() .AddSuccessfulNode(self.master, { constants.NV_DRBDVERSION: "8.3.0" }) .AddSuccessfulNode(node1, { constants.NV_DRBDVERSION: "8.4.0" }) .Build()) self.mcpu.assertLogContainsRegex("DRBD version mismatch: 8.3.0") self.mcpu.assertLogContainsRegex("DRBD version mismatch: 8.4.0") class TestLUClusterVerifyGroupVerifyGroupLVM(TestLUClusterVerifyGroupMethods): @withLockedLU def testNoVgName(self, lu): lu._VerifyGroupLVM(None, None) self.mcpu.assertLogIsEmpty() @withLockedLU def testNoExclusiveStorage(self, lu): lu._VerifyGroupLVM(None, "mock_vg") self.mcpu.assertLogIsEmpty() @withLockedLU def testNoPvInfo(self, lu): lu._exclusive_storage = True nimg = verify.LUClusterVerifyGroup.NodeImage() lu._VerifyGroupLVM({self.master.uuid: nimg}, "mock_vg") self.mcpu.assertLogIsEmpty() @withLockedLU def testValidPvInfos(self, lu): lu._exclusive_storage = True node2 = self.cfg.AddNewNode() nimg1 = verify.LUClusterVerifyGroup.NodeImage(uuid=self.master.uuid) nimg1.pv_min = 10000 nimg1.pv_max = 10010 nimg2 = verify.LUClusterVerifyGroup.NodeImage(uuid=node2.uuid) nimg2.pv_min = 9998 nimg2.pv_max = 10005 lu._VerifyGroupLVM({self.master.uuid: nimg1, node2.uuid: nimg2}, "mock_vg") self.mcpu.assertLogIsEmpty() class TestLUClusterVerifyGroupVerifyNodeBridges( TestLUClusterVerifyGroupMethods): @withLockedLU def testNoBridges(self, lu): lu._VerifyNodeBridges(None, None, None) self.mcpu.assertLogIsEmpty() @withLockedLU def testInvalidBridges(self, lu): for ndata in [{}, {constants.NV_BRIDGES: ""}]: self.mcpu.ClearLogMessages() lu._VerifyNodeBridges(self.master, ndata, ["mock_bridge"]) self.mcpu.assertLogContainsRegex("not return valid bridge information") self.mcpu.ClearLogMessages() lu._VerifyNodeBridges(self.master, {constants.NV_BRIDGES: ["mock_bridge"]}, ["mock_bridge"]) self.mcpu.assertLogContainsRegex("missing bridge") class TestLUClusterVerifyGroupVerifyNodeUserScripts( TestLUClusterVerifyGroupMethods): @withLockedLU def testNoUserScripts(self, lu): lu._VerifyNodeUserScripts(self.master, {}) self.mcpu.assertLogContainsRegex("did not return user scripts information") @withLockedLU def testBrokenUserScripts(self, lu): lu._VerifyNodeUserScripts(self.master, {constants.NV_USERSCRIPTS: ["script"]}) self.mcpu.assertLogContainsRegex("scripts not present or not executable") class TestLUClusterVerifyGroupVerifyNodeNetwork( TestLUClusterVerifyGroupMethods): def setUp(self): super(TestLUClusterVerifyGroupVerifyNodeNetwork, self).setUp() self.VALID_NRESULT = { constants.NV_NODELIST: {}, constants.NV_NODENETTEST: {}, constants.NV_MASTERIP: True } @withLockedLU def testEmptyNodeResult(self, lu): lu._VerifyNodeNetwork(self.master, {}) self.mcpu.assertLogContainsRegex( "node hasn't returned node ssh connectivity data") self.mcpu.assertLogContainsRegex( "node hasn't returned node tcp connectivity data") self.mcpu.assertLogContainsRegex( "node hasn't returned node master IP reachability data") @withLockedLU def testValidResult(self, lu): lu._VerifyNodeNetwork(self.master, self.VALID_NRESULT) self.mcpu.assertLogIsEmpty() @withLockedLU def testSshProblem(self, lu): self.VALID_NRESULT.update({ constants.NV_NODELIST: { "mock_node": "mock_error" } }) lu._VerifyNodeNetwork(self.master, self.VALID_NRESULT) self.mcpu.assertLogContainsRegex("ssh communication with node 'mock_node'") @withLockedLU def testTcpProblem(self, lu): self.VALID_NRESULT.update({ constants.NV_NODENETTEST: { "mock_node": "mock_error" } }) lu._VerifyNodeNetwork(self.master, self.VALID_NRESULT) self.mcpu.assertLogContainsRegex("tcp communication with node 'mock_node'") @withLockedLU def testMasterIpNotReachable(self, lu): self.VALID_NRESULT.update({ constants.NV_MASTERIP: False }) node1 = self.cfg.AddNewNode() lu._VerifyNodeNetwork(self.master, self.VALID_NRESULT) self.mcpu.assertLogContainsRegex( "the master node cannot reach the master IP") self.mcpu.ClearLogMessages() lu._VerifyNodeNetwork(node1, self.VALID_NRESULT) self.mcpu.assertLogContainsRegex("cannot reach the master IP") class TestLUClusterVerifyGroupVerifyInstance(TestLUClusterVerifyGroupMethods): def setUp(self): super(TestLUClusterVerifyGroupVerifyInstance, self).setUp() self.node1 = self.cfg.AddNewNode() self.drbd_inst = self.cfg.AddNewInstance( disks=[self.cfg.CreateDisk(dev_type=constants.DT_DRBD8, primary_node=self.master, secondary_node=self.node1)]) self.running_inst = self.cfg.AddNewInstance( admin_state=constants.ADMINST_UP, disks_active=True) self.diskless_inst = self.cfg.AddNewInstance(disks=[]) self.master_img = \ verify.LUClusterVerifyGroup.NodeImage(uuid=self.master_uuid) self.master_img.volumes = ["/".join(disk.logical_id) for inst in [self.running_inst, self.diskless_inst] for disk in self.cfg.GetInstanceDisks(inst.uuid)] drbd_inst_disks = self.cfg.GetInstanceDisks(self.drbd_inst.uuid) self.master_img.volumes.extend( ["/".join(disk.logical_id) for disk in drbd_inst_disks[0].children]) self.master_img.instances = [self.running_inst.uuid] self.node1_img = \ verify.LUClusterVerifyGroup.NodeImage(uuid=self.node1.uuid) self.node1_img.volumes = \ ["/".join(disk.logical_id) for disk in drbd_inst_disks[0].children] self.node_imgs = { self.master_uuid: self.master_img, self.node1.uuid: self.node1_img } running_inst_disks = self.cfg.GetInstanceDisks(self.running_inst.uuid) self.diskstatus = { self.master_uuid: [ (True, objects.BlockDevStatus(ldisk_status=constants.LDS_OKAY)) for _ in running_inst_disks ] } @withLockedLU def testDisklessInst(self, lu): lu._VerifyInstance(self.diskless_inst, self.node_imgs, {}) self.mcpu.assertLogIsEmpty() @withLockedLU def testOfflineNode(self, lu): self.master_img.offline = True lu._VerifyInstance(self.drbd_inst, self.node_imgs, {}) self.mcpu.assertLogIsEmpty() @withLockedLU def testRunningOnOfflineNode(self, lu): self.master_img.offline = True lu._VerifyInstance(self.running_inst, self.node_imgs, {}) self.mcpu.assertLogContainsRegex( "instance is marked as running and lives on offline node") @withLockedLU def testMissingVolume(self, lu): self.master_img.volumes = [] lu._VerifyInstance(self.running_inst, self.node_imgs, {}) self.mcpu.assertLogContainsRegex("volume .* missing") @withLockedLU def testRunningInstanceOnWrongNode(self, lu): self.master_img.instances = [] self.diskless_inst.admin_state = constants.ADMINST_UP lu._VerifyInstance(self.running_inst, self.node_imgs, {}) self.mcpu.assertLogContainsRegex("instance not running on its primary node") @withLockedLU def testRunningInstanceOnRightNode(self, lu): self.master_img.instances = [self.running_inst.uuid] lu._VerifyInstance(self.running_inst, self.node_imgs, {}) self.mcpu.assertLogIsEmpty() @withLockedLU def testValidDiskStatus(self, lu): lu._VerifyInstance(self.running_inst, self.node_imgs, self.diskstatus) self.mcpu.assertLogIsEmpty() @withLockedLU def testDegradedDiskStatus(self, lu): self.diskstatus[self.master_uuid][0][1].is_degraded = True lu._VerifyInstance(self.running_inst, self.node_imgs, self.diskstatus) self.mcpu.assertLogContainsRegex("instance .* is degraded") @withLockedLU def testNotOkayDiskStatus(self, lu): self.diskstatus[self.master_uuid][0][1].is_degraded = True self.diskstatus[self.master_uuid][0][1].ldisk_status = constants.LDS_FAULTY lu._VerifyInstance(self.running_inst, self.node_imgs, self.diskstatus) self.mcpu.assertLogContainsRegex("instance .* state is 'faulty'") @withLockedLU def testExclusiveStorageWithInvalidInstance(self, lu): self.master.ndparams[constants.ND_EXCLUSIVE_STORAGE] = True lu._VerifyInstance(self.drbd_inst, self.node_imgs, self.diskstatus) self.mcpu.assertLogContainsRegex( "disk types? drbd, which are not supported") @withLockedLU def testExclusiveStorageWithValidInstance(self, lu): self.master.ndparams[constants.ND_EXCLUSIVE_STORAGE] = True running_inst_disks = self.cfg.GetInstanceDisks(self.running_inst.uuid) running_inst_disks[0].spindles = 1 feedback_fn = lambda _: None self.cfg.Update(running_inst_disks[0], feedback_fn) lu._VerifyInstance(self.running_inst, self.node_imgs, self.diskstatus) self.mcpu.assertLogIsEmpty() @withLockedLU def testDrbdInTwoGroups(self, lu): group = self.cfg.AddNewNodeGroup() self.node1.group = group.uuid lu._VerifyInstance(self.drbd_inst, self.node_imgs, self.diskstatus) self.mcpu.assertLogContainsRegex( "instance has primary and secondary nodes in different groups") @withLockedLU def testOfflineSecondary(self, lu): self.node1_img.offline = True lu._VerifyInstance(self.drbd_inst, self.node_imgs, self.diskstatus) self.mcpu.assertLogContainsRegex("instance has offline secondary node\(s\)") class TestLUClusterVerifyGroupVerifyOrphanVolumes( TestLUClusterVerifyGroupMethods): @withLockedLU def testOrphanedVolume(self, lu): master_img = verify.LUClusterVerifyGroup.NodeImage(uuid=self.master_uuid) master_img.volumes = [ "mock_vg/disk_0", # Required, present, no error "mock_vg/disk_1", # Unknown, present, orphan "mock_vg/disk_2", # Reserved, present, no error "other_vg/disk_0", # Required, present, no error "other_vg/disk_1", # Unknown, present, no error ] node_imgs = { self.master_uuid: master_img } node_vol_should = { self.master_uuid: ["mock_vg/disk_0", "other_vg/disk_0", "other_vg/disk_1"] } lu._VerifyOrphanVolumes("mock_vg", node_vol_should, node_imgs, utils.FieldSet("mock_vg/disk_2")) self.mcpu.assertLogContainsRegex("volume mock_vg/disk_1 is unknown") self.mcpu.assertLogDoesNotContainRegex("volume mock_vg/disk_0 is unknown") self.mcpu.assertLogDoesNotContainRegex("volume mock_vg/disk_2 is unknown") self.mcpu.assertLogDoesNotContainRegex("volume other_vg/disk_0 is unknown") self.mcpu.assertLogDoesNotContainRegex("volume other_vg/disk_1 is unknown") class TestLUClusterVerifyGroupVerifyNPlusOneMemory( TestLUClusterVerifyGroupMethods): @withLockedLU def testN1Failure(self, lu): group1 = self.cfg.AddNewNodeGroup() node1 = self.cfg.AddNewNode() node2 = self.cfg.AddNewNode(group=group1) node3 = self.cfg.AddNewNode() inst1 = self.cfg.AddNewInstance() inst2 = self.cfg.AddNewInstance() inst3 = self.cfg.AddNewInstance() node1_img = verify.LUClusterVerifyGroup.NodeImage(uuid=node1.uuid) node1_img.sbp = { self.master_uuid: [inst1.uuid, inst2.uuid, inst3.uuid] } node2_img = verify.LUClusterVerifyGroup.NodeImage(uuid=node2.uuid) node3_img = verify.LUClusterVerifyGroup.NodeImage(uuid=node3.uuid) node3_img.offline = True node_imgs = { node1.uuid: node1_img, node2.uuid: node2_img, node3.uuid: node3_img } lu._VerifyNPlusOneMemory(node_imgs, self.cfg.GetAllInstancesInfo()) self.mcpu.assertLogContainsRegex( "not enough memory to accomodate instance failovers") self.mcpu.ClearLogMessages() node1_img.mfree = 1000 lu._VerifyNPlusOneMemory(node_imgs, self.cfg.GetAllInstancesInfo()) self.mcpu.assertLogIsEmpty() class TestLUClusterVerifyGroupVerifyFiles(TestLUClusterVerifyGroupMethods): @withLockedLU def test(self, lu): node1 = self.cfg.AddNewNode(master_candidate=False, offline=False, vm_capable=True) node2 = self.cfg.AddNewNode(master_candidate=True, vm_capable=False) node3 = self.cfg.AddNewNode(master_candidate=False, offline=False, vm_capable=True) node4 = self.cfg.AddNewNode(master_candidate=False, offline=False, vm_capable=True) node5 = self.cfg.AddNewNode(master_candidate=False, offline=True) nodeinfo = [self.master, node1, node2, node3, node4, node5] files_all = set([ pathutils.CLUSTER_DOMAIN_SECRET_FILE, pathutils.RAPI_CERT_FILE, pathutils.RAPI_USERS_FILE, ]) files_opt = set([ pathutils.RAPI_USERS_FILE, hv_xen.XL_CONFIG_FILE, pathutils.VNC_PASSWORD_FILE, ]) files_mc = set([ pathutils.CLUSTER_CONF_FILE, ]) files_vm = set([ hv_xen.XL_CONFIG_FILE, pathutils.VNC_PASSWORD_FILE, ]) nvinfo = RpcResultsBuilder() \ .AddSuccessfulNode(self.master, { constants.NV_FILELIST: { pathutils.CLUSTER_CONF_FILE: "82314f897f38b35f9dab2f7c6b1593e0", pathutils.RAPI_CERT_FILE: "babbce8f387bc082228e544a2146fee4", pathutils.CLUSTER_DOMAIN_SECRET_FILE: "cds-47b5b3f19202936bb4", hv_xen.XL_CONFIG_FILE: "77935cee92afd26d162f9e525e3d49b9" }}) \ .AddSuccessfulNode(node1, { constants.NV_FILELIST: { pathutils.RAPI_CERT_FILE: "97f0356500e866387f4b84233848cc4a", } }) \ .AddSuccessfulNode(node2, { constants.NV_FILELIST: { pathutils.RAPI_CERT_FILE: "97f0356500e866387f4b84233848cc4a", pathutils.CLUSTER_DOMAIN_SECRET_FILE: "cds-47b5b3f19202936bb4", } }) \ .AddSuccessfulNode(node3, { constants.NV_FILELIST: { pathutils.RAPI_CERT_FILE: "97f0356500e866387f4b84233848cc4a", pathutils.CLUSTER_CONF_FILE: "conf-a6d4b13e407867f7a7b4f0f232a8f527", pathutils.CLUSTER_DOMAIN_SECRET_FILE: "cds-47b5b3f19202936bb4", pathutils.RAPI_USERS_FILE: "rapiusers-ea3271e8d810ef3", hv_xen.XL_CONFIG_FILE: "77935cee92afd26d162f9e525e3d49b9" } }) \ .AddSuccessfulNode(node4, {}) \ .AddOfflineNode(node5) \ .Build() assert set(nvinfo.keys()) == set(ni.uuid for ni in nodeinfo) lu._VerifyFiles(nodeinfo, self.master_uuid, nvinfo, (files_all, files_opt, files_mc, files_vm)) expected_msgs = [ "File %s found with 2 different checksums (variant 1 on" " %s, %s, %s; variant 2 on %s)" % (pathutils.RAPI_CERT_FILE, node1.name, node2.name, node3.name, self.master.name), "File %s is missing from node(s) %s" % (pathutils.CLUSTER_DOMAIN_SECRET_FILE, node1.name), "File %s should not exist on node(s) %s" % (pathutils.CLUSTER_CONF_FILE, node3.name), "File %s is missing from node(s) %s" % (pathutils.CLUSTER_CONF_FILE, node2.name), "File %s found with 2 different checksums (variant 1 on" " %s; variant 2 on %s)" % (pathutils.CLUSTER_CONF_FILE, self.master.name, node3.name), "File %s is optional, but it must exist on all or no nodes (not" " found on %s, %s, %s)" % (pathutils.RAPI_USERS_FILE, self.master.name, node1.name, node2.name), "File %s is optional, but it must exist on all or no nodes (not" " found on %s)" % (hv_xen.XL_CONFIG_FILE, node1.name), "Node did not return file checksum data", ] self.assertEqual(len(self.mcpu.GetLogMessages()), len(expected_msgs)) for expected_msg in expected_msgs: self.mcpu.assertLogContainsInLine(expected_msg) class TestLUClusterVerifyGroupVerifyNodeOs(TestLUClusterVerifyGroupMethods): @withLockedLU def testUpdateNodeOsInvalidNodeResult(self, lu): for ndata in [{}, {constants.NV_OSLIST: ""}, {constants.NV_OSLIST: [""]}, {constants.NV_OSLIST: [["1", "2"]]}]: self.mcpu.ClearLogMessages() nimage = verify.LUClusterVerifyGroup.NodeImage(uuid=self.master_uuid) lu._UpdateNodeOS(self.master, ndata, nimage) self.mcpu.assertLogContainsRegex("node hasn't returned valid OS data") @withLockedLU def testUpdateNodeOsValidNodeResult(self, lu): ndata = { constants.NV_OSLIST: [ ["mock_OS", "/mocked/path", True, "", ["default"], [], [constants.OS_API_V20], True], ["Another_Mock", "/random", True, "", ["var1", "var2"], [{"param1": "val1"}, {"param2": "val2"}], constants.OS_API_VERSIONS, True] ] } nimage = verify.LUClusterVerifyGroup.NodeImage(uuid=self.master_uuid) lu._UpdateNodeOS(self.master, ndata, nimage) self.mcpu.assertLogIsEmpty() @withLockedLU def testVerifyNodeOs(self, lu): node = self.cfg.AddNewNode() nimg_root = verify.LUClusterVerifyGroup.NodeImage(uuid=self.master_uuid) nimg = verify.LUClusterVerifyGroup.NodeImage(uuid=node.uuid) nimg_root.os_fail = False nimg_root.oslist = { "mock_os": [("/mocked/path", True, "", set(["default"]), set(), set([constants.OS_API_V20]), True)], "broken_base_os": [("/broken", False, "", set(), set(), set([constants.OS_API_V20]), True)], "only_on_root": [("/random", True, "", set(), set(), set(), True)], "diffing_os": [("/pinky", True, "", set(["var1", "var2"]), set([("param1", "val1"), ("param2", "val2")]), set([constants.OS_API_V20]), True)], "trust_os": [("/trust/mismatch", True, "", set(), set(), set(), True)], } nimg.os_fail = False nimg.oslist = { "mock_os": [("/mocked/path", True, "", set(["default"]), set(), set([constants.OS_API_V20]), True)], "only_on_test": [("/random", True, "", set(), set(), set(), True)], "diffing_os": [("/bunny", True, "", set(["var1", "var3"]), set([("param1", "val1"), ("param3", "val3")]), set([constants.OS_API_V15]), True)], "broken_os": [("/broken", False, "", set(), set(), set([constants.OS_API_V20]), True)], "multi_entries": [ ("/multi1", True, "", set(), set(), set([constants.OS_API_V20]), True), ("/multi2", True, "", set(), set(), set([constants.OS_API_V20]), True)], "trust_os": [("/trust/mismatch", True, "", set(), set(), set(), False)], } lu._VerifyNodeOS(node, nimg, nimg_root) expected_msgs = [ "Extra OS only_on_test not present on reference node", "OSes present on reference node .* but missing on this node:.*" + " only_on_root", "OS API version for diffing_os differs", "OS variants list for diffing_os differs", "OS parameters for diffing_os differs", "Invalid OS broken_os", "Extra OS broken_os not present on reference node", "OS 'multi_entries' has multiple entries", "Extra OS multi_entries not present on reference node", "OS trusted for trust_os differs from reference node " ] self.assertEqual(len(expected_msgs), len(self.mcpu.GetLogMessages())) for expected_msg in expected_msgs: self.mcpu.assertLogContainsRegex(expected_msg) class TestLUClusterVerifyGroupVerifyAcceptedFileStoragePaths( TestLUClusterVerifyGroupMethods): @withLockedLU def testNotMaster(self, lu): lu._VerifyAcceptedFileStoragePaths(self.master, {}, False) self.mcpu.assertLogIsEmpty() @withLockedLU def testNotMasterButRetunedValue(self, lu): lu._VerifyAcceptedFileStoragePaths( self.master, {constants.NV_ACCEPTED_STORAGE_PATHS: []}, False) self.mcpu.assertLogContainsRegex( "Node should not have returned forbidden file storage paths") @withLockedLU def testMasterInvalidNodeResult(self, lu): lu._VerifyAcceptedFileStoragePaths(self.master, {}, True) self.mcpu.assertLogContainsRegex( "Node did not return forbidden file storage paths") @withLockedLU def testMasterForbiddenPaths(self, lu): lu._VerifyAcceptedFileStoragePaths( self.master, {constants.NV_ACCEPTED_STORAGE_PATHS: ["/forbidden"]}, True) self.mcpu.assertLogContainsRegex("Found forbidden file storage paths") @withLockedLU def testMasterSuccess(self, lu): lu._VerifyAcceptedFileStoragePaths( self.master, {constants.NV_ACCEPTED_STORAGE_PATHS: []}, True) self.mcpu.assertLogIsEmpty() class TestLUClusterVerifyGroupVerifyStoragePaths( TestLUClusterVerifyGroupMethods): @withLockedLU def testVerifyFileStoragePathsSuccess(self, lu): lu._VerifyFileStoragePaths(self.master, {}) self.mcpu.assertLogIsEmpty() @withLockedLU def testVerifyFileStoragePathsFailure(self, lu): lu._VerifyFileStoragePaths(self.master, {constants.NV_FILE_STORAGE_PATH: "/fail/path"}) self.mcpu.assertLogContainsRegex( "The configured file storage path is unusable") @withLockedLU def testVerifySharedFileStoragePathsSuccess(self, lu): lu._VerifySharedFileStoragePaths(self.master, {}) self.mcpu.assertLogIsEmpty() @withLockedLU def testVerifySharedFileStoragePathsFailure(self, lu): lu._VerifySharedFileStoragePaths( self.master, {constants.NV_SHARED_FILE_STORAGE_PATH: "/fail/path"}) self.mcpu.assertLogContainsRegex( "The configured sharedfile storage path is unusable") class TestLUClusterVerifyGroupVerifyOob(TestLUClusterVerifyGroupMethods): @withLockedLU def testEmptyResult(self, lu): lu._VerifyOob(self.master, {}) self.mcpu.assertLogIsEmpty() @withLockedLU def testErrorResults(self, lu): lu._VerifyOob(self.master, {constants.NV_OOB_PATHS: ["path1", "path2"]}) self.mcpu.assertLogContainsRegex("path1") self.mcpu.assertLogContainsRegex("path2") class TestLUClusterVerifyGroupUpdateNodeVolumes( TestLUClusterVerifyGroupMethods): def setUp(self): super(TestLUClusterVerifyGroupUpdateNodeVolumes, self).setUp() self.nimg = verify.LUClusterVerifyGroup.NodeImage(uuid=self.master_uuid) @withLockedLU def testNoVgName(self, lu): lu._UpdateNodeVolumes(self.master, {}, self.nimg, None) self.mcpu.assertLogIsEmpty() self.assertTrue(self.nimg.lvm_fail) @withLockedLU def testErrorMessage(self, lu): lu._UpdateNodeVolumes(self.master, {constants.NV_LVLIST: "mock error"}, self.nimg, "mock_vg") self.mcpu.assertLogContainsRegex("LVM problem on node: mock error") self.assertTrue(self.nimg.lvm_fail) @withLockedLU def testInvalidNodeResult(self, lu): lu._UpdateNodeVolumes(self.master, {constants.NV_LVLIST: [1, 2, 3]}, self.nimg, "mock_vg") self.mcpu.assertLogContainsRegex("rpc call to node failed") self.assertTrue(self.nimg.lvm_fail) @withLockedLU def testValidNodeResult(self, lu): lu._UpdateNodeVolumes(self.master, {constants.NV_LVLIST: {}}, self.nimg, "mock_vg") self.mcpu.assertLogIsEmpty() self.assertFalse(self.nimg.lvm_fail) class TestLUClusterVerifyGroupUpdateNodeInstances( TestLUClusterVerifyGroupMethods): def setUp(self): super(TestLUClusterVerifyGroupUpdateNodeInstances, self).setUp() self.nimg = verify.LUClusterVerifyGroup.NodeImage(uuid=self.master_uuid) @withLockedLU def testInvalidNodeResult(self, lu): lu._UpdateNodeInstances(self.master, {}, self.nimg) self.mcpu.assertLogContainsRegex("rpc call to node failed") @withLockedLU def testValidNodeResult(self, lu): inst = self.cfg.AddNewInstance() lu._UpdateNodeInstances(self.master, {constants.NV_INSTANCELIST: [inst.name]}, self.nimg) self.mcpu.assertLogIsEmpty() class TestLUClusterVerifyGroupUpdateNodeInfo(TestLUClusterVerifyGroupMethods): def setUp(self): super(TestLUClusterVerifyGroupUpdateNodeInfo, self).setUp() self.nimg = verify.LUClusterVerifyGroup.NodeImage(uuid=self.master_uuid) self.valid_hvresult = {constants.NV_HVINFO: {"memory_free": 1024}} @withLockedLU def testInvalidHvNodeResult(self, lu): for ndata in [{}, {constants.NV_HVINFO: ""}]: self.mcpu.ClearLogMessages() lu._UpdateNodeInfo(self.master, ndata, self.nimg, None) self.mcpu.assertLogContainsRegex("rpc call to node failed") @withLockedLU def testInvalidMemoryFreeHvNodeResult(self, lu): lu._UpdateNodeInfo(self.master, {constants.NV_HVINFO: {"memory_free": "abc"}}, self.nimg, None) self.mcpu.assertLogContainsRegex( "node returned invalid nodeinfo, check hypervisor") @withLockedLU def testValidHvNodeResult(self, lu): lu._UpdateNodeInfo(self.master, self.valid_hvresult, self.nimg, None) self.mcpu.assertLogIsEmpty() @withLockedLU def testInvalidVgNodeResult(self, lu): for vgdata in [[], ""]: self.mcpu.ClearLogMessages() ndata = {constants.NV_VGLIST: vgdata} ndata.update(self.valid_hvresult) lu._UpdateNodeInfo(self.master, ndata, self.nimg, "mock_vg") self.mcpu.assertLogContainsRegex( "node didn't return data for the volume group 'mock_vg'") @withLockedLU def testInvalidDiskFreeVgNodeResult(self, lu): self.valid_hvresult.update({ constants.NV_VGLIST: {"mock_vg": "abc"} }) lu._UpdateNodeInfo(self.master, self.valid_hvresult, self.nimg, "mock_vg") self.mcpu.assertLogContainsRegex( "node returned invalid LVM info, check LVM status") @withLockedLU def testValidVgNodeResult(self, lu): self.valid_hvresult.update({ constants.NV_VGLIST: {"mock_vg": 10000} }) lu._UpdateNodeInfo(self.master, self.valid_hvresult, self.nimg, "mock_vg") self.mcpu.assertLogIsEmpty() class TestLUClusterVerifyGroupCollectDiskInfo(TestLUClusterVerifyGroupMethods): def setUp(self): super(TestLUClusterVerifyGroupCollectDiskInfo, self).setUp() self.node1 = self.cfg.AddNewNode() self.node2 = self.cfg.AddNewNode() self.node3 = self.cfg.AddNewNode() self.diskless_inst = \ self.cfg.AddNewInstance(primary_node=self.node1, disk_template=constants.DT_DISKLESS) self.plain_inst = \ self.cfg.AddNewInstance(primary_node=self.node2, disk_template=constants.DT_PLAIN) self.drbd_inst = \ self.cfg.AddNewInstance(primary_node=self.node3, secondary_node=self.node2, disk_template=constants.DT_DRBD8) self.node1_img = verify.LUClusterVerifyGroup.NodeImage( uuid=self.node1.uuid) self.node1_img.pinst = [self.diskless_inst.uuid] self.node1_img.sinst = [] self.node2_img = verify.LUClusterVerifyGroup.NodeImage( uuid=self.node2.uuid) self.node2_img.pinst = [self.plain_inst.uuid] self.node2_img.sinst = [self.drbd_inst.uuid] self.node3_img = verify.LUClusterVerifyGroup.NodeImage( uuid=self.node3.uuid) self.node3_img.pinst = [self.drbd_inst.uuid] self.node3_img.sinst = [] self.node_images = { self.node1.uuid: self.node1_img, self.node2.uuid: self.node2_img, self.node3.uuid: self.node3_img } self.node_uuids = [self.node1.uuid, self.node2.uuid, self.node3.uuid] @withLockedLU def testSuccessfulRun(self, lu): self.rpc.call_blockdev_getmirrorstatus_multi.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.node2, [(True, ""), (True, "")]) \ .AddSuccessfulNode(self.node3, [(True, "")]) \ .Build() lu._CollectDiskInfo(self.node_uuids, self.node_images, self.cfg.GetAllInstancesInfo()) self.mcpu.assertLogIsEmpty() @withLockedLU def testOfflineAndFailingNodes(self, lu): self.rpc.call_blockdev_getmirrorstatus_multi.return_value = \ RpcResultsBuilder() \ .AddOfflineNode(self.node2) \ .AddFailedNode(self.node3) \ .Build() lu._CollectDiskInfo(self.node_uuids, self.node_images, self.cfg.GetAllInstancesInfo()) self.mcpu.assertLogContainsRegex("while getting disk information") @withLockedLU def testInvalidNodeResult(self, lu): self.rpc.call_blockdev_getmirrorstatus_multi.return_value = \ RpcResultsBuilder() \ .AddSuccessfulNode(self.node2, [(True,), (False,)]) \ .AddSuccessfulNode(self.node3, [""]) \ .Build() lu._CollectDiskInfo(self.node_uuids, self.node_images, self.cfg.GetAllInstancesInfo()) # logging is not performed through mcpu self.mcpu.assertLogIsEmpty() class TestLUClusterVerifyGroupHooksCallBack(TestLUClusterVerifyGroupMethods): def setUp(self): super(TestLUClusterVerifyGroupHooksCallBack, self).setUp() self.feedback_fn = lambda _: None def PrepareLU(self, lu): super(TestLUClusterVerifyGroupHooksCallBack, self).PrepareLU(lu) lu.my_node_uuids = list(self.cfg.GetAllNodesInfo()) @withLockedLU def testEmptyGroup(self, lu): lu.my_node_uuids = [] lu.HooksCallBack(constants.HOOKS_PHASE_POST, None, self.feedback_fn, None) @withLockedLU def testFailedResult(self, lu): lu.HooksCallBack(constants.HOOKS_PHASE_POST, RpcResultsBuilder(use_node_names=True) .AddFailedNode(self.master).Build(), self.feedback_fn, None) self.mcpu.assertLogContainsRegex("Communication failure in hooks execution") @withLockedLU def testOfflineNode(self, lu): lu.HooksCallBack(constants.HOOKS_PHASE_POST, RpcResultsBuilder(use_node_names=True) .AddOfflineNode(self.master).Build(), self.feedback_fn, None) @withLockedLU def testValidResult(self, lu): lu.HooksCallBack(constants.HOOKS_PHASE_POST, RpcResultsBuilder(use_node_names=True) .AddSuccessfulNode(self.master, [("mock_script", constants.HKR_SUCCESS, "mock output")]) .Build(), self.feedback_fn, None) @withLockedLU def testFailedScriptResult(self, lu): lu.HooksCallBack(constants.HOOKS_PHASE_POST, RpcResultsBuilder(use_node_names=True) .AddSuccessfulNode(self.master, [("mock_script", constants.HKR_FAIL, "mock output")]) .Build(), self.feedback_fn, None) self.mcpu.assertLogContainsRegex("Script mock_script failed") class TestLUClusterVerifyDisks(CmdlibTestCase): def testVerifyDisks(self): self.cfg.AddNewInstance(uuid="tst1.inst.corp.example.com", disk_template=constants.DT_PLAIN) op = opcodes.OpClusterVerifyDisks() result = self.ExecOpCode(op) self.assertEqual(1, len(result["jobs"])) def testVerifyDisksExt(self): self.cfg.AddNewInstance(uuid="tst1.inst.corp.example.com", disk_template=constants.DT_EXT) self.cfg.AddNewInstance(uuid="tst2.inst.corp.example.com", disk_template=constants.DT_EXT) op = opcodes.OpClusterVerifyDisks() result = self.ExecOpCode(op) self.assertEqual(0, len(result["jobs"])) def testVerifyDisksMixed(self): self.cfg.AddNewInstance(uuid="tst1.inst.corp.example.com", disk_template=constants.DT_EXT) self.cfg.AddNewInstance(uuid="tst2.inst.corp.example.com", disk_template=constants.DT_PLAIN) op = opcodes.OpClusterVerifyDisks() result = self.ExecOpCode(op) self.assertEqual(1, len(result["jobs"])) class TestLUClusterRenewCrypto(CmdlibTestCase): def setUp(self): super(TestLUClusterRenewCrypto, self).setUp() self._node_cert = self._CreateTempFile() shutil.copy(testutils.TestDataFilename("cert1.pem"), self._node_cert) self._client_node_cert = self._CreateTempFile() shutil.copy(testutils.TestDataFilename("cert2.pem"), self._client_node_cert) self._client_node_cert_digest = \ "30:AF:82:D0:00:1C:F2:99:DE:A8:6D:31:7F:C9:D5:46:70:07:EC:4F" def tearDown(self): super(TestLUClusterRenewCrypto, self).tearDown() def _GetFakeDigest(self, uuid): """Creates a fake SSL digest depending on the UUID of a node. @type uuid: string @param uuid: node UUID @returns: a string impersonating a SSL digest """ return "FA:KE:%s:%s:%s:%s" % (uuid[0:2], uuid[2:4], uuid[4:6], uuid[6:8]) def _InitPathutils(self, pathutils): """Patch pathutils to point to temporary files. """ pathutils.NODED_CERT_FILE = self._node_cert pathutils.NODED_CLIENT_CERT_FILE = self._client_node_cert def _AssertCertFiles(self, pathutils): """Check if the correct certificates exist and don't exist on the master. """ self.assertTrue(os.path.exists(pathutils.NODED_CERT_FILE)) self.assertTrue(os.path.exists(pathutils.NODED_CLIENT_CERT_FILE)) def _CompletelySuccessfulRpc(self, node_uuid, _): """Fake RPC call which always returns successfully. """ return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node_uuid, [(constants.CRYPTO_TYPE_SSL_DIGEST, self._GetFakeDigest(node_uuid))]) @patchPathutils("cluster") def testSuccessfulCase(self, pathutils): self._InitPathutils(pathutils) # create a few non-master, online nodes num_nodes = 3 for _ in range(num_nodes): self.cfg.AddNewNode() self.rpc.call_node_crypto_tokens = self._CompletelySuccessfulRpc op = opcodes.OpClusterRenewCrypto(node_certificates=True) self.ExecOpCode(op) self._AssertCertFiles(pathutils) # Check if we have the correct digests in the configuration cluster = self.cfg.GetClusterInfo() self.assertEqual(num_nodes + 1, len(cluster.candidate_certs)) nodes = self.cfg.GetAllNodesInfo() master_uuid = self.cfg.GetMasterNode() for (node_uuid, _) in nodes.items(): if node_uuid == master_uuid: # The master digest is from the actual test certificate. self.assertEqual(self._client_node_cert_digest, cluster.candidate_certs[node_uuid]) else: # The non-master nodes have the fake digest from the # mock RPC. expected_digest = self._GetFakeDigest(node_uuid) self.assertEqual(expected_digest, cluster.candidate_certs[node_uuid]) def _partiallyFailingRpc(self, node_uuid, _): if node_uuid == self._failed_node: return self.RpcResultsBuilder() \ .CreateFailedNodeResult(node_uuid) else: return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node_uuid, [(constants.CRYPTO_TYPE_SSL_DIGEST, self._GetFakeDigest(node_uuid))]) @patchPathutils("cluster") def testNonMasterFails(self, pathutils): self._InitPathutils(pathutils) # create a few non-master, online nodes num_nodes = 3 for _ in range(num_nodes): self.cfg.AddNewNode() nodes = self.cfg.GetAllNodesInfo() # pick one node as the failing one master_uuid = self.cfg.GetMasterNode() self._failed_node = [node_uuid for node_uuid in nodes if node_uuid != master_uuid][1] self.rpc.call_node_crypto_tokens = self._partiallyFailingRpc op = opcodes.OpClusterRenewCrypto(node_certificates=True) self.ExecOpCode(op) self._AssertCertFiles(pathutils) # Check if we have the correct digests in the configuration cluster = self.cfg.GetClusterInfo() # There should be one digest missing. self.assertEqual(num_nodes, len(cluster.candidate_certs)) nodes = self.cfg.GetAllNodesInfo() for (node_uuid, _) in nodes.items(): if node_uuid == self._failed_node: self.assertTrue(node_uuid not in cluster.candidate_certs) else: self.assertTrue(node_uuid in cluster.candidate_certs) @patchPathutils("cluster") def testOfflineNodes(self, pathutils): self._InitPathutils(pathutils) # create a few non-master, online nodes num_nodes = 3 offline_index = 1 for i in range(num_nodes): # Pick one node to be offline. self.cfg.AddNewNode(offline=(i==offline_index)) self.rpc.call_node_crypto_tokens = self._CompletelySuccessfulRpc op = opcodes.OpClusterRenewCrypto(node_certificates=True) self.ExecOpCode(op) self._AssertCertFiles(pathutils) # Check if we have the correct digests in the configuration cluster = self.cfg.GetClusterInfo() # There should be one digest missing. self.assertEqual(num_nodes, len(cluster.candidate_certs)) nodes = self.cfg.GetAllNodesInfo() for (node_uuid, node_info) in nodes.items(): if node_info.offline == True: self.assertTrue(node_uuid not in cluster.candidate_certs) else: self.assertTrue(node_uuid in cluster.candidate_certs) def _RpcSuccessfulAfterRetries(self, node_uuid, _): if self._retries < self._max_retries: self._retries += 1 return self.RpcResultsBuilder() \ .CreateFailedNodeResult(node_uuid) else: return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node_uuid, [(constants.CRYPTO_TYPE_SSL_DIGEST, self._GetFakeDigest(node_uuid))]) def _RpcSuccessfulAfterRetriesNonMaster(self, node_uuid, _): if self._retries < self._max_retries and node_uuid != self._master_uuid: self._retries += 1 return self.RpcResultsBuilder() \ .CreateFailedNodeResult(node_uuid) else: return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node_uuid, [(constants.CRYPTO_TYPE_SSL_DIGEST, self._GetFakeDigest(node_uuid))]) def _NonMasterRetries(self, pathutils, max_retries): self._InitPathutils(pathutils) self._master_uuid = self.cfg.GetMasterNode() self._max_retries = max_retries self._retries = 0 self.rpc.call_node_crypto_tokens = self._RpcSuccessfulAfterRetriesNonMaster # Add one non-master node self.cfg.AddNewNode() op = opcodes.OpClusterRenewCrypto(node_certificates=True) self.ExecOpCode(op) self._AssertCertFiles(pathutils) return self.cfg.GetClusterInfo() @patchPathutils("cluster") def testNonMasterRetriesSuccess(self, pathutils): cluster = self._NonMasterRetries(pathutils, 2) self.assertEqual(2, len(cluster.candidate_certs)) @patchPathutils("cluster") def testNonMasterRetriesFail(self, pathutils): cluster = self._NonMasterRetries(pathutils, 5) # Only the master digest should be in the cert list self.assertEqual(1, len(cluster.candidate_certs.values())) self.assertTrue(self._master_uuid in cluster.candidate_certs) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/cmdlib_unittest.py000075500000000000000000001174131476477700300234660ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2008, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the cmdlib module""" import unittest import itertools import copy from unittest import mock from ganeti import constants from ganeti import mcpu from ganeti import cmdlib from ganeti.cmdlib import cluster from ganeti.cmdlib.cluster import verify from ganeti.cmdlib import instance_storage from ganeti.cmdlib import instance_utils from ganeti.cmdlib import common from ganeti.cmdlib import query from ganeti import opcodes from ganeti import errors from ganeti import luxi from ganeti import ht from ganeti import objects from ganeti import locking from ganeti.masterd import iallocator import testutils import mocks class TestOpcodeParams(testutils.GanetiTestCase): def testParamsStructures(self): for op in mcpu.Processor.DISPATCH_TABLE: lu = mcpu.Processor.DISPATCH_TABLE[op] lu_name = lu.__name__ self.assertFalse(hasattr(lu, "_OP_REQP"), msg=("LU '%s' has old-style _OP_REQP" % lu_name)) self.assertFalse(hasattr(lu, "_OP_DEFS"), msg=("LU '%s' has old-style _OP_DEFS" % lu_name)) self.assertFalse(hasattr(lu, "_OP_PARAMS"), msg=("LU '%s' has old-style _OP_PARAMS" % lu_name)) class TestIAllocatorChecks(testutils.GanetiTestCase): def testFunction(self): class TestLU(object): def __init__(self, opcode): self.cfg = mocks.FakeConfig() self.op = opcode class OpTest(opcodes.OpCode): OP_PARAMS = [ ("iallocator", None, ht.TAny, None), ("node", None, ht.TAny, None), ] default_iallocator = mocks.FakeConfig().GetDefaultIAllocator() other_iallocator = default_iallocator + "_not" op = OpTest() lu = TestLU(op) c_i = lambda: common.CheckIAllocatorOrNode(lu, "iallocator", "node") # Neither node nor iallocator given for n in (None, []): op.iallocator = None op.node = n c_i() self.assertEqual(lu.op.iallocator, default_iallocator) self.assertEqual(lu.op.node, n) # Both, iallocator and node given for a in ("test", constants.DEFAULT_IALLOCATOR_SHORTCUT): op.iallocator = a op.node = "test" self.assertRaises(errors.OpPrereqError, c_i) # Only iallocator given for n in (None, []): op.iallocator = other_iallocator op.node = n c_i() self.assertEqual(lu.op.iallocator, other_iallocator) self.assertEqual(lu.op.node, n) # Only node given op.iallocator = None op.node = "node" c_i() self.assertEqual(lu.op.iallocator, None) self.assertEqual(lu.op.node, "node") # Asked for default iallocator, no node given op.iallocator = constants.DEFAULT_IALLOCATOR_SHORTCUT op.node = None c_i() self.assertEqual(lu.op.iallocator, default_iallocator) self.assertEqual(lu.op.node, None) # No node, iallocator or default iallocator op.iallocator = None op.node = None lu.cfg.GetDefaultIAllocator = lambda: None self.assertRaises(errors.OpPrereqError, c_i) class TestLUTestJqueue(unittest.TestCase): def test(self): self.assertTrue(cmdlib.LUTestJqueue._CLIENT_CONNECT_TIMEOUT < (luxi.WFJC_TIMEOUT * 0.75), msg=("Client timeout too high, might not notice bugs" " in WaitForJobChange")) class TestLUQuery(unittest.TestCase): def test(self): self.assertEqual(sorted(query._QUERY_IMPL.keys()), sorted(constants.QR_VIA_OP)) assert constants.QR_NODE in constants.QR_VIA_LUXI assert constants.QR_INSTANCE in constants.QR_VIA_LUXI for i in constants.QR_VIA_OP: self.assertTrue(query._GetQueryImplementation(i)) self.assertRaises(errors.OpPrereqError, query._GetQueryImplementation, "") self.assertRaises(errors.OpPrereqError, query._GetQueryImplementation, "xyz") class _FakeLU: def __init__(self, cfg=NotImplemented, proc=NotImplemented, rpc=NotImplemented): self.warning_log = [] self.info_log = [] self.cfg = cfg self.proc = proc self.rpc = rpc def LogWarning(self, text, *args): self.warning_log.append((text, args)) def LogInfo(self, text, *args): self.info_log.append((text, args)) class TestLoadNodeEvacResult(unittest.TestCase): def testSuccess(self): for moved in [[], [ ("inst20153.example.com", "grp2", ["nodeA4509", "nodeB2912"]), ]]: for early_release in [False, True]: for use_nodes in [False, True]: jobs = [ [opcodes.OpInstanceReplaceDisks().__getstate__()], [opcodes.OpInstanceMigrate().__getstate__()], ] alloc_result = (moved, [], jobs) assert iallocator._NEVAC_RESULT(alloc_result) lu = _FakeLU() result = common.LoadNodeEvacResult(lu, alloc_result, early_release, use_nodes) if moved: (_, (info_args, )) = lu.info_log.pop(0) for (instname, instgroup, instnodes) in moved: self.assertTrue(instname in info_args) if use_nodes: for i in instnodes: self.assertTrue(i in info_args) else: self.assertTrue(instgroup in info_args) self.assertFalse(lu.info_log) self.assertFalse(lu.warning_log) for op in itertools.chain(*result): if hasattr(op.__class__, "early_release"): self.assertEqual(op.early_release, early_release) else: self.assertFalse(hasattr(op, "early_release")) def testFailed(self): alloc_result = ([], [ ("inst5191.example.com", "errormsg21178"), ], []) assert iallocator._NEVAC_RESULT(alloc_result) lu = _FakeLU() self.assertRaises(errors.OpExecError, common.LoadNodeEvacResult, lu, alloc_result, False, False) self.assertFalse(lu.info_log) (_, (args, )) = lu.warning_log.pop(0) self.assertTrue("inst5191.example.com" in args) self.assertTrue("errormsg21178" in args) self.assertFalse(lu.warning_log) class TestUpdateAndVerifySubDict(unittest.TestCase): def setUp(self): self.type_check = { "a": constants.VTYPE_INT, "b": constants.VTYPE_STRING, "c": constants.VTYPE_BOOL, "d": constants.VTYPE_STRING, } def test(self): old_test = { "foo": { "d": "blubb", "a": 321, }, "baz": { "a": 678, "b": "678", "c": True, }, } test = { "foo": { "a": 123, "b": "123", "c": True, }, "bar": { "a": 321, "b": "321", "c": False, }, } mv = { "foo": { "a": 123, "b": "123", "c": True, "d": "blubb" }, "bar": { "a": 321, "b": "321", "c": False, }, "baz": { "a": 678, "b": "678", "c": True, }, } verified = common._UpdateAndVerifySubDict(old_test, test, self.type_check) self.assertEqual(verified, mv) def testWrong(self): test = { "foo": { "a": "blubb", "b": "123", "c": True, }, "bar": { "a": 321, "b": "321", "c": False, }, } self.assertRaises(errors.TypeEnforcementError, common._UpdateAndVerifySubDict, {}, test, self.type_check) class TestHvStateHelper(unittest.TestCase): def testWithoutOpData(self): self.assertEqual(common.MergeAndVerifyHvState(None, NotImplemented), None) def testWithoutOldData(self): new = { constants.HT_XEN_PVM: { constants.HVST_MEMORY_TOTAL: 4096, }, } self.assertEqual(common.MergeAndVerifyHvState(new, None), new) def testWithWrongHv(self): new = { "i-dont-exist": { constants.HVST_MEMORY_TOTAL: 4096, }, } self.assertRaises(errors.OpPrereqError, common.MergeAndVerifyHvState, new, None) class TestDiskStateHelper(unittest.TestCase): def testWithoutOpData(self): self.assertEqual(common.MergeAndVerifyDiskState(None, NotImplemented), None) def testWithoutOldData(self): new = { constants.DT_PLAIN: { "xenvg": { constants.DS_DISK_RESERVED: 1024, }, }, } self.assertEqual(common.MergeAndVerifyDiskState(new, None), new) def testWithWrongStorageType(self): new = { "i-dont-exist": { "xenvg": { constants.DS_DISK_RESERVED: 1024, }, }, } self.assertRaises(errors.OpPrereqError, common.MergeAndVerifyDiskState, new, None) class TestComputeMinMaxSpec(unittest.TestCase): def setUp(self): self.ispecs = { constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 512, constants.ISPEC_DISK_SIZE: 1024, }, constants.ISPECS_MIN: { constants.ISPEC_MEM_SIZE: 128, constants.ISPEC_DISK_COUNT: 1, }, } def testNoneValue(self): self.assertTrue(common._ComputeMinMaxSpec(constants.ISPEC_MEM_SIZE, None, self.ispecs, None) is None) def testAutoValue(self): self.assertTrue(common._ComputeMinMaxSpec(constants.ISPEC_MEM_SIZE, None, self.ispecs, constants.VALUE_AUTO) is None) def testNotDefined(self): self.assertTrue(common._ComputeMinMaxSpec(constants.ISPEC_NIC_COUNT, None, self.ispecs, 3) is None) def testNoMinDefined(self): self.assertTrue(common._ComputeMinMaxSpec(constants.ISPEC_DISK_SIZE, None, self.ispecs, 128) is None) def testNoMaxDefined(self): self.assertTrue(common._ComputeMinMaxSpec(constants.ISPEC_DISK_COUNT, None, self.ispecs, 16) is None) def testOutOfRange(self): for (name, val) in ((constants.ISPEC_MEM_SIZE, 64), (constants.ISPEC_MEM_SIZE, 768), (constants.ISPEC_DISK_SIZE, 4096), (constants.ISPEC_DISK_COUNT, 0)): min_v = self.ispecs[constants.ISPECS_MIN].get(name, val) max_v = self.ispecs[constants.ISPECS_MAX].get(name, val) self.assertEqual(common._ComputeMinMaxSpec(name, None, self.ispecs, val), "%s value %s is not in range [%s, %s]" % (name, val,min_v, max_v)) self.assertEqual(common._ComputeMinMaxSpec(name, "1", self.ispecs, val), "%s/1 value %s is not in range [%s, %s]" % (name, val,min_v, max_v)) def test(self): for (name, val) in ((constants.ISPEC_MEM_SIZE, 256), (constants.ISPEC_MEM_SIZE, 128), (constants.ISPEC_MEM_SIZE, 512), (constants.ISPEC_DISK_SIZE, 1024), (constants.ISPEC_DISK_SIZE, 0), (constants.ISPEC_DISK_COUNT, 1), (constants.ISPEC_DISK_COUNT, 5)): self.assertTrue(common._ComputeMinMaxSpec(name, None, self.ispecs, val) is None) def _ValidateComputeMinMaxSpec(name, *_): assert name in constants.ISPECS_PARAMETERS return None def _NoDiskComputeMinMaxSpec(name, *_): if name == constants.ISPEC_DISK_COUNT: return name else: return None class _SpecWrapper: def __init__(self, spec): self.spec = spec def ComputeMinMaxSpec(self, *args): return self.spec.pop(0) class TestComputeIPolicySpecViolation(unittest.TestCase): # Minimal policy accepted by _ComputeIPolicySpecViolation() _MICRO_IPOL = { constants.IPOLICY_DTS: [constants.DT_PLAIN, constants.DT_DISKLESS], constants.ISPECS_MINMAX: [NotImplemented], } def test(self): compute_fn = _ValidateComputeMinMaxSpec ret = common.ComputeIPolicySpecViolation(self._MICRO_IPOL, 1024, 1, 1, 1, [1024], 1, [constants.DT_PLAIN], _compute_fn=compute_fn) self.assertEqual(ret, []) def testDiskFull(self): compute_fn = _NoDiskComputeMinMaxSpec ret = common.ComputeIPolicySpecViolation(self._MICRO_IPOL, 1024, 1, 1, 1, [1024], 1, [constants.DT_PLAIN], _compute_fn=compute_fn) self.assertEqual(ret, [constants.ISPEC_DISK_COUNT]) def testDiskLess(self): compute_fn = _NoDiskComputeMinMaxSpec ret = common.ComputeIPolicySpecViolation(self._MICRO_IPOL, 1024, 1, 0, 1, [], 1, [], _compute_fn=compute_fn) self.assertEqual(ret, []) def testWrongTemplates(self): compute_fn = _ValidateComputeMinMaxSpec ret = common.ComputeIPolicySpecViolation(self._MICRO_IPOL, 1024, 1, 1, 1, [1024], 1, [constants.DT_DRBD8], _compute_fn=compute_fn) self.assertEqual(len(ret), 1) self.assertTrue("Disk template" in ret[0]) def testInvalidArguments(self): self.assertRaises(AssertionError, common.ComputeIPolicySpecViolation, self._MICRO_IPOL, 1024, 1, 1, 1, constants.DT_DISKLESS, 1, constants.DT_PLAIN,) def testInvalidSpec(self): spec = _SpecWrapper([None, False, "foo", None, "bar", None]) compute_fn = spec.ComputeMinMaxSpec ret = common.ComputeIPolicySpecViolation(self._MICRO_IPOL, 1024, 1, 1, 1, [1024], 1, [constants.DT_PLAIN], _compute_fn=compute_fn) self.assertEqual(ret, ["foo", "bar"]) self.assertFalse(spec.spec) def testWithIPolicy(self): mem_size = 2048 cpu_count = 2 disk_count = 1 disk_sizes = [512] nic_count = 1 spindle_use = 4 disk_template = "mytemplate" ispec = { constants.ISPEC_MEM_SIZE: mem_size, constants.ISPEC_CPU_COUNT: cpu_count, constants.ISPEC_DISK_COUNT: disk_count, constants.ISPEC_DISK_SIZE: disk_sizes[0], constants.ISPEC_NIC_COUNT: nic_count, constants.ISPEC_SPINDLE_USE: spindle_use, } ipolicy1 = { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: ispec, constants.ISPECS_MAX: ispec, }], constants.IPOLICY_DTS: [disk_template], } ispec_copy = copy.deepcopy(ispec) ipolicy2 = { constants.ISPECS_MINMAX: [ { constants.ISPECS_MIN: ispec_copy, constants.ISPECS_MAX: ispec_copy, }, { constants.ISPECS_MIN: ispec, constants.ISPECS_MAX: ispec, }, ], constants.IPOLICY_DTS: [disk_template], } ipolicy3 = { constants.ISPECS_MINMAX: [ { constants.ISPECS_MIN: ispec, constants.ISPECS_MAX: ispec, }, { constants.ISPECS_MIN: ispec_copy, constants.ISPECS_MAX: ispec_copy, }, ], constants.IPOLICY_DTS: [disk_template], } def AssertComputeViolation(ipolicy, violations): ret = common.ComputeIPolicySpecViolation(ipolicy, mem_size, cpu_count, disk_count, nic_count, disk_sizes, spindle_use, [disk_template]*disk_count) self.assertEqual(len(ret), violations) AssertComputeViolation(ipolicy1, 0) AssertComputeViolation(ipolicy2, 0) AssertComputeViolation(ipolicy3, 0) for par in constants.ISPECS_PARAMETERS: ispec[par] += 1 AssertComputeViolation(ipolicy1, 1) AssertComputeViolation(ipolicy2, 0) AssertComputeViolation(ipolicy3, 0) ispec[par] -= 2 AssertComputeViolation(ipolicy1, 1) AssertComputeViolation(ipolicy2, 0) AssertComputeViolation(ipolicy3, 0) ispec[par] += 1 # Restore ipolicy1[constants.IPOLICY_DTS] = ["another_template"] AssertComputeViolation(ipolicy1, 1) class TestComputeIPolicyDiskSizesViolation(unittest.TestCase): # Minimal policy accepted by _ComputeIPolicyDiskSizesViolation() _MICRO_IPOL = { constants.IPOLICY_DTS: [constants.DT_PLAIN, constants.DT_DISKLESS], constants.ISPECS_MINMAX: [None], } def MakeDisks(self, *dev_types): return [mock.Mock(dev_type=d) for d in dev_types] def test(self): compute_fn = _ValidateComputeMinMaxSpec ret = common.ComputeIPolicyDiskSizesViolation( self._MICRO_IPOL, [1024], self.MakeDisks(constants.DT_PLAIN), _compute_fn=compute_fn) self.assertEqual(ret, []) def testDiskFull(self): compute_fn = _NoDiskComputeMinMaxSpec ret = common.ComputeIPolicyDiskSizesViolation( self._MICRO_IPOL, [1024], self.MakeDisks(constants.DT_PLAIN), _compute_fn=compute_fn) self.assertEqual(ret, [constants.ISPEC_DISK_COUNT]) def testDisksMixed(self): compute_fn = _ValidateComputeMinMaxSpec ipol = copy.deepcopy(self._MICRO_IPOL) ipol[constants.IPOLICY_DTS].append(constants.DT_DRBD8) ret = common.ComputeIPolicyDiskSizesViolation( ipol, [1024, 1024], self.MakeDisks(constants.DT_DRBD8, constants.DT_PLAIN), _compute_fn=compute_fn) self.assertEqual(ret, []) def testDiskLess(self): compute_fn = _NoDiskComputeMinMaxSpec ret = common.ComputeIPolicyDiskSizesViolation(self._MICRO_IPOL, [], [], _compute_fn=compute_fn) self.assertEqual(ret, []) def testWrongTemplates(self): compute_fn = _ValidateComputeMinMaxSpec ret = common.ComputeIPolicyDiskSizesViolation( self._MICRO_IPOL, [1024], self.MakeDisks(constants.DT_DRBD8), _compute_fn=compute_fn) self.assertEqual(len(ret), 1) self.assertTrue("Disk template" in ret[0]) def _AssertComputeViolation(self, ipolicy, disk_sizes, dev_types, violations): ret = common.ComputeIPolicyDiskSizesViolation( ipolicy, disk_sizes, self.MakeDisks(*dev_types)) self.assertEqual(len(ret), violations) def testWithIPolicy(self): mem_size = 2048 cpu_count = 2 disk_count = 1 disk_sizes = [512] nic_count = 1 spindle_use = 4 disk_template = "mytemplate" ispec = { constants.ISPEC_MEM_SIZE: mem_size, constants.ISPEC_CPU_COUNT: cpu_count, constants.ISPEC_DISK_COUNT: disk_count, constants.ISPEC_DISK_SIZE: disk_sizes[0], constants.ISPEC_NIC_COUNT: nic_count, constants.ISPEC_SPINDLE_USE: spindle_use, } ipolicy = { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: ispec, constants.ISPECS_MAX: ispec, }], constants.IPOLICY_DTS: [disk_template], } self._AssertComputeViolation(ipolicy, [512], [disk_template], 0) self._AssertComputeViolation(ipolicy, [], [disk_template], 1) self._AssertComputeViolation(ipolicy, [], [], 1) self._AssertComputeViolation(ipolicy, [512, 512], [disk_template, disk_template], 1) self._AssertComputeViolation(ipolicy, [511], [disk_template], 1) self._AssertComputeViolation(ipolicy, [513], [disk_template], 1) class _StubComputeIPolicySpecViolation: def __init__(self, mem_size, cpu_count, disk_count, nic_count, disk_sizes, spindle_use, disk_template): self.mem_size = mem_size self.cpu_count = cpu_count self.disk_count = disk_count self.nic_count = nic_count self.disk_sizes = disk_sizes self.spindle_use = spindle_use self.disk_template = disk_template def __call__(self, _, mem_size, cpu_count, disk_count, nic_count, disk_sizes, spindle_use, disk_template): assert self.mem_size == mem_size assert self.cpu_count == cpu_count assert self.disk_count == disk_count assert self.nic_count == nic_count assert self.disk_sizes == disk_sizes assert self.spindle_use == spindle_use assert self.disk_template == disk_template return [] class _FakeConfigForComputeIPolicyInstanceViolation: def __init__(self, be, excl_stor): self.cluster = objects.Cluster(beparams={"default": be}) self.excl_stor = excl_stor def GetClusterInfo(self): return self.cluster def GetNodeInfo(self, _): return {} def GetNdParams(self, _): return { constants.ND_EXCLUSIVE_STORAGE: self.excl_stor, } def GetInstanceNodes(self, instance_uuid): return ("pnode_uuid", ) def GetInstanceDisks(self, _): return [objects.Disk(size=512, spindles=13, uuid="disk_uuid", dev_type=constants.DT_PLAIN)] class TestComputeIPolicyInstanceViolation(unittest.TestCase): def setUp(self): self.beparams = { constants.BE_MAXMEM: 2048, constants.BE_VCPUS: 2, constants.BE_SPINDLE_USE: 4, } self.cfg = _FakeConfigForComputeIPolicyInstanceViolation( self.beparams, False) self.cfg_exclusive = _FakeConfigForComputeIPolicyInstanceViolation( self.beparams, True) self.stub = mock.MagicMock() self.stub.return_value = [] def testPlain(self): instance = objects.Instance(beparams=self.beparams, disks=["disk_uuid"], nics=[], primary_node="pnode_uuid", disk_template=constants.DT_PLAIN) ret = common.ComputeIPolicyInstanceViolation( NotImplemented, instance, self.cfg, _compute_fn=self.stub) self.assertEqual(ret, []) self.stub.assert_called_with(NotImplemented, 2048, 2, 1, 0, [512], 4, [constants.DT_PLAIN]) def testNoBeparams(self): instance = objects.Instance(beparams={}, disks=["disk_uuid"], nics=[], primary_node="pnode_uuid", disk_template=constants.DT_PLAIN) ret = common.ComputeIPolicyInstanceViolation( NotImplemented, instance, self.cfg, _compute_fn=self.stub) self.assertEqual(ret, []) self.stub.assert_called_with(NotImplemented, 2048, 2, 1, 0, [512], 4, [constants.DT_PLAIN]) def testExclusiveStorage(self): instance = objects.Instance(beparams=self.beparams, disks=["disk_uuid"], nics=[], primary_node="pnode_uuid", disk_template=constants.DT_PLAIN) ret = common.ComputeIPolicyInstanceViolation( NotImplemented, instance, self.cfg_exclusive, _compute_fn=self.stub) self.assertEqual(ret, []) self.stub.assert_called_with(NotImplemented, 2048, 2, 1, 0, [512], 13, [constants.DT_PLAIN]) def testExclusiveStorageNoBeparams(self): instance = objects.Instance(beparams={}, disks=["disk_uuid"], nics=[], primary_node="pnode_uuid", disk_template=constants.DT_PLAIN) ret = common.ComputeIPolicyInstanceViolation( NotImplemented, instance, self.cfg_exclusive, _compute_fn=self.stub) self.assertEqual(ret, []) self.stub.assert_called_with(NotImplemented, 2048, 2, 1, 0, [512], 13, [constants.DT_PLAIN]) class _CallRecorder: def __init__(self, return_value=None): self.called = False self.return_value = return_value def __call__(self, *args): self.called = True return self.return_value class TestComputeIPolicyNodeViolation(unittest.TestCase): def setUp(self): self.recorder = _CallRecorder(return_value=[]) def testSameGroup(self): ret = instance_utils._ComputeIPolicyNodeViolation( NotImplemented, NotImplemented, "foo", "foo", NotImplemented, _compute_fn=self.recorder) self.assertFalse(self.recorder.called) self.assertEqual(ret, []) def testDifferentGroup(self): ret = instance_utils._ComputeIPolicyNodeViolation( NotImplemented, NotImplemented, "foo", "bar", NotImplemented, _compute_fn=self.recorder) self.assertTrue(self.recorder.called) self.assertEqual(ret, []) class TestDiskSizeInBytesToMebibytes(unittest.TestCase): def testLessThanOneMebibyte(self): for i in [1, 2, 7, 512, 1000, 1023]: lu = _FakeLU() result = instance_storage._DiskSizeInBytesToMebibytes(lu, i) self.assertEqual(result, 1) self.assertEqual(len(lu.warning_log), 1) self.assertEqual(len(lu.warning_log[0]), 2) (_, (warnsize, )) = lu.warning_log[0] self.assertEqual(warnsize, (1024 * 1024) - i) def testEven(self): for i in [1, 2, 7, 512, 1000, 1023]: lu = _FakeLU() result = instance_storage._DiskSizeInBytesToMebibytes(lu, i * 1024 * 1024) self.assertEqual(result, i) self.assertFalse(lu.warning_log) def testLargeNumber(self): for i in [1, 2, 7, 512, 1000, 1023, 2724, 12420]: for j in [1, 2, 486, 326, 986, 1023]: lu = _FakeLU() size = (1024 * 1024 * i) + j result = instance_storage._DiskSizeInBytesToMebibytes(lu, size) self.assertEqual(result, i + 1, msg="Amount was not rounded up") self.assertEqual(len(lu.warning_log), 1) self.assertEqual(len(lu.warning_log[0]), 2) (_, (warnsize, )) = lu.warning_log[0] self.assertEqual(warnsize, (1024 * 1024) - j) class _OpTestVerifyErrors(opcodes.OpCode): OP_PARAMS = [ ("debug_simulate_errors", False, ht.TBool, ""), ("error_codes", False, ht.TBool, ""), ("ignore_errors", [], ht.TListOf(ht.TElemOf(constants.CV_ALL_ECODES_STRINGS)), "") ] class _LuTestVerifyErrors(verify._VerifyErrors): def __init__(self, **kwargs): super(_LuTestVerifyErrors, self).__init__() self.op = _OpTestVerifyErrors(**kwargs) self.op.Validate(True) self.msglist = [] self._feedback_fn = self.Feedback self.bad = False # TODO: Cleanup calling conventions, make them explicit def Feedback(self, *args): # TODO: Remove this once calling conventions are explicit. # Feedback can be called with anything, we interpret ELogMessageList as # messages that have to be individually added to the log list, but pushed # in a single update. Other types are only transparently passed forward. if len(args) == 1: log_type = constants.ELOG_MESSAGE log_msg = args[0] else: log_type, log_msg = args if log_type != constants.ELOG_MESSAGE_LIST: log_msg = [log_msg] self.msglist.extend(log_msg) def DispatchCallError(self, which, *args, **kwargs): if which: self._Error(*args, **kwargs) else: self._ErrorIf(True, *args, **kwargs) def CallErrorIf(self, c, *args, **kwargs): self._ErrorIf(c, *args, **kwargs) class TestVerifyErrors(unittest.TestCase): # Fake cluster-verify error code structures; we use two arbitary real error # codes to pass validation of ignore_errors (_, _ERR1ID, _) = constants.CV_ECLUSTERCFG _NODESTR = "node" _NODENAME = "mynode" _ERR1CODE = (_NODESTR, _ERR1ID, "Error one") (_, _ERR2ID, _) = constants.CV_ECLUSTERCERT _INSTSTR = "instance" _INSTNAME = "myinstance" _ERR2CODE = (_INSTSTR, _ERR2ID, "Error two") # Arguments used to call _Error() or _ErrorIf() _ERR1ARGS = (_ERR1CODE, _NODENAME, "Error1 is %s", "an error") _ERR2ARGS = (_ERR2CODE, _INSTNAME, "Error2 has no argument") # Expected error messages _ERR1MSG = _ERR1ARGS[2] % _ERR1ARGS[3] _ERR2MSG = _ERR2ARGS[2] def testNoError(self): lu = _LuTestVerifyErrors() lu.CallErrorIf(False, self._ERR1CODE, *self._ERR1ARGS) self.assertFalse(lu.bad) self.assertFalse(lu.msglist) def _InitTest(self, **kwargs): self.lu1 = _LuTestVerifyErrors(**kwargs) self.lu2 = _LuTestVerifyErrors(**kwargs) def _CallError(self, *args, **kwargs): # Check that _Error() and _ErrorIf() produce the same results self.lu1.DispatchCallError(True, *args, **kwargs) self.lu2.DispatchCallError(False, *args, **kwargs) self.assertEqual(self.lu1.bad, self.lu2.bad) self.assertEqual(self.lu1.msglist, self.lu2.msglist) # Test-specific checks are made on one LU return self.lu1 def _checkMsgCommon(self, logstr, errmsg, itype, item, warning): self.assertTrue(errmsg in logstr) if warning: self.assertTrue("WARNING" in logstr) else: self.assertTrue("ERROR" in logstr) self.assertTrue(itype in logstr) self.assertTrue(item in logstr) def _checkMsg1(self, logstr, warning=False): self._checkMsgCommon(logstr, self._ERR1MSG, self._NODESTR, self._NODENAME, warning) def _checkMsg2(self, logstr, warning=False): self._checkMsgCommon(logstr, self._ERR2MSG, self._INSTSTR, self._INSTNAME, warning) def testPlain(self): self._InitTest() lu = self._CallError(*self._ERR1ARGS) self.assertTrue(lu.bad) self.assertEqual(len(lu.msglist), 1) self._checkMsg1(lu.msglist[0]) def testMultiple(self): self._InitTest() self._CallError(*self._ERR1ARGS) lu = self._CallError(*self._ERR2ARGS) self.assertTrue(lu.bad) self.assertEqual(len(lu.msglist), 2) self._checkMsg1(lu.msglist[0]) self._checkMsg2(lu.msglist[1]) def testIgnore(self): self._InitTest(ignore_errors=[self._ERR1ID]) lu = self._CallError(*self._ERR1ARGS) self.assertFalse(lu.bad) self.assertEqual(len(lu.msglist), 1) self._checkMsg1(lu.msglist[0], warning=True) def testWarning(self): self._InitTest() lu = self._CallError(*self._ERR1ARGS, code=_LuTestVerifyErrors.ETYPE_WARNING) self.assertFalse(lu.bad) self.assertEqual(len(lu.msglist), 1) self._checkMsg1(lu.msglist[0], warning=True) def testWarning2(self): self._InitTest() self._CallError(*self._ERR1ARGS) lu = self._CallError(*self._ERR2ARGS, code=_LuTestVerifyErrors.ETYPE_WARNING) self.assertTrue(lu.bad) self.assertEqual(len(lu.msglist), 2) self._checkMsg1(lu.msglist[0]) self._checkMsg2(lu.msglist[1], warning=True) def testDebugSimulate(self): lu = _LuTestVerifyErrors(debug_simulate_errors=True) lu.CallErrorIf(False, *self._ERR1ARGS) self.assertTrue(lu.bad) self.assertEqual(len(lu.msglist), 1) self._checkMsg1(lu.msglist[0]) def testErrCodes(self): self._InitTest(error_codes=True) lu = self._CallError(*self._ERR1ARGS) self.assertTrue(lu.bad) self.assertEqual(len(lu.msglist), 1) self._checkMsg1(lu.msglist[0]) self.assertTrue(self._ERR1ID in lu.msglist[0]) class TestGetUpdatedIPolicy(unittest.TestCase): """Tests for cmdlib._GetUpdatedIPolicy()""" _OLD_CLUSTER_POLICY = { constants.IPOLICY_VCPU_RATIO: 1.5, constants.ISPECS_MINMAX: [ { constants.ISPECS_MIN: { constants.ISPEC_MEM_SIZE: 32768, constants.ISPEC_CPU_COUNT: 8, constants.ISPEC_DISK_COUNT: 1, constants.ISPEC_DISK_SIZE: 1024, constants.ISPEC_NIC_COUNT: 1, constants.ISPEC_SPINDLE_USE: 1, }, constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 65536, constants.ISPEC_CPU_COUNT: 10, constants.ISPEC_DISK_COUNT: 5, constants.ISPEC_DISK_SIZE: 1024 * 1024, constants.ISPEC_NIC_COUNT: 3, constants.ISPEC_SPINDLE_USE: 12, }, }, constants.ISPECS_MINMAX_DEFAULTS, ], constants.ISPECS_STD: constants.IPOLICY_DEFAULTS[constants.ISPECS_STD], } _OLD_GROUP_POLICY = { constants.IPOLICY_SPINDLE_RATIO: 2.5, constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: { constants.ISPEC_MEM_SIZE: 128, constants.ISPEC_CPU_COUNT: 1, constants.ISPEC_DISK_COUNT: 1, constants.ISPEC_DISK_SIZE: 1024, constants.ISPEC_NIC_COUNT: 1, constants.ISPEC_SPINDLE_USE: 1, }, constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 32768, constants.ISPEC_CPU_COUNT: 8, constants.ISPEC_DISK_COUNT: 5, constants.ISPEC_DISK_SIZE: 1024 * 1024, constants.ISPEC_NIC_COUNT: 3, constants.ISPEC_SPINDLE_USE: 12, }, }], } def _TestSetSpecs(self, old_policy, isgroup): diff_minmax = [{ constants.ISPECS_MIN: { constants.ISPEC_MEM_SIZE: 64, constants.ISPEC_CPU_COUNT: 1, constants.ISPEC_DISK_COUNT: 2, constants.ISPEC_DISK_SIZE: 64, constants.ISPEC_NIC_COUNT: 1, constants.ISPEC_SPINDLE_USE: 1, }, constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 16384, constants.ISPEC_CPU_COUNT: 10, constants.ISPEC_DISK_COUNT: 12, constants.ISPEC_DISK_SIZE: 1024, constants.ISPEC_NIC_COUNT: 9, constants.ISPEC_SPINDLE_USE: 18, }, }] diff_std = { constants.ISPEC_DISK_COUNT: 10, constants.ISPEC_DISK_SIZE: 512, } diff_policy = { constants.ISPECS_MINMAX: diff_minmax } if not isgroup: diff_policy[constants.ISPECS_STD] = diff_std new_policy = common.GetUpdatedIPolicy(old_policy, diff_policy, group_policy=isgroup) self.assertTrue(constants.ISPECS_MINMAX in new_policy) self.assertEqual(new_policy[constants.ISPECS_MINMAX], diff_minmax) for key in old_policy: if not key in diff_policy: self.assertTrue(key in new_policy) self.assertEqual(new_policy[key], old_policy[key]) if not isgroup: new_std = new_policy[constants.ISPECS_STD] for key in diff_std: self.assertTrue(key in new_std) self.assertEqual(new_std[key], diff_std[key]) old_std = old_policy.get(constants.ISPECS_STD, {}) for key in old_std: self.assertTrue(key in new_std) if key not in diff_std: self.assertEqual(new_std[key], old_std[key]) def _TestSet(self, old_policy, diff_policy, isgroup): new_policy = common.GetUpdatedIPolicy(old_policy, diff_policy, group_policy=isgroup) for key in diff_policy: self.assertTrue(key in new_policy) self.assertEqual(new_policy[key], diff_policy[key]) for key in old_policy: if not key in diff_policy: self.assertTrue(key in new_policy) self.assertEqual(new_policy[key], old_policy[key]) def testSet(self): diff_policy = { constants.IPOLICY_VCPU_RATIO: 3, constants.IPOLICY_DTS: [constants.DT_FILE], } self._TestSet(self._OLD_GROUP_POLICY, diff_policy, True) self._TestSetSpecs(self._OLD_GROUP_POLICY, True) self._TestSet({}, diff_policy, True) self._TestSetSpecs({}, True) self._TestSet(self._OLD_CLUSTER_POLICY, diff_policy, False) self._TestSetSpecs(self._OLD_CLUSTER_POLICY, False) def testUnset(self): old_policy = self._OLD_GROUP_POLICY diff_policy = { constants.IPOLICY_SPINDLE_RATIO: constants.VALUE_DEFAULT, } new_policy = common.GetUpdatedIPolicy(old_policy, diff_policy, group_policy=True) for key in diff_policy: self.assertFalse(key in new_policy) for key in old_policy: if not key in diff_policy: self.assertTrue(key in new_policy) self.assertEqual(new_policy[key], old_policy[key]) self.assertRaises(errors.OpPrereqError, common.GetUpdatedIPolicy, old_policy, diff_policy, group_policy=False) def testUnsetEmpty(self): old_policy = {} for key in constants.IPOLICY_ALL_KEYS: diff_policy = { key: constants.VALUE_DEFAULT, } new_policy = common.GetUpdatedIPolicy(old_policy, diff_policy, group_policy=True) self.assertEqual(new_policy, old_policy) def _TestInvalidKeys(self, old_policy, isgroup): INVALID_KEY = "this_key_shouldnt_be_allowed" INVALID_DICT = { INVALID_KEY: 3, } invalid_policy = INVALID_DICT self.assertRaises(errors.OpPrereqError, common.GetUpdatedIPolicy, old_policy, invalid_policy, group_policy=isgroup) invalid_ispecs = { constants.ISPECS_MINMAX: [INVALID_DICT], } self.assertRaises(errors.TypeEnforcementError, common.GetUpdatedIPolicy, old_policy, invalid_ispecs, group_policy=isgroup) if isgroup: invalid_for_group = { constants.ISPECS_STD: constants.IPOLICY_DEFAULTS[constants.ISPECS_STD], } self.assertRaises(errors.OpPrereqError, common.GetUpdatedIPolicy, old_policy, invalid_for_group, group_policy=isgroup) good_ispecs = self._OLD_CLUSTER_POLICY[constants.ISPECS_MINMAX] invalid_ispecs = copy.deepcopy(good_ispecs) invalid_policy = { constants.ISPECS_MINMAX: invalid_ispecs, } for minmax in invalid_ispecs: for key in constants.ISPECS_MINMAX_KEYS: ispec = minmax[key] ispec[INVALID_KEY] = None self.assertRaises(errors.TypeEnforcementError, common.GetUpdatedIPolicy, old_policy, invalid_policy, group_policy=isgroup) del ispec[INVALID_KEY] for par in constants.ISPECS_PARAMETERS: oldv = ispec[par] ispec[par] = "this_is_not_good" self.assertRaises(errors.TypeEnforcementError, common.GetUpdatedIPolicy, old_policy, invalid_policy, group_policy=isgroup) ispec[par] = oldv # This is to make sure that no two errors were present during the tests common.GetUpdatedIPolicy(old_policy, invalid_policy, group_policy=isgroup) def testInvalidKeys(self): self._TestInvalidKeys(self._OLD_GROUP_POLICY, True) self._TestInvalidKeys(self._OLD_CLUSTER_POLICY, False) def testInvalidValues(self): for par in (constants.IPOLICY_PARAMETERS | frozenset([constants.IPOLICY_DTS])): bad_policy = { par: "invalid_value", } self.assertRaises(errors.OpPrereqError, common.GetUpdatedIPolicy, {}, bad_policy, group_policy=True) class TestCopyLockList(unittest.TestCase): def test(self): self.assertEqual(instance_utils.CopyLockList([]), []) self.assertEqual(instance_utils.CopyLockList(None), None) self.assertEqual(instance_utils.CopyLockList(locking.ALL_SET), locking.ALL_SET) names = ["foo", "bar"] output = instance_utils.CopyLockList(names) self.assertEqual(names, output) self.assertNotEqual(id(names), id(output), msg="List was not copied") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/group_unittest.py000064400000000000000000000351701476477700300233640ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2008, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tests for LUGroup* """ import itertools from ganeti import constants from ganeti import opcodes from ganeti import query from testsupport import * import testutils class TestLUGroupAdd(CmdlibTestCase): def testAddExistingGroup(self): self.cfg.AddNewNodeGroup(name="existing_group") op = opcodes.OpGroupAdd(group_name="existing_group") self.ExecOpCodeExpectOpPrereqError( op, "Desired group name 'existing_group' already exists") def testAddNewGroup(self): op = opcodes.OpGroupAdd(group_name="new_group") self.ExecOpCode(op) self.mcpu.assertLogIsEmpty() def testAddNewGroupParams(self): ndparams = {constants.ND_EXCLUSIVE_STORAGE: True} hv_state = {constants.HT_FAKE: {constants.HVST_CPU_TOTAL: 8}} disk_state = { constants.DT_PLAIN: { "mock_vg": {constants.DS_DISK_TOTAL: 10} } } diskparams = {constants.DT_RBD: {constants.RBD_POOL: "mock_pool"}} ipolicy = constants.IPOLICY_DEFAULTS op = opcodes.OpGroupAdd(group_name="new_group", ndparams=ndparams, hv_state=hv_state, disk_state=disk_state, diskparams=diskparams, ipolicy=ipolicy) self.ExecOpCode(op) self.mcpu.assertLogIsEmpty() def testAddNewGroupInvalidDiskparams(self): diskparams = {constants.DT_RBD: {constants.LV_STRIPES: 1}} op = opcodes.OpGroupAdd(group_name="new_group", diskparams=diskparams) self.ExecOpCodeExpectOpPrereqError( op, "Provided option keys not supported") def testAddNewGroupInvalidIPolic(self): ipolicy = {"invalid_key": "value"} op = opcodes.OpGroupAdd(group_name="new_group", ipolicy=ipolicy) self.ExecOpCodeExpectOpPrereqError(op, "Invalid keys in ipolicy") class TestLUGroupAssignNodes(CmdlibTestCase): def __init__(self, methodName='runTest'): super(TestLUGroupAssignNodes, self).__init__(methodName) self.op = opcodes.OpGroupAssignNodes(group_name="default", nodes=[]) def testAssignSingleNode(self): node = self.cfg.AddNewNode() op = self.CopyOpCode(self.op, nodes=[node.name]) self.ExecOpCode(op) self.mcpu.assertLogIsEmpty() def _BuildSplitInstanceSituation(self): node = self.cfg.AddNewNode() self.cfg.AddNewInstance(disk_template=constants.DT_DRBD8, primary_node=self.master, secondary_node=node) group = self.cfg.AddNewNodeGroup() return (node, group) def testSplitInstanceNoForce(self): (node, group) = self._BuildSplitInstanceSituation() op = opcodes.OpGroupAssignNodes(group_name=group.name, nodes=[node.name]) self.ExecOpCodeExpectOpExecError( op, "instances get split by this change and --force was not given") def testSplitInstanceForce(self): (node, group) = self._BuildSplitInstanceSituation() node2 = self.cfg.AddNewNode(group=group) self.cfg.AddNewInstance(disk_template=constants.DT_DRBD8, primary_node=self.master, secondary_node=node2) op = opcodes.OpGroupAssignNodes(group_name=group.name, nodes=[node.name], force=True) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex("will split the following instances") self.mcpu.assertLogContainsRegex( "instances continue to be split across groups") @withLockedLU def testCheckAssignmentForSplitInstances(self, lu): self.cfg._OpenConfig(True) g1 = self.cfg.AddNewNodeGroup() g2 = self.cfg.AddNewNodeGroup() g3 = self.cfg.AddNewNodeGroup() for (n, g) in [("n1a", g1), ("n1b", g1), ("n2a", g2), ("n2b", g2), ("n3a", g3), ("n3b", g3), ("n3c", g3)]: self.cfg.AddNewNode(uuid=n, group=g.uuid) for uuid, pnode, snode in [("inst1a", "n1a", "n1b"), ("inst1b", "n1b", "n1a"), ("inst2a", "n2a", "n2b"), ("inst3a", "n3a", None), ("inst3b", "n3b", "n1b"), ("inst3c", "n3b", "n2b")]: dt = constants.DT_DISKLESS if snode is None else constants.DT_DRBD8 self.cfg.AddNewInstance(uuid=uuid, disk_template=dt, primary_node=pnode, secondary_node=snode) # Test first with the existing state. (new, prev) = lu.CheckAssignmentForSplitInstances( [], self.cfg.GetAllNodesInfo(), self.cfg.GetAllInstancesInfo()) self.assertEqual([], new) self.assertEqual(set(["inst3b", "inst3c"]), set(prev)) # And now some changes. (new, prev) = lu.CheckAssignmentForSplitInstances( [("n1b", g3.uuid)], self.cfg.GetAllNodesInfo(), self.cfg.GetAllInstancesInfo()) self.assertEqual(set(["inst1a", "inst1b"]), set(new)) self.assertEqual(set(["inst3c"]), set(prev)) class TestLUGroupSetParams(CmdlibTestCase): def testNoModifications(self): op = opcodes.OpGroupSetParams(group_name=self.group.name) self.ExecOpCodeExpectOpPrereqError(op, "Please pass at least one modification") def testModifyingAll(self): ndparams = {constants.ND_EXCLUSIVE_STORAGE: True} hv_state = {constants.HT_FAKE: {constants.HVST_CPU_TOTAL: 8}} disk_state = { constants.DT_PLAIN: { "mock_vg": {constants.DS_DISK_TOTAL: 10} } } diskparams = {constants.DT_RBD: {constants.RBD_POOL: "mock_pool"}} ipolicy = {constants.IPOLICY_DTS: [constants.DT_DRBD8]} op = opcodes.OpGroupSetParams(group_name=self.group.name, ndparams=ndparams, hv_state=hv_state, disk_state=disk_state, diskparams=diskparams, ipolicy=ipolicy) self.ExecOpCode(op) self.mcpu.assertLogIsEmpty() def testInvalidDiskparams(self): diskparams = {constants.DT_RBD: {constants.LV_STRIPES: 1}} op = opcodes.OpGroupSetParams(group_name=self.group.name, diskparams=diskparams) self.ExecOpCodeExpectOpPrereqError( op, "Provided option keys not supported") def testIPolicyNewViolations(self): self.cfg.AddNewInstance(beparams={constants.BE_VCPUS: 8}) min_max = dict(constants.ISPECS_MINMAX_DEFAULTS) min_max[constants.ISPECS_MAX].update({constants.ISPEC_CPU_COUNT: 2}) ipolicy = {constants.ISPECS_MINMAX: [min_max]} op = opcodes.OpGroupSetParams(group_name=self.group.name, ipolicy=ipolicy) self.ExecOpCode(op) self.assertLogContainsRegex( "After the ipolicy change the following instances violate them") class TestLUGroupRemove(CmdlibTestCase): def testNonEmptyGroup(self): group = self.cfg.AddNewNodeGroup() self.cfg.AddNewNode(group=group) op = opcodes.OpGroupRemove(group_name=group.name) self.ExecOpCodeExpectOpPrereqError(op, "Group .* not empty") def testRemoveLastGroup(self): self.master.group = "invalid_group" op = opcodes.OpGroupRemove(group_name=self.group.name) self.ExecOpCodeExpectOpPrereqError( op, "Group .* is the only group, cannot be removed") def testRemoveGroup(self): group = self.cfg.AddNewNodeGroup() op = opcodes.OpGroupRemove(group_name=group.name) self.ExecOpCode(op) self.mcpu.assertLogIsEmpty() class TestLUGroupRename(CmdlibTestCase): def testRenameToExistingName(self): group = self.cfg.AddNewNodeGroup() op = opcodes.OpGroupRename(group_name=group.name, new_name=self.group.name) self.ExecOpCodeExpectOpPrereqError( op, "Desired new name .* clashes with existing node group") def testRename(self): group = self.cfg.AddNewNodeGroup() op = opcodes.OpGroupRename(group_name=group.name, new_name="new_group_name") self.ExecOpCode(op) self.mcpu.assertLogIsEmpty() class TestLUGroupEvacuate(CmdlibTestCase): def testEvacuateEmptyGroup(self): group = self.cfg.AddNewNodeGroup() op = opcodes.OpGroupEvacuate(group_name=group.name) self.iallocator_cls.return_value.result = ([], [], []) self.ExecOpCode(op) def testEvacuateOnlyGroup(self): op = opcodes.OpGroupEvacuate(group_name=self.group.name) self.ExecOpCodeExpectOpPrereqError( op, "There are no possible target groups") def testEvacuateWithTargetGroups(self): group = self.cfg.AddNewNodeGroup() self.cfg.AddNewNode(group=group) self.cfg.AddNewNode(group=group) target_group1 = self.cfg.AddNewNodeGroup() target_group2 = self.cfg.AddNewNodeGroup() op = opcodes.OpGroupEvacuate(group_name=group.name, target_groups=[target_group1.name, target_group2.name]) self.iallocator_cls.return_value.result = ([], [], []) self.ExecOpCode(op) def testFailingIAllocator(self): group = self.cfg.AddNewNodeGroup() op = opcodes.OpGroupEvacuate(group_name=group.name) self.iallocator_cls.return_value.success = False self.ExecOpCodeExpectOpPrereqError( op, "Can't compute group evacuation using iallocator") class TestLUGroupVerifyDisks(CmdlibTestCase): def testNoInstances(self): op = opcodes.OpGroupVerifyDisks(group_name=self.group.name) self.ExecOpCode(op) self.mcpu.assertLogIsEmpty() def testOfflineAndFailingNode(self): node = self.cfg.AddNewNode(offline=True) self.cfg.AddNewInstance(primary_node=node, admin_state=constants.ADMINST_UP) self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP) self.rpc.call_lv_list.return_value = \ self.RpcResultsBuilder() \ .AddFailedNode(self.master) \ .AddOfflineNode(node) \ .Build() op = opcodes.OpGroupVerifyDisks(group_name=self.group.name) (nerrors, offline, missing) = self.ExecOpCode(op) self.assertEqual(1, len(nerrors)) self.assertEqual(0, len(offline)) self.assertEqual(2, len(missing)) def testValidNodeResult(self): self.cfg.AddNewInstance( disks=[self.cfg.CreateDisk(dev_type=constants.DT_PLAIN), self.cfg.CreateDisk(dev_type=constants.DT_PLAIN) ], admin_state=constants.ADMINST_UP) self.rpc.call_lv_list.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, { "mockvg/mock_disk_1": (None, None, True), "mockvg/mock_disk_2": (None, None, False) }) \ .Build() op = opcodes.OpGroupVerifyDisks(group_name=self.group.name) (nerrors, offline, missing) = self.ExecOpCode(op) self.assertEqual(0, len(nerrors)) self.assertEqual(1, len(offline)) self.assertEqual(0, len(missing)) def testDrbdDisk(self): node1 = self.cfg.AddNewNode() node2 = self.cfg.AddNewNode() node3 = self.cfg.AddNewNode() node4 = self.cfg.AddNewNode() valid_disk = self.cfg.CreateDisk(dev_type=constants.DT_DRBD8, primary_node=node1, secondary_node=node2) broken_disk = self.cfg.CreateDisk(dev_type=constants.DT_DRBD8, primary_node=node1, secondary_node=node2) failing_node_disk = self.cfg.CreateDisk(dev_type=constants.DT_DRBD8, primary_node=node3, secondary_node=node4) self.cfg.AddNewInstance(disks=[valid_disk, broken_disk], primary_node=node1, admin_state=constants.ADMINST_UP) self.cfg.AddNewInstance(disks=[failing_node_disk], primary_node=node3, admin_state=constants.ADMINST_UP) lv_list_result = dict(("/".join(disk.logical_id), (None, None, True)) for disk in itertools.chain(valid_disk.children, broken_disk.children)) self.rpc.call_lv_list.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(node1, lv_list_result) \ .AddSuccessfulNode(node2, lv_list_result) \ .AddFailedNode(node3) \ .AddFailedNode(node4) \ .Build() def GetDrbdNeedsActivationResult(node_uuid, *_): if node_uuid == node1.uuid: return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node1, []) elif node_uuid == node2.uuid: return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node2, [broken_disk.uuid]) elif node_uuid == node3.uuid or node_uuid == node4.uuid: return self.RpcResultsBuilder() \ .CreateFailedNodeResult(node_uuid) self.rpc.call_drbd_needs_activation.side_effect = \ GetDrbdNeedsActivationResult op = opcodes.OpGroupVerifyDisks(group_name=self.group.name) (nerrors, offline, missing) = self.ExecOpCode(op) self.assertEqual(2, len(nerrors)) self.assertEqual(1, len(offline)) self.assertEqual(1, len(missing)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/instance_migration_unittest.py000064400000000000000000000144451476477700300261070ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tests for LUInstanceFailover and LUInstanceMigrate """ from ganeti import constants from ganeti import objects from ganeti import opcodes from testsupport import * import testutils class TestLUInstanceMigrate(CmdlibTestCase): def setUp(self): super(TestLUInstanceMigrate, self).setUp() self.snode = self.cfg.AddNewNode() hv_info = ("bootid", [{ "type": constants.ST_LVM_VG, "storage_free": 10000 }], ({"memory_free": 10000}, )) self.rpc.call_node_info.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, hv_info) \ .AddSuccessfulNode(self.snode, hv_info) \ .Build() self.rpc.call_blockdev_find.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, objects.BlockDevStatus()) self.rpc.call_migration_info.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) self.rpc.call_accept_instance.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.snode, True) self.rpc.call_instance_migrate.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) self.rpc.call_instance_get_migration_status.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, objects.MigrationStatus()) self.rpc.call_instance_finalize_migration_dst.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.snode, True) self.rpc.call_instance_finalize_migration_src.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) self.inst = self.cfg.AddNewInstance(disk_template=constants.DT_DRBD8, admin_state=constants.ADMINST_UP, secondary_node=self.snode) self.op = opcodes.OpInstanceMigrate(instance_name=self.inst.name) def testPlainDisk(self): inst = self.cfg.AddNewInstance(disk_template=constants.DT_PLAIN) op = self.CopyOpCode(self.op, instance_name=inst.name) self.ExecOpCodeExpectOpPrereqError( op, "Instance's disk layout 'plain' does not allow migrations") def testMigrationToWrongNode(self): node = self.cfg.AddNewNode() op = self.CopyOpCode(self.op, target_node=node.name) self.ExecOpCodeExpectOpPrereqError( op, "Instances with disk types drbd cannot be migrated to" " arbitrary nodes") def testMigration(self): op = self.CopyOpCode(self.op) self.ExecOpCode(op) class TestLUInstanceFailover(CmdlibTestCase): def setUp(self): super(TestLUInstanceFailover, self).setUp() self.snode = self.cfg.AddNewNode() hv_info = ("bootid", [{ "type": constants.ST_LVM_VG, "storage_free": 10000 }], ({"memory_free": 10000}, )) self.rpc.call_node_info.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, hv_info) \ .AddSuccessfulNode(self.snode, hv_info) \ .Build() self.rpc.call_blockdev_find.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, objects.BlockDevStatus()) self.rpc.call_instance_shutdown.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) self.rpc.call_blockdev_shutdown.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) self.rpc.call_blockdev_assemble.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.snode, ("/dev/mock", "/var/mock", None)) self.rpc.call_instance_start.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.snode, True) self.inst = self.cfg.AddNewInstance(disk_template=constants.DT_DRBD8, admin_state=constants.ADMINST_UP, secondary_node=self.snode) self.op = opcodes.OpInstanceFailover(instance_name=self.inst.name) def testPlainDisk(self): inst = self.cfg.AddNewInstance(disk_template=constants.DT_PLAIN) op = self.CopyOpCode(self.op, instance_name=inst.name) self.ExecOpCodeExpectOpPrereqError( op, "Instance's disk layout 'plain' does not allow failovers") def testMigrationToWrongNode(self): node = self.cfg.AddNewNode() op = self.CopyOpCode(self.op, target_node=node.name) self.ExecOpCodeExpectOpPrereqError( op, "Instances with disk types drbd cannot be failed over to" " arbitrary nodes") def testMigration(self): op = self.CopyOpCode(self.op) self.ExecOpCode(op) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/instance_storage_unittest.py000075500000000000000000000264561476477700300255720ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the cmdlib module 'instance_storage'""" import time import unittest from unittest import mock from ganeti import constants from ganeti.cmdlib import instance_storage from ganeti import errors from ganeti import objects from ganeti import opcodes import testutils from testsupport import CmdlibTestCase class TestCheckNodesFreeDiskOnVG(unittest.TestCase): def setUp(self): self.node_uuid = "12345" self.node_uuids = [self.node_uuid] self.node_info = mock.Mock() self.es = True self.ndparams = {constants.ND_EXCLUSIVE_STORAGE: self.es} mock_rpc = mock.Mock() mock_rpc.call_node_info = mock.Mock() mock_cfg = mock.Mock() mock_cfg.GetNodeInfo = mock.Mock(return_value=self.node_info) mock_cfg.GetNdParams = mock.Mock(return_value=self.ndparams) self.hvname = "myhv" self.hvparams = mock.Mock() self.clusterinfo = mock.Mock() self.clusterinfo.hvparams = {self.hvname: self.hvparams} mock_cfg.GetHypervisorType = mock.Mock(return_value=self.hvname) mock_cfg.GetClusterInfo = mock.Mock(return_value=self.clusterinfo) self.lu = mock.Mock() self.lu.rpc = mock_rpc self.lu.cfg = mock_cfg self.vg = "myvg" self.node_name = "mynode" self.space_info = [{"type": constants.ST_LVM_VG, "name": self.vg, "storage_free": 125, "storage_size": 666}] def testPerformNodeInfoCall(self): expected_hv_arg = [(self.hvname, self.hvparams)] expected_storage_arg = {self.node_uuid: [(constants.ST_LVM_VG, self.vg, [self.es]), (constants.ST_LVM_PV, self.vg, [self.es])]} instance_storage._PerformNodeInfoCall(self.lu, self.node_uuids, self.vg) self.lu.rpc.call_node_info.assert_called_with( self.node_uuids, expected_storage_arg, expected_hv_arg) def testCheckVgCapacityForNode(self): requested = 123 node_info = (None, self.space_info, None) instance_storage._CheckVgCapacityForNode(self.node_name, node_info, self.vg, requested) def testCheckVgCapacityForNodeNotEnough(self): requested = 250 node_info = (None, self.space_info, None) self.assertRaises( errors.OpPrereqError, instance_storage._CheckVgCapacityForNode, self.node_name, node_info, self.vg, requested) def testCheckVgCapacityForNodeNoStorageData(self): node_info = (None, [], None) self.assertRaises( errors.OpPrereqError, instance_storage._CheckVgCapacityForNode, self.node_name, node_info, self.vg, NotImplemented) def testCheckVgCapacityForNodeBogusSize(self): broken_space_info = [{"type": constants.ST_LVM_VG, "name": self.vg, "storage_free": "greenbunny", "storage_size": "redbunny"}] node_info = (None, broken_space_info, None) self.assertRaises( errors.OpPrereqError, instance_storage._CheckVgCapacityForNode, self.node_name, node_info, self.vg, NotImplemented) class TestCheckComputeDisksInfo(unittest.TestCase): """Tests for instance_storage.ComputeDisksInfo() """ def setUp(self): """Set up input data""" self.disks = [ objects.Disk(dev_type=constants.DT_PLAIN, size=1024, logical_id=("ganeti", "disk01234"), name="disk-0", mode="rw", params={}, children=[], uuid="disk0"), objects.Disk(dev_type=constants.DT_PLAIN, size=2048, logical_id=("ganeti", "disk56789"), name="disk-1", mode="ro", params={}, children=[], uuid="disk1") ] self.ext_params = { "provider": "pvdr", "param1" : "value1", "param2" : "value2" } self.default_vg = "ganeti-vg" def testComputeDisksInfo(self): """Test instance_storage.ComputeDisksInfo() method""" disks_info = instance_storage.ComputeDisksInfo(self.disks, constants.DT_EXT, self.default_vg, self.ext_params) for disk, d in zip(disks_info, self.disks): self.assertEqual(disk.get("size"), d.size) self.assertEqual(disk.get("mode"), d.mode) self.assertEqual(disk.get("name"), d.name) self.assertEqual(disk.get("param1"), self.ext_params.get("param1")) self.assertEqual(disk.get("param2"), self.ext_params.get("param2")) self.assertEqual(disk.get("provider"), self.ext_params.get("provider")) def testComputeDisksInfoPlainToDrbd(self): disks = [{constants.IDISK_TYPE: constants.DT_DRBD8, constants.IDISK_SIZE: d.size, constants.IDISK_MODE: d.mode, constants.IDISK_VG: d.logical_id[0], constants.IDISK_NAME: d.name} for d in self.disks] disks_info = instance_storage.ComputeDisksInfo(self.disks, constants.DT_DRBD8, self.default_vg, {}) self.assertEqual(disks, disks_info) def testComputeDisksInfoFails(self): """Test instance_storage.ComputeDisksInfo() method fails""" self.assertRaises( errors.OpPrereqError, instance_storage.ComputeDisksInfo, self.disks, constants.DT_EXT, self.default_vg, {}) self.assertRaises( errors.OpPrereqError, instance_storage.ComputeDisksInfo, self.disks, constants.DT_DRBD8, self.default_vg, self.ext_params) self.ext_params.update({"size": 128}) self.assertRaises( AssertionError, instance_storage.ComputeDisksInfo, self.disks, constants.DT_EXT, self.default_vg, self.ext_params) class TestLUInstanceReplaceDisks(CmdlibTestCase): """Tests for LUInstanceReplaceDisks.""" def setUp(self): super(TestLUInstanceReplaceDisks, self).setUp() self.MockOut(time, 'sleep') self.node1 = self.cfg.AddNewNode() self.node2 = self.cfg.AddNewNode() def MakeOpCode(self, disks, early_release=False, ignore_ipolicy=False, remote_node=False, mode='replace_auto', iallocator=None): return opcodes.OpInstanceReplaceDisks( instance_name=self.instance.name, instance_uuid=self.instance.uuid, early_release=early_release, ignore_ipolicy=ignore_ipolicy, mode=mode, disks=disks, remote_node=self.node2.name if remote_node else None, remote_node_uuid=self.node2.uuid if remote_node else None, iallocator=iallocator) def testInvalidTemplate(self): self.instance = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP, disk_template='diskless', primary_node=self.node1) opcode = self.MakeOpCode([]) self.ExecOpCodeExpectOpPrereqError( opcode, 'strange layout') def SimulateDiskFailure(self, node, disk): def Faulty(node_uuid): disks = self.cfg.GetInstanceDisks(node_uuid) return [i for i,d in enumerate(disks) if i == disk and node.uuid == node_uuid] self.MockOut(instance_storage.TLReplaceDisks, '_FindFaultyDisks', side_effect=Faulty) self.MockOut(instance_storage.TLReplaceDisks, '_CheckDevices') self.MockOut(instance_storage.TLReplaceDisks, '_CheckVolumeGroup') self.MockOut(instance_storage.TLReplaceDisks, '_CheckDisksExistence') self.MockOut(instance_storage.TLReplaceDisks, '_CheckDisksConsistency') self.MockOut(instance_storage.LUInstanceReplaceDisks, 'AssertReleasedLocks') self.MockOut(instance_storage, 'WaitForSync') self.rpc.call_blockdev_addchildren().fail_msg = None def testReplacePrimary(self): self.instance = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP, disk_template='drbd', primary_node=self.node1, secondary_node=self.node2) self.SimulateDiskFailure(self.node1, 0) opcode = self.MakeOpCode([0], mode='replace_on_primary') self.ExecOpCode(opcode) self.rpc.call_blockdev_rename.assert_any_call(self.node1.uuid, []) def testReplaceSecondary(self): self.instance = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP, disk_template='drbd', primary_node=self.node1, secondary_node=self.node2) self.SimulateDiskFailure(self.node2, 0) opcode = self.MakeOpCode([0], mode='replace_on_secondary') self.ExecOpCode(opcode) self.rpc.call_blockdev_rename.assert_any_call(self.node2.uuid, []) def testReplaceSecondaryNew(self): disk = self.cfg.CreateDisk(dev_type=constants.DT_DRBD8, primary_node=self.node1, secondary_node=self.node2) self.instance = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP, disk_template='drbd', disks=[disk], primary_node=self.node1, secondary_node=self.node2) self.SimulateDiskFailure(self.node2, 0) node3 = self.cfg.AddNewNode() self.MockOut(instance_storage.TLReplaceDisks, '_RunAllocator', return_value=node3.uuid) self.rpc.call_drbd_disconnect_net().__getitem__().fail_msg = None self.rpc.call_blockdev_shutdown().fail_msg = None self.rpc.call_drbd_attach_net().fail_msg = None opcode = self.MakeOpCode([], mode='replace_new_secondary', iallocator='hail') self.ExecOpCode(opcode) self.rpc.call_blockdev_shutdown.assert_any_call( self.node2.uuid, (disk, self.instance)) self.rpc.call_drbd_attach_net.assert_any_call( [self.node1.uuid, node3.uuid], ([disk], self.instance), False) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/instance_unittest.py000075500000000000000000003604031476477700300240370ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2008, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tests for LUInstance* """ import copy import itertools import re import unittest import os from unittest import mock from ganeti import backend from ganeti import compat from ganeti import config from ganeti import constants from ganeti import errors from ganeti import ht from ganeti import opcodes from ganeti import objects from ganeti.rpc import node as rpc from ganeti import utils from ganeti.cmdlib import instance from ganeti.cmdlib import instance_storage from ganeti.cmdlib import instance_create from ganeti.cmdlib import instance_set_params from ganeti.cmdlib import instance_utils from cmdlib.cmdlib_unittest import _FakeLU from testsupport import * import testutils from testutils.config_mock import _UpdateIvNames class TestComputeIPolicyInstanceSpecViolation(unittest.TestCase): def setUp(self): self.ispec = { constants.ISPEC_MEM_SIZE: 2048, constants.ISPEC_CPU_COUNT: 2, constants.ISPEC_DISK_COUNT: 1, constants.ISPEC_DISK_SIZE: [512], constants.ISPEC_NIC_COUNT: 0, constants.ISPEC_SPINDLE_USE: 1, } self.stub = mock.MagicMock() self.stub.return_value = [] def testPassThrough(self): ret = instance_utils.ComputeIPolicyInstanceSpecViolation( NotImplemented, self.ispec, [constants.DT_PLAIN], _compute_fn=self.stub) self.assertEqual(ret, []) self.stub.assert_called_with(NotImplemented, 2048, 2, 1, 0, [512], 1, [constants.DT_PLAIN]) class TestLUInstanceCreate(CmdlibTestCase): def _setupOSDiagnose(self): os_result = [(self.os.name, self.os.path, True, "", self.os.supported_variants, self.os.supported_parameters, self.os.api_versions, True)] self.rpc.call_os_diagnose.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, os_result) \ .AddSuccessfulNode(self.node1, os_result) \ .AddSuccessfulNode(self.node2, os_result) \ .Build() def setUp(self): super(TestLUInstanceCreate, self).setUp() self.ResetMocks() self.MockOut(instance_create, 'netutils', self.netutils_mod) self.MockOut(instance_utils, 'netutils', self.netutils_mod) self.net = self.cfg.AddNewNetwork() self.cfg.ConnectNetworkToGroup(self.net, self.group) self.node1 = self.cfg.AddNewNode() self.node2 = self.cfg.AddNewNode() hv_info = ("bootid", [{ "type": constants.ST_LVM_VG, "storage_free": 10000 }], ({"memory_free": 10000}, )) self.rpc.call_node_info.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, hv_info) \ .AddSuccessfulNode(self.node1, hv_info) \ .AddSuccessfulNode(self.node2, hv_info) \ .Build() self._setupOSDiagnose() self.rpc.call_blockdev_getmirrorstatus.side_effect = \ lambda node, _: self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node, []) self.iallocator_cls.return_value.result = [self.node1.name, self.node2.name] self.diskless_op = opcodes.OpInstanceCreate( instance_name="diskless.example.com", pnode=self.master.name, disk_template=constants.DT_DISKLESS, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[], os_type=self.os_name_variant, name_check=True, ip_check=True) self.plain_op = opcodes.OpInstanceCreate( instance_name="plain.example.com", pnode=self.master.name, disk_template=constants.DT_PLAIN, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[{ constants.IDISK_SIZE: 1024 }], os_type=self.os_name_variant) self.block_op = opcodes.OpInstanceCreate( instance_name="block.example.com", pnode=self.master.name, disk_template=constants.DT_BLOCK, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[{ constants.IDISK_SIZE: 1024, constants.IDISK_ADOPT: "/dev/disk/block0" }], os_type=self.os_name_variant) self.drbd_op = opcodes.OpInstanceCreate( instance_name="drbd.example.com", pnode=self.node1.name, snode=self.node2.name, disk_template=constants.DT_DRBD8, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[{ constants.IDISK_SIZE: 1024 }], os_type=self.os_name_variant) self.file_op = opcodes.OpInstanceCreate( instance_name="file.example.com", pnode=self.node1.name, disk_template=constants.DT_FILE, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[{ constants.IDISK_SIZE: 1024 }], os_type=self.os_name_variant) self.shared_file_op = opcodes.OpInstanceCreate( instance_name="shared-file.example.com", pnode=self.node1.name, disk_template=constants.DT_SHARED_FILE, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[{ constants.IDISK_SIZE: 1024 }], os_type=self.os_name_variant) self.gluster_op = opcodes.OpInstanceCreate( instance_name="gluster.example.com", pnode=self.node1.name, disk_template=constants.DT_GLUSTER, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[{ constants.IDISK_SIZE: 1024 }], os_type=self.os_name_variant) self.rbd_op = opcodes.OpInstanceCreate( instance_name="gluster.example.com", pnode=self.node1.name, disk_template=constants.DT_RBD, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[{ constants.IDISK_SIZE: 1024 }], os_type=self.os_name_variant) def testSimpleCreate(self): op = self.CopyOpCode(self.diskless_op) self.ExecOpCode(op) def testStrangeHostnameResolve(self): op = self.CopyOpCode(self.diskless_op) self.netutils_mod.GetHostname.return_value = \ HostnameMock("random.host.example.com", "203.0.113.1") self.ExecOpCodeExpectOpPrereqError( op, "Resolved hostname .* does not look the same as given hostname") def testOpportunisticLockingNoIAllocator(self): op = self.CopyOpCode(self.diskless_op, opportunistic_locking=True, iallocator=None) self.ExecOpCodeExpectOpPrereqError( op, "Opportunistic locking is only available in combination with an" " instance allocator") def testNicWithNetAndMode(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_NETWORK: self.net.name, constants.INIC_MODE: constants.NIC_MODE_BRIDGED }]) self.ExecOpCodeExpectOpPrereqError( op, "If network is given, no mode or link is allowed to be passed") def testAutoIpNoNameCheck(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_IP: constants.VALUE_AUTO }], ip_check=False, name_check=False) self.ExecOpCodeExpectOpPrereqError( op, "IP address set to auto but the name checks are not enabled") def testAutoIp(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_IP: constants.VALUE_AUTO }]) self.ExecOpCode(op) def testPoolIpNoNetwork(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_IP: constants.NIC_IP_POOL }]) self.ExecOpCodeExpectOpPrereqError( op, "if ip=pool, parameter network must be passed too") def testValidIp(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_IP: "203.0.113.1" }]) self.ExecOpCode(op) def testRoutedNoIp(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_NETWORK: constants.VALUE_NONE, constants.INIC_MODE: constants.NIC_MODE_ROUTED }]) self.ExecOpCodeExpectOpPrereqError( op, "Routed nic mode requires an ip address" " if not attached to a network") def testValicMac(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_MAC: "f0:df:f4:a3:d1:cf" }]) self.ExecOpCode(op) def testValidNicParams(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_MODE: constants.NIC_MODE_BRIDGED, constants.INIC_LINK: "br_mock" }]) self.ExecOpCode(op) def testValidNicParamsOpenVSwitch(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_MODE: constants.NIC_MODE_OVS, constants.INIC_VLAN: "1" }]) self.ExecOpCode(op) def testNicNoneName(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_NAME: constants.VALUE_NONE }]) self.ExecOpCode(op) def testConflictingIP(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_IP: self.net.gateway[:-1] + "2" }]) self.ExecOpCodeExpectOpPrereqError( op, "The requested IP address .* belongs to network .*, but the target" " NIC does not.") def testVLanFormat(self): for vlan in [".pinky", ":bunny", ":1:pinky", "bunny"]: self.ResetMocks() op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_VLAN: vlan }]) self.ExecOpCodeExpectOpPrereqError( op, "Specified VLAN parameter is invalid") def testPoolIp(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_IP: constants.NIC_IP_POOL, constants.INIC_NETWORK: self.net.name }]) self.ExecOpCode(op) def testPoolIpUnconnectedNetwork(self): net = self.cfg.AddNewNetwork() op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_IP: constants.NIC_IP_POOL, constants.INIC_NETWORK: net.name }]) self.ExecOpCodeExpectOpPrereqError( op, "No netparams found for network .*.") def testIpNotInNetwork(self): op = self.CopyOpCode(self.diskless_op, nics=[{ constants.INIC_IP: "203.0.113.1", constants.INIC_NETWORK: self.net.name }]) self.ExecOpCodeExpectOpPrereqError( op, "IP address .* already in use or does not belong to network .*") def testMixAdoptAndNotAdopt(self): op = self.CopyOpCode(self.diskless_op, disk_template=constants.DT_PLAIN, disks=[{ constants.IDISK_ADOPT: "lv1" }, {}]) self.ExecOpCodeExpectOpPrereqError( op, "Either all disks are adopted or none is") def testMustAdoptWithoutAdopt(self): op = self.CopyOpCode(self.diskless_op, disk_template=constants.DT_BLOCK, disks=[{}]) self.ExecOpCodeExpectOpPrereqError( op, "Disk template blockdev requires disk adoption, but no 'adopt'" " parameter given") def testDontAdoptWithAdopt(self): op = self.CopyOpCode(self.diskless_op, disk_template=constants.DT_DRBD8, disks=[{ constants.IDISK_ADOPT: "lv1" }]) self.ExecOpCodeExpectOpPrereqError( op, "Disk adoption is not supported for the 'drbd' disk template") def testAdoptWithIAllocator(self): op = self.CopyOpCode(self.diskless_op, disk_template=constants.DT_PLAIN, disks=[{ constants.IDISK_ADOPT: "lv1" }], iallocator="mock") self.ExecOpCodeExpectOpPrereqError( op, "Disk adoption not allowed with an iallocator script") def testAdoptWithImport(self): op = self.CopyOpCode(self.diskless_op, disk_template=constants.DT_PLAIN, disks=[{ constants.IDISK_ADOPT: "lv1" }], mode=constants.INSTANCE_IMPORT) self.ExecOpCodeExpectOpPrereqError( op, "Disk adoption not allowed for instance import") def testArgumentCombinations(self): op = self.CopyOpCode(self.diskless_op, # start flag will be flipped no_install=True, start=True, # no allowed combination ip_check=True, name_check=False) self.ExecOpCodeExpectOpPrereqError( op, "Cannot do IP address check without a name check") def testInvalidFileDriver(self): op = self.CopyOpCode(self.diskless_op, file_driver="invalid_file_driver") self.ExecOpCodeExpectOpPrereqError( op, "Parameter 'OP_INSTANCE_CREATE.file_driver' fails validation") def testMissingSecondaryNode(self): op = self.CopyOpCode(self.diskless_op, pnode=self.master.name, disk_template=constants.DT_DRBD8) self.ExecOpCodeExpectOpPrereqError( op, "The networked disk templates need a mirror node") def testIgnoredSecondaryNode(self): op = self.CopyOpCode(self.diskless_op, pnode=self.master.name, snode=self.node1.name, disk_template=constants.DT_PLAIN) try: self.ExecOpCode(op) except Exception: pass self.mcpu.assertLogContainsRegex( "Secondary node will be ignored on non-mirrored disk template") def testMissingOsType(self): op = self.CopyOpCode(self.diskless_op, os_type=self.REMOVE) self.ExecOpCodeExpectOpPrereqError(op, "No guest OS or OS image specified") def testBlacklistedOs(self): self.cluster.blacklisted_os = [self.os_name_variant] op = self.CopyOpCode(self.diskless_op) self.ExecOpCodeExpectOpPrereqError( op, "Guest OS .* is not allowed for installation") def testMissingDiskTemplate(self): self.cluster.enabled_disk_templates = [constants.DT_DISKLESS] op = self.CopyOpCode(self.diskless_op, disk_template=self.REMOVE) self.ExecOpCode(op) def testExistingInstance(self): inst = self.cfg.AddNewInstance() op = self.CopyOpCode(self.diskless_op, instance_name=inst.name) self.ExecOpCodeExpectOpPrereqError( op, "Instance .* is already in the cluster") def testPlainInstance(self): op = self.CopyOpCode(self.plain_op) self.ExecOpCode(op) def testPlainIAllocator(self): op = self.CopyOpCode(self.plain_op, pnode=self.REMOVE, iallocator="mock") self.ExecOpCode(op) def testIAllocatorOpportunisticLocking(self): op = self.CopyOpCode(self.plain_op, pnode=self.REMOVE, iallocator="mock", opportunistic_locking=True) self.ExecOpCode(op) def testFailingIAllocator(self): self.iallocator_cls.return_value.success = False op = self.CopyOpCode(self.plain_op, pnode=self.REMOVE, iallocator="mock") self.ExecOpCodeExpectOpPrereqError( op, "Can't compute nodes using iallocator") def testDrbdInstance(self): op = self.CopyOpCode(self.drbd_op) self.ExecOpCode(op) def testDrbdIAllocator(self): op = self.CopyOpCode(self.drbd_op, pnode=self.REMOVE, snode=self.REMOVE, iallocator="mock") self.ExecOpCode(op) def testFileInstance(self): op = self.CopyOpCode(self.file_op) self.ExecOpCode(op) def testFileInstanceNoClusterStorage(self): self.cluster.file_storage_dir = None op = self.CopyOpCode(self.file_op) self.ExecOpCodeExpectOpPrereqError( op, "Cluster file storage dir for 'file' storage type not defined") def testFileInstanceAdditionalPath(self): op = self.CopyOpCode(self.file_op, file_storage_dir="mock_dir") self.ExecOpCode(op) def testIdentifyDefaults(self): op = self.CopyOpCode(self.plain_op, hvparams={ constants.HV_BOOT_ORDER: "cd" }, beparams=constants.BEC_DEFAULTS.copy(), nics=[{ constants.NIC_MODE: constants.NIC_MODE_BRIDGED }], osparams={ self.os_name_variant: {} }, osparams_private={}, identify_defaults=True) self.ExecOpCode(op) inst = list(self.cfg.GetAllInstancesInfo().values())[0] self.assertEqual(0, len(inst.hvparams)) self.assertEqual(0, len(inst.beparams)) assert self.os_name_variant not in inst.osparams or \ len(inst.osparams[self.os_name_variant]) == 0 def testOfflineNode(self): self.node1.offline = True op = self.CopyOpCode(self.diskless_op, pnode=self.node1.name) self.ExecOpCodeExpectOpPrereqError(op, "Cannot use offline primary node") def testDrainedNode(self): self.node1.drained = True op = self.CopyOpCode(self.diskless_op, pnode=self.node1.name) self.ExecOpCodeExpectOpPrereqError(op, "Cannot use drained primary node") def testNonVmCapableNode(self): self.node1.vm_capable = False op = self.CopyOpCode(self.diskless_op, pnode=self.node1.name) self.ExecOpCodeExpectOpPrereqError( op, "Cannot use non-vm_capable primary node") def testNonEnabledHypervisor(self): self.cluster.enabled_hypervisors = [constants.HT_XEN_HVM] op = self.CopyOpCode(self.diskless_op, hypervisor=constants.HT_FAKE) self.ExecOpCodeExpectOpPrereqError( op, "Selected hypervisor .* not enabled in the cluster") def testAddTag(self): op = self.CopyOpCode(self.diskless_op, tags=["tag"]) self.ExecOpCode(op) def testInvalidTag(self): op = self.CopyOpCode(self.diskless_op, tags=["too_long" * 20]) self.ExecOpCodeExpectException(op, errors.TagError, "Tag too long") def testPingableInstanceName(self): self.netutils_mod.TcpPing.return_value = True op = self.CopyOpCode(self.diskless_op) self.ExecOpCodeExpectOpPrereqError( op, "IP .* of instance diskless.example.com already in use") def testPrimaryIsSecondaryNode(self): op = self.CopyOpCode(self.drbd_op, snode=self.drbd_op.pnode) self.ExecOpCodeExpectOpPrereqError( op, "The secondary node cannot be the primary node") def testPrimarySecondaryDifferentNodeGroups(self): group = self.cfg.AddNewNodeGroup() self.node2.group = group.uuid op = self.CopyOpCode(self.drbd_op) self.ExecOpCode(op) self.mcpu.assertLogContainsRegex( "The primary and secondary nodes are in two different node groups") def testExclusiveStorageUnsupportedDiskTemplate(self): self.node1.ndparams[constants.ND_EXCLUSIVE_STORAGE] = True op = self.CopyOpCode(self.drbd_op) self.ExecOpCodeExpectOpPrereqError( op, "Disk template drbd not supported with exclusive storage") def testAdoptPlain(self): self.rpc.call_lv_list.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, { "xenvg/mock_disk_1": (10000, None, False) }) \ .Build() op = self.CopyOpCode(self.plain_op) op.disks[0].update({constants.IDISK_ADOPT: "mock_disk_1"}) self.ExecOpCode(op) def testAdoptPlainMissingLv(self): self.rpc.call_lv_list.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {}) \ .Build() op = self.CopyOpCode(self.plain_op) op.disks[0].update({constants.IDISK_ADOPT: "mock_disk_1"}) self.ExecOpCodeExpectOpPrereqError(op, "Missing logical volume") def testAdoptPlainOnlineLv(self): self.rpc.call_lv_list.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, { "xenvg/mock_disk_1": (10000, None, True) }) \ .Build() op = self.CopyOpCode(self.plain_op) op.disks[0].update({constants.IDISK_ADOPT: "mock_disk_1"}) self.ExecOpCodeExpectOpPrereqError( op, "Online logical volumes found, cannot adopt") def testAdoptBlock(self): self.rpc.call_bdev_sizes.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, { "/dev/disk/block0": 10000 }) \ .Build() op = self.CopyOpCode(self.block_op) self.ExecOpCode(op) def testAdoptBlockDuplicateNames(self): op = self.CopyOpCode(self.block_op, disks=[{ constants.IDISK_SIZE: 0, constants.IDISK_ADOPT: "/dev/disk/block0" }, { constants.IDISK_SIZE: 0, constants.IDISK_ADOPT: "/dev/disk/block0" }]) self.ExecOpCodeExpectOpPrereqError( op, "Duplicate disk names given for adoption") def testAdoptBlockInvalidNames(self): op = self.CopyOpCode(self.block_op, disks=[{ constants.IDISK_SIZE: 0, constants.IDISK_ADOPT: "/invalid/block0" }]) self.ExecOpCodeExpectOpPrereqError( op, "Device node.* lie outside .* and cannot be adopted") def testAdoptBlockMissingDisk(self): self.rpc.call_bdev_sizes.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {}) \ .Build() op = self.CopyOpCode(self.block_op) self.ExecOpCodeExpectOpPrereqError(op, "Missing block device") def testNoWaitForSyncDrbd(self): op = self.CopyOpCode(self.drbd_op, wait_for_sync=False) self.ExecOpCode(op) def testNoWaitForSyncPlain(self): op = self.CopyOpCode(self.plain_op, wait_for_sync=False) self.ExecOpCode(op) def testImportPlainFromGivenSrcNode(self): exp_info = """ [export] version=0 os=%s [instance] name=old_name.example.com """ % self.os.name self.rpc.call_export_info.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, exp_info) op = self.CopyOpCode(self.plain_op, mode=constants.INSTANCE_IMPORT, src_node=self.master.name) self.ExecOpCode(op) def testImportPlainWithoutSrcNodeNotFound(self): op = self.CopyOpCode(self.plain_op, mode=constants.INSTANCE_IMPORT) self.ExecOpCodeExpectOpPrereqError( op, "No export found for relative path") def testImportPlainWithoutSrcNode(self): exp_info = """ [export] version=0 os=%s [instance] name=old_name.example.com """ % self.os.name self.rpc.call_export_list.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, {"mock_path": {}}) \ .Build() self.rpc.call_export_info.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, exp_info) op = self.CopyOpCode(self.plain_op, mode=constants.INSTANCE_IMPORT, src_path="mock_path") self.ExecOpCode(op) def testImportPlainCorruptExportInfo(self): exp_info = "" self.rpc.call_export_info.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, exp_info) op = self.CopyOpCode(self.plain_op, mode=constants.INSTANCE_IMPORT, src_node=self.master.name) self.ExecOpCodeExpectException(op, errors.ProgrammerError, "Corrupted export config") def testImportPlainWrongExportInfoVersion(self): exp_info = """ [export] version=1 """ self.rpc.call_export_info.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, exp_info) op = self.CopyOpCode(self.plain_op, mode=constants.INSTANCE_IMPORT, src_node=self.master.name) self.ExecOpCodeExpectOpPrereqError(op, "Wrong export version") def testImportPlainWithParametersAndImport(self): exp_info = """ [export] version=0 os=%s [instance] name=old_name.example.com disk0_size=1024 disk1_size=1500 disk1_dump=mock_path nic0_mode=bridged nic0_link=br_mock nic0_mac=f6:ab:f4:45:d1:af nic0_ip=192.0.2.1 tags=tag1 tag2 hypervisor=xen-hvm [hypervisor] boot_order=cd [backend] memory=1024 vcpus=8 [os] param1=val1 """ % self.os.name self.rpc.call_export_info.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, exp_info) self.rpc.call_import_start.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, "daemon_name") self.rpc.call_impexp_status.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, [ objects.ImportExportStatus(exit_status=0) ]) self.rpc.call_impexp_cleanup.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) op = self.CopyOpCode(self.plain_op, disks=[], nics=[], tags=[], hypervisor=None, hvparams={}, mode=constants.INSTANCE_IMPORT, src_node=self.master.name) self.ExecOpCode(op) class TestDiskTemplateDiskTypeBijection(TestLUInstanceCreate): """Tests that one disk template corresponds to exactly one disk type.""" def GetSingleInstance(self): instances = self.cfg.GetInstancesInfoByFilter(lambda _: True) self.assertEqual(len(instances), 1, "Expected 1 instance, got\n%s" % instances) return list(instances.values())[0] def testDiskTemplateLogicalIdBijectionDiskless(self): op = self.CopyOpCode(self.diskless_op) self.ExecOpCode(op) instance = self.GetSingleInstance() self.assertEqual(instance.disk_template, constants.DT_DISKLESS) self.assertEqual(instance.disks, []) def testDiskTemplateLogicalIdBijectionPlain(self): op = self.CopyOpCode(self.plain_op) self.ExecOpCode(op) instance = self.GetSingleInstance() self.assertEqual(instance.disk_template, constants.DT_PLAIN) disks = self.cfg.GetInstanceDisks(instance.uuid) self.assertEqual(disks[0].dev_type, constants.DT_PLAIN) def testDiskTemplateLogicalIdBijectionBlock(self): self.rpc.call_bdev_sizes.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, { "/dev/disk/block0": 10000 }) \ .Build() op = self.CopyOpCode(self.block_op) self.ExecOpCode(op) instance = self.GetSingleInstance() self.assertEqual(instance.disk_template, constants.DT_BLOCK) disks = self.cfg.GetInstanceDisks(instance.uuid) self.assertEqual(disks[0].dev_type, constants.DT_BLOCK) def testDiskTemplateLogicalIdBijectionDrbd(self): op = self.CopyOpCode(self.drbd_op) self.ExecOpCode(op) instance = self.GetSingleInstance() self.assertEqual(instance.disk_template, constants.DT_DRBD8) disks = self.cfg.GetInstanceDisks(instance.uuid) self.assertEqual(disks[0].dev_type, constants.DT_DRBD8) def testDiskTemplateLogicalIdBijectionFile(self): op = self.CopyOpCode(self.file_op) self.ExecOpCode(op) instance = self.GetSingleInstance() self.assertEqual(instance.disk_template, constants.DT_FILE) disks = self.cfg.GetInstanceDisks(instance.uuid) self.assertEqual(disks[0].dev_type, constants.DT_FILE) def testDiskTemplateLogicalIdBijectionSharedFile(self): self.cluster.shared_file_storage_dir = '/tmp' op = self.CopyOpCode(self.shared_file_op) self.ExecOpCode(op) instance = self.GetSingleInstance() self.assertEqual(instance.disk_template, constants.DT_SHARED_FILE) disks = self.cfg.GetInstanceDisks(instance.uuid) self.assertEqual(disks[0].dev_type, constants.DT_SHARED_FILE) def testDiskTemplateLogicalIdBijectionGluster(self): self.cluster.gluster_storage_dir = '/tmp' op = self.CopyOpCode(self.gluster_op) self.ExecOpCode(op) instance = self.GetSingleInstance() self.assertEqual(instance.disk_template, constants.DT_GLUSTER) disks = self.cfg.GetInstanceDisks(instance.uuid) self.assertEqual(disks[0].dev_type, constants.DT_GLUSTER) def testDiskTemplateLogicalIdBijectionRbd(self): op = self.CopyOpCode(self.rbd_op) self.ExecOpCode(op) instance = self.GetSingleInstance() self.assertEqual(instance.disk_template, constants.DT_RBD) disks = self.cfg.GetInstanceDisks(instance.uuid) self.assertEqual(disks[0].dev_type, constants.DT_RBD) class TestCheckOSVariant(CmdlibTestCase): def testNoVariantsSupported(self): os = self.cfg.CreateOs(supported_variants=[]) self.assertRaises(backend.RPCFail, backend._CheckOSVariant, os, "os+variant") def testNoVariantGiven(self): os = self.cfg.CreateOs(supported_variants=["default"]) self.assertRaises(backend.RPCFail, backend._CheckOSVariant, os, "os") def testWrongVariantGiven(self): os = self.cfg.CreateOs(supported_variants=["default"]) self.assertRaises(backend.RPCFail, backend._CheckOSVariant, os, "os+wrong_variant") def testOkWithVariant(self): os = self.cfg.CreateOs(supported_variants=["default"]) backend._CheckOSVariant(os, "os+default") def testOkWithoutVariant(self): os = self.cfg.CreateOs(supported_variants=[]) backend._CheckOSVariant(os, "os") class TestCheckTargetNodeIPolicy(TestLUInstanceCreate): def setUp(self): super(TestCheckTargetNodeIPolicy, self).setUp() self.op = self.diskless_op self.instance = self.cfg.AddNewInstance() self.target_group = self.cfg.AddNewNodeGroup() self.target_node = self.cfg.AddNewNode(group=self.target_group) @withLockedLU def testNoViolation(self, lu): compute_recoder = mock.Mock(return_value=[]) instance.CheckTargetNodeIPolicy(lu, NotImplemented, self.instance, self.target_node, NotImplemented, _compute_fn=compute_recoder) self.assertTrue(compute_recoder.called) self.mcpu.assertLogIsEmpty() @withLockedLU def testNoIgnore(self, lu): compute_recoder = mock.Mock(return_value=["mem_size not in range"]) self.assertRaises(errors.OpPrereqError, instance.CheckTargetNodeIPolicy, lu, NotImplemented, self.instance, self.target_node, NotImplemented, _compute_fn=compute_recoder) self.assertTrue(compute_recoder.called) self.mcpu.assertLogIsEmpty() @withLockedLU def testIgnoreViolation(self, lu): compute_recoder = mock.Mock(return_value=["mem_size not in range"]) instance.CheckTargetNodeIPolicy(lu, NotImplemented, self.instance, self.target_node, NotImplemented, ignore=True, _compute_fn=compute_recoder) self.assertTrue(compute_recoder.called) msg = ("Instance does not meet target node group's .* instance policy:" " mem_size not in range") self.mcpu.assertLogContainsRegex(msg) class TestIndexOperations(unittest.TestCase): """Test if index operations on containers work as expected.""" def testGetIndexFromIdentifierTail(self): """Check if -1 is translated to tail index.""" container = ['item1134'] idx = instance_utils.GetIndexFromIdentifier("-1", "test", container) self.assertEqual(1, idx) def testGetIndexFromIdentifierEmpty(self): """Check if empty containers return 0 as index.""" container = [] idx = instance_utils.GetIndexFromIdentifier("0", "test", container) self.assertEqual(0, idx) idx = instance_utils.GetIndexFromIdentifier("-1", "test", container) self.assertEqual(0, idx) def testGetIndexFromIdentifierError(self): """Check if wrong input raises an exception.""" container = [] self.assertRaises(errors.OpPrereqError, instance_utils.GetIndexFromIdentifier, "lala", "test", container) def testGetIndexFromIdentifierOffByOne(self): """Check for off-by-one errors.""" container = [] self.assertRaises(IndexError, instance_utils.GetIndexFromIdentifier, "1", "test", container) def testGetIndexFromIdentifierOutOfRange(self): """Check for identifiers out of the container range.""" container = [] self.assertRaises(IndexError, instance_utils.GetIndexFromIdentifier, "-1134", "test", container) self.assertRaises(IndexError, instance_utils.GetIndexFromIdentifier, "1134", "test", container) def testInsertItemtoIndex(self): """Test if we can insert an item to a container at a specified index.""" container = [] instance_utils.InsertItemToIndex(0, 2, container) self.assertEqual([2], container) instance_utils.InsertItemToIndex(0, 1, container) self.assertEqual([1, 2], container) instance_utils.InsertItemToIndex(-1, 3, container) self.assertEqual([1, 2, 3], container) self.assertRaises(AssertionError, instance_utils.InsertItemToIndex, -2, 1134, container) self.assertRaises(AssertionError, instance_utils.InsertItemToIndex, 4, 1134, container) class TestApplyContainerMods(unittest.TestCase): def applyAndAssert(self, container, inp, expected_container, expected_chgdesc=[]): """Apply a list of changes to a container and check the container state Parameters: @type container: List @param container: The container on which we will apply the changes @type inp: List<(action, index, object)> @param inp: The list of changes, a tupple with three elements: i. action, e.g. constants.DDM_ADD ii. index, e.g. -1, 0, 10 iii. object (any type) @type expected: List @param expected: The expected state of the container @type chgdesc: List @param chgdesc: List of applied changes """ chgdesc = [] mods = instance_utils.PrepareContainerMods(inp, None) instance_utils.ApplyContainerMods("test", container, chgdesc, mods, None, None, None, None, None) self.assertEqual(container, expected_container) self.assertEqual(chgdesc, expected_chgdesc) def _insertContainerSuccessFn(self, op): container = [] inp = [(op, -1, "Hello"), (op, -1, "World"), (op, 0, "Start"), (op, -1, "End"), ] expected = ["Start", "Hello", "World", "End"] self.applyAndAssert(container, inp, expected) inp = [(op, 0, "zero"), (op, 3, "Added"), (op, 5, "four"), (op, 7, "xyz"), ] expected = ["zero", "Start", "Hello", "Added", "World", "four", "End", "xyz"] self.applyAndAssert(container, inp, expected) def _insertContainerErrorFn(self, op): container = [] expected = None inp = [(op, 1, "error"), ] self.assertRaises(IndexError, self.applyAndAssert, container, inp, expected) inp = [(op, -2, "error"), ] self.assertRaises(IndexError, self.applyAndAssert, container, inp, expected) def _extractContainerSuccessFn(self, op): container = ["item1", "item2", "item3", "item4", "item5"] inp = [(op, -1, None), (op, -0, None), (op, 1, None), ] expected = ["item2", "item4"] chgdesc = [('test/4', op), ('test/0', op), ('test/1', op) ] self.applyAndAssert(container, inp, expected, chgdesc) def _extractContainerErrorFn(self, op): container = [] expected = None inp = [(op, 0, None), ] self.assertRaises(IndexError, self.applyAndAssert, container, inp, expected) inp = [(op, -1, None), ] self.assertRaises(IndexError, self.applyAndAssert, container, inp, expected) inp = [(op, 2, None), ] self.assertRaises(IndexError, self.applyAndAssert, container, inp, expected) container = [""] inp = [(op, 0, None), ] expected = None self.assertRaises(AssertionError, self.applyAndAssert, container, inp, expected) def testEmptyContainer(self): container = [] chgdesc = [] instance_utils.ApplyContainerMods("test", container, chgdesc, [], None, None, None, None, None) self.assertEqual(container, []) self.assertEqual(chgdesc, []) def testAddSuccess(self): self._insertContainerSuccessFn(constants.DDM_ADD) def testAddError(self): self._insertContainerErrorFn(constants.DDM_ADD) def testAttachSuccess(self): self._insertContainerSuccessFn(constants.DDM_ATTACH) def testAttachError(self): self._insertContainerErrorFn(constants.DDM_ATTACH) def testRemoveSuccess(self): self._extractContainerSuccessFn(constants.DDM_REMOVE) def testRemoveError(self): self._extractContainerErrorFn(constants.DDM_REMOVE) def testDetachSuccess(self): self._extractContainerSuccessFn(constants.DDM_DETACH) def testDetachError(self): self._extractContainerErrorFn(constants.DDM_DETACH) def testModify(self): container = ["item 1", "item 2"] mods = instance_utils.PrepareContainerMods([ (constants.DDM_MODIFY, -1, "a"), (constants.DDM_MODIFY, 0, "b"), (constants.DDM_MODIFY, 1, "c"), ], None) chgdesc = [] instance_utils.ApplyContainerMods("test", container, chgdesc, mods, None, None, None, None, None) self.assertEqual(container, ["item 1", "item 2"]) self.assertEqual(chgdesc, []) for idx in [-2, len(container) + 1]: mods = instance_utils.PrepareContainerMods([ (constants.DDM_MODIFY, idx, "error"), ], None) self.assertRaises(IndexError, instance_utils.ApplyContainerMods, "test", container, None, mods, None, None, None, None, None) @staticmethod def _CreateTestFn(idx, params, private): private.data = ("add", idx, params) return ((100 * idx, params), [ ("test/%s" % idx, hex(idx)), ]) @staticmethod def _AttachTestFn(idx, params, private): private.data = ("attach", idx, params) return ((100 * idx, params), [ ("test/%s" % idx, hex(idx)), ]) @staticmethod def _ModifyTestFn(idx, item, params, private): private.data = ("modify", idx, params) return [ ("test/%s" % idx, "modify %s" % params), ] @staticmethod def _RemoveTestFn(idx, item, private): private.data = ("remove", idx, item) @staticmethod def _DetachTestFn(idx, item, private): private.data = ("detach", idx, item) def testAddWithCreateFunction(self): container = [] chgdesc = [] mods = instance_utils.PrepareContainerMods([ (constants.DDM_ADD, -1, "Hello"), (constants.DDM_ADD, -1, "World"), (constants.DDM_ADD, 0, "Start"), (constants.DDM_ADD, -1, "End"), (constants.DDM_REMOVE, 2, None), (constants.DDM_MODIFY, -1, "foobar"), (constants.DDM_REMOVE, 2, None), (constants.DDM_ADD, 1, "More"), (constants.DDM_DETACH, -1, None), (constants.DDM_ATTACH, 0, "Hello"), ], mock.Mock) instance_utils.ApplyContainerMods("test", container, chgdesc, mods, self._CreateTestFn, self._AttachTestFn, self._ModifyTestFn, self._RemoveTestFn, self._DetachTestFn) self.assertEqual(container, [ (000, "Hello"), (000, "Start"), (100, "More"), ]) self.assertEqual(chgdesc, [ ("test/0", "0x0"), ("test/1", "0x1"), ("test/0", "0x0"), ("test/3", "0x3"), ("test/2", "remove"), ("test/2", "modify foobar"), ("test/2", "remove"), ("test/1", "0x1"), ("test/2", "detach"), ("test/0", "0x0"), ]) self.assertTrue(compat.all(op == private.data[0] for (op, _, _, private) in mods)) self.assertEqual([private.data for (op, _, _, private) in mods], [ ("add", 0, "Hello"), ("add", 1, "World"), ("add", 0, "Start"), ("add", 3, "End"), ("remove", 2, (100, "World")), ("modify", 2, "foobar"), ("remove", 2, (300, "End")), ("add", 1, "More"), ("detach", 2, (000, "Hello")), ("attach", 0, "Hello"), ]) class _FakeConfigForGenDiskTemplate(ConfigMock): def __init__(self): super(_FakeConfigForGenDiskTemplate, self).__init__() self._unique_id = itertools.count() self._drbd_minor = itertools.count(20) self._port = itertools.count(constants.FIRST_DRBD_PORT) self._secret = itertools.count() def GenerateUniqueID(self, ec_id): return "ec%s-uq%s" % (ec_id, next(self._unique_id)) def AllocateDRBDMinor(self, nodes, disk): return [next(self._drbd_minor) for _ in nodes] def AllocatePort(self): return next(self._port) def GenerateDRBDSecret(self, ec_id): return "ec%s-secret%s" % (ec_id, next(self._secret)) class TestGenerateDiskTemplate(CmdlibTestCase): def setUp(self): super(TestGenerateDiskTemplate, self).setUp() self.cfg = _FakeConfigForGenDiskTemplate() self.cluster.enabled_disk_templates = list(constants.DISK_TEMPLATES) self.nodegroup = self.cfg.AddNewNodeGroup(name="ng") self.lu = self.GetMockLU() @staticmethod def GetDiskParams(): return copy.deepcopy(constants.DISK_DT_DEFAULTS) def testWrongDiskTemplate(self): gdt = instance_storage.GenerateDiskTemplate disk_template = "##unknown##" assert disk_template not in constants.DISK_TEMPLATES self.assertRaises(errors.OpPrereqError, gdt, self.lu, disk_template, "inst26831.example.com", "node30113.example.com", [], [], NotImplemented, NotImplemented, 0, self.lu.LogInfo, self.GetDiskParams()) def testDiskless(self): gdt = instance_storage.GenerateDiskTemplate result = gdt(self.lu, constants.DT_DISKLESS, "inst27734.example.com", "node30113.example.com", [], [], NotImplemented, NotImplemented, 0, self.lu.LogInfo, self.GetDiskParams()) self.assertEqual(result, []) def _TestTrivialDisk(self, template, disk_info, base_index, exp_dev_type, file_storage_dir=NotImplemented, file_driver=NotImplemented): gdt = instance_storage.GenerateDiskTemplate for params in disk_info: utils.ForceDictType(params, constants.IDISK_PARAMS_TYPES) # Check if non-empty list of secondaries is rejected self.assertRaises(errors.ProgrammerError, gdt, self.lu, template, "inst25088.example.com", "node185.example.com", ["node323.example.com"], [], NotImplemented, NotImplemented, base_index, self.lu.LogInfo, self.GetDiskParams()) result = gdt(self.lu, template, "inst21662.example.com", "node21741.example.com", [], disk_info, file_storage_dir, file_driver, base_index, self.lu.LogInfo, self.GetDiskParams()) for (idx, disk) in enumerate(result): self.assertTrue(isinstance(disk, objects.Disk)) self.assertEqual(disk.dev_type, exp_dev_type) self.assertEqual(disk.size, disk_info[idx][constants.IDISK_SIZE]) self.assertEqual(disk.mode, disk_info[idx][constants.IDISK_MODE]) self.assertTrue(disk.children is None) self._CheckIvNames(result, base_index, base_index + len(disk_info)) _UpdateIvNames(base_index, result) self._CheckIvNames(result, base_index, base_index + len(disk_info)) return result def _CheckIvNames(self, disks, base_index, end_index): self.assertEqual([d.iv_name for d in disks], ["disk/%s" % i for i in range(base_index, end_index)]) def testPlain(self): disk_info = [{ constants.IDISK_SIZE: 1024, constants.IDISK_MODE: constants.DISK_RDWR, }, { constants.IDISK_SIZE: 4096, constants.IDISK_VG: "othervg", constants.IDISK_MODE: constants.DISK_RDWR, }] result = self._TestTrivialDisk(constants.DT_PLAIN, disk_info, 3, constants.DT_PLAIN) self.assertEqual([d.logical_id for d in result], [ ("xenvg", "ec1-uq0.disk3"), ("othervg", "ec1-uq1.disk4"), ]) self.assertEqual([d.nodes for d in result], [ ["node21741.example.com"], ["node21741.example.com"]]) def testFile(self): # anything != DT_FILE would do here self.cluster.enabled_disk_templates = [constants.DT_PLAIN] self.assertRaises(errors.OpPrereqError, self._TestTrivialDisk, constants.DT_FILE, [], 0, NotImplemented) self.assertRaises(errors.OpPrereqError, self._TestTrivialDisk, constants.DT_SHARED_FILE, [], 0, NotImplemented) for disk_template in constants.DTS_FILEBASED: disk_info = [{ constants.IDISK_SIZE: 80 * 1024, constants.IDISK_MODE: constants.DISK_RDONLY, }, { constants.IDISK_SIZE: 4096, constants.IDISK_MODE: constants.DISK_RDWR, }, { constants.IDISK_SIZE: 6 * 1024, constants.IDISK_MODE: constants.DISK_RDWR, }] self.cluster.enabled_disk_templates = [disk_template] result = self._TestTrivialDisk( disk_template, disk_info, 2, disk_template, file_storage_dir="/tmp", file_driver=constants.FD_BLKTAP) if disk_template == constants.DT_GLUSTER: # Here "inst21662.example.com" is actually the instance UUID, not its # name, so while this result looks wrong, it is actually correct. expected = [(constants.FD_BLKTAP, 'ganeti/inst21662.example.com.%d' % x) for x in (2,3,4)] self.assertEqual([d.logical_id for d in result], expected) self.assertEqual([d.nodes for d in result], [ [], [], []]) else: if disk_template == constants.DT_FILE: self.assertEqual([d.nodes for d in result], [ ["node21741.example.com"], ["node21741.example.com"], ["node21741.example.com"]]) else: self.assertEqual([d.nodes for d in result], [ [], [], []]) for (idx, disk) in enumerate(result): (file_driver, file_storage_dir) = disk.logical_id dir_fmt = r"^/tmp/.*\.%s\.disk%d$" % (disk_template, idx + 2) self.assertEqual(file_driver, constants.FD_BLKTAP) # FIXME: use assertIsNotNone when py 2.7 is minimum supported version self.assertNotEqual(re.match(dir_fmt, file_storage_dir), None) def testBlock(self): disk_info = [{ constants.IDISK_SIZE: 8 * 1024, constants.IDISK_MODE: constants.DISK_RDWR, constants.IDISK_ADOPT: "/tmp/some/block/dev", }] result = self._TestTrivialDisk(constants.DT_BLOCK, disk_info, 10, constants.DT_BLOCK) self.assertEqual([d.logical_id for d in result], [ (constants.BLOCKDEV_DRIVER_MANUAL, "/tmp/some/block/dev"), ]) self.assertEqual([d.nodes for d in result], [[]]) def testRbd(self): disk_info = [{ constants.IDISK_SIZE: 8 * 1024, constants.IDISK_MODE: constants.DISK_RDONLY, }, { constants.IDISK_SIZE: 100 * 1024, constants.IDISK_MODE: constants.DISK_RDWR, }] result = self._TestTrivialDisk(constants.DT_RBD, disk_info, 0, constants.DT_RBD) self.assertEqual([d.logical_id for d in result], [ ("rbd", "ec1-uq0.rbd.disk0"), ("rbd", "ec1-uq1.rbd.disk1"), ]) self.assertEqual([d.nodes for d in result], [[], []]) def testDrbd8(self): gdt = instance_storage.GenerateDiskTemplate drbd8_defaults = constants.DISK_LD_DEFAULTS[constants.DT_DRBD8] drbd8_default_metavg = drbd8_defaults[constants.LDP_DEFAULT_METAVG] disk_info = [{ constants.IDISK_SIZE: 1024, constants.IDISK_MODE: constants.DISK_RDWR, }, { constants.IDISK_SIZE: 100 * 1024, constants.IDISK_MODE: constants.DISK_RDONLY, constants.IDISK_METAVG: "metavg", }, { constants.IDISK_SIZE: 4096, constants.IDISK_MODE: constants.DISK_RDWR, constants.IDISK_VG: "vgxyz", }, ] exp_logical_ids = [ [ (self.lu.cfg.GetVGName(), "ec1-uq0.disk0_data"), (drbd8_default_metavg, "ec1-uq0.disk0_meta"), ], [ (self.lu.cfg.GetVGName(), "ec1-uq1.disk1_data"), ("metavg", "ec1-uq1.disk1_meta"), ], [ ("vgxyz", "ec1-uq2.disk2_data"), (drbd8_default_metavg, "ec1-uq2.disk2_meta"), ]] exp_nodes = ["node1334.example.com", "node12272.example.com"] assert len(exp_logical_ids) == len(disk_info) for params in disk_info: utils.ForceDictType(params, constants.IDISK_PARAMS_TYPES) # Check if empty list of secondaries is rejected self.assertRaises(errors.ProgrammerError, gdt, self.lu, constants.DT_DRBD8, "inst827.example.com", "node1334.example.com", [], disk_info, NotImplemented, NotImplemented, 0, self.lu.LogInfo, self.GetDiskParams()) result = gdt(self.lu, constants.DT_DRBD8, "inst827.example.com", "node1334.example.com", ["node12272.example.com"], disk_info, NotImplemented, NotImplemented, 0, self.lu.LogInfo, self.GetDiskParams()) for (idx, disk) in enumerate(result): self.assertTrue(isinstance(disk, objects.Disk)) self.assertEqual(disk.dev_type, constants.DT_DRBD8) self.assertEqual(disk.size, disk_info[idx][constants.IDISK_SIZE]) self.assertEqual(disk.mode, disk_info[idx][constants.IDISK_MODE]) for child in disk.children: self.assertTrue(isinstance(disk, objects.Disk)) self.assertEqual(child.dev_type, constants.DT_PLAIN) self.assertTrue(child.children is None) self.assertEqual(child.nodes, exp_nodes) self.assertEqual([d.logical_id for d in disk.children], exp_logical_ids[idx]) self.assertEqual(disk.nodes, exp_nodes) self.assertEqual(len(disk.children), 2) self.assertEqual(disk.children[0].size, disk.size) self.assertEqual(disk.children[1].size, constants.DRBD_META_SIZE) self._CheckIvNames(result, 0, len(disk_info)) _UpdateIvNames(0, result) self._CheckIvNames(result, 0, len(disk_info)) self.assertEqual([d.logical_id for d in result], [ ("node1334.example.com", "node12272.example.com", constants.FIRST_DRBD_PORT, 20, 21, "ec1-secret0"), ("node1334.example.com", "node12272.example.com", constants.FIRST_DRBD_PORT + 1, 22, 23, "ec1-secret1"), ("node1334.example.com", "node12272.example.com", constants.FIRST_DRBD_PORT + 2, 24, 25, "ec1-secret2"), ]) class _DiskPauseTracker: def __init__(self): self.history = [] def __call__(self, instance_disks, pause): (disks, instance) = instance_disks disk_uuids = [d.uuid for d in disks] assert not (set(disk_uuids) - set(instance.disks)) self.history.extend((i.logical_id, i.size, pause) for i in disks) return (True, [True] * len(disks)) class _ConfigForDiskWipe: def __init__(self, exp_node_uuid, disks): self._exp_node_uuid = exp_node_uuid self._disks = disks def GetNodeName(self, node_uuid): assert node_uuid == self._exp_node_uuid return "name.of.expected.node" def GetInstanceDisks(self, _): return self._disks class _RpcForDiskWipe: def __init__(self, exp_node, pause_cb, wipe_cb): self._exp_node = exp_node self._pause_cb = pause_cb self._wipe_cb = wipe_cb def call_blockdev_pause_resume_sync(self, node, disks, pause): assert node == self._exp_node return rpc.RpcResult(data=self._pause_cb(disks, pause)) def call_blockdev_wipe(self, node, bdev, offset, size): assert node == self._exp_node return rpc.RpcResult(data=self._wipe_cb(bdev, offset, size)) class _DiskWipeProgressTracker: def __init__(self, start_offset): self._start_offset = start_offset self.progress = {} def __call__(self, disk_info, offset, size): (disk, _) = disk_info assert isinstance(offset, int) assert isinstance(size, int) max_chunk_size = (disk.size / 100.0 * constants.MIN_WIPE_CHUNK_PERCENT) assert offset >= self._start_offset assert (offset + size) <= disk.size assert size > 0 assert size <= constants.MAX_WIPE_CHUNK assert size <= max_chunk_size assert offset == self._start_offset or disk.logical_id in self.progress # Keep track of progress cur_progress = self.progress.setdefault(disk.logical_id, self._start_offset) assert cur_progress == offset # Record progress self.progress[disk.logical_id] += size return (True, None) class TestWipeDisks(unittest.TestCase): def _FailingPauseCb(self, disks_info, pause): (disks, _) = disks_info self.assertEqual(len(disks), 3) self.assertTrue(pause) # Simulate an RPC error return (False, "error") def testPauseFailure(self): node_name = "node1372.example.com" disks = [ objects.Disk(dev_type=constants.DT_PLAIN, uuid="disk0"), objects.Disk(dev_type=constants.DT_PLAIN, uuid="disk1"), objects.Disk(dev_type=constants.DT_PLAIN, uuid="disk2"), ] lu = _FakeLU(rpc=_RpcForDiskWipe(node_name, self._FailingPauseCb, NotImplemented), cfg=_ConfigForDiskWipe(node_name, disks)) inst = objects.Instance(name="inst21201", primary_node=node_name, disk_template=constants.DT_PLAIN, disks=[d.uuid for d in disks]) self.assertRaises(errors.OpExecError, instance_create.WipeDisks, lu, inst) def _FailingWipeCb(self, disk_info, offset, size): # This should only ever be called for the first disk (disk, _) = disk_info self.assertEqual(disk.logical_id, "disk0") return (False, None) def testFailingWipe(self): node_uuid = "node13445-uuid" pt = _DiskPauseTracker() disks = [ objects.Disk(dev_type=constants.DT_PLAIN, logical_id="disk0", size=100 * 1024, uuid="disk0"), objects.Disk(dev_type=constants.DT_PLAIN, logical_id="disk1", size=500 * 1024, uuid="disk1"), objects.Disk(dev_type=constants.DT_PLAIN, logical_id="disk2", size=256, uuid="disk2"), ] lu = _FakeLU(rpc=_RpcForDiskWipe(node_uuid, pt, self._FailingWipeCb), cfg=_ConfigForDiskWipe(node_uuid, disks)) inst = objects.Instance(name="inst562", primary_node=node_uuid, disk_template=constants.DT_PLAIN, disks=[d.uuid for d in disks]) try: instance_create.WipeDisks(lu, inst) except errors.OpExecError as err: self.assertTrue(str(err), "Could not wipe disk 0 at offset 0 ") else: self.fail("Did not raise exception") # Check if all disks were paused and resumed self.assertEqual(pt.history, [ ("disk0", 100 * 1024, True), ("disk1", 500 * 1024, True), ("disk2", 256, True), ("disk0", 100 * 1024, False), ("disk1", 500 * 1024, False), ("disk2", 256, False), ]) def _PrepareWipeTest(self, start_offset, disks): node_name = "node-with-offset%s.example.com" % start_offset pauset = _DiskPauseTracker() progresst = _DiskWipeProgressTracker(start_offset) lu = _FakeLU(rpc=_RpcForDiskWipe(node_name, pauset, progresst), cfg=_ConfigForDiskWipe(node_name, disks)) instance = objects.Instance(name="inst3560", primary_node=node_name, disk_template=constants.DT_PLAIN, disks=[d.uuid for d in disks]) return (lu, instance, pauset, progresst) def testNormalWipe(self): disks = [ objects.Disk(dev_type=constants.DT_PLAIN, logical_id="disk0", size=1024, uuid="disk0"), objects.Disk(dev_type=constants.DT_PLAIN, logical_id="disk1", size=500 * 1024, uuid="disk1"), objects.Disk(dev_type=constants.DT_PLAIN, logical_id="disk2", size=128, uuid="disk2"), objects.Disk(dev_type=constants.DT_PLAIN, logical_id="disk3", size=constants.MAX_WIPE_CHUNK, uuid="disk3"), ] (lu, inst, pauset, progresst) = self._PrepareWipeTest(0, disks) instance_create.WipeDisks(lu, inst) self.assertEqual(pauset.history, [ ("disk0", 1024, True), ("disk1", 500 * 1024, True), ("disk2", 128, True), ("disk3", constants.MAX_WIPE_CHUNK, True), ("disk0", 1024, False), ("disk1", 500 * 1024, False), ("disk2", 128, False), ("disk3", constants.MAX_WIPE_CHUNK, False), ]) # Ensure the complete disk has been wiped self.assertEqual(progresst.progress, dict((i.logical_id, i.size) for i in disks)) def testWipeWithStartOffset(self): for start_offset in [0, 280, 8895, 1563204]: disks = [ objects.Disk(dev_type=constants.DT_PLAIN, logical_id="disk0", size=128, uuid="disk0"), objects.Disk(dev_type=constants.DT_PLAIN, logical_id="disk1", size=start_offset + (100 * 1024), uuid="disk1"), ] (lu, inst, pauset, progresst) = \ self._PrepareWipeTest(start_offset, disks) # Test start offset with only one disk instance_create.WipeDisks(lu, inst, disks=[(1, disks[1], start_offset)]) # Only the second disk may have been paused and wiped self.assertEqual(pauset.history, [ ("disk1", start_offset + (100 * 1024), True), ("disk1", start_offset + (100 * 1024), False), ]) self.assertEqual(progresst.progress, { "disk1": disks[1].size, }) class TestCheckOpportunisticLocking(unittest.TestCase): class OpTest(opcodes.OpCode): OP_PARAMS = [ ("opportunistic_locking", False, ht.TBool, None), ("iallocator", None, ht.TMaybe(ht.TNonEmptyString), "") ] @classmethod def _MakeOp(cls, **kwargs): op = cls.OpTest(**kwargs) op.Validate(True) return op def testMissingAttributes(self): self.assertRaises(AttributeError, instance.CheckOpportunisticLocking, object()) def testDefaults(self): op = self._MakeOp() instance.CheckOpportunisticLocking(op) def test(self): for iallocator in [None, "something", "other"]: for opplock in [False, True]: op = self._MakeOp(iallocator=iallocator, opportunistic_locking=opplock) if opplock and not iallocator: self.assertRaises(errors.OpPrereqError, instance.CheckOpportunisticLocking, op) else: instance.CheckOpportunisticLocking(op) class TestLUInstanceMove(CmdlibTestCase): def setUp(self): super(TestLUInstanceMove, self).setUp() self.node = self.cfg.AddNewNode() self.rpc.call_blockdev_assemble.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.node, ("/dev/mocked_path", "/var/run/ganeti/instance-disks/mocked_d", None)) self.rpc.call_blockdev_remove.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, "") def ImportStart(node_uuid, opt, inst, component, args): return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node_uuid, "deamon_on_%s" % node_uuid) self.rpc.call_import_start.side_effect = ImportStart def ImpExpStatus(node_uuid, name): return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node_uuid, [objects.ImportExportStatus( exit_status=0 )]) self.rpc.call_impexp_status.side_effect = ImpExpStatus def ImpExpCleanup(node_uuid, name): return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node_uuid) self.rpc.call_impexp_cleanup.side_effect = ImpExpCleanup def testMissingInstance(self): op = opcodes.OpInstanceMove(instance_name="missing.inst", target_node=self.node.name) self.ExecOpCodeExpectOpPrereqError(op, "Instance 'missing.inst' not known") def testUncopyableDiskTemplate(self): inst = self.cfg.AddNewInstance(disk_template=constants.DT_SHARED_FILE) op = opcodes.OpInstanceMove(instance_name=inst.name, target_node=self.node.name) self.ExecOpCodeExpectOpPrereqError( op, "Instance disk 0 has disk type sharedfile and is not suitable" " for copying") def testAlreadyOnTargetNode(self): inst = self.cfg.AddNewInstance() op = opcodes.OpInstanceMove(instance_name=inst.name, target_node=self.master.name) self.ExecOpCodeExpectOpPrereqError( op, "Instance .* is already on the node .*") def testMoveStoppedInstance(self): inst = self.cfg.AddNewInstance() op = opcodes.OpInstanceMove(instance_name=inst.name, target_node=self.node.name) self.ExecOpCode(op) def testMoveRunningInstance(self): self.rpc.call_node_info.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.node, (NotImplemented, NotImplemented, ({"memory_free": 10000}, ))) \ .Build() self.rpc.call_instance_start.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.node, "") inst = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP) op = opcodes.OpInstanceMove(instance_name=inst.name, target_node=self.node.name) self.ExecOpCode(op) def testMoveFailingStartInstance(self): self.rpc.call_node_info.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.node, (NotImplemented, NotImplemented, ({"memory_free": 10000}, ))) \ .Build() self.rpc.call_instance_start.return_value = \ self.RpcResultsBuilder() \ .CreateFailedNodeResult(self.node) inst = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP) op = opcodes.OpInstanceMove(instance_name=inst.name, target_node=self.node.name) self.ExecOpCodeExpectOpExecError( op, "Could not start instance .* on node .*") def testMoveFailingImpExpDaemonExitCode(self): inst = self.cfg.AddNewInstance() self.rpc.call_impexp_status.side_effect = None self.rpc.call_impexp_status.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.node, [objects.ImportExportStatus( exit_status=1, recent_output=["mock output"] )]) op = opcodes.OpInstanceMove(instance_name=inst.name, target_node=self.node.name) self.ExecOpCodeExpectOpExecError(op, "Errors during disk copy") def testMoveFailingStartImpExpDaemon(self): inst = self.cfg.AddNewInstance() self.rpc.call_import_start.side_effect = None self.rpc.call_import_start.return_value = \ self.RpcResultsBuilder() \ .CreateFailedNodeResult(self.node) op = opcodes.OpInstanceMove(instance_name=inst.name, target_node=self.node.name) self.ExecOpCodeExpectOpExecError(op, "Errors during disk copy") class TestLUInstanceRename(CmdlibTestCase): def setUp(self): super(TestLUInstanceRename, self).setUp() self.MockOut(instance_utils, 'netutils', self.netutils_mod) self.inst = self.cfg.AddNewInstance() self.op = opcodes.OpInstanceRename(instance_name=self.inst.name, new_name="new_name.example.com") def testIpCheckWithoutNameCheck(self): op = self.CopyOpCode(self.op, ip_check=True, name_check=False) self.ExecOpCodeExpectOpPrereqError( op, "IP address check requires a name check") def testIpAlreadyInUse(self): self.netutils_mod.TcpPing.return_value = True op = self.CopyOpCode(self.op, name_check=True, ip_check=True) self.ExecOpCodeExpectOpPrereqError( op, "IP .* of instance .* already in use") def testExistingInstanceName(self): self.cfg.AddNewInstance(name="new_name.example.com") op = self.CopyOpCode(self.op) self.ExecOpCodeExpectOpPrereqError( op, "Instance .* is already in the cluster") def testFileInstance(self): self.rpc.call_blockdev_assemble.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, (None, None, None)) self.rpc.call_blockdev_shutdown.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, (None, None)) inst = self.cfg.AddNewInstance(disk_template=constants.DT_FILE) op = self.CopyOpCode(self.op, instance_name=inst.name) self.ExecOpCode(op) class TestLUInstanceMultiAlloc(CmdlibTestCase): def setUp(self): super(TestLUInstanceMultiAlloc, self).setUp() self.inst_op = opcodes.OpInstanceCreate(instance_name="inst.example.com", disk_template=constants.DT_DRBD8, disks=[], nics=[], os_type="mock_os", hypervisor=constants.HT_XEN_HVM, mode=constants.INSTANCE_CREATE) def testInstanceWithIAllocator(self): inst = self.CopyOpCode(self.inst_op, iallocator="mock") op = opcodes.OpInstanceMultiAlloc(instances=[inst]) self.ExecOpCodeExpectOpPrereqError( op, "iallocator are not allowed to be set on instance objects") def testOnlySomeNodesGiven(self): inst1 = self.CopyOpCode(self.inst_op, pnode=self.master.name) inst2 = self.CopyOpCode(self.inst_op) op = opcodes.OpInstanceMultiAlloc(instances=[inst1, inst2]) self.ExecOpCodeExpectOpPrereqError( op, "There are instance objects providing pnode/snode while others" " do not") def testMissingIAllocator(self): self.cluster.default_iallocator = None inst = self.CopyOpCode(self.inst_op) op = opcodes.OpInstanceMultiAlloc(instances=[inst]) self.ExecOpCodeExpectOpPrereqError( op, "No iallocator or nodes on the instances given and no cluster-wide" " default iallocator found") def testDuplicateInstanceNames(self): inst1 = self.CopyOpCode(self.inst_op) inst2 = self.CopyOpCode(self.inst_op) op = opcodes.OpInstanceMultiAlloc(instances=[inst1, inst2]) self.ExecOpCodeExpectOpPrereqError( op, "There are duplicate instance names") def testWithGivenNodes(self): snode = self.cfg.AddNewNode() inst = self.CopyOpCode(self.inst_op, pnode=self.master.name, snode=snode.name) op = opcodes.OpInstanceMultiAlloc(instances=[inst]) self.ExecOpCode(op) def testDryRun(self): snode = self.cfg.AddNewNode() inst = self.CopyOpCode(self.inst_op, pnode=self.master.name, snode=snode.name) op = opcodes.OpInstanceMultiAlloc(instances=[inst], dry_run=True) self.ExecOpCode(op) def testWithIAllocator(self): snode = self.cfg.AddNewNode() self.iallocator_cls.return_value.result = \ ([("inst.example.com", [self.master.name, snode.name])], []) inst = self.CopyOpCode(self.inst_op) op = opcodes.OpInstanceMultiAlloc(instances=[inst], iallocator="mock_ialloc") self.ExecOpCode(op) def testManyInstancesWithIAllocator(self): snode = self.cfg.AddNewNode() inst1 = self.CopyOpCode(self.inst_op) inst2 = self.CopyOpCode(self.inst_op, instance_name="inst2.example.com") self.iallocator_cls.return_value.result = \ ([("inst.example.com", [self.master.name, snode.name]), ("inst2.example.com", [self.master.name, snode.name])], []) op = opcodes.OpInstanceMultiAlloc(instances=[inst1, inst2], iallocator="mock_ialloc") self.ExecOpCode(op) def testWithIAllocatorOpportunisticLocking(self): snode = self.cfg.AddNewNode() self.iallocator_cls.return_value.result = \ ([("inst.example.com", [self.master.name, snode.name])], []) inst = self.CopyOpCode(self.inst_op) op = opcodes.OpInstanceMultiAlloc(instances=[inst], iallocator="mock_ialloc", opportunistic_locking=True) self.ExecOpCode(op) def testFailingIAllocator(self): self.iallocator_cls.return_value.success = False inst = self.CopyOpCode(self.inst_op) op = opcodes.OpInstanceMultiAlloc(instances=[inst], iallocator="mock_ialloc") self.ExecOpCodeExpectOpPrereqError( op, "Can't compute nodes using iallocator") class TestLUInstanceSetParams(CmdlibTestCase): def setUp(self): super(TestLUInstanceSetParams, self).setUp() self.MockOut(instance_set_params, 'netutils', self.netutils_mod) self.MockOut(instance_utils, 'netutils', self.netutils_mod) self.dev_type = constants.DT_PLAIN self.inst = self.cfg.AddNewInstance(disk_template=self.dev_type) self.op = opcodes.OpInstanceSetParams(instance_name=self.inst.name) self.cfg._cluster.default_iallocator=None self.running_inst = \ self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP) self.running_op = \ opcodes.OpInstanceSetParams(instance_name=self.running_inst.name) ext_disks = [self.cfg.CreateDisk(dev_type=constants.DT_EXT, params={ constants.IDISK_PROVIDER: "pvdr" })] self.ext_storage_inst = \ self.cfg.AddNewInstance(disk_template=constants.DT_EXT, disks=ext_disks) self.ext_storage_op = \ opcodes.OpInstanceSetParams(instance_name=self.ext_storage_inst.name) self.snode = self.cfg.AddNewNode() self.mocked_storage_type = constants.ST_LVM_VG self.mocked_storage_free = 10000 self.mocked_master_cpu_total = 16 self.mocked_master_memory_free = 2048 self.mocked_snode_cpu_total = 16 self.mocked_snode_memory_free = 512 self.mocked_running_inst_memory = 1024 self.mocked_running_inst_vcpus = 8 self.mocked_running_inst_state = "running" self.mocked_running_inst_time = 10938474 self.mocked_disk_uuid = "mock_uuid_1134" self.mocked_disk_name = "mock_disk_1134" bootid = "mock_bootid" storage_info = [ { "type": self.mocked_storage_type, "storage_free": self.mocked_storage_free } ] hv_info_master = { "cpu_total": self.mocked_master_cpu_total, "memory_free": self.mocked_master_memory_free } hv_info_snode = { "cpu_total": self.mocked_snode_cpu_total, "memory_free": self.mocked_snode_memory_free } self.rpc.call_node_info.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master, (bootid, storage_info, (hv_info_master, ))) \ .AddSuccessfulNode(self.snode, (bootid, storage_info, (hv_info_snode, ))) \ .Build() def _InstanceInfo(_, instance, __, ___): if instance in [self.inst.name, self.ext_storage_inst.name]: return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, None) elif instance == self.running_inst.name: return self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult( self.master, { "memory": self.mocked_running_inst_memory, "vcpus": self.mocked_running_inst_vcpus, "state": self.mocked_running_inst_state, "time": self.mocked_running_inst_time }) else: raise AssertionError() self.rpc.call_instance_info.side_effect = _InstanceInfo self.rpc.call_bridges_exist.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) self.rpc.call_blockdev_getmirrorstatus.side_effect = \ lambda node, _: self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node, []) self.rpc.call_blockdev_shutdown.side_effect = \ lambda node, _: self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(node, []) def testNoChanges(self): op = self.CopyOpCode(self.op) self.ExecOpCodeExpectOpPrereqError(op, "No changes submitted") def testGlobalHvparams(self): op = self.CopyOpCode(self.op, hvparams={constants.HV_MIGRATION_PORT: 1234}) self.ExecOpCodeExpectOpPrereqError( op, "hypervisor parameters are global and cannot be customized") def testHvparams(self): op = self.CopyOpCode(self.op, hvparams={constants.HV_BOOT_ORDER: "cd"}) self.ExecOpCode(op) def testDisksAndDiskTemplate(self): op = self.CopyOpCode(self.op, disk_template=constants.DT_PLAIN, disks=[[constants.DDM_ADD, -1, {}]]) self.ExecOpCodeExpectOpPrereqError( op, "Disk template conversion and other disk changes not supported at" " the same time") def testDiskTemplateToMirroredNoRemoteNode(self): op = self.CopyOpCode(self.op, disk_template=constants.DT_DRBD8) self.ExecOpCodeExpectOpPrereqError( op, "No iallocator or node given and no cluster-wide default iallocator" " found; please specify either an iallocator or a node, or set a" " cluster-wide default iallocator") def testPrimaryNodeToOldPrimaryNode(self): op = self.CopyOpCode(self.op, pnode=self.master.name) self.ExecOpCode(op) def testPrimaryNodeChange(self): node = self.cfg.AddNewNode() op = self.CopyOpCode(self.op, pnode=node.name) self.ExecOpCode(op) def testPrimaryNodeChangeRunningInstance(self): node = self.cfg.AddNewNode() op = self.CopyOpCode(self.running_op, pnode=node.name) self.ExecOpCodeExpectOpPrereqError(op, "Instance is still running") def testOsChange(self): os = self.cfg.CreateOs(supported_variants=[]) self.rpc.call_os_validate.return_value = True op = self.CopyOpCode(self.op, os_name=os.name) self.ExecOpCode(op) def testVCpuChange(self): op = self.CopyOpCode(self.op, beparams={ constants.BE_VCPUS: 4 }) self.ExecOpCode(op) def testWrongCpuMask(self): op = self.CopyOpCode(self.op, beparams={ constants.BE_VCPUS: 4 }, hvparams={ constants.HV_CPU_MASK: "1,2:3,4" }) self.ExecOpCodeExpectOpPrereqError( op, "Number of vCPUs .* does not match the CPU mask .*") def testCorrectCpuMask(self): op = self.CopyOpCode(self.op, beparams={ constants.BE_VCPUS: 4 }, hvparams={ constants.HV_CPU_MASK: "1,2:3,4:all:1,4" }) self.ExecOpCode(op) def testOsParams(self): op = self.CopyOpCode(self.op, osparams={ self.os.supported_parameters[0]: "test_param_val" }) self.ExecOpCode(op) def testIncreaseMemoryTooMuch(self): op = self.CopyOpCode(self.running_op, beparams={ constants.BE_MAXMEM: self.mocked_master_memory_free * 2 }) self.ExecOpCodeExpectOpPrereqError( op, "This change will prevent the instance from starting") def testIncreaseMemory(self): op = self.CopyOpCode(self.running_op, beparams={ constants.BE_MAXMEM: self.mocked_master_memory_free }) self.ExecOpCode(op) def testIncreaseMemoryTooMuchForSecondary(self): inst = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP, disk_template=constants.DT_DRBD8, secondary_node=self.snode) self.rpc.call_instance_info.side_effect = [ self.RpcResultsBuilder() .CreateSuccessfulNodeResult(self.master, { "memory": self.mocked_snode_memory_free * 2, "vcpus": self.mocked_running_inst_vcpus, "state": self.mocked_running_inst_state, "time": self.mocked_running_inst_time })] op = self.CopyOpCode(self.op, instance_name=inst.name, beparams={ constants.BE_MAXMEM: self.mocked_snode_memory_free * 2, constants.BE_AUTO_BALANCE: True }) self.ExecOpCodeExpectOpPrereqError( op, "This change will prevent the instance from failover to its" " secondary node") def testInvalidRuntimeMemory(self): op = self.CopyOpCode(self.running_op, runtime_mem=self.mocked_master_memory_free * 2) self.ExecOpCodeExpectOpPrereqError( op, "Instance .* must have memory between .* and .* of memory") def testIncreaseRuntimeMemory(self): op = self.CopyOpCode(self.running_op, runtime_mem=self.mocked_master_memory_free, beparams={ constants.BE_MAXMEM: self.mocked_master_memory_free }) self.ExecOpCode(op) def testAddNicWithPoolIpNoNetwork(self): op = self.CopyOpCode(self.op, nics=[(constants.DDM_ADD, -1, { constants.INIC_IP: constants.NIC_IP_POOL })]) self.ExecOpCodeExpectOpPrereqError( op, "If ip=pool, parameter network cannot be none") def testAddNicWithPoolIp(self): net = self.cfg.AddNewNetwork() self.cfg.ConnectNetworkToGroup(net, self.group) op = self.CopyOpCode(self.op, nics=[(constants.DDM_ADD, -1, { constants.INIC_IP: constants.NIC_IP_POOL, constants.INIC_NETWORK: net.name })]) self.ExecOpCode(op) def testAddNicWithInvalidIp(self): op = self.CopyOpCode(self.op, nics=[(constants.DDM_ADD, -1, { constants.INIC_IP: "invalid" })]) self.ExecOpCodeExpectOpPrereqError( op, "Invalid IP address") def testAddNic(self): op = self.CopyOpCode(self.op, nics=[(constants.DDM_ADD, -1, {})]) self.ExecOpCode(op) def testAttachNICs(self): msg = "Attach operation is not supported for NICs" op = self.CopyOpCode(self.op, nics=[(constants.DDM_ATTACH, -1, {})]) self.ExecOpCodeExpectOpPrereqError(op, msg) def testNoHotplugSupport(self): op = self.CopyOpCode(self.running_op, nics=[(constants.DDM_ADD, -1, {})]) self.rpc.call_hotplug_supported.return_value = \ self.RpcResultsBuilder() \ .CreateFailedNodeResult(self.master) self.ExecOpCode(op) self.assertFalse(op.hotplug) self.assertTrue(self.rpc.call_hotplug_supported.called) def testHotplugIfPossible(self): op = self.CopyOpCode(self.running_op, nics=[(constants.DDM_ADD, -1, {})] ) self.rpc.call_hotplug_supported.return_value = \ self.RpcResultsBuilder() \ .CreateFailedNodeResult(self.master) self.ExecOpCode(op) self.assertTrue(self.rpc.call_hotplug_supported.called) self.assertFalse(self.rpc.call_hotplug_device.called) def testHotAddNic(self): op = self.CopyOpCode(self.running_op, nics=[(constants.DDM_ADD, -1, {})], hotplug=True) self.rpc.call_hotplug_supported.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.assertTrue(self.rpc.call_hotplug_supported.called) self.assertTrue(self.rpc.call_hotplug_device.called) def testAddNicWithIp(self): op = self.CopyOpCode(self.op, nics=[(constants.DDM_ADD, -1, { constants.INIC_IP: "2.3.1.4" })]) self.ExecOpCode(op) def testModifyNicRoutedWithoutIp(self): op = self.CopyOpCode(self.op, nics=[(constants.DDM_MODIFY, 0, { constants.INIC_NETWORK: constants.VALUE_NONE, constants.INIC_MODE: constants.NIC_MODE_ROUTED })]) self.ExecOpCodeExpectOpPrereqError( op, "Cannot set the NIC IP address to None on a routed NIC" " if not attached to a network") def testModifyNicSetMac(self): op = self.CopyOpCode(self.op, nics=[(constants.DDM_MODIFY, 0, { constants.INIC_MAC: "0a:12:95:15:bf:75" })]) self.ExecOpCode(op) def testModifyNicWithPoolIpNoNetwork(self): op = self.CopyOpCode(self.op, nics=[(constants.DDM_MODIFY, -1, { constants.INIC_IP: constants.NIC_IP_POOL })]) self.ExecOpCodeExpectOpPrereqError( op, "ip=pool, but no network found") def testModifyNicSetNet(self): old_net = self.cfg.AddNewNetwork() self.cfg.ConnectNetworkToGroup(old_net, self.group) inst = self.cfg.AddNewInstance(nics=[ self.cfg.CreateNic(network=old_net, ip="198.51.100.2")]) new_net = self.cfg.AddNewNetwork(mac_prefix="be") self.cfg.ConnectNetworkToGroup(new_net, self.group) op = self.CopyOpCode(self.op, instance_name=inst.name, nics=[(constants.DDM_MODIFY, 0, { constants.INIC_NETWORK: new_net.name })]) self.ExecOpCode(op) def testModifyNicSetLinkWhileConnected(self): old_net = self.cfg.AddNewNetwork() self.cfg.ConnectNetworkToGroup(old_net, self.group) inst = self.cfg.AddNewInstance(nics=[ self.cfg.CreateNic(network=old_net)]) op = self.CopyOpCode(self.op, instance_name=inst.name, nics=[(constants.DDM_MODIFY, 0, { constants.INIC_LINK: "mock_link" })]) self.ExecOpCodeExpectOpPrereqError( op, "Not allowed to change link or mode of a NIC that is connected" " to a network") def testModifyNicSetNetAndIp(self): net = self.cfg.AddNewNetwork(mac_prefix="be", network="123.123.123.0/24") self.cfg.ConnectNetworkToGroup(net, self.group) op = self.CopyOpCode(self.op, nics=[(constants.DDM_MODIFY, 0, { constants.INIC_NETWORK: net.name, constants.INIC_IP: "123.123.123.1" })]) self.ExecOpCode(op) def testModifyNic(self): op = self.CopyOpCode(self.op, nics=[(constants.DDM_MODIFY, 0, {})]) self.ExecOpCode(op) def testHotModifyNic(self): op = self.CopyOpCode(self.running_op, nics=[(constants.DDM_MODIFY, 0, {})], hotplug=True) self.rpc.call_hotplug_supported.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.assertTrue(self.rpc.call_hotplug_supported.called) self.assertTrue(self.rpc.call_hotplug_device.called) def testRemoveLastNic(self): op = self.CopyOpCode(self.op, nics=[(constants.DDM_REMOVE, 0, {})]) self.ExecOpCodeExpectOpPrereqError( op, "violates policy") def testRemoveNic(self): inst = self.cfg.AddNewInstance(nics=[self.cfg.CreateNic(), self.cfg.CreateNic()]) op = self.CopyOpCode(self.op, instance_name=inst.name, nics=[(constants.DDM_REMOVE, 0, {})]) self.ExecOpCode(op) def testDetachNICs(self): msg = "Detach operation is not supported for NICs" op = self.CopyOpCode(self.op, nics=[(constants.DDM_DETACH, -1, {})]) self.ExecOpCodeExpectOpPrereqError(op, msg) def testHotRemoveNic(self): inst = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP, nics=[self.cfg.CreateNic(), self.cfg.CreateNic()]) op = self.CopyOpCode(self.running_op, instance_name=inst.name, nics=[(constants.DDM_REMOVE, 0, {})], hotplug=True) self.rpc.call_hotplug_supported.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.assertTrue(self.rpc.call_hotplug_supported.called) self.assertTrue(self.rpc.call_hotplug_device.called) def testSetOffline(self): op = self.CopyOpCode(self.op, offline=True) self.ExecOpCode(op) def testUnsetOffline(self): op = self.CopyOpCode(self.op, offline=False) self.ExecOpCode(op) def testAddDiskInvalidMode(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_MODE: "invalid" }]]) self.ExecOpCodeExpectOpPrereqError( op, "Invalid disk access mode 'invalid'") def testAddDiskMissingSize(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_ADD, -1, {}]]) self.ExecOpCodeExpectOpPrereqError( op, "Required disk parameter 'size' missing") def testAddDiskInvalidSize(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_SIZE: "invalid" }]]) self.ExecOpCodeExpectException( op, errors.TypeEnforcementError, "is not a valid size") def testAddDiskUnknownParam(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_ADD, -1, { "uuid": self.mocked_disk_uuid }]]) self.ExecOpCodeExpectException( op, errors.TypeEnforcementError, "Unknown parameter 'uuid'") def testAddDiskRunningInstanceNoWaitForSync(self): op = self.CopyOpCode(self.running_op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_SIZE: 1024 }]], wait_for_sync=False) self.ExecOpCode(op) self.assertFalse(self.rpc.call_blockdev_shutdown.called) def testAddDiskDownInstance(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_SIZE: 1024 }]]) self.ExecOpCode(op) self.assertTrue(self.rpc.call_blockdev_shutdown.called) def testAddDiskIndexBased(self): SPECIFIC_SIZE = 435 * 4 insertion_index = len(self.inst.disks) op = self.CopyOpCode(self.op, disks=[[constants.DDM_ADD, insertion_index, { constants.IDISK_SIZE: SPECIFIC_SIZE }]]) self.ExecOpCode(op) self.assertEqual(len(self.inst.disks), insertion_index + 1) new_disk = self.cfg.GetDisk(self.inst.disks[insertion_index]) self.assertEqual(new_disk.size, SPECIFIC_SIZE) def testAddDiskHugeIndex(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_ADD, 5, { constants.IDISK_SIZE: 1024 }]]) self.ExecOpCodeExpectException( op, IndexError, "Got disk index.*but there are only.*" ) def testAddExtDisk(self): op = self.CopyOpCode(self.ext_storage_op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_SIZE: 1024 }]]) self.ExecOpCodeExpectOpPrereqError(op, "Missing provider for template 'ext'") op = self.CopyOpCode(self.ext_storage_op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_SIZE: 1024, constants.IDISK_PROVIDER: "bla" }]]) self.ExecOpCode(op) def testAddDiskDownInstanceNoWaitForSync(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_SIZE: 1024 }]], wait_for_sync=False) self.ExecOpCodeExpectOpPrereqError( op, "Can't add a disk to an instance with deactivated disks" " and --no-wait-for-sync given") def testAddDiskRunningInstance(self): op = self.CopyOpCode(self.running_op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_SIZE: 1024 }]]) self.ExecOpCode(op) self.assertFalse(self.rpc.call_blockdev_shutdown.called) def testAddDiskNoneName(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_SIZE: 1024, constants.IDISK_NAME: constants.VALUE_NONE }]]) self.ExecOpCode(op) def testHotAddDisk(self): self.rpc.call_blockdev_assemble.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, ("/dev/mocked_path", "/var/run/ganeti/instance-disks/mocked_d", None)) op = self.CopyOpCode(self.running_op, disks=[[constants.DDM_ADD, -1, { constants.IDISK_SIZE: 1024, }]], hotplug=True) self.rpc.call_hotplug_supported.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.assertTrue(self.rpc.call_hotplug_supported.called) self.assertTrue(self.rpc.call_blockdev_create.called) self.assertTrue(self.rpc.call_blockdev_assemble.called) self.assertTrue(self.rpc.call_hotplug_device.called) def testAttachDiskWrongParams(self): msg = "Only one argument is permitted in attach op, either name or uuid" op = self.CopyOpCode(self.op, disks=[[constants.DDM_ATTACH, -1, { constants.IDISK_SIZE: 1134 }]], ) self.ExecOpCodeExpectOpPrereqError(op, msg) op = self.CopyOpCode(self.op, disks=[[constants.DDM_ATTACH, -1, { 'uuid': "1134", constants.IDISK_NAME: "1134", }]], ) self.ExecOpCodeExpectOpPrereqError(op, msg) op = self.CopyOpCode(self.op, disks=[[constants.DDM_ATTACH, -1, { 'uuid': "1134", constants.IDISK_SIZE: 1134, }]], ) self.ExecOpCodeExpectOpPrereqError(op, msg) def testAttachDiskWrongTemplate(self): msg = "Instance has '%s' template while disk has '%s' template" % \ (constants.DT_PLAIN, constants.DT_BLOCK) self.cfg.AddOrphanDisk(name=self.mocked_disk_name, dev_type=constants.DT_BLOCK) op = self.CopyOpCode(self.op, disks=[[constants.DDM_ATTACH, -1, { constants.IDISK_NAME: self.mocked_disk_name }]], ) self.ExecOpCodeExpectOpPrereqError(op, msg) def testAttachDiskWrongNodes(self): msg = "Disk nodes are \['mock_node_1134'\]" self.cfg.AddOrphanDisk(name=self.mocked_disk_name, primary_node="mock_node_1134") op = self.CopyOpCode(self.op, disks=[[constants.DDM_ATTACH, -1, { constants.IDISK_NAME: self.mocked_disk_name }]], ) self.ExecOpCodeExpectOpPrereqError(op, msg) def testAttachDiskRunningInstance(self): self.cfg.AddOrphanDisk(name=self.mocked_disk_name, primary_node=self.master.uuid) self.rpc.call_blockdev_assemble.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, ("/dev/mocked_path", "/var/run/ganeti/instance-disks/mocked_d", None)) op = self.CopyOpCode(self.running_op, disks=[[constants.DDM_ATTACH, -1, { constants.IDISK_NAME: self.mocked_disk_name }]], ) self.ExecOpCode(op) self.assertTrue(self.rpc.call_blockdev_assemble.called) self.assertFalse(self.rpc.call_blockdev_shutdown.called) def testAttachDiskRunningInstanceNoWaitForSync(self): self.cfg.AddOrphanDisk(name=self.mocked_disk_name, primary_node=self.master.uuid) self.rpc.call_blockdev_assemble.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, ("/dev/mocked_path", "/var/run/ganeti/instance-disks/mocked_d", None)) op = self.CopyOpCode(self.running_op, disks=[[constants.DDM_ATTACH, -1, { constants.IDISK_NAME: self.mocked_disk_name }]], wait_for_sync=False) self.ExecOpCode(op) self.assertTrue(self.rpc.call_blockdev_assemble.called) self.assertFalse(self.rpc.call_blockdev_shutdown.called) def testAttachDiskDownInstance(self): self.cfg.AddOrphanDisk(name=self.mocked_disk_name, primary_node=self.master.uuid) op = self.CopyOpCode(self.op, disks=[[constants.DDM_ATTACH, -1, { constants.IDISK_NAME: self.mocked_disk_name }]]) self.ExecOpCode(op) self.assertTrue(self.rpc.call_blockdev_assemble.called) self.assertTrue(self.rpc.call_blockdev_shutdown.called) def testAttachDiskDownInstanceNoWaitForSync(self): self.cfg.AddOrphanDisk(name=self.mocked_disk_name) op = self.CopyOpCode(self.op, disks=[[constants.DDM_ATTACH, -1, { constants.IDISK_NAME: self.mocked_disk_name }]], wait_for_sync=False) self.ExecOpCodeExpectOpPrereqError( op, "Can't attach a disk to an instance with deactivated disks" " and --no-wait-for-sync given.") def testHotAttachDisk(self): self.cfg.AddOrphanDisk(name=self.mocked_disk_name, primary_node=self.master.uuid) self.rpc.call_blockdev_assemble.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, ("/dev/mocked_path", "/var/run/ganeti/instance-disks/mocked_d", None)) op = self.CopyOpCode(self.running_op, disks=[[constants.DDM_ATTACH, -1, { constants.IDISK_NAME: self.mocked_disk_name }]], hotplug=True) self.rpc.call_hotplug_supported.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.assertTrue(self.rpc.call_hotplug_supported.called) self.assertTrue(self.rpc.call_blockdev_assemble.called) self.assertTrue(self.rpc.call_hotplug_device.called) def testHotRemoveDisk(self): inst = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP, disks=[self.cfg.CreateDisk(), self.cfg.CreateDisk()]) op = self.CopyOpCode(self.running_op, instance_name=inst.name, disks=[[constants.DDM_REMOVE, -1, {}]], hotplug=True) self.rpc.call_hotplug_supported.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.assertTrue(self.rpc.call_hotplug_supported.called) self.assertTrue(self.rpc.call_hotplug_device.called) self.assertTrue(self.rpc.call_blockdev_shutdown.called) self.assertTrue(self.rpc.call_blockdev_remove.called) def testHotDetachDisk(self): inst = self.cfg.AddNewInstance(admin_state=constants.ADMINST_UP, disks=[self.cfg.CreateDisk(), self.cfg.CreateDisk()]) op = self.CopyOpCode(self.running_op, instance_name=inst.name, disks=[[constants.DDM_DETACH, -1, {}]], hotplug=True) self.rpc.call_hotplug_supported.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.assertTrue(self.rpc.call_hotplug_supported.called) self.assertTrue(self.rpc.call_hotplug_device.called) self.assertTrue(self.rpc.call_blockdev_shutdown.called) def testDetachAttachFileBasedDisk(self): """Detach and re-attach a disk from a file-based instance.""" # Create our disk and calculate the path where it is stored, its name, as # well as the expected path where it will be moved. mock_disk = self.cfg.CreateDisk( name='mock_disk_1134', dev_type=constants.DT_FILE, logical_id=('loop', '/tmp/instance/disk'), primary_node=self.master.uuid) # Create a file-based instance file_disk = self.cfg.CreateDisk( dev_type=constants.DT_FILE, logical_id=('loop', '/tmp/instance/disk2')) inst = self.cfg.AddNewInstance(name='instance', disk_template=constants.DT_FILE, disks=[file_disk, mock_disk], ) # Detach the disk and assert that it has been moved to the upper directory op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_DETACH, -1, {}]], ) self.ExecOpCode(op) mock_disk = self.cfg.GetDiskInfo(mock_disk.uuid) self.assertEqual('/tmp/disk', mock_disk.logical_id[1]) # Re-attach the disk and assert that it has been moved to the original # directory op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_ATTACH, -1, { constants.IDISK_NAME: self.mocked_disk_name }]], ) self.ExecOpCode(op) mock_disk = self.cfg.GetDiskInfo(mock_disk.uuid) self.assertIn('/tmp/instance', mock_disk.logical_id[1]) def testAttachDetachDisk(self): """Check if the disks can be attached and detached in sequence. Also, check if the operations succeed both with name and uuid. """ disk1 = self.cfg.CreateDisk(uuid=self.mocked_disk_uuid, primary_node=self.master.uuid) disk2 = self.cfg.CreateDisk(name="mock_name_1134", primary_node=self.master.uuid) inst = self.cfg.AddNewInstance(disks=[disk1, disk2]) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_DETACH, self.mocked_disk_uuid, {}]]) self.ExecOpCode(op) self.assertEqual([disk2], self.cfg.GetInstanceDisks(inst.uuid)) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_ATTACH, 0, { 'uuid': self.mocked_disk_uuid }]]) self.ExecOpCode(op) self.assertEqual([disk1, disk2], self.cfg.GetInstanceDisks(inst.uuid)) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_DETACH, 1, {}]]) self.ExecOpCode(op) self.assertEqual([disk1], self.cfg.GetInstanceDisks(inst.uuid)) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_ATTACH, 0, { constants.IDISK_NAME: "mock_name_1134" }]]) self.ExecOpCode(op) self.assertEqual([disk2, disk1], self.cfg.GetInstanceDisks(inst.uuid)) def testDetachAndAttachToDisklessInstance(self): """Check if a disk can be detached and then re-attached if the instance is diskless inbetween. """ disk = self.cfg.CreateDisk(uuid=self.mocked_disk_uuid, primary_node=self.master.uuid) inst = self.cfg.AddNewInstance(disks=[disk], primary_node=self.master) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_DETACH, self.mocked_disk_uuid, {}]]) self.ExecOpCode(op) self.assertEqual([], self.cfg.GetInstanceDisks(inst.uuid)) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_ATTACH, 0, { 'uuid': self.mocked_disk_uuid }]]) self.ExecOpCode(op) self.assertEqual([disk], self.cfg.GetInstanceDisks(inst.uuid)) def testDetachAttachDrbdDisk(self): """Check if a DRBD disk can be detached and then re-attached. """ disk = self.cfg.CreateDisk(uuid=self.mocked_disk_uuid, primary_node=self.master.uuid, secondary_node=self.snode.uuid, dev_type=constants.DT_DRBD8) inst = self.cfg.AddNewInstance(disks=[disk], primary_node=self.master) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_DETACH, self.mocked_disk_uuid, {}]]) self.ExecOpCode(op) self.assertEqual([], self.cfg.GetInstanceDisks(inst.uuid)) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_ATTACH, 0, { 'uuid': self.mocked_disk_uuid }]]) self.ExecOpCode(op) self.assertEqual([disk], self.cfg.GetInstanceDisks(inst.uuid)) def testDetachAttachDrbdDiskWithWrongPrimaryNode(self): """Check if disk attachment with a wrong primary node fails. """ disk1 = self.cfg.CreateDisk(uuid=self.mocked_disk_uuid, primary_node=self.master.uuid, secondary_node=self.snode.uuid, dev_type=constants.DT_DRBD8) inst1 = self.cfg.AddNewInstance(disks=[disk1], primary_node=self.master, secondary_node=self.snode) op = self.CopyOpCode(self.op, instance_name=inst1.name, disks=[[constants.DDM_DETACH, self.mocked_disk_uuid, {}]]) self.ExecOpCode(op) self.assertEqual([], self.cfg.GetInstanceDisks(inst1.uuid)) disk2 = self.cfg.CreateDisk(uuid="mock_uuid_1135", primary_node=self.snode.uuid, secondary_node=self.master.uuid, dev_type=constants.DT_DRBD8) inst2 = self.cfg.AddNewInstance(disks=[disk2], primary_node=self.snode, secondary_node=self.master) op = self.CopyOpCode(self.op, instance_name=inst2.name, disks=[[constants.DDM_ATTACH, 0, { 'uuid': self.mocked_disk_uuid }]]) self.assertRaises(errors.OpExecError, self.ExecOpCode, op) def testDetachAttachExtDisk(self): """Check attach/detach functionality of ExtStorage disks. """ disk = self.cfg.CreateDisk(uuid=self.mocked_disk_uuid, dev_type=constants.DT_EXT, params={ constants.IDISK_PROVIDER: "pvdr" }) inst = self.cfg.AddNewInstance(disks=[disk], primary_node=self.master) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_DETACH, self.mocked_disk_uuid, {}]]) self.ExecOpCode(op) self.assertEqual([], self.cfg.GetInstanceDisks(inst.uuid)) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_ATTACH, 0, { 'uuid': self.mocked_disk_uuid }]]) self.ExecOpCode(op) self.assertEqual([disk], self.cfg.GetInstanceDisks(inst.uuid)) def testRemoveDiskRemovesStorageDir(self): inst = self.cfg.AddNewInstance(disks=[self.cfg.CreateDisk(dev_type='file')]) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_REMOVE, -1, {}]]) self.rpc.call_instance_info.side_effect = [ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) ] self.rpc.call_file_storage_dir_remove.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.rpc.call_file_storage_dir_remove.assert_called_with( self.master.uuid, '/file/storage') def testRemoveDiskKeepsStorageForRemaining(self): inst = self.cfg.AddNewInstance(disks=[self.cfg.CreateDisk(dev_type='file'), self.cfg.CreateDisk(dev_type='file')]) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_REMOVE, -1, {}]]) self.rpc.call_instance_info.side_effect = [ self.RpcResultsBuilder() .CreateSuccessfulNodeResult(self.master) ] self.rpc.call_file_storage_dir_remove.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.ExecOpCode(op) self.assertFalse(self.rpc.call_file_storage_dir_remove.called) def testRemoveUsedDiskWithoutHotplug(self): inst = self.cfg.AddNewInstance(disks=[self.cfg.CreateDisk(), self.cfg.CreateDisk()]) op = self.CopyOpCode(self.op, instance_name=inst.name, disks=[[constants.DDM_REMOVE, -1, {}]], hotplug=False) # without hotplug self.rpc.call_instance_info.side_effect = [ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, { "memory": self.mocked_snode_memory_free, "vcpus": self.mocked_running_inst_vcpus, "state": self.mocked_running_inst_state, "time": self.mocked_running_inst_time })] self.ExecOpCodeExpectOpPrereqError( op, "can't remove volume from a running instance without using hotplug") self.assertFalse(self.rpc.call_blockdev_shutdown.called) self.assertFalse(self.rpc.call_blockdev_remove.called) def testModifyDiskWithSize(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_MODIFY, 0, { constants.IDISK_SIZE: 1024 }]]) self.ExecOpCodeExpectOpPrereqError( op, "Disk size change not possible, use grow-disk") def testModifyDiskWithRandomParams(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_MODIFY, 0, { constants.IDISK_METAVG: "new_meta_vg", constants.IDISK_MODE: "invalid", constants.IDISK_NAME: "new_name" }]]) self.ExecOpCodeExpectException(op, errors.TypeEnforcementError, "Unknown parameter 'metavg'") def testModifyDiskUnsetName(self): op = self.CopyOpCode(self.op, disks=[[constants.DDM_MODIFY, 0, { constants.IDISK_NAME: constants.VALUE_NONE }]]) self.ExecOpCode(op) def testModifyExtDiskProvider(self): mod = [[constants.DDM_MODIFY, 0, { constants.IDISK_PROVIDER: "anything" }]] op = self.CopyOpCode(self.op, disks=mod) self.ExecOpCodeExpectException(op, errors.TypeEnforcementError, "Unknown parameter 'provider'") op = self.CopyOpCode(self.ext_storage_op, disks=mod) self.ExecOpCodeExpectOpPrereqError(op, "Disk 'provider' parameter change" " is not possible") def testSetOldDiskTemplate(self): op = self.CopyOpCode(self.op, disk_template=self.dev_type) self.ExecOpCodeExpectOpPrereqError( op, "Instance already has disk template") def testSetDisabledDiskTemplate(self): self.cfg.SetEnabledDiskTemplates([self.inst.disk_template]) op = self.CopyOpCode(self.op, disk_template=constants.DT_EXT) self.ExecOpCodeExpectOpPrereqError( op, "Disk template .* is not enabled for this cluster") def testConvertToExtWithMissingProvider(self): op = self.CopyOpCode(self.op, disk_template=constants.DT_EXT) self.ExecOpCodeExpectOpPrereqError( op, "Missing provider for template .*") def testConvertToNotExtWithProvider(self): op = self.CopyOpCode(self.op, disk_template=constants.DT_FILE, ext_params={constants.IDISK_PROVIDER: "pvdr"}) self.ExecOpCodeExpectOpPrereqError( op, "The 'provider' option is only valid for the ext disk" " template, not .*") def testConvertToExtWithSameProvider(self): op = self.CopyOpCode(self.ext_storage_op, disk_template=constants.DT_EXT, ext_params={constants.IDISK_PROVIDER: "pvdr"}) self.ExecOpCodeExpectOpPrereqError( op, "Not converting, 'disk/0' of type ExtStorage already using" " provider 'pvdr'") def testConvertToInvalidDiskTemplate(self): for disk_template in constants.DTS_NOT_CONVERTIBLE_TO: op = self.CopyOpCode(self.op, disk_template=disk_template) self.ExecOpCodeExpectOpPrereqError( op, "Conversion to the .* disk template is not supported") def testConvertFromInvalidDiskTemplate(self): for disk_template in constants.DTS_NOT_CONVERTIBLE_FROM: inst = self.cfg.AddNewInstance(disk_template=disk_template) op = self.CopyOpCode(self.op, instance_name=inst.name, disk_template=constants.DT_PLAIN) self.ExecOpCodeExpectOpPrereqError( op, "Conversion from the .* disk template is not supported") def testConvertToDRBDWithSecondarySameAsPrimary(self): op = self.CopyOpCode(self.op, disk_template=constants.DT_DRBD8, remote_node=self.master.name) self.ExecOpCodeExpectOpPrereqError( op, "Given new secondary node .* is the same as the primary node" " of the instance") def testConvertPlainToDRBD(self): self.rpc.call_blockdev_shutdown.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) self.rpc.call_blockdev_getmirrorstatus.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, [objects.BlockDevStatus()]) op = self.CopyOpCode(self.op, disk_template=constants.DT_DRBD8, remote_node=self.snode.name) self.ExecOpCode(op) def testConvertDRBDToPlain(self): for disk_uuid in self.inst.disks: self.cfg.RemoveInstanceDisk(self.inst.uuid, disk_uuid) disk = self.cfg.CreateDisk(dev_type=constants.DT_DRBD8, primary_node=self.master, secondary_node=self.snode) self.cfg.AddInstanceDisk(self.inst.uuid, disk) self.inst.disk_template = constants.DT_DRBD8 self.rpc.call_blockdev_shutdown.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, True) self.rpc.call_blockdev_remove.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master) self.rpc.call_blockdev_getmirrorstatus.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.master, [objects.BlockDevStatus()]) op = self.CopyOpCode(self.op, disk_template=constants.DT_PLAIN) self.ExecOpCode(op) class TestLUInstanceChangeGroup(CmdlibTestCase): def setUp(self): super(TestLUInstanceChangeGroup, self).setUp() self.group2 = self.cfg.AddNewNodeGroup() self.node2 = self.cfg.AddNewNode(group=self.group2) self.inst = self.cfg.AddNewInstance() self.op = opcodes.OpInstanceChangeGroup(instance_name=self.inst.name) def testTargetGroupIsInstanceGroup(self): op = self.CopyOpCode(self.op, target_groups=[self.group.name]) self.ExecOpCodeExpectOpPrereqError( op, "Can't use group\(s\) .* as targets, they are used by the" " instance .*") def testNoTargetGroups(self): inst = self.cfg.AddNewInstance(disk_template=constants.DT_DRBD8, primary_node=self.master, secondary_node=self.node2) op = self.CopyOpCode(self.op, instance_name=inst.name) self.ExecOpCodeExpectOpPrereqError( op, "There are no possible target groups") def testFailingIAllocator(self): self.iallocator_cls.return_value.success = False op = self.CopyOpCode(self.op) self.ExecOpCodeExpectOpPrereqError( op, "Can't compute solution for changing group of instance .*" " using iallocator .*") def testChangeGroup(self): self.iallocator_cls.return_value.success = True self.iallocator_cls.return_value.result = ([], [], []) op = self.CopyOpCode(self.op) self.ExecOpCode(op) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/node_unittest.py000064400000000000000000000307341476477700300231560ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tests for LUNode* """ from collections import defaultdict from unittest import mock from ganeti import compat from ganeti import constants from ganeti import objects from ganeti import opcodes from ganeti.cmdlib import node from testsupport import * import testutils # pylint: disable=W0613 def _TcpPingFailSecondary(cfg, mock_fct, target, port, timeout=None, live_port_needed=None, source=None): # This will return True if target is in 192.0.2.0/24 (primary range) # and False if not. return "192.0.2." in target class TestLUNodeAdd(CmdlibTestCase): def setUp(self): super(TestLUNodeAdd, self).setUp() # One node for testing readding: self.node_readd = self.cfg.AddNewNode() self.op_readd = opcodes.OpNodeAdd(node_name=self.node_readd.name, readd=True, primary_ip=self.node_readd.primary_ip, secondary_ip=self.node_readd.secondary_ip) # One node for testing adding: # don't add to configuration now! self.node_add = objects.Node(name="node_add", primary_ip="192.0.2.200", secondary_ip="203.0.113.200") self.op_add = opcodes.OpNodeAdd(node_name=self.node_add.name, primary_ip=self.node_add.primary_ip, secondary_ip=self.node_add.secondary_ip) self.netutils_mod.TcpPing.return_value = True self.mocked_dns_rpc = self.rpc_mod.DnsOnlyRunner.return_value self.mocked_dns_rpc.call_version.return_value = \ self.RpcResultsBuilder(use_node_names=True) \ .AddSuccessfulNode(self.node_add, constants.CONFIG_VERSION) \ .AddSuccessfulNode(self.node_readd, constants.CONFIG_VERSION) \ .Build() node_verify_result = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.node_add, {constants.NV_NODELIST: []}) # we can't know the node's UUID in advance, so use defaultdict here self.rpc.call_node_verify.return_value = \ defaultdict(lambda: node_verify_result, {}) self.rpc.call_node_crypto_tokens.return_value = \ self.RpcResultsBuilder() \ .CreateSuccessfulNodeResult(self.node_add, [(constants.CRYPTO_TYPE_SSL_DIGEST, "IA:MA:FA:KE:DI:GE:ST")]) def testOvsNoLink(self): ndparams = { constants.ND_OVS: True, constants.ND_OVS_NAME: "testswitch", constants.ND_OVS_LINK: None, } op = self.CopyOpCode(self.op_add, ndparams=ndparams) self.ExecOpCode(op) self.assertLogContainsRegex( "No physical interface for OpenvSwitch was given." " OpenvSwitch will not have an outside connection." " This might not be what you want") created_node = self.cfg.GetNodeInfoByName(op.node_name) self.assertEqual(ndparams[constants.ND_OVS], created_node.ndparams.get(constants.ND_OVS, None)) self.assertEqual(ndparams[constants.ND_OVS_NAME], created_node.ndparams.get(constants.ND_OVS_NAME, None)) self.assertEqual(ndparams[constants.ND_OVS_LINK], created_node.ndparams.get(constants.ND_OVS_LINK, None)) def testAddCandidateCert(self): self.ExecOpCode(self.op_add) created_node = self.cfg.GetNodeInfoByName(self.op_add.node_name) cluster = self.cfg.GetClusterInfo() self.assertTrue(created_node.uuid in cluster.candidate_certs) def testReAddCandidateCert(self): cluster = self.cfg.GetClusterInfo() self.ExecOpCode(self.op_readd) created_node = self.cfg.GetNodeInfoByName(self.op_readd.node_name) self.assertTrue(created_node.uuid in cluster.candidate_certs) def testAddNoCandidateCert(self): op = self.CopyOpCode(self.op_add, master_capable=False) self.ExecOpCode(op) created_node = self.cfg.GetNodeInfoByName(self.op_add.node_name) cluster = self.cfg.GetClusterInfo() self.assertFalse(created_node.uuid in cluster.candidate_certs) def testWithoutOVS(self): self.ExecOpCode(self.op_add) created_node = self.cfg.GetNodeInfoByName(self.op_add.node_name) self.assertEqual(None, created_node.ndparams.get(constants.ND_OVS, None)) def testWithOVS(self): ndparams = { constants.ND_OVS: True, constants.ND_OVS_LINK: "eth2", } op = self.CopyOpCode(self.op_add, ndparams=ndparams) self.ExecOpCode(op) created_node = self.cfg.GetNodeInfoByName(op.node_name) self.assertEqual(ndparams[constants.ND_OVS], created_node.ndparams.get(constants.ND_OVS, None)) self.assertEqual(ndparams[constants.ND_OVS_LINK], created_node.ndparams.get(constants.ND_OVS_LINK, None)) def testReaddingMaster(self): op = opcodes.OpNodeAdd(node_name=self.cfg.GetMasterNodeName(), readd=True) self.ExecOpCodeExpectOpPrereqError(op, "Cannot readd the master node") def testReaddNotVmCapableNode(self): self.cfg.AddNewInstance(primary_node=self.node_readd) self.netutils_mod.GetHostname.return_value = \ HostnameMock(self.node_readd.name, self.node_readd.primary_ip) op = self.CopyOpCode(self.op_readd, vm_capable=False) self.ExecOpCodeExpectOpPrereqError(op, "Node .* being re-added with" " vm_capable flag set to false, but it" " already holds instances") def testReaddAndPassNodeGroup(self): op = self.CopyOpCode(self.op_readd,group="groupname") self.ExecOpCodeExpectOpPrereqError(op, "Cannot pass a node group when a" " node is being readded") def testPrimaryIPv6(self): self.master.secondary_ip = self.master.primary_ip op = self.CopyOpCode(self.op_add, primary_ip="2001:DB8::1", secondary_ip=self.REMOVE) self.ExecOpCode(op) def testInvalidSecondaryIP(self): op = self.CopyOpCode(self.op_add, secondary_ip="333.444.555.777") self.ExecOpCodeExpectOpPrereqError(op, "Secondary IP .* needs to be a valid" " IPv4 address") def testNodeAlreadyInCluster(self): op = self.CopyOpCode(self.op_readd, readd=False) self.ExecOpCodeExpectOpPrereqError(op, "Node %s is already in the" " configuration" % self.node_readd.name) def testReaddNodeNotInConfiguration(self): op = self.CopyOpCode(self.op_add, readd=True) self.ExecOpCodeExpectOpPrereqError(op, "Node %s is not in the" " configuration" % self.node_add.name) def testPrimaryIpConflict(self): # In LUNodeAdd, DNS will resolve the node name to an IP address, that is # used to overwrite any given primary_ip value! # Thus we need to mock this DNS resolver here! self.netutils_mod.GetHostname.return_value = \ HostnameMock(self.node_add.name, self.node_readd.primary_ip) op = self.CopyOpCode(self.op_add) self.ExecOpCodeExpectOpPrereqError(op, "New node ip address.* conflict with" " existing node") def testSecondaryIpConflict(self): op = self.CopyOpCode(self.op_add, secondary_ip=self.node_readd.secondary_ip) self.ExecOpCodeExpectOpPrereqError(op, "New node ip address.* conflict with" " existing node") def testReaddWithDifferentIP(self): op = self.CopyOpCode(self.op_readd, primary_ip="192.0.2.100", secondary_ip="230.0.113.100") self.ExecOpCodeExpectOpPrereqError(op, "Readded node doesn't have the same" " IP address configuration as before") def testNodeHasSecondaryIpButNotMaster(self): self.master.secondary_ip = self.master.primary_ip self.ExecOpCodeExpectOpPrereqError(self.op_add, "The master has no" " secondary ip but the new node has one") def testMasterHasSecondaryIpButNotNode(self): op = self.CopyOpCode(self.op_add, secondary_ip=None) self.ExecOpCodeExpectOpPrereqError(op, "The master has a secondary ip but" " the new node doesn't have one") def testNodeNotReachableByPing(self): self.netutils_mod.TcpPing.return_value = False op = self.CopyOpCode(self.op_add) self.ExecOpCodeExpectOpPrereqError(op, "Node not reachable by ping") def testNodeNotReachableByPingOnSecondary(self): self.netutils_mod.GetHostname.return_value = \ HostnameMock(self.node_add.name, self.node_add.primary_ip) self.netutils_mod.TcpPing.side_effect = \ compat.partial(_TcpPingFailSecondary, self.cfg, self.netutils_mod.TcpPing) op = self.CopyOpCode(self.op_add) self.ExecOpCodeExpectOpPrereqError(op, "Node secondary ip not reachable by" " TCP based ping to node daemon port") def testCantGetVersion(self): self.mocked_dns_rpc.call_version.return_value = \ self.RpcResultsBuilder(use_node_names=True) \ .AddErrorNode(self.node_add) \ .Build() op = self.CopyOpCode(self.op_add) self.ExecOpCodeExpectOpPrereqError(op, "Can't get version information from" " node %s" % self.node_add.name) class TestLUNodeSetParams(CmdlibTestCase): def setUp(self): super(TestLUNodeSetParams, self).setUp() self.MockOut(node, 'netutils', self.netutils_mod) node.netutils.TcpPing.return_value = True self.node = self.cfg.AddNewNode( primary_ip='192.168.168.191', secondary_ip='192.168.168.192', master_candidate=True, uuid='blue_bunny') self.snode = self.cfg.AddNewNode( primary_ip='192.168.168.193', secondary_ip='192.168.168.194', master_candidate=True, uuid='pink_bunny') def testSetSecondaryIp(self): self.instance = self.cfg.AddNewInstance(primary_node=self.node, secondary_node=self.snode, disk_template='drbd') op = opcodes.OpNodeSetParams(node_name=self.node.name, secondary_ip='254.254.254.254') self.ExecOpCode(op) self.assertEqual('254.254.254.254', self.node.secondary_ip) self.assertEqual(sorted(self.wconfd.all_locks.items()), [ ('cluster/BGL', 'shared'), ('instance/mock_inst_1.example.com', 'shared'), ('node-res/blue_bunny', 'exclusive'), ('node/blue_bunny', 'exclusive')]) def testSetSecondaryIpNoLock(self): self.instance = self.cfg.AddNewInstance(primary_node=self.node, secondary_node=self.snode, disk_template='file') op = opcodes.OpNodeSetParams(node_name=self.node.name, secondary_ip='254.254.254.254') self.ExecOpCode(op) self.assertEqual('254.254.254.254', self.node.secondary_ip) self.assertEqual(sorted(self.wconfd.all_locks.items()), [ ('cluster/BGL', 'shared'), ('node-res/blue_bunny', 'exclusive'), ('node/blue_bunny', 'exclusive')]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/test_unittest.py000064400000000000000000000211211476477700300231760ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tests for LUTest*""" from unittest import mock from ganeti import constants from ganeti import opcodes from testsupport import * import testutils DELAY_DURATION = 0.01 class TestLUTestDelay(CmdlibTestCase): def testRepeatedInvocation(self): op = opcodes.OpTestDelay(duration=DELAY_DURATION, repeat=3) self.ExecOpCode(op) self.assertLogContainsMessage(" - INFO: Test delay iteration 0/2") self.mcpu.assertLogContainsEntry(constants.ELOG_MESSAGE, " - INFO: Test delay iteration 1/2") self.assertLogContainsRegex("2/2$") def testInvalidDuration(self): op = opcodes.OpTestDelay(duration=-1) self.ExecOpCodeExpectOpPrereqError(op) def testOnNodeUuid(self): node_uuids = [self.master_uuid] op = opcodes.OpTestDelay(duration=DELAY_DURATION, on_node_uuids=node_uuids) self.ExecOpCode(op) self.rpc.call_test_delay.assert_called_once_with(node_uuids, DELAY_DURATION) def testOnNodeName(self): op = opcodes.OpTestDelay(duration=DELAY_DURATION, on_nodes=[self.master.name]) self.ExecOpCode(op) self.rpc.call_test_delay.assert_called_once_with([self.master_uuid], DELAY_DURATION) def testSuccessfulRpc(self): op = opcodes.OpTestDelay(duration=DELAY_DURATION, on_nodes=[self.master.name]) self.rpc.call_test_delay.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(self.master) \ .Build() self.ExecOpCode(op) self.rpc.call_test_delay.assert_called_once() def testFailingRpc(self): op = opcodes.OpTestDelay(duration=DELAY_DURATION, on_nodes=[self.master.name]) self.rpc.call_test_delay.return_value = \ self.RpcResultsBuilder() \ .AddFailedNode(self.master) \ .Build() self.ExecOpCodeExpectOpExecError(op) def testMultipleNodes(self): node1 = self.cfg.AddNewNode() node2 = self.cfg.AddNewNode() op = opcodes.OpTestDelay(duration=DELAY_DURATION, on_nodes=[node1.name, node2.name], on_master=False) self.rpc.call_test_delay.return_value = \ self.RpcResultsBuilder() \ .AddSuccessfulNode(node1) \ .AddSuccessfulNode(node2) \ .Build() self.ExecOpCode(op) self.rpc.call_test_delay.assert_called_once_with([node1.uuid, node2.uuid], DELAY_DURATION) class TestLUTestAllocator(CmdlibTestCase): def setUp(self): super(TestLUTestAllocator, self).setUp() self.base_op = opcodes.OpTestAllocator( name="new-instance.example.com", nics=[], disks=[], disk_template=constants.DT_DISKLESS, direction=constants.IALLOCATOR_DIR_OUT, iallocator="test") self.valid_alloc_op = \ self.CopyOpCode(self.base_op, mode=constants.IALLOCATOR_MODE_ALLOC, memory=0, disk_template=constants.DT_DISKLESS, os="mock_os", group_name="default", vcpus=1) self.valid_multi_alloc_op = \ self.CopyOpCode(self.base_op, mode=constants.IALLOCATOR_MODE_MULTI_ALLOC, instances=["new-instance.example.com"], memory=0, disk_template=constants.DT_DISKLESS, os="mock_os", group_name="default", vcpus=1) self.valid_reloc_op = \ self.CopyOpCode(self.base_op, mode=constants.IALLOCATOR_MODE_RELOC) self.valid_chg_group_op = \ self.CopyOpCode(self.base_op, mode=constants.IALLOCATOR_MODE_CHG_GROUP, instances=["new-instance.example.com"], target_groups=["default"]) self.valid_node_evac_op = \ self.CopyOpCode(self.base_op, mode=constants.IALLOCATOR_MODE_NODE_EVAC, instances=["new-instance.example.com"], evac_mode=constants.NODE_EVAC_PRI) self.iallocator_cls.return_value.in_text = "mock in text" self.iallocator_cls.return_value.out_text = "mock out text" def testMissingDirection(self): op = self.CopyOpCode(self.base_op, direction=self.REMOVE) self.ExecOpCodeExpectOpPrereqError( op, "'OP_TEST_ALLOCATOR.direction' fails validation") def testAllocWrongDisks(self): op = self.CopyOpCode(self.valid_alloc_op, disks=[0, "test"]) self.ExecOpCodeExpectOpPrereqError(op, "Invalid contents") def testAllocWithExistingInstance(self): inst = self.cfg.AddNewInstance() op = self.CopyOpCode(self.valid_alloc_op, name=inst.name) self.ExecOpCodeExpectOpPrereqError(op, "already in the cluster") def testAllocMultiAllocMissingIAllocator(self): for mode in [constants.IALLOCATOR_MODE_ALLOC, constants.IALLOCATOR_MODE_MULTI_ALLOC]: op = self.CopyOpCode(self.base_op, mode=mode, iallocator=None) self.ResetMocks() self.ExecOpCodeExpectOpPrereqError(op, "Missing allocator name") def testChgGroupNodeEvacMissingInstances(self): for mode in [constants.IALLOCATOR_MODE_CHG_GROUP, constants.IALLOCATOR_MODE_NODE_EVAC]: op = self.CopyOpCode(self.base_op, mode=mode) self.ResetMocks() self.ExecOpCodeExpectOpPrereqError(op, "Missing instances") def testAlloc(self): op = self.valid_alloc_op self.ExecOpCode(op) assert self.iallocator_cls.call_count == 1 self.iallocator_cls.return_value.Run \ .assert_called_once_with("test", validate=False) def testReloc(self): op = self.valid_reloc_op self.cfg.AddNewInstance(name=op.name) self.ExecOpCode(op) assert self.iallocator_cls.call_count == 1 self.iallocator_cls.return_value.Run \ .assert_called_once_with("test", validate=False) def testChgGroup(self): op = self.valid_chg_group_op for inst_name in op.instances: self.cfg.AddNewInstance(name=inst_name) self.ExecOpCode(op) assert self.iallocator_cls.call_count == 1 self.iallocator_cls.return_value.Run \ .assert_called_once_with("test", validate=False) def testNodeEvac(self): op = self.valid_node_evac_op for inst_name in op.instances: self.cfg.AddNewInstance(name=inst_name) self.ExecOpCode(op) assert self.iallocator_cls.call_count == 1 self.iallocator_cls.return_value.Run \ .assert_called_once_with("test", validate=False) def testMultiAlloc(self): op = self.valid_multi_alloc_op self.ExecOpCode(op) assert self.iallocator_cls.call_count == 1 self.iallocator_cls.return_value.Run \ .assert_called_once_with("test", validate=False) def testAllocDirectionIn(self): op = self.CopyOpCode(self.valid_alloc_op, direction=constants.IALLOCATOR_DIR_IN) self.ExecOpCode(op) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/000075500000000000000000000000001476477700300223255ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/__init__.py000064400000000000000000000047051476477700300244440ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support classes and functions for testing the cmdlib module. """ from cmdlib.testsupport.cmdlib_testcase import CmdlibTestCase, \ withLockedLU from testutils.config_mock import ConfigMock from cmdlib.testsupport.iallocator_mock import patchIAllocator from cmdlib.testsupport.livelock_mock import LiveLockMock from cmdlib.testsupport.utils_mock import patchUtils from cmdlib.testsupport.netutils_mock import patchNetutils, HostnameMock from cmdlib.testsupport.processor_mock import ProcessorMock from cmdlib.testsupport.pathutils_mock import patchPathutils from cmdlib.testsupport.rpc_runner_mock import CreateRpcRunnerMock, \ RpcResultsBuilder from cmdlib.testsupport.ssh_mock import patchSsh from cmdlib.testsupport.wconfd_mock import WConfdMock __all__ = ["CmdlibTestCase", "withLockedLU", "ConfigMock", "CreateRpcRunnerMock", "HostnameMock", "patchIAllocator", "patchUtils", "patchNetutils", "patchSsh", "patchPathutils", "ProcessorMock", "RpcResultsBuilder", "LiveLockMock", "WConfdMock", ] ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/cmdlib_testcase.py000064400000000000000000000346741476477700300260420ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Main module of the cmdlib test framework""" import inspect import re import traceback import functools import sys from unittest import mock from testutils.config_mock import ConfigMock, ConfigObjectMatcher from cmdlib.testsupport.iallocator_mock import patchIAllocator from cmdlib.testsupport.livelock_mock import LiveLockMock from cmdlib.testsupport.netutils_mock import patchNetutils, \ SetupDefaultNetutilsMock from cmdlib.testsupport.processor_mock import ProcessorMock from cmdlib.testsupport.rpc_runner_mock import CreateRpcRunnerMock, \ RpcResultsBuilder, patchRpc, SetupDefaultRpcModuleMock from cmdlib.testsupport.ssh_mock import patchSsh from cmdlib.testsupport.wconfd_mock import WConfdMock from ganeti.cmdlib.base import LogicalUnit from ganeti import errors from ganeti import objects from ganeti import opcodes from ganeti import runtime import testutils class GanetiContextMock(object): # pylint: disable=W0212 cfg = property(fget=lambda self: self._test_case.cfg) # pylint: disable=W0212 rpc = property(fget=lambda self: self._test_case.rpc) def __init__(self, test_case): self._test_case = test_case self.livelock = LiveLockMock() def GetWConfdContext(self, _ec_id): return (None, None, None) def GetConfig(self, _ec_id): return self._test_case.cfg def GetRpc(self, _cfg): return self._test_case.rpc def AddNode(self, cfg, node, ec_id): cfg.AddNode(node, ec_id) def RemoveNode(self, cfg, node): cfg.RemoveNode(node.uuid) class MockLU(LogicalUnit): def BuildHooksNodes(self): pass def BuildHooksEnv(self): pass # pylint: disable=R0904 class CmdlibTestCase(testutils.GanetiTestCase): """Base class for cmdlib tests. This class sets up a mocked environment for the execution of L{ganeti.cmdlib.base.LogicalUnit} subclasses. The environment can be customized via the following fields: * C{cfg}: @see L{ConfigMock} * C{rpc}: @see L{CreateRpcRunnerMock} * C{iallocator_cls}: @see L{patchIAllocator} * C{mcpu}: @see L{ProcessorMock} * C{netutils_mod}: @see L{patchNetutils} * C{ssh_mod}: @see L{patchSsh} """ REMOVE = object() cluster = property(fget=lambda self: self.cfg.GetClusterInfo(), doc="Cluster configuration object") master = property(fget=lambda self: self.cfg.GetMasterNodeInfo(), doc="Master node") master_uuid = property(fget=lambda self: self.cfg.GetMasterNode(), doc="Master node UUID") # pylint: disable=W0212 group = property(fget=lambda self: self._GetDefaultGroup(), doc="Default node group") os = property(fget=lambda self: self.cfg.GetDefaultOs(), doc="Default OS") os_name_variant = property( fget=lambda self: self.os.name + objects.OS.VARIANT_DELIM + self.os.supported_variants[0], doc="OS name and variant string") def setUp(self): super(CmdlibTestCase, self).setUp() self._iallocator_patcher = None self._netutils_patcher = None self._ssh_patcher = None self._rpc_patcher = None try: runtime.InitArchInfo() except errors.ProgrammerError: # during tests, the arch info can be initialized multiple times pass self.ResetMocks() self._cleanups = [] def _StopPatchers(self): if self._iallocator_patcher is not None: self._iallocator_patcher.stop() self._iallocator_patcher = None if self._netutils_patcher is not None: self._netutils_patcher.stop() self._netutils_patcher = None if self._ssh_patcher is not None: self._ssh_patcher.stop() self._ssh_patcher = None if self._rpc_patcher is not None: self._rpc_patcher.stop() self._rpc_patcher = None def tearDown(self): super(CmdlibTestCase, self).tearDown() self._StopPatchers() while self._cleanups: f, args, kwargs = self._cleanups.pop(-1) try: f(*args, **kwargs) except BaseException as e: sys.stderr.write('Error in cleanup: %s\n' % e) def _GetTestModule(self): module = inspect.getsourcefile(self.__class__).split("/")[-1] suffix = "_unittest.py" assert module.endswith(suffix), "Naming convention for cmdlib test" \ " modules is: %s (found '%s')"\ % (suffix, module) return module[:-len(suffix)] def ResetMocks(self): """Resets all mocks back to their initial state. This is useful if you want to execute more than one opcode in a single test. """ self.cfg = ConfigMock() self.rpc = CreateRpcRunnerMock() self.ctx = GanetiContextMock(self) self.wconfd = WConfdMock() self.mcpu = ProcessorMock(self.ctx, self.wconfd) self._StopPatchers() try: self._iallocator_patcher = patchIAllocator(self._GetTestModule()) self.iallocator_cls = self._iallocator_patcher.start() except (ImportError, AttributeError): # this test module does not use iallocator, no patching performed self._iallocator_patcher = None try: self._netutils_patcher = patchNetutils(self._GetTestModule()) self.netutils_mod = self._netutils_patcher.start() SetupDefaultNetutilsMock(self.netutils_mod, self.cfg) except (ImportError, AttributeError): # this test module does not use netutils, no patching performed self._netutils_patcher = None try: self._ssh_patcher = patchSsh(self._GetTestModule()) self.ssh_mod = self._ssh_patcher.start() except (ImportError, AttributeError): # this test module does not use ssh, no patching performed self._ssh_patcher = None try: self._rpc_patcher = patchRpc(self._GetTestModule()) self.rpc_mod = self._rpc_patcher.start() SetupDefaultRpcModuleMock(self.rpc_mod) except (ImportError, AttributeError): # this test module does not use rpc, no patching performed self._rpc_patcher = None def GetMockLU(self): """Creates a mock L{LogialUnit} with access to the mocked config etc. @rtype: L{LogialUnit} @return: A mock LU """ return MockLU(self.mcpu, mock.MagicMock(), self.cfg, self.rpc, (1234, "/tmp/mock/livelock"), self.wconfd) def RpcResultsBuilder(self, use_node_names=False): """Creates a pre-configured L{RpcResultBuilder} @type use_node_names: bool @param use_node_names: @see L{RpcResultBuilder} @rtype: L{RpcResultBuilder} @return: a pre-configured builder for RPC results """ return RpcResultsBuilder(cfg=self.cfg, use_node_names=use_node_names) def ExecOpCode(self, opcode): """Executes the given opcode. @param opcode: the opcode to execute @return: the result of the LU's C{Exec} method """ return self.mcpu.ExecOpCodeAndRecordOutput(opcode) def ExecOpCodeExpectException(self, opcode, expected_exception, expected_regex=None): """Executes the given opcode and expects an exception. @param opcode: @see L{ExecOpCode} @type expected_exception: class @param expected_exception: the exception which must be raised @type expected_regex: string @param expected_regex: if not C{None}, a regular expression which must be present in the string representation of the exception """ try: self.ExecOpCode(opcode) except expected_exception as e: if expected_regex is not None: assert re.search(expected_regex, str(e)) is not None, \ "Caught exception '%s' did not match '%s'" % \ (str(e), expected_regex) except Exception as e: tb = traceback.format_exc() raise AssertionError("%s\n(See original exception above)\n" "Expected exception '%s' was not raised," " got '%s' of class '%s' instead." % (tb, expected_exception, e, e.__class__)) else: raise AssertionError("Expected exception '%s' was not raised" % expected_exception) def ExecOpCodeExpectOpPrereqError(self, opcode, expected_regex=None): """Executes the given opcode and expects a L{errors.OpPrereqError} @see L{ExecOpCodeExpectException} """ self.ExecOpCodeExpectException(opcode, errors.OpPrereqError, expected_regex) def ExecOpCodeExpectOpExecError(self, opcode, expected_regex=None): """Executes the given opcode and expects a L{errors.OpExecError} @see L{ExecOpCodeExpectException} """ self.ExecOpCodeExpectException(opcode, errors.OpExecError, expected_regex) def RunWithLockedLU(self, opcode, test_func): """Takes the given opcode, creates a LU and runs func on it. The passed LU did already perform locking, but no methods which actually require locking are executed on the LU. @param opcode: the opcode to get the LU for. @param test_func: the function to execute with the LU as parameter. @return: the result of test_func """ return self.mcpu.RunWithLockedLU(opcode, test_func) def assertLogContainsMessage(self, expected_msg): """Shortcut for L{ProcessorMock.assertLogContainsMessage} """ self.mcpu.assertLogContainsMessage(expected_msg) def assertLogContainsRegex(self, expected_regex): """Shortcut for L{ProcessorMock.assertLogContainsRegex} """ self.mcpu.assertLogContainsRegex(expected_regex) def assertHooksCall(self, nodes, hook_path, phase, environment=None, count=None, index=0): """Asserts a call to C{rpc.call_hooks_runner} @type nodes: list of string @param nodes: node UUID's or names hooks run on @type hook_path: string @param hook_path: path (or name) of the hook run @type phase: string @param phase: phase in which the hook runs in @type environment: dict @param environment: the environment passed to the hooks. C{None} to skip asserting it @type count: int @param count: the number of hook invocations. C{None} to skip asserting it @type index: int @param index: the index of the hook invocation to assert """ if count is not None: self.assertEqual(count, self.rpc.call_hooks_runner.call_count) args = self.rpc.call_hooks_runner.call_args[index] self.assertEqual(set(nodes), set(args[0])) self.assertEqual(hook_path, args[1]) self.assertEqual(phase, args[2]) if environment is not None: self.assertEqual(environment, args[3]) def assertSingleHooksCall(self, nodes, hook_path, phase, environment=None): """Asserts a single call to C{rpc.call_hooks_runner} @see L{assertHooksCall} for parameter description. """ self.assertHooksCall(nodes, hook_path, phase, environment=environment, count=1) def CopyOpCode(self, opcode, **kwargs): """Creates a copy of the given opcode and applies modifications to it @type opcode: opcode.OpCode @param opcode: the opcode to copy @type kwargs: dict @param kwargs: dictionary of fields to overwrite in the copy. The special value L{REMOVE} can be used to remove fields from the copy. @return: a copy of the given opcode """ state = opcode.__getstate__() for key, value in kwargs.items(): if value == self.REMOVE and key in state: del state[key] else: state[key] = value return opcodes.OpCode.LoadOpCode(state) def _GetDefaultGroup(self): for group in self.cfg.GetAllNodeGroupsInfo().values(): if group.name == "default": return group assert False def _MatchMasterParams(self): return ConfigObjectMatcher(self.cfg.GetMasterNetworkParameters()) def MockOut(self, *args, **kwargs): """Immediately start mock.patch.object.""" patcher = mock.patch.object(*args, **kwargs) mocked = patcher.start() self.AddCleanup(patcher.stop) return mocked # Simplified backport of 2.7 feature def AddCleanup(self, func, *args, **kwargs): self._cleanups.append((func, args, kwargs)) def assertIn(self, first, second, msg=None): if first not in second: if msg is None: msg = "%r not found in %r" % (first, second) self.fail(msg) # pylint: disable=C0103 def withLockedLU(func): """Convenience decorator which runs the decorated method with the LU. This uses L{CmdlibTestCase.RunWithLockedLU} to run the decorated method. For this to work, the opcode to run has to be an instance field named "op", "_op", "opcode" or "_opcode". If the instance has a method called "PrepareLU", this method is invoked with the LU right before the test method is called. """ @functools.wraps(func) def wrapper(*args, **kwargs): test = args[0] assert isinstance(test, CmdlibTestCase) op = None for attr_name in ["op", "_op", "opcode", "_opcode"]: if hasattr(test, attr_name): op = getattr(test, attr_name) break assert op is not None prepare_fn = None if hasattr(test, "PrepareLU"): prepare_fn = getattr(test, "PrepareLU") assert callable(prepare_fn) def callWithLU(lu): if prepare_fn: prepare_fn(lu) new_args = list(args) new_args.append(lu) func(*new_args, **kwargs) return test.RunWithLockedLU(op, callWithLU) return wrapper ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/iallocator_mock.py000064400000000000000000000035261476477700300260470ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the IAllocator interface""" from cmdlib.testsupport.util import patchModule # pylint: disable=C0103 def patchIAllocator(module_under_test): """Patches the L{ganeti.masterd.iallocator.IAllocator} class for tests. This function is meant to be used as a decorator for test methods. @type module_under_test: string @param module_under_test: the module within cmdlib which is tested. The "ganeti.cmdlib" prefix is optional. """ return patchModule(module_under_test, "iallocator.IAllocator") ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/livelock_mock.py000064400000000000000000000034621476477700300255250ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the live locks""" import os class Mockfile(object): """Mock an opaque file. Only the name field is provided. """ def __init__(self, fname): self.name = fname class LiveLockMock(object): """Lock version of a live lock. Does not actually touch the file system. """ def __init__(self, name=None): if name is None: name = "pid4711" name = "%s_123456" % name fname = os.path.join("/tmp/mock", name) self.lockfile = Mockfile(fname) ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/netutils_mock.py000064400000000000000000000102471476477700300255630ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the netutils module""" from unittest import mock from ganeti import compat from ganeti import netutils from cmdlib.testsupport.util import patchModule # pylint: disable=C0103 def patchNetutils(module_under_test): """Patches the L{ganeti.netutils} module for tests. This function is meant to be used as a decorator for test methods. @type module_under_test: string @param module_under_test: the module within cmdlib which is tested. The "ganeti.cmdlib" prefix is optional. """ return patchModule(module_under_test, "netutils") class HostnameMock(object): """Simple mocked version of L{netutils.Hostname}. """ def __init__(self, name, ip): self.name = name self.ip = ip def _IsOverwrittenReturnValue(value): return value is not None and value != mock.DEFAULT and \ not isinstance(value, mock.Mock) # pylint: disable=W0613 def _GetHostnameMock(cfg, mock_fct, name=None, family=None): if _IsOverwrittenReturnValue(mock_fct.return_value): return mock.DEFAULT if name is None: name = cfg.GetMasterNodeName() if name == cfg.GetClusterName(): cluster = cfg.GetClusterInfo() return HostnameMock(cluster.cluster_name, cluster.master_ip) node = cfg.GetNodeInfoByName(name) if node is not None: return HostnameMock(node.name, node.primary_ip) return HostnameMock(name, "203.0.113.253") # pylint: disable=W0613 def _TcpPingMock(cfg, mock_fct, target, port, timeout=None, live_port_needed=None, source=None): if _IsOverwrittenReturnValue(mock_fct.return_value): return mock.DEFAULT if target == cfg.GetClusterName(): return True if cfg.GetNodeInfoByName(target) is not None: return True if target in [node.primary_ip for node in cfg.GetAllNodesInfo().values()]: return True if target in [node.secondary_ip for node in cfg.GetAllNodesInfo().values()]: return True return False def SetupDefaultNetutilsMock(netutils_mod, cfg): """Configures the given netutils_mod mock to work with the given config. All relevant functions in netutils_mod are stubbed in such a way that they are consistent with the configuration. @param netutils_mod: the mock module to configure @type cfg: cmdlib.testsupport.ConfigMock @param cfg: the configuration to query for netutils request """ netutils_mod.GetHostname.side_effect = \ compat.partial(_GetHostnameMock, cfg, netutils_mod.GetHostname) netutils_mod.TcpPing.side_effect = \ compat.partial(_TcpPingMock, cfg, netutils_mod.TcpPing) netutils_mod.GetDaemonPort.side_effect = netutils.GetDaemonPort netutils_mod.FormatAddress.side_effect = netutils.FormatAddress netutils_mod.Hostname.GetNormalizedName.side_effect = \ netutils.Hostname.GetNormalizedName netutils_mod.IPAddress = netutils.IPAddress netutils_mod.IP4Address = netutils.IP4Address netutils_mod.IP6Address = netutils.IP6Address ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/pathutils_mock.py000064400000000000000000000034541476477700300257330ustar00rootroot00000000000000# # # Copyright (C) 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the ssh module""" from cmdlib.testsupport.util import patchModule # pylint: disable=C0103 def patchPathutils(module_under_test): """Patches the L{ganeti.pathutils} module for tests. This function is meant to be used as a decorator for test methods. @type module_under_test: string @param module_under_test: the module within cmdlib which is tested. The "ganeti.cmdlib" prefix is optional. """ return patchModule(module_under_test, "pathutils") ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/processor_mock.py000064400000000000000000000170741476477700300257400ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the opcode processor""" import re from ganeti import constants from ganeti import mcpu class LogRecordingCallback(mcpu.OpExecCbBase): """Helper class for log output recording. """ def __init__(self, processor): super(LogRecordingCallback, self).__init__() self.processor = processor # TODO: Cleanup calling conventions, make them explicit def Feedback(self, *args): assert len(args) < 3 if len(args) == 1: log_type = constants.ELOG_MESSAGE log_msg = args[0] else: (log_type, log_msg) = args # TODO: Remove this once calling conventions are explicit. # Feedback can be called with anything, we interpret ELogMessageList as # messages that have to be individually added to the log list, but pushed # in a single update. Other types are only transparently passed forward. if log_type == constants.ELOG_MESSAGE_LIST: log_msg_list = [(constants.ELOG_MESSAGE, msg) for msg in log_msg] else: log_msg_list = [(log_type, log_msg)] self.processor.log_entries.extend(log_msg_list) def SubmitManyJobs(self, jobs): results = [] for idx, _ in enumerate(jobs): results.append((True, idx)) return results class ProcessorMock(mcpu.Processor): """Mocked opcode processor for tests. This class actually performs much more than a mock, as it drives the execution of LU's. But it also provides access to the log output of the LU the result of the execution. See L{ExecOpCodeAndRecordOutput} for the main method of this class. """ def __init__(self, context, wconfd): super(ProcessorMock, self).__init__(context, 1, True) self.log_entries = [] self._lu_test_func = None self.wconfd = wconfd def ExecOpCodeAndRecordOutput(self, op): """Executes the given opcode and records the output for further inspection. @param op: the opcode to execute. @return: see L{mcpu.Processor.ExecOpCode} """ return self.ExecOpCode(op, LogRecordingCallback(self)) def _ExecLU(self, lu): # pylint: disable=W0212 if not self._lu_test_func: return super(ProcessorMock, self)._ExecLU(lu) else: # required by a lot LU's, and usually passed in Exec lu._feedback_fn = self.Log return self._lu_test_func(lu) def _CheckLUResult(self, op, result): # pylint: disable=W0212 if not self._lu_test_func: return super(ProcessorMock, self)._CheckLUResult(op, result) else: pass def RunWithLockedLU(self, op, func): """Takes the given opcode, creates a LU and runs func with it. @param op: the opcode to get the LU for. @param func: the function to run with the created and locked LU. @return: the result of func. """ self._lu_test_func = func try: return self.ExecOpCodeAndRecordOutput(op) finally: self._lu_test_func = None def GetLogEntries(self): """Return the list of recorded log entries. @rtype: list of (string, string) tuples @return: the list of recorded log entries """ return self.log_entries def GetLogMessages(self): """Return the list of recorded log messages. @rtype: list of string @return: the list of recorded log messages """ return [msg for _, msg in self.log_entries] def GetLogEntriesString(self): """Return a string with all log entries separated by a newline. """ return "\n".join("%s: %s" % (log_type, msg) for log_type, msg in self.GetLogEntries()) def GetLogMessagesString(self): """Return a string with all log messages separated by a newline. """ return "\n".join("%s" % msg for _, msg in self.GetLogEntries()) def assertLogContainsEntry(self, expected_type, expected_msg): """Asserts that the log contains the exact given entry. @type expected_type: string @param expected_type: the expected type @type expected_msg: string @param expected_msg: the expected message """ for log_type, msg in self.log_entries: if log_type == expected_type and msg == expected_msg: return raise AssertionError( "Could not find '%s' (type '%s') in LU log messages. Log is:\n%s" % (expected_msg, expected_type, self.GetLogEntriesString())) def assertLogContainsMessage(self, expected_msg): """Asserts that the log contains the exact given message. @type expected_msg: string @param expected_msg: the expected message """ for msg in self.GetLogMessages(): if msg == expected_msg: return raise AssertionError( "Could not find '%s' in LU log messages. Log is:\n%s" % (expected_msg, self.GetLogMessagesString())) def assertLogContainsRegex(self, expected_regex): """Asserts that the log contains a message which matches the regex. @type expected_regex: string @param expected_regex: regular expression to match messages with. """ for msg in self.GetLogMessages(): if re.search(expected_regex, msg) is not None: return raise AssertionError( "Could not find '%s' in LU log messages. Log is:\n%s" % (expected_regex, self.GetLogMessagesString()) ) def assertLogContainsInLine(self, expected): """Asserts that the log contains a message which contains a string. @type expected: string @param expected: string to search in messages. """ self.assertLogContainsRegex(re.escape(expected)) def assertLogDoesNotContainRegex(self, expected_regex): """Asserts that the log does not contain a message which matches the regex. @type expected_regex: string @param expected_regex: regular expression to match messages with. """ for msg in self.GetLogMessages(): if re.search(expected_regex, msg) is not None: raise AssertionError( "Found '%s' in LU log messages. Log is:\n%s" % (expected_regex, self.GetLogMessagesString()) ) def assertLogIsEmpty(self): """Asserts that the log does not contain any message. """ if len(self.GetLogMessages()) > 0: raise AssertionError("Log is not empty. Log is:\n%s" % self.GetLogMessagesString()) def ClearLogMessages(self): """Clears all recorded log messages. This is useful if you use L{GetLockedLU} and want to test multiple calls on it. """ self.log_entries = [] ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/rpc_runner_mock.py000064400000000000000000000146331476477700300260740ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the RPC runner""" from unittest import mock from ganeti import objects from ganeti.rpc import node as rpc from cmdlib.testsupport.util import patchModule def CreateRpcRunnerMock(): """Creates a new L{mock.MagicMock} tailored for L{rpc.RpcRunner} """ ret = mock.MagicMock(spec=rpc.RpcRunner) return ret class RpcResultsBuilder(object): """Helper class which assists in constructing L{rpc.RpcResult} objects. This class provides some convenience methods for constructing L{rpc.RpcResult} objects. It is possible to create single results with the C{Create*} methods or to create multi-node results by repeatedly calling the C{Add*} methods and then obtaining the final result with C{Build}. The C{node} parameter of all the methods can either be a L{objects.Node} object, a node UUID or a node name. You have to provide the cluster config in the constructor if you want to use node UUID's/names. A typical usage of this class is as follows:: self.rpc.call_some_rpc.return_value = \ RpcResultsBuilder(cfg=self.cfg) \ .AddSuccessfulNode(node1, { "result_key": "result_data", "another_key": "other_data", }) \ .AddErrorNode(node2) \ .Build() """ def __init__(self, cfg=None, use_node_names=False): """Constructor. @type cfg: L{ganeti.config.ConfigWriter} @param cfg: used to resolve nodes if not C{None} @type use_node_names: bool @param use_node_names: if set to C{True}, the node field in the RPC results will contain the node name instead of the node UUID. """ self._cfg = cfg self._use_node_names = use_node_names self._results = [] def _GetNode(self, node_id): if isinstance(node_id, objects.Node): return node_id node = None if self._cfg is not None: node = self._cfg.GetNodeInfo(node_id) if node is None: node = self._cfg.GetNodeInfoByName(node_id) assert node is not None, "Failed to find '%s' in configuration" % node_id return node def _GetNodeId(self, node_id): node = self._GetNode(node_id) if self._use_node_names: return node.name else: return node.uuid def CreateSuccessfulNodeResult(self, node, data=None): """@see L{RpcResultsBuilder} @param node: @see L{RpcResultsBuilder}. @type data: dict @param data: the data as returned by the RPC @rtype: L{rpc.RpcResult} """ if data is None: data = {} return rpc.RpcResult(data=(True, data), node=self._GetNodeId(node)) def CreateFailedNodeResult(self, node): """@see L{RpcResultsBuilder} @param node: @see L{RpcResultsBuilder}. @rtype: L{rpc.RpcResult} """ return rpc.RpcResult(failed=True, node=self._GetNodeId(node)) def CreateOfflineNodeResult(self, node): """@see L{RpcResultsBuilder} @param node: @see L{RpcResultsBuilder}. @rtype: L{rpc.RpcResult} """ return rpc.RpcResult(failed=True, offline=True, node=self._GetNodeId(node)) def CreateErrorNodeResult(self, node, error_msg=None): """@see L{RpcResultsBuilder} @param node: @see L{RpcResultsBuilder}. @type error_msg: string @param error_msg: the error message as returned by the RPC @rtype: L{rpc.RpcResult} """ return rpc.RpcResult(data=(False, error_msg), node=self._GetNodeId(node)) def AddSuccessfulNode(self, node, data=None): """@see L{CreateSuccessfulNode} @rtype: L{RpcResultsBuilder} @return: self for chaining """ self._results.append(self.CreateSuccessfulNodeResult(node, data)) return self def AddFailedNode(self, node): """@see L{CreateFailedNode} @rtype: L{RpcResultsBuilder} @return: self for chaining """ self._results.append(self.CreateFailedNodeResult(node)) return self def AddOfflineNode(self, node): """@see L{CreateOfflineNode} @rtype: L{RpcResultsBuilder} @return: self for chaining """ self._results.append(self.CreateOfflineNodeResult(node)) return self def AddErrorNode(self, node, error_msg=None): """@see L{CreateErrorNode} @rtype: L{RpcResultsBuilder} @return: self for chaining """ self._results.append(self.CreateErrorNodeResult(node, error_msg=error_msg)) return self def Build(self): """Creates a dictionary holding multi-node results @rtype: dict """ return dict((result.node, result) for result in self._results) # pylint: disable=C0103 def patchRpc(module_under_test): """Patches the L{ganeti.rpc} module for tests. This function is meant to be used as a decorator for test methods. @type module_under_test: string @param module_under_test: the module within cmdlib which is tested. The "ganeti.cmdlib" prefix is optional. """ return patchModule(module_under_test, "rpc", wraps=rpc) def SetupDefaultRpcModuleMock(rpc_mod): """Configures the given rpc_mod. All relevant functions in rpc_mod are stubbed in a sensible way. @param rpc_mod: the mock module to configure """ rpc_mod.DnsOnlyRunner.return_value = CreateRpcRunnerMock() ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/ssh_mock.py000064400000000000000000000034321476477700300245070ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the ssh module""" from cmdlib.testsupport.util import patchModule # pylint: disable=C0103 def patchSsh(module_under_test): """Patches the L{ganeti.ssh} module for tests. This function is meant to be used as a decorator for test methods. @type module_under_test: string @param module_under_test: the module within cmdlib which is tested. The "ganeti.cmdlib" prefix is optional. """ return patchModule(module_under_test, "ssh") ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/util.py000064400000000000000000000037331476477700300236620ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utility functions or the cmdlib test framework""" from unittest import mock # pylint: disable=C0103 def patchModule(module_under_test, mock_module, **kwargs): """Computes the module prefix required to mock parts of the Ganeti code. @type module_under_test: string @param module_under_test: the module within cmdlib which is tested. The "ganeti.cmdlib" prefix is optional. @type mock_module @param mock_module: the module which should be mocked. """ if not module_under_test.startswith("ganeti.cmdlib"): module_under_test = "ganeti.cmdlib." + module_under_test return mock.patch("%s.%s" % (module_under_test, mock_module), **kwargs) ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/utils_mock.py000064400000000000000000000034421476477700300250530ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the utils module""" from cmdlib.testsupport.util import patchModule # pylint: disable=C0103 def patchUtils(module_under_test): """Patches the L{ganeti.utils} module for tests. This function is meant to be used as a decorator for test methods. @type module_under_test: string @param module_under_test: the module within cmdlib which is tested. The "ganeti.cmdlib" prefix is optional. """ return patchModule(module_under_test, "utils") ganeti-3.1.0~rc2/test/py/legacy/cmdlib/testsupport/wconfd_mock.py000064400000000000000000000063041476477700300251730ustar00rootroot00000000000000# # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the wconfd calls""" class MockClient(object): """Mock client calls to wconfd. """ def __init__(self, wconfdmock): self.wconfdmock = wconfdmock def TryUpdateLocks(self, _cid, req): # Note that UpdateLocksWaiting in this mock # relies on TryUpdateLocks to always succeed. for lockrq in req: if lockrq[1] == "release": if lockrq[0] in self.wconfdmock.mylocks: del self.wconfdmock.mylocks[lockrq[0]] else: self.wconfdmock.mylocks[lockrq[0]] = lockrq[1] self.wconfdmock.all_locks[lockrq[0]] = lockrq[1] return [] def UpdateLocksWaiting(self, cid, _prio, req): # as our mock TryUpdateLocks always suceeds, we can # just use it return self.TryUpdateLocks(cid, req) def HasPendingRequest(self, _cid): return False def ListLocks(self, *_): result = [] for lock in self.wconfdmock.mylocks: result.append([lock, self.wconfdmock.mylocks[lock]]) return result def FreeLocksLevel(self, _cid, level): locks = list(self.wconfdmock.mylocks) for lock in locks: if lock.startswith(level + "/"): del self.wconfdmock.mylocks[lock] def OpportunisticLockUnion(self, _cid, req): for lockrq in req: self.wconfdmock.mylocks[lockrq[0]] = lockrq[1] return [lockrq[0] for lockrq in req] def PrepareClusterDestruction(self, _cid): pass class WConfdMock(object): """Mock calls to WConfD. As at various points, LUs are tested in an integration-test fashion, calling it through mcpu, which, in turn, calls wconfd, this mock must be able to live under these circumstances. In particular, it needs to keep track of locks requested and released, as both, mcpu and the individual LUs do consistency checks on the locks they own. """ def __init__(self): self.mylocks = {} self.all_locks = {} def Client(self): return MockClient(self) ganeti-3.1.0~rc2/test/py/legacy/daemon-util_unittest.bash000075500000000000000000000063701476477700300235040ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e daemon_util=daemons/daemon-util err() { echo "$@" echo 'Aborting' exit 1 } if ! grep -q '^ENABLE_MOND = ' lib/_constants.py; then err "Please update $0, mond enable feature is missing" fi DAEMONS_LIST="noded confd wconfd rapi luxid kvmd" STOPDAEMONS_LIST="kvmd luxid rapi wconfd confd noded" if grep -q '^ENABLE_MOND = True' lib/_constants.py; then DAEMONS_LIST="$DAEMONS_LIST mond" STOPDAEMONS_LIST="mond $STOPDAEMONS_LIST" fi STOPDAEMONS_LIST="metad $STOPDAEMONS_LIST" DAEMONS=$(echo $(for d in $DAEMONS_LIST; do echo "ganeti-$d"; done)) STOPDAEMONS=$(echo $(for d in $STOPDAEMONS_LIST; do echo "ganeti-$d"; done)) $daemon_util >/dev/null 2>&1 && err "daemon-util succeeded without command" $daemon_util this-is-an-unimplemented-command >/dev/null 2>&1 && err "daemon-util accepted unimplemented command" $daemon_util list_start_daemons >/dev/null 2>&1 && err "daemon-util accepted command with underscores" $daemon_util check-exitcode 0 || err "check-exitcode 0 failed" for i in 1 2 3 4 20 25 33; do $daemon_util check-exitcode $i >/dev/null 2>&1 && rc=0 || rc=$? test "$rc" == 1 || err "check-exitcode $i didn't return 1" done $daemon_util check-exitcode 11 >/dev/null 2>&1 || err "check-exitcode 11 (not master) didn't return 0" tmp=$(echo $($daemon_util list-start-daemons)) test "$tmp" == "$DAEMONS" || err "list-start-daemons didn't return correct list of daemons" tmp=$(echo $($daemon_util list-stop-daemons)) test "$tmp" == "$STOPDAEMONS" || err "list-stop-daemons didn't return correct list of daemons" $daemon_util is-daemon-name >/dev/null 2>&1 && err "is-daemon-name didn't require daemon name" for i in '' '.' '..' '-' 'not-a-daemon'; do $daemon_util is-daemon-name "$i" >/dev/null 2>&1 && err "is-daemon-name thinks '$i' is a daemon name" done for i in $DAEMONS; do $daemon_util is-daemon-name $i >/dev/null 2>&1 || err "is-daemon-name doesn't think '$i' is a daemon name" done ganeti-3.1.0~rc2/test/py/legacy/docs_unittest.py000075500000000000000000000257201476477700300217310ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting documentation""" import unittest import re import itertools from ganeti import _constants from ganeti import utils from ganeti import cmdlib from ganeti import build from ganeti import compat from ganeti import mcpu from ganeti import opcodes from ganeti import constants from ganeti.rapi import baserlib from ganeti.rapi import rlib2 from ganeti.rapi import connector import testutils VALID_URI_RE = re.compile(r"^[-/a-z0-9]*$") RAPI_OPCODE_EXCLUDE = compat.UniqueFrozenset([ # Not yet implemented opcodes.OpBackupRemove, opcodes.OpClusterConfigQuery, opcodes.OpClusterRepairDiskSizes, opcodes.OpClusterVerify, opcodes.OpClusterVerifyDisks, opcodes.OpInstanceChangeGroup, opcodes.OpInstanceMove, opcodes.OpNodeQueryvols, opcodes.OpOobCommand, opcodes.OpTagsSearch, opcodes.OpClusterActivateMasterIp, opcodes.OpClusterDeactivateMasterIp, opcodes.OpExtStorageDiagnose, # Difficult if not impossible opcodes.OpClusterDestroy, opcodes.OpClusterPostInit, opcodes.OpClusterRename, opcodes.OpNodeAdd, opcodes.OpNodeRemove, # Very sensitive in nature opcodes.OpRestrictedCommand, opcodes.OpClusterRenewCrypto, # Helper opcodes (e.g. submitted by LUs) opcodes.OpClusterVerifyConfig, opcodes.OpClusterVerifyGroup, opcodes.OpGroupEvacuate, opcodes.OpGroupVerifyDisks, # Test opcodes opcodes.OpTestAllocator, opcodes.OpTestDelay, opcodes.OpTestDummy, opcodes.OpTestJqueue, opcodes.OpTestOsParams, ]) def _ReadDocFile(filename): return utils.ReadFile("%s/doc/%s" % (testutils.GetSourceDir(), filename)) class TestHooksDocs(unittest.TestCase): HOOK_PATH_OK = compat.UniqueFrozenset([ "master-ip-turnup", "master-ip-turndown", ]) def test(self): """Check whether all hooks are documented. """ hooksdoc = _ReadDocFile("hooks.rst") # Reverse mapping from LU to opcode lu2opcode = dict((lu, op) for (op, lu) in mcpu.Processor.DISPATCH_TABLE.items()) assert len(lu2opcode) == len(mcpu.Processor.DISPATCH_TABLE), \ "Found duplicate entries" hooks_paths = frozenset(re.findall("^:directory:\s*(.+)\s*$", hooksdoc, re.M)) self.assertTrue(self.HOOK_PATH_OK.issubset(hooks_paths), msg="Whitelisted path not found in documentation") raw_hooks_ops = re.findall("^OP_(?!CODE$).+$", hooksdoc, re.M) hooks_ops = set() duplicate_ops = set() for op in raw_hooks_ops: if op in hooks_ops: duplicate_ops.add(op) else: hooks_ops.add(op) self.assertFalse(duplicate_ops, msg="Found duplicate opcode documentation: %s" % utils.CommaJoin(duplicate_ops)) seen_paths = set() seen_ops = set() self.assertFalse(duplicate_ops, msg="Found duplicated hook documentation: %s" % utils.CommaJoin(duplicate_ops)) for name in dir(cmdlib): lucls = getattr(cmdlib, name) if (isinstance(lucls, type) and issubclass(lucls, cmdlib.LogicalUnit) and hasattr(lucls, "HPATH")): if lucls.HTYPE is None: continue opcls = lu2opcode.get(lucls, None) if opcls: seen_ops.add(opcls.OP_ID) self.assertTrue(opcls.OP_ID in hooks_ops, msg="Missing hook documentation for %s" % opcls.OP_ID) self.assertTrue(lucls.HPATH in hooks_paths, msg="Missing documentation for hook %s/%s" % (lucls.HTYPE, lucls.HPATH)) seen_paths.add(lucls.HPATH) missed_ops = hooks_ops - seen_ops missed_paths = hooks_paths - seen_paths - self.HOOK_PATH_OK self.assertFalse(missed_ops, msg="Op documents hook not existing anymore: %s" % utils.CommaJoin(missed_ops)) self.assertFalse(missed_paths, msg="Hook path does not exist in opcode: %s" % utils.CommaJoin(missed_paths)) class TestRapiDocs(unittest.TestCase): def _CheckRapiResource(self, uri, fixup, handler): docline = "%s resource." % uri self.assertEqual(handler.__doc__.splitlines()[0].strip(), docline, msg=("First line of %r's docstring is not %r" % (handler, docline))) # Apply fixes before testing for (rx, value) in fixup.items(): uri = rx.sub(value, uri) self.assertTrue(VALID_URI_RE.match(uri), msg="Invalid URI %r" % uri) def test(self): """Check whether all RAPI resources are documented. """ rapidoc = _ReadDocFile("rapi.rst") node_name = re.escape("[node_name]") instance_name = re.escape("[instance_name]") group_name = re.escape("[group_name]") network_name = re.escape("[network_name]") job_id = re.escape("[job_id]") disk_index = re.escape("[disk_index]") filter_uuid = re.escape("[filter_uuid]") query_res = re.escape("[resource]") resources = connector.GetHandlers(node_name, instance_name, group_name, network_name, job_id, disk_index, filter_uuid, query_res) handler_dups = utils.FindDuplicates(resources.values()) self.assertFalse(handler_dups, msg=("Resource handlers used more than once: %r" % handler_dups)) uri_check_fixup = { re.compile(node_name): "node1examplecom", re.compile(instance_name): "inst1examplecom", re.compile(group_name): "group4440", re.compile(network_name): "network5550", re.compile(job_id): "9409", re.compile(disk_index): "123", re.compile(filter_uuid): "c863fbb5-f248-47bf-869b-cea259890061", re.compile(query_res): "lock", } assert compat.all(VALID_URI_RE.match(value) for value in uri_check_fixup.values()), \ "Fixup values must be valid URIs, too" titles = [] prevline = None for line in rapidoc.splitlines(): if re.match(r"^\++$", line): titles.append(prevline) prevline = line prefix_exception = compat.UniqueFrozenset(["/", "/version", "/2"]) undocumented = [] used_uris = [] for key, handler in resources.items(): # Regex objects if hasattr(key, "match"): self.assertTrue(key.pattern.startswith("^/2/"), msg="Pattern %r does not start with '^/2/'" % key.pattern) self.assertEqual(key.pattern[-1], "$") found = False for title in titles: if title.startswith("``") and title.endswith("``"): uri = title[2:-2] if key.match(uri): self._CheckRapiResource(uri, uri_check_fixup, handler) used_uris.append(uri) found = True break if not found: # TODO: Find better way of identifying resource undocumented.append(key.pattern) else: self.assertTrue(key.startswith("/2/") or key in prefix_exception, msg="Path %r does not start with '/2/'" % key) if ("``%s``" % key) in titles: self._CheckRapiResource(key, {}, handler) used_uris.append(key) else: undocumented.append(key) self.assertFalse(undocumented, msg=("Missing RAPI resource documentation for %s" % utils.CommaJoin(undocumented))) uri_dups = utils.FindDuplicates(used_uris) self.assertFalse(uri_dups, msg=("URIs matched by more than one resource: %s" % utils.CommaJoin(uri_dups))) self._FindRapiMissing(resources.values()) self._CheckTagHandlers(resources.values()) def _FindRapiMissing(self, handlers): used = frozenset(itertools.chain(*list(map(baserlib.GetResourceOpcodes, handlers)))) unexpected = used & RAPI_OPCODE_EXCLUDE self.assertFalse(unexpected, msg=("Found RAPI resources for excluded opcodes: %s" % utils.CommaJoin(_GetOpIds(unexpected)))) missing = (frozenset(opcodes.OP_MAPPING.values()) - used - RAPI_OPCODE_EXCLUDE) self.assertFalse(missing, msg=("Missing RAPI resources for opcodes: %s" % utils.CommaJoin(_GetOpIds(missing)))) def _CheckTagHandlers(self, handlers): tag_handlers = [x for x in handlers if issubclass(x, rlib2._R_Tags)] self.assertEqual(frozenset(tag.TAG_LEVEL for tag in tag_handlers), constants.VALID_TAG_TYPES) def _GetOpIds(ops): """Returns C{OP_ID} for all opcodes in passed sequence. """ return sorted(opcls.OP_ID for opcls in ops) class TestManpages(unittest.TestCase): """Manpage tests""" @staticmethod def _ReadManFile(name): return utils.ReadFile("%s/man/%s.rst" % (testutils.GetSourceDir(), name)) @staticmethod def _LoadScript(name): return build.LoadModule("scripts/%s" % name) def test(self): for script in _constants.GNT_SCRIPTS: self._CheckManpage(script, self._ReadManFile(script), list(self._LoadScript(script).commands)) def _CheckManpage(self, script, mantext, commands): missing = [] for cmd in commands: pattern = r"^(\| )?\*\*%s\*\*" % re.escape(cmd) if not re.findall(pattern, mantext, re.DOTALL | re.MULTILINE): missing.append(cmd) self.assertFalse(missing, msg=("Manpage for '%s' missing documentation for %s" % (script, utils.CommaJoin(missing)))) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti-cleaner_unittest.bash000075500000000000000000000151501476477700300241400ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e -u set -o pipefail export PYTHON=${PYTHON:=python3} GNTC=daemons/ganeti-cleaner CCE=$PWD/tools/check-cert-expired # Expand relative PYTHONPATH passed as passed by the test environment. PYTHONPATH="${PYTHONPATH/.:./$PWD:$PWD}" err() { echo "$@" echo 'Aborting' exit 1 } upto() { echo "$(date '+%F %T'):" "$@" '...' } gencert() { local path=$1 validity=$2 VALIDITY=$validity $PYTHON \ ${TOP_SRCDIR:-.}/test/py/legacy/import-export_unittest-helper \ $path gencert } check_logfiles() { local n=$1 p=$2 path if [[ "$p" = master ]]; then path=$tmpls/log/ganeti/master-cleaner else path=$tmpls/log/ganeti/cleaner fi test -d $path || \ err "Log file directory '$path' not created" [[ "$(find $path -mindepth 1 | wc -l)" -le "$n" ]] || \ err "Found more than $n logfiles" } count_jobs() { local n=$1 local count=$(find $queuedir -mindepth 1 -type f | wc -l) [[ "$count" -eq "$n" ]] || err "Found $count jobs instead of $n" } count_watcher() { local suffix="$1" n=$2 local count=$(find $watcherdir -maxdepth 1 -type f \ -name "watcher.*-*-*-*.$suffix" | wc -l) [[ "$count" -eq "$n" ]] || \ err "Found $count watcher files with suffix '$suffix' instead of $n" } count_and_check_certs() { local n=$1 local count=$(find $cryptodir -mindepth 1 -type f -name cert | wc -l) [[ "$count" -eq "$n" ]] || err "Found $count certificates instead of $n" find $cryptodir -mindepth 1 -type d | \ while read dir; do [[ ( -e $dir/key && -e $dir/cert ) || ( ! -e $dir/cert && ! -e $dir/key ) ]] || \ err 'Inconsistent cert/key directory found' done } run_cleaner() { CHECK_CERT_EXPIRED=$CCE LOCALSTATEDIR=$tmpls $GNTC $1 } create_archived_jobs() { local i jobdir touchargs local jobarchive=$queuedir/archive local old_ts=$(date -d '25 days ago' +%Y%m%d%H%M) # Remove jobs from previous run find $jobarchive -mindepth 1 -type f | xargs -r rm i=0 for job_id in {1..50} 469581574 19857 1420164 494433 2448521 do jobdir=$jobarchive/$(( job_id / 10 )) test -d $jobdir || mkdir $jobdir if (( i % 3 == 0 || i % 7 == 0 )); then touchargs="-t $old_ts" else touchargs= fi touch $touchargs $jobdir/job-$job_id let ++i done } create_watcher_state() { local uuids=( 6792a0d5-f8b6-4531-8d8c-3680c86b8a53 ab74da37-f5f7-44c4-83ad-074159772593 fced2e48-ffff-43ae-919e-2b77d37ecafa 6e89ac57-2eb1-4a16-85a1-94daa815d643 8714e8f5-59c4-47db-b2cb-196ec37978e5 91763d73-e1f3-47c7-a735-57025d4e2a7d e27d3ff8-9546-4e86-86a4-04151223e140 aa3f63dd-be17-4ac8-bd01-d71790e124cb 05b6d7e2-003b-40d9-a6d6-ab61bf123a15 54c93e4c-61fe-40de-b47e-2a8e6c805d02 ) i=0 for uuid in ${uuids[@]}; do touch -d "$(( 5 * i )) days ago" \ $watcherdir/watcher.$uuid.{data,instance-status} let ++i done } create_certdirs() { local cert=$1; shift local certdir for name in "$@"; do certdir=$cryptodir/$name mkdir $certdir if [[ -n "$cert" ]]; then cp $cert $certdir/cert cp $cert $certdir/key fi done } tmpdir=$(mktemp -d) trap "rm -rf $tmpdir" EXIT # Temporary localstatedir tmpls=$tmpdir/var queuedir=$tmpls/lib/ganeti/queue cryptodir=$tmpls/run/ganeti/crypto watcherdir=$tmpls/lib/ganeti mkdir -p $tmpls/{lib,log,run}/ganeti $queuedir/archive $cryptodir maxlog=50 upto 'Checking log directory creation' test -d $tmpls/log/ganeti || err 'log/ganeti does not exist' test -d $tmpls/log/ganeti/cleaner && \ err 'log/ganeti/cleaner should not exist yet' run_cleaner node check_logfiles 1 node test -d $tmpls/log/ganeti/master-cleaner && \ err 'log/ganeti/master-cleaner should not exist yet' run_cleaner master check_logfiles 1 master upto 'Checking number of retained log files (master)' for (( i=0; i < (maxlog + 10); ++i )); do run_cleaner master check_logfiles 1 node check_logfiles $(( (i + 2) > $maxlog?$maxlog:(i + 2) )) master done upto 'Checking number of retained log files (node)' for (( i=0; i < (maxlog + 10); ++i )); do run_cleaner node check_logfiles $(( (i + 2) > $maxlog?$maxlog:(i + 2) )) node check_logfiles $maxlog master done upto 'Removal of archived jobs (non-master)' create_archived_jobs count_jobs 55 test -f $tmpls/lib/ganeti/ssconf_master_node && \ err 'ssconf_master_node should not exist' run_cleaner node count_jobs 55 run_cleaner master count_jobs 55 upto 'Removal of archived jobs (master node)' create_archived_jobs count_jobs 55 echo $HOSTNAME > $tmpls/lib/ganeti/ssconf_master_node run_cleaner node count_jobs 55 run_cleaner master count_jobs 31 upto 'Certificate expiration' gencert $tmpdir/validcert 30 & vcpid=${!} gencert $tmpdir/expcert -30 & ecpid=${!} wait $vcpid $ecpid create_certdirs $tmpdir/validcert foo{a,b,c}123 trvRMH4Wvt OfDlh6Pc2n create_certdirs $tmpdir/expcert bar{x,y,z}999 fx0ljoImWr em3RBC0U8c create_certdirs '' empty{1,2,3} gd2HCvRc iFG55Z0a PP28v5kg count_and_check_certs 10 run_cleaner master count_and_check_certs 10 run_cleaner node count_and_check_certs 5 check_logfiles $maxlog node check_logfiles $maxlog master count_jobs 31 upto 'Watcher status files' create_watcher_state count_watcher data 10 count_watcher instance-status 10 run_cleaner master count_watcher data 10 count_watcher instance-status 10 run_cleaner node count_watcher data 5 count_watcher instance-status 5 exit 0 ganeti-3.1.0~rc2/test/py/legacy/ganeti-cli.test000064400000000000000000000006641476477700300214020ustar00rootroot00000000000000# test the various gnt-commands for common options sh -c "$DAEMONS/ganeti-noded --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$DAEMONS/ganeti-noded --version" >>>/^ganeti-/ >>>2 >>>= 0 sh -c "$DAEMONS/ganeti-rapi --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$DAEMONS/ganeti-rapi --version" >>>/^ganeti-/ >>>2 >>>= 0 sh -c "$DAEMONS/ganeti-watcher --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$DAEMONS/ganeti-watcher --version" >>>/^ganeti-/ >>>2 >>>= 0 ganeti-3.1.0~rc2/test/py/legacy/ganeti.asyncnotifier_unittest.py000075500000000000000000000166151476477700300251270ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the asyncnotifier module""" import logging import unittest import signal import os import tempfile import shutil try: # pylint: disable=E0611 from pyinotify import pyinotify except ImportError: import pyinotify from ganeti import asyncnotifier from ganeti import daemon from ganeti import utils from ganeti import errors import testutils class _MyErrorLoggingAsyncNotifier(asyncnotifier.ErrorLoggingAsyncNotifier): def __init__(self, *args, **kwargs): asyncnotifier.ErrorLoggingAsyncNotifier.__init__(self, *args, **kwargs) self.error_count = 0 def handle_error(self): self.error_count += 1 raise class TestSingleFileEventHandler(testutils.GanetiTestCase): """Test daemon.Mainloop""" NOTIFIERS = [NOTIFIER_TERM, NOTIFIER_NORM, NOTIFIER_ERR] = range(3) def setUp(self): testutils.GanetiTestCase.setUp(self) self.mainloop = daemon.Mainloop() self.chk_files = [self._CreateTempFile() for i in self.NOTIFIERS] self.notified = [False for i in self.NOTIFIERS] # We need one watch manager per notifier, as those contain the file # descriptor which is monitored by asyncore self.wms = [pyinotify.WatchManager() for i in self.NOTIFIERS] self.cbk = [self.OnInotifyCallback(self, i) for i in self.NOTIFIERS] self.ihandler = [asyncnotifier.SingleFileEventHandler(wm, cb, cf) for (wm, cb, cf) in zip(self.wms, self.cbk, self.chk_files)] self.notifiers = [_MyErrorLoggingAsyncNotifier(wm, ih) for (wm, ih) in zip(self.wms, self.ihandler)] # TERM notifier is enabled by default, as we use it to get out of the loop self.ihandler[self.NOTIFIER_TERM].enable() def tearDown(self): # disable the inotifiers, before removing the files for i in self.ihandler: i.disable() testutils.GanetiTestCase.tearDown(self) # and unregister the fd's being polled for n in self.notifiers: n.del_channel() class OnInotifyCallback: def __init__(self, testobj, i): self.testobj = testobj self.notified = testobj.notified self.i = i def __call__(self, enabled): self.notified[self.i] = True if self.i == self.testobj.NOTIFIER_TERM: os.kill(os.getpid(), signal.SIGTERM) elif self.i == self.testobj.NOTIFIER_ERR: raise errors.GenericError("an error") def testReplace(self): utils.WriteFile(self.chk_files[self.NOTIFIER_TERM], data="dummy") self.mainloop.Run() self.assertTrue(self.notified[self.NOTIFIER_TERM]) self.assertFalse(self.notified[self.NOTIFIER_NORM]) self.assertEqual(self.notifiers[self.NOTIFIER_TERM].error_count, 0) self.assertEqual(self.notifiers[self.NOTIFIER_NORM].error_count, 0) def testEnableDisable(self): self.ihandler[self.NOTIFIER_TERM].enable() self.ihandler[self.NOTIFIER_TERM].disable() self.ihandler[self.NOTIFIER_TERM].disable() self.ihandler[self.NOTIFIER_TERM].enable() self.ihandler[self.NOTIFIER_TERM].disable() self.ihandler[self.NOTIFIER_TERM].enable() utils.WriteFile(self.chk_files[self.NOTIFIER_TERM], data="dummy") self.mainloop.Run() self.assertTrue(self.notified[self.NOTIFIER_TERM]) self.assertFalse(self.notified[self.NOTIFIER_NORM]) self.assertEqual(self.notifiers[self.NOTIFIER_TERM].error_count, 0) self.assertEqual(self.notifiers[self.NOTIFIER_NORM].error_count, 0) def testDoubleEnable(self): self.ihandler[self.NOTIFIER_TERM].enable() self.ihandler[self.NOTIFIER_TERM].enable() utils.WriteFile(self.chk_files[self.NOTIFIER_TERM], data="dummy") self.mainloop.Run() self.assertTrue(self.notified[self.NOTIFIER_TERM]) self.assertFalse(self.notified[self.NOTIFIER_NORM]) self.assertEqual(self.notifiers[self.NOTIFIER_TERM].error_count, 0) self.assertEqual(self.notifiers[self.NOTIFIER_NORM].error_count, 0) def testDefaultDisabled(self): utils.WriteFile(self.chk_files[self.NOTIFIER_NORM], data="dummy") utils.WriteFile(self.chk_files[self.NOTIFIER_TERM], data="dummy") self.mainloop.Run() self.assertTrue(self.notified[self.NOTIFIER_TERM]) # NORM notifier is disabled by default self.assertFalse(self.notified[self.NOTIFIER_NORM]) self.assertEqual(self.notifiers[self.NOTIFIER_TERM].error_count, 0) self.assertEqual(self.notifiers[self.NOTIFIER_NORM].error_count, 0) def testBothEnabled(self): self.ihandler[self.NOTIFIER_NORM].enable() utils.WriteFile(self.chk_files[self.NOTIFIER_NORM], data="dummy") utils.WriteFile(self.chk_files[self.NOTIFIER_TERM], data="dummy") self.mainloop.Run() self.assertTrue(self.notified[self.NOTIFIER_TERM]) self.assertTrue(self.notified[self.NOTIFIER_NORM]) self.assertEqual(self.notifiers[self.NOTIFIER_TERM].error_count, 0) self.assertEqual(self.notifiers[self.NOTIFIER_NORM].error_count, 0) def testError(self): self.ihandler[self.NOTIFIER_ERR].enable() utils.WriteFile(self.chk_files[self.NOTIFIER_ERR], data="dummy") self.assertRaises(errors.GenericError, self.mainloop.Run) self.assertTrue(self.notified[self.NOTIFIER_ERR]) self.assertEqual(self.notifiers[self.NOTIFIER_ERR].error_count, 1) self.assertEqual(self.notifiers[self.NOTIFIER_NORM].error_count, 0) self.assertEqual(self.notifiers[self.NOTIFIER_TERM].error_count, 0) class TestSingleFileEventHandlerError(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def test(self): wm = pyinotify.WatchManager() handler = asyncnotifier.SingleFileEventHandler(wm, None, utils.PathJoin(self.tmpdir, "nonexist")) logger = logging.getLogger('pyinotify') logger.disabled = True try: self.assertRaises(errors.InotifyError, handler.enable) self.assertRaises(errors.InotifyError, handler.enable) handler.disable() self.assertRaises(errors.InotifyError, handler.enable) finally: logger.disabled = False if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.backend_unittest-runasroot.py000075500000000000000000000051201476477700300256600ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.backend (tests requiring root access)""" import os import tempfile import shutil import errno from ganeti import constants from ganeti import utils from ganeti import compat from ganeti import backend import testutils class TestCommonRestrictedCmdCheck(testutils.GanetiTestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def _PrepareTest(self): tmpname = utils.PathJoin(self.tmpdir, "foobar") os.mkdir(tmpname) os.chmod(tmpname, 0o700) return tmpname def testCorrectOwner(self): tmpname = self._PrepareTest() os.chown(tmpname, 0, 0) (status, value) = backend._CommonRestrictedCmdCheck(tmpname, None) self.assertTrue(status) self.assertTrue(value) def testWrongOwner(self): tmpname = self._PrepareTest() tests = [ (1, 0), (0, 1), (100, 50), ] for (uid, gid) in tests: self.assertFalse(uid == os.getuid() and gid == os.getgid()) os.chown(tmpname, uid, gid) (status, errmsg) = backend._CommonRestrictedCmdCheck(tmpname, None) self.assertFalse(status) self.assertTrue("foobar' is not owned by " in errmsg) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.backend_unittest.py000075500000000000000000002424511476477700300236400ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.backend""" import collections import copy import os import shutil import tempfile import testutils import testutils_ssh import unittest from unittest import mock from ganeti import backend from ganeti import constants from ganeti import errors from ganeti import hypervisor from ganeti import netutils from ganeti import objects from ganeti import serializer from ganeti import ssh from ganeti import utils from testutils.config_mock import ConfigMock class TestX509Certificates(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def test(self): (name, cert_pem) = backend.CreateX509Certificate(300, cryptodir=self.tmpdir) self.assertEqual(utils.ReadBinaryFile(os.path.join(self.tmpdir, name, backend._X509_CERT_FILE)), cert_pem) self.assertTrue(0 < os.path.getsize(os.path.join(self.tmpdir, name, backend._X509_KEY_FILE))) (name2, cert_pem2) = \ backend.CreateX509Certificate(300, cryptodir=self.tmpdir) backend.RemoveX509Certificate(name, cryptodir=self.tmpdir) backend.RemoveX509Certificate(name2, cryptodir=self.tmpdir) self.assertEqual(utils.ListVisibleFiles(self.tmpdir), []) def testNonEmpty(self): (name, _) = backend.CreateX509Certificate(300, cryptodir=self.tmpdir) utils.WriteFile(utils.PathJoin(self.tmpdir, name, "hello-world"), data="Hello World") self.assertRaises(backend.RPCFail, backend.RemoveX509Certificate, name, cryptodir=self.tmpdir) self.assertEqual(utils.ListVisibleFiles(self.tmpdir), [name]) class TestGetCryptoTokens(testutils.GanetiTestCase): def setUp(self): self._get_digest_fn_orig = utils.GetCertificateDigest self._create_digest_fn_orig = utils.GenerateNewSslCert self._ssl_digest = "12345" utils.GetCertificateDigest = mock.Mock( return_value=self._ssl_digest) utils.GenerateNewSslCert = mock.Mock() def tearDown(self): utils.GetCertificateDigest = self._get_digest_fn_orig utils.GenerateNewSslCert = self._create_digest_fn_orig def testGetSslToken(self): result = backend.GetCryptoTokens( [(constants.CRYPTO_TYPE_SSL_DIGEST, constants.CRYPTO_ACTION_GET, None)]) self.assertTrue((constants.CRYPTO_TYPE_SSL_DIGEST, self._ssl_digest) in result) def testUnknownTokenType(self): self.assertRaises(errors.ProgrammerError, backend.GetCryptoTokens, [("pink_bunny", constants.CRYPTO_ACTION_GET, None)]) def testUnknownAction(self): self.assertRaises(errors.ProgrammerError, backend.GetCryptoTokens, [(constants.CRYPTO_TYPE_SSL_DIGEST, "illuminate", None)]) class TestNodeVerify(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self._mock_hv = None def _GetHypervisor(self, hv_name): self._mock_hv = hypervisor.GetHypervisor(hv_name) self._mock_hv.ValidateParameters = mock.Mock() self._mock_hv.Verify = mock.Mock() return self._mock_hv def testMasterIPLocalhost(self): # this a real functional test, but requires localhost to be reachable my_name = netutils.Hostname.GetSysName() local_data = (my_name,constants.IP4_ADDRESS_LOCALHOST, [my_name]) result = backend.VerifyNode({constants.NV_MASTERIP: local_data}, None, {}) self.assertTrue(constants.NV_MASTERIP in result, "Master IP data not returned") self.assertTrue(result[constants.NV_MASTERIP], "Cannot reach localhost") def testMasterIPSkipTest(self): # this a real functional test, but requires localhost to be reachable local_data = (netutils.Hostname.GetSysName(), constants.IP4_ADDRESS_LOCALHOST, []) result = backend.VerifyNode({constants.NV_MASTERIP: local_data}, None, {}) self.assertTrue(constants.NV_MASTERIP in result, "Master IP data not returned") self.assertTrue(result[constants.NV_MASTERIP] == None, "Test ran by non master candidate") def testMasterIPUnreachable(self): # Network 192.0.2.0/24 is reserved for test/documentation as per # RFC 5737 my_name = "master.example.com" bad_data = (my_name, "192.0.2.1", [my_name]) # we just test that whatever TcpPing returns, VerifyNode returns too netutils.TcpPing = lambda a, b, source=None: False result = backend.VerifyNode({constants.NV_MASTERIP: bad_data}, None, {}) self.assertTrue(constants.NV_MASTERIP in result, "Master IP data not returned") self.assertFalse(result[constants.NV_MASTERIP], "Result from netutils.TcpPing corrupted") def testVerifyNodeNetTestMissingSelf(self): my_name = netutils.Hostname.GetSysName() local_data = ([('n1.example.com', "any", "any")], [my_name]) result = backend.VerifyNode({constants.NV_NODENETTEST: local_data}, None, {}) self.assertTrue(constants.NV_NODENETTEST in result, "NodeNetTest data not returned") self.assertTrue(my_name in result[constants.NV_NODENETTEST], "Missing failure in net test") def testVerifyNodeNetTest(self): my_name = netutils.Hostname.GetSysName() local_data = ([(my_name, "any", "any")], [my_name]) # we just test that whatever TcpPing returns, VerifyNode returns too netutils.TcpPing = lambda a, b, source=None: True result = backend.VerifyNode({constants.NV_NODENETTEST: local_data}, None, {}) self.assertTrue(constants.NV_NODENETTEST in result, "NodeNetTest data not returned") self.assertTrue(result[constants.NV_NODENETTEST] == {}, "NodeNetTest failed") def testVerifyNodeNetSkipTest(self): local_data = ([('n1.example.com', "any", "any")], []) result = backend.VerifyNode({constants.NV_NODENETTEST: local_data}, None, {}) self.assertTrue(constants.NV_NODENETTEST in result, "NodeNetTest data not returned") self.assertTrue(result[constants.NV_NODENETTEST] == {}, "Test ran by non master candidate") def testVerifyHvparams(self): test_hvparams = {} test_what = {constants.NV_HVPARAMS: \ [("mynode", constants.HT_XEN_PVM, test_hvparams)]} result = {} backend._VerifyHvparams(test_what, True, result, get_hv_fn=self._GetHypervisor) self._mock_hv.ValidateParameters.assert_called_with(test_hvparams) def testVerifyHypervisors(self): hvname = constants.HT_XEN_PVM hvparams = {} all_hvparams = {hvname: hvparams} test_what = {constants.NV_HYPERVISOR: [hvname]} result = {} backend._VerifyHypervisors( test_what, True, result, all_hvparams=all_hvparams, get_hv_fn=self._GetHypervisor) self._mock_hv.Verify.assert_called_with(hvparams=hvparams) @testutils.patch_object(utils, "VerifyCertificate") def testVerifyClientCertificateSuccess(self, verif_cert): # mock the underlying x509 verification because the test cert is expired verif_cert.return_value = (None, None) cert_file = testutils.TestDataFilename("cert2.pem") (errcode, digest) = backend._VerifyClientCertificate(cert_file=cert_file) self.assertEqual(constants.CV_WARNING, errcode) self.assertTrue(isinstance(digest, str)) @testutils.patch_object(utils, "VerifyCertificate") def testVerifyClientCertificateFailed(self, verif_cert): expected_errcode = 666 verif_cert.return_value = (expected_errcode, "The devil created this certificate.") cert_file = testutils.TestDataFilename("cert2.pem") (errcode, digest) = backend._VerifyClientCertificate(cert_file=cert_file) self.assertEqual(expected_errcode, errcode) def testVerifyClientCertificateNoCert(self): cert_file = testutils.TestDataFilename("cert-that-does-not-exist.pem") (errcode, digest) = backend._VerifyClientCertificate(cert_file=cert_file) self.assertEqual(constants.CV_ERROR, errcode) def _DefRestrictedCmdOwner(): return (os.getuid(), os.getgid()) class TestVerifyRestrictedCmdName(unittest.TestCase): def testAcceptableName(self): for i in ["foo", "bar", "z1", "000first", "hello-world"]: for fn in [lambda s: s, lambda s: s.upper(), lambda s: s.title()]: (status, msg) = backend._VerifyRestrictedCmdName(fn(i)) self.assertTrue(status) self.assertTrue(msg is None) def testEmptyAndSpace(self): for i in ["", " ", "\t", "\n"]: (status, msg) = backend._VerifyRestrictedCmdName(i) self.assertFalse(status) self.assertEqual(msg, "Missing command name") def testNameWithSlashes(self): for i in ["/", "./foo", "../moo", "some/name"]: (status, msg) = backend._VerifyRestrictedCmdName(i) self.assertFalse(status) self.assertEqual(msg, "Invalid command name") def testForbiddenCharacters(self): for i in ["#", ".", "..", "bash -c ls", "'"]: (status, msg) = backend._VerifyRestrictedCmdName(i) self.assertFalse(status) self.assertEqual(msg, "Command name contains forbidden characters") class TestVerifyRestrictedCmdDirectory(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testCanNotStat(self): tmpname = utils.PathJoin(self.tmpdir, "foobar") self.assertFalse(os.path.exists(tmpname)) (status, msg) = \ backend._VerifyRestrictedCmdDirectory(tmpname, _owner=NotImplemented) self.assertFalse(status) self.assertTrue(msg.startswith("Can't stat(2) '")) def testTooPermissive(self): tmpname = utils.PathJoin(self.tmpdir, "foobar") os.mkdir(tmpname) for mode in [0o777, 0o706, 0o760, 0o722]: os.chmod(tmpname, mode) self.assertTrue(os.path.isdir(tmpname)) (status, msg) = \ backend._VerifyRestrictedCmdDirectory(tmpname, _owner=NotImplemented) self.assertFalse(status) self.assertTrue(msg.startswith("Permissions on '")) def testNoDirectory(self): tmpname = utils.PathJoin(self.tmpdir, "foobar") utils.WriteFile(tmpname, data="empty\n") self.assertTrue(os.path.isfile(tmpname)) (status, msg) = \ backend._VerifyRestrictedCmdDirectory(tmpname, _owner=_DefRestrictedCmdOwner()) self.assertFalse(status) self.assertTrue(msg.endswith("is not a directory")) def testNormal(self): tmpname = utils.PathJoin(self.tmpdir, "foobar") os.mkdir(tmpname) os.chmod(tmpname, 0o755) self.assertTrue(os.path.isdir(tmpname)) (status, msg) = \ backend._VerifyRestrictedCmdDirectory(tmpname, _owner=_DefRestrictedCmdOwner()) self.assertTrue(status) self.assertTrue(msg is None) class TestVerifyRestrictedCmd(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testCanNotStat(self): tmpname = utils.PathJoin(self.tmpdir, "helloworld") self.assertFalse(os.path.exists(tmpname)) (status, msg) = \ backend._VerifyRestrictedCmd(self.tmpdir, "helloworld", _owner=NotImplemented) self.assertFalse(status) self.assertTrue(msg.startswith("Can't stat(2) '")) def testNotExecutable(self): tmpname = utils.PathJoin(self.tmpdir, "cmdname") utils.WriteFile(tmpname, data="empty\n") (status, msg) = \ backend._VerifyRestrictedCmd(self.tmpdir, "cmdname", _owner=_DefRestrictedCmdOwner()) self.assertFalse(status) self.assertTrue(msg.startswith("access(2) thinks '")) def testExecutable(self): tmpname = utils.PathJoin(self.tmpdir, "cmdname") utils.WriteFile(tmpname, data="empty\n", mode=0o700) (status, executable) = \ backend._VerifyRestrictedCmd(self.tmpdir, "cmdname", _owner=_DefRestrictedCmdOwner()) self.assertTrue(status) self.assertEqual(executable, tmpname) class TestPrepareRestrictedCmd(unittest.TestCase): _TEST_PATH = "/tmp/some/test/path" def testDirFails(self): def fn(path): self.assertEqual(path, self._TEST_PATH) return (False, "test error 31420") (status, msg) = \ backend._PrepareRestrictedCmd(self._TEST_PATH, "cmd21152", _verify_dir=fn, _verify_name=NotImplemented, _verify_cmd=NotImplemented) self.assertFalse(status) self.assertEqual(msg, "test error 31420") def testNameFails(self): def fn(cmd): self.assertEqual(cmd, "cmd4617") return (False, "test error 591") (status, msg) = \ backend._PrepareRestrictedCmd(self._TEST_PATH, "cmd4617", _verify_dir=lambda _: (True, None), _verify_name=fn, _verify_cmd=NotImplemented) self.assertFalse(status) self.assertEqual(msg, "test error 591") def testCommandFails(self): def fn(path, cmd): self.assertEqual(path, self._TEST_PATH) self.assertEqual(cmd, "cmd17577") return (False, "test error 25524") (status, msg) = \ backend._PrepareRestrictedCmd(self._TEST_PATH, "cmd17577", _verify_dir=lambda _: (True, None), _verify_name=lambda _: (True, None), _verify_cmd=fn) self.assertFalse(status) self.assertEqual(msg, "test error 25524") def testSuccess(self): def fn(path, cmd): return (True, utils.PathJoin(path, cmd)) (status, executable) = \ backend._PrepareRestrictedCmd(self._TEST_PATH, "cmd22633", _verify_dir=lambda _: (True, None), _verify_name=lambda _: (True, None), _verify_cmd=fn) self.assertTrue(status) self.assertEqual(executable, utils.PathJoin(self._TEST_PATH, "cmd22633")) def _SleepForRestrictedCmd(duration): assert duration > 5 def _GenericRestrictedCmdError(cmd): return "Executing command '%s' failed" % cmd class TestRunRestrictedCmd(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testNonExistantLockDirectory(self): lockfile = utils.PathJoin(self.tmpdir, "does", "not", "exist") sleep_fn = testutils.CallCounter(_SleepForRestrictedCmd) self.assertFalse(os.path.exists(lockfile)) self.assertRaises(backend.RPCFail, backend.RunRestrictedCmd, "test", _lock_timeout=NotImplemented, _lock_file=lockfile, _path=NotImplemented, _sleep_fn=sleep_fn, _prepare_fn=NotImplemented, _runcmd_fn=NotImplemented, _enabled=True) self.assertEqual(sleep_fn.Count(), 1) @staticmethod def _TryLock(lockfile): sleep_fn = testutils.CallCounter(_SleepForRestrictedCmd) result = False try: backend.RunRestrictedCmd("test22717", _lock_timeout=0.1, _lock_file=lockfile, _path=NotImplemented, _sleep_fn=sleep_fn, _prepare_fn=NotImplemented, _runcmd_fn=NotImplemented, _enabled=True) except backend.RPCFail as err: assert str(err) == _GenericRestrictedCmdError("test22717"), \ "Did not fail with generic error message" result = True assert sleep_fn.Count() == 1 return result def testLockHeldByOtherProcess(self): lockfile = utils.PathJoin(self.tmpdir, "lock") lock = utils.FileLock.Open(lockfile) lock.Exclusive(blocking=True, timeout=1.0) try: self.assertTrue(utils.RunInSeparateProcess(self._TryLock, lockfile)) finally: lock.Close() @staticmethod def _PrepareRaisingException(path, cmd): assert cmd == "test23122" raise Exception("test") def testPrepareRaisesException(self): lockfile = utils.PathJoin(self.tmpdir, "lock") sleep_fn = testutils.CallCounter(_SleepForRestrictedCmd) prepare_fn = testutils.CallCounter(self._PrepareRaisingException) try: backend.RunRestrictedCmd("test23122", _lock_timeout=1.0, _lock_file=lockfile, _path=NotImplemented, _runcmd_fn=NotImplemented, _sleep_fn=sleep_fn, _prepare_fn=prepare_fn, _enabled=True) except backend.RPCFail as err: self.assertEqual(str(err), _GenericRestrictedCmdError("test23122")) else: self.fail("Didn't fail") self.assertEqual(sleep_fn.Count(), 1) self.assertEqual(prepare_fn.Count(), 1) @staticmethod def _PrepareFails(path, cmd): assert cmd == "test29327" return ("some error message", None) def testPrepareFails(self): lockfile = utils.PathJoin(self.tmpdir, "lock") sleep_fn = testutils.CallCounter(_SleepForRestrictedCmd) prepare_fn = testutils.CallCounter(self._PrepareFails) try: backend.RunRestrictedCmd("test29327", _lock_timeout=1.0, _lock_file=lockfile, _path=NotImplemented, _runcmd_fn=NotImplemented, _sleep_fn=sleep_fn, _prepare_fn=prepare_fn, _enabled=True) except backend.RPCFail as err: self.assertEqual(str(err), _GenericRestrictedCmdError("test29327")) else: self.fail("Didn't fail") self.assertEqual(sleep_fn.Count(), 1) self.assertEqual(prepare_fn.Count(), 1) @staticmethod def _SuccessfulPrepare(path, cmd): return (True, utils.PathJoin(path, cmd)) def testRunCmdFails(self): lockfile = utils.PathJoin(self.tmpdir, "lock") def fn(args, env=NotImplemented, reset_env=NotImplemented, postfork_fn=NotImplemented): self.assertEqual(args, [utils.PathJoin(self.tmpdir, "test3079")]) self.assertEqual(env, {}) self.assertTrue(reset_env) self.assertTrue(callable(postfork_fn)) trylock = utils.FileLock.Open(lockfile) try: # See if lockfile is still held self.assertRaises(EnvironmentError, trylock.Exclusive, blocking=False) # Call back to release lock postfork_fn(NotImplemented) # See if lockfile can be acquired trylock.Exclusive(blocking=False) finally: trylock.Close() # Simulate a failed command return utils.RunResult(constants.EXIT_FAILURE, None, "stdout", "stderr406328567", utils.ShellQuoteArgs(args), NotImplemented, NotImplemented) sleep_fn = testutils.CallCounter(_SleepForRestrictedCmd) prepare_fn = testutils.CallCounter(self._SuccessfulPrepare) runcmd_fn = testutils.CallCounter(fn) try: backend.RunRestrictedCmd("test3079", _lock_timeout=1.0, _lock_file=lockfile, _path=self.tmpdir, _runcmd_fn=runcmd_fn, _sleep_fn=sleep_fn, _prepare_fn=prepare_fn, _enabled=True) except backend.RPCFail as err: self.assertTrue(str(err).startswith("Restricted command 'test3079'" " failed:")) self.assertTrue("stderr406328567" in str(err), msg="Error did not include output") else: self.fail("Didn't fail") self.assertEqual(sleep_fn.Count(), 0) self.assertEqual(prepare_fn.Count(), 1) self.assertEqual(runcmd_fn.Count(), 1) def testRunCmdSucceeds(self): lockfile = utils.PathJoin(self.tmpdir, "lock") def fn(args, env=NotImplemented, reset_env=NotImplemented, postfork_fn=NotImplemented): self.assertEqual(args, [utils.PathJoin(self.tmpdir, "test5667")]) self.assertEqual(env, {}) self.assertTrue(reset_env) # Call back to release lock postfork_fn(NotImplemented) # Simulate a successful command return utils.RunResult(constants.EXIT_SUCCESS, None, "stdout14463", "", utils.ShellQuoteArgs(args), NotImplemented, NotImplemented) sleep_fn = testutils.CallCounter(_SleepForRestrictedCmd) prepare_fn = testutils.CallCounter(self._SuccessfulPrepare) runcmd_fn = testutils.CallCounter(fn) result = backend.RunRestrictedCmd("test5667", _lock_timeout=1.0, _lock_file=lockfile, _path=self.tmpdir, _runcmd_fn=runcmd_fn, _sleep_fn=sleep_fn, _prepare_fn=prepare_fn, _enabled=True) self.assertEqual(result, "stdout14463") self.assertEqual(sleep_fn.Count(), 0) self.assertEqual(prepare_fn.Count(), 1) self.assertEqual(runcmd_fn.Count(), 1) def testCommandsDisabled(self): try: backend.RunRestrictedCmd("test", _lock_timeout=NotImplemented, _lock_file=NotImplemented, _path=NotImplemented, _sleep_fn=NotImplemented, _prepare_fn=NotImplemented, _runcmd_fn=NotImplemented, _enabled=False) except backend.RPCFail as err: self.assertEqual(str(err), "Restricted commands disabled at configure time") else: self.fail("Did not raise exception") class TestSetWatcherPause(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() self.filename = utils.PathJoin(self.tmpdir, "pause") def tearDown(self): shutil.rmtree(self.tmpdir) def testUnsetNonExisting(self): self.assertFalse(os.path.exists(self.filename)) backend.SetWatcherPause(None, _filename=self.filename) self.assertFalse(os.path.exists(self.filename)) def testSetNonNumeric(self): for i in ["", [], {}, "Hello World", "0", "1.0"]: self.assertFalse(os.path.exists(self.filename)) try: backend.SetWatcherPause(i, _filename=self.filename) except backend.RPCFail as err: self.assertEqual(str(err), "Duration must be numeric") else: self.fail("Did not raise exception") self.assertFalse(os.path.exists(self.filename)) def testSet(self): self.assertFalse(os.path.exists(self.filename)) for i in range(10): backend.SetWatcherPause(i, _filename=self.filename) self.assertEqual(utils.ReadFile(self.filename), "%s\n" % i) self.assertEqual(os.stat(self.filename).st_mode & 0o777, 0o644) class TestGetBlockDevSymlinkPath(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def _Test(self, name, idx): self.assertEqual(backend._GetBlockDevSymlinkPath(name, idx, _dir=self.tmpdir), ("%s/%s%s%s" % (self.tmpdir, name, constants.DISK_SEPARATOR, idx))) def test(self): for idx in range(100): self._Test("inst1.example.com", idx) class TestGetInstanceList(unittest.TestCase): def setUp(self): self._test_hv = self._TestHypervisor() self._test_hv.ListInstances = mock.Mock( return_value=["instance1", "instance2", "instance3"] ) class _TestHypervisor(hypervisor.hv_base.BaseHypervisor): def __init__(self): hypervisor.hv_base.BaseHypervisor.__init__(self) def _GetHypervisor(self, name): return self._test_hv def testHvparams(self): fake_hvparams = {} hvparams = {constants.HT_FAKE: fake_hvparams} backend.GetInstanceList([constants.HT_FAKE], all_hvparams=hvparams, get_hv_fn=self._GetHypervisor) self._test_hv.ListInstances.assert_called_with(hvparams=fake_hvparams) class TestInstanceConsoleInfo(unittest.TestCase): def setUp(self): self._test_hv_a = self._TestHypervisor() self._test_hv_a.GetInstanceConsole = mock.Mock( return_value = objects.InstanceConsole(instance="inst", kind="aHy") ) self._test_hv_b = self._TestHypervisor() self._test_hv_b.GetInstanceConsole = mock.Mock( return_value = objects.InstanceConsole(instance="inst", kind="bHy") ) class _TestHypervisor(hypervisor.hv_base.BaseHypervisor): def __init__(self): hypervisor.hv_base.BaseHypervisor.__init__(self) def _GetHypervisor(self, name): if name == "a": return self._test_hv_a else: return self._test_hv_b def testRightHypervisor(self): dictMaker = lambda hyName: { "instance":{"hypervisor":hyName}, "node":{}, "group":{}, "hvParams":{}, "beParams":{}, } call = { 'i1':dictMaker("a"), 'i2':dictMaker("b"), } res = backend.GetInstanceConsoleInfo(call, get_hv_fn=self._GetHypervisor) self.assertTrue(res["i1"]["kind"] == "aHy") self.assertTrue(res["i2"]["kind"] == "bHy") class TestGetHvInfo(unittest.TestCase): def setUp(self): self._test_hv = self._TestHypervisor() self._test_hv.GetNodeInfo = mock.Mock() class _TestHypervisor(hypervisor.hv_base.BaseHypervisor): def __init__(self): hypervisor.hv_base.BaseHypervisor.__init__(self) def _GetHypervisor(self, name): return self._test_hv def testGetHvInfoAllNone(self): result = backend._GetHvInfoAll(None) self.assertTrue(result is None) def testGetHvInfoAll(self): hvname = constants.HT_XEN_PVM hvparams = {} hv_specs = [(hvname, hvparams)] backend._GetHvInfoAll(hv_specs, self._GetHypervisor) self._test_hv.GetNodeInfo.assert_called_with(hvparams=hvparams) class TestApplyStorageInfoFunction(unittest.TestCase): _STORAGE_KEY = "some_key" _SOME_ARGS = ["some_args"] def setUp(self): self.mock_storage_fn = mock.Mock() def testApplyValidStorageType(self): storage_type = constants.ST_LVM_VG info_fn_orig = backend._STORAGE_TYPE_INFO_FN backend._STORAGE_TYPE_INFO_FN = { storage_type: self.mock_storage_fn } backend._ApplyStorageInfoFunction( storage_type, self._STORAGE_KEY, self._SOME_ARGS) self.mock_storage_fn.assert_called_with(self._STORAGE_KEY, self._SOME_ARGS) backend._STORAGE_TYPE_INFO_FN = info_fn_orig def testApplyInValidStorageType(self): storage_type = "invalid_storage_type" info_fn_orig = backend._STORAGE_TYPE_INFO_FN backend._STORAGE_TYPE_INFO_FN = {} self.assertRaises(KeyError, backend._ApplyStorageInfoFunction, storage_type, self._STORAGE_KEY, self._SOME_ARGS) backend._STORAGE_TYPE_INFO_FN = info_fn_orig def testApplyNotImplementedStorageType(self): storage_type = "not_implemented_storage_type" info_fn_orig = backend._STORAGE_TYPE_INFO_FN backend._STORAGE_TYPE_INFO_FN = {storage_type: None} self.assertRaises(NotImplementedError, backend._ApplyStorageInfoFunction, storage_type, self._STORAGE_KEY, self._SOME_ARGS) backend._STORAGE_TYPE_INFO_FN = info_fn_orig class TestGetLvmVgSpaceInfo(unittest.TestCase): def testValid(self): path = "somepath" excl_stor = True orig_fn = backend._GetVgInfo backend._GetVgInfo = mock.Mock() backend._GetLvmVgSpaceInfo(path, [excl_stor]) backend._GetVgInfo.assert_called_with(path, excl_stor) backend._GetVgInfo = orig_fn def testNoExclStorageNotBool(self): path = "somepath" excl_stor = "123" self.assertRaises(errors.ProgrammerError, backend._GetLvmVgSpaceInfo, path, [excl_stor]) def testNoExclStorageNotInList(self): path = "somepath" excl_stor = "123" self.assertRaises(errors.ProgrammerError, backend._GetLvmVgSpaceInfo, path, excl_stor) class TestGetLvmPvSpaceInfo(unittest.TestCase): def testValid(self): path = "somepath" excl_stor = True orig_fn = backend._GetVgSpindlesInfo backend._GetVgSpindlesInfo = mock.Mock() backend._GetLvmPvSpaceInfo(path, [excl_stor]) backend._GetVgSpindlesInfo.assert_called_with(path, excl_stor) backend._GetVgSpindlesInfo = orig_fn class TestCheckStorageParams(unittest.TestCase): def testParamsNone(self): self.assertRaises(errors.ProgrammerError, backend._CheckStorageParams, None, NotImplemented) def testParamsWrongType(self): self.assertRaises(errors.ProgrammerError, backend._CheckStorageParams, "string", NotImplemented) def testParamsEmpty(self): backend._CheckStorageParams([], 0) def testParamsValidNumber(self): backend._CheckStorageParams(["a", True], 2) def testParamsInvalidNumber(self): self.assertRaises(errors.ProgrammerError, backend._CheckStorageParams, ["b", False], 3) class TestGetVgSpindlesInfo(unittest.TestCase): def setUp(self): self.vg_free = 13 self.vg_size = 31 self.mock_fn = mock.Mock(return_value=(self.vg_free, self.vg_size)) def testValidInput(self): name = "myvg" excl_stor = True result = backend._GetVgSpindlesInfo(name, excl_stor, info_fn=self.mock_fn) self.mock_fn.assert_called_with(name) self.assertEqual(name, result["name"]) self.assertEqual(constants.ST_LVM_PV, result["type"]) self.assertEqual(self.vg_free, result["storage_free"]) self.assertEqual(self.vg_size, result["storage_size"]) def testNoExclStor(self): name = "myvg" excl_stor = False result = backend._GetVgSpindlesInfo(name, excl_stor, info_fn=self.mock_fn) self.mock_fn.assert_not_called() self.assertEqual(name, result["name"]) self.assertEqual(constants.ST_LVM_PV, result["type"]) self.assertEqual(0, result["storage_free"]) self.assertEqual(0, result["storage_size"]) class TestGetVgSpindlesInfo(unittest.TestCase): def testValidInput(self): self.vg_free = 13 self.vg_size = 31 self.mock_fn = mock.Mock(return_value=[(self.vg_free, self.vg_size)]) name = "myvg" excl_stor = True result = backend._GetVgInfo(name, excl_stor, info_fn=self.mock_fn) self.mock_fn.assert_called_with([name], excl_stor) self.assertEqual(name, result["name"]) self.assertEqual(constants.ST_LVM_VG, result["type"]) self.assertEqual(self.vg_free, result["storage_free"]) self.assertEqual(self.vg_size, result["storage_size"]) def testNoExclStor(self): name = "myvg" excl_stor = True self.mock_fn = mock.Mock(return_value=None) result = backend._GetVgInfo(name, excl_stor, info_fn=self.mock_fn) self.mock_fn.assert_called_with([name], excl_stor) self.assertEqual(name, result["name"]) self.assertEqual(constants.ST_LVM_VG, result["type"]) self.assertEqual(None, result["storage_free"]) self.assertEqual(None, result["storage_size"]) class TestGetNodeInfo(unittest.TestCase): _SOME_RESULT = None def testApplyStorageInfoFunction(self): orig_fn = backend._ApplyStorageInfoFunction backend._ApplyStorageInfoFunction = mock.Mock( return_value=self._SOME_RESULT) storage_units = [(st, st + "_key", [st + "_params"]) for st in constants.STORAGE_TYPES] backend.GetNodeInfo(storage_units, None) call_args_list = backend._ApplyStorageInfoFunction.call_args_list self.assertEqual(len(constants.STORAGE_TYPES), len(call_args_list)) for call in call_args_list: storage_type, storage_key, storage_params = call[0] self.assertEqual(storage_type + "_key", storage_key) self.assertEqual([storage_type + "_params"], storage_params) self.assertTrue(storage_type in constants.STORAGE_TYPES) backend._ApplyStorageInfoFunction = orig_fn class TestSpaceReportingConstants(unittest.TestCase): """Ensures consistency between STS_REPORT and backend. These tests ensure, that the constant 'STS_REPORT' is consistent with the implementation of invoking space reporting functions in backend.py. Once space reporting is available for all types, the constant can be removed and these tests as well. """ REPORTING = set(constants.STS_REPORT) NOT_REPORTING = set(constants.STORAGE_TYPES) - REPORTING def testAllReportingTypesHaveAReportingFunction(self): for storage_type in TestSpaceReportingConstants.REPORTING: self.assertTrue(backend._STORAGE_TYPE_INFO_FN[storage_type] is not None) def testAllNotReportingTypesDontHaveFunction(self): for storage_type in TestSpaceReportingConstants.NOT_REPORTING: self.assertEqual(None, backend._STORAGE_TYPE_INFO_FN[storage_type]) class TestAddRemoveGenerateNodeSshKey(testutils.GanetiTestCase): _CLUSTER_NAME = "mycluster" _SSH_PORT = 22 def setUp(self): self._ssh_file_manager = testutils_ssh.FakeSshFileManager() testutils.GanetiTestCase.setUp(self) self._ssh_add_authorized_patcher = testutils \ .patch_object(ssh, "AddAuthorizedKeys") self._ssh_remove_authorized_patcher = testutils \ .patch_object(ssh, "RemoveAuthorizedKeys") self._ssh_add_authorized_mock = self._ssh_add_authorized_patcher.start() self._ssh_add_authorized_mock.side_effect = \ self._ssh_file_manager.AddAuthorizedKeys self._ssconf_mock = mock.Mock() self._ssconf_mock.GetNodeList = mock.Mock() self._ssconf_mock.GetMasterNode = mock.Mock() self._ssconf_mock.GetClusterName = mock.Mock() self._ssconf_mock.GetOnlineNodeList = mock.Mock() self._ssconf_mock.GetSshPortMap = mock.Mock() self._run_cmd_mock = mock.Mock() self._run_cmd_mock.side_effect = self._ssh_file_manager.RunCommand self._ssh_remove_authorized_mock = \ self._ssh_remove_authorized_patcher.start() self._ssh_remove_authorized_mock.side_effect = \ self._ssh_file_manager.RemoveAuthorizedKeys self._ssh_add_public_key_patcher = testutils \ .patch_object(ssh, "AddPublicKey") self._ssh_add_public_key_mock = \ self._ssh_add_public_key_patcher.start() self._ssh_add_public_key_mock.side_effect = \ self._ssh_file_manager.AddPublicKey self._ssh_remove_public_key_patcher = testutils \ .patch_object(ssh, "RemovePublicKey") self._ssh_remove_public_key_mock = \ self._ssh_remove_public_key_patcher.start() self._ssh_remove_public_key_mock.side_effect = \ self._ssh_file_manager.RemovePublicKey self._ssh_query_pub_key_file_patcher = testutils \ .patch_object(ssh, "QueryPubKeyFile") self._ssh_query_pub_key_file_mock = \ self._ssh_query_pub_key_file_patcher.start() self._ssh_query_pub_key_file_mock.side_effect = \ self._ssh_file_manager.QueryPubKeyFile self._ssh_replace_name_by_uuid_patcher = testutils \ .patch_object(ssh, "ReplaceNameByUuid") self._ssh_replace_name_by_uuid_mock = \ self._ssh_replace_name_by_uuid_patcher.start() self._ssh_replace_name_by_uuid_mock.side_effect = \ self._ssh_file_manager.ReplaceNameByUuid self.noded_cert_file = testutils.TestDataFilename("cert1.pem") self._SetupTestData() def tearDown(self): super(testutils.GanetiTestCase, self).tearDown() self._ssh_add_authorized_patcher.stop() self._ssh_remove_authorized_patcher.stop() self._ssh_add_public_key_patcher.stop() self._ssh_remove_public_key_patcher.stop() self._ssh_query_pub_key_file_patcher.stop() self._ssh_replace_name_by_uuid_patcher.stop() self._TearDownTestData() def _SetupTestData(self, number_of_nodes=15, number_of_pot_mcs=5, number_of_mcs=5): """Sets up consistent test data for a cluster with a couple of nodes. """ self._pub_key_file = self._CreateTempFile() self._all_nodes = [] self._potential_master_candidates = [] self._master_candidate_uuids = [] self._ssconf_mock.reset_mock() self._ssconf_mock.GetNodeList.reset_mock() self._ssconf_mock.GetMasterNode.reset_mock() self._ssconf_mock.GetClusterName.reset_mock() self._ssconf_mock.GetOnlineNodeList.reset_mock() self._run_cmd_mock.reset_mock() self._ssh_file_manager.InitAllNodes(15, 10, 5) self._master_node = self._ssh_file_manager.GetMasterNodeName() self._ssconf_mock.GetSshPortMap.return_value = \ self._ssh_file_manager.GetSshPortMap(self._SSH_PORT) self._potential_master_candidates = \ self._ssh_file_manager.GetAllPotentialMasterCandidateNodeNames() self._master_candidate_uuids = \ self._ssh_file_manager.GetAllMasterCandidateUuids() self._all_nodes = self._ssh_file_manager.GetAllNodeNames() self._ssconf_mock.GetNodeList.side_effect = \ self._ssh_file_manager.GetAllNodeNames self._ssconf_mock.GetOnlineNodeList.side_effect = \ self._ssh_file_manager.GetAllNodeNames self._ssconf_mock.GetMasterNode.side_effect = \ self._ssh_file_manager.GetMasterNodeName def _TearDownTestData(self): os.remove(self._pub_key_file) def _GetCallsPerNode(self): calls_per_node = {} for (pos, keyword) in self._run_cmd_mock.call_args_list: (cluster_name, node, _, _, data) = pos if not node in calls_per_node: calls_per_node[node] = [] calls_per_node[node].append(data) return calls_per_node def testGenerateKey(self): test_node_name = "node_name_7" test_node_uuid = "node_uuid_7" self._SetupTestData() ssh.AddPublicKey(test_node_uuid, "some_old_key", key_file=self._pub_key_file) backend._GenerateNodeSshKey( test_node_uuid, test_node_name, self._ssh_file_manager.GetSshPortMap(self._SSH_PORT), "rsa", 2048, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) calls_per_node = self._GetCallsPerNode() for node, calls in calls_per_node.items(): self.assertEqual(node, test_node_name) for call in calls: self.assertTrue(constants.SSHS_GENERATE in call) def _AddNewNodeToTestData(self, name, uuid, key, pot_mc, mc, master): self._ssh_file_manager.SetOrAddNode(name, uuid, key, pot_mc, mc, master) if pot_mc: ssh.AddPublicKey(name, key, key_file=self._pub_key_file) self._potential_master_candidates.append(name) self._ssconf_mock.GetSshPortMap.return_value = \ self._ssh_file_manager.GetSshPortMap(self._SSH_PORT) def _GetNewMasterCandidate(self): """Returns the properties of a new master candidate node.""" return ("new_node_name", "new_node_uuid", "new_node_key", True, True, False) def _GetNewNumberedMasterCandidate(self, num): """Returns the properties of a new master candidate node.""" return ("new_node_name_%s" % num, "new_node_uuid_%s" % num, "new_node_key_%s" % num, True, True, False) def _GetNewNumberedPotentialMasterCandidate(self, num): """Returns the properties of a new potential master candidate node.""" return ("new_node_name_%s" % num, "new_node_uuid_%s" % num, "new_node_key_%s" % num, False, True, False) def _GetNewNumberedNormalNode(self, num): """Returns the properties of a new normal node.""" return ("new_node_name_%s" % num, "new_node_uuid_%s" % num, "new_node_key_%s" % num, False, False, False) def testAddMasterCandidate(self): (new_node_name, new_node_uuid, new_node_key, is_master_candidate, is_potential_master_candidate, is_master) = self._GetNewMasterCandidate() self._AddNewNodeToTestData( new_node_name, new_node_uuid, new_node_key, is_potential_master_candidate, is_master_candidate, is_master) backend.AddNodeSshKey(new_node_uuid, new_node_name, self._potential_master_candidates, to_authorized_keys=is_master_candidate, to_public_keys=is_potential_master_candidate, get_public_keys=is_potential_master_candidate, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertPotentialMasterCandidatesOnlyHavePublicKey( new_node_name) self._ssh_file_manager.AssertAllNodesHaveAuthorizedKey(new_node_key) def _SetupNodeBulk(self, num_nodes, node_fn): """Sets up the test data for a bulk of nodes. @param num_nodes: number of nodes @type num_nodes: integer @param node_fn: function @param node_fn: function to generate data of one node, taking an integer as only argument """ node_list = [] key_map = {} for i in range(num_nodes): (new_node_name, new_node_uuid, new_node_key, is_master_candidate, is_potential_master_candidate, is_master) = \ node_fn(i) self._AddNewNodeToTestData( new_node_name, new_node_uuid, new_node_key, is_potential_master_candidate, is_master_candidate, is_master) node_list.append( backend.SshAddNodeInfo( uuid=new_node_uuid, name=new_node_name, to_authorized_keys=is_master_candidate, to_public_keys=is_potential_master_candidate, get_public_keys=is_potential_master_candidate)) key_map[new_node_name] = new_node_key return (node_list, key_map) def testAddMasterCandidateBulk(self): num_nodes = 3 (node_list, key_map) = self._SetupNodeBulk( num_nodes, self._GetNewNumberedMasterCandidate) backend.AddNodeSshKeyBulk(node_list, self._potential_master_candidates, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) for node_info in node_list: self._ssh_file_manager.AssertPotentialMasterCandidatesOnlyHavePublicKey( node_info.name) self._ssh_file_manager.AssertAllNodesHaveAuthorizedKey( key_map[node_info.name]) def testAddPotentialMasterCandidateBulk(self): num_nodes = 3 (node_list, key_map) = self._SetupNodeBulk( num_nodes, self._GetNewNumberedPotentialMasterCandidate) backend.AddNodeSshKeyBulk(node_list, self._potential_master_candidates, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) for node_info in node_list: self._ssh_file_manager.AssertPotentialMasterCandidatesOnlyHavePublicKey( node_info.name) self._ssh_file_manager.AssertNoNodeHasAuthorizedKey( key_map[node_info.name]) def testAddPotentialMasterCandidate(self): new_node_name = "new_node_name" new_node_uuid = "new_node_uuid" new_node_key = "new_node_key" is_master_candidate = False is_potential_master_candidate = True is_master = False self._AddNewNodeToTestData( new_node_name, new_node_uuid, new_node_key, is_potential_master_candidate, is_master_candidate, is_master) backend.AddNodeSshKey(new_node_uuid, new_node_name, self._potential_master_candidates, to_authorized_keys=is_master_candidate, to_public_keys=is_potential_master_candidate, get_public_keys=is_potential_master_candidate, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertPotentialMasterCandidatesOnlyHavePublicKey( new_node_name) self._ssh_file_manager.AssertNoNodeHasAuthorizedKey(new_node_key) def testAddNormalNode(self): new_node_name = "new_node_name" new_node_uuid = "new_node_uuid" new_node_key = "new_node_key" is_master_candidate = False is_potential_master_candidate = False is_master = False self._AddNewNodeToTestData( new_node_name, new_node_uuid, new_node_key, is_potential_master_candidate, is_master_candidate, is_master) backend.AddNodeSshKey(new_node_uuid, new_node_name, self._potential_master_candidates, to_authorized_keys=is_master_candidate, to_public_keys=is_potential_master_candidate, get_public_keys=is_potential_master_candidate, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertNoNodeHasPublicKey(new_node_uuid, new_node_key) self._ssh_file_manager.AssertNoNodeHasAuthorizedKey(new_node_key) def testAddNormalBulk(self): num_nodes = 3 (node_list, key_map) = self._SetupNodeBulk( num_nodes, self._GetNewNumberedNormalNode) backend.AddNodeSshKeyBulk(node_list, self._potential_master_candidates, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) for node_info in node_list: self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, key_map[node_info.name]) self._ssh_file_manager.AssertNoNodeHasAuthorizedKey( key_map[node_info.name]) def _GetNewNumberedNode(self, num): """Returns the properties of a node. This will in round-robin style return a master candidate, a potential master candiate and a normal node. """ is_master_candidate = num % 3 == 0 is_potential_master_candidate = num % 3 == 0 or num % 3 == 1 is_master = False return ("new_node_name_%s" % num, "new_node_uuid_%s" % num, "new_node_key_%s" % num, is_master_candidate, is_potential_master_candidate, is_master) def testAddDiverseNodeBulk(self): """Tests adding keys of several nodes with several qualities. This tests subsumes previous tests. However, we leave the previous tests here, because debugging problems with this all-embracing test is much more tedious than having one of the one-purpose tests fail. """ num_nodes = 9 (node_list, key_map) = self._SetupNodeBulk( num_nodes, self._GetNewNumberedNode) backend.AddNodeSshKeyBulk(node_list, self._potential_master_candidates, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) for node_info in node_list: if node_info.to_authorized_keys: self._ssh_file_manager.AssertAllNodesHaveAuthorizedKey( key_map[node_info.name]) else: self._ssh_file_manager.AssertNoNodeHasAuthorizedKey( key_map[node_info.name]) if node_info.to_public_keys: self._ssh_file_manager.AssertPotentialMasterCandidatesOnlyHavePublicKey( node_info.name) else: self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, key_map[node_info.name]) def testPromoteToMasterCandidate(self): # Get one of the potential master candidates node_name, node_info = \ self._ssh_file_manager.GetAllPurePotentialMasterCandidates()[0] # Update it's role to master candidate in the test data self._ssh_file_manager.SetOrAddNode( node_name, node_info.uuid, node_info.key, node_info.is_potential_master_candidate, True, node_info.is_master) backend.AddNodeSshKey(node_info.uuid, node_name, self._potential_master_candidates, to_authorized_keys=True, to_public_keys=False, get_public_keys=False, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertPotentialMasterCandidatesOnlyHavePublicKey( node_name) self._ssh_file_manager.AssertAllNodesHaveAuthorizedKey(node_info.key) def testRemoveMasterCandidate(self): node_name, (node_uuid, node_key, is_potential_master_candidate, is_master_candidate, is_master) = \ self._ssh_file_manager.GetAllMasterCandidates()[0] backend.RemoveNodeSshKey(node_uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=True, from_public_keys=True, clear_authorized_keys=True, clear_public_keys=True, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertNoNodeHasPublicKey(node_uuid, node_key) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_name], node_key) self.assertEqual(0, len(self._ssh_file_manager.GetPublicKeysOfNode(node_name))) self.assertEqual(1, len(self._ssh_file_manager.GetAuthorizedKeysOfNode(node_name))) def testRemoveMasterCandidateBulk(self): node_list = [] key_map = {} for node_name, (node_uuid, node_key, _, _, _) in \ self._ssh_file_manager.GetAllMasterCandidates()[:3]: node_list.append(backend.SshRemoveNodeInfo(uuid=node_uuid, name=node_name, from_authorized_keys=True, from_public_keys=True, clear_authorized_keys=True, clear_public_keys=True)) key_map[node_name] = node_key backend.RemoveNodeSshKeyBulk(node_list, self._master_candidate_uuids, self._potential_master_candidates, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) for node_info in node_list: self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, key_map[node_info.name]) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_info.name], key_map[node_info.name]) self.assertEqual(0, len(self._ssh_file_manager.GetPublicKeysOfNode(node_info.name))) self.assertEqual(1, len(self._ssh_file_manager.GetAuthorizedKeysOfNode(node_info.name))) def testRemovePotentialMasterCandidate(self): (node_name, node_info) = \ self._ssh_file_manager.GetAllPurePotentialMasterCandidates()[0] backend.RemoveNodeSshKey(node_info.uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=False, from_public_keys=True, clear_authorized_keys=True, clear_public_keys=True, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, node_info.key) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_name], node_info.key) self.assertEqual(0, len(self._ssh_file_manager.GetPublicKeysOfNode(node_name))) self.assertEqual(1, len(self._ssh_file_manager.GetAuthorizedKeysOfNode(node_name))) def testRemovePotentialMasterCandidateBulk(self): node_list = [] key_map = {} for node_name, (node_uuid, node_key, _, _, _) in \ self._ssh_file_manager.GetAllPurePotentialMasterCandidates()[:3]: node_list.append(backend.SshRemoveNodeInfo(uuid=node_uuid, name=node_name, from_authorized_keys=False, from_public_keys=True, clear_authorized_keys=True, clear_public_keys=True)) key_map[node_name] = node_key backend.RemoveNodeSshKeyBulk(node_list, self._master_candidate_uuids, self._potential_master_candidates, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) for node_info in node_list: self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, key_map[node_info.name]) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_info.name], key_map[node_info.name]) self.assertEqual(0, len(self._ssh_file_manager.GetPublicKeysOfNode(node_info.name))) self.assertEqual(1, len(self._ssh_file_manager.GetAuthorizedKeysOfNode(node_info.name))) def testRemoveNormalNode(self): node_name, node_info = self._ssh_file_manager.GetAllNormalNodes()[0] backend.RemoveNodeSshKey(node_info.uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=False, from_public_keys=False, clear_authorized_keys=True, clear_public_keys=True, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, node_info.key) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_name], node_info.key) self.assertEqual(0, len(self._ssh_file_manager.GetPublicKeysOfNode(node_name))) self.assertEqual(1, len(self._ssh_file_manager.GetAuthorizedKeysOfNode(node_name))) def testRemoveNormalNodeBulk(self): node_list = [] key_map = {} for node_name, (node_uuid, node_key, _, _, _) in \ self._ssh_file_manager.GetAllNormalNodes()[:3]: node_list.append(backend.SshRemoveNodeInfo(uuid=node_uuid, name=node_name, from_authorized_keys=False, from_public_keys=False, clear_authorized_keys=True, clear_public_keys=True)) key_map[node_name] = node_key backend.RemoveNodeSshKeyBulk(node_list, self._master_candidate_uuids, self._potential_master_candidates, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) for node_info in node_list: self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, key_map[node_info.name]) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_info.name], key_map[node_info.name]) self.assertEqual(0, len(self._ssh_file_manager.GetPublicKeysOfNode(node_info.name))) self.assertEqual(1, len(self._ssh_file_manager.GetAuthorizedKeysOfNode(node_info.name))) def testRemoveDiverseNodesBulk(self): node_list = [] key_map = {} for node_name, (node_uuid, node_key, is_potential_master_candidate, is_master_candidate, _) in \ self._ssh_file_manager.GetAllNodesDiverse()[:3]: node_list.append(backend.SshRemoveNodeInfo( uuid=node_uuid, name=node_name, from_authorized_keys=is_master_candidate, from_public_keys=is_potential_master_candidate, clear_authorized_keys=True, clear_public_keys=True)) key_map[node_name] = node_key backend.RemoveNodeSshKeyBulk(node_list, self._master_candidate_uuids, self._potential_master_candidates, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) for node_info in node_list: self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, key_map[node_info.name]) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_info.name], key_map[node_info.name]) self.assertEqual(0, len(self._ssh_file_manager.GetPublicKeysOfNode(node_info.name))) self.assertEqual(1, len(self._ssh_file_manager.GetAuthorizedKeysOfNode(node_info.name))) def testDemoteMasterCandidateToPotentialMasterCandidate(self): node_name, node_info = self._ssh_file_manager.GetAllMasterCandidates()[0] self._ssh_file_manager.SetOrAddNode( node_name, node_info.uuid, node_info.key, node_info.is_potential_master_candidate, False, node_info.is_master) backend.RemoveNodeSshKey(node_info.uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=True, from_public_keys=False, clear_authorized_keys=False, clear_public_keys=False, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertPotentialMasterCandidatesOnlyHavePublicKey( node_name) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_name], node_info.key) def testDemotePotentialMasterCandidateToNormalNode(self): (node_name, node_info) = \ self._ssh_file_manager.GetAllPurePotentialMasterCandidates()[0] self._ssh_file_manager.SetOrAddNode( node_name, node_info.uuid, node_info.key, False, node_info.is_master_candidate, node_info.is_master) backend.RemoveNodeSshKey(node_info.uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=False, from_public_keys=True, clear_authorized_keys=False, clear_public_keys=False, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, node_info.key) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_name], node_info.key) def _GetReducedOnlineNodeList(self): """'Randomly' mark some nodes as offline.""" return [name for name in self._all_nodes if '3' not in name and '5' not in name] def testAddKeyWithOfflineNodes(self): (new_node_name, new_node_uuid, new_node_key, is_master_candidate, is_potential_master_candidate, is_master) = self._GetNewMasterCandidate() self._AddNewNodeToTestData( new_node_name, new_node_uuid, new_node_key, is_potential_master_candidate, is_master_candidate, is_master) self._online_nodes = self._GetReducedOnlineNodeList() self._ssconf_mock.GetOnlineNodeList.side_effect = \ lambda : self._online_nodes backend.AddNodeSshKey(new_node_uuid, new_node_name, self._potential_master_candidates, to_authorized_keys=is_master_candidate, to_public_keys=is_potential_master_candidate, get_public_keys=is_potential_master_candidate, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) for node in self._all_nodes: if node in self._online_nodes: self.assertTrue(self._ssh_file_manager.NodeHasAuthorizedKey( node, new_node_key)) else: self.assertFalse(self._ssh_file_manager.NodeHasAuthorizedKey( node, new_node_key)) def testRemoveKeyWithOfflineNodes(self): (node_name, node_info) = \ self._ssh_file_manager.GetAllMasterCandidates()[0] self._online_nodes = self._GetReducedOnlineNodeList() self._ssconf_mock.GetOnlineNodeList.side_effect = \ lambda : self._online_nodes backend.RemoveNodeSshKey(node_info.uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=True, from_public_keys=True, clear_authorized_keys=True, clear_public_keys=True, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) offline_nodes = [node for node in self._all_nodes if node not in self._online_nodes] self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( offline_nodes + [node_name], node_info.key) def testAddKeySuccessfullyOnNewNodeWithRetries(self): """Tests adding a new node's key when updating that node takes retries. This test checks whether adding a new node's key successfully updates the SSH key files of all nodes, even if updating the new node's key files itself takes a couple of retries to succeed. """ (new_node_name, new_node_uuid, new_node_key, is_master_candidate, is_potential_master_candidate, is_master) = self._GetNewMasterCandidate() self._AddNewNodeToTestData( new_node_name, new_node_uuid, new_node_key, is_potential_master_candidate, is_master_candidate, is_master) self._ssh_file_manager.SetMaxRetries( new_node_name, constants.SSHS_MAX_RETRIES) backend.AddNodeSshKey(new_node_uuid, new_node_name, self._potential_master_candidates, to_authorized_keys=is_master_candidate, to_public_keys=is_potential_master_candidate, get_public_keys=is_potential_master_candidate, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertPotentialMasterCandidatesOnlyHavePublicKey( new_node_name) self._ssh_file_manager.AssertAllNodesHaveAuthorizedKey( new_node_key) def testAddKeyFailedOnNewNodeWithRetries(self): """Tests clean up if updating a new node's SSH setup fails. If adding the keys of a new node fails, because updating the SSH key files of that new node fails, check whether already carried out operations are successfully rolled back and thus the state of the cluster is cleaned up. """ (new_node_name, new_node_uuid, new_node_key, is_master_candidate, is_potential_master_candidate, is_master) = self._GetNewMasterCandidate() self._AddNewNodeToTestData( new_node_name, new_node_uuid, new_node_key, is_potential_master_candidate, is_master_candidate, is_master) self._ssh_file_manager.SetMaxRetries( new_node_name, constants.SSHS_MAX_RETRIES + 1) self.assertRaises( errors.SshUpdateError, backend.AddNodeSshKey, new_node_uuid, new_node_name, self._potential_master_candidates, to_authorized_keys=is_master_candidate, to_public_keys=is_potential_master_candidate, get_public_keys=is_potential_master_candidate, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) master_node = self._ssh_file_manager.GetMasterNodeName() for node in self._all_nodes: if node in [new_node_name, master_node]: self.assertTrue(self._ssh_file_manager.NodeHasAuthorizedKey( node, new_node_key)) else: self.assertFalse(self._ssh_file_manager.NodeHasAuthorizedKey( node, new_node_key)) self._ssh_file_manager.AssertNoNodeHasPublicKey(new_node_uuid, new_node_key) def testAddKeySuccessfullyOnOldNodeWithRetries(self): """Tests adding a new key even if updating nodes takes retries. This tests whether adding a new node's key successfully finishes, even if one of the other cluster nodes takes a couple of retries to succeed. """ (new_node_name, new_node_uuid, new_node_key, is_master_candidate, is_potential_master_candidate, is_master) = self._GetNewMasterCandidate() other_node_name, _ = self._ssh_file_manager.GetAllMasterCandidates()[0] self._ssh_file_manager.SetMaxRetries( other_node_name, constants.SSHS_MAX_RETRIES) assert other_node_name != new_node_name self._AddNewNodeToTestData( new_node_name, new_node_uuid, new_node_key, is_potential_master_candidate, is_master_candidate, is_master) backend.AddNodeSshKey(new_node_uuid, new_node_name, self._potential_master_candidates, to_authorized_keys=is_master_candidate, to_public_keys=is_potential_master_candidate, get_public_keys=is_potential_master_candidate, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertAllNodesHaveAuthorizedKey(new_node_key) def testAddKeyFailedOnOldNodeWithRetries(self): """Tests adding keys when updating one node's SSH setup fails. This tests whether when adding a new node's key and one node is unreachable (but not marked as offline) the operation still finishes properly and only that unreachable node's SSH key setup did not get updated. """ (new_node_name, new_node_uuid, new_node_key, is_master_candidate, is_potential_master_candidate, is_master) = self._GetNewMasterCandidate() other_node_name, _ = self._ssh_file_manager.GetAllMasterCandidates()[0] self._ssh_file_manager.SetMaxRetries( other_node_name, constants.SSHS_MAX_RETRIES + 1) assert other_node_name != new_node_name self._AddNewNodeToTestData( new_node_name, new_node_uuid, new_node_key, is_potential_master_candidate, is_master_candidate, is_master) node_errors = backend.AddNodeSshKey( new_node_uuid, new_node_name, self._potential_master_candidates, to_authorized_keys=is_master_candidate, to_public_keys=is_potential_master_candidate, get_public_keys=is_potential_master_candidate, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) rest_nodes = [node for node in self._all_nodes if node != other_node_name] rest_nodes.append(new_node_name) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( rest_nodes, new_node_key) self.assertTrue([error_msg for (node, error_msg) in node_errors if node == other_node_name]) def testRemoveKeySuccessfullyWithRetriesOnOtherNode(self): """Test removing keys even if one of the old nodes needs retries. This tests checks whether a key can be removed successfully even when one of the other nodes needs to be contacted with several retries. """ all_master_candidates = self._ssh_file_manager.GetAllMasterCandidates() node_name, node_info = all_master_candidates[0] other_node_name, _ = all_master_candidates[1] assert node_name != self._master_node assert other_node_name != self._master_node assert node_name != other_node_name self._ssh_file_manager.SetMaxRetries( other_node_name, constants.SSHS_MAX_RETRIES) backend.RemoveNodeSshKey(node_info.uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=True, from_public_keys=True, clear_authorized_keys=True, clear_public_keys=True, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, node_info.key) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_name], node_info.key) def testRemoveKeyFailedWithRetriesOnOtherNode(self): """Test removing keys even if one of the old nodes fails even with retries. This tests checks whether the removal of a key finishes properly, even if the update of the key files on one of the other nodes fails despite several retries. """ all_master_candidates = self._ssh_file_manager.GetAllMasterCandidates() node_name, node_info = all_master_candidates[0] other_node_name, _ = all_master_candidates[1] assert node_name != self._master_node assert other_node_name != self._master_node assert node_name != other_node_name self._ssh_file_manager.SetMaxRetries( other_node_name, constants.SSHS_MAX_RETRIES + 1) error_msgs = backend.RemoveNodeSshKey( node_info.uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=True, from_public_keys=True, clear_authorized_keys=True, clear_public_keys=True, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [other_node_name, node_name], node_info.key) self.assertTrue([error_msg for (node, error_msg) in error_msgs if node == other_node_name]) def testRemoveKeySuccessfullyWithRetriesOnTargetNode(self): """Test removing keys even if the target nodes needs retries. This tests checks whether a key can be removed successfully even when removing the key on the node itself needs retries. """ all_master_candidates = self._ssh_file_manager.GetAllMasterCandidates() node_name, node_info = all_master_candidates[0] assert node_name != self._master_node self._ssh_file_manager.SetMaxRetries( node_name, constants.SSHS_MAX_RETRIES) backend.RemoveNodeSshKey(node_info.uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=True, from_public_keys=True, clear_authorized_keys=True, clear_public_keys=True, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertNoNodeHasPublicKey( node_info.uuid, node_info.key) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_name], node_info.key) def testRemoveKeyFailedWithRetriesOnTargetNode(self): """Test removing keys even if contacting the node fails with retries. This tests checks whether the removal of a key finishes properly, even if the update of the key files on the node itself fails despite several retries. """ all_master_candidates = self._ssh_file_manager.GetAllMasterCandidates() node_name, node_info = all_master_candidates[0] assert node_name != self._master_node self._ssh_file_manager.SetMaxRetries( node_name, constants.SSHS_MAX_RETRIES + 1) error_msgs = backend.RemoveNodeSshKey( node_info.uuid, node_name, self._master_candidate_uuids, self._potential_master_candidates, from_authorized_keys=True, from_public_keys=True, clear_authorized_keys=True, clear_public_keys=True, pub_key_file=self._pub_key_file, ssconf_store=self._ssconf_mock, noded_cert_file=self.noded_cert_file, run_cmd_fn=self._run_cmd_mock) self._ssh_file_manager.AssertNodeSetOnlyHasAuthorizedKey( [node_name], node_info.key) self.assertTrue([error_msg for (node, error_msg) in error_msgs if node == node_name]) class TestVerifySshSetup(testutils.GanetiTestCase): _NODE1_UUID = "uuid1" _NODE2_UUID = "uuid2" _NODE3_UUID = "uuid3" _NODE1_NAME = "name1" _NODE2_NAME = "name2" _NODE3_NAME = "name3" _NODE1_KEYS = ["key11"] _NODE2_KEYS = ["key21"] _NODE3_KEYS = ["key31"] _NODE_STATUS_LIST = [ (_NODE1_UUID, _NODE1_NAME, True, True, True), (_NODE2_UUID, _NODE2_NAME, False, True, True), (_NODE3_UUID, _NODE3_NAME, False, False, True), ] _PUB_KEY_RESULT = { _NODE1_UUID: _NODE1_KEYS, _NODE2_UUID: _NODE2_KEYS, _NODE3_UUID: _NODE3_KEYS, } _AUTH_RESULT = { _NODE1_KEYS[0]: True, _NODE2_KEYS[0]: False, _NODE3_KEYS[0]: False, } def setUp(self): testutils.GanetiTestCase.setUp(self) self._has_authorized_patcher = testutils \ .patch_object(ssh, "HasAuthorizedKey") self._has_authorized_mock = self._has_authorized_patcher.start() self._query_patcher = testutils \ .patch_object(ssh, "QueryPubKeyFile") self._query_mock = self._query_patcher.start() self._read_file_patcher = testutils \ .patch_object(utils, "ReadFile") self._read_file_mock = self._read_file_patcher.start() self._read_file_mock.return_value = self._NODE1_KEYS[0] self.tmpdir = tempfile.mkdtemp() self.pub_keys_file = os.path.join(self.tmpdir, "pub_keys_file") open(self.pub_keys_file, "w").close() def tearDown(self): super(testutils.GanetiTestCase, self).tearDown() self._has_authorized_patcher.stop() self._query_patcher.stop() self._read_file_patcher.stop() shutil.rmtree(self.tmpdir) def testValidData(self): self._has_authorized_mock.side_effect = \ lambda _, key : self._AUTH_RESULT[key] self._query_mock.return_value = self._PUB_KEY_RESULT result = backend._VerifySshSetup(self._NODE_STATUS_LIST, self._NODE1_NAME, "dsa", ganeti_pub_keys_file=self.pub_keys_file) self.assertEqual(result, []) def testMissingKey(self): self._has_authorized_mock.side_effect = \ lambda _, key : self._AUTH_RESULT[key] pub_key_missing = copy.deepcopy(self._PUB_KEY_RESULT) del pub_key_missing[self._NODE2_UUID] self._query_mock.return_value = pub_key_missing result = backend._VerifySshSetup(self._NODE_STATUS_LIST, self._NODE1_NAME, "dsa", ganeti_pub_keys_file=self.pub_keys_file) self.assertTrue(self._NODE2_UUID in result[0]) def testUnknownKey(self): self._has_authorized_mock.side_effect = \ lambda _, key : self._AUTH_RESULT[key] pub_key_missing = copy.deepcopy(self._PUB_KEY_RESULT) pub_key_missing["unkownnodeuuid"] = "pinkbunny" self._query_mock.return_value = pub_key_missing result = backend._VerifySshSetup(self._NODE_STATUS_LIST, self._NODE1_NAME, "dsa", ganeti_pub_keys_file=self.pub_keys_file) self.assertTrue("unkownnodeuuid" in result[0]) def testMissingMasterCandidate(self): auth_result = copy.deepcopy(self._AUTH_RESULT) auth_result["key11"] = False self._has_authorized_mock.side_effect = \ lambda _, key : auth_result[key] self._query_mock.return_value = self._PUB_KEY_RESULT result = backend._VerifySshSetup(self._NODE_STATUS_LIST, self._NODE1_NAME, "dsa", ganeti_pub_keys_file=self.pub_keys_file) self.assertTrue(self._NODE1_UUID in result[0]) def testSuperfluousNormalNode(self): auth_result = copy.deepcopy(self._AUTH_RESULT) auth_result["key31"] = True self._has_authorized_mock.side_effect = \ lambda _, key : auth_result[key] self._query_mock.return_value = self._PUB_KEY_RESULT result = backend._VerifySshSetup(self._NODE_STATUS_LIST, self._NODE1_NAME, "dsa", ganeti_pub_keys_file=self.pub_keys_file) self.assertTrue(self._NODE3_UUID in result[0]) class TestOSEnvironment(unittest.TestCase): """Ensure the presence of public and private parameters. They have to be present inside os environment variables. """ def _CreateEnv(self): """Create and return an environment.""" config_mock = ConfigMock() inst = config_mock.AddNewInstance( osparams={"public_param": "public_info"}, osparams_private=serializer.PrivateDict({"private_param": "private_info", "another_private_param": "more_privacy"}), nics = []) inst.disks_info = "" inst.secondary_nodes = [] return backend.OSEnvironment(inst, config_mock.CreateOs()) def testParamPresence(self): env = self._CreateEnv() env_keys = list(env) self.assertTrue("OSP_PUBLIC_PARAM" in env) self.assertTrue("OSP_PRIVATE_PARAM" in env) self.assertTrue("OSP_ANOTHER_PRIVATE_PARAM" in env) self.assertEqual("public_info", env["OSP_PUBLIC_PARAM"]) self.assertEqual("private_info", env["OSP_PRIVATE_PARAM"]) self.assertEqual("more_privacy", env["OSP_ANOTHER_PRIVATE_PARAM"]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.bootstrap_unittest.py000075500000000000000000000161321476477700300242610ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.bootstrap""" import shutil import tempfile import unittest from unittest import mock from ganeti import bootstrap from ganeti import constants from ganeti.storage import drbd from ganeti import errors from ganeti import pathutils import testutils class TestPrepareFileStorage(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def enableFileStorage(self, enable): self.enabled_disk_templates = [] if enable: self.enabled_disk_templates.append(constants.DT_FILE) else: # anything != DT_FILE would do here self.enabled_disk_templates.append(constants.DT_DISKLESS) def testFallBackToDefaultPathAcceptedFileStorageEnabled(self): expected_file_storage_dir = pathutils.DEFAULT_FILE_STORAGE_DIR acceptance_fn = mock.Mock() init_fn = mock.Mock(return_value=expected_file_storage_dir) self.enableFileStorage(True) file_storage_dir = bootstrap._PrepareFileStorage( self.enabled_disk_templates, None, acceptance_fn=acceptance_fn, init_fn=init_fn) self.assertEqual(expected_file_storage_dir, file_storage_dir) acceptance_fn.assert_called_with(expected_file_storage_dir) init_fn.assert_called_with(expected_file_storage_dir) def testPathAcceptedFileStorageEnabled(self): acceptance_fn = mock.Mock() init_fn = mock.Mock(return_value=self.tmpdir) self.enableFileStorage(True) file_storage_dir = bootstrap._PrepareFileStorage( self.enabled_disk_templates, self.tmpdir, acceptance_fn=acceptance_fn, init_fn=init_fn) self.assertEqual(self.tmpdir, file_storage_dir) acceptance_fn.assert_called_with(self.tmpdir) init_fn.assert_called_with(self.tmpdir) def testPathAcceptedFileStorageDisabled(self): acceptance_fn = mock.Mock() init_fn = mock.Mock() self.enableFileStorage(False) file_storage_dir = bootstrap._PrepareFileStorage( self.enabled_disk_templates, self.tmpdir, acceptance_fn=acceptance_fn, init_fn=init_fn) self.assertEqual(self.tmpdir, file_storage_dir) self.assertFalse(init_fn.called) self.assertFalse(acceptance_fn.called) def testPathNotAccepted(self): acceptance_fn = mock.Mock() acceptance_fn.side_effect = errors.FileStoragePathError init_fn = mock.Mock() self.enableFileStorage(True) self.assertRaises(errors.OpPrereqError, bootstrap._PrepareFileStorage, self.enabled_disk_templates, self.tmpdir, acceptance_fn=acceptance_fn, init_fn=init_fn) acceptance_fn.assert_called_with(self.tmpdir) class TestInitCheckEnabledDiskTemplates(unittest.TestCase): def testValidTemplates(self): enabled_disk_templates = list(constants.DISK_TEMPLATES) bootstrap._InitCheckEnabledDiskTemplates(enabled_disk_templates) def testInvalidTemplates(self): enabled_disk_templates = ["pinkbunny"] self.assertRaises(errors.OpPrereqError, bootstrap._InitCheckEnabledDiskTemplates, enabled_disk_templates) def testEmptyTemplates(self): enabled_disk_templates = [] self.assertRaises(errors.OpPrereqError, bootstrap._InitCheckEnabledDiskTemplates, enabled_disk_templates) class TestRestrictIpolicyToEnabledDiskTemplates(unittest.TestCase): def testNoRestriction(self): allowed_disk_templates = list(constants.DISK_TEMPLATES) ipolicy = {constants.IPOLICY_DTS: allowed_disk_templates} enabled_disk_templates = list(constants.DISK_TEMPLATES) bootstrap._RestrictIpolicyToEnabledDiskTemplates( ipolicy, enabled_disk_templates) self.assertCountEqual(ipolicy[constants.IPOLICY_DTS], allowed_disk_templates) def testRestriction(self): allowed_disk_templates = [constants.DT_DRBD8, constants.DT_PLAIN] ipolicy = {constants.IPOLICY_DTS: allowed_disk_templates} enabled_disk_templates = [constants.DT_PLAIN, constants.DT_FILE] bootstrap._RestrictIpolicyToEnabledDiskTemplates( ipolicy, enabled_disk_templates) self.assertEqual(ipolicy[constants.IPOLICY_DTS], [constants.DT_PLAIN]) class TestInitCheckDrbdHelper(unittest.TestCase): @testutils.patch_object(drbd.DRBD8, "GetUsermodeHelper") def testNoDrbd(self, drbd_mock_get_usermode_helper): drbd_enabled = False drbd_helper = None bootstrap._InitCheckDrbdHelper(drbd_helper, drbd_enabled) @testutils.patch_object(drbd.DRBD8, "GetUsermodeHelper") def testHelperNone(self, drbd_mock_get_usermode_helper): drbd_enabled = True current_helper = "/bin/helper" drbd_helper = None drbd_mock_get_usermode_helper.return_value = current_helper bootstrap._InitCheckDrbdHelper(drbd_helper, drbd_enabled) @testutils.patch_object(drbd.DRBD8, "GetUsermodeHelper") def testHelperOk(self, drbd_mock_get_usermode_helper): drbd_enabled = True current_helper = "/bin/helper" drbd_helper = "/bin/helper" drbd_mock_get_usermode_helper.return_value = current_helper bootstrap._InitCheckDrbdHelper(drbd_helper, drbd_enabled) @testutils.patch_object(drbd.DRBD8, "GetUsermodeHelper") def testWrongHelper(self, drbd_mock_get_usermode_helper): drbd_enabled = True current_helper = "/bin/otherhelper" drbd_helper = "/bin/helper" drbd_mock_get_usermode_helper.return_value = current_helper self.assertRaises(errors.OpPrereqError, bootstrap._InitCheckDrbdHelper, drbd_helper, drbd_enabled) @testutils.patch_object(drbd.DRBD8, "GetUsermodeHelper") def testHelperCheckFails(self, drbd_mock_get_usermode_helper): drbd_enabled = True drbd_helper = "/bin/helper" drbd_mock_get_usermode_helper.side_effect=errors.BlockDeviceError self.assertRaises(errors.OpPrereqError, bootstrap._InitCheckDrbdHelper, drbd_helper, drbd_enabled) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.cli_opts_unittest.py000064400000000000000000000137301476477700300240560ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the cli module""" import copy import testutils import time import unittest import yaml from io import StringIO from ganeti import constants from ganeti import cli_opts from ganeti import errors from ganeti import utils from ganeti import objects from ganeti import qlang from ganeti.errors import OpPrereqError, ParameterError class TestSplitKeyVal(unittest.TestCase): """Testing case for cli_opts._SplitKeyVal""" DATA = "a=b,c,no_d,-e" RESULT = {"a": "b", "c": True, "d": False, "e": None} RESULT_NOPREFIX = {"a": "b", "c": {}, "no_d": {}, "-e": {}} def testSplitKeyVal(self): """Test splitting""" self.assertEqual(cli_opts._SplitKeyVal("option", self.DATA, True), self.RESULT) def testDuplicateParam(self): """Test duplicate parameters""" for data in ("a=1,a=2", "a,no_a"): self.assertRaises(ParameterError, cli_opts._SplitKeyVal, "option", data, True) def testEmptyData(self): """Test how we handle splitting an empty string""" self.assertEqual(cli_opts._SplitKeyVal("option", "", True), {}) class TestIdentKeyVal(unittest.TestCase): """Testing case for cli_opts.check_ident_key_val""" def testIdentKeyVal(self): """Test identkeyval""" def cikv(value): return cli_opts.check_ident_key_val("option", "opt", value) self.assertEqual(cikv("foo:bar"), ("foo", {"bar": True})) self.assertEqual(cikv("foo:bar=baz"), ("foo", {"bar": "baz"})) self.assertEqual(cikv("bar:b=c,c=a"), ("bar", {"b": "c", "c": "a"})) self.assertEqual(cikv("no_bar"), ("bar", False)) self.assertRaises(ParameterError, cikv, "no_bar:foo") self.assertRaises(ParameterError, cikv, "no_bar:foo=baz") self.assertRaises(ParameterError, cikv, "bar:foo=baz,foo=baz") self.assertEqual(cikv("-foo"), ("foo", None)) self.assertRaises(ParameterError, cikv, "-foo:a=c") # Check negative numbers self.assertEqual(cikv("-1:remove"), ("-1", { "remove": True, })) self.assertEqual(cikv("-29447:add,size=4G"), ("-29447", { "add": True, "size": "4G", })) for i in ["-:", "-"]: self.assertEqual(cikv(i), ("", None)) @staticmethod def _csikv(value): return cli_opts._SplitIdentKeyVal("opt", value, False) def testIdentKeyValNoPrefix(self): """Test identkeyval without prefixes""" test_cases = [ ("foo:bar", None), ("foo:no_bar", None), ("foo:bar=baz,bar=baz", None), ("foo", ("foo", {})), ("foo:bar=baz", ("foo", {"bar": "baz"})), ("no_foo:-1=baz,no_op=3", ("no_foo", {"-1": "baz", "no_op": "3"})), ] for (arg, res) in test_cases: if res is None: self.assertRaises(ParameterError, self._csikv, arg) else: self.assertEqual(self._csikv(arg), res) class TestMultilistIdentKeyVal(unittest.TestCase): """Test for cli_opts.check_multilist_ident_key_val()""" @staticmethod def _cmikv(value): return cli_opts.check_multilist_ident_key_val("option", "opt", value) def testListIdentKeyVal(self): test_cases = [ ("", None), ("foo", [ {"foo": {}} ]), ("foo:bar=baz", [ {"foo": {"bar": "baz"}} ]), ("foo:bar=baz/foo:bat=bad", None), ("foo:abc=42/bar:def=11", [ {"foo": {"abc": "42"}, "bar": {"def": "11"}} ]), ("foo:abc=42/bar:def=11,ghi=07", [ {"foo": {"abc": "42"}, "bar": {"def": "11", "ghi": "07"}} ]), ("foo:abc=42/bar:def=11//", None), ("foo:abc=42/bar:def=11,ghi=07//foobar", [ {"foo": {"abc": "42"}, "bar": {"def": "11", "ghi": "07"}}, {"foobar": {}} ]), ("foo:abc=42/bar:def=11,ghi=07//foobar:xyz=88", [ {"foo": {"abc": "42"}, "bar": {"def": "11", "ghi": "07"}}, {"foobar": {"xyz": "88"}} ]), ("foo:abc=42/bar:def=11,ghi=07//foobar:xyz=88/foo:uvw=314", [ {"foo": {"abc": "42"}, "bar": {"def": "11", "ghi": "07"}}, {"foobar": {"xyz": "88"}, "foo": {"uvw": "314"}} ]), ] for (arg, res) in test_cases: if res is None: self.assertRaises(ParameterError, self._cmikv, arg) else: self.assertEqual(res, self._cmikv(arg)) class TestConstants(unittest.TestCase): def testPriority(self): self.assertEqual(set(cli_opts._PRIONAME_TO_VALUE.values()), set(constants.OP_PRIO_SUBMIT_VALID)) self.assertEqual(list(value for _, value in cli_opts._PRIORITY_NAMES), sorted(constants.OP_PRIO_SUBMIT_VALID, reverse=True)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.cli_unittest.py000075500000000000000000001601711476477700300230160ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2008, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the cli module""" import copy import testutils import time import unittest import yaml from io import StringIO from ganeti import constants from ganeti import cli from ganeti import errors from ganeti import utils from ganeti import objects from ganeti import qlang from ganeti.errors import OpPrereqError, ParameterError class TestParseTimespec(unittest.TestCase): """Testing case for ParseTimespec""" def testValidTimes(self): """Test valid timespecs""" test_data = [ ("1s", 1), ("1", 1), ("1m", 60), ("1h", 60 * 60), ("1d", 60 * 60 * 24), ("1w", 60 * 60 * 24 * 7), ("4h", 4 * 60 * 60), ("61m", 61 * 60), ] for value, expected_result in test_data: self.assertEqual(cli.ParseTimespec(value), expected_result) def testInvalidTime(self): """Test invalid timespecs""" test_data = [ "1y", "", "aaa", "s", ] for value in test_data: self.assertRaises(OpPrereqError, cli.ParseTimespec, value) class TestToStream(unittest.TestCase): """Test the ToStream functions""" def testBasic(self): for data in ["foo", "foo %s", "foo %(test)s", "foo %s %s", "", ]: buf = StringIO() cli._ToStream(buf, data) self.assertEqual(buf.getvalue(), data + "\n") def testParams(self): buf = StringIO() cli._ToStream(buf, "foo %s", 1) self.assertEqual(buf.getvalue(), "foo 1\n") buf = StringIO() cli._ToStream(buf, "foo %s", (15,16)) self.assertEqual(buf.getvalue(), "foo (15, 16)\n") buf = StringIO() cli._ToStream(buf, "foo %s %s", "a", "b") self.assertEqual(buf.getvalue(), "foo a b\n") class TestGenerateTable(unittest.TestCase): HEADERS = dict([("f%s" % i, "Field%s" % i) for i in range(5)]) FIELDS1 = ["f1", "f2"] DATA1 = [ ["abc", 1234], ["foobar", 56], ["b", -14], ] def _test(self, headers, fields, separator, data, numfields, unitfields, units, expected): table = cli.GenerateTable(headers, fields, separator, data, numfields=numfields, unitfields=unitfields, units=units) self.assertEqual(table, expected) def testPlain(self): exp = [ "Field1 Field2", "abc 1234", "foobar 56", "b -14", ] self._test(self.HEADERS, self.FIELDS1, None, self.DATA1, None, None, "m", exp) def testNoFields(self): self._test(self.HEADERS, [], None, [[], []], None, None, "m", ["", "", ""]) self._test(None, [], None, [[], []], None, None, "m", ["", ""]) def testSeparator(self): for sep in ["#", ":", ",", "^", "!", "%", "|", "###", "%%", "!!!", "||"]: exp = [ "Field1%sField2" % sep, "abc%s1234" % sep, "foobar%s56" % sep, "b%s-14" % sep, ] self._test(self.HEADERS, self.FIELDS1, sep, self.DATA1, None, None, "m", exp) def testNoHeader(self): exp = [ "abc 1234", "foobar 56", "b -14", ] self._test(None, self.FIELDS1, None, self.DATA1, None, None, "m", exp) def testUnknownField(self): headers = { "f1": "Field1", } exp = [ "Field1 UNKNOWN", "abc 1234", "foobar 56", "b -14", ] self._test(headers, ["f1", "UNKNOWN"], None, self.DATA1, None, None, "m", exp) def testNumfields(self): fields = ["f1", "f2", "f3"] data = [ ["abc", 1234, 0], ["foobar", 56, 3], ["b", -14, "-"], ] exp = [ "Field1 Field2 Field3", "abc 1234 0", "foobar 56 3", "b -14 -", ] self._test(self.HEADERS, fields, None, data, ["f2", "f3"], None, "m", exp) def testUnitfields(self): expnosep = [ "Field1 Field2 Field3", "abc 1234 0M", "foobar 56 3M", "b -14 -", ] expsep = [ "Field1:Field2:Field3", "abc:1234:0M", "foobar:56:3M", "b:-14:-", ] for sep, expected in [(None, expnosep), (":", expsep)]: fields = ["f1", "f2", "f3"] data = [ ["abc", 1234, 0], ["foobar", 56, 3], ["b", -14, "-"], ] self._test(self.HEADERS, fields, sep, data, ["f2", "f3"], ["f3"], "h", expected) def testUnusual(self): data = [ ["%", "xyz"], ["%%", "abc"], ] exp = [ "Field1 Field2", "% xyz", "%% abc", ] self._test(self.HEADERS, ["f1", "f2"], None, data, None, None, "m", exp) class TestFormatQueryResult(unittest.TestCase): def test(self): fields = [ objects.QueryFieldDefinition(name="name", title="Name", kind=constants.QFT_TEXT), objects.QueryFieldDefinition(name="size", title="Size", kind=constants.QFT_NUMBER), objects.QueryFieldDefinition(name="act", title="Active", kind=constants.QFT_BOOL), objects.QueryFieldDefinition(name="mem", title="Memory", kind=constants.QFT_UNIT), objects.QueryFieldDefinition(name="other", title="SomeList", kind=constants.QFT_OTHER), ] response = objects.QueryResponse(fields=fields, data=[ [(constants.RS_NORMAL, "nodeA"), (constants.RS_NORMAL, 128), (constants.RS_NORMAL, False), (constants.RS_NORMAL, 1468006), (constants.RS_NORMAL, [])], [(constants.RS_NORMAL, "other"), (constants.RS_NORMAL, 512), (constants.RS_NORMAL, True), (constants.RS_NORMAL, 16), (constants.RS_NORMAL, [1, 2, 3])], [(constants.RS_NORMAL, "xyz"), (constants.RS_NORMAL, 1024), (constants.RS_NORMAL, True), (constants.RS_NORMAL, 4096), (constants.RS_NORMAL, [{}, {}])], ]) self.assertEqual(cli.FormatQueryResult(response, unit="h", header=True), (cli.QR_NORMAL, [ "Name Size Active Memory SomeList", "nodeA 128 N 1.4T []", "other 512 Y 16M [1, 2, 3]", "xyz 1024 Y 4.0G [{}, {}]", ])) def testTimestampAndUnit(self): fields = [ objects.QueryFieldDefinition(name="name", title="Name", kind=constants.QFT_TEXT), objects.QueryFieldDefinition(name="size", title="Size", kind=constants.QFT_UNIT), objects.QueryFieldDefinition(name="mtime", title="ModTime", kind=constants.QFT_TIMESTAMP), ] response = objects.QueryResponse(fields=fields, data=[ [(constants.RS_NORMAL, "a"), (constants.RS_NORMAL, 1024), (constants.RS_NORMAL, 0)], [(constants.RS_NORMAL, "b"), (constants.RS_NORMAL, 144996), (constants.RS_NORMAL, 1291746295)], ]) self.assertEqual(cli.FormatQueryResult(response, unit="m", header=True), (cli.QR_NORMAL, [ "Name Size ModTime", "a 1024 %s" % utils.FormatTime(0), "b 144996 %s" % utils.FormatTime(1291746295), ])) def testOverride(self): fields = [ objects.QueryFieldDefinition(name="name", title="Name", kind=constants.QFT_TEXT), objects.QueryFieldDefinition(name="cust", title="Custom", kind=constants.QFT_OTHER), objects.QueryFieldDefinition(name="xt", title="XTime", kind=constants.QFT_TIMESTAMP), ] response = objects.QueryResponse(fields=fields, data=[ [(constants.RS_NORMAL, "x"), (constants.RS_NORMAL, ["a", "b", "c"]), (constants.RS_NORMAL, 1234)], [(constants.RS_NORMAL, "y"), (constants.RS_NORMAL, range(10)), (constants.RS_NORMAL, 1291746295)], ]) override = { "cust": (utils.CommaJoin, False), "xt": (hex, True), } self.assertEqual(cli.FormatQueryResult(response, unit="h", header=True, format_override=override), (cli.QR_NORMAL, [ "Name Custom XTime", "x a, b, c 0x4d2", "y 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 0x4cfe7bf7", ])) def testSeparator(self): fields = [ objects.QueryFieldDefinition(name="name", title="Name", kind=constants.QFT_TEXT), objects.QueryFieldDefinition(name="count", title="Count", kind=constants.QFT_NUMBER), objects.QueryFieldDefinition(name="desc", title="Description", kind=constants.QFT_TEXT), ] response = objects.QueryResponse(fields=fields, data=[ [(constants.RS_NORMAL, "instance1.example.com"), (constants.RS_NORMAL, 21125), (constants.RS_NORMAL, "Hello World!")], [(constants.RS_NORMAL, "mail.other.net"), (constants.RS_NORMAL, -9000), (constants.RS_NORMAL, "a,b,c")], ]) for sep in [":", "|", "#", "|||", "###", "@@@", "@#@"]: for header in [None, "Name%sCount%sDescription" % (sep, sep)]: exp = [] if header: exp.append(header) exp.extend([ "instance1.example.com%s21125%sHello World!" % (sep, sep), "mail.other.net%s-9000%sa,b,c" % (sep, sep), ]) self.assertEqual(cli.FormatQueryResult(response, separator=sep, header=bool(header)), (cli.QR_NORMAL, exp)) def testStatusWithUnknown(self): fields = [ objects.QueryFieldDefinition(name="id", title="ID", kind=constants.QFT_NUMBER), objects.QueryFieldDefinition(name="unk", title="unk", kind=constants.QFT_UNKNOWN), objects.QueryFieldDefinition(name="unavail", title="Unavail", kind=constants.QFT_BOOL), objects.QueryFieldDefinition(name="nodata", title="NoData", kind=constants.QFT_TEXT), objects.QueryFieldDefinition(name="offline", title="OffLine", kind=constants.QFT_TEXT), ] response = objects.QueryResponse(fields=fields, data=[ [(constants.RS_NORMAL, 1), (constants.RS_UNKNOWN, None), (constants.RS_NORMAL, False), (constants.RS_NORMAL, ""), (constants.RS_OFFLINE, None)], [(constants.RS_NORMAL, 2), (constants.RS_UNKNOWN, None), (constants.RS_NODATA, None), (constants.RS_NORMAL, "x"), (constants.RS_OFFLINE, None)], [(constants.RS_NORMAL, 3), (constants.RS_UNKNOWN, None), (constants.RS_NORMAL, False), (constants.RS_UNAVAIL, None), (constants.RS_OFFLINE, None)], ]) self.assertEqual(cli.FormatQueryResult(response, header=True, separator="|", verbose=True), (cli.QR_UNKNOWN, [ "ID|unk|Unavail|NoData|OffLine", "1|(unknown)|N||(offline)", "2|(unknown)|(nodata)|x|(offline)", "3|(unknown)|N|(unavail)|(offline)", ])) self.assertEqual(cli.FormatQueryResult(response, header=True, separator="|", verbose=False), (cli.QR_UNKNOWN, [ "ID|unk|Unavail|NoData|OffLine", "1|??|N||*", "2|??|?|x|*", "3|??|N|-|*", ])) def testNoData(self): fields = [ objects.QueryFieldDefinition(name="id", title="ID", kind=constants.QFT_NUMBER), objects.QueryFieldDefinition(name="name", title="Name", kind=constants.QFT_TEXT), ] response = objects.QueryResponse(fields=fields, data=[]) self.assertEqual(cli.FormatQueryResult(response, header=True), (cli.QR_NORMAL, ["ID Name"])) def testNoDataWithUnknown(self): fields = [ objects.QueryFieldDefinition(name="id", title="ID", kind=constants.QFT_NUMBER), objects.QueryFieldDefinition(name="unk", title="unk", kind=constants.QFT_UNKNOWN), ] response = objects.QueryResponse(fields=fields, data=[]) self.assertEqual(cli.FormatQueryResult(response, header=False), (cli.QR_UNKNOWN, [])) def testStatus(self): fields = [ objects.QueryFieldDefinition(name="id", title="ID", kind=constants.QFT_NUMBER), objects.QueryFieldDefinition(name="unavail", title="Unavail", kind=constants.QFT_BOOL), objects.QueryFieldDefinition(name="nodata", title="NoData", kind=constants.QFT_TEXT), objects.QueryFieldDefinition(name="offline", title="OffLine", kind=constants.QFT_TEXT), ] response = objects.QueryResponse(fields=fields, data=[ [(constants.RS_NORMAL, 1), (constants.RS_NORMAL, False), (constants.RS_NORMAL, ""), (constants.RS_OFFLINE, None)], [(constants.RS_NORMAL, 2), (constants.RS_NODATA, None), (constants.RS_NORMAL, "x"), (constants.RS_NORMAL, "abc")], [(constants.RS_NORMAL, 3), (constants.RS_NORMAL, False), (constants.RS_UNAVAIL, None), (constants.RS_OFFLINE, None)], ]) self.assertEqual(cli.FormatQueryResult(response, header=False, separator="|", verbose=True), (cli.QR_INCOMPLETE, [ "1|N||(offline)", "2|(nodata)|x|abc", "3|N|(unavail)|(offline)", ])) self.assertEqual(cli.FormatQueryResult(response, header=False, separator="|", verbose=False), (cli.QR_INCOMPLETE, [ "1|N||*", "2|?|x|abc", "3|N|-|*", ])) def testInvalidFieldType(self): fields = [ objects.QueryFieldDefinition(name="x", title="x", kind="#some#other#type"), ] response = objects.QueryResponse(fields=fields, data=[]) self.assertRaises(NotImplementedError, cli.FormatQueryResult, response) def testInvalidFieldStatus(self): fields = [ objects.QueryFieldDefinition(name="x", title="x", kind=constants.QFT_TEXT), ] response = objects.QueryResponse(fields=fields, data=[[(-1, None)]]) self.assertRaises(NotImplementedError, cli.FormatQueryResult, response) response = objects.QueryResponse(fields=fields, data=[[(-1, "x")]]) self.assertRaises(AssertionError, cli.FormatQueryResult, response) def testEmptyFieldTitle(self): fields = [ objects.QueryFieldDefinition(name="x", title="", kind=constants.QFT_TEXT), ] response = objects.QueryResponse(fields=fields, data=[]) self.assertRaises(AssertionError, cli.FormatQueryResult, response) class _MockJobPollCb(cli.JobPollCbBase, cli.JobPollReportCbBase): def __init__(self, tc, job_id): self.tc = tc self.job_id = job_id self._wfjcr = [] self._jobstatus = [] self._expect_notchanged = False self._expect_log = [] def CheckEmpty(self): self.tc.assertFalse(self._wfjcr) self.tc.assertFalse(self._jobstatus) self.tc.assertFalse(self._expect_notchanged) self.tc.assertFalse(self._expect_log) def AddWfjcResult(self, *args): self._wfjcr.append(args) def AddQueryJobsResult(self, *args): self._jobstatus.append(args) def WaitForJobChangeOnce(self, job_id, fields, prev_job_info, prev_log_serial, timeout=constants.DEFAULT_WFJC_TIMEOUT): self.tc.assertEqual(job_id, self.job_id) self.tc.assertEqualValues(fields, ["status"]) self.tc.assertFalse(self._expect_notchanged) self.tc.assertFalse(self._expect_log) (exp_prev_job_info, exp_prev_log_serial, result) = self._wfjcr.pop(0) self.tc.assertEqualValues(prev_job_info, exp_prev_job_info) self.tc.assertEqual(prev_log_serial, exp_prev_log_serial) if result == constants.JOB_NOTCHANGED: self._expect_notchanged = True elif result: (_, logmsgs) = result if logmsgs: self._expect_log.extend(logmsgs) return result def QueryJobs(self, job_ids, fields): self.tc.assertEqual(job_ids, [self.job_id]) self.tc.assertEqualValues(fields, ["status", "opstatus", "opresult"]) self.tc.assertFalse(self._expect_notchanged) self.tc.assertFalse(self._expect_log) result = self._jobstatus.pop(0) self.tc.assertEqual(len(fields), len(result)) return [result] def CancelJob(self, job_id): self.tc.assertEqual(job_id, self.job_id) def ReportLogMessage(self, job_id, serial, timestamp, log_type, log_msg): self.tc.assertEqual(job_id, self.job_id) self.tc.assertEqualValues((serial, timestamp, log_type, log_msg), self._expect_log.pop(0)) def ReportNotChanged(self, job_id, status): self.tc.assertEqual(job_id, self.job_id) self.tc.assertTrue(self._expect_notchanged) self._expect_notchanged = False class TestGenericPollJob(testutils.GanetiTestCase): def testSuccessWithLog(self): job_id = 29609 cbs = _MockJobPollCb(self, job_id) cbs.AddWfjcResult(None, None, constants.JOB_NOTCHANGED) cbs.AddWfjcResult(None, None, ((constants.JOB_STATUS_QUEUED, ), None)) cbs.AddWfjcResult((constants.JOB_STATUS_QUEUED, ), None, constants.JOB_NOTCHANGED) cbs.AddWfjcResult((constants.JOB_STATUS_QUEUED, ), None, ((constants.JOB_STATUS_RUNNING, ), [(1, utils.SplitTime(1273491611.0), constants.ELOG_MESSAGE, "Step 1"), (2, utils.SplitTime(1273491615.9), constants.ELOG_MESSAGE, "Step 2"), (3, utils.SplitTime(1273491625.02), constants.ELOG_MESSAGE, "Step 3"), (4, utils.SplitTime(1273491635.05), constants.ELOG_MESSAGE, "Step 4"), (37, utils.SplitTime(1273491645.0), constants.ELOG_MESSAGE, "Step 5"), (203, utils.SplitTime(127349155.0), constants.ELOG_MESSAGE, "Step 6")])) cbs.AddWfjcResult((constants.JOB_STATUS_RUNNING, ), 203, ((constants.JOB_STATUS_RUNNING, ), [(300, utils.SplitTime(1273491711.01), constants.ELOG_MESSAGE, "Step X"), (302, utils.SplitTime(1273491815.8), constants.ELOG_MESSAGE, "Step Y"), (303, utils.SplitTime(1273491925.32), constants.ELOG_MESSAGE, "Step Z")])) cbs.AddWfjcResult((constants.JOB_STATUS_RUNNING, ), 303, ((constants.JOB_STATUS_SUCCESS, ), None)) cbs.AddQueryJobsResult(constants.JOB_STATUS_SUCCESS, [constants.OP_STATUS_SUCCESS, constants.OP_STATUS_SUCCESS], ["Hello World", "Foo man bar"]) self.assertEqual(["Hello World", "Foo man bar"], cli.GenericPollJob(job_id, cbs, cbs)) cbs.CheckEmpty() def testJobLost(self): job_id = 13746 cbs = _MockJobPollCb(self, job_id) cbs.AddWfjcResult(None, None, constants.JOB_NOTCHANGED) cbs.AddWfjcResult(None, None, None) self.assertRaises(errors.JobLost, cli.GenericPollJob, job_id, cbs, cbs) cbs.CheckEmpty() def testError(self): job_id = 31088 cbs = _MockJobPollCb(self, job_id) cbs.AddWfjcResult(None, None, constants.JOB_NOTCHANGED) cbs.AddWfjcResult(None, None, ((constants.JOB_STATUS_ERROR, ), None)) cbs.AddQueryJobsResult(constants.JOB_STATUS_ERROR, [constants.OP_STATUS_SUCCESS, constants.OP_STATUS_ERROR], ["Hello World", "Error code 123"]) self.assertRaises(errors.OpExecError, cli.GenericPollJob, job_id, cbs, cbs) cbs.CheckEmpty() def testError2(self): job_id = 22235 cbs = _MockJobPollCb(self, job_id) cbs.AddWfjcResult(None, None, ((constants.JOB_STATUS_ERROR, ), None)) encexc = errors.EncodeException(errors.LockError("problem")) cbs.AddQueryJobsResult(constants.JOB_STATUS_ERROR, [constants.OP_STATUS_ERROR], [encexc]) self.assertRaises(errors.LockError, cli.GenericPollJob, job_id, cbs, cbs) cbs.CheckEmpty() def testWeirdError(self): job_id = 28847 cbs = _MockJobPollCb(self, job_id) cbs.AddWfjcResult(None, None, ((constants.JOB_STATUS_ERROR, ), None)) cbs.AddQueryJobsResult(constants.JOB_STATUS_ERROR, [constants.OP_STATUS_RUNNING, constants.OP_STATUS_RUNNING], [None, None]) self.assertRaises(errors.OpExecError, cli.GenericPollJob, job_id, cbs, cbs) cbs.CheckEmpty() def testCancel(self): job_id = 4275 cbs = _MockJobPollCb(self, job_id) cbs.AddWfjcResult(None, None, constants.JOB_NOTCHANGED) cbs.AddWfjcResult(None, None, ((constants.JOB_STATUS_CANCELING, ), None)) cbs.AddQueryJobsResult(constants.JOB_STATUS_CANCELING, [constants.OP_STATUS_CANCELING, constants.OP_STATUS_CANCELING], [None, None]) self.assertRaises(errors.JobCanceled, cli.GenericPollJob, job_id, cbs, cbs) cbs.CheckEmpty() def testNegativeUpdateFreqParameter(self): job_id = 12345 cbs = _MockJobPollCb(self, job_id) self.assertRaises(errors.ParameterError, cli.GenericPollJob, job_id, cbs, cbs, update_freq=-30) def testZeroUpdateFreqParameter(self): job_id = 12345 cbs = _MockJobPollCb(self, job_id) self.assertRaises(errors.ParameterError, cli.GenericPollJob, job_id, cbs, cbs, update_freq=0) def testShouldCancel(self): job_id = 12345 cbs = _MockJobPollCb(self, job_id) cbs.AddWfjcResult(None, None, constants.JOB_NOTCHANGED) self.assertRaises(errors.JobCanceled, cli.GenericPollJob, job_id, cbs, cbs, cancel_fn=(lambda: True)) def testIgnoreCancel(self): job_id = 12345 cbs = _MockJobPollCb(self, job_id) cbs.AddWfjcResult(None, None, ((constants.JOB_STATUS_SUCCESS, ), None)) cbs.AddQueryJobsResult(constants.JOB_STATUS_SUCCESS, [constants.OP_STATUS_SUCCESS, constants.OP_STATUS_SUCCESS], ["Hello World", "Foo man bar"]) self.assertEqual(["Hello World", "Foo man bar"], cli.GenericPollJob( job_id, cbs, cbs, cancel_fn=(lambda: False))) cbs.CheckEmpty() class TestFormatLogMessage(unittest.TestCase): def test(self): self.assertEqual(cli.FormatLogMessage(constants.ELOG_MESSAGE, "Hello World"), "Hello World") self.assertRaises(TypeError, cli.FormatLogMessage, constants.ELOG_MESSAGE, [1, 2, 3]) self.assertTrue(cli.FormatLogMessage("some other type", (1, 2, 3))) class TestParseFields(unittest.TestCase): def test(self): self.assertEqual(cli.ParseFields(None, []), []) self.assertEqual(cli.ParseFields("name,foo,hello", []), ["name", "foo", "hello"]) self.assertEqual(cli.ParseFields(None, ["def", "ault", "fields", "here"]), ["def", "ault", "fields", "here"]) self.assertEqual(cli.ParseFields("name,foo", ["def", "ault"]), ["name", "foo"]) self.assertEqual(cli.ParseFields("+name,foo", ["def", "ault"]), ["def", "ault", "name", "foo"]) class TestParseNicOption(unittest.TestCase): def test(self): self.assertEqual(cli.ParseNicOption([("0", { "link": "eth0", })]), [{ "link": "eth0", }]) self.assertEqual(cli.ParseNicOption([("5", { "ip": "192.0.2.7", })]), [{}, {}, {}, {}, {}, { "ip": "192.0.2.7", }]) def testErrors(self): for i in [None, "", "abc", "zero", "Hello World", "\0", []]: self.assertRaises(errors.OpPrereqError, cli.ParseNicOption, [(i, { "link": "eth0", })]) self.assertRaises(errors.OpPrereqError, cli.ParseNicOption, [("0", i)]) self.assertRaises(errors.TypeEnforcementError, cli.ParseNicOption, [(0, { True: False, })]) self.assertRaises(errors.TypeEnforcementError, cli.ParseNicOption, [(3, { "mode": [], })]) class TestFormatResultError(unittest.TestCase): def testNormal(self): for verbose in [False, True]: self.assertRaises(AssertionError, cli.FormatResultError, constants.RS_NORMAL, verbose) def testUnknown(self): for verbose in [False, True]: self.assertRaises(NotImplementedError, cli.FormatResultError, "#some!other!status#", verbose) def test(self): for status in constants.RS_ALL: if status == constants.RS_NORMAL: continue self.assertNotEqual(cli.FormatResultError(status, False), cli.FormatResultError(status, True)) result = cli.FormatResultError(status, True) self.assertTrue(result.startswith("(")) self.assertTrue(result.endswith(")")) class TestGetOnlineNodes(unittest.TestCase): class _FakeClient: def __init__(self): self._query = [] def AddQueryResult(self, *args): self._query.append(args) def CountPending(self): return len(self._query) def Query(self, res, fields, qfilter): if res != constants.QR_NODE: raise Exception("Querying wrong resource") (exp_fields, check_filter, result) = self._query.pop(0) if exp_fields != fields: raise Exception("Expected fields %s, got %s" % (exp_fields, fields)) if not (qfilter is None or check_filter(qfilter)): raise Exception("Filter doesn't match expectations") return objects.QueryResponse(fields=None, data=result) def testEmpty(self): cl = self._FakeClient() cl.AddQueryResult(["name", "offline", "sip"], None, []) self.assertEqual(cli.GetOnlineNodes(None, cl=cl), []) self.assertEqual(cl.CountPending(), 0) def testNoSpecialFilter(self): cl = self._FakeClient() cl.AddQueryResult(["name", "offline", "sip"], None, [ [(constants.RS_NORMAL, "master.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.1")], [(constants.RS_NORMAL, "node2.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.2")], ]) self.assertEqual(cli.GetOnlineNodes(None, cl=cl), ["master.example.com", "node2.example.com"]) self.assertEqual(cl.CountPending(), 0) def testNoMaster(self): cl = self._FakeClient() def _CheckFilter(qfilter): self.assertEqual(qfilter, [qlang.OP_NOT, [qlang.OP_TRUE, "master"]]) return True cl.AddQueryResult(["name", "offline", "sip"], _CheckFilter, [ [(constants.RS_NORMAL, "node2.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.2")], ]) self.assertEqual(cli.GetOnlineNodes(None, cl=cl, filter_master=True), ["node2.example.com"]) self.assertEqual(cl.CountPending(), 0) def testSecondaryIpAddress(self): cl = self._FakeClient() cl.AddQueryResult(["name", "offline", "sip"], None, [ [(constants.RS_NORMAL, "master.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.1")], [(constants.RS_NORMAL, "node2.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.2")], ]) self.assertEqual(cli.GetOnlineNodes(None, cl=cl, secondary_ips=True), ["192.0.2.1", "192.0.2.2"]) self.assertEqual(cl.CountPending(), 0) def testNoMasterFilterNodeName(self): cl = self._FakeClient() def _CheckFilter(qfilter): self.assertEqual(qfilter, [qlang.OP_AND, [qlang.OP_OR] + [[qlang.OP_EQUAL, "name", name] for name in ["node2", "node3"]], [qlang.OP_NOT, [qlang.OP_TRUE, "master"]]]) return True cl.AddQueryResult(["name", "offline", "sip"], _CheckFilter, [ [(constants.RS_NORMAL, "node2.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.12")], [(constants.RS_NORMAL, "node3.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.13")], ]) self.assertEqual(cli.GetOnlineNodes(["node2", "node3"], cl=cl, secondary_ips=True, filter_master=True), ["192.0.2.12", "192.0.2.13"]) self.assertEqual(cl.CountPending(), 0) def testOfflineNodes(self): cl = self._FakeClient() cl.AddQueryResult(["name", "offline", "sip"], None, [ [(constants.RS_NORMAL, "master.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.1")], [(constants.RS_NORMAL, "node2.example.com"), (constants.RS_NORMAL, True), (constants.RS_NORMAL, "192.0.2.2")], [(constants.RS_NORMAL, "node3.example.com"), (constants.RS_NORMAL, True), (constants.RS_NORMAL, "192.0.2.3")], ]) self.assertEqual(cli.GetOnlineNodes(None, cl=cl, nowarn=True), ["master.example.com"]) self.assertEqual(cl.CountPending(), 0) def testNodeGroup(self): cl = self._FakeClient() def _CheckFilter(qfilter): self.assertEqual(qfilter, [qlang.OP_OR, [qlang.OP_EQUAL, "group", "foobar"], [qlang.OP_EQUAL, "group.uuid", "foobar"]]) return True cl.AddQueryResult(["name", "offline", "sip"], _CheckFilter, [ [(constants.RS_NORMAL, "master.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.1")], [(constants.RS_NORMAL, "node3.example.com"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, "192.0.2.3")], ]) self.assertEqual(cli.GetOnlineNodes(None, cl=cl, nodegroup="foobar"), ["master.example.com", "node3.example.com"]) self.assertEqual(cl.CountPending(), 0) class TestFormatTimestamp(unittest.TestCase): def testGood(self): self.assertEqual(cli.FormatTimestamp((0, 1)), time.strftime("%F %T", time.localtime(0)) + ".000001") self.assertEqual(cli.FormatTimestamp((1332944009, 17376)), (time.strftime("%F %T", time.localtime(1332944009)) + ".017376")) def testWrong(self): for i in [0, [], {}, "", [1]]: self.assertEqual(cli.FormatTimestamp(i), "?") class TestFormatUsage(unittest.TestCase): def test(self): binary = "gnt-unittest" commands = { "cmdA": (NotImplemented, NotImplemented, NotImplemented, NotImplemented, "description of A"), "bbb": (NotImplemented, NotImplemented, NotImplemented, NotImplemented, "Hello World," * 10), "longname": (NotImplemented, NotImplemented, NotImplemented, NotImplemented, "Another description"), } self.assertEqual(list(cli._FormatUsage(binary, commands)), [ "Usage: gnt-unittest {command} [options...] [argument...]", "gnt-unittest --help to see details, or man gnt-unittest", "", "Commands:", (" bbb - Hello World,Hello World,Hello World,Hello World,Hello" " World,Hello"), " World,Hello World,Hello World,Hello World,Hello World,", " cmdA - description of A", " longname - Another description", "", ]) class TestParseArgs(unittest.TestCase): def testNoArguments(self): for argv in [[], ["gnt-unittest"]]: try: cli._ParseArgs("gnt-unittest", argv, {}, {}, set()) except cli._ShowUsage as err: self.assertTrue(err.exit_error) else: self.fail("Did not raise exception") def testVersion(self): for argv in [["test", "--version"], ["test", "--version", "somethingelse"]]: try: cli._ParseArgs("test", argv, {}, {}, set()) except cli._ShowVersion: pass else: self.fail("Did not raise exception") def testHelp(self): for argv in [["test", "--help"], ["test", "--help", "somethingelse"]]: try: cli._ParseArgs("test", argv, {}, {}, set()) except cli._ShowUsage as err: self.assertFalse(err.exit_error) else: self.fail("Did not raise exception") def testUnknownCommandOrAlias(self): for argv in [["test", "list"], ["test", "somethingelse", "--help"]]: try: cli._ParseArgs("test", argv, {}, {}, set()) except cli._ShowUsage as err: self.assertTrue(err.exit_error) else: self.fail("Did not raise exception") def testInvalidAliasList(self): cmd = { "list": NotImplemented, "foo": NotImplemented, } aliases = { "list": NotImplemented, "foo": NotImplemented, } assert sorted(cmd.keys()) == sorted(aliases.keys()) self.assertRaises(AssertionError, cli._ParseArgs, "test", ["test", "list"], cmd, aliases, set()) def testAliasForNonExistantCommand(self): cmd = {} aliases = { "list": NotImplemented, } self.assertRaises(errors.ProgrammerError, cli._ParseArgs, "test", ["test", "list"], cmd, aliases, set()) class TestQftNames(unittest.TestCase): def testComplete(self): self.assertEqual(frozenset(cli._QFT_NAMES), constants.QFT_ALL) def testUnique(self): lcnames = [s.lower() for s in cli._QFT_NAMES.values()] self.assertFalse(utils.FindDuplicates(lcnames)) def testUppercase(self): for name in cli._QFT_NAMES.values(): self.assertEqual(name[0], name[0].upper()) class TestFieldDescValues(unittest.TestCase): def testKnownKind(self): fdef = objects.QueryFieldDefinition(name="aname", title="Atitle", kind=constants.QFT_TEXT, doc="aaa doc aaa") self.assertEqual(cli._FieldDescValues(fdef), ["aname", "Text", "Atitle", "aaa doc aaa"]) def testUnknownKind(self): kind = "#foo#" self.assertFalse(kind in constants.QFT_ALL) self.assertFalse(kind in cli._QFT_NAMES) fdef = objects.QueryFieldDefinition(name="zname", title="Ztitle", kind=kind, doc="zzz doc zzz") self.assertEqual(cli._FieldDescValues(fdef), ["zname", kind, "Ztitle", "zzz doc zzz"]) class TestSerializeGenericInfo(unittest.TestCase): """Test case for cli._SerializeGenericInfo""" def _RunTest(self, data, expected): buf = StringIO() cli._SerializeGenericInfo(buf, data, 0) self.assertEqual(buf.getvalue(), expected) def testSimple(self): test_samples = [ ("abc", "abc\n"), ([], "\n"), ({}, "\n"), (["1", "2", "3"], "- 1\n- 2\n- 3\n"), ([("z", "26")], "z: 26\n"), ({"z": "26"}, "z: 26\n"), ([("z", "26"), ("a", "1")], "z: 26\na: 1\n"), ({"z": "26", "a": "1"}, "a: 1\nz: 26\n"), ] for (data, expected) in test_samples: self._RunTest(data, expected) def testLists(self): adict = { "aa": "11", "bb": "22", "cc": "33", } adict_exp = ("- aa: 11\n" " bb: 22\n" " cc: 33\n") anobj = [ ("zz", "11"), ("ww", "33"), ("xx", "22"), ] anobj_exp = ("- zz: 11\n" " ww: 33\n" " xx: 22\n") alist = ["aa", "cc", "bb"] alist_exp = ("- - aa\n" " - cc\n" " - bb\n") test_samples = [ (adict, adict_exp), (anobj, anobj_exp), (alist, alist_exp), ] for (base_data, base_expected) in test_samples: for k in range(1, 4): data = k * [base_data] expected = k * base_expected self._RunTest(data, expected) def testDictionaries(self): data = [ ("aaa", ["x", "y"]), ("bbb", { "w": "1", "z": "2", }), ("ccc", [ ("xyz", "123"), ("efg", "456"), ]), ] expected = ("aaa: \n" " - x\n" " - y\n" "bbb: \n" " w: 1\n" " z: 2\n" "ccc: \n" " xyz: 123\n" " efg: 456\n") self._RunTest(data, expected) self._RunTest(dict(data), expected) class TestFormatPolicyInfo(unittest.TestCase): """Test case for cli.FormatPolicyInfo. These tests rely on cli._SerializeGenericInfo (tested elsewhere). """ def setUp(self): # Policies are big, and we want to see the difference in case of an error self.maxDiff = None def _RenameDictItem(self, parsed, old, new): self.assertTrue(old in parsed) self.assertTrue(new not in parsed) parsed[new] = parsed[old] del parsed[old] def _TranslateParsedNames(self, parsed): for (pretty, raw) in [ ("bounds specs", constants.ISPECS_MINMAX), ("allowed disk templates", constants.IPOLICY_DTS) ]: self._RenameDictItem(parsed, pretty, raw) for minmax in parsed[constants.ISPECS_MINMAX]: for key in set(minmax.keys()): keyparts = key.split("/", 1) if len(keyparts) > 1: self._RenameDictItem(minmax, key, keyparts[0]) self.assertTrue(constants.IPOLICY_DTS in parsed) parsed[constants.IPOLICY_DTS] = yaml.load("[%s]" % parsed[constants.IPOLICY_DTS], Loader=yaml.SafeLoader) @staticmethod def _PrintAndParsePolicy(custom, effective, iscluster): formatted = cli.FormatPolicyInfo(custom, effective, iscluster) buf = StringIO() cli._SerializeGenericInfo(buf, formatted, 0) return yaml.load(buf.getvalue(), Loader=yaml.SafeLoader) def _PrintAndCheckParsed(self, policy): parsed = self._PrintAndParsePolicy(policy, NotImplemented, True) self._TranslateParsedNames(parsed) self.assertEqual(parsed, policy) def _CompareClusterGroupItems(self, cluster, group, skip=None): if isinstance(group, dict): self.assertTrue(isinstance(cluster, dict)) if skip is None: skip = frozenset() self.assertEqual(frozenset(cluster).difference(skip), frozenset(group)) for key in group: self._CompareClusterGroupItems(cluster[key], group[key]) elif isinstance(group, list): self.assertTrue(isinstance(cluster, list)) self.assertEqual(len(cluster), len(group)) for (cval, gval) in zip(cluster, group): self._CompareClusterGroupItems(cval, gval) else: self.assertTrue(isinstance(group, str)) self.assertEqual("default (%s)" % cluster, group) def _TestClusterVsGroup(self, policy): cluster = self._PrintAndParsePolicy(policy, NotImplemented, True) group = self._PrintAndParsePolicy({}, policy, False) self._CompareClusterGroupItems(cluster, group, ["std"]) def testWithDefaults(self): self._PrintAndCheckParsed(constants.IPOLICY_DEFAULTS) self._TestClusterVsGroup(constants.IPOLICY_DEFAULTS) class TestCreateIPolicyFromOpts(unittest.TestCase): """Test case for cli.CreateIPolicyFromOpts.""" def setUp(self): # Policies are big, and we want to see the difference in case of an error self.maxDiff = None def _RecursiveCheckMergedDicts(self, default_pol, diff_pol, merged_pol, merge_minmax=False): self.assertTrue(type(default_pol) is dict) self.assertTrue(type(diff_pol) is dict) self.assertTrue(type(merged_pol) is dict) self.assertEqual(frozenset(default_pol), frozenset(merged_pol)) for (key, val) in merged_pol.items(): if key in diff_pol: if type(val) is dict: self._RecursiveCheckMergedDicts(default_pol[key], diff_pol[key], val) elif (merge_minmax and key == "minmax" and type(val) is list and len(val) == 1): self.assertEqual(len(default_pol[key]), 1) self.assertEqual(len(diff_pol[key]), 1) self._RecursiveCheckMergedDicts(default_pol[key][0], diff_pol[key][0], val[0]) else: self.assertEqual(val, diff_pol[key]) else: self.assertEqual(val, default_pol[key]) def testClusterPolicy(self): pol0 = cli.CreateIPolicyFromOpts( ispecs_mem_size={}, ispecs_cpu_count={}, ispecs_disk_count={}, ispecs_disk_size={}, ispecs_nic_count={}, ipolicy_disk_templates=None, ipolicy_vcpu_ratio=None, ipolicy_spindle_ratio=None, fill_all=True ) self.assertEqual(pol0, constants.IPOLICY_DEFAULTS) exp_pol1 = { constants.ISPECS_MINMAX: [ { constants.ISPECS_MIN: { constants.ISPEC_CPU_COUNT: 2, constants.ISPEC_DISK_COUNT: 1, }, constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 12*1024, constants.ISPEC_DISK_COUNT: 2, }, }, ], constants.ISPECS_STD: { constants.ISPEC_CPU_COUNT: 2, constants.ISPEC_DISK_COUNT: 2, }, constants.IPOLICY_VCPU_RATIO: 3.1, } pol1 = cli.CreateIPolicyFromOpts( ispecs_mem_size={"max": "12g"}, ispecs_cpu_count={"min": 2, "std": 2}, ispecs_disk_count={"min": 1, "max": 2, "std": 2}, ispecs_disk_size={}, ispecs_nic_count={}, ipolicy_disk_templates=None, ipolicy_vcpu_ratio=3.1, ipolicy_spindle_ratio=None, fill_all=True ) self._RecursiveCheckMergedDicts(constants.IPOLICY_DEFAULTS, exp_pol1, pol1, merge_minmax=True) exp_pol2 = { constants.ISPECS_MINMAX: [ { constants.ISPECS_MIN: { constants.ISPEC_DISK_SIZE: 512, constants.ISPEC_NIC_COUNT: 2, }, constants.ISPECS_MAX: { constants.ISPEC_NIC_COUNT: 3, }, }, ], constants.ISPECS_STD: { constants.ISPEC_CPU_COUNT: 2, constants.ISPEC_NIC_COUNT: 3, }, constants.IPOLICY_SPINDLE_RATIO: 1.3, constants.IPOLICY_DTS: ["templates"], } pol2 = cli.CreateIPolicyFromOpts( ispecs_mem_size={}, ispecs_cpu_count={"std": 2}, ispecs_disk_count={}, ispecs_disk_size={"min": "0.5g"}, ispecs_nic_count={"min": 2, "max": 3, "std": 3}, ipolicy_disk_templates=["templates"], ipolicy_vcpu_ratio=None, ipolicy_spindle_ratio=1.3, fill_all=True ) self._RecursiveCheckMergedDicts(constants.IPOLICY_DEFAULTS, exp_pol2, pol2, merge_minmax=True) for fill_all in [False, True]: exp_pol3 = { constants.ISPECS_STD: { constants.ISPEC_CPU_COUNT: 2, constants.ISPEC_NIC_COUNT: 3, }, } pol3 = cli.CreateIPolicyFromOpts( std_ispecs={ constants.ISPEC_CPU_COUNT: "2", constants.ISPEC_NIC_COUNT: "3", }, ipolicy_disk_templates=None, ipolicy_vcpu_ratio=None, ipolicy_spindle_ratio=None, fill_all=fill_all ) if fill_all: self._RecursiveCheckMergedDicts(constants.IPOLICY_DEFAULTS, exp_pol3, pol3, merge_minmax=True) else: self.assertEqual(pol3, exp_pol3) def testPartialPolicy(self): exp_pol0 = objects.MakeEmptyIPolicy() pol0 = cli.CreateIPolicyFromOpts( minmax_ispecs=None, std_ispecs=None, ipolicy_disk_templates=None, ipolicy_vcpu_ratio=None, ipolicy_spindle_ratio=None, fill_all=False ) self.assertEqual(pol0, exp_pol0) exp_pol1 = { constants.IPOLICY_VCPU_RATIO: 3.1, } pol1 = cli.CreateIPolicyFromOpts( minmax_ispecs=None, std_ispecs=None, ipolicy_disk_templates=None, ipolicy_vcpu_ratio=3.1, ipolicy_spindle_ratio=None, fill_all=False ) self.assertEqual(pol1, exp_pol1) exp_pol2 = { constants.IPOLICY_SPINDLE_RATIO: 1.3, constants.IPOLICY_DTS: ["templates"], } pol2 = cli.CreateIPolicyFromOpts( minmax_ispecs=None, std_ispecs=None, ipolicy_disk_templates=["templates"], ipolicy_vcpu_ratio=None, ipolicy_spindle_ratio=1.3, fill_all=False ) self.assertEqual(pol2, exp_pol2) def _TestInvalidISpecs(self, minmax_ispecs, std_ispecs, fail=True): for fill_all in [False, True]: if fail: self.assertRaises((errors.OpPrereqError, errors.UnitParseError, errors.TypeEnforcementError), cli.CreateIPolicyFromOpts, minmax_ispecs=minmax_ispecs, std_ispecs=std_ispecs, fill_all=fill_all) else: cli.CreateIPolicyFromOpts(minmax_ispecs=minmax_ispecs, std_ispecs=std_ispecs, fill_all=fill_all) def testInvalidPolicies(self): self.assertRaises(AssertionError, cli.CreateIPolicyFromOpts, std_ispecs={constants.ISPEC_MEM_SIZE: 1024}, ipolicy_disk_templates=None, ipolicy_vcpu_ratio=None, ipolicy_spindle_ratio=None, group_ipolicy=True) self.assertRaises(errors.OpPrereqError, cli.CreateIPolicyFromOpts, ispecs_mem_size={"wrong": "x"}, ispecs_cpu_count={}, ispecs_disk_count={}, ispecs_disk_size={}, ispecs_nic_count={}, ipolicy_disk_templates=None, ipolicy_vcpu_ratio=None, ipolicy_spindle_ratio=None, fill_all=True) self.assertRaises(errors.TypeEnforcementError, cli.CreateIPolicyFromOpts, ispecs_mem_size={}, ispecs_cpu_count={"min": "default"}, ispecs_disk_count={}, ispecs_disk_size={}, ispecs_nic_count={}, ipolicy_disk_templates=None, ipolicy_vcpu_ratio=None, ipolicy_spindle_ratio=None, fill_all=True) good_mmspecs = [ constants.ISPECS_MINMAX_DEFAULTS, constants.ISPECS_MINMAX_DEFAULTS, ] self._TestInvalidISpecs(good_mmspecs, None, fail=False) broken_mmspecs = copy.deepcopy(good_mmspecs) for minmaxpair in broken_mmspecs: for key in constants.ISPECS_MINMAX_KEYS: for par in constants.ISPECS_PARAMETERS: old = minmaxpair[key][par] del minmaxpair[key][par] self._TestInvalidISpecs(broken_mmspecs, None) minmaxpair[key][par] = "invalid" self._TestInvalidISpecs(broken_mmspecs, None) minmaxpair[key][par] = old minmaxpair[key]["invalid_key"] = None self._TestInvalidISpecs(broken_mmspecs, None) del minmaxpair[key]["invalid_key"] minmaxpair["invalid_key"] = None self._TestInvalidISpecs(broken_mmspecs, None) del minmaxpair["invalid_key"] assert broken_mmspecs == good_mmspecs good_stdspecs = constants.IPOLICY_DEFAULTS[constants.ISPECS_STD] self._TestInvalidISpecs(None, good_stdspecs, fail=False) broken_stdspecs = copy.deepcopy(good_stdspecs) for par in constants.ISPECS_PARAMETERS: old = broken_stdspecs[par] broken_stdspecs[par] = "invalid" self._TestInvalidISpecs(None, broken_stdspecs) broken_stdspecs[par] = old broken_stdspecs["invalid_key"] = None self._TestInvalidISpecs(None, broken_stdspecs) del broken_stdspecs["invalid_key"] assert broken_stdspecs == good_stdspecs def testAllowedValues(self): allowedv = "blah" exp_pol1 = { constants.ISPECS_MINMAX: allowedv, constants.IPOLICY_DTS: allowedv, constants.IPOLICY_VCPU_RATIO: allowedv, constants.IPOLICY_SPINDLE_RATIO: allowedv, } pol1 = cli.CreateIPolicyFromOpts(minmax_ispecs=[{allowedv: {}}], std_ispecs=None, ipolicy_disk_templates=allowedv, ipolicy_vcpu_ratio=allowedv, ipolicy_spindle_ratio=allowedv, allowed_values=[allowedv]) self.assertEqual(pol1, exp_pol1) @staticmethod def _ConvertSpecToStrings(spec): ret = {} for (par, val) in spec.items(): ret[par] = str(val) return ret def _CheckNewStyleSpecsCall(self, exp_ipolicy, minmax_ispecs, std_ispecs, group_ipolicy, fill_all): ipolicy = cli.CreateIPolicyFromOpts(minmax_ispecs=minmax_ispecs, std_ispecs=std_ispecs, group_ipolicy=group_ipolicy, fill_all=fill_all) self.assertEqual(ipolicy, exp_ipolicy) def _TestFullISpecsInner(self, skel_exp_ipol, exp_minmax, exp_std, group_ipolicy, fill_all): exp_ipol = skel_exp_ipol.copy() if exp_minmax is not None: minmax_ispecs = [] for exp_mm_pair in exp_minmax: mmpair = {} for (key, spec) in exp_mm_pair.items(): mmpair[key] = self._ConvertSpecToStrings(spec) minmax_ispecs.append(mmpair) exp_ipol[constants.ISPECS_MINMAX] = exp_minmax else: minmax_ispecs = None if exp_std is not None: std_ispecs = self._ConvertSpecToStrings(exp_std) exp_ipol[constants.ISPECS_STD] = exp_std else: std_ispecs = None self._CheckNewStyleSpecsCall(exp_ipol, minmax_ispecs, std_ispecs, group_ipolicy, fill_all) if minmax_ispecs: for mmpair in minmax_ispecs: for (key, spec) in mmpair.items(): for par in [constants.ISPEC_MEM_SIZE, constants.ISPEC_DISK_SIZE]: if par in spec: spec[par] += "m" self._CheckNewStyleSpecsCall(exp_ipol, minmax_ispecs, std_ispecs, group_ipolicy, fill_all) if std_ispecs: for par in [constants.ISPEC_MEM_SIZE, constants.ISPEC_DISK_SIZE]: if par in std_ispecs: std_ispecs[par] += "m" self._CheckNewStyleSpecsCall(exp_ipol, minmax_ispecs, std_ispecs, group_ipolicy, fill_all) def testFullISpecs(self): exp_minmax1 = [ { constants.ISPECS_MIN: { constants.ISPEC_MEM_SIZE: 512, constants.ISPEC_CPU_COUNT: 2, constants.ISPEC_DISK_COUNT: 2, constants.ISPEC_DISK_SIZE: 512, constants.ISPEC_NIC_COUNT: 2, constants.ISPEC_SPINDLE_USE: 2, }, constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 768*1024, constants.ISPEC_CPU_COUNT: 7, constants.ISPEC_DISK_COUNT: 6, constants.ISPEC_DISK_SIZE: 2048*1024, constants.ISPEC_NIC_COUNT: 3, constants.ISPEC_SPINDLE_USE: 3, }, }, ] exp_minmax2 = [ { constants.ISPECS_MIN: { constants.ISPEC_MEM_SIZE: 512, constants.ISPEC_CPU_COUNT: 2, constants.ISPEC_DISK_COUNT: 2, constants.ISPEC_DISK_SIZE: 512, constants.ISPEC_NIC_COUNT: 2, constants.ISPEC_SPINDLE_USE: 2, }, constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 768*1024, constants.ISPEC_CPU_COUNT: 7, constants.ISPEC_DISK_COUNT: 6, constants.ISPEC_DISK_SIZE: 2048*1024, constants.ISPEC_NIC_COUNT: 3, constants.ISPEC_SPINDLE_USE: 3, }, }, { constants.ISPECS_MIN: { constants.ISPEC_MEM_SIZE: 1024*1024, constants.ISPEC_CPU_COUNT: 3, constants.ISPEC_DISK_COUNT: 3, constants.ISPEC_DISK_SIZE: 256, constants.ISPEC_NIC_COUNT: 4, constants.ISPEC_SPINDLE_USE: 5, }, constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 2048*1024, constants.ISPEC_CPU_COUNT: 5, constants.ISPEC_DISK_COUNT: 5, constants.ISPEC_DISK_SIZE: 1024*1024, constants.ISPEC_NIC_COUNT: 5, constants.ISPEC_SPINDLE_USE: 7, }, }, ] exp_std1 = { constants.ISPEC_MEM_SIZE: 768*1024, constants.ISPEC_CPU_COUNT: 7, constants.ISPEC_DISK_COUNT: 6, constants.ISPEC_DISK_SIZE: 2048*1024, constants.ISPEC_NIC_COUNT: 3, constants.ISPEC_SPINDLE_USE: 1, } for fill_all in [False, True]: if fill_all: skel_ipolicy = constants.IPOLICY_DEFAULTS else: skel_ipolicy = {} self._TestFullISpecsInner(skel_ipolicy, None, exp_std1, False, fill_all) for exp_minmax in [exp_minmax1, exp_minmax2]: self._TestFullISpecsInner(skel_ipolicy, exp_minmax, exp_std1, False, fill_all) self._TestFullISpecsInner(skel_ipolicy, exp_minmax, None, False, fill_all) class TestPrintIPolicyCommand(unittest.TestCase): """Test case for cli.PrintIPolicyCommand""" _SPECS1 = { "par1": 42, "par2": "xyz", } _SPECS1_STR = "par1=42,par2=xyz" _SPECS2 = { "param": 10, "another_param": 101, } _SPECS2_STR = "another_param=101,param=10" _SPECS3 = { "par1": 1024, "param": "abc", } _SPECS3_STR = "par1=1024,param=abc" def _CheckPrintIPolicyCommand(self, ipolicy, isgroup, expected): buf = StringIO() cli.PrintIPolicyCommand(buf, ipolicy, isgroup) self.assertEqual(buf.getvalue(), expected) def testIgnoreStdForGroup(self): self._CheckPrintIPolicyCommand({"std": self._SPECS1}, True, "") def testIgnoreEmpty(self): policies = [ {}, {"std": {}}, {"minmax": []}, {"minmax": [{}]}, {"minmax": [{ "min": {}, "max": {}, }]}, {"minmax": [{ "min": self._SPECS1, "max": {}, }]}, ] for pol in policies: self._CheckPrintIPolicyCommand(pol, False, "") def testFullPolicies(self): cases = [ ({"std": self._SPECS1}, " %s %s" % (cli.IPOLICY_STD_SPECS_STR, self._SPECS1_STR)), ({"minmax": [{ "min": self._SPECS1, "max": self._SPECS2, }]}, " %s min:%s/max:%s" % (cli.IPOLICY_BOUNDS_SPECS_STR, self._SPECS1_STR, self._SPECS2_STR)), ({"minmax": [ { "min": self._SPECS1, "max": self._SPECS2, }, { "min": self._SPECS2, "max": self._SPECS3, }, ]}, " %s min:%s/max:%s//min:%s/max:%s" % (cli.IPOLICY_BOUNDS_SPECS_STR, self._SPECS1_STR, self._SPECS2_STR, self._SPECS2_STR, self._SPECS3_STR)), ] for (pol, exp) in cases: self._CheckPrintIPolicyCommand(pol, False, exp) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.client.gnt_cluster_unittest.py000075500000000000000000000412221476477700300260500ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.client.gnt_cluster""" import unittest import optparse import os import shutil import tempfile from unittest import mock from ganeti import errors from ganeti.client import gnt_cluster from ganeti import utils from ganeti import compat from ganeti import constants from ganeti import ssh from ganeti import cli import testutils class TestEpoUtilities(unittest.TestCase): def setUp(self): self.nodes2ip = dict(("node%s" % i, "192.0.2.%s" % i) for i in range(1, 10)) self.nodes = set(self.nodes2ip.keys()) self.ips2node = dict((v, k) for (k, v) in self.nodes2ip.items()) def _FakeAction(*args): return True def _FakePing(ip, port, live_port_needed=False): self.assertTrue(live_port_needed) self.assertEqual(port, 0) return True def _FakeSleep(secs): self.assertTrue(secs >= 0 and secs <= 5) return def _NoopFeedback(self, text): return def testPingFnRemoveHostsUp(self): seen = set() def _FakeSeenPing(ip, *args, **kwargs): node = self.ips2node[ip] self.assertFalse(node in seen) seen.add(node) return True helper = gnt_cluster._RunWhenNodesReachableHelper(self.nodes, self._FakeAction, self.nodes2ip, 0, self._NoopFeedback, _ping_fn=_FakeSeenPing, _sleep_fn=self._FakeSleep) nodes_len = len(self.nodes) for (num, _) in enumerate(self.nodes): helper.Wait(5) if num < nodes_len - 1: self.assertRaises(utils.RetryAgain, helper) else: helper() self.assertEqual(seen, self.nodes) self.assertFalse(helper.down) self.assertEqual(helper.up, self.nodes) def testActionReturnFalseSetsHelperFalse(self): called = False def _FalseAction(*args): return called helper = gnt_cluster._RunWhenNodesReachableHelper(self.nodes, _FalseAction, self.nodes2ip, 0, self._NoopFeedback, _ping_fn=self._FakePing, _sleep_fn=self._FakeSleep) for _ in self.nodes: try: helper() except utils.RetryAgain: called = True self.assertFalse(helper.success) def testMaybeInstanceStartup(self): instances_arg = [] def _FakeInstanceStart(opts, instances, start): instances_arg.append(set(instances)) return None inst_map = { "inst1": set(["node1", "node2"]), "inst2": set(["node1", "node3"]), "inst3": set(["node2", "node1"]), "inst4": set(["node2", "node1", "node3"]), "inst5": set(["node4"]), } fn = _FakeInstanceStart self.assertTrue(gnt_cluster._MaybeInstanceStartup(None, inst_map, set(), _instance_start_fn=fn)) self.assertFalse(instances_arg) result = gnt_cluster._MaybeInstanceStartup(None, inst_map, set(["node1"]), _instance_start_fn=fn) self.assertTrue(result) self.assertFalse(instances_arg) result = gnt_cluster._MaybeInstanceStartup(None, inst_map, set(["node1", "node3"]), _instance_start_fn=fn) self.assertTrue(result is None) self.assertEqual(instances_arg.pop(0), set(["inst2"])) self.assertFalse("inst2" in inst_map) result = gnt_cluster._MaybeInstanceStartup(None, inst_map, set(["node1", "node3"]), _instance_start_fn=fn) self.assertTrue(result) self.assertFalse(instances_arg) result = gnt_cluster._MaybeInstanceStartup(None, inst_map, set(["node1", "node3", "node2"]), _instance_start_fn=fn) self.assertEqual(instances_arg.pop(0), set(["inst1", "inst3", "inst4"])) self.assertTrue(result is None) result = gnt_cluster._MaybeInstanceStartup(None, inst_map, set(["node1", "node3", "node2", "node4"]), _instance_start_fn=fn) self.assertTrue(result is None) self.assertEqual(instances_arg.pop(0), set(["inst5"])) self.assertFalse(inst_map) class _ClientForEpo: def __init__(self, groups, nodes): self._groups = groups self._nodes = nodes def QueryGroups(self, names, fields, use_locking): assert not use_locking assert fields == ["node_list"] return self._groups def QueryNodes(self, names, fields, use_locking): assert not use_locking assert fields == ["name", "master", "pinst_list", "sinst_list", "powered", "offline"] return self._nodes class TestEpo(unittest.TestCase): _ON_EXITCODE = 253 _OFF_EXITCODE = 254 def _ConfirmForce(self, *args): self.fail("Shouldn't need confirmation") def _Confirm(self, exp_names, result, names, ltype, text): self.assertEqual(names, exp_names) self.assertFalse(result is NotImplemented) return result def _Off(self, exp_node_list, opts, node_list, inst_map): self.assertEqual(node_list, exp_node_list) self.assertFalse(inst_map) return self._OFF_EXITCODE def _Test(self, *args, **kwargs): defaults = dict(qcl=NotImplemented, _on_fn=NotImplemented, _off_fn=NotImplemented, _stdout_fn=lambda *args: None, _stderr_fn=lambda *args: None) defaults.update(kwargs) return gnt_cluster.Epo(*args, **defaults) def testShowAllWithGroups(self): opts = optparse.Values(dict(groups=True, show_all=True)) result = self._Test(opts, NotImplemented) self.assertEqual(result, constants.EXIT_FAILURE) def testShowAllWithArgs(self): opts = optparse.Values(dict(groups=False, show_all=True)) result = self._Test(opts, ["a", "b", "c"]) self.assertEqual(result, constants.EXIT_FAILURE) def testNoArgumentsNoParameters(self): for (force, confirm_result) in [(True, NotImplemented), (False, False), (False, True)]: opts = optparse.Values(dict(groups=False, show_all=False, force=force, on=False)) client = _ClientForEpo(NotImplemented, [ ("node1.example.com", False, [], [], True, False), ]) if force: confirm_fn = self._ConfirmForce else: confirm_fn = compat.partial(self._Confirm, ["node1.example.com"], confirm_result) off_fn = compat.partial(self._Off, ["node1.example.com"]) result = self._Test(opts, [], qcl=client, _off_fn=off_fn, _confirm_fn=confirm_fn) if force or confirm_result: self.assertEqual(result, self._OFF_EXITCODE) else: self.assertEqual(result, constants.EXIT_FAILURE) def testPowerOn(self): for master in [False, True]: opts = optparse.Values(dict(groups=False, show_all=True, force=True, on=True)) client = _ClientForEpo(NotImplemented, [ ("node1.example.com", False, [], [], True, False), ("node2.example.com", False, [], [], False, False), ("node3.example.com", False, [], [], True, True), ("node4.example.com", False, [], [], None, True), ("node5.example.com", master, [], [], False, False), ]) def _On(_, all_nodes, node_list, inst_map): self.assertEqual(all_nodes, ["node%s.example.com" % i for i in range(1, 6)]) if master: self.assertEqual(node_list, ["node2.example.com"]) else: self.assertEqual(node_list, ["node2.example.com", "node5.example.com"]) self.assertFalse(inst_map) return self._ON_EXITCODE result = self._Test(opts, [], qcl=client, _on_fn=_On, _confirm_fn=self._ConfirmForce) self.assertEqual(result, self._ON_EXITCODE) def testMasterWithoutShowAll(self): opts = optparse.Values(dict(groups=False, show_all=False, force=True, on=False)) client = _ClientForEpo(NotImplemented, [ ("node1.example.com", True, [], [], True, False), ]) result = self._Test(opts, [], qcl=client, _confirm_fn=self._ConfirmForce) self.assertEqual(result, constants.EXIT_FAILURE) class DrbdHelperTestCase(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.enabled_disk_templates = [] def enableDrbd(self): self.enabled_disk_templates = [constants.DT_DRBD8] def disableDrbd(self): self.enabled_disk_templates = [constants.DT_DISKLESS] class InitDrbdHelper(DrbdHelperTestCase): def testNoDrbdNoHelper(self): opts = mock.Mock() opts.drbd_helper = None self.disableDrbd() helper = gnt_cluster._InitDrbdHelper(opts, self.enabled_disk_templates, feedback_fn=mock.Mock()) self.assertEqual(None, helper) def testNoDrbdHelper(self): opts = mock.Mock() self.disableDrbd() opts.drbd_helper = "/bin/true" helper = gnt_cluster._InitDrbdHelper(opts, self.enabled_disk_templates, feedback_fn=mock.Mock()) self.assertEqual(opts.drbd_helper, helper) def testDrbdHelperNone(self): opts = mock.Mock() self.enableDrbd() opts.drbd_helper = None helper = gnt_cluster._InitDrbdHelper(opts, self.enabled_disk_templates, feedback_fn=mock.Mock()) self.assertEqual(constants.DEFAULT_DRBD_HELPER, helper) def testDrbdHelperEmpty(self): opts = mock.Mock() self.enableDrbd() opts.drbd_helper = '' self.assertRaises(errors.OpPrereqError, gnt_cluster._InitDrbdHelper, opts, self.enabled_disk_templates, feedback_fn=mock.Mock()) def testDrbdHelper(self): opts = mock.Mock() self.enableDrbd() opts.drbd_helper = "/bin/true" helper = gnt_cluster._InitDrbdHelper(opts, self.enabled_disk_templates, feedback_fn=mock.Mock()) self.assertEqual(opts.drbd_helper, helper) class GetDrbdHelper(DrbdHelperTestCase): def testNoDrbdNoHelper(self): opts = mock.Mock() self.disableDrbd() opts.drbd_helper = None helper = gnt_cluster._GetDrbdHelper(opts, self.enabled_disk_templates) self.assertEqual(None, helper) def testNoTemplateInfoNoHelper(self): opts = mock.Mock() opts.drbd_helper = None helper = gnt_cluster._GetDrbdHelper(opts, None) self.assertEqual(None, helper) def testNoTemplateInfoHelper(self): opts = mock.Mock() opts.drbd_helper = "/bin/true" helper = gnt_cluster._GetDrbdHelper(opts, None) self.assertEqual(opts.drbd_helper, helper) def testNoDrbdHelper(self): opts = mock.Mock() self.disableDrbd() opts.drbd_helper = "/bin/true" helper = gnt_cluster._GetDrbdHelper(opts, None) self.assertEqual(opts.drbd_helper, helper) def testDrbdNoHelper(self): opts = mock.Mock() self.enableDrbd() opts.drbd_helper = None helper = gnt_cluster._GetDrbdHelper(opts, self.enabled_disk_templates) self.assertEqual(None, helper) def testDrbdHelper(self): opts = mock.Mock() self.enableDrbd() opts.drbd_helper = "/bin/true" helper = gnt_cluster._GetDrbdHelper(opts, self.enabled_disk_templates) self.assertEqual(opts.drbd_helper, helper) class TestBuildGanetiPubKeys(testutils.GanetiTestCase): _SOME_KEY_DICT = {"rsa": "key_rsa", "dsa": "key_dsa"} _MASTER_NODE_NAME = "master_node" _MASTER_NODE_UUID = "master_uuid" _NUM_NODES = 2 # excluding master node _ONLINE_NODE_NAMES = ["node%s_name" % i for i in range(_NUM_NODES)] _ONLINE_NODE_UUIDS = ["node%s_uuid" % i for i in range(_NUM_NODES)] _CLUSTER_NAME = "cluster_name" _PRIV_KEY = "master_private_key" _PUB_KEY = "master_public_key" _MODIFY_SSH_SETUP = True _AUTH_KEYS = "a\nb\nc" _SSH_KEY_TYPE = "dsa" def _setUpFakeKeys(self): os.makedirs(os.path.join(self.tmpdir, ".ssh")) for key_type in ["rsa", "dsa"]: self.priv_filename = os.path.join(self.tmpdir, ".ssh", "id_%s" % key_type) utils.WriteFile(self.priv_filename, data=self._PRIV_KEY) self.pub_filename = os.path.join( self.tmpdir, ".ssh", "id_%s.pub" % key_type) utils.WriteFile(self.pub_filename, data=self._PUB_KEY) self.auth_filename = os.path.join(self.tmpdir, ".ssh", "authorized_keys") utils.WriteFile(self.auth_filename, data=self._AUTH_KEYS) def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() self.pub_key_filename = os.path.join(self.tmpdir, "ganeti_test_pub_keys") self._setUpFakeKeys() self._ssh_read_remote_ssh_pub_keys_patcher = testutils \ .patch_object(ssh, "ReadRemoteSshPubKeys") self._ssh_read_remote_ssh_pub_keys_mock = \ self._ssh_read_remote_ssh_pub_keys_patcher.start() self._ssh_read_remote_ssh_pub_keys_mock.return_value = self._SOME_KEY_DICT self.mock_cl = mock.Mock() self.mock_cl.QueryConfigValues = mock.Mock() self.mock_cl.QueryConfigValues.return_value = \ (self._CLUSTER_NAME, self._MASTER_NODE_NAME, self._MODIFY_SSH_SETUP, self._SSH_KEY_TYPE) self._get_online_nodes_mock = mock.Mock() self._get_online_nodes_mock.return_value = \ self._ONLINE_NODE_NAMES self._get_nodes_ssh_ports_mock = mock.Mock() self._get_nodes_ssh_ports_mock.return_value = \ [22 for i in range(self._NUM_NODES + 1)] self._get_node_uuids_mock = mock.Mock() self._get_node_uuids_mock.return_value = \ self._ONLINE_NODE_UUIDS + [self._MASTER_NODE_UUID] self._options = mock.Mock() self._options.ssh_key_check = False def _GetTempHomedir(self, _): return self.tmpdir def tearDown(self): super(testutils.GanetiTestCase, self).tearDown() shutil.rmtree(self.tmpdir) self._ssh_read_remote_ssh_pub_keys_patcher.stop() def testNewPubKeyFile(self): gnt_cluster._BuildGanetiPubKeys( self._options, pub_key_file=self.pub_key_filename, cl=self.mock_cl, get_online_nodes_fn=self._get_online_nodes_mock, get_nodes_ssh_ports_fn=self._get_nodes_ssh_ports_mock, get_node_uuids_fn=self._get_node_uuids_mock, homedir_fn=self._GetTempHomedir) key_file_result = utils.ReadFile(self.pub_key_filename) for node_uuid in self._ONLINE_NODE_UUIDS + [self._MASTER_NODE_UUID]: self.assertTrue(node_uuid in key_file_result) self.assertTrue(self._PUB_KEY in key_file_result) def testOverridePubKeyFile(self): fd = open(self.pub_key_filename, "w") fd.write("Pink Bunny") fd.close() gnt_cluster._BuildGanetiPubKeys( self._options, pub_key_file=self.pub_key_filename, cl=self.mock_cl, get_online_nodes_fn=self._get_online_nodes_mock, get_nodes_ssh_ports_fn=self._get_nodes_ssh_ports_mock, get_node_uuids_fn=self._get_node_uuids_mock, homedir_fn=self._GetTempHomedir) self.assertFalse("Pink Bunny" in self.pub_key_filename) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.client.gnt_instance_unittest.py000075500000000000000000000224601476477700300261760ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.client.gnt_instance""" import unittest from ganeti import constants from ganeti import utils from ganeti import errors from ganeti import objects from ganeti.client import gnt_instance import testutils class TestConsole(unittest.TestCase): def setUp(self): self._output = [] self._cmds = [] self._next_cmd_exitcode = 0 def _Test(self, console, show_command, cluster_name): return gnt_instance._DoConsole(console, show_command, cluster_name, feedback_fn=self._Feedback, _runcmd_fn=self._FakeRunCmd) def _Feedback(self, msg, *args): if args: msg = msg % args self._output.append(msg) def _FakeRunCmd(self, cmd, interactive=None): self.assertTrue(interactive) self.assertTrue(isinstance(cmd, list)) self._cmds.append(cmd) return utils.RunResult(self._next_cmd_exitcode, None, "", "", "cmd", utils.process._TIMEOUT_NONE, 5) def testMessage(self): cons = objects.InstanceConsole(instance="inst98.example.com", kind=constants.CONS_MESSAGE, message="Hello World") self.assertEqual(self._Test(cons, False, "cluster.example.com"), constants.EXIT_SUCCESS) self.assertEqual(len(self._cmds), 0) self.assertEqual(self._output, ["Hello World"]) def testVnc(self): cons = objects.InstanceConsole(instance="inst1.example.com", kind=constants.CONS_VNC, host="node1.example.com", port=5901, display=1) self.assertEqual(self._Test(cons, False, "cluster.example.com"), constants.EXIT_SUCCESS) self.assertEqual(len(self._cmds), 0) self.assertEqual(len(self._output), 1) self.assertTrue(" inst1.example.com " in self._output[0]) self.assertTrue(" node1.example.com:5901 " in self._output[0]) self.assertTrue("vnc://node1.example.com:5901/" in self._output[0]) def testSshShow(self): cons = objects.InstanceConsole(instance="inst31.example.com", kind=constants.CONS_SSH, host="node93.example.com", user="user_abc", command="xl console x.y.z") self.assertEqual(self._Test(cons, True, "cluster.example.com"), constants.EXIT_SUCCESS) self.assertEqual(len(self._cmds), 0) self.assertEqual(len(self._output), 1) self.assertTrue(" user_abc@node93.example.com " in self._output[0]) self.assertTrue("'xl console x.y.z'" in self._output[0]) def testSshRun(self): cons = objects.InstanceConsole(instance="inst31.example.com", kind=constants.CONS_SSH, host="node93.example.com", user="user_abc", command=["xl", "console", "x.y.z"]) self.assertEqual(self._Test(cons, False, "cluster.example.com"), constants.EXIT_SUCCESS) self.assertEqual(len(self._cmds), 1) self.assertEqual(len(self._output), 0) # This is very important to prevent escapes from the console self.assertTrue("-oEscapeChar=none" in self._cmds[0]) def testSshRunFail(self): cons = objects.InstanceConsole(instance="inst31.example.com", kind=constants.CONS_SSH, host="node93.example.com", user="user_abc", command=["xl", "console", "x.y.z"]) self._next_cmd_exitcode = 100 self.assertRaises(errors.OpExecError, self._Test, cons, False, "cluster.example.com") self.assertEqual(len(self._cmds), 1) self.assertEqual(len(self._output), 0) class TestConvertNicDiskModifications(unittest.TestCase): def testErrorMods(self): fn = gnt_instance._ConvertNicDiskModifications self.assertEqual(fn([]), []) # Error cases self.assertRaises(errors.OpPrereqError, fn, [ (constants.DDM_REMOVE, {"param": "value", }), ]) self.assertRaises(errors.OpPrereqError, fn, [ (0, {constants.DDM_REMOVE: True, "param": "value", }), ]) self.assertRaises(errors.OpPrereqError, fn, [ (constants.DDM_DETACH, {"param": "value", }), ]) self.assertRaises(errors.OpPrereqError, fn, [ (0, {constants.DDM_DETACH: True, "param": "value", }), ]) self.assertRaises(errors.OpPrereqError, fn, [ (0, { constants.DDM_REMOVE: True, constants.DDM_ADD: True, }), ]) self.assertRaises(errors.OpPrereqError, fn, [ (0, { constants.DDM_DETACH: True, constants.DDM_MODIFY: True, }), ]) def testLegacyCalls(self): fn = gnt_instance._ConvertNicDiskModifications for action in constants.DDMS_VALUES: self.assertEqual(fn([ (action, {}), ]), [ (action, -1, {}), ]) self.assertRaises(errors.OpPrereqError, fn, [ (0, { action: True, constants.DDM_MODIFY: True, }), ]) self.assertEqual(fn([ (constants.DDM_ADD, { constants.IDISK_SIZE: 1024, }), ]), [ (constants.DDM_ADD, -1, { constants.IDISK_SIZE: 1024, }), ]) def testNewStyleCalls(self): fn = gnt_instance._ConvertNicDiskModifications self.assertEqual(fn([ (2, { constants.IDISK_MODE: constants.DISK_RDWR, }), ]), [ (constants.DDM_MODIFY, 2, { constants.IDISK_MODE: constants.DISK_RDWR, }), ]) self.assertEqual(fn([ (0, { constants.DDM_ADD: True, constants.IDISK_SIZE: 4096, }), ]), [ (constants.DDM_ADD, 0, { constants.IDISK_SIZE: 4096, }), ]) self.assertEqual(fn([ (-1, { constants.DDM_REMOVE: True, }), ]), [ (constants.DDM_REMOVE, -1, {}), ]) self.assertEqual(fn([ (-1, { constants.DDM_MODIFY: True, constants.IDISK_SIZE: 1024, }), ]), [ (constants.DDM_MODIFY, -1, { constants.IDISK_SIZE: 1024, }), ]) def testNamesUUIDs(self): fn = gnt_instance._ConvertNicDiskModifications self.assertEqual(fn([ ('name', { constants.IDISK_MODE: constants.DISK_RDWR, constants.IDISK_NAME: "rename", }), ]), [ (constants.DDM_MODIFY, 'name', { constants.IDISK_MODE: constants.DISK_RDWR, constants.IDISK_NAME: "rename", }), ]) self.assertEqual(fn([ ('024ef14d-4879-400e-8767-d61c051950bf', { constants.DDM_MODIFY: True, constants.IDISK_SIZE: 1024, constants.IDISK_NAME: "name", }), ]), [ (constants.DDM_MODIFY, '024ef14d-4879-400e-8767-d61c051950bf', { constants.IDISK_SIZE: 1024, constants.IDISK_NAME: "name", }), ]) self.assertEqual(fn([ ('name', { constants.DDM_REMOVE: True, }), ]), [ (constants.DDM_REMOVE, 'name', {}), ]) class TestParseDiskSizes(unittest.TestCase): def test(self): fn = gnt_instance._ParseDiskSizes self.assertEqual(fn([]), []) # Missing size parameter self.assertRaises(errors.OpPrereqError, fn, [ (constants.DDM_ADD, 0, {}), ]) # Converting disk size self.assertEqual(fn([ (constants.DDM_ADD, 11, { constants.IDISK_SIZE: "9G", }), ]), [ (constants.DDM_ADD, 11, { constants.IDISK_SIZE: 9216, }), ]) # No size parameter self.assertEqual(fn([ (constants.DDM_REMOVE, 11, { "other": "24M", }), ]), [ (constants.DDM_REMOVE, 11, { "other": "24M", }), ]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.client.gnt_job_unittest.py000075500000000000000000000133651476477700300251500ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.client.gnt_job""" import unittest import optparse from ganeti.client import gnt_job from ganeti import utils from ganeti import errors from ganeti import query from ganeti import qlang from ganeti import objects from ganeti import compat from ganeti import constants import testutils class _ClientForCancelJob: def __init__(self, cancel_cb, query_cb): self.cancelled = [] self._cancel_cb = cancel_cb self._query_cb = query_cb def CancelJob(self, job_id, kill=False): self.cancelled.append(job_id) return self._cancel_cb(job_id) def Query(self, kind, selected, qfilter): assert kind == constants.QR_JOB assert selected == ["id", "status", "summary"] fields = query.GetAllFields(query._GetQueryFields(query.JOB_FIELDS, selected)) return objects.QueryResponse(data=self._query_cb(qfilter), fields=fields) class TestCancelJob(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.stdout = [] def _ToStdout(self, line): self.stdout.append(line) def _Ask(self, answer, question): self.assertTrue(question.endswith("?")) return answer def testStatusFilterAndArguments(self): opts = optparse.Values(dict(status_filter=frozenset(), force=False, kill=False)) try: gnt_job.CancelJobs(opts, ["a"], cl=NotImplemented, _stdout_fn=NotImplemented, _ask_fn=NotImplemented) except errors.OpPrereqError as err: self.assertEqual(err.args[1], errors.ECODE_INVAL) else: self.fail("Did not raise exception") def _TestArguments(self, force): opts = optparse.Values(dict(status_filter=None, force=force, kill=False)) def _CancelCb(job_id): self.assertTrue(job_id in ("24185", "3252")) return (True, "%s will be cancelled" % job_id) cl = _ClientForCancelJob(_CancelCb, NotImplemented) self.assertEqual(gnt_job.CancelJobs(opts, ["24185", "3252"], cl=cl, _stdout_fn=self._ToStdout, _ask_fn=NotImplemented), constants.EXIT_SUCCESS) self.assertEqual(cl.cancelled, ["24185", "3252"]) self.assertEqual(self.stdout, [ "24185 will be cancelled", "3252 will be cancelled", ]) def testArgumentsWithForce(self): self._TestArguments(True) def testArgumentsNoForce(self): self._TestArguments(False) def testArgumentsWithError(self): opts = optparse.Values(dict(status_filter=None, force=True, kill=False)) def _CancelCb(job_id): if job_id == "10788": return (False, "error %s" % job_id) else: return (True, "%s will be cancelled" % job_id) cl = _ClientForCancelJob(_CancelCb, NotImplemented) self.assertEqual(gnt_job.CancelJobs(opts, ["203", "10788", "30801"], cl=cl, _stdout_fn=self._ToStdout, _ask_fn=NotImplemented), constants.EXIT_FAILURE) self.assertEqual(cl.cancelled, ["203", "10788", "30801"]) self.assertEqual(self.stdout, [ "203 will be cancelled", "error 10788", "30801 will be cancelled", ]) def testFilterPending(self): opts = optparse.Values(dict(status_filter=constants.JOBS_PENDING, force=False, kill=False)) def _Query(qfilter): assert isinstance(constants.JOBS_PENDING, frozenset) ref_qfilter = qlang.MakeSimpleFilter("status", constants.JOBS_PENDING) self.assertEqual(qfilter[0], ref_qfilter[0]) # Need to sort as constants.JOBS_PENDING has no stable order self.assertEqual(sorted(qfilter[1:]), sorted(ref_qfilter[1:])) return [ [(constants.RS_UNAVAIL, None), (constants.RS_UNAVAIL, None), (constants.RS_UNAVAIL, None)], [(constants.RS_NORMAL, 32532), (constants.RS_NORMAL, constants.JOB_STATUS_QUEUED), (constants.RS_NORMAL, ["op1", "op2", "op3"])], ] cl = _ClientForCancelJob(NotImplemented, _Query) result = gnt_job.CancelJobs(opts, [], cl=cl, _stdout_fn=self._ToStdout, _ask_fn=compat.partial(self._Ask, False)) self.assertEqual(result, constants.EXIT_CONFIRMATION) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.compat_unittest.py000075500000000000000000000111271476477700300235260ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the compat module""" import inspect import unittest from ganeti import compat import testutils class TestPartial(testutils.GanetiTestCase): def test(self): # Test standard version self._Test(compat.partial) # Test our version self._Test(compat._partial) def _Test(self, fn): def _TestFunc1(x, power=2): return x ** power cubic = fn(_TestFunc1, power=3) self.assertEqual(cubic(1), 1) self.assertEqual(cubic(3), 27) self.assertEqual(cubic(4), 64) def _TestFunc2(*args, **kwargs): return (args, kwargs) self.assertEqualValues(fn(_TestFunc2, "Hello", "World")("Foo"), (("Hello", "World", "Foo"), {})) self.assertEqualValues(fn(_TestFunc2, "Hello", xyz=123)("Foo"), (("Hello", "Foo"), {"xyz": 123})) self.assertEqualValues(fn(_TestFunc2, xyz=123)("Foo", xyz=999), (("Foo", ), {"xyz": 999,})) class TestTryToRoman(testutils.GanetiTestCase): """test the compat.TryToRoman function""" def setUp(self): testutils.GanetiTestCase.setUp(self) # Save the compat.roman module so we can alter it with a fake... self.compat_roman_module = compat.roman def tearDown(self): # ...and restore it at the end of the test compat.roman = self.compat_roman_module testutils.GanetiTestCase.tearDown(self) def testAFewIntegers(self): # This test only works is the roman module is installed if compat.roman is not None: # starting with roman 3.2 single-digit 0 now converts to 'N' instead of 0 self.assertIn(compat.TryToRoman(0), [0, 'N']) self.assertEqual(compat.TryToRoman(1), "I") self.assertEqual(compat.TryToRoman(4), "IV") self.assertEqual(compat.TryToRoman(5), "V") def testWithNoRoman(self): # compat.roman is saved/restored in setUp/tearDown compat.roman = None self.assertEqual(compat.TryToRoman(0), 0) self.assertEqual(compat.TryToRoman(1), 1) self.assertEqual(compat.TryToRoman(4), 4) self.assertEqual(compat.TryToRoman(5), 5) def testStrings(self): self.assertEqual(compat.TryToRoman("astring"), "astring") self.assertEqual(compat.TryToRoman("5"), "5") def testDontConvert(self): self.assertEqual(compat.TryToRoman(0, convert=False), 0) self.assertEqual(compat.TryToRoman(1, convert=False), 1) self.assertEqual(compat.TryToRoman(7, convert=False), 7) self.assertEqual(compat.TryToRoman("astring", convert=False), "astring") self.assertEqual(compat.TryToRoman("19", convert=False), "19") class TestUniqueFrozenset(unittest.TestCase): def testDuplicates(self): for values in [["", ""], ["Hello", "World", "Hello"]]: self.assertRaises(ValueError, compat.UniqueFrozenset, values) def testEmpty(self): self.assertEqual(compat.UniqueFrozenset([]), frozenset([])) def testUnique(self): self.assertEqual(compat.UniqueFrozenset([1, 2, 3]), frozenset([1, 2, 3])) def testGenerator(self): seq = ("Foo%s" % i for i in range(10)) self.assertTrue(inspect.isgenerator(seq)) self.assertFalse(isinstance(seq, (list, tuple))) self.assertEqual(compat.UniqueFrozenset(seq), frozenset(["Foo%s" % i for i in range(10)])) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.confd.client_unittest.py000075500000000000000000000173061476477700300246160ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the confd client module""" import socket import unittest from ganeti import confd from ganeti import constants from ganeti import errors import ganeti.confd.client import testutils class ResettableMock(object): def __init__(self, *args, **kwargs): self.Reset() def Reset(self): pass class MockLogger(ResettableMock): def Reset(self): self.debug_count = 0 self.warn_count = 0 self.error_count = 0 def debug(string): self.debug_count += 1 def warning(string): self.warn_count += 1 def error(string): self.error_count += 1 class MockConfdAsyncUDPClient(ResettableMock): def Reset(self): self.send_count = 0 self.last_address = '' self.last_port = -1 self.last_sent = '' def enqueue_send(self, address, port, payload): self.send_count += 1 self.last_payload = payload self.last_port = port self.last_address = address class MockCallback(ResettableMock): def Reset(self): self.call_count = 0 self.last_up = None def __call__(self, up): """Callback @type up: L{ConfdUpcallPayload} @param up: upper callback """ self.call_count += 1 self.last_up = up class MockTime(ResettableMock): def Reset(self): self.mytime = 1254213006.5175071 def time(self): return self.mytime def increase(self, delta): self.mytime += delta class _BaseClientTest: """Base class for client tests""" mc_list = None new_peers = None family = None def setUp(self): self.mock_time = MockTime() confd.client.time = self.mock_time confd.client.ConfdAsyncUDPClient = MockConfdAsyncUDPClient self.logger = MockLogger() hmac_key = "mykeydata" self.callback = MockCallback() self.client = confd.client.ConfdClient(hmac_key, self.mc_list, self.callback, logger=self.logger) def testRequest(self): req1 = confd.client.ConfdClientRequest(type=constants.CONFD_REQ_PING) req2 = confd.client.ConfdClientRequest(type=constants.CONFD_REQ_PING) self.assertNotEqual(req1.rsalt, req2.rsalt) self.assertEqual(req1.protocol, constants.CONFD_PROTOCOL_VERSION) self.assertEqual(req2.protocol, constants.CONFD_PROTOCOL_VERSION) self.assertRaises(errors.ConfdClientError, confd.client.ConfdClientRequest, type=-33) def testClientSend(self): req = confd.client.ConfdClientRequest(type=constants.CONFD_REQ_PING) self.client.SendRequest(req) # Cannot send the same request twice self.assertRaises(errors.ConfdClientError, self.client.SendRequest, req) req2 = confd.client.ConfdClientRequest(type=constants.CONFD_REQ_PING) # Coverage is too big self.assertRaises(errors.ConfdClientError, self.client.SendRequest, req2, coverage=15) self.assertEqual(self.client._socket.send_count, constants.CONFD_DEFAULT_REQ_COVERAGE) # Send with max coverage self.client.SendRequest(req2, coverage=-1) self.assertEqual(self.client._socket.send_count, constants.CONFD_DEFAULT_REQ_COVERAGE + len(self.mc_list)) self.assertTrue(self.client._socket.last_address in self.mc_list) def testClientExpire(self): req = confd.client.ConfdClientRequest(type=constants.CONFD_REQ_PING) self.client.SendRequest(req) # Make a couple of seconds pass ;) self.mock_time.increase(2) # Now sending the second request req2 = confd.client.ConfdClientRequest(type=constants.CONFD_REQ_PING) self.client.SendRequest(req2) self.mock_time.increase(constants.CONFD_CLIENT_EXPIRE_TIMEOUT - 1) # First request should be expired, second one should not self.client.ExpireRequests() self.assertEqual(self.callback.call_count, 1) self.assertEqual(self.callback.last_up.type, confd.client.UPCALL_EXPIRE) self.assertEqual(self.callback.last_up.salt, req.rsalt) self.assertEqual(self.callback.last_up.orig_request, req) self.mock_time.increase(3) self.assertEqual(self.callback.call_count, 1) self.client.ExpireRequests() self.assertEqual(self.callback.call_count, 2) self.assertEqual(self.callback.last_up.type, confd.client.UPCALL_EXPIRE) self.assertEqual(self.callback.last_up.salt, req2.rsalt) self.assertEqual(self.callback.last_up.orig_request, req2) def testClientCascadeExpire(self): req = confd.client.ConfdClientRequest(type=constants.CONFD_REQ_PING) self.client.SendRequest(req) self.mock_time.increase(constants.CONFD_CLIENT_EXPIRE_TIMEOUT +1) req2 = confd.client.ConfdClientRequest(type=constants.CONFD_REQ_PING) self.client.SendRequest(req2) self.assertEqual(self.callback.call_count, 1) def testUpdatePeerList(self): self.client.UpdatePeerList(self.new_peers) self.assertEqual(self.client._peers, self.new_peers) req = confd.client.ConfdClientRequest(type=constants.CONFD_REQ_PING) self.client.SendRequest(req) self.assertEqual(self.client._socket.send_count, len(self.new_peers)) self.assertTrue(self.client._socket.last_address in self.new_peers) def testSetPeersFamily(self): self.client._SetPeersAddressFamily() self.assertEqual(self.client._family, self.family) mixed_peers = ["192.0.2.99", "2001:db8:beef::13"] self.client.UpdatePeerList(mixed_peers) self.assertRaises(errors.ConfdClientError, self.client._SetPeersAddressFamily) class TestIP4Client(unittest.TestCase, _BaseClientTest): """Client tests""" mc_list = ["192.0.2.1", "192.0.2.2", "192.0.2.3", "192.0.2.4", "192.0.2.5", "192.0.2.6", "192.0.2.7", "192.0.2.8", "192.0.2.9", ] new_peers = ["198.51.100.1", "198.51.100.2"] family = socket.AF_INET def setUp(self): unittest.TestCase.setUp(self) _BaseClientTest.setUp(self) class TestIP6Client(unittest.TestCase, _BaseClientTest): """Client tests""" mc_list = ["2001:db8::1", "2001:db8::2", "2001:db8::3", "2001:db8::4", "2001:db8::5", "2001:db8::6", "2001:db8::7", "2001:db8::8", "2001:db8::9", ] new_peers = ["2001:db8:beef::11", "2001:db8:beef::12"] family = socket.AF_INET6 def setUp(self): unittest.TestCase.setUp(self) _BaseClientTest.setUp(self) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.config_unittest.py000075500000000000000000000614211476477700300235120ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the config module""" import unittest import os import tempfile import operator from unittest import mock from ganeti import bootstrap from ganeti import config from ganeti import constants from ganeti import errors from ganeti import objects from ganeti import utils from ganeti import netutils from ganeti import compat from ganeti import serializer from ganeti.config import TemporaryReservationManager import testutils import mocks from testutils.config_mock import ConfigMock, _UpdateIvNames def _StubGetEntResolver(): return mocks.FakeGetentResolver() class TestConfigRunner(unittest.TestCase): """Testing case for HooksRunner""" def setUp(self): fd, self.cfg_file = tempfile.mkstemp() os.close(fd) self._init_cluster(self.cfg_file) def tearDown(self): try: os.unlink(self.cfg_file) except OSError: pass def _get_object(self): """Returns an instance of ConfigWriter""" cfg = config.ConfigWriter(cfg_file=self.cfg_file, offline=True, _getents=_StubGetEntResolver) return cfg def _get_object_mock(self): """Returns a mocked instance of ConfigWriter""" cfg = ConfigMock(cfg_file=self.cfg_file) return cfg def _init_cluster(self, cfg): """Initializes the cfg object""" me = netutils.Hostname() ip = constants.IP4_ADDRESS_LOCALHOST # master_ip must not conflict with the node ip address master_ip = "127.0.0.2" cluster_config = objects.Cluster( serial_no=1, rsahostkeypub="", dsahostkeypub="", highest_used_port=(constants.FIRST_DRBD_PORT - 1), mac_prefix="aa:00:00", volume_group_name="xenvg", drbd_usermode_helper="/bin/true", nicparams={constants.PP_DEFAULT: constants.NICC_DEFAULTS}, ndparams=constants.NDC_DEFAULTS, tcpudp_port_pool=set(), enabled_hypervisors=[constants.HT_FAKE], master_node=me.name, master_ip=master_ip, master_netdev=constants.DEFAULT_BRIDGE, cluster_name="cluster.local", file_storage_dir="/tmp", uid_pool=[], ) master_node_config = objects.Node(name=me.name, primary_ip=me.ip, secondary_ip=ip, serial_no=1, master_candidate=True) bootstrap.InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config, self.cfg_file) def _create_instance(self, cfg): """Create and return an instance object""" inst = objects.Instance(name="test.example.com", uuid="test-uuid", disks=[], nics=[], disk_template=constants.DT_DISKLESS, primary_node=cfg.GetMasterNode(), osparams_private=serializer.PrivateDict(), beparams={}) return inst def testEmpty(self): """Test instantiate config object""" self._get_object() def testInit(self): """Test initialize the config file""" cfg = self._get_object() self.assertEqual(1, len(cfg.GetNodeList())) self.assertEqual(0, len(cfg.GetInstanceList())) def _GenericNodesCheck(self, iobj, all_nodes, secondary_nodes): for i in [all_nodes, secondary_nodes]: self.assertTrue(isinstance(i, (list, tuple)), msg="Data type doesn't guarantee order") self.assertTrue(iobj.primary_node not in secondary_nodes) self.assertEqual(all_nodes[0], iobj.primary_node, msg="Primary node not first node in list") def _CreateInstanceDisk(self, cfg): # Construct instance and add a plain disk inst = self._create_instance(cfg) cfg.AddInstance(inst, "my-job") disk = objects.Disk(dev_type=constants.DT_PLAIN, size=128, logical_id=("myxenvg", "disk25494"), uuid="disk0", name="name0") cfg.AddInstanceDisk(inst.uuid, disk) return inst, disk def testDiskInfoByUUID(self): """Check if the GetDiskInfo works with UUIDs.""" # Create mock config writer cfg = self._get_object_mock() # Create an instance and attach a disk to it inst, disk = self._CreateInstanceDisk(cfg) result = cfg.GetDiskInfo("disk0") self.assertEqual(disk, result) def testDiskInfoByName(self): """Check if the GetDiskInfo works with names.""" # Create mock config writer cfg = self._get_object_mock() # Create an instance and attach a disk to it inst, disk = self._CreateInstanceDisk(cfg) result = cfg.GetDiskInfoByName("name0") self.assertEqual(disk, result) def testDiskInfoByWrongUUID(self): """Assert that GetDiskInfo raises an exception when given a wrong UUID.""" # Create mock config writer cfg = self._get_object_mock() # Create an instance and attach a disk to it inst, disk = self._CreateInstanceDisk(cfg) result = cfg.GetDiskInfo("disk1134") self.assertEqual(None, result) def testDiskInfoByWrongName(self): """Assert that GetDiskInfo returns None when given a wrong name.""" # Create mock config writer cfg = self._get_object_mock() # Create an instance and attach a disk to it inst, disk = self._CreateInstanceDisk(cfg) result = cfg.GetDiskInfoByName("name1134") self.assertEqual(None, result) def testDiskInfoDuplicateName(self): """Assert that GetDiskInfo raises exception on duplicate names.""" # Create mock config writer cfg = self._get_object_mock() # Create an instance and attach a disk to it inst, disk = self._CreateInstanceDisk(cfg) # Create a disk with the same name and attach it to the instance. disk = objects.Disk(dev_type=constants.DT_PLAIN, size=128, logical_id=("myxenvg", "disk25494"), uuid="disk1", name="name0") cfg.AddInstanceDisk(inst.uuid, disk) self.assertRaises(errors.ConfigurationError, cfg.GetDiskInfoByName, "name0") def testInstNodesNoDisks(self): """Test all_nodes/secondary_nodes when there are no disks""" # construct instance cfg = self._get_object_mock() inst = self._create_instance(cfg) cfg.AddInstance(inst, "my-job") # No disks all_nodes = cfg.GetInstanceNodes(inst.uuid) secondary_nodes = cfg.GetInstanceSecondaryNodes(inst.uuid) self._GenericNodesCheck(inst, all_nodes, secondary_nodes) self.assertEqual(len(secondary_nodes), 0) self.assertEqual(set(all_nodes), set([inst.primary_node])) self.assertEqual(cfg.GetInstanceLVsByNode(inst.uuid), { inst.primary_node: [], }) def testInstNodesPlainDisks(self): # construct instance cfg = self._get_object_mock() inst = self._create_instance(cfg) disks = [ objects.Disk(dev_type=constants.DT_PLAIN, size=128, logical_id=("myxenvg", "disk25494"), uuid="disk0"), objects.Disk(dev_type=constants.DT_PLAIN, size=512, logical_id=("myxenvg", "disk29071"), uuid="disk1"), ] cfg.AddInstance(inst, "my-job") for disk in disks: cfg.AddInstanceDisk(inst.uuid, disk) # Plain disks all_nodes = cfg.GetInstanceNodes(inst.uuid) secondary_nodes = cfg.GetInstanceSecondaryNodes(inst.uuid) self._GenericNodesCheck(inst, all_nodes, secondary_nodes) self.assertEqual(len(secondary_nodes), 0) self.assertEqual(set(all_nodes), set([inst.primary_node])) self.assertEqual(cfg.GetInstanceLVsByNode(inst.uuid), { inst.primary_node: ["myxenvg/disk25494", "myxenvg/disk29071"], }) def testInstNodesDrbdDisks(self): # construct a second node cfg = self._get_object_mock() node_group = cfg.LookupNodeGroup(None) master_uuid = cfg.GetMasterNode() node2 = objects.Node(name="node2.example.com", group=node_group, ndparams={}, uuid="node2-uuid") cfg.AddNode(node2, "my-job") # construct instance inst = self._create_instance(cfg) disks = [ objects.Disk(dev_type=constants.DT_DRBD8, size=786432, logical_id=(master_uuid, node2.uuid, 12300, 0, 0, "secret"), children=[ objects.Disk(dev_type=constants.DT_PLAIN, size=786432, logical_id=("myxenvg", "disk0"), uuid="data0"), objects.Disk(dev_type=constants.DT_PLAIN, size=128, logical_id=("myxenvg", "meta0"), uuid="meta0") ], iv_name="disk/0", uuid="disk0") ] cfg.AddInstance(inst, "my-job") for disk in disks: cfg.AddInstanceDisk(inst.uuid, disk) # Drbd Disks all_nodes = cfg.GetInstanceNodes(inst.uuid) secondary_nodes = cfg.GetInstanceSecondaryNodes(inst.uuid) self._GenericNodesCheck(inst, all_nodes, secondary_nodes) self.assertEqual(set(secondary_nodes), set([node2.uuid])) self.assertEqual(set(all_nodes), set([inst.primary_node, node2.uuid])) self.assertEqual(cfg.GetInstanceLVsByNode(inst.uuid), { master_uuid: ["myxenvg/disk0", "myxenvg/meta0"], node2.uuid: ["myxenvg/disk0", "myxenvg/meta0"], }) def testUpgradeSave(self): """Test that any modification done during upgrading is saved back""" cfg = self._get_object() # Remove an element, run upgrade, and check if the element is # back and the file upgraded node = cfg.GetNodeInfo(cfg.GetNodeList()[0]) # For a ConfigObject, None is the same as a missing field node.ndparams = None oldsaved = utils.ReadFile(self.cfg_file) cfg._UpgradeConfig(saveafter=True) self.assertTrue(node.ndparams is not None) newsaved = utils.ReadFile(self.cfg_file) # We rely on the fact that at least the serial number changes self.assertNotEqual(oldsaved, newsaved) # Add something that should not be there this time key = list(constants.NDC_GLOBALS)[0] node.ndparams[key] = constants.NDC_DEFAULTS[key] cfg._WriteConfig(None) oldsaved = utils.ReadFile(self.cfg_file) cfg._UpgradeConfig(saveafter=True) self.assertTrue(node.ndparams.get(key) is None) newsaved = utils.ReadFile(self.cfg_file) self.assertNotEqual(oldsaved, newsaved) # Do the upgrade again, this time there should be no update oldsaved = newsaved cfg._UpgradeConfig(saveafter=True) newsaved = utils.ReadFile(self.cfg_file) self.assertEqual(oldsaved, newsaved) # Reload the configuration again: it shouldn't change the file oldsaved = newsaved self._get_object() newsaved = utils.ReadFile(self.cfg_file) self.assertEqual(oldsaved, newsaved) def testNICParameterSyntaxCheck(self): """Test the NIC's CheckParameterSyntax function""" mode = constants.NIC_MODE link = constants.NIC_LINK m_bridged = constants.NIC_MODE_BRIDGED m_routed = constants.NIC_MODE_ROUTED CheckSyntax = objects.NIC.CheckParameterSyntax CheckSyntax(constants.NICC_DEFAULTS) CheckSyntax({mode: m_bridged, link: "br1"}) CheckSyntax({mode: m_routed, link: "default"}) self.assertRaises(errors.ConfigurationError, CheckSyntax, {mode: "000invalid", link: "any"}) self.assertRaises(errors.ConfigurationError, CheckSyntax, {mode: m_bridged, link: None}) self.assertRaises(errors.ConfigurationError, CheckSyntax, {mode: m_bridged, link: ""}) def testGetNdParamsDefault(self): cfg = self._get_object() node = cfg.GetNodeInfo(cfg.GetNodeList()[0]) self.assertEqual(cfg.GetNdParams(node), constants.NDC_DEFAULTS) def testGetNdParamsModifiedNode(self): my_ndparams = { constants.ND_OOB_PROGRAM: "/bin/node-oob", constants.ND_SPINDLE_COUNT: 1, constants.ND_EXCLUSIVE_STORAGE: False, constants.ND_OVS: True, constants.ND_OVS_NAME: "openvswitch", constants.ND_OVS_LINK: "eth1", constants.ND_SSH_PORT: 22, constants.ND_CPU_SPEED: 1.0, } cfg = self._get_object_mock() node = cfg.GetNodeInfo(cfg.GetNodeList()[0]) node.ndparams = my_ndparams cfg.Update(node, None) self.assertEqual(cfg.GetNdParams(node), my_ndparams) def testGetNdParamsInheritance(self): node_ndparams = { constants.ND_OOB_PROGRAM: "/bin/node-oob", constants.ND_OVS_LINK: "eth3" } group_ndparams = { constants.ND_SPINDLE_COUNT: 10, constants.ND_OVS: True, constants.ND_OVS_NAME: "openvswitch", constants.ND_SSH_PORT: 222, } expected_ndparams = { constants.ND_OOB_PROGRAM: "/bin/node-oob", constants.ND_SPINDLE_COUNT: 10, constants.ND_EXCLUSIVE_STORAGE: constants.NDC_DEFAULTS[constants.ND_EXCLUSIVE_STORAGE], constants.ND_OVS: True, constants.ND_OVS_NAME: "openvswitch", constants.ND_OVS_LINK: "eth3", constants.ND_SSH_PORT: 222, constants.ND_CPU_SPEED: 1.0, } cfg = self._get_object_mock() node = cfg.GetNodeInfo(cfg.GetNodeList()[0]) node.ndparams = node_ndparams cfg.Update(node, None) group = cfg.GetNodeGroup(node.group) group.ndparams = group_ndparams cfg.Update(group, None) self.assertEqual(cfg.GetNdParams(node), expected_ndparams) def testAddGroupFillsFieldsIfMissing(self): cfg = self._get_object() group = objects.NodeGroup(name="test", members=[]) cfg.AddNodeGroup(group, "my-job") self.assertTrue(utils.UUID_RE.match(group.uuid)) self.assertEqual(constants.ALLOC_POLICY_PREFERRED, group.alloc_policy) def testAddGroupPreservesFields(self): cfg = self._get_object() group = objects.NodeGroup(name="test", members=[], alloc_policy=constants.ALLOC_POLICY_LAST_RESORT) cfg.AddNodeGroup(group, "my-job") self.assertEqual(constants.ALLOC_POLICY_LAST_RESORT, group.alloc_policy) def testAddGroupDoesNotPreserveFields(self): cfg = self._get_object() group = objects.NodeGroup(name="test", members=[], serial_no=17, ctime=123, mtime=456) cfg.AddNodeGroup(group, "my-job") self.assertEqual(1, group.serial_no) self.assertTrue(group.ctime > 1200000000) self.assertTrue(group.mtime > 1200000000) def testAddGroupCanSkipUUIDCheck(self): cfg = self._get_object() uuid = cfg.GenerateUniqueID("my-job") group = objects.NodeGroup(name="test", members=[], uuid=uuid, serial_no=17, ctime=123, mtime=456) self.assertRaises(errors.ConfigurationError, cfg.AddNodeGroup, group, "my-job") cfg.AddNodeGroup(group, "my-job", check_uuid=False) # Does not raise. self.assertEqual(uuid, group.uuid) def testAssignGroupNodes(self): me = netutils.Hostname() cfg = self._get_object() # Create two groups grp1 = objects.NodeGroup(name="grp1", members=[], uuid="2f2fadf7-2a70-4a23-9ab5-2568c252032c") grp1_serial = 1 cfg.AddNodeGroup(grp1, "job") grp2 = objects.NodeGroup(name="grp2", members=[], uuid="798d0de3-680f-4a0e-b29a-0f54f693b3f1") grp2_serial = 1 cfg.AddNodeGroup(grp2, "job") self.assertEqual(set(ng.name for ng in cfg.GetAllNodeGroupsInfo().values()), set(["grp1", "grp2", constants.INITIAL_NODE_GROUP_NAME])) # No-op cluster_serial = cfg.GetClusterInfo().serial_no cfg.AssignGroupNodes([]) cluster_serial += 1 # Create two nodes node1 = objects.Node(name="node1", group=grp1.uuid, ndparams={}, uuid="node1-uuid") node1_serial = 1 node2 = objects.Node(name="node2", group=grp2.uuid, ndparams={}, uuid="node2-uuid") node2_serial = 1 cfg.AddNode(node1, "job") cfg.AddNode(node2, "job") cluster_serial += 2 self.assertEqual(set(cfg.GetNodeList()), set(["node1-uuid", "node2-uuid", cfg.GetNodeInfoByName(me.name).uuid])) (grp1, grp2) = [cfg.GetNodeGroup(grp.uuid) for grp in (grp1, grp2)] def _VerifySerials(): self.assertEqual(cfg.GetClusterInfo().serial_no, cluster_serial) self.assertEqual(node1.serial_no, node1_serial) self.assertEqual(node2.serial_no, node2_serial) self.assertEqual(grp1.serial_no, grp1_serial) self.assertEqual(grp2.serial_no, grp2_serial) _VerifySerials() self.assertEqual(set(grp1.members), set(["node1-uuid"])) self.assertEqual(set(grp2.members), set(["node2-uuid"])) # Check invalid nodes and groups self.assertRaises(errors.ConfigurationError, cfg.AssignGroupNodes, [ ("unknown.node.example.com", grp2.uuid), ]) self.assertRaises(errors.ConfigurationError, cfg.AssignGroupNodes, [ (node1.name, "unknown-uuid"), ]) self.assertEqual(node1.group, grp1.uuid) self.assertEqual(node2.group, grp2.uuid) self.assertEqual(set(grp1.members), set(["node1-uuid"])) self.assertEqual(set(grp2.members), set(["node2-uuid"])) # Another no-op cfg.AssignGroupNodes([]) cluster_serial += 1 _VerifySerials() # Assign to the same group (should be a no-op) self.assertEqual(node2.group, grp2.uuid) cfg.AssignGroupNodes([ (node2.uuid, grp2.uuid), ]) cluster_serial += 1 self.assertEqual(node2.group, grp2.uuid) _VerifySerials() self.assertEqual(set(grp1.members), set(["node1-uuid"])) self.assertEqual(set(grp2.members), set(["node2-uuid"])) # Assign node 2 to group 1 self.assertEqual(node2.group, grp2.uuid) cfg.AssignGroupNodes([ (node2.uuid, grp1.uuid), ]) (grp1, grp2) = [cfg.GetNodeGroup(grp.uuid) for grp in (grp1, grp2)] (node1, node2) = [cfg.GetNodeInfo(node.uuid) for node in (node1, node2)] cluster_serial += 1 node2_serial += 1 grp1_serial += 1 grp2_serial += 1 self.assertEqual(node2.group, grp1.uuid) _VerifySerials() self.assertEqual(set(grp1.members), set(["node1-uuid", "node2-uuid"])) self.assertFalse(grp2.members) # And assign both nodes to group 2 self.assertEqual(node1.group, grp1.uuid) self.assertEqual(node2.group, grp1.uuid) self.assertNotEqual(grp1.uuid, grp2.uuid) cfg.AssignGroupNodes([ (node1.uuid, grp2.uuid), (node2.uuid, grp2.uuid), ]) (grp1, grp2) = [cfg.GetNodeGroup(grp.uuid) for grp in (grp1, grp2)] (node1, node2) = [cfg.GetNodeInfo(node.uuid) for node in (node1, node2)] cluster_serial += 1 node1_serial += 1 node2_serial += 1 grp1_serial += 1 grp2_serial += 1 self.assertEqual(node1.group, grp2.uuid) self.assertEqual(node2.group, grp2.uuid) _VerifySerials() self.assertFalse(grp1.members) self.assertEqual(set(grp2.members), set(["node1-uuid", "node2-uuid"])) # Tests for Ssconf helper functions def testUnlockedGetHvparamsString(self): hvparams = {"a": "A", "b": "B", "c": "C"} hvname = "myhv" cfg_writer = self._get_object() cfg_writer._SetConfigData(mock.Mock()) cfg_writer._ConfigData().cluster = mock.Mock() cfg_writer._ConfigData().cluster.hvparams = {hvname: hvparams} result = cfg_writer._UnlockedGetHvparamsString(hvname) self.assertTrue("a=A" in result) lines = [line for line in result.split('\n') if line != ''] self.assertEqual(len(hvparams), len(lines)) def testExtendByAllHvparamsStrings(self): all_hvparams = {constants.HT_XEN_PVM: "foo"} ssconf_values = {} cfg_writer = self._get_object() cfg_writer._ExtendByAllHvparamsStrings(ssconf_values, all_hvparams) expected_key = constants.SS_HVPARAMS_PREF + constants.HT_XEN_PVM self.assertTrue(expected_key in ssconf_values) def testAddAndRemoveCerts(self): cfg = self._get_object() self.assertEqual(0, len(cfg.GetCandidateCerts())) node_uuid = "1234" cert_digest = "foobar" cfg.AddNodeToCandidateCerts(node_uuid, cert_digest, warn_fn=None, info_fn=None) self.assertEqual(1, len(cfg.GetCandidateCerts())) # Try adding the same cert again cfg.AddNodeToCandidateCerts(node_uuid, cert_digest, warn_fn=None, info_fn=None) self.assertEqual(1, len(cfg.GetCandidateCerts())) self.assertTrue(cfg.GetCandidateCerts()[node_uuid] == cert_digest) # Overriding cert other_digest = "barfoo" cfg.AddNodeToCandidateCerts(node_uuid, other_digest, warn_fn=None, info_fn=None) self.assertEqual(1, len(cfg.GetCandidateCerts())) self.assertTrue(cfg.GetCandidateCerts()[node_uuid] == other_digest) # Try removing a certificate from a node that is not in the list other_node_uuid = "5678" cfg.RemoveNodeFromCandidateCerts(other_node_uuid, warn_fn=None) self.assertEqual(1, len(cfg.GetCandidateCerts())) # Remove a certificate from a node that is in the list cfg.RemoveNodeFromCandidateCerts(node_uuid, warn_fn=None) self.assertEqual(0, len(cfg.GetCandidateCerts())) def testAttachDetachDisks(self): """Test if the attach/detach wrappers work properly. This test checks if the configuration remains in a consistent state after a series of detach/attach ops """ # construct instance cfg = self._get_object_mock() inst = self._create_instance(cfg) disk = objects.Disk(dev_type=constants.DT_PLAIN, size=128, logical_id=("myxenvg", "disk25494"), uuid="disk0") cfg.AddInstance(inst, "my-job") cfg.AddInstanceDisk(inst.uuid, disk) # Detach disk from non-existent instance self.assertRaises(errors.ConfigurationError, cfg.DetachInstanceDisk, "1134", "disk0") # Detach non-existent disk self.assertRaises(errors.ConfigurationError, cfg.DetachInstanceDisk, "test-uuid", "disk1") # Detach disk cfg.DetachInstanceDisk("test-uuid", "disk0") instance_disks = cfg.GetInstanceDisks("test-uuid") self.assertEqual(instance_disks, []) # Detach disk again self.assertRaises(errors.ProgrammerError, cfg.DetachInstanceDisk, "test-uuid", "disk0") # Attach disk cfg.AttachInstanceDisk("test-uuid", "disk0") instance_disks = cfg.GetInstanceDisks("test-uuid") self.assertEqual(instance_disks, [disk]) def _IsErrorInList(err_str, err_list): return any((err_str in e) for e in err_list) class TestTRM(unittest.TestCase): EC_ID = 1 def testEmpty(self): t = TemporaryReservationManager() t.Reserve(self.EC_ID, "a") self.assertFalse(t.Reserved(self.EC_ID)) self.assertTrue(t.Reserved("a")) self.assertEqual(len(t.GetReserved()), 1) def testDuplicate(self): t = TemporaryReservationManager() t.Reserve(self.EC_ID, "a") self.assertRaises(errors.ReservationError, t.Reserve, 2, "a") t.DropECReservations(self.EC_ID) self.assertFalse(t.Reserved("a")) class TestCheckInstanceDiskIvNames(unittest.TestCase): @staticmethod def _MakeDisks(names): return [objects.Disk(iv_name=name) for name in names] def testNoError(self): disks = self._MakeDisks(["disk/0", "disk/1"]) self.assertEqual(config._CheckInstanceDiskIvNames(disks), []) _UpdateIvNames(0, disks) self.assertEqual(config._CheckInstanceDiskIvNames(disks), []) def testWrongNames(self): disks = self._MakeDisks(["disk/1", "disk/3", "disk/2"]) self.assertEqual(config._CheckInstanceDiskIvNames(disks), [ (0, "disk/0", "disk/1"), (1, "disk/1", "disk/3"), ]) # Fix names _UpdateIvNames(0, disks) self.assertEqual(config._CheckInstanceDiskIvNames(disks), []) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.constants_unittest.py000075500000000000000000000157751476477700300242740ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the constants module""" import unittest import re import itertools from ganeti import constants from ganeti import locking from ganeti import utils from ganeti.utils import version import testutils class TestConstants(unittest.TestCase): """Constants tests""" def testConfigVersion(self): self.assertTrue(constants.CONFIG_MAJOR >= 0 and constants.CONFIG_MAJOR <= 99) self.assertTrue(constants.CONFIG_MINOR >= 0 and constants.CONFIG_MINOR <= 99) self.assertTrue(constants.CONFIG_REVISION >= 0 and constants.CONFIG_REVISION <= 9999) self.assertTrue(constants.CONFIG_VERSION >= 0 and constants.CONFIG_VERSION <= 99999999) self.assertTrue(version.BuildVersion(0, 0, 0) == 0) self.assertTrue(version.BuildVersion(10, 10, 1010) == 10101010) self.assertTrue(version.BuildVersion(12, 34, 5678) == 12345678) self.assertTrue(version.BuildVersion(99, 99, 9999) == 99999999) self.assertTrue(version.SplitVersion(00000000) == (0, 0, 0)) self.assertTrue(version.SplitVersion(10101010) == (10, 10, 1010)) self.assertTrue(version.SplitVersion(12345678) == (12, 34, 5678)) self.assertTrue(version.SplitVersion(99999999) == (99, 99, 9999)) self.assertTrue(version.SplitVersion(constants.CONFIG_VERSION) == (constants.CONFIG_MAJOR, constants.CONFIG_MINOR, constants.CONFIG_REVISION)) def testDiskStatus(self): self.assertTrue(constants.LDS_OKAY < constants.LDS_UNKNOWN) self.assertTrue(constants.LDS_UNKNOWN < constants.LDS_FAULTY) def testClockSkew(self): self.assertTrue(constants.NODE_MAX_CLOCK_SKEW < (0.8 * constants.CONFD_MAX_CLOCK_SKEW)) def testSslCertExpiration(self): self.assertTrue(constants.SSL_CERT_EXPIRATION_ERROR < constants.SSL_CERT_EXPIRATION_WARN) def testOpCodePriority(self): self.assertTrue(constants.OP_PRIO_LOWEST > constants.OP_PRIO_LOW) self.assertTrue(constants.OP_PRIO_LOW > constants.OP_PRIO_NORMAL) self.assertEqual(constants.OP_PRIO_NORMAL, locking._DEFAULT_PRIORITY) self.assertEqual(constants.OP_PRIO_DEFAULT, locking._DEFAULT_PRIORITY) self.assertTrue(constants.OP_PRIO_NORMAL > constants.OP_PRIO_HIGH) self.assertTrue(constants.OP_PRIO_HIGH > constants.OP_PRIO_HIGHEST) def testDiskDefaults(self): self.assertTrue( set(constants.DISK_LD_DEFAULTS.keys()) == set(constants.DISK_TEMPLATES) - set([constants.DT_DISKLESS])) self.assertTrue(set(constants.DISK_DT_DEFAULTS.keys()) == constants.DISK_TEMPLATES) def testJobStatus(self): self.assertFalse(constants.JOBS_PENDING & constants.JOBS_FINALIZED) self.assertFalse(constants.JOBS_PENDING - constants.JOB_STATUS_ALL) self.assertFalse(constants.JOBS_FINALIZED - constants.JOB_STATUS_ALL) def testDefaultsForAllHypervisors(self): self.assertEqual(frozenset(constants.HVC_DEFAULTS), constants.HYPER_TYPES) def testDefaultHypervisor(self): self.assertTrue(constants.DEFAULT_ENABLED_HYPERVISOR in constants.HYPER_TYPES) class TestExportedNames(unittest.TestCase): _VALID_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]+$") _BUILTIN_NAME_RE = re.compile(r"^__\w+__$") _EXCEPTIONS = frozenset([ "SplitVersion", "BuildVersion", ]) def test(self): wrong = \ set(itertools.filterfalse(self._BUILTIN_NAME_RE.match, itertools.filterfalse(self._VALID_NAME_RE.match, dir(constants)))) wrong -= self._EXCEPTIONS self.assertFalse(wrong, msg=("Invalid names exported from constants module: %s" % utils.CommaJoin(sorted(wrong)))) class TestParameterNames(unittest.TestCase): """HV/BE parameter tests""" VALID_NAME = re.compile("^[a-zA-Z_][a-zA-Z0-9_]*$") def testNoDashes(self): for kind, source in [("hypervisor", constants.HVS_PARAMETER_TYPES), ("backend", constants.BES_PARAMETER_TYPES), ("nic", constants.NICS_PARAMETER_TYPES), ("instdisk", constants.IDISK_PARAMS_TYPES), ("instnic", constants.INIC_PARAMS_TYPES), ]: for key in source: self.assertTrue(self.VALID_NAME.match(key), "The %s parameter '%s' contains invalid characters" % (kind, key)) class TestConfdConstants(unittest.TestCase): """Test the confd constants""" def testFourCc(self): self.assertEqual(len(constants.CONFD_MAGIC_FOURCC_BYTES), 4, msg="Invalid fourcc len, should be 4") def testReqs(self): self.assertFalse(utils.FindDuplicates(constants.CONFD_REQS), msg="Duplicated confd request code") def testReplStatuses(self): self.assertFalse(utils.FindDuplicates(constants.CONFD_REPL_STATUSES), msg="Duplicated confd reply status code") class TestDiskTemplateConstants(unittest.TestCase): def testPreference(self): self.assertEqual(set(constants.DISK_TEMPLATE_PREFERENCE), set(constants.DISK_TEMPLATES)) def testMapToStorageTypes(self): for disk_template in constants.DISK_TEMPLATES: self.assertTrue( constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[disk_template] is not None) def testLvmDiskTemplates(self): lvm_by_storage_type = [ dt for dt in constants.DISK_TEMPLATES if constants.ST_LVM_VG == constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[dt]] self.assertEqual(set(lvm_by_storage_type), set(constants.DTS_LVM)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.daemon_unittest.py000075500000000000000000000204011476477700300235010ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the daemon module""" import unittest import signal import os import socket import time import tempfile import shutil from ganeti import daemon from ganeti import errors from ganeti import constants from ganeti import utils import testutils class _MyAsyncUDPSocket(daemon.AsyncUDPSocket): def __init__(self, family): daemon.AsyncUDPSocket.__init__(self, family) self.received = [] self.error_count = 0 def handle_datagram(self, payload, ip, port): payload = payload.decode("utf-8") self.received.append(payload) if payload == "terminate": os.kill(os.getpid(), signal.SIGTERM) elif payload == "error": raise errors.GenericError("error") def handle_error(self): self.error_count += 1 raise class _BaseAsyncUDPSocketTest: """Base class for AsyncUDPSocket tests""" family = None address = None def setUp(self): self.mainloop = daemon.Mainloop() self.server = _MyAsyncUDPSocket(self.family) self.client = _MyAsyncUDPSocket(self.family) self.server.bind((self.address, 0)) self.port = self.server.socket.getsockname()[1] # Save utils.IgnoreSignals so we can do evil things to it... self.saved_utils_ignoresignals = utils.IgnoreSignals def tearDown(self): self.server.close() self.client.close() # ...and restore it as well utils.IgnoreSignals = self.saved_utils_ignoresignals testutils.GanetiTestCase.tearDown(self) def testNoDoubleBind(self): self.assertRaises(socket.error, self.client.bind, (self.address, self.port)) def testAsyncClientServer(self): self.client.enqueue_send(self.address, self.port, "p1") self.client.enqueue_send(self.address, self.port, "p2") self.client.enqueue_send(self.address, self.port, "terminate") self.mainloop.Run() self.assertEqual(self.server.received, ["p1", "p2", "terminate"]) def testSyncClientServer(self): self.client.handle_write() self.client.enqueue_send(self.address, self.port, "p1") self.client.enqueue_send(self.address, self.port, "p2") while self.client.writable(): self.client.handle_write() self.server.process_next_packet() self.assertEqual(self.server.received, ["p1"]) self.server.process_next_packet() self.assertEqual(self.server.received, ["p1", "p2"]) self.client.enqueue_send(self.address, self.port, "p3") while self.client.writable(): self.client.handle_write() self.server.process_next_packet() self.assertEqual(self.server.received, ["p1", "p2", "p3"]) def testErrorHandling(self): self.client.enqueue_send(self.address, self.port, "p1") self.client.enqueue_send(self.address, self.port, "p2") self.client.enqueue_send(self.address, self.port, "error") self.client.enqueue_send(self.address, self.port, "p3") self.client.enqueue_send(self.address, self.port, "error") self.client.enqueue_send(self.address, self.port, "terminate") self.assertRaises(errors.GenericError, self.mainloop.Run) self.assertEqual(self.server.received, ["p1", "p2", "error"]) self.assertEqual(self.server.error_count, 1) self.assertRaises(errors.GenericError, self.mainloop.Run) self.assertEqual(self.server.received, ["p1", "p2", "error", "p3", "error"]) self.assertEqual(self.server.error_count, 2) self.mainloop.Run() self.assertEqual(self.server.received, ["p1", "p2", "error", "p3", "error", "terminate"]) self.assertEqual(self.server.error_count, 2) def testSignaledWhileReceiving(self): utils.IgnoreSignals = lambda fn, *args, **kwargs: None self.client.enqueue_send(self.address, self.port, "p1") self.client.enqueue_send(self.address, self.port, "p2") self.server.handle_read() self.assertEqual(self.server.received, []) self.client.enqueue_send(self.address, self.port, "terminate") utils.IgnoreSignals = self.saved_utils_ignoresignals self.mainloop.Run() self.assertEqual(self.server.received, ["p1", "p2", "terminate"]) def testOversizedDatagram(self): oversized_data = (constants.MAX_UDP_DATA_SIZE + 1) * "a" self.assertRaises(errors.UdpDataSizeError, self.client.enqueue_send, self.address, self.port, oversized_data) class TestAsyncIP4UDPSocket(testutils.GanetiTestCase, _BaseAsyncUDPSocketTest): """Test IP4 daemon.AsyncUDPSocket""" family = socket.AF_INET address = "127.0.0.1" def setUp(self): testutils.GanetiTestCase.setUp(self) _BaseAsyncUDPSocketTest.setUp(self) def tearDown(self): testutils.GanetiTestCase.tearDown(self) _BaseAsyncUDPSocketTest.tearDown(self) @testutils.RequiresIPv6() class TestAsyncIP6UDPSocket(testutils.GanetiTestCase, _BaseAsyncUDPSocketTest): """Test IP6 daemon.AsyncUDPSocket""" family = socket.AF_INET6 address = "::1" def setUp(self): testutils.GanetiTestCase.setUp(self) _BaseAsyncUDPSocketTest.setUp(self) def tearDown(self): testutils.GanetiTestCase.tearDown(self) _BaseAsyncUDPSocketTest.tearDown(self) class TestAsyncAwaker(testutils.GanetiTestCase): """Test daemon.AsyncAwaker""" family = socket.AF_INET def setUp(self): testutils.GanetiTestCase.setUp(self) self.mainloop = daemon.Mainloop() self.awaker = daemon.AsyncAwaker(signal_fn=self.handle_signal) self.signal_count = 0 self.signal_terminate_count = 1 def tearDown(self): self.awaker.close() def handle_signal(self): self.signal_count += 1 self.signal_terminate_count -= 1 if self.signal_terminate_count <= 0: os.kill(os.getpid(), signal.SIGTERM) def testBasicSignaling(self): self.awaker.signal() self.mainloop.Run() self.assertEqual(self.signal_count, 1) def testDoubleSignaling(self): self.awaker.signal() self.awaker.signal() self.mainloop.Run() # The second signal is never delivered self.assertEqual(self.signal_count, 1) def testReallyDoubleSignaling(self): self.assertTrue(self.awaker.readable()) self.awaker.signal() # Let's suppose two threads overlap, and both find need_signal True self.awaker.need_signal = True self.awaker.signal() self.mainloop.Run() # We still get only one signaling self.assertEqual(self.signal_count, 1) def testNoSignalFnArgument(self): myawaker = daemon.AsyncAwaker() self.assertRaises(socket.error, myawaker.handle_read) myawaker.signal() myawaker.handle_read() self.assertRaises(socket.error, myawaker.handle_read) myawaker.signal() myawaker.signal() myawaker.handle_read() self.assertRaises(socket.error, myawaker.handle_read) myawaker.close() def testWrongSignalFnArgument(self): self.assertRaises(AssertionError, daemon.AsyncAwaker, 1) self.assertRaises(AssertionError, daemon.AsyncAwaker, "string") self.assertRaises(AssertionError, daemon.AsyncAwaker, signal_fn=1) self.assertRaises(AssertionError, daemon.AsyncAwaker, signal_fn="string") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.errors_unittest.py000075500000000000000000000070201476477700300235540ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.backend""" import os import sys import unittest from ganeti import errors import testutils class TestErrors(testutils.GanetiTestCase): def testGetErrorClass(self): tdata = { "": None, ".": None, "-": None, "ECODE_INVAL": None, "NoErrorClassName": None, "GenericError": errors.GenericError, "ProgrammerError": errors.ProgrammerError, } for name, cls in tdata.items(): self.assertTrue(errors.GetErrorClass(name) is cls) def testEncodeException(self): self.assertEqualValues(errors.EncodeException(Exception("Foobar")), ("Exception", ("Foobar", ))) err = errors.GenericError(True, 100, "foo", ["x", "y"]) self.assertEqualValues(errors.EncodeException(err), ("GenericError", (True, 100, "foo", ["x", "y"]))) def testMaybeRaise(self): testvals = [None, 1, 2, 3, "Hello World", (1, ), (1, 2, 3), ("NoErrorClassName", []), ("NoErrorClassName", None), ("GenericError", [1, 2, 3], None), ("GenericError", 1)] # These shouldn't raise for i in testvals: errors.MaybeRaise(i) self.assertRaises(errors.GenericError, errors.MaybeRaise, ("GenericError", ["Hello"])) # Check error encoding for i in testvals: src = errors.GenericError(i) try: errors.MaybeRaise(errors.EncodeException(src)) except errors.GenericError as dst: self.assertEqual(src.args, dst.args) self.assertEqual(src.__class__, dst.__class__) else: self.fail("Exception %s not raised" % repr(src)) def testGetEncodedError(self): self.assertEqualValues(errors.GetEncodedError(["GenericError", ("Hello", 123, "World")]), (errors.GenericError, ("Hello", 123, "World"))) self.assertEqualValues(errors.GetEncodedError(["GenericError", []]), (errors.GenericError, ())) self.assertFalse(errors.GetEncodedError(["NoErrorClass", ("Hello", 123, "World")])) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.hooks_unittest.py000075500000000000000000000463331476477700300233750ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the hooks module""" import unittest import os import time import tempfile import os.path from ganeti import errors from ganeti import opcodes from ganeti import hooksmaster from ganeti import backend from ganeti import constants from ganeti import cmdlib from ganeti.rpc import node as rpc from ganeti import compat from ganeti import pathutils from ganeti.constants import HKR_SUCCESS, HKR_FAIL, HKR_SKIP from mocks import FakeConfig, FakeProc, FakeContext import testutils class FakeLU(cmdlib.LogicalUnit): HPATH = "test" def BuildHooksEnv(self): return {} def BuildHooksNodes(self): return ["a"], ["a"] class TestHooksRunner(unittest.TestCase): """Testing case for HooksRunner""" def setUp(self): self.torm = [] self.tmpdir = tempfile.mkdtemp() self.torm.append((self.tmpdir, True)) self.logdir = tempfile.mkdtemp() self.torm.append((self.logdir, True)) self.hpath = "fake" self.ph_dirs = {} for i in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): dname = "%s/%s-%s.d" % (self.tmpdir, self.hpath, i) os.mkdir(dname) self.torm.append((dname, True)) self.ph_dirs[i] = dname self.hr = backend.HooksRunner(hooks_base_dir=self.tmpdir) def tearDown(self): self.torm.reverse() for path, kind in self.torm: if kind: os.rmdir(path) else: os.unlink(path) def _rname(self, fname): return "/".join(fname.split("/")[-2:]) def testEmpty(self): """Test no hooks""" for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): self.assertEqual(self.hr.RunHooks(self.hpath, phase, {}), []) def testSkipNonExec(self): """Test skip non-exec file""" for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): fname = "%s/test" % self.ph_dirs[phase] f = open(fname, "w") f.close() self.torm.append((fname, False)) self.assertEqual(self.hr.RunHooks(self.hpath, phase, {}), [(self._rname(fname), HKR_SKIP, "")]) def testSkipInvalidName(self): """Test skip script with invalid name""" for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): fname = "%s/a.off" % self.ph_dirs[phase] f = open(fname, "w") f.write("#!/bin/sh\nexit 0\n") f.close() os.chmod(fname, 0o700) self.torm.append((fname, False)) self.assertEqual(self.hr.RunHooks(self.hpath, phase, {}), [(self._rname(fname), HKR_SKIP, "")]) def testSkipDir(self): """Test skip directory""" for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): fname = "%s/testdir" % self.ph_dirs[phase] os.mkdir(fname) self.torm.append((fname, True)) self.assertEqual(self.hr.RunHooks(self.hpath, phase, {}), [(self._rname(fname), HKR_SKIP, "")]) def testSuccess(self): """Test success execution""" for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): fname = "%s/success" % self.ph_dirs[phase] f = open(fname, "w") f.write("#!/bin/sh\nexit 0\n") f.close() self.torm.append((fname, False)) os.chmod(fname, 0o700) self.assertEqual(self.hr.RunHooks(self.hpath, phase, {}), [(self._rname(fname), HKR_SUCCESS, "")]) def testSymlink(self): """Test running a symlink""" for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): fname = "%s/success" % self.ph_dirs[phase] os.symlink("/bin/true", fname) self.torm.append((fname, False)) self.assertEqual(self.hr.RunHooks(self.hpath, phase, {}), [(self._rname(fname), HKR_SUCCESS, "")]) def testFail(self): """Test success execution""" for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): fname = "%s/success" % self.ph_dirs[phase] f = open(fname, "w") f.write("#!/bin/sh\nexit 1\n") f.close() self.torm.append((fname, False)) os.chmod(fname, 0o700) self.assertEqual(self.hr.RunHooks(self.hpath, phase, {}), [(self._rname(fname), HKR_FAIL, "")]) def testCombined(self): """Test success, failure and skip all in one test""" for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): expect = [] for fbase, ecode, rs in [("00succ", 0, HKR_SUCCESS), ("10fail", 1, HKR_FAIL), ("20inv.", 0, HKR_SKIP), ]: fname = "%s/%s" % (self.ph_dirs[phase], fbase) f = open(fname, "w") f.write("#!/bin/sh\nexit %d\n" % ecode) f.close() self.torm.append((fname, False)) os.chmod(fname, 0o700) expect.append((self._rname(fname), rs, "")) self.assertEqual(self.hr.RunHooks(self.hpath, phase, {}), expect) def testOrdering(self): for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): expect = [] for fbase in ["10s1", "00s0", "10sa", "80sc", "60sd", ]: fname = "%s/%s" % (self.ph_dirs[phase], fbase) os.symlink("/bin/true", fname) self.torm.append((fname, False)) expect.append((self._rname(fname), HKR_SUCCESS, "")) expect.sort() self.assertEqual(self.hr.RunHooks(self.hpath, phase, {}), expect) def testEnv(self): """Test environment execution""" for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): fbase = "success" fname = "%s/%s" % (self.ph_dirs[phase], fbase) os.symlink("/usr/bin/env", fname) self.torm.append((fname, False)) env_snt = {"PHASE": phase} env_exp = "PHASE=%s" % phase self.assertEqual(self.hr.RunHooks(self.hpath, phase, env_snt), [(self._rname(fname), HKR_SUCCESS, env_exp)]) def FakeHooksRpcSuccess(node_list, hpath, phase, env): """Fake call_hooks_runner function. @rtype: dict of node -> L{rpc.RpcResult} with a successful script result @return: script execution from all nodes """ rr = rpc.RpcResult return dict([(node, rr((True, [("utest", constants.HKR_SUCCESS, "ok")]), node=node, call="FakeScriptOk")) for node in node_list]) class TestHooksMaster(unittest.TestCase): """Testing case for HooksMaster""" def _call_false(*args): """Fake call_hooks_runner function which returns False.""" return False @staticmethod def _call_nodes_false(node_list, hpath, phase, env): """Fake call_hooks_runner function. @rtype: dict of node -> L{rpc.RpcResult} with an rpc error @return: rpc failure from all nodes """ return dict([(node, rpc.RpcResult("error", failed=True, node=node, call="FakeError")) for node in node_list]) @staticmethod def _call_script_fail(node_list, hpath, phase, env): """Fake call_hooks_runner function. @rtype: dict of node -> L{rpc.RpcResult} with a failed script result @return: script execution failure from all nodes """ rr = rpc.RpcResult return dict([(node, rr((True, [("utest", constants.HKR_FAIL, "err")]), node=node, call="FakeScriptFail")) for node in node_list]) def setUp(self): self.op = opcodes.OpCode() # WARNING: here we pass None as RpcRunner instance since we know # our usage via HooksMaster will not use lu.rpc self.lu = FakeLU(FakeProc(), self.op, FakeConfig(), None, (123, "/foo/bar"), None) def testTotalFalse(self): """Test complete rpc failure""" hm = hooksmaster.HooksMaster.BuildFromLu(self._call_false, self.lu) self.assertRaises(errors.HooksFailure, hm.RunPhase, constants.HOOKS_PHASE_PRE) hm.RunPhase(constants.HOOKS_PHASE_POST) def testIndividualFalse(self): """Test individual node failure""" hm = hooksmaster.HooksMaster.BuildFromLu(self._call_nodes_false, self.lu) hm.RunPhase(constants.HOOKS_PHASE_PRE) #self.failUnlessRaises(errors.HooksFailure, # hm.RunPhase, constants.HOOKS_PHASE_PRE) hm.RunPhase(constants.HOOKS_PHASE_POST) def testScriptFalse(self): """Test individual rpc failure""" hm = hooksmaster.HooksMaster.BuildFromLu(self._call_script_fail, self.lu) self.assertRaises(errors.HooksAbort, hm.RunPhase, constants.HOOKS_PHASE_PRE) hm.RunPhase(constants.HOOKS_PHASE_POST) def testScriptSucceed(self): """Test individual rpc failure""" hm = hooksmaster.HooksMaster.BuildFromLu(FakeHooksRpcSuccess, self.lu) for phase in (constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST): hm.RunPhase(phase) class FakeEnvLU(cmdlib.LogicalUnit): HPATH = "env_test_lu" HTYPE = constants.HTYPE_GROUP def __init__(self, *args): cmdlib.LogicalUnit.__init__(self, *args) self.hook_env = None def BuildHooksEnv(self): assert self.hook_env is not None return self.hook_env def BuildHooksNodes(self): return (["a"], ["a"]) class FakeNoHooksLU(cmdlib.NoHooksLU): pass class TestHooksRunnerEnv(unittest.TestCase): def setUp(self): self._rpcs = [] self.op = opcodes.OpTestDummy(result=False, messages=[], fail=False) self.lu = FakeEnvLU(FakeProc(), self.op, FakeContext(), None) def _HooksRpc(self, *args): self._rpcs.append(args) return FakeHooksRpcSuccess(*args) def _CheckEnv(self, env, phase, hpath): self.assertTrue(env["PATH"].startswith("/sbin")) self.assertEqual(env["GANETI_HOOKS_PHASE"], phase) self.assertEqual(env["GANETI_HOOKS_PATH"], hpath) self.assertEqual(env["GANETI_OP_CODE"], self.op.OP_ID) self.assertEqual(env["GANETI_HOOKS_VERSION"], str(constants.HOOKS_VERSION)) self.assertEqual(env["GANETI_DATA_DIR"], pathutils.DATA_DIR) if "GANETI_OBJECT_TYPE" in env: self.assertEqual(env["GANETI_OBJECT_TYPE"], constants.HTYPE_GROUP) else: self.assertTrue(self.lu.HTYPE is None) def testEmptyEnv(self): # Check pre-phase hook self.lu.hook_env = {} hm = hooksmaster.HooksMaster.BuildFromLu(self._HooksRpc, self.lu) hm.RunPhase(constants.HOOKS_PHASE_PRE) (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(node_list, set(["node_a.example.com"])) self.assertEqual(hpath, self.lu.HPATH) self.assertEqual(phase, constants.HOOKS_PHASE_PRE) self._CheckEnv(env, constants.HOOKS_PHASE_PRE, self.lu.HPATH) # Check post-phase hook self.lu.hook_env = {} hm.RunPhase(constants.HOOKS_PHASE_POST) (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(node_list, set(["node_a.example.com"])) self.assertEqual(hpath, self.lu.HPATH) self.assertEqual(phase, constants.HOOKS_PHASE_POST) self._CheckEnv(env, constants.HOOKS_PHASE_POST, self.lu.HPATH) self.assertRaises(IndexError, self._rpcs.pop) def testEnv(self): # Check pre-phase hook self.lu.hook_env = { "FOO": "pre-foo-value", } hm = hooksmaster.HooksMaster.BuildFromLu(self._HooksRpc, self.lu) hm.RunPhase(constants.HOOKS_PHASE_PRE) (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(node_list, set(["node_a.example.com"])) self.assertEqual(hpath, self.lu.HPATH) self.assertEqual(phase, constants.HOOKS_PHASE_PRE) self.assertEqual(env["GANETI_FOO"], "pre-foo-value") self.assertFalse(compat.any(key.startswith("GANETI_POST") for key in env)) self._CheckEnv(env, constants.HOOKS_PHASE_PRE, self.lu.HPATH) # Check post-phase hook self.lu.hook_env = { "FOO": "post-value", "BAR": 123, } hm.RunPhase(constants.HOOKS_PHASE_POST) (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(node_list, set(["node_a.example.com"])) self.assertEqual(hpath, self.lu.HPATH) self.assertEqual(phase, constants.HOOKS_PHASE_POST) self.assertEqual(env["GANETI_FOO"], "pre-foo-value") self.assertEqual(env["GANETI_POST_FOO"], "post-value") self.assertEqual(env["GANETI_POST_BAR"], "123") self.assertFalse("GANETI_BAR" in env) self._CheckEnv(env, constants.HOOKS_PHASE_POST, self.lu.HPATH) self.assertRaises(IndexError, self._rpcs.pop) # Check configuration update hook hm.RunConfigUpdate() (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(set(node_list), set([self.lu.cfg.GetMasterNodeName()])) self.assertEqual(hpath, constants.HOOKS_NAME_CFGUPDATE) self.assertEqual(phase, constants.HOOKS_PHASE_POST) self._CheckEnv(env, constants.HOOKS_PHASE_POST, constants.HOOKS_NAME_CFGUPDATE) self.assertFalse(compat.any(key.startswith("GANETI_POST") for key in env)) self.assertEqual(env["GANETI_FOO"], "pre-foo-value") self.assertRaises(IndexError, self._rpcs.pop) def testConflict(self): for name in ["DATA_DIR", "OP_CODE"]: self.lu.hook_env = { name: "value" } # Test using a clean HooksMaster instance hm = hooksmaster.HooksMaster.BuildFromLu(self._HooksRpc, self.lu) for phase in [constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST]: self.assertRaises(AssertionError, hm.RunPhase, phase) self.assertRaises(IndexError, self._rpcs.pop) def testNoNodes(self): self.lu.hook_env = {} hm = hooksmaster.HooksMaster.BuildFromLu(self._HooksRpc, self.lu) hm.RunPhase(constants.HOOKS_PHASE_PRE, node_names=[]) self.assertRaises(IndexError, self._rpcs.pop) def testSpecificNodes(self): self.lu.hook_env = {} nodes = [ "node1.example.com", "node93782.example.net", ] hm = hooksmaster.HooksMaster.BuildFromLu(self._HooksRpc, self.lu) for phase in [constants.HOOKS_PHASE_PRE, constants.HOOKS_PHASE_POST]: hm.RunPhase(phase, node_names=nodes) (node_list, hpath, rpc_phase, env) = self._rpcs.pop(0) self.assertEqual(set(node_list), set(nodes)) self.assertEqual(hpath, self.lu.HPATH) self.assertEqual(rpc_phase, phase) self._CheckEnv(env, phase, self.lu.HPATH) self.assertRaises(IndexError, self._rpcs.pop) def testRunConfigUpdateNoPre(self): self.lu.hook_env = { "FOO": "value", } hm = hooksmaster.HooksMaster.BuildFromLu(self._HooksRpc, self.lu) hm.RunConfigUpdate() (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(set(node_list), set([self.lu.cfg.GetMasterNodeName()])) self.assertEqual(hpath, constants.HOOKS_NAME_CFGUPDATE) self.assertEqual(phase, constants.HOOKS_PHASE_POST) self.assertEqual(env["GANETI_FOO"], "value") self.assertFalse(compat.any(key.startswith("GANETI_POST") for key in env)) self._CheckEnv(env, constants.HOOKS_PHASE_POST, constants.HOOKS_NAME_CFGUPDATE) self.assertRaises(IndexError, self._rpcs.pop) def testNoPreBeforePost(self): self.lu.hook_env = { "FOO": "value", } hm = hooksmaster.HooksMaster.BuildFromLu(self._HooksRpc, self.lu) hm.RunPhase(constants.HOOKS_PHASE_POST) (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(node_list, set(["node_a.example.com"])) self.assertEqual(hpath, self.lu.HPATH) self.assertEqual(phase, constants.HOOKS_PHASE_POST) self.assertEqual(env["GANETI_FOO"], "value") self.assertEqual(env["GANETI_POST_FOO"], "value") self._CheckEnv(env, constants.HOOKS_PHASE_POST, self.lu.HPATH) self.assertRaises(IndexError, self._rpcs.pop) def testNoHooksLU(self): self.lu = FakeNoHooksLU(FakeProc(), self.op, FakeContext(), None) self.assertRaises(AssertionError, self.lu.BuildHooksEnv) self.assertRaises(AssertionError, self.lu.BuildHooksNodes) hm = hooksmaster.HooksMaster.BuildFromLu(self._HooksRpc, self.lu) self.assertEqual(hm.pre_env, {}) self.assertRaises(IndexError, self._rpcs.pop) hm.RunPhase(constants.HOOKS_PHASE_PRE) self.assertRaises(IndexError, self._rpcs.pop) hm.RunPhase(constants.HOOKS_PHASE_POST) self.assertRaises(IndexError, self._rpcs.pop) hm.RunConfigUpdate() (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(set(node_list), set([self.lu.cfg.GetMasterNodeName()])) self.assertEqual(hpath, constants.HOOKS_NAME_CFGUPDATE) self.assertEqual(phase, constants.HOOKS_PHASE_POST) self.assertFalse(compat.any(key.startswith("GANETI_POST") for key in env)) self._CheckEnv(env, constants.HOOKS_PHASE_POST, constants.HOOKS_NAME_CFGUPDATE) self.assertRaises(IndexError, self._rpcs.pop) assert isinstance(self.lu, FakeNoHooksLU), "LU was replaced" class FakeEnvWithCustomPostHookNodesLU(cmdlib.LogicalUnit): HPATH = "env_test_lu" HTYPE = constants.HTYPE_GROUP def __init__(self, *args): cmdlib.LogicalUnit.__init__(self, *args) def BuildHooksEnv(self): return {} def BuildHooksNodes(self): return (["a"], ["a"]) def PreparePostHookNodes(self, post_hook_node_uuids): return post_hook_node_uuids + ["b"] class TestHooksRunnerEnv(unittest.TestCase): def setUp(self): self._rpcs = [] self.op = opcodes.OpTestDummy(result=False, messages=[], fail=False) self.lu = FakeEnvWithCustomPostHookNodesLU(FakeProc(), self.op, FakeConfig(), None, (123, "/foo/bar"), None) def _HooksRpc(self, *args): self._rpcs.append(args) return FakeHooksRpcSuccess(*args) def testEmptyEnv(self): # Check pre-phase hook hm = hooksmaster.HooksMaster.BuildFromLu(self._HooksRpc, self.lu) hm.RunPhase(constants.HOOKS_PHASE_PRE) (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(node_list, set(["a"])) # Check post-phase hook hm.RunPhase(constants.HOOKS_PHASE_POST) (node_list, hpath, phase, env) = self._rpcs.pop(0) self.assertEqual(node_list, set(["a", "b"])) self.assertRaises(IndexError, self._rpcs.pop) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.ht_unittest.py000075500000000000000000000234751476477700300226670ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.ht""" import unittest from ganeti import constants from ganeti import ht import testutils class TestTypeChecks(unittest.TestCase): def testNone(self): self.assertFalse(ht.TNotNone(None)) self.assertTrue(ht.TNone(None)) for val in [0, True, "", "Hello World", [], range(5)]: self.assertTrue(ht.TNotNone(val)) self.assertFalse(ht.TNone(val)) def testBool(self): self.assertTrue(ht.TBool(True)) self.assertTrue(ht.TBool(False)) for val in [0, None, "", [], "Hello"]: self.assertFalse(ht.TBool(val)) for val in [True, -449, 1, 3, "x", "abc", [1, 2]]: self.assertTrue(ht.TTrue(val)) for val in [False, 0, None, []]: self.assertFalse(ht.TTrue(val)) def testInt(self): for val in [-100, -3, 0, 16, 128, 923874]: self.assertTrue(ht.TInt(val)) self.assertTrue(ht.TNumber(val)) for val in [False, True, None, "", [], "Hello", 0.0, 0.23, -3818.163]: self.assertFalse(ht.TInt(val)) for val in range(0, 100, 4): self.assertTrue(ht.TNonNegativeInt(val)) neg = -(val + 1) self.assertFalse(ht.TNonNegativeInt(neg)) self.assertFalse(ht.TPositiveInt(neg)) self.assertFalse(ht.TNonNegativeInt(0.1 + val)) self.assertFalse(ht.TPositiveInt(0.1 + val)) for val in [0, 0.1, 0.9, -0.3]: self.assertFalse(ht.TPositiveInt(val)) for val in range(1, 100, 4): self.assertTrue(ht.TPositiveInt(val)) self.assertFalse(ht.TPositiveInt(0.1 + val)) def testFloat(self): for val in [-100.21, -3.0, 0.0, 16.12, 128.3433, 923874.928]: self.assertTrue(ht.TFloat(val)) self.assertTrue(ht.TNumber(val)) for val in [False, True, None, "", [], "Hello", 0, 28, -1, -3281]: self.assertFalse(ht.TFloat(val)) def testNumber(self): for val in [-100, -3, 0, 16, 128, 923874, -100.21, -3.0, 0.0, 16.12, 128.3433, 923874.928]: self.assertTrue(ht.TNumber(val)) for val in [False, True, None, "", [], "Hello", "1"]: self.assertFalse(ht.TNumber(val)) def testString(self): for val in ["", "abc", "Hello World", "123", "", "\u272C", "abc"]: self.assertTrue(ht.TString(val)) for val in [False, True, None, [], 0, 1, 5, -193, 93.8582]: self.assertFalse(ht.TString(val)) def testElemOf(self): fn = ht.TElemOf(range(10)) self.assertTrue(fn(0)) self.assertTrue(fn(3)) self.assertTrue(fn(9)) self.assertFalse(fn(-1)) self.assertFalse(fn(100)) fn = ht.TElemOf([]) self.assertFalse(fn(0)) self.assertFalse(fn(100)) self.assertFalse(fn(True)) fn = ht.TElemOf(["Hello", "World"]) self.assertTrue(fn("Hello")) self.assertTrue(fn("World")) self.assertFalse(fn("e")) def testList(self): for val in [[], list(range(10)), ["Hello", "World", "!"]]: self.assertTrue(ht.TList(val)) for val in [False, True, None, {}, 0, 1, 5, -193, 93.8582]: self.assertFalse(ht.TList(val)) def testDict(self): for val in [{}, dict.fromkeys(range(10)), {"Hello": [], "World": "!"}]: self.assertTrue(ht.TDict(val)) for val in [False, True, None, [], 0, 1, 5, -193, 93.8582]: self.assertFalse(ht.TDict(val)) def testIsLength(self): fn = ht.TIsLength(10) self.assertTrue(fn(range(10))) self.assertFalse(fn(range(1))) self.assertFalse(fn(range(100))) def testAnd(self): fn = ht.TAnd(ht.TNotNone, ht.TString) self.assertTrue(fn("")) self.assertFalse(fn(1)) self.assertFalse(fn(None)) def testOr(self): fn = ht.TMaybe(ht.TAnd(ht.TString, ht.TIsLength(5))) self.assertTrue(fn("12345")) self.assertTrue(fn(None)) self.assertFalse(fn(1)) self.assertFalse(fn("")) self.assertFalse(fn("abc")) def testMap(self): self.assertTrue(ht.TMap(str, ht.TString)(123)) self.assertTrue(ht.TMap(int, ht.TInt)("9999")) self.assertFalse(ht.TMap(lambda x: x + 100, ht.TString)(123)) def testNonEmptyString(self): self.assertTrue(ht.TNonEmptyString("xyz")) self.assertTrue(ht.TNonEmptyString("Hello World")) self.assertFalse(ht.TNonEmptyString("")) self.assertFalse(ht.TNonEmptyString(None)) self.assertFalse(ht.TNonEmptyString([])) def testMaybeString(self): self.assertTrue(ht.TMaybeString("xyz")) self.assertTrue(ht.TMaybeString("Hello World")) self.assertTrue(ht.TMaybeString(None)) self.assertFalse(ht.TMaybeString("")) self.assertFalse(ht.TMaybeString([])) def testMaybeBool(self): self.assertTrue(ht.TMaybeBool(False)) self.assertTrue(ht.TMaybeBool(True)) self.assertTrue(ht.TMaybeBool(None)) self.assertFalse(ht.TMaybeBool([])) self.assertFalse(ht.TMaybeBool("0")) self.assertFalse(ht.TMaybeBool("False")) def testListOf(self): fn = ht.TListOf(ht.TNonEmptyString) self.assertTrue(fn([])) self.assertTrue(fn(["x"])) self.assertTrue(fn(["Hello", "World"])) self.assertFalse(fn(None)) self.assertFalse(fn(False)) self.assertFalse(fn(range(3))) self.assertFalse(fn(["x", None])) def testDictOf(self): fn = ht.TDictOf(ht.TNonEmptyString, ht.TInt) self.assertTrue(fn({})) self.assertTrue(fn({"x": 123, "y": 999})) self.assertFalse(fn(None)) self.assertFalse(fn({1: "x"})) self.assertFalse(fn({"x": ""})) self.assertFalse(fn({"x": None})) self.assertFalse(fn({"": 8234})) def testStrictDictRequireAllExclusive(self): fn = ht.TStrictDict(True, True, { "a": ht.TInt, }) self.assertFalse(fn(1)) self.assertFalse(fn(None)) self.assertFalse(fn({})) self.assertFalse(fn({"a": "Hello", })) self.assertFalse(fn({"unknown": 999,})) self.assertFalse(fn({"unknown": None,})) self.assertTrue(fn({"a": 123, })) self.assertTrue(fn({"a": -5, })) fn = ht.TStrictDict(True, True, { "a": ht.TInt, "x": ht.TString, }) self.assertFalse(fn({})) self.assertFalse(fn({"a": -5, })) self.assertTrue(fn({"a": 123, "x": "", })) self.assertFalse(fn({"a": 123, "x": None, })) def testStrictDictExclusive(self): fn = ht.TStrictDict(False, True, { "a": ht.TInt, "b": ht.TList, }) self.assertTrue(fn({})) self.assertTrue(fn({"a": 123, })) self.assertTrue(fn({"b": list(range(4)), })) self.assertFalse(fn({"b": 123, })) self.assertFalse(fn({"foo": {}, })) self.assertFalse(fn({"bar": object(), })) def testStrictDictRequireAll(self): fn = ht.TStrictDict(True, False, { "a": ht.TInt, "m": ht.TInt, }) self.assertTrue(fn({"a": 1, "m": 2, "bar": object(), })) self.assertFalse(fn({})) self.assertFalse(fn({"a": 1, "bar": object(), })) self.assertFalse(fn({"a": 1, "m": [], "bar": object(), })) def testStrictDict(self): fn = ht.TStrictDict(False, False, { "a": ht.TInt, }) self.assertTrue(fn({})) self.assertFalse(fn({"a": ""})) self.assertTrue(fn({"a": 11})) self.assertTrue(fn({"other": 11})) self.assertTrue(fn({"other": object()})) def testJobId(self): for i in [0, 1, 4395, 2347625220]: self.assertTrue(ht.TJobId(i)) self.assertTrue(ht.TJobId(str(i))) self.assertFalse(ht.TJobId(-(i + 1))) for i in ["", "-", ".", ",", "a", "99j", "job-123", "\t", " 83 ", None, [], {}, object()]: self.assertFalse(ht.TJobId(i)) def testRelativeJobId(self): for i in [-1, -93, -4395]: self.assertTrue(ht.TRelativeJobId(i)) self.assertFalse(ht.TRelativeJobId(str(i))) for i in [0, 1, 2, 10, 9289, "", "0", "-1", "-999"]: self.assertFalse(ht.TRelativeJobId(i)) self.assertFalse(ht.TRelativeJobId(str(i))) def testItems(self): self.assertRaises(AssertionError, ht.TItems, []) fn = ht.TItems([ht.TString]) self.assertFalse(fn([0])) self.assertFalse(fn([None])) self.assertTrue(fn(["Hello"])) self.assertTrue(fn(["Hello", "World"])) self.assertTrue(fn(["Hello", 0, 1, 2, "anything"])) fn = ht.TItems([ht.TAny, ht.TInt, ht.TAny]) self.assertTrue(fn(["Hello", 0, []])) self.assertTrue(fn(["Hello", 893782])) self.assertTrue(fn([{}, -938210858947, None])) self.assertFalse(fn(["Hello", []])) def testInstanceOf(self): fn = ht.TInstanceOf(self.__class__) self.assertTrue(fn(self)) self.assertTrue(str(fn).startswith("Instance of ")) self.assertFalse(fn(None)) def testMaybeValueNone(self): fn = ht.TMaybeValueNone(ht.TInt) self.assertTrue(fn(None)) self.assertTrue(fn(0)) self.assertTrue(fn(constants.VALUE_NONE)) self.assertFalse(fn("")) self.assertFalse(fn([])) self.assertFalse(fn(constants.VALUE_DEFAULT)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.http_unittest.py000075500000000000000000000650211476477700300232240ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the http module""" import os import unittest import time import tempfile import pycurl import itertools import threading from io import StringIO from ganeti import http from ganeti import compat import ganeti.http.server import ganeti.http.client import ganeti.http.auth import testutils class TestStartLines(unittest.TestCase): """Test cases for start line classes""" def testClientToServerStartLine(self): """Test client to server start line (HTTP request)""" start_line = http.HttpClientToServerStartLine("GET", "/", "HTTP/1.1") self.assertEqual(str(start_line), "GET / HTTP/1.1") def testServerToClientStartLine(self): """Test server to client start line (HTTP response)""" start_line = http.HttpServerToClientStartLine("HTTP/1.1", 200, "OK") self.assertEqual(str(start_line), "HTTP/1.1 200 OK") class TestMisc(unittest.TestCase): """Miscellaneous tests""" def _TestDateTimeHeader(self, gmnow, expected): self.assertEqual(http.server._DateTimeHeader(gmnow=gmnow), expected) def testDateTimeHeader(self): """Test ganeti.http._DateTimeHeader""" self._TestDateTimeHeader((2008, 1, 2, 3, 4, 5, 3, 0, 0), "Thu, 02 Jan 2008 03:04:05 GMT") self._TestDateTimeHeader((2008, 1, 1, 0, 0, 0, 0, 0, 0), "Mon, 01 Jan 2008 00:00:00 GMT") self._TestDateTimeHeader((2008, 12, 31, 0, 0, 0, 0, 0, 0), "Mon, 31 Dec 2008 00:00:00 GMT") self._TestDateTimeHeader((2008, 12, 31, 23, 59, 59, 0, 0, 0), "Mon, 31 Dec 2008 23:59:59 GMT") self._TestDateTimeHeader((2008, 12, 31, 0, 0, 0, 6, 0, 0), "Sun, 31 Dec 2008 00:00:00 GMT") def testHttpServerRequest(self): """Test ganeti.http.server._HttpServerRequest""" server_request = \ http.server._HttpServerRequest("GET", "/", None, None, None) # These are expected by users of the HTTP server self.assertTrue(hasattr(server_request, "request_method")) self.assertTrue(hasattr(server_request, "request_path")) self.assertTrue(hasattr(server_request, "request_headers")) self.assertTrue(hasattr(server_request, "request_body")) self.assertTrue(isinstance(server_request.resp_headers, dict)) self.assertTrue(hasattr(server_request, "private")) def testServerSizeLimits(self): """Test HTTP server size limits""" message_reader_class = http.server._HttpClientToServerMessageReader self.assertTrue(message_reader_class.START_LINE_LENGTH_MAX > 0) self.assertTrue(message_reader_class.HEADER_LENGTH_MAX > 0) def testFormatAuthHeader(self): self.assertEqual(http.auth._FormatAuthHeader("Basic", {}), "Basic") self.assertEqual(http.auth._FormatAuthHeader("Basic", { "foo": "bar", }), "Basic foo=bar") self.assertEqual(http.auth._FormatAuthHeader("Basic", { "foo": "", }), "Basic foo=\"\"") self.assertEqual(http.auth._FormatAuthHeader("Basic", { "foo": "x,y", }), "Basic foo=\"x,y\"") params = { "foo": "x,y", "realm": "secure", } # It's a dict whose order isn't guaranteed, hence checking a list self.assertTrue(http.auth._FormatAuthHeader("Digest", params) in ("Digest foo=\"x,y\" realm=secure", "Digest realm=secure foo=\"x,y\"")) class _FakeRequestAuth(http.auth.HttpServerRequestAuthentication): def __init__(self, realm, authreq, authenticate_fn): http.auth.HttpServerRequestAuthentication.__init__(self) self.realm = realm self.authreq = authreq self.authenticate_fn = authenticate_fn def AuthenticationRequired(self, req): return self.authreq def GetAuthRealm(self, req): return self.realm def Authenticate(self, *args): if self.authenticate_fn: return self.authenticate_fn(*args) raise NotImplementedError() class TestAuth(unittest.TestCase): """Authentication tests""" hsra = http.auth.HttpServerRequestAuthentication def testConstants(self): for scheme in [self.hsra._CLEARTEXT_SCHEME, self.hsra._HA1_SCHEME]: self.assertEqual(scheme, scheme.upper()) self.assertTrue(scheme.startswith("{")) self.assertTrue(scheme.endswith("}")) def _testVerifyBasicAuthPassword(self, realm, user, password, expected): ra = _FakeRequestAuth(realm, False, None) return ra.VerifyBasicAuthPassword(None, user, password, expected) def testVerifyBasicAuthPassword(self): tvbap = self._testVerifyBasicAuthPassword good_pws = ["pw", "pw{", "pw}", "pw{}", "pw{x}y", "}pw", "0", "123", "foo...:xyz", "TeST"] for pw in good_pws: # Try cleartext passwords self.assertTrue(tvbap("abc", "user", pw, pw)) self.assertTrue(tvbap("abc", "user", pw, "{cleartext}" + pw)) self.assertTrue(tvbap("abc", "user", pw, "{ClearText}" + pw)) self.assertTrue(tvbap("abc", "user", pw, "{CLEARTEXT}" + pw)) # Try with invalid password self.assertFalse(tvbap("abc", "user", pw, "something")) # Try with invalid scheme self.assertFalse(tvbap("abc", "user", pw, "{000}" + pw)) self.assertFalse(tvbap("abc", "user", pw, "{unk}" + pw)) self.assertFalse(tvbap("abc", "user", pw, "{Unk}" + pw)) self.assertFalse(tvbap("abc", "user", pw, "{UNK}" + pw)) # Try with invalid scheme format self.assertFalse(tvbap("abc", "user", "pw", "{something")) # Hash is MD5("user:This is only a test:pw") self.assertTrue(tvbap("This is only a test", "user", "pw", "{ha1}92ea58ae804481498c257b2f65561a17")) self.assertTrue(tvbap("This is only a test", "user", "pw", "{HA1}92ea58ae804481498c257b2f65561a17")) self.assertRaises(AssertionError, tvbap, None, "user", "pw", "{HA1}92ea58ae804481498c257b2f65561a17") self.assertFalse(tvbap("Admin area", "user", "pw", "{HA1}92ea58ae804481498c257b2f65561a17")) self.assertFalse(tvbap("This is only a test", "someone", "pw", "{HA1}92ea58ae804481498c257b2f65561a17")) self.assertFalse(tvbap("This is only a test", "user", "something", "{HA1}92ea58ae804481498c257b2f65561a17")) class _SimpleAuthenticator: def __init__(self, user, password): self.user = user self.password = password self.called = False def __call__(self, req, user, password): self.called = True return self.user == user and self.password == password class TestHttpServerRequestAuthentication(unittest.TestCase): def testNoAuth(self): req = http.server._HttpServerRequest("GET", "/", None, None, None) _FakeRequestAuth("area1", False, None).PreHandleRequest(req) def testNoRealm(self): headers = { http.HTTP_AUTHORIZATION: "", } req = http.server._HttpServerRequest("GET", "/", headers, None, None) ra = _FakeRequestAuth(None, False, None) self.assertRaises(AssertionError, ra.PreHandleRequest, req) def testNoScheme(self): headers = { http.HTTP_AUTHORIZATION: "", } req = http.server._HttpServerRequest("GET", "/", headers, None, None) ra = _FakeRequestAuth("area1", False, None) self.assertRaises(http.HttpUnauthorized, ra.PreHandleRequest, req) def testUnknownScheme(self): headers = { http.HTTP_AUTHORIZATION: "NewStyleAuth abc", } req = http.server._HttpServerRequest("GET", "/", headers, None, None) ra = _FakeRequestAuth("area1", False, None) self.assertRaises(http.HttpUnauthorized, ra.PreHandleRequest, req) def testInvalidBase64(self): headers = { http.HTTP_AUTHORIZATION: "Basic x_=_", } req = http.server._HttpServerRequest("GET", "/", headers, None, None) ra = _FakeRequestAuth("area1", False, None) self.assertRaises(http.HttpUnauthorized, ra.PreHandleRequest, req) def testAuthForPublicResource(self): headers = { http.HTTP_AUTHORIZATION: "Basic %s" % testutils.b64encode_string("foo"), } req = http.server._HttpServerRequest("GET", "/", headers, None, None) ra = _FakeRequestAuth("area1", False, None) self.assertRaises(http.HttpUnauthorized, ra.PreHandleRequest, req) def testAuthForPublicResource(self): headers = { http.HTTP_AUTHORIZATION: "Basic %s" % testutils.b64encode_string("foo:bar"), } req = http.server._HttpServerRequest("GET", "/", headers, None, None) ac = _SimpleAuthenticator("foo", "bar") ra = _FakeRequestAuth("area1", False, ac) ra.PreHandleRequest(req) req = http.server._HttpServerRequest("GET", "/", headers, None, None) ac = _SimpleAuthenticator("something", "else") ra = _FakeRequestAuth("area1", False, ac) self.assertRaises(http.HttpUnauthorized, ra.PreHandleRequest, req) def testInvalidRequestHeader(self): checks = { http.HttpUnauthorized: ["", "\t", "-", ".", "@", "<", ">", "Digest", "basic %s" % testutils.b64encode_string("foobar")], http.HttpBadRequest: ["Basic"], } for exc, headers in checks.items(): for i in headers: headers = { http.HTTP_AUTHORIZATION: i, } req = http.server._HttpServerRequest("GET", "/", headers, None, None) ra = _FakeRequestAuth("area1", False, None) self.assertRaises(exc, ra.PreHandleRequest, req) def testBasicAuth(self): for user in ["", "joe", "user name with spaces"]: for pw in ["", "-", ":", "foobar", "Foo Bar Baz", "@@@", "###", "foo:bar:baz"]: for wrong_pw in [True, False]: basic_auth = "%s:%s" % (user, pw) if wrong_pw: basic_auth += "WRONG" headers = { http.HTTP_AUTHORIZATION: "Basic %s" % testutils.b64encode_string(basic_auth), } req = http.server._HttpServerRequest("GET", "/", headers, None, None) ac = _SimpleAuthenticator(user, pw) self.assertFalse(ac.called) ra = _FakeRequestAuth("area1", True, ac) if wrong_pw: try: ra.PreHandleRequest(req) except http.HttpUnauthorized as err: www_auth = err.headers[http.HTTP_WWW_AUTHENTICATE] self.assertTrue(www_auth.startswith(http.auth.HTTP_BASIC_AUTH)) else: self.fail("Didn't raise HttpUnauthorized") else: ra.PreHandleRequest(req) self.assertTrue(ac.called) class TestReadPasswordFile(unittest.TestCase): def testSimple(self): users = http.auth.ParsePasswordFile("user1 password") self.assertEqual(len(users), 1) self.assertEqual(users["user1"].password, "password") self.assertEqual(len(users["user1"].options), 0) def testOptions(self): buf = StringIO() buf.write("# Passwords\n") buf.write("user1 password\n") buf.write("\n") buf.write("# Comment\n") buf.write("user2 pw write,read\n") buf.write(" \t# Another comment\n") buf.write("invalidline\n") users = http.auth.ParsePasswordFile(buf.getvalue()) self.assertEqual(len(users), 2) self.assertEqual(users["user1"].password, "password") self.assertEqual(len(users["user1"].options), 0) self.assertEqual(users["user2"].password, "pw") self.assertEqual(users["user2"].options, ["write", "read"]) class TestClientRequest(unittest.TestCase): def testRepr(self): cr = http.client.HttpClientRequest("localhost", 1234, "GET", "/version", headers=[], post_data="Hello World") self.assertTrue(repr(cr).startswith("<")) def testNoHeaders(self): cr = http.client.HttpClientRequest("localhost", 1234, "GET", "/version", headers=None) self.assertTrue(isinstance(cr.headers, list)) self.assertEqual(cr.headers, []) self.assertEqual(cr.url, "https://localhost:1234/version") def testPlainAddressIPv4(self): cr = http.client.HttpClientRequest("192.0.2.9", 19956, "GET", "/version") self.assertEqual(cr.url, "https://192.0.2.9:19956/version") def testPlainAddressIPv6(self): cr = http.client.HttpClientRequest("2001:db8::cafe", 15110, "GET", "/info") self.assertEqual(cr.url, "https://[2001:db8::cafe]:15110/info") def testOldStyleHeaders(self): headers = { "Content-type": "text/plain", "Accept": "text/html", } cr = http.client.HttpClientRequest("localhost", 16481, "GET", "/vg_list", headers=headers) self.assertTrue(isinstance(cr.headers, list)) self.assertEqual(sorted(cr.headers), [ "Accept: text/html", "Content-type: text/plain", ]) self.assertEqual(cr.url, "https://localhost:16481/vg_list") def testNewStyleHeaders(self): headers = [ "Accept: text/html", "Content-type: text/plain; charset=ascii", "Server: httpd 1.0", ] cr = http.client.HttpClientRequest("localhost", 1234, "GET", "/version", headers=headers) self.assertTrue(isinstance(cr.headers, list)) self.assertEqual(sorted(cr.headers), sorted(headers)) self.assertEqual(cr.url, "https://localhost:1234/version") def testPostData(self): cr = http.client.HttpClientRequest("localhost", 1234, "GET", "/version", post_data="Hello World") self.assertEqual(cr.post_data, "Hello World") def testNoPostData(self): cr = http.client.HttpClientRequest("localhost", 1234, "GET", "/version") self.assertEqual(cr.post_data, "") def testCompletionCallback(self): for argname in ["completion_cb", "curl_config_fn"]: kwargs = { argname: NotImplementedError, } cr = http.client.HttpClientRequest("localhost", 14038, "GET", "/version", **kwargs) self.assertEqual(getattr(cr, argname), NotImplementedError) for fn in [NotImplemented, {}, 1]: kwargs = { argname: fn, } self.assertRaises(AssertionError, http.client.HttpClientRequest, "localhost", 23150, "GET", "/version", **kwargs) class _FakeCurl: def __init__(self): self.opts = {} self.info = NotImplemented def setopt(self, opt, value): assert opt not in self.opts, "Option set more than once" self.opts[opt] = value def getinfo(self, info): return self.info.pop(info) class TestClientStartRequest(unittest.TestCase): @staticmethod def _TestCurlConfig(curl): curl.setopt(pycurl.SSLKEYTYPE, "PEM") def test(self): for method in [http.HTTP_GET, http.HTTP_PUT, "CUSTOM"]: for port in [8761, 29796, 19528]: for curl_config_fn in [None, self._TestCurlConfig]: for read_timeout in [None, 0, 1, 123, 36000]: self._TestInner(method, port, curl_config_fn, read_timeout) def _TestInner(self, method, port, curl_config_fn, read_timeout): for response_code in [http.HTTP_OK, http.HttpNotFound.code, http.HTTP_NOT_MODIFIED]: for response_body in [None, "Hello World", "Very Long\tContent here\n" * 171]: for errmsg in [None, "error"]: req = http.client.HttpClientRequest("localhost", port, method, "/version", curl_config_fn=curl_config_fn, read_timeout=read_timeout) curl = _FakeCurl() pending = http.client._StartRequest(curl, req) self.assertEqual(pending.GetCurlHandle(), curl) self.assertEqual(pending.GetCurrentRequest(), req) # Check options opts = curl.opts self.assertEqual(opts.pop(pycurl.CUSTOMREQUEST), method) self.assertEqual(opts.pop(pycurl.URL), "https://localhost:%s/version" % port) if read_timeout is None: self.assertEqual(opts.pop(pycurl.TIMEOUT), 0) else: self.assertEqual(opts.pop(pycurl.TIMEOUT), read_timeout) self.assertFalse(opts.pop(pycurl.VERBOSE)) self.assertTrue(opts.pop(pycurl.NOSIGNAL)) self.assertEqual(opts.pop(pycurl.USERAGENT), http.HTTP_GANETI_VERSION) self.assertEqual(opts.pop(pycurl.PROXY), "") self.assertFalse(opts.pop(pycurl.POSTFIELDS)) self.assertFalse(opts.pop(pycurl.HTTPHEADER)) write_fn = opts.pop(pycurl.WRITEFUNCTION) self.assertTrue(callable(write_fn)) if hasattr(pycurl, "SSL_SESSIONID_CACHE"): self.assertFalse(opts.pop(pycurl.SSL_SESSIONID_CACHE)) if curl_config_fn: self.assertEqual(opts.pop(pycurl.SSLKEYTYPE), "PEM") else: self.assertFalse(pycurl.SSLKEYTYPE in opts) self.assertFalse(opts) if response_body is not None: offset = 0 while offset < len(response_body): piece = response_body[offset:offset + 10] write_fn(piece.encode("utf-8")) offset += len(piece) curl.info = { pycurl.RESPONSE_CODE: response_code, } if hasattr(pycurl, 'LOCAL_IP'): curl.info[pycurl.LOCAL_IP] = '127.0.0.1' if hasattr(pycurl, 'LOCAL_PORT'): curl.info[pycurl.LOCAL_PORT] = port # Finalize request pending.Done(errmsg) self.assertFalse(curl.info) # Can only finalize once self.assertRaises(AssertionError, pending.Done, True) if errmsg: self.assertFalse(req.success) else: self.assertTrue(req.success) self.assertEqual(req.error, errmsg) self.assertEqual(req.resp_status_code, response_code) if response_body is None: self.assertEqual(req.resp_body, "") else: self.assertEqual(req.resp_body, response_body) # Check if resetting worked assert not hasattr(curl, "reset") opts = curl.opts self.assertFalse(opts.pop(pycurl.POSTFIELDS)) self.assertTrue(callable(opts.pop(pycurl.WRITEFUNCTION))) self.assertFalse(opts) self.assertFalse(curl.opts, msg="Previous checks did not consume all options") assert id(opts) == id(curl.opts) class _EmptyCurlMulti: def perform(self): return (pycurl.E_MULTI_OK, 0) def info_read(self): return (0, [], []) class TestClientProcessRequests(unittest.TestCase): def testEmpty(self): requests = [] http.client.ProcessRequests(requests, _curl=NotImplemented, _curl_multi=_EmptyCurlMulti) self.assertEqual(requests, []) class TestProcessCurlRequests(unittest.TestCase): class _FakeCurlMulti: def __init__(self): self.handles = [] self.will_fail = [] self._expect = ["perform"] self._counter = itertools.count() def add_handle(self, curl): assert curl not in self.handles self.handles.append(curl) if next(self._counter) % 3 == 0: self.will_fail.append(curl) def remove_handle(self, curl): self.handles.remove(curl) def perform(self): assert self._expect.pop(0) == "perform" if next(self._counter) % 2 == 0: self._expect.append("perform") return (pycurl.E_CALL_MULTI_PERFORM, None) self._expect.append("info_read") return (pycurl.E_MULTI_OK, len(self.handles)) def info_read(self): assert self._expect.pop(0) == "info_read" successful = [] failed = [] if self.handles: if next(self._counter) % 17 == 0: curl = self.handles[0] if curl in self.will_fail: failed.append((curl, -1, "test error")) else: successful.append(curl) remaining_messages = len(self.handles) % 3 if remaining_messages > 0: self._expect.append("info_read") else: self._expect.append("select") else: remaining_messages = 0 self._expect.append("select") return (remaining_messages, successful, failed) def select(self, timeout): # Never compare floats for equality assert timeout >= 0.95 and timeout <= 1.05 assert self._expect.pop(0) == "select" self._expect.append("perform") def test(self): requests = [_FakeCurl() for _ in range(10)] multi = self._FakeCurlMulti() for (curl, errmsg) in http.client._ProcessCurlRequests(multi, requests): self.assertTrue(curl not in multi.handles) if curl in multi.will_fail: self.assertTrue("test error" in errmsg) else: self.assertTrue(errmsg is None) self.assertFalse(multi.handles) self.assertEqual(multi._expect, ["select"]) class TestProcessRequests(unittest.TestCase): class _DummyCurlMulti: pass def testNoMonitor(self): self._Test(False) def testWithMonitor(self): self._Test(True) class _MonitorChecker: def __init__(self): self._monitor = None def GetMonitor(self): return self._monitor def __call__(self, monitor): assert callable(monitor.GetLockInfo) self._monitor = monitor def _Test(self, use_monitor): def cfg_fn(port, curl): curl.opts["__port__"] = port def _LockCheckReset(monitor, req): self.assertTrue(monitor._lock.is_owned(shared=0), msg="Lock must be owned in exclusive mode") assert not hasattr(req, "lockcheck__") setattr(req, "lockcheck__", True) def _BuildNiceName(port, default=None): if port % 5 == 0: return "nicename%s" % port else: # Use standard name return default requests = \ [http.client.HttpClientRequest("localhost", i, "POST", "/version%s" % i, curl_config_fn=compat.partial(cfg_fn, i), completion_cb=NotImplementedError, nicename=_BuildNiceName(i)) for i in range(15176, 15501)] requests_count = len(requests) if use_monitor: lock_monitor_cb = self._MonitorChecker() else: lock_monitor_cb = None def _ProcessRequests(multi, handles): self.assertTrue(isinstance(multi, self._DummyCurlMulti)) self.assertEqual(len(requests), len(handles)) self.assertTrue(compat.all(isinstance(curl, _FakeCurl) for curl in handles)) # Prepare for lock check for req in requests: assert req.completion_cb is NotImplementedError if use_monitor: req.completion_cb = \ compat.partial(_LockCheckReset, lock_monitor_cb.GetMonitor()) for idx, curl in enumerate(handles): try: port = curl.opts["__port__"] except KeyError: self.fail("Per-request config function was not called") if use_monitor: # Check if lock information is correct lock_info = lock_monitor_cb.GetMonitor().GetLockInfo(None) expected = \ [("rpc/%s" % (_BuildNiceName(handle.opts["__port__"], default=("localhost/version%s" % handle.opts["__port__"]))), None, [threading.current_thread().name], None) for handle in handles[idx:]] self.assertEqual(sorted(lock_info), sorted(expected)) if port % 3 == 0: response_code = http.HTTP_OK msg = None else: response_code = http.HttpNotFound.code msg = "test error" curl.info = { pycurl.RESPONSE_CODE: response_code, } if hasattr(pycurl, 'LOCAL_IP'): curl.info[pycurl.LOCAL_IP] = '127.0.0.1' if hasattr(pycurl, 'LOCAL_PORT'): curl.info[pycurl.LOCAL_PORT] = port # Prepare for reset self.assertFalse(curl.opts.pop(pycurl.POSTFIELDS)) self.assertTrue(callable(curl.opts.pop(pycurl.WRITEFUNCTION))) yield (curl, msg) if use_monitor: self.assertTrue(compat.all(req.lockcheck__ for req in requests)) if use_monitor: self.assertEqual(lock_monitor_cb.GetMonitor(), None) http.client.ProcessRequests(requests, lock_monitor_cb=lock_monitor_cb, _curl=_FakeCurl, _curl_multi=self._DummyCurlMulti, _curl_process=_ProcessRequests) for req in requests: if req.port % 3 == 0: self.assertTrue(req.success) self.assertEqual(req.error, None) else: self.assertFalse(req.success) self.assertTrue("test error" in req.error) # See if monitor was disabled if use_monitor: monitor = lock_monitor_cb.GetMonitor() self.assertEqual(monitor._pending_fn, None) self.assertEqual(monitor.GetLockInfo(None), []) else: self.assertEqual(lock_monitor_cb, None) self.assertEqual(len(requests), requests_count) def testBadRequest(self): bad_request = http.client.HttpClientRequest("localhost", 27784, "POST", "/version") bad_request.success = False self.assertRaises(AssertionError, http.client.ProcessRequests, [bad_request], _curl=NotImplemented, _curl_multi=NotImplemented, _curl_process=NotImplemented) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.hypervisor.hv_chroot_unittest.py000075500000000000000000000046111476477700300264470ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.hypervisor.hv_chroot""" import unittest import tempfile import shutil from ganeti import constants from ganeti import objects from ganeti import hypervisor from ganeti.hypervisor import hv_chroot import testutils class TestConsole(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def test(self): instance = objects.Instance(name="fake.example.com", primary_node="node837-uuid") node = objects.Node(name="node837", uuid="node837-uuid", ndparams={}) group = objects.NodeGroup(name="group164", ndparams={}) cons = hv_chroot.ChrootManager.GetInstanceConsole(instance, node, group, {}, {}, root_dir=self.tmpdir) self.assertEqual(cons.Validate(), None) self.assertEqual(cons.kind, constants.CONS_SSH) self.assertEqual(cons.host, node.name) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.hypervisor.hv_fake_unittest.py000075500000000000000000000040751476477700300260630ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.hypervisor.hv_fake""" import unittest from ganeti import constants from ganeti import objects from ganeti import hypervisor from ganeti.hypervisor import hv_fake import testutils class TestConsole(unittest.TestCase): def test(self): instance = objects.Instance(name="fake.example.com") node = objects.Node(name="fakenode.example.com", ndparams={}) group = objects.NodeGroup(name="default", ndparams={}) cons = hv_fake.FakeHypervisor.GetInstanceConsole(instance, node, group, {}, {}) self.assertEqual(cons.Validate(), None) self.assertEqual(cons.kind, constants.CONS_MESSAGE) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.hypervisor.hv_kvm_unittest.py000075500000000000000000000764341476477700300257620ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing the hypervisor.hv_kvm module""" import threading import tempfile import unittest import socket import os import struct import re from unittest import mock from ganeti import serializer from ganeti import constants from ganeti import compat from ganeti import objects from ganeti import errors from ganeti import utils from ganeti import pathutils from ganeti.hypervisor import hv_kvm import ganeti.hypervisor.hv_kvm.netdev as netdev import ganeti.hypervisor.hv_kvm.monitor as monitor import ganeti.hypervisor.hv_kvm.validation as validation import testutils from lib.hypervisor.hv_kvm.kvm_runtime import KVMRuntime from testutils.config_mock import ConfigMock class TestParameterCheck(testutils.GanetiTestCase): def testInvalidVncParameters(self): invalid_data = { constants.HV_VNC_X509_VERIFY: True, constants.HV_VNC_X509: None } self.assertRaises(errors.HypervisorError, validation.check_vnc_parameters, invalid_data) def testValidVncParameters(self): valid_data = { constants.HV_VNC_X509_VERIFY: True, constants.HV_VNC_X509: "mycert.pem" } self.assertTrue(validation.check_vnc_parameters(valid_data)) def testInvalidSecurityModel(self): invalid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_USER, constants.HV_SECURITY_DOMAIN: None } self.assertRaises(errors.HypervisorError, validation.check_security_model, invalid_data) invalid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_NONE, constants.HV_SECURITY_DOMAIN: "secure_user" } self.assertRaises(errors.HypervisorError, validation.check_security_model, invalid_data) invalid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_POOL, constants.HV_SECURITY_DOMAIN: "secure_user" } self.assertRaises(errors.HypervisorError, validation.check_security_model, invalid_data) def testValidSecurityModel(self): valid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_USER, constants.HV_SECURITY_DOMAIN: "secure_user" } self.assertTrue(validation.check_security_model(valid_data)) valid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_POOL, constants.HV_SECURITY_DOMAIN: None } self.assertTrue(validation.check_security_model(valid_data)) valid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_NONE, constants.HV_SECURITY_DOMAIN: None } self.assertTrue(validation.check_security_model(valid_data)) def testInvalidBootParameters(self): invalid_data = { constants.HV_BOOT_ORDER: constants.HT_BO_CDROM, constants.HV_CDROM_IMAGE_PATH: None, constants.HV_KERNEL_PATH: "/some/path", constants.HV_ROOT_PATH: "/" } self.assertRaises(errors.HypervisorError, validation.check_boot_parameters, invalid_data) invalid_data = { constants.HV_BOOT_ORDER: constants.HT_BO_CDROM, constants.HV_CDROM_IMAGE_PATH: "/cd.iso", constants.HV_KERNEL_PATH: "/some/path", constants.HV_ROOT_PATH: None } self.assertRaises(errors.HypervisorError, validation.check_boot_parameters, invalid_data) def testValidBootParameters(self): valid_data = { constants.HV_BOOT_ORDER: constants.HT_BO_CDROM, constants.HV_CDROM_IMAGE_PATH: "/cd.iso", constants.HV_KERNEL_PATH: "/some/path", constants.HV_ROOT_PATH: "/" } self.assertTrue(validation.check_boot_parameters(valid_data)) valid_data = { constants.HV_BOOT_ORDER: constants.HT_BO_DISK, constants.HV_CDROM_IMAGE_PATH: None, constants.HV_KERNEL_PATH: "/some/path", constants.HV_ROOT_PATH: "/" } self.assertTrue(validation.check_boot_parameters(valid_data)) valid_data = { constants.HV_BOOT_ORDER: constants.HT_BO_DISK, constants.HV_CDROM_IMAGE_PATH: None, constants.HV_KERNEL_PATH: None, constants.HV_ROOT_PATH: None } self.assertTrue(validation.check_boot_parameters(valid_data)) def testInvalidConsoleParameters(self): invalid_data = { constants.HV_SERIAL_CONSOLE: True, constants.HV_SERIAL_SPEED: None, } self.assertRaises(errors.HypervisorError, validation.check_console_parameters, invalid_data) invalid_data = { constants.HV_SERIAL_CONSOLE: True, constants.HV_SERIAL_SPEED: 1, } self.assertRaises(errors.HypervisorError, validation.check_console_parameters, invalid_data) def testValidConsoleParameters(self): valid_data = { constants.HV_SERIAL_CONSOLE: False } self.assertTrue(validation.check_console_parameters(valid_data)) for speed in constants.VALID_SERIAL_SPEEDS: valid_data = { constants.HV_SERIAL_CONSOLE: True, constants.HV_SERIAL_SPEED: speed } self.assertTrue(validation.check_console_parameters(valid_data), "Testing serial console speed %d" % speed) def testInvalidSpiceParameters(self): invalid_data = { constants.HV_KVM_SPICE_BIND: "0.0.0.0", constants.HV_KVM_SPICE_IP_VERSION: constants.IP6_VERSION } self.assertRaises(errors.HypervisorError, validation.check_spice_parameters, invalid_data) invalid_data = { constants.HV_KVM_SPICE_BIND: "::", constants.HV_KVM_SPICE_IP_VERSION: constants.IP4_VERSION } self.assertRaises(errors.HypervisorError, validation.check_spice_parameters, invalid_data) invalid_data = { constants.HV_KVM_SPICE_BIND: None, constants.HV_KVM_SPICE_IP_VERSION: None, constants.HV_KVM_SPICE_PASSWORD_FILE: "password.txt", constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR: None, constants.HV_KVM_SPICE_JPEG_IMG_COMPR: None, constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR: None, constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION: None, constants.HV_KVM_SPICE_USE_TLS: True } self.assertRaises(errors.HypervisorError, validation.check_spice_parameters, invalid_data) def testValidSpiceParameters(self): valid_data = { constants.HV_KVM_SPICE_BIND: None, constants.HV_KVM_SPICE_IP_VERSION: None, constants.HV_KVM_SPICE_PASSWORD_FILE: None, constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR: None, constants.HV_KVM_SPICE_JPEG_IMG_COMPR: None, constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR: None, constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION: None, constants.HV_KVM_SPICE_USE_TLS: None } self.assertTrue(validation.check_spice_parameters(valid_data)) valid_data = { constants.HV_KVM_SPICE_BIND: "0.0.0.0", constants.HV_KVM_SPICE_IP_VERSION: constants.IP4_VERSION, constants.HV_KVM_SPICE_PASSWORD_FILE: "password.txt", constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR: "glz", constants.HV_KVM_SPICE_JPEG_IMG_COMPR: "never", constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR: "never", constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION: "off", constants.HV_KVM_SPICE_USE_TLS: True } self.assertTrue(validation.check_spice_parameters(valid_data)) valid_data = { constants.HV_KVM_SPICE_BIND: "::", constants.HV_KVM_SPICE_IP_VERSION: constants.IP6_VERSION, constants.HV_KVM_SPICE_PASSWORD_FILE: "password.txt", constants.HV_KVM_SPICE_LOSSLESS_IMG_COMPR: "glz", constants.HV_KVM_SPICE_JPEG_IMG_COMPR: "never", constants.HV_KVM_SPICE_ZLIB_GLZ_IMG_COMPR: "never", constants.HV_KVM_SPICE_STREAMING_VIDEO_DETECTION: "off", constants.HV_KVM_SPICE_USE_TLS: True } self.assertTrue(validation.check_spice_parameters(valid_data)) def testInvalidDiskCacheParameters(self): invalid_data = { constants.HV_KVM_DISK_AIO: constants.HT_KVM_AIO_NATIVE, constants.HV_DISK_CACHE: constants.HT_CACHE_WBACK } self.assertRaises(errors.HypervisorError, validation.check_disk_cache_parameters, invalid_data) def testValidDiskCacheParameters(self): valid_data = { constants.HV_KVM_DISK_AIO: constants.HT_KVM_AIO_THREADS, constants.HV_DISK_CACHE: constants.HT_CACHE_WBACK } self.assertTrue(validation.check_disk_cache_parameters(valid_data)) valid_data = { constants.HV_KVM_DISK_AIO: constants.HT_KVM_AIO_THREADS, constants.HV_DISK_CACHE: constants.HT_CACHE_DEFAULT } self.assertTrue(validation.check_disk_cache_parameters(valid_data)) valid_data = { constants.HV_KVM_DISK_AIO: constants.HT_KVM_AIO_THREADS, constants.HV_DISK_CACHE: constants.HT_CACHE_WTHROUGH } self.assertTrue(validation.check_disk_cache_parameters(valid_data)) valid_data = { constants.HV_KVM_DISK_AIO: constants.HT_KVM_AIO_NATIVE, constants.HV_DISK_CACHE: constants.HT_CACHE_NONE } self.assertTrue(validation.check_disk_cache_parameters(valid_data)) class TestParameterValidation(testutils.GanetiTestCase): def testInvalidVncParameters(self): # invalid IPv4 address invalid_data = { constants.HV_VNC_BIND_ADDRESS: "192.0.2.5.5", } self.assertRaises(errors.HypervisorError, validation.validate_vnc_parameters, invalid_data) # invalid network interface invalid_data = { constants.HV_VNC_BIND_ADDRESS: "doesnotexist0", } self.assertRaises(errors.HypervisorError, validation.validate_vnc_parameters, invalid_data) def testValidVncParameters(self): valid_data = { constants.HV_VNC_BIND_ADDRESS: "127.0.0.1" } self.assertTrue(validation.validate_vnc_parameters(valid_data)) valid_data = { constants.HV_VNC_BIND_ADDRESS: "lo" } self.assertTrue(validation.validate_vnc_parameters(valid_data)) def testInvalidSecurityModelParameters(self): invalid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_USER, constants.HV_SECURITY_DOMAIN: "really-non-existing-user" } self.assertRaises(errors.HypervisorError, validation.validate_security_model, invalid_data) def testValidSecurityModelParameters(self): valid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_NONE } self.assertTrue(validation.validate_security_model(valid_data)) valid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_POOL } self.assertTrue(validation.validate_security_model(valid_data)) valid_data = { constants.HV_SECURITY_MODEL: constants.HT_SM_USER, constants.HV_SECURITY_DOMAIN: "root" } self.assertTrue(validation.validate_security_model(valid_data)) def testInvalidMachineVersion(self): kvm_machine_output = testutils.ReadTestData("kvm_6.0.0_machine.txt") invalid_data = { constants.HV_KVM_MACHINE_VERSION: "some-invalid-machine-type" } self.assertRaises(errors.HypervisorError, validation.validate_machine_version, invalid_data, kvm_machine_output) def testValidMachineVersion(self): kvm_machine_output = testutils.ReadTestData("kvm_6.0.0_machine.txt") valid_data = { constants.HV_KVM_MACHINE_VERSION: "pc-i440fx-6.0" } self.assertTrue(validation.validate_machine_version(valid_data, kvm_machine_output)) def testInvalidSpiceParameters(self): kvm_help_too_old = testutils.ReadTestData("kvm_0.9.1_help.txt") kvm_help_working = testutils.ReadTestData("kvm_1.1.2_help.txt") invalid_data = { constants.HV_KVM_SPICE_BIND: "0.0.0.0", constants.HV_VNC_BIND_ADDRESS: "0.0.0.0" } self.assertRaises(errors.HypervisorError, validation.validate_spice_parameters, invalid_data, kvm_help_working) invalid_data = { constants.HV_KVM_SPICE_BIND: "0.0.0.0", constants.HV_VNC_BIND_ADDRESS: None } self.assertRaises(errors.HypervisorError, validation.validate_spice_parameters, invalid_data, kvm_help_too_old) invalid_data = { constants.HV_KVM_SPICE_BIND: "invalid-interface0", constants.HV_VNC_BIND_ADDRESS: None } self.assertRaises(errors.HypervisorError, validation.validate_spice_parameters, invalid_data, kvm_help_working) def testValidSpiceParameters(self): kvm_help_working = testutils.ReadTestData("kvm_1.1.2_help.txt") valid_data = { constants.HV_KVM_SPICE_BIND: "0.0.0.0", constants.HV_VNC_BIND_ADDRESS: None } self.assertTrue(validation.validate_spice_parameters(valid_data, kvm_help_working)) valid_data = { constants.HV_KVM_SPICE_BIND: "::", constants.HV_VNC_BIND_ADDRESS: None } self.assertTrue(validation.validate_spice_parameters(valid_data, kvm_help_working)) valid_data = { constants.HV_KVM_SPICE_BIND: "lo", constants.HV_VNC_BIND_ADDRESS: None } self.assertTrue(validation.validate_spice_parameters(valid_data, kvm_help_working)) class TestQmpMessage(testutils.GanetiTestCase): def testSerialization(self): test_data = { "execute": "command", "arguments": ["a", "b", "c"], } message = hv_kvm.QmpMessage(test_data) for k, v in test_data.items(): self.assertEqual(message[k], v) serialized = message.to_bytes() self.assertEqual(len(serialized.splitlines()), 1, msg="Got multi-line message") rebuilt_message = hv_kvm.QmpMessage.build_from_json_string(serialized) self.assertEqual(rebuilt_message, message) self.assertEqual(len(rebuilt_message), len(test_data)) def testDelete(self): toDelete = "execute" test_data = { toDelete: "command", "arguments": ["a", "b", "c"], } message = hv_kvm.QmpMessage(test_data) oldLen = len(message) del message[toDelete] newLen = len(message) self.assertEqual(oldLen - 1, newLen) class TestConsole(unittest.TestCase): def MakeConsole(self, instance, node, group, hvparams): cons = hv_kvm.KVMHypervisor.GetInstanceConsole(instance, node, group, hvparams, {}) self.assertEqual(cons.Validate(), None) return cons def testSerial(self): instance = objects.Instance(name="kvm.example.com", primary_node="node6017-uuid") node = objects.Node(name="node6017", uuid="node6017-uuid", ndparams={}) group = objects.NodeGroup(name="group6134", ndparams={}) hvparams = { constants.HV_SERIAL_CONSOLE: True, constants.HV_VNC_BIND_ADDRESS: None, constants.HV_KVM_SPICE_BIND: None, } cons = self.MakeConsole(instance, node, group, hvparams) self.assertEqual(cons.kind, constants.CONS_SSH) self.assertEqual(cons.host, node.name) self.assertEqual(cons.command[0], pathutils.KVM_CONSOLE_WRAPPER) self.assertEqual(cons.command[1], constants.SOCAT_PATH) def testVnc(self): instance = objects.Instance(name="kvm.example.com", primary_node="node7235-uuid", network_port=constants.VNC_BASE_PORT + 10) node = objects.Node(name="node7235", uuid="node7235-uuid", ndparams={}) group = objects.NodeGroup(name="group3632", ndparams={}) hvparams = { constants.HV_SERIAL_CONSOLE: False, constants.HV_VNC_BIND_ADDRESS: "192.0.2.1", constants.HV_KVM_SPICE_BIND: None, } cons = self.MakeConsole(instance, node, group, hvparams) self.assertEqual(cons.kind, constants.CONS_VNC) self.assertEqual(cons.host, "192.0.2.1") self.assertEqual(cons.port, constants.VNC_BASE_PORT + 10) self.assertEqual(cons.display, 10) def testSpice(self): instance = objects.Instance(name="kvm.example.com", primary_node="node7235", network_port=11000) node = objects.Node(name="node7235", uuid="node7235-uuid", ndparams={}) group = objects.NodeGroup(name="group0132", ndparams={}) hvparams = { constants.HV_SERIAL_CONSOLE: False, constants.HV_VNC_BIND_ADDRESS: None, constants.HV_KVM_SPICE_BIND: "192.0.2.1", } cons = self.MakeConsole(instance, node, group, hvparams) self.assertEqual(cons.kind, constants.CONS_SPICE) self.assertEqual(cons.host, "192.0.2.1") self.assertEqual(cons.port, 11000) def testNoConsole(self): instance = objects.Instance(name="kvm.example.com", primary_node="node24325", network_port=0) node = objects.Node(name="node24325", uuid="node24325-uuid", ndparams={}) group = objects.NodeGroup(name="group9184", ndparams={}) hvparams = { constants.HV_SERIAL_CONSOLE: False, constants.HV_VNC_BIND_ADDRESS: None, constants.HV_KVM_SPICE_BIND: None, } cons = self.MakeConsole(instance, node, group, hvparams) self.assertEqual(cons.kind, constants.CONS_MESSAGE) class TestVersionChecking(testutils.GanetiTestCase): @staticmethod def ParseTestData(name): help = testutils.ReadTestData(name) return hv_kvm.KVMHypervisor._ParseKVMVersion(help) def testParseVersion112(self): self.assertEqual( self.ParseTestData("kvm_1.1.2_help.txt"), ("1.1.2", 1, 1, 2)) def testParseVersion10(self): self.assertEqual(self.ParseTestData("kvm_1.0_help.txt"), ("1.0", 1, 0, 0)) def testParseVersion01590(self): self.assertEqual( self.ParseTestData("kvm_0.15.90_help.txt"), ("0.15.90", 0, 15, 90)) def testParseVersion0125(self): self.assertEqual( self.ParseTestData("kvm_0.12.5_help.txt"), ("0.12.5", 0, 12, 5)) def testParseVersion091(self): self.assertEqual( self.ParseTestData("kvm_0.9.1_help.txt"), ("0.9.1", 0, 9, 1)) class TestSpiceParameterList(unittest.TestCase): def setUp(self): self.defaults = constants.HVC_DEFAULTS[constants.HT_KVM] def testAudioCompressionDefaultOn(self): self.assertTrue(self.defaults[constants.HV_KVM_SPICE_AUDIO_COMPR]) def testVdAgentDefaultOn(self): self.assertTrue(self.defaults[constants.HV_KVM_SPICE_USE_VDAGENT]) def testTlsCiphersDefaultOn(self): self.assertTrue(self.defaults[constants.HV_KVM_SPICE_TLS_CIPHERS]) def testBindDefaultOff(self): self.assertFalse(self.defaults[constants.HV_KVM_SPICE_BIND]) def testAdditionalParams(self): params = compat.UniqueFrozenset( getattr(constants, name) for name in dir(constants) if name.startswith("HV_KVM_SPICE_")) fixed = set([ constants.HV_KVM_SPICE_BIND, constants.HV_KVM_SPICE_TLS_CIPHERS, constants.HV_KVM_SPICE_USE_VDAGENT, constants.HV_KVM_SPICE_AUDIO_COMPR]) self.assertEqual(hv_kvm.validation._SPICE_ADDITIONAL_PARAMS, params - fixed) class TestHelpRegexps(testutils.GanetiTestCase): """Check _BOOT_RE It has to match -drive.*boot=on|off except if there is another dash-option at the beginning of the line. """ @staticmethod def SearchTestData(name): boot_re = hv_kvm.KVMHypervisor._BOOT_RE help = testutils.ReadTestData(name) return boot_re.search(help) def testBootRe112(self): self.assertFalse(self.SearchTestData("kvm_1.1.2_help.txt")) def testBootRe10(self): self.assertFalse(self.SearchTestData("kvm_1.0_help.txt")) def testBootRe01590(self): self.assertFalse(self.SearchTestData("kvm_0.15.90_help.txt")) def testBootRe0125(self): self.assertTrue(self.SearchTestData("kvm_0.12.5_help.txt")) def testBootRe091(self): self.assertTrue(self.SearchTestData("kvm_0.9.1_help.txt")) def testBootRe091_fake(self): self.assertFalse(self.SearchTestData("kvm_0.9.1_help_boot_test.txt")) class TestGetTunFeatures(unittest.TestCase): def testWrongIoctl(self): tmpfile = tempfile.NamedTemporaryFile() # A file does not have the right ioctls, so this must always fail result = netdev._GetTunFeatures(tmpfile.fileno()) self.assertTrue(result is None) def _FakeIoctl(self, features, fd, request, buf): self.assertEqual(request, netdev.TUNGETFEATURES) (reqno, ) = struct.unpack("I", buf) self.assertEqual(reqno, 0) return struct.pack("I", features) def test(self): tmpfile = tempfile.NamedTemporaryFile() fd = tmpfile.fileno() for features in [0, netdev.IFF_VNET_HDR]: fn = compat.partial(self._FakeIoctl, features) result = netdev._GetTunFeatures(fd, _ioctl=fn) self.assertEqual(result, features) class TestProbeTapVnetHdr(unittest.TestCase): def _FakeTunFeatures(self, expected_fd, flags, fd): self.assertEqual(fd, expected_fd) return flags def test(self): tmpfile = tempfile.NamedTemporaryFile() fd = tmpfile.fileno() for flags in [0, netdev.IFF_VNET_HDR]: fn = compat.partial(self._FakeTunFeatures, fd, flags) result = netdev._ProbeTapVnetHdr(fd, _features_fn=fn) if flags == 0: self.assertFalse(result) else: self.assertTrue(result) def testUnsupported(self): tmpfile = tempfile.NamedTemporaryFile() fd = tmpfile.fileno() self.assertFalse(netdev._ProbeTapVnetHdr(fd, _features_fn=lambda _: None)) class TestGenerateDeviceKVMId(unittest.TestCase): def test(self): device = objects.NIC() target = constants.HOTPLUG_TARGET_NIC fn = hv_kvm._GenerateDeviceKVMId device.uuid = "003fc157-66a8-4e6d-8b7e-ec4f69751396" self.assertTrue(re.match("nic-003fc157-66a8-4e6d", fn(target, device))) class TestGenerateDeviceHVInfo(testutils.GanetiTestCase): def testPCI(self): """Test the placement of the first PCI device during startup.""" self.MockOut(mock.patch('ganeti.utils.EnsureDirs')) hypervisor = hv_kvm.KVMHypervisor() dev_type = constants.HOTPLUG_TARGET_NIC kvm_devid = "nic-9e7c85f6-b6e5-4243" hv_dev_type = constants.HT_NIC_PARAVIRTUAL bus_slots = hypervisor._GetBusSlots() hvinfo = hv_kvm._GenerateDeviceHVInfo(dev_type, kvm_devid, hv_dev_type, bus_slots) # NOTE: The PCI slot is zero-based, i.e. 13th slot has addr hex(12) expected_hvinfo = { "driver": "virtio-net-pci", "id": kvm_devid, "bus": "pci.0", "addr": hex(constants.QEMU_DEFAULT_PCI_RESERVATIONS), } self.assertTrue(hvinfo == expected_hvinfo) def testSCSI(self): """Test the placement of the first SCSI device during startup.""" self.MockOut(mock.patch('ganeti.utils.EnsureDirs')) hypervisor = hv_kvm.KVMHypervisor() dev_type = constants.HOTPLUG_TARGET_DISK kvm_devid = "disk-932df160-7a22-4067" hv_dev_type = constants.HT_DISK_SCSI_BLOCK bus_slots = hypervisor._GetBusSlots() hvinfo = hv_kvm._GenerateDeviceHVInfo(dev_type, kvm_devid, hv_dev_type, bus_slots) expected_hvinfo = { "driver": "scsi-block", "id": kvm_devid, "bus": "scsi.0", "channel": 0, "scsi-id": 0, "lun": 0, } self.assertTrue(hvinfo == expected_hvinfo) class TestGetRuntimeInfo(unittest.TestCase): @classmethod def _GetRuntime(cls): data = testutils.ReadTestData("kvm_runtime.json") return KVMRuntime.from_serialized(data) def _fail(self, target, device, runtime): device.uuid = "aaaaaaaa-66a8-4e6d-8b7e-ec4f69751396" self.assertRaises(errors.HotplugError, hv_kvm._GetExistingDeviceInfo, target, device, runtime) def testNIC(self): device = objects.NIC() target = constants.HOTPLUG_TARGET_NIC runtime = self._GetRuntime() self._fail(target, device, runtime) device.uuid = "003fc157-66a8-4e6d-8b7e-ec4f69751396" devinfo = hv_kvm._GetExistingDeviceInfo(target, device, runtime) self.assertTrue(devinfo.hvinfo["addr"] == "0x8") def testDisk(self): device = objects.Disk() target = constants.HOTPLUG_TARGET_DISK runtime = self._GetRuntime() self._fail(target, device, runtime) device.uuid = "9f5c5bd4-6f60-480b-acdc-9bb1a4b7df79" (devinfo, _, __) = hv_kvm._GetExistingDeviceInfo(target, device, runtime) self.assertTrue(devinfo.hvinfo["addr"] == "0xa") class TestDictToQemuStringNotation(unittest.TestCase): def test(self): tests = [ { "blockdev": { 'driver': 'raw', 'node-name': 'disk-3edc32a2-8127-4b9d', 'discard': 'ignore', 'cache': { 'direct': True, 'no-flush': False }, 'file': { 'driver': 'rbd', 'pool': 'ganeti', 'image': '32eb97b3-ec20-401d-b48c-a1f24c0385a2.rbd.disk0' }, 'auto-read-only': False }, "result": "driver=raw,node-name=disk-3edc32a2-8127-4b9d," "discard=ignore,cache.direct=on,cache.no-flush=off," "file.driver=rbd,file.pool=ganeti," "file.image=32eb97b3-ec20-401d-b48c-a1f24c0385a2.rbd.disk0," "auto-read-only=off" }, { "blockdev": { 'driver': 'raw', 'node-name': 'disk-c44790ad-e9e2-46a3', 'discard': 'ignore', 'cache': { 'direct': True, 'no-flush': False }, 'file': { 'driver': 'host_device', 'filename': '/path/to/disk', 'aio': 'native' }, 'auto-read-only': False }, "result": "driver=raw,node-name=disk-c44790ad-e9e2-46a3," "discard=ignore,cache.direct=on,cache.no-flush=off," "file.driver=host_device," "file.filename=/path/to/disk,file.aio=native," "auto-read-only=off" } ] for test in tests: self.assertEqual(test["result"], hv_kvm.kvm_utils.DictToQemuStringNotation( test["blockdev"])) class TestParseStorageUriToBlockdevParam(unittest.TestCase): def testRbd(self): uri = "rbd:cephpool/image-1234-xyz" expected_data = { "driver": "rbd", "pool": "cephpool", "image": "image-1234-xyz" } blockdev_driver = hv_kvm.kvm_utils.ParseStorageUriToBlockdevParam(uri) self.assertDictEqual(expected_data, blockdev_driver) def testGluster(self): uri = "gluster://server:1234/gluster-volume/path" expected_data = { "driver": "gluster", "server": [ { "type": "inet", "host": "server", "port": "1234", } ], "volume": "gluster-volume", "path": "path" } blockdev_driver = hv_kvm.kvm_utils.ParseStorageUriToBlockdevParam(uri) self.assertDictEqual(expected_data, blockdev_driver) def testBadURI(self): uri = "gopher://storage/file" self.assertRaises(errors.HypervisorError, hv_kvm.kvm_utils.ParseStorageUriToBlockdevParam, uri) class PostfixMatcher(object): def __init__(self, string): self.string = string def __eq__(self, other): return other.endswith(self.string) def __repr__(self): return "" % self.string class TestKvmRuntime(testutils.GanetiTestCase): """The _ExecuteKvmRuntime is at the core of all KVM operations.""" def setUp(self): super(TestKvmRuntime, self).setUp() kvm_class = 'ganeti.hypervisor.hv_kvm.KVMHypervisor' self.MockOut('qmp', mock.patch('ganeti.hypervisor.hv_kvm.QmpConnection')) self.MockOut('run_cmd', mock.patch('ganeti.utils.RunCmd')) self.MockOut('ensure_dirs', mock.patch('ganeti.utils.EnsureDirs')) self.MockOut('write_file', mock.patch('ganeti.utils.WriteFile')) self.MockOut(mock.patch(kvm_class + '.ValidateParameters')) self.MockOut(mock.patch('ganeti.hypervisor.hv_kvm.OpenTap', return_value=('test_nic', [], []))) self.MockOut(mock.patch(kvm_class + '._ConfigureNIC')) self.MockOut('pid_alive', mock.patch(kvm_class + '._InstancePidAlive', return_value=('file', -1, False))) self.MockOut(mock.patch(kvm_class + '._ExecuteCpuAffinity')) self.cfg = ConfigMock() params = constants.HVC_DEFAULTS[constants.HT_KVM].copy() beparams = constants.BEC_DEFAULTS.copy() self.instance = self.cfg.AddNewInstance(name='name.example.com', hypervisor='kvm', hvparams=params, beparams=beparams) def testDirectoriesCreated(self): hypervisor = hv_kvm.KVMHypervisor() self.mocks['ensure_dirs'].assert_called_with([ (PostfixMatcher('/run/ganeti/kvm-hypervisor'), 0o775), (PostfixMatcher('/run/ganeti/kvm-hypervisor/pid'), 0o775), (PostfixMatcher('/run/ganeti/kvm-hypervisor/uid'), 0o775), (PostfixMatcher('/run/ganeti/kvm-hypervisor/ctrl'), 0o775), (PostfixMatcher('/run/ganeti/kvm-hypervisor/conf'), 0o775), (PostfixMatcher('/run/ganeti/kvm-hypervisor/nic'), 0o775), (PostfixMatcher('/run/ganeti/kvm-hypervisor/chroot'), 0o775), (PostfixMatcher('/run/ganeti/kvm-hypervisor/chroot-quarantine'), 0o775) ]) def testStartInstance(self): hypervisor = hv_kvm.KVMHypervisor() def RunCmd(cmd, **kwargs): if '--help' in cmd: return mock.Mock( failed=False, output=testutils.ReadTestData("kvm_current_help.txt")) if '-S' in cmd: self.mocks['pid_alive'].return_value = ('file', -1, True) return mock.Mock(failed=False) elif '-machine' in cmd: return mock.Mock(failed=False, output='') elif '-device' in cmd: return mock.Mock(failed=False, output='name "virtio-blk-pci"') else: raise errors.ProgrammerError('Unexpected command: %s' % cmd) self.mocks['run_cmd'].side_effect = RunCmd hypervisor.StartInstance(self.instance, [], False) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.hypervisor.hv_lxc_unittest.py000075500000000000000000000261471476477700300257470ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011, 2014, 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.hypervisor.hv_lxc""" import unittest from ganeti import constants from ganeti import errors from ganeti import objects from ganeti import hypervisor from ganeti import utils from ganeti.hypervisor import hv_base from ganeti.hypervisor import hv_lxc from ganeti.hypervisor.hv_lxc import LXCHypervisor, LXCVersion from unittest import mock import os import shutil import tempfile import testutils from testutils import patch_object def setUpModule(): # Creating instance of LXCHypervisor will fail by permission issue of # instance directories global temp_dir temp_dir = tempfile.mkdtemp() LXCHypervisor._ROOT_DIR = utils.PathJoin(temp_dir, "root") LXCHypervisor._LOG_DIR = utils.PathJoin(temp_dir, "log") def tearDownModule(): shutil.rmtree(temp_dir) def RunResultOk(stdout): return utils.RunResult(0, None, stdout, "", [], None, None) class TestLXCVersion(unittest.TestCase): def testParseLXCVersion(self): self.assertEqual(LXCVersion("1.0.0"), (1, 0, 0)) self.assertEqual(LXCVersion("1.0.0.alpha1"), (1, 0, 0)) self.assertRaises(ValueError, LXCVersion, "1.0") self.assertRaises(ValueError, LXCVersion, "1.2a.0") class LXCHypervisorTestCase(unittest.TestCase): """Used to test classes instantiating the LXC hypervisor class. """ def setUp(self): self.ensure_fn = LXCHypervisor._EnsureDirectoryExistence LXCHypervisor._EnsureDirectoryExistence = mock.Mock(return_value=False) self.hv = LXCHypervisor() def tearDown(self): LXCHypervisor._EnsureDirectoryExistence = self.ensure_fn class TestConsole(unittest.TestCase): def test(self): instance = objects.Instance(name="lxc.example.com", primary_node="node199-uuid") node = objects.Node(name="node199", uuid="node199-uuid", ndparams={}) group = objects.NodeGroup(name="group991", ndparams={}) cons = hv_lxc.LXCHypervisor.GetInstanceConsole(instance, node, group, {}, {}) self.assertEqual(cons.Validate(), None) self.assertEqual(cons.kind, constants.CONS_SSH) self.assertEqual(cons.host, node.name) self.assertEqual(cons.command[-1], instance.name) class TestLXCIsInstanceAlive(unittest.TestCase): @patch_object(utils, "RunCmd") def testActive(self, runcmd_mock): runcmd_mock.return_value = RunResultOk("inst1 inst2 inst3\ninst4 inst5") self.assertTrue(LXCHypervisor._IsInstanceAlive("inst4")) @patch_object(utils, "RunCmd") def testInactive(self, runcmd_mock): runcmd_mock.return_value = RunResultOk("inst1 inst2foo") self.assertFalse(LXCHypervisor._IsInstanceAlive("inst2")) class TestLXCListInstances(LXCHypervisorTestCase): @patch_object(utils, "RunCmd") def testRunningInstnaces(self, runcmd_mock): instance_list = ["inst1", "inst2", "inst3", "inst4", "inst5"] runcmd_mock.return_value = RunResultOk("inst1 inst2 inst3\ninst4 inst5") self.assertEqual(self.hv.ListInstances(), instance_list) @patch_object(utils, "RunCmd") def testEmpty(self, runcmd_mock): runcmd_mock.return_value = RunResultOk(" ") self.assertEqual(self.hv.ListInstances(), []) class TestLXCHypervisorGetInstanceInfo(LXCHypervisorTestCase): def setUp(self): super(TestLXCHypervisorGetInstanceInfo, self).setUp() self.hv._GetCgroupCpuList = mock.Mock(return_value=[1, 3]) self.hv._GetCgroupMemoryLimit = mock.Mock(return_value=128*(1024**2)) self.hv._GetCgroupCpuUsage = mock.Mock(return_value=5.01) @patch_object(LXCHypervisor, "_IsInstanceAlive") def testRunningInstance(self, isalive_mock): isalive_mock.return_value = True self.assertEqual(self.hv.GetInstanceInfo("inst1"), ("inst1", 0, 128, 2, hv_base.HvInstanceState.RUNNING, 5.01)) @patch_object(LXCHypervisor, "_IsInstanceAlive") def testInactiveOrNonexistentInstance(self, isalive_mock): isalive_mock.return_value = False self.assertEqual(self.hv.GetInstanceInfo("inst1"), None) class TestCgroupMount(LXCHypervisorTestCase): @patch_object(utils, "GetMounts") @patch_object(LXCHypervisor, "_MountCgroupSubsystem") def testGetOrPrepareCgroupSubsysMountPoint(self, mntcgsub_mock, getmnt_mock): getmnt_mock.return_value = [ ("/dev/foo", "/foo", "foo", "cpuset"), ("cpuset", "/sys/fs/cgroup/cpuset", "cgroup", "rw,relatime,cpuset"), ("devices", "/sys/fs/cgroup/devices", "cgroup", "rw,devices,relatime"), ("cpumem", "/sys/fs/cgroup/cpumem", "cgroup", "cpu,memory,rw,relatime"), ] mntcgsub_mock.return_value = "/foo" self.assertEqual(self.hv._GetOrPrepareCgroupSubsysMountPoint("cpuset"), "/sys/fs/cgroup/cpuset") self.assertEqual(self.hv._GetOrPrepareCgroupSubsysMountPoint("devices"), "/sys/fs/cgroup/devices") self.assertEqual(self.hv._GetOrPrepareCgroupSubsysMountPoint("cpu"), "/sys/fs/cgroup/cpumem") self.assertEqual(self.hv._GetOrPrepareCgroupSubsysMountPoint("memory"), "/sys/fs/cgroup/cpumem") self.assertEqual(self.hv._GetOrPrepareCgroupSubsysMountPoint("freezer"), "/foo") mntcgsub_mock.assert_called_with("freezer") class TestCgroupReadData(LXCHypervisorTestCase): cgroot = os.path.abspath(testutils.TestDataFilename("cgroup_root")) @patch_object(LXCHypervisor, "_CGROUP_ROOT_DIR", cgroot) def testGetCgroupMountPoint(self): self.assertEqual(self.hv._GetCgroupMountPoint(), self.cgroot) @patch_object(LXCHypervisor, "_PROC_SELF_CGROUP_FILE", testutils.TestDataFilename("proc_cgroup.txt")) def testGetCurrentCgroupSubsysGroups(self): expected_groups = { "memory": "", # root "cpuset": "some_group", "devices": "some_group", } self.assertEqual(self.hv._GetCurrentCgroupSubsysGroups(), expected_groups) @patch_object(LXCHypervisor, "_GetOrPrepareCgroupSubsysMountPoint") @patch_object(LXCHypervisor, "_GetCurrentCgroupSubsysGroups") def testGetCgroupSubsysDir(self, getcgg_mock, getmp_mock): getmp_mock.return_value = "/cg" getcgg_mock.return_value = {"cpuset": "grp"} self.assertEqual( self.hv._GetCgroupSubsysDir("memory"), "/cg/lxc" ) self.assertEqual( self.hv._GetCgroupSubsysDir("cpuset"), "/cg/grp/lxc" ) @patch_object(LXCHypervisor, "_GetOrPrepareCgroupSubsysMountPoint") @patch_object(LXCHypervisor, "_GetCurrentCgroupSubsysGroups") def testGetCgroupParamPath(self, getcgg_mock, getmp_mock): getmp_mock.return_value = "/cg" getcgg_mock.return_value = {"cpuset": "grp"} self.assertEqual( self.hv._GetCgroupParamPath("memory.memsw.limit_in_bytes", instance_name="instance1"), "/cg/lxc/instance1/memory.memsw.limit_in_bytes" ) self.assertEqual( self.hv._GetCgroupParamPath("cpuset.cpus"), "/cg/grp/lxc/cpuset.cpus" ) @patch_object(LXCHypervisor, "_GetCgroupSubsysDir") def testGetCgroupInstanceValue(self, getdir_mock): getdir_mock.return_value = utils.PathJoin(self.cgroot, "memory", "lxc") self.assertEqual(self.hv._GetCgroupInstanceValue("instance1", "memory.limit_in_bytes"), "128") getdir_mock.return_value = utils.PathJoin(self.cgroot, "cpuset", "some_group", "lxc") self.assertEqual(self.hv._GetCgroupInstanceValue("instance1", "cpuset.cpus"), "0-1") @patch_object(LXCHypervisor, "_GetCgroupInstanceValue") def testGetCgroupCpuList(self, getval_mock): getval_mock.return_value = "0-1" self.assertEqual(self.hv._GetCgroupCpuList("instance1"), [0, 1]) @patch_object(LXCHypervisor, "_GetCgroupInstanceValue") def testGetCgroupMemoryLimit(self, getval_mock): getval_mock.return_value = "128" self.assertEqual(self.hv._GetCgroupMemoryLimit("instance1"), 128) class TestVerifyLXCCommands(unittest.TestCase): def setUp(self): runcmd_mock = mock.Mock(return_value="") self.RunCmdPatch = patch_object(utils, "RunCmd", runcmd_mock) self.RunCmdPatch.start() version_patch = patch_object(LXCHypervisor, "_LXC_MIN_VERSION_REQUIRED", LXCVersion("1.2.3")) self._LXC_MIN_VERSION_REQUIRED_Patch = version_patch self._LXC_MIN_VERSION_REQUIRED_Patch.start() self.hvc = LXCHypervisor def tearDown(self): self.RunCmdPatch.stop() self._LXC_MIN_VERSION_REQUIRED_Patch.stop() @patch_object(LXCHypervisor, "_LXC_COMMANDS_REQUIRED", ["lxc-stop"]) def testCommandVersion(self): utils.RunCmd.return_value = RunResultOk("1.2.3\n") self.assertFalse(self.hvc._VerifyLXCCommands()) utils.RunCmd.return_value = RunResultOk("1.10.0\n") self.assertFalse(self.hvc._VerifyLXCCommands()) utils.RunCmd.return_value = RunResultOk("1.2.2\n") self.assertTrue(self.hvc._VerifyLXCCommands()) @patch_object(LXCHypervisor, "_LXC_COMMANDS_REQUIRED", ["lxc-stop"]) def testCommandVersionInvalid(self): utils.RunCmd.return_value = utils.RunResult(1, None, "", "", [], None, None) self.assertTrue(self.hvc._VerifyLXCCommands()) utils.RunCmd.return_value = RunResultOk("1.2a.0\n") self.assertTrue(self.hvc._VerifyLXCCommands()) @patch_object(LXCHypervisor, "_LXC_COMMANDS_REQUIRED", ["lxc-stop"]) def testCommandNotExists(self): utils.RunCmd.side_effect = errors.OpExecError self.assertTrue(self.hvc._VerifyLXCCommands()) @patch_object(LXCHypervisor, "_LXC_COMMANDS_REQUIRED", ["lxc-ls"]) def testVerifyLXCLs(self): utils.RunCmd.return_value = RunResultOk("garbage\n--running\ngarbage") self.assertFalse(self.hvc._VerifyLXCCommands()) utils.RunCmd.return_value = RunResultOk("foo") self.assertTrue(self.hvc._VerifyLXCCommands()) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.hypervisor.hv_xen_unittest.py000075500000000000000000000740201476477700300257440ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.hypervisor.hv_xen""" import string # pylint: disable=W0402 import unittest import tempfile import shutil import random import os from unittest import mock from ganeti import constants from ganeti import objects from ganeti import pathutils from ganeti import hypervisor from ganeti import utils from ganeti import errors from ganeti import compat from ganeti.hypervisor import hv_base from ganeti.hypervisor import hv_xen import testutils # Map from hypervisor class to hypervisor name HVCLASS_TO_HVNAME = utils.InvertDict(hypervisor._HYPERVISOR_MAP) class TestConsole(unittest.TestCase): def test(self): hvparams = {} for cls in [hv_xen.XenPvmHypervisor(), hv_xen.XenHvmHypervisor()]: instance = objects.Instance(name="xen.example.com", primary_node="node24828-uuid") node = objects.Node(name="node24828", uuid="node24828-uuid", ndparams={}) group = objects.NodeGroup(name="group52341", ndparams={}) cons = cls.GetInstanceConsole(instance, node, group, hvparams, {}) self.assertEqual(cons.Validate(), None) self.assertEqual(cons.kind, constants.CONS_SSH) self.assertEqual(cons.host, node.name) self.assertEqual(cons.command[-1], instance.name) class TestCreateConfigCpus(unittest.TestCase): def testEmpty(self): for cpu_mask in [None, ""]: self.assertEqual(hv_xen._CreateConfigCpus(cpu_mask), "cpus = [ ]") def testAll(self): self.assertEqual(hv_xen._CreateConfigCpus(constants.CPU_PINNING_ALL), None) def testOne(self): self.assertEqual(hv_xen._CreateConfigCpus("9"), "cpu = \"9\"") def testMultiple(self): self.assertEqual(hv_xen._CreateConfigCpus("0-2,4,5-5:3:all"), ("cpus = [ \"0,1,2,4,5\", \"3\", \"%s\" ]" % constants.CPU_PINNING_ALL_XEN)) class TestParseInstanceList(testutils.GanetiTestCase): def test(self): data = testutils.ReadTestData("xen-xl-list-4.0.1-dom0-only.txt") # Exclude node self.assertEqual(hv_xen._ParseInstanceList(data.splitlines(), False), []) # Include node result = hv_xen._ParseInstanceList(data.splitlines(), True) self.assertEqual(len(result), 1) self.assertEqual(len(result[0]), 6) # Name self.assertEqual(result[0][0], hv_xen._DOM0_NAME) # ID self.assertEqual(result[0][1], 0) # Memory self.assertEqual(result[0][2], 1023) # VCPUs self.assertEqual(result[0][3], 1) # State self.assertEqual(result[0][4], hv_base.HvInstanceState.RUNNING) # Time self.assertAlmostEqual(result[0][5], 121152.6) def testWrongLineFormat(self): tests = [ ["three fields only"], ["name InvalidID 128 1 r----- 12345"], ] for lines in tests: try: hv_xen._ParseInstanceList(["Header would be here"] + lines, False) except errors.HypervisorError as err: self.assertTrue("Can't parse instance list" in str(err)) else: self.fail("Exception was not raised") class TestInstanceStateParsing(unittest.TestCase): def testRunningStates(self): states = [ "r-----", "r-p---", "rb----", "rbp---", "-b----", "-bp---", "-----d", "--p--d", "------", "--p---", "r--ss-", "r-pss-", "rb-ss-", "rbpss-", "-b-ss-", "-bpss-", "---ss-", "r--sr-", "r-psr-", "rb-sr-", "rbpsr-", "-b-sr-", "-bpsr-", "---sr-", ] for state in states: self.assertEqual(hv_xen._XenToHypervisorInstanceState(state), hv_base.HvInstanceState.RUNNING) def testShutdownStates(self): states = [ "---s--", "--ps--", "---s-d", "--ps-d", ] for state in states: self.assertEqual(hv_xen._XenToHypervisorInstanceState(state), hv_base.HvInstanceState.SHUTDOWN) def testCrashingStates(self): states = [ "--psc-", "---sc-", "---scd", "--p-c-", "----c-", "----cd", ] for state in states: self.assertRaises(hv_xen._InstanceCrashed, hv_xen._XenToHypervisorInstanceState, state) class TestGetInstanceList(testutils.GanetiTestCase): def _Fail(self): return utils.RunResult(constants.EXIT_FAILURE, None, "stdout", "stderr", None, NotImplemented, NotImplemented) def testTimeout(self): fn = testutils.CallCounter(self._Fail) try: hv_xen._GetRunningInstanceList(fn, False, delays=(0.02, 1.0, 0.03), timeout=0.1) except errors.HypervisorError as err: self.assertTrue("timeout exceeded" in str(err)) else: self.fail("Exception was not raised") self.assertTrue(fn.Count() < 10, msg="'xl list' was called too many times") def _Success(self, stdout): return utils.RunResult(constants.EXIT_SUCCESS, None, stdout, "", None, NotImplemented, NotImplemented) def testSuccess(self): data = testutils.ReadTestData("xen-xl-list-4.0.1-four-instances.txt") fn = testutils.CallCounter(compat.partial(self._Success, data)) result = hv_xen._GetRunningInstanceList(fn, True, delays=(0.02, 1.0, 0.03), timeout=0.1) self.assertEqual(len(result), 4) self.assertEqual([r[0] for r in result], [ "Domain-0", "server01.example.com", "web3106215069.example.com", "testinstance.example.com", ]) self.assertEqual(fn.Count(), 1) def testOmitCrashed(self): data = testutils.ReadTestData("xen-xl-list-4.4-crashed-instances.txt") fn = testutils.CallCounter(compat.partial(self._Success, data)) result = hv_xen._GetAllInstanceList(fn, True, delays=(0.02, 1.0, 0.03), timeout=0.1) self.assertEqual(len(result), 2) self.assertEqual([r[0] for r in result], [ "Domain-0", "server01.example.com", ]) self.assertEqual(fn.Count(), 1) class TestParseNodeInfo(testutils.GanetiTestCase): def testEmpty(self): self.assertEqual(hv_xen._ParseNodeInfo(""), {}) def testUnknownInput(self): data = "\n".join([ "foo bar", "something else goes", "here", ]) self.assertEqual(hv_xen._ParseNodeInfo(data), {}) def testBasicInfo(self): data = testutils.ReadTestData("xen-xl-info-4.0.1.txt") result = hv_xen._ParseNodeInfo(data) self.assertEqual(result, { "cpu_nodes": 1, "cpu_sockets": 2, "cpu_total": 4, "hv_version": (4, 0), "memory_free": 8004, "memory_total": 16378, }) class TestMergeInstanceInfo(testutils.GanetiTestCase): def testEmpty(self): self.assertEqual(hv_xen._MergeInstanceInfo({}, []), {}) def _FakeXlList(self, include_node): return [ (hv_xen._DOM0_NAME, NotImplemented, 4096, 7, NotImplemented, NotImplemented), ("inst1.example.com", NotImplemented, 2048, 4, NotImplemented, NotImplemented), ] def testMissingNodeInfo(self): instance_list = self._FakeXlList(True) result = hv_xen._MergeInstanceInfo({}, instance_list) self.assertEqual(result, { "memory_dom0": 4096, "cpu_dom0": 7, }) def testWithNodeInfo(self): info = testutils.ReadTestData("xen-xl-info-4.0.1.txt") instance_list = self._FakeXlList(True) result = hv_xen._GetNodeInfo(info, instance_list) self.assertEqual(result, { "cpu_nodes": 1, "cpu_sockets": 2, "cpu_total": 4, "cpu_dom0": 7, "hv_version": (4, 0), "memory_dom0": 4096, "memory_free": 8004, "memory_hv": 2230, "memory_total": 16378, }) class TestGetConfigFileDiskData(unittest.TestCase): def testLetterCount(self): self.assertEqual(len(hv_xen._DISK_LETTERS), 26) def testNoDisks(self): self.assertEqual(hv_xen._GetConfigFileDiskData([], "hd"), []) def testManyDisks(self): for offset in [0, 1, 10]: disks = [(objects.Disk(dev_type=constants.DT_PLAIN), "/tmp/disk/%s" % idx, NotImplemented) for idx in range(len(hv_xen._DISK_LETTERS) + offset)] if offset == 0: result = hv_xen._GetConfigFileDiskData(disks, "hd") self.assertEqual(result, [ "'phy:/tmp/disk/%s,hd%s,r'" % (idx, string.ascii_lowercase[idx]) for idx in range(len(hv_xen._DISK_LETTERS) + offset) ]) else: try: hv_xen._GetConfigFileDiskData(disks, "hd") except errors.HypervisorError as err: self.assertEqual(str(err), "Too many disks") else: self.fail("Exception was not raised") def testTwoLvDisksWithMode(self): disks = [ (objects.Disk(dev_type=constants.DT_PLAIN, mode=constants.DISK_RDWR), "/tmp/diskFirst", NotImplemented), (objects.Disk(dev_type=constants.DT_PLAIN, mode=constants.DISK_RDONLY), "/tmp/diskLast", NotImplemented), ] result = hv_xen._GetConfigFileDiskData(disks, "hd") self.assertEqual(result, [ "'phy:/tmp/diskFirst,hda,w'", "'phy:/tmp/diskLast,hdb,r'", ]) def testFileDisks(self): disks = [ (objects.Disk(dev_type=constants.DT_FILE, mode=constants.DISK_RDWR, logical_id=[constants.FD_LOOP]), "/tmp/diskFirst", NotImplemented), (objects.Disk(dev_type=constants.DT_FILE, mode=constants.DISK_RDONLY, logical_id=[constants.FD_BLKTAP]), "/tmp/diskTwo", NotImplemented), (objects.Disk(dev_type=constants.DT_FILE, mode=constants.DISK_RDWR, logical_id=[constants.FD_LOOP]), "/tmp/diskThree", NotImplemented), (objects.Disk(dev_type=constants.DT_FILE, mode=constants.DISK_RDONLY, logical_id=[constants.FD_BLKTAP2]), "/tmp/diskFour", NotImplemented), (objects.Disk(dev_type=constants.DT_FILE, mode=constants.DISK_RDWR, logical_id=[constants.FD_BLKTAP]), "/tmp/diskLast", NotImplemented), ] result = hv_xen._GetConfigFileDiskData(disks, "sd") self.assertEqual(result, [ "'file:/tmp/diskFirst,sda,w'", "'tap:aio:/tmp/diskTwo,sdb,r'", "'file:/tmp/diskThree,sdc,w'", "'tap2:tapdisk:aio:/tmp/diskFour,sdd,r'", "'tap:aio:/tmp/diskLast,sde,w'", ]) def testInvalidFileDisk(self): disks = [ (objects.Disk(dev_type=constants.DT_FILE, mode=constants.DISK_RDWR, logical_id=["#unknown#"]), "/tmp/diskinvalid", NotImplemented), ] self.assertRaises(KeyError, hv_xen._GetConfigFileDiskData, disks, "sd") class TestXenHypervisorGetInstanceList(unittest.TestCase): RESULT_OK = utils.RunResult(0, None, "", "", "", None, None) XEN_LIST = "list" def testFromHvparams(self): expected_xen_cmd = "xl" hvparams = {} mock_run_cmd = mock.Mock(return_value=self.RESULT_OK) hv = hv_xen.XenHypervisor(_cfgdir=NotImplemented, _run_cmd_fn=mock_run_cmd) hv._GetInstanceList(True) mock_run_cmd.assert_called_with([expected_xen_cmd, self.XEN_LIST]) class TestXenHypervisorListInstances(unittest.TestCase): RESULT_OK = utils.RunResult(0, None, "", "", "", None, None) XEN_LIST = "list" def testHvparamsXl(self): expected_xen_cmd = "xl" hvparams = {} mock_run_cmd = mock.Mock(return_value=self.RESULT_OK) hv = hv_xen.XenHypervisor(_cfgdir=NotImplemented, _run_cmd_fn=mock_run_cmd) hv.ListInstances(hvparams=hvparams) mock_run_cmd.assert_called_with([expected_xen_cmd, self.XEN_LIST]) class TestXenHypervisorCheckToolstack(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() self.cfg_name = "xen_config" self.cfg_path = utils.PathJoin(self.tmpdir, self.cfg_name) self.hv = hv_xen.XenHypervisor() def tearDown(self): shutil.rmtree(self.tmpdir) def testCheckToolstackXlConfigured(self): RESULT_OK = utils.RunResult(0, None, "", "", "", None, None) mock_run_cmd = mock.Mock(return_value=RESULT_OK) hv = hv_xen.XenHypervisor(_cfgdir=NotImplemented, _run_cmd_fn=mock_run_cmd) result = hv._CheckToolstackXlConfigured() self.assertTrue(result) def testCheckToolstackXlFails(self): RESULT_FAILED = utils.RunResult( 1, None, "", "ERROR: The pink bunny hid the binary.", "", None, None) mock_run_cmd = mock.Mock(return_value=RESULT_FAILED) hv = hv_xen.XenHypervisor(_cfgdir=NotImplemented, _run_cmd_fn=mock_run_cmd) self.assertRaises(errors.HypervisorError, hv._CheckToolstackXlConfigured) class TestXenHypervisorWriteConfigFile(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testWriteError(self): cfgdir = utils.PathJoin(self.tmpdir, "foobar") hv = hv_xen.XenHypervisor(_cfgdir=cfgdir, _run_cmd_fn=NotImplemented, _cmd=NotImplemented) self.assertFalse(os.path.exists(cfgdir)) try: hv._WriteConfigFile("name", "data") except errors.HypervisorError as err: self.assertTrue(str(err).startswith("Cannot write Xen instance")) else: self.fail("Exception was not raised") class TestXenHypervisorVerify(unittest.TestCase): def setUp(self): output = testutils.ReadTestData("xen-xl-info-4.0.1.txt") self._result_ok = utils.RunResult(0, None, output, "", "", None, None) def testVerify(self): hvparams = {} mock_run_cmd = mock.Mock(return_value=self._result_ok) hv = hv_xen.XenHypervisor(_cfgdir=NotImplemented, _run_cmd_fn=mock_run_cmd) hv._CheckToolstackXlConfigured = mock.Mock(return_value=True) result = hv.Verify(hvparams) self.assertTrue(result is None) def testVerifyToolstackNotOk(self): hvparams = {} mock_run_cmd = mock.Mock(return_value=self._result_ok) hv = hv_xen.XenHypervisor(_cfgdir=NotImplemented, _run_cmd_fn=mock_run_cmd) hv._CheckToolstackXlConfigured = mock.Mock() hv._CheckToolstackXlConfigured.side_effect = errors.HypervisorError("foo") result = hv.Verify(hvparams) self.assertTrue(result is not None) def testVerifyFailing(self): result_failed = utils.RunResult(1, None, "", "", "", None, None) mock_run_cmd = mock.Mock(return_value=result_failed) hv = hv_xen.XenHypervisor(_cfgdir=NotImplemented, _run_cmd_fn=mock_run_cmd) hv._CheckToolstackXlConfigured = mock.Mock(return_value=True) result = hv.Verify() self.assertTrue(result is not None) class _TestXenHypervisor(object): TARGET = NotImplemented CMD = NotImplemented HVNAME = NotImplemented VALID_HVPARAMS = {} def setUp(self): super(_TestXenHypervisor, self).setUp() self.tmpdir = tempfile.mkdtemp() self.vncpw = "".join(random.sample(string.ascii_letters, 10)) self._xen_delay = self.TARGET._INSTANCE_LIST_DELAYS self.TARGET._INSTANCE_LIST_DELAYS = (0.01, 1.0, 0.05) self._list_timeout = self.TARGET._INSTANCE_LIST_TIMEOUT self.TARGET._INSTANCE_LIST_TIMEOUT = 0.1 self.vncpw_path = utils.PathJoin(self.tmpdir, "vncpw") utils.WriteFile(self.vncpw_path, data=self.vncpw) def tearDown(self): super(_TestXenHypervisor, self).tearDown() shutil.rmtree(self.tmpdir) self.TARGET._INSTANCE_LIST_DELAYS = self._xen_delay self.TARGET._INSTANCE_LIST_TIMEOUT = self._list_timeout def _GetHv(self, run_cmd=NotImplemented): return self.TARGET(_cfgdir=self.tmpdir, _run_cmd_fn=run_cmd, _cmd=self.CMD) def _SuccessCommand(self, stdout, cmd): self.assertEqual(cmd[0], self.CMD) return utils.RunResult(constants.EXIT_SUCCESS, None, stdout, "", None, NotImplemented, NotImplemented) def _FailingCommand(self, cmd): self.assertEqual(cmd[0], self.CMD) return utils.RunResult(constants.EXIT_FAILURE, None, "", "This command failed", None, NotImplemented, NotImplemented) def _FakeTcpPing(self, expected, result, target, port, **kwargs): self.assertEqual((target, port), expected) return result def testReadingNonExistentConfigFile(self): hv = self._GetHv() try: hv._ReadConfigFile("inst15780.example.com") except errors.HypervisorError as err: self.assertTrue(str(err).startswith("Failed to load Xen config file:")) else: self.fail("Exception was not raised") def testRemovingAutoConfigFile(self): name = "inst8206.example.com" cfgfile = utils.PathJoin(self.tmpdir, name) autodir = utils.PathJoin(self.tmpdir, "auto") autocfgfile = utils.PathJoin(autodir, name) os.mkdir(autodir) utils.WriteFile(autocfgfile, data="") hv = self._GetHv() self.assertTrue(os.path.isfile(autocfgfile)) hv._WriteConfigFile(name, "content") self.assertFalse(os.path.exists(autocfgfile)) self.assertEqual(utils.ReadFile(cfgfile), "content") def _XenList(self, cmd): self.assertEqual(cmd, [self.CMD, "list"]) # TODO: Use actual data from "xl" command output = testutils.ReadTestData("xen-xl-list-4.0.1-four-instances.txt") return self._SuccessCommand(output, cmd) def testGetInstanceInfo(self): hv = self._GetHv(run_cmd=self._XenList) (name, instid, memory, vcpus, state, runtime) = \ hv.GetInstanceInfo("server01.example.com") self.assertEqual(name, "server01.example.com") self.assertEqual(instid, 1) self.assertEqual(memory, 1024) self.assertEqual(vcpus, 1) self.assertEqual(state, hv_base.HvInstanceState.RUNNING) self.assertAlmostEqual(runtime, 167643.2) def testGetInstanceInfoDom0(self): hv = self._GetHv(run_cmd=self._XenList) # TODO: Not sure if this is actually used anywhere (can't find it), but the # code supports querying for Dom0 (name, instid, memory, vcpus, state, runtime) = \ hv.GetInstanceInfo(hv_xen._DOM0_NAME) self.assertEqual(name, "Domain-0") self.assertEqual(instid, 0) self.assertEqual(memory, 1023) self.assertEqual(vcpus, 1) self.assertEqual(state, hv_base.HvInstanceState.RUNNING) self.assertAlmostEqual(runtime, 154706.1) def testGetInstanceInfoUnknown(self): hv = self._GetHv(run_cmd=self._XenList) result = hv.GetInstanceInfo("unknown.example.com") self.assertTrue(result is None) def testGetAllInstancesInfo(self): hv = self._GetHv(run_cmd=self._XenList) result = hv.GetAllInstancesInfo() self.assertEqual([r[0] for r in result], [ "server01.example.com", "web3106215069.example.com", "testinstance.example.com", ]) def testListInstances(self): hv = self._GetHv(run_cmd=self._XenList) self.assertEqual(hv.ListInstances(), [ "server01.example.com", "web3106215069.example.com", "testinstance.example.com", ]) def _StartInstanceCommand(self, inst, paused, failcreate, cmd): if cmd == [self.CMD, "info"]: output = testutils.ReadTestData("xen-xl-info-4.0.1.txt") elif cmd == [self.CMD, "list"]: output = testutils.ReadTestData("xen-xl-list-4.0.1-dom0-only.txt") elif cmd[:2] == [self.CMD, "create"]: args = cmd[2:] cfgfile = utils.PathJoin(self.tmpdir, inst.name) if paused: self.assertEqual(args, ["-p", cfgfile]) else: self.assertEqual(args, [cfgfile]) if failcreate: return self._FailingCommand(cmd) output = "" else: self.fail("Unhandled command: %s" % (cmd, )) return self._SuccessCommand(output, cmd) def _MakeInstance(self): # Copy default parameters bep = objects.FillDict(constants.BEC_DEFAULTS, {}) hvp = objects.FillDict(constants.HVC_DEFAULTS[self.HVNAME], {}) # Override default VNC password file path if constants.HV_VNC_PASSWORD_FILE in hvp: hvp[constants.HV_VNC_PASSWORD_FILE] = self.vncpw_path disks = [ (objects.Disk(dev_type=constants.DT_PLAIN, mode=constants.DISK_RDWR), utils.PathJoin(self.tmpdir, "disk0"), NotImplemented), (objects.Disk(dev_type=constants.DT_PLAIN, mode=constants.DISK_RDONLY), utils.PathJoin(self.tmpdir, "disk1"), NotImplemented), ] inst = objects.Instance(name="server01.example.com", hvparams=hvp, beparams=bep, osparams={}, nics=[], os="deb1", disks=[d[0] for d in disks]) inst.UpgradeConfig() return (inst, disks) def testStartInstance(self): (inst, disks) = self._MakeInstance() pathutils.LOG_XEN_DIR = self.tmpdir for failcreate in [False, True]: for paused in [False, True]: run_cmd = compat.partial(self._StartInstanceCommand, inst, paused, failcreate) hv = self._GetHv(run_cmd=run_cmd) # Ensure instance is not listed self.assertTrue(inst.name not in hv.ListInstances()) # Remove configuration cfgfile = utils.PathJoin(self.tmpdir, inst.name) utils.RemoveFile(cfgfile) if failcreate: self.assertRaises(errors.HypervisorError, hv.StartInstance, inst, disks, paused) # Check whether a stale config file is left behind self.assertFalse(os.path.exists(cfgfile)) else: hv.StartInstance(inst, disks, paused) # Check if configuration was updated lines = utils.ReadFile(cfgfile).splitlines() if constants.HV_VNC_PASSWORD_FILE in inst.hvparams: self.assertTrue(("vncpasswd = '%s'" % self.vncpw) in lines) else: extra = inst.hvparams[constants.HV_KERNEL_ARGS] self.assertTrue(("extra = '%s'" % extra) in lines) def _StopInstanceCommand(self, instance_name, force, fail, full_cmd): # Remove the timeout (and its number of seconds) if it's there if full_cmd[:1][0] == "timeout": cmd = full_cmd[2:] else: cmd = full_cmd # Test the actual command if (cmd == [self.CMD, "list"]): output = "Name ID Mem VCPUs State Time(s)\n" \ "Domain-0 0 1023 1 r----- 142691.0\n" \ "%s 417 128 1 r----- 3.2\n" % instance_name elif cmd[:2] == [self.CMD, "destroy"]: self.assertEqual(cmd[2:], [instance_name]) output = "" elif not force and cmd[:3] == [self.CMD, "shutdown", "-w"]: self.assertEqual(cmd[3:], [instance_name]) output = "" else: self.fail("Unhandled command: %s" % (cmd, )) if fail: # Simulate a failing command return self._FailingCommand(cmd) else: return self._SuccessCommand(output, cmd) def testStopInstance(self): name = "inst4284.example.com" cfgfile = utils.PathJoin(self.tmpdir, name) cfgdata = "config file content\n" for force in [False, True]: for fail in [False, True]: utils.WriteFile(cfgfile, data=cfgdata) run_cmd = compat.partial(self._StopInstanceCommand, name, force, fail) hv = self._GetHv(run_cmd=run_cmd) self.assertTrue(os.path.isfile(cfgfile)) if fail: try: hv._StopInstance(name, force, None, constants.DEFAULT_SHUTDOWN_TIMEOUT) except errors.HypervisorError as err: self.assertTrue(str(err).startswith("listing instances failed"), msg=str(err)) else: self.fail("Exception was not raised") self.assertEqual(utils.ReadFile(cfgfile), cfgdata, msg=("Configuration was removed when stopping" " instance failed")) else: hv._StopInstance(name, force, None, constants.DEFAULT_SHUTDOWN_TIMEOUT) self.assertFalse(os.path.exists(cfgfile)) def _MigrateNonRunningInstCmd(self, cmd): if cmd == [self.CMD, "list"]: output = testutils.ReadTestData("xen-xl-list-4.0.1-dom0-only.txt") else: self.fail("Unhandled command: %s" % (cmd, )) return self._SuccessCommand(output, cmd) def testMigrateInstanceNotRunning(self): name = "nonexistinginstance.example.com" target = constants.IP4_ADDRESS_LOCALHOST port = 14618 hv = self._GetHv(run_cmd=self._MigrateNonRunningInstCmd) try: hv._MigrateInstance(name, target, port, self.VALID_HVPARAMS, _ping_fn=NotImplemented) except errors.HypervisorError as err: self.assertEqual(str(err), "Instance not running, cannot migrate") else: self.fail("Exception was not raised") def _MigrateInstanceCmd(self, instance_name, target, port, fail, cmd): if cmd == [self.CMD, "list"]: output = testutils.ReadTestData("xen-xl-list-4.0.1-four-instances.txt") elif cmd[:2] == [self.CMD, "migrate"]: args = [ "-s", constants.XL_SOCAT_CMD % (target, port), "-C", utils.PathJoin(self.tmpdir, instance_name), ] args.extend([instance_name, target]) self.assertEqual(cmd[2:], args) if fail: return self._FailingCommand(cmd) output = "" else: self.fail("Unhandled command: %s" % (cmd, )) return self._SuccessCommand(output, cmd) def testMigrateInstance(self): instname = "server01.example.com" target = constants.IP4_ADDRESS_LOCALHOST port = 22364 hvparams = {} for fail in [False, True]: ping_fn = \ testutils.CallCounter(compat.partial(self._FakeTcpPing, (target, port), True)) run_cmd = \ compat.partial(self._MigrateInstanceCmd, instname, target, port, fail) hv = self._GetHv(run_cmd=run_cmd) if fail: try: hv._MigrateInstance(instname, target, port, hvparams, _ping_fn=ping_fn) except errors.HypervisorError as err: self.assertTrue(str(err).startswith("Failed to migrate instance")) else: self.fail("Exception was not raised") else: hv._MigrateInstance(instname, target, port, hvparams, _ping_fn=ping_fn) expected_pings = 0 self.assertEqual(ping_fn.Count(), expected_pings) def _GetNodeInfoCmd(self, fail, cmd): if cmd == [self.CMD, "info"]: if fail: return self._FailingCommand(cmd) else: output = testutils.ReadTestData("xen-xl-info-4.0.1.txt") elif cmd == [self.CMD, "list"]: if fail: self.fail("'xl list' shouldn't be called when 'xl info' failed") else: output = testutils.ReadTestData("xen-xl-list-4.0.1-four-instances.txt") else: self.fail("Unhandled command: %s" % (cmd, )) return self._SuccessCommand(output, cmd) def testGetNodeInfo(self): run_cmd = compat.partial(self._GetNodeInfoCmd, False) hv = self._GetHv(run_cmd=run_cmd) result = hv.GetNodeInfo() self.assertEqual(result["hv_version"], (4, 0)) self.assertEqual(result["memory_free"], 8004) def testGetNodeInfoFailing(self): run_cmd = compat.partial(self._GetNodeInfoCmd, True) hv = self._GetHv(run_cmd=run_cmd) self.assertTrue(hv.GetNodeInfo() is None) class TestXenVersionsSafeForMigration(unittest.TestCase): def testHVVersionsLikelySafeForMigration(self): hv = hv_xen.XenHypervisor() self.assertTrue(hv.VersionsSafeForMigration([4, 0], [4, 1])) self.assertFalse(hv.VersionsSafeForMigration([4, 1], [4, 0])) self.assertFalse(hv.VersionsSafeForMigration([4, 0], [4, 2])) self.assertTrue(hv.VersionsSafeForMigration([4, 2, 7], [4, 2, 9])) self.assertTrue(hv.VersionsSafeForMigration([4, 2, 9], [4, 2, 7])) self.assertTrue(hv.VersionsSafeForMigration([4], [4])) self.assertFalse(hv.VersionsSafeForMigration([4], [5])) def _MakeTestClass(cls, cmd): """Makes a class for testing. The returned class has structure as shown in the following pseudo code: class Test{cls.__name__}{cmd}(_TestXenHypervisor, unittest.TestCase): TARGET = {cls} CMD = {cmd} HVNAME = {Hypervisor name retrieved using class} @type cls: class @param cls: Hypervisor class to be tested @type cmd: string @param cmd: Hypervisor command @rtype: tuple @return: Class name and class object (not instance) """ name = "Test%sCmd%s" % (cls.__name__, cmd.title()) bases = (_TestXenHypervisor, unittest.TestCase) hvname = HVCLASS_TO_HVNAME[cls] return (name, type(name, bases, dict(TARGET=cls, CMD=cmd, HVNAME=hvname))) # Create test classes programmatically instead of manually to reduce the risk # of forgetting some combinations for cls in [hv_xen.XenPvmHypervisor, hv_xen.XenHvmHypervisor]: (name, testcls) = _MakeTestClass(cls, "xl") assert name not in locals() locals()[name] = testcls if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.hypervisor_unittest.py000075500000000000000000000062221476477700300244550ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing hypervisor functionality""" import unittest from ganeti import constants from ganeti import compat from ganeti import objects from ganeti import errors from ganeti import hypervisor from ganeti.hypervisor import hv_base import testutils class TestParameters(unittest.TestCase): def test(self): for hv, const_params in constants.HVC_DEFAULTS.items(): hyp = hypervisor.GetHypervisorClass(hv) for pname in const_params: self.assertTrue(pname in hyp.PARAMETERS, "Hypervisor %s: parameter %s defined in constants" " but not in the permitted hypervisor parameters" % (hv, pname)) for pname in hyp.PARAMETERS: self.assertTrue(pname in const_params, "Hypervisor %s: parameter %s defined in the hypervisor" " but missing a default value" % (hv, pname)) class TestBase(unittest.TestCase): def testVerifyResults(self): fn = hv_base.BaseHypervisor._FormatVerifyResults # FIXME: use assertIsNone when py 2.7 is minimum supported version self.assertEqual(fn([]), None) self.assertEqual(fn(["a"]), "a") self.assertEqual(fn(["a", "b"]), "a; b") def testGetLinuxNodeInfo(self): meminfo = testutils.TestDataFilename("proc_meminfo.txt") cpuinfo = testutils.TestDataFilename("proc_cpuinfo.txt") result = hv_base.BaseHypervisor.GetLinuxNodeInfo(meminfo, cpuinfo) self.assertEqual(result["memory_total"], 31506) self.assertEqual(result["memory_free"], 21098) self.assertEqual(result["memory_dom0"], 31506 - 21098) self.assertEqual(result["cpu_total"], 4) self.assertEqual(result["cpu_nodes"], 1) self.assertEqual(result["cpu_sockets"], 1) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.impexpd_unittest.py000075500000000000000000000246031476477700300237140ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.impexpd""" import os import sys import re import unittest import socket from ganeti import constants from ganeti import objects from ganeti import compat from ganeti import utils from ganeti import errors from ganeti import impexpd import testutils import unittest.mock class CmdBuilderConfig(objects.ConfigObject): __slots__ = [ "bind", "key", "cert", "ca", "host", "port", "ipv4", "ipv6", "compress", "magic", "connect_timeout", "connect_retries", "cmd_prefix", "cmd_suffix", ] def CheckCmdWord(cmd, word): wre = re.compile(r"\b%s\b" % re.escape(word)) return compat.any(wre.search(i) for i in cmd) class TestCommandBuilder(unittest.TestCase): def test(self): # The commands various compressions should use compress_import = { constants.IEC_GZIP: "gzip -d", constants.IEC_GZIP_FAST: "gzip -d", constants.IEC_GZIP_SLOW: "gzip -d", constants.IEC_LZOP: "lzop -d", } compress_export = { constants.IEC_GZIP: "gzip -1", constants.IEC_GZIP_FAST: "gzip -1", constants.IEC_GZIP_SLOW: "gzip", constants.IEC_LZOP: "lzop", } for mode in [constants.IEM_IMPORT, constants.IEM_EXPORT]: if mode == constants.IEM_IMPORT: compress_dict = compress_import elif mode == constants.IEM_EXPORT: compress_dict = compress_export for compress in constants.IEC_ALL: for magic in [None, 10 * "-", "HelloWorld", "J9plh4nFo2", "24A02A81-2264-4B51-A882-A2AB9D85B420"]: opts = CmdBuilderConfig( magic=magic, compress=compress, host="localhost", ) builder = impexpd.CommandBuilder(mode, opts, 1, 2, 3) magic_cmd = builder._GetMagicCommand() dd_cmd = builder._GetDdCommand() if magic: self.assertTrue(("M=%s" % magic) in magic_cmd) self.assertTrue(("M=%s" % magic) in dd_cmd) else: self.assertFalse(magic_cmd) testcases = [ { "target": "localhost", "expected_hostname": "localhost" }, { "target": "198.51.100.4", "expected_hostname": "my.ganeti.org" }, ] for host in testcases: socket_patcher = unittest.mock.patch( "socket.gethostbyaddr", return_value=(host["expected_hostname"], [], []), ) socket_patcher.start() for port in [0, 1, 1234, 7856, 45452]: for cmd_prefix in [None, "PrefixCommandGoesHere|", "dd if=/dev/hda bs=1048576 |"]: for cmd_suffix in [None, "< /some/file/name", "| dd of=/dev/null"]: opts = CmdBuilderConfig( host=host["target"], port=port, compress=compress, cmd_prefix=cmd_prefix, cmd_suffix=cmd_suffix, ) builder = impexpd.CommandBuilder(mode, opts, 1, 2, 3) # Check complete command cmd = builder.GetCommand() self.assertTrue(isinstance(cmd, list)) if compress != constants.IEC_NONE: self.assertTrue(CheckCmdWord(cmd, compress_dict[compress])) if cmd_prefix is not None: self.assertTrue(compat.any(cmd_prefix in i for i in cmd)) if cmd_suffix is not None: self.assertTrue(compat.any(cmd_suffix in i for i in cmd)) # Check socat command socat_cmd = builder._GetSocatCommand() if mode == constants.IEM_IMPORT: ssl_addr = socat_cmd[-2].split(",") self.assertTrue(("OPENSSL-LISTEN:%s" % port) in ssl_addr) elif mode == constants.IEM_EXPORT: ssl_addr = socat_cmd[-1].split(",") self.assertTrue( ("OPENSSL:%s:%s" % (host["target"], port)) in ssl_addr ) if impexpd.CommandBuilder._GetSocatVersion() >= (1, 7, 3): self.assertTrue("openssl-commonname=%s" % host["expected_hostname"] in ssl_addr) else: self.assertTrue("openssl-commonname=%s" % constants.X509_CERT_CN not in ssl_addr) self.assertTrue("verify=1" in ssl_addr) socket_patcher.stop() @testutils.RequiresIPv6() def testIPv6(self): for mode in [constants.IEM_IMPORT, constants.IEM_EXPORT]: opts = CmdBuilderConfig(host="localhost", port=6789, ipv4=False, ipv6=False) builder = impexpd.CommandBuilder(mode, opts, 1, 2, 3) cmd = builder._GetSocatCommand() self.assertTrue(compat.all("pf=" not in i for i in cmd)) # IPv4 opts = CmdBuilderConfig(host="localhost", port=6789, ipv4=True, ipv6=False) builder = impexpd.CommandBuilder(mode, opts, 1, 2, 3) cmd = builder._GetSocatCommand() self.assertTrue(compat.any(",pf=ipv4" in i for i in cmd)) # IPv6 opts = CmdBuilderConfig(host="localhost", port=6789, ipv4=False, ipv6=True) builder = impexpd.CommandBuilder(mode, opts, 1, 2, 3) cmd = builder._GetSocatCommand() self.assertTrue(compat.any(",pf=ipv6" in i for i in cmd)) # IPv4 and IPv6 opts = CmdBuilderConfig(host="localhost", port=6789, ipv4=True, ipv6=True) builder = impexpd.CommandBuilder(mode, opts, 1, 2, 3) self.assertRaises(AssertionError, builder._GetSocatCommand) def testCommaError(self): opts = CmdBuilderConfig(host="localhost", port=1234, ca="/some/path/with,a/,comma") for mode in [constants.IEM_IMPORT, constants.IEM_EXPORT]: builder = impexpd.CommandBuilder(mode, opts, 1, 2, 3) self.assertRaises(errors.GenericError, builder.GetCommand) def testOptionLengthError(self): testopts = [ CmdBuilderConfig(host="localhost", port=1234, ca="/tmp/ca" + ("B" * impexpd.SOCAT_OPTION_MAXLEN)), CmdBuilderConfig(host="localhost", port=1234, key="/tmp/key" + ("B" * impexpd.SOCAT_OPTION_MAXLEN)), ] for opts in testopts: for mode in [constants.IEM_IMPORT, constants.IEM_EXPORT]: builder = impexpd.CommandBuilder(mode, opts, 1, 2, 3) self.assertRaises(errors.GenericError, builder.GetCommand) opts.host = "localhost" + ("A" * impexpd.SOCAT_OPTION_MAXLEN) builder = impexpd.CommandBuilder(constants.IEM_EXPORT, opts, 1, 2, 3) self.assertRaises(errors.GenericError, builder.GetCommand) def testModeError(self): mode = "foobarbaz" assert mode not in [constants.IEM_IMPORT, constants.IEM_EXPORT] opts = CmdBuilderConfig(host="localhost", port=1234) builder = impexpd.CommandBuilder(mode, opts, 1, 2, 3) self.assertRaises(errors.GenericError, builder.GetCommand) class TestVerifyListening(unittest.TestCase): def test(self): self.assertEqual(impexpd._VerifyListening(socket.AF_INET, "192.0.2.7", 1234), ("192.0.2.7", 1234)) self.assertEqual(impexpd._VerifyListening(socket.AF_INET6, "::1", 9876), ("::1", 9876)) self.assertEqual(impexpd._VerifyListening(socket.AF_INET6, "[::1]", 4563), ("::1", 4563)) self.assertEqual(impexpd._VerifyListening(socket.AF_INET6, "[2001:db8::1:4563]", 4563), ("2001:db8::1:4563", 4563)) def testError(self): for family in [socket.AF_UNIX, socket.AF_INET, socket.AF_INET6]: self.assertRaises(errors.GenericError, impexpd._VerifyListening, family, "", 1234) self.assertRaises(errors.GenericError, impexpd._VerifyListening, family, "192", 999) for family in [socket.AF_UNIX, socket.AF_INET6]: self.assertRaises(errors.GenericError, impexpd._VerifyListening, family, "192.0.2.7", 1234) self.assertRaises(errors.GenericError, impexpd._VerifyListening, family, "[2001:db8::1", 1234) self.assertRaises(errors.GenericError, impexpd._VerifyListening, family, "2001:db8::1]", 1234) for family in [socket.AF_UNIX, socket.AF_INET]: self.assertRaises(errors.GenericError, impexpd._VerifyListening, family, "::1", 1234) class TestCalcThroughput(unittest.TestCase): def test(self): self.assertEqual(impexpd._CalcThroughput([]), None) self.assertEqual(impexpd._CalcThroughput([(0, 0)]), None) samples = [ (0.0, 0.0), (10.0, 100.0), ] self.assertAlmostEqual(impexpd._CalcThroughput(samples), 10.0, 3) samples = [ (5.0, 7.0), (10.0, 100.0), (16.0, 181.0), ] self.assertAlmostEqual(impexpd._CalcThroughput(samples), 15.818, 3) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.jqueue_unittest.py000075500000000000000000002076211476477700300235470ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.jqueue""" import os import sys import unittest import tempfile import shutil import errno import itertools import random try: # pylint: disable=E0611 from pyinotify import pyinotify except ImportError: import pyinotify from ganeti import constants from ganeti import utils from ganeti import errors from ganeti import jqueue from ganeti import opcodes from ganeti import compat from ganeti import mcpu from ganeti import query from ganeti import workerpool import testutils class _FakeJob: def __init__(self, job_id, status): self.id = job_id self.writable = False self._status = status self._log = [] def SetStatus(self, status): self._status = status def AddLogEntry(self, msg): self._log.append((len(self._log), msg)) def CalcStatus(self): return self._status def GetLogEntries(self, newer_than): assert newer_than is None or newer_than >= 0 if newer_than is None: return self._log return self._log[newer_than:] class TestEncodeOpError(unittest.TestCase): def test(self): encerr = jqueue._EncodeOpError(errors.LockError("Test 1")) self.assertTrue(isinstance(encerr, tuple)) self.assertRaises(errors.LockError, errors.MaybeRaise, encerr) encerr = jqueue._EncodeOpError(errors.GenericError("Test 2")) self.assertTrue(isinstance(encerr, tuple)) self.assertRaises(errors.GenericError, errors.MaybeRaise, encerr) encerr = jqueue._EncodeOpError(NotImplementedError("Foo")) self.assertTrue(isinstance(encerr, tuple)) self.assertRaises(errors.OpExecError, errors.MaybeRaise, encerr) encerr = jqueue._EncodeOpError("Hello World") self.assertTrue(isinstance(encerr, tuple)) self.assertRaises(errors.OpExecError, errors.MaybeRaise, encerr) class TestQueuedOpCode(unittest.TestCase): def testDefaults(self): def _Check(op): self.assertFalse(op.input.dry_run) self.assertEqual(op.priority, constants.OP_PRIO_DEFAULT) self.assertFalse(op.log) self.assertTrue(op.start_timestamp is None) self.assertTrue(op.exec_timestamp is None) self.assertTrue(op.end_timestamp is None) self.assertTrue(op.result is None) self.assertEqual(op.status, constants.OP_STATUS_QUEUED) op1 = jqueue._QueuedOpCode(opcodes.OpTestDelay()) _Check(op1) op2 = jqueue._QueuedOpCode.Restore(op1.Serialize()) _Check(op2) self.assertEqual(op1.Serialize(), op2.Serialize()) def testPriority(self): def _Check(op): assert constants.OP_PRIO_DEFAULT != constants.OP_PRIO_HIGH, \ "Default priority equals high priority; test can't work" self.assertEqual(op.priority, constants.OP_PRIO_HIGH) self.assertEqual(op.status, constants.OP_STATUS_QUEUED) inpop = opcodes.OpTagsGet(priority=constants.OP_PRIO_HIGH) op1 = jqueue._QueuedOpCode(inpop) _Check(op1) op2 = jqueue._QueuedOpCode.Restore(op1.Serialize()) _Check(op2) self.assertEqual(op1.Serialize(), op2.Serialize()) class TestQueuedJob(unittest.TestCase): def testNoOpCodes(self): self.assertRaises(errors.GenericError, jqueue._QueuedJob, None, 1, [], False) def testDefaults(self): job_id = 4260 ops = [ opcodes.OpTagsGet(), opcodes.OpTestDelay(), ] def _Check(job): self.assertTrue(job.writable) self.assertEqual(job.id, job_id) self.assertEqual(job.log_serial, 0) self.assertTrue(job.received_timestamp) self.assertTrue(job.start_timestamp is None) self.assertTrue(job.end_timestamp is None) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) self.assertTrue(repr(job).startswith("<")) self.assertEqual(len(job.ops), len(ops)) self.assertTrue(compat.all(inp.__getstate__() == op.input.__getstate__() for (inp, op) in zip(ops, job.ops))) self.assertFalse(job.archived) job1 = jqueue._QueuedJob(None, job_id, ops, True) _Check(job1) job2 = jqueue._QueuedJob.Restore(None, job1.Serialize(), True, False) _Check(job2) self.assertEqual(job1.Serialize(), job2.Serialize()) def testWritable(self): job = jqueue._QueuedJob(None, 1, [opcodes.OpTestDelay()], False) self.assertFalse(job.writable) job = jqueue._QueuedJob(None, 1, [opcodes.OpTestDelay()], True) self.assertTrue(job.writable) def testArchived(self): job = jqueue._QueuedJob(None, 1, [opcodes.OpTestDelay()], False) self.assertFalse(job.archived) newjob = jqueue._QueuedJob.Restore(None, job.Serialize(), True, True) self.assertTrue(newjob.archived) newjob2 = jqueue._QueuedJob.Restore(None, newjob.Serialize(), True, False) self.assertFalse(newjob2.archived) def testPriority(self): job_id = 4283 ops = [ opcodes.OpTagsGet(priority=constants.OP_PRIO_DEFAULT), opcodes.OpTestDelay(), ] def _Check(job): self.assertEqual(job.id, job_id) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertTrue(repr(job).startswith("<")) job = jqueue._QueuedJob(None, job_id, ops, True) _Check(job) self.assertTrue(compat.all(op.priority == constants.OP_PRIO_DEFAULT for op in job.ops)) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) # Increase first job.ops[0].priority -= 1 _Check(job) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT - 1) # Mark opcode as finished job.ops[0].status = constants.OP_STATUS_SUCCESS _Check(job) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) # Increase second job.ops[1].priority -= 10 self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT - 10) # Test increasing first job.ops[0].status = constants.OP_STATUS_RUNNING job.ops[0].priority -= 19 self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT - 20) def _JobForPriority(self, job_id): ops = [ opcodes.OpTagsGet(), opcodes.OpTestDelay(), opcodes.OpTagsGet(), opcodes.OpTestDelay(), ] job = jqueue._QueuedJob(None, job_id, ops, True) self.assertTrue(compat.all(op.priority == constants.OP_PRIO_DEFAULT for op in job.ops)) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) return job def testChangePriorityAllQueued(self): job = self._JobForPriority(24984) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertTrue(compat.all(op.status == constants.OP_STATUS_QUEUED for op in job.ops)) result = job.ChangePriority(-10) self.assertEqual(job.CalcPriority(), -10) self.assertTrue(compat.all(op.priority == -10 for op in job.ops)) self.assertEqual(result, (True, ("Priorities of pending opcodes for job 24984 have" " been changed to -10"))) def testChangePriorityAllFinished(self): job = self._JobForPriority(16405) for (idx, op) in enumerate(job.ops): if idx > 2: op.status = constants.OP_STATUS_ERROR else: op.status = constants.OP_STATUS_SUCCESS self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_ERROR) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) result = job.ChangePriority(-10) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) self.assertTrue(compat.all(op.priority == constants.OP_PRIO_DEFAULT for op in job.ops)) self.assertEqual([op.status for op in job.ops], [ constants.OP_STATUS_SUCCESS, constants.OP_STATUS_SUCCESS, constants.OP_STATUS_SUCCESS, constants.OP_STATUS_ERROR, ]) self.assertEqual(result, (False, "Job 16405 is finished")) def testChangePriorityCancelling(self): job = self._JobForPriority(31572) for (idx, op) in enumerate(job.ops): if idx > 1: op.status = constants.OP_STATUS_CANCELING else: op.status = constants.OP_STATUS_SUCCESS self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_CANCELING) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) result = job.ChangePriority(5) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) self.assertTrue(compat.all(op.priority == constants.OP_PRIO_DEFAULT for op in job.ops)) self.assertEqual([op.status for op in job.ops], [ constants.OP_STATUS_SUCCESS, constants.OP_STATUS_SUCCESS, constants.OP_STATUS_CANCELING, constants.OP_STATUS_CANCELING, ]) self.assertEqual(result, (False, "Job 31572 is cancelling")) def testChangePriorityFirstRunning(self): job = self._JobForPriority(1716215889) for (idx, op) in enumerate(job.ops): if idx == 0: op.status = constants.OP_STATUS_RUNNING else: op.status = constants.OP_STATUS_QUEUED self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) result = job.ChangePriority(7) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) self.assertEqual([op.priority for op in job.ops], [constants.OP_PRIO_DEFAULT, 7, 7, 7]) self.assertEqual([op.status for op in job.ops], [ constants.OP_STATUS_RUNNING, constants.OP_STATUS_QUEUED, constants.OP_STATUS_QUEUED, constants.OP_STATUS_QUEUED, ]) self.assertEqual(result, (True, ("Priorities of pending opcodes for job" " 1716215889 have been changed to 7"))) def testChangePriorityLastRunning(self): job = self._JobForPriority(1308) for (idx, op) in enumerate(job.ops): if idx == (len(job.ops) - 1): op.status = constants.OP_STATUS_RUNNING else: op.status = constants.OP_STATUS_SUCCESS self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) result = job.ChangePriority(-3) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) self.assertTrue(compat.all(op.priority == constants.OP_PRIO_DEFAULT for op in job.ops)) self.assertEqual([op.status for op in job.ops], [ constants.OP_STATUS_SUCCESS, constants.OP_STATUS_SUCCESS, constants.OP_STATUS_SUCCESS, constants.OP_STATUS_RUNNING, ]) self.assertEqual(result, (False, "Job 1308 had no pending opcodes")) def testChangePrioritySecondOpcodeRunning(self): job = self._JobForPriority(27701) self.assertEqual(len(job.ops), 4) job.ops[0].status = constants.OP_STATUS_SUCCESS job.ops[1].status = constants.OP_STATUS_RUNNING job.ops[2].status = constants.OP_STATUS_QUEUED job.ops[3].status = constants.OP_STATUS_QUEUED self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) result = job.ChangePriority(-19) self.assertEqual(job.CalcPriority(), -19) self.assertEqual([op.priority for op in job.ops], [constants.OP_PRIO_DEFAULT, constants.OP_PRIO_DEFAULT, -19, -19]) self.assertEqual([op.status for op in job.ops], [ constants.OP_STATUS_SUCCESS, constants.OP_STATUS_RUNNING, constants.OP_STATUS_QUEUED, constants.OP_STATUS_QUEUED, ]) self.assertEqual(result, (True, ("Priorities of pending opcodes for job" " 27701 have been changed to -19"))) def testChangePriorityWithInconsistentJob(self): job = self._JobForPriority(30097) self.assertEqual(len(job.ops), 4) # This job is invalid (as it has two opcodes marked as running) and make # the call fail because an unprocessed opcode precedes a running one (which # should never happen in reality) job.ops[0].status = constants.OP_STATUS_SUCCESS job.ops[1].status = constants.OP_STATUS_RUNNING job.ops[2].status = constants.OP_STATUS_QUEUED job.ops[3].status = constants.OP_STATUS_RUNNING self.assertRaises(AssertionError, job.ChangePriority, 19) def testCalcStatus(self): def _Queued(ops): # The default status is "queued" self.assertTrue(compat.all(op.status == constants.OP_STATUS_QUEUED for op in ops)) def _Waitlock1(ops): ops[0].status = constants.OP_STATUS_WAITING def _Waitlock2(ops): ops[0].status = constants.OP_STATUS_SUCCESS ops[1].status = constants.OP_STATUS_SUCCESS ops[2].status = constants.OP_STATUS_WAITING def _Running(ops): ops[0].status = constants.OP_STATUS_SUCCESS ops[1].status = constants.OP_STATUS_RUNNING for op in ops[2:]: op.status = constants.OP_STATUS_QUEUED def _Canceling1(ops): ops[0].status = constants.OP_STATUS_SUCCESS ops[1].status = constants.OP_STATUS_SUCCESS for op in ops[2:]: op.status = constants.OP_STATUS_CANCELING def _Canceling2(ops): for op in ops: op.status = constants.OP_STATUS_CANCELING def _Canceled(ops): for op in ops: op.status = constants.OP_STATUS_CANCELED def _Error1(ops): for idx, op in enumerate(ops): if idx > 3: op.status = constants.OP_STATUS_ERROR else: op.status = constants.OP_STATUS_SUCCESS def _Error2(ops): for op in ops: op.status = constants.OP_STATUS_ERROR def _Success(ops): for op in ops: op.status = constants.OP_STATUS_SUCCESS tests = { constants.JOB_STATUS_QUEUED: [_Queued], constants.JOB_STATUS_WAITING: [_Waitlock1, _Waitlock2], constants.JOB_STATUS_RUNNING: [_Running], constants.JOB_STATUS_CANCELING: [_Canceling1, _Canceling2], constants.JOB_STATUS_CANCELED: [_Canceled], constants.JOB_STATUS_ERROR: [_Error1, _Error2], constants.JOB_STATUS_SUCCESS: [_Success], } def _NewJob(): job = jqueue._QueuedJob(None, 1, [opcodes.OpTestDelay() for _ in range(10)], True) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertTrue(compat.all(op.status == constants.OP_STATUS_QUEUED for op in job.ops)) return job for status in constants.JOB_STATUS_ALL: sttests = tests[status] assert sttests for fn in sttests: job = _NewJob() fn(job.ops) self.assertEqual(job.CalcStatus(), status) class _FakeDependencyManager: def __init__(self): self._checks = [] self._notifications = [] self._waiting = set() def AddCheckResult(self, job, dep_job_id, dep_status, result): self._checks.append((job, dep_job_id, dep_status, result)) def CountPendingResults(self): return len(self._checks) def CountWaitingJobs(self): return len(self._waiting) def GetNextNotification(self): return self._notifications.pop(0) def JobWaiting(self, job): return job in self._waiting def CheckAndRegister(self, job, dep_job_id, dep_status): (exp_job, exp_dep_job_id, exp_dep_status, result) = self._checks.pop(0) assert exp_job == job assert exp_dep_job_id == dep_job_id assert exp_dep_status == dep_status (result_status, _) = result if result_status == jqueue._JobDependencyManager.WAIT: self._waiting.add(job) elif result_status == jqueue._JobDependencyManager.CONTINUE: self._waiting.remove(job) return result def NotifyWaiters(self, job_id): self._notifications.append(job_id) class _DisabledFakeDependencyManager: def JobWaiting(self, _): return False def CheckAndRegister(self, *args): assert False, "Should not be called" def NotifyWaiters(self, _): pass class _FakeQueueForProc: def __init__(self, depmgr=None): self._updates = [] self._submitted = [] self._submit_count = itertools.count(1000) if depmgr: self.depmgr = depmgr else: self.depmgr = _DisabledFakeDependencyManager() def GetNextUpdate(self): return self._updates.pop(0) def GetNextSubmittedJob(self): return self._submitted.pop(0) def UpdateJobUnlocked(self, job, replicate=True): self._updates.append((job, bool(replicate))) def SubmitManyJobs(self, jobs): job_ids = [next(self._submit_count) for _ in jobs] self._submitted.extend(zip(job_ids, jobs)) return job_ids class _FakeExecOpCodeForProc: def __init__(self, queue, before_start, after_start): self._queue = queue self._before_start = before_start self._after_start = after_start def __call__(self, op, cbs, timeout=None): assert isinstance(op, opcodes.OpTestDummy) if self._before_start: self._before_start(timeout, cbs.CurrentPriority()) cbs.NotifyStart() if self._after_start: self._after_start(op, cbs) if op.fail: raise errors.OpExecError("Error requested (%s)" % op.result) if hasattr(op, "submit_jobs") and op.submit_jobs is not None: return cbs.SubmitManyJobs(op.submit_jobs) return op.result class _JobProcessorTestUtils: def _CreateJob(self, queue, job_id, ops): job = jqueue._QueuedJob(queue, job_id, ops, True) self.assertFalse(job.start_timestamp) self.assertFalse(job.end_timestamp) self.assertEqual(len(ops), len(job.ops)) self.assertTrue(compat.all(op.input == inp for (op, inp) in zip(job.ops, ops))) return job class TestJobProcessor(unittest.TestCase, _JobProcessorTestUtils): def _GenericCheckJob(self, job): assert compat.all(isinstance(op.input, opcodes.OpTestDummy) for op in job.ops) self.assertTrue(job.start_timestamp) self.assertTrue(job.end_timestamp) self.assertEqual(job.start_timestamp, job.ops[0].start_timestamp) def testSuccess(self): queue = _FakeQueueForProc() for (job_id, opcount) in [(25351, 1), (6637, 3), (24644, 10), (32207, 100)]: ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=False) for i in range(opcount)] # Create job job = self._CreateJob(queue, job_id, ops) def _BeforeStart(timeout, priority): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) self.assertFalse(job.cur_opctx) def _AfterStart(op, cbs): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) self.assertFalse(job.cur_opctx) # Job is running, cancelling shouldn't be possible (success, _) = job.Cancel() self.assertFalse(success) opexec = _FakeExecOpCodeForProc(queue, _BeforeStart, _AfterStart) for idx in range(len(ops)): self.assertRaises(IndexError, queue.GetNextUpdate) result = jqueue._JobProcessor(queue, opexec, job)() self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) if idx == len(ops) - 1: # Last opcode self.assertEqual(result, jqueue._JobProcessor.FINISHED) else: self.assertEqual(result, jqueue._JobProcessor.DEFER) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertTrue(job.start_timestamp) self.assertFalse(job.end_timestamp) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_SUCCESS) self.assertTrue(compat.all(op.start_timestamp and op.end_timestamp for op in job.ops)) self._GenericCheckJob(job) # Calling the processor on a finished job should be a no-op self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertRaises(IndexError, queue.GetNextUpdate) def testOpcodeError(self): queue = _FakeQueueForProc() testdata = [ (17077, 1, 0, 0), (1782, 5, 2, 2), (18179, 10, 9, 9), (4744, 10, 3, 8), (23816, 100, 39, 45), ] for (job_id, opcount, failfrom, failto) in testdata: # Prepare opcodes ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=(failfrom <= i and i <= failto)) for i in range(opcount)] # Create job job = self._CreateJob(queue, str(job_id), ops) opexec = _FakeExecOpCodeForProc(queue, None, None) for idx in range(len(ops)): self.assertRaises(IndexError, queue.GetNextUpdate) result = jqueue._JobProcessor(queue, opexec, job)() # queued to waitlock self.assertEqual(queue.GetNextUpdate(), (job, True)) # waitlock to running self.assertEqual(queue.GetNextUpdate(), (job, True)) # Opcode result self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) if idx in (failfrom, len(ops) - 1): # Last opcode self.assertEqual(result, jqueue._JobProcessor.FINISHED) break self.assertEqual(result, jqueue._JobProcessor.DEFER) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertRaises(IndexError, queue.GetNextUpdate) # Check job status self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_ERROR) # Check opcode status self.assertTrue(compat.all(op.start_timestamp and op.end_timestamp for op in job.ops[:failfrom])) self._GenericCheckJob(job) # Calling the processor on a finished job should be a no-op self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertRaises(IndexError, queue.GetNextUpdate) def testCancelWhileInQueue(self): queue = _FakeQueueForProc() ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=False) for i in range(5)] # Create job job_id = 17045 job = self._CreateJob(queue, job_id, ops) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) # Mark as cancelled (success, _) = job.Cancel() self.assertTrue(success) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertFalse(job.start_timestamp) self.assertTrue(job.end_timestamp) self.assertTrue(compat.all(op.status == constants.OP_STATUS_CANCELED for op in job.ops)) # Serialize to check for differences before_proc = job.Serialize() # Simulate processor called in workerpool opexec = _FakeExecOpCodeForProc(queue, None, None) self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) # Check result self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_CANCELED) self.assertFalse(job.start_timestamp) self.assertTrue(job.end_timestamp) self.assertFalse(compat.any(op.start_timestamp or op.end_timestamp for op in job.ops)) # Must not have changed or written self.assertEqual(before_proc, job.Serialize()) self.assertRaises(IndexError, queue.GetNextUpdate) def testCancelWhileWaitlockInQueue(self): queue = _FakeQueueForProc() ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=False) for i in range(5)] # Create job job_id = 8645 job = self._CreateJob(queue, job_id, ops) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) job.ops[0].status = constants.OP_STATUS_WAITING assert len(job.ops) == 5 self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) # Mark as cancelling (success, _) = job.Cancel() self.assertTrue(success) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertTrue(compat.all(op.status == constants.OP_STATUS_CANCELING for op in job.ops)) opexec = _FakeExecOpCodeForProc(queue, None, None) self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) # Check result self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_CANCELED) self.assertFalse(job.start_timestamp) self.assertTrue(job.end_timestamp) self.assertFalse(compat.any(op.start_timestamp or op.end_timestamp for op in job.ops)) def testCancelWhileWaitlock(self): queue = _FakeQueueForProc() ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=False) for i in range(5)] # Create job job_id = 11009 job = self._CreateJob(queue, job_id, ops) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) def _BeforeStart(timeout, priority): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) # Mark as cancelled (success, _) = job.Cancel() self.assertTrue(success) self.assertTrue(compat.all(op.status == constants.OP_STATUS_CANCELING for op in job.ops)) self.assertRaises(IndexError, queue.GetNextUpdate) def _AfterStart(op, cbs): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) opexec = _FakeExecOpCodeForProc(queue, _BeforeStart, _AfterStart) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) # Check result self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_CANCELED) self.assertTrue(job.start_timestamp) self.assertTrue(job.end_timestamp) self.assertFalse(compat.all(op.start_timestamp and op.end_timestamp for op in job.ops)) def _TestCancelWhileSomething(self, cb): queue = _FakeQueueForProc() ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=False) for i in range(5)] # Create job job_id = 24314 job = self._CreateJob(queue, job_id, ops) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) def _BeforeStart(timeout, priority): self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) # Mark as cancelled (success, _) = job.Cancel() self.assertTrue(success) self.assertTrue(compat.all(op.status == constants.OP_STATUS_CANCELING for op in job.ops)) cb(queue) def _AfterStart(op, cbs): self.fail("Should not reach this") opexec = _FakeExecOpCodeForProc(queue, _BeforeStart, _AfterStart) self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) # Check result self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_CANCELED) self.assertTrue(job.start_timestamp) self.assertTrue(job.end_timestamp) self.assertFalse(compat.all(op.start_timestamp and op.end_timestamp for op in job.ops)) return queue def testCancelWhileWaitlockWithTimeout(self): def fn(_): # Fake an acquire attempt timing out raise mcpu.LockAcquireTimeout() self._TestCancelWhileSomething(fn) def testCancelWhileRunning(self): # Tests canceling a job with finished opcodes and more, unprocessed ones queue = _FakeQueueForProc() ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=False) for i in range(3)] # Create job job_id = 28492 job = self._CreateJob(queue, job_id, ops) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) opexec = _FakeExecOpCodeForProc(queue, None, None) # Run one opcode self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.DEFER) # Job goes back to queued self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) # Mark as cancelled (success, _) = job.Cancel() self.assertTrue(success) # Try processing another opcode (this will actually cancel the job) self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) # Check result self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_CANCELED) def testPartiallyRun(self): # Tests calling the processor on a job that's been partially run before the # program was restarted queue = _FakeQueueForProc() opexec = _FakeExecOpCodeForProc(queue, None, None) for job_id, successcount in [(30697, 1), (2552, 4), (12489, 9)]: ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=False) for i in range(10)] # Create job job = self._CreateJob(queue, job_id, ops) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) for _ in range(successcount): self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.DEFER) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertTrue(job.ops_iter) # Serialize and restore (simulates program restart) newjob = jqueue._QueuedJob.Restore(queue, job.Serialize(), True, False) self.assertFalse(newjob.ops_iter) self._TestPartial(newjob, successcount) def _TestPartial(self, job, successcount): self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertEqual(job.start_timestamp, job.ops[0].start_timestamp) queue = _FakeQueueForProc() opexec = _FakeExecOpCodeForProc(queue, None, None) for remaining in reversed(range(len(job.ops) - successcount)): result = jqueue._JobProcessor(queue, opexec, job)() self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) if remaining == 0: # Last opcode self.assertEqual(result, jqueue._JobProcessor.FINISHED) break self.assertEqual(result, jqueue._JobProcessor.DEFER) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_SUCCESS) self.assertTrue(compat.all(op.start_timestamp and op.end_timestamp for op in job.ops)) self._GenericCheckJob(job) # Calling the processor on a finished job should be a no-op self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertRaises(IndexError, queue.GetNextUpdate) # ... also after being restored job2 = jqueue._QueuedJob.Restore(queue, job.Serialize(), True, False) # Calling the processor on a finished job should be a no-op self.assertEqual(jqueue._JobProcessor(queue, opexec, job2)(), jqueue._JobProcessor.FINISHED) self.assertRaises(IndexError, queue.GetNextUpdate) def testProcessorOnRunningJob(self): ops = [opcodes.OpTestDummy(result="result", fail=False)] queue = _FakeQueueForProc() opexec = _FakeExecOpCodeForProc(queue, None, None) # Create job job = self._CreateJob(queue, 9571, ops) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) job.ops[0].status = constants.OP_STATUS_RUNNING assert len(job.ops) == 1 self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) # Calling on running job must fail self.assertRaises(errors.ProgrammerError, jqueue._JobProcessor(queue, opexec, job)) def testLogMessages(self): # Tests the "Feedback" callback function queue = _FakeQueueForProc() messages = { 1: [ (None, "Hello"), (None, "World"), (constants.ELOG_MESSAGE, "there"), ], 4: [ (constants.ELOG_JQUEUE_TEST, (1, 2, 3)), (constants.ELOG_JQUEUE_TEST, ("other", "type")), ], } ops = [opcodes.OpTestDummy(result="Logtest%s" % i, fail=False, messages=messages.get(i, [])) for i in range(5)] # Create job job = self._CreateJob(queue, 29386, ops) def _BeforeStart(timeout, priority): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) def _AfterStart(op, cbs): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) self.assertRaises(AssertionError, cbs.Feedback, "too", "many", "arguments") for (log_type, msg) in op.messages: self.assertRaises(IndexError, queue.GetNextUpdate) if log_type: cbs.Feedback(log_type, msg) else: cbs.Feedback(msg) # Check for job update without replication self.assertEqual(queue.GetNextUpdate(), (job, False)) self.assertRaises(IndexError, queue.GetNextUpdate) opexec = _FakeExecOpCodeForProc(queue, _BeforeStart, _AfterStart) for remaining in reversed(range(len(job.ops))): self.assertRaises(IndexError, queue.GetNextUpdate) result = jqueue._JobProcessor(queue, opexec, job)() self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) if remaining == 0: # Last opcode self.assertEqual(result, jqueue._JobProcessor.FINISHED) break self.assertEqual(result, jqueue._JobProcessor.DEFER) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_SUCCESS) logmsgcount = sum(len(m) for m in messages.values()) self._CheckLogMessages(job, logmsgcount) # Serialize and restore (simulates program restart) newjob = jqueue._QueuedJob.Restore(queue, job.Serialize(), True, False) self._CheckLogMessages(newjob, logmsgcount) def _CheckLogMessages(self, job, count): # Check serial self.assertEqual(job.log_serial, count) # Filter with serial assert count > 3 self.assertTrue(job.GetLogEntries(3)) # No log message after highest serial self.assertFalse(job.GetLogEntries(count)) self.assertFalse(job.GetLogEntries(count + 3)) def testSubmitManyJobs(self): queue = _FakeQueueForProc() job_id = 15656 ops = [ opcodes.OpTestDummy(result="Res0", fail=False, submit_jobs=[]), opcodes.OpTestDummy(result="Res1", fail=False, submit_jobs=[ [opcodes.OpTestDummy(result="r1j0", fail=False)], ]), opcodes.OpTestDummy(result="Res2", fail=False, submit_jobs=[ [opcodes.OpTestDummy(result="r2j0o0", fail=False), opcodes.OpTestDummy(result="r2j0o1", fail=False), opcodes.OpTestDummy(result="r2j0o2", fail=False), opcodes.OpTestDummy(result="r2j0o3", fail=False)], [opcodes.OpTestDummy(result="r2j1", fail=False)], [opcodes.OpTestDummy(result="r2j3o0", fail=False), opcodes.OpTestDummy(result="r2j3o1", fail=False)], ]), ] # Create job job = self._CreateJob(queue, job_id, ops) def _BeforeStart(timeout, priority): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) self.assertFalse(job.cur_opctx) def _AfterStart(op, cbs): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) self.assertFalse(job.cur_opctx) # Job is running, cancelling shouldn't be possible (success, _) = job.Cancel() self.assertFalse(success) opexec = _FakeExecOpCodeForProc(queue, _BeforeStart, _AfterStart) for idx in range(len(ops)): self.assertRaises(IndexError, queue.GetNextUpdate) result = jqueue._JobProcessor(queue, opexec, job)() self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) if idx == len(ops) - 1: # Last opcode self.assertEqual(result, jqueue._JobProcessor.FINISHED) else: self.assertEqual(result, jqueue._JobProcessor.DEFER) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertTrue(job.start_timestamp) self.assertFalse(job.end_timestamp) self.assertRaises(IndexError, queue.GetNextUpdate) for idx, submitted_ops in enumerate(job_ops for op in ops for job_ops in op.submit_jobs): self.assertEqual(queue.GetNextSubmittedJob(), (1000 + idx, submitted_ops)) self.assertRaises(IndexError, queue.GetNextSubmittedJob) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_SUCCESS) self._GenericCheckJob(job) # Calling the processor on a finished job should be a no-op self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertRaises(IndexError, queue.GetNextUpdate) def testJobDependency(self): depmgr = _FakeDependencyManager() queue = _FakeQueueForProc(depmgr=depmgr) self.assertEqual(queue.depmgr, depmgr) prev_job_id = 22113 prev_job_id2 = 28102 job_id = 29929 ops = [ opcodes.OpTestDummy(result="Res0", fail=False, depends=[ [prev_job_id2, None], [prev_job_id, None], ]), opcodes.OpTestDummy(result="Res1", fail=False), ] # Create job job = self._CreateJob(queue, job_id, ops) def _BeforeStart(timeout, priority): if attempt == 0 or attempt > 5: # Job should only be updated when it wasn't waiting for another job self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) self.assertFalse(job.cur_opctx) def _AfterStart(op, cbs): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) self.assertFalse(job.cur_opctx) # Job is running, cancelling shouldn't be possible (success, _) = job.Cancel() self.assertFalse(success) opexec = _FakeExecOpCodeForProc(queue, _BeforeStart, _AfterStart) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) counter = itertools.count() while True: attempt = next(counter) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertRaises(IndexError, depmgr.GetNextNotification) if attempt < 2: depmgr.AddCheckResult(job, prev_job_id2, None, (jqueue._JobDependencyManager.WAIT, "wait2")) elif attempt == 2: depmgr.AddCheckResult(job, prev_job_id2, None, (jqueue._JobDependencyManager.CONTINUE, "cont")) # The processor will ask for the next dependency immediately depmgr.AddCheckResult(job, prev_job_id, None, (jqueue._JobDependencyManager.WAIT, "wait")) elif attempt < 5: depmgr.AddCheckResult(job, prev_job_id, None, (jqueue._JobDependencyManager.WAIT, "wait")) elif attempt == 5: depmgr.AddCheckResult(job, prev_job_id, None, (jqueue._JobDependencyManager.CONTINUE, "cont")) if attempt == 2: self.assertEqual(depmgr.CountPendingResults(), 2) elif attempt > 5: self.assertEqual(depmgr.CountPendingResults(), 0) else: self.assertEqual(depmgr.CountPendingResults(), 1) result = jqueue._JobProcessor(queue, opexec, job)() if attempt == 0 or attempt >= 5: # Job should only be updated if there was an actual change self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertFalse(depmgr.CountPendingResults()) if attempt < 5: # Simulate waiting for other job self.assertEqual(result, jqueue._JobProcessor.WAITDEP) self.assertTrue(job.cur_opctx) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) self.assertRaises(IndexError, depmgr.GetNextNotification) self.assertTrue(job.start_timestamp) self.assertFalse(job.end_timestamp) continue if result == jqueue._JobProcessor.FINISHED: # Last opcode self.assertFalse(job.cur_opctx) break self.assertRaises(IndexError, depmgr.GetNextNotification) self.assertEqual(result, jqueue._JobProcessor.DEFER) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertTrue(job.start_timestamp) self.assertFalse(job.end_timestamp) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_SUCCESS) self.assertTrue(compat.all(op.start_timestamp and op.end_timestamp for op in job.ops)) self._GenericCheckJob(job) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertRaises(IndexError, depmgr.GetNextNotification) self.assertFalse(depmgr.CountPendingResults()) self.assertFalse(depmgr.CountWaitingJobs()) # Calling the processor on a finished job should be a no-op self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertRaises(IndexError, queue.GetNextUpdate) def testJobDependencyCancel(self): depmgr = _FakeDependencyManager() queue = _FakeQueueForProc(depmgr=depmgr) self.assertEqual(queue.depmgr, depmgr) prev_job_id = 13623 job_id = 30876 ops = [ opcodes.OpTestDummy(result="Res0", fail=False), opcodes.OpTestDummy(result="Res1", fail=False, depends=[ [prev_job_id, None], ]), opcodes.OpTestDummy(result="Res2", fail=False), ] # Create job job = self._CreateJob(queue, job_id, ops) def _BeforeStart(timeout, priority): if attempt == 0 or attempt > 5: # Job should only be updated when it wasn't waiting for another job self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) self.assertFalse(job.cur_opctx) def _AfterStart(op, cbs): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) self.assertFalse(job.cur_opctx) # Job is running, cancelling shouldn't be possible (success, _) = job.Cancel() self.assertFalse(success) opexec = _FakeExecOpCodeForProc(queue, _BeforeStart, _AfterStart) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) counter = itertools.count() while True: attempt = next(counter) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertRaises(IndexError, depmgr.GetNextNotification) if attempt == 0: # This will handle the first opcode pass elif attempt < 4: depmgr.AddCheckResult(job, prev_job_id, None, (jqueue._JobDependencyManager.WAIT, "wait")) elif attempt == 4: # Other job was cancelled depmgr.AddCheckResult(job, prev_job_id, None, (jqueue._JobDependencyManager.CANCEL, "cancel")) if attempt == 0: self.assertEqual(depmgr.CountPendingResults(), 0) else: self.assertEqual(depmgr.CountPendingResults(), 1) result = jqueue._JobProcessor(queue, opexec, job)() if attempt <= 1 or attempt >= 4: # Job should only be updated if there was an actual change self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertFalse(depmgr.CountPendingResults()) if attempt > 0 and attempt < 4: # Simulate waiting for other job self.assertEqual(result, jqueue._JobProcessor.WAITDEP) self.assertTrue(job.cur_opctx) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) self.assertRaises(IndexError, depmgr.GetNextNotification) self.assertTrue(job.start_timestamp) self.assertFalse(job.end_timestamp) continue if result == jqueue._JobProcessor.FINISHED: # Last opcode self.assertFalse(job.cur_opctx) break self.assertRaises(IndexError, depmgr.GetNextNotification) self.assertEqual(result, jqueue._JobProcessor.DEFER) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertTrue(job.start_timestamp) self.assertFalse(job.end_timestamp) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_CANCELED) self._GenericCheckJob(job) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertRaises(IndexError, depmgr.GetNextNotification) self.assertFalse(depmgr.CountPendingResults()) # Calling the processor on a finished job should be a no-op self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertRaises(IndexError, queue.GetNextUpdate) def testJobDependencyWrongstatus(self): depmgr = _FakeDependencyManager() queue = _FakeQueueForProc(depmgr=depmgr) self.assertEqual(queue.depmgr, depmgr) prev_job_id = 9741 job_id = 11763 ops = [ opcodes.OpTestDummy(result="Res0", fail=False), opcodes.OpTestDummy(result="Res1", fail=False, depends=[ [prev_job_id, None], ]), opcodes.OpTestDummy(result="Res2", fail=False), ] # Create job job = self._CreateJob(queue, job_id, ops) def _BeforeStart(timeout, priority): if attempt == 0 or attempt > 5: # Job should only be updated when it wasn't waiting for another job self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) self.assertFalse(job.cur_opctx) def _AfterStart(op, cbs): self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) self.assertFalse(job.cur_opctx) # Job is running, cancelling shouldn't be possible (success, _) = job.Cancel() self.assertFalse(success) opexec = _FakeExecOpCodeForProc(queue, _BeforeStart, _AfterStart) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) counter = itertools.count() while True: attempt = next(counter) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertRaises(IndexError, depmgr.GetNextNotification) if attempt == 0: # This will handle the first opcode pass elif attempt < 4: depmgr.AddCheckResult(job, prev_job_id, None, (jqueue._JobDependencyManager.WAIT, "wait")) elif attempt == 4: # Other job failed depmgr.AddCheckResult(job, prev_job_id, None, (jqueue._JobDependencyManager.WRONGSTATUS, "w")) if attempt == 0: self.assertEqual(depmgr.CountPendingResults(), 0) else: self.assertEqual(depmgr.CountPendingResults(), 1) result = jqueue._JobProcessor(queue, opexec, job)() if attempt <= 1 or attempt >= 4: # Job should only be updated if there was an actual change self.assertEqual(queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertFalse(depmgr.CountPendingResults()) if attempt > 0 and attempt < 4: # Simulate waiting for other job self.assertEqual(result, jqueue._JobProcessor.WAITDEP) self.assertTrue(job.cur_opctx) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) self.assertRaises(IndexError, depmgr.GetNextNotification) self.assertTrue(job.start_timestamp) self.assertFalse(job.end_timestamp) continue if result == jqueue._JobProcessor.FINISHED: # Last opcode self.assertFalse(job.cur_opctx) break self.assertRaises(IndexError, depmgr.GetNextNotification) self.assertEqual(result, jqueue._JobProcessor.DEFER) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertTrue(job.start_timestamp) self.assertFalse(job.end_timestamp) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_ERROR) self._GenericCheckJob(job) self.assertRaises(IndexError, queue.GetNextUpdate) self.assertRaises(IndexError, depmgr.GetNextNotification) self.assertFalse(depmgr.CountPendingResults()) # Calling the processor on a finished job should be a no-op self.assertEqual(jqueue._JobProcessor(queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertRaises(IndexError, queue.GetNextUpdate) class _FakeTimeoutStrategy: def __init__(self, timeouts): self.timeouts = timeouts self.attempts = 0 self.last_timeout = None def NextAttempt(self): self.attempts += 1 if self.timeouts: timeout = self.timeouts.pop(0) else: timeout = None self.last_timeout = timeout return timeout class TestJobProcessorTimeouts(unittest.TestCase, _JobProcessorTestUtils): def setUp(self): self.queue = _FakeQueueForProc() self.job = None self.curop = None self.opcounter = None self.timeout_strategy = None self.retries = 0 self.prev_tsop = None self.prev_prio = None self.prev_status = None self.lock_acq_prio = None self.gave_lock = None self.done_lock_before_blocking = False def _BeforeStart(self, timeout, priority): job = self.job # If status has changed, job must've been written if self.prev_status != self.job.ops[self.curop].status: self.assertEqual(self.queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, self.queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) ts = self.timeout_strategy self.assertTrue(timeout is None or isinstance(timeout, (int, float))) self.assertEqual(timeout, ts.last_timeout) self.assertEqual(priority, job.ops[self.curop].priority) self.gave_lock = True self.lock_acq_prio = priority if (self.curop == 3 and job.ops[self.curop].priority == constants.OP_PRIO_HIGHEST + 3): # Give locks before running into blocking acquire assert self.retries == 7 self.retries = 0 self.done_lock_before_blocking = True return if self.retries > 0: self.assertTrue(timeout is not None) self.retries -= 1 self.gave_lock = False raise mcpu.LockAcquireTimeout() if job.ops[self.curop].priority == constants.OP_PRIO_HIGHEST: assert self.retries == 0, "Didn't exhaust all retries at highest priority" assert not ts.timeouts self.assertTrue(timeout is None) def _AfterStart(self, op, cbs): job = self.job # Setting to "running" requires an update self.assertEqual(self.queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, self.queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_RUNNING) # Job is running, cancelling shouldn't be possible (success, _) = job.Cancel() self.assertFalse(success) def _NextOpcode(self): self.curop = next(self.opcounter) self.prev_prio = self.job.ops[self.curop].priority self.prev_status = self.job.ops[self.curop].status def _NewTimeoutStrategy(self): job = self.job self.assertEqual(self.retries, 0) if self.prev_tsop == self.curop: # Still on the same opcode, priority must've been increased self.assertEqual(self.prev_prio, job.ops[self.curop].priority + 1) if self.curop == 1: # Normal retry timeouts = list(range(10, 31, 10)) self.retries = len(timeouts) - 1 elif self.curop == 2: # Let this run into a blocking acquire timeouts = list(range(11, 61, 12)) self.retries = len(timeouts) elif self.curop == 3: # Wait for priority to increase, but give lock before blocking acquire timeouts = list(range(12, 100, 14)) self.retries = len(timeouts) self.assertFalse(self.done_lock_before_blocking) elif self.curop == 4: self.assertTrue(self.done_lock_before_blocking) # Timeouts, but no need to retry timeouts = list(range(10, 31, 10)) self.retries = 0 elif self.curop == 5: # Normal retry timeouts = list(range(19, 100, 11)) self.retries = len(timeouts) else: timeouts = [] self.retries = 0 assert len(job.ops) == 10 assert self.retries <= len(timeouts) ts = _FakeTimeoutStrategy(timeouts) self.timeout_strategy = ts self.prev_tsop = self.curop self.prev_prio = job.ops[self.curop].priority return ts def testTimeout(self): ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=False) for i in range(10)] # Create job job_id = 15801 job = self._CreateJob(self.queue, job_id, ops) self.job = job self.opcounter = itertools.count(0) opexec = _FakeExecOpCodeForProc(self.queue, self._BeforeStart, self._AfterStart) tsf = self._NewTimeoutStrategy self.assertFalse(self.done_lock_before_blocking) while True: proc = jqueue._JobProcessor(self.queue, opexec, job, _timeout_strategy_factory=tsf) self.assertRaises(IndexError, self.queue.GetNextUpdate) if self.curop is not None: self.prev_status = self.job.ops[self.curop].status self.lock_acq_prio = None result = proc(_nextop_fn=self._NextOpcode) assert self.curop is not None if result == jqueue._JobProcessor.FINISHED or self.gave_lock: # Got lock and/or job is done, result must've been written self.assertFalse(job.cur_opctx) self.assertEqual(self.queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, self.queue.GetNextUpdate) self.assertEqual(self.lock_acq_prio, job.ops[self.curop].priority) self.assertTrue(job.ops[self.curop].exec_timestamp) if result == jqueue._JobProcessor.FINISHED: self.assertFalse(job.cur_opctx) break self.assertEqual(result, jqueue._JobProcessor.DEFER) if self.curop == 0: self.assertEqual(job.ops[self.curop].start_timestamp, job.start_timestamp) if self.gave_lock: # Opcode finished, but job not yet done self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) else: # Did not get locks self.assertTrue(job.cur_opctx) self.assertEqual(job.cur_opctx._timeout_strategy._fn, self.timeout_strategy.NextAttempt) self.assertFalse(job.ops[self.curop].exec_timestamp) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_WAITING) # If priority has changed since acquiring locks, the job must've been # updated if self.lock_acq_prio != job.ops[self.curop].priority: self.assertEqual(self.queue.GetNextUpdate(), (job, True)) self.assertRaises(IndexError, self.queue.GetNextUpdate) self.assertTrue(job.start_timestamp) self.assertFalse(job.end_timestamp) self.assertEqual(self.curop, len(job.ops) - 1) self.assertEqual(self.job, job) self.assertEqual(next(self.opcounter), len(job.ops)) self.assertTrue(self.done_lock_before_blocking) self.assertRaises(IndexError, self.queue.GetNextUpdate) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_SUCCESS) self.assertTrue(compat.all(op.start_timestamp and op.end_timestamp for op in job.ops)) # Calling the processor on a finished job should be a no-op self.assertEqual(jqueue._JobProcessor(self.queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertRaises(IndexError, self.queue.GetNextUpdate) class TestJobProcessorChangePriority(unittest.TestCase, _JobProcessorTestUtils): def setUp(self): self.queue = _FakeQueueForProc() self.opexecprio = [] def _BeforeStart(self, timeout, priority): self.opexecprio.append(priority) def testChangePriorityWhileRunning(self): # Tests changing the priority on a job while it has finished opcodes # (successful) and more, unprocessed ones ops = [opcodes.OpTestDummy(result="Res%s" % i, fail=False) for i in range(3)] # Create job job_id = 3499 job = self._CreateJob(self.queue, job_id, ops) self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) opexec = _FakeExecOpCodeForProc(self.queue, self._BeforeStart, None) # Run first opcode self.assertEqual(jqueue._JobProcessor(self.queue, opexec, job)(), jqueue._JobProcessor.DEFER) # Job goes back to queued self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) self.assertEqual(self.opexecprio.pop(0), constants.OP_PRIO_DEFAULT) self.assertRaises(IndexError, self.opexecprio.pop, 0) # Change priority self.assertEqual(job.ChangePriority(-10), (True, ("Priorities of pending opcodes for job 3499 have" " been changed to -10"))) self.assertEqual(job.CalcPriority(), -10) # Process second opcode self.assertEqual(jqueue._JobProcessor(self.queue, opexec, job)(), jqueue._JobProcessor.DEFER) self.assertEqual(self.opexecprio.pop(0), -10) self.assertRaises(IndexError, self.opexecprio.pop, 0) # Check status self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED) self.assertEqual(job.CalcPriority(), -10) # Change priority once more self.assertEqual(job.ChangePriority(5), (True, ("Priorities of pending opcodes for job 3499 have" " been changed to 5"))) self.assertEqual(job.CalcPriority(), 5) # Process third opcode self.assertEqual(jqueue._JobProcessor(self.queue, opexec, job)(), jqueue._JobProcessor.FINISHED) self.assertEqual(self.opexecprio.pop(0), 5) self.assertRaises(IndexError, self.opexecprio.pop, 0) # Check status self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_SUCCESS) self.assertEqual(job.CalcPriority(), constants.OP_PRIO_DEFAULT) self.assertEqual([op.priority for op in job.ops], [constants.OP_PRIO_DEFAULT, -10, 5]) class _IdOnlyFakeJob: def __init__(self, job_id, priority=NotImplemented): self.id = str(job_id) self._priority = priority def CalcPriority(self): return self._priority class TestJobDependencyManager(unittest.TestCase): def setUp(self): self._status = [] self._queue = [] self.jdm = jqueue._JobDependencyManager(self._GetStatus) def _GetStatus(self, job_id): (exp_job_id, result) = self._status.pop(0) self.assertEqual(exp_job_id, job_id) return result def testNotFinalizedThenCancel(self): job = _IdOnlyFakeJob(17697) job_id = str(28625) self._status.append((job_id, constants.JOB_STATUS_RUNNING)) (result, _) = self.jdm.CheckAndRegister(job, job_id, []) self.assertEqual(result, self.jdm.WAIT) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertTrue(self.jdm.JobWaiting(job)) self.assertEqual(self.jdm._waiters, { job_id: set([job]), }) self._status.append((job_id, constants.JOB_STATUS_CANCELED)) (result, _) = self.jdm.CheckAndRegister(job, job_id, []) self.assertEqual(result, self.jdm.CANCEL) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertFalse(self.jdm.JobWaiting(job)) def testNotFinalizedThenQueued(self): # This can happen on a queue shutdown job = _IdOnlyFakeJob(1320) job_id = str(22971) for i in range(5): if i > 2: self._status.append((job_id, constants.JOB_STATUS_QUEUED)) else: self._status.append((job_id, constants.JOB_STATUS_RUNNING)) (result, _) = self.jdm.CheckAndRegister(job, job_id, []) self.assertEqual(result, self.jdm.WAIT) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertTrue(self.jdm.JobWaiting(job)) self.assertEqual(self.jdm._waiters, { job_id: set([job]), }) def testRequireCancel(self): job = _IdOnlyFakeJob(5278) job_id = str(9610) dep_status = [constants.JOB_STATUS_CANCELED] self._status.append((job_id, constants.JOB_STATUS_WAITING)) (result, _) = self.jdm.CheckAndRegister(job, job_id, dep_status) self.assertEqual(result, self.jdm.WAIT) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertTrue(self.jdm.JobWaiting(job)) self.assertEqual(self.jdm._waiters, { job_id: set([job]), }) self._status.append((job_id, constants.JOB_STATUS_CANCELED)) (result, _) = self.jdm.CheckAndRegister(job, job_id, dep_status) self.assertEqual(result, self.jdm.CONTINUE) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertFalse(self.jdm.JobWaiting(job)) def testRequireError(self): job = _IdOnlyFakeJob(21459) job_id = str(25519) dep_status = [constants.JOB_STATUS_ERROR] self._status.append((job_id, constants.JOB_STATUS_WAITING)) (result, _) = self.jdm.CheckAndRegister(job, job_id, dep_status) self.assertEqual(result, self.jdm.WAIT) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertTrue(self.jdm.JobWaiting(job)) self.assertEqual(self.jdm._waiters, { job_id: set([job]), }) self._status.append((job_id, constants.JOB_STATUS_ERROR)) (result, _) = self.jdm.CheckAndRegister(job, job_id, dep_status) self.assertEqual(result, self.jdm.CONTINUE) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertFalse(self.jdm.JobWaiting(job)) def testRequireMultiple(self): dep_status = list(constants.JOBS_FINALIZED) for end_status in dep_status: job = _IdOnlyFakeJob(21343) job_id = str(14609) self._status.append((job_id, constants.JOB_STATUS_WAITING)) (result, _) = self.jdm.CheckAndRegister(job, job_id, dep_status) self.assertEqual(result, self.jdm.WAIT) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertTrue(self.jdm.JobWaiting(job)) self.assertEqual(self.jdm._waiters, { job_id: set([job]), }) self._status.append((job_id, end_status)) (result, _) = self.jdm.CheckAndRegister(job, job_id, dep_status) self.assertEqual(result, self.jdm.CONTINUE) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertFalse(self.jdm.JobWaiting(job)) def testWrongStatus(self): job = _IdOnlyFakeJob(10102) job_id = str(1271) self._status.append((job_id, constants.JOB_STATUS_QUEUED)) (result, _) = self.jdm.CheckAndRegister(job, job_id, [constants.JOB_STATUS_SUCCESS]) self.assertEqual(result, self.jdm.WAIT) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertTrue(self.jdm.JobWaiting(job)) self.assertEqual(self.jdm._waiters, { job_id: set([job]), }) self._status.append((job_id, constants.JOB_STATUS_ERROR)) (result, _) = self.jdm.CheckAndRegister(job, job_id, [constants.JOB_STATUS_SUCCESS]) self.assertEqual(result, self.jdm.WRONGSTATUS) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertFalse(self.jdm.JobWaiting(job)) def testCorrectStatus(self): job = _IdOnlyFakeJob(24273) job_id = str(23885) self._status.append((job_id, constants.JOB_STATUS_QUEUED)) (result, _) = self.jdm.CheckAndRegister(job, job_id, [constants.JOB_STATUS_SUCCESS]) self.assertEqual(result, self.jdm.WAIT) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertTrue(self.jdm.JobWaiting(job)) self.assertEqual(self.jdm._waiters, { job_id: set([job]), }) self._status.append((job_id, constants.JOB_STATUS_SUCCESS)) (result, _) = self.jdm.CheckAndRegister(job, job_id, [constants.JOB_STATUS_SUCCESS]) self.assertEqual(result, self.jdm.CONTINUE) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertFalse(self.jdm.JobWaiting(job)) def testFinalizedRightAway(self): job = _IdOnlyFakeJob(224) job_id = str(3081) self._status.append((job_id, constants.JOB_STATUS_SUCCESS)) (result, _) = self.jdm.CheckAndRegister(job, job_id, [constants.JOB_STATUS_SUCCESS]) self.assertEqual(result, self.jdm.CONTINUE) self.assertFalse(self._status) self.assertFalse(self._queue) self.assertFalse(self.jdm.JobWaiting(job)) self.assertEqual(self.jdm._waiters, { job_id: set(), }) def testSelfDependency(self): job = _IdOnlyFakeJob(18937) self._status.append((job.id, constants.JOB_STATUS_SUCCESS)) (result, _) = self.jdm.CheckAndRegister(job, job.id, []) self.assertEqual(result, self.jdm.ERROR) def testJobDisappears(self): job = _IdOnlyFakeJob(30540) job_id = str(23769) def _FakeStatus(_): raise errors.JobLost("#msg#") jdm = jqueue._JobDependencyManager(_FakeStatus) (result, _) = jdm.CheckAndRegister(job, job_id, []) self.assertEqual(result, self.jdm.ERROR) self.assertFalse(jdm.JobWaiting(job)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.jstore_unittest.py000075500000000000000000000074051476477700300235550ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.jstore""" import re import unittest import random from ganeti import constants from ganeti import utils from ganeti import compat from ganeti import errors from ganeti import jstore import testutils class TestFormatJobID(testutils.GanetiTestCase): def test(self): self.assertEqual(jstore.FormatJobID(0), 0) self.assertEqual(jstore.FormatJobID(30498), 30498) self.assertEqual(jstore.FormatJobID(319472592764518609), 319472592764518609) def testErrors(self): for i in [-1, -2288, -9667, -0.205641, 0.0, 0.1, 13041.4472, "", "Hello", [], [1], {}]: self.assertRaises(errors.ProgrammerError, jstore.FormatJobID, i) class TestGetArchiveDirectory(testutils.GanetiTestCase): def test(self): tests = [ ("0", [0, 1, 3343, 9712, 9999]), ("1", [10000, 13188, 19999]), ("29", [290000, 296041, 298796, 299999]), ("30", [300000, 309384]), ] for (exp, job_ids) in tests: for job_id in job_ids: fmt_id = jstore.FormatJobID(job_id) self.assertEqual(jstore.GetArchiveDirectory(fmt_id), exp) self.assertEqual(jstore.ParseJobId(fmt_id), job_id) def testErrors(self): self.assertRaises(errors.ParameterError, jstore.GetArchiveDirectory, None) self.assertRaises(errors.ParameterError, jstore.GetArchiveDirectory, "foo") class TestParseJobId(testutils.GanetiTestCase): def test(self): self.assertEqual(jstore.ParseJobId(29981), 29981) self.assertEqual(jstore.ParseJobId("12918"), 12918) def testErrors(self): self.assertRaises(errors.ParameterError, jstore.ParseJobId, "") self.assertRaises(errors.ParameterError, jstore.ParseJobId, "MXXI") self.assertRaises(errors.ParameterError, jstore.ParseJobId, []) class TestReadNumericFile(testutils.GanetiTestCase): def testNonExistingFile(self): result = jstore._ReadNumericFile("/tmp/this/file/does/not/exist") self.assertTrue(result is None) def testValidFile(self): tmpfile = self._CreateTempFile() for (data, exp) in [("123", 123), ("0\n", 0)]: utils.WriteFile(tmpfile, data=data) result = jstore._ReadNumericFile(tmpfile) self.assertEqual(result, exp) def testInvalidContent(self): tmpfile = self._CreateTempFile() utils.WriteFile(tmpfile, data="{wrong content") self.assertRaises(errors.JobQueueError, jstore._ReadNumericFile, tmpfile) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.locking_unittest.py000075500000000000000000001034651476477700300237000ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the locking module""" import os import unittest import time import queue import threading import random import gc import itertools from ganeti import constants from ganeti import locking from ganeti import errors from ganeti import utils from ganeti import compat from ganeti import objects from ganeti import query import testutils # This is used to test the ssynchronize decorator. # Since it's passed as input to a decorator it must be declared as a global. _decoratorlock = locking.SharedLock("decorator lock") #: List for looping tests ITERATIONS = range(8) def _Repeat(fn): """Decorator for executing a function many times""" def wrapper(*args, **kwargs): for i in ITERATIONS: fn(*args, **kwargs) return wrapper def SafeSleep(duration): start = time.time() while True: delay = start + duration - time.time() if delay <= 0.0: break time.sleep(delay) class _ThreadedTestCase(unittest.TestCase): """Test class that supports adding/waiting on threads""" def setUp(self): unittest.TestCase.setUp(self) self.done = queue.Queue(0) self.threads = [] def _addThread(self, *args, **kwargs): """Create and remember a new thread""" t = threading.Thread(*args, **kwargs) self.threads.append(t) t.start() return t def _waitThreads(self): """Wait for all our threads to finish""" for t in self.threads: t.join(60) self.assertFalse(t.is_alive()) self.threads = [] class _ConditionTestCase(_ThreadedTestCase): """Common test case for conditions""" def setUp(self, cls): _ThreadedTestCase.setUp(self) self.lock = threading.Lock() self.cond = cls(self.lock) def _testAcquireRelease(self): self.assertFalse(self.cond._is_owned()) self.assertRaises(RuntimeError, self.cond.wait, None) self.assertRaises(RuntimeError, self.cond.notify_all) self.cond.acquire() self.assertTrue(self.cond._is_owned()) self.cond.notify_all() self.assertTrue(self.cond._is_owned()) self.cond.release() self.assertFalse(self.cond._is_owned()) self.assertRaises(RuntimeError, self.cond.wait, None) self.assertRaises(RuntimeError, self.cond.notify_all) def _testNotification(self): def _NotifyAll(): self.done.put("NE") self.cond.acquire() self.done.put("NA") self.cond.notify_all() self.done.put("NN") self.cond.release() self.cond.acquire() self._addThread(target=_NotifyAll) self.assertEqual(self.done.get(True, 1), "NE") self.assertRaises(queue.Empty, self.done.get_nowait) self.cond.wait(None) self.assertEqual(self.done.get(True, 1), "NA") self.assertEqual(self.done.get(True, 1), "NN") self.assertTrue(self.cond._is_owned()) self.cond.release() self.assertFalse(self.cond._is_owned()) class TestSingleNotifyPipeCondition(_ConditionTestCase): """SingleNotifyPipeCondition tests""" def setUp(self): _ConditionTestCase.setUp(self, locking.SingleNotifyPipeCondition) def testAcquireRelease(self): self._testAcquireRelease() def testNotification(self): self._testNotification() def testWaitReuse(self): self.cond.acquire() self.cond.wait(0) self.cond.wait(0.1) self.cond.release() def testNoNotifyReuse(self): self.cond.acquire() self.cond.notify_all() self.assertRaises(RuntimeError, self.cond.wait, None) self.assertRaises(RuntimeError, self.cond.notify_all) self.cond.release() class TestPipeCondition(_ConditionTestCase): """PipeCondition tests""" def setUp(self): _ConditionTestCase.setUp(self, locking.PipeCondition) def testAcquireRelease(self): self._testAcquireRelease() def testNotification(self): self._testNotification() def _TestWait(self, fn): threads = [ self._addThread(target=fn), self._addThread(target=fn), self._addThread(target=fn), ] # Wait for threads to be waiting for _ in threads: self.assertEqual(self.done.get(True, 1), "A") self.assertRaises(queue.Empty, self.done.get_nowait) self.cond.acquire() self.assertEqual(len(self.cond._waiters), 3) self.assertEqual(self.cond._waiters, set(threads)) self.assertTrue(repr(self.cond).startswith("<")) self.assertTrue("waiters=" in repr(self.cond)) # This new thread can't acquire the lock, and thus call wait, before we # release it self._addThread(target=fn) self.cond.notify_all() self.assertRaises(queue.Empty, self.done.get_nowait) self.cond.release() # We should now get 3 W and 1 A (for the new thread) in whatever order w = 0 a = 0 for i in range(4): got = self.done.get(True, 1) if got == "W": w += 1 elif got == "A": a += 1 else: self.fail("Got %s on the done queue" % got) self.assertEqual(w, 3) self.assertEqual(a, 1) self.cond.acquire() self.cond.notify_all() self.cond.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "W") self.assertRaises(queue.Empty, self.done.get_nowait) def testBlockingWait(self): def _BlockingWait(): self.cond.acquire() self.done.put("A") self.cond.wait(None) self.cond.release() self.done.put("W") self._TestWait(_BlockingWait) def testLongTimeoutWait(self): def _Helper(): self.cond.acquire() self.done.put("A") self.cond.wait(15.0) self.cond.release() self.done.put("W") self._TestWait(_Helper) def _TimeoutWait(self, timeout, check): self.cond.acquire() self.cond.wait(timeout) self.cond.release() self.done.put(check) def testShortTimeoutWait(self): self._addThread(target=self._TimeoutWait, args=(0.1, "T1")) self._addThread(target=self._TimeoutWait, args=(0.1, "T1")) self._waitThreads() self.assertEqual(self.done.get_nowait(), "T1") self.assertEqual(self.done.get_nowait(), "T1") self.assertRaises(queue.Empty, self.done.get_nowait) def testZeroTimeoutWait(self): self._addThread(target=self._TimeoutWait, args=(0, "T0")) self._addThread(target=self._TimeoutWait, args=(0, "T0")) self._addThread(target=self._TimeoutWait, args=(0, "T0")) self._waitThreads() self.assertEqual(self.done.get_nowait(), "T0") self.assertEqual(self.done.get_nowait(), "T0") self.assertEqual(self.done.get_nowait(), "T0") self.assertRaises(queue.Empty, self.done.get_nowait) class TestSharedLock(_ThreadedTestCase): """SharedLock tests""" def setUp(self): _ThreadedTestCase.setUp(self) self.sl = locking.SharedLock("TestSharedLock") self.assertTrue(repr(self.sl).startswith("<")) self.assertTrue("name=TestSharedLock" in repr(self.sl)) def testSequenceAndOwnership(self): self.assertFalse(self.sl.is_owned()) self.sl.acquire(shared=1) self.assertTrue(self.sl.is_owned()) self.assertTrue(self.sl.is_owned(shared=1)) self.assertFalse(self.sl.is_owned(shared=0)) self.sl.release() self.assertFalse(self.sl.is_owned()) self.sl.acquire() self.assertTrue(self.sl.is_owned()) self.assertFalse(self.sl.is_owned(shared=1)) self.assertTrue(self.sl.is_owned(shared=0)) self.sl.release() self.assertFalse(self.sl.is_owned()) self.sl.acquire(shared=1) self.assertTrue(self.sl.is_owned()) self.assertTrue(self.sl.is_owned(shared=1)) self.assertFalse(self.sl.is_owned(shared=0)) self.sl.release() self.assertFalse(self.sl.is_owned()) def testBooleanValue(self): # semaphores are supposed to return a true value on a successful acquire self.assertTrue(self.sl.acquire(shared=1)) self.sl.release() self.assertTrue(self.sl.acquire()) self.sl.release() def testDoubleLockingStoE(self): self.sl.acquire(shared=1) self.assertRaises(AssertionError, self.sl.acquire) def testDoubleLockingEtoS(self): self.sl.acquire() self.assertRaises(AssertionError, self.sl.acquire, shared=1) def testDoubleLockingStoS(self): self.sl.acquire(shared=1) self.assertRaises(AssertionError, self.sl.acquire, shared=1) def testDoubleLockingEtoE(self): self.sl.acquire() self.assertRaises(AssertionError, self.sl.acquire) # helper functions: called in a separate thread they acquire the lock, send # their identifier on the done queue, then release it. def _doItSharer(self): try: self.sl.acquire(shared=1) self.done.put("SHR") self.sl.release() except errors.LockError: self.done.put("ERR") def _doItExclusive(self): try: self.sl.acquire() self.done.put("EXC") self.sl.release() except errors.LockError: self.done.put("ERR") def _doItDelete(self): try: self.sl.delete() self.done.put("DEL") except errors.LockError: self.done.put("ERR") def testSharersCanCoexist(self): self.sl.acquire(shared=1) threading.Thread(target=self._doItSharer).start() self.assertTrue(self.done.get(True, 1)) self.sl.release() @_Repeat def testExclusiveBlocksExclusive(self): self.sl.acquire() self._addThread(target=self._doItExclusive) self.assertRaises(queue.Empty, self.done.get_nowait) self.sl.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "EXC") @_Repeat def testExclusiveBlocksDelete(self): self.sl.acquire() self._addThread(target=self._doItDelete) self.assertRaises(queue.Empty, self.done.get_nowait) self.sl.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "DEL") self.sl = locking.SharedLock(self.sl.name) @_Repeat def testExclusiveBlocksSharer(self): self.sl.acquire() self._addThread(target=self._doItSharer) self.assertRaises(queue.Empty, self.done.get_nowait) self.sl.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "SHR") @_Repeat def testSharerBlocksExclusive(self): self.sl.acquire(shared=1) self._addThread(target=self._doItExclusive) self.assertRaises(queue.Empty, self.done.get_nowait) self.sl.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "EXC") @_Repeat def testSharerBlocksDelete(self): self.sl.acquire(shared=1) self._addThread(target=self._doItDelete) self.assertRaises(queue.Empty, self.done.get_nowait) self.sl.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "DEL") self.sl = locking.SharedLock(self.sl.name) @_Repeat def testWaitingExclusiveBlocksSharer(self): """SKIPPED testWaitingExclusiveBlockSharer""" return self.sl.acquire(shared=1) # the lock is acquired in shared mode... self._addThread(target=self._doItExclusive) # ...but now an exclusive is waiting... self._addThread(target=self._doItSharer) # ...so the sharer should be blocked as well self.assertRaises(queue.Empty, self.done.get_nowait) self.sl.release() self._waitThreads() # The exclusive passed before self.assertEqual(self.done.get_nowait(), "EXC") self.assertEqual(self.done.get_nowait(), "SHR") @_Repeat def testWaitingSharerBlocksExclusive(self): """SKIPPED testWaitingSharerBlocksExclusive""" return self.sl.acquire() # the lock is acquired in exclusive mode... self._addThread(target=self._doItSharer) # ...but now a sharer is waiting... self._addThread(target=self._doItExclusive) # ...the exclusive is waiting too... self.assertRaises(queue.Empty, self.done.get_nowait) self.sl.release() self._waitThreads() # The sharer passed before self.assertEqual(self.done.get_nowait(), "SHR") self.assertEqual(self.done.get_nowait(), "EXC") def testDelete(self): self.sl.delete() self.assertRaises(errors.LockError, self.sl.acquire) self.assertRaises(errors.LockError, self.sl.acquire, shared=1) self.assertRaises(errors.LockError, self.sl.delete) def testDeleteTimeout(self): self.assertTrue(self.sl.delete(timeout=60)) def testDeleteTimeoutFail(self): ready = threading.Event() finish = threading.Event() def fn(): self.sl.acquire(shared=0) ready.set() finish.wait() self.sl.release() self._addThread(target=fn) ready.wait() # Test if deleting a lock owned in exclusive mode by another thread fails # to delete when a timeout is used self.assertFalse(self.sl.delete(timeout=0.02)) finish.set() self._waitThreads() self.assertTrue(self.sl.delete()) self.assertRaises(errors.LockError, self.sl.acquire) def testNoDeleteIfSharer(self): self.sl.acquire(shared=1) self.assertRaises(AssertionError, self.sl.delete) @_Repeat def testDeletePendingSharersExclusiveDelete(self): self.sl.acquire() self._addThread(target=self._doItSharer) self._addThread(target=self._doItSharer) self._addThread(target=self._doItExclusive) self._addThread(target=self._doItDelete) self.sl.delete() self._waitThreads() # The threads who were pending return ERR for _ in range(4): self.assertEqual(self.done.get_nowait(), "ERR") self.sl = locking.SharedLock(self.sl.name) @_Repeat def testDeletePendingDeleteExclusiveSharers(self): self.sl.acquire() self._addThread(target=self._doItDelete) self._addThread(target=self._doItExclusive) self._addThread(target=self._doItSharer) self._addThread(target=self._doItSharer) self.sl.delete() self._waitThreads() # The two threads who were pending return both ERR self.assertEqual(self.done.get_nowait(), "ERR") self.assertEqual(self.done.get_nowait(), "ERR") self.assertEqual(self.done.get_nowait(), "ERR") self.assertEqual(self.done.get_nowait(), "ERR") self.sl = locking.SharedLock(self.sl.name) @_Repeat def testExclusiveAcquireTimeout(self): for shared in [0, 1]: on_queue = threading.Event() release_exclusive = threading.Event() def _LockExclusive(): self.sl.acquire(shared=0, test_notify=on_queue.set) self.done.put("A: start wait") release_exclusive.wait() self.done.put("A: end wait") self.sl.release() # Start thread to hold lock in exclusive mode self._addThread(target=_LockExclusive) # Wait for wait to begin self.assertEqual(self.done.get(timeout=60), "A: start wait") # Wait up to 60s to get lock, but release exclusive lock as soon as we're # on the queue self.assertTrue(self.sl.acquire(shared=shared, timeout=60, test_notify=release_exclusive.set)) self.done.put("got 2nd") self.sl.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "A: end wait") self.assertEqual(self.done.get_nowait(), "got 2nd") self.assertRaises(queue.Empty, self.done.get_nowait) @_Repeat def testAcquireExpiringTimeout(self): def _AcquireWithTimeout(shared, timeout): if not self.sl.acquire(shared=shared, timeout=timeout): self.done.put("timeout") for shared in [0, 1]: # Lock exclusively self.sl.acquire() # Start shared acquires with timeout between 0 and 20 ms for i in range(11): self._addThread(target=_AcquireWithTimeout, args=(shared, i * 2.0 / 1000.0)) # Wait for threads to finish (makes sure the acquire timeout expires # before releasing the lock) self._waitThreads() # Release lock self.sl.release() for _ in range(11): self.assertEqual(self.done.get_nowait(), "timeout") self.assertRaises(queue.Empty, self.done.get_nowait) @_Repeat def testSharedSkipExclusiveAcquires(self): # Tests whether shared acquires jump in front of exclusive acquires in the # queue. def _Acquire(shared, name, notify_ev, wait_ev): if notify_ev: notify_fn = notify_ev.set else: notify_fn = None if wait_ev: wait_ev.wait() if not self.sl.acquire(shared=shared, test_notify=notify_fn): return self.done.put(name) self.sl.release() # Get exclusive lock while we fill the queue self.sl.acquire() shrcnt1 = 5 shrcnt2 = 7 shrcnt3 = 9 shrcnt4 = 2 # Add acquires using threading.Event for synchronization. They'll be # acquired exactly in the order defined in this list. acquires = (shrcnt1 * [(1, "shared 1")] + 3 * [(0, "exclusive 1")] + shrcnt2 * [(1, "shared 2")] + shrcnt3 * [(1, "shared 3")] + shrcnt4 * [(1, "shared 4")] + 3 * [(0, "exclusive 2")]) ev_cur = None ev_prev = None for args in acquires: ev_cur = threading.Event() self._addThread(target=_Acquire, args=args + (ev_cur, ev_prev)) ev_prev = ev_cur # Wait for last acquire to start ev_prev.wait() # Expect 6 pending exclusive acquires and 1 for all shared acquires # together self.assertEqual(self.sl._count_pending(), 7) # Release exclusive lock and wait self.sl.release() self._waitThreads() # Check sequence for _ in range(shrcnt1 + shrcnt2 + shrcnt3 + shrcnt4): # Shared locks aren't guaranteed to be notified in order, but they'll be # first tmp = self.done.get_nowait() if tmp == "shared 1": shrcnt1 -= 1 elif tmp == "shared 2": shrcnt2 -= 1 elif tmp == "shared 3": shrcnt3 -= 1 elif tmp == "shared 4": shrcnt4 -= 1 self.assertEqual(shrcnt1, 0) self.assertEqual(shrcnt2, 0) self.assertEqual(shrcnt3, 0) self.assertEqual(shrcnt3, 0) for _ in range(3): self.assertEqual(self.done.get_nowait(), "exclusive 1") for _ in range(3): self.assertEqual(self.done.get_nowait(), "exclusive 2") self.assertRaises(queue.Empty, self.done.get_nowait) def testIllegalDowngrade(self): # Not yet acquired self.assertRaises(AssertionError, self.sl.downgrade) # Acquire in shared mode, downgrade should be no-op self.assertTrue(self.sl.acquire(shared=1)) self.assertTrue(self.sl.is_owned(shared=1)) self.assertTrue(self.sl.downgrade()) self.assertTrue(self.sl.is_owned(shared=1)) self.sl.release() def testDowngrade(self): self.assertTrue(self.sl.acquire()) self.assertTrue(self.sl.is_owned(shared=0)) self.assertTrue(self.sl.downgrade()) self.assertTrue(self.sl.is_owned(shared=1)) self.sl.release() @_Repeat def testDowngradeJumpsAheadOfExclusive(self): def _KeepExclusive(ev_got, ev_downgrade, ev_release): self.assertTrue(self.sl.acquire()) self.assertTrue(self.sl.is_owned(shared=0)) ev_got.set() ev_downgrade.wait() self.assertTrue(self.sl.is_owned(shared=0)) self.assertTrue(self.sl.downgrade()) self.assertTrue(self.sl.is_owned(shared=1)) ev_release.wait() self.assertTrue(self.sl.is_owned(shared=1)) self.sl.release() def _KeepExclusive2(ev_started, ev_release): self.assertTrue(self.sl.acquire(test_notify=ev_started.set)) self.assertTrue(self.sl.is_owned(shared=0)) ev_release.wait() self.assertTrue(self.sl.is_owned(shared=0)) self.sl.release() def _KeepShared(ev_started, ev_got, ev_release): self.assertTrue(self.sl.acquire(shared=1, test_notify=ev_started.set)) self.assertTrue(self.sl.is_owned(shared=1)) ev_got.set() ev_release.wait() self.assertTrue(self.sl.is_owned(shared=1)) self.sl.release() # Acquire lock in exclusive mode ev_got_excl1 = threading.Event() ev_downgrade_excl1 = threading.Event() ev_release_excl1 = threading.Event() th_excl1 = self._addThread(target=_KeepExclusive, args=(ev_got_excl1, ev_downgrade_excl1, ev_release_excl1)) ev_got_excl1.wait() # Start a second exclusive acquire ev_started_excl2 = threading.Event() ev_release_excl2 = threading.Event() th_excl2 = self._addThread(target=_KeepExclusive2, args=(ev_started_excl2, ev_release_excl2)) ev_started_excl2.wait() # Start shared acquires, will jump ahead of second exclusive acquire when # first exclusive acquire downgrades ev_shared = [(threading.Event(), threading.Event()) for _ in range(5)] ev_release_shared = threading.Event() th_shared = [self._addThread(target=_KeepShared, args=(ev_started, ev_got, ev_release_shared)) for (ev_started, ev_got) in ev_shared] # Wait for all shared acquires to start for (ev, _) in ev_shared: ev.wait() # Check lock information self.assertEqual(self.sl.GetLockInfo(set([query.LQ_MODE, query.LQ_OWNER])), [(self.sl.name, "exclusive", [th_excl1.name], None)]) [(_, _, _, pending), ] = self.sl.GetLockInfo(set([query.LQ_PENDING])) self.assertEqual([(pendmode, sorted(waiting)) for (pendmode, waiting) in pending], [("exclusive", [th_excl2.name]), ("shared", sorted(th.name for th in th_shared))]) # Shared acquires won't start until the exclusive lock is downgraded ev_downgrade_excl1.set() # Wait for all shared acquires to be successful for (_, ev) in ev_shared: ev.wait() # Check lock information again self.assertEqual(self.sl.GetLockInfo(set([query.LQ_MODE, query.LQ_PENDING])), [(self.sl.name, "shared", None, [("exclusive", [th_excl2.name])])]) [(_, _, owner, _), ] = self.sl.GetLockInfo(set([query.LQ_OWNER])) self.assertEqual(set(owner), set([th_excl1.name] + [th.name for th in th_shared])) ev_release_excl1.set() ev_release_excl2.set() ev_release_shared.set() self._waitThreads() self.assertEqual(self.sl.GetLockInfo(set([query.LQ_MODE, query.LQ_OWNER, query.LQ_PENDING])), [(self.sl.name, None, None, [])]) @_Repeat def testMixedAcquireTimeout(self): sync = threading.Event() def _AcquireShared(ev): if not self.sl.acquire(shared=1, timeout=None): return self.done.put("shared") # Notify main thread ev.set() # Wait for notification from main thread sync.wait() # Release lock self.sl.release() acquires = [] for _ in range(3): ev = threading.Event() self._addThread(target=_AcquireShared, args=(ev, )) acquires.append(ev) # Wait for all acquires to finish for i in acquires: i.wait() self.assertEqual(self.sl._count_pending(), 0) # Try to get exclusive lock self.assertFalse(self.sl.acquire(shared=0, timeout=0.02)) # Acquire exclusive without timeout exclsync = threading.Event() exclev = threading.Event() def _AcquireExclusive(): if not self.sl.acquire(shared=0): return self.done.put("exclusive") # Notify main thread exclev.set() # Wait for notification from main thread exclsync.wait() self.sl.release() self._addThread(target=_AcquireExclusive) # Try to get exclusive lock self.assertFalse(self.sl.acquire(shared=0, timeout=0.02)) # Make all shared holders release their locks sync.set() # Wait for exclusive acquire to succeed exclev.wait() self.assertEqual(self.sl._count_pending(), 0) # Try to get exclusive lock self.assertFalse(self.sl.acquire(shared=0, timeout=0.02)) def _AcquireSharedSimple(): if self.sl.acquire(shared=1, timeout=None): self.done.put("shared2") self.sl.release() for _ in range(10): self._addThread(target=_AcquireSharedSimple) # Tell exclusive lock to release exclsync.set() # Wait for everything to finish self._waitThreads() self.assertEqual(self.sl._count_pending(), 0) # Check sequence for _ in range(3): self.assertEqual(self.done.get_nowait(), "shared") self.assertEqual(self.done.get_nowait(), "exclusive") for _ in range(10): self.assertEqual(self.done.get_nowait(), "shared2") self.assertRaises(queue.Empty, self.done.get_nowait) def testPriority(self): # Acquire in exclusive mode self.assertTrue(self.sl.acquire(shared=0)) # Queue acquires def _Acquire(prev, next, shared, priority, result): prev.wait() self.sl.acquire(shared=shared, priority=priority, test_notify=next.set) try: self.done.put(result) finally: self.sl.release() counter = itertools.count(0) priorities = range(-20, 30) first = threading.Event() prev = first # Data structure: # { # priority: # [(shared/exclusive, set(acquire names), set(pending threads)), # (shared/exclusive, ...), # ..., # ], # } perprio = {} # References shared acquire per priority in L{perprio}. Data structure: # { # priority: (shared=1, set(acquire names), set(pending threads)), # } prioshared = {} for seed in [4979, 9523, 14902, 32440]: # Use a deterministic random generator rnd = random.Random(seed) for priority in [rnd.choice(priorities) for _ in range(30)]: modes = [0, 1] rnd.shuffle(modes) for shared in modes: # Unique name acqname = "%s/shr=%s/prio=%s" % (next(counter), shared, priority) ev = threading.Event() thread = self._addThread(target=_Acquire, args=(prev, ev, shared, priority, acqname)) prev = ev # Record expected aqcuire, see above for structure data = (shared, set([acqname]), set([thread])) priolist = perprio.setdefault(priority, []) if shared: priosh = prioshared.get(priority, None) if priosh: # Shared acquires are merged for i, j in zip(priosh[1:], data[1:]): i.update(j) assert data[0] == priosh[0] else: prioshared[priority] = data priolist.append(data) else: priolist.append(data) # Start all acquires and wait for them first.set() prev.wait() # Check lock information self.assertEqual(self.sl.GetLockInfo(set()), [(self.sl.name, None, None, None)]) self.assertEqual(self.sl.GetLockInfo(set([query.LQ_MODE, query.LQ_OWNER])), [(self.sl.name, "exclusive", [threading.current_thread().name], None)]) self._VerifyPrioPending(self.sl.GetLockInfo(set([query.LQ_PENDING])), perprio) # Let threads acquire the lock self.sl.release() # Wait for everything to finish self._waitThreads() self.assertTrue(self.sl._check_empty()) # Check acquires by priority for acquires in [perprio[i] for i in sorted(perprio.keys())]: for (_, names, _) in acquires: # For shared acquires, the set will contain 1..n entries. For exclusive # acquires only one. while names: names.remove(self.done.get_nowait()) self.assertFalse(compat.any(names for (_, names, _) in acquires)) self.assertRaises(queue.Empty, self.done.get_nowait) def _VerifyPrioPending(self, lockinfo, perprio): ((name, mode, owner, pending), ) = lockinfo self.assertEqual(name, self.sl.name) self.assertTrue(mode is None) self.assertTrue(owner is None) self.assertEqual([(pendmode, sorted(waiting)) for (pendmode, waiting) in pending], [(["exclusive", "shared"][int(bool(shared))], sorted(t.name for t in threads)) for acquires in [perprio[i] for i in sorted(perprio.keys())] for (shared, _, threads) in acquires]) class _FakeTimeForSpuriousNotifications: def __init__(self, now, check_end): self.now = now self.check_end = check_end # Deterministic random number generator self.rnd = random.Random(15086) def time(self): # Advance time if the random number generator thinks so (this is to test # multiple notifications without advancing the time) if self.rnd.random() < 0.3: self.now += self.rnd.random() self.check_end(self.now) return self.now @_Repeat def testAcquireTimeoutWithSpuriousNotifications(self): ready = threading.Event() locked = threading.Event() req = queue.Queue(0) epoch = 4000.0 timeout = 60.0 def check_end(now): self.assertFalse(locked.is_set()) # If we waited long enough (in virtual time), tell main thread to release # lock, otherwise tell it to notify once more req.put(now < (epoch + (timeout * 0.8))) time_fn = self._FakeTimeForSpuriousNotifications(epoch, check_end).time sl = locking.SharedLock("test", _time_fn=time_fn) # Acquire in exclusive mode sl.acquire(shared=0) def fn(): self.assertTrue(sl.acquire(shared=0, timeout=timeout, test_notify=ready.set)) locked.set() sl.release() self.done.put("success") # Start acquire with timeout and wait for it to be ready self._addThread(target=fn) ready.wait() # The separate thread is now waiting to acquire the lock, so start sending # spurious notifications. # Wait for separate thread to ask for another notification count = 0 while req.get(): # After sending the notification, the lock will take a short amount of # time to notice and to retrieve the current time sl._notify_topmost() count += 1 self.assertTrue(count > 100, "Not enough notifications were sent") self.assertFalse(locked.is_set()) # Some notifications have been sent, now actually release the lock sl.release() # Wait for lock to be acquired locked.wait() self._waitThreads() self.assertEqual(self.done.get_nowait(), "success") self.assertRaises(queue.Empty, self.done.get_nowait) class TestSharedLockInCondition(_ThreadedTestCase): """SharedLock as a condition lock tests""" def setUp(self): _ThreadedTestCase.setUp(self) self.sl = locking.SharedLock("TestSharedLockInCondition") self.setCondition() def setCondition(self): self.cond = threading.Condition(self.sl) def testKeepMode(self): self.cond.acquire(shared=1) self.assertTrue(self.sl.is_owned(shared=1)) self.cond.wait(0) self.assertTrue(self.sl.is_owned(shared=1)) self.cond.release() self.cond.acquire(shared=0) self.assertTrue(self.sl.is_owned(shared=0)) self.cond.wait(0) self.assertTrue(self.sl.is_owned(shared=0)) self.cond.release() class TestSharedLockInPipeCondition(TestSharedLockInCondition): """SharedLock as a pipe condition lock tests""" def setCondition(self): self.cond = locking.PipeCondition(self.sl) class TestSSynchronizedDecorator(_ThreadedTestCase): """Shared Lock Synchronized decorator test""" def setUp(self): _ThreadedTestCase.setUp(self) @locking.ssynchronized(_decoratorlock) def _doItExclusive(self): self.assertTrue(_decoratorlock.is_owned()) self.done.put("EXC") @locking.ssynchronized(_decoratorlock, shared=1) def _doItSharer(self): self.assertTrue(_decoratorlock.is_owned(shared=1)) self.done.put("SHR") def testDecoratedFunctions(self): self._doItExclusive() self.assertFalse(_decoratorlock.is_owned()) self._doItSharer() self.assertFalse(_decoratorlock.is_owned()) def testSharersCanCoexist(self): _decoratorlock.acquire(shared=1) threading.Thread(target=self._doItSharer).start() self.assertTrue(self.done.get(True, 1)) _decoratorlock.release() @_Repeat def testExclusiveBlocksExclusive(self): _decoratorlock.acquire() self._addThread(target=self._doItExclusive) # give it a bit of time to check that it's not actually doing anything self.assertRaises(queue.Empty, self.done.get_nowait) _decoratorlock.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "EXC") @_Repeat def testExclusiveBlocksSharer(self): _decoratorlock.acquire() self._addThread(target=self._doItSharer) self.assertRaises(queue.Empty, self.done.get_nowait) _decoratorlock.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "SHR") @_Repeat def testSharerBlocksExclusive(self): _decoratorlock.acquire(shared=1) self._addThread(target=self._doItExclusive) self.assertRaises(queue.Empty, self.done.get_nowait) _decoratorlock.release() self._waitThreads() self.assertEqual(self.done.get_nowait(), "EXC") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.luxi_unittest.py000075500000000000000000000031441476477700300232240ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the luxi module. Currently empty (after all the tests moved to ganeti.rpc.client_unittest.py).""" import unittest from ganeti import constants from ganeti import errors from ganeti import luxi from ganeti import serializer import testutils ganeti-3.1.0~rc2/test/py/legacy/ganeti.masterd.iallocator_unittest.py000075500000000000000000000175771476477700300260510ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.masterd.iallocator""" import unittest from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import objects from ganeti import ht from ganeti.masterd import iallocator import testutils class _StubIAllocator(object): def __init__(self, success): self.success = success class TestIAReqMultiInstanceAlloc(unittest.TestCase): def testResult(self): good_results = [ # First result (all instances "allocate") [ [["foo", ["a", "b"]], ["bar", ["c"]], ["baz", []]], [] ], # Second result (partial "allocate", partial "fail") [ [["bar", ["c", "b"]], ["baz", ["a"]]], ["foo"] ], # Third result (all instances "fail") [ [], ["foo", "bar", "baz"] ], ] bad_results = [ "foobar", 1234, [], [[]], [[], [], []], ] result_fn = iallocator.IAReqMultiInstanceAlloc.REQ_RESULT self.assertTrue(compat.all(map(result_fn, good_results))) self.assertFalse(compat.any(map(result_fn, bad_results))) class TestIARequestBase(unittest.TestCase): def testValidateResult(self): class _StubReqBase(iallocator.IARequestBase): MODE = constants.IALLOCATOR_MODE_ALLOC REQ_RESULT = ht.TBool stub = _StubReqBase() stub.ValidateResult(_StubIAllocator(True), True) self.assertRaises(errors.ResultValidationError, stub.ValidateResult, _StubIAllocator(True), "foo") stub.ValidateResult(_StubIAllocator(False), True) # We don't validate the result if the iallocation request was not successful stub.ValidateResult(_StubIAllocator(False), "foo") class _FakeConfigWithNdParams: def GetNdParams(self, _): return None class TestComputeBasicNodeData(unittest.TestCase): def setUp(self): self.fn = compat.partial(iallocator.IAllocator._ComputeBasicNodeData, _FakeConfigWithNdParams()) def testEmpty(self): self.assertEqual(self.fn({}), {}) def testSimple(self): node1 = objects.Node(name="node1", primary_ip="192.0.2.1", secondary_ip="192.0.2.2", offline=False, drained=False, master_candidate=True, master_capable=True, group="11112222", vm_capable=False) node2 = objects.Node(name="node2", primary_ip="192.0.2.3", secondary_ip="192.0.2.4", offline=True, drained=False, master_candidate=False, master_capable=False, group="11112222", vm_capable=True) assert node1 != node2 ninfo = { "#unused-1#": node1, "#unused-2#": node2, } self.assertEqual(self.fn(ninfo), { "node1": { "tags": [], "primary_ip": "192.0.2.1", "secondary_ip": "192.0.2.2", "offline": False, "drained": False, "master_candidate": True, "group": "11112222", "master_capable": True, "vm_capable": False, "ndparams": None, }, "node2": { "tags": [], "primary_ip": "192.0.2.3", "secondary_ip": "192.0.2.4", "offline": True, "drained": False, "master_candidate": False, "group": "11112222", "master_capable": False, "vm_capable": True, "ndparams": None, }, }) class TestProcessStorageInfo(unittest.TestCase): def setUp(self): self.free_storage_file = 23 self.total_storage_file = 42 self.free_storage_lvm = 69 self.total_storage_lvm = 666 self.space_info = [{"name": "mynode", "type": constants.ST_FILE, "storage_free": self.free_storage_file, "storage_size": self.total_storage_file}, {"name": "mynode", "type": constants.ST_LVM_VG, "storage_free": self.free_storage_lvm, "storage_size": self.total_storage_lvm}, {"name": "mynode", "type": constants.ST_LVM_PV, "storage_free": 33, "storage_size": 44}] def testComputeStorageDataFromNodeInfoDefault(self): has_lvm = False node_name = "mynode" (total_disk, free_disk, total_spindles, free_spindles) = \ iallocator.IAllocator._ComputeStorageDataFromSpaceInfo( self.space_info, node_name, has_lvm) # FIXME: right now, iallocator ignores anything else than LVM, adjust # this test once that arbitrary storage is supported self.assertEqual(0, free_disk) self.assertEqual(0, total_disk) def testComputeStorageDataFromNodeInfoLvm(self): has_lvm = True node_name = "mynode" (total_disk, free_disk, total_spindles, free_spindles) = \ iallocator.IAllocator._ComputeStorageDataFromSpaceInfo( self.space_info, node_name, has_lvm) self.assertEqual(self.free_storage_lvm, free_disk) self.assertEqual(self.total_storage_lvm, total_disk) def testComputeStorageDataFromSpaceInfoByTemplate(self): disk_template = constants.DT_FILE node_name = "mynode" (total_disk, free_disk, total_spindles, free_spindles) = \ iallocator.IAllocator._ComputeStorageDataFromSpaceInfoByTemplate( self.space_info, node_name, disk_template) self.assertEqual(self.free_storage_file, free_disk) self.assertEqual(self.total_storage_file, total_disk) def testComputeStorageDataFromSpaceInfoByTemplateLvm(self): disk_template = constants.DT_PLAIN node_name = "mynode" (total_disk, free_disk, total_spindles, free_spindles) = \ iallocator.IAllocator._ComputeStorageDataFromSpaceInfoByTemplate( self.space_info, node_name, disk_template) self.assertEqual(self.free_storage_lvm, free_disk) self.assertEqual(self.total_storage_lvm, total_disk) def testComputeStorageDataFromSpaceInfoByTemplateNoReport(self): disk_template = constants.DT_DISKLESS node_name = "mynode" (total_disk, free_disk, total_spindles, free_spindles) = \ iallocator.IAllocator._ComputeStorageDataFromSpaceInfoByTemplate( self.space_info, node_name, disk_template) self.assertEqual(0, free_disk) self.assertEqual(0, total_disk) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.masterd.instance_unittest.py000075500000000000000000000144001476477700300255020ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.masterd.instance""" import os import sys import unittest from ganeti import constants from ganeti import errors from ganeti import utils from ganeti import masterd from ganeti.masterd.instance import \ ImportExportTimeouts, _DiskImportExportBase, \ ComputeRemoteExportHandshake, CheckRemoteExportHandshake, \ ComputeRemoteImportDiskInfo, CheckRemoteExportDiskInfo, \ FormatProgress import testutils class TestMisc(unittest.TestCase): def testTimeouts(self): tmo = ImportExportTimeouts(0) self.assertEqual(tmo.connect, 0) self.assertEqual(tmo.listen, ImportExportTimeouts.DEFAULT_LISTEN_TIMEOUT) self.assertEqual(tmo.ready, ImportExportTimeouts.DEFAULT_READY_TIMEOUT) self.assertEqual(tmo.error, ImportExportTimeouts.DEFAULT_ERROR_TIMEOUT) self.assertEqual(tmo.progress, ImportExportTimeouts.DEFAULT_PROGRESS_INTERVAL) tmo = ImportExportTimeouts(999) self.assertEqual(tmo.connect, 999) tmo = ImportExportTimeouts(1, listen=2, error=3, ready=4, progress=5) self.assertEqual(tmo.connect, 1) self.assertEqual(tmo.listen, 2) self.assertEqual(tmo.error, 3) self.assertEqual(tmo.ready, 4) self.assertEqual(tmo.progress, 5) def testTimeoutExpired(self): self.assertTrue(utils.TimeoutExpired(100, 300, _time_fn=lambda: 500)) self.assertFalse(utils.TimeoutExpired(100, 300, _time_fn=lambda: 0)) self.assertFalse(utils.TimeoutExpired(100, 300, _time_fn=lambda: 100)) self.assertFalse(utils.TimeoutExpired(100, 300, _time_fn=lambda: 400)) def testDiskImportExportBaseDirect(self): self.assertRaises(AssertionError, _DiskImportExportBase, None, None, None, None, None, None, None) class TestRieHandshake(unittest.TestCase): def test(self): cds = "cd-secret" hs = ComputeRemoteExportHandshake(cds) self.assertEqual(len(hs), 3) self.assertEqual(hs[0], constants.RIE_VERSION) self.assertEqual(CheckRemoteExportHandshake(cds, hs), None) def testCheckErrors(self): self.assertTrue(CheckRemoteExportHandshake(None, None)) self.assertTrue(CheckRemoteExportHandshake("", "")) self.assertTrue(CheckRemoteExportHandshake("", ("xyz", "foo"))) def testCheckWrongHash(self): cds = "cd-secret999" self.assertTrue(CheckRemoteExportHandshake(cds, (0, "fakehash", "xyz"))) def testCheckWrongVersion(self): version = 14887 self.assertNotEqual(version, constants.RIE_VERSION) cds = "c28ac99" salt = "a19cf8cc06" msg = "%s:%s" % (version, constants.RIE_HANDSHAKE) hs = (version, utils.Sha1Hmac(cds, msg, salt=salt), salt) self.assertTrue(CheckRemoteExportHandshake(cds, hs)) class TestRieDiskInfo(unittest.TestCase): def test(self): cds = "bbf46ea9a" salt = "ee5ad9" di = ComputeRemoteImportDiskInfo(cds, salt, 0, "node1", 1234, "mag111") self.assertEqual(CheckRemoteExportDiskInfo(cds, 0, di), ("node1", 1234, "mag111")) for i in range(1, 100): # Wrong disk index self.assertRaises(errors.GenericError, CheckRemoteExportDiskInfo, cds, i, di) def testInvalidHostPort(self): cds = "3ZoJY8KtGJ" salt = "drK5oYiHWD" for host in [",", "...", "Hello World", "`", "!", "#", "\\"]: di = ComputeRemoteImportDiskInfo(cds, salt, 0, host, 1234, "magic") self.assertRaises(errors.OpPrereqError, CheckRemoteExportDiskInfo, cds, 0, di) for port in [-1, 792825908, "HelloWorld!", "`#", "\\\"", "_?_"]: di = ComputeRemoteImportDiskInfo(cds, salt, 0, "localhost", port, "magic") self.assertRaises(errors.OpPrereqError, CheckRemoteExportDiskInfo, cds, 0, di) def testCheckErrors(self): cds = "0776450535a" self.assertRaises(errors.GenericError, CheckRemoteExportDiskInfo, cds, 0, "") self.assertRaises(errors.GenericError, CheckRemoteExportDiskInfo, cds, 0, ()) self.assertRaises(errors.GenericError, CheckRemoteExportDiskInfo, cds, 0, ("", 1, 2, 3, 4, 5)) # No host/port self.assertRaises(errors.GenericError, CheckRemoteExportDiskInfo, cds, 0, ("", 1234, "magic", "", "")) self.assertRaises(errors.GenericError, CheckRemoteExportDiskInfo, cds, 0, ("host", 0, "magic", "", "")) self.assertRaises(errors.GenericError, CheckRemoteExportDiskInfo, cds, 0, ("host", 1234, "", "", "")) # Wrong hash self.assertRaises(errors.GenericError, CheckRemoteExportDiskInfo, cds, 0, ("nodeX", 123, "magic", "fakehash", "xyz")) class TestFormatProgress(unittest.TestCase): def test(self): FormatProgress((0, 0, None, None)) FormatProgress((100, 3.3, 30, None)) FormatProgress((100, 3.3, 30, 900)) self.assertEqual(FormatProgress((1500, 12, 30, None)), "1.5G, 12.0 MiB/s, 30%") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.mcpu_unittest.py000075500000000000000000000203131476477700300232040ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2009, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the mcpu module""" import unittest import itertools import mocks from cmdlib.testsupport.rpc_runner_mock import CreateRpcRunnerMock from ganeti import compat from ganeti import errors from ganeti import mcpu from ganeti import opcodes from ganeti import cmdlib from ganeti import locking from ganeti import serializer from ganeti import ht from ganeti import constants from ganeti.constants import \ LOCK_ATTEMPTS_TIMEOUT, \ LOCK_ATTEMPTS_MAXWAIT, \ LOCK_ATTEMPTS_MINWAIT import testutils # FIXME: Document what BGL whitelist means REQ_BGL_WHITELIST = compat.UniqueFrozenset([ opcodes.OpClusterActivateMasterIp, opcodes.OpClusterDeactivateMasterIp, opcodes.OpClusterDestroy, opcodes.OpClusterPostInit, opcodes.OpClusterRename, opcodes.OpNodeAdd, opcodes.OpNodeRemove, opcodes.OpTestAllocator, ]) class TestLockAttemptTimeoutStrategy(unittest.TestCase): def testConstants(self): tpa = mcpu.LockAttemptTimeoutStrategy._TIMEOUT_PER_ATTEMPT self.assertTrue(len(tpa) > LOCK_ATTEMPTS_TIMEOUT / LOCK_ATTEMPTS_MAXWAIT) self.assertTrue(sum(tpa) >= LOCK_ATTEMPTS_TIMEOUT) self.assertTrue(LOCK_ATTEMPTS_TIMEOUT >= 1800, msg="Waiting less than half an hour per priority") self.assertTrue(LOCK_ATTEMPTS_TIMEOUT <= 3600, msg="Waiting more than an hour per priority") def testSimple(self): strat = mcpu.LockAttemptTimeoutStrategy(_random_fn=lambda: 0.5, _time_fn=lambda: 0.0) prev = None for i in range(len(strat._TIMEOUT_PER_ATTEMPT)): timeout = strat.NextAttempt() self.assertTrue(timeout is not None) self.assertTrue(timeout <= LOCK_ATTEMPTS_MAXWAIT) self.assertTrue(timeout >= LOCK_ATTEMPTS_MINWAIT) self.assertTrue(prev is None or timeout >= prev) prev = timeout for _ in range(10): self.assertTrue(strat.NextAttempt() is None) class TestDispatchTable(unittest.TestCase): def test(self): for opcls in opcodes.OP_MAPPING.values(): if not opcls.WITH_LU: continue self.assertTrue(opcls in mcpu.Processor.DISPATCH_TABLE, msg="%s missing handler class" % opcls) # Check against BGL whitelist lucls = mcpu.Processor.DISPATCH_TABLE[opcls] if lucls.REQ_BGL: self.assertTrue(opcls in REQ_BGL_WHITELIST, msg=("%s not whitelisted for BGL" % opcls.OP_ID)) else: self.assertFalse(opcls in REQ_BGL_WHITELIST, msg=("%s whitelisted for BGL, but doesn't use it" % opcls.OP_ID)) class TestProcessResult(unittest.TestCase): def setUp(self): self._submitted = [] self._count = itertools.count(200) def _Submit(self, jobs): job_ids = [next(self._count) for _ in jobs] self._submitted.extend(zip(job_ids, jobs)) return job_ids def testNoJobs(self): for i in [object(), [], False, True, None, 1, 929, {}]: self.assertEqual(mcpu._ProcessResult(NotImplemented, NotImplemented, i), i) def testDefaults(self): src = opcodes.OpTestDummy() res = mcpu._ProcessResult(self._Submit, src, cmdlib.ResultWithJobs([[ opcodes.OpTestDelay(), opcodes.OpTestDelay(), ], [ opcodes.OpTestDelay(), ]])) self.assertEqual(res, { constants.JOB_IDS_KEY: [200, 201], }) (_, (op1, op2)) = self._submitted.pop(0) (_, (op3, )) = self._submitted.pop(0) self.assertRaises(IndexError, self._submitted.pop) for op in [op1, op2, op3]: self.assertTrue("OP_TEST_DUMMY" in op.comment) def testParams(self): src = opcodes.OpTestDummy(priority=constants.OP_PRIO_HIGH, debug_level=3) res = mcpu._ProcessResult(self._Submit, src, cmdlib.ResultWithJobs([[ opcodes.OpTestDelay(priority=constants.OP_PRIO_LOW), ], [ opcodes.OpTestDelay(comment="foobar", debug_level=10), ]], other=True, value=range(10))) self.assertEqual(res, { constants.JOB_IDS_KEY: [200, 201], "other": True, "value": range(10), }) (_, (op1, )) = self._submitted.pop(0) (_, (op2, )) = self._submitted.pop(0) self.assertRaises(IndexError, self._submitted.pop) self.assertEqual(op1.priority, constants.OP_PRIO_LOW) self.assertTrue("OP_TEST_DUMMY" in op1.comment) self.assertEqual(op1.debug_level, 3) # FIXME: as priority is mandatory, there is no way # of specifying "just inherit the priority". self.assertEqual(op2.comment, "foobar") self.assertEqual(op2.debug_level, 3) class TestExecLU(unittest.TestCase): class OpTest(opcodes.OpCode): OP_DSC_FIELD = "data" OP_PARAMS = [ ("data", ht.NoDefault, ht.TString, None), ] def setUp(self): self.ctx = mocks.FakeContext() self.cfg = self.ctx.GetConfig("ec_id") self.rpc = CreateRpcRunnerMock() self.proc = mcpu.Processor(self.ctx, "ec_id", enable_locks = False) self.op = self.OpTest() self.calc_timeout = lambda: 42 def testRunLU(self): lu = mocks.FakeLU(self.proc, self.op, self.cfg, self.rpc, None) self.proc._ExecLU(lu) def testRunLUWithPrereqError(self): prereq = errors.OpPrereqError(self.op, errors.ECODE_INVAL) lu = mocks.FakeLU(self.proc, self.op, self.cfg, self.rpc, prereq) self.assertRaises(errors.OpPrereqError, self.proc._LockAndExecLU, lu, locking.LEVEL_CLUSTER, self.calc_timeout) def testRunLUWithPrereqErrorMissingECode(self): prereq = errors.OpPrereqError(self.op) lu = mocks.FakeLU(self.proc, self.op, self.cfg, self.rpc, prereq) self.assertRaises(errors.OpPrereqError, self.proc._LockAndExecLU, lu, locking.LEVEL_CLUSTER, self.calc_timeout) class TestSecretParams(unittest.TestCase): def testSecretParamsCheckNoError(self): op = opcodes.OpInstanceCreate( instance_name="plain.example.com", pnode="master.example.com", disk_template=constants.DT_PLAIN, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[{ constants.IDISK_SIZE: 1024 }], osparams_secret= serializer.PrivateDict({"foo":"bar", "foo2":"bar2"}), os_type="debian-image") try: mcpu._CheckSecretParameters(op) except errors.OpPrereqError: self.fail("OpPrereqError raised unexpectedly in _CheckSecretParameters") def testSecretParamsCheckWithError(self): op = opcodes.OpInstanceCreate( instance_name="plain.example.com", pnode="master.example.com", disk_template=constants.DT_PLAIN, mode=constants.INSTANCE_CREATE, nics=[{}], disks=[{ constants.IDISK_SIZE: 1024 }], osparams_secret= serializer.PrivateDict({"foo":"bar", "secret_param":""}), os_type="debian-image") self.assertRaises(errors.OpPrereqError, mcpu._CheckSecretParameters, op) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.netutils_unittest.py000075500000000000000000000476751476477700300241330ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the netutils module""" import os import re import shutil import socket import tempfile import unittest import testutils from ganeti import constants from ganeti import errors from ganeti import netutils from ganeti import serializer from ganeti import utils def _GetSocketCredentials(path): """Connect to a Unix socket and return remote credentials. """ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: sock.settimeout(10) sock.connect(path) return netutils.GetSocketCredentials(sock) finally: sock.close() class TestGetSocketCredentials(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() self.sockpath = utils.PathJoin(self.tmpdir, "sock") self.listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.listener.settimeout(10) self.listener.bind(self.sockpath) self.listener.listen(1) def tearDown(self): self.listener.shutdown(socket.SHUT_RDWR) self.listener.close() shutil.rmtree(self.tmpdir) def test(self): (c2pr, c2pw) = os.pipe() # Start child process child = os.fork() if child == 0: try: data = serializer.DumpJson(_GetSocketCredentials(self.sockpath)) os.write(c2pw, data) os.close(c2pw) os._exit(0) finally: os._exit(1) os.close(c2pw) # Wait for one connection (conn, _) = self.listener.accept() conn.recv(1) conn.close() # Wait for result result = os.read(c2pr, 4096) os.close(c2pr) # Check child's exit code (_, status) = os.waitpid(child, 0) self.assertFalse(os.WIFSIGNALED(status)) self.assertEqual(os.WEXITSTATUS(status), 0) # Check result (pid, uid, gid) = serializer.LoadJson(result) self.assertEqual(pid, os.getpid()) self.assertEqual(uid, os.getuid()) self.assertEqual(gid, os.getgid()) class TestHostname(unittest.TestCase): """Testing case for Hostname""" def testUppercase(self): data = "AbC.example.com" self.assertEqual(netutils.Hostname.GetNormalizedName(data), data.lower()) def testTooLongName(self): data = "a.b." + "c" * 255 self.assertRaises(errors.OpPrereqError, netutils.Hostname.GetNormalizedName, data) def testTrailingDot(self): data = "a.b.c" self.assertEqual(netutils.Hostname.GetNormalizedName(data + "."), data) def testInvalidName(self): data = [ "a b", "a/b", ".a.b", "a..b", ] for value in data: self.assertRaises(errors.OpPrereqError, netutils.Hostname.GetNormalizedName, value) def testValidName(self): data = [ "a.b", "a-b", "a_b", "a.b.c", ] for value in data: self.assertEqual(netutils.Hostname.GetNormalizedName(value), value) class TestIPAddress(unittest.TestCase): def testIsValid(self): self.assertTrue(netutils.IPAddress.IsValid("0.0.0.0")) self.assertTrue(netutils.IPAddress.IsValid("127.0.0.1")) self.assertTrue(netutils.IPAddress.IsValid("::")) self.assertTrue(netutils.IPAddress.IsValid("::1")) def testNotIsValid(self): self.assertFalse(netutils.IPAddress.IsValid("0")) self.assertFalse(netutils.IPAddress.IsValid("1.1.1.256")) self.assertFalse(netutils.IPAddress.IsValid("a:g::1")) def testGetAddressFamily(self): fn = netutils.IPAddress.GetAddressFamily self.assertEqual(fn("127.0.0.1"), socket.AF_INET) self.assertEqual(fn("10.2.0.127"), socket.AF_INET) self.assertEqual(fn("::1"), socket.AF_INET6) self.assertEqual(fn("2001:db8::1"), socket.AF_INET6) self.assertRaises(errors.IPAddressError, fn, "0") def testValidateNetmask(self): for netmask in [0, 33]: self.assertFalse(netutils.IP4Address.ValidateNetmask(netmask)) for netmask in [1, 32]: self.assertTrue(netutils.IP4Address.ValidateNetmask(netmask)) for netmask in [0, 129]: self.assertFalse(netutils.IP6Address.ValidateNetmask(netmask)) for netmask in [1, 128]: self.assertTrue(netutils.IP6Address.ValidateNetmask(netmask)) def testGetClassFromX(self): self.assertTrue( netutils.IPAddress.GetClassFromIpVersion(constants.IP4_VERSION) == netutils.IP4Address) self.assertTrue( netutils.IPAddress.GetClassFromIpVersion(constants.IP6_VERSION) == netutils.IP6Address) self.assertTrue( netutils.IPAddress.GetClassFromIpFamily(socket.AF_INET) == netutils.IP4Address) self.assertTrue( netutils.IPAddress.GetClassFromIpFamily(socket.AF_INET6) == netutils.IP6Address) def testOwnLoopback(self): # FIXME: In a pure IPv6 environment this is no longer true self.assertTrue(netutils.IPAddress.Own("127.0.0.1"), "Should own 127.0.0.1 address") def testNotOwnAddress(self): self.assertFalse(netutils.IPAddress.Own("2001:db8::1"), "Should not own IP address 2001:db8::1") self.assertFalse(netutils.IPAddress.Own("192.0.2.1"), "Should not own IP address 192.0.2.1") def testFamilyVersionConversions(self): # IPAddress.GetAddressFamilyFromVersion self.assertEqual( netutils.IPAddress.GetAddressFamilyFromVersion(constants.IP4_VERSION), socket.AF_INET) self.assertEqual( netutils.IPAddress.GetAddressFamilyFromVersion(constants.IP6_VERSION), socket.AF_INET6) self.assertRaises(errors.ProgrammerError, netutils.IPAddress.GetAddressFamilyFromVersion, 3) # IPAddress.GetVersionFromAddressFamily self.assertEqual( netutils.IPAddress.GetVersionFromAddressFamily(socket.AF_INET), constants.IP4_VERSION) self.assertEqual( netutils.IPAddress.GetVersionFromAddressFamily(socket.AF_INET6), constants.IP6_VERSION) self.assertRaises(errors.ProgrammerError, netutils.IPAddress.GetVersionFromAddressFamily, socket.AF_UNIX) class TestIP4Address(unittest.TestCase): def testGetIPIntFromString(self): fn = netutils.IP4Address._GetIPIntFromString self.assertEqual(fn("0.0.0.0"), 0) self.assertEqual(fn("0.0.0.1"), 1) self.assertEqual(fn("127.0.0.1"), 2130706433) self.assertEqual(fn("192.0.2.129"), 3221226113) self.assertEqual(fn("255.255.255.255"), 2**32 - 1) self.assertNotEqual(fn("0.0.0.0"), 1) self.assertNotEqual(fn("0.0.0.0"), 1) def testIsValid(self): self.assertTrue(netutils.IP4Address.IsValid("0.0.0.0")) self.assertTrue(netutils.IP4Address.IsValid("127.0.0.1")) self.assertTrue(netutils.IP4Address.IsValid("192.0.2.199")) self.assertTrue(netutils.IP4Address.IsValid("255.255.255.255")) def testNotIsValid(self): self.assertFalse(netutils.IP4Address.IsValid("0")) self.assertFalse(netutils.IP4Address.IsValid("1")) self.assertFalse(netutils.IP4Address.IsValid("1.1.1")) self.assertFalse(netutils.IP4Address.IsValid("255.255.255.256")) self.assertFalse(netutils.IP4Address.IsValid("::1")) def testInNetwork(self): self.assertTrue(netutils.IP4Address.InNetwork("127.0.0.0/8", "127.0.0.1")) def testNotInNetwork(self): self.assertFalse(netutils.IP4Address.InNetwork("192.0.2.0/24", "127.0.0.1")) def testIsLoopback(self): self.assertTrue(netutils.IP4Address.IsLoopback("127.0.0.1")) def testNotIsLoopback(self): self.assertFalse(netutils.IP4Address.IsLoopback("192.0.2.1")) class TestIP6Address(unittest.TestCase): def testGetIPIntFromString(self): fn = netutils.IP6Address._GetIPIntFromString self.assertEqual(fn("::"), 0) self.assertEqual(fn("::1"), 1) self.assertEqual(fn("2001:db8::1"), 42540766411282592856903984951653826561) self.assertEqual(fn("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"), 2**128-1) self.assertNotEqual(netutils.IP6Address._GetIPIntFromString("::2"), 1) def testIsValid(self): self.assertTrue(netutils.IP6Address.IsValid("::")) self.assertTrue(netutils.IP6Address.IsValid("::1")) self.assertTrue(netutils.IP6Address.IsValid("1" + (":1" * 7))) self.assertTrue(netutils.IP6Address.IsValid("ffff" + (":ffff" * 7))) self.assertTrue(netutils.IP6Address.IsValid("::")) def testNotIsValid(self): self.assertFalse(netutils.IP6Address.IsValid("0")) self.assertFalse(netutils.IP6Address.IsValid(":1")) self.assertFalse(netutils.IP6Address.IsValid("f" + (":f" * 6))) self.assertFalse(netutils.IP6Address.IsValid("fffg" + (":ffff" * 7))) self.assertFalse(netutils.IP6Address.IsValid("fffff" + (":ffff" * 7))) self.assertFalse(netutils.IP6Address.IsValid("1" + (":1" * 8))) self.assertFalse(netutils.IP6Address.IsValid("127.0.0.1")) def testInNetwork(self): self.assertTrue(netutils.IP6Address.InNetwork("::1/128", "::1")) def testNotInNetwork(self): self.assertFalse(netutils.IP6Address.InNetwork("2001:db8::1/128", "::1")) def testIsLoopback(self): self.assertTrue(netutils.IP6Address.IsLoopback("::1")) def testNotIsLoopback(self): self.assertFalse(netutils.IP6Address.IsLoopback("2001:db8::1")) class _BaseTcpPingTest: """Base class for TcpPing tests against listen(2)ing port""" family = None address = None def setUp(self): self.listener = socket.socket(self.family, socket.SOCK_STREAM) self.listener.bind((self.address, 0)) self.listenerport = self.listener.getsockname()[1] self.listener.listen(1) def tearDown(self): self.listener.shutdown(socket.SHUT_RDWR) self.listener.close() del self.listener del self.listenerport def testTcpPingToLocalHostAccept(self): self.assertTrue(netutils.TcpPing(self.address, self.listenerport, timeout=constants.TCP_PING_TIMEOUT, live_port_needed=True, source=self.address, ), "failed to connect to test listener") self.assertTrue(netutils.TcpPing(self.address, self.listenerport, timeout=constants.TCP_PING_TIMEOUT, live_port_needed=True), "failed to connect to test listener (no source)") class TestIP4TcpPing(unittest.TestCase, _BaseTcpPingTest): """Testcase for IPv4 TCP version of ping - against listen(2)ing port""" family = socket.AF_INET address = constants.IP4_ADDRESS_LOCALHOST def setUp(self): unittest.TestCase.setUp(self) _BaseTcpPingTest.setUp(self) def tearDown(self): unittest.TestCase.tearDown(self) _BaseTcpPingTest.tearDown(self) @testutils.RequiresIPv6() class TestIP6TcpPing(unittest.TestCase, _BaseTcpPingTest): """Testcase for IPv6 TCP version of ping - against listen(2)ing port""" family = socket.AF_INET6 address = constants.IP6_ADDRESS_LOCALHOST def setUp(self): unittest.TestCase.setUp(self) _BaseTcpPingTest.setUp(self) def tearDown(self): unittest.TestCase.tearDown(self) _BaseTcpPingTest.tearDown(self) class _BaseTcpPingDeafTest: """Base class for TcpPing tests against non listen(2)ing port""" family = None address = None def setUp(self): self.deaflistener = socket.socket(self.family, socket.SOCK_STREAM) self.deaflistener.bind((self.address, 0)) self.deaflistenerport = self.deaflistener.getsockname()[1] def tearDown(self): self.deaflistener.close() del self.deaflistener del self.deaflistenerport def testTcpPingToLocalHostAcceptDeaf(self): self.assertFalse(netutils.TcpPing(self.address, self.deaflistenerport, timeout=constants.TCP_PING_TIMEOUT, live_port_needed=True, source=self.address, ), # need successful connect(2) "successfully connected to deaf listener") self.assertFalse(netutils.TcpPing(self.address, self.deaflistenerport, timeout=constants.TCP_PING_TIMEOUT, live_port_needed=True, ), # need successful connect(2) "successfully connected to deaf listener (no source)") def testTcpPingToLocalHostNoAccept(self): self.assertTrue(netutils.TcpPing(self.address, self.deaflistenerport, timeout=constants.TCP_PING_TIMEOUT, live_port_needed=False, source=self.address, ), # ECONNREFUSED is OK "failed to ping alive host on deaf port") self.assertTrue(netutils.TcpPing(self.address, self.deaflistenerport, timeout=constants.TCP_PING_TIMEOUT, live_port_needed=False, ), # ECONNREFUSED is OK "failed to ping alive host on deaf port (no source)") class TestIP4TcpPingDeaf(unittest.TestCase, _BaseTcpPingDeafTest): """Testcase for IPv4 TCP version of ping - against non listen(2)ing port""" family = socket.AF_INET address = constants.IP4_ADDRESS_LOCALHOST def setUp(self): self.deaflistener = socket.socket(self.family, socket.SOCK_STREAM) self.deaflistener.bind((self.address, 0)) self.deaflistenerport = self.deaflistener.getsockname()[1] def tearDown(self): self.deaflistener.close() del self.deaflistener del self.deaflistenerport @testutils.RequiresIPv6() class TestIP6TcpPingDeaf(unittest.TestCase, _BaseTcpPingDeafTest): """Testcase for IPv6 TCP version of ping - against non listen(2)ing port""" family = socket.AF_INET6 address = constants.IP6_ADDRESS_LOCALHOST def setUp(self): unittest.TestCase.setUp(self) _BaseTcpPingDeafTest.setUp(self) def tearDown(self): unittest.TestCase.tearDown(self) _BaseTcpPingDeafTest.tearDown(self) class TestFormatAddress(unittest.TestCase): """Testcase for FormatAddress""" def testFormatAddressUnixSocket(self): res1 = netutils.FormatAddress(("12352", 0, 0), family=socket.AF_UNIX) self.assertEqual(res1, "pid=12352, uid=0, gid=0") def testFormatAddressIP4(self): res1 = netutils.FormatAddress(("127.0.0.1", 1234), family=socket.AF_INET) self.assertEqual(res1, "127.0.0.1:1234") res2 = netutils.FormatAddress(("192.0.2.32", None), family=socket.AF_INET) self.assertEqual(res2, "192.0.2.32") def testFormatAddressIP6(self): res1 = netutils.FormatAddress(("::1", 1234), family=socket.AF_INET6) self.assertEqual(res1, "[::1]:1234") res2 = netutils.FormatAddress(("::1", None), family=socket.AF_INET6) self.assertEqual(res2, "[::1]") res2 = netutils.FormatAddress(("2001:db8::beef", "80"), family=socket.AF_INET6) self.assertEqual(res2, "[2001:db8::beef]:80") def testFormatAddressWithoutFamily(self): res1 = netutils.FormatAddress(("127.0.0.1", 1234)) self.assertEqual(res1, "127.0.0.1:1234") res2 = netutils.FormatAddress(("::1", 1234)) self.assertEqual(res2, "[::1]:1234") def testInvalidFormatAddress(self): self.assertRaises(errors.ParameterError, netutils.FormatAddress, "127.0.0.1") self.assertRaises(errors.ParameterError, netutils.FormatAddress, "127.0.0.1", family=socket.AF_INET) self.assertRaises(errors.ParameterError, netutils.FormatAddress, ("::1"), family=socket.AF_INET ) class TestIpParsing(testutils.GanetiTestCase): """Test the code that parses the ip command output""" def testIp4(self): valid_addresses = [constants.IP4_ADDRESS_ANY, constants.IP4_ADDRESS_LOCALHOST, "192.0.2.1", # RFC5737, IPv4 address blocks for docs "198.51.100.1", "203.0.113.1", ] for addr in valid_addresses: self.assertTrue(re.search(netutils._IP_RE_TEXT, addr)) def testIp6(self): valid_addresses = [constants.IP6_ADDRESS_ANY, constants.IP6_ADDRESS_LOCALHOST, "0:0:0:0:0:0:0:1", # other form for IP6_ADDRESS_LOCALHOST "0:0:0:0:0:0:0:0", # other form for IP6_ADDRESS_ANY "2001:db8:85a3::8a2e:370:7334", # RFC3849 IP6 docs block "2001:0db8:85a3:0000:0000:8a2e:0370:7334", "0:0:0:0:0:FFFF:192.0.2.1", # IPv4-compatible IPv6 "::FFFF:192.0.2.1", "0:0:0:0:0:0:203.0.113.1", # IPv4-mapped IPv6 "::203.0.113.1", ] for addr in valid_addresses: self.assertTrue(re.search(netutils._IP_RE_TEXT, addr)) def testParseIpCommandOutput(self): # IPv4-only, fake loopback interface tests = ["ip-addr-show-lo-ipv4.txt", "ip-addr-show-lo-oneline-ipv4.txt"] for test_file in tests: data = testutils.ReadTestData(test_file) addr = netutils._GetIpAddressesFromIpOutput(data) self.assertTrue(len(addr[4]) == 1 and addr[4][0] == "127.0.0.1" and not addr[6]) # IPv6-only, fake loopback interface tests = ["ip-addr-show-lo-ipv6.txt", "ip-addr-show-lo-ipv6.txt"] for test_file in tests: data = testutils.ReadTestData(test_file) addr = netutils._GetIpAddressesFromIpOutput(data) self.assertTrue(len(addr[6]) == 1 and addr[6][0] == "::1" and not addr[4]) # IPv4 and IPv6, fake loopback interface tests = ["ip-addr-show-lo.txt", "ip-addr-show-lo-oneline.txt"] for test_file in tests: data = testutils.ReadTestData(test_file) addr = netutils._GetIpAddressesFromIpOutput(data) self.assertTrue(len(addr[6]) == 1 and addr[6][0] == "::1" and len(addr[4]) == 1 and addr[4][0] == "127.0.0.1") # IPv4 and IPv6, dummy interface data = testutils.ReadTestData("ip-addr-show-dummy0.txt") addr = netutils._GetIpAddressesFromIpOutput(data) self.assertTrue(len(addr[6]) == 1 and addr[6][0] == "2001:db8:85a3::8a2e:370:7334" and len(addr[4]) == 1 and addr[4][0] == "192.0.2.1") class TestValidatePortNumber(unittest.TestCase): """Test netutils.ValidatePortNumber""" def testPortNumberInt(self): self.assertRaises(ValueError, lambda: \ netutils.ValidatePortNumber(500000)) self.assertEqual(netutils.ValidatePortNumber(5000), 5000) def testPortNumberStr(self): self.assertRaises(ValueError, lambda: \ netutils.ValidatePortNumber("pinky bunny")) self.assertEqual(netutils.ValidatePortNumber("5000"), 5000) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.objects_unittest.py000075500000000000000000000754711476477700300237100ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2008, 2010, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the objects module""" import copy import pprint import unittest from ganeti import constants from ganeti import objects from ganeti import errors from ganeti import serializer import testutils class SimpleObject(objects.ConfigObject): __slots__ = ["a", "b"] class TestDictState(unittest.TestCase): """Simple dict tansformation tests""" def testSimpleObjectToDict(self): o1 = SimpleObject(a="1") self.assertEqual(o1.ToDict(), {"a": "1"}) self.assertEqual(o1.__getstate__(), {"a": "1"}) self.assertEqual(o1.__getstate__(), o1.ToDict()) o1.a = 2 o1.b = 5 self.assertEqual(o1.ToDict(), {"a": 2, "b": 5}) o2 = SimpleObject.FromDict(o1.ToDict()) self.assertEqual(o1.ToDict(), {"a": 2, "b": 5}) class TestClusterObject(unittest.TestCase): """Tests done on a L{objects.Cluster}""" def setUp(self): hvparams = { constants.HT_FAKE: { "foo": "bar", "bar": "foo", "foobar": "barfoo", }, } os_hvp = { "lenny-image": { constants.HT_FAKE: { "foo": "baz", "foobar": "foobar", "blah": "blibb", "blubb": "blah", }, constants.HT_XEN_PVM: { "root_path": "/dev/sda5", "foo": "foobar", }, }, "ubuntu-hardy": { }, } ndparams = { constants.ND_OOB_PROGRAM: "/bin/cluster-oob", constants.ND_SPINDLE_COUNT: 1, constants.ND_EXCLUSIVE_STORAGE: False, } self.fake_cl = objects.Cluster(hvparams=hvparams, os_hvp=os_hvp, ndparams=ndparams) self.fake_cl.UpgradeConfig() def testGetHVDefaults(self): cl = self.fake_cl self.assertEqual(cl.GetHVDefaults(constants.HT_FAKE), cl.hvparams[constants.HT_FAKE]) self.assertEqual(cl.GetHVDefaults(None), {}) defaults = cl.GetHVDefaults(constants.HT_XEN_PVM, os_name="lenny-image") for param, value in cl.os_hvp["lenny-image"][constants.HT_XEN_PVM].items(): self.assertEqual(value, defaults[param]) def testFillHvFullMerge(self): inst_hvparams = { "blah": "blubb", } fake_dict = constants.HVC_DEFAULTS[constants.HT_FAKE].copy() fake_dict.update({ "foo": "baz", "bar": "foo", "foobar": "foobar", "blah": "blubb", "blubb": "blah", }) fake_inst = objects.Instance(name="foobar", os="lenny-image", hypervisor=constants.HT_FAKE, hvparams=inst_hvparams) self.assertEqual(fake_dict, self.fake_cl.FillHV(fake_inst)) def testFillHvGlobalParams(self): fake_inst = objects.Instance(name="foobar", os="ubuntu-hardy", hypervisor=constants.HT_FAKE, hvparams={}) self.assertEqual(self.fake_cl.hvparams[constants.HT_FAKE], self.fake_cl.FillHV(fake_inst)) def testFillHvInstParams(self): inst_hvparams = { "blah": "blubb", } fake_inst = objects.Instance(name="foobar", os="ubuntu-hardy", hypervisor=constants.HT_XEN_PVM, hvparams=inst_hvparams) filled_conf = self.fake_cl.FillHV(fake_inst) for param, value in constants.HVC_DEFAULTS[constants.HT_XEN_PVM].items(): if param == "blah": value = "blubb" self.assertEqual(value, filled_conf[param]) def testFillHvDefaultParams(self): fake_inst = objects.Instance(name="foobar", os="ubuntu-hardy", hypervisor=constants.HT_XEN_PVM, hvparams={}) self.assertEqual(constants.HVC_DEFAULTS[constants.HT_XEN_PVM], self.fake_cl.FillHV(fake_inst)) def testFillHvPartialParams(self): os = "lenny-image" fake_inst = objects.Instance(name="foobar", os=os, hypervisor=constants.HT_XEN_PVM, hvparams={}) filled_conf = self.fake_cl.FillHV(fake_inst) for param, value in self.fake_cl.os_hvp[os][constants.HT_XEN_PVM].items(): self.assertEqual(value, filled_conf[param]) def testFillNdParamsCluster(self): fake_node = objects.Node(name="test", ndparams={}, group="testgroup") fake_group = objects.NodeGroup(name="testgroup", ndparams={}) self.assertEqual(self.fake_cl.ndparams, self.fake_cl.FillND(fake_node, fake_group)) def testFillNdParamsNodeGroup(self): fake_node = objects.Node(name="test", ndparams={}, group="testgroup") group_ndparams = { constants.ND_OOB_PROGRAM: "/bin/group-oob", constants.ND_SPINDLE_COUNT: 10, constants.ND_EXCLUSIVE_STORAGE: True, constants.ND_OVS: True, constants.ND_OVS_LINK: "eth2", constants.ND_OVS_NAME: "openvswitch", constants.ND_SSH_PORT: 122, constants.ND_CPU_SPEED: 1.1, } fake_group = objects.NodeGroup(name="testgroup", ndparams=group_ndparams) self.assertEqual(group_ndparams, self.fake_cl.FillND(fake_node, fake_group)) def testFillNdParamsNode(self): node_ndparams = { constants.ND_OOB_PROGRAM: "/bin/node-oob", constants.ND_SPINDLE_COUNT: 2, constants.ND_EXCLUSIVE_STORAGE: True, constants.ND_OVS: True, constants.ND_OVS_LINK: "eth2", constants.ND_OVS_NAME: "openvswitch", constants.ND_SSH_PORT: 222, constants.ND_CPU_SPEED: 1.1, } fake_node = objects.Node(name="test", ndparams=node_ndparams, group="testgroup") fake_group = objects.NodeGroup(name="testgroup", ndparams={}) self.assertEqual(node_ndparams, self.fake_cl.FillND(fake_node, fake_group)) def testFillNdParamsAll(self): node_ndparams = { constants.ND_OOB_PROGRAM: "/bin/node-oob", constants.ND_SPINDLE_COUNT: 5, constants.ND_EXCLUSIVE_STORAGE: True, constants.ND_OVS: True, constants.ND_OVS_LINK: "eth2", constants.ND_OVS_NAME: "openvswitch", constants.ND_SSH_PORT: 322, constants.ND_CPU_SPEED: 1.1, } fake_node = objects.Node(name="test", ndparams=node_ndparams, group="testgroup") group_ndparams = { constants.ND_OOB_PROGRAM: "/bin/group-oob", constants.ND_SPINDLE_COUNT: 4, constants.ND_SSH_PORT: 422, } fake_group = objects.NodeGroup(name="testgroup", ndparams=group_ndparams) self.assertEqual(node_ndparams, self.fake_cl.FillND(fake_node, fake_group)) def testPrimaryHypervisor(self): assert self.fake_cl.enabled_hypervisors is None self.fake_cl.enabled_hypervisors = [constants.HT_XEN_HVM] self.assertEqual(self.fake_cl.primary_hypervisor, constants.HT_XEN_HVM) self.fake_cl.enabled_hypervisors = [constants.HT_XEN_PVM, constants.HT_KVM] self.assertEqual(self.fake_cl.primary_hypervisor, constants.HT_XEN_PVM) self.fake_cl.enabled_hypervisors = sorted(constants.HYPER_TYPES) self.assertEqual(self.fake_cl.primary_hypervisor, constants.HT_CHROOT) def testUpgradeConfig(self): # FIXME: This test is incomplete cluster = objects.Cluster() cluster.UpgradeConfig() cluster = objects.Cluster(ipolicy={"unknown_key": None}) self.assertRaises(errors.ConfigurationError, cluster.UpgradeConfig) def testUpgradeEnabledDiskTemplates(self): cfg = objects.ConfigData() cfg.cluster = objects.Cluster() cfg.cluster.volume_group_name = "myvg" instance1 = objects.Instance() instance1.disk_template = constants.DT_DISKLESS instance2 = objects.Instance() instance2.disk_template = constants.DT_RBD cfg.instances = { "myinstance1": instance1, "myinstance2": instance2 } disk2 = objects.Disk(dev_type=constants.DT_RBD) cfg.disks = { "pinkbunnydisk": disk2 } nodegroup = objects.NodeGroup() nodegroup.ipolicy = {} nodegroup.ipolicy[constants.IPOLICY_DTS] = [instance1.disk_template, \ constants.DT_BLOCK] cfg.cluster.ipolicy = {} cfg.cluster.ipolicy[constants.IPOLICY_DTS] = \ [constants.DT_EXT, constants.DT_DISKLESS] cfg.nodegroups = { "mynodegroup": nodegroup } cfg._UpgradeEnabledDiskTemplates() expected_disk_templates = [constants.DT_DRBD8, constants.DT_PLAIN, instance1.disk_template, instance2.disk_template] self.assertEqual(set(expected_disk_templates), set(cfg.cluster.enabled_disk_templates)) self.assertEqual(set([instance1.disk_template]), set(cfg.cluster.ipolicy[constants.IPOLICY_DTS])) class TestClusterObjectTcpUdpPortPool(unittest.TestCase): def testNewCluster(self): self.assertTrue(objects.Cluster().tcpudp_port_pool is None) def testSerializingEmpty(self): self.assertEqual(objects.Cluster().ToDict(), { "tcpudp_port_pool": [], }) def testSerializing(self): cluster = objects.Cluster.FromDict({}) self.assertEqual(cluster.tcpudp_port_pool, set()) cluster.tcpudp_port_pool.add(3546) cluster.tcpudp_port_pool.add(62511) data = cluster.ToDict() self.assertEqual(list(data), ["tcpudp_port_pool"]) self.assertEqual(sorted(data["tcpudp_port_pool"]), sorted([3546, 62511])) def testDeserializingEmpty(self): cluster = objects.Cluster.FromDict({}) self.assertEqual(cluster.tcpudp_port_pool, set()) def testDeserialize(self): cluster = objects.Cluster.FromDict({ "tcpudp_port_pool": [26214, 10039, 267], }) self.assertEqual(cluster.tcpudp_port_pool, set([26214, 10039, 267])) class TestOS(unittest.TestCase): ALL_DATA = [ "debootstrap", "debootstrap+default", "debootstrap++default", ] def testSplitNameVariant(self): for name in self.ALL_DATA: self.assertEqual(len(objects.OS.SplitNameVariant(name)), 2) def testVariant(self): self.assertEqual(objects.OS.GetVariant("debootstrap"), "") self.assertEqual(objects.OS.GetVariant("debootstrap+default"), "default") class TestInstance(unittest.TestCase): def testFindDisk(self): inst = objects.Instance(name="fakeinstdrbd.example.com", primary_node="node20.example.com", disks=[ objects.Disk(dev_type=constants.DT_DRBD8, size=786432, logical_id=("node20.example.com", "node15.example.com", 12300, 0, 0, "secret"), children=[ objects.Disk(dev_type=constants.DT_PLAIN, size=786432, logical_id=("myxenvg", "disk0")), objects.Disk(dev_type=constants.DT_PLAIN, size=128, logical_id=("myxenvg", "meta0")) ], iv_name="disk/0") ]) self.assertEqual(inst.FindDisk(0), inst.disks[0]) self.assertRaises(errors.OpPrereqError, inst.FindDisk, "hello") self.assertRaises(errors.OpPrereqError, inst.FindDisk, 100) self.assertRaises(errors.OpPrereqError, inst.FindDisk, 1) class TestNode(unittest.TestCase): def testEmpty(self): self.assertEqual(objects.Node().ToDict(), {}) self.assertTrue(isinstance(objects.Node.FromDict({}), objects.Node)) def testHvState(self): node = objects.Node(name="node18157.example.com", hv_state={ constants.HT_XEN_HVM: objects.NodeHvState(cpu_total=64), constants.HT_KVM: objects.NodeHvState(cpu_node=1), }) node2 = objects.Node.FromDict(node.ToDict()) # Make sure nothing can reference it anymore del node self.assertEqual(node2.name, "node18157.example.com") self.assertEqual(frozenset(node2.hv_state), frozenset([ constants.HT_XEN_HVM, constants.HT_KVM, ])) self.assertEqual(node2.hv_state[constants.HT_KVM].cpu_node, 1) self.assertEqual(node2.hv_state[constants.HT_XEN_HVM].cpu_total, 64) def testDiskState(self): node = objects.Node(name="node32087.example.com", disk_state={ constants.DT_PLAIN: { "lv32352": objects.NodeDiskState(total=128), "lv2082": objects.NodeDiskState(total=512), }, }) node2 = objects.Node.FromDict(node.ToDict()) # Make sure nothing can reference it anymore del node self.assertEqual(node2.name, "node32087.example.com") self.assertEqual(frozenset(node2.disk_state), frozenset([ constants.DT_PLAIN, ])) self.assertEqual(frozenset(node2.disk_state[constants.DT_PLAIN]), frozenset(["lv32352", "lv2082"])) self.assertEqual(node2.disk_state[constants.DT_PLAIN]["lv2082"].total, 512) self.assertEqual(node2.disk_state[constants.DT_PLAIN]["lv32352"].total, 128) def testFilterEsNdp(self): node1 = objects.Node(name="node11673.example.com", ndparams={ constants.ND_EXCLUSIVE_STORAGE: True, }) node2 = objects.Node(name="node11674.example.com", ndparams={ constants.ND_SPINDLE_COUNT: 3, constants.ND_EXCLUSIVE_STORAGE: False, }) self.assertTrue(constants.ND_EXCLUSIVE_STORAGE in node1.ndparams) node1.UpgradeConfig() self.assertFalse(constants.ND_EXCLUSIVE_STORAGE in node1.ndparams) self.assertTrue(constants.ND_EXCLUSIVE_STORAGE in node2.ndparams) self.assertTrue(constants.ND_SPINDLE_COUNT in node2.ndparams) node2.UpgradeConfig() self.assertFalse(constants.ND_EXCLUSIVE_STORAGE in node2.ndparams) self.assertTrue(constants.ND_SPINDLE_COUNT in node2.ndparams) class TestInstancePolicy(unittest.TestCase): def setUp(self): # Policies are big, and we want to see the difference in case of an error self.maxDiff = None def _AssertIPolicyIsFull(self, policy): self.assertEqual(frozenset(policy), constants.IPOLICY_ALL_KEYS) self.assertTrue(len(policy[constants.ISPECS_MINMAX]) > 0) for minmax in policy[constants.ISPECS_MINMAX]: self.assertEqual(frozenset(minmax), constants.ISPECS_MINMAX_KEYS) for key in constants.ISPECS_MINMAX_KEYS: self.assertEqual(frozenset(minmax[key]), constants.ISPECS_PARAMETERS) self.assertEqual(frozenset(policy[constants.ISPECS_STD]), constants.ISPECS_PARAMETERS) def testDefaultIPolicy(self): objects.InstancePolicy.CheckParameterSyntax(constants.IPOLICY_DEFAULTS, True) self._AssertIPolicyIsFull(constants.IPOLICY_DEFAULTS) def _AssertPolicyIsBad(self, ipolicy, do_check_std=None): if do_check_std is None: check_std_vals = [False, True] else: check_std_vals = [do_check_std] for check_std in check_std_vals: self.assertRaises(errors.ConfigurationError, objects.InstancePolicy.CheckISpecSyntax, ipolicy, check_std) def testCheckISpecSyntax(self): default_stdspec = constants.IPOLICY_DEFAULTS[constants.ISPECS_STD] incomplete_ipolicies = [ { constants.ISPECS_MINMAX: [], constants.ISPECS_STD: default_stdspec, }, { constants.ISPECS_MINMAX: [{}], constants.ISPECS_STD: default_stdspec, }, { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: NotImplemented, }], constants.ISPECS_STD: default_stdspec, }, { constants.ISPECS_MINMAX: [{ constants.ISPECS_MAX: NotImplemented, }], constants.ISPECS_STD: default_stdspec, }, { constants.ISPECS_MINMAX: [{ constants.ISPECS_MIN: NotImplemented, constants.ISPECS_MAX: NotImplemented, }], }, ] for ipol in incomplete_ipolicies: self.assertRaises(errors.ConfigurationError, objects.InstancePolicy.CheckISpecSyntax, ipol, True) oldminmax = ipol[constants.ISPECS_MINMAX] if oldminmax: # Prepending valid specs shouldn't change the error ipol[constants.ISPECS_MINMAX] = ([constants.ISPECS_MINMAX_DEFAULTS] + oldminmax) self.assertRaises(errors.ConfigurationError, objects.InstancePolicy.CheckISpecSyntax, ipol, True) good_ipolicy = { constants.ISPECS_MINMAX: [ { constants.ISPECS_MIN: { constants.ISPEC_MEM_SIZE: 64, constants.ISPEC_CPU_COUNT: 1, constants.ISPEC_DISK_COUNT: 2, constants.ISPEC_DISK_SIZE: 64, constants.ISPEC_NIC_COUNT: 1, constants.ISPEC_SPINDLE_USE: 1, }, constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 16384, constants.ISPEC_CPU_COUNT: 5, constants.ISPEC_DISK_COUNT: 12, constants.ISPEC_DISK_SIZE: 1024, constants.ISPEC_NIC_COUNT: 9, constants.ISPEC_SPINDLE_USE: 18, }, }, { constants.ISPECS_MIN: { constants.ISPEC_MEM_SIZE: 32768, constants.ISPEC_CPU_COUNT: 8, constants.ISPEC_DISK_COUNT: 1, constants.ISPEC_DISK_SIZE: 1024, constants.ISPEC_NIC_COUNT: 1, constants.ISPEC_SPINDLE_USE: 1, }, constants.ISPECS_MAX: { constants.ISPEC_MEM_SIZE: 65536, constants.ISPEC_CPU_COUNT: 10, constants.ISPEC_DISK_COUNT: 5, constants.ISPEC_DISK_SIZE: 1024 * 1024, constants.ISPEC_NIC_COUNT: 3, constants.ISPEC_SPINDLE_USE: 12, }, }, ], } good_ipolicy[constants.ISPECS_STD] = copy.deepcopy( good_ipolicy[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX]) # Check that it's really good before making it bad objects.InstancePolicy.CheckISpecSyntax(good_ipolicy, True) bad_ipolicy = copy.deepcopy(good_ipolicy) for minmax in bad_ipolicy[constants.ISPECS_MINMAX]: for (key, spec) in minmax.items(): for param in set(spec.keys()): oldv = spec[param] del spec[param] self._AssertPolicyIsBad(bad_ipolicy) if key == constants.ISPECS_MIN: spec[param] = minmax[constants.ISPECS_MAX][param] + 1 self._AssertPolicyIsBad(bad_ipolicy) spec[param] = oldv assert bad_ipolicy == good_ipolicy stdspec = bad_ipolicy[constants.ISPECS_STD] for param in set(stdspec.keys()): oldv = stdspec[param] del stdspec[param] self._AssertPolicyIsBad(bad_ipolicy, True) # Note that std spec is the same as a max spec stdspec[param] = oldv + 1 self._AssertPolicyIsBad(bad_ipolicy, True) stdspec[param] = oldv assert bad_ipolicy == good_ipolicy for minmax in good_ipolicy[constants.ISPECS_MINMAX]: for spec in minmax.values(): good_ipolicy[constants.ISPECS_STD] = spec objects.InstancePolicy.CheckISpecSyntax(good_ipolicy, True) def testCheckISpecParamSyntax(self): par = "my_parameter" for check_std in [True, False]: # Min and max only good_values = [(11, 11), (11, 40), (0, 0)] for (mn, mx) in good_values: minmax = dict((k, {}) for k in constants.ISPECS_MINMAX_KEYS) minmax[constants.ISPECS_MIN][par] = mn minmax[constants.ISPECS_MAX][par] = mx objects.InstancePolicy._CheckISpecParamSyntax(minmax, {}, par, check_std) minmax = dict((k, {}) for k in constants.ISPECS_MINMAX_KEYS) minmax[constants.ISPECS_MIN][par] = 11 minmax[constants.ISPECS_MAX][par] = 5 self.assertRaises(errors.ConfigurationError, objects.InstancePolicy._CheckISpecParamSyntax, minmax, {}, par, check_std) # Min, std, max good_values = [ (11, 11, 11), (11, 11, 40), (11, 40, 40), ] for (mn, st, mx) in good_values: minmax = { constants.ISPECS_MIN: {par: mn}, constants.ISPECS_MAX: {par: mx}, } stdspec = {par: st} objects.InstancePolicy._CheckISpecParamSyntax(minmax, stdspec, par, True) bad_values = [ (11, 11, 5, True), (40, 11, 11, True), (11, 80, 40, False), (11, 5, 40, False,), (11, 5, 5, True), (40, 40, 11, True), ] for (mn, st, mx, excp) in bad_values: minmax = { constants.ISPECS_MIN: {par: mn}, constants.ISPECS_MAX: {par: mx}, } stdspec = {par: st} if excp: self.assertRaises(errors.ConfigurationError, objects.InstancePolicy._CheckISpecParamSyntax, minmax, stdspec, par, True) else: ret = objects.InstancePolicy._CheckISpecParamSyntax(minmax, stdspec, par, True) self.assertFalse(ret) def testCheckDiskTemplates(self): invalid = "this_is_not_a_good_template" for dt in constants.DISK_TEMPLATES: objects.InstancePolicy.CheckDiskTemplates([dt]) objects.InstancePolicy.CheckDiskTemplates(list(constants.DISK_TEMPLATES)) bad_examples = [ [invalid], [constants.DT_DRBD8, invalid], list(constants.DISK_TEMPLATES) + [invalid], [], None, ] for dtl in bad_examples: self.assertRaises(errors.ConfigurationError, objects.InstancePolicy.CheckDiskTemplates, dtl) def testCheckParameterSyntax(self): invalid = "this_key_shouldnt_be_here" for check_std in [True, False]: objects.InstancePolicy.CheckParameterSyntax({}, check_std) policy = {invalid: None} self.assertRaises(errors.ConfigurationError, objects.InstancePolicy.CheckParameterSyntax, policy, check_std) for par in constants.IPOLICY_PARAMETERS: for val in ("blah", None, {}, [42]): policy = {par: val} self.assertRaises(errors.ConfigurationError, objects.InstancePolicy.CheckParameterSyntax, policy, check_std) def testFillIPolicyEmpty(self): policy = objects.FillIPolicy(constants.IPOLICY_DEFAULTS, {}) objects.InstancePolicy.CheckParameterSyntax(policy, True) self.assertEqual(policy, constants.IPOLICY_DEFAULTS) def _AssertISpecsMerged(self, default_spec, diff_spec, merged_spec): for (param, value) in merged_spec.items(): if param in diff_spec: self.assertEqual(value, diff_spec[param]) else: self.assertEqual(value, default_spec[param]) def _AssertIPolicyMerged(self, default_pol, diff_pol, merged_pol): for (key, value) in merged_pol.items(): if key in diff_pol: if key == constants.ISPECS_STD: self._AssertISpecsMerged(default_pol[key], diff_pol[key], value) else: self.assertEqual(value, diff_pol[key]) else: self.assertEqual(value, default_pol[key]) def testFillIPolicy(self): partial_policies = [ {constants.IPOLICY_VCPU_RATIO: 3.14}, {constants.IPOLICY_SPINDLE_RATIO: 2.72}, {constants.IPOLICY_DTS: [constants.DT_FILE]}, {constants.ISPECS_STD: {constants.ISPEC_DISK_COUNT: 3}}, {constants.ISPECS_MINMAX: [constants.ISPECS_MINMAX_DEFAULTS, constants.ISPECS_MINMAX_DEFAULTS]} ] for diff_pol in partial_policies: policy = objects.FillIPolicy(constants.IPOLICY_DEFAULTS, diff_pol) objects.InstancePolicy.CheckParameterSyntax(policy, True) self._AssertIPolicyIsFull(policy) self._AssertIPolicyMerged(constants.IPOLICY_DEFAULTS, diff_pol, policy) def testFillIPolicyKeepsUnknown(self): INVALID_KEY = "invalid_ipolicy_key" diff_pol = { INVALID_KEY: None, } policy = objects.FillIPolicy(constants.IPOLICY_DEFAULTS, diff_pol) self.assertTrue(INVALID_KEY in policy) class TestDisk(unittest.TestCase): def addChild(self, disk): """Adds a child of the same device type as the parent.""" disk.children = [] child = objects.Disk() child.dev_type = disk.dev_type disk.children.append(child) def testUpgradeConfigDevTypeLegacy(self): for old, new in [("drbd8", constants.DT_DRBD8), ("lvm", constants.DT_PLAIN)]: disk = objects.Disk() disk.dev_type = old self.addChild(disk) disk.UpgradeConfig() self.assertEqual(new, disk.dev_type) self.assertEqual(new, disk.children[0].dev_type) def testUpgradeConfigDevTypeLegacyUnchanged(self): dev_types = [constants.DT_FILE, constants.DT_SHARED_FILE, constants.DT_BLOCK, constants.DT_EXT, constants.DT_RBD, constants.DT_GLUSTER] for dev_type in dev_types: disk = objects.Disk() disk.dev_type = dev_type self.addChild(disk) disk.UpgradeConfig() self.assertEqual(dev_type, disk.dev_type) self.assertEqual(dev_type, disk.children[0].dev_type) class TestSimpleFillOS(unittest.TestCase): # We have to make sure that: # * From within the configuration, variants override defaults # * Temporary values override configuration # * No overlap between public, private and secret dicts is allowed # # As a result, here are the actors in this test: # # A: temporary public # B: temporary private # C: temporary secret # X: temporary public private secret # D: configuration public variant # E: configuration public base # F: configuration private variant # G: configuration private base # # Every time a param is assigned "ERROR", we expect FillOSParams to override # it. If it doesn't, it's an error. # # Every time a param is assigned itself as a value, it's the value we expect # FillOSParams to give us back. def setUp(self): self.fake_cl = objects.Cluster() self.fake_cl.UpgradeConfig() self.fake_cl.osparams = {"os": {"A": "ERROR", "D": "ERROR", "E": "E"}, "os+a": {"D": "D"}} self.fake_cl.osparams_private_cluster = {"os": {"B": "ERROR", "F": "ERROR", "G": "G"}, "os+a": {"F": "F"}} def testConflictPublicPrivate(self): "Make sure we disallow attempts to override params based on visibility." public_dict = {"A": "A", "X": "X"} private_dict = {"B": "B", "X": "X"} secret_dict = {"C": "C"} dicts_pp = (public_dict, private_dict) dicts_pps = (public_dict, private_dict, secret_dict) # Without secret parameters self.assertRaises(errors.OpPrereqError, lambda: self.fake_cl.SimpleFillOS("os+a", *dicts_pp)) # but also with those. self.assertRaises(errors.OpPrereqError, lambda: self.fake_cl.SimpleFillOS("os+a", *dicts_pps)) def testConflictPublicSecret(self): "Make sure we disallow attempts to override params based on visibility." public_dict = {"A": "A", "X": "X"} private_dict = {"B": "B"} secret_dict = {"C": "C", "X": "X"} dicts_pps = (public_dict, private_dict, secret_dict) self.assertRaises(errors.OpPrereqError, lambda: self.fake_cl.SimpleFillOS("os+a", *dicts_pps)) def testConflictPrivateSecret(self): "Make sure we disallow attempts to override params based on visibility." public_dict = {"A": "A"} private_dict = {"B": "B", "X": "X"} secret_dict = {"C": "C", "X": "X"} dicts_pps = (public_dict, private_dict, secret_dict) self.assertRaises(errors.OpPrereqError, lambda: self.fake_cl.SimpleFillOS("os+a", *dicts_pps)) def testValidValues(self): "Make sure we handle all overriding we do allow correctly." public_dict = {"A": "A"} private_dict = {"B": "B"} secret_dict = {"C": "C"} dicts_p = (public_dict,) dicts_pp = (public_dict, private_dict) dicts_pps = (public_dict, private_dict, secret_dict) expected_keys_p = ("A", "D", "E") # nothing private, secret expected_keys_pp = ("A", "B", "D", "E", "F", "G") # nothing secret expected_keys_pps = ("A", "B", "C", "D", "E", "F", "G") # all of them for (dicts, expected_keys) in [(dicts_p, expected_keys_p), (dicts_pp, expected_keys_pp), (dicts_pps, expected_keys_pps)]: result = self.fake_cl.SimpleFillOS("os+a", *dicts) # Values for key in result: if not result[key] == key: self.fail("Invalid public-private fill with input:\n%s\n%s" % (pprint.pformat(dicts), result)) # Completeness if set(result) != set(expected_keys): self.fail("Problem with key %s from merge result of:\n%s\n%s" % (set(expected_keys) ^ set(result), # symmetric difference pprint.pformat(dicts), result)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.opcodes_unittest.py000075500000000000000000000352551476477700300237070ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.backend""" import os import sys import unittest from ganeti import utils from ganeti import opcodes from ganeti import opcodes_base from ganeti import ht from ganeti import constants from ganeti import errors from ganeti import compat import testutils class TestOpcodes(unittest.TestCase): def test(self): self.assertRaises(ValueError, opcodes.OpCode.LoadOpCode, None) self.assertRaises(ValueError, opcodes.OpCode.LoadOpCode, "") self.assertRaises(ValueError, opcodes.OpCode.LoadOpCode, {}) self.assertRaises(ValueError, opcodes.OpCode.LoadOpCode, {"OP_ID": ""}) for cls in opcodes.OP_MAPPING.values(): self.assertTrue(cls.OP_ID.startswith("OP_")) self.assertTrue(len(cls.OP_ID) > 3) self.assertEqual(cls.OP_ID, cls.OP_ID.upper()) self.assertEqual(cls.OP_ID, opcodes_base._NameToId(cls.__name__)) self.assertFalse( compat.any(cls.OP_ID.startswith(prefix) for prefix in opcodes_base.SUMMARY_PREFIX.keys())) self.assertTrue(callable(cls.OP_RESULT), msg=("%s should have a result check" % cls.OP_ID)) self.assertRaises(TypeError, cls, unsupported_parameter="some value") args = [ # No variables {}, # Variables supported by all opcodes {"dry_run": False, "debug_level": 0, }, # All variables dict([(name, []) for name in cls.GetAllSlots()]) ] for i in args: op = cls(**i) self.assertEqual(op.OP_ID, cls.OP_ID) self._checkSummary(op) # Try a restore state = op.__getstate__() self.assertTrue(isinstance(state, dict)) restored = opcodes.OpCode.LoadOpCode(state) self.assertTrue(isinstance(restored, cls)) self._checkSummary(restored) for name in ["x_y_z", "hello_world"]: assert name not in cls.GetAllSlots() for value in [None, True, False, [], "Hello World"]: self.assertRaises(AttributeError, setattr, op, name, value) def _checkSummary(self, op): summary = op.Summary() if hasattr(op, "OP_DSC_FIELD"): self.assertTrue(("OP_%s" % summary).startswith("%s(" % op.OP_ID)) self.assertTrue(summary.endswith(")")) else: self.assertEqual("OP_%s" % summary, op.OP_ID) def testSummary(self): class OpTest(opcodes.OpCode): OP_DSC_FIELD = "data" OP_PARAMS = [ ("data", ht.NoDefault, ht.TString, None), ] self.assertEqual(OpTest(data="").Summary(), "TEST()") self.assertEqual(OpTest(data="Hello World").Summary(), "TEST(Hello World)") self.assertEqual(OpTest(data="node1.example.com").Summary(), "TEST(node1.example.com)") def testSummaryFormatter(self): class OpTest(opcodes.OpCode): OP_DSC_FIELD = "data" OP_DSC_FORMATTER = lambda _, v: "a" OP_PARAMS = [ ("data", ht.NoDefault, ht.TString, None), ] self.assertEqual(OpTest(data="").Summary(), "TEST(a)") self.assertEqual(OpTest(data="b").Summary(), "TEST(a)") def testTinySummary(self): self.assertFalse( utils.FindDuplicates(opcodes_base.SUMMARY_PREFIX.values())) self.assertTrue(compat.all(prefix.endswith("_") and supplement.endswith("_") for (prefix, supplement) in opcodes_base.SUMMARY_PREFIX.items())) self.assertEqual(opcodes.OpClusterPostInit().TinySummary(), "C_POST_INIT") self.assertEqual(opcodes.OpNodeRemove().TinySummary(), "N_REMOVE") self.assertEqual(opcodes.OpInstanceMigrate().TinySummary(), "I_MIGRATE") self.assertEqual(opcodes.OpTestJqueue().TinySummary(), "TEST_JQUEUE") def testListSummary(self): class OpTest(opcodes.OpCode): OP_DSC_FIELD = "data" OP_PARAMS = [ ("data", ht.NoDefault, ht.TList, None), ] self.assertEqual(OpTest(data=["a", "b", "c"]).Summary(), "TEST(a,b,c)") self.assertEqual(OpTest(data=["a", None, "c"]).Summary(), "TEST(a,None,c)") self.assertEqual(OpTest(data=[1, 2, 3, 4]).Summary(), "TEST(1,2,3,4)") def testOpId(self): self.assertFalse(utils.FindDuplicates(cls.OP_ID for cls in opcodes._GetOpList())) self.assertEqual(len(opcodes._GetOpList()), len(opcodes.OP_MAPPING)) def testParams(self): supported_by_all = set(["debug_level", "dry_run", "priority"]) self.assertTrue(opcodes_base.BaseOpCode not in opcodes.OP_MAPPING.values()) self.assertTrue(opcodes.OpCode not in opcodes.OP_MAPPING.values()) for cls in list(opcodes.OP_MAPPING.values()) + [opcodes.OpCode]: all_slots = cls.GetAllSlots() self.assertEqual(len(set(all_slots) & supported_by_all), 3, msg=("Opcode %s doesn't support all base" " parameters (%r)" % (cls.OP_ID, supported_by_all))) # All opcodes must have OP_PARAMS self.assertTrue(hasattr(cls, "OP_PARAMS"), msg="%s doesn't have OP_PARAMS" % cls.OP_ID) param_names = [name for (name, _, _, _) in cls.GetAllParams()] self.assertEqual(all_slots, param_names) # Without inheritance self.assertEqual(cls.__slots__, [name for (name, _, _, _) in cls.OP_PARAMS]) # This won't work if parameters are converted to a dictionary duplicates = utils.FindDuplicates(param_names) self.assertFalse(duplicates, msg=("Found duplicate parameters %r in %s" % (duplicates, cls.OP_ID))) # Check parameter definitions for attr_name, aval, test, doc in cls.GetAllParams(): self.assertTrue(attr_name) self.assertTrue(callable(test), msg=("Invalid type check for %s.%s" % (cls.OP_ID, attr_name))) self.assertTrue(doc is None or isinstance(doc, str)) if callable(aval): default_value = aval() self.assertFalse(callable(default_value), msg=("Default value of %s.%s returned by function" " is callable" % (cls.OP_ID, attr_name))) else: default_value = aval if aval is not ht.NoDefault and aval is not None: self.assertTrue(test(default_value), msg=("Default value of %s.%s does not verify" % (cls.OP_ID, attr_name))) # If any parameter has documentation, all others need to have it as well has_doc = [doc is not None for (_, _, _, doc) in cls.OP_PARAMS] self.assertTrue(not compat.any(has_doc) or compat.all(has_doc), msg="%s does not document all parameters" % cls) def testValidateNoModification(self): class OpTest(opcodes.OpCode): OP_PARAMS = [ ("nodef", None, ht.TString, None), ("wdef", "default", ht.TMaybeString, None), ("number", 0, ht.TInt, None), ("notype", None, ht.TAny, None), ] # Missing required parameter "nodef" op = OpTest() before = op.__getstate__() self.assertRaises(errors.OpPrereqError, op.Validate, False) self.assertTrue(op.nodef is None) self.assertEqual(op.wdef, "default") self.assertEqual(op.number, 0) self.assertTrue(op.notype is None) self.assertEqual(op.__getstate__(), before, msg="Opcode was modified") # Required parameter "nodef" is provided op = OpTest(nodef="foo") before = op.__getstate__() op.Validate(False) self.assertEqual(op.__getstate__(), before, msg="Opcode was modified") self.assertEqual(op.nodef, "foo") self.assertEqual(op.wdef, "default") self.assertEqual(op.number, 0) self.assertTrue(op.notype is None) # Missing required parameter "nodef" op = OpTest(wdef="hello", number=999) before = op.__getstate__() self.assertRaises(errors.OpPrereqError, op.Validate, False) self.assertTrue(op.nodef is None) self.assertTrue(op.notype is None) self.assertEqual(op.__getstate__(), before, msg="Opcode was modified") # Wrong type for "nodef" op = OpTest(nodef=987) before = op.__getstate__() self.assertRaises(errors.OpPrereqError, op.Validate, False) self.assertEqual(op.nodef, 987) self.assertTrue(op.notype is None) self.assertEqual(op.__getstate__(), before, msg="Opcode was modified") # Testing different types for "notype" op = OpTest(nodef="foo", notype=[1, 2, 3]) before = op.__getstate__() op.Validate(False) self.assertEqual(op.nodef, "foo") self.assertEqual(op.notype, [1, 2, 3]) self.assertEqual(op.__getstate__(), before, msg="Opcode was modified") op = OpTest(nodef="foo", notype="Hello World") before = op.__getstate__() op.Validate(False) self.assertEqual(op.nodef, "foo") self.assertEqual(op.notype, "Hello World") self.assertEqual(op.__getstate__(), before, msg="Opcode was modified") def testValidateSetDefaults(self): class OpTest(opcodes.OpCode): OP_PARAMS = [ ("value1", "default", ht.TMaybeString, None), ("value2", "result", ht.TMaybeString, None), ] op = OpTest() op.Validate(True) self.assertEqual(op.value1, "default") self.assertEqual(op.value2, "result") self.assertTrue(op.dry_run is None) self.assertTrue(op.debug_level is None) self.assertEqual(op.priority, constants.OP_PRIO_DEFAULT) op = OpTest(value1="hello", value2="world", debug_level=123) op.Validate(True) self.assertEqual(op.value1, "hello") self.assertEqual(op.value2, "world") self.assertEqual(op.debug_level, 123) def testOpInstanceMultiAlloc(self): inst = dict([(name, []) for name in opcodes.OpInstanceCreate.GetAllSlots()]) inst_op = opcodes.OpInstanceCreate(**inst) inst_state = inst_op.__getstate__() multialloc = opcodes.OpInstanceMultiAlloc(instances=[inst_op]) state = multialloc.__getstate__() self.assertEqual(state["instances"], [inst_state]) loaded_multialloc = opcodes.OpCode.LoadOpCode(state) (loaded_inst,) = loaded_multialloc.instances self.assertNotEqual(loaded_inst, inst_op) self.assertEqual(loaded_inst.__getstate__(), inst_state) class TestOpcodeDepends(unittest.TestCase): def test(self): check_relative = opcodes_base.BuildJobDepCheck(True) check_norelative = opcodes_base.TNoRelativeJobDependencies for fn in [check_relative, check_norelative]: self.assertTrue(fn(None)) self.assertTrue(fn([])) self.assertTrue(fn([(1, [])])) self.assertTrue(fn([(719833, [])])) self.assertTrue(fn([("24879", [])])) self.assertTrue(fn([(2028, [constants.JOB_STATUS_ERROR])])) self.assertTrue(fn([ (2028, [constants.JOB_STATUS_ERROR]), (18750, []), (5063, [constants.JOB_STATUS_SUCCESS, constants.JOB_STATUS_ERROR]), ])) self.assertFalse(fn(1)) self.assertFalse(fn([(9, )])) self.assertFalse(fn([(15194, constants.JOB_STATUS_ERROR)])) for i in [ [(-1, [])], [(-27740, [constants.JOB_STATUS_CANCELED, constants.JOB_STATUS_ERROR]), (-1, [constants.JOB_STATUS_ERROR]), (9921, [])], ]: self.assertTrue(check_relative(i)) self.assertFalse(check_norelative(i)) class TestResultChecks(unittest.TestCase): def testJobIdList(self): for i in [[], [(False, "error")], [(False, "")], [(True, 123), (True, "999")]]: self.assertTrue(ht.TJobIdList(i)) for i in ["", [("x", 1)], [[], []], [[False, "", None], [True, 123]]]: self.assertFalse(ht.TJobIdList(i)) def testJobIdListOnly(self): self.assertTrue(ht.TJobIdListOnly({ constants.JOB_IDS_KEY: [], })) self.assertTrue(ht.TJobIdListOnly({ constants.JOB_IDS_KEY: [(True, "9282")], })) self.assertFalse(ht.TJobIdListOnly({ "x": None, })) self.assertFalse(ht.TJobIdListOnly({ constants.JOB_IDS_KEY: [], "x": None, })) self.assertFalse(ht.TJobIdListOnly({ constants.JOB_IDS_KEY: [("foo", "bar")], })) self.assertFalse(ht.TJobIdListOnly({ constants.JOB_IDS_KEY: [("one", "two", "three")], })) class TestOpInstanceSetParams(unittest.TestCase): def _GenericTests(self, fn): self.assertTrue(fn([])) self.assertTrue(fn([(constants.DDM_ADD, {})])) self.assertTrue(fn([(constants.DDM_ATTACH, {})])) self.assertTrue(fn([(constants.DDM_REMOVE, {})])) self.assertTrue(fn([(constants.DDM_DETACH, {})])) for i in [0, 1, 2, 3, 9, 10, 1024]: self.assertTrue(fn([(i, {})])) self.assertFalse(fn(None)) self.assertFalse(fn({})) self.assertFalse(fn("")) self.assertFalse(fn(0)) self.assertFalse(fn([(-100, {})])) self.assertFalse(fn([(constants.DDM_ADD, 2, 3)])) self.assertFalse(fn([[constants.DDM_ADD]])) def testNicModifications(self): fn = ht.TSetParamsMods(ht.TINicParams) self._GenericTests(fn) for param in constants.INIC_PARAMS: self.assertTrue(fn([[constants.DDM_ADD, {param: None}]])) self.assertTrue(fn([[constants.DDM_ADD, {param: param}]])) def testDiskModifications(self): fn = ht.TSetParamsMods(ht.TIDiskParams) self._GenericTests(fn) for param in constants.IDISK_PARAMS: self.assertTrue(fn([[constants.DDM_ADD, {param: 0}]])) self.assertTrue(fn([[constants.DDM_ADD, {param: param}]])) self.assertTrue(fn([[constants.DDM_ATTACH, {param: param}]])) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.outils_unittest.py000075500000000000000000000070761476477700300235720ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the outils module""" import unittest from ganeti import outils import testutils class SlotsAutoSlot(outils.AutoSlots): @classmethod def _GetSlots(mcs, attr): return attr["SLOTS"] class AutoSlotted(object, metaclass=SlotsAutoSlot): SLOTS = ["foo", "bar", "baz"] class TestAutoSlot(unittest.TestCase): def test(self): slotted = AutoSlotted() self.assertEqual(slotted.__slots__, AutoSlotted.SLOTS) class TestContainerToDicts(unittest.TestCase): def testUnknownType(self): for value in [None, 19410, "xyz"]: try: outils.ContainerToDicts(value) except TypeError as err: self.assertTrue(str(err).startswith("Unknown container type")) else: self.fail("Exception was not raised") def testEmptyDict(self): value = {} self.assertFalse(type(value) in outils._SEQUENCE_TYPES) self.assertEqual(outils.ContainerToDicts(value), {}) def testEmptySequences(self): for cls in [list, tuple, set, frozenset]: self.assertEqual(outils.ContainerToDicts(cls()), []) class _FakeWithFromDict: def FromDict(self, _): raise NotImplemented class TestContainerFromDicts(unittest.TestCase): def testUnknownType(self): for cls in [str, int, bool]: try: outils.ContainerFromDicts(None, cls, NotImplemented) except TypeError as err: self.assertTrue(str(err).startswith("Unknown container type")) else: self.fail("Exception was not raised") try: outils.ContainerFromDicts(None, cls(), NotImplemented) except TypeError as err: self.assertTrue(str(err).endswith("is not a type")) else: self.fail("Exception was not raised") def testEmptyDict(self): value = {} self.assertFalse(type(value) in outils._SEQUENCE_TYPES) self.assertEqual(outils.ContainerFromDicts(value, dict, NotImplemented), {}) def testEmptySequences(self): for cls in [list, tuple, set, frozenset]: self.assertEqual(outils.ContainerFromDicts([], cls, _FakeWithFromDict), cls()) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.ovf_unittest.py000075500000000000000000000763231476477700300230460ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.ovf. """ import optparse import os import os.path import re import shutil import sys import tempfile import unittest try: import xml.etree.ElementTree as ET except ImportError: import elementtree.ElementTree as ET from ganeti import constants from ganeti import errors from ganeti import ovf from ganeti import utils from ganeti import pathutils import testutils OUTPUT_DIR = tempfile.mkdtemp() GANETI_DISKS = { "disk_count": "1", "disk0_dump": "new_disk.raw", "disk0_size": "0", "disk0_ivname": "disk/0", } GANETI_NETWORKS = { "nic_count": "1", "nic0_mode": "bridged", "nic0_ip": "none", "nic0_mac": "aa:00:00:d8:2c:1e", "nic0_link": "xen-br0", "nic0_network": "auto", } GANETI_HYPERVISOR = { "hypervisor_name": "xen-pvm", "root-path": "/dev/sda", "kernel_args": "ro", } GANETI_OS = {"os_name": "lenny-image"} GANETI_BACKEND = { "vcpus": "1", "memory" : "2048", "auto_balance": "False", } GANETI_NAME = "ganeti-test-xen" GANETI_TEMPLATE = "plain" GANETI_TAGS = None GANETI_VERSION = "0" VIRTUALBOX_DISKS = { "disk_count": "2", "disk0_ivname": "disk/0", "disk0_dump": "new_disk.raw", "disk0_size": "0", "disk1_ivname": "disk/1", "disk1_dump": "second_disk.raw", "disk1_size": "0", } VIRTUALBOX_NETWORKS = { "nic_count": "1", "nic0_mode": "bridged", "nic0_ip": "none", "nic0_link": "auto", "nic0_mac": "auto", "nic0_network": "auto", } VIRTUALBOX_HYPERVISOR = {"hypervisor_name": "auto"} VIRTUALBOX_OS = {"os_name": None} VIRTUALBOX_BACKEND = { "vcpus": "1", "memory" : "2048", "auto_balance": "auto", } VIRTUALBOX_NAME = None VIRTUALBOX_TEMPLATE = None VIRTUALBOX_TAGS = None VIRTUALBOX_VERSION = None EMPTY_DISKS = {} EMPTY_NETWORKS = {} EMPTY_HYPERVISOR = {"hypervisor_name": "auto"} EMPTY_OS = {} EMPTY_BACKEND = { "vcpus": "auto", "memory" : "auto", "auto_balance": "auto", } EMPTY_NAME = None EMPTY_TEMPLATE = None EMPTY_TAGS = None EMPTY_VERSION = None CMDARGS_DISKS = { "disk_count": "1", "disk0_ivname": "disk/0", "disk0_dump": "disk0.raw", "disk0_size": "8", } CMDARGS_NETWORKS = { "nic0_link": "auto", "nic0_mode": "bridged", "nic0_ip": "none", "nic0_mac": "auto", "nic_count": "1", "nic0_network": "auto", } CMDARGS_HYPERVISOR = { "hypervisor_name": "xen-pvm" } CMDARGS_OS = {"os_name": "lenny-image"} CMDARGS_BACKEND = { "auto_balance": False, "vcpus": "1", "memory": "256", } CMDARGS_NAME = "test-instance" CMDARGS_TEMPLATE = "plain" CMDARGS_TAGS = "test-tag-1,test-tag-2" ARGS_EMPTY = { "output_dir": None, "nics": [], "disks": [], "name": "test-instance", "ova_package": False, "ext_usage": False, "disk_format": "cow", "compression": False, } ARGS_EXPORT_DIR = dict(ARGS_EMPTY, **{ "output_dir": OUTPUT_DIR, "name": None, "hypervisor": None, "os": None, "beparams": {}, "no_nics": False, "disk_template": None, "tags": None, }) ARGS_VBOX = dict(ARGS_EXPORT_DIR, **{ "output_dir": OUTPUT_DIR, "name": "test-instance", "os": "lenny-image", "hypervisor": ("xen-pvm", {}), "osparams":{}, "osparams_private":{}, "disks": [], }) ARGS_COMPLETE = dict(ARGS_VBOX, **{ "beparams": {"vcpus":"1", "memory":"256", "auto_balance": False}, "disks": [(0,{"size":"5mb"})], "nics": [("0",{"mode":"bridged"})], "disk_template": "plain", "tags": "test-tag-1,test-tag-2", }) ARGS_BROKEN = dict(ARGS_EXPORT_DIR , **{ "no_nics": True, "disk_template": "diskless", "name": "test-instance", "os": "lenny-image", "osparams": {}, "osparams_private":{}, }) EXP_ARGS_COMPRESSED = dict(ARGS_EXPORT_DIR, **{ "compression": True, }) EXP_DISKS_LIST = [ { "format": "vmdk", "compression": "gzip", "virt-size": 90000, "real-size": 203, "path": "new_disk.cow.gz", }, { "format": "cow", "virt-size": 15, "real-size": 15, "path": "new_disk.cow", }, ] EXP_NETWORKS_LIST = [ {"mac": "aa:00:00:d8:2c:1e", "ip":"None", "link":"br0", "mode":"routed", "network": "test"}, ] EXP_PARTIAL_GANETI_DICT = { "hypervisor": {"name": "xen-kvm"}, "os": {"name": "lenny-image"}, "auto_balance": "True", "version": "0", } EXP_GANETI_DICT = { "tags": None, "auto_balance": "False", "hypervisor": { "root-path": "/dev/sda", "name": "xen-pvm", "kernel_args": "ro" }, "version": "0", "disk_template": None, "os": {"name": "lenny-image"} } EXP_NAME ="xen-dev-i1" EXP_VCPUS = 1 EXP_MEMORY = 512 EXPORT_EMPTY = ("") EXPORT_DISKS_EMPTY = ("Virtual disk" " information") EXPORT_DISKS = ("" "Virtual disk information" "") EXPORT_NETWORKS_EMPTY = ("List of logical networks" "") EXPORT_NETWORKS = ("List of logical networks" "") EXPORT_GANETI_INCOMPLETE = ("0" "Truelenny-image" "xen-kvmroutedaa:00:00:d8:2c:1eNone" "br0test" "") EXPORT_GANETI = ("0False" "lenny-imagexen-pvm" "/dev/sdaroroutedaa:00:00:d8:2c:1eNonebr0" "test" "") EXPORT_SYSTEM = ("" "Virtual disk information" "" "" "List of logical networks" "A virtual machinexen-dev-i1" "Installed guest" " operating systemVirtual hardware requirements" "Virtual Hardware Family" "0xen-dev-i1ganeti-ovf1 virtual CPU(s)" "131" "" "byte * 2^20512MB of" " memory24512" "0scsi" "_controller03" "lsilogic6" "disk0ovf:/disk/disk043" "17disk1ovf:/" "disk/disk15317" "aa:00" ":00:d8:2c:1erouted0routed0610" "") def _GetArgs(args, with_name=False): options = optparse.Values() needed = args if with_name: needed["name"] = "test-instance" options._update_loose(needed) return options def _SortAttrs(tree): """Sort XML node attributes by name As of Python 3.8, ElementTree will sort output according to insertion order, leveraging the stable dict sorting available since Python 3.6. Some of the tests in this file rely on the previous behavior of ElementTree, which produced stable output by sorting the node attributes. This function updates node attribute dictionaries so that the insertion order matches the alphabetical order and the tree thus produces identical output in all Python 3 versions. """ for el in tree.iter(): attrib = el.attrib if len(attrib) > 1: attribs = sorted(attrib.items()) attrib.clear() attrib.update(attribs) OPTS_EMPTY = _GetArgs(ARGS_EMPTY) OPTS_EXPORT_NO_NAME = _GetArgs(ARGS_EXPORT_DIR) OPTS_EXPORT = _GetArgs(ARGS_EXPORT_DIR, with_name=True) EXP_OPTS = OPTS_EXPORT_NO_NAME EXP_OPTS_COMPRESSED = _GetArgs(EXP_ARGS_COMPRESSED) OPTS_VBOX = _GetArgs(ARGS_VBOX) OPTS_COMPLETE = _GetArgs(ARGS_COMPLETE) OPTS_NONIC_NODISK = _GetArgs(ARGS_BROKEN) def _GetFullFilename(file_name): file_path = "%s/test/data/ovfdata/%s" % (testutils.GetSourceDir(), file_name) file_path = os.path.abspath(file_path) return file_path class BetterUnitTest(unittest.TestCase): def assertRaisesRegexp(self, exception, regexp_val, function, *args): try: function(*args) self.fail("Expected raising %s" % exception) except exception as err: regexp = re.compile(regexp_val) if re.search(regexp, str(err)) == None: self.fail("Expected matching '%s', got '%s'" % (regexp_val, str(err))) class TestOVFImporter(BetterUnitTest): def setUp(self): self.non_existing_file = _GetFullFilename("not_the_file.ovf") self.ganeti_ovf = _GetFullFilename("ganeti.ovf") self.virtualbox_ovf = _GetFullFilename("virtualbox.ovf") self.ova_package = _GetFullFilename("ova.ova") self.empty_ovf = _GetFullFilename("empty.ovf") self.wrong_extension = _GetFullFilename("wrong_extension.ovd") self.wrong_ova_archive = _GetFullFilename("wrong_ova.ova") self.no_ovf_in_ova = _GetFullFilename("no_ovf.ova") self.importer = None def tearDown(self): if self.importer: self.importer.Cleanup() del_dir = os.path.abspath(OUTPUT_DIR) try: shutil.rmtree(del_dir) except OSError: pass def testFileDoesNotExistError(self): self.assertRaisesRegex(errors.OpPrereqError, "does not exist", ovf.OVFImporter, self.non_existing_file, None) def testWrongInputFileExtensionError(self): self.assertRaisesRegex(errors.OpPrereqError, "Unknown file extension", ovf.OVFImporter, self.wrong_extension, None) def testOVAUnpackingDirectories(self): self.importer = ovf.OVFImporter(self.ova_package, OPTS_EMPTY) self.assertTrue(self.importer.input_dir != None) self.assertEqual(self.importer.output_dir , pathutils.EXPORT_DIR) self.assertTrue(self.importer.temp_dir != None) def testOVFUnpackingDirectories(self): self.importer = ovf.OVFImporter(self.virtualbox_ovf, OPTS_EMPTY) self.assertEqual(self.importer.input_dir , _GetFullFilename("")) self.assertEqual(self.importer.output_dir , pathutils.EXPORT_DIR) self.assertEqual(self.importer.temp_dir , None) def testOVFSetOutputDirDirectories(self): self.importer = ovf.OVFImporter(self.ganeti_ovf, OPTS_EXPORT) self.assertEqual(self.importer.input_dir , _GetFullFilename("")) self.assertTrue(OUTPUT_DIR in self.importer.output_dir) self.assertEqual(self.importer.temp_dir , None) def testWrongOVAArchiveError(self): self.assertRaisesRegex(errors.OpPrereqError, "not a proper tar", ovf.OVFImporter, self.wrong_ova_archive, None) def testNoOVFFileInOVAPackageError(self): self.assertRaisesRegex(errors.OpPrereqError, "No .ovf file", ovf.OVFImporter, self.no_ovf_in_ova, None) def testParseGanetiOvf(self): self.importer = ovf.OVFImporter(self.ganeti_ovf, OPTS_EXPORT_NO_NAME) self.importer.Parse() self.assertTrue("%s/ganeti-test-xen" % OUTPUT_DIR in self.importer.output_dir) self.assertEqual(self.importer.results_disk, GANETI_DISKS) self.assertEqual(self.importer.results_network, GANETI_NETWORKS) self.assertEqual(self.importer.results_hypervisor, GANETI_HYPERVISOR) self.assertEqual(self.importer.results_os, GANETI_OS) self.assertEqual(self.importer.results_backend, GANETI_BACKEND) self.assertEqual(self.importer.results_name, GANETI_NAME) self.assertEqual(self.importer.results_template, GANETI_TEMPLATE) self.assertEqual(self.importer.results_tags, GANETI_TAGS) self.assertEqual(self.importer.results_version, GANETI_VERSION) def testParseVirtualboxOvf(self): self.importer = ovf.OVFImporter(self.virtualbox_ovf, OPTS_VBOX) self.importer.Parse() self.assertTrue("%s/test-instance" % OUTPUT_DIR in self.importer.output_dir) self.assertEqual(self.importer.results_disk, VIRTUALBOX_DISKS) self.assertEqual(self.importer.results_network, VIRTUALBOX_NETWORKS) self.assertEqual(self.importer.results_hypervisor, CMDARGS_HYPERVISOR) self.assertEqual(self.importer.results_os, CMDARGS_OS) self.assertEqual(self.importer.results_backend, VIRTUALBOX_BACKEND) self.assertEqual(self.importer.results_name, CMDARGS_NAME) self.assertEqual(self.importer.results_template, VIRTUALBOX_TEMPLATE) self.assertEqual(self.importer.results_tags, VIRTUALBOX_TAGS) self.assertEqual(self.importer.results_version, constants.EXPORT_VERSION) def testParseEmptyOvf(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_COMPLETE) self.importer.Parse() self.assertTrue("%s/test-instance" % OUTPUT_DIR in self.importer.output_dir) self.assertEqual(self.importer.results_disk, CMDARGS_DISKS) self.assertEqual(self.importer.results_network, CMDARGS_NETWORKS) self.assertEqual(self.importer.results_hypervisor, CMDARGS_HYPERVISOR) self.assertEqual(self.importer.results_os, CMDARGS_OS) self.assertEqual(self.importer.results_backend, CMDARGS_BACKEND) self.assertEqual(self.importer.results_name, CMDARGS_NAME) self.assertEqual(self.importer.results_template, CMDARGS_TEMPLATE) self.assertEqual(self.importer.results_tags, CMDARGS_TAGS) self.assertEqual(self.importer.results_version, constants.EXPORT_VERSION) def testParseNameOptions(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_COMPLETE) results = self.importer._ParseNameOptions() self.assertEqual(results, CMDARGS_NAME) def testParseHypervisorOptions(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_COMPLETE) results = self.importer._ParseHypervisorOptions() self.assertEqual(results, CMDARGS_HYPERVISOR) def testParseOSOptions(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_COMPLETE) results = self.importer._ParseOSOptions() self.assertEqual(results, CMDARGS_OS) def testParseBackendOptions(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_COMPLETE) results = self.importer._ParseBackendOptions() self.assertEqual(results, CMDARGS_BACKEND) def testParseTags(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_COMPLETE) results = self.importer._ParseTags() self.assertEqual(results, CMDARGS_TAGS) def testParseNicOptions(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_COMPLETE) results = self.importer._ParseNicOptions() self.assertEqual(results, CMDARGS_NETWORKS) def testParseDiskOptionsFromGanetiOVF(self): self.importer = ovf.OVFImporter(self.ganeti_ovf, OPTS_EXPORT) os.mkdir(OUTPUT_DIR) results = self.importer._GetDiskInfo() self.assertEqual(results, GANETI_DISKS) def testParseTemplateOptions(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_COMPLETE) results = self.importer._ParseTemplateOptions() self.assertEqual(results, GANETI_TEMPLATE) def testParseDiskOptionsFromCmdLine(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_COMPLETE) os.mkdir(OUTPUT_DIR) results = self.importer._ParseDiskOptions() self.assertEqual(results, CMDARGS_DISKS) def testGetDiskFormat(self): self.importer = ovf.OVFImporter(self.ganeti_ovf, OPTS_EXPORT) disks_list = self.importer.ovf_reader.GetDisksNames() results = [self.importer._GetDiskQemuInfo("%s/%s" % (self.importer.input_dir, path), "file format: (\S+)") for (path, _) in disks_list] self.assertEqual(results, ["vmdk"]) def testNoInstanceNameOVF(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_EXPORT_NO_NAME) self.assertRaisesRegex(errors.OpPrereqError, "Name of instance", self.importer.Parse) def testErrorNoOSNameOVF(self): self.importer = ovf.OVFImporter(self.virtualbox_ovf, OPTS_EXPORT) self.assertRaisesRegex(errors.OpPrereqError, "OS name", self.importer.Parse) def testErrorNoDiskAndNoNetwork(self): self.importer = ovf.OVFImporter(self.empty_ovf, OPTS_NONIC_NODISK) self.assertRaisesRegex(errors.OpPrereqError, "Either disk specification or network" " description", self.importer.Parse) class TestOVFExporter(BetterUnitTest): def setUp(self): self.exporter = None self.wrong_config_file = _GetFullFilename("wrong_config.ini") self.unsafe_path_to_disk = _GetFullFilename("unsafe_path.ini") self.disk_image_not_exist = _GetFullFilename("no_disk.ini") self.empty_config = _GetFullFilename("empty.ini") self.standard_export = _GetFullFilename("config.ini") self.wrong_network_mode = self.disk_image_not_exist self.no_memory = self.disk_image_not_exist self.no_vcpus = self.disk_image_not_exist self.no_os = _GetFullFilename("no_os.ini") self.no_hypervisor = self.disk_image_not_exist def tearDown(self): if self.exporter: self.exporter.Cleanup() del_dir = os.path.abspath(OUTPUT_DIR) try: shutil.rmtree(del_dir) except OSError: pass def testErrorWrongConfigFile(self): self.assertRaisesRegex(errors.OpPrereqError, "Error when trying to read", ovf.OVFExporter, self.wrong_config_file, EXP_OPTS) def testErrorPathToTheDiskIncorrect(self): self.exporter = ovf.OVFExporter(self.unsafe_path_to_disk, EXP_OPTS) self.assertRaisesRegex(errors.OpPrereqError, "contains a directory name", self.exporter._ParseDisks) def testErrorDiskImageNotExist(self): self.exporter = ovf.OVFExporter(self.disk_image_not_exist, EXP_OPTS) self.assertRaisesRegex(errors.OpPrereqError, "Disk image does not exist", self.exporter._ParseDisks) def testParseNetworks(self): self.exporter = ovf.OVFExporter(self.standard_export, EXP_OPTS) results = self.exporter._ParseNetworks() self.assertEqual(results, EXP_NETWORKS_LIST) def testErrorWrongNetworkMode(self): self.exporter = ovf.OVFExporter(self.wrong_network_mode, EXP_OPTS) self.assertRaisesRegex(errors.OpPrereqError, "Network mode nic not recognized", self.exporter._ParseNetworks) def testParseVCPusMem(self): self.exporter = ovf.OVFExporter(self.standard_export, EXP_OPTS) vcpus = self.exporter._ParseVCPUs() memory = self.exporter._ParseMemory() self.assertEqual(vcpus, EXP_VCPUS) self.assertEqual(memory, EXP_MEMORY) def testErrorNoVCPUs(self): self.exporter = ovf.OVFExporter(self.no_vcpus, EXP_OPTS) self.assertRaisesRegex(errors.OpPrereqError, "No CPU information found", self.exporter._ParseVCPUs) def testErrorNoMemory(self): self.exporter = ovf.OVFExporter(self.no_memory, EXP_OPTS) self.assertRaisesRegex(errors.OpPrereqError, "No memory information found", self.exporter._ParseMemory) def testParseGaneti(self): self.exporter = ovf.OVFExporter(self.standard_export, EXP_OPTS) results = self.exporter._ParseGaneti() self.assertEqual(results, EXP_GANETI_DICT) def testErrorNoHypervisor(self): self.exporter = ovf.OVFExporter(self.no_hypervisor, EXP_OPTS) self.assertRaisesRegex(errors.OpPrereqError, "No hypervisor information found", self.exporter._ParseGaneti) def testErrorNoOS(self): self.exporter = ovf.OVFExporter(self.no_os, EXP_OPTS) self.assertRaisesRegex(errors.OpPrereqError, "No operating system information found", self.exporter._ParseGaneti) def testErrorParseNoInstanceName(self): self.exporter = ovf.OVFExporter(self.empty_config, EXP_OPTS) self.assertRaisesRegex(errors.OpPrereqError, "No instance name found", self.exporter.Parse) class TestOVFReader(BetterUnitTest): def setUp(self): self.wrong_xml_file = _GetFullFilename("wrong_xml.ovf") self.ganeti_ovf = _GetFullFilename("ganeti.ovf") self.virtualbox_ovf = _GetFullFilename("virtualbox.ovf") self.corrupted_ovf = _GetFullFilename("corrupted_resources.ovf") self.wrong_manifest_ovf = _GetFullFilename("wrong_manifest.ovf") self.no_disk_in_ref_ovf = _GetFullFilename("no_disk_in_ref.ovf") self.empty_ovf = _GetFullFilename("empty.ovf") self.compressed_disk = _GetFullFilename("gzip_disk.ovf") def tearDown(self): pass def testXMLParsingError(self): self.assertRaisesRegex(errors.OpPrereqError, "Error while reading .ovf", ovf.OVFReader, self.wrong_xml_file) def testFileInResourcesDoesNotExistError(self): self.assertRaisesRegex(errors.OpPrereqError, "does not exist", ovf.OVFReader, self.corrupted_ovf) def testWrongManifestChecksumError(self): reader = ovf.OVFReader(self.wrong_manifest_ovf) self.assertRaisesRegex(errors.OpPrereqError, "does not match the value in manifest file", reader.VerifyManifest) def testGoodManifestChecksum(self): reader = ovf.OVFReader(self.ganeti_ovf) self.assertEqual(reader.VerifyManifest(), None) def testGetDisksNamesOVFCorruptedError(self): reader = ovf.OVFReader(self.no_disk_in_ref_ovf) self.assertRaisesRegex(errors.OpPrereqError, "not found in references", reader.GetDisksNames) def testGetDisksNamesVirtualbox(self): reader = ovf.OVFReader(self.virtualbox_ovf) disk_names = reader.GetDisksNames() expected_names = [ ("new_disk.vmdk", None) , ("second_disk.vmdk", None), ] self.assertEqual(sorted(disk_names), sorted(expected_names)) def testGetDisksNamesEmpty(self): reader = ovf.OVFReader(self.empty_ovf) disk_names = reader.GetDisksNames() self.assertEqual(disk_names, []) def testGetDisksNamesCompressed(self): reader = ovf.OVFReader(self.compressed_disk) disk_names = reader.GetDisksNames() self.assertEqual(disk_names, [("compr_disk.vmdk.gz", "gzip")]) def testGetNetworkDataGaneti(self): reader = ovf.OVFReader(self.ganeti_ovf) networks = reader.GetNetworkData() self.assertEqual(networks, GANETI_NETWORKS) def testGetNetworkDataVirtualbox(self): reader = ovf.OVFReader(self.virtualbox_ovf) networks = reader.GetNetworkData() self.assertEqual(networks, VIRTUALBOX_NETWORKS) def testGetNetworkDataEmpty(self): reader = ovf.OVFReader(self.empty_ovf) networks = reader.GetNetworkData() self.assertEqual(networks, EMPTY_NETWORKS) def testGetHypervisorDataGaneti(self): reader = ovf.OVFReader(self.ganeti_ovf) hypervisor = reader.GetHypervisorData() self.assertEqual(hypervisor, GANETI_HYPERVISOR) def testGetHypervisorDataEmptyOvf(self): reader = ovf.OVFReader(self.empty_ovf) hypervisor = reader.GetHypervisorData() self.assertEqual(hypervisor, EMPTY_HYPERVISOR) def testGetOSDataGaneti(self): reader = ovf.OVFReader(self.ganeti_ovf) osys = reader.GetOSData() self.assertEqual(osys, GANETI_OS) def testGetOSDataEmptyOvf(self): reader = ovf.OVFReader(self.empty_ovf) osys = reader.GetOSData() self.assertEqual(osys, EMPTY_OS) def testGetBackendDataGaneti(self): reader = ovf.OVFReader(self.ganeti_ovf) backend = reader.GetBackendData() self.assertEqual(backend, GANETI_BACKEND) def testGetBackendDataVirtualbox(self): reader = ovf.OVFReader(self.virtualbox_ovf) backend = reader.GetBackendData() self.assertEqual(backend, VIRTUALBOX_BACKEND) def testGetBackendDataEmptyOvf(self): reader = ovf.OVFReader(self.empty_ovf) backend = reader.GetBackendData() self.assertEqual(backend, EMPTY_BACKEND) def testGetInstanceNameGaneti(self): reader = ovf.OVFReader(self.ganeti_ovf) name = reader.GetInstanceName() self.assertEqual(name, GANETI_NAME) def testGetInstanceNameDataEmptyOvf(self): reader = ovf.OVFReader(self.empty_ovf) name = reader.GetInstanceName() self.assertEqual(name, EMPTY_NAME) def testGetDiskTemplateGaneti(self): reader = ovf.OVFReader(self.ganeti_ovf) name = reader.GetDiskTemplate() self.assertEqual(name, GANETI_TEMPLATE) def testGetDiskTemplateEmpty(self): reader = ovf.OVFReader(self.empty_ovf) name = reader.GetDiskTemplate() self.assertEqual(name, EMPTY_TEMPLATE) def testGetTagsGaneti(self): reader = ovf.OVFReader(self.ganeti_ovf) tags = reader.GetTagsData() self.assertEqual(tags, GANETI_TAGS) def testGetTagsEmpty(self): reader = ovf.OVFReader(self.empty_ovf) tags = reader.GetTagsData() self.assertEqual(tags, EMPTY_TAGS) def testGetVersionGaneti(self): reader = ovf.OVFReader(self.ganeti_ovf) version = reader.GetVersionData() self.assertEqual(version, GANETI_VERSION) def testGetVersionEmpty(self): reader = ovf.OVFReader(self.empty_ovf) version = reader.GetVersionData() self.assertEqual(version, EMPTY_VERSION) class TestOVFWriter(BetterUnitTest): def setUp(self): self.writer = ovf.OVFWriter(True) def tearDown(self): pass def assertXMLOutputContains(self, text): _SortAttrs(self.writer.tree) result = ET.tostring(self.writer.tree, encoding="unicode") self.assertIn(text, result) def testOVFWriterInit(self): self.assertXMLOutputContains(EXPORT_EMPTY) def testSaveDisksDataEmpty(self): self.writer.SaveDisksData([]) self.assertXMLOutputContains(EXPORT_DISKS_EMPTY) def testSaveDisksData(self): self.writer.SaveDisksData(EXP_DISKS_LIST) self.assertXMLOutputContains(EXPORT_DISKS) def testSaveNetworkDataEmpty(self): self.writer.SaveNetworksData([]) self.assertXMLOutputContains(EXPORT_NETWORKS_EMPTY) def testSaveNetworksData(self): self.writer.SaveNetworksData(EXP_NETWORKS_LIST) self.assertXMLOutputContains(EXPORT_NETWORKS) def testSaveGanetiDataIncomplete(self): self.writer.SaveGanetiData(EXP_PARTIAL_GANETI_DICT, EXP_NETWORKS_LIST) self.assertXMLOutputContains(EXPORT_GANETI_INCOMPLETE) def testSaveGanetiDataComplete(self): self.writer.SaveGanetiData(EXP_GANETI_DICT, EXP_NETWORKS_LIST) self.assertXMLOutputContains(EXPORT_GANETI) def testSaveVirtualSystem(self): self.writer.SaveDisksData(EXP_DISKS_LIST) self.writer.SaveNetworksData(EXP_NETWORKS_LIST) self.writer.SaveVirtualSystemData(EXP_NAME, EXP_VCPUS, EXP_MEMORY) self.assertXMLOutputContains(EXPORT_SYSTEM) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.qlang_unittest.py000075500000000000000000000257341476477700300233560ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.qlang""" import unittest import string from ganeti import utils from ganeti import errors from ganeti import qlang from ganeti import query import testutils class TestMakeSimpleFilter(unittest.TestCase): def _Test(self, field, names, expected, parse_exp=None): if parse_exp is None: parse_exp = names qfilter = qlang.MakeSimpleFilter(field, names) self.assertEqual(qfilter, expected) def test(self): self._Test("name", None, None, parse_exp=[]) self._Test("name", [], None) self._Test("name", ["node1.example.com"], ["|", ["==", "name", "node1.example.com"]]) self._Test("xyz", ["a", "b", "c"], ["|", ["==", "xyz", "a"], ["==", "xyz", "b"], ["==", "xyz", "c"]]) class TestParseFilter(unittest.TestCase): def setUp(self): self.parser = qlang.BuildFilterParser() def _Test(self, qfilter, expected, expect_filter=True): self.assertEqual(qlang.MakeFilter([qfilter], not expect_filter), expected) self.assertEqual(qlang.ParseFilter(qfilter, parser=self.parser), expected) def test(self): self._Test("name==\"foobar\"", [qlang.OP_EQUAL, "name", "foobar"]) self._Test("name=='foobar'", [qlang.OP_EQUAL, "name", "foobar"]) # Legacy "=" self._Test("name=\"foobar\"", [qlang.OP_EQUAL, "name", "foobar"]) self._Test("name='foobar'", [qlang.OP_EQUAL, "name", "foobar"]) self._Test("valA==1 and valB==2 or valC==3", [qlang.OP_OR, [qlang.OP_AND, [qlang.OP_EQUAL, "valA", 1], [qlang.OP_EQUAL, "valB", 2]], [qlang.OP_EQUAL, "valC", 3]]) self._Test(("(name\n==\"foobar\") and (xyz==\"va)ue\" and k == 256 or" " x ==\t\"y\"\n) and mc"), [qlang.OP_AND, [qlang.OP_EQUAL, "name", "foobar"], [qlang.OP_OR, [qlang.OP_AND, [qlang.OP_EQUAL, "xyz", "va)ue"], [qlang.OP_EQUAL, "k", 256]], [qlang.OP_EQUAL, "x", "y"]], [qlang.OP_TRUE, "mc"]]) self._Test("(xyz==\"v\" or k == 256 and x == \"y\")", [qlang.OP_OR, [qlang.OP_EQUAL, "xyz", "v"], [qlang.OP_AND, [qlang.OP_EQUAL, "k", 256], [qlang.OP_EQUAL, "x", "y"]]]) self._Test("valA==1 and valB==2 and valC==3", [qlang.OP_AND, [qlang.OP_EQUAL, "valA", 1], [qlang.OP_EQUAL, "valB", 2], [qlang.OP_EQUAL, "valC", 3]]) self._Test("master or field", [qlang.OP_OR, [qlang.OP_TRUE, "master"], [qlang.OP_TRUE, "field"]]) self._Test("mem == 128", [qlang.OP_EQUAL, "mem", 128]) self._Test("negfield != -1", [qlang.OP_NOT_EQUAL, "negfield", -1]) self._Test("master", [qlang.OP_TRUE, "master"], expect_filter=False) self._Test("not master", [qlang.OP_NOT, [qlang.OP_TRUE, "master"]]) for op in ["not", "and", "or"]: self._Test("%sxyz" % op, [qlang.OP_TRUE, "%sxyz" % op], expect_filter=False) self._Test("not %sxyz" % op, [qlang.OP_NOT, [qlang.OP_TRUE, "%sxyz" % op]]) self._Test(" not \t%sfoo" % op, [qlang.OP_NOT, [qlang.OP_TRUE, "%sfoo" % op]]) self._Test("%sname =~ m/abc/" % op, [qlang.OP_REGEXP, "%sname" % op, "abc"]) self._Test("master and not other", [qlang.OP_AND, [qlang.OP_TRUE, "master"], [qlang.OP_NOT, [qlang.OP_TRUE, "other"]]]) self._Test("not (master or other == 4)", [qlang.OP_NOT, [qlang.OP_OR, [qlang.OP_TRUE, "master"], [qlang.OP_EQUAL, "other", 4]]]) self._Test("some==\"val\\\"ue\"", [qlang.OP_EQUAL, "some", "val\\\"ue"]) self._Test("123 in ips", [qlang.OP_CONTAINS, "ips", 123]) self._Test("99 not in ips", [qlang.OP_NOT, [qlang.OP_CONTAINS, "ips", 99]]) self._Test("\"a\" in valA and \"b\" not in valB", [qlang.OP_AND, [qlang.OP_CONTAINS, "valA", "a"], [qlang.OP_NOT, [qlang.OP_CONTAINS, "valB", "b"]]]) self._Test("name =~ m/test/", [qlang.OP_REGEXP, "name", "test"]) self._Test("name =~ m/^node.*example.com$/i", [qlang.OP_REGEXP, "name", "(?i)^node.*example.com$"]) self._Test("(name =~ m/^node.*example.com$/s and master) or pip =~ |^3.*|", [qlang.OP_OR, [qlang.OP_AND, [qlang.OP_REGEXP, "name", "(?s)^node.*example.com$"], [qlang.OP_TRUE, "master"]], [qlang.OP_REGEXP, "pip", "^3.*"]]) for flags in ["si", "is", "ssss", "iiiisiii"]: self._Test("name =~ m/gi/%s" % flags, [qlang.OP_REGEXP, "name", "(?%s)gi" % "".join(sorted(flags))]) for i in qlang._KNOWN_REGEXP_DELIM: self._Test("name =~ m%stest%s" % (i, i), [qlang.OP_REGEXP, "name", "test"]) self._Test("name !~ m%stest%s" % (i, i), [qlang.OP_NOT, [qlang.OP_REGEXP, "name", "test"]]) self._Test("not\tname =~ m%stest%s" % (i, i), [qlang.OP_NOT, [qlang.OP_REGEXP, "name", "test"]]) self._Test("notname =~ m%stest%s" % (i, i), [qlang.OP_REGEXP, "notname", "test"]) self._Test("name =* '*.site'", [qlang.OP_REGEXP, "name", utils.DnsNameGlobPattern("*.site")]) self._Test("field !* '*.example.*'", [qlang.OP_NOT, [qlang.OP_REGEXP, "field", utils.DnsNameGlobPattern("*.example.*")]]) self._Test("ctime < 1234", [qlang.OP_LT, "ctime", 1234]) self._Test("ctime > 1234", [qlang.OP_GT, "ctime", 1234]) self._Test("mtime <= 9999", [qlang.OP_LE, "mtime", 9999]) self._Test("mtime >= 9999", [qlang.OP_GE, "mtime", 9999]) def testAllFields(self): for name in frozenset(i for d in query.ALL_FIELD_LISTS for i in d): self._Test("%s == \"value\"" % name, [qlang.OP_EQUAL, name, "value"]) def testError(self): # Invalid field names, meaning no boolean check is done tests = ["#invalid!filter#", "m/x/,"] # Unknown regexp flag tests.append("name=~m#a#g") # Incomplete regexp group tests.append("name=~^[^") # Valid flag, but in uppercase tests.append("asdf =~ m|abc|I") # Non-matching regexp delimiters tests.append("name =~ /foobarbaz#") # Invalid operators tests.append("name <> value") tests.append("name => value") tests.append("name =< value") for qfilter in tests: try: qlang.ParseFilter(qfilter, parser=self.parser) except errors.QueryFilterParseError as err: self.assertEqual(len(err.GetDetails()), 3) else: self.fail("Invalid filter '%s' did not raise exception" % qfilter) class TestMakeFilter(unittest.TestCase): def testNoNames(self): self.assertEqual(qlang.MakeFilter([], False), None) self.assertEqual(qlang.MakeFilter(None, False), None) def testPlainNames(self): self.assertEqual(qlang.MakeFilter(["web1", "web2"], False), [qlang.OP_OR, [qlang.OP_EQUAL, "name", "web1"], [qlang.OP_EQUAL, "name", "web2"]]) def testPlainNamesOtherNamefield(self): self.assertEqual(qlang.MakeFilter(["mailA", "mailB"], False, namefield="id"), [qlang.OP_OR, [qlang.OP_EQUAL, "id", "mailA"], [qlang.OP_EQUAL, "id", "mailB"]]) def testForcedFilter(self): for i in [None, [], ["1", "2"], ["", "", ""], ["a", "b", "c", "d"]]: self.assertRaises(errors.OpPrereqError, qlang.MakeFilter, i, True) # Glob pattern shouldn't parse as filter self.assertRaises(errors.QueryFilterParseError, qlang.MakeFilter, ["*.site"], True) # Plain name parses as boolean filter self.assertEqual(qlang.MakeFilter(["web1"], True), [qlang.OP_TRUE, "web1"]) def testFilter(self): self.assertEqual(qlang.MakeFilter(["foo/bar"], False), [qlang.OP_TRUE, "foo/bar"]) self.assertEqual(qlang.MakeFilter(["foo=='bar'"], False), [qlang.OP_EQUAL, "foo", "bar"]) self.assertEqual(qlang.MakeFilter(["field=*'*.site'"], False), [qlang.OP_REGEXP, "field", utils.DnsNameGlobPattern("*.site")]) # Plain name parses as name filter, not boolean for name in ["node1", "n-o-d-e", "n_o_d_e", "node1.example.com", "node1.example.com."]: self.assertEqual(qlang.MakeFilter([name], False), [qlang.OP_OR, [qlang.OP_EQUAL, "name", name]]) # Invalid filters for i in ["foo==bar", "foo+=1"]: self.assertRaises(errors.QueryFilterParseError, qlang.MakeFilter, [i], False) def testGlob(self): self.assertEqual(qlang.MakeFilter(["*.site"], False), [qlang.OP_OR, [qlang.OP_REGEXP, "name", utils.DnsNameGlobPattern("*.site")]]) self.assertEqual(qlang.MakeFilter(["web?.example"], False), [qlang.OP_OR, [qlang.OP_REGEXP, "name", utils.DnsNameGlobPattern("web?.example")]]) self.assertEqual(qlang.MakeFilter(["*.a", "*.b", "?.c"], False), [qlang.OP_OR, [qlang.OP_REGEXP, "name", utils.DnsNameGlobPattern("*.a")], [qlang.OP_REGEXP, "name", utils.DnsNameGlobPattern("*.b")], [qlang.OP_REGEXP, "name", utils.DnsNameGlobPattern("?.c")]]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.query_unittest.py000075500000000000000000002342061476477700300234150ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.query""" import re import unittest import random import uuid as uuid_module from ganeti import constants from ganeti import utils from ganeti import compat from ganeti import errors from ganeti import query from ganeti import objects from ganeti import cmdlib import ganeti.masterd.instance as gmi from ganeti.hypervisor import hv_base import testutils class TestConstants(unittest.TestCase): def test(self): self.assertEqual(set(query._VERIFY_FN.keys()), constants.QFT_ALL) class _QueryData: def __init__(self, data, **kwargs): self.data = data for name, value in kwargs.items(): setattr(self, name, value) def __iter__(self): return iter(self.data) def _GetDiskSize(nr, ctx, item): disks = item["disks"] try: return disks[nr] except IndexError: return query._FS_UNAVAIL class TestQuery(unittest.TestCase): def test(self): (STATIC, DISK) = range(10, 12) fielddef = query._PrepareFieldList([ (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"), STATIC, 0, lambda ctx, item: item["name"]), (query._MakeField("master", "Master", constants.QFT_BOOL, "Master"), STATIC, 0, lambda ctx, item: ctx.mastername == item["name"]), ] + [(query._MakeField("disk%s.size" % i, "DiskSize%s" % i, constants.QFT_UNIT, "Disk size %s" % i), DISK, 0, compat.partial(_GetDiskSize, i)) for i in range(4)], []) q = query.Query(fielddef, ["name"]) self.assertEqual(q.RequestedData(), set([STATIC])) self.assertEqual(len(q._fields), 1) self.assertEqual(len(q.GetFields()), 1) self.assertEqual(q.GetFields()[0].ToDict(), objects.QueryFieldDefinition(name="name", title="Name", kind=constants.QFT_TEXT, doc="Name").ToDict()) # Create data only once query has been prepared data = [ { "name": "node1", "disks": [0, 1, 2], }, { "name": "node2", "disks": [3, 4], }, { "name": "node3", "disks": [5, 6, 7], }, ] self.assertEqual(q.Query(_QueryData(data, mastername="node3")), [[(constants.RS_NORMAL, "node1")], [(constants.RS_NORMAL, "node2")], [(constants.RS_NORMAL, "node3")]]) self.assertEqual(q.OldStyleQuery(_QueryData(data, mastername="node3")), [["node1"], ["node2"], ["node3"]]) q = query.Query(fielddef, ["name", "master"]) self.assertEqual(q.RequestedData(), set([STATIC])) self.assertEqual(len(q._fields), 2) self.assertEqual(q.Query(_QueryData(data, mastername="node3")), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, False)], [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, True)], ]) q = query.Query(fielddef, ["name", "master", "disk0.size"]) self.assertEqual(q.RequestedData(), set([STATIC, DISK])) self.assertEqual(len(q._fields), 3) self.assertEqual(q.Query(_QueryData(data, mastername="node2")), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, 0)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True), (constants.RS_NORMAL, 3)], [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, 5)], ]) # With unknown column q = query.Query(fielddef, ["disk2.size", "disk1.size", "disk99.size", "disk0.size"]) self.assertEqual(q.RequestedData(), set([DISK])) self.assertEqual(len(q._fields), 4) self.assertEqual(q.Query(_QueryData(data, mastername="node2")), [[(constants.RS_NORMAL, 2), (constants.RS_NORMAL, 1), (constants.RS_UNKNOWN, None), (constants.RS_NORMAL, 0)], [(constants.RS_UNAVAIL, None), (constants.RS_NORMAL, 4), (constants.RS_UNKNOWN, None), (constants.RS_NORMAL, 3)], [(constants.RS_NORMAL, 7), (constants.RS_NORMAL, 6), (constants.RS_UNKNOWN, None), (constants.RS_NORMAL, 5)], ]) self.assertRaises(errors.OpPrereqError, q.OldStyleQuery, _QueryData(data, mastername="node2")) self.assertEqual([fdef.ToDict() for fdef in q.GetFields()], [ { "name": "disk2.size", "title": "DiskSize2", "kind": constants.QFT_UNIT, "doc": "Disk size 2", }, { "name": "disk1.size", "title": "DiskSize1", "kind": constants.QFT_UNIT, "doc": "Disk size 1", }, { "name": "disk99.size", "title": "disk99.size", "kind": constants.QFT_UNKNOWN, "doc": "Unknown field 'disk99.size'", }, { "name": "disk0.size", "title": "DiskSize0", "kind": constants.QFT_UNIT, "doc": "Disk size 0", }, ]) # Empty query q = query.Query(fielddef, []) self.assertEqual(q.RequestedData(), set([])) self.assertEqual(len(q._fields), 0) self.assertEqual(q.Query(_QueryData(data, mastername="node2")), [[], [], []]) self.assertEqual(q.OldStyleQuery(_QueryData(data, mastername="node2")), [[], [], []]) self.assertEqual(q.GetFields(), []) def testPrepareFieldList(self): # Duplicate titles for (a, b) in [("name", "name"), ("NAME", "name")]: self.assertRaises(AssertionError, query._PrepareFieldList, [ (query._MakeField("name", b, constants.QFT_TEXT, "Name"), None, 0, lambda *args: None), (query._MakeField("other", a, constants.QFT_TEXT, "Other"), None, 0, lambda *args: None), ], []) # Non-lowercase names self.assertRaises(AssertionError, query._PrepareFieldList, [ (query._MakeField("NAME", "Name", constants.QFT_TEXT, "Name"), None, 0, lambda *args: None), ], []) self.assertRaises(AssertionError, query._PrepareFieldList, [ (query._MakeField("Name", "Name", constants.QFT_TEXT, "Name"), None, 0, lambda *args: None), ], []) # Empty name self.assertRaises(AssertionError, query._PrepareFieldList, [ (query._MakeField("", "Name", constants.QFT_TEXT, "Name"), None, 0, lambda *args: None), ], []) # Empty title self.assertRaises(AssertionError, query._PrepareFieldList, [ (query._MakeField("name", "", constants.QFT_TEXT, "Name"), None, 0, lambda *args: None), ], []) # Whitespace in title self.assertRaises(AssertionError, query._PrepareFieldList, [ (query._MakeField("name", "Co lu mn", constants.QFT_TEXT, "Name"), None, 0, lambda *args: None), ], []) # No callable function self.assertRaises(AssertionError, query._PrepareFieldList, [ (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"), None, 0, None), ], []) # Invalid documentation for doc in ["", ".", "Hello world\n", "Hello\nWo\nrld", "Hello World!", "HelloWorld.", "only lowercase", ",", " x y z .\t", " "]: self.assertRaises(AssertionError, query._PrepareFieldList, [ (query._MakeField("name", "Name", constants.QFT_TEXT, doc), None, 0, lambda *args: None), ], []) # Duplicate field name self.assertRaises(ValueError, query._PrepareFieldList, [ (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"), None, 0, lambda *args: None), (query._MakeField("name", "Other", constants.QFT_OTHER, "Other"), None, 0, lambda *args: None), ], []) def testUnknown(self): fielddef = query._PrepareFieldList([ (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"), None, 0, lambda _, item: "name%s" % item), (query._MakeField("other0", "Other0", constants.QFT_TIMESTAMP, "Other"), None, 0, lambda *args: 1234), (query._MakeField("nodata", "NoData", constants.QFT_NUMBER, "No data"), None, 0, lambda *args: query._FS_NODATA ), (query._MakeField("unavail", "Unavail", constants.QFT_BOOL, "Unavail"), None, 0, lambda *args: query._FS_UNAVAIL), ], []) for selected in [["foo"], ["Hello", "World"], ["name1", "other", "foo"]]: q = query.Query(fielddef, selected) self.assertEqual(len(q._fields), len(selected)) self.assertTrue(compat.all(len(row) == len(selected) for row in q.Query(_QueryData(range(1, 10))))) self.assertEqual(q.Query(_QueryData(range(1, 10))), [[(constants.RS_UNKNOWN, None)] * len(selected) for i in range(1, 10)]) self.assertEqual([fdef.ToDict() for fdef in q.GetFields()], [{ "name": name, "title": name, "kind": constants.QFT_UNKNOWN, "doc": "Unknown field '%s'" % name} for name in selected]) q = query.Query(fielddef, ["name", "other0", "nodata", "unavail"]) self.assertEqual(len(q._fields), 4) self.assertEqual(q.OldStyleQuery(_QueryData(range(1, 10))), [ ["name%s" % i, 1234, None, None] for i in range(1, 10) ]) q = query.Query(fielddef, ["name", "other0", "nodata", "unavail", "unk"]) self.assertEqual(len(q._fields), 5) self.assertEqual(q.Query(_QueryData(range(1, 10))), [[(constants.RS_NORMAL, "name%s" % i), (constants.RS_NORMAL, 1234), (constants.RS_NODATA, None), (constants.RS_UNAVAIL, None), (constants.RS_UNKNOWN, None)] for i in range(1, 10)]) def testAliases(self): fields = [ (query._MakeField("a", "a-title", constants.QFT_TEXT, "Field A"), None, 0, lambda *args: None), (query._MakeField("b", "b-title", constants.QFT_TEXT, "Field B"), None, 0, lambda *args: None), ] # duplicate field self.assertRaises(AssertionError, query._PrepareFieldList, fields, [("b", "a")]) self.assertRaises(AssertionError, query._PrepareFieldList, fields, [("c", "b"), ("c", "a")]) # missing target self.assertRaises(AssertionError, query._PrepareFieldList, fields, [("c", "d")]) fdefs = query._PrepareFieldList(fields, [("c", "b")]) self.assertEqual(len(fdefs), 3) self.assertEqual(fdefs["b"][1:], fdefs["c"][1:]) class TestGetNodeRole(unittest.TestCase): def test(self): tested_role = set() master_uuid = "969502b9-f632-4d3d-83a5-a78b0ca8cdf6" node_uuid = "d75499b5-83e3-4b80-b6fe-3e1aee7e5a35" checks = [ (constants.NR_MASTER, objects.Node(name="node1", uuid=master_uuid)), (constants.NR_MCANDIDATE, objects.Node(name="node1", uuid=node_uuid, master_candidate=True)), (constants.NR_REGULAR, objects.Node(name="node1", uuid=node_uuid)), (constants.NR_DRAINED, objects.Node(name="node1", uuid=node_uuid, drained=True)), (constants.NR_OFFLINE, objects.Node(name="node1", uuid=node_uuid, offline=True)), ] for (role, node) in checks: result = query._GetNodeRole(node, master_uuid) self.assertEqual(result, role) tested_role.add(result) self.assertEqual(tested_role, constants.NR_ALL) class TestNodeQuery(unittest.TestCase): def _Create(self, selected): return query.Query(query.NODE_FIELDS, selected) def testSimple(self): cluster = objects.Cluster(cluster_name="testcluster", ndparams=constants.NDC_DEFAULTS.copy()) grp1 = objects.NodeGroup(name="default", uuid="c0e89160-18e7-11e0-a46e-001d0904baeb", alloc_policy=constants.ALLOC_POLICY_PREFERRED, ipolicy=objects.MakeEmptyIPolicy(), ndparams={}, ) grp2 = objects.NodeGroup(name="group2", uuid="c0e89160-18e7-11e0-a46e-001d0904babe", alloc_policy=constants.ALLOC_POLICY_PREFERRED, ipolicy=objects.MakeEmptyIPolicy(), ndparams={constants.ND_SPINDLE_COUNT: 2}, ) groups = {grp1.uuid: grp1, grp2.uuid: grp2} nodes = [ objects.Node(name="node1", drained=False, group=grp1.uuid, ndparams={}), objects.Node(name="node2", drained=True, group=grp2.uuid, ndparams={}), objects.Node(name="node3", drained=False, group=grp1.uuid, ndparams={constants.ND_SPINDLE_COUNT: 4}), ] for live_data in [None, dict.fromkeys([node.name for node in nodes], {})]: nqd = query.NodeQueryData(nodes, live_data, None, None, None, None, groups, None, cluster) q = self._Create(["name", "drained"]) self.assertEqual(q.RequestedData(), set([query.NQ_CONFIG])) self.assertEqual(q.Query(nqd), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True)], [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, False)], ]) self.assertEqual(q.OldStyleQuery(nqd), [["node1", False], ["node2", True], ["node3", False]]) q = self._Create(["ndp/spindle_count"]) self.assertEqual(q.RequestedData(), set([query.NQ_GROUP])) self.assertEqual(q.Query(nqd), [[(constants.RS_NORMAL, constants.NDC_DEFAULTS[constants.ND_SPINDLE_COUNT])], [(constants.RS_NORMAL, grp2.ndparams[constants.ND_SPINDLE_COUNT])], [(constants.RS_NORMAL, nodes[2].ndparams[constants.ND_SPINDLE_COUNT])], ]) def test(self): selected = list(query.NODE_FIELDS) field_index = dict((field, idx) for idx, field in enumerate(selected)) q = self._Create(selected) self.assertEqual(q.RequestedData(), set([query.NQ_CONFIG, query.NQ_LIVE, query.NQ_INST, query.NQ_GROUP, query.NQ_OOB])) cluster = objects.Cluster(cluster_name="testcluster", hvparams=constants.HVC_DEFAULTS, beparams={ constants.PP_DEFAULT: constants.BEC_DEFAULTS, }, nicparams={ constants.PP_DEFAULT: constants.NICC_DEFAULTS, }, ndparams=constants.NDC_DEFAULTS, ) node_names = ["node%s" % i for i in range(20)] master_name = node_names[3] nodes = [ objects.Node(name=name, primary_ip="192.0.2.%s" % idx, secondary_ip="192.0.100.%s" % idx, serial_no=7789 * idx, master_candidate=(name != master_name and idx % 3 == 0), offline=False, drained=False, powered=True, vm_capable=True, master_capable=False, ndparams={}, group="default", ctime=1290006900, mtime=1290006913, uuid="fd9ccebe-6339-43c9-a82e-94bbe575%04d" % idx) for idx, name in enumerate(node_names) ] master_node = nodes[3] master_node.AddTag("masternode") master_node.AddTag("another") master_node.AddTag("tag") master_node.ctime = None master_node.mtime = None assert master_node.name == master_name live_data_node = nodes[4] assert live_data_node.name != master_name fake_live_data = { "bootid": "a2504766-498e-4b25-b21e-d23098dc3af4", "cnodes": 4, "cnos": 3, "csockets": 4, "ctotal": 8, "mnode": 128, "mfree": 100, "mtotal": 4096, "dfree": 5 * 1024 * 1024, "dtotal": 100 * 1024 * 1024, "spfree": 0, "sptotal": 0, } assert (sorted(query._NODE_LIVE_FIELDS.keys()) == sorted(fake_live_data.keys())) live_data = dict.fromkeys([node.uuid for node in nodes], {}) live_data[live_data_node.uuid] = \ dict((query._NODE_LIVE_FIELDS[name][2], value) for name, value in fake_live_data.items()) node_to_primary_uuid = dict((node.uuid, set()) for node in nodes) node_to_primary_uuid[master_node.uuid].update(["inst1", "inst2"]) node_to_secondary_uuid = dict((node.uuid, set()) for node in nodes) node_to_secondary_uuid[live_data_node.uuid].update(["instX", "instY", "instZ"]) inst_uuid_to_inst_name = { "inst1": "inst1-name", "inst2": "inst2-name", "instX": "instX-name", "instY": "instY-name", "instZ": "instZ-name" } ng_uuid = "492b4b74-8670-478a-b98d-4c53a76238e6" groups = { ng_uuid: objects.NodeGroup(name="ng1", uuid=ng_uuid, ndparams={}), } oob_not_powered_node = nodes[0] oob_not_powered_node.powered = False oob_support = dict((node.uuid, False) for node in nodes) oob_support[master_node.uuid] = True oob_support[oob_not_powered_node.uuid] = True master_node.group = ng_uuid nqd = query.NodeQueryData(nodes, live_data, master_node.uuid, node_to_primary_uuid, node_to_secondary_uuid, inst_uuid_to_inst_name, groups, oob_support, cluster) result = q.Query(nqd) self.assertTrue(compat.all(len(row) == len(selected) for row in result)) self.assertEqual([row[field_index["name"]] for row in result], [(constants.RS_NORMAL, name) for name in node_names]) node_to_row = dict((row[field_index["name"]][1], idx) for idx, row in enumerate(result)) master_row = result[node_to_row[master_name]] self.assertTrue(master_row[field_index["master"]]) self.assertTrue(master_row[field_index["role"]], "M") self.assertEqual(master_row[field_index["group"]], (constants.RS_NORMAL, "ng1")) self.assertEqual(master_row[field_index["group.uuid"]], (constants.RS_NORMAL, ng_uuid)) self.assertEqual(master_row[field_index["ctime"]], (constants.RS_UNAVAIL, None)) self.assertEqual(master_row[field_index["mtime"]], (constants.RS_UNAVAIL, None)) self.assertTrue(row[field_index["pip"]] == node.primary_ip and row[field_index["sip"]] == node.secondary_ip and set(row[field_index["tags"]]) == node.GetTags() and row[field_index["serial_no"]] == node.serial_no and row[field_index["role"]] == query._GetNodeRole(node, master_name) and (node.name == master_name or (row[field_index["group"]] == "" and row[field_index["group.uuid"]] is None and row[field_index["ctime"]] == (constants.RS_NORMAL, node.ctime) and row[field_index["mtime"]] == (constants.RS_NORMAL, node.mtime) and row[field_index["powered"]] == (constants.RS_NORMAL, True))) or (node.name == oob_not_powered_node and row[field_index["powered"]] == (constants.RS_NORMAL, False)) or row[field_index["powered"]] == (constants.RS_UNAVAIL, None) for row, node in zip(result, nodes)) live_data_row = result[node_to_row[live_data_node.name]] for (field, value) in fake_live_data.items(): self.assertEqual(live_data_row[field_index[field]], (constants.RS_NORMAL, value)) self.assertEqual(master_row[field_index["pinst_cnt"]], (constants.RS_NORMAL, 2)) self.assertEqual(live_data_row[field_index["sinst_cnt"]], (constants.RS_NORMAL, 3)) self.assertEqual(len(master_row[field_index["pinst_list"]]), 2) self.assertEqual(master_row[field_index["pinst_list"]][0], constants.RS_NORMAL) self.assertCountEqual(master_row[field_index["pinst_list"]][1], [inst_uuid_to_inst_name[uuid] for uuid in node_to_primary_uuid[master_node.uuid]]) self.assertEqual(live_data_row[field_index["sinst_list"]], (constants.RS_NORMAL, utils.NiceSort( [inst_uuid_to_inst_name[uuid] for uuid in node_to_secondary_uuid[live_data_node.uuid]]))) def testGetLiveNodeField(self): nodes = [ objects.Node(name="node1", drained=False, offline=False, vm_capable=True), objects.Node(name="node2", drained=True, offline=False, vm_capable=True), objects.Node(name="node3", drained=False, offline=False, vm_capable=True), objects.Node(name="node4", drained=False, offline=True, vm_capable=True), objects.Node(name="node5", drained=False, offline=False, vm_capable=False), ] live_data = dict.fromkeys([node.name for node in nodes], {}) # No data nqd = query.NodeQueryData(None, None, None, None, None, None, None, None, None) self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER, nqd, nodes[0]), query._FS_NODATA) # Missing field ctx = _QueryData(None, curlive_data={ "some": 1, "other": 2, }) self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER, ctx, nodes[0]), query._FS_UNAVAIL) # Wrong format/datatype ctx = _QueryData(None, curlive_data={ "hello": ["Hello World"], "other": 2, }) self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER, ctx, nodes[0]), query._FS_UNAVAIL) # Offline node assert nodes[3].offline ctx = _QueryData(None, curlive_data={}) self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER, ctx, nodes[3]), query._FS_OFFLINE, None) # Wrong field type ctx = _QueryData(None, curlive_data={"hello": 123}) self.assertRaises(AssertionError, query._GetLiveNodeField, "hello", constants.QFT_BOOL, ctx, nodes[0]) # Non-vm_capable node assert not nodes[4].vm_capable ctx = _QueryData(None, curlive_data={}) self.assertEqual(query._GetLiveNodeField("hello", constants.QFT_NUMBER, ctx, nodes[4]), query._FS_UNAVAIL, None) class TestInstanceQuery(unittest.TestCase): def _Create(self, selected): return query.Query(query.INSTANCE_FIELDS, selected) def testSimple(self): q = self._Create(["name", "be/maxmem", "ip"]) self.assertEqual(q.RequestedData(), set([query.IQ_CONFIG])) cluster = objects.Cluster(cluster_name="testcluster", hvparams=constants.HVC_DEFAULTS, beparams={ constants.PP_DEFAULT: constants.BEC_DEFAULTS, }, nicparams={ constants.PP_DEFAULT: constants.NICC_DEFAULTS, }, os_hvp={}, osparams={}) instances = [ objects.Instance(name="inst1", hvparams={}, beparams={}, osparams={}, nics=[], os="deb1"), objects.Instance(name="inst2", hvparams={}, nics=[], osparams={}, os="foomoo", beparams={ constants.BE_MAXMEM: 512, }), objects.Instance(name="inst3", hvparams={}, beparams={}, osparams={}, os="dos", nics=[objects.NIC(ip="192.0.2.99", nicparams={})]), ] iqd = query.InstanceQueryData(instances, cluster, None, [], [], {}, set(), {}, None, None, None) self.assertEqual(q.Query(iqd), [[(constants.RS_NORMAL, "inst1"), (constants.RS_NORMAL, 128), (constants.RS_UNAVAIL, None), ], [(constants.RS_NORMAL, "inst2"), (constants.RS_NORMAL, 512), (constants.RS_UNAVAIL, None), ], [(constants.RS_NORMAL, "inst3"), (constants.RS_NORMAL, 128), (constants.RS_NORMAL, "192.0.2.99"), ]]) self.assertEqual(q.OldStyleQuery(iqd), [["inst1", 128, None], ["inst2", 512, None], ["inst3", 128, "192.0.2.99"]]) def test(self): selected = list(query.INSTANCE_FIELDS) fieldidx = dict((field, idx) for idx, field in enumerate(selected)) macs = ["00:11:22:%02x:%02x:%02x" % (i % 255, i % 3, (i * 123) % 255) for i in range(20)] q = self._Create(selected) self.assertEqual(q.RequestedData(), set([query.IQ_CONFIG, query.IQ_LIVE, query.IQ_DISKUSAGE, query.IQ_CONSOLE, query.IQ_NODES, query.IQ_NETWORKS])) cluster = objects.Cluster(cluster_name="testcluster", enabled_user_shutdown=True, hvparams=constants.HVC_DEFAULTS, beparams={ constants.PP_DEFAULT: constants.BEC_DEFAULTS, }, nicparams={ constants.PP_DEFAULT: constants.NICC_DEFAULTS, }, os_hvp={}, tcpudp_port_pool=set(), osparams={ "deb99": { "clean_install": "yes", }, }) offline_nodes = ["nodeoff1-uuid", "nodeoff2-uuid"] bad_nodes = ["nodebad1-uuid", "nodebad2-uuid", "nodebad3-uuid"] +\ offline_nodes node_uuids = ["node%s-uuid" % i for i in range(10)] + bad_nodes instances = [ objects.Instance(name="inst1", hvparams={}, beparams={}, nics=[], uuid="inst1-uuid", ctime=1291244000, mtime=1291244400, serial_no=30, admin_state=constants.ADMINST_UP, admin_state_source=constants.ADMIN_SOURCE, hypervisor=constants.HT_XEN_PVM, os="linux1", primary_node="node1-uuid", secondary_nodes=[], disk_template=constants.DT_PLAIN, disks=[], disks_active=True, osparams={}), objects.Instance(name="inst2", hvparams={}, nics=[], uuid="inst2-uuid", ctime=1291211000, mtime=1291211077, serial_no=1, admin_state=constants.ADMINST_UP, admin_state_source=constants.ADMIN_SOURCE, hypervisor=constants.HT_XEN_HVM, os="deb99", primary_node="node5-uuid", secondary_nodes=[], disk_template=constants.DT_DISKLESS, disks=[], disks_active=True, beparams={ constants.BE_MAXMEM: 512, constants.BE_MINMEM: 256, }, osparams={}), objects.Instance(name="inst3", hvparams={}, beparams={}, uuid="inst3-uuid", ctime=1291011000, mtime=1291013000, serial_no=1923, admin_state=constants.ADMINST_DOWN, admin_state_source=constants.ADMIN_SOURCE, hypervisor=constants.HT_KVM, os="busybox", primary_node="node6-uuid", secondary_nodes=[], disk_template=constants.DT_DRBD8, disks=[], disks_active=False, nics=[ objects.NIC(ip="192.0.2.99", mac=macs.pop(), nicparams={ constants.NIC_LINK: constants.DEFAULT_BRIDGE, }), objects.NIC(ip=None, mac=macs.pop(), nicparams={}), ], osparams={}), objects.Instance(name="inst4", hvparams={}, beparams={}, uuid="inst4-uuid", ctime=1291244390, mtime=1291244395, serial_no=25, admin_state=constants.ADMINST_DOWN, admin_state_source=constants.ADMIN_SOURCE, hypervisor=constants.HT_XEN_PVM, os="linux1", primary_node="nodeoff2-uuid", secondary_nodes=[], disk_template=constants.DT_DRBD8, disks=[], disks_active=True, nics=[ objects.NIC(ip="192.0.2.1", mac=macs.pop(), nicparams={ constants.NIC_LINK: constants.DEFAULT_BRIDGE, }), objects.NIC(ip="192.0.2.2", mac=macs.pop(), nicparams={}), objects.NIC(ip="192.0.2.3", mac=macs.pop(), nicparams={ constants.NIC_MODE: constants.NIC_MODE_ROUTED, }), objects.NIC(ip="192.0.2.4", mac=macs.pop(), nicparams={ constants.NIC_MODE: constants.NIC_MODE_BRIDGED, constants.NIC_LINK: "eth123", }), ], osparams={}), objects.Instance(name="inst5", hvparams={}, nics=[], uuid="inst5-uuid", ctime=1231211000, mtime=1261200000, serial_no=3, admin_state=constants.ADMINST_UP, admin_state_source=constants.ADMIN_SOURCE, hypervisor=constants.HT_XEN_HVM, os="deb99", primary_node="nodebad2-uuid", secondary_nodes=[], disk_template=constants.DT_DISKLESS, disks=[], disks_active=True, beparams={ constants.BE_MAXMEM: 512, constants.BE_MINMEM: 512, }, osparams={}), objects.Instance(name="inst6", hvparams={}, nics=[], uuid="inst6-uuid", ctime=7513, mtime=11501, serial_no=13390, admin_state=constants.ADMINST_DOWN, admin_state_source=constants.ADMIN_SOURCE, hypervisor=constants.HT_XEN_HVM, os="deb99", primary_node="node7-uuid", secondary_nodes=[], disk_template=constants.DT_DISKLESS, disks=[], disks_active=False, beparams={ constants.BE_MAXMEM: 768, constants.BE_MINMEM: 256, }, osparams={ "clean_install": "no", }), objects.Instance(name="inst7", hvparams={}, nics=[], uuid="inst7-uuid", ctime=None, mtime=None, serial_no=1947, admin_state=constants.ADMINST_DOWN, admin_state_source=constants.ADMIN_SOURCE, hypervisor=constants.HT_XEN_HVM, os="deb99", primary_node="node6-uuid", secondary_nodes=[], disk_template=constants.DT_DISKLESS, disks=[], disks_active=False, beparams={}, osparams={}), objects.Instance(name="inst8", hvparams={}, nics=[], uuid="inst8-uuid", ctime=None, mtime=None, serial_no=19478, admin_state=constants.ADMINST_OFFLINE, admin_state_source=constants.ADMIN_SOURCE, hypervisor=constants.HT_XEN_HVM, os="deb99", primary_node="node6-uuid", secondary_nodes=[], disk_template=constants.DT_DISKLESS, disks=[], disks_active=False, beparams={}, osparams={}), objects.Instance( name="inst9", hvparams={constants.HV_KVM_USER_SHUTDOWN: True}, nics=[], uuid="inst9-uuid", ctime=None, mtime=None, serial_no=19478, admin_state=constants.ADMINST_UP, admin_state_source=constants.ADMIN_SOURCE, hypervisor=constants.HT_XEN_HVM, os="deb99", primary_node="node6-uuid", secondary_nodes=[], disk_template=constants.DT_DISKLESS, disks=[], disks_active=False, beparams={}, osparams={}), ] assert not utils.FindDuplicates(inst.uuid for inst in instances) assert not utils.FindDuplicates(inst.name for inst in instances) instbyname = dict((inst.name, inst) for inst in instances) disk_usage = dict((inst.uuid, gmi.ComputeDiskSize([{"size": disk.size, "dev_type": disk.dev_type} for disk in inst.disks])) for inst in instances) inst_bridges = { "inst3-uuid": [constants.DEFAULT_BRIDGE, constants.DEFAULT_BRIDGE], "inst4-uuid": [constants.DEFAULT_BRIDGE, constants.DEFAULT_BRIDGE, None, "eth123"], } live_data = { "inst2-uuid": { "vcpus": 3, "state": hv_base.HvInstanceState.RUNNING, }, "inst4-uuid": { "memory": 123, "state": hv_base.HvInstanceState.RUNNING, }, "inst6-uuid": { "memory": 768, "state": hv_base.HvInstanceState.RUNNING, }, "inst7-uuid": { "vcpus": 3, "state": hv_base.HvInstanceState.RUNNING, }, "inst9-uuid": { "vcpus": 3, "state": hv_base.HvInstanceState.SHUTDOWN, }, } wrongnode_inst = set(["inst7-uuid"]) consinfo = dict((inst.uuid, None) for inst in instances) consinfo["inst7-uuid"] = \ objects.InstanceConsole(instance="inst7", kind=constants.CONS_SSH, host=instbyname["inst7"].primary_node, user="root", command=["hostname"]).ToDict() nodes = dict([(uuid, objects.Node( name="%s.example.com" % uuid, uuid=uuid, group="default-uuid")) for uuid in node_uuids]) iqd = query.InstanceQueryData(instances, cluster, disk_usage, offline_nodes, bad_nodes, live_data, wrongnode_inst, consinfo, nodes, {}, {}) result = q.Query(iqd) self.assertEqual(len(result), len(instances)) self.assertTrue(compat.all(len(row) == len(selected) for row in result)) assert len(set(bad_nodes) & set(offline_nodes)) == len(offline_nodes), \ "Offline nodes not included in bad nodes" tested_status = set() for (inst, row) in zip(instances, result): assert inst.primary_node in node_uuids self.assertEqual(row[fieldidx["name"]], (constants.RS_NORMAL, inst.name)) if inst.primary_node in offline_nodes: exp_status = constants.INSTST_NODEOFFLINE elif inst.primary_node in bad_nodes: exp_status = constants.INSTST_NODEDOWN elif inst.uuid in live_data: if inst.uuid in wrongnode_inst: exp_status = constants.INSTST_WRONGNODE else: instance_state = live_data[inst.uuid]["state"] if hv_base.HvInstanceState.IsShutdown(instance_state): if inst.admin_state == constants.ADMINST_UP: exp_status = constants.INSTST_USERDOWN else: exp_status = constants.INSTST_ADMINDOWN else: if inst.admin_state == constants.ADMINST_UP: exp_status = constants.INSTST_RUNNING else: exp_status = constants.INSTST_ERRORUP else: if inst.admin_state == constants.ADMINST_UP: exp_status = constants.INSTST_ERRORDOWN elif inst.admin_state == constants.ADMINST_DOWN: if inst.admin_state_source == constants.USER_SOURCE: exp_status = constants.INSTST_USERDOWN else: exp_status = constants.INSTST_ADMINDOWN else: exp_status = constants.INSTST_ADMINOFFLINE self.assertEqual(row[fieldidx["status"]], (constants.RS_NORMAL, exp_status)) (_, status) = row[fieldidx["status"]] tested_status.add(status) #FIXME(dynmem): check oper_ram vs min/max mem for (field, livefield) in [("oper_vcpus", "vcpus")]: if inst.primary_node in bad_nodes: exp = (constants.RS_NODATA, None) elif inst.uuid in live_data: value = live_data[inst.uuid].get(livefield, None) if value is None: exp = (constants.RS_UNAVAIL, None) else: exp = (constants.RS_NORMAL, value) else: exp = (constants.RS_UNAVAIL, None) self.assertEqual(row[fieldidx[field]], exp) bridges = inst_bridges.get(inst.uuid, []) self.assertEqual(row[fieldidx["nic.bridges"]], (constants.RS_NORMAL, bridges)) if bridges: self.assertEqual(row[fieldidx["bridge"]], (constants.RS_NORMAL, bridges[0])) else: self.assertEqual(row[fieldidx["bridge"]], (constants.RS_UNAVAIL, None)) for i in range(constants.MAX_NICS): if i < len(bridges) and bridges[i] is not None: exp = (constants.RS_NORMAL, bridges[i]) else: exp = (constants.RS_UNAVAIL, None) self.assertEqual(row[fieldidx["nic.bridge/%s" % i]], exp) if inst.primary_node in bad_nodes: exp = (constants.RS_NODATA, None) else: exp = (constants.RS_NORMAL, inst.uuid in live_data) self.assertEqual(row[fieldidx["oper_state"]], exp) cust_exp = (constants.RS_NORMAL, {}) if inst.os == "deb99": if inst.uuid == "inst6-uuid": exp = (constants.RS_NORMAL, {"clean_install": "no"}) cust_exp = exp else: exp = (constants.RS_NORMAL, {"clean_install": "yes"}) else: exp = (constants.RS_NORMAL, {}) self.assertEqual(row[fieldidx["osparams"]], exp) self.assertEqual(row[fieldidx["custom_osparams"]], cust_exp) usage = disk_usage[inst.uuid] if usage is None: usage = 0 self.assertEqual(row[fieldidx["disk_usage"]], (constants.RS_NORMAL, usage)) for alias, target in [("sda_size", "disk.size/0"), ("sdb_size", "disk.size/1"), ("vcpus", "be/vcpus"), ("ip", "nic.ip/0"), ("mac", "nic.mac/0"), ("bridge", "nic.bridge/0"), ("nic_mode", "nic.mode/0"), ("nic_link", "nic.link/0"), ]: self.assertEqual(row[fieldidx[alias]], row[fieldidx[target]]) for field in ["ctime", "mtime"]: if getattr(inst, field) is None: # No ctime/mtime exp = (constants.RS_UNAVAIL, None) else: exp = (constants.RS_NORMAL, getattr(inst, field)) self.assertEqual(row[fieldidx[field]], exp) self._CheckInstanceConsole(inst, row[fieldidx["console"]]) # Ensure all possible status' have been tested self.assertEqual(tested_status, set(constants.INSTST_ALL)) def _CheckInstanceConsole(self, instance, console_info): (status, consdata) = console_info if instance.name == "inst7": self.assertEqual(status, constants.RS_NORMAL) console = objects.InstanceConsole.FromDict(consdata) self.assertEqual(console.Validate(), None) self.assertEqual(console.host, instance.primary_node) else: self.assertEqual(status, constants.RS_UNAVAIL) class TestGroupQuery(unittest.TestCase): def setUp(self): self.custom_diskparams = { constants.DT_DRBD8: { constants.DRBD_DEFAULT_METAVG: "foobar", }, } self.groups = [ objects.NodeGroup(name="default", uuid="c0e89160-18e7-11e0-a46e-001d0904baeb", alloc_policy=constants.ALLOC_POLICY_PREFERRED, ipolicy=objects.MakeEmptyIPolicy(), ndparams={}, diskparams={}, ), objects.NodeGroup(name="restricted", uuid="d2a40a74-18e7-11e0-9143-001d0904baeb", alloc_policy=constants.ALLOC_POLICY_LAST_RESORT, ipolicy=objects.MakeEmptyIPolicy(), ndparams={}, diskparams=self.custom_diskparams, ), ] self.cluster = objects.Cluster(cluster_name="testcluster", hvparams=constants.HVC_DEFAULTS, beparams={ constants.PP_DEFAULT: constants.BEC_DEFAULTS, }, nicparams={ constants.PP_DEFAULT: constants.NICC_DEFAULTS, }, ndparams=constants.NDC_DEFAULTS, ipolicy=constants.IPOLICY_DEFAULTS, diskparams=constants.DISK_DT_DEFAULTS, ) def _Create(self, selected): return query.Query(query.GROUP_FIELDS, selected) def testSimple(self): q = self._Create(["name", "uuid", "alloc_policy"]) gqd = query.GroupQueryData(self.cluster, self.groups, None, None, False) self.assertEqual(q.RequestedData(), set([query.GQ_CONFIG])) self.assertEqual(q.Query(gqd), [[(constants.RS_NORMAL, "default"), (constants.RS_NORMAL, "c0e89160-18e7-11e0-a46e-001d0904baeb"), (constants.RS_NORMAL, constants.ALLOC_POLICY_PREFERRED) ], [(constants.RS_NORMAL, "restricted"), (constants.RS_NORMAL, "d2a40a74-18e7-11e0-9143-001d0904baeb"), (constants.RS_NORMAL, constants.ALLOC_POLICY_LAST_RESORT) ], ]) def testNodes(self): groups_to_nodes = { "c0e89160-18e7-11e0-a46e-001d0904baeb": ["node1", "node2"], "d2a40a74-18e7-11e0-9143-001d0904baeb": ["node1", "node10", "node9"], } q = self._Create(["name", "node_cnt", "node_list"]) gqd = query.GroupQueryData(self.cluster, self.groups, groups_to_nodes, None, False) self.assertEqual(q.RequestedData(), set([query.GQ_CONFIG, query.GQ_NODE])) self.assertEqual(q.Query(gqd), [[(constants.RS_NORMAL, "default"), (constants.RS_NORMAL, 2), (constants.RS_NORMAL, ["node1", "node2"]), ], [(constants.RS_NORMAL, "restricted"), (constants.RS_NORMAL, 3), (constants.RS_NORMAL, ["node1", "node9", "node10"]), ], ]) def testInstances(self): groups_to_instances = { "c0e89160-18e7-11e0-a46e-001d0904baeb": ["inst1", "inst2"], "d2a40a74-18e7-11e0-9143-001d0904baeb": ["inst1", "inst10", "inst9"], } q = self._Create(["pinst_cnt", "pinst_list"]) gqd = query.GroupQueryData(self.cluster, self.groups, None, groups_to_instances, False) self.assertEqual(q.RequestedData(), set([query.GQ_INST])) self.assertEqual(q.Query(gqd), [[(constants.RS_NORMAL, 2), (constants.RS_NORMAL, ["inst1", "inst2"]), ], [(constants.RS_NORMAL, 3), (constants.RS_NORMAL, ["inst1", "inst9", "inst10"]), ], ]) def testDiskparams(self): q = self._Create(["name", "uuid", "diskparams", "custom_diskparams"]) gqd = query.GroupQueryData(self.cluster, self.groups, None, None, True) self.assertEqual(q.RequestedData(), set([query.GQ_CONFIG, query.GQ_DISKPARAMS])) self.assertEqual(q.Query(gqd), [[(constants.RS_NORMAL, "default"), (constants.RS_NORMAL, "c0e89160-18e7-11e0-a46e-001d0904baeb"), (constants.RS_NORMAL, constants.DISK_DT_DEFAULTS), (constants.RS_NORMAL, {}), ], [(constants.RS_NORMAL, "restricted"), (constants.RS_NORMAL, "d2a40a74-18e7-11e0-9143-001d0904baeb"), (constants.RS_NORMAL, objects.FillDiskParams(constants.DISK_DT_DEFAULTS, self.custom_diskparams)), (constants.RS_NORMAL, self.custom_diskparams), ], ]) class TestOsQuery(unittest.TestCase): def _Create(self, selected): return query.Query(query.OS_FIELDS, selected) def test(self): variants = ["v00", "plain", "v3", "var0", "v33", "v20"] api_versions = [10, 0, 15, 5] parameters = ["zpar3", "apar9"] os_hvps = { "os+variant1": { "kvm": { "acpi": False, "migration_downtime": 35,} }, "os+variant2": { "xen": { "acpi": "noirq", "console": "com1",} }, } osparameters = { "os+variant3": { "img_id": "Debian", "img_passwd": "1234", "img_format": "diskdump", }, } assert variants != sorted(variants) and variants != utils.NiceSort(variants) assert (api_versions != sorted(api_versions) and api_versions != utils.NiceSort(variants)) assert (parameters != sorted(parameters) and parameters != utils.NiceSort(parameters)) data = [ query.OsInfo(name="debian", valid=False, hidden=False, blacklisted=False, variants=set(), api_versions=set(), parameters=set(), node_status={ "some": "status", }, os_hvp={}, osparams={}), query.OsInfo(name="dos", valid=True, hidden=False, blacklisted=True, variants=set(variants), api_versions=set(api_versions), parameters=set(parameters), node_status={ "some": "other", "status": None, }, os_hvp=os_hvps, osparams=osparameters), ] q = self._Create(["name", "valid", "hidden", "blacklisted", "variants", "api_versions", "parameters", "node_status", "os_hvp", "osparams"]) self.assertEqual(q.RequestedData(), set([])) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "debian"), (constants.RS_NORMAL, False), (constants.RS_NORMAL, False), (constants.RS_NORMAL, False), (constants.RS_NORMAL, []), (constants.RS_NORMAL, []), (constants.RS_NORMAL, []), (constants.RS_NORMAL, {"some": "status"}), (constants.RS_NORMAL, {}), (constants.RS_NORMAL, {})], [(constants.RS_NORMAL, "dos"), (constants.RS_NORMAL, True), (constants.RS_NORMAL, False), (constants.RS_NORMAL, True), (constants.RS_NORMAL, ["plain", "v00", "v3", "v20", "v33", "var0"]), (constants.RS_NORMAL, [0, 5, 10, 15]), (constants.RS_NORMAL, ["apar9", "zpar3"]), (constants.RS_NORMAL, { "some": "other", "status": None, }), (constants.RS_NORMAL, os_hvps), (constants.RS_NORMAL, osparameters) ]]) class TestQueryFields(unittest.TestCase): def testAllFields(self): for fielddefs in query.ALL_FIELD_LISTS: result = query.QueryFields(fielddefs, None) self.assertTrue(isinstance(result, dict)) response = objects.QueryFieldsResponse.FromDict(result) self.assertEqual([(fdef.name, fdef.title) for fdef in response.fields], [(fdef2.name, fdef2.title) for (fdef2, _, _, _) in utils.NiceSort(fielddefs.values(), key=lambda x: x[0].name)]) def testSomeFields(self): rnd = random.Random(5357) for _ in range(10): for fielddefs in query.ALL_FIELD_LISTS: if len(fielddefs) > 20: sample_size = rnd.randint(5, 20) else: sample_size = rnd.randint(1, max(1, len(fielddefs) - 1)) fields = [fdef for (fdef, _, _, _) in rnd.sample(list(fielddefs.values()), sample_size)] result = query.QueryFields(fielddefs, [fdef.name for fdef in fields]) self.assertTrue(isinstance(result, dict)) response = objects.QueryFieldsResponse.FromDict(result) self.assertEqual([(fdef.name, fdef.title) for fdef in response.fields], [(fdef2.name, fdef2.title) for fdef2 in fields]) class TestQueryFilter(unittest.TestCase): def testRequestedNames(self): for (what, fielddefs) in query.ALL_FIELDS.items(): if what == constants.QR_JOB: namefield = "id" nameval = 123 namevalempty = 0 genval = lambda i: i * 10 randvals = [17361, 22015, 13193, 15215] else: nameval = "abc" namevalempty = "" genval = lambda i: "x%s" % i randvals = ["x17361", "x22015", "x13193", "x15215"] namefield = { constants.QR_EXPORT: "export", constants.QR_FILTER: "uuid", }.get(what, "name") assert namefield in fielddefs reqnames = [genval(i) for i in range(4)] innerfilter = [["=", namefield, v] for v in reqnames] # No name field q = query.Query(fielddefs, [namefield], qfilter=["=", namefield, nameval], namefield=None) self.assertEqual(q.RequestedNames(), None) # No filter q = query.Query(fielddefs, [namefield], qfilter=None, namefield=namefield) self.assertEqual(q.RequestedNames(), None) # Check empty query q = query.Query(fielddefs, [namefield], qfilter=["|"], namefield=namefield) self.assertEqual(q.RequestedNames(), None) # Check order q = query.Query(fielddefs, [namefield], qfilter=["|"] + innerfilter, namefield=namefield) self.assertEqual(q.RequestedNames(), reqnames) # Check reverse order q = query.Query(fielddefs, [namefield], qfilter=["|"] + list(reversed(innerfilter)), namefield=namefield) self.assertEqual(q.RequestedNames(), list(reversed(reqnames))) # Duplicates q = query.Query(fielddefs, [namefield], qfilter=["|"] + innerfilter + list(reversed(innerfilter)), namefield=namefield) self.assertEqual(q.RequestedNames(), reqnames) # Unknown name field self.assertRaises(AssertionError, query.Query, fielddefs, [namefield], namefield="_unknown_field_") # Filter with AND q = query.Query(fielddefs, [namefield], qfilter=["|", ["=", namefield, nameval], ["&", ["=", namefield, namevalempty]]], namefield=namefield) self.assertTrue(q.RequestedNames() is None) # Filter with NOT q = query.Query(fielddefs, [namefield], qfilter=["|", ["=", namefield, nameval], ["!", ["=", namefield, namevalempty]]], namefield=namefield) self.assertTrue(q.RequestedNames() is None) # Filter with only OR (names must be in correct order) q = query.Query(fielddefs, [namefield], qfilter=["|", ["=", namefield, randvals[0]], ["|", ["=", namefield, randvals[1]]], ["|", ["|", ["=", namefield, randvals[2]]]], ["=", namefield, randvals[3]]], namefield=namefield) self.assertEqual(q.RequestedNames(), randvals) @staticmethod def _GenNestedFilter(namefield, op, depth, nameval): nested = ["=", namefield, nameval] for i in range(depth): nested = [op, nested] return nested def testCompileFilter(self): levels_max = query._FilterCompilerHelper._LEVELS_MAX for (what, fielddefs) in query.ALL_FIELDS.items(): namefield, nameval = { constants.QR_JOB: ("id", 123), constants.QR_EXPORT: ("export", "value"), constants.QR_FILTER: ("uuid", str(uuid_module.uuid4())), }.get(what, ("name", "value")) checks = [ [], ["="], ["=", "foo"], ["unknownop"], ["!"], ["=", "_unknown_field", "value"], self._GenNestedFilter(namefield, "|", levels_max, nameval), self._GenNestedFilter(namefield, "|", levels_max * 3, nameval), self._GenNestedFilter(namefield, "!", levels_max, nameval), ] for qfilter in checks: self.assertRaises(errors.ParameterError, query._CompileFilter, fielddefs, None, qfilter) for op in ["|", "!"]: qfilter = self._GenNestedFilter(namefield, op, levels_max - 1, nameval) self.assertTrue(callable(query._CompileFilter(fielddefs, None, qfilter))) def testQueryInputOrder(self): fielddefs = query._PrepareFieldList([ (query._MakeField("pnode", "PNode", constants.QFT_TEXT, "Primary"), None, 0, lambda ctx, item: item["pnode"]), (query._MakeField("snode", "SNode", constants.QFT_TEXT, "Secondary"), None, 0, lambda ctx, item: item["snode"]), ], []) data = [ { "pnode": "node1", "snode": "node44", }, { "pnode": "node30", "snode": "node90", }, { "pnode": "node25", "snode": "node1", }, { "pnode": "node20", "snode": "node1", }, ] qfilter = ["|", ["=", "pnode", "node1"], ["=", "snode", "node1"]] q = query.Query(fielddefs, ["pnode", "snode"], namefield="pnode", qfilter=qfilter) self.assertTrue(q.RequestedNames() is None) self.assertFalse(q.RequestedData()) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")], [(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")], [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")]]) # Try again with reversed input data self.assertEqual(q.Query(reversed(data)), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")], [(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")], [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")]]) # No name field, result must be in incoming order q = query.Query(fielddefs, ["pnode", "snode"], namefield=None, qfilter=qfilter) self.assertFalse(q.RequestedData()) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")], [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")], [(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")]]) self.assertEqual(q.OldStyleQuery(data), [ ["node1", "node44"], ["node25", "node1"], ["node20", "node1"], ]) self.assertEqual(q.Query(reversed(data)), [[(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")], [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")], [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")]]) self.assertEqual(q.OldStyleQuery(reversed(data)), [ ["node20", "node1"], ["node25", "node1"], ["node1", "node44"], ]) # Name field, but no sorting, result must be in incoming order q = query.Query(fielddefs, ["pnode", "snode"], namefield="pnode") self.assertFalse(q.RequestedData()) self.assertEqual(q.Query(data, sort_by_name=False), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")], [(constants.RS_NORMAL, "node30"), (constants.RS_NORMAL, "node90")], [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")], [(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")]]) self.assertEqual(q.OldStyleQuery(data, sort_by_name=False), [ ["node1", "node44"], ["node30", "node90"], ["node25", "node1"], ["node20", "node1"], ]) self.assertEqual(q.Query(reversed(data), sort_by_name=False), [[(constants.RS_NORMAL, "node20"), (constants.RS_NORMAL, "node1")], [(constants.RS_NORMAL, "node25"), (constants.RS_NORMAL, "node1")], [(constants.RS_NORMAL, "node30"), (constants.RS_NORMAL, "node90")], [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "node44")]]) self.assertEqual(q.OldStyleQuery(reversed(data), sort_by_name=False), [ ["node20", "node1"], ["node25", "node1"], ["node30", "node90"], ["node1", "node44"], ]) def testEqualNamesOrder(self): fielddefs = query._PrepareFieldList([ (query._MakeField("pnode", "PNode", constants.QFT_TEXT, "Primary"), None, 0, lambda ctx, item: item["pnode"]), (query._MakeField("num", "Num", constants.QFT_NUMBER, "Num"), None, 0, lambda ctx, item: item["num"]), ], []) data = [ { "pnode": "node1", "num": 100, }, { "pnode": "node1", "num": 25, }, { "pnode": "node2", "num": 90, }, { "pnode": "node2", "num": 30, }, ] q = query.Query(fielddefs, ["pnode", "num"], namefield="pnode", qfilter=["|", ["=", "pnode", "node1"], ["=", "pnode", "node2"], ["=", "pnode", "node1"]]) self.assertEqual(q.RequestedNames(), ["node1", "node2"], msg="Did not return unique names") self.assertFalse(q.RequestedData()) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, 100)], [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, 25)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, 90)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, 30)]]) self.assertEqual(q.Query(data, sort_by_name=False), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, 100)], [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, 25)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, 90)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, 30)]]) data = [ { "pnode": "nodeX", "num": 50, }, { "pnode": "nodeY", "num": 40, }, { "pnode": "nodeX", "num": 30, }, { "pnode": "nodeX", "num": 20, }, { "pnode": "nodeM", "num": 10, }, ] q = query.Query(fielddefs, ["pnode", "num"], namefield="pnode", qfilter=["|", ["=", "pnode", "nodeX"], ["=", "pnode", "nodeY"], ["=", "pnode", "nodeY"], ["=", "pnode", "nodeY"], ["=", "pnode", "nodeM"]]) self.assertEqual(q.RequestedNames(), ["nodeX", "nodeY", "nodeM"], msg="Did not return unique names") self.assertFalse(q.RequestedData()) # First sorted by name, then input order self.assertEqual(q.Query(data, sort_by_name=True), [[(constants.RS_NORMAL, "nodeM"), (constants.RS_NORMAL, 10)], [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 50)], [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 30)], [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 20)], [(constants.RS_NORMAL, "nodeY"), (constants.RS_NORMAL, 40)]]) # Input order self.assertEqual(q.Query(data, sort_by_name=False), [[(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 50)], [(constants.RS_NORMAL, "nodeY"), (constants.RS_NORMAL, 40)], [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 30)], [(constants.RS_NORMAL, "nodeX"), (constants.RS_NORMAL, 20)], [(constants.RS_NORMAL, "nodeM"), (constants.RS_NORMAL, 10)]]) def testFilter(self): (DK_A, DK_B) = range(1000, 1002) fielddefs = query._PrepareFieldList([ (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"), DK_A, 0, lambda ctx, item: item["name"]), (query._MakeField("other", "Other", constants.QFT_TEXT, "Other"), DK_B, 0, lambda ctx, item: item["other"]), ], []) data = [ { "name": "node1", "other": "foo", }, { "name": "node2", "other": "bar", }, { "name": "node3", "other": "Hello", }, ] # Empty filter q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=["|"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.RequestedData(), set([DK_A, DK_B])) self.assertEqual(q.Query(data), []) # Normal filter q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=["=", "name", "node1"]) self.assertEqual(q.RequestedNames(), ["node1"]) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "foo")]]) q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=(["|", ["=", "name", "node1"], ["=", "name", "node3"]])) self.assertEqual(q.RequestedNames(), ["node1", "node3"]) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "foo")], [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, "Hello")]]) # Complex filter q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=(["|", ["=", "name", "node1"], ["|", ["=", "name", "node3"], ["=", "name", "node2"]], ["=", "name", "node3"]])) self.assertEqual(q.RequestedNames(), ["node1", "node3", "node2"]) self.assertEqual(q.RequestedData(), set([DK_A, DK_B])) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "foo")], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, "bar")], [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, "Hello")]]) # Filter data type mismatch for i in [-1, 0, 1, 123, [], None, True, False]: self.assertRaises(errors.ParameterError, query.Query, fielddefs, ["name", "other"], namefield="name", qfilter=["=", "name", i]) # Negative filter q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=["!", ["|", ["=", "name", "node1"], ["=", "name", "node3"]]]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, "bar")]]) # Not equal q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=["!=", "name", "node3"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, "foo")], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, "bar")]]) # Data type q = query.Query(fielddefs, [], namefield="name", qfilter=["|", ["=", "other", "bar"], ["=", "name", "foo"]]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.RequestedData(), set([DK_A, DK_B])) self.assertEqual(q.Query(data), [[]]) # Only one data type q = query.Query(fielddefs, ["other"], namefield="name", qfilter=["=", "other", "bar"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.RequestedData(), set([DK_B])) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, "bar")]]) q = query.Query(fielddefs, [], namefield="name", qfilter=["=", "other", "bar"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.RequestedData(), set([DK_B])) self.assertEqual(q.Query(data), [[]]) # Data type in boolean operator q = query.Query(fielddefs, [], namefield="name", qfilter=["?", "name"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.RequestedData(), set([DK_A])) self.assertEqual(q.Query(data), [[], [], []]) q = query.Query(fielddefs, [], namefield="name", qfilter=["!", ["?", "name"]]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.RequestedData(), set([DK_A])) self.assertEqual(q.Query(data), []) def testFilterContains(self): fielddefs = query._PrepareFieldList([ (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"), None, 0, lambda ctx, item: item["name"]), (query._MakeField("other", "Other", constants.QFT_OTHER, "Other"), None, 0, lambda ctx, item: item["other"]), ], []) data = [ { "name": "node2", "other": ["x", "y", "bar"], }, { "name": "node3", "other": "Hello", }, { "name": "node1", "other": ["a", "b", "foo"], }, { "name": "empty", "other": []}, ] q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=["=[]", "other", "bar"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, ["x", "y", "bar"])], ]) q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=["|", ["=[]", "other", "bar"], ["=[]", "other", "a"], ["=[]", "other", "b"]]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, ["a", "b", "foo"])], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, ["x", "y", "bar"])], ]) self.assertEqual(q.OldStyleQuery(data), [ ["node1", ["a", "b", "foo"]], ["node2", ["x", "y", "bar"]], ]) # Boolean test q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=["?", "other"]) self.assertEqual(q.OldStyleQuery(data), [ ["node1", ["a", "b", "foo"]], ["node2", ["x", "y", "bar"]], ["node3", "Hello"], ]) q = query.Query(fielddefs, ["name", "other"], namefield="name", qfilter=["!", ["?", "other"]]) self.assertEqual(q.OldStyleQuery(data), [ ["empty", []], ]) def testFilterHostname(self): fielddefs = query._PrepareFieldList([ (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"), None, query.QFF_HOSTNAME, lambda ctx, item: item["name"]), ], []) data = [ { "name": "node1.example.com", }, { "name": "node2.example.com", }, { "name": "node2.example.net", }, ] q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["=", "name", "node2"]) self.assertEqual(q.RequestedNames(), ["node2"]) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node2.example.com")], [(constants.RS_NORMAL, "node2.example.net")], ]) q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["=", "name", "node1"]) self.assertEqual(q.RequestedNames(), ["node1"]) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node1.example.com")], ]) q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["=", "name", "othername"]) self.assertEqual(q.RequestedNames(), ["othername"]) self.assertEqual(q.Query(data), []) q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["|", ["=", "name", "node1.example.com"], ["=", "name", "node2"]]) self.assertEqual(q.RequestedNames(), ["node1.example.com", "node2"]) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node1.example.com")], [(constants.RS_NORMAL, "node2.example.com")], [(constants.RS_NORMAL, "node2.example.net")], ]) self.assertEqual(q.OldStyleQuery(data), [ ["node1.example.com"], ["node2.example.com"], ["node2.example.net"], ]) q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["!=", "name", "node1"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node2.example.com")], [(constants.RS_NORMAL, "node2.example.net")], ]) self.assertEqual(q.OldStyleQuery(data), [ ["node2.example.com"], ["node2.example.net"], ]) def testFilterBoolean(self): fielddefs = query._PrepareFieldList([ (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"), None, query.QFF_HOSTNAME, lambda ctx, item: item["name"]), (query._MakeField("value", "Value", constants.QFT_BOOL, "Value"), None, 0, lambda ctx, item: item["value"]), ], []) data = [ { "name": "node1", "value": False, }, { "name": "node2", "value": True, }, { "name": "node3", "value": True, }, ] q = query.Query(fielddefs, ["name", "value"], qfilter=["|", ["=", "value", False], ["=", "value", True]]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True)], [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, True)], ]) q = query.Query(fielddefs, ["name", "value"], qfilter=["|", ["=", "value", False], ["!", ["=", "value", False]]]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True)], [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, True)], ]) # Comparing bool with string for i in ["False", "True", "0", "1", "no", "yes", "N", "Y"]: self.assertRaises(errors.ParameterError, query.Query, fielddefs, ["name", "value"], qfilter=["=", "value", i]) # Truth filter q = query.Query(fielddefs, ["name", "value"], qfilter=["?", "value"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True)], [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, True)], ]) # Negative bool filter q = query.Query(fielddefs, ["name", "value"], qfilter=["!", ["?", "value"]]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)], ]) # Complex truth filter q = query.Query(fielddefs, ["name", "value"], qfilter=["|", ["&", ["=", "name", "node1"], ["!", ["?", "value"]]], ["?", "value"]]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node1"), (constants.RS_NORMAL, False)], [(constants.RS_NORMAL, "node2"), (constants.RS_NORMAL, True)], [(constants.RS_NORMAL, "node3"), (constants.RS_NORMAL, True)], ]) def testFilterRegex(self): fielddefs = query._PrepareFieldList([ (query._MakeField("name", "Name", constants.QFT_TEXT, "Name"), None, 0, lambda ctx, item: item["name"]), ], []) data = [ { "name": "node1.example.com", }, { "name": "node2.site.example.com", }, { "name": "node2.example.net", }, # Empty name { "name": "", }, ] q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["=~", "name", "site"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node2.site.example.com")], ]) q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["=~", "name", "^node2"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node2.example.net")], [(constants.RS_NORMAL, "node2.site.example.com")], ]) q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["=~", "name", r"(?i)\.COM$"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node1.example.com")], [(constants.RS_NORMAL, "node2.site.example.com")], ]) q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["=~", "name", r"."]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "node1.example.com")], [(constants.RS_NORMAL, "node2.example.net")], [(constants.RS_NORMAL, "node2.site.example.com")], ]) q = query.Query(fielddefs, ["name"], namefield="name", qfilter=["=~", "name", r"^$"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "")], ]) # Invalid regular expression self.assertRaises(errors.ParameterError, query.Query, fielddefs, ["name"], qfilter=["=~", "name", r"["]) def testFilterLessGreater(self): fielddefs = query._PrepareFieldList([ (query._MakeField("value", "Value", constants.QFT_NUMBER, "Value"), None, 0, lambda ctx, item: item), ], []) data = range(100) q = query.Query(fielddefs, ["value"], qfilter=["<", "value", 20]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, i)] for i in range(20)]) q = query.Query(fielddefs, ["value"], qfilter=["<=", "value", 30]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, i)] for i in range(31)]) q = query.Query(fielddefs, ["value"], qfilter=[">", "value", 40]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, i)] for i in range(41, 100)]) q = query.Query(fielddefs, ["value"], qfilter=[">=", "value", 50]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [[(constants.RS_NORMAL, i)] for i in range(50, 100)]) def testFilterLessGreaterJobId(self): fielddefs = query._PrepareFieldList([ (query._MakeField("id", "ID", constants.QFT_TEXT, "Job ID"), None, query.QFF_JOB_ID, lambda ctx, item: item), ], []) data = ["1", "2", "3", "10", "102", "120", "125", "15", "100", "7"] assert data != utils.NiceSort(data), "Test data should not be sorted" q = query.Query(fielddefs, ["id"], qfilter=["<", "id", "20"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "1")], [(constants.RS_NORMAL, "2")], [(constants.RS_NORMAL, "3")], [(constants.RS_NORMAL, "10")], [(constants.RS_NORMAL, "15")], [(constants.RS_NORMAL, "7")], ]) q = query.Query(fielddefs, ["id"], qfilter=[">=", "id", "100"]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, "102")], [(constants.RS_NORMAL, "120")], [(constants.RS_NORMAL, "125")], [(constants.RS_NORMAL, "100")], ]) # Integers are no valid job IDs self.assertRaises(errors.ParameterError, query.Query, fielddefs, ["id"], qfilter=[">=", "id", 10]) def testFilterLessGreaterSplitTimestamp(self): fielddefs = query._PrepareFieldList([ (query._MakeField("ts", "Timestamp", constants.QFT_OTHER, "Timestamp"), None, query.QFF_SPLIT_TIMESTAMP, lambda ctx, item: item), ], []) data = [ utils.SplitTime(0), utils.SplitTime(0.1), utils.SplitTime(18224.7872), utils.SplitTime(919896.12623), utils.SplitTime(999), utils.SplitTime(989.9999), ] for i in [0, [0, 0]]: q = query.Query(fielddefs, ["ts"], qfilter=["<", "ts", i]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), []) q = query.Query(fielddefs, ["ts"], qfilter=["<", "ts", 1000]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, (0, 0))], [(constants.RS_NORMAL, (0, 100000))], [(constants.RS_NORMAL, (999, 0))], [(constants.RS_NORMAL, (989, 999900))], ]) q = query.Query(fielddefs, ["ts"], qfilter=[">=", "ts", 5000.3]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, (18224, 787200))], [(constants.RS_NORMAL, (919896, 126230))], ]) for i in [18224.7772, utils.SplitTime(18224.7772)]: q = query.Query(fielddefs, ["ts"], qfilter=[">=", "ts", i]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, (18224, 787200))], [(constants.RS_NORMAL, (919896, 126230))], ]) q = query.Query(fielddefs, ["ts"], qfilter=[">", "ts", 18224.7880]) self.assertTrue(q.RequestedNames() is None) self.assertEqual(q.Query(data), [ [(constants.RS_NORMAL, (919896, 126230))], ]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.rapi.baserlib_unittest.py000075500000000000000000000141621476477700300247620ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.rapi.baserlib""" import unittest import itertools from ganeti import errors from ganeti import opcodes from ganeti import ht from ganeti import http from ganeti import compat from ganeti.rapi import baserlib import testutils class TestFillOpcode(unittest.TestCase): class OpTest(opcodes.OpCode): OP_PARAMS = [ ("test", None, ht.TMaybeString, None), ] def test(self): for static in [None, {}]: op = baserlib.FillOpcode(self.OpTest, {}, static) self.assertTrue(isinstance(op, self.OpTest)) self.assertTrue(op.test is None) def testStatic(self): op = baserlib.FillOpcode(self.OpTest, {}, {"test": "abc"}) self.assertTrue(isinstance(op, self.OpTest)) self.assertEqual(op.test, "abc") # Overwrite static parameter self.assertRaises(http.HttpBadRequest, baserlib.FillOpcode, self.OpTest, {"test": 123}, {"test": "abc"}) def testType(self): self.assertRaises(http.HttpBadRequest, baserlib.FillOpcode, self.OpTest, {"test": [1, 2, 3]}, {}) def testStaticType(self): self.assertRaises(http.HttpBadRequest, baserlib.FillOpcode, self.OpTest, {}, {"test": [1, 2, 3]}) def testUnicode(self): op = baserlib.FillOpcode(self.OpTest, {"test": "abc"}, {}) self.assertTrue(isinstance(op, self.OpTest)) self.assertEqual(op.test, "abc") op = baserlib.FillOpcode(self.OpTest, {}, {"test": "abc"}) self.assertTrue(isinstance(op, self.OpTest)) self.assertEqual(op.test, "abc") def testUnknownParameter(self): self.assertRaises(http.HttpBadRequest, baserlib.FillOpcode, self.OpTest, {"othervalue": 123}, None) def testInvalidBody(self): self.assertRaises(http.HttpBadRequest, baserlib.FillOpcode, self.OpTest, "", None) self.assertRaises(http.HttpBadRequest, baserlib.FillOpcode, self.OpTest, range(10), None) def testRenameBothSpecified(self): self.assertRaises(http.HttpBadRequest, baserlib.FillOpcode, self.OpTest, { "old": 123, "new": 999, }, None, rename={ "old": "new", }) def testRename(self): value = "Hello World" op = baserlib.FillOpcode(self.OpTest, { "data": value, }, None, rename={ "data": "test", }) self.assertEqual(op.test, value) def testRenameStatic(self): self.assertRaises(http.HttpBadRequest, baserlib.FillOpcode, self.OpTest, { "data": 0, }, { "test": None, }, rename={ "data": "test", }) class TestOpcodeResource(unittest.TestCase): @staticmethod def _MakeClass(method, attrs): return type("Test%s" % method, (baserlib.OpcodeResource, ), attrs) @staticmethod def _GetMethodAttributes(method): attrs = ["%s_OPCODE" % method, "%s_RENAME" % method, "%s_ALIASES" % method, "%s_FORBIDDEN" % method, "Get%sOpInput" % method.capitalize()] assert attrs == dict((opattrs.method, opattrs.GetModifiers()) for opattrs in baserlib.OPCODE_ATTRS)[method] return attrs def test(self): for method in baserlib._SUPPORTED_METHODS: # Empty handler obj = self._MakeClass(method, {})(None, {}, None) for m_attr in baserlib.OPCODE_ATTRS: for attr in m_attr.GetAll(): self.assertFalse(hasattr(obj, attr)) # Direct handler function obj = self._MakeClass(method, { method: lambda _: None, })(None, {}, None) self.assertFalse(compat.all(hasattr(obj, attr) for i in baserlib._SUPPORTED_METHODS for attr in self._GetMethodAttributes(i))) # Let metaclass define handler function for opcls in [None, object()]: obj = self._MakeClass(method, { "%s_OPCODE" % method: opcls, })(None, {}, None) self.assertTrue(callable(getattr(obj, method))) self.assertEqual(getattr(obj, "%s_OPCODE" % method), opcls) self.assertFalse(hasattr(obj, "%s_RENAME" % method)) self.assertFalse(compat.any(hasattr(obj, attr) for i in baserlib._SUPPORTED_METHODS if i != method for attr in self._GetMethodAttributes(i))) def testIllegalRename(self): class _TClass(baserlib.OpcodeResource): PUT_RENAME = None def PUT(self): pass self.assertRaises(AssertionError, _TClass, None, None, None) def testEmpty(self): class _Empty(baserlib.OpcodeResource): pass obj = _Empty(None, {}, None) for m_attr in baserlib.OPCODE_ATTRS: for attr in m_attr.GetAll(): self.assertFalse(hasattr(obj, attr)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.rapi.client_unittest.py000075500000000000000000001746241476477700300244670ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the RAPI client module""" import unittest import warnings import pycurl from ganeti import opcodes from ganeti import constants from ganeti import http from ganeti import serializer from ganeti import utils from ganeti import query from ganeti import objects from ganeti import rapi from ganeti import errors import ganeti.rapi.testutils from ganeti.rapi import connector from ganeti.rapi import rlib2 from ganeti.rapi import client import testutils # List of resource handlers which aren't used by the RAPI client _KNOWN_UNUSED = set([ rlib2.R_root, rlib2.R_2, ]) # Global variable for collecting used handlers _used_handlers = None class RapiMock(object): def __init__(self): self._mapper = connector.Mapper() self._responses = [] self._last_handler = None self._last_req_data = None def ResetResponses(self): del self._responses[:] def AddResponse(self, response, code=200): self._responses.insert(0, (code, response)) def CountPending(self): return len(self._responses) def GetLastHandler(self): return self._last_handler def GetLastRequestData(self): return self._last_req_data def FetchResponse(self, path, method, headers, request_body): self._last_req_data = request_body try: (handler_cls, items, args) = self._mapper.getController(path) # Record handler as used _used_handlers.add(handler_cls) self._last_handler = handler_cls(items, args, None) if not hasattr(self._last_handler, method.upper()): raise http.HttpNotImplemented(message="Method not implemented") except http.HttpException as ex: code = ex.code response = ex.message else: if not self._responses: raise Exception("No responses") (code, response) = self._responses.pop() return (code, NotImplemented, response) class TestConstants(unittest.TestCase): def test(self): self.assertEqual(client.GANETI_RAPI_PORT, constants.DEFAULT_RAPI_PORT) self.assertEqual(client.GANETI_RAPI_VERSION, constants.RAPI_VERSION) self.assertEqual(client.HTTP_APP_JSON, http.HTTP_APP_JSON) self.assertEqual(client._REQ_DATA_VERSION_FIELD, rlib2._REQ_DATA_VERSION) self.assertEqual(client.JOB_STATUS_QUEUED, constants.JOB_STATUS_QUEUED) self.assertEqual(client.JOB_STATUS_WAITING, constants.JOB_STATUS_WAITING) self.assertEqual(client.JOB_STATUS_CANCELING, constants.JOB_STATUS_CANCELING) self.assertEqual(client.JOB_STATUS_RUNNING, constants.JOB_STATUS_RUNNING) self.assertEqual(client.JOB_STATUS_CANCELED, constants.JOB_STATUS_CANCELED) self.assertEqual(client.JOB_STATUS_SUCCESS, constants.JOB_STATUS_SUCCESS) self.assertEqual(client.JOB_STATUS_ERROR, constants.JOB_STATUS_ERROR) self.assertEqual(client.JOB_STATUS_PENDING, constants.JOBS_PENDING) self.assertEqual(client.JOB_STATUS_FINALIZED, constants.JOBS_FINALIZED) self.assertEqual(client.JOB_STATUS_ALL, constants.JOB_STATUS_ALL) # Node evacuation self.assertEqual(client.NODE_EVAC_PRI, constants.NODE_EVAC_PRI) self.assertEqual(client.NODE_EVAC_SEC, constants.NODE_EVAC_SEC) self.assertEqual(client.NODE_EVAC_ALL, constants.NODE_EVAC_ALL) # Legacy name self.assertEqual(client.JOB_STATUS_WAITLOCK, constants.JOB_STATUS_WAITING) # RAPI feature strings self.assertEqual(client._INST_CREATE_REQV1, rlib2._INST_CREATE_REQV1) self.assertEqual(client.INST_CREATE_REQV1, rlib2._INST_CREATE_REQV1) self.assertEqual(client._INST_REINSTALL_REQV1, rlib2._INST_REINSTALL_REQV1) self.assertEqual(client.INST_REINSTALL_REQV1, rlib2._INST_REINSTALL_REQV1) self.assertEqual(client._NODE_MIGRATE_REQV1, rlib2._NODE_MIGRATE_REQV1) self.assertEqual(client.NODE_MIGRATE_REQV1, rlib2._NODE_MIGRATE_REQV1) self.assertEqual(client._NODE_EVAC_RES1, rlib2._NODE_EVAC_RES1) self.assertEqual(client.NODE_EVAC_RES1, rlib2._NODE_EVAC_RES1) # Error codes self.assertEqual(client.ECODE_RESOLVER, errors.ECODE_RESOLVER) self.assertEqual(client.ECODE_NORES, errors.ECODE_NORES) self.assertEqual(client.ECODE_TEMP_NORES, errors.ECODE_TEMP_NORES) self.assertEqual(client.ECODE_INVAL, errors.ECODE_INVAL) self.assertEqual(client.ECODE_STATE, errors.ECODE_STATE) self.assertEqual(client.ECODE_NOENT, errors.ECODE_NOENT) self.assertEqual(client.ECODE_EXISTS, errors.ECODE_EXISTS) self.assertEqual(client.ECODE_NOTUNIQUE, errors.ECODE_NOTUNIQUE) self.assertEqual(client.ECODE_FAULT, errors.ECODE_FAULT) self.assertEqual(client.ECODE_ENVIRON, errors.ECODE_ENVIRON) def testErrors(self): self.assertEqual(client.ECODE_ALL, errors.ECODE_ALL) # Make sure all error codes are in both RAPI client and errors module for name in [s for s in dir(client) if (s.startswith("ECODE_") and s != "ECODE_ALL")]: value = getattr(client, name) self.assertEqual(value, getattr(errors, name)) self.assertTrue(value in client.ECODE_ALL) self.assertTrue(value in errors.ECODE_ALL) class RapiMockTest(unittest.TestCase): def test404(self): (code, _, body) = RapiMock().FetchResponse("/foo", "GET", None, None) self.assertEqual(code, 404) self.assertTrue(body is None) def test501(self): (code, _, body) = RapiMock().FetchResponse("/version", "POST", None, None) self.assertEqual(code, 501) self.assertEqual(body, "Method not implemented") def test200(self): rapi = RapiMock() rapi.AddResponse("2") (code, _, response) = rapi.FetchResponse("/version", "GET", None, None) self.assertEqual(200, code) self.assertEqual("2", response) self.assertTrue(isinstance(rapi.GetLastHandler(), rlib2.R_version)) def _FakeNoSslPycurlVersion(): # Note: incomplete version tuple return (3, "7.16.0", 462848, "mysystem", 1581, None, 0) def _FakeFancySslPycurlVersion(): # Note: incomplete version tuple return (3, "7.16.0", 462848, "mysystem", 1581, "FancySSL/1.2.3", 0) def _FakeOpenSslPycurlVersion(): # Note: incomplete version tuple return (2, "7.15.5", 462597, "othersystem", 668, "OpenSSL/0.9.8c", 0) def _FakeGnuTlsPycurlVersion(): # Note: incomplete version tuple return (3, "7.18.0", 463360, "somesystem", 1581, "GnuTLS/2.0.4", 0) class TestExtendedConfig(unittest.TestCase): def testAuth(self): cl = client.GanetiRapiClient("master.example.com", username="user", password="pw", curl_factory=lambda: rapi.testutils.FakeCurl(RapiMock())) curl = cl._CreateCurl() self.assertEqual(curl.getopt(pycurl.HTTPAUTH), pycurl.HTTPAUTH_BASIC) self.assertEqual(curl.getopt(pycurl.USERPWD), "user:pw") def testInvalidAuth(self): # No username self.assertRaises(client.Error, client.GanetiRapiClient, "master-a.example.com", password="pw") # No password self.assertRaises(client.Error, client.GanetiRapiClient, "master-b.example.com", username="user") def testCertVerifyInvalidCombinations(self): self.assertRaises(client.Error, client.GenericCurlConfig, use_curl_cabundle=True, cafile="cert1.pem") self.assertRaises(client.Error, client.GenericCurlConfig, use_curl_cabundle=True, capath="certs/") self.assertRaises(client.Error, client.GenericCurlConfig, use_curl_cabundle=True, cafile="cert1.pem", capath="certs/") def testProxySignalVerifyHostname(self): for use_gnutls in [False, True]: if use_gnutls: pcverfn = _FakeGnuTlsPycurlVersion else: pcverfn = _FakeOpenSslPycurlVersion for proxy in ["", "http://127.0.0.1:1234"]: for use_signal in [False, True]: for verify_hostname in [False, True]: cfgfn = client.GenericCurlConfig(proxy=proxy, use_signal=use_signal, verify_hostname=verify_hostname, _pycurl_version_fn=pcverfn) curl_factory = lambda: rapi.testutils.FakeCurl(RapiMock()) cl = client.GanetiRapiClient("master.example.com", curl_config_fn=cfgfn, curl_factory=curl_factory) curl = cl._CreateCurl() self.assertEqual(curl.getopt(pycurl.PROXY), proxy) self.assertEqual(curl.getopt(pycurl.NOSIGNAL), not use_signal) if verify_hostname: self.assertEqual(curl.getopt(pycurl.SSL_VERIFYHOST), 2) else: self.assertEqual(curl.getopt(pycurl.SSL_VERIFYHOST), 0) def testNoCertVerify(self): cfgfn = client.GenericCurlConfig() curl_factory = lambda: rapi.testutils.FakeCurl(RapiMock()) cl = client.GanetiRapiClient("master.example.com", curl_config_fn=cfgfn, curl_factory=curl_factory) curl = cl._CreateCurl() self.assertFalse(curl.getopt(pycurl.SSL_VERIFYPEER)) self.assertFalse(curl.getopt(pycurl.CAINFO)) self.assertFalse(curl.getopt(pycurl.CAPATH)) def testCertVerifyCurlBundle(self): cfgfn = client.GenericCurlConfig(use_curl_cabundle=True) curl_factory = lambda: rapi.testutils.FakeCurl(RapiMock()) cl = client.GanetiRapiClient("master.example.com", curl_config_fn=cfgfn, curl_factory=curl_factory) curl = cl._CreateCurl() self.assertTrue(curl.getopt(pycurl.SSL_VERIFYPEER)) self.assertFalse(curl.getopt(pycurl.CAINFO)) self.assertFalse(curl.getopt(pycurl.CAPATH)) def testCertVerifyCafile(self): mycert = "/tmp/some/UNUSED/cert/file.pem" cfgfn = client.GenericCurlConfig(cafile=mycert) curl_factory = lambda: rapi.testutils.FakeCurl(RapiMock()) cl = client.GanetiRapiClient("master.example.com", curl_config_fn=cfgfn, curl_factory=curl_factory) curl = cl._CreateCurl() self.assertTrue(curl.getopt(pycurl.SSL_VERIFYPEER)) self.assertEqual(curl.getopt(pycurl.CAINFO), mycert) self.assertFalse(curl.getopt(pycurl.CAPATH)) def testCertVerifyCapath(self): certdir = "/tmp/some/UNUSED/cert/directory" pcverfn = _FakeOpenSslPycurlVersion cfgfn = client.GenericCurlConfig(capath=certdir, _pycurl_version_fn=pcverfn) curl_factory = lambda: rapi.testutils.FakeCurl(RapiMock()) cl = client.GanetiRapiClient("master.example.com", curl_config_fn=cfgfn, curl_factory=curl_factory) curl = cl._CreateCurl() self.assertTrue(curl.getopt(pycurl.SSL_VERIFYPEER)) self.assertEqual(curl.getopt(pycurl.CAPATH), certdir) self.assertFalse(curl.getopt(pycurl.CAINFO)) def testCertVerifyCapathGnuTls(self): certdir = "/tmp/some/UNUSED/cert/directory" pcverfn = _FakeGnuTlsPycurlVersion cfgfn = client.GenericCurlConfig(capath=certdir, _pycurl_version_fn=pcverfn) curl_factory = lambda: rapi.testutils.FakeCurl(RapiMock()) cl = client.GanetiRapiClient("master.example.com", curl_config_fn=cfgfn, curl_factory=curl_factory) self.assertRaises(client.Error, cl._CreateCurl) def testCertVerifyNoSsl(self): certdir = "/tmp/some/UNUSED/cert/directory" pcverfn = _FakeNoSslPycurlVersion cfgfn = client.GenericCurlConfig(capath=certdir, _pycurl_version_fn=pcverfn) curl_factory = lambda: rapi.testutils.FakeCurl(RapiMock()) cl = client.GanetiRapiClient("master.example.com", curl_config_fn=cfgfn, curl_factory=curl_factory) self.assertRaises(client.Error, cl._CreateCurl) def testCertVerifyFancySsl(self): certdir = "/tmp/some/UNUSED/cert/directory" pcverfn = _FakeFancySslPycurlVersion cfgfn = client.GenericCurlConfig(capath=certdir, _pycurl_version_fn=pcverfn) curl_factory = lambda: rapi.testutils.FakeCurl(RapiMock()) cl = client.GanetiRapiClient("master.example.com", curl_config_fn=cfgfn, curl_factory=curl_factory) self.assertRaises(NotImplementedError, cl._CreateCurl) def testCertVerifyCapath(self): for connect_timeout in [None, 1, 5, 10, 30, 60, 300]: for timeout in [None, 1, 30, 60, 3600, 24 * 3600]: cfgfn = client.GenericCurlConfig(connect_timeout=connect_timeout, timeout=timeout) curl_factory = lambda: rapi.testutils.FakeCurl(RapiMock()) cl = client.GanetiRapiClient("master.example.com", curl_config_fn=cfgfn, curl_factory=curl_factory) curl = cl._CreateCurl() self.assertEqual(curl.getopt(pycurl.CONNECTTIMEOUT), connect_timeout) self.assertEqual(curl.getopt(pycurl.TIMEOUT), timeout) class GanetiRapiClientTests(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.rapi = RapiMock() self.curl = rapi.testutils.FakeCurl(self.rapi) self.client = client.GanetiRapiClient("master.example.com", curl_factory=lambda: self.curl) def assertHandler(self, handler_cls): self.assertTrue(isinstance(self.rapi.GetLastHandler(), handler_cls)) def assertQuery(self, key, value): self.assertEqual(value, self.rapi.GetLastHandler().queryargs.get(key, None)) def assertItems(self, items): self.assertEqual(items, self.rapi.GetLastHandler().items) def assertBulk(self): self.assertTrue(self.rapi.GetLastHandler().useBulk()) def assertDryRun(self): self.assertTrue(self.rapi.GetLastHandler().dryRun()) def assertUseForce(self): self.assertTrue(self.rapi.GetLastHandler().useForce()) def testEncodeQuery(self): query = [ ("a", None), ("b", 1), ("c", 2), ("d", "Foo"), ("e", True), ] expected = [ ("a", ""), ("b", 1), ("c", 2), ("d", "Foo"), ("e", 1), ] self.assertEqualValues(self.client._EncodeQuery(query), expected) # invalid types for i in [[1, 2, 3], {"moo": "boo"}, (1, 2, 3)]: self.assertRaises(ValueError, self.client._EncodeQuery, [("x", i)]) def testCurlSettings(self): self.rapi.AddResponse("2") self.assertEqual(2, self.client.GetVersion()) self.assertHandler(rlib2.R_version) # Signals should be disabled by default self.assertTrue(self.curl.getopt(pycurl.NOSIGNAL)) # No auth and no proxy self.assertFalse(self.curl.getopt(pycurl.USERPWD)) self.assertTrue(self.curl.getopt(pycurl.PROXY) is None) # Content-type is required for requests headers = self.curl.getopt(pycurl.HTTPHEADER) self.assertTrue("Content-type: application/json" in headers) def testHttpError(self): self.rapi.AddResponse(None, code=404) try: self.client.GetJobStatus(15140) except client.GanetiApiError as err: self.assertEqual(err.code, 404) else: self.fail("Didn't raise exception") def testGetVersion(self): self.rapi.AddResponse("2") self.assertEqual(2, self.client.GetVersion()) self.assertHandler(rlib2.R_version) def testGetFeatures(self): for features in [[], ["foo", "bar", "baz"]]: self.rapi.AddResponse(serializer.DumpJson(features)) self.assertEqual(features, self.client.GetFeatures()) self.assertHandler(rlib2.R_2_features) def testGetFeaturesNotFound(self): self.rapi.AddResponse(None, code=404) self.assertEqual([], self.client.GetFeatures()) def testGetOperatingSystems(self): self.rapi.AddResponse("[\"beos\"]") self.assertEqual(["beos"], self.client.GetOperatingSystems()) self.assertHandler(rlib2.R_2_os) def testGetClusterTags(self): self.rapi.AddResponse("[\"tag\"]") self.assertEqual(["tag"], self.client.GetClusterTags()) self.assertHandler(rlib2.R_2_tags) def testAddClusterTags(self): self.rapi.AddResponse("1234") self.assertEqual(1234, self.client.AddClusterTags(["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_tags) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testDeleteClusterTags(self): self.rapi.AddResponse("5107") self.assertEqual(5107, self.client.DeleteClusterTags(["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_tags) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testGetInfo(self): self.rapi.AddResponse("{}") self.assertEqual({}, self.client.GetInfo()) self.assertHandler(rlib2.R_2_info) def testGetInstances(self): self.rapi.AddResponse("[]") self.assertEqual([], self.client.GetInstances(bulk=True)) self.assertHandler(rlib2.R_2_instances) self.assertBulk() def testGetInstance(self): self.rapi.AddResponse("[]") self.assertEqual([], self.client.GetInstance("instance")) self.assertHandler(rlib2.R_2_instances_name) self.assertItems(["instance"]) def testGetInstanceInfo(self): self.rapi.AddResponse("21291") self.assertEqual(21291, self.client.GetInstanceInfo("inst3")) self.assertHandler(rlib2.R_2_instances_name_info) self.assertItems(["inst3"]) self.assertQuery("static", None) self.rapi.AddResponse("3428") self.assertEqual(3428, self.client.GetInstanceInfo("inst31", static=False)) self.assertHandler(rlib2.R_2_instances_name_info) self.assertItems(["inst31"]) self.assertQuery("static", ["0"]) self.rapi.AddResponse("15665") self.assertEqual(15665, self.client.GetInstanceInfo("inst32", static=True)) self.assertHandler(rlib2.R_2_instances_name_info) self.assertItems(["inst32"]) self.assertQuery("static", ["1"]) def testInstancesMultiAlloc(self): response = { constants.JOB_IDS_KEY: ["23423"], constants.ALLOCATABLE_KEY: ["foobar"], constants.FAILED_KEY: ["foobar2"], } self.rapi.AddResponse(serializer.DumpJson(response)) insts = [self.client.InstanceAllocation("create", "foobar", "plain", [], []), self.client.InstanceAllocation("create", "foobar2", "drbd8", [{"size": 100}], [])] resp = self.client.InstancesMultiAlloc(insts) self.assertEqual(resp, response) self.assertHandler(rlib2.R_2_instances_multi_alloc) def testCreateInstanceOldVersion(self): # The old request format, version 0, is no longer supported self.rapi.AddResponse(None, code=404) self.assertRaises(client.GanetiApiError, self.client.CreateInstance, "create", "inst1.example.com", "plain", [], []) self.assertEqual(self.rapi.CountPending(), 0) def testCreateInstance(self): self.rapi.AddResponse(serializer.DumpJson([rlib2._INST_CREATE_REQV1])) self.rapi.AddResponse("23030") job_id = self.client.CreateInstance("create", "inst1.example.com", "plain", [], [], dry_run=True) self.assertEqual(job_id, 23030) self.assertHandler(rlib2.R_2_instances) self.assertDryRun() data = serializer.LoadJson(self.rapi.GetLastRequestData()) for field in ["dry_run", "beparams", "hvparams", "start"]: self.assertFalse(field in data) self.assertEqual(data["name"], "inst1.example.com") self.assertEqual(data["disk_template"], "plain") def testCreateInstance2(self): self.rapi.AddResponse(serializer.DumpJson([rlib2._INST_CREATE_REQV1])) self.rapi.AddResponse("24740") job_id = self.client.CreateInstance("import", "inst2.example.com", "drbd8", [{"size": 100,}], [{}, {"bridge": "br1", }], dry_run=False, start=True, pnode="node1", snode="node9", ip_check=False) self.assertEqual(job_id, 24740) self.assertHandler(rlib2.R_2_instances) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual(data[rlib2._REQ_DATA_VERSION], 1) self.assertEqual(data["name"], "inst2.example.com") self.assertEqual(data["disk_template"], "drbd8") self.assertEqual(data["start"], True) self.assertEqual(data["ip_check"], False) self.assertEqualValues(data["disks"], [{"size": 100,}]) self.assertEqualValues(data["nics"], [{}, {"bridge": "br1", }]) def testDeleteInstance(self): self.rapi.AddResponse("1234") self.assertEqual(1234, self.client.DeleteInstance("instance", dry_run=True)) self.assertHandler(rlib2.R_2_instances_name) self.assertItems(["instance"]) self.assertDryRun() def testGetInstanceTags(self): self.rapi.AddResponse("[]") self.assertEqual([], self.client.GetInstanceTags("fooinstance")) self.assertHandler(rlib2.R_2_instances_name_tags) self.assertItems(["fooinstance"]) def testAddInstanceTags(self): self.rapi.AddResponse("1234") self.assertEqual(1234, self.client.AddInstanceTags("fooinstance", ["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_instances_name_tags) self.assertItems(["fooinstance"]) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testDeleteInstanceTags(self): self.rapi.AddResponse("25826") self.assertEqual(25826, self.client.DeleteInstanceTags("foo", ["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_instances_name_tags) self.assertItems(["foo"]) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testRebootInstance(self): self.rapi.AddResponse("6146") job_id = self.client.RebootInstance("i-bar", reboot_type="hard", ignore_secondaries=True, dry_run=True, reason="Updates") self.assertEqual(6146, job_id) self.assertHandler(rlib2.R_2_instances_name_reboot) self.assertItems(["i-bar"]) self.assertDryRun() self.assertQuery("type", ["hard"]) self.assertQuery("ignore_secondaries", ["1"]) self.assertQuery("reason", ["Updates"]) def testRebootInstanceDefaultReason(self): self.rapi.AddResponse("6146") job_id = self.client.RebootInstance("i-bar", reboot_type="hard", ignore_secondaries=True, dry_run=True) self.assertEqual(6146, job_id) self.assertHandler(rlib2.R_2_instances_name_reboot) self.assertItems(["i-bar"]) self.assertDryRun() self.assertQuery("type", ["hard"]) self.assertQuery("ignore_secondaries", ["1"]) self.assertQuery("reason", None) def testShutdownInstance(self): self.rapi.AddResponse("1487") self.assertEqual(1487, self.client.ShutdownInstance("foo-instance", dry_run=True, reason="NoMore")) self.assertHandler(rlib2.R_2_instances_name_shutdown) self.assertItems(["foo-instance"]) self.assertDryRun() self.assertQuery("reason", ["NoMore"]) def testShutdownInstanceDefaultReason(self): self.rapi.AddResponse("1487") self.assertEqual(1487, self.client.ShutdownInstance("foo-instance", dry_run=True)) self.assertHandler(rlib2.R_2_instances_name_shutdown) self.assertItems(["foo-instance"]) self.assertDryRun() self.assertQuery("reason", None) def testStartupInstance(self): self.rapi.AddResponse("27149") self.assertEqual(27149, self.client.StartupInstance("bar-instance", dry_run=True, reason="New")) self.assertHandler(rlib2.R_2_instances_name_startup) self.assertItems(["bar-instance"]) self.assertDryRun() self.assertQuery("reason", ["New"]) def testStartupInstanceDefaultReason(self): self.rapi.AddResponse("27149") self.assertEqual(27149, self.client.StartupInstance("bar-instance", dry_run=True)) self.assertHandler(rlib2.R_2_instances_name_startup) self.assertItems(["bar-instance"]) self.assertDryRun() self.assertQuery("reason", None) def testReinstallInstance(self): self.rapi.AddResponse(serializer.DumpJson([])) self.rapi.AddResponse("19119") self.assertEqual(19119, self.client.ReinstallInstance("baz-instance", os="DOS", no_startup=True)) self.assertHandler(rlib2.R_2_instances_name_reinstall) self.assertItems(["baz-instance"]) self.assertQuery("os", ["DOS"]) self.assertQuery("nostartup", ["1"]) self.assertEqual(self.rapi.CountPending(), 0) def testReinstallInstanceNew(self): self.rapi.AddResponse(serializer.DumpJson([rlib2._INST_REINSTALL_REQV1])) self.rapi.AddResponse("25689") self.assertEqual(25689, self.client.ReinstallInstance("moo-instance", os="Debian", no_startup=True)) self.assertHandler(rlib2.R_2_instances_name_reinstall) self.assertItems(["moo-instance"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual(len(data), 2) self.assertEqual(data["os"], "Debian") self.assertEqual(data["start"], False) self.assertEqual(self.rapi.CountPending(), 0) def testReinstallInstanceWithOsparams1(self): self.rapi.AddResponse(serializer.DumpJson([])) self.assertRaises(client.GanetiApiError, self.client.ReinstallInstance, "doo-instance", osparams={"x": "y"}) self.assertEqual(self.rapi.CountPending(), 0) def testReinstallInstanceWithOsparams2(self): osparams = { "Hello": "World", "foo": "bar", } self.rapi.AddResponse(serializer.DumpJson([rlib2._INST_REINSTALL_REQV1])) self.rapi.AddResponse("1717") self.assertEqual(1717, self.client.ReinstallInstance("zoo-instance", osparams=osparams)) self.assertHandler(rlib2.R_2_instances_name_reinstall) self.assertItems(["zoo-instance"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual(len(data), 2) self.assertEqual(data["osparams"], osparams) self.assertEqual(data["start"], True) self.assertEqual(self.rapi.CountPending(), 0) def testReplaceInstanceDisks(self): self.rapi.AddResponse("999") job_id = self.client.ReplaceInstanceDisks("instance-name", disks=[0, 1], iallocator="hail") self.assertEqual(999, job_id) self.assertHandler(rlib2.R_2_instances_name_replace_disks) self.assertItems(["instance-name"]) self.assertQuery("disks", ["0,1"]) self.assertQuery("mode", ["replace_auto"]) self.assertQuery("iallocator", ["hail"]) self.rapi.AddResponse("1000") job_id = self.client.ReplaceInstanceDisks("instance-bar", disks=[1], mode="replace_on_secondary", remote_node="foo-node") self.assertEqual(1000, job_id) self.assertItems(["instance-bar"]) self.assertQuery("disks", ["1"]) self.assertQuery("remote_node", ["foo-node"]) self.rapi.AddResponse("5175") self.assertEqual(5175, self.client.ReplaceInstanceDisks("instance-moo")) self.assertItems(["instance-moo"]) self.assertQuery("disks", None) def testPrepareExport(self): self.rapi.AddResponse("8326") self.assertEqual(8326, self.client.PrepareExport("inst1", "local")) self.assertHandler(rlib2.R_2_instances_name_prepare_export) self.assertItems(["inst1"]) self.assertQuery("mode", ["local"]) def testExportInstance(self): self.rapi.AddResponse("19695") job_id = self.client.ExportInstance("inst2", "local", "nodeX", shutdown=True) self.assertEqual(job_id, 19695) self.assertHandler(rlib2.R_2_instances_name_export) self.assertItems(["inst2"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual(data["mode"], "local") self.assertEqual(data["destination"], "nodeX") self.assertEqual(data["shutdown"], True) def testMigrateInstanceDefaults(self): self.rapi.AddResponse("24873") job_id = self.client.MigrateInstance("inst91") self.assertEqual(job_id, 24873) self.assertHandler(rlib2.R_2_instances_name_migrate) self.assertItems(["inst91"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertFalse(data) def testMigrateInstance(self): for mode in constants.HT_MIGRATION_MODES: for cleanup in [False, True]: self.rapi.AddResponse("31910") job_id = self.client.MigrateInstance("inst289", mode=mode, cleanup=cleanup) self.assertEqual(job_id, 31910) self.assertHandler(rlib2.R_2_instances_name_migrate) self.assertItems(["inst289"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual(len(data), 2) self.assertEqual(data["mode"], mode) self.assertEqual(data["cleanup"], cleanup) def testFailoverInstanceDefaults(self): self.rapi.AddResponse("7639") job_id = self.client.FailoverInstance("inst13579") self.assertEqual(job_id, 7639) self.assertHandler(rlib2.R_2_instances_name_failover) self.assertItems(["inst13579"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertFalse(data) def testFailoverInstance(self): for iallocator in ["dumb", "hail"]: for ignore_consistency in [False, True]: for target_node in ["node-a", "node2"]: self.rapi.AddResponse("19161") job_id = \ self.client.FailoverInstance("inst251", iallocator=iallocator, ignore_consistency=ignore_consistency, target_node=target_node) self.assertEqual(job_id, 19161) self.assertHandler(rlib2.R_2_instances_name_failover) self.assertItems(["inst251"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual(len(data), 3) self.assertEqual(data["iallocator"], iallocator) self.assertEqual(data["ignore_consistency"], ignore_consistency) self.assertEqual(data["target_node"], target_node) self.assertEqual(self.rapi.CountPending(), 0) def testRenameInstanceDefaults(self): new_name = "newnametha7euqu" self.rapi.AddResponse("8791") job_id = self.client.RenameInstance("inst18821", new_name) self.assertEqual(job_id, 8791) self.assertHandler(rlib2.R_2_instances_name_rename) self.assertItems(["inst18821"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqualValues(data, {"new_name": new_name, }) def testRenameInstance(self): new_name = "new-name-yiux1iin" for ip_check in [False, True]: for name_check in [False, True]: self.rapi.AddResponse("24776") job_id = self.client.RenameInstance("inst20967", new_name, ip_check=ip_check, name_check=name_check) self.assertEqual(job_id, 24776) self.assertHandler(rlib2.R_2_instances_name_rename) self.assertItems(["inst20967"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual(len(data), 3) self.assertEqual(data["new_name"], new_name) self.assertEqual(data["ip_check"], ip_check) self.assertEqual(data["name_check"], name_check) def testGetJobs(self): self.rapi.AddResponse('[ { "id": "123", "uri": "\\/2\\/jobs\\/123" },' ' { "id": "124", "uri": "\\/2\\/jobs\\/124" } ]') self.assertEqual([123, 124], self.client.GetJobs()) self.assertHandler(rlib2.R_2_jobs) self.rapi.AddResponse('[ { "id": "123", "uri": "\\/2\\/jobs\\/123" },' ' { "id": "124", "uri": "\\/2\\/jobs\\/124" } ]') self.assertEqual([{"id": "123", "uri": "/2/jobs/123"}, {"id": "124", "uri": "/2/jobs/124"}], self.client.GetJobs(bulk=True)) self.assertHandler(rlib2.R_2_jobs) self.assertBulk() def testGetJobStatus(self): self.rapi.AddResponse("{\"foo\": \"bar\"}") self.assertEqual({"foo": "bar"}, self.client.GetJobStatus(1234)) self.assertHandler(rlib2.R_2_jobs_id) self.assertItems(["1234"]) def testWaitForJobChange(self): fields = ["id", "summary"] expected = { "job_info": [123, "something"], "log_entries": [], } self.rapi.AddResponse(serializer.DumpJson(expected)) result = self.client.WaitForJobChange(123, fields, [], -1) self.assertEqualValues(expected, result) self.assertHandler(rlib2.R_2_jobs_id_wait) self.assertItems(["123"]) def testCancelJob(self): self.rapi.AddResponse("[true, \"Job 123 will be canceled\"]") self.assertEqual([True, "Job 123 will be canceled"], self.client.CancelJob(999, dry_run=True)) self.assertHandler(rlib2.R_2_jobs_id) self.assertItems(["999"]) self.assertDryRun() def testGetNodes(self): self.rapi.AddResponse("[ { \"id\": \"node1\", \"uri\": \"uri1\" }," " { \"id\": \"node2\", \"uri\": \"uri2\" } ]") self.assertEqual(["node1", "node2"], self.client.GetNodes()) self.assertHandler(rlib2.R_2_nodes) self.rapi.AddResponse("[ { \"id\": \"node1\", \"uri\": \"uri1\" }," " { \"id\": \"node2\", \"uri\": \"uri2\" } ]") self.assertEqual([{"id": "node1", "uri": "uri1"}, {"id": "node2", "uri": "uri2"}], self.client.GetNodes(bulk=True)) self.assertHandler(rlib2.R_2_nodes) self.assertBulk() def testGetNode(self): self.rapi.AddResponse("{}") self.assertEqual({}, self.client.GetNode("node-foo")) self.assertHandler(rlib2.R_2_nodes_name) self.assertItems(["node-foo"]) def testEvacuateNode(self): self.rapi.AddResponse(serializer.DumpJson([rlib2._NODE_EVAC_RES1])) self.rapi.AddResponse("9876") job_id = self.client.EvacuateNode("node-1", remote_node="node-2") self.assertEqual(9876, job_id) self.assertHandler(rlib2.R_2_nodes_name_evacuate) self.assertItems(["node-1"]) self.assertEqual(serializer.LoadJson(self.rapi.GetLastRequestData()), { "remote_node": "node-2", }) self.assertEqual(self.rapi.CountPending(), 0) self.rapi.AddResponse(serializer.DumpJson([rlib2._NODE_EVAC_RES1])) self.rapi.AddResponse("8888") job_id = self.client.EvacuateNode("node-3", iallocator="hail", dry_run=True, mode=constants.NODE_EVAC_ALL, early_release=True) self.assertEqual(8888, job_id) self.assertItems(["node-3"]) self.assertEqual(serializer.LoadJson(self.rapi.GetLastRequestData()), { "iallocator": "hail", "mode": "all", "early_release": True, }) self.assertDryRun() self.assertRaises(client.GanetiApiError, self.client.EvacuateNode, "node-4", iallocator="hail", remote_node="node-5") self.assertEqual(self.rapi.CountPending(), 0) def testEvacuateNodeOldResponse(self): self.rapi.AddResponse(serializer.DumpJson([])) self.assertRaises(client.GanetiApiError, self.client.EvacuateNode, "node-4", accept_old=False) self.assertEqual(self.rapi.CountPending(), 0) for mode in [client.NODE_EVAC_PRI, client.NODE_EVAC_ALL]: self.rapi.AddResponse(serializer.DumpJson([])) self.assertRaises(client.GanetiApiError, self.client.EvacuateNode, "node-4", accept_old=True, mode=mode) self.assertEqual(self.rapi.CountPending(), 0) self.rapi.AddResponse(serializer.DumpJson([])) self.rapi.AddResponse(serializer.DumpJson("21533")) result = self.client.EvacuateNode("node-3", iallocator="hail", dry_run=True, accept_old=True, mode=client.NODE_EVAC_SEC, early_release=True) self.assertEqual(result, "21533") self.assertItems(["node-3"]) self.assertQuery("iallocator", ["hail"]) self.assertQuery("early_release", ["1"]) self.assertFalse(self.rapi.GetLastRequestData()) self.assertDryRun() self.assertEqual(self.rapi.CountPending(), 0) def testMigrateNode(self): self.rapi.AddResponse(serializer.DumpJson([])) self.rapi.AddResponse("1111") self.assertEqual(1111, self.client.MigrateNode("node-a", dry_run=True)) self.assertHandler(rlib2.R_2_nodes_name_migrate) self.assertItems(["node-a"]) self.assertTrue("mode" not in self.rapi.GetLastHandler().queryargs) self.assertDryRun() self.assertFalse(self.rapi.GetLastRequestData()) self.rapi.AddResponse(serializer.DumpJson([])) self.rapi.AddResponse("1112") self.assertEqual(1112, self.client.MigrateNode("node-a", dry_run=True, mode="live")) self.assertHandler(rlib2.R_2_nodes_name_migrate) self.assertItems(["node-a"]) self.assertQuery("mode", ["live"]) self.assertDryRun() self.assertFalse(self.rapi.GetLastRequestData()) self.rapi.AddResponse(serializer.DumpJson([])) self.assertRaises(client.GanetiApiError, self.client.MigrateNode, "node-c", target_node="foonode") self.assertEqual(self.rapi.CountPending(), 0) def testMigrateNodeBodyData(self): self.rapi.AddResponse(serializer.DumpJson([rlib2._NODE_MIGRATE_REQV1])) self.rapi.AddResponse("27539") self.assertEqual(27539, self.client.MigrateNode("node-a", dry_run=False, mode="live")) self.assertHandler(rlib2.R_2_nodes_name_migrate) self.assertItems(["node-a"]) self.assertFalse(self.rapi.GetLastHandler().queryargs) self.assertEqual(serializer.LoadJson(self.rapi.GetLastRequestData()), { "mode": "live", }) self.rapi.AddResponse(serializer.DumpJson([rlib2._NODE_MIGRATE_REQV1])) self.rapi.AddResponse("14219") self.assertEqual(14219, self.client.MigrateNode("node-x", dry_run=True, target_node="node9", iallocator="ial")) self.assertHandler(rlib2.R_2_nodes_name_migrate) self.assertItems(["node-x"]) self.assertDryRun() self.assertEqual(serializer.LoadJson(self.rapi.GetLastRequestData()), { "target_node": "node9", "iallocator": "ial", }) self.assertEqual(self.rapi.CountPending(), 0) def testGetNodeRole(self): self.rapi.AddResponse("\"master\"") self.assertEqual("master", self.client.GetNodeRole("node-a")) self.assertHandler(rlib2.R_2_nodes_name_role) self.assertItems(["node-a"]) def testSetNodeRole(self): self.rapi.AddResponse("789") self.assertEqual(789, self.client.SetNodeRole("node-foo", "master-candidate", force=True)) self.assertHandler(rlib2.R_2_nodes_name_role) self.assertItems(["node-foo"]) self.assertQuery("force", ["1"]) self.assertEqual("\"master-candidate\"", self.rapi.GetLastRequestData()) def testPowercycleNode(self): self.rapi.AddResponse("23051") self.assertEqual(23051, self.client.PowercycleNode("node5468", force=True)) self.assertHandler(rlib2.R_2_nodes_name_powercycle) self.assertItems(["node5468"]) self.assertQuery("force", ["1"]) self.assertFalse(self.rapi.GetLastRequestData()) self.assertEqual(self.rapi.CountPending(), 0) def testModifyNode(self): self.rapi.AddResponse("3783") job_id = self.client.ModifyNode("node16979.example.com", drained=True) self.assertEqual(job_id, 3783) self.assertHandler(rlib2.R_2_nodes_name_modify) self.assertItems(["node16979.example.com"]) self.assertEqual(self.rapi.CountPending(), 0) def testGetNodeStorageUnits(self): self.rapi.AddResponse("42") self.assertEqual(42, self.client.GetNodeStorageUnits("node-x", "lvm-pv", "fields")) self.assertHandler(rlib2.R_2_nodes_name_storage) self.assertItems(["node-x"]) self.assertQuery("storage_type", ["lvm-pv"]) self.assertQuery("output_fields", ["fields"]) def testModifyNodeStorageUnits(self): self.rapi.AddResponse("14") self.assertEqual(14, self.client.ModifyNodeStorageUnits("node-z", "lvm-pv", "hda")) self.assertHandler(rlib2.R_2_nodes_name_storage_modify) self.assertItems(["node-z"]) self.assertQuery("storage_type", ["lvm-pv"]) self.assertQuery("name", ["hda"]) self.assertQuery("allocatable", None) for allocatable, query_allocatable in [(True, "1"), (False, "0")]: self.rapi.AddResponse("7205") job_id = self.client.ModifyNodeStorageUnits("node-z", "lvm-pv", "hda", allocatable=allocatable) self.assertEqual(7205, job_id) self.assertHandler(rlib2.R_2_nodes_name_storage_modify) self.assertItems(["node-z"]) self.assertQuery("storage_type", ["lvm-pv"]) self.assertQuery("name", ["hda"]) self.assertQuery("allocatable", [query_allocatable]) def testRepairNodeStorageUnits(self): self.rapi.AddResponse("99") self.assertEqual(99, self.client.RepairNodeStorageUnits("node-z", "lvm-pv", "hda")) self.assertHandler(rlib2.R_2_nodes_name_storage_repair) self.assertItems(["node-z"]) self.assertQuery("storage_type", ["lvm-pv"]) self.assertQuery("name", ["hda"]) def testGetNodeTags(self): self.rapi.AddResponse("[\"fry\", \"bender\"]") self.assertEqual(["fry", "bender"], self.client.GetNodeTags("node-k")) self.assertHandler(rlib2.R_2_nodes_name_tags) self.assertItems(["node-k"]) def testAddNodeTags(self): self.rapi.AddResponse("1234") self.assertEqual(1234, self.client.AddNodeTags("node-v", ["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_nodes_name_tags) self.assertItems(["node-v"]) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testDeleteNodeTags(self): self.rapi.AddResponse("16861") self.assertEqual(16861, self.client.DeleteNodeTags("node-w", ["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_nodes_name_tags) self.assertItems(["node-w"]) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testGetGroups(self): groups = [{"name": "group1", "uri": "/2/groups/group1", }, {"name": "group2", "uri": "/2/groups/group2", }, ] self.rapi.AddResponse(serializer.DumpJson(groups)) self.assertEqual(["group1", "group2"], self.client.GetGroups()) self.assertHandler(rlib2.R_2_groups) def testGetGroupsBulk(self): groups = [{"name": "group1", "uri": "/2/groups/group1", "node_cnt": 2, "node_list": ["gnt1.test", "gnt2.test", ], }, {"name": "group2", "uri": "/2/groups/group2", "node_cnt": 1, "node_list": ["gnt3.test", ], }, ] self.rapi.AddResponse(serializer.DumpJson(groups)) self.assertEqual(groups, self.client.GetGroups(bulk=True)) self.assertHandler(rlib2.R_2_groups) self.assertBulk() def testGetGroup(self): group = {"ctime": None, "name": "default", } self.rapi.AddResponse(serializer.DumpJson(group)) self.assertEqual({"ctime": None, "name": "default"}, self.client.GetGroup("default")) self.assertHandler(rlib2.R_2_groups_name) self.assertItems(["default"]) def testCreateGroup(self): self.rapi.AddResponse("12345") job_id = self.client.CreateGroup("newgroup", dry_run=True) self.assertEqual(job_id, 12345) self.assertHandler(rlib2.R_2_groups) self.assertDryRun() def testDeleteGroup(self): self.rapi.AddResponse("12346") job_id = self.client.DeleteGroup("newgroup", dry_run=True) self.assertEqual(job_id, 12346) self.assertHandler(rlib2.R_2_groups_name) self.assertDryRun() def testRenameGroup(self): self.rapi.AddResponse("12347") job_id = self.client.RenameGroup("oldname", "newname") self.assertEqual(job_id, 12347) self.assertHandler(rlib2.R_2_groups_name_rename) def testModifyGroup(self): self.rapi.AddResponse("12348") job_id = self.client.ModifyGroup("mygroup", alloc_policy="foo") self.assertEqual(job_id, 12348) self.assertHandler(rlib2.R_2_groups_name_modify) def testAssignGroupNodes(self): self.rapi.AddResponse("12349") job_id = self.client.AssignGroupNodes("mygroup", ["node1", "node2"], force=True, dry_run=True) self.assertEqual(job_id, 12349) self.assertHandler(rlib2.R_2_groups_name_assign_nodes) self.assertDryRun() self.assertUseForce() def testGetNetworksBulk(self): networks = [{"name": "network1", "uri": "/2/networks/network1", "network": "192.168.0.0/24", }, {"name": "network2", "uri": "/2/networks/network2", "network": "192.168.0.0/24", }, ] self.rapi.AddResponse(serializer.DumpJson(networks)) self.assertEqual(networks, self.client.GetNetworks(bulk=True)) self.assertHandler(rlib2.R_2_networks) self.assertBulk() def testGetNetwork(self): network = {"ctime": None, "name": "network1", } self.rapi.AddResponse(serializer.DumpJson(network)) self.assertEqual({"ctime": None, "name": "network1"}, self.client.GetNetwork("network1")) self.assertHandler(rlib2.R_2_networks_name) self.assertItems(["network1"]) def testCreateNetwork(self): self.rapi.AddResponse("12345") job_id = self.client.CreateNetwork("newnetwork", network="192.168.0.0/24", dry_run=True) self.assertEqual(job_id, 12345) self.assertHandler(rlib2.R_2_networks) self.assertDryRun() def testModifyNetwork(self): self.rapi.AddResponse("12346") job_id = self.client.ModifyNetwork("mynetwork", gateway="192.168.0.10", dry_run=True) self.assertEqual(job_id, 12346) self.assertHandler(rlib2.R_2_networks_name_modify) def testDeleteNetwork(self): self.rapi.AddResponse("12347") job_id = self.client.DeleteNetwork("newnetwork", dry_run=True) self.assertEqual(job_id, 12347) self.assertHandler(rlib2.R_2_networks_name) self.assertDryRun() def testRenameNetwork(self): self.rapi.AddResponse("12347") job_id = self.client.RenameNetwork("oldname", "newname") self.assertEqual(job_id, 12347) self.assertHandler(rlib2.R_2_networks_name_rename) def testConnectNetwork(self): self.rapi.AddResponse("12348") job_id = self.client.ConnectNetwork("mynetwork", "default", "bridged", "br0", dry_run=True) self.assertEqual(job_id, 12348) self.assertHandler(rlib2.R_2_networks_name_connect) self.assertDryRun() def testDisconnectNetwork(self): self.rapi.AddResponse("12349") job_id = self.client.DisconnectNetwork("mynetwork", "default", dry_run=True) self.assertEqual(job_id, 12349) self.assertHandler(rlib2.R_2_networks_name_disconnect) self.assertDryRun() def testGetNetworkTags(self): self.rapi.AddResponse("[]") self.assertEqual([], self.client.GetNetworkTags("fooNetwork")) self.assertHandler(rlib2.R_2_networks_name_tags) self.assertItems(["fooNetwork"]) def testAddNetworkTags(self): self.rapi.AddResponse("1234") self.assertEqual(1234, self.client.AddNetworkTags("fooNetwork", ["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_networks_name_tags) self.assertItems(["fooNetwork"]) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testDeleteNetworkTags(self): self.rapi.AddResponse("25826") self.assertEqual(25826, self.client.DeleteNetworkTags("foo", ["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_networks_name_tags) self.assertItems(["foo"]) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testModifyInstance(self): self.rapi.AddResponse("23681") job_id = self.client.ModifyInstance("inst7210", os_name="linux") self.assertEqual(job_id, 23681) self.assertItems(["inst7210"]) self.assertHandler(rlib2.R_2_instances_name_modify) self.assertEqual(serializer.LoadJson(self.rapi.GetLastRequestData()), { "os_name": "linux", }) def testModifyCluster(self): for mnh in [None, False, True]: self.rapi.AddResponse("14470") self.assertEqual(14470, self.client.ModifyCluster(maintain_node_health=mnh, reason="PinkBunniesInvasion")) self.assertHandler(rlib2.R_2_cluster_modify) self.assertItems([]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual(len(data), 1) self.assertEqual(data["maintain_node_health"], mnh) self.assertEqual(self.rapi.CountPending(), 0) self.assertQuery("reason", ["PinkBunniesInvasion"]) def testRedistributeConfig(self): self.rapi.AddResponse("3364") job_id = self.client.RedistributeConfig() self.assertEqual(job_id, 3364) self.assertItems([]) self.assertHandler(rlib2.R_2_redist_config) def testActivateInstanceDisks(self): self.rapi.AddResponse("23547") job_id = self.client.ActivateInstanceDisks("inst28204") self.assertEqual(job_id, 23547) self.assertItems(["inst28204"]) self.assertHandler(rlib2.R_2_instances_name_activate_disks) self.assertFalse(self.rapi.GetLastHandler().queryargs) def testActivateInstanceDisksIgnoreSize(self): self.rapi.AddResponse("11044") job_id = self.client.ActivateInstanceDisks("inst28204", ignore_size=True) self.assertEqual(job_id, 11044) self.assertItems(["inst28204"]) self.assertHandler(rlib2.R_2_instances_name_activate_disks) self.assertQuery("ignore_size", ["1"]) def testDeactivateInstanceDisks(self): self.rapi.AddResponse("14591") job_id = self.client.DeactivateInstanceDisks("inst28234") self.assertEqual(job_id, 14591) self.assertItems(["inst28234"]) self.assertHandler(rlib2.R_2_instances_name_deactivate_disks) self.assertFalse(self.rapi.GetLastHandler().queryargs) def testRecreateInstanceDisks(self): self.rapi.AddResponse("13553") job_id = self.client.RecreateInstanceDisks("inst23153", iallocator="hail") self.assertEqual(job_id, 13553) self.assertItems(["inst23153"]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual("hail", data["iallocator"]) self.assertHandler(rlib2.R_2_instances_name_recreate_disks) self.assertFalse(self.rapi.GetLastHandler().queryargs) def testGetInstanceConsole(self): self.rapi.AddResponse("26876") job_id = self.client.GetInstanceConsole("inst21491") self.assertEqual(job_id, 26876) self.assertItems(["inst21491"]) self.assertHandler(rlib2.R_2_instances_name_console) self.assertFalse(self.rapi.GetLastHandler().queryargs) self.assertFalse(self.rapi.GetLastRequestData()) def testGrowInstanceDisk(self): for idx, wait_for_sync in enumerate([None, False, True]): amount = 128 + (512 * idx) self.assertEqual(self.rapi.CountPending(), 0) self.rapi.AddResponse("30783") self.assertEqual(30783, self.client.GrowInstanceDisk("eze8ch", idx, amount, wait_for_sync=wait_for_sync)) self.assertHandler(rlib2.R_2_instances_name_disk_grow) self.assertItems(["eze8ch", str(idx)]) data = serializer.LoadJson(self.rapi.GetLastRequestData()) if wait_for_sync is None: self.assertEqual(len(data), 1) self.assertTrue("wait_for_sync" not in data) else: self.assertEqual(len(data), 2) self.assertEqual(data["wait_for_sync"], wait_for_sync) self.assertEqual(data["amount"], amount) self.assertEqual(self.rapi.CountPending(), 0) def testGetGroupTags(self): self.rapi.AddResponse("[]") self.assertEqual([], self.client.GetGroupTags("fooGroup")) self.assertHandler(rlib2.R_2_groups_name_tags) self.assertItems(["fooGroup"]) def testAddGroupTags(self): self.rapi.AddResponse("1234") self.assertEqual(1234, self.client.AddGroupTags("fooGroup", ["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_groups_name_tags) self.assertItems(["fooGroup"]) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testDeleteGroupTags(self): self.rapi.AddResponse("25826") self.assertEqual(25826, self.client.DeleteGroupTags("foo", ["awesome"], dry_run=True)) self.assertHandler(rlib2.R_2_groups_name_tags) self.assertItems(["foo"]) self.assertDryRun() self.assertQuery("tag", ["awesome"]) def testQuery(self): for idx, what in enumerate(constants.QR_VIA_RAPI): for idx2, qfilter in enumerate([None, ["?", "name"]]): job_id = 11010 + (idx << 4) + (idx2 << 16) fields = sorted(query.ALL_FIELDS[what].keys())[:10] self.rapi.AddResponse(str(job_id)) self.assertEqual(self.client.Query(what, fields, qfilter=qfilter), job_id) self.assertItems([what]) self.assertHandler(rlib2.R_2_query) self.assertFalse(self.rapi.GetLastHandler().queryargs) data = serializer.LoadJson(self.rapi.GetLastRequestData()) self.assertEqual(data["fields"], fields) if qfilter is None: self.assertTrue("qfilter" not in data) else: self.assertEqual(data["qfilter"], qfilter) self.assertEqual(self.rapi.CountPending(), 0) def testQueryFields(self): exp_result = objects.QueryFieldsResponse(fields=[ objects.QueryFieldDefinition(name="pnode", title="PNode", kind=constants.QFT_NUMBER), objects.QueryFieldDefinition(name="other", title="Other", kind=constants.QFT_BOOL), ]) for what in constants.QR_VIA_RAPI: for fields in [None, ["name", "_unknown_"], ["&", "?|"]]: self.rapi.AddResponse(serializer.DumpJson(exp_result.ToDict())) result = self.client.QueryFields(what, fields=fields) self.assertItems([what]) self.assertHandler(rlib2.R_2_query_fields) self.assertFalse(self.rapi.GetLastRequestData()) queryargs = self.rapi.GetLastHandler().queryargs if fields is None: self.assertFalse(queryargs) else: self.assertEqual(queryargs, { "fields": [",".join(fields)], }) self.assertEqual(objects.QueryFieldsResponse.FromDict(result).ToDict(), exp_result.ToDict()) self.assertEqual(self.rapi.CountPending(), 0) def testWaitForJobCompletionNoChange(self): resp = serializer.DumpJson({ "status": constants.JOB_STATUS_WAITING, }) for retries in [1, 5, 25]: for _ in range(retries): self.rapi.AddResponse(resp) self.assertFalse(self.client.WaitForJobCompletion(22789, period=None, retries=retries)) self.assertHandler(rlib2.R_2_jobs_id) self.assertItems(["22789"]) self.assertEqual(self.rapi.CountPending(), 0) def testWaitForJobCompletionAlreadyFinished(self): self.rapi.AddResponse(serializer.DumpJson({ "status": constants.JOB_STATUS_SUCCESS, })) self.assertTrue(self.client.WaitForJobCompletion(22793, period=None, retries=1)) self.assertHandler(rlib2.R_2_jobs_id) self.assertItems(["22793"]) self.assertEqual(self.rapi.CountPending(), 0) def testWaitForJobCompletionEmptyResponse(self): self.rapi.AddResponse("{}") self.assertFalse(self.client.WaitForJobCompletion(22793, period=None, retries=10)) self.assertHandler(rlib2.R_2_jobs_id) self.assertItems(["22793"]) self.assertEqual(self.rapi.CountPending(), 0) def testWaitForJobCompletionOutOfRetries(self): for retries in [3, 10, 21]: for _ in range(retries): self.rapi.AddResponse(serializer.DumpJson({ "status": constants.JOB_STATUS_RUNNING, })) self.assertFalse(self.client.WaitForJobCompletion(30948, period=None, retries=retries - 1)) self.assertHandler(rlib2.R_2_jobs_id) self.assertItems(["30948"]) self.assertEqual(self.rapi.CountPending(), 1) self.rapi.ResetResponses() def testWaitForJobCompletionSuccessAndFailure(self): for retries in [1, 4, 13]: for (success, end_status) in [(False, constants.JOB_STATUS_ERROR), (True, constants.JOB_STATUS_SUCCESS)]: for _ in range(retries): self.rapi.AddResponse(serializer.DumpJson({ "status": constants.JOB_STATUS_RUNNING, })) self.rapi.AddResponse(serializer.DumpJson({ "status": end_status, })) result = self.client.WaitForJobCompletion(3187, period=None, retries=retries + 1) self.assertEqual(result, success) self.assertHandler(rlib2.R_2_jobs_id) self.assertItems(["3187"]) self.assertEqual(self.rapi.CountPending(), 0) def testGetFilters(self): self.rapi.AddResponse( "[ { \"uuid\": \"4364c043-f232-41e3-837f-f1ce846f21d2\"," " \"uri\": \"uri1\" }," " { \"uuid\": \"eceb3f7f-fee8-447a-8277-031b32a20e6b\"," " \"uri\": \"uri2\" } ]") self.assertEqual(["4364c043-f232-41e3-837f-f1ce846f21d2", "eceb3f7f-fee8-447a-8277-031b32a20e6b", ], self.client.GetFilters()) self.assertHandler(rlib2.R_2_filters) self.rapi.AddResponse( "[ { \"uuid\": \"4364c043-f232-41e3-837f-f1ce846f21d2\"," " \"uri\": \"uri1\" }," " { \"uuid\": \"eceb3f7f-fee8-447a-8277-031b32a20e6b\"," " \"uri\": \"uri2\" } ]") self.assertEqual([{"uuid": "4364c043-f232-41e3-837f-f1ce846f21d2", "uri": "uri1"}, {"uuid": "eceb3f7f-fee8-447a-8277-031b32a20e6b", "uri": "uri2"}, ], self.client.GetFilters(bulk=True)) self.assertHandler(rlib2.R_2_filters) self.assertBulk() def testGetFilter(self): filter_rule = { "uuid": "4364c043-f232-41e3-837f-f1ce846f21d2", "priority": 1, "predicates": [["jobid", [">", "id", "watermark"]]], "action": "CONTINUE", "reason_trail": ["testReplaceFilter", "myreason", 1412159589686391000], } self.rapi.AddResponse(serializer.DumpJson(filter_rule)) self.assertEqual( filter_rule, self.client.GetFilter("4364c043-f232-41e3-837f-f1ce846f21d2") ) self.assertHandler(rlib2.R_2_filters_uuid) self.assertItems(["4364c043-f232-41e3-837f-f1ce846f21d2"]) def testAddFilter(self): self.rapi.AddResponse("\"4364c043-f232-41e3-837f-f1ce846f21d2\"") self.assertEqual("4364c043-f232-41e3-837f-f1ce846f21d2", self.client.AddFilter( priority=1, predicates=[["jobid", [">", "id", "watermark"]]], action="CONTINUE", reason_trail=["testAddFilter", "myreason", utils.EpochNano()], )) self.assertHandler(rlib2.R_2_filters) def testReplaceFilter(self): self.rapi.AddResponse("\"4364c043-f232-41e3-837f-f1ce846f21d2\"") self.assertEqual("4364c043-f232-41e3-837f-f1ce846f21d2", self.client.ReplaceFilter( uuid="4364c043-f232-41e3-837f-f1ce846f21d2", priority=1, predicates=[["jobid", [">", "id", "watermark"]]], action="CONTINUE", reason_trail=["testReplaceFilter", "myreason", utils.EpochNano()], )) self.assertHandler(rlib2.R_2_filters_uuid) def testDeleteFilter(self): self.rapi.AddResponse("null") self.assertEqual(None, self.client.DeleteFilter( uuid="4364c043-f232-41e3-837f-f1ce846f21d2", )) self.assertHandler(rlib2.R_2_filters_uuid) class RapiTestRunner(unittest.TextTestRunner): def run(self, *args): global _used_handlers assert _used_handlers is None _used_handlers = set() try: # Run actual tests result = unittest.TextTestRunner.run(self, *args) diff = (set(connector.CONNECTOR.values()) - _used_handlers - _KNOWN_UNUSED) if diff: raise AssertionError("The following RAPI resources were not used by the" " RAPI client: %r" % utils.CommaJoin(diff)) finally: # Reset global variable _used_handlers = None return result if __name__ == "__main__": client.UsesRapiClient(testutils.GanetiTestProgram)(testRunner=RapiTestRunner) ganeti-3.1.0~rc2/test/py/legacy/ganeti.rapi.resources_unittest.py000075500000000000000000000051601476477700300252070ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the RAPI resources module""" import unittest import tempfile from ganeti import errors from ganeti import http from ganeti.rapi import connector from ganeti.rapi import rlib2 import testutils class MapperTests(unittest.TestCase): """Tests for remote API URI mapper.""" def setUp(self): self.map = connector.Mapper() def _TestUri(self, uri, result): self.assertEqual(self.map.getController(uri), result) def _TestFailingUri(self, uri): self.assertRaises(http.HttpNotFound, self.map.getController, uri) def testMapper(self): """Testing Mapper""" self._TestFailingUri("/tags") self._TestFailingUri("/instances") self._TestUri("/version", (rlib2.R_version, [], {})) self._TestUri("/2/instances/www.example.com", (rlib2.R_2_instances_name, ["www.example.com"], {})) self._TestUri("/2/instances/www.example.com/tags?f=5&f=6&alt=html", (rlib2.R_2_instances_name_tags, ["www.example.com"], {"alt": ["html"], "f": ["5", "6"], })) self._TestFailingUri("/tag") self._TestFailingUri("/instances/does/not/exist") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.rapi.rlib2_unittest.py000075500000000000000000001446621476477700300242220ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the RAPI rlib2 module """ import unittest import itertools import random from ganeti import constants from ganeti import opcodes from ganeti import compat from ganeti import ht from ganeti import http from ganeti import query import ganeti.rpc.errors as rpcerr from ganeti import errors from ganeti import rapi from ganeti.rapi import rlib2 from ganeti.rapi import baserlib from ganeti.rapi import connector import testutils class _FakeRequestPrivateData: def __init__(self, body_data): self.body_data = body_data class _FakeRequest: def __init__(self, body_data): self.private = _FakeRequestPrivateData(body_data) def _CreateHandler(cls, items, queryargs, body_data, client_cls): return cls(items, queryargs, _FakeRequest(body_data), _client_cls=client_cls) class _FakeClient: def __init__(self, address=None): self._jobs = [] def GetNextSubmittedJob(self): return self._jobs.pop(0) def SubmitJob(self, ops): job_id = str(1 + int(random.random() * 1000000)) self._jobs.append((job_id, ops)) return job_id class _FakeClientFactory: def __init__(self, cls): self._client_cls = cls self._clients = [] def GetNextClient(self): return self._clients.pop(0) def __call__(self, address=None): cl = self._client_cls(address=address) self._clients.append(cl) return cl class RAPITestCase(testutils.GanetiTestCase): """Provides a few helper methods specific to RAPI testing. """ def __init__(self, *args, **kwargs): """Creates a fake client factory the test may or may not use. """ unittest.TestCase.__init__(self, *args, **kwargs) self._clfactory = _FakeClientFactory(_FakeClient) def assertNoNextClient(self, clfactory=None): """Insures that no further clients are present. """ if clfactory is None: clfactory = self._clfactory self.assertRaises(IndexError, clfactory.GetNextClient) def getSubmittedOpcode(self, rapi_cls, items, query_args, body_data, method_name, opcode_cls): """Submits a RAPI request and fetches the resulting opcode. """ handler = _CreateHandler(rapi_cls, items, query_args, body_data, self._clfactory) self.assertTrue(hasattr(handler, method_name), "Handler lacks target method %s" % method_name) job_id = getattr(handler, method_name)() try: int(job_id) except ValueError: raise AssertionError("Returned value not job id; received %s" % job_id) cl = self._clfactory.GetNextClient() self.assertNoNextClient() (exp_job_id, (op, )) = cl.GetNextSubmittedJob() self.assertEqual(job_id, exp_job_id, "Job IDs do not match: %s != %s" % (job_id, exp_job_id)) self.assertRaises(IndexError, cl.GetNextSubmittedJob) self.assertTrue(isinstance(op, opcode_cls), "Wrong opcode class: expected %s, got %s" % (opcode_cls.__name__, op.__class__.__name__)) return op class TestConstants(unittest.TestCase): def testConsole(self): # Exporting the console field without authentication might expose # information assert "console" in query.INSTANCE_FIELDS self.assertTrue("console" not in rlib2.I_FIELDS) def testFields(self): checks = { constants.QR_INSTANCE: rlib2.I_FIELDS, constants.QR_NODE: rlib2.N_FIELDS, constants.QR_GROUP: rlib2.G_FIELDS, } for (qr, fields) in checks.items(): self.assertFalse(set(fields) - set(query.ALL_FIELDS[qr].keys())) class TestClientConnectError(unittest.TestCase): @staticmethod def _FailingClient(address=None): raise rpcerr.NoMasterError("test") def test(self): resources = [ rlib2.R_2_groups, rlib2.R_2_instances, rlib2.R_2_nodes, ] for cls in resources: handler = _CreateHandler(cls, ["name"], {}, None, self._FailingClient) self.assertRaises(http.HttpBadGateway, handler.GET) class TestJobSubmitError(unittest.TestCase): class _SubmitErrorClient: def __init__(self, address=None): pass @staticmethod def SubmitJob(ops): raise errors.JobQueueFull("test") def test(self): handler = _CreateHandler(rlib2.R_2_redist_config, [], {}, None, self._SubmitErrorClient) self.assertRaises(http.HttpServiceUnavailable, handler.PUT) class TestClusterModify(RAPITestCase): def test(self): body_data = { "vg_name": "testvg", "candidate_pool_size": 100, } op = self.getSubmittedOpcode(rlib2.R_2_cluster_modify, [], {}, body_data, "PUT", opcodes.OpClusterSetParams) self.assertEqual(op.vg_name, "testvg") self.assertEqual(op.candidate_pool_size, 100) def testInvalidValue(self): for attr in ["vg_name", "candidate_pool_size", "beparams", "_-Unknown#"]: handler = _CreateHandler(rlib2.R_2_cluster_modify, [], {}, { attr: True, }, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) self.assertNoNextClient() def testForbiddenParams(self): for attr, value in [ ("compression_tools", ["lzop"]), ]: handler = _CreateHandler(rlib2.R_2_cluster_modify, [], {}, { attr: value, }, self._clfactory) self.assertRaises(http.HttpForbidden, handler.PUT) self.assertNoNextClient() class TestRedistConfig(RAPITestCase): def test(self): self.getSubmittedOpcode(rlib2.R_2_redist_config, [], {}, None, "PUT", opcodes.OpClusterRedistConf) class TestNodeMigrate(RAPITestCase): def test(self): body_data = { "iallocator": "fooalloc", } op = self.getSubmittedOpcode(rlib2.R_2_nodes_name_migrate, ["node1"], {}, body_data, "POST", opcodes.OpNodeMigrate) self.assertEqual(op.node_name, "node1") self.assertEqual(op.iallocator, "fooalloc") def testQueryArgsConflict(self): query_args = { "live": True, "mode": constants.HT_MIGRATION_NONLIVE, } handler = _CreateHandler(rlib2.R_2_nodes_name_migrate, ["node2"], query_args, None, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) self.assertNoNextClient() def testQueryArgsMode(self): query_args = { "mode": [constants.HT_MIGRATION_LIVE], } op = self.getSubmittedOpcode(rlib2.R_2_nodes_name_migrate, ["node17292"], query_args, None, "POST", opcodes.OpNodeMigrate) self.assertEqual(op.node_name, "node17292") self.assertEqual(op.mode, constants.HT_MIGRATION_LIVE) def testQueryArgsLive(self): clfactory = _FakeClientFactory(_FakeClient) for live in [False, True]: query_args = { "live": [str(int(live))], } op = self.getSubmittedOpcode(rlib2.R_2_nodes_name_migrate, ["node6940"], query_args, None, "POST", opcodes.OpNodeMigrate) self.assertEqual(op.node_name, "node6940") if live: self.assertEqual(op.mode, constants.HT_MIGRATION_LIVE) else: self.assertEqual(op.mode, constants.HT_MIGRATION_NONLIVE) class TestNodeEvacuate(RAPITestCase): def test(self): query_args = { "dry-run": ["1"], } body_data = { "mode": constants.NODE_EVAC_SEC, } op = self.getSubmittedOpcode(rlib2.R_2_nodes_name_evacuate, ["node92"], query_args, body_data, "POST", opcodes.OpNodeEvacuate) self.assertEqual(op.node_name, "node92") self.assertEqual(op.mode, constants.NODE_EVAC_SEC) class TestNodePowercycle(RAPITestCase): def test(self): query_args = { "force": ["1"], } op = self.getSubmittedOpcode(rlib2.R_2_nodes_name_powercycle, ["node20744"], query_args, None, "POST", opcodes.OpNodePowercycle) self.assertEqual(op.node_name, "node20744") self.assertTrue(op.force) class TestGroupAssignNodes(RAPITestCase): def test(self): clfactory = _FakeClientFactory(_FakeClient) query_args = { "dry-run": ["1"], "force": ["1"], } body_data = { "nodes": ["n2", "n3"], } op = self.getSubmittedOpcode(rlib2.R_2_groups_name_assign_nodes, ["grp-a"], query_args, body_data, "PUT", opcodes.OpGroupAssignNodes) self.assertEqual(op.group_name, "grp-a") self.assertEqual(op.nodes, ["n2", "n3"]) self.assertTrue(op.dry_run) self.assertTrue(op.force) class TestInstanceDelete(RAPITestCase): def test(self): query_args = { "dry-run": ["1"], } op = self.getSubmittedOpcode(rlib2.R_2_instances_name, ["inst30965"], query_args, {}, "DELETE", opcodes.OpInstanceRemove) self.assertEqual(op.instance_name, "inst30965") self.assertTrue(op.dry_run) self.assertFalse(op.ignore_failures) class TestInstanceInfo(RAPITestCase): def test(self): query_args = { "static": ["1"], } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_info, ["inst31217"], query_args, {}, "GET", opcodes.OpInstanceQueryData) self.assertEqual(op.instances, ["inst31217"]) self.assertTrue(op.static) class TestInstanceReboot(RAPITestCase): def test(self): query_args = { "dry-run": ["1"], "ignore_secondaries": ["1"], "reason": ["System update"], } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_reboot, ["inst847"], query_args, {}, "POST", opcodes.OpInstanceReboot) self.assertEqual(op.instance_name, "inst847") self.assertEqual(op.reboot_type, constants.INSTANCE_REBOOT_HARD) self.assertTrue(op.ignore_secondaries) self.assertTrue(op.dry_run) self.assertEqual(op.reason[0][0], constants.OPCODE_REASON_SRC_USER) self.assertEqual(op.reason[0][1], "System update") self.assertEqual(op.reason[1][0], "%s:%s" % (constants.OPCODE_REASON_SRC_RLIB2, "instances_name_reboot")) self.assertEqual(op.reason[1][1], "") class TestInstanceStartup(RAPITestCase): def test(self): query_args = { "force": ["1"], "no_remember": ["1"], "reason": ["Newly created instance"], } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_startup, ["inst31083"], query_args, {}, "PUT", opcodes.OpInstanceStartup) self.assertEqual(op.instance_name, "inst31083") self.assertTrue(op.no_remember) self.assertTrue(op.force) self.assertFalse(op.dry_run) self.assertEqual(op.reason[0][0], constants.OPCODE_REASON_SRC_USER) self.assertEqual(op.reason[0][1], "Newly created instance") self.assertEqual(op.reason[1][0], "%s:%s" % (constants.OPCODE_REASON_SRC_RLIB2, "instances_name_startup")) self.assertEqual(op.reason[1][1], "") class TestInstanceShutdown(RAPITestCase): def test(self): query_args = { "no_remember": ["0"], "reason": ["Not used anymore"], } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_shutdown, ["inst26791"], query_args, {}, "PUT", opcodes.OpInstanceShutdown) self.assertEqual(op.instance_name, "inst26791") self.assertFalse(op.no_remember) self.assertFalse(op.dry_run) self.assertEqual(op.reason[0][0], constants.OPCODE_REASON_SRC_USER) self.assertEqual(op.reason[0][1], "Not used anymore") self.assertEqual(op.reason[1][0], "%s:%s" % (constants.OPCODE_REASON_SRC_RLIB2, "instances_name_shutdown")) self.assertEqual(op.reason[1][1], "") class TestInstanceActivateDisks(RAPITestCase): def test(self): query_args = { "ignore_size": ["1"], } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_activate_disks, ["xyz"], query_args, {}, "PUT", opcodes.OpInstanceActivateDisks) self.assertEqual(op.instance_name, "xyz") self.assertTrue(op.ignore_size) self.assertFalse(op.dry_run) class TestInstanceDeactivateDisks(RAPITestCase): def test(self): op = self.getSubmittedOpcode(rlib2.R_2_instances_name_deactivate_disks, ["inst22357"], {}, {}, "PUT", opcodes.OpInstanceDeactivateDisks) self.assertEqual(op.instance_name, "inst22357") self.assertFalse(op.dry_run) self.assertFalse(op.force) class TestInstanceRecreateDisks(RAPITestCase): def test(self): op = self.getSubmittedOpcode(rlib2.R_2_instances_name_recreate_disks, ["inst22357"], {}, {}, "POST", opcodes.OpInstanceRecreateDisks) self.assertEqual(op.instance_name, "inst22357") self.assertFalse(op.dry_run) self.assertTrue(op.iallocator is None) self.assertFalse(hasattr(op, "force")) def testCustomParameters(self): data = { "iallocator": "hail", } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_recreate_disks, ["inst22357"], {}, data, "POST", opcodes.OpInstanceRecreateDisks) self.assertEqual(op.instance_name, "inst22357") self.assertEqual(op.iallocator, "hail") self.assertFalse(op.dry_run) class TestInstanceFailover(RAPITestCase): def test(self): op = self.getSubmittedOpcode(rlib2.R_2_instances_name_failover, ["inst12794"], {}, {}, "PUT", opcodes.OpInstanceFailover) self.assertEqual(op.instance_name, "inst12794") self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) class TestInstanceDiskGrow(RAPITestCase): def test(self): data = { "amount": 1024, } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_disk_grow, ["inst10742", "3"], {}, data, "POST", opcodes.OpInstanceGrowDisk) self.assertEqual(op.instance_name, "inst10742") self.assertEqual(op.disk, 3) self.assertEqual(op.amount, 1024) self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) class TestInstanceModify(RAPITestCase): def testCustomParamRename(self): name = "instant_instance" data = { "custom_beparams": {}, "custom_hvparams": {}, } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_modify, [name], {}, data, "PUT", opcodes.OpInstanceSetParams) self.assertEqual(op.beparams, {}) self.assertEqual(op.hvparams, {}) # Define both data["beparams"] = {} assert "beparams" in data and "custom_beparams" in data handler = _CreateHandler(rlib2.R_2_instances_name_modify, [name], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) class TestBackupPrepare(RAPITestCase): def test(self): query_args = { "mode": constants.EXPORT_MODE_REMOTE, } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_prepare_export, ["inst17925"], query_args, {}, "PUT", opcodes.OpBackupPrepare) self.assertEqual(op.instance_name, "inst17925") self.assertEqual(op.mode, constants.EXPORT_MODE_REMOTE) self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) class TestGroupRemove(RAPITestCase): def test(self): op = self.getSubmittedOpcode(rlib2.R_2_groups_name, ["grp28575"], {}, {}, "DELETE", opcodes.OpGroupRemove) self.assertEqual(op.group_name, "grp28575") self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) class TestStorageQuery(RAPITestCase): def test(self): query_args = { "storage_type": constants.ST_LVM_PV, "output_fields": "name,other", } op = self.getSubmittedOpcode(rlib2.R_2_nodes_name_storage, ["node21075"], query_args, {}, "GET", opcodes.OpNodeQueryStorage) self.assertEqual(op.nodes, ["node21075"]) self.assertEqual(op.storage_type, constants.ST_LVM_PV) self.assertEqual(op.output_fields, ["name", "other"]) self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) def testErrors(self): # storage type which does not support space reporting queryargs = { "storage_type": constants.ST_DISKLESS, } handler = _CreateHandler(rlib2.R_2_nodes_name_storage, ["node21273"], queryargs, {}, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.GET) queryargs = { "storage_type": constants.ST_LVM_VG, } handler = _CreateHandler(rlib2.R_2_nodes_name_storage, ["node21273"], queryargs, {}, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.GET) queryargs = { "storage_type": "##unknown_storage##", "output_fields": "name,other", } handler = _CreateHandler(rlib2.R_2_nodes_name_storage, ["node10315"], queryargs, {}, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.GET) class TestStorageModify(RAPITestCase): def test(self): clfactory = _FakeClientFactory(_FakeClient) for allocatable in [None, "1", "0"]: query_args = { "storage_type": constants.ST_LVM_VG, "name": "pv-a", } if allocatable is not None: query_args["allocatable"] = allocatable op = self.getSubmittedOpcode(rlib2.R_2_nodes_name_storage_modify, ["node9292"], query_args, {}, "PUT", opcodes.OpNodeModifyStorage) self.assertEqual(op.node_name, "node9292") self.assertEqual(op.storage_type, constants.ST_LVM_VG) self.assertEqual(op.name, "pv-a") if allocatable is None: self.assertFalse(op.changes) else: assert allocatable in ("0", "1") self.assertEqual(op.changes, { constants.SF_ALLOCATABLE: (allocatable == "1"), }) self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) def testErrors(self): # No storage type queryargs = { "name": "xyz", } handler = _CreateHandler(rlib2.R_2_nodes_name_storage_modify, ["node26016"], queryargs, {}, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) # No name queryargs = { "storage_type": constants.ST_LVM_VG, } handler = _CreateHandler(rlib2.R_2_nodes_name_storage_modify, ["node21218"], queryargs, {}, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) # Invalid value queryargs = { "storage_type": constants.ST_LVM_VG, "name": "pv-b", "allocatable": "noint", } handler = _CreateHandler(rlib2.R_2_nodes_name_storage_modify, ["node30685"], queryargs, {}, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) class TestStorageRepair(RAPITestCase): def test(self): query_args = { "storage_type": constants.ST_LVM_PV, "name": "pv16611", } op = self.getSubmittedOpcode(rlib2.R_2_nodes_name_storage_repair, ["node19265"], query_args, {}, "PUT", opcodes.OpRepairNodeStorage) self.assertEqual(op.node_name, "node19265") self.assertEqual(op.storage_type, constants.ST_LVM_PV) self.assertEqual(op.name, "pv16611") self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) def testErrors(self): # No storage type queryargs = { "name": "xyz", } handler = _CreateHandler(rlib2.R_2_nodes_name_storage_repair, ["node11275"], queryargs, {}, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) # No name queryargs = { "storage_type": constants.ST_LVM_VG, } handler = _CreateHandler(rlib2.R_2_nodes_name_storage_repair, ["node21218"], queryargs, {}, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) class TestTags(RAPITestCase): TAG_HANDLERS = [ rlib2.R_2_instances_name_tags, rlib2.R_2_nodes_name_tags, rlib2.R_2_groups_name_tags, rlib2.R_2_tags, ] def testSetAndDelete(self): for method, opcls in [("PUT", opcodes.OpTagsSet), ("DELETE", opcodes.OpTagsDel)]: for idx, handler in enumerate(self.TAG_HANDLERS): dry_run = bool(idx % 2) name = "test%s" % idx queryargs = { "tag": ["foo", "bar", "baz"], "dry-run": str(int(dry_run)), } op = self.getSubmittedOpcode(handler, [name], queryargs, {}, method, opcls) self.assertEqual(op.kind, handler.TAG_LEVEL) if handler.TAG_LEVEL == constants.TAG_CLUSTER: self.assertTrue(op.name is None) else: self.assertEqual(op.name, name) self.assertEqual(op.tags, ["foo", "bar", "baz"]) self.assertEqual(op.dry_run, dry_run) self.assertFalse(hasattr(op, "force")) class TestInstanceCreation(RAPITestCase): def test(self): name = "inst863.example.com" disk_variants = [ # No disks [], # Two disks [{"size": 5, }, {"size": 100, }], # Disk with mode [{"size": 123, "mode": constants.DISK_RDWR, }], ] nic_variants = [ # No NIC [], # Three NICs [{}, {}, {}], # Two NICs [ { "ip": "192.0.2.6", "mode": constants.NIC_MODE_ROUTED, "mac": "01:23:45:67:68:9A", }, { "mode": constants.NIC_MODE_BRIDGED, "link": "br1" }, ], ] beparam_variants = [ None, {}, { constants.BE_VCPUS: 2, }, { constants.BE_MAXMEM: 200, }, { constants.BE_MEMORY: 256, }, { constants.BE_VCPUS: 2, constants.BE_MAXMEM: 1024, constants.BE_MINMEM: 1024, constants.BE_AUTO_BALANCE: True, constants.BE_ALWAYS_FAILOVER: True, } ] hvparam_variants = [ None, { constants.HV_BOOT_ORDER: "anc", }, { constants.HV_KERNEL_PATH: "/boot/fookernel", constants.HV_ROOT_PATH: "/dev/hda1", }, ] for mode in [constants.INSTANCE_CREATE, constants.INSTANCE_IMPORT]: for nics in nic_variants: for disk_template in constants.DISK_TEMPLATES: for disks in disk_variants: for beparams in beparam_variants: for hvparams in hvparam_variants: for dry_run in [False, True]: query_args = { "dry-run": str(int(dry_run)), } data = { rlib2._REQ_DATA_VERSION: 1, "name": name, "hypervisor": constants.HT_FAKE, "disks": disks, "nics": nics, "mode": mode, "disk_template": disk_template, "os": "debootstrap", } if beparams is not None: data["beparams"] = beparams if hvparams is not None: data["hvparams"] = hvparams op = self.getSubmittedOpcode( rlib2.R_2_instances, [], query_args, data, "POST", opcodes.OpInstanceCreate ) self.assertEqual(op.instance_name, name) self.assertEqual(op.mode, mode) self.assertEqual(op.disk_template, disk_template) self.assertEqual(op.dry_run, dry_run) self.assertEqual(len(op.disks), len(disks)) self.assertEqual(len(op.nics), len(nics)) for opdisk, disk in zip(op.disks, disks): for key in constants.IDISK_PARAMS: self.assertEqual(opdisk.get(key), disk.get(key)) self.assertFalse("unknown" in opdisk) for opnic, nic in zip(op.nics, nics): for key in constants.INIC_PARAMS: self.assertEqual(opnic.get(key), nic.get(key)) self.assertFalse("unknown" in opnic) self.assertFalse("foobar" in opnic) if beparams is None: self.assertTrue(op.beparams in [None, {}]) else: self.assertEqualValues(op.beparams, beparams) if hvparams is None: self.assertTrue(op.hvparams in [None, {}]) else: self.assertEqualValues(op.hvparams, hvparams) def testLegacyName(self): name = "inst29128.example.com" data = { rlib2._REQ_DATA_VERSION: 1, "name": name, "disks": [], "nics": [], "mode": constants.INSTANCE_CREATE, "disk_template": constants.DT_PLAIN, } op = self.getSubmittedOpcode(rlib2.R_2_instances, [], {}, data, "POST", opcodes.OpInstanceCreate) self.assertEqual(op.instance_name, name) self.assertFalse(hasattr(op, "name")) self.assertFalse(op.dry_run) # Define both data["instance_name"] = "other.example.com" assert "name" in data and "instance_name" in data handler = _CreateHandler(rlib2.R_2_instances, [], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) self.assertNoNextClient() def testLegacyOs(self): name = "inst4673.example.com" os = "linux29206" data = { rlib2._REQ_DATA_VERSION: 1, "name": name, "os_type": os, "disks": [], "nics": [], "mode": constants.INSTANCE_CREATE, "disk_template": constants.DT_PLAIN, } op = self.getSubmittedOpcode(rlib2.R_2_instances, [], {}, data, "POST", opcodes.OpInstanceCreate) self.assertEqual(op.instance_name, name) self.assertEqual(op.os_type, os) self.assertFalse(hasattr(op, "os")) self.assertFalse(op.dry_run) # Define both data["os"] = "linux9584" assert "os" in data and "os_type" in data handler = _CreateHandler(rlib2.R_2_instances, [], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) def testErrors(self): # Test all required fields reqfields = { rlib2._REQ_DATA_VERSION: 1, "name": "inst1.example.com", "disks": [], "nics": [], "mode": constants.INSTANCE_CREATE, } for name in reqfields.keys(): data = dict(i for i in reqfields.items() if i[0] != name) handler = _CreateHandler(rlib2.R_2_instances, [], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) self.assertNoNextClient() # Invalid disks and nics for field in ["disks", "nics"]: invalid_values = [None, 1, "", {}, [1, 2, 3], ["hda1", "hda2"], [{"_unknown_": False, }]] for invvalue in invalid_values: data = reqfields.copy() data[field] = invvalue handler = _CreateHandler(rlib2.R_2_instances, [], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) self.assertNoNextClient() def testVersion(self): # No version field data = { "name": "inst1.example.com", "disks": [], "nics": [], "mode": constants.INSTANCE_CREATE, "disk_template": constants.DT_PLAIN, } handler = _CreateHandler(rlib2.R_2_instances, [], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) # Old and incorrect versions for version in [0, -1, 10483, "Hello World"]: data[rlib2._REQ_DATA_VERSION] = version handler = _CreateHandler(rlib2.R_2_instances, [], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) self.assertNoNextClient() # Correct version data[rlib2._REQ_DATA_VERSION] = 1 self.getSubmittedOpcode(rlib2.R_2_instances, [], {}, data, "POST", opcodes.OpInstanceCreate) class TestBackupExport(RAPITestCase): def test(self): name = "instmoo" data = { "mode": constants.EXPORT_MODE_REMOTE, "destination": [(1, 2, 3), (99, 99, 99)], "shutdown": True, "remove_instance": True, "x509_key_name": ["name", "hash"], "destination_x509_ca": "---cert---" } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_export, [name], {}, data, "PUT", opcodes.OpBackupExport) self.assertEqual(op.instance_name, name) self.assertEqual(op.mode, constants.EXPORT_MODE_REMOTE) self.assertEqual(op.target_node, [(1, 2, 3), (99, 99, 99)]) self.assertEqual(op.shutdown, True) self.assertEqual(op.remove_instance, True) self.assertEqual(op.x509_key_name, ["name", "hash"]) self.assertEqual(op.destination_x509_ca, "---cert---") self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) def testDefaults(self): name = "inst1" data = { "destination": "node2", "shutdown": False, } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_export, [name], {}, data, "PUT", opcodes.OpBackupExport) self.assertEqual(op.instance_name, name) self.assertEqual(op.target_node, "node2") self.assertEqual(op.mode, "local") self.assertFalse(op.remove_instance) self.assertFalse(hasattr(op, "destination")) self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) def testErrors(self): clfactory = _FakeClientFactory(_FakeClient) for value in ["True", "False"]: data = { "remove_instance": value, } handler = _CreateHandler(rlib2.R_2_instances_name_export, ["err1"], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) class TestInstanceMigrate(RAPITestCase): def test(self): name = "instYooho6ek" for cleanup in [False, True]: for mode in constants.HT_MIGRATION_MODES: data = { "cleanup": cleanup, "mode": mode, } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_migrate, [name], {}, data, "PUT", opcodes.OpInstanceMigrate) self.assertEqual(op.instance_name, name) self.assertEqual(op.mode, mode) self.assertEqual(op.cleanup, cleanup) self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) def testDefaults(self): name = "instnohZeex0" op = self.getSubmittedOpcode(rlib2.R_2_instances_name_migrate, [name], {}, {}, "PUT", opcodes.OpInstanceMigrate) self.assertEqual(op.instance_name, name) self.assertTrue(op.mode is None) self.assertFalse(op.cleanup) self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) class TestParseRenameInstanceRequest(RAPITestCase): def test(self): name = "instij0eeph7" for new_name in ["ua0aiyoo", "fai3ongi"]: for ip_check in [False, True]: for name_check in [False, True]: data = { "new_name": new_name, "ip_check": ip_check, "name_check": name_check, } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_rename, [name], {}, data, "PUT", opcodes.OpInstanceRename) self.assertEqual(op.instance_name, name) self.assertEqual(op.new_name, new_name) self.assertEqual(op.ip_check, ip_check) self.assertEqual(op.name_check, name_check) self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) def testDefaults(self): name = "instahchie3t" for new_name in ["thag9mek", "quees7oh"]: data = { "new_name": new_name, } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_rename, [name], {}, data, "PUT", opcodes.OpInstanceRename) self.assertEqual(op.instance_name, name) self.assertEqual(op.new_name, new_name) self.assertFalse(op.ip_check) self.assertFalse(op.name_check) self.assertFalse(op.dry_run) self.assertFalse(hasattr(op, "force")) class TestParseModifyInstanceRequest(RAPITestCase): def test(self): name = "instush8gah" test_disks = [ [], [(1, { constants.IDISK_MODE: constants.DISK_RDWR, })], ] for osparams in [{}, { "some": "value", "other": "Hello World", }]: for hvparams in [{}, { constants.HV_KERNEL_PATH: "/some/kernel", }]: for beparams in [{}, { constants.BE_MAXMEM: 128, }]: for force in [False, True]: for nics in [[], [(0, { constants.INIC_IP: "192.0.2.1", })]]: for disks in test_disks: for disk_template in constants.DISK_TEMPLATES: data = { "osparams": osparams, "hvparams": hvparams, "beparams": beparams, "nics": nics, "disks": disks, "force": force, "disk_template": disk_template, } op = self.getSubmittedOpcode( rlib2.R_2_instances_name_modify, [name], {}, data, "PUT", opcodes.OpInstanceSetParams ) self.assertEqual(op.instance_name, name) self.assertEqual(op.hvparams, hvparams) self.assertEqual(op.beparams, beparams) self.assertEqual(op.osparams, osparams) self.assertEqual(op.force, force) self.assertEqual(op.nics, nics) self.assertEqual(op.disks, disks) self.assertEqual(op.disk_template, disk_template) self.assertTrue(op.remote_node is None) self.assertTrue(op.os_name is None) self.assertFalse(op.force_variant) self.assertFalse(op.dry_run) def testDefaults(self): name = "instir8aish31" op = self.getSubmittedOpcode(rlib2.R_2_instances_name_modify, [name], {}, {}, "PUT", opcodes.OpInstanceSetParams) for i in ["hvparams", "beparams", "osparams", "force", "nics", "disks", "disk_template", "remote_node", "os_name", "force_variant"]: self.assertTrue(hasattr(op, i)) class TestParseInstanceReinstallRequest(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.Parse = rlib2._ParseInstanceReinstallRequest def _Check(self, ops, name): expcls = [ opcodes.OpInstanceShutdown, opcodes.OpInstanceReinstall, opcodes.OpInstanceStartup, ] self.assertTrue(compat.all(isinstance(op, exp) for op, exp in zip(ops, expcls))) self.assertTrue(compat.all(op.instance_name == name for op in ops)) def test(self): name = "shoo0tihohma" ops = self.Parse(name, {"os": "sys1", "start": True,}) self.assertEqual(len(ops), 3) self._Check(ops, name) self.assertEqual(ops[1].os_type, "sys1") self.assertFalse(ops[1].osparams) ops = self.Parse(name, {"os": "sys2", "start": False,}) self.assertEqual(len(ops), 2) self._Check(ops, name) self.assertEqual(ops[1].os_type, "sys2") osparams = { "reformat": "1", } ops = self.Parse(name, {"os": "sys4035", "start": True, "osparams": osparams,}) self.assertEqual(len(ops), 3) self._Check(ops, name) self.assertEqual(ops[1].os_type, "sys4035") self.assertEqual(ops[1].osparams, osparams) def testDefaults(self): name = "noolee0g" ops = self.Parse(name, {"os": "linux1"}) self.assertEqual(len(ops), 3) self._Check(ops, name) self.assertEqual(ops[1].os_type, "linux1") self.assertFalse(ops[1].osparams) def testErrors(self): self.assertRaises(http.HttpBadRequest, self.Parse, "foo", "not a dictionary") class TestGroupRename(RAPITestCase): def test(self): name = "group608242564" data = { "new_name": "ua0aiyoo15112", } op = self.getSubmittedOpcode(rlib2.R_2_groups_name_rename, [name], {}, data, "PUT", opcodes.OpGroupRename) self.assertEqual(op.group_name, name) self.assertEqual(op.new_name, "ua0aiyoo15112") self.assertFalse(op.dry_run) def testDryRun(self): name = "group28548" query_args = { "dry-run": ["1"], } data = { "new_name": "ua0aiyoo", } op = self.getSubmittedOpcode(rlib2.R_2_groups_name_rename, [name], query_args, data, "PUT", opcodes.OpGroupRename) self.assertEqual(op.group_name, name) self.assertEqual(op.new_name, "ua0aiyoo") self.assertTrue(op.dry_run) class TestInstanceReplaceDisks(RAPITestCase): def test(self): name = "inst22568" for disks in [list(range(1, 4)), "1,2,3", "1, 2, 3"]: data = { "mode": constants.REPLACE_DISK_SEC, "disks": disks, "iallocator": "myalloc", } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_replace_disks, [name], {}, data, "POST", opcodes.OpInstanceReplaceDisks) self.assertEqual(op.instance_name, name) self.assertEqual(op.mode, constants.REPLACE_DISK_SEC) self.assertEqual(op.disks, [1, 2, 3]) self.assertEqual(op.iallocator, "myalloc") def testDefaults(self): name = "inst11413" data = { "mode": constants.REPLACE_DISK_AUTO, } op = self.getSubmittedOpcode(rlib2.R_2_instances_name_replace_disks, [name], {}, data, "POST", opcodes.OpInstanceReplaceDisks) self.assertEqual(op.instance_name, name) self.assertEqual(op.mode, constants.REPLACE_DISK_AUTO) self.assertTrue(op.iallocator is None) self.assertEqual(op.disks, []) def testNoDisks(self): handler = _CreateHandler(rlib2.R_2_instances_name_replace_disks, ["inst20661"], {}, {}, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) for disks in [None, "", {}]: handler = _CreateHandler(rlib2.R_2_instances_name_replace_disks, ["inst20661"], {}, { "disks": disks, }, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) def testWrong(self): data = { "mode": constants.REPLACE_DISK_AUTO, "disks": "hello world", } handler = _CreateHandler(rlib2.R_2_instances_name_replace_disks, ["foo"], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) class TestGroupModify(RAPITestCase): def test(self): name = "group6002" for policy in constants.VALID_ALLOC_POLICIES: data = { "alloc_policy": policy, } op = self.getSubmittedOpcode(rlib2.R_2_groups_name_modify, [name], {}, data, "PUT", opcodes.OpGroupSetParams) self.assertEqual(op.group_name, name) self.assertEqual(op.alloc_policy, policy) self.assertFalse(op.dry_run) def testUnknownPolicy(self): data = { "alloc_policy": "_unknown_policy_", } handler = _CreateHandler(rlib2.R_2_groups_name_modify, ["xyz"], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) self.assertNoNextClient() def testDefaults(self): name = "group6679" op = self.getSubmittedOpcode(rlib2.R_2_groups_name_modify, [name], {}, {}, "PUT", opcodes.OpGroupSetParams) self.assertEqual(op.group_name, name) self.assertTrue(op.alloc_policy is None) self.assertFalse(op.dry_run) def testCustomParamRename(self): name = "groupie" data = { "custom_diskparams": {}, "custom_ipolicy": {}, "custom_ndparams": {}, } op = self.getSubmittedOpcode(rlib2.R_2_groups_name_modify, [name], {}, data, "PUT", opcodes.OpGroupSetParams) self.assertEqual(op.diskparams, {}) self.assertEqual(op.ipolicy, {}) self.assertEqual(op.ndparams, {}) # Define both data["diskparams"] = {} assert "diskparams" in data and "custom_diskparams" in data handler = _CreateHandler(rlib2.R_2_groups_name_modify, [name], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.PUT) class TestGroupAdd(RAPITestCase): def test(self): name = "group3618" for policy in constants.VALID_ALLOC_POLICIES: data = { "group_name": name, "alloc_policy": policy, } op = self.getSubmittedOpcode(rlib2.R_2_groups, [], {}, data, "POST", opcodes.OpGroupAdd) self.assertEqual(op.group_name, name) self.assertEqual(op.alloc_policy, policy) self.assertFalse(op.dry_run) def testUnknownPolicy(self): data = { "alloc_policy": "_unknown_policy_", } handler = _CreateHandler(rlib2.R_2_groups, [], {}, data, self._clfactory) self.assertRaises(http.HttpBadRequest, handler.POST) self.assertNoNextClient() def testDefaults(self): name = "group15395" data = { "group_name": name, } op = self.getSubmittedOpcode(rlib2.R_2_groups, [], {}, data, "POST", opcodes.OpGroupAdd) self.assertEqual(op.group_name, name) self.assertTrue(op.alloc_policy is None) self.assertFalse(op.dry_run) def testLegacyName(self): name = "group29852" query_args = { "dry-run": ["1"], } data = { "name": name, } op = self.getSubmittedOpcode(rlib2.R_2_groups, [], query_args, data, "POST", opcodes.OpGroupAdd) self.assertEqual(op.group_name, name) self.assertTrue(op.alloc_policy is None) self.assertTrue(op.dry_run) class TestNodeRole(RAPITestCase): def test(self): for role in rlib2._NR_MAP.values(): handler = _CreateHandler(rlib2.R_2_nodes_name_role, ["node-z"], {}, role, self._clfactory) if role == rlib2._NR_MASTER: self.assertRaises(http.HttpBadRequest, handler.PUT) else: job_id = handler.PUT() cl = self._clfactory.GetNextClient() self.assertNoNextClient() (exp_job_id, (op, )) = cl.GetNextSubmittedJob() self.assertEqual(job_id, exp_job_id) self.assertTrue(isinstance(op, opcodes.OpNodeSetParams)) self.assertEqual(op.node_name, "node-z") self.assertFalse(op.force) self.assertFalse(op.dry_run) if role == rlib2._NR_REGULAR: self.assertFalse(op.drained) self.assertFalse(op.offline) self.assertFalse(op.master_candidate) elif role == rlib2._NR_MASTER_CANDIDATE: self.assertFalse(op.drained) self.assertFalse(op.offline) self.assertTrue(op.master_candidate) elif role == rlib2._NR_DRAINED: self.assertTrue(op.drained) self.assertFalse(op.offline) self.assertFalse(op.master_candidate) elif role == rlib2._NR_OFFLINE: self.assertFalse(op.drained) self.assertTrue(op.offline) self.assertFalse(op.master_candidate) else: self.fail("Unknown role '%s'" % role) self.assertRaises(IndexError, cl.GetNextSubmittedJob) class TestSimpleResources(RAPITestCase): def tearDown(self): self.assertNoNextClient() def testFeatures(self): handler = _CreateHandler(rlib2.R_2_features, [], {}, None, self._clfactory) self.assertEqual(set(handler.GET()), rlib2.ALL_FEATURES) def testEmpty(self): for cls in [rlib2.R_root, rlib2.R_2]: handler = _CreateHandler(cls, [], {}, None, self._clfactory) self.assertTrue(handler.GET() is None) def testVersion(self): handler = _CreateHandler(rlib2.R_version, [], {}, None, self._clfactory) self.assertEqual(handler.GET(), constants.RAPI_VERSION) class TestClusterInfo(unittest.TestCase): class _ClusterInfoClient: def __init__(self, address=None): self.cluster_info = None def QueryClusterInfo(self): assert self.cluster_info is None self.cluster_info = {} return self.cluster_info def test(self): clfactory = _FakeClientFactory(self._ClusterInfoClient) handler = _CreateHandler(rlib2.R_2_info, [], {}, None, clfactory) result = handler.GET() cl = clfactory.GetNextClient() self.assertRaises(IndexError, clfactory.GetNextClient) self.assertEqual(result, cl.cluster_info) class TestInstancesMultiAlloc(unittest.TestCase): def testInstanceUpdate(self): clfactory = _FakeClientFactory(_FakeClient) data = { "instances": [{ "name": "bar", "mode": "create", "disks": [{"size": 1024}], "disk_template": "plain", "nics": [{}], }, { "name": "foo", "mode": "create", "disks": [{"size": 1024}], "disk_template": "drbd", "nics": [{}], }], } handler = _CreateHandler(rlib2.R_2_instances_multi_alloc, [], {}, data, clfactory) (body, _) = handler.GetPostOpInput() self.assertTrue(compat.all( [isinstance(inst, opcodes.OpInstanceCreate) for inst in body["instances"]] )) class TestPermissions(unittest.TestCase): def testEquality(self): self.assertEqual(rlib2.R_2_query.GET_ACCESS, rlib2.R_2_query.PUT_ACCESS) self.assertEqual(rlib2.R_2_query.GET_ACCESS, rlib2.R_2_instances_name_console.GET_ACCESS) def testMethodAccess(self): for handler in connector.CONNECTOR.values(): for method in baserlib._SUPPORTED_METHODS: access = baserlib.GetHandlerAccess(handler, method) self.assertFalse(access is None) self.assertFalse(set(access) - rapi.RAPI_ACCESS_ALL, msg=("Handler '%s' uses unknown access options for" " method %s" % (handler, method))) self.assertTrue(rapi.RAPI_ACCESS_READ not in access or rapi.RAPI_ACCESS_WRITE in access, msg=("Handler '%s' gives query, but not write access" " for method %s (the latter includes query and" " should therefore be given as well)" % (handler, method))) class ForbiddenOpcode(opcodes.OpCode): OP_PARAMS = [ ("obligatory", None, ht.TString, None), ("forbidden", None, ht.TMaybe(ht.TString), None), ("forbidden_true", None, ht.TMaybe(ht.TBool), None), ] class ForbiddenRAPI(baserlib.OpcodeResource): POST_OPCODE = ForbiddenOpcode POST_FORBIDDEN = [ "forbidden", ("forbidden_true", [True]), ] POST_RENAME = { "totally_not_forbidden": "forbidden" } class TestForbiddenParams(RAPITestCase): def testTestOpcode(self): obligatory_value = "a" forbidden_value = "b" forbidden_true_value = True op = ForbiddenOpcode( obligatory=obligatory_value, forbidden=forbidden_value, forbidden_true=forbidden_true_value ) self.assertEqualValues(op.obligatory, obligatory_value) self.assertEqualValues(op.forbidden, forbidden_value) self.assertEqualValues(op.forbidden_true, forbidden_true_value) def testCorrectRequest(self): value = "o" op = self.getSubmittedOpcode(ForbiddenRAPI, [], {}, {"obligatory": value}, "POST", ForbiddenOpcode) self.assertEqual(op.obligatory, value) def testValueForbidden(self): value = "o" data = { "obligatory": value, "forbidden": value, } handler = _CreateHandler(ForbiddenRAPI, [], {}, data, self._clfactory) self.assertRaises(http.HttpForbidden, handler.POST) def testSpecificValueForbidden(self): for value in [True, False, "True"]: data = { "obligatory": "o", "forbidden_true": value, } handler = _CreateHandler(ForbiddenRAPI, [], {}, data, self._clfactory) if value == True: self.assertRaises(http.HttpForbidden, handler.POST) elif value == "True": self.assertRaises(http.HttpBadRequest, handler.POST) else: handler.POST() cl = self._clfactory.GetNextClient() (_, (op, )) = cl.GetNextSubmittedJob() self.assertTrue(isinstance(op, ForbiddenOpcode)) self.assertEqual(op.forbidden_true, value) def testRenameIntoForbidden(self): data = { "obligatory": "o", "totally_not_forbidden": "o", } handler = _CreateHandler(ForbiddenRAPI, [], {}, data, self._clfactory) self.assertRaises(http.HttpForbidden, handler.POST) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.rapi.testutils_unittest.py000075500000000000000000000156241476477700300252430ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.rapi.testutils""" import unittest from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import opcodes from ganeti import luxi from ganeti import rapi from ganeti import utils import ganeti.rapi.testutils import ganeti.rapi.client import testutils KNOWN_UNUSED_LUXI = compat.UniqueFrozenset([ luxi.REQ_SUBMIT_MANY_JOBS, luxi.REQ_SUBMIT_JOB_TO_DRAINED_QUEUE, luxi.REQ_ARCHIVE_JOB, luxi.REQ_AUTO_ARCHIVE_JOBS, luxi.REQ_CHANGE_JOB_PRIORITY, luxi.REQ_PICKUP_JOB, luxi.REQ_QUERY_EXPORTS, luxi.REQ_QUERY_CONFIG_VALUES, luxi.REQ_QUERY_NETWORKS, luxi.REQ_QUERY_TAGS, luxi.REQ_SET_DRAIN_FLAG, luxi.REQ_SET_WATCHER_PAUSE, ]) # Global variable for storing used LUXI calls _used_luxi_calls = None class TestHideInternalErrors(unittest.TestCase): def test(self): def inner(): raise errors.GenericError("error") fn = rapi.testutils._HideInternalErrors(inner) self.assertRaises(rapi.testutils.VerificationError, fn) class TestVerifyOpInput(unittest.TestCase): def testUnknownOpId(self): voi = rapi.testutils.VerifyOpInput self.assertRaises(rapi.testutils.VerificationError, voi, "UNK_OP_ID", None) def testUnknownParameter(self): voi = rapi.testutils.VerifyOpInput self.assertRaises(rapi.testutils.VerificationError, voi, opcodes.OpClusterRename.OP_ID, { "unk": "unk", }) def testWrongParameterValue(self): voi = rapi.testutils.VerifyOpInput self.assertRaises(rapi.testutils.VerificationError, voi, opcodes.OpClusterRename.OP_ID, { "name": object(), }) def testSuccess(self): voi = rapi.testutils.VerifyOpInput voi(opcodes.OpClusterRename.OP_ID, { "name": "new-name.example.com", }) class TestVerifyOpResult(unittest.TestCase): def testSuccess(self): vor = rapi.testutils.VerifyOpResult vor(opcodes.OpClusterVerify.OP_ID, { constants.JOB_IDS_KEY: [ (False, "error message"), ], }) def testWrongResult(self): vor = rapi.testutils.VerifyOpResult self.assertRaises(rapi.testutils.VerificationError, vor, opcodes.OpClusterVerify.OP_ID, []) def testNoResultCheck(self): vor = rapi.testutils.VerifyOpResult vor(opcodes.OpTestDummy.OP_ID, None) class TestInputTestClient(unittest.TestCase): def setUp(self): self.cl = rapi.testutils.InputTestClient() def tearDown(self): _used_luxi_calls.update(self.cl._GetLuxiCalls()) def testGetInfo(self): self.assertTrue(self.cl.GetInfo() is NotImplemented) def testPrepareExport(self): result = self.cl.PrepareExport("inst1.example.com", constants.EXPORT_MODE_LOCAL) self.assertTrue(result is NotImplemented) self.assertRaises(rapi.testutils.VerificationError, self.cl.PrepareExport, "inst1.example.com", "###invalid###") def testGetJobs(self): self.assertTrue(self.cl.GetJobs() is NotImplemented) def testQuery(self): result = self.cl.Query(constants.QR_NODE, ["name"]) self.assertTrue(result is NotImplemented) def testQueryFields(self): result = self.cl.QueryFields(constants.QR_INSTANCE) self.assertTrue(result is NotImplemented) def testCancelJob(self): self.assertTrue(self.cl.CancelJob("1") is NotImplemented) def testGetNodes(self): self.assertTrue(self.cl.GetNodes() is NotImplemented) def testGetInstances(self): self.assertTrue(self.cl.GetInstances() is NotImplemented) def testGetGroups(self): self.assertTrue(self.cl.GetGroups() is NotImplemented) def testWaitForJobChange(self): result = self.cl.WaitForJobChange("1", ["id"], None, None) self.assertTrue(result is NotImplemented) def testGetFilters(self): self.assertTrue(self.cl.GetFilters() is NotImplemented) def testGetFilter(self): result = self.cl.GetFilter("4364c043-f232-41e3-837f-f1ce846f21d2") self.assertTrue(result is NotImplemented) def testReplaceFilter(self): self.assertTrue(self.cl.ReplaceFilter( uuid="c6a70f02-facb-4e37-b344-54f146dd0396", priority=1, predicates=[["jobid", [">", "id", "watermark"]]], action="CONTINUE", reason_trail=["testReplaceFilter", "myreason", utils.EpochNano()], ) is NotImplemented) def testAddFilter(self): self.assertTrue(self.cl.AddFilter( priority=1, predicates=[["jobid", [">", "id", "watermark"]]], action="CONTINUE", reason_trail=["testAddFilter", "myreason", utils.EpochNano()], ) is NotImplemented) def testDeleteFilter(self): self.assertTrue(self.cl.DeleteFilter( uuid="c6a70f02-facb-4e37-b344-54f146dd0396", ) is NotImplemented) class CustomTestRunner(unittest.TextTestRunner): def run(self, *args): global _used_luxi_calls assert _used_luxi_calls is None diff = (KNOWN_UNUSED_LUXI - luxi.REQ_ALL) assert not diff, "Non-existing LUXI calls listed as unused: %s" % diff _used_luxi_calls = set() try: # Run actual tests result = unittest.TextTestRunner.run(self, *args) diff = _used_luxi_calls & KNOWN_UNUSED_LUXI if diff: raise AssertionError("LUXI methods marked as unused were called: %s" % utils.CommaJoin(diff)) diff = (luxi.REQ_ALL - KNOWN_UNUSED_LUXI - _used_luxi_calls) if diff: raise AssertionError("The following LUXI methods were not used: %s" % utils.CommaJoin(diff)) finally: # Reset global variable _used_luxi_calls = None return result if __name__ == "__main__": testutils.GanetiTestProgram(testRunner=CustomTestRunner) ganeti-3.1.0~rc2/test/py/legacy/ganeti.rpc.client_unittest.py000075500000000000000000000233631476477700300243110ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the RPC client module""" import unittest from ganeti import constants from ganeti import errors from ganeti import serializer from ganeti.rpc import client import testutils class TextRPCParsing(testutils.GanetiTestCase): def testParseRequest(self): msg = serializer.DumpJson({ client.KEY_METHOD: "foo", client.KEY_ARGS: ("bar", "baz", 123), }) self.assertEqualValues(client.ParseRequest(msg), ("foo", ["bar", "baz", 123], None)) self.assertRaises(client.ProtocolError, client.ParseRequest, "this\"is {invalid, ]json data") # No dict self.assertRaises(client.ProtocolError, client.ParseRequest, serializer.DumpJson(123)) # Empty dict self.assertRaises(client.ProtocolError, client.ParseRequest, serializer.DumpJson({ })) # No arguments self.assertRaises(client.ProtocolError, client.ParseRequest, serializer.DumpJson({ client.KEY_METHOD: "foo", })) # No method self.assertRaises(client.ProtocolError, client.ParseRequest, serializer.DumpJson({ client.KEY_ARGS: [], })) # No method or arguments self.assertRaises(client.ProtocolError, client.ParseRequest, serializer.DumpJson({ client.KEY_VERSION: 1, })) def testParseRequestWithVersion(self): msg = serializer.DumpJson({ client.KEY_METHOD: "version", client.KEY_ARGS: (["some"], "args", 0, "here"), client.KEY_VERSION: 20100101, }) self.assertEqualValues(client.ParseRequest(msg), ("version", [["some"], "args", 0, "here"], 20100101)) def testParseResponse(self): msg = serializer.DumpJson({ client.KEY_SUCCESS: True, client.KEY_RESULT: None, }) self.assertEqual(client.ParseResponse(msg), (True, None, None)) self.assertRaises(client.ProtocolError, client.ParseResponse, "this\"is {invalid, ]json data") # No dict self.assertRaises(client.ProtocolError, client.ParseResponse, serializer.DumpJson(123)) # Empty dict self.assertRaises(client.ProtocolError, client.ParseResponse, serializer.DumpJson({ })) # No success self.assertRaises(client.ProtocolError, client.ParseResponse, serializer.DumpJson({ client.KEY_RESULT: True, })) # No result self.assertRaises(client.ProtocolError, client.ParseResponse, serializer.DumpJson({ client.KEY_SUCCESS: True, })) # No result or success self.assertRaises(client.ProtocolError, client.ParseResponse, serializer.DumpJson({ client.KEY_VERSION: 123, })) def testParseResponseWithVersion(self): msg = serializer.DumpJson({ client.KEY_SUCCESS: True, client.KEY_RESULT: "Hello World", client.KEY_VERSION: 19991234, }) self.assertEqual(client.ParseResponse(msg), (True, "Hello World", 19991234)) def testFormatResponse(self): for success, result in [(False, "error"), (True, "abc"), (True, { "a": 123, "b": None, })]: msg = client.FormatResponse(success, result) msgdata = serializer.LoadJson(msg) self.assertTrue(client.KEY_SUCCESS in msgdata) self.assertTrue(client.KEY_RESULT in msgdata) self.assertTrue(client.KEY_VERSION not in msgdata) self.assertEqualValues(msgdata, { client.KEY_SUCCESS: success, client.KEY_RESULT: result, }) def testFormatResponseWithVersion(self): for success, result, version in [(False, "error", 123), (True, "abc", 999), (True, { "a": 123, "b": None, }, 2010)]: msg = client.FormatResponse(success, result, version=version) msgdata = serializer.LoadJson(msg) self.assertTrue(client.KEY_SUCCESS in msgdata) self.assertTrue(client.KEY_RESULT in msgdata) self.assertTrue(client.KEY_VERSION in msgdata) self.assertEqualValues(msgdata, { client.KEY_SUCCESS: success, client.KEY_RESULT: result, client.KEY_VERSION: version, }) def testFormatRequest(self): for method, args in [("a", []), ("b", [1, 2, 3])]: msg = client.FormatRequest(method, args) msgdata = serializer.LoadJson(msg) self.assertTrue(client.KEY_METHOD in msgdata) self.assertTrue(client.KEY_ARGS in msgdata) self.assertTrue(client.KEY_VERSION not in msgdata) self.assertEqualValues(msgdata, { client.KEY_METHOD: method, client.KEY_ARGS: args, }) def testFormatRequestWithVersion(self): for method, args, version in [("fn1", [], 123), ("fn2", [1, 2, 3], 999)]: msg = client.FormatRequest(method, args, version=version) msgdata = serializer.LoadJson(msg) self.assertTrue(client.KEY_METHOD in msgdata) self.assertTrue(client.KEY_ARGS in msgdata) self.assertTrue(client.KEY_VERSION in msgdata) self.assertEqualValues(msgdata, { client.KEY_METHOD: method, client.KEY_ARGS: args, client.KEY_VERSION: version, }) class TestCallRPCMethod(unittest.TestCase): MY_LUXI_VERSION = 1234 assert constants.LUXI_VERSION != MY_LUXI_VERSION def testSuccessNoVersion(self): def _Cb(msg): (method, args, version) = client.ParseRequest(msg) self.assertEqual(method, "fn1") self.assertEqual(args, "Hello World") return client.FormatResponse(True, "x") result = client.CallRPCMethod(_Cb, "fn1", "Hello World") def testServerVersionOnly(self): def _Cb(msg): (method, args, version) = client.ParseRequest(msg) self.assertEqual(method, "fn1") self.assertEqual(args, "Hello World") return client.FormatResponse(True, "x", version=self.MY_LUXI_VERSION) self.assertRaises(errors.LuxiError, client.CallRPCMethod, _Cb, "fn1", "Hello World") def testWithVersion(self): def _Cb(msg): (method, args, version) = client.ParseRequest(msg) self.assertEqual(method, "fn99") self.assertEqual(args, "xyz") return client.FormatResponse(True, "y", version=self.MY_LUXI_VERSION) self.assertEqual("y", client.CallRPCMethod(_Cb, "fn99", "xyz", version=self.MY_LUXI_VERSION)) def testVersionMismatch(self): def _Cb(msg): (method, args, version) = client.ParseRequest(msg) self.assertEqual(method, "fn5") self.assertEqual(args, "xyz") return client.FormatResponse(True, "F", version=self.MY_LUXI_VERSION * 2) self.assertRaises(errors.LuxiError, client.CallRPCMethod, _Cb, "fn5", "xyz", version=self.MY_LUXI_VERSION) def testError(self): def _Cb(msg): (method, args, version) = client.ParseRequest(msg) self.assertEqual(method, "fnErr") self.assertEqual(args, []) err = errors.OpPrereqError("Test") return client.FormatResponse(False, errors.EncodeException(err)) self.assertRaises(errors.OpPrereqError, client.CallRPCMethod, _Cb, "fnErr", []) def testErrorWithVersionMismatch(self): def _Cb(msg): (method, args, version) = client.ParseRequest(msg) self.assertEqual(method, "fnErr") self.assertEqual(args, []) err = errors.OpPrereqError("TestVer") return client.FormatResponse(False, errors.EncodeException(err), version=self.MY_LUXI_VERSION * 2) self.assertRaises(errors.LuxiError, client.CallRPCMethod, _Cb, "fnErr", [], version=self.MY_LUXI_VERSION) def testErrorWithVersion(self): def _Cb(msg): (method, args, version) = client.ParseRequest(msg) self.assertEqual(method, "fn9") self.assertEqual(args, []) err = errors.OpPrereqError("TestVer") return client.FormatResponse(False, errors.EncodeException(err), version=self.MY_LUXI_VERSION) self.assertRaises(errors.OpPrereqError, client.CallRPCMethod, _Cb, "fn9", [], version=self.MY_LUXI_VERSION) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.rpc_unittest.py000075500000000000000000001062731476477700300230360ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.rpc""" import os import sys import unittest import random import tempfile from ganeti import constants from ganeti import compat from ganeti.rpc import node as rpc from ganeti import rpc_defs from ganeti import http from ganeti import errors from ganeti import serializer from ganeti import objects from ganeti import backend import testutils import mocks class _FakeRequestProcessor: def __init__(self, response_fn): self._response_fn = response_fn self.reqcount = 0 def __call__(self, reqs, lock_monitor_cb=None): assert lock_monitor_cb is None or callable(lock_monitor_cb) for req in reqs: self.reqcount += 1 self._response_fn(req) def GetFakeSimpleStoreClass(fn): class FakeSimpleStore: GetNodePrimaryIPList = fn GetPrimaryIPFamily = lambda _: None return FakeSimpleStore def _RaiseNotImplemented(): """Simple wrapper to raise NotImplementedError. """ raise NotImplementedError class TestRpcProcessor(unittest.TestCase): def _FakeAddressLookup(self, map): return lambda node_list: [map.get(node) for node in node_list] def _GetVersionResponse(self, req): self.assertEqual(req.host, "127.0.0.1") self.assertEqual(req.port, 24094) self.assertEqual(req.path, "/version") self.assertEqual(req.read_timeout, constants.RPC_TMO_URGENT) req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, 123)) def testVersionSuccess(self): resolver = rpc._StaticResolver(["127.0.0.1"]) http_proc = _FakeRequestProcessor(self._GetVersionResponse) proc = rpc._RpcProcessor(resolver, 24094) result = proc(["localhost"], "version", {"localhost": ""}, 60, NotImplemented, _req_process_fn=http_proc) self.assertEqual(list(result), ["localhost"]) lhresp = result["localhost"] self.assertFalse(lhresp.offline) self.assertEqual(lhresp.node, "localhost") self.assertFalse(lhresp.fail_msg) self.assertEqual(lhresp.payload, 123) self.assertEqual(lhresp.call, "version") lhresp.Raise("should not raise") self.assertEqual(http_proc.reqcount, 1) def _ReadTimeoutResponse(self, req): self.assertEqual(req.host, "192.0.2.13") self.assertEqual(req.port, 19176) self.assertEqual(req.path, "/version") self.assertEqual(req.read_timeout, 12356) req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, -1)) def testReadTimeout(self): resolver = rpc._StaticResolver(["192.0.2.13"]) http_proc = _FakeRequestProcessor(self._ReadTimeoutResponse) proc = rpc._RpcProcessor(resolver, 19176) host = "node31856" body = {host: ""} result = proc([host], "version", body, 12356, NotImplemented, _req_process_fn=http_proc) self.assertEqual(list(result), [host]) lhresp = result[host] self.assertFalse(lhresp.offline) self.assertEqual(lhresp.node, host) self.assertFalse(lhresp.fail_msg) self.assertEqual(lhresp.payload, -1) self.assertEqual(lhresp.call, "version") lhresp.Raise("should not raise") self.assertEqual(http_proc.reqcount, 1) def testOfflineNode(self): resolver = rpc._StaticResolver([rpc._OFFLINE]) http_proc = _FakeRequestProcessor(NotImplemented) proc = rpc._RpcProcessor(resolver, 30668) host = "n17296" body = {host: ""} result = proc([host], "version", body, 60, NotImplemented, _req_process_fn=http_proc) self.assertEqual(list(result), [host]) lhresp = result[host] self.assertTrue(lhresp.offline) self.assertEqual(lhresp.node, host) self.assertTrue(lhresp.fail_msg) self.assertFalse(lhresp.payload) self.assertEqual(lhresp.call, "version") # With a message self.assertRaises(errors.OpExecError, lhresp.Raise, "should raise") # No message self.assertRaises(errors.OpExecError, lhresp.Raise, None) self.assertEqual(http_proc.reqcount, 0) def _GetMultiVersionResponse(self, req): self.assertTrue(req.host.startswith("node")) self.assertEqual(req.port, 23245) self.assertEqual(req.path, "/version") req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, 987)) def testMultiVersionSuccess(self): nodes = ["node%s" % i for i in range(50)] body = dict((n, "") for n in nodes) resolver = rpc._StaticResolver(nodes) http_proc = _FakeRequestProcessor(self._GetMultiVersionResponse) proc = rpc._RpcProcessor(resolver, 23245) result = proc(nodes, "version", body, 60, NotImplemented, _req_process_fn=http_proc) self.assertEqual(sorted(result.keys()), sorted(nodes)) for name in nodes: lhresp = result[name] self.assertFalse(lhresp.offline) self.assertEqual(lhresp.node, name) self.assertFalse(lhresp.fail_msg) self.assertEqual(lhresp.payload, 987) self.assertEqual(lhresp.call, "version") lhresp.Raise("should not raise") self.assertEqual(http_proc.reqcount, len(nodes)) def _GetVersionResponseFail(self, errinfo, req): self.assertEqual(req.path, "/version") req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((False, errinfo)) def testVersionFailure(self): resolver = rpc._StaticResolver(["aef9ur4i.example.com"]) proc = rpc._RpcProcessor(resolver, 5903) for errinfo in [None, "Unknown error"]: http_proc = \ _FakeRequestProcessor(compat.partial(self._GetVersionResponseFail, errinfo)) host = "aef9ur4i.example.com" body = {host: ""} result = proc(list(body), "version", body, 60, NotImplemented, _req_process_fn=http_proc) self.assertEqual(list(result), [host]) lhresp = result[host] self.assertFalse(lhresp.offline) self.assertEqual(lhresp.node, host) self.assertTrue(lhresp.fail_msg) self.assertFalse(lhresp.payload) self.assertEqual(lhresp.call, "version") self.assertRaises(errors.OpExecError, lhresp.Raise, "failed") self.assertEqual(http_proc.reqcount, 1) def _GetHttpErrorResponse(self, httperrnodes, failnodes, req): self.assertEqual(req.path, "/vg_list") self.assertEqual(req.port, 15165) if req.host in httperrnodes: req.success = False req.error = "Node set up for HTTP errors" elif req.host in failnodes: req.success = True req.resp_status_code = 404 req.resp_body = serializer.DumpJson({ "code": 404, "message": "Method not found", "explain": "Explanation goes here", }) else: req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, hash(req.host))) def testHttpError(self): nodes = ["uaf6pbbv%s" % i for i in range(50)] body = dict((n, "") for n in nodes) resolver = rpc._StaticResolver(nodes) httperrnodes = set(nodes[1::7]) self.assertEqual(len(httperrnodes), 7) failnodes = set(nodes[2::3]) - httperrnodes self.assertEqual(len(failnodes), 14) self.assertEqual(len(set(nodes) - failnodes - httperrnodes), 29) proc = rpc._RpcProcessor(resolver, 15165) http_proc = \ _FakeRequestProcessor(compat.partial(self._GetHttpErrorResponse, httperrnodes, failnodes)) result = proc(nodes, "vg_list", body, constants.RPC_TMO_URGENT, NotImplemented, _req_process_fn=http_proc) self.assertEqual(sorted(result.keys()), sorted(nodes)) for name in nodes: lhresp = result[name] self.assertFalse(lhresp.offline) self.assertEqual(lhresp.node, name) self.assertEqual(lhresp.call, "vg_list") if name in httperrnodes: self.assertTrue(lhresp.fail_msg) self.assertRaises(errors.OpExecError, lhresp.Raise, "failed") elif name in failnodes: self.assertTrue(lhresp.fail_msg) self.assertRaises(errors.OpPrereqError, lhresp.Raise, "failed", prereq=True, ecode=errors.ECODE_INVAL) else: self.assertFalse(lhresp.fail_msg) self.assertEqual(lhresp.payload, hash(name)) lhresp.Raise("should not raise") self.assertEqual(http_proc.reqcount, len(nodes)) def _GetInvalidResponseA(self, req): self.assertEqual(req.path, "/version") req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson(("This", "is", "an", "invalid", "response", "!", 1, 2, 3)) def _GetInvalidResponseB(self, req): self.assertEqual(req.path, "/version") req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson("invalid response") def testInvalidResponse(self): resolver = rpc._StaticResolver(["oqo7lanhly.example.com"]) proc = rpc._RpcProcessor(resolver, 19978) for fn in [self._GetInvalidResponseA, self._GetInvalidResponseB]: http_proc = _FakeRequestProcessor(fn) host = "oqo7lanhly.example.com" body = {host: ""} result = proc([host], "version", body, 60, NotImplemented, _req_process_fn=http_proc) self.assertEqual(list(result), [host]) lhresp = result[host] self.assertFalse(lhresp.offline) self.assertEqual(lhresp.node, host) self.assertTrue(lhresp.fail_msg) self.assertFalse(lhresp.payload) self.assertEqual(lhresp.call, "version") self.assertRaises(errors.OpExecError, lhresp.Raise, "failed") self.assertEqual(http_proc.reqcount, 1) def _GetBodyTestResponse(self, test_data, req): self.assertEqual(req.host, "192.0.2.84") self.assertEqual(req.port, 18700) self.assertEqual(req.path, "/upload_file") self.assertEqual(serializer.LoadJson(req.post_data), test_data) req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, None)) def testResponseBody(self): test_data = { "Hello": "World", "xyz": list(range(10)), } resolver = rpc._StaticResolver(["192.0.2.84"]) http_proc = _FakeRequestProcessor(compat.partial(self._GetBodyTestResponse, test_data)) proc = rpc._RpcProcessor(resolver, 18700) host = "node19759" body = {host: serializer.DumpJson(test_data)} result = proc([host], "upload_file", body, 30, NotImplemented, _req_process_fn=http_proc) self.assertEqual(list(result), [host]) lhresp = result[host] self.assertFalse(lhresp.offline) self.assertEqual(lhresp.node, host) self.assertFalse(lhresp.fail_msg) self.assertEqual(lhresp.payload, None) self.assertEqual(lhresp.call, "upload_file") lhresp.Raise("should not raise") self.assertEqual(http_proc.reqcount, 1) class TestSsconfResolver(unittest.TestCase): def testSsconfLookup(self): addr_list = ["192.0.2.%d" % n for n in range(0, 255, 13)] node_list = ["node%d.example.com" % n for n in range(0, 255, 13)] node_addr_list = [" ".join(t) for t in zip(node_list, addr_list)] ssc = GetFakeSimpleStoreClass(lambda _: node_addr_list) result = rpc._SsconfResolver(True, node_list, NotImplemented, ssc=ssc, nslookup_fn=NotImplemented) self.assertEqual(result, list(zip(node_list, addr_list, node_list))) def testNsLookup(self): addr_list = ["192.0.2.%d" % n for n in range(0, 255, 13)] node_list = ["node%d.example.com" % n for n in range(0, 255, 13)] ssc = GetFakeSimpleStoreClass(lambda _: []) node_addr_map = dict(list(zip(node_list, addr_list))) nslookup_fn = lambda name, family=None: node_addr_map.get(name) result = rpc._SsconfResolver(True, node_list, NotImplemented, ssc=ssc, nslookup_fn=nslookup_fn) self.assertEqual(result, list(zip(node_list, addr_list, node_list))) def testDisabledSsconfIp(self): addr_list = ["192.0.2.%d" % n for n in range(0, 255, 13)] node_list = ["node%d.example.com" % n for n in range(0, 255, 13)] ssc = GetFakeSimpleStoreClass(_RaiseNotImplemented) node_addr_map = dict(list(zip(node_list, addr_list))) nslookup_fn = lambda name, family=None: node_addr_map.get(name) result = rpc._SsconfResolver(False, node_list, NotImplemented, ssc=ssc, nslookup_fn=nslookup_fn) self.assertEqual(result, list(zip(node_list, addr_list, node_list))) def testBothLookups(self): addr_list = ["192.0.2.%d" % n for n in range(0, 255, 13)] node_list = ["node%d.example.com" % n for n in range(0, 255, 13)] n = len(addr_list) // 2 node_addr_list = [" ".join(t) for t in zip(node_list[n:], addr_list[n:])] ssc = GetFakeSimpleStoreClass(lambda _: node_addr_list) node_addr_map = dict(list(zip(node_list[:n], addr_list[:n]))) nslookup_fn = lambda name, family=None: node_addr_map.get(name) result = rpc._SsconfResolver(True, node_list, NotImplemented, ssc=ssc, nslookup_fn=nslookup_fn) self.assertEqual(result, list(zip(node_list, addr_list, node_list))) def testAddressLookupIPv6(self): addr_list = ["2001:db8::%d" % n for n in range(0, 255, 11)] node_list = ["node%d.example.com" % n for n in range(0, 255, 11)] node_addr_list = [" ".join(t) for t in zip(node_list, addr_list)] ssc = GetFakeSimpleStoreClass(lambda _: node_addr_list) result = rpc._SsconfResolver(True, node_list, NotImplemented, ssc=ssc, nslookup_fn=NotImplemented) self.assertEqual(result, list(zip(node_list, addr_list, node_list))) class TestStaticResolver(unittest.TestCase): def test(self): addresses = ["192.0.2.%d" % n for n in range(0, 123, 7)] nodes = ["node%s.example.com" % n for n in range(0, 123, 7)] res = rpc._StaticResolver(addresses) self.assertEqual(res(nodes, NotImplemented), list(zip(nodes, addresses, nodes))) def testWrongLength(self): res = rpc._StaticResolver([]) self.assertRaises(AssertionError, res, ["abc"], NotImplemented) class TestNodeConfigResolver(unittest.TestCase): @staticmethod def _GetSingleOnlineNode(uuid): assert uuid == "node90-uuid" return objects.Node(name="node90.example.com", uuid=uuid, offline=False, primary_ip="192.0.2.90") @staticmethod def _GetSingleOfflineNode(uuid): assert uuid == "node100-uuid" return objects.Node(name="node100.example.com", uuid=uuid, offline=True, primary_ip="192.0.2.100") def testSingleOnline(self): self.assertEqual(rpc._NodeConfigResolver(self._GetSingleOnlineNode, NotImplemented, ["node90-uuid"], None), [("node90.example.com", "192.0.2.90", "node90-uuid")]) def testSingleOffline(self): self.assertEqual(rpc._NodeConfigResolver(self._GetSingleOfflineNode, NotImplemented, ["node100-uuid"], None), [("node100.example.com", rpc._OFFLINE, "node100-uuid")]) def testSingleOfflineWithAcceptOffline(self): fn = self._GetSingleOfflineNode assert fn("node100-uuid").offline self.assertEqual(rpc._NodeConfigResolver(fn, NotImplemented, ["node100-uuid"], rpc_defs.ACCEPT_OFFLINE_NODE), [("node100.example.com", "192.0.2.100", "node100-uuid")]) for i in [False, True, "", "Hello", 0, 1]: self.assertRaises(AssertionError, rpc._NodeConfigResolver, fn, NotImplemented, ["node100.example.com"], i) def testUnknownSingleNode(self): self.assertEqual(rpc._NodeConfigResolver(lambda _: None, NotImplemented, ["node110.example.com"], None), [("node110.example.com", "node110.example.com", "node110.example.com")]) def testMultiEmpty(self): self.assertEqual(rpc._NodeConfigResolver(NotImplemented, lambda: {}, [], None), []) def testMultiSomeOffline(self): nodes = dict(("node%s-uuid" % i, objects.Node(name="node%s.example.com" % i, offline=((i % 3) == 0), primary_ip="192.0.2.%s" % i, uuid="node%s-uuid" % i)) for i in range(1, 255)) # Resolve no names self.assertEqual(rpc._NodeConfigResolver(NotImplemented, lambda: nodes, [], None), []) # Offline, online and unknown hosts self.assertEqual(rpc._NodeConfigResolver(NotImplemented, lambda: nodes, ["node3-uuid", "node92-uuid", "node54-uuid", "unknown.example.com",], None), [ ("node3.example.com", rpc._OFFLINE, "node3-uuid"), ("node92.example.com", "192.0.2.92", "node92-uuid"), ("node54.example.com", rpc._OFFLINE, "node54-uuid"), ("unknown.example.com", "unknown.example.com", "unknown.example.com"), ]) class TestCompress(unittest.TestCase): def test(self): for data in ["", "Hello", "Hello World!\nnew\nlines"]: self.assertEqual(rpc._Compress(NotImplemented, data), (constants.RPC_ENCODING_NONE, data)) for data in [512 * b" ", 5242 * b"Hello World!\n"]: compressed = rpc._Compress(NotImplemented, data) self.assertEqual(len(compressed), 2) self.assertEqual(backend._Decompress(compressed), data) def testDecompression(self): self.assertRaises(AssertionError, backend._Decompress, "") self.assertRaises(AssertionError, backend._Decompress, [""]) self.assertRaises(AssertionError, backend._Decompress, ("unknown compression", "data")) self.assertRaises(Exception, backend._Decompress, (constants.RPC_ENCODING_ZLIB_BASE64, "invalid zlib data")) class TestRpcClientBase(unittest.TestCase): def testNoHosts(self): cdef = ("test_call", NotImplemented, None, constants.RPC_TMO_SLOW, [], None, None, NotImplemented) http_proc = _FakeRequestProcessor(NotImplemented) client = rpc._RpcClientBase(rpc._StaticResolver([]), NotImplemented, _req_process_fn=http_proc) self.assertEqual(client._Call(cdef, [], []), {}) # Test wrong number of arguments self.assertRaises(errors.ProgrammerError, client._Call, cdef, [], [0, 1, 2]) def testTimeout(self): def _CalcTimeout(args): (arg1, arg2) = args return arg1 + arg2 def _VerifyRequest(exp_timeout, req): self.assertEqual(req.read_timeout, exp_timeout) req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, hex(req.read_timeout))) resolver = rpc._StaticResolver([ "192.0.2.1", "192.0.2.2", ]) nodes = [ "node1.example.com", "node2.example.com", ] tests = [(100, None, 100), (30, None, 30)] tests.extend((_CalcTimeout, i, i + 300) for i in [0, 5, 16485, 30516]) for timeout, arg1, exp_timeout in tests: cdef = ("test_call", NotImplemented, None, timeout, [ ("arg1", None, NotImplemented), ("arg2", None, NotImplemented), ], None, None, NotImplemented) http_proc = _FakeRequestProcessor(compat.partial(_VerifyRequest, exp_timeout)) client = rpc._RpcClientBase(resolver, NotImplemented, _req_process_fn=http_proc) result = client._Call(cdef, nodes, [arg1, 300]) self.assertEqual(len(result), len(nodes)) self.assertTrue(compat.all(not res.fail_msg and res.payload == hex(exp_timeout) for res in result.values())) def testArgumentEncoder(self): (AT1, AT2) = range(1, 3) resolver = rpc._StaticResolver([ "192.0.2.5", "192.0.2.6", ]) nodes = [ "node5.example.com", "node6.example.com", ] encoders = { AT1: lambda _, value: hex(value), AT2: lambda _, value: hash(value), } cdef = ("test_call", NotImplemented, None, constants.RPC_TMO_NORMAL, [ ("arg0", None, NotImplemented), ("arg1", AT1, NotImplemented), ("arg1", AT2, NotImplemented), ], None, None, NotImplemented) def _VerifyRequest(req): req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, req.post_data)) http_proc = _FakeRequestProcessor(_VerifyRequest) for num in [0, 3796, 9032119]: client = rpc._RpcClientBase(resolver, encoders.get, _req_process_fn=http_proc) result = client._Call(cdef, nodes, ["foo", num, "Hello%s" % num]) self.assertEqual(len(result), len(nodes)) for res in result.values(): self.assertFalse(res.fail_msg) self.assertEqual(serializer.LoadJson(res.payload), ["foo", hex(num), hash("Hello%s" % num)]) def testPostProc(self): def _VerifyRequest(nums, req): req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, nums)) resolver = rpc._StaticResolver([ "192.0.2.90", "192.0.2.95", ]) nodes = [ "node90.example.com", "node95.example.com", ] def _PostProc(res): self.assertFalse(res.fail_msg) res.payload = sum(res.payload) return res cdef = ("test_call", NotImplemented, None, constants.RPC_TMO_NORMAL, [], None, _PostProc, NotImplemented) # Seeded random generator rnd = random.Random(20299) for i in [0, 4, 74, 1391]: nums = [rnd.randint(0, 1000) for _ in range(i)] http_proc = _FakeRequestProcessor(compat.partial(_VerifyRequest, nums)) client = rpc._RpcClientBase(resolver, NotImplemented, _req_process_fn=http_proc) result = client._Call(cdef, nodes, []) self.assertEqual(len(result), len(nodes)) for res in result.values(): self.assertFalse(res.fail_msg) self.assertEqual(res.payload, sum(nums)) def testPreProc(self): def _VerifyRequest(req): req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, req.post_data)) resolver = rpc._StaticResolver([ "192.0.2.30", "192.0.2.35", ]) nodes = [ "node30.example.com", "node35.example.com", ] def _PreProc(node, data): self.assertEqual(len(data), 1) return data[0] + node cdef = ("test_call", NotImplemented, None, constants.RPC_TMO_NORMAL, [ ("arg0", None, NotImplemented), ], _PreProc, None, NotImplemented) http_proc = _FakeRequestProcessor(_VerifyRequest) client = rpc._RpcClientBase(resolver, NotImplemented, _req_process_fn=http_proc) for prefix in ["foo", "bar", "baz"]: result = client._Call(cdef, nodes, [prefix]) self.assertEqual(len(result), len(nodes)) for (idx, (node, res)) in enumerate(result.items()): self.assertFalse(res.fail_msg) self.assertEqual(serializer.LoadJson(res.payload), prefix + node) def testResolverOptions(self): def _VerifyRequest(req): req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, req.post_data)) nodes = [ "node30.example.com", "node35.example.com", ] def _Resolver(expected, hosts, options): self.assertEqual(hosts, nodes) self.assertEqual(options, expected) return list(zip(hosts, nodes, hosts)) def _DynamicResolverOptions(args): (arg0, ) = args return sum(arg0) tests = [ (None, None, None), (rpc_defs.ACCEPT_OFFLINE_NODE, None, rpc_defs.ACCEPT_OFFLINE_NODE), (False, None, False), (True, None, True), (0, None, 0), (_DynamicResolverOptions, [1, 2, 3], 6), (_DynamicResolverOptions, list(range(4, 19)), 165), ] for (resolver_opts, arg0, expected) in tests: cdef = ("test_call", NotImplemented, resolver_opts, constants.RPC_TMO_NORMAL, [ ("arg0", None, NotImplemented), ], None, None, NotImplemented) http_proc = _FakeRequestProcessor(_VerifyRequest) client = rpc._RpcClientBase(compat.partial(_Resolver, expected), NotImplemented, _req_process_fn=http_proc) result = client._Call(cdef, nodes, [arg0]) self.assertEqual(len(result), len(nodes)) for (idx, (node, res)) in enumerate(result.items()): self.assertFalse(res.fail_msg) class _FakeConfigForRpcRunner: GetAllNodesInfo = NotImplemented def __init__(self, cluster=NotImplemented): self._cluster = cluster self._disks = [ objects.Disk(dev_type=constants.DT_PLAIN, size=4096, logical_id=("vg", "disk6120"), uuid="disk_uuid_1"), objects.Disk(dev_type=constants.DT_PLAIN, size=1024, logical_id=("vg", "disk8508"), uuid="disk_uuid_2"), ] for disk in self._disks: disk.UpgradeConfig() def GetNodeInfo(self, name): return objects.Node(name=name) def GetMultiNodeInfo(self, names): return [(name, self.GetNodeInfo(name)) for name in names] def GetClusterInfo(self): return self._cluster def GetInstanceDiskParams(self, _): return constants.DISK_DT_DEFAULTS def GetInstanceSecondaryNodes(self, _): return [] def GetInstanceDisks(self, _): return self._disks class TestRpcRunner(unittest.TestCase): def testUploadFile(self): data = 1779 * b"Hello World\n" tmpfile = tempfile.NamedTemporaryFile() tmpfile.write(data) tmpfile.flush() st = os.stat(tmpfile.name) nodes = [ "node1.example.com", ] def _VerifyRequest(req): (uldata, ) = serializer.LoadJson(req.post_data) self.assertEqual(len(uldata), 7) self.assertEqual(uldata[0], tmpfile.name) compressed = [x.decode("ascii") if isinstance(x, bytes) else x for x in rpc._Compress(nodes[0], data)] self.assertEqual(list(uldata[1]), compressed) self.assertEqual(uldata[2], st.st_mode) self.assertEqual(uldata[3], "user%s" % os.getuid()) self.assertEqual(uldata[4], "group%s" % os.getgid()) self.assertTrue(uldata[5] is not None) self.assertEqual(uldata[6], st.st_mtime) req.success = True req.resp_status_code = http.HTTP_OK req.resp_body = serializer.DumpJson((True, None)) http_proc = _FakeRequestProcessor(_VerifyRequest) std_runner = rpc.RpcRunner(_FakeConfigForRpcRunner(), None, _req_process_fn=http_proc, _getents=mocks.FakeGetentResolver) cfg_runner = rpc.ConfigRunner(None, ["192.0.2.13"], _req_process_fn=http_proc, _getents=mocks.FakeGetentResolver) for runner in [std_runner, cfg_runner]: result = runner.call_upload_file(nodes, tmpfile.name) self.assertEqual(len(result), len(nodes)) for (idx, (node, res)) in enumerate(result.items()): self.assertFalse(res.fail_msg) def testEncodeInstance(self): cluster = objects.Cluster(hvparams={ constants.HT_KVM: { constants.HV_CDROM_IMAGE_PATH: "foo", }, }, beparams={ constants.PP_DEFAULT: { constants.BE_MAXMEM: 8192, }, }, os_hvp={}, osparams={ "linux": { "role": "unknown", }, }) cluster.UpgradeConfig() inst = objects.Instance(name="inst1.example.com", hypervisor=constants.HT_KVM, os="linux", hvparams={ constants.HV_CDROM_IMAGE_PATH: "bar", constants.HV_ROOT_PATH: "/tmp", }, beparams={ constants.BE_MINMEM: 128, constants.BE_MAXMEM: 256, }, nics=[ objects.NIC(nicparams={ constants.NIC_MODE: "mymode", }), ], disk_template=constants.DT_PLAIN, disks=["disk_uuid_1", "disk_uuid_2"] ) inst.UpgradeConfig() cfg = _FakeConfigForRpcRunner(cluster=cluster) runner = rpc.RpcRunner(cfg, None, _req_process_fn=NotImplemented, _getents=mocks.FakeGetentResolver) def _CheckBasics(result): self.assertEqual(result["name"], "inst1.example.com") self.assertEqual(result["os"], "linux") self.assertEqual(result["beparams"][constants.BE_MINMEM], 128) self.assertEqual(len(result["nics"]), 1) self.assertEqual(result["nics"][0]["nicparams"][constants.NIC_MODE], "mymode") # Generic object serialization result = runner._encoder(NotImplemented, (rpc_defs.ED_OBJECT_DICT, inst)) _CheckBasics(result) self.assertEqual(len(result["hvparams"]), 2) result = runner._encoder(NotImplemented, (rpc_defs.ED_OBJECT_DICT_LIST, 5 * [inst])) for r in result: _CheckBasics(r) self.assertEqual(len(r["hvparams"]), 2) # Just an instance result = runner._encoder(NotImplemented, (rpc_defs.ED_INST_DICT, inst)) _CheckBasics(result) self.assertEqual(result["beparams"][constants.BE_MAXMEM], 256) self.assertEqual(result["hvparams"][constants.HV_CDROM_IMAGE_PATH], "bar") self.assertEqual(result["hvparams"][constants.HV_ROOT_PATH], "/tmp") self.assertEqual(result["osparams"], { "role": "unknown", }) self.assertEqual(len(result["hvparams"]), len(constants.HVC_DEFAULTS[constants.HT_KVM])) # Instance with OS parameters result = runner._encoder(NotImplemented, (rpc_defs.ED_INST_DICT_OSP_DP, (inst, { "role": "webserver", "other": "field", }))) _CheckBasics(result) self.assertEqual(result["beparams"][constants.BE_MAXMEM], 256) self.assertEqual(result["hvparams"][constants.HV_CDROM_IMAGE_PATH], "bar") self.assertEqual(result["hvparams"][constants.HV_ROOT_PATH], "/tmp") self.assertEqual(result["osparams"], { "role": "webserver", "other": "field", }) # Instance with hypervisor and backend parameters result = runner._encoder(NotImplemented, (rpc_defs.ED_INST_DICT_HVP_BEP_DP, (inst, { constants.HT_KVM: { constants.HV_BOOT_ORDER: "xyz", }, }, { constants.BE_VCPUS: 100, constants.BE_MAXMEM: 4096, }))) _CheckBasics(result) self.assertEqual(result["beparams"][constants.BE_MAXMEM], 4096) self.assertEqual(result["beparams"][constants.BE_VCPUS], 100) self.assertEqual(result["hvparams"][constants.HT_KVM], { constants.HV_BOOT_ORDER: "xyz", }) del result["disks_info"][0]["ctime"] del result["disks_info"][0]["mtime"] del result["disks_info"][1]["ctime"] del result["disks_info"][1]["mtime"] self.assertEqual(result["disks_info"], [{ "dev_type": constants.DT_PLAIN, "dynamic_params": {}, "size": 4096, "logical_id": ("vg", "disk6120"), "params": constants.DISK_DT_DEFAULTS[inst.disk_template], "serial_no": 1, "uuid": "disk_uuid_1", }, { "dev_type": constants.DT_PLAIN, "dynamic_params": {}, "size": 1024, "logical_id": ("vg", "disk8508"), "params": constants.DISK_DT_DEFAULTS[inst.disk_template], "serial_no": 1, "uuid": "disk_uuid_2", }]) inst_disks = cfg.GetInstanceDisks(inst.uuid) self.assertTrue(compat.all(disk.params == {} for disk in inst_disks), msg="Configuration objects were modified") class TestLegacyNodeInfo(unittest.TestCase): KEY_BOOT = "bootid" KEY_NAME = "name" KEY_STORAGE_FREE = "storage_free" KEY_STORAGE_TOTAL = "storage_size" KEY_CPU_COUNT = "cpu_count" KEY_SPINDLES_FREE = "spindles_free" KEY_SPINDLES_TOTAL = "spindles_total" KEY_STORAGE_TYPE = "type" # key for storage type VAL_BOOT = 0 VAL_VG_NAME = "xy" VAL_VG_FREE = 11 VAL_VG_TOTAL = 12 VAL_VG_TYPE = "lvm-vg" VAL_CPU_COUNT = 2 VAL_PV_NAME = "ab" VAL_PV_FREE = 31 VAL_PV_TOTAL = 32 VAL_PV_TYPE = "lvm-pv" DICT_VG = { KEY_NAME: VAL_VG_NAME, KEY_STORAGE_FREE: VAL_VG_FREE, KEY_STORAGE_TOTAL: VAL_VG_TOTAL, KEY_STORAGE_TYPE: VAL_VG_TYPE, } DICT_HV = {KEY_CPU_COUNT: VAL_CPU_COUNT} DICT_SP = { KEY_STORAGE_TYPE: VAL_PV_TYPE, KEY_NAME: VAL_PV_NAME, KEY_STORAGE_FREE: VAL_PV_FREE, KEY_STORAGE_TOTAL: VAL_PV_TOTAL, } STD_LST = [VAL_BOOT, [DICT_VG, DICT_SP], [DICT_HV]] STD_DICT = { KEY_BOOT: VAL_BOOT, KEY_NAME: VAL_VG_NAME, KEY_STORAGE_FREE: VAL_VG_FREE, KEY_STORAGE_TOTAL: VAL_VG_TOTAL, KEY_SPINDLES_FREE: VAL_PV_FREE, KEY_SPINDLES_TOTAL: VAL_PV_TOTAL, KEY_CPU_COUNT: VAL_CPU_COUNT, } def testWithSpindles(self): result = rpc.MakeLegacyNodeInfo(self.STD_LST, constants.DT_PLAIN) self.assertEqual(result, self.STD_DICT) def testNoSpindles(self): my_lst = [self.VAL_BOOT, [self.DICT_VG], [self.DICT_HV]] result = rpc.MakeLegacyNodeInfo(my_lst, constants.DT_PLAIN) expected_dict = dict((k,v) for k, v in self.STD_DICT.items()) expected_dict[self.KEY_SPINDLES_FREE] = 0 expected_dict[self.KEY_SPINDLES_TOTAL] = 0 self.assertEqual(result, expected_dict) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.runtime_unittest.py000075500000000000000000000172501476477700300237310ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.runtime""" from ganeti import constants from ganeti import errors from ganeti import runtime from ganeti import ht import testutils import unittest class _EntStub: def __init__(self, uid=None, gid=None): self.pw_uid = uid self.gr_gid = gid def _StubGetpwnam(user): users = { constants.MASTERD_USER: _EntStub(uid=0), constants.CONFD_USER: _EntStub(uid=1), constants.RAPI_USER: _EntStub(uid=2), constants.NODED_USER: _EntStub(uid=3), constants.LUXID_USER: _EntStub(uid=4), constants.WCONFD_USER: _EntStub(uid=5), constants.MOND_USER: _EntStub(uid=6), constants.METAD_USER: _EntStub(uid=7), } return users[user] def _StubGetgrnam(group): groups = { constants.MASTERD_GROUP: _EntStub(gid=0), constants.CONFD_GROUP: _EntStub(gid=1), constants.RAPI_GROUP: _EntStub(gid=2), constants.DAEMONS_GROUP: _EntStub(gid=3), constants.ADMIN_GROUP: _EntStub(gid=4), constants.NODED_GROUP: _EntStub(gid=5), constants.LUXID_GROUP: _EntStub(gid=6), constants.WCONFD_GROUP: _EntStub(gid=7), constants.MOND_GROUP: _EntStub(gid=8), constants.METAD_GROUP: _EntStub(gid=9), } return groups[group] def _RaisingStubGetpwnam(user): raise KeyError("user not found") def _RaisingStubGetgrnam(group): raise KeyError("group not found") class ResolverStubRaising(object): def __init__(self): raise errors.ConfigurationError("No entries") class TestErrors(unittest.TestCase): def setUp(self): self.resolver = runtime.GetentResolver(_getpwnam=_StubGetpwnam, _getgrnam=_StubGetgrnam) def testEverythingSuccessful(self): self.assertEqual(self.resolver.masterd_uid, _StubGetpwnam(constants.MASTERD_USER).pw_uid) self.assertEqual(self.resolver.masterd_gid, _StubGetgrnam(constants.MASTERD_GROUP).gr_gid) self.assertEqual(self.resolver.confd_uid, _StubGetpwnam(constants.CONFD_USER).pw_uid) self.assertEqual(self.resolver.confd_gid, _StubGetgrnam(constants.CONFD_GROUP).gr_gid) self.assertEqual(self.resolver.wconfd_uid, _StubGetpwnam(constants.WCONFD_USER).pw_uid) self.assertEqual(self.resolver.wconfd_gid, _StubGetgrnam(constants.WCONFD_GROUP).gr_gid) self.assertEqual(self.resolver.luxid_uid, _StubGetpwnam(constants.LUXID_USER).pw_uid) self.assertEqual(self.resolver.luxid_gid, _StubGetgrnam(constants.LUXID_GROUP).gr_gid) self.assertEqual(self.resolver.rapi_uid, _StubGetpwnam(constants.RAPI_USER).pw_uid) self.assertEqual(self.resolver.rapi_gid, _StubGetgrnam(constants.RAPI_GROUP).gr_gid) self.assertEqual(self.resolver.noded_uid, _StubGetpwnam(constants.NODED_USER).pw_uid) self.assertEqual(self.resolver.noded_gid, _StubGetgrnam(constants.NODED_GROUP).gr_gid) self.assertEqual(self.resolver.mond_uid, _StubGetpwnam(constants.MOND_USER).pw_uid) self.assertEqual(self.resolver.mond_gid, _StubGetgrnam(constants.MOND_GROUP).gr_gid) self.assertEqual(self.resolver.metad_uid, _StubGetpwnam(constants.METAD_USER).pw_uid) self.assertEqual(self.resolver.metad_gid, _StubGetgrnam(constants.METAD_GROUP).gr_gid) self.assertEqual(self.resolver.daemons_gid, _StubGetgrnam(constants.DAEMONS_GROUP).gr_gid) self.assertEqual(self.resolver.admin_gid, _StubGetgrnam(constants.ADMIN_GROUP).gr_gid) def testUserNotFound(self): self.assertRaises(errors.ConfigurationError, runtime.GetentResolver, _getpwnam=_RaisingStubGetpwnam, _getgrnam=_StubGetgrnam) def testGroupNotFound(self): self.assertRaises(errors.ConfigurationError, runtime.GetentResolver, _getpwnam=_StubGetpwnam, _getgrnam=_RaisingStubGetgrnam) def testUserNotFoundGetEnts(self): self.assertRaises(errors.ConfigurationError, runtime.GetEnts, resolver=ResolverStubRaising) def testLookupForUser(self): master_stub = _StubGetpwnam(constants.MASTERD_USER) rapi_stub = _StubGetpwnam(constants.RAPI_USER) self.assertEqual(self.resolver.LookupUid(master_stub.pw_uid), constants.MASTERD_USER) self.assertEqual(self.resolver.LookupUid(rapi_stub.pw_uid), constants.RAPI_USER) self.assertEqual(self.resolver.LookupUser(constants.MASTERD_USER), master_stub.pw_uid) self.assertEqual(self.resolver.LookupUser(constants.RAPI_USER), rapi_stub.pw_uid) def testLookupForGroup(self): master_stub = _StubGetgrnam(constants.MASTERD_GROUP) rapi_stub = _StubGetgrnam(constants.RAPI_GROUP) self.assertEqual(self.resolver.LookupGid(master_stub.gr_gid), constants.MASTERD_GROUP) self.assertEqual(self.resolver.LookupGid(rapi_stub.gr_gid), constants.RAPI_GROUP) def testLookupForUserNotFound(self): self.assertRaises(errors.ConfigurationError, self.resolver.LookupUid, 9999) self.assertRaises(errors.ConfigurationError, self.resolver.LookupUser, "does-not-exist-foo") def testLookupForGroupNotFound(self): self.assertRaises(errors.ConfigurationError, self.resolver.LookupGid, 9999) self.assertRaises(errors.ConfigurationError, self.resolver.LookupGroup, "does-not-exist-foo") class TestArchInfo(unittest.TestCase): EXP_TYPES = \ ht.TAnd(ht.TIsLength(2), ht.TItems([ ht.TNonEmptyString, ht.TNonEmptyString, ])) def setUp(self): self.assertTrue(runtime._arch is None) def tearDown(self): runtime._arch = None def testNotInitialized(self): self.assertRaises(errors.ProgrammerError, runtime.GetArchInfo) def testInitializeMultiple(self): runtime.InitArchInfo() self.assertRaises(errors.ProgrammerError, runtime.InitArchInfo) def testNormal(self): runtime.InitArchInfo() info = runtime.GetArchInfo() self.assertTrue(self.EXP_TYPES(info), msg=("Doesn't match expected type description: %s" % self.EXP_TYPES)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.serializer_unittest.py000075500000000000000000000200651476477700300244150ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the serializer module""" import doctest import unittest from ganeti import errors from ganeti import ht from ganeti import objects from ganeti import serializer import testutils class TestSerializer(testutils.GanetiTestCase): """Serializer tests""" _TESTDATA = [ "test", 255, [1, 2, 3], (1, 2, 3), {"1": 2, "foo": "bar"}, ["abc", 1, 2, 3, 999, { "a1": ("Hello", "World"), "a2": "This is only a test", "a3": None, "osparams:": serializer.PrivateDict({ "foo": 5, }) } ] ] def _TestSerializer(self, dump_fn, load_fn): _dump_fn = lambda data: dump_fn( data, private_encoder=serializer.EncodeWithPrivateFields ) for data in self._TESTDATA: self.assertTrue(_dump_fn(data).endswith(b"\n")) self.assertEqualValues(load_fn(_dump_fn(data)), data) def testGeneric(self): self._TestSerializer(serializer.Dump, serializer.Load) def testSignedGeneric(self): self._TestSigned(serializer.DumpSigned, serializer.LoadSigned) def testJson(self): self._TestSerializer(serializer.DumpJson, serializer.LoadJson) def testSignedJson(self): self._TestSigned(serializer.DumpSignedJson, serializer.LoadSignedJson) def _TestSigned(self, dump_fn, load_fn): _dump_fn = lambda *args, **kwargs: dump_fn( *args, private_encoder=serializer.EncodeWithPrivateFields, **kwargs ) for data in self._TESTDATA: self.assertEqualValues(load_fn(_dump_fn(data, "mykey"), "mykey"), (data, "")) self.assertEqualValues(load_fn(_dump_fn(data, "myprivatekey", salt="mysalt"), "myprivatekey"), (data, "mysalt")) keydict = { "mykey_id": "myprivatekey", } self.assertEqualValues(load_fn(_dump_fn(data, "myprivatekey", salt="mysalt", key_selector="mykey_id"), keydict.get), (data, "mysalt")) self.assertRaises(errors.SignatureError, load_fn, _dump_fn(data, "myprivatekey", salt="mysalt", key_selector="mykey_id"), {}.get) self.assertRaises(errors.SignatureError, load_fn, _dump_fn("test", "myprivatekey"), "myotherkey") self.assertRaises(errors.SignatureError, load_fn, serializer.DumpJson("This is a test"), "mykey") self.assertRaises(errors.SignatureError, load_fn, serializer.DumpJson({}), "mykey") # Message missing salt and HMAC tdata = { "msg": "Foo", } self.assertRaises(errors.SignatureError, load_fn, serializer.DumpJson(tdata), "mykey") class TestLoadAndVerifyJson(unittest.TestCase): def testNoJson(self): self.assertRaises(errors.ParseError, serializer.LoadAndVerifyJson, "", NotImplemented) self.assertRaises(errors.ParseError, serializer.LoadAndVerifyJson, "}", NotImplemented) def testVerificationFails(self): self.assertRaises(errors.ParseError, serializer.LoadAndVerifyJson, "{}", lambda _: False) verify_fn = ht.TListOf(ht.TNonEmptyString) try: serializer.LoadAndVerifyJson("{}", verify_fn) except errors.ParseError as err: self.assertTrue(str(err).endswith(str(verify_fn))) else: self.fail("Exception not raised") def testSuccess(self): self.assertEqual(serializer.LoadAndVerifyJson("{}", ht.TAny), {}) self.assertEqual(serializer.LoadAndVerifyJson("\"Foo\"", ht.TAny), "Foo") class TestPrivate(unittest.TestCase): def testEquality(self): pDict = serializer.PrivateDict() pDict["bar"] = "egg" nDict = {"bar": "egg"} self.assertEqual(pDict, nDict, "PrivateDict-dict equality failure") def testPrivateDictUnprivate(self): pDict = serializer.PrivateDict() pDict["bar"] = "egg" uDict = pDict.Unprivate() nDict = {"bar": "egg"} self.assertEqual(type(uDict), dict, "PrivateDict.Unprivate() did not return a dict") self.assertEqual(pDict, uDict, "PrivateDict.Unprivate() equality failure") self.assertEqual(nDict, uDict, "PrivateDict.Unprivate() failed to return") def testAttributeTransparency(self): class Dummy(object): pass dummy = Dummy() dummy.bar = "egg" pDummy = serializer.Private(dummy) self.assertEqual(pDummy.bar, "egg", "Failed to access attribute of Private") def testCallTransparency(self): foo = serializer.Private("egg") self.assertEqual(foo.upper(), "EGG", "Failed to call Private instance") def testFillDict(self): pDict = serializer.PrivateDict() pDict["bar"] = "egg" self.assertEqual(pDict, objects.FillDict({}, pDict)) def testLeak(self): pDict = serializer.PrivateDict() pDict["bar"] = "egg" self.assertTrue("egg" not in str(pDict), "Value leaked in str(PrivateDict)") self.assertTrue("egg" not in repr(pDict), "Value leak in repr(PrivateDict)") self.assertTrue("egg" not in "{0}".format(pDict), "Value leaked in PrivateDict.__format__") self.assertTrue(b"egg" not in serializer.Dump(pDict), "Value leaked in serializer.Dump(PrivateDict)") def testProperAccess(self): pDict = serializer.PrivateDict() pDict["bar"] = "egg" self.assertTrue("egg" == pDict["bar"].Get(), "Value not returned by Private.Get()") self.assertTrue("egg" == pDict.GetPrivate("bar"), "Value not returned by Private.GetPrivate()") self.assertTrue("egg" == pDict.Unprivate()["bar"], "Value not returned by PrivateDict.Unprivate()") json = serializer.Dump(pDict, private_encoder=serializer.EncodeWithPrivateFields) self.assertTrue(b"egg" in json) def testDictGet(self): result = serializer.PrivateDict().GetPrivate("bar", "tar") self.assertTrue("tar" == result, "Private.GetPrivate() did not handle the default case") def testZeronessPrivate(self): self.assertTrue(serializer.Private("foo"), "Private of non-empty string is false") self.assertFalse(serializer.Private(""), "Private empty string is true") class TestCheckDoctests(unittest.TestCase): def testCheckSerializer(self): results = doctest.testmod(serializer) self.assertEqual(results.failed, 0, "Doctest failures detected") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.server.rapi_unittest.py000075500000000000000000000232351476477700300245060ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.server.rapi""" import re import unittest import random from io import StringIO from ganeti import constants from ganeti import utils from ganeti import compat from ganeti import errors from ganeti import serializer from ganeti import rapi from ganeti import http from ganeti import objects import ganeti.rapi.baserlib import ganeti.rapi.testutils import ganeti.rapi.rlib2 import ganeti.http.auth import testutils class TestRemoteApiHandler(unittest.TestCase): @staticmethod def _LookupWrongUser(_): return None def _Test(self, method, path, headers, reqbody, user_fn=NotImplemented, luxi_client=NotImplemented, reqauth=False): rm = rapi.testutils._RapiMock(user_fn, luxi_client, reqauth=reqauth) (resp_code, resp_headers, resp_body) = \ rm.FetchResponse(path, method, http.ParseHeaders(StringIO(headers)), reqbody) self.assertTrue(resp_headers[http.HTTP_DATE]) self.assertEqual(resp_headers[http.HTTP_CONNECTION], "close") self.assertEqual(resp_headers[http.HTTP_CONTENT_TYPE], http.HTTP_APP_JSON) self.assertEqual(resp_headers[http.HTTP_SERVER], http.HTTP_GANETI_VERSION) return (resp_code, resp_headers, serializer.LoadJson(resp_body)) def testRoot(self): (code, _, data) = self._Test(http.HTTP_GET, "/", "", None) self.assertEqual(code, http.HTTP_OK) self.assertTrue(data is None) def testRootReqAuth(self): (code, _, _) = self._Test(http.HTTP_GET, "/", "", None, reqauth=True) self.assertEqual(code, http.HttpUnauthorized.code) def testVersion(self): (code, _, data) = self._Test(http.HTTP_GET, "/version", "", None) self.assertEqual(code, http.HTTP_OK) self.assertEqual(data, constants.RAPI_VERSION) def testSlashTwo(self): (code, _, data) = self._Test(http.HTTP_GET, "/2", "", None) self.assertEqual(code, http.HTTP_OK) self.assertTrue(data is None) def testFeatures(self): (code, _, data) = self._Test(http.HTTP_GET, "/2/features", "", None) self.assertEqual(code, http.HTTP_OK) self.assertEqual(set(data), set(rapi.rlib2.ALL_FEATURES)) def testPutInstances(self): (code, _, data) = self._Test(http.HTTP_PUT, "/2/instances", "", None) self.assertEqual(code, http.HttpNotImplemented.code) self.assertTrue(data["message"].startswith("Method PUT is unsupported")) def testPostInstancesNoAuth(self): (code, _, _) = self._Test(http.HTTP_POST, "/2/instances", "", None) self.assertEqual(code, http.HttpUnauthorized.code) def testRequestWithUnsupportedMediaType(self): for fn in [lambda s: s, lambda s: s.upper(), lambda s: s.title()]: headers = rapi.testutils._FormatHeaders([ "%s: %s" % (http.HTTP_CONTENT_TYPE, fn("un/supported/media/type")), ]) (code, _, data) = self._Test(http.HTTP_GET, "/", headers, "body") self.assertEqual(code, http.HttpUnsupportedMediaType.code) self.assertEqual(data["message"], "Unsupported Media Type") def testRequestWithInvalidJsonData(self): body = "_this/is/no'valid.json" self.assertRaises(Exception, serializer.LoadJson, body) headers = rapi.testutils._FormatHeaders([ "%s: %s" % (http.HTTP_CONTENT_TYPE, http.HTTP_APP_JSON), ]) (code, _, data) = self._Test(http.HTTP_GET, "/", headers, body) self.assertEqual(code, http.HttpBadRequest.code) self.assertEqual(data["message"], "Unable to parse JSON data") def testUnsupportedAuthScheme(self): headers = rapi.testutils._FormatHeaders([ "%s: %s" % (http.HTTP_AUTHORIZATION, "Unsupported scheme"), ]) (code, _, _) = self._Test(http.HTTP_POST, "/2/instances", headers, "") self.assertEqual(code, http.HttpUnauthorized.code) def testIncompleteBasicAuth(self): headers = rapi.testutils._FormatHeaders([ "%s: Basic" % http.HTTP_AUTHORIZATION, ]) (code, _, data) = self._Test(http.HTTP_POST, "/2/instances", headers, "") self.assertEqual(code, http.HttpBadRequest.code) self.assertEqual(data["message"], "Basic authentication requires credentials") def testInvalidBasicAuth(self): for auth in ["!invalid=base!64.", testutils.b64encode_string(" "), testutils.b64encode_string("missingcolonchar")]: headers = rapi.testutils._FormatHeaders([ "%s: Basic %s" % (http.HTTP_AUTHORIZATION, auth), ]) (code, _, data) = self._Test(http.HTTP_POST, "/2/instances", headers, "") self.assertEqual(code, http.HttpUnauthorized.code) @staticmethod def _MakeAuthHeaders(username, password, correct_password): if correct_password: pw = password else: pw = "wrongpass" authtok = "%s:%s" % (username, pw) return rapi.testutils._FormatHeaders([ "%s: Basic %s" % (http.HTTP_AUTHORIZATION, testutils.b64encode_string(authtok)), "%s: %s" % (http.HTTP_CONTENT_TYPE, http.HTTP_APP_JSON), ]) def testQueryAuth(self): username = "admin" password = "2046920054" header_fn = compat.partial(self._MakeAuthHeaders, username, password) def _LookupUserNoWrite(name): if name == username: return http.auth.PasswordFileUser(name, password, []) else: return None for access in [rapi.RAPI_ACCESS_WRITE, rapi.RAPI_ACCESS_READ]: def _LookupUserWithWrite(name): if name == username: return http.auth.PasswordFileUser(name, password, [ access, ]) else: return None for qr in constants.QR_VIA_RAPI: # The /2/query resource has somewhat special rules for authentication as # it can be used to retrieve critical information path = "/2/query/%s" % qr for method in rapi.baserlib._SUPPORTED_METHODS: # No authorization (code, _, _) = self._Test(method, path, "", "") if method in (http.HTTP_DELETE, http.HTTP_POST): self.assertEqual(code, http.HttpNotImplemented.code) continue self.assertEqual(code, http.HttpUnauthorized.code) # Incorrect user (code, _, _) = self._Test(method, path, header_fn(True), "", user_fn=self._LookupWrongUser) self.assertEqual(code, http.HttpUnauthorized.code) # User has no write access, but the password is correct (code, _, _) = self._Test(method, path, header_fn(True), "", user_fn=_LookupUserNoWrite) self.assertEqual(code, http.HttpForbidden.code) # Wrong password and no write access (code, _, _) = self._Test(method, path, header_fn(False), "", user_fn=_LookupUserNoWrite) self.assertEqual(code, http.HttpUnauthorized.code) # Wrong password with write access (code, _, _) = self._Test(method, path, header_fn(False), "", user_fn=_LookupUserWithWrite) self.assertEqual(code, http.HttpUnauthorized.code) # Prepare request information if method == http.HTTP_PUT: reqpath = path body = serializer.DumpJson({ "fields": ["name"], }) elif method == http.HTTP_GET: reqpath = "%s?fields=name" % path body = "" else: self.fail("Unknown method '%s'" % method) # User has write access, password is correct (code, _, data) = self._Test(method, reqpath, header_fn(True), body, user_fn=_LookupUserWithWrite, luxi_client=_FakeLuxiClientForQuery) self.assertEqual(code, http.HTTP_OK) self.assertTrue(objects.QueryResponse.FromDict(data)) def testConsole(self): path = "/2/instances/inst1.example.com/console" for method in rapi.baserlib._SUPPORTED_METHODS: for reqauth in [False, True]: # No authorization (code, _, _) = self._Test(method, path, "", "", reqauth=reqauth) if method == http.HTTP_GET or reqauth: self.assertEqual(code, http.HttpUnauthorized.code) else: self.assertEqual(code, http.HttpNotImplemented.code) class _FakeLuxiClientForQuery: def __init__(self, *args, **kwargs): pass def Query(self, *args): return objects.QueryResponse(fields=[]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.ssconf_unittest.py000075500000000000000000000244061476477700300235420ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.ssconf""" import os import unittest import tempfile import shutil import errno from unittest import mock from ganeti import utils from ganeti import constants from ganeti import errors from ganeti import ssconf import testutils class TestReadSsconfFile(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testReadDirectory(self): self.assertRaises(EnvironmentError, ssconf.ReadSsconfFile, self.tmpdir) def testNonExistantFile(self): testfile = utils.PathJoin(self.tmpdir, "does.not.exist") self.assertFalse(os.path.exists(testfile)) try: ssconf.ReadSsconfFile(testfile) except EnvironmentError as err: self.assertEqual(err.errno, errno.ENOENT) else: self.fail("Exception was not raised") def testEmptyFile(self): testfile = utils.PathJoin(self.tmpdir, "empty") utils.WriteFile(testfile, data="") self.assertEqual(ssconf.ReadSsconfFile(testfile), "") def testSingleLine(self): testfile = utils.PathJoin(self.tmpdir, "data") for nl in range(0, 10): utils.WriteFile(testfile, data="Hello World" + ("\n" * nl)) self.assertEqual(ssconf.ReadSsconfFile(testfile), "Hello World") def testExactlyMaxSize(self): testfile = utils.PathJoin(self.tmpdir, "data") data = "A" * ssconf._MAX_SIZE utils.WriteFile(testfile, data=data) self.assertEqual(os.path.getsize(testfile), ssconf._MAX_SIZE) self.assertEqual(ssconf.ReadSsconfFile(testfile), data) def testLargeFile(self): testfile = utils.PathJoin(self.tmpdir, "data") for size in [ssconf._MAX_SIZE + 1, ssconf._MAX_SIZE * 2]: utils.WriteFile(testfile, data="A" * size) self.assertTrue(os.path.getsize(testfile) > ssconf._MAX_SIZE) self.assertRaises(RuntimeError, ssconf.ReadSsconfFile, testfile) class TestSimpleStore(unittest.TestCase): def setUp(self): self._tmpdir = tempfile.mkdtemp() self.ssdir = utils.PathJoin(self._tmpdir, "files") lockfile = utils.PathJoin(self._tmpdir, "lock") os.mkdir(self.ssdir) self.sstore = ssconf.SimpleStore(cfg_location=self.ssdir, _lockfile=lockfile) def tearDown(self): shutil.rmtree(self._tmpdir) def _ReadSsFile(self, filename): return utils.ReadFile(utils.PathJoin(self.ssdir, "ssconf_%s" % filename)) def testInvalidKey(self): self.assertRaises(errors.ProgrammerError, self.sstore.KeyToFilename, "not a valid key") self.assertRaises(errors.ProgrammerError, self.sstore._ReadFile, "not a valid key") def testKeyToFilename(self): for key in ssconf._VALID_KEYS: result = self.sstore.KeyToFilename(key) self.assertTrue(utils.IsBelowDir(self.ssdir, result)) self.assertTrue(os.path.basename(result).startswith("ssconf_")) def testReadFileNonExistingFile(self): filename = self.sstore.KeyToFilename(constants.SS_CLUSTER_NAME) self.assertFalse(os.path.exists(filename)) try: self.sstore._ReadFile(constants.SS_CLUSTER_NAME) except errors.ConfigurationError as err: self.assertTrue(str(err).startswith("Can't read ssconf file")) else: self.fail("Exception was not raised") for default in ["", "Hello World", 0, 100]: self.assertFalse(os.path.exists(filename)) result = self.sstore._ReadFile(constants.SS_CLUSTER_NAME, default=default) self.assertEqual(result, default) def testReadFile(self): utils.WriteFile(self.sstore.KeyToFilename(constants.SS_CLUSTER_NAME), data="cluster.example.com") self.assertEqual(self.sstore._ReadFile(constants.SS_CLUSTER_NAME), "cluster.example.com") self.assertEqual(self.sstore._ReadFile(constants.SS_CLUSTER_NAME, default="something.example.com"), "cluster.example.com") def testReadAllNoFiles(self): self.assertEqual(self.sstore.ReadAll(), {}) def testReadAllSingleFile(self): utils.WriteFile(self.sstore.KeyToFilename(constants.SS_CLUSTER_NAME), data="cluster.example.com") self.assertEqual(self.sstore.ReadAll(), { constants.SS_CLUSTER_NAME: "cluster.example.com", }) def testWriteFiles(self): values = { constants.SS_CLUSTER_NAME: "cluster.example.com", constants.SS_CLUSTER_TAGS: "value\nwith\nnewlines\n", constants.SS_INSTANCE_LIST: "", } self.sstore.WriteFiles(values) self.assertEqual(sorted(os.listdir(self.ssdir)), sorted([ "ssconf_cluster_name", "ssconf_cluster_tags", "ssconf_instance_list", ])) self.assertEqual(self._ReadSsFile(constants.SS_CLUSTER_NAME), "cluster.example.com\n") self.assertEqual(self._ReadSsFile(constants.SS_CLUSTER_TAGS), "value\nwith\nnewlines\n") self.assertEqual(self._ReadSsFile(constants.SS_INSTANCE_LIST), "") def testWriteFilesUnknownKey(self): values = { "unknown key": "value", } self.assertRaises(errors.ProgrammerError, self.sstore.WriteFiles, values, dry_run=True) self.assertEqual(os.listdir(self.ssdir), []) def testWriteFilesDryRun(self): values = { constants.SS_CLUSTER_NAME: "cluster.example.com", } self.sstore.WriteFiles(values, dry_run=True) self.assertEqual(os.listdir(self.ssdir), []) def testWriteFilesNoValues(self): for dry_run in [False, True]: self.sstore.WriteFiles({}, dry_run=dry_run) self.assertEqual(os.listdir(self.ssdir), []) def testWriteFilesTooLong(self): values = { constants.SS_INSTANCE_LIST: "A" * ssconf._MAX_SIZE, } for dry_run in [False, True]: try: self.sstore.WriteFiles(values, dry_run=dry_run) except errors.ConfigurationError as err: self.assertTrue(str(err).startswith("Value 'instance_list' has")) else: self.fail("Exception was not raised") self.assertEqual(os.listdir(self.ssdir), []) def testEnabledUserShutdown(self): self.sstore._ReadFile = mock.Mock(return_value="True") result = self.sstore.GetEnabledUserShutdown() self.assertTrue(isinstance(result, bool)) self.assertEqual(result, True) self.sstore._ReadFile = mock.Mock(return_value="False") result = self.sstore.GetEnabledUserShutdown() self.assertTrue(isinstance(result, bool)) self.assertEqual(result, False) def testGetNodesVmCapable(self): def _ToBool(x): return x == "True" vm_capable = [("node1.example.com", "True"), ("node2.example.com", "False")] ssconf_file_content = '\n'.join("%s=%s" % (key, value) for (key, value) in vm_capable) self.sstore._ReadFile = mock.Mock(return_value=ssconf_file_content) result = self.sstore.GetNodesVmCapable() for (key, value) in vm_capable: self.assertTrue(isinstance(key, str)) self.assertTrue(key in result) self.assertTrue(isinstance(result[key], bool)) self.assertEqual(_ToBool(value), result[key]) def testGetHvparamsForHypervisor(self): hvparams = [("a", "A"), ("b", "B"), ("c", "C")] ssconf_file_content = '\n'.join("%s=%s" % (key, value) for (key, value) in hvparams) self.sstore._ReadFile = mock.Mock(return_value=ssconf_file_content) result = self.sstore.GetHvparamsForHypervisor("foo") for (key, value) in hvparams: self.assertTrue(key in result) self.assertEqual(value, result[key]) class TestVerifyClusterName(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testMissingFile(self): tmploc = utils.PathJoin(self.tmpdir, "does-not-exist") ssconf.VerifyClusterName(NotImplemented, _cfg_location=tmploc) def testMatchingName(self): tmpfile = utils.PathJoin(self.tmpdir, "ssconf_cluster_name") for content in ["cluster.example.com", "cluster.example.com\n\n"]: utils.WriteFile(tmpfile, data=content) ssconf.VerifyClusterName("cluster.example.com", _cfg_location=self.tmpdir) def testNameMismatch(self): tmpfile = utils.PathJoin(self.tmpdir, "ssconf_cluster_name") for content in ["something.example.com", "foobar\n\ncluster.example.com"]: utils.WriteFile(tmpfile, data=content) self.assertRaises(errors.GenericError, ssconf.VerifyClusterName, "cluster.example.com", _cfg_location=self.tmpdir) class TestVerifyKeys(unittest.TestCase): def testNoKeys(self): ssconf.VerifyKeys({}) def testValidKeys(self): ssconf.VerifyKeys(ssconf._VALID_KEYS) for key in ssconf._VALID_KEYS: ssconf.VerifyKeys([key]) def testInvalidKeys(self): for key in ["", ".", " ", "foo", "bar", "HelloWorld"]: self.assertRaises(errors.GenericError, ssconf.VerifyKeys, [key]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.ssh_unittest.py000075500000000000000000000507031476477700300230430ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the ssh module""" import os import tempfile import unittest import shutil import testutils import mocks from ganeti import constants from ganeti import utils from ganeti import ssh from ganeti import errors class TestKnownHosts(testutils.GanetiTestCase): """Test case for function writing the known_hosts file""" def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpfile = self._CreateTempFile() def test(self): cfg = mocks.FakeConfig() ssh.WriteKnownHostsFile(cfg, self.tmpfile) self.assertFileContent(self.tmpfile, "%s ssh-rsa %s\n%s ssh-dss %s\n" % (cfg.GetClusterName(), mocks.FAKE_CLUSTER_KEY, cfg.GetClusterName(), mocks.FAKE_CLUSTER_KEY)) class TestGetUserFiles(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) @staticmethod def _GetNoHomedir(_): return None def _GetTempHomedir(self, _): return self.tmpdir def testNonExistantUser(self): for kind in constants.SSHK_ALL: self.assertRaises(errors.OpExecError, ssh.GetUserFiles, "example", kind=kind, _homedir_fn=self._GetNoHomedir) def testUnknownKind(self): kind = "something-else" assert kind not in constants.SSHK_ALL self.assertRaises(errors.ProgrammerError, ssh.GetUserFiles, "example4645", kind=kind, _homedir_fn=self._GetTempHomedir) self.assertEqual(os.listdir(self.tmpdir), []) def testNoSshDirectory(self): for kind in constants.SSHK_ALL: self.assertRaises(errors.OpExecError, ssh.GetUserFiles, "example29694", kind=kind, _homedir_fn=self._GetTempHomedir) self.assertEqual(os.listdir(self.tmpdir), []) def testSshIsFile(self): utils.WriteFile(os.path.join(self.tmpdir, ".ssh"), data="") for kind in constants.SSHK_ALL: self.assertRaises(errors.OpExecError, ssh.GetUserFiles, "example26237", kind=kind, _homedir_fn=self._GetTempHomedir) self.assertEqual(os.listdir(self.tmpdir), [".ssh"]) def testMakeSshDirectory(self): sshdir = os.path.join(self.tmpdir, ".ssh") self.assertEqual(os.listdir(self.tmpdir), []) for kind in constants.SSHK_ALL: ssh.GetUserFiles("example20745", mkdir=True, kind=kind, _homedir_fn=self._GetTempHomedir) self.assertEqual(os.listdir(self.tmpdir), [".ssh"]) self.assertEqual(os.stat(sshdir).st_mode & 0o777, 0o700) def testFilenames(self): sshdir = os.path.join(self.tmpdir, ".ssh") os.mkdir(sshdir) for kind in constants.SSHK_ALL: result = ssh.GetUserFiles("example15103", mkdir=False, kind=kind, _homedir_fn=self._GetTempHomedir) self.assertEqual(result, [ os.path.join(self.tmpdir, ".ssh", "id_%s" % kind), os.path.join(self.tmpdir, ".ssh", "id_%s.pub" % kind), os.path.join(self.tmpdir, ".ssh", "authorized_keys"), ]) self.assertEqual(os.listdir(self.tmpdir), [".ssh"]) self.assertEqual(os.listdir(sshdir), []) def testNoDirCheck(self): self.assertEqual(os.listdir(self.tmpdir), []) for kind in constants.SSHK_ALL: ssh.GetUserFiles("example14528", mkdir=False, dircheck=False, kind=kind, _homedir_fn=self._GetTempHomedir) self.assertEqual(os.listdir(self.tmpdir), []) def testGetAllUserFiles(self): result = ssh.GetAllUserFiles("example7475", mkdir=False, dircheck=False, _homedir_fn=self._GetTempHomedir) self.assertEqual(result, (os.path.join(self.tmpdir, ".ssh", "authorized_keys"), { constants.SSHK_RSA: (os.path.join(self.tmpdir, ".ssh", "id_rsa"), os.path.join(self.tmpdir, ".ssh", "id_rsa.pub")), constants.SSHK_DSA: (os.path.join(self.tmpdir, ".ssh", "id_dsa"), os.path.join(self.tmpdir, ".ssh", "id_dsa.pub")), constants.SSHK_ECDSA: (os.path.join(self.tmpdir, ".ssh", "id_ecdsa"), os.path.join(self.tmpdir, ".ssh", "id_ecdsa.pub")), })) self.assertEqual(os.listdir(self.tmpdir), []) def testGetAllUserFilesNoDirectoryNoMkdir(self): self.assertRaises(errors.OpExecError, ssh.GetAllUserFiles, "example17270", mkdir=False, dircheck=True, _homedir_fn=self._GetTempHomedir) self.assertEqual(os.listdir(self.tmpdir), []) class TestSshKeys(testutils.GanetiTestCase): """Test case for the AddAuthorizedKey function""" KEY_A = "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a" KEY_B = ('command="/usr/bin/fooserver -t --verbose",from="198.51.100.4" ' "ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b") def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpname = self._CreateTempFile() handle = open(self.tmpname, "w") try: handle.write("%s\n" % TestSshKeys.KEY_A) handle.write("%s\n" % TestSshKeys.KEY_B) finally: handle.close() def testHasAuthorizedKey(self): self.assertTrue(ssh.HasAuthorizedKey(self.tmpname, self.KEY_A)) self.assertFalse(ssh.HasAuthorizedKey( self.tmpname, "I am the key of the pink bunny!")) def testAddingNewKey(self): ssh.AddAuthorizedKey(self.tmpname, "ssh-dss AAAAB3NzaC1kc3MAAACB root@test") self.assertFileContent(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n" "ssh-dss AAAAB3NzaC1kc3MAAACB root@test\n") def testAddingDuplicateKeys(self): ssh.AddAuthorizedKey(self.tmpname, "ssh-dss AAAAB3NzaC1kc3MAAACB root@test") ssh.AddAuthorizedKeys(self.tmpname, ["ssh-dss AAAAB3NzaC1kc3MAAACB root@test", "ssh-dss AAAAB3NzaC1kc3MAAACB root@test"]) self.assertFileContent(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n" "ssh-dss AAAAB3NzaC1kc3MAAACB root@test\n") def testAddingSeveralKeysAtOnce(self): ssh.AddAuthorizedKeys(self.tmpname, ["aaa", "bbb", "ccc"]) self.assertFileContent(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n" "aaa\nbbb\nccc\n") ssh.AddAuthorizedKeys(self.tmpname, ["bbb", "ddd", "eee"]) self.assertFileContent(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n" "aaa\nbbb\nccc\nddd\neee\n") def testAddingAlmostButNotCompletelyTheSameKey(self): ssh.AddAuthorizedKey(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@test") # Only significant fields are compared, therefore the key won't be # updated/added self.assertFileContent(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n") def testAddingExistingKeyWithSomeMoreSpaces(self): ssh.AddAuthorizedKey(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a") ssh.AddAuthorizedKey(self.tmpname, "ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22") self.assertFileContent(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n" "ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22\n") def testRemovingExistingKeyWithSomeMoreSpaces(self): ssh.RemoveAuthorizedKey(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a") self.assertFileContent(self.tmpname, 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n") def testRemovingNonExistingKey(self): ssh.RemoveAuthorizedKey(self.tmpname, "ssh-dss AAAAB3Nsdfj230xxjxJjsjwjsjdjU root@test") self.assertFileContent(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n") def testAddingNewKeys(self): ssh.AddAuthorizedKeys(self.tmpname, ["ssh-dss AAAAB3NzaC1kc3MAAACB root@test"]) self.assertFileContent(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n" "ssh-dss AAAAB3NzaC1kc3MAAACB root@test\n") ssh.AddAuthorizedKeys(self.tmpname, ["ssh-dss AAAAB3asdfasdfaYTUCB laracroft@test", "ssh-dss AasdfliuobaosfMAAACB frodo@test"]) self.assertFileContent(self.tmpname, "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" 'command="/usr/bin/fooserver -t --verbose",from="198.51.100.4"' " ssh-dss AAAAB3NzaC1w520smc01ms0jfJs22 root@key-b\n" "ssh-dss AAAAB3NzaC1kc3MAAACB root@test\n" "ssh-dss AAAAB3asdfasdfaYTUCB laracroft@test\n" "ssh-dss AasdfliuobaosfMAAACB frodo@test\n") def testOtherKeyTypes(self): key_rsa = "ssh-rsa AAAAimnottypingallofthathere0jfJs22 test@test" key_ed25519 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOlcZ6cpQTGow0LZECRHWn9"\ "7Yvn16J5un501T/RcbfuF fast@secure" key_ecdsa = "ecdsa-sha2-nistp256 AAAAE2VjZHNtoolongk/TNhVbEg= secure@secure" def _ToFileContent(keys): return '\n'.join(keys) + '\n' ssh.AddAuthorizedKeys(self.tmpname, [key_rsa, key_ed25519, key_ecdsa]) self.assertFileContent(self.tmpname, _ToFileContent([self.KEY_A, self.KEY_B, key_rsa, key_ed25519, key_ecdsa])) ssh.RemoveAuthorizedKey(self.tmpname, key_ed25519) self.assertFileContent(self.tmpname, _ToFileContent([self.KEY_A, self.KEY_B, key_rsa, key_ecdsa])) ssh.RemoveAuthorizedKey(self.tmpname, key_rsa) ssh.RemoveAuthorizedKey(self.tmpname, key_ecdsa) self.assertFileContent(self.tmpname, _ToFileContent([self.KEY_A, self.KEY_B])) class TestPublicSshKeys(testutils.GanetiTestCase): """Test case for the handling of the list of public ssh keys.""" KEY_A = "ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a" KEY_B = "ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b" UUID_1 = "123-456" UUID_2 = "789-ABC" def setUp(self): testutils.GanetiTestCase.setUp(self) def testAddingAndRemovingPubKey(self): pub_key_file = self._CreateTempFile() ssh.AddPublicKey(self.UUID_1, self.KEY_A, key_file=pub_key_file) ssh.AddPublicKey(self.UUID_2, self.KEY_B, key_file=pub_key_file) self.assertFileContent(pub_key_file, "123-456 ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" "789-ABC ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n") ssh.RemovePublicKey(self.UUID_2, key_file=pub_key_file) self.assertFileContent(pub_key_file, "123-456 ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n") def testAddingExistingPubKey(self): expected_file_content = \ "123-456 ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" + \ "789-ABC ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n" pub_key_file = self._CreateTempFile() ssh.AddPublicKey(self.UUID_1, self.KEY_A, key_file=pub_key_file) ssh.AddPublicKey(self.UUID_2, self.KEY_B, key_file=pub_key_file) self.assertFileContent(pub_key_file, expected_file_content) ssh.AddPublicKey(self.UUID_1, self.KEY_A, key_file=pub_key_file) self.assertFileContent(pub_key_file, expected_file_content) ssh.AddPublicKey(self.UUID_1, self.KEY_B, key_file=pub_key_file) self.assertFileContent(pub_key_file, "123-456 ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" "789-ABC ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n" "123-456 ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n") def testRemoveNonexistingKey(self): pub_key_file = self._CreateTempFile() ssh.AddPublicKey(self.UUID_1, self.KEY_B, key_file=pub_key_file) self.assertFileContent(pub_key_file, "123-456 ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n") ssh.RemovePublicKey(self.UUID_2, key_file=pub_key_file) self.assertFileContent(pub_key_file, "123-456 ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n") def testRemoveAllExistingKeys(self): pub_key_file = self._CreateTempFile() ssh.AddPublicKey(self.UUID_1, self.KEY_A, key_file=pub_key_file) ssh.AddPublicKey(self.UUID_1, self.KEY_B, key_file=pub_key_file) self.assertFileContent(pub_key_file, "123-456 ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" "123-456 ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n") ssh.RemovePublicKey(self.UUID_1, key_file=pub_key_file) self.assertFileContent(pub_key_file, "") def testRemoveKeyFromEmptyFile(self): pub_key_file = self._CreateTempFile() ssh.RemovePublicKey(self.UUID_2, key_file=pub_key_file) self.assertFileContent(pub_key_file, "") def testRetrieveKeys(self): pub_key_file = self._CreateTempFile() ssh.AddPublicKey(self.UUID_1, self.KEY_A, key_file=pub_key_file) ssh.AddPublicKey(self.UUID_2, self.KEY_B, key_file=pub_key_file) result = ssh.QueryPubKeyFile(self.UUID_1, key_file=pub_key_file) self.assertEqual([self.KEY_A], result[self.UUID_1]) target_uuids = [self.UUID_1, self.UUID_2, "non-existing-UUID"] result = ssh.QueryPubKeyFile(target_uuids, key_file=pub_key_file) self.assertEqual([self.KEY_A], result[self.UUID_1]) self.assertEqual([self.KEY_B], result[self.UUID_2]) self.assertEqual(2, len(result)) # Query all keys target_uuids = None result = ssh.QueryPubKeyFile(target_uuids, key_file=pub_key_file) self.assertEqual([self.KEY_A], result[self.UUID_1]) self.assertEqual([self.KEY_B], result[self.UUID_2]) def testReplaceNameByUuid(self): pub_key_file = self._CreateTempFile() name = "my.precious.node" ssh.AddPublicKey(name, self.KEY_A, key_file=pub_key_file) ssh.AddPublicKey(self.UUID_2, self.KEY_A, key_file=pub_key_file) ssh.AddPublicKey(name, self.KEY_B, key_file=pub_key_file) self.assertFileContent(pub_key_file, "my.precious.node ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" "789-ABC ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" "my.precious.node ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n") ssh.ReplaceNameByUuid(self.UUID_1, name, key_file=pub_key_file) self.assertFileContent(pub_key_file, "123-456 ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" "789-ABC ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" "123-456 ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n") def testParseEmptyLines(self): pub_key_file = self._CreateTempFile() ssh.AddPublicKey(self.UUID_1, self.KEY_A, key_file=pub_key_file) # Add an empty line fd = open(pub_key_file, 'a') fd.write("\n") fd.close() ssh.AddPublicKey(self.UUID_2, self.KEY_B, key_file=pub_key_file) # Add a whitespace line fd = open(pub_key_file, 'a') fd.write(" \n") fd.close() result = ssh.QueryPubKeyFile(self.UUID_1, key_file=pub_key_file) self.assertEqual([self.KEY_A], result[self.UUID_1]) def testClearPubKeyFile(self): pub_key_file = self._CreateTempFile() ssh.AddPublicKey(self.UUID_2, self.KEY_A, key_file=pub_key_file) ssh.ClearPubKeyFile(key_file=pub_key_file) self.assertFileContent(pub_key_file, "") def testOverridePubKeyFile(self): pub_key_file = self._CreateTempFile() key_map = {self.UUID_1: [self.KEY_A, self.KEY_B], self.UUID_2: [self.KEY_A]} ssh.OverridePubKeyFile(key_map, key_file=pub_key_file) self.assertFileContent(pub_key_file, "123-456 ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n" "123-456 ssh-dss BAasjkakfa234SFSFDA345462AAAB root@key-b\n" "789-ABC ssh-dss AAAAB3NzaC1w5256closdj32mZaQU root@key-a\n") class TestGetUserFiles(testutils.GanetiTestCase): _PRIV_KEY = "my private key" _PUB_KEY = "my public key" _AUTH_KEYS = "a\nb\nc" def _setUpFakeKeys(self): ssh_tmpdir = os.path.join(self.tmpdir, ".ssh") os.makedirs(ssh_tmpdir) self.priv_filename = os.path.join(ssh_tmpdir, "id_rsa") utils.WriteFile(self.priv_filename, data=self._PRIV_KEY) self.pub_filename = os.path.join(ssh_tmpdir, "id_rsa.pub") utils.WriteFile(self.pub_filename, data=self._PUB_KEY) self.auth_filename = os.path.join(ssh_tmpdir, "authorized_keys") utils.WriteFile(self.auth_filename, data=self._AUTH_KEYS) def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() self._setUpFakeKeys() def tearDown(self): shutil.rmtree(self.tmpdir) def _GetTempHomedir(self, _): return self.tmpdir def testNewKeysOverrideOldKeys(self): ssh.InitSSHSetup("rsa", 2048, _homedir_fn=self._GetTempHomedir) self.assertFileContentNotEqual(self.priv_filename, self._PRIV_KEY) self.assertFileContentNotEqual(self.pub_filename, self._PUB_KEY) def testSuffix(self): suffix = "_pinkbunny" ssh.InitSSHSetup("rsa", 2048, _homedir_fn=self._GetTempHomedir, _suffix=suffix) self.assertFileContent(self.priv_filename, self._PRIV_KEY) self.assertFileContent(self.pub_filename, self._PUB_KEY) self.assertTrue(os.path.exists(self.priv_filename + suffix)) self.assertTrue(os.path.exists(self.priv_filename + suffix + ".pub")) class TestDetermineKeyBits(): def testCompleteness(self): self.assertEqual(constants.SSHK_ALL, list(ssh.SSH_KEY_VALID_BITS)) def testAdoptDefault(self): self.assertEqual(2048, DetermineKeyBits("rsa", None, None, None)) self.assertEqual(1024, DetermineKeyBits("dsa", None, None, None)) def testAdoptOldKeySize(self): self.assertEqual(4098, DetermineKeyBits("rsa", None, "rsa", 4098)) self.assertEqual(2048, DetermineKeyBits("rsa", None, "dsa", 1024)) def testDsaSpecificValues(self): self.assertRaises(errors.OpPrereqError, DetermineKeyBits, "dsa", 2048, None, None) self.assertRaises(errors.OpPrereqError, DetermineKeyBits, "dsa", 512, None, None) self.assertEqual(1024, DetermineKeyBits("dsa", None, None, None)) def testEcdsaSpecificValues(self): self.assertRaises(errors.OpPrereqError, DetermineKeyBits, "ecdsa", 2048, None, None) for b in [256, 384, 521]: self.assertEqual(b, DetermineKeyBits("ecdsa", b, None, None)) def testRsaSpecificValues(self): self.assertRaises(errors.OpPrereqError, DetermineKeyBits, "dsa", 766, None, None) for b in [768, 769, 2048, 2049, 4096]: self.assertEqual(b, DetermineKeyBits("rsa", b, None, None)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.storage.bdev_unittest.py000075500000000000000000000657621476477700300246440ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2012, 2013, 2016 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the bdev module""" import os import random import unittest from ganeti import compat from ganeti import constants from ganeti import errors from ganeti import objects from ganeti import utils from ganeti.storage import bdev import testutils def _FakeRunCmd(success, stdout, cmd): if success: exit_code = 0 else: exit_code = 1 return utils.RunResult(exit_code, None, stdout, "", cmd, utils.process._TIMEOUT_NONE, 5) class FakeStatResult(object): def __init__(self, st_mode): self.st_mode = st_mode self.st_rdev = 0 class TestRADOSBlockDevice(testutils.GanetiTestCase): """Tests for bdev.RADOSBlockDevice volumes """ def setUp(self): """Set up input data""" testutils.GanetiTestCase.setUp(self) self.plain_output_old_ok = \ testutils.ReadTestData("bdev-rbd/plain_output_old_ok.txt") self.plain_output_old_no_matches = \ testutils.ReadTestData("bdev-rbd/plain_output_old_no_matches.txt") self.plain_output_old_extra_matches = \ testutils.ReadTestData("bdev-rbd/plain_output_old_extra_matches.txt") self.plain_output_old_empty = \ testutils.ReadTestData("bdev-rbd/plain_output_old_empty.txt") self.plain_output_new_ok = \ testutils.ReadTestData("bdev-rbd/plain_output_new_ok.txt") self.plain_output_new_no_matches = \ testutils.ReadTestData("bdev-rbd/plain_output_new_no_matches.txt") self.plain_output_new_extra_matches = \ testutils.ReadTestData("bdev-rbd/plain_output_new_extra_matches.txt") # This file is completely empty, and as such it's not shipped. self.plain_output_new_empty = "" self.json_output_ok = testutils.ReadTestData("bdev-rbd/json_output_ok.txt") self.json_output_no_matches = \ testutils.ReadTestData("bdev-rbd/json_output_no_matches.txt") self.json_output_extra_matches = \ testutils.ReadTestData("bdev-rbd/json_output_extra_matches.txt") self.json_output_empty = \ testutils.ReadTestData("bdev-rbd/json_output_empty.txt") self.output_invalid = testutils.ReadTestData("bdev-rbd/output_invalid.txt") self.volume_name = "d7ab910a-4933-4ffe-88d0-faf2ce31390a.rbd.disk0" self.pool_name = "rbd" self.test_unique_id = ("rbd", self.volume_name) self.test_params = { constants.LDP_POOL: "fake_pool" } def testParseRbdShowmappedJson(self): parse_function = bdev.RADOSBlockDevice._ParseRbdShowmappedJson self.assertEqual(parse_function(self.json_output_ok, self.pool_name, self.volume_name), "/dev/rbd3") self.assertEqual(parse_function(self.json_output_ok, "fake_pool", self.volume_name), None) self.assertEqual(parse_function(self.json_output_empty, self.pool_name, self.volume_name), None) self.assertEqual(parse_function(self.json_output_no_matches, self.pool_name, self.volume_name), None) self.assertRaises(errors.BlockDeviceError, parse_function, self.json_output_extra_matches, self.pool_name, self.volume_name) self.assertRaises(errors.BlockDeviceError, parse_function, self.output_invalid, self.pool_name, self.volume_name) def testParseRbdShowmappedPlain(self): parse_function = bdev.RADOSBlockDevice._ParseRbdShowmappedPlain self.assertEqual(parse_function(self.plain_output_new_ok, self.volume_name), "/dev/rbd3") self.assertEqual(parse_function(self.plain_output_old_ok, self.volume_name), "/dev/rbd3") self.assertEqual(parse_function(self.plain_output_new_empty, self.volume_name), None) self.assertEqual(parse_function(self.plain_output_old_empty, self.volume_name), None) self.assertEqual(parse_function(self.plain_output_new_no_matches, self.volume_name), None) self.assertEqual(parse_function(self.plain_output_old_no_matches, self.volume_name), None) self.assertRaises(errors.BlockDeviceError, parse_function, self.plain_output_new_extra_matches, self.volume_name) self.assertRaises(errors.BlockDeviceError, parse_function, self.plain_output_old_extra_matches, self.volume_name) self.assertRaises(errors.BlockDeviceError, parse_function, self.output_invalid, self.volume_name) @testutils.patch_object(utils, "RunCmd") @testutils.patch_object(bdev.RADOSBlockDevice, "_UnmapVolumeFromBlockdev") @testutils.patch_object(bdev.RADOSBlockDevice, "Attach") def testRADOSBlockDeviceImport(self, attach_mock, unmap_mock, run_cmd_mock): """Test for bdev.RADOSBlockDevice.Import()""" # Set up the mock objects return values attach_mock.return_value = True run_cmd_mock.return_value = _FakeRunCmd(True, "", "") # Create a fake rbd volume inst = bdev.RADOSBlockDevice(self.test_unique_id, [], 1024, self.test_params, {}) # Desired output command import_cmd = [constants.RBD_CMD, "import", "-p", inst.rbd_pool, "-", inst.rbd_name] self.assertEqual(inst.Import(), import_cmd) @testutils.patch_object(bdev.RADOSBlockDevice, "Attach") def testRADOSBlockDeviceExport(self, attach_mock): """Test for bdev.RADOSBlockDevice.Export()""" # Set up the mock object return value attach_mock.return_value = True # Create a fake rbd volume inst = bdev.RADOSBlockDevice(self.test_unique_id, [], 1024, self.test_params, {}) # Desired output command export_cmd = [constants.RBD_CMD, "export", "-p", inst.rbd_pool, inst.rbd_name, "-"] self.assertEqual(inst.Export(), export_cmd) @testutils.patch_object(utils, "RunCmd") @testutils.patch_object(bdev.RADOSBlockDevice, "Attach") def testRADOSBlockDeviceCreate(self, attach_mock, run_cmd_mock): """Test for bdev.RADOSBlockDevice.Create() success""" attach_mock.return_value = True # This returns a successful RunCmd result run_cmd_mock.return_value = _FakeRunCmd(True, "", "") expect = bdev.RADOSBlockDevice(self.test_unique_id, [], 1024, self.test_params, {}) got = bdev.RADOSBlockDevice.Create(self.test_unique_id, [], 1024, None, self.test_params, False, {}, test_kwarg="test") self.assertEqual(expect, got) @testutils.patch_object(bdev.RADOSBlockDevice, "Attach") def testRADOSBlockDeviceCreateFailure(self, attach_mock): """Test for bdev.RADOSBlockDevice.Create() failure with exclusive_storage enabled """ attach_mock.return_value = True self.assertRaises(errors.ProgrammerError, bdev.RADOSBlockDevice.Create, self.test_unique_id, [], 1024, None, self.test_params, True, {}) @testutils.patch_object(bdev.RADOSBlockDevice, "_MapVolumeToBlockdev") @testutils.patch_object(os, "stat") def testAttach(self, stat_mock, map_mock): """Test for bdev.RADOSBlockDevice.Attach()""" stat_mock.return_value = FakeStatResult(0x6000) # bitmask for S_ISBLK map_mock.return_value = "/fake/path" dev = bdev.RADOSBlockDevice.__new__(bdev.RADOSBlockDevice) dev.unique_id = self.test_unique_id self.assertEqual(dev.Attach(), True) @testutils.patch_object(bdev.RADOSBlockDevice, "_MapVolumeToBlockdev") @testutils.patch_object(os, "stat") def testAttachFailureNotBlockdev(self, stat_mock, map_mock): """Test for bdev.RADOSBlockDevice.Attach() failure, not a blockdev""" stat_mock.return_value = FakeStatResult(0x0) map_mock.return_value = "/fake/path" dev = bdev.RADOSBlockDevice.__new__(bdev.RADOSBlockDevice) dev.unique_id = self.test_unique_id self.assertEqual(dev.Attach(), False) @testutils.patch_object(bdev.RADOSBlockDevice, "_MapVolumeToBlockdev") @testutils.patch_object(os, "stat") def testAttachFailureNoDevice(self, stat_mock, map_mock): """Test for bdev.RADOSBlockDevice.Attach() failure, no device found""" stat_mock.side_effect = OSError("No device found") map_mock.return_value = "/fake/path" dev = bdev.RADOSBlockDevice.__new__(bdev.RADOSBlockDevice) dev.unique_id = self.test_unique_id self.assertEqual(dev.Attach(), False) class TestExclusiveStoragePvs(unittest.TestCase): """Test cases for functions dealing with LVM PV and exclusive storage""" # Allowance for rounding _EPS = 1e-4 _MARGIN = constants.PART_MARGIN + constants.PART_RESERVED + _EPS @staticmethod def _GenerateRandomPvInfo(rnd, name, vg): # Granularity is .01 MiB size = rnd.randint(1024 * 100, 10 * 1024 * 1024 * 100) if rnd.choice([False, True]): free = float(rnd.randint(0, size)) / 100.0 else: free = float(size) / 100.0 size = float(size) / 100.0 attr = "a-" return objects.LvmPvInfo(name=name, vg_name=vg, size=size, free=free, attributes=attr) def testGetStdPvSize(self): """Test cases for bdev.LogicalVolume._GetStdPvSize()""" rnd = random.Random(9517) for _ in range(0, 50): # Identical volumes pvi = self._GenerateRandomPvInfo(rnd, "disk", "myvg") onesize = bdev.LogicalVolume._GetStdPvSize([pvi]) self.assertTrue(onesize <= pvi.size) self.assertTrue(onesize > pvi.size * (1 - self._MARGIN)) for length in range(2, 10): n_size = bdev.LogicalVolume._GetStdPvSize([pvi] * length) self.assertEqual(onesize, n_size) # Mixed volumes for length in range(1, 10): pvlist = [self._GenerateRandomPvInfo(rnd, "disk", "myvg") for _ in range(0, length)] std_size = bdev.LogicalVolume._GetStdPvSize(pvlist) self.assertTrue(compat.all(std_size <= pvi.size for pvi in pvlist)) self.assertTrue(compat.any(std_size > pvi.size * (1 - self._MARGIN) for pvi in pvlist)) pvlist.append(pvlist[0]) p1_size = bdev.LogicalVolume._GetStdPvSize(pvlist) self.assertEqual(std_size, p1_size) def testComputeNumPvs(self): """Test cases for bdev.LogicalVolume._ComputeNumPvs()""" rnd = random.Random(8067) for _ in range(0, 1000): pvlist = [self._GenerateRandomPvInfo(rnd, "disk", "myvg")] lv_size = float(rnd.randint(10 * 100, 1024 * 1024 * 100)) / 100.0 num_pv = bdev.LogicalVolume._ComputeNumPvs(lv_size, pvlist) std_size = bdev.LogicalVolume._GetStdPvSize(pvlist) self.assertTrue(num_pv >= 1) self.assertTrue(num_pv * std_size >= lv_size) self.assertTrue((num_pv - 1) * std_size < lv_size * (1 + self._EPS)) def testGetEmptyPvNames(self): """Test cases for bdev.LogicalVolume._GetEmptyPvNames()""" rnd = random.Random(21126) for _ in range(0, 100): num_pvs = rnd.randint(1, 20) pvlist = [self._GenerateRandomPvInfo(rnd, "disk%d" % n, "myvg") for n in range(0, num_pvs)] for num_req in range(1, num_pvs + 2): epvs = bdev.LogicalVolume._GetEmptyPvNames(pvlist, num_req) epvs_set = compat.UniqueFrozenset(epvs) if len(epvs) > 1: self.assertEqual(len(epvs), len(epvs_set)) for pvi in pvlist: if pvi.name in epvs_set: self.assertEqual(pvi.size, pvi.free) else: # There should be no remaining empty PV when less than the # requeste number of PVs has been returned self.assertTrue(len(epvs) == num_req or pvi.free != pvi.size) class TestLogicalVolume(testutils.GanetiTestCase): """Tests for bdev.LogicalVolume.""" def setUp(self): """Set up test data""" testutils.GanetiTestCase.setUp(self) self.volume_name = "31225655-5775-4356-c212-e8b1e137550a.disk0" self.test_unique_id = ("ganeti", self.volume_name) self.test_params = { constants.LDP_STRIPES: 1 } self.pv_info_return = [objects.LvmPvInfo(name="/dev/sda5", vg_name="xenvg", size=3500000.00, free=5000000.00, attributes="wz--n-", lv_list=[])] self.pv_info_invalid = [objects.LvmPvInfo(name="/dev/s:da5", vg_name="xenvg", size=3500000.00, free=5000000.00, attributes="wz--n-", lv_list=[])] self.pv_info_no_space = [objects.LvmPvInfo(name="/dev/sda5", vg_name="xenvg", size=3500000.00, free=0.00, attributes="wz--n-", lv_list=[])] def testParseLvInfoLine(self): """Tests for LogicalVolume._ParseLvInfoLine.""" broken_lines = [ " toomuch#vg#lv#-wi-ao#253#3#4096.00#2#/dev/abc(20)", " vg#lv#-wi-ao#253#3#4096.00#/dev/abc(20)", " vg#lv#-wi-a#253#3#4096.00#2#/dev/abc(20)", " vg#lv#-wi-ao#25.3#3#4096.00#2#/dev/abc(20)", " vg#lv#-wi-ao#twenty#3#4096.00#2#/dev/abc(20)", " vg#lv#-wi-ao#253#3.1#4096.00#2#/dev/abc(20)", " vg#lv#-wi-ao#253#three#4096.00#2#/dev/abc(20)", " vg#lv#-wi-ao#253#3#four#2#/dev/abc(20)", " vg#lv#-wi-ao#253#3#4096..00#2#/dev/abc(20)", " vg#lv#-wi-ao#253#3#4096.00#2.0#/dev/abc(20)", " vg#lv#-wi-ao#253#3#4096.00#two#/dev/abc(20)", " vg#lv#-wi-ao#253#3#4096.00#2#/dev/abc20", ] for broken in broken_lines: self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume._ParseLvInfoLine, broken, "#") # Examples of good lines from "lvs": # # /dev/something|-wi-ao|253|3|4096.00|2|/dev/sdb(144),/dev/sdc(0) # /dev/somethingelse|-wi-a-|253|4|4096.00|1|/dev/sdb(208) true_out = [ (("vg", "lv"), ("-wi-ao", 253, 3, 4096.00, 2, ["/dev/abc"])), (("vg", "lv"), ("-wi-a-", 253, 7, 4096.00, 4, ["/dev/abc"])), (("vg", "lv"), ("-ri-a-", 253, 4, 4.00, 5, ["/dev/abc", "/dev/def"])), (("vg", "lv"), ("-wc-ao", 15, 18, 4096.00, 32, ["/dev/abc", "/dev/def", "/dev/ghi0"])), # Physical devices might be missing with thin volumes (("vg", "lv"), ("twc-ao", 15, 18, 4096.00, 32, [])), ] for exp in true_out: for sep in "#;|": # NB We get lvs to return vg_name and lv_name separately, but # _ParseLvInfoLine returns a pathname built from these, so we # need to do some extra munging to round-trip this properly. vg_name, lv_name = exp[0] dev = os.environ.get('DM_DEV_DIR', '/dev') devpath = os.path.join(dev, vg_name, lv_name) lvs = exp[1] pvs = ",".join("%s(%s)" % (d, i * 12) for (i, d) in enumerate(lvs[-1])) fmt_str = sep.join((" %s", "%s", "%s", "%d", "%d", "%.2f", "%d", "%s")) lvs_line = fmt_str % ((vg_name, lv_name) + lvs[0:-1] + (pvs,)) parsed = bdev.LogicalVolume._ParseLvInfoLine(lvs_line, sep) self.assertEqual(parsed, (devpath,) + exp[1:]) def testGetLvGlobalInfo(self): """Tests for LogicalVolume.GetLvGlobalInfo.""" good_lines="vg|1|-wi-ao|253|3|4096.00|2|/dev/sda(20)\n" \ "vg|2|-wi-ao|253|3|4096.00|2|/dev/sda(21)\n" expected_output = {"/dev/vg/1": ("-wi-ao", 253, 3, 4096, 2, ["/dev/sda"]), "/dev/vg/2": ("-wi-ao", 253, 3, 4096, 2, ["/dev/sda"])} self.assertEqual({}, bdev.LogicalVolume.GetLvGlobalInfo( _run_cmd=lambda cmd: _FakeRunCmd(False, "Fake error msg", cmd))) self.assertEqual({}, bdev.LogicalVolume.GetLvGlobalInfo( _run_cmd=lambda cmd: _FakeRunCmd(True, "", cmd))) self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume.GetLvGlobalInfo, _run_cmd=lambda cmd: _FakeRunCmd(True, "BadStdOut", cmd)) fake_cmd = lambda cmd: _FakeRunCmd(True, good_lines, cmd) good_res = bdev.LogicalVolume.GetLvGlobalInfo(_run_cmd=fake_cmd) self.assertEqual(expected_output, good_res) @testutils.patch_object(bdev.LogicalVolume, "Attach") def testLogicalVolumeImport(self, attach_mock): """Tests for bdev.LogicalVolume.Import()""" # Set up the mock object return value attach_mock.return_value = True # Create a fake logical volume inst = bdev.LogicalVolume(self.test_unique_id, [], 1024, {}, {}) # Desired output command import_cmd = [constants.DD_CMD, "of=%s" % inst.dev_path, "bs=%s" % constants.DD_BLOCK_SIZE, "oflag=direct", "conv=notrunc"] self.assertEqual(inst.Import(), import_cmd) @testutils.patch_object(bdev.LogicalVolume, "Attach") def testLogicalVolumeExport(self, attach_mock): """Test for bdev.LogicalVolume.Export()""" # Set up the mock object return value attach_mock.return_value = True # Create a fake logical volume inst = bdev.LogicalVolume(self.test_unique_id, [], 1024, {}, {}) # Desired output command export_cmd = [constants.DD_CMD, "if=%s" % inst.dev_path, "bs=%s" % constants.DD_BLOCK_SIZE, "count=%s" % inst.size, "iflag=direct"] self.assertEqual(inst.Export(), export_cmd) @testutils.patch_object(bdev.LogicalVolume, "GetPVInfo") @testutils.patch_object(utils, "RunCmd") @testutils.patch_object(bdev.LogicalVolume, "Attach") def testCreate(self, attach_mock, run_cmd_mock, pv_info_mock): """Test for bdev.LogicalVolume.Create() success""" attach_mock.return_value = True # This returns a successful RunCmd result run_cmd_mock.return_value = _FakeRunCmd(True, "", "") pv_info_mock.return_value = self.pv_info_return expect = bdev.LogicalVolume(self.test_unique_id, [], 1024, self.test_params, {}) got = bdev.LogicalVolume.Create(self.test_unique_id, [], 1024, None, self.test_params, False, {}, test_kwarg="test") self.assertEqual(expect, got) @testutils.patch_object(bdev.LogicalVolume, "GetPVInfo") @testutils.patch_object(bdev.LogicalVolume, "Attach") def testCreateFailurePvsInfoExclStor(self, attach_mock, pv_info_mock): """Test for bdev.LogicalVolume.Create() failure when pv_info is empty and exclusive storage is enabled """ attach_mock.return_value = True pv_info_mock.return_value = [] self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume.Create, self.test_unique_id, [], 1024, None, {}, True, {}) @testutils.patch_object(bdev.LogicalVolume, "GetPVInfo") @testutils.patch_object(bdev.LogicalVolume, "Attach") def testCreateFailurePvsInfoNoExclStor(self, attach_mock, pv_info_mock): """Test for bdev.LogicalVolume.Create() failure when pv_info is empty and exclusive storage is disabled """ attach_mock.return_value = True pv_info_mock.return_value = [] self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume.Create, self.test_unique_id, [], 1024, None, {}, False, {}) @testutils.patch_object(bdev.LogicalVolume, "GetPVInfo") @testutils.patch_object(bdev.LogicalVolume, "Attach") def testCreateFailurePvsInvalid(self, attach_mock, pv_info_mock): """Test for bdev.LogicalVolume.Create() failure when pvs_info output is invalid """ attach_mock.return_value = True pv_info_mock.return_value = self.pv_info_invalid self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume.Create, self.test_unique_id, [], 1024, None, {}, False, {}) @testutils.patch_object(bdev.LogicalVolume, "GetPVInfo") @testutils.patch_object(bdev.LogicalVolume, "Attach") def testCreateFailureNoSpindles(self, attach_mock, pv_info_mock): """Test for bdev.LogicalVolume.Create() failure when there are no spindles """ attach_mock.return_value = True pv_info_mock.return_value = self.pv_info_return self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume.Create, self.test_unique_id, [], 1024, None, self.test_params,True, {}) @testutils.patch_object(bdev.LogicalVolume, "GetPVInfo") @testutils.patch_object(bdev.LogicalVolume, "Attach") def testCreateFailureNotEnoughSpindles(self, attach_mock, pv_info_mock): """Test for bdev.LogicalVolume.Create() failure when there are not enough spindles """ attach_mock.return_value = True pv_info_mock.return_value = self.pv_info_return self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume.Create, self.test_unique_id, [], 1024, 0, self.test_params, True, {}) @testutils.patch_object(bdev.LogicalVolume, "GetPVInfo") @testutils.patch_object(bdev.LogicalVolume, "Attach") def testCreateFailureNotEnoughEmptyPvs(self, attach_mock, pv_info_mock): """Test for bdev.LogicalVolume.Create() failure when there are not enough empty pvs """ attach_mock.return_value = True pv_info_mock.return_value = self.pv_info_return self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume.Create, self.test_unique_id, [], 1024, 2, self.test_params, True, {}) @testutils.patch_object(bdev.LogicalVolume, "GetPVInfo") @testutils.patch_object(bdev.LogicalVolume, "Attach") def testCreateFailureNoFreeSpace(self, attach_mock, pv_info_mock): """Test for bdev.LogicalVolume.Create() failure when there is no free space """ attach_mock.return_value = True pv_info_mock.return_value = self.pv_info_no_space self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume.Create, self.test_unique_id, [], 1024, None, self.test_params, False, {}) @testutils.patch_object(utils, "RunCmd") @testutils.patch_object(bdev.LogicalVolume, "GetPVInfo") @testutils.patch_object(bdev.LogicalVolume, "Attach") def testCreateFailureCommand(self, attach_mock, pv_info_mock, run_cmd_mock): """Test for bdev.LogicalVolume.Create() failure when the runcmd is incorrect """ attach_mock.return_value = True pv_info_mock.return_value = self.pv_info_return run_cmd_mock = _FakeRunCmd(False, "", "") self.assertRaises(errors.BlockDeviceError, bdev.LogicalVolume.Create, self.test_unique_id, [], 1024, None, self.test_params, False, {}) @testutils.patch_object(bdev.LogicalVolume, "GetLvGlobalInfo") def testAttach(self, info_mock): """Test for bdev.LogicalVolume.Attach()""" info_mock.return_value = {"/dev/fake/path": ("v", 1, 0, 1024, 0, ["test"])} dev = bdev.LogicalVolume.__new__(bdev.LogicalVolume) dev.dev_path = "/dev/fake/path" self.assertEqual(dev.Attach(), True) @testutils.patch_object(bdev.LogicalVolume, "GetLvGlobalInfo") def testAttachFalse(self, info_mock): """Test for bdev.LogicalVolume.Attach() with missing lv_info""" info_mock.return_value = {} dev = bdev.LogicalVolume.__new__(bdev.LogicalVolume) dev.dev_path = "/dev/fake/path" self.assertEqual(dev.Attach(), False) class TestPersistentBlockDevice(testutils.GanetiTestCase): """Tests for bdev.PersistentBlockDevice volumes """ def setUp(self): """Set up test data""" testutils.GanetiTestCase.setUp(self) self.test_unique_id = (constants.BLOCKDEV_DRIVER_MANUAL, "/dev/abc") def testPersistentBlockDeviceImport(self): """Test case for bdev.PersistentBlockDevice.Import()""" # Create a fake block device inst = bdev.PersistentBlockDevice(self.test_unique_id, [], 1024, {}, {}) self.assertRaises(errors.BlockDeviceError, bdev.PersistentBlockDevice.Import, inst) @testutils.patch_object(bdev.PersistentBlockDevice, "Attach") def testCreate(self, attach_mock): """Test for bdev.PersistentBlockDevice.Create()""" attach_mock.return_value = True expect = bdev.PersistentBlockDevice(self.test_unique_id, [], 0, {}, {}) got = bdev.PersistentBlockDevice.Create(self.test_unique_id, [], 1024, None, {}, False, {}, test_kwarg="test") self.assertEqual(expect, got) def testCreateFailure(self): """Test for bdev.PersistentBlockDevice.Create() failure""" self.assertRaises(errors.ProgrammerError, bdev.PersistentBlockDevice.Create, self.test_unique_id, [], 1024, None, {}, True, {}) @testutils.patch_object(os, "stat") def testAttach(self, stat_mock): """Test for bdev.PersistentBlockDevice.Attach()""" stat_mock.return_value = FakeStatResult(0x6000) # bitmask for S_ISBLK dev = bdev.PersistentBlockDevice.__new__(bdev.PersistentBlockDevice) dev.dev_path = "/dev/fake/path" self.assertEqual(dev.Attach(), True) @testutils.patch_object(os, "stat") def testAttachFailureNotBlockdev(self, stat_mock): """Test for bdev.PersistentBlockDevice.Attach() failure, not a blockdev""" stat_mock.return_value = FakeStatResult(0x0) dev = bdev.PersistentBlockDevice.__new__(bdev.PersistentBlockDevice) dev.dev_path = "/dev/fake/path" self.assertEqual(dev.Attach(), False) @testutils.patch_object(os, "stat") def testAttachFailureNoDevice(self, stat_mock): """Test for bdev.PersistentBlockDevice.Attach() failure, no device found""" stat_mock.side_effect = OSError("No device found") dev = bdev.PersistentBlockDevice.__new__(bdev.PersistentBlockDevice) dev.dev_path = "/dev/fake/path" self.assertEqual(dev.Attach(), False) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.storage.container_unittest.py000075500000000000000000000121131476477700300256640ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.storage.container""" import re import unittest import random from ganeti import constants from ganeti import utils from ganeti import compat from ganeti import errors from ganeti.storage import container import testutils class TestVGReduce(testutils.GanetiTestCase): VGNAME = "xenvg" LIST_CMD = container.LvmVgStorage.LIST_COMMAND VGREDUCE_CMD = container.LvmVgStorage.VGREDUCE_COMMAND def _runCmd(self, cmd, **kwargs): if not self.run_history: self.fail("Empty run results") exp_cmd, result = self.run_history.pop(0) self.assertEqual(cmd, exp_cmd) return result def testOldVersion(self): lvmvg = container.LvmVgStorage() stdout = testutils.ReadTestData("vgreduce-removemissing-2.02.02.txt") vgs_fail = testutils.ReadTestData("vgs-missing-pvs-2.02.02.txt") self.run_history = [ ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME], utils.RunResult(0, None, stdout, "", "", None, None)), ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME], utils.RunResult(0, None, "", "", "", None, None)), ] lvmvg._RemoveMissing(self.VGNAME, _runcmd_fn=self._runCmd) self.assertEqual(self.run_history, []) for ecode, out in [(1, ""), (0, vgs_fail)]: self.run_history = [ ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME], utils.RunResult(0, None, stdout, "", "", None, None)), ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME], utils.RunResult(ecode, None, out, "", "", None, None)), ] self.assertRaises(errors.StorageError, lvmvg._RemoveMissing, self.VGNAME, _runcmd_fn=self._runCmd) self.assertEqual(self.run_history, []) def testNewVersion(self): lvmvg = container.LvmVgStorage() stdout1 = testutils.ReadTestData("vgreduce-removemissing-2.02.66-fail.txt") stdout2 = testutils.ReadTestData("vgreduce-removemissing-2.02.66-ok.txt") vgs_fail = testutils.ReadTestData("vgs-missing-pvs-2.02.66.txt") # first: require --fail, check that it's used self.run_history = [ ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME], utils.RunResult(0, None, stdout1, "", "", None, None)), ([self.VGREDUCE_CMD, "--removemissing", "--force", self.VGNAME], utils.RunResult(0, None, stdout2, "", "", None, None)), ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME], utils.RunResult(0, None, "", "", "", None, None)), ] lvmvg._RemoveMissing(self.VGNAME, _runcmd_fn=self._runCmd) self.assertEqual(self.run_history, []) # second: make sure --fail is not used if not needed self.run_history = [ ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME], utils.RunResult(0, None, stdout2, "", "", None, None)), ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME], utils.RunResult(0, None, "", "", "", None, None)), ] lvmvg._RemoveMissing(self.VGNAME, _runcmd_fn=self._runCmd) self.assertEqual(self.run_history, []) # third: make sure we error out if vgs doesn't find the volume for ecode, out in [(1, ""), (0, vgs_fail)]: self.run_history = [ ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME], utils.RunResult(0, None, stdout1, "", "", None, None)), ([self.VGREDUCE_CMD, "--removemissing", "--force", self.VGNAME], utils.RunResult(0, None, stdout2, "", "", None, None)), ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME], utils.RunResult(ecode, None, out, "", "", None, None)), ] self.assertRaises(errors.StorageError, lvmvg._RemoveMissing, self.VGNAME, _runcmd_fn=self._runCmd) self.assertEqual(self.run_history, []) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.storage.drbd_unittest.py000075500000000000000000000544141476477700300246270ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2012, 2013, 2016 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the drbd module""" import os from ganeti import constants from ganeti import errors from ganeti import serializer from ganeti.storage import drbd from ganeti.storage import drbd_info from ganeti.storage import drbd_cmdgen import testutils class TestDRBD8(testutils.GanetiTestCase): def testGetVersion(self): data = [ "version: 8.0.0 (api:76/proto:80)", "version: 8.0.12 (api:76/proto:86-91)", "version: 8.2.7 (api:88/proto:0-100)", "version: 8.3.7.49 (api:188/proto:13-191)", "version: 8.4.8-1 (api:1/proto:86-101)", ] result = [ { "k_major": 8, "k_minor": 0, "k_point": 0, "api": 76, "proto": 80, }, { "k_major": 8, "k_minor": 0, "k_point": 12, "api": 76, "proto": 86, "proto2": "91", }, { "k_major": 8, "k_minor": 2, "k_point": 7, "api": 88, "proto": 0, "proto2": "100", }, { "k_major": 8, "k_minor": 3, "k_point": 7, "k_fix_separator": ".", "k_fix": "49", "api": 188, "proto": 13, "proto2": "191", }, { "k_major": 8, "k_minor": 4, "k_point": 8, "k_fix_separator": "-", "k_fix": "1", "api": 1, "proto": 86, "proto2": "101", } ] for d, r in zip(data, result): info = drbd.DRBD8Info.CreateFromLines([d]) self.assertEqual(info.GetVersion(), r) self.assertEqual(info.GetVersionString(), d.replace("version: ", "")) class TestDRBD8Runner(testutils.GanetiTestCase): """Testing case for drbd.DRBD8Dev""" @staticmethod def _has_disk(data, dname, mname, meta_index=0): """Check local disk corectness""" retval = ( "local_dev" in data and data["local_dev"] == dname and "meta_dev" in data and data["meta_dev"] == mname and ((meta_index is None and "meta_index" not in data) or ("meta_index" in data and data["meta_index"] == meta_index) ) ) return retval @staticmethod def _has_net(data, local, remote): """Check network connection parameters""" retval = ( "local_addr" in data and data["local_addr"] == local and "remote_addr" in data and data["remote_addr"] == remote ) return retval def testParser83Creation(self): """Test drbdsetup show parser creation""" drbd_info.DRBD83ShowInfo._GetShowParser() def testParser84Creation(self): """Test drbdsetup show parser creation""" drbd_info.DRBD84ShowInfo._GetShowParser() def testParser80(self): """Test drbdsetup show parser for disk and network version 8.0""" data = testutils.ReadTestData("bdev-drbd-8.0.txt") result = drbd_info.DRBD83ShowInfo.GetDevInfo(data) self.assertTrue(self._has_disk(result, "/dev/xenvg/test.data", "/dev/xenvg/test.meta"), "Wrong local disk info") self.assertTrue(self._has_net(result, ("192.0.2.1", 11000), ("192.0.2.2", 11000)), "Wrong network info (8.0.x)") def testParser83(self): """Test drbdsetup show parser for disk and network version 8.3""" data = testutils.ReadTestData("bdev-drbd-8.3.txt") result = drbd_info.DRBD83ShowInfo.GetDevInfo(data) self.assertTrue(self._has_disk(result, "/dev/xenvg/test.data", "/dev/xenvg/test.meta"), "Wrong local disk info") self.assertTrue(self._has_net(result, ("192.0.2.1", 11000), ("192.0.2.2", 11000)), "Wrong network info (8.3.x)") def testParser84(self): """Test drbdsetup show parser for disk and network version 8.4""" data = testutils.ReadTestData("bdev-drbd-8.4.txt") result = drbd_info.DRBD84ShowInfo.GetDevInfo(data) self.assertTrue(self._has_disk(result, "/dev/xenvg/test.data", "/dev/xenvg/test.meta"), "Wrong local disk info") self.assertTrue(self._has_net(result, ("192.0.2.1", 11000), ("192.0.2.2", 11000)), "Wrong network info (8.4.x)") def testParser84NoDiskParams(self): """Test drbdsetup show parser for 8.4 without disk params The missing disk parameters occur after re-attaching a local disk but before setting the disk params. """ data = testutils.ReadTestData("bdev-drbd-8.4-no-disk-params.txt") result = drbd_info.DRBD84ShowInfo.GetDevInfo(data) self.assertTrue(self._has_disk(result, "/dev/xenvg/test.data", "/dev/xenvg/test.meta", meta_index=None), "Wrong local disk info") self.assertTrue(self._has_net(result, ("192.0.2.1", 11000), ("192.0.2.2", 11000)), "Wrong network info (8.4.x)") def testParserNetIP4(self): """Test drbdsetup show parser for IPv4 network""" data = testutils.ReadTestData("bdev-drbd-net-ip4.txt") result = drbd_info.DRBD83ShowInfo.GetDevInfo(data) self.assertTrue(("local_dev" not in result and "meta_dev" not in result and "meta_index" not in result), "Should not find local disk info") self.assertTrue(self._has_net(result, ("192.0.2.1", 11002), ("192.0.2.2", 11002)), "Wrong network info (IPv4)") def testParserNetIP6(self): """Test drbdsetup show parser for IPv6 network""" data = testutils.ReadTestData("bdev-drbd-net-ip6.txt") result = drbd_info.DRBD83ShowInfo.GetDevInfo(data) self.assertTrue(("local_dev" not in result and "meta_dev" not in result and "meta_index" not in result), "Should not find local disk info") self.assertTrue(self._has_net(result, ("2001:db8:65::1", 11048), ("2001:db8:66::1", 11048)), "Wrong network info (IPv6)") def testParserDisk(self): """Test drbdsetup show parser for disk""" data = testutils.ReadTestData("bdev-drbd-disk.txt") result = drbd_info.DRBD83ShowInfo.GetDevInfo(data) self.assertTrue(self._has_disk(result, "/dev/xenvg/test.data", "/dev/xenvg/test.meta"), "Wrong local disk info") self.assertTrue(("local_addr" not in result and "remote_addr" not in result), "Should not find network info") def testBarriersOptions(self): """Test class method that generates drbdsetup options for disk barriers""" # Tests that should fail because of wrong version/options combinations should_fail = [ (8, 0, 12, "bfd", True), (8, 0, 12, "fd", False), (8, 0, 12, "b", True), (8, 2, 7, "bfd", True), (8, 2, 7, "b", True) ] for vmaj, vmin, vrel, opts, meta in should_fail: self.assertRaises(errors.BlockDeviceError, drbd_cmdgen.DRBD83CmdGenerator._ComputeDiskBarrierArgs, vmaj, vmin, vrel, opts, meta) # get the valid options from the frozenset(frozenset()) in constants. valid_options = [list(x)[0] for x in constants.DRBD_VALID_BARRIER_OPT] # Versions that do not support anything for vmaj, vmin, vrel in ((8, 0, 0), (8, 0, 11), (8, 2, 6)): for opts in valid_options: self.assertRaises( errors.BlockDeviceError, drbd_cmdgen.DRBD83CmdGenerator._ComputeDiskBarrierArgs, vmaj, vmin, vrel, opts, True) # Versions with partial support (testing only options that are supported) tests = [ (8, 0, 12, "n", False, []), (8, 0, 12, "n", True, ["--no-md-flushes"]), (8, 2, 7, "n", False, []), (8, 2, 7, "fd", False, ["--no-disk-flushes", "--no-disk-drain"]), (8, 0, 12, "n", True, ["--no-md-flushes"]), ] # Versions that support everything for vmaj, vmin, vrel in ((8, 3, 0), (8, 3, 12)): tests.append((vmaj, vmin, vrel, "bfd", True, ["--no-disk-barrier", "--no-disk-drain", "--no-disk-flushes", "--no-md-flushes"])) tests.append((vmaj, vmin, vrel, "n", False, [])) tests.append((vmaj, vmin, vrel, "b", True, ["--no-disk-barrier", "--no-md-flushes"])) tests.append((vmaj, vmin, vrel, "fd", False, ["--no-disk-flushes", "--no-disk-drain"])) tests.append((vmaj, vmin, vrel, "n", True, ["--no-md-flushes"])) # Test execution for test in tests: vmaj, vmin, vrel, disabled_barriers, disable_meta_flush, expected = test args = \ drbd_cmdgen.DRBD83CmdGenerator._ComputeDiskBarrierArgs( vmaj, vmin, vrel, disabled_barriers, disable_meta_flush) self.assertTrue(set(args) == set(expected), "For test %s, got wrong results %s" % (test, args)) # Unsupported or invalid versions for vmaj, vmin, vrel in ((0, 7, 25), (9, 0, 0), (7, 0, 0), (8, 4, 0)): self.assertRaises(errors.BlockDeviceError, drbd_cmdgen.DRBD83CmdGenerator._ComputeDiskBarrierArgs, vmaj, vmin, vrel, "n", True) # Invalid options for option in ("", "c", "whatever", "nbdfc", "nf"): self.assertRaises(errors.BlockDeviceError, drbd_cmdgen.DRBD83CmdGenerator._ComputeDiskBarrierArgs, 8, 3, 11, option, True) class TestDRBD8Status(testutils.GanetiTestCase): """Testing case for DRBD8Dev /proc status""" def setUp(self): """Read in txt data""" testutils.GanetiTestCase.setUp(self) proc_data = testutils.TestDataFilename("proc_drbd8.txt") proc80e_data = testutils.TestDataFilename("proc_drbd80-emptyline.txt") proc83_data = testutils.TestDataFilename("proc_drbd83.txt") proc83_sync_data = testutils.TestDataFilename("proc_drbd83_sync.txt") proc83_sync_krnl_data = \ testutils.TestDataFilename("proc_drbd83_sync_krnl2.6.39.txt") proc84_data = testutils.TestDataFilename("proc_drbd84.txt") proc84_sync_data = testutils.TestDataFilename("proc_drbd84_sync.txt") proc84_emptyfirst_data = \ testutils.TestDataFilename("proc_drbd84_emptyfirst.txt") self.proc80ev_data = \ testutils.TestDataFilename("proc_drbd80-emptyversion.txt") self.drbd_info = drbd.DRBD8Info.CreateFromFile(filename=proc_data) self.drbd_info80e = drbd.DRBD8Info.CreateFromFile(filename=proc80e_data) self.drbd_info83 = drbd.DRBD8Info.CreateFromFile(filename=proc83_data) self.drbd_info83_sync = \ drbd.DRBD8Info.CreateFromFile(filename=proc83_sync_data) self.drbd_info83_sync_krnl = \ drbd.DRBD8Info.CreateFromFile(filename=proc83_sync_krnl_data) self.drbd_info84 = drbd.DRBD8Info.CreateFromFile(filename=proc84_data) self.drbd_info84_sync = \ drbd.DRBD8Info.CreateFromFile(filename=proc84_sync_data) self.drbd_info84_emptyfirst = \ drbd.DRBD8Info.CreateFromFile(filename=proc84_emptyfirst_data) def testIOErrors(self): """Test handling of errors while reading the proc file.""" temp_file = self._CreateTempFile() os.unlink(temp_file) self.assertRaises(errors.BlockDeviceError, drbd.DRBD8Info.CreateFromFile, filename=temp_file) def testHelper(self): """Test reading usermode_helper in /sys.""" sys_drbd_helper = testutils.TestDataFilename("sys_drbd_usermode_helper.txt") drbd_helper = drbd.DRBD8.GetUsermodeHelper(filename=sys_drbd_helper) self.assertEqual(drbd_helper, "/bin/true") def testHelperIOErrors(self): """Test handling of errors while reading usermode_helper in /sys.""" temp_file = self._CreateTempFile() os.unlink(temp_file) self.assertRaises(errors.BlockDeviceError, drbd.DRBD8.GetUsermodeHelper, filename=temp_file) def testMinorNotFound(self): """Test not-found-minor in /proc""" self.assertTrue(not self.drbd_info.HasMinorStatus(9)) self.assertTrue(not self.drbd_info83.HasMinorStatus(9)) self.assertTrue(not self.drbd_info80e.HasMinorStatus(3)) self.assertTrue(not self.drbd_info84_emptyfirst.HasMinorStatus(0)) def testLineNotMatch(self): """Test wrong line passed to drbd_info.DRBD8Status""" self.assertRaises(errors.BlockDeviceError, drbd_info.DRBD8Status, "foo") def testMinor0(self): """Test connected, primary device""" for info in [self.drbd_info, self.drbd_info83, self.drbd_info84]: stats = info.GetMinorStatus(0) self.assertTrue(stats.is_in_use) self.assertTrue(stats.is_connected and stats.is_primary and stats.peer_secondary and stats.is_disk_uptodate) def testMinor1(self): """Test connected, secondary device""" for info in [self.drbd_info, self.drbd_info83, self.drbd_info84, self.drbd_info84_emptyfirst]: stats = info.GetMinorStatus(1) self.assertTrue(stats.is_in_use) self.assertTrue(stats.is_connected and stats.is_secondary and stats.peer_primary and stats.is_disk_uptodate) def testMinor2(self): """Test unconfigured device""" for info in [self.drbd_info, self.drbd_info83, self.drbd_info80e, self.drbd_info84, self.drbd_info84_emptyfirst]: stats = info.GetMinorStatus(2) self.assertFalse(stats.is_in_use) def testMinor4(self): """Test WFconn device""" for info in [self.drbd_info, self.drbd_info83, self.drbd_info84, self.drbd_info84_emptyfirst]: stats = info.GetMinorStatus(4) self.assertTrue(stats.is_in_use) self.assertTrue(stats.is_wfconn and stats.is_primary and stats.rrole == "Unknown" and stats.is_disk_uptodate) def testMinor6(self): """Test diskless device""" for info in [self.drbd_info, self.drbd_info83, self.drbd_info84, self.drbd_info84_emptyfirst]: stats = info.GetMinorStatus(6) self.assertTrue(stats.is_in_use) self.assertTrue(stats.is_connected and stats.is_secondary and stats.peer_primary and stats.is_diskless) def testMinor8(self): """Test standalone device""" for info in [self.drbd_info, self.drbd_info83, self.drbd_info84]: stats = info.GetMinorStatus(8) self.assertTrue(stats.is_in_use) self.assertTrue(stats.is_standalone and stats.rrole == "Unknown" and stats.is_disk_uptodate) def testDRBD83SyncFine(self): stats = self.drbd_info83_sync.GetMinorStatus(3) self.assertTrue(stats.is_in_resync) self.assertAlmostEqual(stats.sync_percent, 34.9) def testDRBD83SyncBroken(self): stats = self.drbd_info83_sync_krnl.GetMinorStatus(3) self.assertTrue(stats.is_in_resync) self.assertAlmostEqual(stats.sync_percent, 2.4) def testDRBD84Sync(self): stats = self.drbd_info84_sync.GetMinorStatus(5) self.assertTrue(stats.is_in_resync) self.assertAlmostEqual(stats.sync_percent, 68.5) def testDRBDEmptyVersion(self): self.assertRaises(errors.BlockDeviceError, drbd.DRBD8Info.CreateFromFile, filename=self.proc80ev_data) class TestDRBD8Construction(testutils.GanetiTestCase): def setUp(self): """Read in txt data""" testutils.GanetiTestCase.setUp(self) self.proc80_info = \ drbd_info.DRBD8Info.CreateFromFile( filename=testutils.TestDataFilename("proc_drbd8.txt")) self.proc83_info = \ drbd_info.DRBD8Info.CreateFromFile( filename=testutils.TestDataFilename("proc_drbd83.txt")) self.proc84_info = \ drbd_info.DRBD8Info.CreateFromFile( filename=testutils.TestDataFilename("proc_drbd84.txt")) self.test_unique_id = ("hosta.com", 123, "host2.com", 123, 0, serializer.Private("secret")) self.test_dyn_params = { constants.DDP_LOCAL_IP: "192.0.2.1", constants.DDP_LOCAL_MINOR: 0, constants.DDP_REMOTE_IP: "192.0.2.2", constants.DDP_REMOTE_MINOR: 0, } @testutils.patch_object(drbd.DRBD8, "GetProcInfo") def testConstructionWith80Data(self, mock_create_from_file): mock_create_from_file.return_value = self.proc80_info inst = drbd.DRBD8Dev(self.test_unique_id, [], 123, {}, self.test_dyn_params) self.assertEqual(inst._show_info_cls, drbd_info.DRBD83ShowInfo) self.assertTrue(isinstance(inst._cmd_gen, drbd_cmdgen.DRBD83CmdGenerator)) @testutils.patch_object(drbd.DRBD8, "GetProcInfo") def testConstructionWith83Data(self, mock_create_from_file): mock_create_from_file.return_value = self.proc83_info inst = drbd.DRBD8Dev(self.test_unique_id, [], 123, {}, self.test_dyn_params) self.assertEqual(inst._show_info_cls, drbd_info.DRBD83ShowInfo) self.assertTrue(isinstance(inst._cmd_gen, drbd_cmdgen.DRBD83CmdGenerator)) @testutils.patch_object(drbd.DRBD8, "GetProcInfo") def testConstructionWith84Data(self, mock_create_from_file): mock_create_from_file.return_value = self.proc84_info inst = drbd.DRBD8Dev(self.test_unique_id, [], 123, {}, self.test_dyn_params) self.assertEqual(inst._show_info_cls, drbd_info.DRBD84ShowInfo) self.assertTrue(isinstance(inst._cmd_gen, drbd_cmdgen.DRBD84CmdGenerator)) class TestDRBD8Create(testutils.GanetiTestCase): class fake_disk(object): def __init__(self, dev_path): self.dev_path = dev_path def Assemble(self): pass def Attach(self): return True def setUp(self): """Set up test data""" testutils.GanetiTestCase.setUp(self) self.proc84_info = \ drbd_info.DRBD8Info.CreateFromFile( filename=testutils.TestDataFilename("proc_drbd84.txt")) self.test_unique_id = ("hosta.com", 123, "host2.com", 123, 0, serializer.Private("secret")) self.test_dyn_params = { constants.DDP_LOCAL_IP: "192.0.2.1", constants.DDP_LOCAL_MINOR: 0, constants.DDP_REMOTE_IP: "192.0.2.2", constants.DDP_REMOTE_MINOR: 0, } fake_child_1 = self.fake_disk("/dev/sda5") fake_child_2 = self.fake_disk("/dev/sda6") self.children = [fake_child_1, fake_child_2] @testutils.patch_object(drbd.DRBD8Dev, "_InitMeta") @testutils.patch_object(drbd.DRBD8Dev, "_CheckMetaSize") @testutils.patch_object(drbd.DRBD8, "GetProcInfo") def testCreate(self, proc_info, check_meta_size, init_meta): proc_info.return_value = self.proc84_info check_meta_size.return_value = None init_meta.return_value = None self.test_dyn_params[constants.DDP_LOCAL_MINOR] = 2 expected = drbd.DRBD8Dev(self.test_unique_id, [], 123, {}, self.test_dyn_params) got = drbd.DRBD8Dev.Create(self.test_unique_id, self.children, 123, None, {}, False, self.test_dyn_params, test_kwarg="test") self.assertEqual(got, expected) def testCreateFailureChildrenLength(self): self.assertRaises(errors.ProgrammerError, drbd.DRBD8Dev.Create, self.test_unique_id, [], 123, None, {}, False, self.test_dyn_params) def testCreateFailureExclStorage(self): self.assertRaises(errors.ProgrammerError, drbd.DRBD8Dev.Create, self.test_unique_id, self.children, 123, None, {}, True, self.test_dyn_params) def testCreateFailureNoMinor(self): self.assertRaises(errors.ProgrammerError, drbd.DRBD8Dev.Create, self.test_unique_id, self.children, 123, None, {}, False, {}) @testutils.patch_object(drbd.DRBD8, "GetProcInfo") def testCreateFailureInUse(self, proc_info): # The proc84_info config has a local minor in use, which triggers our # failure test. proc_info.return_value = self.proc84_info self.assertRaises(errors.BlockDeviceError, drbd.DRBD8Dev.Create, self.test_unique_id, self.children, 123, None, {}, False, self.test_dyn_params) @testutils.patch_object(drbd.DRBD8, "GetProcInfo") def testCreateFailureMetaAttach(self, proc_info): proc_info.return_value = self.proc84_info self.test_dyn_params[constants.DDP_LOCAL_MINOR] = 2 self.children[1].Attach = lambda: False self.assertRaises(errors.BlockDeviceError, drbd.DRBD8Dev.Create, self.test_unique_id, self.children, 123, None, {}, False, self.test_dyn_params) @testutils.patch_object(drbd.DRBD8, "GetUsedDevs") def testAttach(self, drbd_mock): """Test for drbd.DRBD8Dev.Attach()""" drbd_mock.return_value = [1] dev = drbd.DRBD8Dev.__new__(drbd.DRBD8Dev) dev._aminor = 1 self.assertEqual(dev.Attach(), True) @testutils.patch_object(drbd.DRBD8, "GetUsedDevs") def testAttach(self, drbd_mock): """Test for drbd.DRBD8Dev.Attach() not finding a minor""" drbd_mock.return_value = [] dev = drbd.DRBD8Dev.__new__(drbd.DRBD8Dev) dev._aminor = 1 self.assertEqual(dev.Attach(), False) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.storage.extstorage_unittest.py000075500000000000000000000127461476477700300261030ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2016 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the extstorage module""" import os from ganeti import errors from ganeti.storage import extstorage import testutils class FakeStatResult(object): def __init__(self, st_mode): self.st_mode = st_mode self.st_rdev = 0 class TestExtStorageDevice(testutils.GanetiTestCase): """Testing case for extstorage.ExtStorageDevice""" def setUp(self): """Set up test data""" testutils.GanetiTestCase.setUp(self) self.name = "testname" self.uuid = "testuuid" self.test_unique_id = ("testdriver", "testvolumename") @testutils.patch_object(extstorage.ExtStorageDevice, "Attach") @testutils.patch_object(extstorage, "_ExtStorageAction") def testCreate(self, action_mock, attach_mock): action_mock.return_value = None attach_mock.return_value = True expected = extstorage.ExtStorageDevice(self.test_unique_id, [], 123, {}, {}, name=self.name, uuid=self.uuid) got = extstorage.ExtStorageDevice.Create(self.test_unique_id, [], 123, None, {}, False, {}, name=self.name, uuid=self.uuid) self.assertEqual(got, expected) def testCreateFailure(self): self.assertRaises(errors.ProgrammerError, extstorage.ExtStorageDevice.Create, self.test_unique_id, [], 123, None, {}, True, {}, name=self.name, uuid=self.uuid) @testutils.patch_object(extstorage.ExtStorageDevice, "_ExtStorageAction") @testutils.patch_object(os, "stat") def testAttach(self, stat_mock, action_mock): """Test for extstorage.ExtStorageDevice.Attach()""" stat_mock.return_value = FakeStatResult(0x6000) # bitmask for S_ISBLK action_mock.return_value = "/dev/path\nURI" dev = extstorage.ExtStorageDevice.__new__(extstorage.ExtStorageDevice) dev.unique_id = self.test_unique_id dev.ext_params = {} dev.name = self.name dev.uuid = self.uuid self.assertEqual(dev.Attach(), True) @testutils.patch_object(extstorage.ExtStorageDevice, "_ExtStorageAction") def testAttachNoUserspaceURI(self, action_mock): """Test for extstorage.ExtStorageDevice.Attach() with no userspace URI""" action_mock.return_value = "" dev = extstorage.ExtStorageDevice.__new__(extstorage.ExtStorageDevice) dev.unique_id = self.test_unique_id dev.ext_params = {} dev.name = self.name dev.uuid = self.uuid self.assertEqual(dev.Attach(), False) @testutils.patch_object(extstorage.ExtStorageDevice, "_ExtStorageAction") def testAttachWithUserspaceURI(self, action_mock): """Test for extstorage.ExtStorageDevice.Attach() with userspace URI""" action_mock.return_value = "\nURI" dev = extstorage.ExtStorageDevice.__new__(extstorage.ExtStorageDevice) dev.unique_id = self.test_unique_id dev.ext_params = {} dev.name = self.name dev.uuid = self.uuid self.assertEqual(dev.Attach(), True) @testutils.patch_object(extstorage.ExtStorageDevice, "_ExtStorageAction") @testutils.patch_object(os, "stat") def testAttachFailureNotBlockdev(self, stat_mock, action_mock): """Test for extstorage.ExtStorageDevice.Attach() failure, not a blockdev""" stat_mock.return_value = FakeStatResult(0x0) action_mock.return_value = "/dev/path\nURI" dev = extstorage.ExtStorageDevice.__new__(extstorage.ExtStorageDevice) dev.unique_id = self.test_unique_id dev.ext_params = {} dev.name = self.name dev.uuid = self.uuid self.assertEqual(dev.Attach(), False) @testutils.patch_object(extstorage.ExtStorageDevice, "_ExtStorageAction") @testutils.patch_object(os, "stat") def testAttachFailureNoDevice(self, stat_mock, action_mock): """Test for extstorage.ExtStorageDevice.Attach() failure, no device found""" stat_mock.side_effect = OSError("No device found") action_mock.return_value = "/dev/path\nURI" dev = extstorage.ExtStorageDevice.__new__(extstorage.ExtStorageDevice) dev.unique_id = self.test_unique_id dev.ext_params = {} dev.name = self.name dev.uuid = self.uuid self.assertEqual(dev.Attach(), False) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.storage.filestorage_unittest.py000075500000000000000000000303761476477700300262210ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013, 2016 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the ganeti.storage.filestorage module""" import os import shutil import tempfile import unittest from ganeti import errors from ganeti.storage import filestorage from ganeti.utils import io from ganeti import utils from ganeti import constants import testutils class TestFileStorageSpaceInfo(unittest.TestCase): def testSpaceInfoPathInvalid(self): """Tests that an error is raised when the given path is not existing. """ self.assertRaises(errors.CommandError, filestorage.GetFileStorageSpaceInfo, "/path/does/not/exist/") def testSpaceInfoPathValid(self): """Smoke test run on a directory that exists for sure. """ filestorage.GetFileStorageSpaceInfo("/") class TestCheckFileStoragePath(unittest.TestCase): def _WriteAllowedFile(self, allowed_paths_filename, allowed_paths): allowed_paths_file = open(allowed_paths_filename, 'w') allowed_paths_file.write('\n'.join(allowed_paths)) allowed_paths_file.close() def setUp(self): self.tmpdir = tempfile.mkdtemp() self.allowed_paths = [os.path.join(self.tmpdir, "allowed")] for path in self.allowed_paths: os.mkdir(path) self.allowed_paths_filename = os.path.join(self.tmpdir, "allowed-path-file") self._WriteAllowedFile(self.allowed_paths_filename, self.allowed_paths) def tearDown(self): shutil.rmtree(self.tmpdir) def testCheckFileStoragePathExistance(self): filestorage._CheckFileStoragePathExistance(self.tmpdir) def testCheckFileStoragePathExistanceFail(self): path = os.path.join(self.tmpdir, "does/not/exist") self.assertRaises(errors.FileStoragePathError, filestorage._CheckFileStoragePathExistance, path) @unittest.skipIf(os.getuid() == 0, "must be run as a regular user") def testCheckFileStoragePathNotWritable(self): path = os.path.join(self.tmpdir, "isnotwritable/") os.mkdir(path) os.chmod(path, 0) self.assertRaises(errors.FileStoragePathError, filestorage._CheckFileStoragePathExistance, path) os.chmod(path, 777) def testCheckFileStoragePath(self): path = os.path.join(self.allowed_paths[0], "allowedsubdir") os.mkdir(path) result = filestorage.CheckFileStoragePath( path, _allowed_paths_file=self.allowed_paths_filename) self.assertEqual(None, result) def testCheckFileStoragePathNotAllowed(self): path = os.path.join(self.tmpdir, "notallowed") result = filestorage.CheckFileStoragePath( path, _allowed_paths_file=self.allowed_paths_filename) self.assertTrue("not acceptable" in result) class TestLoadAllowedFileStoragePaths(testutils.GanetiTestCase): def testDevNull(self): self.assertEqual(filestorage._LoadAllowedFileStoragePaths("/dev/null"), []) def testNonExistantFile(self): filename = "/tmp/this/file/does/not/exist" assert not os.path.exists(filename) self.assertEqual(filestorage._LoadAllowedFileStoragePaths(filename), []) def test(self): tmpfile = self._CreateTempFile() utils.WriteFile(tmpfile, data=""" # This is a test file /tmp /srv/storage relative/path """) self.assertEqual(filestorage._LoadAllowedFileStoragePaths(tmpfile), [ "/tmp", "/srv/storage", "relative/path", ]) class TestComputeWrongFileStoragePathsInternal(unittest.TestCase): def testPaths(self): paths = filestorage._GetForbiddenFileStoragePaths() for path in ["/bin", "/usr/local/sbin", "/lib64", "/etc", "/sys"]: self.assertTrue(path in paths) self.assertEqual(set(map(os.path.normpath, paths)), paths) def test(self): vfsp = filestorage._ComputeWrongFileStoragePaths self.assertEqual(vfsp([]), []) self.assertEqual(vfsp(["/tmp"]), []) self.assertEqual(vfsp(["/bin/ls"]), ["/bin/ls"]) self.assertEqual(vfsp(["/bin"]), ["/bin"]) self.assertEqual(vfsp(["/usr/sbin/vim", "/srv/file-storage"]), ["/usr/sbin/vim"]) class TestComputeWrongFileStoragePaths(testutils.GanetiTestCase): def test(self): tmpfile = self._CreateTempFile() utils.WriteFile(tmpfile, data=""" /tmp x/y///z/relative # This is a test file /srv/storage /bin /usr/local/lib32/ relative/path """) self.assertEqual( filestorage.ComputeWrongFileStoragePaths(_filename=tmpfile), ["/bin", "/usr/local/lib32", "relative/path", "x/y/z/relative", ]) class TestCheckFileStoragePathInternal(unittest.TestCase): def testNonAbsolute(self): for i in ["", "tmp", "foo/bar/baz"]: self.assertRaises(errors.FileStoragePathError, filestorage._CheckFileStoragePath, i, ["/tmp"]) self.assertRaises(errors.FileStoragePathError, filestorage._CheckFileStoragePath, "/tmp", ["tmp", "xyz"]) def testNoAllowed(self): self.assertRaises(errors.FileStoragePathError, filestorage._CheckFileStoragePath, "/tmp", []) def testNoAdditionalPathComponent(self): self.assertRaises(errors.FileStoragePathError, filestorage._CheckFileStoragePath, "/tmp/foo", ["/tmp/foo"]) def testAllowed(self): filestorage._CheckFileStoragePath("/tmp/foo/a", ["/tmp/foo"]) filestorage._CheckFileStoragePath("/tmp/foo/a/x", ["/tmp/foo"]) class TestCheckFileStoragePathExistance(testutils.GanetiTestCase): def testNonExistantFile(self): filename = "/tmp/this/file/does/not/exist" assert not os.path.exists(filename) self.assertRaises(errors.FileStoragePathError, filestorage.CheckFileStoragePathAcceptance, "/bin/", _filename=filename) self.assertRaises(errors.FileStoragePathError, filestorage.CheckFileStoragePathAcceptance, "/srv/file-storage", _filename=filename) def testAllowedPath(self): tmpfile = self._CreateTempFile() utils.WriteFile(tmpfile, data=""" /srv/storage """) filestorage.CheckFileStoragePathAcceptance( "/srv/storage/inst1", _filename=tmpfile) # No additional path component self.assertRaises(errors.FileStoragePathError, filestorage.CheckFileStoragePathAcceptance, "/srv/storage", _filename=tmpfile) # Forbidden path self.assertRaises(errors.FileStoragePathError, filestorage.CheckFileStoragePathAcceptance, "/usr/lib64/xyz", _filename=tmpfile) class TestFileDeviceHelper(testutils.GanetiTestCase): @staticmethod def _Make(path, create_with_size=None, create_folders=False): skip_checks = lambda path: None if create_with_size: return filestorage.FileDeviceHelper.CreateFile( path, create_with_size, create_folders=create_folders, _file_path_acceptance_fn=skip_checks ) else: return filestorage.FileDeviceHelper(path, _file_path_acceptance_fn=skip_checks) class TempEnvironment(object): def __init__(self, create_file=False, delete_file=True): self.create_file = create_file self.delete_file = delete_file def __enter__(self): self.directory = tempfile.mkdtemp() self.subdirectory = io.PathJoin(self.directory, "pinky") os.mkdir(self.subdirectory) self.path = io.PathJoin(self.subdirectory, "bunny") self.volume = TestFileDeviceHelper._Make(self.path) if self.create_file: open(self.path, mode="w").close() return self def __exit__(self, *args): if self.delete_file: os.unlink(self.path) os.rmdir(self.subdirectory) os.rmdir(self.directory) return False #don't swallow exceptions def testOperationsOnNonExistingFiles(self): path = "/e/no/ent" volume = TestFileDeviceHelper._Make(path) # These should fail horribly. volume.Exists(assert_exists=False) self.assertRaises(errors.BlockDeviceError, volume.Exists, assert_exists=True) self.assertRaises(errors.BlockDeviceError, volume.Size) self.assertRaises(errors.BlockDeviceError, volume.Grow, 0.020, True, False, None) # Removing however fails silently. volume.Remove() # Make sure we don't create all directories for you unless we ask for it self.assertRaises(errors.BlockDeviceError, TestFileDeviceHelper._Make, path, create_with_size=1) @unittest.skipIf(os.getuid() == 0, "must be run as a regular user") def testFileCreation(self): with TestFileDeviceHelper.TempEnvironment() as env: TestFileDeviceHelper._Make(env.path, create_with_size=1) self.assertTrue(env.volume.Exists()) env.volume.Exists(assert_exists=True) self.assertRaises(errors.BlockDeviceError, env.volume.Exists, assert_exists=False) self.assertRaises(errors.BlockDeviceError, TestFileDeviceHelper._Make, "/enoent", create_with_size=0.042) def testFailSizeDirectory(self): # This should still fail. with TestFileDeviceHelper.TempEnvironment(delete_file=False) as env: test_helper = TestFileDeviceHelper._Make(env.subdirectory) self.assertRaises(errors.BlockDeviceError, test_helper.Size) def testGrowFile(self): with TestFileDeviceHelper.TempEnvironment(create_file=True) as env: self.assertRaises(errors.BlockDeviceError, env.volume.Grow, -1, False, True, None) env.volume.Grow(2, False, True, None) self.assertEqual(2.0, env.volume.Size() / 1024.0**2) def testRemoveFile(self): with TestFileDeviceHelper.TempEnvironment(create_file=True, delete_file=False) as env: env.volume.Remove() env.volume.Exists(assert_exists=False) def testRenameFile(self): """Test if we can rename a file.""" with TestFileDeviceHelper.TempEnvironment(create_file=True) as env: new_path = os.path.join(env.subdirectory, 'middle') env.volume.Move(new_path) self.assertEqual(new_path, env.volume.path) env.volume.Exists(assert_exists=True) env.path = new_path # update the path for the context manager class TestFileStorage(testutils.GanetiTestCase): @testutils.patch_object(filestorage, "FileDeviceHelper") @testutils.patch_object(filestorage.FileStorage, "Attach") def testCreate(self, attach_mock, helper_mock): attach_mock.return_value = True helper_mock.return_value = None test_unique_id = ("test_driver", "/test/file") expect = filestorage.FileStorage(test_unique_id, [], 123, {}, {}) got = filestorage.FileStorage.Create(test_unique_id, [], 123, None, {}, False, {}, test_kwarg="test") self.assertEqual(expect, got) def testCreateFailure(self): test_unique_id = ("test_driver", "/test/file") self.assertRaises(errors.ProgrammerError, filestorage.FileStorage.Create, test_unique_id, [], 123, None, {}, True, {}) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.storage.gluster_unittest.py000064400000000000000000000156361476477700300254010ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013, 2016 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the ganeti.storage.gluster module""" import os import shutil import tempfile import unittest from unittest import mock from ganeti import constants from ganeti import errors from ganeti.storage import gluster from ganeti import ssconf from ganeti import utils import testutils class TestGlusterVolume(testutils.GanetiTestCase): testAddrIpv = {4: "203.0.113.42", 6: "2001:DB8::74:65:28:6:69", } @staticmethod def _MakeVolume(addr=None, port=9001, run_cmd=NotImplemented, vol_name="pinky"): addr = addr if addr is not None else TestGlusterVolume.testAddrIpv[4] return gluster.GlusterVolume(addr, port, vol_name, _run_cmd=run_cmd, _mount_point="/invalid") def setUp(self): testutils.GanetiTestCase.setUp(self) # Create some volumes. self.vol_a = TestGlusterVolume._MakeVolume() self.vol_a_clone = TestGlusterVolume._MakeVolume() self.vol_b = TestGlusterVolume._MakeVolume(vol_name="pinker") def testEquality(self): self.assertEqual(self.vol_a, self.vol_a_clone) def testInequality(self): self.assertNotEqual(self.vol_a, self.vol_b) def testHostnameResolution(self): vol_1 = TestGlusterVolume._MakeVolume(addr="localhost") self.assertTrue(vol_1.server_ip in ["127.0.0.1", "::1"], msg="%s not an IP of localhost" % (vol_1.server_ip,)) self.assertRaises(errors.ResolverError, lambda: \ TestGlusterVolume._MakeVolume(addr="E_NOENT")) def testKVMMountStrings(self): # The only source of documentation I can find is: # https://github.com/qemu/qemu/commit/8d6d89c # This test gets as close as possible to the examples given there, # within the limits of our implementation (no transport specification, # no default port version). vol_1 = TestGlusterVolume._MakeVolume(addr=TestGlusterVolume.testAddrIpv[4], port=24007, vol_name="testvol") self.assertEqual( vol_1.GetKVMMountString("dir/a.img"), "gluster://203.0.113.42:24007/testvol/dir/a.img" ) vol_2 = TestGlusterVolume._MakeVolume(addr=TestGlusterVolume.testAddrIpv[6], port=24007, vol_name="testvol") self.assertEqual( vol_2.GetKVMMountString("dir/a.img"), "gluster://[2001:db8:0:74:65:28:6:69]:24007/testvol/dir/a.img" ) vol_3 = TestGlusterVolume._MakeVolume(addr="localhost", port=9001, vol_name="testvol") kvmMountString = vol_3.GetKVMMountString("dir/a.img") self.assertTrue( kvmMountString in ["gluster://127.0.0.1:9001/testvol/dir/a.img", "gluster://[::1]:9001/testvol/dir/a.img"], msg="%s is not volume testvol/dir/a.img on localhost" % (kvmMountString,) ) def testFUSEMountStrings(self): vol_1 = TestGlusterVolume._MakeVolume(addr=TestGlusterVolume.testAddrIpv[4], port=24007, vol_name="testvol") self.assertEqual( vol_1._GetFUSEMountString(), "-o server-port=24007 203.0.113.42:/testvol" ) vol_2 = TestGlusterVolume._MakeVolume(addr=TestGlusterVolume.testAddrIpv[6], port=24007, vol_name="testvol") # This _ought_ to work. https://bugzilla.redhat.com/show_bug.cgi?id=764188 self.assertEqual( vol_2._GetFUSEMountString(), "-o server-port=24007 2001:db8:0:74:65:28:6:69:/testvol" ) vol_3 = TestGlusterVolume._MakeVolume(addr="localhost", port=9001, vol_name="testvol") fuseMountString = vol_3._GetFUSEMountString() self.assertTrue(fuseMountString in ["-o server-port=9001 127.0.0.1:/testvol", "-o server-port=9001 ::1:/testvol"], msg="%s not testvol on localhost:9001" % (fuseMountString,)) class TestGlusterStorage(testutils.GanetiTestCase): def setUp(self): """Set up test data""" testutils.GanetiTestCase.setUp(self) self.test_params = { constants.GLUSTER_HOST: "127.0.0.1", constants.GLUSTER_PORT: "24007", constants.GLUSTER_VOLUME: "/testvol" } self.test_unique_id = ("testdriver", "testpath") @testutils.patch_object(gluster.FileDeviceHelper, "CreateFile") @testutils.patch_object(gluster.GlusterVolume, "Mount") @testutils.patch_object(ssconf.SimpleStore, "GetGlusterStorageDir") @testutils.patch_object(gluster.GlusterStorage, "Attach") def testCreate(self, attach_mock, storage_dir_mock, mount_mock, create_file_mock): attach_mock.return_value = True storage_dir_mock.return_value = "/testmount" expect = gluster.GlusterStorage(self.test_unique_id, [], 123, self.test_params, {}) got = gluster.GlusterStorage.Create(self.test_unique_id, [], 123, None, self.test_params, False, {}, test_kwarg="test") self.assertEqual(expect, got) def testCreateFailure(self): self.assertRaises(errors.ProgrammerError, gluster.GlusterStorage.Create, self.test_unique_id, [], 123, None, self.test_params, True, {}) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.tools.burnin_unittest.py000075500000000000000000000034771476477700300247100ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.tools.burnin""" import unittest from ganeti import constants from ganeti.tools import burnin import testutils class TestConstants(unittest.TestCase): def testSupportedDiskTemplates(self): # Ignore disk templates not supported by burnin supported = (constants.DISK_TEMPLATES - frozenset([ constants.DT_BLOCK, ])) self.assertEqual(burnin._SUPPORTED_DISK_TEMPLATES, supported) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.tools.common_unittest.py000075500000000000000000000165321476477700300246770ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.tools.ssl_update""" import unittest import shutil import tempfile import os.path import OpenSSL import time from ganeti import constants from ganeti import errors from ganeti import serializer from ganeti import utils from ganeti.tools import common import testutils class TestGenerateClientCert(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() self.client_cert = os.path.join(self.tmpdir, "client.pem") self.server_cert = os.path.join(self.tmpdir, "server.pem") some_serial_no = int(time.time()) utils.GenerateSelfSignedSslCert(self.server_cert, some_serial_no) def tearDown(self): shutil.rmtree(self.tmpdir) def testRegnerateClientCertificate(self): my_node_name = "mynode.example.com" data = {constants.NDS_CLUSTER_NAME: "winnie_poohs_cluster", constants.NDS_NODE_DAEMON_CERTIFICATE: "some_cert", constants.NDS_NODE_NAME: my_node_name} common.GenerateClientCertificate(data, Exception, client_cert=self.client_cert, signing_cert=self.server_cert) client_cert_pem = utils.ReadFile(self.client_cert) server_cert_pem = utils.ReadFile(self.server_cert) client_cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, client_cert_pem) signing_cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, server_cert_pem) self.assertEqual(client_cert.get_issuer().CN, signing_cert.get_subject().CN) self.assertEqual(client_cert.get_subject().CN, my_node_name) class TestLoadData(unittest.TestCase): def testNoJson(self): self.assertRaises(errors.ParseError, common.LoadData, Exception, "") self.assertRaises(errors.ParseError, common.LoadData, Exception, "}") def testInvalidDataStructure(self): raw = serializer.DumpJson({ "some other thing": False, }) self.assertRaises(errors.ParseError, common.LoadData, Exception, raw) raw = serializer.DumpJson([]) self.assertRaises(errors.ParseError, common.LoadData, Exception, raw) def testValidData(self): raw = serializer.DumpJson({}) self.assertEqual(common.LoadData(raw, Exception), {}) class TestVerifyClusterName(unittest.TestCase): class MyException(Exception): pass def setUp(self): unittest.TestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() def tearDown(self): unittest.TestCase.tearDown(self) shutil.rmtree(self.tmpdir) def testNoName(self): self.assertRaises(self.MyException, common.VerifyClusterName, {}, self.MyException, "cluster_name", _verify_fn=NotImplemented) @staticmethod def _FailingVerify(name): assert name == "cluster.example.com" raise errors.GenericError() def testFailingVerification(self): data = { constants.SSHS_CLUSTER_NAME: "cluster.example.com", } self.assertRaises(errors.GenericError, common.VerifyClusterName, data, self.MyException, "cluster_name", _verify_fn=self._FailingVerify) class TestVerifyCertificateStrong(testutils.GanetiTestCase): class MyException(Exception): pass def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() def tearDown(self): testutils.GanetiTestCase.tearDown(self) shutil.rmtree(self.tmpdir) def testNoCert(self): self.assertRaises(self.MyException, common.VerifyCertificateStrong, {}, self.MyException, _verify_fn=NotImplemented) def testVerificationSuccessWithCert(self): common.VerifyCertificateStrong({ constants.NDS_NODE_DAEMON_CERTIFICATE: "something", }, self.MyException, _verify_fn=lambda x,y: None) def testNoPrivateKey(self): cert_filename = testutils.TestDataFilename("cert1.pem") cert_pem = utils.ReadFile(cert_filename) self.assertRaises(self.MyException, common._VerifyCertificateStrong, cert_pem, self.MyException, _check_fn=NotImplemented) def testInvalidCertificate(self): self.assertRaises(self.MyException, common._VerifyCertificateStrong, "Something that's not a certificate", self.MyException, _check_fn=NotImplemented) @staticmethod def _Check(cert): assert cert.get_subject() def testSuccessfulCheck(self): cert_filename = testutils.TestDataFilename("cert2.pem") cert_pem = utils.ReadFile(cert_filename) result = \ common._VerifyCertificateStrong(cert_pem, self.MyException, _check_fn=self._Check) cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, result) self.assertTrue(cert) key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, result) self.assertTrue(key) def testMismatchingKey(self): cert1_path = testutils.TestDataFilename("cert1.pem") cert2_path = testutils.TestDataFilename("cert2.pem") # Extract certificate cert1 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, utils.ReadFile(cert1_path)) cert1_pem = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert1) # Extract mismatching key key2 = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, utils.ReadFile(cert2_path)) key2_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key2) try: common._VerifyCertificateStrong(cert1_pem + key2_pem, self.MyException, _check_fn=NotImplemented) except self.MyException as err: self.assertTrue("not signed with given key" in str(err)) else: self.fail("Exception was not raised") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.tools.ensure_dirs_unittest.py000075500000000000000000000062071476477700300257270ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.tools.ensure_dirs""" import unittest import os.path from ganeti import utils from ganeti.tools import ensure_dirs import testutils class TestGetPaths(unittest.TestCase): def testEntryOrder(self): paths = [(path[0], path[1]) for path in ensure_dirs.GetPaths()] # Directories for which permissions have been set seen = set() # Current directory (changes when an entry of type C{DIR} or C{QUEUE_DIR} # is encountered) current_dir = None for (path, pathtype) in paths: self.assertTrue(pathtype in ensure_dirs.ALL_TYPES) self.assertTrue(utils.IsNormAbsPath(path), msg=("Path '%s' is not absolute and/or normalized" % path)) dirname = os.path.dirname(path) if pathtype == ensure_dirs.DIR: self.assertFalse(path in seen, msg=("Directory '%s' was seen before" % path)) current_dir = path seen.add(path) elif pathtype == ensure_dirs.QUEUE_DIR: self.assertTrue(dirname in seen, msg=("Queue directory '%s' was not seen before" % path)) current_dir = path elif pathtype == ensure_dirs.FILE: self.assertFalse(current_dir is None) self.assertTrue(dirname in seen, msg=("Directory '%s' of path '%s' has not been seen" " yet" % (dirname, path))) self.assertTrue((utils.IsBelowDir(current_dir, path) and current_dir == dirname), msg=("File '%s' not below current directory '%s'" % (path, current_dir))) else: self.fail("Unknown path type '%s'" % (pathtype, )) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.tools.node_daemon_setup_unittest.py000075500000000000000000000055261476477700300271000ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.tools.node_daemon_setup""" import unittest from ganeti import errors from ganeti import constants from ganeti.tools import node_daemon_setup import testutils _SetupError = node_daemon_setup.SetupError class TestVerifySsconf(unittest.TestCase): def testNoSsconf(self): self.assertRaises(_SetupError, node_daemon_setup.VerifySsconf, {}, NotImplemented, _verify_fn=NotImplemented) for items in [None, {}]: self.assertRaises(_SetupError, node_daemon_setup.VerifySsconf, { constants.NDS_SSCONF: items, }, NotImplemented, _verify_fn=NotImplemented) def _Check(self, names): self.assertEqual(frozenset(names), frozenset([ constants.SS_CLUSTER_NAME, constants.SS_INSTANCE_LIST, ])) def testSuccess(self): ssdata = { constants.SS_CLUSTER_NAME: "cluster.example.com", constants.SS_INSTANCE_LIST: [], } result = node_daemon_setup.VerifySsconf({ constants.NDS_SSCONF: ssdata, }, "cluster.example.com", _verify_fn=self._Check) self.assertEqual(result, ssdata) self.assertRaises(_SetupError, node_daemon_setup.VerifySsconf, { constants.NDS_SSCONF: ssdata, }, "wrong.example.com", _verify_fn=self._Check) def testInvalidKey(self): self.assertRaises(errors.GenericError, node_daemon_setup.VerifySsconf, { constants.NDS_SSCONF: { "no-valid-ssconf-key": "value", }, }, NotImplemented) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.tools.prepare_node_join_unittest.py000075500000000000000000000207511476477700300270670ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.tools.prepare_node_join""" import unittest import shutil import tempfile import os.path from ganeti import errors from ganeti import constants from ganeti import pathutils from ganeti import compat from ganeti import utils from ganeti.tools import prepare_node_join from ganeti.tools import common import testutils _JoinError = prepare_node_join.JoinError _DATA_CHECK = prepare_node_join._DATA_CHECK class TestVerifyCertificate(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() def tearDown(self): testutils.GanetiTestCase.tearDown(self) shutil.rmtree(self.tmpdir) def testNoCert(self): common.VerifyCertificateSoft({}, error_fn=prepare_node_join.JoinError, _verify_fn=NotImplemented) def testGivenPrivateKey(self): cert_filename = testutils.TestDataFilename("cert2.pem") cert_pem = utils.ReadFile(cert_filename) self.assertRaises(_JoinError, common._VerifyCertificateSoft, cert_pem, _JoinError, _check_fn=NotImplemented) def testInvalidCertificate(self): self.assertRaises(errors.X509CertError, common._VerifyCertificateSoft, "Something that's not a certificate", _JoinError, _check_fn=NotImplemented) @staticmethod def _Check(cert): assert cert.get_subject() def testSuccessfulCheck(self): cert_filename = testutils.TestDataFilename("cert1.pem") cert_pem = utils.ReadFile(cert_filename) common._VerifyCertificateSoft(cert_pem, _JoinError, _check_fn=self._Check) class TestUpdateSshDaemon(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() self.keyfiles = { constants.SSHK_RSA: (utils.PathJoin(self.tmpdir, "rsa.private"), utils.PathJoin(self.tmpdir, "rsa.public")), constants.SSHK_DSA: (utils.PathJoin(self.tmpdir, "dsa.private"), utils.PathJoin(self.tmpdir, "dsa.public")), constants.SSHK_ECDSA: (utils.PathJoin(self.tmpdir, "ecdsa.private"), utils.PathJoin(self.tmpdir, "ecdsa.public")), } def tearDown(self): unittest.TestCase.tearDown(self) shutil.rmtree(self.tmpdir) def testNoKeys(self): data_empty_keys = { constants.SSHS_SSH_HOST_KEY: [], } for data in [{}, data_empty_keys]: for dry_run in [False, True]: prepare_node_join.UpdateSshDaemon(data, dry_run, _runcmd_fn=NotImplemented, _keyfiles=NotImplemented) self.assertEqual(os.listdir(self.tmpdir), []) def _TestDryRun(self, data): prepare_node_join.UpdateSshDaemon(data, True, _runcmd_fn=NotImplemented, _keyfiles=self.keyfiles) self.assertEqual(os.listdir(self.tmpdir), []) def testDryRunRsa(self): self._TestDryRun({ constants.SSHS_SSH_HOST_KEY: [ (constants.SSHK_RSA, "rsapriv", "rsapub"), ], }) def testDryRunDsa(self): self._TestDryRun({ constants.SSHS_SSH_HOST_KEY: [ (constants.SSHK_DSA, "dsapriv", "dsapub"), ], }) def testDryRunEcdsa(self): self._TestDryRun({ constants.SSHS_SSH_HOST_KEY: [ (constants.SSHK_ECDSA, "ecdsapriv", "ecdsapub"), ], }) def _RunCmd(self, fail, cmd, interactive=NotImplemented): self.assertTrue(interactive) self.assertEqual(cmd, [pathutils.DAEMON_UTIL, "reload-ssh-keys"]) if fail: exit_code = constants.EXIT_FAILURE else: exit_code = constants.EXIT_SUCCESS return utils.RunResult(exit_code, None, "stdout", "stderr", utils.ShellQuoteArgs(cmd), NotImplemented, NotImplemented) def _TestUpdate(self, failcmd): data = { constants.SSHS_SSH_HOST_KEY: [ (constants.SSHK_DSA, "dsapriv", "dsapub"), (constants.SSHK_ECDSA, "ecdsapriv", "ecdsapub"), (constants.SSHK_RSA, "rsapriv", "rsapub"), ], constants.SSHS_SSH_KEY_TYPE: "dsa", constants.SSHS_SSH_KEY_BITS: 1024, } runcmd_fn = compat.partial(self._RunCmd, failcmd) if failcmd: self.assertRaises(_JoinError, prepare_node_join.UpdateSshDaemon, data, False, _runcmd_fn=runcmd_fn, _keyfiles=self.keyfiles) else: prepare_node_join.UpdateSshDaemon(data, False, _runcmd_fn=runcmd_fn, _keyfiles=self.keyfiles) self.assertEqual(sorted(os.listdir(self.tmpdir)), sorted([ "rsa.public", "rsa.private", "dsa.public", "dsa.private", "ecdsa.public", "ecdsa.private", ])) self.assertEqual(utils.ReadFile(utils.PathJoin(self.tmpdir, "rsa.public")), "rsapub") self.assertEqual(utils.ReadFile(utils.PathJoin(self.tmpdir, "rsa.private")), "rsapriv") self.assertEqual(utils.ReadFile(utils.PathJoin(self.tmpdir, "dsa.public")), "dsapub") self.assertEqual(utils.ReadFile(utils.PathJoin(self.tmpdir, "dsa.private")), "dsapriv") self.assertEqual(utils.ReadFile(utils.PathJoin( self.tmpdir, "ecdsa.public")), "ecdsapub") self.assertEqual(utils.ReadFile(utils.PathJoin( self.tmpdir, "ecdsa.private")), "ecdsapriv") def testSuccess(self): self._TestUpdate(False) def testFailure(self): self._TestUpdate(True) class TestUpdateSshRoot(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() self.sshdir = utils.PathJoin(self.tmpdir, ".ssh") def tearDown(self): unittest.TestCase.tearDown(self) shutil.rmtree(self.tmpdir) def _GetHomeDir(self, user): self.assertEqual(user, constants.SSH_LOGIN_USER) return self.tmpdir def testDryRun(self): data = { constants.SSHS_SSH_ROOT_KEY: [ (constants.SSHK_RSA, "aaa", "bbb"), ] } prepare_node_join.UpdateSshRoot(data, True, _homedir_fn=self._GetHomeDir) self.assertEqual(os.listdir(self.tmpdir), [".ssh"]) self.assertEqual(os.listdir(self.sshdir), []) def testUpdate(self): data = { constants.SSHS_SSH_ROOT_KEY: [ (constants.SSHK_RSA, "privatersa", "ssh-rsa pubrsa"), ], constants.SSHS_SSH_KEY_TYPE: "rsa", constants.SSHS_SSH_KEY_BITS: 2048, } prepare_node_join.UpdateSshRoot(data, False, _homedir_fn=self._GetHomeDir) self.assertEqual(os.listdir(self.tmpdir), [".ssh"]) self.assertEqual(sorted(os.listdir(self.sshdir)), sorted(["authorized_keys", "id_rsa", "id_rsa.pub"])) self.assertTrue(utils.ReadFile(utils.PathJoin(self.sshdir, "id_rsa")) is not None) pub_key = utils.ReadFile(utils.PathJoin(self.sshdir, "id_rsa.pub")) self.assertTrue(pub_key is not None) self.assertEqual(utils.ReadFile(utils.PathJoin(self.sshdir, "authorized_keys")), pub_key) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.tools.ssh_update_unittest.py000075500000000000000000000137621476477700300255500ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.tools.ssh_update""" import unittest import shutil import tempfile import os.path from ganeti import constants from ganeti import utils from ganeti.tools import ssh_update import testutils _JoinError = ssh_update.SshUpdateError _DATA_CHECK = ssh_update._DATA_CHECK class TestUpdateAuthorizedKeys(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() self.sshdir = utils.PathJoin(self.tmpdir, ".ssh") def tearDown(self): unittest.TestCase.tearDown(self) shutil.rmtree(self.tmpdir) def _GetHomeDir(self, user): self.assertEqual(user, constants.SSH_LOGIN_USER) return self.tmpdir def testNoop(self): data_empty_keys = {} for data in [{}, data_empty_keys]: for dry_run in [False, True]: ssh_update.UpdateAuthorizedKeys(data, dry_run, _homedir_fn=NotImplemented) self.assertEqual(os.listdir(self.tmpdir), []) def testDryRun(self): data = { constants.SSHS_SSH_AUTHORIZED_KEYS: (constants.SSHS_ADD, { "node1" : ["key11", "key12", "key13"], "node2" : ["key21", "key22"]}), } ssh_update.UpdateAuthorizedKeys(data, True, _homedir_fn=self._GetHomeDir) self.assertEqual(os.listdir(self.tmpdir), [".ssh"]) self.assertEqual(os.listdir(self.sshdir), []) def testAddAndRemove(self): data = { constants.SSHS_SSH_AUTHORIZED_KEYS: (constants.SSHS_ADD, { "node1": ["key11", "key12"], "node2": ["key21"]}), } ssh_update.UpdateAuthorizedKeys(data, False, _homedir_fn=self._GetHomeDir) self.assertEqual(os.listdir(self.tmpdir), [".ssh"]) self.assertEqual(sorted(os.listdir(self.sshdir)), sorted(["authorized_keys"])) self.assertEqual(utils.ReadFile(utils.PathJoin(self.sshdir, "authorized_keys")), "key11\nkey12\nkey21\n") data = { constants.SSHS_SSH_AUTHORIZED_KEYS: (constants.SSHS_REMOVE, { "node1": ["key12"], "node2": ["key21"]}), } ssh_update.UpdateAuthorizedKeys(data, False, _homedir_fn=self._GetHomeDir) self.assertEqual(utils.ReadFile(utils.PathJoin(self.sshdir, "authorized_keys")), "key11\n") def testAddAndRemoveDuplicates(self): data = { constants.SSHS_SSH_AUTHORIZED_KEYS: (constants.SSHS_ADD, { "node1": ["key11", "key12"], "node2": ["key12"]}), } ssh_update.UpdateAuthorizedKeys(data, False, _homedir_fn=self._GetHomeDir) self.assertEqual(os.listdir(self.tmpdir), [".ssh"]) self.assertEqual(sorted(os.listdir(self.sshdir)), sorted(["authorized_keys"])) self.assertEqual(utils.ReadFile(utils.PathJoin(self.sshdir, "authorized_keys")), "key11\nkey12\nkey12\n") data = { constants.SSHS_SSH_AUTHORIZED_KEYS: (constants.SSHS_REMOVE, { "node1": ["key12"]}), } ssh_update.UpdateAuthorizedKeys(data, False, _homedir_fn=self._GetHomeDir) self.assertEqual(utils.ReadFile(utils.PathJoin(self.sshdir, "authorized_keys")), "key11\n") class TestUpdatePubKeyFile(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) def testNoKeys(self): pub_key_file = self._CreateTempFile() data_empty_keys = {} for data in [{}, data_empty_keys]: for dry_run in [False, True]: ssh_update.UpdatePubKeyFile(data, dry_run, key_file=pub_key_file) self.assertEqual(utils.ReadFile(pub_key_file), "") def testAddAndRemoveKeys(self): pub_key_file = self._CreateTempFile() data = { constants.SSHS_SSH_PUBLIC_KEYS: (constants.SSHS_OVERRIDE, { "node1": ["key11", "key12"], "node2": ["key21"]}), } ssh_update.UpdatePubKeyFile(data, False, key_file=pub_key_file) self.assertEqual(utils.ReadFile(pub_key_file), "node1 key11\nnode1 key12\nnode2 key21\n") data = { constants.SSHS_SSH_PUBLIC_KEYS: (constants.SSHS_REMOVE, { "node1": ["key12"], "node3": ["key21"], "node4": ["key33"]}), } ssh_update.UpdatePubKeyFile(data, False, key_file=pub_key_file) self.assertEqual(utils.ReadFile(pub_key_file), "node2 key21\n") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.uidpool_unittest.py000075500000000000000000000106141476477700300237160ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the uidpool module""" import os import tempfile import unittest from ganeti import constants from ganeti import uidpool from ganeti import errors from ganeti import pathutils import testutils class TestUidPool(testutils.GanetiTestCase): """Uid-pool tests""" def setUp(self): self.old_uid_min = constants.UIDPOOL_UID_MIN self.old_uid_max = constants.UIDPOOL_UID_MAX constants.UIDPOOL_UID_MIN = 1 constants.UIDPOOL_UID_MAX = 10 pathutils.UIDPOOL_LOCKDIR = tempfile.mkdtemp() def tearDown(self): constants.UIDPOOL_UID_MIN = self.old_uid_min constants.UIDPOOL_UID_MAX = self.old_uid_max for name in os.listdir(pathutils.UIDPOOL_LOCKDIR): os.unlink(os.path.join(pathutils.UIDPOOL_LOCKDIR, name)) os.rmdir(pathutils.UIDPOOL_LOCKDIR) def testParseUidPool(self): self.assertEqualValues( uidpool.ParseUidPool("1-100,200,"), [(1, 100), (200, 200)]) self.assertEqualValues( uidpool.ParseUidPool("1000:2000-2500", separator=":"), [(1000, 1000), (2000, 2500)]) self.assertEqualValues( uidpool.ParseUidPool("1000\n2000-2500", separator="\n"), [(1000, 1000), (2000, 2500)]) def testCheckUidPool(self): # UID < UIDPOOL_UID_MIN self.assertRaises(errors.OpPrereqError, uidpool.CheckUidPool, [(0, 0)]) # UID > UIDPOOL_UID_MAX self.assertRaises(errors.OpPrereqError, uidpool.CheckUidPool, [(11, 11)]) # lower boundary > higher boundary self.assertRaises(errors.OpPrereqError, uidpool.CheckUidPool, [(2, 1)]) def testFormatUidPool(self): self.assertEqualValues( uidpool.FormatUidPool([(1, 100), (200, 200)]), "1-100, 200") self.assertEqualValues( uidpool.FormatUidPool([(1, 100), (200, 200)], separator=":"), "1-100:200") self.assertEqualValues( uidpool.FormatUidPool([(1, 100), (200, 200)], separator="\n"), "1-100\n200") def testRequestUnusedUid(self): # Check with known used user-ids # # Test with our own user-id which is guaranteed to be used self.assertRaises(errors.LockError, uidpool.RequestUnusedUid, set([os.getuid()])) # Check with a single, known unused user-id # # We use 2^30+42 here, which is a valid UID, but unlikely to be used on # most systems (even as a subuid). free_uid = 2**30 + 42 uid = uidpool.RequestUnusedUid(set([free_uid])) self.assertEqualValues(uid.GetUid(), free_uid) # Check uid-pool exhaustion # # free_uid is locked now, so RequestUnusedUid is expected to fail self.assertRaises(errors.LockError, uidpool.RequestUnusedUid, set([free_uid])) # Check unlocking uid.Unlock() # After unlocking, "-1" should be available again uid = uidpool.RequestUnusedUid(set([free_uid])) self.assertEqualValues(uid.GetUid(), free_uid) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.algo_unittest.py000075500000000000000000000313471476477700300243320ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.algo""" import unittest import random import operator from ganeti import constants from ganeti import compat from ganeti.utils import algo import testutils class TestUniqueSequence(unittest.TestCase): """Test case for UniqueSequence""" def _test(self, input, expected): self.assertEqual(algo.UniqueSequence(input), expected) def runTest(self): # Ordered input self._test([1, 2, 3], [1, 2, 3]) self._test([1, 1, 2, 2, 3, 3], [1, 2, 3]) self._test([1, 2, 2, 3], [1, 2, 3]) self._test([1, 2, 3, 3], [1, 2, 3]) # Unordered input self._test([1, 2, 3, 1, 2, 3], [1, 2, 3]) self._test([1, 1, 2, 3, 3, 1, 2], [1, 2, 3]) # Strings self._test(["a", "a"], ["a"]) self._test(["a", "b"], ["a", "b"]) self._test(["a", "b", "a"], ["a", "b"]) class TestFindDuplicates(unittest.TestCase): """Test case for FindDuplicates""" def _Test(self, seq, expected): result = algo.FindDuplicates(seq) self.assertEqual(result, algo.UniqueSequence(result)) self.assertEqual(set(result), set(expected)) def test(self): self._Test([], []) self._Test([1, 2, 3], []) self._Test([9, 8, 8, 0, 5, 1, 7, 0, 6, 7], [8, 0, 7]) for exp in [[1, 2, 3], [3, 2, 1]]: self._Test([1, 1, 2, 2, 3, 3], exp) self._Test(["A", "a", "B"], []) self._Test(["a", "A", "a", "B"], ["a"]) self._Test("Hello World out there!", ["e", " ", "o", "r", "t", "l"]) self._Test(self._Gen(False), []) self._Test(self._Gen(True), range(1, 10)) @staticmethod def _Gen(dup): for i in range(10): yield i if dup: for _ in range(i): yield i class TestNiceSort(unittest.TestCase): def test(self): self.assertEqual(algo.NiceSort([]), []) self.assertEqual(algo.NiceSort(["foo"]), ["foo"]) self.assertEqual(algo.NiceSort(["bar", ""]), ["", "bar"]) self.assertEqual(algo.NiceSort([",", "."]), [",", "."]) self.assertEqual(algo.NiceSort(["0.1", "0.2"]), ["0.1", "0.2"]) self.assertEqual(algo.NiceSort(["0;099", "0,099", "0.1", "0.2"]), ["0,099", "0.1", "0.2", "0;099"]) data = ["a0", "a1", "a99", "a20", "a2", "b10", "b70", "b00", "0000"] self.assertEqual(algo.NiceSort(data), ["0000", "a0", "a1", "a2", "a20", "a99", "b00", "b10", "b70"]) data = ["a0-0", "a1-0", "a99-10", "a20-3", "a0-4", "a99-3", "a09-2", "Z", "a9-1", "A", "b"] self.assertEqual(algo.NiceSort(data), ["A", "Z", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2", "a20-3", "a99-3", "a99-10", "b"]) self.assertEqual(algo.NiceSort(data, key=str.lower), ["A", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2", "a20-3", "a99-3", "a99-10", "b", "Z"]) self.assertEqual(algo.NiceSort(data, key=str.upper), ["A", "a0-0", "a0-4", "a1-0", "a9-1", "a09-2", "a20-3", "a99-3", "a99-10", "b", "Z"]) def testLargeA(self): data = [ "Eegah9ei", "xij88brTulHYAv8IEOyU", "3jTwJPtrXOY22bwL2YoW", "Z8Ljf1Pf5eBfNg171wJR", "WvNJd91OoXvLzdEiEXa6", "uHXAyYYftCSG1o7qcCqe", "xpIUJeVT1Rp", "KOt7vn1dWXi", "a07h8feON165N67PIE", "bH4Q7aCu3PUPjK3JtH", "cPRi0lM7HLnSuWA2G9", "KVQqLPDjcPjf8T3oyzjcOsfkb", "guKJkXnkULealVC8CyF1xefym", "pqF8dkU5B1cMnyZuREaSOADYx", ] self.assertEqual(algo.NiceSort(data), [ "3jTwJPtrXOY22bwL2YoW", "Eegah9ei", "KOt7vn1dWXi", "KVQqLPDjcPjf8T3oyzjcOsfkb", "WvNJd91OoXvLzdEiEXa6", "Z8Ljf1Pf5eBfNg171wJR", "a07h8feON165N67PIE", "bH4Q7aCu3PUPjK3JtH", "cPRi0lM7HLnSuWA2G9", "guKJkXnkULealVC8CyF1xefym", "pqF8dkU5B1cMnyZuREaSOADYx", "uHXAyYYftCSG1o7qcCqe", "xij88brTulHYAv8IEOyU", "xpIUJeVT1Rp" ]) def testLargeB(self): data = [ "inst-0.0.0.0-0.0.0.0", "inst-0.1.0.0-0.0.0.0", "inst-0.2.0.0-0.0.0.0", "inst-0.2.1.0-0.0.0.0", "inst-0.2.2.0-0.0.0.0", "inst-0.2.2.0-0.0.0.9", "inst-0.2.2.0-0.0.3.9", "inst-0.2.2.0-0.2.0.9", "inst-0.2.2.0-0.9.0.9", "inst-0.20.2.0-0.0.0.0", "inst-0.20.2.0-0.9.0.9", "inst-10.020.2.0-0.9.0.10", "inst-15.020.2.0-0.9.1.00", "inst-100.020.2.0-0.9.0.9", # Only the last group, not converted to a number anymore, differs "inst-100.020.2.0a999", "inst-100.020.2.0b000", "inst-100.020.2.0c10", "inst-100.020.2.0c101", "inst-100.020.2.0c2", "inst-100.020.2.0c20", "inst-100.020.2.0c3", "inst-100.020.2.0c39123", ] rnd = random.Random(16205) for _ in range(10): testdata = data[:] rnd.shuffle(testdata) assert testdata != data self.assertEqual(algo.NiceSort(testdata), data) class _CallCount: def __init__(self, fn): self.count = 0 self.fn = fn def __call__(self, *args): self.count += 1 return self.fn(*args) def testKeyfuncA(self): # Generate some random numbers rnd = random.Random(21131) numbers = [rnd.randint(0, 10000) for _ in range(999)] assert numbers != sorted(numbers) # Convert to hex data = [hex(i) for i in numbers] datacopy = data[:] keyfn = self._CallCount(lambda value: str(int(value, 16))) # Sort with key function converting hex to decimal result = algo.NiceSort(data, key=keyfn) self.assertEqual([hex(i) for i in sorted(numbers)], result) self.assertEqual(data, datacopy, msg="Input data was modified in NiceSort") self.assertEqual(keyfn.count, len(numbers), msg="Key function was not called once per value") class _TestData: def __init__(self, name, value): self.name = name self.value = value def testKeyfuncB(self): rnd = random.Random(27396) data = [] for i in range(123): v1 = rnd.randint(0, 5) v2 = rnd.randint(0, 5) data.append(self._TestData("inst-%s-%s-%s" % (v1, v2, i), (v1, v2, i))) rnd.shuffle(data) assert data != sorted(data, key=operator.attrgetter("name")) keyfn = self._CallCount(operator.attrgetter("name")) # Sort by name result = algo.NiceSort(data, key=keyfn) self.assertEqual(result, sorted(data, key=operator.attrgetter("value"))) self.assertEqual(keyfn.count, len(data), msg="Key function was not called once per value") def testNiceSortKey(self): key = algo.NiceSortKey("") self.assertEqual([k._obj for k in key], ([None] * algo._SORTER_GROUPS) + [""]) key = algo.NiceSortKey("Hello World") self.assertEqual([k._obj for k in key], ["Hello World"] + ([None] * int(algo._SORTER_GROUPS - 1)) + [""]) key = algo.NiceSortKey("node1.net75.bld3.example.com") self.assertEqual([k._obj for k in key], ["node", 1, ".net", 75, ".bld", 3, ".example.com", None, ""]) class TestInvertDict(unittest.TestCase): def testInvertDict(self): test_dict = { "foo": 1, "bar": 2, "baz": 5 } self.assertEqual(algo.InvertDict(test_dict), { 1: "foo", 2: "bar", 5: "baz"}) class TestInsertAtPos(unittest.TestCase): def test(self): a = [1, 5, 6] b = [2, 3, 4] self.assertEqual(algo.InsertAtPos(a, 1, b), [1, 2, 3, 4, 5, 6]) self.assertEqual(algo.InsertAtPos(a, 0, b), b + a) self.assertEqual(algo.InsertAtPos(a, len(a), b), a + b) self.assertEqual(algo.InsertAtPos(a, 2, b), [1, 5, 2, 3, 4, 6]) class TimeMock: def __init__(self, values): self.values = values def __call__(self): return self.values.pop(0) class TestRunningTimeout(unittest.TestCase): def setUp(self): self.time_fn = TimeMock([0.0, 0.3, 4.6, 6.5]) def testRemainingFloat(self): timeout = algo.RunningTimeout(5.0, True, _time_fn=self.time_fn) self.assertAlmostEqual(timeout.Remaining(), 4.7) self.assertAlmostEqual(timeout.Remaining(), 0.4) self.assertAlmostEqual(timeout.Remaining(), -1.5) def testRemaining(self): self.time_fn = TimeMock([0, 2, 4, 5, 6]) timeout = algo.RunningTimeout(5, True, _time_fn=self.time_fn) self.assertEqual(timeout.Remaining(), 3) self.assertEqual(timeout.Remaining(), 1) self.assertEqual(timeout.Remaining(), 0) self.assertEqual(timeout.Remaining(), -1) def testRemainingNonNegative(self): timeout = algo.RunningTimeout(5.0, False, _time_fn=self.time_fn) self.assertAlmostEqual(timeout.Remaining(), 4.7) self.assertAlmostEqual(timeout.Remaining(), 0.4) self.assertEqual(timeout.Remaining(), 0.0) def testNegativeTimeout(self): self.assertRaises(ValueError, algo.RunningTimeout, -1.0, True) class TestJoinDisjointDicts(unittest.TestCase): def setUp(self): self.non_empty_dict = {"a": 1, "b": 2} self.empty_dict = {} def testWithEmptyDicts(self): self.assertEqual(self.empty_dict, algo.JoinDisjointDicts(self.empty_dict, self.empty_dict)) self.assertEqual(self.non_empty_dict, algo.JoinDisjointDicts( self.empty_dict, self.non_empty_dict)) self.assertEqual(self.non_empty_dict, algo.JoinDisjointDicts( self.non_empty_dict, self.empty_dict)) def testNonDisjoint(self): self.assertRaises(AssertionError, algo.JoinDisjointDicts, self.non_empty_dict, self.non_empty_dict) def testCommonCase(self): dict_a = {"TEST1": 1, "TEST2": 2} dict_b = {"TEST3": 3, "TEST4": 4} result = dict_a.copy() result.update(dict_b) self.assertEqual(result, algo.JoinDisjointDicts(dict_a, dict_b)) self.assertEqual(result, algo.JoinDisjointDicts(dict_b, dict_a)) class TestSequenceToDict(unittest.TestCase): def testEmpty(self): self.assertEqual(algo.SequenceToDict([]), {}) self.assertEqual(algo.SequenceToDict({}), {}) def testSimple(self): data = [(i, str(i), "test%s" % i) for i in range(391)] self.assertEqual(algo.SequenceToDict(data), dict((i, (i, str(i), "test%s" % i)) for i in range(391))) def testCustomKey(self): data = [(i, hex(i), "test%s" % i) for i in range(100)] self.assertEqual(algo.SequenceToDict(data, key=compat.snd), dict((hex(i), (i, hex(i), "test%s" % i)) for i in range(100))) self.assertEqual(algo.SequenceToDict(data, key=lambda a_b_val: hash(a_b_val[2])), dict((hash("test%s" % i), (i, hex(i), "test%s" % i)) for i in range(100))) def testDuplicate(self): self.assertRaises(ValueError, algo.SequenceToDict, [(0, 0), (0, 0)]) self.assertRaises(ValueError, algo.SequenceToDict, [(i, ) for i in range(200)] + [(10, )]) class TestFlatToDict(unittest.TestCase): def testNormal(self): data = [ ("lv/xenvg", {"foo": "bar", "bar": "baz"}), ("lv/xenfoo", {"foo": "bar", "baz": "blubb"}), ("san/foo", {"ip": "127.0.0.1", "port": 1337}), ("san/blubb/blibb", 54), ] reference = { "lv": { "xenvg": {"foo": "bar", "bar": "baz"}, "xenfoo": {"foo": "bar", "baz": "blubb"}, }, "san": { "foo": {"ip": "127.0.0.1", "port": 1337}, "blubb": {"blibb": 54}, }, } self.assertEqual(algo.FlatToDict(data), reference) def testUnlikeDepth(self): data = [ ("san/foo", {"ip": "127.0.0.1", "port": 1337}), ("san/foo/blubb", 23), # Another foo entry under san ("san/blubb/blibb", 54), ] self.assertRaises(AssertionError, algo.FlatToDict, data) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.bitarrays_unittest.py000075500000000000000000000044121476477700300254010ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the bitarray utility functions""" import unittest import testutils from bitarray import bitarray from ganeti import errors from ganeti.utils import bitarrays _FREE = bitarray("11100010") _FULL = bitarray("11111111") class GetFreeSlotTest(unittest.TestCase): """Test function that finds a free slot in a bitarray""" def testFreeSlot(self): self.assertEqual(bitarrays.GetFreeSlot(_FREE), 3) def testReservedSlot(self): self.assertRaises(errors.GenericError, bitarrays.GetFreeSlot, _FREE, slot=1) def testNoFreeSlot(self): self.assertRaises(errors.GenericError, bitarrays.GetFreeSlot, _FULL) def testGetAndReserveSlot(self): self.assertEqual(bitarrays.GetFreeSlot(_FREE, slot=5, reserve=True), 5) self.assertEqual(_FREE, bitarray("11100110")) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.filelock_unittest.py000075500000000000000000000115671476477700300252020ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.filelock""" import os import tempfile import unittest from ganeti import constants from ganeti import utils from ganeti import errors import testutils class _BaseFileLockTest: """Test case for the FileLock class""" def testSharedNonblocking(self): self.lock.Shared(blocking=False) self.lock.Close() def testExclusiveNonblocking(self): self.lock.Exclusive(blocking=False) self.lock.Close() def testUnlockNonblocking(self): self.lock.Unlock(blocking=False) self.lock.Close() def testSharedBlocking(self): self.lock.Shared(blocking=True) self.lock.Close() def testExclusiveBlocking(self): self.lock.Exclusive(blocking=True) self.lock.Close() def testUnlockBlocking(self): self.lock.Unlock(blocking=True) self.lock.Close() def testSharedExclusiveUnlock(self): self.lock.Shared(blocking=False) self.lock.Exclusive(blocking=False) self.lock.Unlock(blocking=False) self.lock.Close() def testExclusiveSharedUnlock(self): self.lock.Exclusive(blocking=False) self.lock.Shared(blocking=False) self.lock.Unlock(blocking=False) self.lock.Close() def testSimpleTimeout(self): # These will succeed on the first attempt, hence a short timeout self.lock.Shared(blocking=True, timeout=10.0) self.lock.Exclusive(blocking=False, timeout=10.0) self.lock.Unlock(blocking=True, timeout=10.0) self.lock.Close() @staticmethod def _TryLockInner(filename, shared, blocking): lock = utils.FileLock.Open(filename) if shared: fn = lock.Shared else: fn = lock.Exclusive try: # The timeout doesn't really matter as the parent process waits for us to # finish anyway. fn(blocking=blocking, timeout=0.01) except errors.LockError as err: return False return True def _TryLock(self, *args): return utils.RunInSeparateProcess(self._TryLockInner, self.tmpfile.name, *args) def testTimeout(self): for blocking in [True, False]: self.lock.Exclusive(blocking=True) self.assertFalse(self._TryLock(False, blocking)) self.assertFalse(self._TryLock(True, blocking)) self.lock.Shared(blocking=True) self.assertTrue(self._TryLock(True, blocking)) self.assertFalse(self._TryLock(False, blocking)) def testCloseShared(self): self.lock.Close() self.assertRaises(AssertionError, self.lock.Shared, blocking=False) def testCloseExclusive(self): self.lock.Close() self.assertRaises(AssertionError, self.lock.Exclusive, blocking=False) def testCloseUnlock(self): self.lock.Close() self.assertRaises(AssertionError, self.lock.Unlock, blocking=False) class TestFileLockWithFilename(testutils.GanetiTestCase, _BaseFileLockTest): TESTDATA = "Hello World\n" * 10 def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpfile = tempfile.NamedTemporaryFile() utils.WriteFile(self.tmpfile.name, data=self.TESTDATA) self.lock = utils.FileLock.Open(self.tmpfile.name) # Ensure "Open" didn't truncate file self.assertFileContent(self.tmpfile.name, self.TESTDATA) def tearDown(self): self.assertFileContent(self.tmpfile.name, self.TESTDATA) testutils.GanetiTestCase.tearDown(self) class TestFileLockWithFileObject(unittest.TestCase, _BaseFileLockTest): def setUp(self): self.tmpfile = tempfile.NamedTemporaryFile() self.lock = utils.FileLock(open(self.tmpfile.name, "w"), self.tmpfile.name) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.hash_unittest.py000075500000000000000000000120051476477700300243210ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.hash""" import unittest import random import tempfile from ganeti import constants from ganeti import utils import testutils class TestHmacFunctions(unittest.TestCase): # Digests can be checked with "openssl sha1 -hmac $key" def testSha1Hmac(self): self.assertEqual(utils.Sha1Hmac("", ""), "fbdb1d1b18aa6c08324b7d64b71fb76370690e1d") self.assertEqual(utils.Sha1Hmac("3YzMxZWE", "Hello World"), "ef4f3bda82212ecb2f7ce868888a19092481f1fd") self.assertEqual(utils.Sha1Hmac("TguMTA2K", ""), "f904c2476527c6d3e6609ab683c66fa0652cb1dc") longtext = 1500 * "The quick brown fox jumps over the lazy dog\n" self.assertEqual(utils.Sha1Hmac("3YzMxZWE", longtext), "35901b9a3001a7cdcf8e0e9d7c2e79df2223af54") def testSha1HmacSalt(self): self.assertEqual(utils.Sha1Hmac("TguMTA2K", "", salt="abc0"), "4999bf342470eadb11dfcd24ca5680cf9fd7cdce") self.assertEqual(utils.Sha1Hmac("TguMTA2K", "", salt="abc9"), "17a4adc34d69c0d367d4ffbef96fd41d4df7a6e8") self.assertEqual(utils.Sha1Hmac("3YzMxZWE", "Hello World", salt="xyz0"), "7f264f8114c9066afc9bb7636e1786d996d3cc0d") def testVerifySha1Hmac(self): self.assertTrue(utils.VerifySha1Hmac("", "", ("fbdb1d1b18aa6c08324b" "7d64b71fb76370690e1d"))) self.assertTrue(utils.VerifySha1Hmac("TguMTA2K", "", ("f904c2476527c6d3e660" "9ab683c66fa0652cb1dc"))) digest = "ef4f3bda82212ecb2f7ce868888a19092481f1fd" self.assertTrue(utils.VerifySha1Hmac("3YzMxZWE", "Hello World", digest)) self.assertTrue(utils.VerifySha1Hmac("3YzMxZWE", "Hello World", digest.lower())) self.assertTrue(utils.VerifySha1Hmac("3YzMxZWE", "Hello World", digest.upper())) self.assertTrue(utils.VerifySha1Hmac("3YzMxZWE", "Hello World", digest.title())) def testVerifySha1HmacSalt(self): self.assertTrue(utils.VerifySha1Hmac("TguMTA2K", "", ("17a4adc34d69c0d367d4" "ffbef96fd41d4df7a6e8"), salt="abc9")) self.assertTrue(utils.VerifySha1Hmac("3YzMxZWE", "Hello World", ("7f264f8114c9066afc9b" "b7636e1786d996d3cc0d"), salt="xyz0")) class TestFingerprintFiles(unittest.TestCase): def setUp(self): self.tmpfile = tempfile.NamedTemporaryFile() self.tmpfile2 = tempfile.NamedTemporaryFile() utils.WriteFile(self.tmpfile2.name, data="Hello World\n") self.results = { self.tmpfile.name: "da39a3ee5e6b4b0d3255bfef95601890afd80709", self.tmpfile2.name: "648a6a6ffffdaa0badb23b8baf90b6168dd16b3a", } def testSingleFile(self): self.assertEqual(utils.hash._FingerprintFile(self.tmpfile.name), self.results[self.tmpfile.name]) self.assertEqual(utils.hash._FingerprintFile("/no/such/file"), None) def testBigFile(self): self.tmpfile.write(b"A" * 8192) self.tmpfile.flush() self.assertEqual(utils.hash._FingerprintFile(self.tmpfile.name), "35b6795ca20d6dc0aff8c7c110c96cd1070b8c38") def testMultiple(self): all_files = list(self.results) all_files.append("/no/such/file") self.assertEqual(utils.FingerprintFiles(list(self.results)), self.results) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.io_unittest-runasroot.py000064400000000000000000000122701476477700300260400ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.io (tests that require root access)""" import os import tempfile import shutil import errno import grp import pwd import stat from ganeti import constants from ganeti import utils from ganeti import compat from ganeti import errors import testutils class TestWriteFile(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = None self.tfile = tempfile.NamedTemporaryFile() self.did_pre = False self.did_post = False self.did_write = False def tearDown(self): testutils.GanetiTestCase.tearDown(self) if self.tmpdir: shutil.rmtree(self.tmpdir) def testFileUid(self): self.tmpdir = tempfile.mkdtemp() target = utils.PathJoin(self.tmpdir, "target") tuid = os.geteuid() + 1 utils.WriteFile(target, data="data", uid=tuid + 1) self.assertFileUid(target, tuid + 1) utils.WriteFile(target, data="data", uid=tuid) self.assertFileUid(target, tuid) utils.WriteFile(target, data="data", uid=tuid + 1, keep_perms=utils.KP_IF_EXISTS) self.assertFileUid(target, tuid) utils.WriteFile(target, data="data", keep_perms=utils.KP_ALWAYS) self.assertFileUid(target, tuid) def testNewFileUid(self): self.tmpdir = tempfile.mkdtemp() target = utils.PathJoin(self.tmpdir, "target") tuid = os.geteuid() + 1 utils.WriteFile(target, data="data", uid=tuid, keep_perms=utils.KP_IF_EXISTS) self.assertFileUid(target, tuid) def testFileGid(self): self.tmpdir = tempfile.mkdtemp() target = utils.PathJoin(self.tmpdir, "target") tgid = os.getegid() + 1 utils.WriteFile(target, data="data", gid=tgid + 1) self.assertFileGid(target, tgid + 1) utils.WriteFile(target, data="data", gid=tgid) self.assertFileGid(target, tgid) utils.WriteFile(target, data="data", gid=tgid + 1, keep_perms=utils.KP_IF_EXISTS) self.assertFileGid(target, tgid) utils.WriteFile(target, data="data", keep_perms=utils.KP_ALWAYS) self.assertFileGid(target, tgid) def testNewFileGid(self): self.tmpdir = tempfile.mkdtemp() target = utils.PathJoin(self.tmpdir, "target") tgid = os.getegid() + 1 utils.WriteFile(target, data="data", gid=tgid, keep_perms=utils.KP_IF_EXISTS) self.assertFileGid(target, tgid) class TestCanRead(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() self.confdUid = pwd.getpwnam(constants.CONFD_USER).pw_uid self.masterdUid = pwd.getpwnam(constants.MASTERD_USER).pw_uid self.masterdGid = grp.getgrnam(constants.MASTERD_GROUP).gr_gid def tearDown(self): testutils.GanetiTestCase.tearDown(self) if self.tmpdir: shutil.rmtree(self.tmpdir) def testUserCanRead(self): target = utils.PathJoin(self.tmpdir, "target1") f=open(target, "w") f.close() utils.EnforcePermission(target, 0o400, uid=self.confdUid, gid=self.masterdGid) self.assertTrue(utils.CanRead(constants.CONFD_USER, target)) if constants.CONFD_USER != constants.MASTERD_USER: self.assertFalse(utils.CanRead(constants.MASTERD_USER, target)) def testGroupCanRead(self): target = utils.PathJoin(self.tmpdir, "target2") f=open(target, "w") f.close() utils.EnforcePermission(target, 0o040, uid=self.confdUid, gid=self.masterdGid) self.assertFalse(utils.CanRead(constants.CONFD_USER, target)) if constants.CONFD_USER != constants.MASTERD_USER: self.assertTrue(utils.CanRead(constants.MASTERD_USER, target)) utils.EnforcePermission(target, 0o040, uid=self.masterdUid+1, gid=self.masterdGid) self.assertTrue(utils.CanRead(constants.MASTERD_USER, target)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.io_unittest.py000075500000000000000000001042301476477700300240070ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.io""" import os import tempfile import unittest import shutil import glob import time import signal import stat import errno from hashlib import md5 from ganeti import constants from ganeti import utils from ganeti import compat from ganeti import errors import testutils class TestReadBinaryFile(testutils.GanetiTestCase): def testReadAll(self): data = utils.ReadBinaryFile(testutils.TestDataFilename("cert1.pem")) self.assertEqual(len(data), 1229) h = md5() h.update(data) self.assertEqual(h.hexdigest(), "a02be485db0d82b70c0ae7913b26894e") def testReadSize(self): data = utils.ReadBinaryFile(testutils.TestDataFilename("cert1.pem"), size=100) self.assertEqual(len(data), 100) h = md5() h.update(data) self.assertEqual(h.hexdigest(), "256d28505448898d4741b10c5f5dbc12") def testCallback(self): def _Cb(fh): self.assertEqual(fh.tell(), 0) data = utils.ReadBinaryFile(testutils.TestDataFilename("cert1.pem"), preread=_Cb) self.assertEqual(len(data), 1229) def testError(self): self.assertRaises(EnvironmentError, utils.ReadBinaryFile, "/dev/null/does-not-exist") class TestReadFile(testutils.GanetiTestCase): def testReadAll(self): data = utils.ReadFile(testutils.TestDataFilename("cert1.pem")) self.assertEqual(len(data), 1229) def testReadSize(self): data = utils.ReadFile(testutils.TestDataFilename("cert1.pem"), size=100) self.assertEqual(len(data), 100) def testCallback(self): def _Cb(fh): self.assertEqual(fh.tell(), 0) data = utils.ReadFile(testutils.TestDataFilename("cert1.pem"), preread=_Cb) self.assertEqual(len(data), 1229) def testError(self): self.assertRaises(EnvironmentError, utils.ReadFile, "/dev/null/does-not-exist") class TestReadOneLineFile(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) def testDefault(self): data = utils.ReadOneLineFile(testutils.TestDataFilename("cert1.pem")) self.assertEqual(len(data), 27) self.assertEqual(data, "-----BEGIN CERTIFICATE-----") def testNotStrict(self): data = utils.ReadOneLineFile(testutils.TestDataFilename("cert1.pem"), strict=False) self.assertEqual(len(data), 27) self.assertEqual(data, "-----BEGIN CERTIFICATE-----") def testStrictFailure(self): self.assertRaises(errors.GenericError, utils.ReadOneLineFile, testutils.TestDataFilename("cert1.pem"), strict=True) def testLongLine(self): dummydata = (1024 * "Hello World! ") myfile = self._CreateTempFile() utils.WriteFile(myfile, data=dummydata) datastrict = utils.ReadOneLineFile(myfile, strict=True) datalax = utils.ReadOneLineFile(myfile, strict=False) self.assertEqual(dummydata, datastrict) self.assertEqual(dummydata, datalax) def testNewline(self): myfile = self._CreateTempFile() myline = "myline" for nl in ["", "\n", "\r\n"]: dummydata = "%s%s" % (myline, nl) utils.WriteFile(myfile, data=dummydata) datalax = utils.ReadOneLineFile(myfile, strict=False) self.assertEqual(myline, datalax) datastrict = utils.ReadOneLineFile(myfile, strict=True) self.assertEqual(myline, datastrict) def testWhitespaceAndMultipleLines(self): myfile = self._CreateTempFile() for nl in ["", "\n", "\r\n"]: for ws in [" ", "\t", "\t\t \t", "\t "]: dummydata = (1024 * ("Foo bar baz %s%s" % (ws, nl))) utils.WriteFile(myfile, data=dummydata) datalax = utils.ReadOneLineFile(myfile, strict=False) if nl: self.assertTrue(set("\r\n") & set(dummydata)) self.assertRaises(errors.GenericError, utils.ReadOneLineFile, myfile, strict=True) explen = len("Foo bar baz ") + len(ws) self.assertEqual(len(datalax), explen) self.assertEqual(datalax, dummydata[:explen]) self.assertFalse(set("\r\n") & set(datalax)) else: datastrict = utils.ReadOneLineFile(myfile, strict=True) self.assertEqual(dummydata, datastrict) self.assertEqual(dummydata, datalax) def testEmptylines(self): myfile = self._CreateTempFile() myline = "myline" for nl in ["\n", "\r\n"]: for ol in ["", "otherline"]: dummydata = "%s%s%s%s%s%s" % (nl, nl, myline, nl, ol, nl) utils.WriteFile(myfile, data=dummydata) self.assertTrue(set("\r\n") & set(dummydata)) datalax = utils.ReadOneLineFile(myfile, strict=False) self.assertEqual(myline, datalax) if ol: self.assertRaises(errors.GenericError, utils.ReadOneLineFile, myfile, strict=True) else: datastrict = utils.ReadOneLineFile(myfile, strict=True) self.assertEqual(myline, datastrict) def testEmptyfile(self): myfile = self._CreateTempFile() self.assertRaises(errors.GenericError, utils.ReadOneLineFile, myfile) class TestTimestampForFilename(unittest.TestCase): def test(self): self.assertTrue("." not in utils.TimestampForFilename()) self.assertTrue(":" not in utils.TimestampForFilename()) class TestCreateBackup(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() def tearDown(self): testutils.GanetiTestCase.tearDown(self) shutil.rmtree(self.tmpdir) def testEmpty(self): filename = utils.PathJoin(self.tmpdir, "config.data") utils.WriteFile(filename, data="") bname = utils.CreateBackup(filename) self.assertFileContent(bname, "") self.assertEqual(len(glob.glob("%s*" % filename)), 2) utils.CreateBackup(filename) self.assertEqual(len(glob.glob("%s*" % filename)), 3) utils.CreateBackup(filename) self.assertEqual(len(glob.glob("%s*" % filename)), 4) fifoname = utils.PathJoin(self.tmpdir, "fifo") os.mkfifo(fifoname) self.assertRaises(errors.ProgrammerError, utils.CreateBackup, fifoname) def testContent(self): bkpcount = 0 for data in ["", "X", "Hello World!\n" * 100, "Binary data\0\x01\x02\n"]: for rep in [1, 2, 10, 127]: testdata = data * rep filename = utils.PathJoin(self.tmpdir, "test.data_") utils.WriteFile(filename, data=testdata) self.assertFileContent(filename, testdata) for _ in range(3): bname = utils.CreateBackup(filename) bkpcount += 1 self.assertFileContent(bname, testdata) self.assertEqual(len(glob.glob("%s*" % filename)), 1 + bkpcount) class TestListVisibleFiles(unittest.TestCase): """Test case for ListVisibleFiles""" def setUp(self): self.path = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.path) def _CreateFiles(self, files): for name in files: utils.WriteFile(os.path.join(self.path, name), data="test") def _test(self, files, expected): self._CreateFiles(files) found = utils.ListVisibleFiles(self.path) self.assertEqual(set(found), set(expected)) def testAllVisible(self): files = ["a", "b", "c"] expected = files self._test(files, expected) def testNoneVisible(self): files = [".a", ".b", ".c"] expected = [] self._test(files, expected) def testSomeVisible(self): files = ["a", "b", ".c"] expected = ["a", "b"] self._test(files, expected) def testNonAbsolutePath(self): self.assertRaises(errors.ProgrammerError, utils.ListVisibleFiles, "abc") def testNonNormalizedPath(self): self.assertRaises(errors.ProgrammerError, utils.ListVisibleFiles, "/bin/../tmp") def testMountpoint(self): lvfmp_fn = compat.partial(utils.ListVisibleFiles, _is_mountpoint=lambda _: True) self.assertEqual(lvfmp_fn(self.path), []) # Create "lost+found" as a regular file self._CreateFiles(["foo", "bar", ".baz", "lost+found"]) self.assertEqual(set(lvfmp_fn(self.path)), set(["foo", "bar", "lost+found"])) # Replace "lost+found" with a directory laf_path = utils.PathJoin(self.path, "lost+found") utils.RemoveFile(laf_path) os.mkdir(laf_path) self.assertEqual(set(lvfmp_fn(self.path)), set(["foo", "bar"])) def testLostAndFoundNoMountpoint(self): files = ["foo", "bar", ".Hello World", "lost+found"] expected = ["foo", "bar", "lost+found"] self._test(files, expected) class TestWriteFile(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = None self.tfile = tempfile.NamedTemporaryFile() self.did_pre = False self.did_post = False self.did_write = False def tearDown(self): testutils.GanetiTestCase.tearDown(self) if self.tmpdir: shutil.rmtree(self.tmpdir) def markPre(self, fd): self.did_pre = True def markPost(self, fd): self.did_post = True def markWrite(self, fd): self.did_write = True def testWrite(self): data = "abc" utils.WriteFile(self.tfile.name, data=data) self.assertEqual(utils.ReadFile(self.tfile.name), data) def testWriteSimpleUnicode(self): data = "ιβÎŗ" utils.WriteFile(self.tfile.name, data=data) self.assertEqual(utils.ReadFile(self.tfile.name), data) def testErrors(self): self.assertRaises(errors.ProgrammerError, utils.WriteFile, self.tfile.name, data="test", fn=lambda fd: None) self.assertRaises(errors.ProgrammerError, utils.WriteFile, self.tfile.name) self.assertRaises(errors.ProgrammerError, utils.WriteFile, self.tfile.name, data="test", atime=0) self.assertRaises(errors.ProgrammerError, utils.WriteFile, self.tfile.name, mode=0o400, keep_perms=utils.KP_ALWAYS) self.assertRaises(errors.ProgrammerError, utils.WriteFile, self.tfile.name, uid=0, keep_perms=utils.KP_ALWAYS) self.assertRaises(errors.ProgrammerError, utils.WriteFile, self.tfile.name, gid=0, keep_perms=utils.KP_ALWAYS) self.assertRaises(errors.ProgrammerError, utils.WriteFile, self.tfile.name, mode=0o400, uid=0, keep_perms=utils.KP_ALWAYS) def testPreWrite(self): utils.WriteFile(self.tfile.name, data="", prewrite=self.markPre) self.assertTrue(self.did_pre) self.assertFalse(self.did_post) self.assertFalse(self.did_write) def testPostWrite(self): utils.WriteFile(self.tfile.name, data="", postwrite=self.markPost) self.assertFalse(self.did_pre) self.assertTrue(self.did_post) self.assertFalse(self.did_write) def testWriteFunction(self): utils.WriteFile(self.tfile.name, fn=self.markWrite) self.assertFalse(self.did_pre) self.assertFalse(self.did_post) self.assertTrue(self.did_write) def testDryRun(self): orig = "abc" self.tfile.write(orig.encode()) self.tfile.flush() utils.WriteFile(self.tfile.name, data="hello", dry_run=True) self.assertEqual(utils.ReadFile(self.tfile.name), orig) def testTimes(self): f = self.tfile.name for at, mt in [(0, 0), (1000, 1000), (2000, 3000), (int(time.time()), 5000)]: utils.WriteFile(f, data="hello", atime=at, mtime=mt) st = os.stat(f) self.assertEqual(st.st_atime, at) self.assertEqual(st.st_mtime, mt) def testNoClose(self): data = b"hello" self.assertEqual(utils.WriteFile(self.tfile.name, data="abc"), None) fd = utils.WriteFile(self.tfile.name, data=data, close=False) try: os.lseek(fd, 0, 0) self.assertEqual(os.read(fd, 4096), data) finally: os.close(fd) def testNoLeftovers(self): self.tmpdir = tempfile.mkdtemp() self.assertEqual(utils.WriteFile(utils.PathJoin(self.tmpdir, "test"), data="abc"), None) self.assertEqual(os.listdir(self.tmpdir), ["test"]) def testFailRename(self): self.tmpdir = tempfile.mkdtemp() target = utils.PathJoin(self.tmpdir, "target") os.mkdir(target) self.assertRaises(OSError, utils.WriteFile, target, data="abc") self.assertTrue(os.path.isdir(target)) self.assertEqual(os.listdir(self.tmpdir), ["target"]) self.assertFalse(os.listdir(target)) def testFailRenameDryRun(self): self.tmpdir = tempfile.mkdtemp() target = utils.PathJoin(self.tmpdir, "target") os.mkdir(target) self.assertEqual(utils.WriteFile(target, data="abc", dry_run=True), None) self.assertTrue(os.path.isdir(target)) self.assertEqual(os.listdir(self.tmpdir), ["target"]) self.assertFalse(os.listdir(target)) def testBackup(self): self.tmpdir = tempfile.mkdtemp() testfile = utils.PathJoin(self.tmpdir, "test") self.assertEqual(utils.WriteFile(testfile, data="foo", backup=True), None) self.assertEqual(utils.ReadFile(testfile), "foo") self.assertEqual(os.listdir(self.tmpdir), ["test"]) # Write again assert os.path.isfile(testfile) self.assertEqual(utils.WriteFile(testfile, data="bar", backup=True), None) self.assertEqual(utils.ReadFile(testfile), "bar") self.assertEqual(len(glob.glob("%s.backup*" % testfile)), 1) self.assertTrue("test" in os.listdir(self.tmpdir)) self.assertEqual(len(os.listdir(self.tmpdir)), 2) # Write again as dry-run assert os.path.isfile(testfile) self.assertEqual(utils.WriteFile(testfile, data="000", backup=True, dry_run=True), None) self.assertEqual(utils.ReadFile(testfile), "bar") self.assertEqual(len(glob.glob("%s.backup*" % testfile)), 1) self.assertTrue("test" in os.listdir(self.tmpdir)) self.assertEqual(len(os.listdir(self.tmpdir)), 2) def testFileMode(self): self.tmpdir = tempfile.mkdtemp() target = utils.PathJoin(self.tmpdir, "target") self.assertRaises(OSError, utils.WriteFile, target, data="data", keep_perms=utils.KP_ALWAYS) # All masks have only user bits set, to avoid interactions with umask utils.WriteFile(target, data="data", mode=0o200) self.assertFileMode(target, 0o200) utils.WriteFile(target, data="data", mode=0o400, keep_perms=utils.KP_IF_EXISTS) self.assertFileMode(target, 0o200) utils.WriteFile(target, data="data", keep_perms=utils.KP_ALWAYS) self.assertFileMode(target, 0o200) utils.WriteFile(target, data="data", mode=0o700) self.assertFileMode(target, 0o700) def testNewFileMode(self): self.tmpdir = tempfile.mkdtemp() target = utils.PathJoin(self.tmpdir, "target") utils.WriteFile(target, data="data", mode=0o400, keep_perms=utils.KP_IF_EXISTS) self.assertFileMode(target, 0o400) class TestFileID(testutils.GanetiTestCase): def testEquality(self): name = self._CreateTempFile() oldi = utils.GetFileID(path=name) self.assertTrue(utils.VerifyFileID(oldi, oldi)) def testUpdate(self): name = self._CreateTempFile() oldi = utils.GetFileID(path=name) fd = os.open(name, os.O_RDWR) try: newi = utils.GetFileID(fd=fd) self.assertTrue(utils.VerifyFileID(oldi, newi)) self.assertTrue(utils.VerifyFileID(newi, oldi)) finally: os.close(fd) def testWriteFile(self): name = self._CreateTempFile() oldi = utils.GetFileID(path=name) mtime = oldi[2] os.utime(name, (mtime + 10, mtime + 10)) self.assertRaises(errors.LockError, utils.SafeWriteFile, name, oldi, data="") os.utime(name, (mtime - 10, mtime - 10)) utils.SafeWriteFile(name, oldi, data="") oldi = utils.GetFileID(path=name) mtime = oldi[2] os.utime(name, (mtime + 10, mtime + 10)) # this doesn't raise, since we passed None utils.SafeWriteFile(name, None, data="") def testError(self): t = tempfile.NamedTemporaryFile() self.assertRaises(errors.ProgrammerError, utils.GetFileID, path=t.name, fd=t.fileno()) class TestRemoveFile(unittest.TestCase): """Test case for the RemoveFile function""" def setUp(self): """Create a temp dir and file for each case""" self.tmpdir = tempfile.mkdtemp("", "ganeti-unittest-") fd, self.tmpfile = tempfile.mkstemp("", "", self.tmpdir) os.close(fd) def tearDown(self): if os.path.exists(self.tmpfile): os.unlink(self.tmpfile) os.rmdir(self.tmpdir) def testIgnoreDirs(self): """Test that RemoveFile() ignores directories""" self.assertEqual(None, utils.RemoveFile(self.tmpdir)) def testIgnoreNotExisting(self): """Test that RemoveFile() ignores non-existing files""" utils.RemoveFile(self.tmpfile) utils.RemoveFile(self.tmpfile) def testRemoveFile(self): """Test that RemoveFile does remove a file""" utils.RemoveFile(self.tmpfile) if os.path.exists(self.tmpfile): self.fail("File '%s' not removed" % self.tmpfile) def testRemoveSymlink(self): """Test that RemoveFile does remove symlinks""" symlink = self.tmpdir + "/symlink" os.symlink("no-such-file", symlink) utils.RemoveFile(symlink) if os.path.exists(symlink): self.fail("File '%s' not removed" % symlink) os.symlink(self.tmpfile, symlink) utils.RemoveFile(symlink) if os.path.exists(symlink): self.fail("File '%s' not removed" % symlink) class TestRemoveDir(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): try: shutil.rmtree(self.tmpdir) except EnvironmentError: pass def testEmptyDir(self): utils.RemoveDir(self.tmpdir) self.assertFalse(os.path.isdir(self.tmpdir)) def testNonEmptyDir(self): self.tmpfile = os.path.join(self.tmpdir, "test1") open(self.tmpfile, "w").close() self.assertRaises(EnvironmentError, utils.RemoveDir, self.tmpdir) class TestRename(unittest.TestCase): """Test case for RenameFile""" def setUp(self): """Create a temporary directory""" self.tmpdir = tempfile.mkdtemp() self.tmpfile = os.path.join(self.tmpdir, "test1") # Touch the file open(self.tmpfile, "w").close() def tearDown(self): """Remove temporary directory""" shutil.rmtree(self.tmpdir) def testSimpleRename1(self): """Simple rename 1""" utils.RenameFile(self.tmpfile, os.path.join(self.tmpdir, "xyz")) self.assertTrue(os.path.isfile(os.path.join(self.tmpdir, "xyz"))) def testSimpleRename2(self): """Simple rename 2""" utils.RenameFile(self.tmpfile, os.path.join(self.tmpdir, "xyz"), mkdir=True) self.assertTrue(os.path.isfile(os.path.join(self.tmpdir, "xyz"))) def testRenameMkdir(self): """Rename with mkdir""" utils.RenameFile(self.tmpfile, os.path.join(self.tmpdir, "test/xyz"), mkdir=True) self.assertTrue(os.path.isdir(os.path.join(self.tmpdir, "test"))) self.assertTrue(os.path.isfile(os.path.join(self.tmpdir, "test/xyz"))) self.assertRaises(EnvironmentError, utils.RenameFile, os.path.join(self.tmpdir, "test/xyz"), os.path.join(self.tmpdir, "test/foo/bar/baz"), mkdir=True) self.assertTrue(os.path.exists(os.path.join(self.tmpdir, "test/xyz"))) self.assertFalse(os.path.exists(os.path.join(self.tmpdir, "test/foo/bar"))) self.assertFalse(os.path.exists(os.path.join(self.tmpdir, "test/foo/bar/baz"))) class TestMakedirs(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testNonExisting(self): path = utils.PathJoin(self.tmpdir, "foo") utils.Makedirs(path) self.assertTrue(os.path.isdir(path)) def testExisting(self): path = utils.PathJoin(self.tmpdir, "foo") os.mkdir(path) utils.Makedirs(path) self.assertTrue(os.path.isdir(path)) def testRecursiveNonExisting(self): path = utils.PathJoin(self.tmpdir, "foo/bar/baz") utils.Makedirs(path) self.assertTrue(os.path.isdir(path)) def testRecursiveExisting(self): path = utils.PathJoin(self.tmpdir, "B/moo/xyz") self.assertFalse(os.path.exists(path)) os.mkdir(utils.PathJoin(self.tmpdir, "B")) utils.Makedirs(path) self.assertTrue(os.path.isdir(path)) class TestEnsureDirs(unittest.TestCase): """Tests for EnsureDirs""" def setUp(self): self.dir = tempfile.mkdtemp() self.old_umask = os.umask(0o777) def testEnsureDirs(self): utils.EnsureDirs([ (utils.PathJoin(self.dir, "foo"), 0o777), (utils.PathJoin(self.dir, "bar"), 0000), ]) self.assertEqual(os.stat(utils.PathJoin(self.dir, "foo"))[0] & 0o777, 0o777) self.assertEqual(os.stat(utils.PathJoin(self.dir, "bar"))[0] & 0o777, 0000) def tearDown(self): os.rmdir(utils.PathJoin(self.dir, "foo")) os.rmdir(utils.PathJoin(self.dir, "bar")) os.rmdir(self.dir) os.umask(self.old_umask) class TestIsNormAbsPath(unittest.TestCase): """Testing case for IsNormAbsPath""" def _pathTestHelper(self, path, result): if result: self.assertTrue(utils.IsNormAbsPath(path), msg="Path %s should result absolute and normalized" % path) else: self.assertFalse(utils.IsNormAbsPath(path), msg="Path %s should not result absolute and normalized" % path) def testBase(self): self._pathTestHelper("/etc", True) self._pathTestHelper("/srv", True) self._pathTestHelper("etc", False) self._pathTestHelper("/etc/../root", False) self._pathTestHelper("/etc/", False) def testSlashes(self): # Root directory self._pathTestHelper("/", True) # POSIX' "implementation-defined" double slashes self._pathTestHelper("//", True) # Three and more slashes count as one, so the path is not normalized for i in range(3, 10): self._pathTestHelper("/" * i, False) class TestIsBelowDir(unittest.TestCase): """Testing case for IsBelowDir""" def testExactlyTheSame(self): self.assertFalse(utils.IsBelowDir("/a/b", "/a/b")) self.assertFalse(utils.IsBelowDir("/a/b", "/a/b/")) self.assertFalse(utils.IsBelowDir("/a/b/", "/a/b")) self.assertFalse(utils.IsBelowDir("/a/b/", "/a/b/")) def testSamePrefix(self): self.assertTrue(utils.IsBelowDir("/a/b", "/a/b/c")) self.assertTrue(utils.IsBelowDir("/a/b/", "/a/b/e")) def testSamePrefixButDifferentDir(self): self.assertFalse(utils.IsBelowDir("/a/b", "/a/bc/d")) self.assertFalse(utils.IsBelowDir("/a/b/", "/a/bc/e")) def testSamePrefixButDirTraversal(self): self.assertFalse(utils.IsBelowDir("/a/b", "/a/b/../c")) self.assertFalse(utils.IsBelowDir("/a/b/", "/a/b/../d")) def testSamePrefixAndTraversal(self): self.assertTrue(utils.IsBelowDir("/a/b", "/a/b/c/../d")) self.assertTrue(utils.IsBelowDir("/a/b", "/a/b/c/./e")) self.assertTrue(utils.IsBelowDir("/a/b", "/a/b/../b/./e")) def testBothAbsPath(self): self.assertRaises(ValueError, utils.IsBelowDir, "/a/b/c", "d") self.assertRaises(ValueError, utils.IsBelowDir, "a/b/c", "/d") self.assertRaises(ValueError, utils.IsBelowDir, "a/b/c", "d") self.assertRaises(ValueError, utils.IsBelowDir, "", "/") self.assertRaises(ValueError, utils.IsBelowDir, "/", "") def testRoot(self): self.assertFalse(utils.IsBelowDir("/", "/")) for i in ["/a", "/tmp", "/tmp/foo/bar", "/tmp/"]: self.assertTrue(utils.IsBelowDir("/", i)) def testSlashes(self): # In POSIX a double slash is "implementation-defined". self.assertFalse(utils.IsBelowDir("//", "//")) self.assertFalse(utils.IsBelowDir("//", "/tmp")) self.assertTrue(utils.IsBelowDir("//tmp", "//tmp/x")) # Three (or more) slashes count as one self.assertFalse(utils.IsBelowDir("/", "///")) self.assertTrue(utils.IsBelowDir("/", "///tmp")) self.assertTrue(utils.IsBelowDir("/tmp", "///tmp/a/b")) class TestPathJoin(unittest.TestCase): """Testing case for PathJoin""" def testBasicItems(self): mlist = ["/a", "b", "c"] self.assertEqual(utils.PathJoin(*mlist), "/".join(mlist)) def testNonAbsPrefix(self): self.assertRaises(ValueError, utils.PathJoin, "a", "b") def testBackTrack(self): self.assertRaises(ValueError, utils.PathJoin, "/a", "b/../c") def testMultiAbs(self): self.assertRaises(ValueError, utils.PathJoin, "/a", "/b") class TestTailFile(testutils.GanetiTestCase): """Test case for the TailFile function""" def testEmpty(self): fname = self._CreateTempFile() self.assertEqual(utils.TailFile(fname), []) self.assertEqual(utils.TailFile(fname, lines=25), []) def testAllLines(self): data = ["test %d" % i for i in range(30)] for i in range(30): fname = self._CreateTempFile() fd = open(fname, "w") fd.write("\n".join(data[:i])) if i > 0: fd.write("\n") fd.close() self.assertEqual(utils.TailFile(fname, lines=i), data[:i]) def testPartialLines(self): data = ["test %d" % i for i in range(30)] fname = self._CreateTempFile() fd = open(fname, "w") fd.write("\n".join(data)) fd.write("\n") fd.close() for i in range(1, 30): self.assertEqual(utils.TailFile(fname, lines=i), data[-i:]) def testBigFile(self): data = ["test %d" % i for i in range(30)] fname = self._CreateTempFile() fd = open(fname, "w") fd.write("X" * 1048576) fd.write("\n") fd.write("\n".join(data)) fd.write("\n") fd.close() for i in range(1, 30): self.assertEqual(utils.TailFile(fname, lines=i), data[-i:]) class TestPidFileFunctions(unittest.TestCase): """Tests for WritePidFile and ReadPidFile""" def setUp(self): self.dir = tempfile.mkdtemp() self.f_dpn = lambda name: os.path.join(self.dir, "%s.pid" % name) def testPidFileFunctions(self): pid_file = self.f_dpn("test") fd = utils.WritePidFile(self.f_dpn("test")) self.assertTrue(os.path.exists(pid_file), "PID file should have been created") read_pid = utils.ReadPidFile(pid_file) self.assertEqual(read_pid, os.getpid()) self.assertTrue(utils.IsProcessAlive(read_pid)) self.assertRaises(errors.PidFileLockError, utils.WritePidFile, self.f_dpn("test")) os.close(fd) utils.RemoveFile(self.f_dpn("test")) self.assertFalse(os.path.exists(pid_file), "PID file should not exist anymore") self.assertEqual(utils.ReadPidFile(pid_file), 0, "ReadPidFile should return 0 for missing pid file") fh = open(pid_file, "w") fh.write("blah\n") fh.close() self.assertEqual(utils.ReadPidFile(pid_file), 0, "ReadPidFile should return 0 for invalid pid file") # but now, even with the file existing, we should be able to lock it fd = utils.WritePidFile(self.f_dpn("test")) os.close(fd) utils.RemoveFile(self.f_dpn("test")) self.assertFalse(os.path.exists(pid_file), "PID file should not exist anymore") def testKill(self): pid_file = self.f_dpn("child") r_fd, w_fd = os.pipe() new_pid = os.fork() if new_pid == 0: #child utils.WritePidFile(self.f_dpn("child")) os.write(w_fd, b"a") signal.pause() os._exit(0) return # else we are in the parent # wait until the child has written the pid file os.read(r_fd, 1) read_pid = utils.ReadPidFile(pid_file) self.assertEqual(read_pid, new_pid) self.assertTrue(utils.IsProcessAlive(new_pid)) # Try writing to locked file try: utils.WritePidFile(pid_file) except errors.PidFileLockError as err: errmsg = str(err) self.assertTrue(errmsg.endswith(" %s" % new_pid), msg=("Error message ('%s') didn't contain correct" " PID (%s)" % (errmsg, new_pid))) else: self.fail("Writing to locked file didn't fail") utils.KillProcess(new_pid, waitpid=True) self.assertFalse(utils.IsProcessAlive(new_pid)) utils.RemoveFile(self.f_dpn("child")) self.assertRaises(errors.ProgrammerError, utils.KillProcess, 0) def testExceptionType(self): # Make sure the PID lock error is a subclass of LockError in case some code # depends on it self.assertTrue(issubclass(errors.PidFileLockError, errors.LockError)) def tearDown(self): shutil.rmtree(self.dir) class TestNewUUID(unittest.TestCase): """Test case for NewUUID""" def runTest(self): self.assertTrue(utils.UUID_RE.match(utils.NewUUID())) def _MockStatResult(cb, mode, uid, gid): def _fn(path): if cb: cb() return { stat.ST_MODE: mode, stat.ST_UID: uid, stat.ST_GID: gid, } return _fn def _RaiseNoEntError(): raise EnvironmentError(errno.ENOENT, "not found") def _OtherStatRaise(): raise EnvironmentError() class TestPermissionEnforcements(unittest.TestCase): UID_A = 16024 UID_B = 25850 GID_A = 14028 GID_B = 29801 def setUp(self): self._chown_calls = [] self._chmod_calls = [] self._mkdir_calls = [] def tearDown(self): self.assertRaises(IndexError, self._mkdir_calls.pop) self.assertRaises(IndexError, self._chmod_calls.pop) self.assertRaises(IndexError, self._chown_calls.pop) def _FakeMkdir(self, path): self._mkdir_calls.append(path) def _FakeChown(self, path, uid, gid): self._chown_calls.append((path, uid, gid)) def _ChmodWrapper(self, cb): def _fn(path, mode): self._chmod_calls.append((path, mode)) if cb: cb() return _fn def _VerifyPerm(self, path, mode, uid=-1, gid=-1): self.assertEqual(path, "/ganeti-qa-non-test") self.assertEqual(mode, 0o700) self.assertEqual(uid, self.UID_A) self.assertEqual(gid, self.GID_A) def testMakeDirWithPerm(self): is_dir_stat = _MockStatResult(None, stat.S_IFDIR, 0, 0) utils.MakeDirWithPerm("/ganeti-qa-non-test", 0o700, self.UID_A, self.GID_A, _lstat_fn=is_dir_stat, _perm_fn=self._VerifyPerm) def testDirErrors(self): self.assertRaises(errors.GenericError, utils.MakeDirWithPerm, "/ganeti-qa-non-test", 0o700, 0, 0, _lstat_fn=_MockStatResult(None, 0, 0, 0)) self.assertRaises(IndexError, self._mkdir_calls.pop) other_stat_raise = _MockStatResult(_OtherStatRaise, stat.S_IFDIR, 0, 0) self.assertRaises(errors.GenericError, utils.MakeDirWithPerm, "/ganeti-qa-non-test", 0o700, 0, 0, _lstat_fn=other_stat_raise) self.assertRaises(IndexError, self._mkdir_calls.pop) non_exist_stat = _MockStatResult(_RaiseNoEntError, stat.S_IFDIR, 0, 0) utils.MakeDirWithPerm("/ganeti-qa-non-test", 0o700, self.UID_A, self.GID_A, _lstat_fn=non_exist_stat, _mkdir_fn=self._FakeMkdir, _perm_fn=self._VerifyPerm) self.assertEqual(self._mkdir_calls.pop(0), "/ganeti-qa-non-test") def testEnforcePermissionNoEnt(self): self.assertRaises(errors.GenericError, utils.EnforcePermission, "/ganeti-qa-non-test", 0o600, _chmod_fn=NotImplemented, _chown_fn=NotImplemented, _stat_fn=_MockStatResult(_RaiseNoEntError, 0, 0, 0)) def testEnforcePermissionNoEntMustNotExist(self): utils.EnforcePermission("/ganeti-qa-non-test", 0o600, must_exist=False, _chmod_fn=NotImplemented, _chown_fn=NotImplemented, _stat_fn=_MockStatResult(_RaiseNoEntError, 0, 0, 0)) def testEnforcePermissionOtherErrorMustNotExist(self): self.assertRaises(errors.GenericError, utils.EnforcePermission, "/ganeti-qa-non-test", 0o600, must_exist=False, _chmod_fn=NotImplemented, _chown_fn=NotImplemented, _stat_fn=_MockStatResult(_OtherStatRaise, 0, 0, 0)) def testEnforcePermissionNoChanges(self): utils.EnforcePermission("/ganeti-qa-non-test", 0o600, _stat_fn=_MockStatResult(None, 0o600, 0, 0), _chmod_fn=self._ChmodWrapper(None), _chown_fn=self._FakeChown) def testEnforcePermissionChangeMode(self): utils.EnforcePermission("/ganeti-qa-non-test", 0o444, _stat_fn=_MockStatResult(None, 0o600, 0, 0), _chmod_fn=self._ChmodWrapper(None), _chown_fn=self._FakeChown) self.assertEqual(self._chmod_calls.pop(0), ("/ganeti-qa-non-test", 0o444)) def testEnforcePermissionSetUidGid(self): utils.EnforcePermission("/ganeti-qa-non-test", 0o600, uid=self.UID_B, gid=self.GID_B, _stat_fn=_MockStatResult(None, 0o600, self.UID_A, self.GID_A), _chmod_fn=self._ChmodWrapper(None), _chown_fn=self._FakeChown) self.assertEqual(self._chown_calls.pop(0), ("/ganeti-qa-non-test", self.UID_B, self.GID_B)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.livelock_unittest.py000075500000000000000000000043551476477700300252170ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2018 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing utils.LiveLock """ import os import unittest from ganeti.utils.livelock import LiveLock from ganeti import pathutils import testutils class TestLiveLock(unittest.TestCase): """Whether LiveLock() works """ @testutils.patch_object(pathutils, 'LIVELOCK_DIR', '/tmp') def test(self): lock = LiveLock() lock_path = lock.GetPath() self.assertTrue(os.path.exists(lock_path)) self.assertTrue(lock_path.startswith('/tmp/pid')) lock.close() self.assertFalse(os.path.exists(lock_path)) @testutils.patch_object(pathutils, 'LIVELOCK_DIR', '/tmp') def testName(self): lock = LiveLock('unittest.lock') lock_path = lock.GetPath() self.assertTrue(os.path.exists(lock_path)) self.assertTrue(lock_path.startswith('/tmp/unittest.lock_')) lock.close() self.assertFalse(os.path.exists(lock_path)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.log_unittest.py000075500000000000000000000221141476477700300241610ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.log""" import os import unittest import logging import tempfile import shutil import threading from io import FileIO, StringIO from ganeti import constants from ganeti import errors from ganeti import compat from ganeti import utils import testutils class TestLogHandler(unittest.TestCase): def testNormal(self): tmpfile = tempfile.NamedTemporaryFile() handler = utils.log._ReopenableLogHandler(tmpfile.name) handler.setFormatter(logging.Formatter("%(asctime)s: %(message)s")) logger = logging.Logger("TestLogger") logger.addHandler(handler) self.assertEqual(len(logger.handlers), 1) logger.error("Test message ERROR") logger.info("Test message INFO") logger.removeHandler(handler) self.assertFalse(logger.handlers) handler.close() self.assertEqual(len(utils.ReadFile(tmpfile.name).splitlines()), 2) def testReopen(self): tmpfile = tempfile.NamedTemporaryFile() tmpfile2 = tempfile.NamedTemporaryFile() handler = utils.log._ReopenableLogHandler(tmpfile.name) self.assertFalse(utils.ReadFile(tmpfile.name)) self.assertFalse(utils.ReadFile(tmpfile2.name)) logger = logging.Logger("TestLoggerReopen") logger.addHandler(handler) for _ in range(3): logger.error("Test message ERROR") handler.flush() self.assertEqual(len(utils.ReadFile(tmpfile.name).splitlines()), 3) before_id = utils.GetFileID(tmpfile.name) handler.RequestReopen() self.assertTrue(handler._reopen) self.assertTrue(utils.VerifyFileID(utils.GetFileID(tmpfile.name), before_id)) # Rename only after requesting reopen os.rename(tmpfile.name, tmpfile2.name) assert not os.path.exists(tmpfile.name) # Write another message, should reopen for _ in range(4): logger.info("Test message INFO") # Flag must be reset self.assertFalse(handler._reopen) self.assertFalse(utils.VerifyFileID(utils.GetFileID(tmpfile.name), before_id)) logger.removeHandler(handler) self.assertFalse(logger.handlers) handler.close() self.assertEqual(len(utils.ReadFile(tmpfile.name).splitlines()), 4) self.assertEqual(len(utils.ReadFile(tmpfile2.name).splitlines()), 3) def testConsole(self): temp_file = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") failing_file = self._FailingFile(os.devnull, "w") for (console, check) in [(None, False), (temp_file, True), (failing_file, False)]: # Create a handler which will fail when handling errors cls = utils.log._LogErrorsToConsole(self._FailingHandler) # Instantiate handler with file which will fail when writing, # provoking a write to the console failing_output = self._FailingFile(os.devnull) handler = cls(console, failing_output) logger = logging.Logger("TestLogger") logger.addHandler(handler) self.assertEqual(len(logger.handlers), 1) # Provoke write logger.error("Test message ERROR") # Take everything apart logger.removeHandler(handler) self.assertFalse(logger.handlers) handler.close() failing_output.close() if console and check: console.flush() # Check console output consout = utils.ReadFile(console.name) self.assertTrue("Cannot log message" in consout) self.assertTrue("Test message ERROR" in consout) temp_file.close() failing_file.close() class _FailingFile(FileIO): def write(self, _): raise Exception class _FailingHandler(logging.StreamHandler): def handleError(self, _): raise Exception class TestSetupLogging(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testSimple(self): logfile = utils.PathJoin(self.tmpdir, "basic.log") logger = logging.Logger("TestLogger") self.assertTrue(callable(utils.SetupLogging(logfile, "test", console_logging=False, syslog=constants.SYSLOG_NO, stderr_logging=False, multithreaded=False, root_logger=logger))) self.assertEqual(utils.ReadFile(logfile), "") logger.error("This is a test") # Ensure SetupLogging used custom logger logging.error("This message should not show up in the test log file") self.assertTrue(utils.ReadFile(logfile).endswith("This is a test\n")) def testReopen(self): logfile = utils.PathJoin(self.tmpdir, "reopen.log") logfile2 = utils.PathJoin(self.tmpdir, "reopen.log.OLD") logger = logging.Logger("TestLogger") reopen_fn = utils.SetupLogging(logfile, "test", console_logging=False, syslog=constants.SYSLOG_NO, stderr_logging=False, multithreaded=False, root_logger=logger) self.assertTrue(callable(reopen_fn)) self.assertEqual(utils.ReadFile(logfile), "") logger.error("This is a test") self.assertTrue(utils.ReadFile(logfile).endswith("This is a test\n")) os.rename(logfile, logfile2) assert not os.path.exists(logfile) # Notify logger to reopen on the next message reopen_fn() assert not os.path.exists(logfile) # Provoke actual reopen logger.error("First message") self.assertTrue(utils.ReadFile(logfile).endswith("First message\n")) self.assertTrue(utils.ReadFile(logfile2).endswith("This is a test\n")) class TestSetupToolLogging(unittest.TestCase): def test(self): error_name = logging.getLevelName(logging.ERROR) warn_name = logging.getLevelName(logging.WARNING) info_name = logging.getLevelName(logging.INFO) debug_name = logging.getLevelName(logging.DEBUG) for debug in [False, True]: for verbose in [False, True]: logger = logging.Logger("TestLogger") buf = StringIO() utils.SetupToolLogging(debug, verbose, _root_logger=logger, _stream=buf) logger.error("level=error") logger.warning("level=warning") logger.info("level=info") logger.debug("level=debug") lines = buf.getvalue().splitlines() self.assertTrue(compat.all(line.count(":") == 3 for line in lines)) messages = [line.split(":", 3)[-1].strip() for line in lines] if debug: self.assertEqual(messages, [ "%s level=error" % error_name, "%s level=warning" % warn_name, "%s level=info" % info_name, "%s level=debug" % debug_name, ]) elif verbose: self.assertEqual(messages, [ "%s level=error" % error_name, "%s level=warning" % warn_name, "%s level=info" % info_name, ]) else: self.assertEqual(messages, [ "level=error", "level=warning", ]) def testThreadName(self): thread_name = threading.current_thread().name for enable_threadname in [False, True]: logger = logging.Logger("TestLogger") buf = StringIO() utils.SetupToolLogging(True, True, threadname=enable_threadname, _root_logger=logger, _stream=buf) logger.debug("test134042376") lines = buf.getvalue().splitlines() self.assertEqual(len(lines), 1) if enable_threadname: self.assertTrue((" %s " % thread_name) in lines[0]) else: self.assertTrue(thread_name not in lines[0]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.lvm_unittest.py000075500000000000000000000111501476477700300241740ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.lvm""" import unittest from ganeti import constants from ganeti import utils from ganeti.objects import LvmPvInfo import testutils class TestLvmExclusiveCheckNodePvs(unittest.TestCase): """Test cases for LvmExclusiveCheckNodePvs()""" _VG = "vg" _SMALL_PV = LvmPvInfo(name="small", vg_name=_VG, size=100e3, free=40e3, attributes="a-") _MED_PV = LvmPvInfo(name="medium", vg_name=_VG, size=400e3, free=40e3, attributes="a-") _BIG_PV = LvmPvInfo(name="big", vg_name=_VG, size=1e6, free=400e3, attributes="a-") # Allowance for rounding _EPS = 1e-4 def testOnePv(self): (errmsgs, (small, big)) = utils.LvmExclusiveCheckNodePvs([self._MED_PV]) self.assertFalse(errmsgs) self.assertEqual(small, self._MED_PV.size) self.assertEqual(big, self._MED_PV.size) def testEqualPvs(self): (errmsgs, (small, big)) = utils.LvmExclusiveCheckNodePvs( [self._MED_PV] * 2) self.assertFalse(errmsgs) self.assertEqual(small, self._MED_PV.size) self.assertEqual(big, self._MED_PV.size) (errmsgs, (small, big)) = utils.LvmExclusiveCheckNodePvs( [self._SMALL_PV] * 3) self.assertFalse(errmsgs) self.assertEqual(small, self._SMALL_PV.size) self.assertEqual(big, self._SMALL_PV.size) def testTooDifferentPvs(self): (errmsgs, (small, big)) = utils.LvmExclusiveCheckNodePvs( [self._MED_PV, self._BIG_PV]) self.assertEqual(len(errmsgs), 1) self.assertEqual(small, self._MED_PV.size) self.assertEqual(big, self._BIG_PV.size) (errmsgs, (small, big)) = utils.LvmExclusiveCheckNodePvs( [self._MED_PV, self._SMALL_PV]) self.assertEqual(len(errmsgs), 1) self.assertEqual(small, self._SMALL_PV.size) self.assertEqual(big, self._MED_PV.size) def testBoundarySizeCases(self): medpv1 = self._MED_PV.Copy() medpv2 = self._MED_PV.Copy() (errmsgs, (small, big)) = utils.LvmExclusiveCheckNodePvs( [medpv1, medpv2, self._MED_PV]) self.assertFalse(errmsgs) self.assertEqual(small, self._MED_PV.size) self.assertEqual(big, self._MED_PV.size) # Just within the margins medpv1.size = self._MED_PV.size * (1 - constants.PART_MARGIN + self._EPS) medpv2.size = self._MED_PV.size * (1 + constants.PART_MARGIN - self._EPS) (errmsgs, (small, big)) = utils.LvmExclusiveCheckNodePvs( [medpv1, medpv2, self._MED_PV]) self.assertFalse(errmsgs) self.assertEqual(small, medpv1.size) self.assertEqual(big, medpv2.size) # Just outside the margins medpv1.size = self._MED_PV.size * (1 - constants.PART_MARGIN - self._EPS) medpv2.size = self._MED_PV.size * (1 + constants.PART_MARGIN) (errmsgs, (small, big)) = utils.LvmExclusiveCheckNodePvs( [medpv1, medpv2, self._MED_PV]) self.assertTrue(errmsgs) self.assertEqual(small, medpv1.size) self.assertEqual(big, medpv2.size) medpv1.size = self._MED_PV.size * (1 - constants.PART_MARGIN) medpv2.size = self._MED_PV.size * (1 + constants.PART_MARGIN + self._EPS) (errmsgs, (small, big)) = utils.LvmExclusiveCheckNodePvs( [medpv1, medpv2, self._MED_PV]) self.assertTrue(errmsgs) self.assertEqual(small, medpv1.size) self.assertEqual(big, medpv2.size) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.mlock_unittest.py000075500000000000000000000041231476477700300245050ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing utils.Mlockall This test is run in a separate process because it changes memory behaviour. """ import unittest from ganeti import utils from ganeti import errors import testutils # WARNING: The following tests modify the memory behaviour at runtime. Don't # add unrelated tests here. class TestMlockallWithCtypes(unittest.TestCase): """Whether Mlockall() works if ctypes is present. """ def test(self): if utils.ctypes: utils.Mlockall() class TestMlockallWithNoCtypes(unittest.TestCase): """Whether Mlockall() raises an error if ctypes is not present. """ def test(self): self.assertRaises(errors.NoCtypesError, utils.Mlockall, _ctypes=None) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.nodesetup_unittest.py000075500000000000000000000110561476477700300254110ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.nodesetup""" import os import tempfile import unittest from ganeti import constants from ganeti import utils import testutils class TestEtcHosts(testutils.GanetiTestCase): """Test functions modifying /etc/hosts""" def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpname = self._CreateTempFile() handle = open(self.tmpname, "w") try: handle.write("# This is a test file for /etc/hosts\n") handle.write("127.0.0.1\tlocalhost\n") handle.write("192.0.2.1 router gw\n") finally: handle.close() os.chmod(self.tmpname, 0o644) def testSettingNewIp(self): utils.SetEtcHostsEntry(self.tmpname, "198.51.100.4", "myhost.example.com", ["myhost"]) self.assertFileContent(self.tmpname, "# This is a test file for /etc/hosts\n" "127.0.0.1\tlocalhost\n" "192.0.2.1 router gw\n" "198.51.100.4\tmyhost.example.com myhost\n") self.assertFileMode(self.tmpname, 0o644) def testSettingExistingIp(self): utils.SetEtcHostsEntry(self.tmpname, "192.0.2.1", "myhost.example.com", ["myhost"]) self.assertFileContent(self.tmpname, "# This is a test file for /etc/hosts\n" "127.0.0.1\tlocalhost\n" "192.0.2.1\tmyhost.example.com myhost\n") self.assertFileMode(self.tmpname, 0o644) def testSettingDuplicateName(self): utils.SetEtcHostsEntry(self.tmpname, "198.51.100.4", "myhost", ["myhost"]) self.assertFileContent(self.tmpname, "# This is a test file for /etc/hosts\n" "127.0.0.1\tlocalhost\n" "192.0.2.1 router gw\n" "198.51.100.4\tmyhost\n") self.assertFileMode(self.tmpname, 0o644) def testSettingOrdering(self): utils.SetEtcHostsEntry(self.tmpname, "127.0.0.1", "localhost.localdomain", ["localhost"]) self.assertFileContent(self.tmpname, "# This is a test file for /etc/hosts\n" "127.0.0.1\tlocalhost.localdomain localhost\n" "192.0.2.1 router gw\n") self.assertFileMode(self.tmpname, 0o644) def testRemovingExistingHost(self): utils.RemoveEtcHostsEntry(self.tmpname, "router") self.assertFileContent(self.tmpname, "# This is a test file for /etc/hosts\n" "127.0.0.1\tlocalhost\n" "192.0.2.1 gw\n") self.assertFileMode(self.tmpname, 0o644) def testRemovingSingleExistingHost(self): utils.RemoveEtcHostsEntry(self.tmpname, "localhost") self.assertFileContent(self.tmpname, "# This is a test file for /etc/hosts\n" "192.0.2.1 router gw\n") self.assertFileMode(self.tmpname, 0o644) def testRemovingNonExistingHost(self): utils.RemoveEtcHostsEntry(self.tmpname, "myhost") self.assertFileContent(self.tmpname, "# This is a test file for /etc/hosts\n" "127.0.0.1\tlocalhost\n" "192.0.2.1 router gw\n") self.assertFileMode(self.tmpname, 0o644) def testRemovingAlias(self): utils.RemoveEtcHostsEntry(self.tmpname, "gw") self.assertFileContent(self.tmpname, "# This is a test file for /etc/hosts\n" "127.0.0.1\tlocalhost\n" "192.0.2.1 router\n") self.assertFileMode(self.tmpname, 0o644) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.process_unittest.py000075500000000000000000000646111476477700300250660ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.process""" import os import select import shutil import signal import stat import subprocess import tempfile import time import unittest from ganeti import constants from ganeti import utils from ganeti import errors import testutils class TestIsProcessAlive(unittest.TestCase): """Testing case for IsProcessAlive""" def testExists(self): mypid = os.getpid() self.assertTrue(utils.IsProcessAlive(mypid), "can't find myself running") def testNotExisting(self): pid_non_existing = os.fork() if pid_non_existing == 0: os._exit(0) elif pid_non_existing < 0: raise SystemError("can't fork") os.waitpid(pid_non_existing, 0) self.assertFalse(utils.IsProcessAlive(pid_non_existing), "nonexisting process detected") class TestGetProcStatusPath(unittest.TestCase): def test(self): self.assertTrue("/1234/" in utils.process._GetProcStatusPath(1234)) self.assertNotEqual(utils.process._GetProcStatusPath(1), utils.process._GetProcStatusPath(2)) class TestIsProcessHandlingSignal(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testParseSigsetT(self): parse_sigset_t_fn = utils.process._ParseSigsetT self.assertEqual(len(parse_sigset_t_fn("0")), 0) self.assertEqual(parse_sigset_t_fn("1"), set([1])) self.assertEqual(parse_sigset_t_fn("1000a"), set([2, 4, 17])) self.assertEqual(parse_sigset_t_fn("810002"), set([2, 17, 24, ])) self.assertEqual(parse_sigset_t_fn("0000000180000202"), set([2, 10, 32, 33])) self.assertEqual(parse_sigset_t_fn("0000000180000002"), set([2, 32, 33])) self.assertEqual(parse_sigset_t_fn("0000000188000002"), set([2, 28, 32, 33])) self.assertEqual(parse_sigset_t_fn("000000004b813efb"), set([1, 2, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 17, 24, 25, 26, 28, 31])) self.assertEqual(parse_sigset_t_fn("ffffff"), set(range(1, 25))) def testGetProcStatusField(self): for field in ["SigCgt", "Name", "FDSize"]: for value in ["", "0", "cat", " 1234 KB"]: pstatus = "\n".join([ "VmPeak: 999 kB", "%s: %s" % (field, value), "TracerPid: 0", ]) result = utils.process._GetProcStatusField(pstatus, field) self.assertEqual(result, value.strip()) def test(self): sp = utils.PathJoin(self.tmpdir, "status") utils.WriteFile(sp, data="\n".join([ "Name: bash", "State: S (sleeping)", "SleepAVG: 98%", "Pid: 22250", "PPid: 10858", "TracerPid: 0", "SigBlk: 0000000000010000", "SigIgn: 0000000000384004", "SigCgt: 000000004b813efb", "CapEff: 0000000000000000", ])) self.assertTrue(utils.IsProcessHandlingSignal(1234, 10, status_path=sp)) def testNoSigCgt(self): sp = utils.PathJoin(self.tmpdir, "status") utils.WriteFile(sp, data="\n".join([ "Name: bash", ])) self.assertRaises(RuntimeError, utils.IsProcessHandlingSignal, 1234, 10, status_path=sp) def testNoSuchFile(self): sp = utils.PathJoin(self.tmpdir, "notexist") self.assertFalse(utils.IsProcessHandlingSignal(1234, 10, status_path=sp)) @staticmethod def _TestRealProcess(): signal.signal(signal.SIGUSR1, signal.SIG_DFL) if utils.IsProcessHandlingSignal(os.getpid(), signal.SIGUSR1): raise Exception("SIGUSR1 is handled when it should not be") signal.signal(signal.SIGUSR1, lambda signum, frame: None) if not utils.IsProcessHandlingSignal(os.getpid(), signal.SIGUSR1): raise Exception("SIGUSR1 is not handled when it should be") signal.signal(signal.SIGUSR1, signal.SIG_IGN) if utils.IsProcessHandlingSignal(os.getpid(), signal.SIGUSR1): raise Exception("SIGUSR1 is not handled when it should be") signal.signal(signal.SIGUSR1, signal.SIG_DFL) if utils.IsProcessHandlingSignal(os.getpid(), signal.SIGUSR1): raise Exception("SIGUSR1 is handled when it should not be") return True def testRealProcess(self): self.assertTrue(utils.RunInSeparateProcess(self._TestRealProcess)) class _PostforkProcessReadyHelper: """A helper to use with C{postfork_fn} in RunCmd. It makes sure a process has reached a certain state by reading from a fifo. @ivar write_fd: The fd number to write to """ def __init__(self, timeout): """Initialize the helper. @param fifo_dir: The dir where we can create the fifo @param timeout: The time in seconds to wait before giving up """ self.timeout = timeout (self.read_fd, self.write_fd) = os.pipe() def Ready(self, pid): """Waits until the process is ready. @param pid: The pid of the process """ (read_ready, _, _) = select.select([self.read_fd], [], [], self.timeout) if not read_ready: # We hit the timeout raise AssertionError("Timeout %d reached while waiting for process %d" " to become ready" % (self.timeout, pid)) def Cleanup(self): """Cleans up the helper. """ os.close(self.read_fd) os.close(self.write_fd) class TestRunCmd(testutils.GanetiTestCase): """Testing case for the RunCmd function""" def setUp(self): testutils.GanetiTestCase.setUp(self) self.magic = time.ctime() + " ganeti test" self.fname = self._CreateTempFile() self.fifo_tmpdir = tempfile.mkdtemp() self.fifo_file = os.path.join(self.fifo_tmpdir, "ganeti_test_fifo") os.mkfifo(self.fifo_file) # If the process is not ready after 20 seconds we have bigger issues self.proc_ready_helper = _PostforkProcessReadyHelper(20) def tearDown(self): self.proc_ready_helper.Cleanup() shutil.rmtree(self.fifo_tmpdir) testutils.GanetiTestCase.tearDown(self) def testOk(self): """Test successful exit code""" result = utils.RunCmd("/bin/sh -c 'exit 0'") self.assertEqual(result.exit_code, 0) self.assertEqual(result.output, "") def testFail(self): """Test fail exit code""" result = utils.RunCmd("/bin/sh -c 'exit 1'") self.assertEqual(result.exit_code, 1) self.assertEqual(result.output, "") def testStdout(self): """Test standard output""" cmd = 'echo -n "%s"' % self.magic result = utils.RunCmd("/bin/sh -c '%s'" % cmd) self.assertEqual(result.stdout, self.magic) result = utils.RunCmd("/bin/sh -c '%s'" % cmd, output=self.fname) self.assertEqual(result.output, "") self.assertFileContent(self.fname, self.magic) def testStderr(self): """Test standard error""" cmd = 'echo -n "%s"' % self.magic result = utils.RunCmd("/bin/sh -c '%s' 1>&2" % cmd) self.assertEqual(result.stderr, self.magic) result = utils.RunCmd("/bin/sh -c '%s' 1>&2" % cmd, output=self.fname) self.assertEqual(result.output, "") self.assertFileContent(self.fname, self.magic) def testCombined(self): """Test combined output""" cmd = 'echo -n "A%s"; echo -n "B%s" 1>&2' % (self.magic, self.magic) expected = "A" + self.magic + "B" + self.magic result = utils.RunCmd("/bin/sh -c '%s'" % cmd) self.assertEqual(result.output, expected) result = utils.RunCmd("/bin/sh -c '%s'" % cmd, output=self.fname) self.assertEqual(result.output, "") self.assertFileContent(self.fname, expected) def testSignal(self): """Test signal""" result = utils.RunCmd(["python3", "-c", "import os; os.kill(os.getpid(), 15)"]) self.assertEqual(result.signal, 15) self.assertEqual(result.output, "") def testTimeoutFlagTrue(self): result = utils.RunCmd(["sleep", "2"], timeout=0.1) self.assertTrue(result.failed) self.assertTrue(result.failed_by_timeout) def testTimeoutFlagFalse(self): result = utils.RunCmd(["false"], timeout=5) self.assertTrue(result.failed) self.assertFalse(result.failed_by_timeout) def testTimeoutClean(self): cmd = ("trap 'exit 0' TERM; echo >&%d; read < %s" % (self.proc_ready_helper.write_fd, self.fifo_file)) result = utils.RunCmd(["/bin/sh", "-c", cmd], timeout=0.2, noclose_fds=[self.proc_ready_helper.write_fd], postfork_fn=self.proc_ready_helper.Ready) self.assertEqual(result.exit_code, 0) def testTimeoutKill(self): cmd = ["/bin/sh", "-c", "trap '' TERM; echo >&%d; read < %s" % (self.proc_ready_helper.write_fd, self.fifo_file)] timeout = 0.2 (out, err, status, ta) = \ utils.process._RunCmdPipe(cmd, {}, False, "/", False, timeout, [self.proc_ready_helper.write_fd], None, _linger_timeout=0.2, postfork_fn=self.proc_ready_helper.Ready) self.assertTrue(status < 0) self.assertEqual(-status, signal.SIGKILL) def testTimeoutOutputAfterTerm(self): cmd = ("trap 'echo sigtermed; exit 1' TERM; echo >&%d; read < %s" % (self.proc_ready_helper.write_fd, self.fifo_file)) result = utils.RunCmd(["/bin/sh", "-c", cmd], timeout=0.2, noclose_fds=[self.proc_ready_helper.write_fd], postfork_fn=self.proc_ready_helper.Ready) self.assertTrue(result.failed) self.assertEqual(result.stdout, "sigtermed\n") def testListRun(self): """Test list runs""" result = utils.RunCmd(["true"]) self.assertEqual(result.signal, None) self.assertEqual(result.exit_code, 0) result = utils.RunCmd(["/bin/sh", "-c", "exit 1"]) self.assertEqual(result.signal, None) self.assertEqual(result.exit_code, 1) result = utils.RunCmd(["echo", "-n", self.magic]) self.assertEqual(result.signal, None) self.assertEqual(result.exit_code, 0) self.assertEqual(result.stdout, self.magic) def testFileEmptyOutput(self): """Test file output""" result = utils.RunCmd(["true"], output=self.fname) self.assertEqual(result.signal, None) self.assertEqual(result.exit_code, 0) self.assertFileContent(self.fname, "") def testLang(self): """Test locale environment""" old_env = os.environ.copy() try: os.environ["LANG"] = "en_US.UTF-8" os.environ["LC_ALL"] = "en_US.UTF-8" result = utils.RunCmd(["locale"]) for line in result.output.splitlines(): key, value = line.split("=", 1) # Ignore these variables, they're overridden by LC_ALL if key == "LANG" or key == "LANGUAGE": continue self.assertFalse(value and value != "C" and value != '"C"', "Variable %s is set to the invalid value '%s'" % (key, value)) finally: os.environ = old_env def testDefaultCwd(self): """Test default working directory""" self.assertEqual(utils.RunCmd(["pwd"]).stdout.strip(), "/") def testCwd(self): """Test default working directory""" self.assertEqual(utils.RunCmd(["pwd"], cwd="/").stdout.strip(), "/") self.assertEqual(utils.RunCmd(["pwd"], cwd="/tmp").stdout.strip(), "/tmp") cwd = os.getcwd() self.assertEqual(utils.RunCmd(["pwd"], cwd=cwd).stdout.strip(), cwd) def testResetEnv(self): """Test environment reset functionality""" self.assertEqual(utils.RunCmd(["env"], reset_env=True).stdout.strip(), "") self.assertEqual(utils.RunCmd(["env"], reset_env=True, env={"FOO": "bar",}).stdout.strip(), "FOO=bar") def testNoFork(self): """Test that nofork raise an error""" self.assertFalse(utils.process._no_fork) utils.DisableFork() try: self.assertTrue(utils.process._no_fork) self.assertRaises(errors.ProgrammerError, utils.RunCmd, ["true"]) finally: utils.process._no_fork = False self.assertFalse(utils.process._no_fork) def testWrongParams(self): """Test wrong parameters""" self.assertRaises(errors.ProgrammerError, utils.RunCmd, ["true"], output="/dev/null", interactive=True) def testNocloseFds(self): """Test selective fd retention (noclose_fds)""" temp = open(self.fname, "r+") try: temp.write("test") temp.seek(0) cmd = "read -u %d; echo $REPLY" % temp.fileno() result = utils.RunCmd(["/bin/bash", "-c", cmd]) self.assertEqual(result.stdout.strip(), "") temp.seek(0) result = utils.RunCmd(["/bin/bash", "-c", cmd], noclose_fds=[temp.fileno()]) self.assertEqual(result.stdout.strip(), "test") finally: temp.close() def testNoInputRead(self): testfile = testutils.TestDataFilename("cert1.pem") result = utils.RunCmd(["cat"], timeout=10.0) self.assertFalse(result.failed) self.assertEqual(result.stderr, "") self.assertEqual(result.stdout, "") def testInputFileHandle(self): testfile = testutils.TestDataFilename("cert1.pem") with open(testfile, "r") as input_file: result = utils.RunCmd(["cat"], input_fd=input_file) self.assertFalse(result.failed) self.assertEqual(result.stdout, utils.ReadFile(testfile)) self.assertEqual(result.stderr, "") def testInputNumericFileDescriptor(self): testfile = testutils.TestDataFilename("cert2.pem") fh = open(testfile, "r") try: result = utils.RunCmd(["cat"], input_fd=fh.fileno()) finally: fh.close() self.assertFalse(result.failed) self.assertEqual(result.stdout, utils.ReadFile(testfile)) self.assertEqual(result.stderr, "") def testInputWithCloseFds(self): testfile = testutils.TestDataFilename("cert1.pem") temp = open(self.fname, "r+") try: temp.write("test283523367") temp.seek(0) with open(testfile, "r") as input_file: result = utils.RunCmd(["/bin/bash", "-c", ("cat && read -u %s; echo $REPLY" % temp.fileno())], input_fd=input_file, noclose_fds=[temp.fileno()]) self.assertFalse(result.failed) self.assertEqual(result.stdout.strip(), utils.ReadFile(testfile) + "test283523367") self.assertEqual(result.stderr, "") finally: temp.close() def testOutputAndInteractive(self): self.assertRaises(errors.ProgrammerError, utils.RunCmd, [], output=self.fname, interactive=True) def testOutputAndInput(self): with open(self.fname) as input_file: self.assertRaises(errors.ProgrammerError, utils.RunCmd, [], output=self.fname, input_fd=input_file) class TestRunParts(testutils.GanetiTestCase): """Testing case for the RunParts function""" def setUp(self): self.rundir = tempfile.mkdtemp(prefix="ganeti-test", suffix=".tmp") def tearDown(self): shutil.rmtree(self.rundir) def testEmpty(self): """Test on an empty dir""" self.assertEqual(utils.RunParts(self.rundir, reset_env=True), []) def testSkipWrongName(self): """Test that wrong files are skipped""" fname = os.path.join(self.rundir, "00test.dot") utils.WriteFile(fname, data="") os.chmod(fname, stat.S_IREAD | stat.S_IEXEC) relname = os.path.basename(fname) self.assertEqual(utils.RunParts(self.rundir, reset_env=True), [(relname, constants.RUNPARTS_SKIP, None)]) def testSkipNonExec(self): """Test that non executable files are skipped""" fname = os.path.join(self.rundir, "00test") utils.WriteFile(fname, data="") relname = os.path.basename(fname) self.assertEqual(utils.RunParts(self.rundir, reset_env=True), [(relname, constants.RUNPARTS_SKIP, None)]) def testError(self): """Test error on a broken executable""" fname = os.path.join(self.rundir, "00test") utils.WriteFile(fname, data="") os.chmod(fname, stat.S_IREAD | stat.S_IEXEC) (relname, status, error) = utils.RunParts(self.rundir, reset_env=True)[0] self.assertEqual(relname, os.path.basename(fname)) self.assertEqual(status, constants.RUNPARTS_ERR) self.assertTrue(error) def testSorted(self): """Test executions are sorted""" files = [] files.append(os.path.join(self.rundir, "64test")) files.append(os.path.join(self.rundir, "00test")) files.append(os.path.join(self.rundir, "42test")) for fname in files: utils.WriteFile(fname, data="") results = utils.RunParts(self.rundir, reset_env=True) for fname in sorted(files): self.assertEqual(os.path.basename(fname), results.pop(0)[0]) def testOk(self): """Test correct execution""" fname = os.path.join(self.rundir, "00test") utils.WriteFile(fname, data="#!/bin/sh\n\necho -n ciao") os.chmod(fname, stat.S_IREAD | stat.S_IEXEC) (relname, status, runresult) = \ utils.RunParts(self.rundir, reset_env=True)[0] self.assertEqual(relname, os.path.basename(fname)) self.assertEqual(status, constants.RUNPARTS_RUN) self.assertEqual(runresult.stdout, "ciao") def testRunFail(self): """Test correct execution, with run failure""" fname = os.path.join(self.rundir, "00test") utils.WriteFile(fname, data="#!/bin/sh\n\nexit 1") os.chmod(fname, stat.S_IREAD | stat.S_IEXEC) (relname, status, runresult) = \ utils.RunParts(self.rundir, reset_env=True)[0] self.assertEqual(relname, os.path.basename(fname)) self.assertEqual(status, constants.RUNPARTS_RUN) self.assertEqual(runresult.exit_code, 1) self.assertTrue(runresult.failed) def testRunMix(self): files = [] files.append(os.path.join(self.rundir, "00test")) files.append(os.path.join(self.rundir, "42test")) files.append(os.path.join(self.rundir, "64test")) files.append(os.path.join(self.rundir, "99test")) files.sort() # 1st has errors in execution utils.WriteFile(files[0], data="#!/bin/sh\n\nexit 1") os.chmod(files[0], stat.S_IREAD | stat.S_IEXEC) # 2nd is skipped utils.WriteFile(files[1], data="") # 3rd cannot execute properly utils.WriteFile(files[2], data="") os.chmod(files[2], stat.S_IREAD | stat.S_IEXEC) # 4th execs utils.WriteFile(files[3], data="#!/bin/sh\n\necho -n ciao") os.chmod(files[3], stat.S_IREAD | stat.S_IEXEC) results = utils.RunParts(self.rundir, reset_env=True) (relname, status, runresult) = results[0] self.assertEqual(relname, os.path.basename(files[0])) self.assertEqual(status, constants.RUNPARTS_RUN) self.assertEqual(runresult.exit_code, 1) self.assertTrue(runresult.failed) (relname, status, runresult) = results[1] self.assertEqual(relname, os.path.basename(files[1])) self.assertEqual(status, constants.RUNPARTS_SKIP) self.assertEqual(runresult, None) (relname, status, runresult) = results[2] self.assertEqual(relname, os.path.basename(files[2])) self.assertEqual(status, constants.RUNPARTS_ERR) self.assertTrue(runresult) (relname, status, runresult) = results[3] self.assertEqual(relname, os.path.basename(files[3])) self.assertEqual(status, constants.RUNPARTS_RUN) self.assertEqual(runresult.output, "ciao") self.assertEqual(runresult.exit_code, 0) self.assertTrue(not runresult.failed) def testMissingDirectory(self): nosuchdir = utils.PathJoin(self.rundir, "no/such/directory") self.assertEqual(utils.RunParts(nosuchdir), []) class TestStartDaemon(testutils.GanetiTestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix="ganeti-test") self.tmpfile = os.path.join(self.tmpdir, "test") def tearDown(self): shutil.rmtree(self.tmpdir) def testShell(self): utils.StartDaemon("echo Hello World > %s" % self.tmpfile) self._wait(self.tmpfile, 60.0, "Hello World") def testShellOutput(self): utils.StartDaemon("echo Hello World", output=self.tmpfile) self._wait(self.tmpfile, 60.0, "Hello World") def testNoShellNoOutput(self): utils.StartDaemon(["pwd"]) def testNoShellNoOutputTouch(self): testfile = os.path.join(self.tmpdir, "check") self.assertFalse(os.path.exists(testfile)) utils.StartDaemon(["touch", testfile]) self._wait(testfile, 60.0, "") def testNoShellOutput(self): utils.StartDaemon(["pwd"], output=self.tmpfile) self._wait(self.tmpfile, 60.0, "/") def testNoShellOutputCwd(self): utils.StartDaemon(["pwd"], output=self.tmpfile, cwd=os.getcwd()) self._wait(self.tmpfile, 60.0, os.getcwd()) def testShellEnv(self): utils.StartDaemon("echo \"$GNT_TEST_VAR\"", output=self.tmpfile, env={ "GNT_TEST_VAR": "Hello World", }) self._wait(self.tmpfile, 60.0, "Hello World") def testNoShellEnv(self): utils.StartDaemon(["printenv", "GNT_TEST_VAR"], output=self.tmpfile, env={ "GNT_TEST_VAR": "Hello World", }) self._wait(self.tmpfile, 60.0, "Hello World") def testOutputFd(self): fd = os.open(self.tmpfile, os.O_WRONLY | os.O_CREAT) try: utils.StartDaemon(["pwd"], output_fd=fd, cwd=os.getcwd()) finally: os.close(fd) self._wait(self.tmpfile, 60.0, os.getcwd()) def testPid(self): pid = utils.StartDaemon("echo $$ > %s" % self.tmpfile) self._wait(self.tmpfile, 60.0, str(pid)) def testPidFile(self): pidfile = os.path.join(self.tmpdir, "pid") checkfile = os.path.join(self.tmpdir, "abort") pid = utils.StartDaemon("while sleep 5; do :; done", pidfile=pidfile, output=self.tmpfile) try: fd = os.open(pidfile, os.O_RDONLY) try: # Check file is locked self.assertRaises(errors.LockError, utils.LockFile, fd) pidtext = os.read(fd, 100) finally: os.close(fd) self.assertEqual(int(pidtext.strip()), pid) self.assertTrue(utils.IsProcessAlive(pid)) finally: # No matter what happens, kill daemon utils.KillProcess(pid, timeout=5.0, waitpid=False) self.assertFalse(utils.IsProcessAlive(pid)) self.assertEqual(utils.ReadFile(self.tmpfile), "") def _wait(self, path, timeout, expected): # Due to the asynchronous nature of daemon processes, polling is necessary. # A timeout makes sure the test doesn't hang forever. def _CheckFile(): if not (os.path.isfile(path) and utils.ReadFile(path).strip() == expected): raise utils.RetryAgain() try: utils.Retry(_CheckFile, (0.01, 1.5, 1.0), timeout) except utils.RetryTimeout: self.fail("Apparently the daemon didn't run in %s seconds and/or" " didn't write the correct output" % timeout) def testError(self): self.assertRaises(errors.OpExecError, utils.StartDaemon, ["./does-NOT-EXIST/here/0123456789"]) self.assertRaises(errors.OpExecError, utils.StartDaemon, ["./does-NOT-EXIST/here/0123456789"], output=os.path.join(self.tmpdir, "DIR/NOT/EXIST")) self.assertRaises(errors.OpExecError, utils.StartDaemon, ["./does-NOT-EXIST/here/0123456789"], cwd=os.path.join(self.tmpdir, "DIR/NOT/EXIST")) self.assertRaises(errors.OpExecError, utils.StartDaemon, ["./does-NOT-EXIST/here/0123456789"], output=os.path.join(self.tmpdir, "DIR/NOT/EXIST")) fd = os.open(self.tmpfile, os.O_WRONLY | os.O_CREAT) try: self.assertRaises(errors.ProgrammerError, utils.StartDaemon, ["./does-NOT-EXIST/here/0123456789"], output=self.tmpfile, output_fd=fd) finally: os.close(fd) class RunInSeparateProcess(unittest.TestCase): def test(self): for exp in [True, False]: def _child(): return exp self.assertEqual(exp, utils.RunInSeparateProcess(_child)) def testArgs(self): for arg in [0, 1, 999, "Hello World", (1, 2, 3)]: def _child(carg1, carg2): return carg1 == "Foo" and carg2 == arg self.assertTrue(utils.RunInSeparateProcess(_child, "Foo", arg)) def testPid(self): parent_pid = os.getpid() def _check(): return os.getpid() == parent_pid self.assertFalse(utils.RunInSeparateProcess(_check)) def testSignal(self): def _kill(): os.kill(os.getpid(), signal.SIGTERM) self.assertRaises(errors.GenericError, utils.RunInSeparateProcess, _kill) def testException(self): def _exc(): raise errors.GenericError("This is a test") self.assertRaises(errors.GenericError, utils.RunInSeparateProcess, _exc) class GetCmdline(unittest.TestCase): def test(self): sample_cmd = "sleep 20; true" child = subprocess.Popen(sample_cmd, shell=True) pid = child.pid # this is somewhat silly, but apparently sometimes the cmdline has not # yet been set and the call to GetProcCmdline() will return an emptry # string. This problem has surfaced recently and especially on Docker # containers used for testing via Github Actions (could not reproduce # locally) time.sleep(1) cmdline = utils.GetProcCmdline(pid) # As the popen will quote and pass on the sample_cmd, it should be returned # by the function as an element in the list of arguments self.assertTrue(sample_cmd in cmdline) child.kill() child.wait() if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.retry_unittest.py000075500000000000000000000172641476477700300245570ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.retry""" import unittest from ganeti import constants from ganeti import errors from ganeti import utils import testutils class TestRetry(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.retries = 0 self.called = 0 self.time = 1379601882.0 self.time_for_time_fn = 0 self.time_for_retry_and_succeed = 0 def _time_fn(self): self.time += self.time_for_time_fn return self.time def _wait_fn(self, delay): self.time += delay @staticmethod def _RaiseRetryAgain(): raise utils.RetryAgain() @staticmethod def _RaiseRetryAgainWithArg(args): raise utils.RetryAgain(*args) def _WrongNestedLoop(self): return utils.Retry(self._RaiseRetryAgain, 0.01, 0.02) def _RetryAndSucceed(self, retries): self.time += self.time_for_retry_and_succeed if self.retries < retries: self.retries += 1 raise utils.RetryAgain() else: return True def _SimpleRetryAndSucceed(self, retries): self.called += 1 if self.retries < retries: self.retries += 1 return False else: return True def testRaiseTimeout(self): self.assertRaises(utils.RetryTimeout, utils.Retry, self._RaiseRetryAgain, 0.01, 0.02, wait_fn = self._wait_fn, _time_fn = self._time_fn) self.assertRaises(utils.RetryTimeout, utils.Retry, self._RetryAndSucceed, 0.01, 0, args=[1], wait_fn = self._wait_fn, _time_fn = self._time_fn) self.assertEqual(self.retries, 1) def testComplete(self): self.assertEqual(utils.Retry(lambda: True, 0, 1, wait_fn = self._wait_fn, _time_fn = self._time_fn), True) self.assertEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2], wait_fn = self._wait_fn, _time_fn = self._time_fn), True) self.assertEqual(self.retries, 2) def testCompleteNontrivialTimes(self): self.time_for_time_fn = 0.01 self.time_for_retry_and_succeed = 0.1 self.assertEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2], wait_fn = self._wait_fn, _time_fn = self._time_fn), True) self.assertEqual(self.retries, 2) def testNestedLoop(self): try: self.assertRaises(errors.ProgrammerError, utils.Retry, self._WrongNestedLoop, 0, 1, wait_fn = self._wait_fn, _time_fn = self._time_fn) except utils.RetryTimeout: self.fail("Didn't detect inner loop's exception") def testTimeoutArgument(self): retry_arg="my_important_debugging_message" try: utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]], wait_fn = self._wait_fn, _time_fn = self._time_fn) except utils.RetryTimeout as err: self.assertEqual(err.args, (retry_arg, )) else: self.fail("Expected timeout didn't happen") def testTimeout(self): self.time_for_time_fn = 0.01 self.time_for_retry_and_succeed = 10 try: utils.Retry(self._RetryAndSucceed, 1, 18, args=[2], wait_fn = self._wait_fn, _time_fn = self._time_fn) except utils.RetryTimeout as err: self.assertEqual(err.args, ()) else: self.fail("Expected timeout didn't happen") def testNoTimeout(self): self.time_for_time_fn = 0.01 self.time_for_retry_and_succeed = 8 self.assertEqual( utils.Retry(self._RetryAndSucceed, 1, 18, args=[2], wait_fn = self._wait_fn, _time_fn = self._time_fn), True) def testRaiseInnerWithExc(self): retry_arg="my_important_debugging_message" try: try: utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[errors.GenericError(retry_arg, retry_arg)]], wait_fn = self._wait_fn, _time_fn = self._time_fn) except utils.RetryTimeout as err: err.RaiseInner() else: self.fail("Expected timeout didn't happen") except errors.GenericError as err: self.assertEqual(err.args, (retry_arg, retry_arg)) else: self.fail("Expected GenericError didn't happen") def testRaiseInnerWithMsg(self): retry_arg="my_important_debugging_message" try: try: utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg, retry_arg]], wait_fn = self._wait_fn, _time_fn = self._time_fn) except utils.RetryTimeout as err: err.RaiseInner() else: self.fail("Expected timeout didn't happen") except utils.RetryTimeout as err: self.assertEqual(err.args, (retry_arg, retry_arg)) else: self.fail("Expected RetryTimeout didn't happen") def testSimpleRetry(self): self.assertFalse(utils.SimpleRetry(True, lambda: False, 0.01, 0.02, wait_fn = self._wait_fn, _time_fn = self._time_fn)) self.assertFalse(utils.SimpleRetry(lambda x: x, lambda: False, 0.01, 0.02, wait_fn = self._wait_fn, _time_fn = self._time_fn)) self.assertTrue(utils.SimpleRetry(True, lambda: True, 0, 1, wait_fn = self._wait_fn, _time_fn = self._time_fn)) self.assertTrue(utils.SimpleRetry(lambda x: x, lambda: True, 0, 1, wait_fn = self._wait_fn, _time_fn = self._time_fn)) self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed, 0, 1, args=[1], wait_fn = self._wait_fn, _time_fn = self._time_fn)) self.assertEqual(self.retries, 1) self.assertEqual(self.called, 2) self.called = self.retries = 0 self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed, 0, 1, args=[2], wait_fn = self._wait_fn, _time_fn = self._time_fn)) self.assertEqual(self.called, 3) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.security_unittest.py000075500000000000000000000062301476477700300252500ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the ganeti.utils.storage module""" import os import shutil import tempfile import unittest from unittest import mock from ganeti import constants from ganeti.utils import security import testutils class TestUuidConversion(unittest.TestCase): def testUuidConversion(self): uuid_as_int = security.UuidToInt("5cd037f4-9587-49c4-a23e-142f8b7e909d") self.assertEqual(uuid_as_int, int(uuid_as_int)) class TestGetCertificateDigest(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) # certificate file that contains the certificate only self._certfilename1 = testutils.TestDataFilename("cert1.pem") # (different) certificate file that contains both, certificate # and private key self._certfilename2 = testutils.TestDataFilename("cert2.pem") def testGetCertificateDigest(self): digest1 = security.GetCertificateDigest( cert_filename=self._certfilename1) digest2 = security.GetCertificateDigest( cert_filename=self._certfilename2) self.assertFalse(digest1 == digest2) class TestCertVerification(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testVerifyCertificate(self): security.VerifyCertificate(testutils.TestDataFilename("cert1.pem")) nonexist_filename = os.path.join(self.tmpdir, "does-not-exist") (errcode, msg) = security.VerifyCertificate(nonexist_filename) self.assertEqual(errcode, constants.CV_ERROR) # Try to load non-certificate file invalid_cert = testutils.TestDataFilename("bdev-net.txt") (errcode, msg) = security.VerifyCertificate(invalid_cert) self.assertEqual(errcode, constants.CV_ERROR) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.storage_unittest.py000075500000000000000000000162711476477700300250530ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the ganeti.utils.storage module""" import unittest from unittest import mock from ganeti import constants from ganeti.utils import storage import testutils class TestGetStorageUnitForDiskTemplate(unittest.TestCase): def setUp(self): self._default_vg_name = "some_vg_name" self._cluster = mock.Mock() self._cluster.file_storage_dir = "my/file/storage/dir" self._cluster.shared_file_storage_dir = "my/shared/file/storage/dir" self._cfg = mock.Mock() self._cfg.GetVGName = mock.Mock(return_value=self._default_vg_name) self._cfg.GetClusterInfo = mock.Mock(return_value=self._cluster) def testGetDefaultStorageUnitForDiskTemplateLvm(self): for disk_template in [constants.DT_DRBD8, constants.DT_PLAIN]: (storage_type, storage_key) = \ storage._GetDefaultStorageUnitForDiskTemplate(self._cfg, disk_template) self.assertEqual(storage_type, constants.ST_LVM_VG) self.assertEqual(storage_key, self._default_vg_name) def testGetDefaultStorageUnitForDiskTemplateFile(self): (storage_type, storage_key) = \ storage._GetDefaultStorageUnitForDiskTemplate(self._cfg, constants.DT_FILE) self.assertEqual(storage_type, constants.ST_FILE) self.assertEqual(storage_key, self._cluster.file_storage_dir) def testGetDefaultStorageUnitForDiskTemplateSharedFile(self): (storage_type, storage_key) = \ storage._GetDefaultStorageUnitForDiskTemplate(self._cfg, constants.DT_SHARED_FILE) self.assertEqual(storage_type, constants.ST_SHARED_FILE) self.assertEqual(storage_key, self._cluster.shared_file_storage_dir) def testGetDefaultStorageUnitForDiskTemplateGluster(self): (storage_type, storage_key) = \ storage._GetDefaultStorageUnitForDiskTemplate(self._cfg, constants.DT_GLUSTER) self.assertEqual(storage_type, constants.ST_GLUSTER) self.assertEqual(storage_key, self._cluster.gluster_storage_dir) def testGetDefaultStorageUnitForDiskTemplateDiskless(self): (storage_type, storage_key) = \ storage._GetDefaultStorageUnitForDiskTemplate(self._cfg, constants.DT_DISKLESS) self.assertEqual(storage_type, constants.ST_DISKLESS) self.assertEqual(storage_key, None) class TestGetStorageUnits(unittest.TestCase): def setUp(self): storage._GetDefaultStorageUnitForDiskTemplate = \ mock.Mock(return_value=("foo", "bar")) self._cfg = mock.Mock() def testGetStorageUnits(self): sts_non_reporting = \ storage.GetDiskTemplatesOfStorageTypes(constants.ST_GLUSTER, constants.ST_SHARED_FILE) disk_templates = constants.DTS_FILEBASED - frozenset(sts_non_reporting) storage_units = storage.GetStorageUnits(self._cfg, disk_templates) self.assertEqual(len(storage_units), len(disk_templates)) def testGetStorageUnitsLvm(self): disk_templates = [constants.DT_PLAIN, constants.DT_DRBD8] storage_units = storage.GetStorageUnits(self._cfg, disk_templates) self.assertEqual(len(storage_units), len(disk_templates)) class TestLookupSpaceInfoByStorageType(unittest.TestCase): def setUp(self): self._space_info = [ {"type": st, "name": st + "_key", "storage_size": 0, "storage_free": 0} for st in constants.STORAGE_TYPES] def testValidLookup(self): query_type = constants.ST_LVM_PV result = storage.LookupSpaceInfoByStorageType(self._space_info, query_type) self.assertEqual(query_type, result["type"]) def testNotInList(self): result = storage.LookupSpaceInfoByStorageType(self._space_info, "non_existing_type") self.assertEqual(None, result) class TestGetDiskLabels(unittest.TestCase): def setUp(self): pass def testNormalPrefix(self): labels = ["/dev/sda", "/dev/sdb", "/dev/sdc", "/dev/sdd", "/dev/sde", "/dev/sdf", "/dev/sdg", "/dev/sdh", "/dev/sdi", "/dev/sdj", "/dev/sdk", "/dev/sdl", "/dev/sdm", "/dev/sdn", "/dev/sdo", "/dev/sdp", "/dev/sdq", "/dev/sdr", "/dev/sds", "/dev/sdt", "/dev/sdu", "/dev/sdv", "/dev/sdw", "/dev/sdx", "/dev/sdy", "/dev/sdz", "/dev/sdaa", "/dev/sdab", "/dev/sdac", "/dev/sdad", "/dev/sdae", "/dev/sdaf", "/dev/sdag", "/dev/sdah", "/dev/sdai", "/dev/sdaj", "/dev/sdak", "/dev/sdal", "/dev/sdam", "/dev/sdan", "/dev/sdao", "/dev/sdap", "/dev/sdaq", "/dev/sdar", "/dev/sdas", "/dev/sdat", "/dev/sdau", "/dev/sdav", "/dev/sdaw", "/dev/sdax", "/dev/sday", "/dev/sdaz", "/dev/sdba", "/dev/sdbb", "/dev/sdbc", "/dev/sdbd", "/dev/sdbe", "/dev/sdbf", "/dev/sdbg", "/dev/sdbh"] result = list(storage.GetDiskLabels("/dev/sd", 60)) self.assertEqual(labels, result) def testEmptyPrefix(self): labels = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "aa", "ab", "ac", "ad", "ae", "af", "ag", "ah", "ai", "aj", "ak", "al", "am", "an", "ao", "ap", "aq", "ar", "as", "at", "au", "av", "aw", "ax", "ay", "az", "ba", "bb", "bc", "bd", "be", "bf", "bg", "bh"] result = list(storage.GetDiskLabels("", 60)) self.assertEqual(labels, result) def testWrapAt2To3(self): start1 = 27 start2 = 703 result1 = list(storage.GetDiskLabels("", start1)) result2 = [x[1:] for x in list(storage.GetDiskLabels("", start2, start=start2 - start1))] self.assertEqual(result1, result2) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.text_unittest.py000075500000000000000000000563241476477700300243760ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.text""" import re import string import time import unittest import os from io import StringIO from ganeti import constants from ganeti import utils from ganeti import errors import testutils class TestMatchNameComponent(unittest.TestCase): """Test case for the MatchNameComponent function""" def testEmptyList(self): """Test that there is no match against an empty list""" self.assertEqual(utils.MatchNameComponent("", []), None) self.assertEqual(utils.MatchNameComponent("test", []), None) def testSingleMatch(self): """Test that a single match is performed correctly""" mlist = ["test1.example.com", "test2.example.com", "test3.example.com"] for key in "test2", "test2.example", "test2.example.com": self.assertEqual(utils.MatchNameComponent(key, mlist), mlist[1]) def testMultipleMatches(self): """Test that a multiple match is returned as None""" mlist = ["test1.example.com", "test1.example.org", "test1.example.net"] for key in "test1", "test1.example": self.assertEqual(utils.MatchNameComponent(key, mlist), None) def testFullMatch(self): """Test that a full match is returned correctly""" key1 = "test1" key2 = "test1.example" mlist = [key2, key2 + ".com"] self.assertEqual(utils.MatchNameComponent(key1, mlist), None) self.assertEqual(utils.MatchNameComponent(key2, mlist), key2) def testCaseInsensitivePartialMatch(self): """Test for the case_insensitive keyword""" mlist = ["test1.example.com", "test2.example.net"] self.assertEqual(utils.MatchNameComponent("test2", mlist, case_sensitive=False), "test2.example.net") self.assertEqual(utils.MatchNameComponent("Test2", mlist, case_sensitive=False), "test2.example.net") self.assertEqual(utils.MatchNameComponent("teSt2", mlist, case_sensitive=False), "test2.example.net") self.assertEqual(utils.MatchNameComponent("TeSt2", mlist, case_sensitive=False), "test2.example.net") def testCaseInsensitiveFullMatch(self): mlist = ["ts1.ex", "ts1.ex.org", "ts2.ex", "Ts2.ex"] # Between the two ts1 a full string match non-case insensitive should work self.assertEqual(utils.MatchNameComponent("Ts1", mlist, case_sensitive=False), None) self.assertEqual(utils.MatchNameComponent("Ts1.ex", mlist, case_sensitive=False), "ts1.ex") self.assertEqual(utils.MatchNameComponent("ts1.ex", mlist, case_sensitive=False), "ts1.ex") # Between the two ts2 only case differs, so only case-match works self.assertEqual(utils.MatchNameComponent("ts2.ex", mlist, case_sensitive=False), "ts2.ex") self.assertEqual(utils.MatchNameComponent("Ts2.ex", mlist, case_sensitive=False), "Ts2.ex") self.assertEqual(utils.MatchNameComponent("TS2.ex", mlist, case_sensitive=False), None) class TestDnsNameGlobPattern(unittest.TestCase): def setUp(self): self.names = [ "node1.example.com", "node2-0.example.com", "node2-1.example.com", "node1.example.net", "web1.example.com", "web2.example.com", "sub.site.example.com", ] def _Test(self, pattern): re_pat = re.compile(utils.DnsNameGlobPattern(pattern)) return [n for n in self.names if re_pat.match(n)] def test(self): for pattern in ["xyz", "node", " ", "example.net", "x*.example.*", "x*.example.com"]: self.assertEqual(self._Test(pattern), []) for pattern in ["*", "???*"]: self.assertEqual(self._Test(pattern), self.names) self.assertEqual(self._Test("node1.*.net"), ["node1.example.net"]) self.assertEqual(self._Test("*.example.net"), ["node1.example.net"]) self.assertEqual(self._Test("web1.example.com"), ["web1.example.com"]) for pattern in ["*.*.*.*", "???", "*.site"]: self.assertEqual(self._Test(pattern), ["sub.site.example.com"]) self.assertEqual(self._Test("node1"), [ "node1.example.com", "node1.example.net", ]) self.assertEqual(self._Test("node?*.example.*"), [ "node1.example.com", "node2-0.example.com", "node2-1.example.com", "node1.example.net", ]) self.assertEqual(self._Test("*-?"), [ "node2-0.example.com", "node2-1.example.com", ]) self.assertEqual(self._Test("node2-?.example.com"), [ "node2-0.example.com", "node2-1.example.com", ]) class TestFormatUnit(unittest.TestCase): """Test case for the FormatUnit function""" def testMiB(self): self.assertEqual(utils.FormatUnit(1, "h"), "1M") self.assertEqual(utils.FormatUnit(100, "h"), "100M") self.assertEqual(utils.FormatUnit(1023, "h"), "1023M") self.assertEqual(utils.FormatUnit(1, "m"), "1") self.assertEqual(utils.FormatUnit(100, "m"), "100") self.assertEqual(utils.FormatUnit(1023, "m"), "1023") self.assertEqual(utils.FormatUnit(1024, "m"), "1024") self.assertEqual(utils.FormatUnit(1536, "m"), "1536") self.assertEqual(utils.FormatUnit(17133, "m"), "17133") self.assertEqual(utils.FormatUnit(1024 * 1024 - 1, "m"), "1048575") def testGiB(self): self.assertEqual(utils.FormatUnit(1024, "h"), "1.0G") self.assertEqual(utils.FormatUnit(1536, "h"), "1.5G") self.assertEqual(utils.FormatUnit(17133, "h"), "16.7G") self.assertEqual(utils.FormatUnit(1024 * 1024 - 1, "h"), "1024.0G") self.assertEqual(utils.FormatUnit(1024, "g"), "1.0") self.assertEqual(utils.FormatUnit(1536, "g"), "1.5") self.assertEqual(utils.FormatUnit(17133, "g"), "16.7") self.assertEqual(utils.FormatUnit(1024 * 1024 - 1, "g"), "1024.0") self.assertEqual(utils.FormatUnit(1024 * 1024, "g"), "1024.0") self.assertEqual(utils.FormatUnit(5120 * 1024, "g"), "5120.0") self.assertEqual(utils.FormatUnit(29829 * 1024, "g"), "29829.0") def testTiB(self): self.assertEqual(utils.FormatUnit(1024 * 1024, "h"), "1.0T") self.assertEqual(utils.FormatUnit(5120 * 1024, "h"), "5.0T") self.assertEqual(utils.FormatUnit(29829 * 1024, "h"), "29.1T") self.assertEqual(utils.FormatUnit(1024 * 1024, "t"), "1.0") self.assertEqual(utils.FormatUnit(5120 * 1024, "t"), "5.0") self.assertEqual(utils.FormatUnit(29829 * 1024, "t"), "29.1") def testErrors(self): self.assertRaises(errors.ProgrammerError, utils.FormatUnit, 1, "a") class TestParseUnit(unittest.TestCase): """Test case for the ParseUnit function""" SCALES = (("", 1), ("M", 1), ("G", 1024), ("T", 1024 * 1024), ("MB", 1), ("GB", 1024), ("TB", 1024 * 1024), ("MiB", 1), ("GiB", 1024), ("TiB", 1024 * 1024)) def testRounding(self): self.assertEqual(utils.ParseUnit("0"), 0) self.assertEqual(utils.ParseUnit("1"), 4) self.assertEqual(utils.ParseUnit("2"), 4) self.assertEqual(utils.ParseUnit("3"), 4) self.assertEqual(utils.ParseUnit("124"), 124) self.assertEqual(utils.ParseUnit("125"), 128) self.assertEqual(utils.ParseUnit("126"), 128) self.assertEqual(utils.ParseUnit("127"), 128) self.assertEqual(utils.ParseUnit("128"), 128) self.assertEqual(utils.ParseUnit("129"), 132) self.assertEqual(utils.ParseUnit("130"), 132) def testFloating(self): self.assertEqual(utils.ParseUnit("0"), 0) self.assertEqual(utils.ParseUnit("0.5"), 4) self.assertEqual(utils.ParseUnit("1.75"), 4) self.assertEqual(utils.ParseUnit("1.99"), 4) self.assertEqual(utils.ParseUnit("2.00"), 4) self.assertEqual(utils.ParseUnit("2.01"), 4) self.assertEqual(utils.ParseUnit("3.99"), 4) self.assertEqual(utils.ParseUnit("4.00"), 4) self.assertEqual(utils.ParseUnit("4.01"), 8) self.assertEqual(utils.ParseUnit("1.5G"), 1536) self.assertEqual(utils.ParseUnit("1.8G"), 1844) self.assertEqual(utils.ParseUnit("8.28T"), 8682212) def testSuffixes(self): for sep in ("", " ", " ", "\t", "\t "): for suffix, scale in self.SCALES: for func in (lambda x: x, str.lower, str.upper): self.assertEqual(utils.ParseUnit("1024" + sep + func(suffix)), 1024 * scale) def testInvalidInput(self): for sep in ("-", "_", ",", "a"): for suffix, _ in self.SCALES: self.assertRaises(errors.UnitParseError, utils.ParseUnit, "1" + sep + suffix) for suffix, _ in self.SCALES: self.assertRaises(errors.UnitParseError, utils.ParseUnit, "1,3" + suffix) class TestShellQuoting(unittest.TestCase): """Test case for shell quoting functions""" def testShellQuote(self): self.assertEqual(utils.ShellQuote("abc"), "abc") self.assertEqual(utils.ShellQuote('ab"c'), "'ab\"c'") self.assertEqual(utils.ShellQuote("a'bc"), "'a'\\''bc'") self.assertEqual(utils.ShellQuote("a b c"), "'a b c'") self.assertEqual(utils.ShellQuote("a b\\ c"), "'a b\\ c'") def testShellQuoteArgs(self): self.assertEqual(utils.ShellQuoteArgs(["a", "b", "c"]), "a b c") self.assertEqual(utils.ShellQuoteArgs(['a', 'b"', 'c']), "a 'b\"' c") self.assertEqual(utils.ShellQuoteArgs(['a', 'b\'', 'c']), "a 'b'\\\''' c") class TestShellWriter(unittest.TestCase): def test(self): buf = StringIO() sw = utils.ShellWriter(buf) sw.Write("#!/bin/bash") sw.Write("if true; then") sw.IncIndent() try: sw.Write("echo true") sw.Write("for i in 1 2 3") sw.Write("do") sw.IncIndent() try: self.assertEqual(sw._indent, 2) sw.Write("date") finally: sw.DecIndent() sw.Write("done") finally: sw.DecIndent() sw.Write("echo %s", utils.ShellQuote("Hello World")) sw.Write("exit 0") self.assertEqual(sw._indent, 0) output = buf.getvalue() self.assertTrue(output.endswith("\n")) lines = output.splitlines() self.assertEqual(len(lines), 9) self.assertEqual(lines[0], "#!/bin/bash") self.assertTrue(re.match(r"^\s+date$", lines[5])) self.assertEqual(lines[7], "echo 'Hello World'") def testEmpty(self): buf = StringIO() sw = utils.ShellWriter(buf) sw = None self.assertEqual(buf.getvalue(), "") def testEmptyNoIndent(self): buf = StringIO() sw = utils.ShellWriter(buf, indent=False) sw = None self.assertEqual(buf.getvalue(), "") @classmethod def _AddLevel(cls, sw, level): if level == 6: return sw.IncIndent() try: # Add empty line, it should not be indented sw.Write("") sw.Write(str(level)) cls._AddLevel(sw, level + 1) finally: sw.DecIndent() def testEmptyLines(self): buf = StringIO() sw = utils.ShellWriter(buf) self._AddLevel(sw, 1) self.assertEqual(buf.getvalue(), "".join("\n%s%s\n" % (i * " ", i) for i in range(1, 6))) def testEmptyLinesNoIndent(self): buf = StringIO() sw = utils.ShellWriter(buf, indent=False) self._AddLevel(sw, 1) self.assertEqual(buf.getvalue(), "".join("\n%s\n" % i for i in range(1, 6))) class TestNormalizeAndValidateMac(unittest.TestCase): def testInvalid(self): for i in ["xxx", "00:11:22:33:44:55:66", "zz:zz:zz:zz:zz:zz"]: self.assertRaises(errors.OpPrereqError, utils.NormalizeAndValidateMac, i) def testNormalization(self): for mac in ["aa:bb:cc:dd:ee:ff", "00:AA:11:bB:22:cc"]: self.assertEqual(utils.NormalizeAndValidateMac(mac), mac.lower()) class TestNormalizeAndValidateThreeOctetMacPrefix(unittest.TestCase): def testInvalid(self): for i in ["xxx", "00:11:22:33:44:55:66", "zz:zz:zz:zz:zz:zz", "aa:bb:cc:dd:ee:ff", "00:AA:11:bB:22:cc", "00:11:"]: self.assertRaises(errors.OpPrereqError, utils.NormalizeAndValidateThreeOctetMacPrefix, i) def testNormalization(self): for mac in ["aa:bb:cc", "00:AA:11"]: self.assertEqual(utils.NormalizeAndValidateThreeOctetMacPrefix(mac), mac.lower()) class TestSafeEncode(unittest.TestCase): """Test case for SafeEncode""" def testAscii(self): for txt in [string.digits, string.ascii_letters, string.punctuation]: self.assertEqual(txt, utils.SafeEncode(txt)) def testDoubleEncode(self): for i in range(255): txt = utils.SafeEncode(chr(i)) self.assertEqual(txt, utils.SafeEncode(txt)) def testUnicode(self): # 1024 is high enough to catch non-direct ASCII mappings for i in range(1024): txt = utils.SafeEncode(chr(i)) self.assertEqual(txt, utils.SafeEncode(txt)) class TestUnescapeAndSplit(unittest.TestCase): """Testing case for UnescapeAndSplit""" def setUp(self): # testing more that one separator for regexp safety self._seps = [",", "+", ".", ":"] def testSimple(self): a = ["a", "b", "c", "d"] for sep in self._seps: self.assertEqual(utils.UnescapeAndSplit(sep.join(a), sep=sep), a) def testEscape(self): for sep in self._seps: a = ["a", "b\\" + sep + "c", "d"] b = ["a", "b" + sep + "c", "d"] self.assertEqual(utils.UnescapeAndSplit(sep.join(a), sep=sep), b) def testDoubleEscape(self): for sep in self._seps: a = ["a", "b\\\\", "c", "d"] b = ["a", "b\\", "c", "d"] self.assertEqual(utils.UnescapeAndSplit(sep.join(a), sep=sep), b) def testThreeEscape(self): for sep in self._seps: a = ["a", "b\\\\\\" + sep + "c", "d"] b = ["a", "b\\" + sep + "c", "d"] self.assertEqual(utils.UnescapeAndSplit(sep.join(a), sep=sep), b) def testEscapeAtEnd(self): for sep in self._seps: self.assertEqual(utils.UnescapeAndSplit("\\", sep=sep), ["\\"]) a = ["a", "b\\", "c"] b = ["a", "b" + sep + "c\\"] self.assertEqual(utils.UnescapeAndSplit("%s\\" % sep.join(a), sep=sep), b) a = ["\\" + sep, "\\" + sep, "c", "d\\.moo"] b = [sep, sep, "c", "d.moo\\"] self.assertEqual(utils.UnescapeAndSplit("%s\\" % sep.join(a), sep=sep), b) def testMultipleEscapes(self): for sep in self._seps: a = ["a", "b\\" + sep + "c", "d\\" + sep + "e\\" + sep + "f", "g"] b = ["a", "b" + sep + "c", "d" + sep + "e" + sep + "f", "g"] self.assertEqual(utils.UnescapeAndSplit(sep.join(a), sep=sep), b) class TestEscapeAndJoin(unittest.TestCase): def verifyParsesCorrect(self, args): for sep in [",", "+", ".", ":"]: self.assertEqual(utils.UnescapeAndSplit( utils.EscapeAndJoin(args, sep=sep), sep=sep), args) def test(self): self.verifyParsesCorrect(["a", "b", "c"]) self.verifyParsesCorrect(["2.10.0", "12345"]) self.verifyParsesCorrect(["2.10.0~alpha1", "12345"]) self.verifyParsesCorrect(["..:", ",,+"]) self.verifyParsesCorrect(["a\\", "b\\\\", "c"]) self.verifyParsesCorrect(["a"]) self.verifyParsesCorrect(["+"]) self.verifyParsesCorrect(["\\"]) self.verifyParsesCorrect(["\\\\"]) class TestCommaJoin(unittest.TestCase): def test(self): self.assertEqual(utils.CommaJoin([]), "") self.assertEqual(utils.CommaJoin([1, 2, 3]), "1, 2, 3") self.assertEqual(utils.CommaJoin(["Hello"]), "Hello") self.assertEqual(utils.CommaJoin(["Hello", "World"]), "Hello, World") self.assertEqual(utils.CommaJoin(["Hello", "World", 99]), "Hello, World, 99") class TestFormatTime(unittest.TestCase): """Testing case for FormatTime""" @staticmethod def _TestInProcess(tz, timestamp, usecs, expected): os.environ["TZ"] = tz time.tzset() return utils.FormatTime(timestamp, usecs=usecs) == expected def _Test(self, *args): # Need to use separate process as we want to change TZ self.assertTrue(utils.RunInSeparateProcess(self._TestInProcess, *args)) def test(self): self._Test("UTC", 0, None, "1970-01-01 00:00:00") self._Test("America/Sao_Paulo", 1292606926, None, "2010-12-17 15:28:46") self._Test("Europe/London", 1292606926, None, "2010-12-17 17:28:46") self._Test("Europe/Zurich", 1292606926, None, "2010-12-17 18:28:46") self._Test("Europe/Zurich", 1332944288, 8787, "2012-03-28 16:18:08.008787") self._Test("Australia/Sydney", 1292606926, None, "2010-12-18 04:28:46") self._Test("Australia/Sydney", 1292606926, None, "2010-12-18 04:28:46") self._Test("Australia/Sydney", 1292606926, 999999, "2010-12-18 04:28:46.999999") def testNone(self): self.assertEqual(utils.FormatTime(None), "N/A") def testInvalid(self): self.assertEqual(utils.FormatTime(()), "N/A") def testNow(self): # tests that we accept time.time input utils.FormatTime(time.time()) # tests that we accept int input utils.FormatTime(int(time.time())) class TestFormatSeconds(unittest.TestCase): def test(self): self.assertEqual(utils.FormatSeconds(1), "1s") self.assertEqual(utils.FormatSeconds(3600), "1h 0m 0s") self.assertEqual(utils.FormatSeconds(3599), "59m 59s") self.assertEqual(utils.FormatSeconds(7200), "2h 0m 0s") self.assertEqual(utils.FormatSeconds(7201), "2h 0m 1s") self.assertEqual(utils.FormatSeconds(7281), "2h 1m 21s") self.assertEqual(utils.FormatSeconds(29119), "8h 5m 19s") self.assertEqual(utils.FormatSeconds(19431228), "224d 21h 33m 48s") self.assertEqual(utils.FormatSeconds(-1), "-1s") self.assertEqual(utils.FormatSeconds(-282), "-282s") self.assertEqual(utils.FormatSeconds(-29119), "-29119s") def testFloat(self): self.assertEqual(utils.FormatSeconds(1.3), "1s") self.assertEqual(utils.FormatSeconds(1.9), "2s") self.assertEqual(utils.FormatSeconds(3912.12311), "1h 5m 12s") self.assertEqual(utils.FormatSeconds(3912.8), "1h 5m 13s") class TestLineSplitter(unittest.TestCase): def test(self): lines = [] ls = utils.LineSplitter(lines.append) ls.write(b"Hello World\n") self.assertEqual(lines, []) ls.write(b"Foo\n Bar\r\n ") ls.write(b"Baz") ls.write(b"Moo") self.assertEqual(lines, []) ls.flush() self.assertEqual(lines, ["Hello World", "Foo", " Bar"]) ls.close() self.assertEqual(lines, ["Hello World", "Foo", " Bar", " BazMoo"]) def _testExtra(self, line, all_lines, p1, p2): self.assertEqual(p1, 999) self.assertEqual(p2, "extra") all_lines.append(line) def testExtraArgsNoFlush(self): lines = [] ls = utils.LineSplitter(self._testExtra, lines, 999, "extra") ls.write(b"\n\nHello World\n") ls.write(b"Foo\n Bar\r\n ") ls.write(b"") ls.write(b"Baz") ls.write(b"Moo\n\nx\n") self.assertEqual(lines, []) ls.close() self.assertEqual(lines, ["", "", "Hello World", "Foo", " Bar", " BazMoo", "", "x"]) class TestIsValidShellParam(unittest.TestCase): def test(self): for val, result in [ ("abc", True), ("ab;cd", False), ]: self.assertEqual(utils.IsValidShellParam(val), result) class TestBuildShellCmd(unittest.TestCase): def test(self): self.assertRaises(errors.ProgrammerError, utils.BuildShellCmd, "ls %s", "ab;cd") self.assertEqual(utils.BuildShellCmd("ls %s", "ab"), "ls ab") class TestOrdinal(unittest.TestCase): def test(self): checks = { 0: "0th", 1: "1st", 2: "2nd", 3: "3rd", 4: "4th", 5: "5th", 6: "6th", 7: "7th", 8: "8th", 9: "9th", 10: "10th", 11: "11th", 12: "12th", 13: "13th", 14: "14th", 15: "15th", 16: "16th", 17: "17th", 18: "18th", 19: "19th", 20: "20th", 21: "21st", 25: "25th", 30: "30th", 32: "32nd", 40: "40th", 50: "50th", 55: "55th", 60: "60th", 62: "62nd", 70: "70th", 80: "80th", 83: "83rd", 90: "90th", 91: "91st", 582: "582nd", 999: "999th", } for value, ordinal in checks.items(): self.assertEqual(utils.FormatOrdinal(value), ordinal) class TestTruncate(unittest.TestCase): def _Test(self, text, length): result = utils.Truncate(text, length) self.assertTrue(len(result) <= length) return result def test(self): self.assertEqual(self._Test("", 80), "") self.assertEqual(self._Test("abc", 4), "abc") self.assertEqual(self._Test("Hello World", 80), "Hello World") self.assertEqual(self._Test("Hello World", 4), "H...") self.assertEqual(self._Test("Hello World", 5), "He...") for i in [4, 10, 100]: data = i * "FooBarBaz" self.assertEqual(self._Test(data, len(data)), data) for (length, exp) in [(8, "T\u00e4st\u2026xyz"), (7, "T\u00e4st...")]: self.assertEqual(self._Test("T\u00e4st\u2026xyz", length), exp) self.assertEqual(self._Test(list(range(100)), 20), "[0, 1, 2, 3, 4, 5...") def testError(self): for i in range(4): self.assertRaises(AssertionError, utils.Truncate, "", i) class TestFilterEmptyLinesAndComments(unittest.TestCase): def testEmpty(self): self.assertEqual(utils.FilterEmptyLinesAndComments(""), []) self.assertEqual(utils.FilterEmptyLinesAndComments("\n"), []) self.assertEqual(utils.FilterEmptyLinesAndComments("\n" * 100), []) self.assertEqual(utils.FilterEmptyLinesAndComments("\n \n\t \n"), []) def test(self): text = """ This is # with comments a test # in # saying ...#... # multiple places Hello World! """ self.assertEqual(utils.FilterEmptyLinesAndComments(text), [ "This", "is", "a", "test", "saying", "...#...", "Hello World!", ]) class TestFormatKeyValue(unittest.TestCase): def test(self): self.assertEqual(utils.FormatKeyValue({}), []) self.assertEqual(utils.FormatKeyValue({1: 2}), ["1=2"]) self.assertEqual(utils.FormatKeyValue({ "zzz": "0", "aaa": "1", }), ["aaa=1", "zzz=0"]) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.version_unittest.py000075500000000000000000000125271476477700300250740ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the version utility functions""" import unittest from ganeti.utils import version import testutils class ParseVersionTest(unittest.TestCase): def testParseVersion(self): self.assertEqual(version.ParseVersion("2.10"), (2, 10, 0)) self.assertEqual(version.ParseVersion("2.10.1"), (2, 10, 1)) self.assertEqual(version.ParseVersion("2.10.1~beta2"), (2, 10, 1)) self.assertEqual(version.ParseVersion("2.10.1-3"), (2, 10, 1)) self.assertEqual(version.ParseVersion("2"), None) self.assertEqual(version.ParseVersion("pink bunny"), None) class UpgradeRangeTest(unittest.TestCase): def testUpgradeRange(self): self.assertEqual(version.UpgradeRange((2,11,0), current=(2,10,0)), None) self.assertEqual(version.UpgradeRange((2,10,0), current=(2,10,0)), None) self.assertEqual(version.UpgradeRange((2,11,3), current=(2,12,0)), None) self.assertEqual(version.UpgradeRange((2,11,3), current=(2,12,99)), None) self.assertEqual(version.UpgradeRange((3,0,0), current=(2,12,0)), "major version up- or downgrades are only supported" " between 2.16 and 3.0") self.assertEqual(version.UpgradeRange((2,12,0), current=(3,0,0)), "major version up- or downgrades are only supported" " between 2.16 and 3.0") self.assertEqual(version.UpgradeRange((2,10,0), current=(2,12,0)), "can only downgrade one minor version at a time") self.assertEqual(version.UpgradeRange((2,9,0), current=(2,10,0)), "automatic upgrades only supported from 2.10 onwards") self.assertEqual(version.UpgradeRange((2,10,0), current=(2,9,0)), "automatic upgrades only supported from 2.10 onwards") self.assertEqual(version.UpgradeRange((3,0,0), current=(2,16,1)), None) self.assertEqual(version.UpgradeRange((2,16,1), current=(3,0,0)), None) class ShouldCfgdowngradeTest(unittest.TestCase): def testShouldCfgDowngrade(self): self.assertTrue(version.ShouldCfgdowngrade((2,9,3), current=(2,10,0))) self.assertTrue(version.ShouldCfgdowngrade((2,9,0), current=(2,10,4))) self.assertFalse(version.ShouldCfgdowngrade((2,9,0), current=(2,11,0))) self.assertFalse(version.ShouldCfgdowngrade((2,9,0), current=(3,10,0))) self.assertFalse(version.ShouldCfgdowngrade((2,10,0), current=(3,10,0))) class IsCorrectConfigVersionTest(unittest.TestCase): def testIsCorrectConfigVersion(self): self.assertTrue(version.IsCorrectConfigVersion((2,10,1), (2,10,0))) self.assertFalse(version.IsCorrectConfigVersion((2,11,0), (2,10,0))) self.assertFalse(version.IsCorrectConfigVersion((3,10,0), (2,10,0))) class IsBeforeTest(unittest.TestCase): def testIsBefore(self): self.assertTrue(version.IsBefore(None, 2, 10, 0)) self.assertFalse(version.IsBefore((2, 10, 0), 2, 10, 0)) self.assertTrue(version.IsBefore((2, 10, 0), 2, 10, 1)) self.assertFalse(version.IsBefore((2, 10, 1), 2, 10, 0)) self.assertTrue(version.IsBefore((2, 10, 1), 2, 11, 0)) self.assertFalse(version.IsBefore((2, 11, 0), 2, 10, 3)) class IsEqualTest(unittest.TestCase): def testIsEqual(self): self.assertTrue(version.IsEqual((2, 10, 0), 2, 10, 0)) self.assertFalse(version.IsEqual((2, 10, 0), 2, 10, 2)) self.assertFalse(version.IsEqual((2, 10, 0), 2, 12, 0)) self.assertFalse(version.IsEqual((2, 10, 0), 3, 10, 0)) self.assertTrue(version.IsEqual((2, 10, 0), 2, 10, None)) self.assertTrue(version.IsEqual((2, 10, 5), 2, 10, None)) self.assertFalse(version.IsEqual((2, 11, 5), 2, 10, None)) self.assertFalse(version.IsEqual((3, 10, 5), 2, 10, None)) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.wrapper_unittest.py000075500000000000000000000133711476477700300250650ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.wrapper""" import errno import fcntl import os import socket import tempfile import unittest import shutil from ganeti import constants from ganeti import utils import testutils class TestSetCloseOnExecFlag(unittest.TestCase): """Tests for SetCloseOnExecFlag""" def setUp(self): self.tmpfile = tempfile.TemporaryFile() def testEnable(self): utils.SetCloseOnExecFlag(self.tmpfile.fileno(), True) self.assertTrue(fcntl.fcntl(self.tmpfile.fileno(), fcntl.F_GETFD) & fcntl.FD_CLOEXEC) def testDisable(self): utils.SetCloseOnExecFlag(self.tmpfile.fileno(), False) self.assertFalse(fcntl.fcntl(self.tmpfile.fileno(), fcntl.F_GETFD) & fcntl.FD_CLOEXEC) def tearDown(self): self.tmpfile.close() class TestSetNonblockFlag(unittest.TestCase): def setUp(self): self.tmpfile = tempfile.TemporaryFile() def tearDown(self): self.tmpfile.close() def testEnable(self): utils.SetNonblockFlag(self.tmpfile.fileno(), True) self.assertTrue(fcntl.fcntl(self.tmpfile.fileno(), fcntl.F_GETFL) & os.O_NONBLOCK) def testDisable(self): utils.SetNonblockFlag(self.tmpfile.fileno(), False) self.assertFalse(fcntl.fcntl(self.tmpfile.fileno(), fcntl.F_GETFL) & os.O_NONBLOCK) class TestIgnoreProcessNotFound(unittest.TestCase): @staticmethod def _WritePid(fd): os.write(fd, b"%d" % os.getpid()) os.close(fd) return True def test(self): (pid_read_fd, pid_write_fd) = os.pipe() # Start short-lived process which writes its PID to pipe self.assertTrue(utils.RunInSeparateProcess(self._WritePid, pid_write_fd)) os.close(pid_write_fd) # Read PID from pipe pid = int(os.read(pid_read_fd, 1024)) os.close(pid_read_fd) # Try to send signal to process which exited recently self.assertFalse(utils.IgnoreProcessNotFound(os.kill, pid, 0)) class TestIgnoreSignals(unittest.TestCase): """Test the IgnoreSignals decorator""" @staticmethod def _Raise(exception): raise exception @staticmethod def _Return(rval): return rval def testIgnoreSignals(self): sock_err_intr = socket.error(errno.EINTR, "Message") sock_err_inval = socket.error(errno.EINVAL, "Message") env_err_intr = EnvironmentError(errno.EINTR, "Message") env_err_inval = EnvironmentError(errno.EINVAL, "Message") self.assertRaises(socket.error, self._Raise, sock_err_intr) self.assertRaises(socket.error, self._Raise, sock_err_inval) self.assertRaises(EnvironmentError, self._Raise, env_err_intr) self.assertRaises(EnvironmentError, self._Raise, env_err_inval) self.assertEqual(utils.IgnoreSignals(self._Raise, sock_err_intr), None) self.assertEqual(utils.IgnoreSignals(self._Raise, env_err_intr), None) self.assertRaises(socket.error, utils.IgnoreSignals, self._Raise, sock_err_inval) self.assertRaises(EnvironmentError, utils.IgnoreSignals, self._Raise, env_err_inval) self.assertEqual(utils.IgnoreSignals(self._Return, True), True) self.assertEqual(utils.IgnoreSignals(self._Return, 33), 33) class TestIsExecutable(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testNonExisting(self): fname = utils.PathJoin(self.tmpdir, "file") assert not os.path.exists(fname) self.assertFalse(utils.IsExecutable(fname)) def testNoFile(self): path = utils.PathJoin(self.tmpdir, "something") os.mkdir(path) assert os.path.isdir(path) self.assertFalse(utils.IsExecutable(path)) def testExecutable(self): fname = utils.PathJoin(self.tmpdir, "file") utils.WriteFile(fname, data="#!/bin/bash", mode=0o700) assert os.path.exists(fname) self.assertTrue(utils.IsExecutable(fname)) self.assertTrue(self._TestSymlink(fname)) def testFileNotExecutable(self): fname = utils.PathJoin(self.tmpdir, "file") utils.WriteFile(fname, data="#!/bin/bash", mode=0o600) assert os.path.exists(fname) self.assertFalse(utils.IsExecutable(fname)) self.assertFalse(self._TestSymlink(fname)) def _TestSymlink(self, fname): assert os.path.exists(fname) linkname = utils.PathJoin(self.tmpdir, "cmd") os.symlink(fname, linkname) assert os.path.islink(linkname) return utils.IsExecutable(linkname) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils.x509_unittest.py000075500000000000000000000367411476477700300241200ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.utils.x509""" import os import tempfile import unittest import shutil import string import OpenSSL import packaging.version from ganeti import constants from ganeti import utils from ganeti import errors import testutils class TestParseAsn1Generalizedtime(unittest.TestCase): def setUp(self): self._Parse = utils.x509._ParseAsn1Generalizedtime def test(self): # UTC self.assertEqual(self._Parse("19700101000000Z"), 0) self.assertEqual(self._Parse("20100222174152Z"), 1266860512) self.assertEqual(self._Parse("20380119031407Z"), (2**31) - 1) # With offset self.assertEqual(self._Parse("20100222174152+0000"), 1266860512) self.assertEqual(self._Parse("20100223131652+0000"), 1266931012) self.assertEqual(self._Parse("20100223051808-0800"), 1266931088) self.assertEqual(self._Parse("20100224002135+1100"), 1266931295) self.assertEqual(self._Parse("19700101000000-0100"), 3600) # Leap seconds are not supported by datetime.datetime self.assertRaises(ValueError, self._Parse, "19841231235960+0000") self.assertRaises(ValueError, self._Parse, "19920630235960+0000") # Errors self.assertRaises(ValueError, self._Parse, "") self.assertRaises(ValueError, self._Parse, "invalid") self.assertRaises(ValueError, self._Parse, "20100222174152") self.assertRaises(ValueError, self._Parse, "Mon Feb 22 17:47:02 UTC 2010") self.assertRaises(ValueError, self._Parse, "2010-02-22 17:42:02") class TestGetX509CertValidity(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) pyopenssl_version = packaging.version.parse(OpenSSL.__version__) minimal_pyopenssl_version = packaging.version.parse("0.7") # Test whether we have pyOpenSSL in desired_pyopenssl_version (0.7) or above self.pyopenssl0_7 = pyopenssl_version >= minimal_pyopenssl_version if not self.pyopenssl0_7: warnings.warn("This test requires pyOpenSSL 0.7 or above to" " function correctly") def _LoadCert(self, name): return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, testutils.ReadTestData(name)) def test(self): validity = utils.GetX509CertValidity(self._LoadCert("cert1.pem")) if self.pyopenssl0_7: self.assertEqual(validity, (1519816700, 1519903100)) else: self.assertEqual(validity, (None, None)) class TestSignX509Certificate(unittest.TestCase): KEY = "My private key!" KEY_OTHER = "Another key" def test(self): # Generate certificate valid for 5 minutes (_, cert_pem) = utils.GenerateSelfSignedX509Cert(None, 300, 1) cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_pem) # No signature at all self.assertRaises(errors.GenericError, utils.LoadSignedX509Certificate, cert_pem, self.KEY) # Invalid input self.assertRaises(errors.GenericError, utils.LoadSignedX509Certificate, "", self.KEY) self.assertRaises(errors.GenericError, utils.LoadSignedX509Certificate, "X-Ganeti-Signature: \n", self.KEY) self.assertRaises(errors.GenericError, utils.LoadSignedX509Certificate, "X-Ganeti-Sign: $1234$abcdef\n", self.KEY) self.assertRaises(errors.GenericError, utils.LoadSignedX509Certificate, "X-Ganeti-Signature: $1234567890$abcdef\n", self.KEY) self.assertRaises(errors.GenericError, utils.LoadSignedX509Certificate, b"X-Ganeti-Signature: $1234$abc\n\n" + cert_pem, self.KEY) # Invalid salt for salt in list("-_@$,:;/\\ \t\n"): self.assertRaises(errors.GenericError, utils.SignX509Certificate, cert_pem, self.KEY, "foo%sbar" % salt) for salt in ["HelloWorld", "salt", string.ascii_letters, string.digits, utils.GenerateSecret(numbytes=4), utils.GenerateSecret(numbytes=16), "{123:456}".encode("ascii").hex()]: signed_pem = utils.SignX509Certificate(cert, self.KEY, salt) self._Check(cert, salt, signed_pem) self._Check(cert, salt, "X-Another-Header: with a value\n" + signed_pem) self._Check(cert, salt, (10 * "Hello World!\n") + signed_pem) self._Check(cert, salt, (signed_pem + "\n\na few more\n" "lines----\n------ at\nthe end!")) def _Check(self, cert, salt, pem): (cert2, salt2) = utils.LoadSignedX509Certificate(pem, self.KEY) self.assertEqual(salt, salt2) self.assertEqual(cert.digest("sha1"), cert2.digest("sha1")) # Other key self.assertRaises(errors.GenericError, utils.LoadSignedX509Certificate, pem, self.KEY_OTHER) class TestCertVerification(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testVerifyCertificate(self): cert_pem = testutils.ReadTestData("cert1.pem") cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_pem) # Not checking return value as this certificate is expired utils.VerifyX509Certificate(cert, 30, 7) @staticmethod def _GenCert(key, before, validity): # Urgh... mostly copied from x509.py :( # Create self-signed certificate cert = OpenSSL.crypto.X509() cert.set_serial_number(1) if before != 0: cert.gmtime_adj_notBefore(int(before)) cert.gmtime_adj_notAfter(validity) cert.set_issuer(cert.get_subject()) cert.set_pubkey(key) cert.sign(key, constants.X509_CERT_SIGN_DIGEST) return cert def testClockSkew(self): SKEW = constants.NODE_MAX_CLOCK_SKEW # Create private and public key key = OpenSSL.crypto.PKey() key.generate_key(OpenSSL.crypto.TYPE_RSA, constants.RSA_KEY_BITS) validity = 7 * 86400 # skew small enough, accepting cert; note that this is a timed # test, and could fail if the machine is so loaded that the next # few lines take more than NODE_MAX_CLOCK_SKEW / 2 for before in [-1, 0, SKEW // 4, SKEW // 2]: cert = self._GenCert(key, before, validity) result = utils.VerifyX509Certificate(cert, 1, 2) self.assertEqual(result, (None, None)) # skew too great, not accepting certs for before in [SKEW * 2, SKEW * 10]: cert = self._GenCert(key, before, validity) (status, msg) = utils.VerifyX509Certificate(cert, 1, 2) self.assertEqual(status, utils.CERT_WARNING) self.assertTrue(msg.startswith("Certificate not yet valid")) class TestVerifyCertificateInner(unittest.TestCase): def test(self): vci = utils.x509._VerifyCertificateInner # Valid self.assertEqual(vci(False, 1263916313, 1298476313, 1266940313, 30, 7), (None, None)) # Not yet valid (errcode, msg) = vci(False, 1266507600, 1267544400, 1266075600, 30, 7) self.assertEqual(errcode, utils.CERT_WARNING) # Expiring soon (errcode, msg) = vci(False, 1266507600, 1267544400, 1266939600, 30, 7) self.assertEqual(errcode, utils.CERT_ERROR) (errcode, msg) = vci(False, 1266507600, 1267544400, 1266939600, 30, 1) self.assertEqual(errcode, utils.CERT_WARNING) (errcode, msg) = vci(False, 1266507600, None, 1266939600, 30, 7) self.assertEqual(errcode, None) # Expired (errcode, msg) = vci(True, 1266507600, 1267544400, 1266939600, 30, 7) self.assertEqual(errcode, utils.CERT_ERROR) (errcode, msg) = vci(True, None, 1267544400, 1266939600, 30, 7) self.assertEqual(errcode, utils.CERT_ERROR) (errcode, msg) = vci(True, 1266507600, None, 1266939600, 30, 7) self.assertEqual(errcode, utils.CERT_ERROR) (errcode, msg) = vci(True, None, None, 1266939600, 30, 7) self.assertEqual(errcode, utils.CERT_ERROR) class TestGenerateX509Certs(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def _checkRsaPrivateKey(self, key): lines = key.splitlines() return (("-----BEGIN RSA PRIVATE KEY-----" in lines and "-----END RSA PRIVATE KEY-----" in lines) or ("-----BEGIN PRIVATE KEY-----" in lines and "-----END PRIVATE KEY-----" in lines)) def _checkCertificate(self, cert): lines = cert.splitlines() return ("-----BEGIN CERTIFICATE-----" in lines and "-----END CERTIFICATE-----" in lines) def test(self): for common_name in [None, ".", "Ganeti", "node1.example.com"]: (key_pem, cert_pem) = utils.GenerateSelfSignedX509Cert(common_name, 300, 1) self._checkRsaPrivateKey(key_pem) self._checkCertificate(cert_pem) key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, key_pem) self.assertTrue(key.bits() >= 1024) self.assertEqual(key.bits(), constants.RSA_KEY_BITS) self.assertEqual(key.type(), OpenSSL.crypto.TYPE_RSA) x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_pem) self.assertFalse(x509.has_expired()) self.assertEqual(x509.get_issuer().CN, common_name) self.assertEqual(x509.get_subject().CN, common_name) self.assertEqual(x509.get_pubkey().bits(), constants.RSA_KEY_BITS) def testLegacy(self): cert1_filename = os.path.join(self.tmpdir, "cert1.pem") utils.GenerateSelfSignedSslCert(cert1_filename, 1, validity=1) cert1 = utils.ReadFile(cert1_filename) self.assertTrue(self._checkRsaPrivateKey(cert1)) self.assertTrue(self._checkCertificate(cert1)) def _checkKeyMatchesCert(self, key, cert): ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD) ctx.use_privatekey(key) ctx.use_certificate(cert) try: ctx.check_privatekey() except OpenSSL.SSL.Error: return False else: return True def testSignedSslCertificate(self): server_cert_filename = os.path.join(self.tmpdir, "server.pem") utils.GenerateSelfSignedSslCert(server_cert_filename, 123456) client_hostname = "myhost.example.com" client_cert_filename = os.path.join(self.tmpdir, "client.pem") utils.GenerateSignedSslCert(client_cert_filename, 666, server_cert_filename, common_name=client_hostname) client_cert_pem = utils.ReadFile(client_cert_filename) self._checkRsaPrivateKey(client_cert_pem) self._checkCertificate(client_cert_pem) priv_key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, client_cert_pem) client_cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, client_cert_pem) self.assertTrue(self._checkKeyMatchesCert(priv_key, client_cert)) self.assertEqual(client_cert.get_issuer().CN, "ganeti.example.com") self.assertEqual(client_cert.get_subject().CN, client_hostname) class TestCheckNodeCertificate(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) self.tmpdir = tempfile.mkdtemp() def tearDown(self): testutils.GanetiTestCase.tearDown(self) shutil.rmtree(self.tmpdir) def testMismatchingKey(self): other_cert = testutils.TestDataFilename("cert1.pem") node_cert = testutils.TestDataFilename("cert2.pem") cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, utils.ReadFile(other_cert)) try: utils.CheckNodeCertificate(cert, _noded_cert_file=node_cert) except errors.GenericError as err: self.assertEqual(str(err), "Given cluster certificate does not match local key") else: self.fail("Exception was not raised") def testMatchingKey(self): cert_filename = testutils.TestDataFilename("cert2.pem") # Extract certificate cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, utils.ReadFile(cert_filename)) cert_pem = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert) utils.CheckNodeCertificate(cert, _noded_cert_file=cert_filename) def testMissingFile(self): cert_path = testutils.TestDataFilename("cert1.pem") nodecert = utils.PathJoin(self.tmpdir, "does-not-exist") utils.CheckNodeCertificate(NotImplemented, _noded_cert_file=nodecert) self.assertFalse(os.path.exists(nodecert)) def testInvalidCertificate(self): tmpfile = utils.PathJoin(self.tmpdir, "cert") utils.WriteFile(tmpfile, data="not a certificate") self.assertRaises(errors.X509CertError, utils.CheckNodeCertificate, NotImplemented, _noded_cert_file=tmpfile) def testNoPrivateKey(self): cert = testutils.TestDataFilename("cert1.pem") self.assertRaises(errors.X509CertError, utils.CheckNodeCertificate, NotImplemented, _noded_cert_file=cert) def testMismatchInNodeCert(self): cert1_path = testutils.TestDataFilename("cert1.pem") cert2_path = testutils.TestDataFilename("cert2.pem") tmpfile = utils.PathJoin(self.tmpdir, "cert") # Extract certificate cert1 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, utils.ReadFile(cert1_path)) cert1_pem = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert1) # Extract mismatching key key2 = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, utils.ReadFile(cert2_path)) key2_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key2) # Write to file utils.WriteFile(tmpfile, data=cert1_pem + key2_pem) try: utils.CheckNodeCertificate(cert1, _noded_cert_file=tmpfile) except errors.X509CertError as err: self.assertEqual(err.args, (tmpfile, "Certificate does not match with private key")) else: self.fail("Exception was not raised") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.utils_unittest.py000075500000000000000000000424601476477700300234070ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the utils module""" import array import errno import fcntl import glob import os import os.path import random import re import shutil import signal import socket import stat import tempfile import time import unittest import warnings from unittest import mock import testutils from ganeti import constants from ganeti import compat from ganeti import utils from ganeti import errors from ganeti import constants from ganeti.utils import RunCmd, \ FirstFree, \ RunParts class TestParseCpuMask(unittest.TestCase): """Test case for the ParseCpuMask function.""" def testWellFormed(self): self.assertEqual(utils.ParseCpuMask(""), []) self.assertEqual(utils.ParseCpuMask("1"), [1]) self.assertEqual(utils.ParseCpuMask("0-2,4,5-5"), [0,1,2,4,5]) def testInvalidInput(self): for data in ["garbage", "0,", "0-1-2", "2-1", "1-a"]: self.assertRaises(errors.ParseError, utils.ParseCpuMask, data) class TestParseMultiCpuMask(unittest.TestCase): """Test case for the ParseMultiCpuMask function.""" def testWellFormed(self): self.assertEqual(utils.ParseMultiCpuMask(""), []) self.assertEqual(utils.ParseMultiCpuMask("1"), [[1]]) self.assertEqual(utils.ParseMultiCpuMask("0-2,4,5-5"), [[0, 1, 2, 4, 5]]) self.assertEqual(utils.ParseMultiCpuMask("all"), [[-1]]) self.assertEqual(utils.ParseMultiCpuMask("0-2:all:4,6-8"), [[0, 1, 2], [-1], [4, 6, 7, 8]]) def testInvalidInput(self): for data in ["garbage", "0,", "0-1-2", "2-1", "1-a", "all-all"]: self.assertRaises(errors.ParseError, utils.ParseCpuMask, data) class TestGetMounts(unittest.TestCase): """Test case for GetMounts().""" TESTDATA = ( "rootfs / rootfs rw 0 0\n" "none /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0\n" "none /proc proc rw,nosuid,nodev,noexec,relatime 0 0\n") def setUp(self): self.tmpfile = tempfile.NamedTemporaryFile() utils.WriteFile(self.tmpfile.name, data=self.TESTDATA) def testGetMounts(self): self.assertEqual(utils.GetMounts(filename=self.tmpfile.name), [ ("rootfs", "/", "rootfs", "rw"), ("none", "/sys", "sysfs", "rw,nosuid,nodev,noexec,relatime"), ("none", "/proc", "proc", "rw,nosuid,nodev,noexec,relatime"), ]) class TestFirstFree(unittest.TestCase): """Test case for the FirstFree function""" def test(self): """Test FirstFree""" self.assertEqual(FirstFree([0, 1, 3]), 2) self.assertEqual(FirstFree([]), None) self.assertEqual(FirstFree([3, 4, 6]), 0) self.assertEqual(FirstFree([3, 4, 6], base=3), 5) self.assertRaises(AssertionError, FirstFree, [0, 3, 4, 6], base=3) class TestTimeFunctions(unittest.TestCase): """Test case for time functions""" def runTest(self): self.assertEqual(utils.SplitTime(1), (1, 0)) self.assertEqual(utils.SplitTime(1.5), (1, 500000)) self.assertEqual(utils.SplitTime(1218448917.4809151), (1218448917, 480915)) self.assertEqual(utils.SplitTime(123.48012), (123, 480120)) self.assertEqual(utils.SplitTime(123.9996), (123, 999600)) self.assertEqual(utils.SplitTime(123.9995), (123, 999500)) self.assertEqual(utils.SplitTime(123.9994), (123, 999400)) self.assertEqual(utils.SplitTime(123.999999999), (123, 999999)) self.assertRaises(AssertionError, utils.SplitTime, -1) self.assertEqual(utils.MergeTime((1, 0)), 1.0) self.assertEqual(utils.MergeTime((1, 500000)), 1.5) self.assertEqual(utils.MergeTime((1218448917, 500000)), 1218448917.5) self.assertEqual(round(utils.MergeTime((1218448917, 481000)), 3), 1218448917.481) self.assertEqual(round(utils.MergeTime((1, 801000)), 3), 1.801) self.assertRaises(AssertionError, utils.MergeTime, (0, -1)) self.assertRaises(AssertionError, utils.MergeTime, (0, 1000000)) self.assertRaises(AssertionError, utils.MergeTime, (0, 9999999)) self.assertRaises(AssertionError, utils.MergeTime, (-1, 0)) self.assertRaises(AssertionError, utils.MergeTime, (-9999, 0)) class FieldSetTestCase(unittest.TestCase): """Test case for FieldSets""" def testSimpleMatch(self): f = utils.FieldSet("a", "b", "c", "def") self.assertTrue(f.Matches("a")) self.assertFalse(f.Matches("d"), "Substring matched") self.assertFalse(f.Matches("defghi"), "Prefix string matched") self.assertFalse(f.NonMatching(["b", "c"])) self.assertFalse(f.NonMatching(["a", "b", "c", "def"])) self.assertTrue(f.NonMatching(["a", "d"])) def testRegexMatch(self): f = utils.FieldSet("a", "b([0-9]+)", "c") self.assertTrue(f.Matches("b1")) self.assertTrue(f.Matches("b99")) self.assertFalse(f.Matches("b/1")) self.assertFalse(f.NonMatching(["b12", "c"])) self.assertTrue(f.NonMatching(["a", "1"])) class TestForceDictType(unittest.TestCase): """Test case for ForceDictType""" KEY_TYPES = { "a": constants.VTYPE_INT, "b": constants.VTYPE_BOOL, "c": constants.VTYPE_STRING, "d": constants.VTYPE_SIZE, "e": constants.VTYPE_MAYBE_STRING, } def _fdt(self, dict, allowed_values=None): if allowed_values is None: utils.ForceDictType(dict, self.KEY_TYPES) else: utils.ForceDictType(dict, self.KEY_TYPES, allowed_values=allowed_values) return dict def testSimpleDict(self): self.assertEqual(self._fdt({}), {}) self.assertEqual(self._fdt({"a": 1}), {"a": 1}) self.assertEqual(self._fdt({"a": "1"}), {"a": 1}) self.assertEqual(self._fdt({"a": 1, "b": 1}), {"a":1, "b": True}) self.assertEqual(self._fdt({"b": 1, "c": "foo"}), {"b": True, "c": "foo"}) self.assertEqual(self._fdt({"b": 1, "c": False}), {"b": True, "c": ""}) self.assertEqual(self._fdt({"b": "false"}), {"b": False}) self.assertEqual(self._fdt({"b": "False"}), {"b": False}) self.assertEqual(self._fdt({"b": False}), {"b": False}) self.assertEqual(self._fdt({"b": "true"}), {"b": True}) self.assertEqual(self._fdt({"b": "True"}), {"b": True}) self.assertEqual(self._fdt({"d": "4"}), {"d": 4}) self.assertEqual(self._fdt({"d": "4M"}), {"d": 4}) self.assertEqual(self._fdt({"e": None, }), {"e": None, }) self.assertEqual(self._fdt({"e": "Hello World", }), {"e": "Hello World", }) self.assertEqual(self._fdt({"e": False, }), {"e": "", }) self.assertEqual(self._fdt({"b": "hello", }, ["hello"]), {"b": "hello"}) def testErrors(self): self.assertRaises(errors.TypeEnforcementError, self._fdt, {"a": "astring"}) self.assertRaises(errors.TypeEnforcementError, self._fdt, {"b": "hello"}) self.assertRaises(errors.TypeEnforcementError, self._fdt, {"c": True}) self.assertRaises(errors.TypeEnforcementError, self._fdt, {"d": "astring"}) self.assertRaises(errors.TypeEnforcementError, self._fdt, {"d": "4 L"}) self.assertRaises(errors.TypeEnforcementError, self._fdt, {"e": object(), }) self.assertRaises(errors.TypeEnforcementError, self._fdt, {"e": [], }) self.assertRaises(errors.TypeEnforcementError, self._fdt, {"x": None, }) self.assertRaises(errors.TypeEnforcementError, self._fdt, []) self.assertRaises(errors.ProgrammerError, utils.ForceDictType, {"b": "hello"}, {"b": "no-such-type"}) class TestValidateServiceName(unittest.TestCase): def testValid(self): testnames = [ 0, 1, 2, 3, 1024, 65000, 65534, 65535, "ganeti", "gnt-masterd", "HELLO_WORLD_SVC", "hello.world.1", "0", "80", "1111", "65535", ] for name in testnames: self.assertEqual(utils.ValidateServiceName(name), name) def testInvalid(self): testnames = [ -15756, -1, 65536, 133428083, "", "Hello World!", "!", "'", "\"", "\t", "\n", "`", "-8546", "-1", "65536", (129 * "A"), ] for name in testnames: self.assertRaises(errors.OpPrereqError, utils.ValidateServiceName, name) class TestReadLockedPidFile(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testNonExistent(self): path = utils.PathJoin(self.tmpdir, "nonexist") self.assertTrue(utils.ReadLockedPidFile(path) is None) def testUnlocked(self): path = utils.PathJoin(self.tmpdir, "pid") utils.WriteFile(path, data="123") self.assertTrue(utils.ReadLockedPidFile(path) is None) def testLocked(self): path = utils.PathJoin(self.tmpdir, "pid") utils.WriteFile(path, data="123") fl = utils.FileLock.Open(path) try: fl.Exclusive(blocking=True) self.assertEqual(utils.ReadLockedPidFile(path), 123) finally: fl.Close() self.assertTrue(utils.ReadLockedPidFile(path) is None) def testError(self): path = utils.PathJoin(self.tmpdir, "foobar", "pid") utils.WriteFile(utils.PathJoin(self.tmpdir, "foobar"), data="") # open(2) should return ENOTDIR self.assertRaises(EnvironmentError, utils.ReadLockedPidFile, path) class TestFindMatch(unittest.TestCase): def test(self): data = { "aaaa": "Four A", "bb": {"Two B": True}, re.compile(r"^x(foo|bar|bazX)([0-9]+)$"): (1, 2, 3), } self.assertEqual(utils.FindMatch(data, "aaaa"), ("Four A", [])) self.assertEqual(utils.FindMatch(data, "bb"), ({"Two B": True}, [])) for i in ["foo", "bar", "bazX"]: for j in range(1, 100, 7): self.assertEqual(utils.FindMatch(data, "x%s%s" % (i, j)), ((1, 2, 3), [i, str(j)])) def testNoMatch(self): self.assertTrue(utils.FindMatch({}, "") is None) self.assertTrue(utils.FindMatch({}, "foo") is None) self.assertTrue(utils.FindMatch({}, 1234) is None) data = { "X": "Hello World", re.compile("^(something)$"): "Hello World", } self.assertTrue(utils.FindMatch(data, "") is None) self.assertTrue(utils.FindMatch(data, "Hello World") is None) class TestTryConvert(unittest.TestCase): def test(self): for src, fn, result in [ ("1", int, 1), ("a", int, "a"), ("", bool, False), ("a", bool, True), ]: self.assertEqual(utils.TryConvert(fn, src), result) class TestVerifyDictOptions(unittest.TestCase): def setUp(self): self.defaults = { "first_key": "foobar", "foobar": { "key1": "value2", "key2": "value1", }, "another_key": "another_value", } def test(self): some_keys = { "first_key": "blubb", "foobar": { "key2": "foo", }, } utils.VerifyDictOptions(some_keys, self.defaults) def testInvalid(self): some_keys = { "invalid_key": "blubb", "foobar": { "key2": "foo", }, } self.assertRaises(errors.OpPrereqError, utils.VerifyDictOptions, some_keys, self.defaults) def testNestedInvalid(self): some_keys = { "foobar": { "key2": "foo", "key3": "blibb" }, } self.assertRaises(errors.OpPrereqError, utils.VerifyDictOptions, some_keys, self.defaults) def testMultiInvalid(self): some_keys = { "foobar": { "key1": "value3", "key6": "Right here", }, "invalid_with_sub": { "sub1": "value3", }, } self.assertRaises(errors.OpPrereqError, utils.VerifyDictOptions, some_keys, self.defaults) class TestValidateDeviceNames(unittest.TestCase): def testEmpty(self): utils.ValidateDeviceNames("NIC", []) utils.ValidateDeviceNames("disk", []) def testNoName(self): nics = [{}, {}] utils.ValidateDeviceNames("NIC", nics) def testInvalidName(self): self.assertRaises(errors.OpPrereqError, utils.ValidateDeviceNames, "disk", [{constants.IDISK_NAME: "42"}]) self.assertRaises(errors.OpPrereqError, utils.ValidateDeviceNames, "NIC", [{constants.INIC_NAME: "42"}]) def testUsedName(self): disks = [{constants.IDISK_NAME: "name1"}, {constants.IDISK_NAME: "name1"}] self.assertRaises(errors.OpPrereqError, utils.ValidateDeviceNames, "disk", disks) def Disk(dev_type): return mock.Mock(dev_type=dev_type) def Drbd(): return Disk(constants.DT_DRBD8) def Rbd(): return Disk(constants.DT_RBD) class AllDiskTemplateTest(unittest.TestCase): def testAllDiskless(self): self.assertTrue(utils.AllDiskOfType([], [constants.DT_DISKLESS])) def testOrDiskless(self): self.assertTrue(utils.AllDiskOfType( [], [constants.DT_DISKLESS, constants.DT_DRBD8])) def testOrDrbd(self): self.assertTrue(utils.AllDiskOfType( [Drbd()], [constants.DT_DISKLESS, constants.DT_DRBD8])) def testOrRbd(self): self.assertTrue(utils.AllDiskOfType( [Rbd()], [constants.DT_RBD, constants.DT_DRBD8])) def testNotRbd(self): self.assertFalse(utils.AllDiskOfType( [Rbd()], [constants.DT_DRBD8])) def testNotDiskless(self): self.assertFalse(utils.AllDiskOfType( [], [constants.DT_DRBD8])) def testNotRbdDiskless(self): self.assertFalse(utils.AllDiskOfType( [Rbd()], [constants.DT_DISKLESS])) def testHeterogeneous(self): self.assertFalse(utils.AllDiskOfType( [Rbd(), Drbd()], [constants.DT_DRBD8])) def testHeterogeneousDiskless(self): self.assertFalse(utils.AllDiskOfType( [Rbd(), Drbd()], [constants.DT_DISKLESS])) class AnyDiskTemplateTest(unittest.TestCase): def testAnyDiskless(self): self.assertTrue(utils.AnyDiskOfType([], [constants.DT_DISKLESS])) def testOrDiskless(self): self.assertTrue(utils.AnyDiskOfType( [], [constants.DT_DISKLESS, constants.DT_DRBD8])) def testOrDrbd(self): self.assertTrue(utils.AnyDiskOfType( [Drbd()], [constants.DT_DISKLESS, constants.DT_DRBD8])) def testOrRbd(self): self.assertTrue(utils.AnyDiskOfType( [Rbd()], [constants.DT_RBD, constants.DT_DRBD8])) def testNotRbd(self): self.assertFalse(utils.AnyDiskOfType( [Rbd()], [constants.DT_DRBD8])) def testNotDiskless(self): self.assertFalse(utils.AnyDiskOfType( [], [constants.DT_DRBD8])) def testNotRbdDiskless(self): self.assertFalse(utils.AnyDiskOfType( [Rbd()], [constants.DT_DISKLESS])) def testHeterogeneous(self): self.assertTrue(utils.AnyDiskOfType( [Rbd(), Drbd()], [constants.DT_DRBD8])) def testHeterogeneousDiskless(self): self.assertFalse(utils.AnyDiskOfType( [Rbd(), Drbd()], [constants.DT_DISKLESS])) class GetDiskTemplateTest(unittest.TestCase): def testUnique(self): self.assertEqual(utils.GetDiskTemplate([Rbd()]), constants.DT_RBD) def testDiskless(self): self.assertEqual(utils.GetDiskTemplate([]), constants.DT_DISKLESS) def testMultiple(self): self.assertEqual(utils.GetDiskTemplate([Rbd(), Rbd()]), constants.DT_RBD) def testMixed(self): self.assertEqual(utils.GetDiskTemplate([Rbd(), Drbd()]), constants.DT_MIXED) class TestSendFds(unittest.TestCase): def testSendFds(self): sender, receiver = socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM) # Attempt to send both, file-like objects and fds tempfiles = [tempfile.TemporaryFile() for _ in range(3)] tempfds = [tempfile.mkstemp()[0] for _ in range(3)] utils.SendFds(sender, b" ", tempfiles + tempfds) _, ancdata, __, ___ = receiver.recvmsg(10, 1024) self.assertEqual(len(ancdata), 1) cmsg_level, cmsg_type, cmsg_data = ancdata[0] self.assertEqual(cmsg_level, socket.SOL_SOCKET) self.assertEqual(cmsg_type, socket.SCM_RIGHTS) received_fds = array.array("i") received_fds.frombytes(cmsg_data) sent_fds = tempfds + [f.fileno() for f in tempfiles] # The received file descriptors are essentially dup()'d, so we can't # compare them directly. Instead we need to check that they are referring # to the same files. received_inodes = set(os.fstat(fd) for fd in received_fds) sent_inodes = set(os.fstat(fd) for fd in sent_fds) self.assertSetEqual(sent_inodes, received_inodes) sender.close() receiver.close() for fd in received_fds.tolist() + tempfds: os.close(fd) for file_ in tempfiles: file_.close() if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.vcluster_unittest.py000075500000000000000000000223711476477700300241150ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing ganeti.vcluster""" import os import unittest from ganeti import utils from ganeti import compat from ganeti import vcluster from ganeti import pathutils import testutils _ENV_DOES_NOT_EXIST = "GANETI_TEST_DOES_NOT_EXIST" _ENV_TEST = "GANETI_TESTVAR" class _EnvVarTest(testutils.GanetiTestCase): def setUp(self): testutils.GanetiTestCase.setUp(self) os.environ.pop(_ENV_DOES_NOT_EXIST, None) os.environ.pop(_ENV_TEST, None) class TestGetRootDirectory(_EnvVarTest): def test(self): assert os.getenv(_ENV_TEST) is None self.assertEqual(vcluster._GetRootDirectory(_ENV_DOES_NOT_EXIST), "") self.assertEqual(vcluster._GetRootDirectory(_ENV_TEST), "") # Absolute path os.environ[_ENV_TEST] = "/tmp/xy11" self.assertEqual(vcluster._GetRootDirectory(_ENV_TEST), "/tmp/xy11") # Relative path os.environ[_ENV_TEST] = "foobar" self.assertRaises(RuntimeError, vcluster._GetRootDirectory, _ENV_TEST) class TestGetHostname(_EnvVarTest): def test(self): assert os.getenv(_ENV_TEST) is None self.assertEqual(vcluster._GetRootDirectory(_ENV_DOES_NOT_EXIST), "") self.assertEqual(vcluster._GetRootDirectory(_ENV_TEST), "") os.environ[_ENV_TEST] = "some.host.example.com" self.assertEqual(vcluster._GetHostname(_ENV_TEST), "some.host.example.com") class TestCheckHostname(_EnvVarTest): def test(self): for i in ["/", "/tmp"]: self.assertRaises(RuntimeError, vcluster._CheckHostname, i) class TestPreparePaths(_EnvVarTest): def testInvalidParameters(self): self.assertRaises(RuntimeError, vcluster._PreparePaths, None, "host.example.com") self.assertRaises(RuntimeError, vcluster._PreparePaths, "/tmp/", "") def testNonNormalizedRootDir(self): self.assertRaises(AssertionError, vcluster._PreparePaths, "/tmp////xyz//", "host.example.com") def testInvalidHostname(self): self.assertRaises(RuntimeError, vcluster._PreparePaths, "/tmp", "/") def testPathHostnameMismatch(self): self.assertRaises(RuntimeError, vcluster._PreparePaths, "/tmp/host.example.com", "server.example.com") def testNoVirtCluster(self): for i in ["", None]: self.assertEqual(vcluster._PreparePaths(i, i), ("", "", None)) def testVirtCluster(self): self.assertEqual(vcluster._PreparePaths("/tmp/host.example.com", "host.example.com"), ("/tmp", "/tmp/host.example.com", "host.example.com")) class TestMakeNodeRoot(unittest.TestCase): def test(self): self.assertRaises(RuntimeError, vcluster.MakeNodeRoot, "/tmp", "/") for i in ["/tmp", "/tmp/", "/tmp///"]: self.assertEqual(vcluster.MakeNodeRoot(i, "other.example.com"), "/tmp/other.example.com") class TestEnvironmentForHost(unittest.TestCase): def test(self): self.assertEqual(vcluster.EnvironmentForHost("host.example.com", _basedir=None), {}) for i in ["host.example.com", "other.example.com"]: self.assertEqual(vcluster.EnvironmentForHost(i, _basedir="/tmp"), { vcluster._ROOTDIR_ENVNAME: "/tmp/%s" % i, vcluster._HOSTNAME_ENVNAME: i, }) class TestExchangeNodeRoot(unittest.TestCase): def test(self): result = vcluster.ExchangeNodeRoot("node1.example.com", "/tmp/file", _basedir=None, _noderoot=None) self.assertEqual(result, "/tmp/file") self.assertRaises(RuntimeError, vcluster.ExchangeNodeRoot, "node1.example.com", "/tmp/node1.example.com", _basedir="/tmp", _noderoot="/tmp/nodeZZ.example.com") result = vcluster.ExchangeNodeRoot("node2.example.com", "/tmp/node1.example.com/file", _basedir="/tmp", _noderoot="/tmp/node1.example.com") self.assertEqual(result, "/tmp/node2.example.com/file") class TestAddNodePrefix(unittest.TestCase): def testRelativePath(self): self.assertRaises(AssertionError, vcluster.AddNodePrefix, "foobar", _noderoot=None) def testRelativeNodeRoot(self): self.assertRaises(AssertionError, vcluster.AddNodePrefix, "/tmp", _noderoot="foobar") def test(self): path = vcluster.AddNodePrefix("/file/path", _noderoot="/tmp/node1.example.com/") self.assertEqual(path, "/tmp/node1.example.com/file/path") self.assertEqual(vcluster.AddNodePrefix("/file/path", _noderoot=""), "/file/path") class TestRemoveNodePrefix(unittest.TestCase): def testRelativePath(self): self.assertRaises(AssertionError, vcluster._RemoveNodePrefix, "foobar", _noderoot=None) def testOutsideNodeRoot(self): self.assertRaises(RuntimeError, vcluster._RemoveNodePrefix, "/file/path", _noderoot="/tmp/node1.example.com") self.assertRaises(RuntimeError, vcluster._RemoveNodePrefix, "/tmp/xyzfile", _noderoot="/tmp/xyz") def test(self): path = vcluster._RemoveNodePrefix("/tmp/node1.example.com/file/path", _noderoot="/tmp/node1.example.com") self.assertEqual(path, "/file/path") path = vcluster._RemoveNodePrefix("/file/path", _noderoot=None) self.assertEqual(path, "/file/path") class TestMakeVirtualPath(unittest.TestCase): def testRelativePath(self): self.assertRaises(AssertionError, vcluster.MakeVirtualPath, "foobar", _noderoot=None) def testOutsideNodeRoot(self): self.assertRaises(RuntimeError, vcluster.MakeVirtualPath, "/file/path", _noderoot="/tmp/node1.example.com") def testWithNodeRoot(self): path = vcluster.MakeVirtualPath("/tmp/node1.example.com/tmp/file", _noderoot="/tmp/node1.example.com") self.assertEqual(path, "%s/tmp/file" % vcluster._VIRT_PATH_PREFIX) def testNormal(self): self.assertEqual(vcluster.MakeVirtualPath("/tmp/file", _noderoot=None), "/tmp/file") def testWhitelisted(self): mvp = vcluster.MakeVirtualPath for path in vcluster._VPATH_WHITELIST: self.assertEqual(mvp(path), path) self.assertEqual(mvp(path, _noderoot=None), path) self.assertEqual(mvp(path, _noderoot="/tmp"), path) class TestLocalizeVirtualPath(unittest.TestCase): def testWrongPrefix(self): self.assertRaises(RuntimeError, vcluster.LocalizeVirtualPath, "/tmp/some/path", _noderoot="/tmp/node1.example.com") def testCorrectPrefixRelativePath(self): self.assertRaises(AssertionError, vcluster.LocalizeVirtualPath, vcluster._VIRT_PATH_PREFIX + "foobar", _noderoot="/tmp/node1.example.com") def testWithNodeRoot(self): lvp = vcluster.LocalizeVirtualPath virtpath1 = "%s/tmp/file" % vcluster._VIRT_PATH_PREFIX virtpath2 = "%s////tmp////file" % vcluster._VIRT_PATH_PREFIX for i in [virtpath1, virtpath2]: result = lvp(i, _noderoot="/tmp/node1.example.com") self.assertEqual(result, "/tmp/node1.example.com/tmp/file") def testNormal(self): self.assertEqual(vcluster.LocalizeVirtualPath("/tmp/file", _noderoot=None), "/tmp/file") def testWhitelisted(self): lvp = vcluster.LocalizeVirtualPath for path in vcluster._VPATH_WHITELIST: self.assertEqual(lvp(path), path) self.assertEqual(lvp(path, _noderoot=None), path) self.assertEqual(lvp(path, _noderoot="/tmp"), path) class TestVirtualPathPrefix(unittest.TestCase): def test(self): self.assertTrue(os.path.isabs(vcluster._VIRT_PATH_PREFIX)) self.assertEqual(os.path.normcase(vcluster._VIRT_PATH_PREFIX), vcluster._VIRT_PATH_PREFIX) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/ganeti.workerpool_unittest.py000075500000000000000000000322021476477700300244430ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2008, 2009, 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for unittesting the workerpool module""" import unittest import threading import time import sys import zlib import random from ganeti import workerpool from ganeti import errors from ganeti import utils from ganeti import compat import testutils class CountingContext(object): def __init__(self): self._lock = threading.Condition(threading.Lock()) self.done = 0 def DoneTask(self): self._lock.acquire() try: self.done += 1 finally: self._lock.release() def GetDoneTasks(self): self._lock.acquire() try: return self.done finally: self._lock.release() @staticmethod def UpdateChecksum(current, value): return zlib.adler32(str(value).encode("utf-8"), current) class CountingBaseWorker(workerpool.BaseWorker): def RunTask(self, ctx, text): ctx.DoneTask() class ChecksumContext: CHECKSUM_START = zlib.adler32(b"") def __init__(self): self.lock = threading.Condition(threading.Lock()) self.checksum = self.CHECKSUM_START @staticmethod def UpdateChecksum(current, value): return zlib.adler32(str(value).encode("utf-8"), current) class ChecksumBaseWorker(workerpool.BaseWorker): def RunTask(self, ctx, number): name = "number%s" % number self.SetTaskName(name) # This assertion needs to be checked before updating the checksum. A # failing assertion will then cause the result to be wrong. assert self.name == ("%s/%s" % (self._worker_id, name)) ctx.lock.acquire() try: ctx.checksum = ctx.UpdateChecksum(ctx.checksum, number) finally: ctx.lock.release() class ListBuilderContext: def __init__(self): self.lock = threading.Lock() self.result = [] self.prioresult = {} class ListBuilderWorker(workerpool.BaseWorker): def RunTask(self, ctx, data): ctx.lock.acquire() try: ctx.result.append((self.GetCurrentPriority(), data)) ctx.prioresult.setdefault(self.GetCurrentPriority(), []).append(data) finally: ctx.lock.release() class DeferringTaskContext: def __init__(self): self.lock = threading.Lock() self.prioresult = {} self.samepriodefer = {} self.num2ordertaskid = {} class DeferringWorker(workerpool.BaseWorker): def RunTask(self, ctx, num, targetprio): ctx.lock.acquire() try: otilst = ctx.num2ordertaskid.setdefault(num, []) otilst.append(self._GetCurrentOrderAndTaskId()) if num in ctx.samepriodefer: del ctx.samepriodefer[num] raise workerpool.DeferTask() if self.GetCurrentPriority() > targetprio: raise workerpool.DeferTask(priority=self.GetCurrentPriority() - 1) ctx.prioresult.setdefault(self.GetCurrentPriority(), set()).add(num) finally: ctx.lock.release() class PriorityContext: def __init__(self): self.lock = threading.Lock() self.result = [] class PriorityWorker(workerpool.BaseWorker): def RunTask(self, ctx, data): ctx.lock.acquire() try: ctx.result.append((self.GetCurrentPriority(), data)) finally: ctx.lock.release() class NotImplementedWorker(workerpool.BaseWorker): def RunTask(self): raise NotImplementedError class TestWorkerpool(unittest.TestCase): """Workerpool tests""" def testCounting(self): ctx = CountingContext() wp = workerpool.WorkerPool("Test", 3, CountingBaseWorker) try: self._CheckWorkerCount(wp, 3) for i in range(10): wp.AddTask((ctx, "Hello world %s" % i)) wp.Quiesce() finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) self.assertEqual(ctx.GetDoneTasks(), 10) def testNoTasks(self): wp = workerpool.WorkerPool("Test", 3, CountingBaseWorker) try: self._CheckWorkerCount(wp, 3) self._CheckNoTasks(wp) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) def testNoTasksQuiesce(self): wp = workerpool.WorkerPool("Test", 3, CountingBaseWorker) try: self._CheckWorkerCount(wp, 3) self._CheckNoTasks(wp) wp.Quiesce() self._CheckNoTasks(wp) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) def testChecksum(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order. wp = workerpool.WorkerPool("Test", 1, ChecksumBaseWorker) try: self._CheckWorkerCount(wp, 1) ctx = ChecksumContext() checksum = ChecksumContext.CHECKSUM_START for i in range(1, 100): checksum = ChecksumContext.UpdateChecksum(checksum, i) wp.AddTask((ctx, i)) wp.Quiesce() self._CheckNoTasks(wp) # Check sum ctx.lock.acquire() try: self.assertEqual(checksum, ctx.checksum) finally: ctx.lock.release() finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) def testAddManyTasks(self): ctx = CountingContext() wp = workerpool.WorkerPool("Test", 3, CountingBaseWorker) try: self._CheckWorkerCount(wp, 3) wp.AddManyTasks([(ctx, "Hello world %s" % i, ) for i in range(10)]) wp.AddTask((ctx, "A separate hello")) wp.AddTask((ctx, "Once more, hi!")) wp.AddManyTasks([(ctx, "Hello world %s" % i, ) for i in range(10)]) wp.Quiesce() self._CheckNoTasks(wp) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) self.assertEqual(ctx.GetDoneTasks(), 22) def testManyTasksSequence(self): ctx = CountingContext() wp = workerpool.WorkerPool("Test", 3, CountingBaseWorker) try: self._CheckWorkerCount(wp, 3) self.assertRaises(AssertionError, wp.AddManyTasks, ["Hello world %s" % i for i in range(10)]) self.assertRaises(AssertionError, wp.AddManyTasks, [i for i in range(10)]) self.assertRaises(AssertionError, wp.AddManyTasks, [], task_id=0) wp.AddManyTasks([(ctx, "Hello world %s" % i, ) for i in range(10)]) wp.AddTask((ctx, "A separate hello")) wp.Quiesce() self._CheckNoTasks(wp) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) self.assertEqual(ctx.GetDoneTasks(), 11) def _CheckNoTasks(self, wp): wp._lock.acquire() try: # The task queue must be empty now self.assertFalse(wp._tasks) self.assertFalse(wp._taskdata) finally: wp._lock.release() def _CheckWorkerCount(self, wp, num_workers): wp._lock.acquire() try: self.assertEqual(len(wp._workers), num_workers) finally: wp._lock.release() def testPriorityChecksum(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order and respects the priority wp = workerpool.WorkerPool("Test", 1, ChecksumBaseWorker) try: self._CheckWorkerCount(wp, 1) ctx = ChecksumContext() data = {} tasks = [] priorities = [] for i in range(1, 333): prio = i % 7 tasks.append((ctx, i)) priorities.append(prio) data.setdefault(prio, []).append(i) wp.AddManyTasks(tasks, priority=priorities) wp.Quiesce() self._CheckNoTasks(wp) # Check sum ctx.lock.acquire() try: checksum = ChecksumContext.CHECKSUM_START for priority in sorted(data.keys()): for i in data[priority]: checksum = ChecksumContext.UpdateChecksum(checksum, i) self.assertEqual(checksum, ctx.checksum) finally: ctx.lock.release() self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) def testPriorityListManyTasks(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order and respects the priority wp = workerpool.WorkerPool("Test", 1, ListBuilderWorker) try: self._CheckWorkerCount(wp, 1) ctx = ListBuilderContext() # Use static seed for this test rnd = random.Random(0) data = {} tasks = [] priorities = [] for i in range(1, 333): prio = int(rnd.random() * 10) tasks.append((ctx, i)) priorities.append(prio) data.setdefault(prio, []).append((prio, i)) wp.AddManyTasks(tasks, priority=priorities) self.assertRaises(errors.ProgrammerError, wp.AddManyTasks, [("x", ), ("y", )], priority=[1] * 5) self.assertRaises(errors.ProgrammerError, wp.AddManyTasks, [("x", ), ("y", )], task_id=[1] * 5) wp.Quiesce() self._CheckNoTasks(wp) # Check result ctx.lock.acquire() try: expresult = [] for priority in sorted(data.keys()): expresult.extend(data[priority]) self.assertEqual(expresult, ctx.result) finally: ctx.lock.release() self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) def testPriorityListSingleTasks(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order and respects the priority wp = workerpool.WorkerPool("Test", 1, ListBuilderWorker) try: self._CheckWorkerCount(wp, 1) ctx = ListBuilderContext() # Use static seed for this test rnd = random.Random(26279) data = {} for i in range(1, 333): prio = int(rnd.random() * 30) wp.AddTask((ctx, i), priority=prio) data.setdefault(prio, []).append(i) # Cause some distortion if i % 11 == 0: time.sleep(.001) if i % 41 == 0: wp.Quiesce() wp.Quiesce() self._CheckNoTasks(wp) # Check result ctx.lock.acquire() try: self.assertEqual(data, ctx.prioresult) finally: ctx.lock.release() self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) def testDeferTask(self): # Tests whether all tasks are run and, since we're only using a single # thread, whether everything is started in order and respects the priority wp = workerpool.WorkerPool("Test", 1, DeferringWorker) try: self._CheckWorkerCount(wp, 1) ctx = DeferringTaskContext() # Use static seed for this test rnd = random.Random(14921) data = {} num2taskid = {} for i in range(1, 333): ctx.lock.acquire() try: if i % 5 == 0: ctx.samepriodefer[i] = True finally: ctx.lock.release() prio = int(rnd.random() * 30) num2taskid[i] = 1000 * i wp.AddTask((ctx, i, prio), priority=50, task_id=num2taskid[i]) data.setdefault(prio, set()).add(i) # Cause some distortion if i % 24 == 0: time.sleep(.001) if i % 31 == 0: wp.Quiesce() wp.Quiesce() self._CheckNoTasks(wp) # Check result ctx.lock.acquire() try: self.assertEqual(data, ctx.prioresult) all_order_ids = [] for (num, numordertaskid) in ctx.num2ordertaskid.items(): order_ids = [n[0] for n in numordertaskid] self.assertFalse(utils.FindDuplicates(order_ids), msg="Order ID has been reused") all_order_ids.extend(order_ids) for task_id in map(compat.snd, numordertaskid): self.assertEqual(task_id, num2taskid[num], msg=("Task %s used different task IDs" % num)) self.assertFalse(utils.FindDuplicates(all_order_ids), msg="Order ID has been reused") finally: ctx.lock.release() self._CheckWorkerCount(wp, 1) finally: wp.TerminateWorkers() self._CheckWorkerCount(wp, 0) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/gnt-cli.test000064400000000000000000000033531476477700300207210ustar00rootroot00000000000000# test the various gnt-commands for common options sh -c "$SCRIPTS/gnt-node --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-node UNKNOWN" >>>/Usage:/ >>>2 >>>= 1 sh -c "$SCRIPTS/gnt-node --version" >>>/^gnt-/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-instance --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-instance UNKNOWN" >>>/Usage:/ >>>2 >>>= 1 sh -c "$SCRIPTS/gnt-instance --version" >>>/^gnt-instance/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-os --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-os UNKNOWN" >>>/Usage:/ >>>2 >>>= 1 sh -c "$SCRIPTS/gnt-os --version" >>>/^gnt-/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-group --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-group UNKNOWN" >>>/Usage:/ >>>2 >>>= 1 sh -c "$SCRIPTS/gnt-group --version" >>>/^gnt-/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-job --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-job UNKNOWN" >>>/Usage:/ >>>2 >>>= 1 sh -c "$SCRIPTS/gnt-job --version" >>>/^gnt-/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-cluster --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-cluster UNKNOWN" >>>/Usage:/ >>>2 >>>= 1 sh -c "$SCRIPTS/gnt-cluster --version" >>>/^gnt-/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-backup --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-backup UNKNOWN" >>>/Usage:/ >>>2 >>>= 1 sh -c "$SCRIPTS/gnt-backup --version" >>>/^gnt-/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-debug --help" >>>/Usage:/ >>>2 >>>= 0 sh -c "$SCRIPTS/gnt-debug UNKNOWN" >>>/Usage:/ >>>2 >>>= 1 sh -c "$SCRIPTS/gnt-debug --version" >>>/^gnt-/ >>>2 >>>= 0 # test that verifies all sub-commands can be run with --help, checking # that optparse doesn't reject the options list set -e; for c in scripts/gnt-*; do for i in $($c --help|grep '^ [^ ]'|awk '{print $1}'); do echo Checking command ${c##/}/$i; $c $i --help >/dev/null; done; done >>>= 0 ganeti-3.1.0~rc2/test/py/legacy/import-export_unittest-helper000075500000000000000000000063121476477700300244540ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Helpers for testing import-export daemon""" import os import sys import errno import time from ganeti import constants from ganeti import utils from ganeti import objects from ganeti import serializer RETRY_INTERVAL = (0.1, 1.1, 1) TIMEOUT = int(os.getenv("TIMEOUT", 30)) VALIDITY = int(os.getenv("VALIDITY", 1)) def Log(msg, *args): if args: line = msg % args else: line = msg sys.stderr.write("%0.6f, pid %s: %s\n" % (time.time(), os.getpid(), line)) sys.stderr.flush() def _GetImportExportData(filename): try: data = utils.ReadFile(filename) except EnvironmentError as err: Log("%s = %s", filename, err) if err.errno != errno.ENOENT: raise raise utils.RetryAgain() Log("%s = %s", filename, data.strip()) return objects.ImportExportStatus.FromDict(serializer.LoadJson(data)) def _CheckConnected(filename): if not _GetImportExportData(filename).connected: Log("Not connected") raise utils.RetryAgain() Log("Connected") def _CheckListenPort(filename): port = _GetImportExportData(filename).listen_port if not port: Log("No port") raise utils.RetryAgain() Log("Listening on %s", port) return port def WaitForListenPort(filename): return utils.Retry(_CheckListenPort, RETRY_INTERVAL, TIMEOUT, args=(filename, )) def WaitForConnected(filename): utils.Retry(_CheckConnected, RETRY_INTERVAL, TIMEOUT, args=(filename, )) def main(): (filename, what) = sys.argv[1:] Log("Running helper for %s %s", filename, what) if what == "listen-port": print(WaitForListenPort(filename)) elif what == "connected": WaitForConnected(filename) elif what == "gencert": utils.GenerateSelfSignedSslCert(filename, 1, validity=VALIDITY, common_name="localhost") else: raise Exception("Unknown command '%s'" % what) if __name__ == "__main__": main() ganeti-3.1.0~rc2/test/py/legacy/import-export_unittest.bash000075500000000000000000000276631476477700300241270ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e set -o pipefail export PYTHON=${PYTHON:=python3} impexpd="$PYTHON daemons/import-export -d" err() { echo "$@" echo 'Aborting' show_output exit 1 } show_output() { if [[ -s "$gencert_output" ]]; then echo echo 'Generating certificates:' cat $gencert_output fi if [[ -s "$dst_output" ]]; then echo echo 'Import output:' cat $dst_output fi if [[ -s "$src_output" ]]; then echo echo 'Export output:' cat $src_output fi } checkpids() { local result=0 # Unlike combining the "wait" commands using || or &&, this ensures we # actually wait for all PIDs. for pid in "$@"; do if ! wait $pid; then result=1 fi done return $result } get_testpath() { echo "${TOP_SRCDIR:-.}/test" } get_testfile() { echo "$(get_testpath)/data/$1" } upto() { echo "$(date '+%F %T'):" "$@" '...' } statusdir=$(mktemp -d) trap "rm -rf $statusdir" EXIT gencert_output=$statusdir/gencert.output src_statusfile=$statusdir/src.status src_output=$statusdir/src.output src_x509=$statusdir/src.pem dst_statusfile=$statusdir/dst.status dst_output=$statusdir/dst.output dst_x509=$statusdir/dst.pem other_x509=$statusdir/other.pem testdata=$statusdir/data1 largetestdata=$statusdir/data2 upto 'Command line parameter tests' $impexpd >/dev/null 2>&1 && err "daemon-util succeeded without parameters" $impexpd foo bar baz moo boo >/dev/null 2>&1 && err "daemon-util succeeded with wrong parameters" $impexpd $src_statusfile >/dev/null 2>&1 && err "daemon-util succeeded with insufficient parameters" $impexpd $src_statusfile invalidmode >/dev/null 2>&1 && err "daemon-util succeeded with invalid mode" for mode in import export; do $impexpd $src_statusfile $mode --compression=rot13 >/dev/null 2>&1 && err "daemon-util succeeded with invalid compression" for host in '' ' ' ' s p a c e' ... , foo.example.net... \ 'some"evil"name' 'x\ny\tmoo'; do $impexpd $src_statusfile $mode --host="$host" >/dev/null 2>&1 && err "daemon-util succeeded with invalid host '$host'" done for port in '' ' ' -1234 'some ` port " here'; do $impexpd $src_statusfile $mode --port="$port" >/dev/null 2>&1 && err "daemon-util succeeded with invalid port '$port'" done for magic in '' ' ' 'this`is' 'invalid!magic' 'he"re'; do $impexpd $src_statusfile $mode --magic="$magic" >/dev/null 2>&1 && err "daemon-util succeeded with invalid magic '$magic'" done done upto 'Generate test data' cat $(get_testfile proc_drbd8.txt) $(get_testfile cert1.pem) > $testdata # Generate about 7.5 MB of test data { tmp="$(<$testdata)" for (( i=0; i < 100; ++i )); do echo "$tmp $tmp $tmp $tmp $tmp $tmp" done dd if=/dev/zero bs=1024 count=4096 2>/dev/null for (( i=0; i < 100; ++i )); do echo "$tmp $tmp $tmp $tmp $tmp $tmp" done } > $largetestdata impexpd_helper() { $PYTHON $(get_testpath)/py/legacy/import-export_unittest-helper "$@" } start_test() { upto "$@" rm -f $src_statusfile $dst_output $dst_statusfile $dst_output rm -f $gencert_output imppid= exppid= cmd_prefix= cmd_suffix= connect_timeout=30 connect_retries=1 compress=gzip magic= } wait_import_ready() { # Wait for listening port impexpd_helper $dst_statusfile listen-port } do_export() { local port=$1 $impexpd $src_statusfile export --bind=127.0.0.1 \ --host=127.0.0.1 --port=$port \ --key=$src_x509 --cert=$src_x509 --ca=$dst_x509 \ --cmd-prefix="$cmd_prefix" --cmd-suffix="$cmd_suffix" \ --connect-timeout=$connect_timeout \ --connect-retries=$connect_retries \ --compress=$compress ${magic:+--magic="$magic"} } do_import() { $impexpd $dst_statusfile import --bind=127.0.0.1 \ --host=127.0.0.1 \ --key=$dst_x509 --cert=$dst_x509 --ca=$src_x509 \ --cmd-prefix="$cmd_prefix" --cmd-suffix="$cmd_suffix" \ --connect-timeout=$connect_timeout \ --connect-retries=$connect_retries \ --compress=$compress ${magic:+--magic="$magic"} } upto 'Generate X509 certificates and keys' impexpd_helper $src_x509 gencert 2>$gencert_output & srccertpid=$! impexpd_helper $dst_x509 gencert 2>$gencert_output & dstcertpid=$! impexpd_helper $other_x509 gencert 2>$gencert_output & othercertpid=$! checkpids $srccertpid $dstcertpid $othercertpid || \ err 'Failed to generate certificates' start_test 'Normal case' do_import > $statusdir/recv1 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then do_export $port < $testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid || err 'An error occurred' cmp $testdata $statusdir/recv1 || err 'Received data does not match input' start_test 'Export using wrong CA' # Setting lower timeout to not wait for too long connect_timeout=1 do_import &>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then : | dst_x509=$other_x509 do_export $port >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid && err 'Export did not fail when using wrong CA' start_test 'Import using wrong CA' # Setting lower timeout to not wait for too long src_x509=$other_x509 connect_timeout=1 do_import &>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then : | do_export $port >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid && err 'Import did not fail when using wrong CA' start_test 'Suffix command on import' cmd_suffix="| cksum > $statusdir/recv2" do_import &>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then do_export $port < $testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid || err 'Testing additional commands failed' cmp $statusdir/recv2 <(cksum < $testdata) || \ err 'Checksum of received data does not match' start_test 'Prefix command on export' do_import > $statusdir/recv3 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then cmd_prefix='cksum |' do_export $port <$testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid || err 'Testing additional commands failed' cmp $statusdir/recv3 <(cksum < $testdata) || \ err 'Received checksum does not match' start_test 'Failing prefix command on export' : | cmd_prefix='exit 1;' do_export 0 &>$src_output & exppid=$! checkpids $exppid && err 'Prefix command on export did not fail when it should' start_test 'Failing suffix command on export' do_import >&$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then : | cmd_suffix='| exit 1' do_export $port >>$src_output 2>&1 & exppid=$! fi checkpids $imppid $exppid && \ err 'Suffix command on export did not fail when it should' start_test 'Failing prefix command on import' cmd_prefix='exit 1;' do_import &>$dst_output & imppid=$! checkpids $imppid && err 'Prefix command on import did not fail when it should' start_test 'Failing suffix command on import' cmd_suffix='| exit 1' do_import &>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then : | do_export $port >>$src_output 2>&1 & exppid=$! fi checkpids $imppid $exppid && \ err 'Suffix command on import did not fail when it should' start_test 'Listen timeout A' # Setting lower timeout to not wait too long (there won't be anything trying to # connect) connect_timeout=1 do_import &>$dst_output & imppid=$! checkpids $imppid && \ err 'Listening with timeout did not fail when it should' start_test 'Listen timeout B' do_import &>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then { sleep 1; : | do_export $port; } >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid || \ err 'Listening with timeout failed when it should not' start_test 'Connect timeout' # Setting lower timeout as nothing will be listening on port 0 : | connect_timeout=1 do_export 0 &>$src_output & exppid=$! checkpids $exppid && err 'Connection did not time out when it should' start_test 'No compression' compress=none do_import > $statusdir/recv-nocompr 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then compress=none do_export $port < $testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid || err 'An error occurred' cmp $testdata $statusdir/recv-nocompr || \ err 'Received data does not match input' start_test 'Compression mismatch A' compress=none do_import > $statusdir/recv-miscompr 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then compress=gzip do_export $port < $testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid || err 'An error occurred' cmp -s $testdata $statusdir/recv-miscompr && \ err 'Received data matches input when it should not' start_test 'Compression mismatch B' compress=gzip do_import > $statusdir/recv-miscompr2 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then compress=none do_export $port < $testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid && err 'Did not fail when it should' cmp -s $testdata $statusdir/recv-miscompr2 && \ err 'Received data matches input when it should not' start_test 'Magic without compression' compress=none magic=MagicValue13582 \ do_import > $statusdir/recv-magic1 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then compress=none magic=MagicValue13582 \ do_export $port < $testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid || err 'An error occurred' cmp $testdata $statusdir/recv-magic1 || err 'Received data does not match input' start_test 'Magic with compression' compress=gzip magic=yzD1FBH7Iw \ do_import > $statusdir/recv-magic2 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then compress=gzip magic=yzD1FBH7Iw \ do_export $port < $testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid || err 'An error occurred' cmp $testdata $statusdir/recv-magic2 || err 'Received data does not match input' start_test 'Magic mismatch A (same length)' magic=h0tmIKXK do_import > $statusdir/recv-magic3 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then magic=bo6m9uAw do_export $port < $testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid && err 'Did not fail when it should' start_test 'Magic mismatch B' magic=AUxVEWXVr5GK do_import > $statusdir/recv-magic4 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then magic=74RiP9KP do_export $port < $testdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid && err 'Did not fail when it should' start_test 'Large transfer' do_import > $statusdir/recv-large 2>$dst_output & imppid=$! if port=$(wait_import_ready 2>$src_output); then do_export $port < $largetestdata >>$src_output 2>&1 & exppid=$! fi checkpids $exppid $imppid || err 'An error occurred' cmp $largetestdata $statusdir/recv-large || \ err 'Received data does not match input' exit 0 ganeti-3.1.0~rc2/test/py/legacy/lockperf.py000075500000000000000000000101651476477700300206440ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing lock performance""" import os import sys import time import optparse import threading import resource from ganeti import locking def ParseOptions(): """Parses the command line options. In case of command line errors, it will show the usage and exit the program. @return: the options in a tuple """ parser = optparse.OptionParser() parser.add_option("-t", dest="thread_count", default=1, type="int", help="Number of threads", metavar="NUM") parser.add_option("-d", dest="duration", default=5, type="float", help="Duration", metavar="SECS") (opts, args) = parser.parse_args() if opts.thread_count < 1: parser.error("Number of threads must be at least 1") return (opts, args) class State(object): def __init__(self, thread_count): """Initializes this class. """ self.verify = [0 for _ in range(thread_count)] self.counts = [0 for _ in range(thread_count)] self.total_count = 0 def _Counter(lock, state, me): """Thread function for acquiring locks. """ counts = state.counts verify = state.verify while True: lock.acquire() try: verify[me] = 1 counts[me] += 1 state.total_count += 1 if state.total_count % 1000 == 0: sys.stdout.write(" %8d\r" % state.total_count) sys.stdout.flush() if sum(verify) != 1: print("Inconsistent state!") os._exit(1) # pylint: disable=W0212 verify[me] = 0 finally: lock.release() def main(): (opts, _) = ParseOptions() lock = locking.SharedLock("TestLock") state = State(opts.thread_count) lock.acquire(shared=0) try: for i in range(opts.thread_count): t = threading.Thread(target=_Counter, args=(lock, state, i)) t.setDaemon(True) t.start() start = time.clock() finally: lock.release() while True: if (time.clock() - start) > opts.duration: break time.sleep(0.1) # Make sure we get a consistent view lock.acquire(shared=0) lock_cputime = time.clock() - start res = resource.getrusage(resource.RUSAGE_SELF) print("Total number of acquisitions: %s" % state.total_count) print("Per-thread acquisitions:") for (i, count) in enumerate(state.counts): print(" Thread %s: %d (%0.1f%%)" % (i, count, (100.0 * count / state.total_count))) print("Benchmark CPU time: %0.3fs" % lock_cputime) print("Average time per lock acquisition: %0.5fms" % (1000.0 * lock_cputime / state.total_count)) print("Process:") print(" User time: %0.3fs" % res.ru_utime) print(" System time: %0.3fs" % res.ru_stime) print(" Total time: %0.3fs" % (res.ru_utime + res.ru_stime)) # Exit directly without attempting to clean up threads os._exit(0) # pylint: disable=W0212 if __name__ == "__main__": main() ganeti-3.1.0~rc2/test/py/legacy/mocks.py000064400000000000000000000121671476477700300201540ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Module implementing a fake ConfigWriter""" import os from ganeti import locking from ganeti import netutils FAKE_CLUSTER_KEY = ("AAAAB3NzaC1yc2EAAAABIwAAAQEAsuGLw70et3eApJ/ZEJkAVZogIrm" "EYPQJvb1ll52Ti0nr80Wztxibaa8bYGzY22rQIAloIlePeTGcJceAYK" "PZgm0I/Mp2EUGg2NVsQZIzasz6cW0vYuiUbF9GkVlROmvOAykT58RfM" "L8RhPrjrQxZc+NXgZtgDugYSZcXHDLUyWM1xKUoYy0MqYG6ZXCC/Zno" "RThhmjOJgEmvwrMcTWQjmzH3NeJAxaBsEHR8tiVZ/Y23C/ULWLyNT6R" "fB+DE7IovsMQaS+83AK1Teg7RWNyQczachatf/JT8VjUqFYjJepPjMb" "vYdB2nQds7/+Bf40C/OpbvnAxna1kVtgFHAo18cQ==") class FakeConfig(object): """Fake configuration object""" def __init__(self): self.write_count = 0 def OutDate(self): pass def IsCluster(self): return True def GetNodeList(self): return ["a", "b", "c"] def GetRsaHostKey(self): return FAKE_CLUSTER_KEY def GetDsaHostKey(self): return FAKE_CLUSTER_KEY def GetClusterName(self): return "test.cluster" def GetMasterNode(self): return "a" def GetMasterNodeName(self): return netutils.Hostname.GetSysName() def GetDefaultIAllocator(self): return "testallocator" def GetNodeName(self, node_uuid): if node_uuid in self.GetNodeList(): return "node_%s.example.com" % (node_uuid,) else: return None def GetNodeNames(self, node_uuids): return [self.GetNodeName(uuid) for uuid in node_uuids] class FakeProc(object): """Fake processor object""" def Log(self, msg, *args, **kwargs): pass def LogWarning(self, msg, *args, **kwargs): pass def LogInfo(self, msg, *args, **kwargs): pass def LogStep(self, current, total, message): pass class FakeGLM(object): """Fake global lock manager object""" def list_owned(self, _): return set() class FakeContext(object): """Fake context object""" # pylint: disable=W0613 def __init__(self): self.cfg = FakeConfig() self.glm = FakeGLM() def GetConfig(self, ec_id): return self.cfg def GetRpc(self, cfg): return None def GetWConfdContext(self, ec_id): return (None, None, None) class FakeGetentResolver(object): """Fake runtime.GetentResolver""" def __init__(self): # As we nomally don't run under root we use our own uid/gid for all # fields. This way we don't run into permission denied problems. uid = os.getuid() gid = os.getgid() self.masterd_uid = uid self.masterd_gid = gid self.confd_uid = uid self.confd_gid = gid self.rapi_uid = uid self.rapi_gid = gid self.noded_uid = uid self.noded_gid = gid self.daemons_gid = gid self.admin_gid = gid def LookupUid(self, uid): return "user%s" % uid def LookupGid(self, gid): return "group%s" % gid class FakeLU(object): HPATH = "fake-lu" HTYPE = None def __init__(self, processor, op, cfg, rpc_runner, prereq_err): self.proc = processor self.cfg = cfg self.op = op self.rpc = rpc_runner self.prereq_err = prereq_err self.needed_locks = {} self.opportunistic_locks = dict.fromkeys(locking.LEVELS, False) self.dont_collate_locks = dict.fromkeys(locking.LEVELS, False) self.add_locks = {} self.LogWarning = processor.LogWarning # pylint: disable=C0103 def CheckArguments(self): pass def ExpandNames(self): pass def DeclareLocks(self, level): pass def CheckPrereq(self): if self.prereq_err: raise self.prereq_err def Exec(self, feedback_fn): pass def BuildHooksNodes(self): return ([], []) def BuildHooksEnv(self): return {} def PreparePostHookNodes(self, post_hook_node_uuids): # pylint: disable=W0613 return [] def HooksCallBack(self, phase, hook_results, feedback_fn, lu_result): # pylint: disable=W0613 return lu_result ganeti-3.1.0~rc2/test/py/legacy/qa.qa_config_unittest.py000075500000000000000000000342421476477700300233260ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing qa.qa_config""" import unittest import tempfile import shutil import os from ganeti import utils from ganeti import serializer from ganeti import constants from ganeti import compat from qa import qa_config from qa import qa_error import testutils class TestTestEnabled(unittest.TestCase): def testSimple(self): for name in ["test", ["foobar"], ["a", "b"]]: self.assertTrue(qa_config.TestEnabled(name, _cfg={})) for default in [False, True]: self.assertFalse(qa_config.TestEnabled("foo", _cfg={ "tests": { "default": default, "foo": False, }, })) self.assertTrue(qa_config.TestEnabled("bar", _cfg={ "tests": { "default": default, "bar": True, }, })) def testEitherWithDefault(self): names = qa_config.Either("one") self.assertTrue(qa_config.TestEnabled(names, _cfg={ "tests": { "default": True, }, })) self.assertFalse(qa_config.TestEnabled(names, _cfg={ "tests": { "default": False, }, })) def testEither(self): names = [qa_config.Either(["one", "two"]), qa_config.Either("foo"), "hello", ["bar", "baz"]] self.assertTrue(qa_config.TestEnabled(names, _cfg={ "tests": { "default": True, }, })) self.assertFalse(qa_config.TestEnabled(names, _cfg={ "tests": { "default": False, }, })) for name in ["foo", "bar", "baz", "hello"]: self.assertFalse(qa_config.TestEnabled(names, _cfg={ "tests": { "default": True, name: False, }, })) self.assertFalse(qa_config.TestEnabled(names, _cfg={ "tests": { "default": True, "one": False, "two": False, }, })) self.assertTrue(qa_config.TestEnabled(names, _cfg={ "tests": { "default": True, "one": False, "two": True, }, })) self.assertFalse(qa_config.TestEnabled(names, _cfg={ "tests": { "default": True, "one": True, "two": True, "foo": False, }, })) def testEitherNestedWithAnd(self): names = qa_config.Either([["one", "two"], "foo"]) self.assertTrue(qa_config.TestEnabled(names, _cfg={ "tests": { "default": True, }, })) for name in ["one", "two"]: self.assertFalse(qa_config.TestEnabled(names, _cfg={ "tests": { "default": True, "foo": False, name: False, }, })) def testCallable(self): self.assertTrue(qa_config.TestEnabled([lambda: True], _cfg={})) for value in [None, False, "", 0]: self.assertFalse(qa_config.TestEnabled(lambda: value, _cfg={})) class TestQaConfigLoad(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testLoadNonExistent(self): filename = utils.PathJoin(self.tmpdir, "does.not.exist") self.assertRaises(EnvironmentError, qa_config._QaConfig.Load, filename) @staticmethod def _WriteConfig(filename, data): utils.WriteFile(filename, data=serializer.DumpJson(data)) def _CheckLoadError(self, filename, data, expected): self._WriteConfig(filename, data) try: qa_config._QaConfig.Load(filename) except qa_error.Error as err: self.assertTrue(str(err).startswith(expected)) else: self.fail("Exception was not raised") def testFailsValidation(self): filename = utils.PathJoin(self.tmpdir, "qa.json") testconfig = {} check_fn = compat.partial(self._CheckLoadError, filename, testconfig) # No cluster name check_fn("Cluster name is required") testconfig["name"] = "cluster.example.com" # No nodes check_fn("Need at least one node") testconfig["nodes"] = [ { "primary": "xen-test-0", "secondary": "192.0.2.1", }, ] # No instances check_fn("Need at least one instance") testconfig["instances"] = [ { "name": "xen-test-inst1", }, ] # Missing "disk" and "disk-growth" check_fn("Config option 'disks'") testconfig["disks"] = [] # Minimal accepted configuration self._WriteConfig(filename, testconfig) result = qa_config._QaConfig.Load(filename) self.assertTrue(result.get("nodes")) # Non-existent instance check script testconfig[qa_config._INSTANCE_CHECK_KEY] = \ utils.PathJoin(self.tmpdir, "instcheck") check_fn("Can't find instance check script") del testconfig[qa_config._INSTANCE_CHECK_KEY] # No enabled hypervisor testconfig[qa_config._ENABLED_HV_KEY] = None check_fn("No hypervisor is enabled") # Unknown hypervisor testconfig[qa_config._ENABLED_HV_KEY] = ["#unknownhv#"] check_fn("Unknown hypervisor(s) enabled:") del testconfig[qa_config._ENABLED_HV_KEY] # Invalid path for virtual cluster base directory testconfig[qa_config._VCLUSTER_MASTER_KEY] = "value" testconfig[qa_config._VCLUSTER_BASEDIR_KEY] = "./not//normalized/" check_fn("Path given in option 'vcluster-basedir' must be") # Inconsistent virtual cluster settings testconfig.pop(qa_config._VCLUSTER_MASTER_KEY) testconfig[qa_config._VCLUSTER_BASEDIR_KEY] = "/tmp" check_fn("All or none of the") testconfig[qa_config._VCLUSTER_MASTER_KEY] = "master.example.com" testconfig.pop(qa_config._VCLUSTER_BASEDIR_KEY) check_fn("All or none of the") # Accepted virtual cluster settings testconfig[qa_config._VCLUSTER_MASTER_KEY] = "master.example.com" testconfig[qa_config._VCLUSTER_BASEDIR_KEY] = "/tmp" self._WriteConfig(filename, testconfig) result = qa_config._QaConfig.Load(filename) self.assertEqual(result.GetVclusterSettings(), ("master.example.com", "/tmp")) class TestQaConfigWithSampleConfig(unittest.TestCase): """Tests using C{qa-sample.json}. This test case serves two purposes: - Ensure shipped C{qa-sample.json} file is considered a valid QA configuration - Test some functions of L{qa_config._QaConfig} without having to mock a whole configuration file """ def setUp(self): filename = "%s/qa/qa-sample.json" % testutils.GetSourceDir() self.config = qa_config._QaConfig.Load(filename) def testGetEnabledHypervisors(self): self.assertEqual(self.config.GetEnabledHypervisors(), [constants.DEFAULT_ENABLED_HYPERVISOR]) def testGetDefaultHypervisor(self): self.assertEqual(self.config.GetDefaultHypervisor(), constants.DEFAULT_ENABLED_HYPERVISOR) def testGetInstanceCheckScript(self): self.assertTrue(self.config.GetInstanceCheckScript() is None) def testGetAndGetItem(self): self.assertEqual(self.config["nodes"], self.config.get("nodes")) def testGetMasterNode(self): self.assertEqual(self.config.GetMasterNode(), self.config["nodes"][0]) def testGetVclusterSettings(self): # Shipped default settings should be to not use a virtual cluster self.assertEqual(self.config.GetVclusterSettings(), (None, None)) self.assertFalse(qa_config.UseVirtualCluster(_cfg=self.config)) class TestQaConfig(unittest.TestCase): def setUp(self): filename = \ testutils.TestDataFilename("qa-minimal-nodes-instances-only.json") self.config = qa_config._QaConfig.Load(filename) def testExclusiveStorage(self): self.assertRaises(AssertionError, self.config.GetExclusiveStorage) for value in [False, True, 0, 1, 30804, ""]: self.config.SetExclusiveStorage(value) self.assertEqual(self.config.GetExclusiveStorage(), bool(value)) def testIsTemplateSupported(self): enabled_dts = self.config.GetEnabledDiskTemplates() for e_s in [False, True]: self.config.SetExclusiveStorage(e_s) for template in constants.DISK_TEMPLATES: if (template not in enabled_dts or e_s and template not in constants.DTS_EXCL_STORAGE): self.assertFalse(self.config.IsTemplateSupported(template)) else: self.assertTrue(self.config.IsTemplateSupported(template)) def testInstanceConversion(self): self.assertTrue(isinstance(self.config["instances"][0], qa_config._QaInstance)) def testNodeConversion(self): self.assertTrue(isinstance(self.config["nodes"][0], qa_config._QaNode)) def testAcquireAndReleaseInstance(self): self.assertFalse(compat.any(i.used for i in self.config["instances"])) inst = qa_config.AcquireInstance(_cfg=self.config) self.assertTrue(inst.used) self.assertTrue(inst.disk_template is None) inst.Release() self.assertFalse(inst.used) self.assertTrue(inst.disk_template is None) self.assertFalse(compat.any(i.used for i in self.config["instances"])) def testAcquireInstanceTooMany(self): # Acquire all instances for _ in range(len(self.config["instances"])): inst = qa_config.AcquireInstance(_cfg=self.config) self.assertTrue(inst.used) self.assertTrue(inst.disk_template is None) # The next acquisition must fail self.assertRaises(qa_error.OutOfInstancesError, qa_config.AcquireInstance, _cfg=self.config) def testAcquireNodeNoneAdded(self): self.assertFalse(compat.any(n.added for n in self.config["nodes"])) # First call must return master node node = qa_config.AcquireNode(_cfg=self.config) self.assertEqual(node, self.config.GetMasterNode()) # Next call with exclusion list fails self.assertRaises(qa_error.OutOfNodesError, qa_config.AcquireNode, exclude=[node], _cfg=self.config) def testAcquireNodeTooMany(self): # Mark all nodes as marked (master excluded) for node in self.config["nodes"]: if node != self.config.GetMasterNode(): node.MarkAdded() nodecount = len(self.config["nodes"]) self.assertTrue(nodecount > 1) acquired = [] for _ in range(nodecount): node = qa_config.AcquireNode(exclude=acquired, _cfg=self.config) if node == self.config.GetMasterNode(): self.assertFalse(node.added) else: self.assertTrue(node.added) self.assertEqual(node.use_count, 1) acquired.append(node) self.assertRaises(qa_error.OutOfNodesError, qa_config.AcquireNode, exclude=acquired, _cfg=self.config) def testAcquireNodeOrder(self): # Mark all nodes as marked (master excluded) for node in self.config["nodes"]: if node != self.config.GetMasterNode(): node.MarkAdded() nodecount = len(self.config["nodes"]) for iterations in [0, 1, 3, 100, 127, 7964]: acquired = [] for i in range(iterations): node = qa_config.AcquireNode(_cfg=self.config) self.assertTrue(node.use_count > 0) self.assertEqual(node.use_count, (i // nodecount + 1)) acquired.append((node.use_count, node.primary, node)) # Check if returned nodes were in correct order key_fn = lambda a_b_c: (a_b_c[0], utils.NiceSortKey(a_b_c[1]), a_b_c[2]) self.assertEqual(acquired, sorted(acquired, key=key_fn)) # Release previously acquired nodes qa_config.ReleaseManyNodes([a[2] for a in acquired]) # Check if nodes were actually released for node in self.config["nodes"]: self.assertEqual(node.use_count, 0) self.assertTrue(node.added or node == self.config.GetMasterNode()) class TestRepresentation(unittest.TestCase): def _Check(self, target, part): self.assertTrue(part in repr(target).split()) def testQaInstance(self): inst = qa_config._QaInstance("inst1.example.com", []) self._Check(inst, "name=inst1.example.com") self._Check(inst, "nicmac=[]") # Default values self._Check(inst, "disk_template=None") self._Check(inst, "used=None") # Use instance inst.Use() self._Check(inst, "used=True") # Disk template inst.SetDiskTemplate(constants.DT_DRBD8) self._Check(inst, "disk_template=%s" % constants.DT_DRBD8) # Release instance inst.Release() self._Check(inst, "used=False") self._Check(inst, "disk_template=None") def testQaNode(self): node = qa_config._QaNode("primary.example.com", "192.0.2.1") self._Check(node, "primary=primary.example.com") self._Check(node, "secondary=192.0.2.1") self._Check(node, "added=False") self._Check(node, "use_count=0") # Mark as added node.MarkAdded() self._Check(node, "added=True") # Use node for i in range(1, 5): node.Use() self._Check(node, "use_count=%s" % i) # Release node for i in reversed(range(1, 5)): node.Release() self._Check(node, "use_count=%s" % (i - 1)) self._Check(node, "use_count=0") # Mark as added node.MarkRemoved() self._Check(node, "added=False") if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/systemd_unittest.bash000075500000000000000000000042131476477700300227500ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e daemon_util=daemons/daemon-util err() { echo "$@" echo 'Aborting' exit 1 } for dname in $($daemon_util list-start-daemons; $daemon_util list-stop-daemons); do service="doc/examples/systemd/${dname}.service" test -f "$service" || err "No systemd unit found for ${dname}" usergroup=$($daemon_util -daemon-usergroup ${dname#ganeti-}) user=${usergroup%:*} group=${usergroup#*:} service_user=$(grep ^User $service | cut -d '=' -f 2 | tr -d ' ') service_group=$(grep ^Group $service | cut -d '=' -f 2 | tr -d ' ') test "${service_user:-root}" == "$user" || err "Systemd service for ${dname} has user ${service_user}" \ "instead of ${user}" test "${service_group:-root}" == "$group" || err "Systemd service for ${dname} has group ${service_group}" \ "instead of ${group}" done ganeti-3.1.0~rc2/test/py/legacy/tempfile_fork_unittest.py000075500000000000000000000100331476477700300236160ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Script for testing utils.ResetTempfileModule""" import os import sys import errno import shutil import tempfile import unittest import logging from ganeti import utils import testutils # This constant is usually at a much higher value. Setting it lower for test # purposes. tempfile.TMP_MAX = 3 class TestResetTempfileModule(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tmpdir) def testNoReset(self): if ((sys.hexversion >= 0x020703F0 and sys.hexversion < 0x03000000) or sys.hexversion >= 0x030203F0): # We can't test the no_reset case on Python 2.7+ return # evil Debian sid... if (hasattr(tempfile._RandomNameSequence, "rng") and type(tempfile._RandomNameSequence.rng) == property): return self._Test(False) def testReset(self): self._Test(True) def _Test(self, reset): self.assertFalse(tempfile.TMP_MAX > 10) # Initialize tempfile module (fd, _) = tempfile.mkstemp(dir=self.tmpdir, prefix="init.", suffix="") os.close(fd) (notify_read, notify_write) = os.pipe() pid = os.fork() if pid == 0: # Child try: try: if reset: utils.ResetTempfileModule() os.close(notify_write) # Wait for parent to close pipe os.read(notify_read, 1) try: # This is a short-lived process, not caring about closing file # descriptors (_, path) = tempfile.mkstemp(dir=self.tmpdir, prefix="test.", suffix="") except EnvironmentError as err: if err.errno == errno.EEXIST: # Couldnt' create temporary file (e.g. because we run out of # retries) os._exit(2) raise logging.debug("Child created %s", path) os._exit(0) except Exception: logging.exception("Unhandled error") finally: os._exit(1) # Parent os.close(notify_read) # Create parent's temporary files for _ in range(tempfile.TMP_MAX): (fd, path) = tempfile.mkstemp(dir=self.tmpdir, prefix="test.", suffix="") os.close(fd) logging.debug("Parent created %s", path) # Notify child by closing pipe os.close(notify_write) (_, status) = os.waitpid(pid, 0) self.assertFalse(os.WIFSIGNALED(status)) if reset: # If the tempfile module was reset, it should not fail to create # temporary files expected = 0 else: expected = 2 self.assertEqual(os.WEXITSTATUS(status), expected) if __name__ == "__main__": testutils.GanetiTestProgram() ganeti-3.1.0~rc2/test/py/legacy/testutils/000075500000000000000000000000001476477700300205175ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/legacy/testutils/__init__.py000064400000000000000000000206301476477700300226310ustar00rootroot00000000000000# # # Copyright (C) 2006, 2007, 2008 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Utilities for unit testing""" import os import sys import stat import errno import base64 import socket import tempfile import unittest import logging from unittest import mock _patcher = mock._patch_object from ganeti import utils def GetSourceDir(): return os.environ.get("TOP_SRCDIR", ".") def TestDataFilename(name): """Returns the filename of a given test data file. @type name: str @param name: the 'base' of the file name, as present in the test/data directory @rtype: str @return: the full path to the filename, such that it can be used in 'make distcheck' rules """ return "%s/test/data/%s" % (GetSourceDir(), name) def ReadTestData(name): """Returns the content of a test data file. This is just a very simple wrapper over utils.ReadFile with the proper test file name. """ return utils.ReadFile(TestDataFilename(name)) def _SetupLogging(verbose): """Setupup logging infrastructure. """ fmt = logging.Formatter("%(asctime)s: %(threadName)s" " %(levelname)s %(message)s") if verbose: handler = logging.StreamHandler() else: handler = logging.FileHandler(os.devnull, "a") handler.setLevel(logging.NOTSET) handler.setFormatter(fmt) root_logger = logging.getLogger("") root_logger.setLevel(logging.NOTSET) root_logger.addHandler(handler) def RequiresIPv6(): """Decorator for tests requiring IPv6 support Decorated tests will be skipped if no IPv6 networking support is available on the host system. """ try: # Try to bind a DGRAM socket on IPv6 localhost sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) sock.bind(('::1', 0)) sock.close() except socket.error as err: if err.errno in (errno.EADDRNOTAVAIL, errno.EAFNOSUPPORT): return unittest.skip("IPv6 not available") return lambda thing: thing class GanetiTestProgram(unittest.TestProgram): def runTests(self): """Runs all tests. """ _SetupLogging("LOGTOSTDERR" in os.environ) sys.stderr.write("Running %s\n" % self.progName) sys.stderr.flush() # Ensure assertions will be evaluated if not __debug__: raise Exception("Not running in debug mode, assertions would not be" " evaluated") # Check again, this time with a real assertion try: assert False except AssertionError: pass else: raise Exception("Assertion not evaluated") return unittest.TestProgram.runTests(self) # pylint: disable=R0904 class GanetiTestCase(unittest.TestCase): """Helper class for unittesting. This class defines a few utility functions that help in building unittests. Child classes must call the parent setup and cleanup. """ def setUp(self): self._temp_files = [] self.patches = {} self.mocks = {} def MockOut(self, name, patch=None): if patch is None: patch = name self.patches[name] = patch self.mocks[name] = patch.start() def tearDown(self): while self._temp_files: try: utils.RemoveFile(self._temp_files.pop()) except EnvironmentError: pass for patch in self.patches.values(): patch.stop() self.patches = {} self.mocks = {} def assertFileContent(self, file_name, expected_content): """Checks that the content of a file is what we expect. @type file_name: str @param file_name: the file whose contents we should check @type expected_content: str @param expected_content: the content we expect """ actual_content = utils.ReadFile(file_name) self.assertEqual(actual_content, expected_content) def assertFileContentNotEqual(self, file_name, reference_content): """Checks that the content of a file is different to the reference. @type file_name: str @param file_name: the file whose contents we should check @type reference_content: str @param reference_content: the content we use as reference """ actual_content = utils.ReadFile(file_name) self.assertNotEqual(actual_content, reference_content) def assertFileMode(self, file_name, expected_mode): """Checks that the mode of a file is what we expect. @type file_name: str @param file_name: the file whose contents we should check @type expected_mode: int @param expected_mode: the mode we expect """ st = os.stat(file_name) actual_mode = stat.S_IMODE(st.st_mode) self.assertEqual(actual_mode, expected_mode) def assertFileUid(self, file_name, expected_uid): """Checks that the user id of a file is what we expect. @type file_name: str @param file_name: the file whose contents we should check @type expected_uid: int @param expected_uid: the user id we expect """ st = os.stat(file_name) actual_uid = st.st_uid self.assertEqual(actual_uid, expected_uid) def assertFileGid(self, file_name, expected_gid): """Checks that the group id of a file is what we expect. @type file_name: str @param file_name: the file whose contents we should check @type expected_gid: int @param expected_gid: the group id we expect """ st = os.stat(file_name) actual_gid = st.st_gid self.assertEqual(actual_gid, expected_gid) def assertEqualValues(self, first, second, msg=None): """Compares two values whether they're equal. Tuples are automatically converted to lists before comparing. """ return self.assertEqual(UnifyValueType(first), UnifyValueType(second), msg=msg) def _CreateTempFile(self): """Creates a temporary file and adds it to the internal cleanup list. This method simplifies the creation and cleanup of temporary files during tests. """ fh, fname = tempfile.mkstemp(prefix="ganeti-test", suffix=".tmp") os.close(fh) self._temp_files.append(fname) return fname # pylint: enable=R0904 def patch_object(*args, **kwargs): """Unified patch_object for various versions of Python Mock.""" return _patcher(*args, **kwargs) def UnifyValueType(data): """Converts all tuples into lists. This is useful for unittests where an external library doesn't keep types. """ if isinstance(data, (tuple, list)): return [UnifyValueType(i) for i in data] elif isinstance(data, dict): return dict([(UnifyValueType(key), UnifyValueType(value)) for (key, value) in data.items()]) return data class CallCounter(object): """Utility class to count number of calls to a function/method. """ def __init__(self, fn): """Initializes this class. @type fn: Callable """ self._fn = fn self._count = 0 def __call__(self, *args, **kwargs): """Calls wrapped function with given parameters. """ self._count += 1 return self._fn(*args, **kwargs) def Count(self): """Returns number of calls. @rtype: number """ return self._count def b64encode_string(text, encoding="utf-8"): """Utility to base64-encode a string This exposes a string interface for Python3's b64encode @type text: string @param text: input string """ return base64.b64encode(text.encode(encoding)).decode("ascii").strip() ganeti-3.1.0~rc2/test/py/legacy/testutils/config_mock.py000064400000000000000000001017171476477700300233560ustar00rootroot00000000000000# # # Copyright (C) 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Support for mocking the cluster configuration""" import random import time import uuid as uuid_module from ganeti import config from ganeti import constants from ganeti import errors from ganeti.network import AddressPool from ganeti import objects from ganeti import utils import mocks RESERVE_ACTION = "reserve" RELEASE_ACTION = "release" def _StubGetEntResolver(): return mocks.FakeGetentResolver() def _UpdateIvNames(base_idx, disks): """Update the C{iv_name} attribute of disks. @type disks: list of L{objects.Disk} """ for (idx, disk) in enumerate(disks): disk.iv_name = "disk/%s" % (base_idx + idx) # pylint: disable=R0904 class ConfigMock(config.ConfigWriter): """A mocked cluster configuration with added methods for easy customization. """ def __init__(self, cfg_file="/dev/null"): self._cur_group_id = 1 self._cur_node_id = 1 self._cur_inst_id = 1 self._cur_disk_id = 1 self._cur_os_id = 1 self._cur_nic_id = 1 self._cur_net_id = 1 self._default_os = None self._mocked_config_store = None self._temporary_macs = config.TemporaryReservationManager() self._temporary_secrets = config.TemporaryReservationManager() self._temporary_lvs = config.TemporaryReservationManager() self._temporary_ips = config.TemporaryReservationManager() super(ConfigMock, self).__init__(cfg_file=cfg_file, _getents=_StubGetEntResolver(), offline=True) with self.GetConfigManager(): self._CreateConfig() def _GetUuid(self): return str(uuid_module.uuid4()) def _GetObjUuid(self, obj): if obj is None: return None elif isinstance(obj, objects.ConfigObject): return obj.uuid else: return obj def AddNewNodeGroup(self, uuid=None, name=None, ndparams=None, diskparams=None, ipolicy=None, hv_state_static=None, disk_state_static=None, alloc_policy=None, networks=None): """Add a new L{objects.NodeGroup} to the cluster configuration See L{objects.NodeGroup} for parameter documentation. @rtype: L{objects.NodeGroup} @return: the newly added node group """ group_id = self._cur_group_id self._cur_group_id += 1 if uuid is None: uuid = self._GetUuid() if name is None: name = "mock_group_%d" % group_id if networks is None: networks = {} group = objects.NodeGroup(uuid=uuid, name=name, ndparams=ndparams, diskparams=diskparams, ipolicy=ipolicy, hv_state_static=hv_state_static, disk_state_static=disk_state_static, alloc_policy=alloc_policy, networks=networks, members=[]) self._UnlockedAddNodeGroup(group, None, True) return group # pylint: disable=R0913 def AddNewNode(self, uuid=None, name=None, primary_ip=None, secondary_ip=None, master_candidate=True, offline=False, drained=False, group=None, master_capable=True, vm_capable=True, ndparams=None, powered=True, hv_state=None, hv_state_static=None, disk_state=None, disk_state_static=None): """Add a new L{objects.Node} to the cluster configuration See L{objects.Node} for parameter documentation. @rtype: L{objects.Node} @return: the newly added node """ node_id = self._cur_node_id self._cur_node_id += 1 if uuid is None: uuid = self._GetUuid() if name is None: name = "mock_node_%d.example.com" % node_id if primary_ip is None: primary_ip = "192.0.2.%d" % node_id if secondary_ip is None: secondary_ip = "203.0.113.%d" % node_id if group is None: group = self._default_group.uuid group = self._GetObjUuid(group) if ndparams is None: ndparams = {} node = objects.Node(uuid=uuid, name=name, primary_ip=primary_ip, secondary_ip=secondary_ip, master_candidate=master_candidate, offline=offline, drained=drained, group=group, master_capable=master_capable, vm_capable=vm_capable, ndparams=ndparams, powered=powered, hv_state=hv_state, hv_state_static=hv_state_static, disk_state=disk_state, disk_state_static=disk_state_static) self._UnlockedAddNode(node, None) return node def AddNewInstance(self, uuid=None, name=None, primary_node=None, os=None, hypervisor=None, hvparams=None, beparams=None, osparams=None, osparams_private=None, admin_state=None, admin_state_source=None, nics=None, disks=None, disk_template=None, disks_active=None, network_port=None, secondary_node=None): """Add a new L{objects.Instance} to the cluster configuration See L{objects.Instance} for parameter documentation. @rtype: L{objects.Instance} @return: the newly added instance """ inst_id = self._cur_inst_id self._cur_inst_id += 1 if uuid is None: uuid = self._GetUuid() if name is None: name = "mock_inst_%d.example.com" % inst_id if primary_node is None: primary_node = self._master_node.uuid primary_node = self._GetObjUuid(primary_node) if os is None: os = self.GetDefaultOs().name + objects.OS.VARIANT_DELIM +\ self.GetDefaultOs().supported_variants[0] if hypervisor is None: hypervisor = self.GetClusterInfo().enabled_hypervisors[0] if hvparams is None: hvparams = {} if beparams is None: beparams = {} if osparams is None: osparams = {} if osparams_private is None: osparams_private = {} if admin_state is None: admin_state = constants.ADMINST_DOWN if admin_state_source is None: admin_state_source = constants.ADMIN_SOURCE if nics is None: nics = [self.CreateNic()] if disk_template is None: if disks is None: # user chose nothing, so create a plain disk for him disk_template = constants.DT_PLAIN elif len(disks) == 0: disk_template = constants.DT_DISKLESS else: disk_template = disks[0].dev_type if disks is None: if disk_template == constants.DT_DISKLESS: disks = [] elif disk_template == constants.DT_EXT: provider = "mock_provider" disks = [self.CreateDisk(dev_type=disk_template, primary_node=primary_node, secondary_node=secondary_node, params={constants.IDISK_PROVIDER: provider})] else: disks = [self.CreateDisk(dev_type=disk_template, primary_node=primary_node, secondary_node=secondary_node)] if disks_active is None: disks_active = admin_state == constants.ADMINST_UP inst = objects.Instance(uuid=uuid, name=name, primary_node=primary_node, os=os, hypervisor=hypervisor, hvparams=hvparams, beparams=beparams, osparams=osparams, osparams_private=osparams_private, admin_state=admin_state, admin_state_source=admin_state_source, nics=nics, disks=[], disks_active=disks_active, network_port=network_port) self.AddInstance(inst, None) for disk in disks: self.AddInstanceDisk(inst.uuid, disk) return inst def AddNewNetwork(self, uuid=None, name=None, mac_prefix=None, network=None, network6=None, gateway=None, gateway6=None, reservations=None, ext_reservations=None): """Add a new L{objects.Network} to the cluster configuration See L{objects.Network} for parameter documentation. @rtype: L{objects.Network} @return: the newly added network """ net_id = self._cur_net_id self._cur_net_id += 1 if uuid is None: uuid = self._GetUuid() if name is None: name = "mock_net_%d" % net_id if network is None: network = "198.51.100.0/24" if gateway is None: if network[-3:] == "/24": gateway = network[:-4] + "1" else: gateway = "198.51.100.1" if network[-3:] == "/24" and gateway == network[:-4] + "1": if reservations is None: reservations = "0" * 256 if ext_reservations: ext_reservations = "11" + ("0" * 253) + "1" elif reservations is None or ext_reservations is None: raise AssertionError("You have to specify 'reservations' and" " 'ext_reservations'!") net = objects.Network(uuid=uuid, name=name, mac_prefix=mac_prefix, network=network, network6=network6, gateway=gateway, gateway6=gateway6, reservations=reservations, ext_reservations=ext_reservations) self.AddNetwork(net, None) return net def AddOrphanDisk(self, **params): disk = self.CreateDisk(**params) self._UnlockedAddDisk(disk) def ConnectNetworkToGroup(self, net, group, netparams=None): """Connect the given network to the group. @type net: string or L{objects.Network} @param net: network object or UUID @type group: string of L{objects.NodeGroup} @param group: node group object of UUID @type netparams: dict @param netparams: network parameters for this connection """ net_obj = None if isinstance(net, objects.Network): net_obj = net else: net_obj = self.GetNetwork(net) group_obj = None if isinstance(group, objects.NodeGroup): group_obj = group else: group_obj = self.GetNodeGroup(group) if net_obj is None or group_obj is None: raise AssertionError("Failed to get network or node group") if netparams is None: netparams = { constants.NIC_MODE: constants.NIC_MODE_BRIDGED, constants.NIC_LINK: "br_mock" } group_obj.networks[net_obj.uuid] = netparams def CreateDisk(self, uuid=None, name=None, dev_type=constants.DT_PLAIN, logical_id=None, children=None, nodes=None, iv_name=None, size=1024, mode=constants.DISK_RDWR, params=None, spindles=None, primary_node=None, secondary_node=None, create_nodes=False, instance_disk_index=0): """Create a new L{objecs.Disk} object @rtype: L{objects.Disk} @return: the newly create disk object """ disk_id = self._cur_disk_id self._cur_disk_id += 1 if uuid is None: uuid = self._GetUuid() if name is None: name = "mock_disk_%d" % disk_id if params is None: params = {} if dev_type == constants.DT_DRBD8: pnode_uuid = self._GetObjUuid(primary_node) snode_uuid = self._GetObjUuid(secondary_node) if logical_id is not None: pnode_uuid = logical_id[0] snode_uuid = logical_id[1] if pnode_uuid is None and create_nodes: pnode_uuid = self.AddNewNode().uuid if snode_uuid is None and create_nodes: snode_uuid = self.AddNewNode().uuid if pnode_uuid is None or snode_uuid is None: raise AssertionError("Trying to create DRBD disk without nodes!") if logical_id is None: logical_id = (pnode_uuid, snode_uuid, constants.FIRST_DRBD_PORT + disk_id, disk_id, disk_id, "mock_secret") if children is None: data_child = self.CreateDisk(dev_type=constants.DT_PLAIN, size=size) meta_child = self.CreateDisk(dev_type=constants.DT_PLAIN, size=constants.DRBD_META_SIZE) children = [data_child, meta_child] if nodes is None: nodes = [pnode_uuid, snode_uuid] elif dev_type == constants.DT_PLAIN: if logical_id is None: logical_id = ("mockvg", "mock_disk_%d" % disk_id) if nodes is None and primary_node is not None: nodes = [primary_node] elif dev_type in constants.DTS_FILEBASED: if logical_id is None: logical_id = (constants.FD_LOOP, "/file/storage/disk%d" % disk_id) if (nodes is None and primary_node is not None and dev_type == constants.DT_FILE): nodes = [primary_node] elif dev_type == constants.DT_BLOCK: if logical_id is None: logical_id = (constants.BLOCKDEV_DRIVER_MANUAL, "/dev/disk/disk%d" % disk_id) elif dev_type == constants.DT_EXT: if logical_id is None: provider = params.get(constants.IDISK_PROVIDER, None) if provider is None: raise AssertionError("You must specify a 'provider' for 'ext' disks") logical_id = (provider, "mock_disk_%d" % disk_id) elif logical_id is None: raise NotImplementedError if children is None: children = [] if nodes is None: nodes = [] if iv_name is None: iv_name = "disk/%d" % instance_disk_index return objects.Disk(uuid=uuid, name=name, dev_type=dev_type, logical_id=logical_id, children=children, nodes=nodes, iv_name=iv_name, size=size, mode=mode, params=params, spindles=spindles) def GetDefaultOs(self): if self._default_os is None: self._default_os = self.CreateOs(name="mocked_os") return self._default_os def CreateOs(self, name=None, path=None, api_versions=None, create_script=None, export_script=None, import_script=None, rename_script=None, verify_script=None, supported_variants=None, supported_parameters=None): """Create a new L{objects.OS} object @rtype: L{object.OS} @return: the newly create OS objects """ os_id = self._cur_os_id self._cur_os_id += 1 if name is None: name = "mock_os_%d" % os_id if path is None: path = "/mocked/path/%d" % os_id if api_versions is None: api_versions = [constants.OS_API_V20] if create_script is None: create_script = "mock_create.sh" if export_script is None: export_script = "mock_export.sh" if import_script is None: import_script = "mock_import.sh" if rename_script is None: rename_script = "mock_rename.sh" if verify_script is None: verify_script = "mock_verify.sh" if supported_variants is None: supported_variants = ["default"] if supported_parameters is None: supported_parameters = ["mock_param"] return objects.OS(name=name, path=path, api_versions=api_versions, create_script=create_script, export_script=export_script, import_script=import_script, rename_script=rename_script, verify_script=verify_script, supported_variants=supported_variants, supported_parameters=supported_parameters) def CreateNic(self, uuid=None, name=None, mac=None, ip=None, network=None, nicparams=None, netinfo=None): """Create a new L{objecs.NIC} object @rtype: L{objects.NIC} @return: the newly create NIC object """ nic_id = self._cur_nic_id self._cur_nic_id += 1 if uuid is None: uuid = self._GetUuid() if name is None: name = "mock_nic_%d" % nic_id if mac is None: mac = "aa:00:00:aa:%02x:%02x" % (nic_id // 0xff, nic_id % 0xff) if isinstance(network, objects.Network): if ip: pool = AddressPool(network) pool.Reserve(ip) network = network.uuid if nicparams is None: nicparams = {} return objects.NIC(uuid=uuid, name=name, mac=mac, ip=ip, network=network, nicparams=nicparams, netinfo=netinfo) def SetEnabledDiskTemplates(self, enabled_disk_templates): """Set the enabled disk templates in the cluster. This also takes care of required IPolicy updates. @type enabled_disk_templates: list of string @param enabled_disk_templates: list of disk templates to enable """ cluster = self.GetClusterInfo() cluster.enabled_disk_templates = list(enabled_disk_templates) cluster.ipolicy[constants.IPOLICY_DTS] = list(enabled_disk_templates) def ComputeDRBDMap(self): return dict((node_uuid, {}) for node_uuid in self._ConfigData().nodes) def AllocateDRBDMinor(self, node_uuids, disk_uuid): return [0] * len(node_uuids) def ReleaseDRBDMinors(self, disk_uuid): pass def SetIPolicyField(self, category, field, value): """Set a value of a desired ipolicy field. @type category: one of L{constants.ISPECS_MAX}, L{constants.ISPECS_MIN}, L{constants.ISPECS_STD} @param category: Whether to change the default value, or the upper or lower bound. @type field: string @param field: The field to change. @type value: any @param value: The value to assign. """ if category not in [constants.ISPECS_MAX, constants.ISPECS_MIN, constants.ISPECS_STD]: raise ValueError("Invalid ipolicy category %s" % category) ipolicy_dict = self.GetClusterInfo().ipolicy[constants.ISPECS_MINMAX][0] ipolicy_dict[category][field] = value def _CreateConfig(self): self._config_data = objects.ConfigData( version=constants.CONFIG_VERSION, cluster=None, nodegroups={}, nodes={}, instances={}, networks={}, disks={}) master_node_uuid = self._GetUuid() self._cluster = objects.Cluster( serial_no=1, rsahostkeypub="", highest_used_port=(constants.FIRST_DRBD_PORT - 1), tcpudp_port_pool=set(), mac_prefix="aa:00:00", volume_group_name="xenvg", reserved_lvs=None, drbd_usermode_helper="/bin/true", master_node=master_node_uuid, master_ip="192.0.2.254", master_netdev=constants.DEFAULT_BRIDGE, master_netmask=None, use_external_mip_script=None, cluster_name="cluster.example.com", file_storage_dir="/tmp", shared_file_storage_dir=None, enabled_hypervisors=[constants.HT_XEN_HVM, constants.HT_XEN_PVM, constants.HT_KVM], hvparams=constants.HVC_DEFAULTS.copy(), ipolicy=None, os_hvp={self.GetDefaultOs().name: constants.HVC_DEFAULTS.copy()}, beparams=None, osparams=None, osparams_private_cluster=None, nicparams={constants.PP_DEFAULT: constants.NICC_DEFAULTS}, ndparams=None, diskparams=None, candidate_pool_size=3, modify_etc_hosts=False, modify_ssh_setup=False, maintain_node_health=False, uid_pool=None, default_iallocator="mock_iallocator", hidden_os=None, blacklisted_os=None, primary_ip_family=None, prealloc_wipe_disks=None, enabled_disk_templates=list(constants.DISK_TEMPLATE_PREFERENCE), ) self._cluster.ctime = self._cluster.mtime = time.time() self._cluster.UpgradeConfig() self._ConfigData().cluster = self._cluster self._default_group = self.AddNewNodeGroup(name="default") self._master_node = self.AddNewNode(uuid=master_node_uuid) def _OpenConfig(self, _accept_foreign, force=False): self._config_data = self._mocked_config_store def _WriteConfig(self, destination=None, releaselock=False): self._mocked_config_store = self._ConfigData() def _GetRpc(self, _address_list): raise AssertionError("This should not be used during tests!") def _UnlockedGetNetworkMACPrefix(self, net_uuid): """Return the network mac prefix if it exists or the cluster level default. """ prefix = None if net_uuid: nobj = self._UnlockedGetNetwork(net_uuid) if nobj.mac_prefix: prefix = nobj.mac_prefix return prefix def _GenerateOneMAC(self, prefix=None): """Return a function that randomly generates a MAC suffic and appends it to the given prefix. If prefix is not given get the cluster level default. """ if not prefix: prefix = self._ConfigData().cluster.mac_prefix def GenMac(): byte1 = random.randrange(0, 256) byte2 = random.randrange(0, 256) byte3 = random.randrange(0, 256) mac = "%s:%02x:%02x:%02x" % (prefix, byte1, byte2, byte3) return mac return GenMac def GenerateMAC(self, net_uuid, ec_id): """Generate a MAC for an instance. This should check the current instances for duplicates. """ existing = self._AllMACs() prefix = self._UnlockedGetNetworkMACPrefix(net_uuid) gen_mac = self._GenerateOneMAC(prefix) return self._temporary_macs.Generate(existing, gen_mac, ec_id) def ReserveMAC(self, mac, ec_id): """Reserve a MAC for an instance. This only checks instances managed by this cluster, it does not check for potential collisions elsewhere. """ all_macs = self._AllMACs() if mac in all_macs: raise errors.ReservationError("mac already in use") else: self._temporary_macs.Reserve(ec_id, mac) def GenerateDRBDSecret(self, ec_id): """Generate a DRBD secret. This checks the current disks for duplicates. """ return self._temporary_secrets.Generate(self._AllDRBDSecrets(), utils.GenerateSecret, ec_id) def ReserveLV(self, lv_name, ec_id): """Reserve an VG/LV pair for an instance. @type lv_name: string @param lv_name: the logical volume name to reserve """ all_lvs = self._AllLVs() if lv_name in all_lvs: raise errors.ReservationError("LV already in use") else: self._temporary_lvs.Reserve(ec_id, lv_name) def _UnlockedCommitTemporaryIps(self, ec_id): """Commit all reserved IP address to their respective pools """ for action, address, net_uuid in self._temporary_ips.GetECReserved(ec_id): self._UnlockedCommitIp(action, net_uuid, address) def _UnlockedCommitIp(self, action, net_uuid, address): """Commit a reserved IP address to an IP pool. The IP address is taken from the network's IP pool and marked as reserved. """ nobj = self._UnlockedGetNetwork(net_uuid) pool = AddressPool(nobj) if action == RESERVE_ACTION: pool.Reserve(address) elif action == RELEASE_ACTION: pool.Release(address) def _UnlockedReleaseIp(self, net_uuid, address, ec_id): """Give a specific IP address back to an IP pool. The IP address is returned to the IP pool designated by pool_id and marked as reserved. """ self._temporary_ips.Reserve(ec_id, (RELEASE_ACTION, address, net_uuid)) def ReleaseIp(self, net_uuid, address, ec_id): """Give a specified IP address back to an IP pool. This is just a wrapper around _UnlockedReleaseIp. """ if net_uuid: self._UnlockedReleaseIp(net_uuid, address, ec_id) def GenerateIp(self, net_uuid, ec_id): """Find a free IPv4 address for an instance. """ nobj = self._UnlockedGetNetwork(net_uuid) pool = AddressPool(nobj) def gen_one(): try: ip = pool.GenerateFree() except errors.AddressPoolError: raise errors.ReservationError("Cannot generate IP. Network is full") return (RESERVE_ACTION, ip, net_uuid) _, address, _ = self._temporary_ips.Generate([], gen_one, ec_id) return address def _UnlockedReserveIp(self, net_uuid, address, ec_id, check=True): """Reserve a given IPv4 address for use by an instance. """ nobj = self._UnlockedGetNetwork(net_uuid) pool = AddressPool(nobj) try: isreserved = pool.IsReserved(address) isextreserved = pool.IsReserved(address, external=True) except errors.AddressPoolError: raise errors.ReservationError("IP address not in network") if isreserved: raise errors.ReservationError("IP address already in use") if check and isextreserved: raise errors.ReservationError("IP is externally reserved") return self._temporary_ips.Reserve(ec_id, (RESERVE_ACTION, address, net_uuid)) def ReserveIp(self, net_uuid, address, ec_id, check=True): """Reserve a given IPv4 address for use by an instance. """ if net_uuid: return self._UnlockedReserveIp(net_uuid, address, ec_id, check) def AddInstance(self, instance, ec_id, replace=False): """Add an instance to the config. """ instance.serial_no = 1 instance.ctime = instance.mtime = time.time() self._ConfigData().instances[instance.uuid] = instance self._ConfigData().cluster.serial_no += 1 # pylint: disable=E1103 self.ReleaseDRBDMinors(instance.uuid) self._UnlockedCommitTemporaryIps(ec_id) def _UnlockedAddDisk(self, disk): disk.UpgradeConfig() self._ConfigData().disks[disk.uuid] = disk self._ConfigData().cluster.serial_no += 1 # pylint: disable=E1103 self.ReleaseDRBDMinors(disk.uuid) def _UnlockedAttachInstanceDisk(self, inst_uuid, disk_uuid, idx=None): instance = self._UnlockedGetInstanceInfo(inst_uuid) if idx is None: idx = len(instance.disks) instance.disks.insert(idx, disk_uuid) instance_disks = self._UnlockedGetInstanceDisks(inst_uuid) for (disk_idx, disk) in enumerate(instance_disks[idx:]): disk.iv_name = "disk/%s" % (idx + disk_idx) instance.serial_no += 1 instance.mtime = time.time() def AddInstanceDisk(self, inst_uuid, disk, idx=None, replace=False): self._UnlockedAddDisk(disk) self._UnlockedAttachInstanceDisk(inst_uuid, disk.uuid, idx) def AttachInstanceDisk(self, inst_uuid, disk_uuid, idx=None): self._UnlockedAttachInstanceDisk(inst_uuid, disk_uuid, idx) def GetDisk(self, disk_uuid): """Retrieves a disk object if present. """ return self._ConfigData().disks[disk_uuid] def AllocatePort(self): return 1 def Update(self, target, feedback_fn, ec_id=None): def replace_in(target, tdict): tdict[target.uuid] = target update_serial = False if isinstance(target, objects.Cluster): self._ConfigData().cluster = target elif isinstance(target, objects.Node): replace_in(target, self._ConfigData().nodes) update_serial = True elif isinstance(target, objects.Instance): replace_in(target, self._ConfigData().instances) elif isinstance(target, objects.NodeGroup): replace_in(target, self._ConfigData().nodegroups) elif isinstance(target, objects.Network): replace_in(target, self._ConfigData().networks) elif isinstance(target, objects.Disk): replace_in(target, self._ConfigData().disks) target.serial_no += 1 target.mtime = now = time.time() if update_serial: self._ConfigData().cluster.serial_no += 1 # pylint: disable=E1103 self._ConfigData().cluster.mtime = now def SetInstancePrimaryNode(self, inst_uuid, target_node_uuid): self._UnlockedGetInstanceInfo(inst_uuid).primary_node = target_node_uuid def _SetInstanceStatus(self, inst_uuid, status, disks_active, admin_state_source): if inst_uuid not in self._ConfigData().instances: raise errors.ConfigurationError("Unknown instance '%s'" % inst_uuid) instance = self._ConfigData().instances[inst_uuid] if status is None: status = instance.admin_state if disks_active is None: disks_active = instance.disks_active if admin_state_source is None: admin_state_source = instance.admin_state_source assert status in constants.ADMINST_ALL, \ "Invalid status '%s' passed to SetInstanceStatus" % (status,) if instance.admin_state != status or \ instance.disks_active != disks_active or \ instance.admin_state_source != admin_state_source: instance.admin_state = status instance.disks_active = disks_active instance.admin_state_source = admin_state_source instance.serial_no += 1 instance.mtime = time.time() return instance def _UnlockedDetachInstanceDisk(self, inst_uuid, disk_uuid): """Detach a disk from an instance. @type inst_uuid: string @param inst_uuid: The UUID of the instance object @type disk_uuid: string @param disk_uuid: The UUID of the disk object """ instance = self._UnlockedGetInstanceInfo(inst_uuid) if instance is None: raise errors.ConfigurationError("Instance %s doesn't exist" % inst_uuid) if disk_uuid not in self._ConfigData().disks: raise errors.ConfigurationError("Disk %s doesn't exist" % disk_uuid) # Check if disk is attached to the instance if disk_uuid not in instance.disks: raise errors.ProgrammerError("Disk %s is not attached to an instance" % disk_uuid) idx = instance.disks.index(disk_uuid) instance.disks.remove(disk_uuid) instance_disks = self._UnlockedGetInstanceDisks(inst_uuid) _UpdateIvNames(idx, instance_disks[idx:]) instance.serial_no += 1 instance.mtime = time.time() def DetachInstanceDisk(self, inst_uuid, disk_uuid): self._UnlockedDetachInstanceDisk(inst_uuid, disk_uuid) def RemoveInstanceDisk(self, inst_uuid, disk_uuid): self._UnlockedDetachInstanceDisk(inst_uuid, disk_uuid) self._UnlockedRemoveDisk(disk_uuid) def RemoveInstance(self, inst_uuid): del self._ConfigData().instances[inst_uuid] def AddTcpUdpPort(self, port): self._ConfigData().cluster.tcpudp_port_pool.add(port) class ConfigObjectMatcher(object): """Wraps a ConfigObject to provide __eq__ """ def __init__(self, obj): if not isinstance(obj, objects.ConfigObject): raise TypeError self.obj = obj def __eq__(self, other): if not isinstance(other, objects.ConfigObject): return False return self.obj.ToDict() == other.ToDict() ganeti-3.1.0~rc2/test/py/legacy/testutils_ssh.py000064400000000000000000000642121476477700300217530ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2013, 2015 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Helper class to test ssh-related code.""" from ganeti import constants from ganeti import pathutils from ganeti import errors from collections import namedtuple class FakeSshFileManager(object): """Class which 'fakes' the lowest layer of SSH key manipulation. There are various operations which touch the nodes' SSH keys and their respective key files (authorized_keys and ganeti_pub_keys). Those are tedious to test as file operations have to be mocked on different levels (direct access to the authorized_keys and ganeti_pub_keys) of the master node, indirect access to those files of the non-master nodes (via the ssh_update tool). In order to make unit tests of those operations more readable and managable, we introduce this class, which mocks all direct and indirect access to SSH key files on all nodes. This way, the state of this FakeSshFileManager represents the state of a cluster's nodes' SSH key files in a consise and easily accessible way. """ def __init__(self): # Dictionary mapping node name to node properties. The properties # are a named tuple of (node_uuid, ssh_key, is_potential_master_candidate, # is_master_candidate, is_master). self._all_node_data = {} # Dictionary emulating the authorized keys files of all nodes. The # indices of the dictionary are the node names, the values are sets # of keys (strings). self._authorized_keys = {} # Dictionary emulating the public keys file of all nodes. The indices # of the dictionary are the node names where the public key file is # 'located' (if it wasn't faked). The values of the dictionary are # dictionaries itself. Each of those dictionaries is indexed by the # node UUIDs mapping to a list of public keys. self._public_keys = {} # dict of dicts # Node name of the master node self._master_node_name = None # Dictionary mapping nodes by name to number of retries where 'RunCommand' # succeeds. For example if set to '3', RunCommand will fail two times when # called for this node before it succeeds in the 3rd retry. self._max_retries = {} # Dictionary mapping nodes by name to number of retries which # 'RunCommand' has already carried out. self._retries = {} self._AssertTypePublicKeys() self._AssertTypeAuthorizedKeys() _NodeInfo = namedtuple( "NodeInfo", ["uuid", "key", "is_potential_master_candidate", "is_master_candidate", "is_master"]) def _SetMasterNodeName(self): self._master_node_name = [name for name, node_info in self._all_node_data.items() if node_info.is_master][0] def GetMasterNodeName(self): return self._master_node_name def _CreateNodeDict(self, num_nodes, num_pot_mcs, num_mcs): """Creates a dictionary of all nodes and their properties.""" self._all_node_data = {} for i in range(num_nodes): name = "node_name_%i" % i uuid = "node_uuid_%i" % i key = "key%s" % i self._public_keys[name] = {} self._authorized_keys[name] = set() pot_mc = i < num_pot_mcs mc = i < num_mcs master = i == num_mcs // 2 self._all_node_data[name] = self._NodeInfo(uuid, key, pot_mc, mc, master) self._AssertTypePublicKeys() self._AssertTypeAuthorizedKeys() def _FillPublicKeyOfOneNode(self, receiving_node_name): node_info = self._all_node_data[receiving_node_name] # Nodes which are not potential master candidates receive no keys if not node_info.is_potential_master_candidate: return for node_info in self._all_node_data.values(): if node_info.is_potential_master_candidate: self._public_keys[receiving_node_name][node_info.uuid] = [node_info.key] def _FillAuthorizedKeyOfOneNode(self, receiving_node_name): for node_name, node_info in self._all_node_data.items(): if node_info.is_master_candidate \ or node_name == receiving_node_name: self._authorized_keys[receiving_node_name].add(node_info.key) def InitAllNodes(self, num_nodes, num_pot_mcs, num_mcs): """Initializes the entire state of the cluster wrt SSH keys. @type num_nodes: int @param num_nodes: number of nodes in the cluster @type num_pot_mcs: int @param num_pot_mcs: number of potential master candidates in the cluster @type num_mcs: in @param num_mcs: number of master candidates in the cluster. """ self._public_keys = {} self._authorized_keys = {} self._CreateNodeDict(num_nodes, num_pot_mcs, num_mcs) for node in self._all_node_data.keys(): self._FillPublicKeyOfOneNode(node) self._FillAuthorizedKeyOfOneNode(node) self._SetMasterNodeName() self._AssertTypePublicKeys() self._AssertTypeAuthorizedKeys() def SetMaxRetries(self, node_name, retries): """Set the number of unsuccessful retries of 'RunCommand' per node. @type node_name: string @param node_name: name of the node @type retries: integer @param retries: number of unsuccessful retries """ self._max_retries[node_name] = retries def GetSshPortMap(self, port): """Creates a SSH port map with all nodes mapped to the given port. @type port: int @param port: SSH port number for all nodes """ port_map = {} for node in self._all_node_data: port_map[node] = port return port_map def GetAllNodeNames(self): """Returns all node names of the cluster. @rtype: list of str @returns: list of all node names """ return list(self._all_node_data) def GetAllPotentialMasterCandidateNodeNames(self): return [name for name, node_info in self._all_node_data.items() if node_info.is_potential_master_candidate] def GetAllMasterCandidateUuids(self): return [node_info.uuid for node_info in self._all_node_data.values() if node_info.is_master_candidate] def GetAllPurePotentialMasterCandidates(self): """Get the potential master candidates which are not master candidates. @rtype: list of tuples (string, C{_NodeInfo}) @returns: list of tuples of node name and node information of nodes which are potential master candidates but not master candidates """ return [(name, node_info) for name, node_info in self._all_node_data.items() if node_info.is_potential_master_candidate and not node_info.is_master_candidate] def GetAllMasterCandidates(self): """Get all master candidate nodes. @rtype: list of tuples (string, C{_NodeInfo}) @returns: list of tuples of node name and node information of master candidate nodes. """ # Sort MCs so that the master node comes last # This is implicitly relied upon by the backend unittests # TODO: refine the test infrastructure so that this is not necessary keyfunc = lambda n_ni: n_ni[1].is_master return [(name, node_info) for name, node_info in sorted(self._all_node_data.items(), key=keyfunc) if node_info.is_master_candidate] def GetAllNormalNodes(self): """Get all normal nodes. Normal nodes are nodes that are neither master, master candidate nor potential master candidate. @rtype: list of tuples (string, C{_NodeInfo}) @returns: list of tuples of node name and node information of normal nodes """ return [(name, node_info) for name, node_info in self._all_node_data.items() if not node_info.is_master_candidate and not node_info.is_potential_master_candidate] def GetAllNodesDiverse(self): """This returns all nodes in a diverse order. This will return all nodes, but makes sure that they are ordered so that the list will contain in a round-robin fashion, a master candidate, a potential master candidate, a normal node, then again a master candidate, etc. @rtype: list of tuples (string, C{_NodeInfo}) @returns: list of tuples of node name and node information """ master_candidates = self.GetAllMasterCandidates() potential_master_candidates = self.GetAllPurePotentialMasterCandidates() normal_nodes = self.GetAllNormalNodes() mixed_list = [] i = 0 assert (len(self._all_node_data) == len(master_candidates) + len(potential_master_candidates) + len(normal_nodes)) while len(mixed_list) < len(self._all_node_data): if i % 3 == 0: if master_candidates: mixed_list.append(master_candidates[0]) master_candidates = master_candidates[1:] elif i % 3 == 1: if potential_master_candidates: mixed_list.append(potential_master_candidates[0]) potential_master_candidates = potential_master_candidates[1:] else: # i % 3 == 2 if normal_nodes: mixed_list.append(normal_nodes[0]) normal_nodes = normal_nodes[1:] i += 1 return mixed_list def GetPublicKeysOfNode(self, node): """Returns the public keys that are stored on the given node. @rtype: dict of str to list of str @returns: a mapping of node names to a list of public keys """ return self._public_keys[node] def GetAuthorizedKeysOfNode(self, node): """Returns the authorized keys of the given node. @rtype: list of str @returns: a list of authorized keys that are stored on that node """ return self._authorized_keys[node] def SetOrAddNode(self, name, uuid, key, pot_mc, mc, master): """Adds a new node to the state of the file manager. This is necessary when testing to add new nodes to the cluster. Otherwise this new node's state would not be evaluated properly with the assertion functions. @type name: string @param name: name of the new node @type uuid: string @param uuid: UUID of the new node @type key: string @param key: SSH key of the new node @type pot_mc: boolean @param pot_mc: whether the new node is a potential master candidate @type mc: boolean @param mc: whether the new node is a master candidate @type master: boolean @param master: whether the new node is the master """ self._all_node_data[name] = self._NodeInfo(uuid, key, pot_mc, mc, master) if name not in self._authorized_keys: self._authorized_keys[name] = set() if mc: self._authorized_keys[name].add(key) if name not in self._public_keys: self._public_keys[name] = {} self._AssertTypePublicKeys() self._AssertTypeAuthorizedKeys() def NodeHasPublicKey(self, file_node_name, key_node_uuid, key): """Checks whether a node has another node's public key. @type file_node_name: string @param file_node_name: name of the node whose public key file is inspected @type key_node_uuid: string @param key_node_uuid: UUID of the node whose key is checked for @rtype: boolean @return: True if the key_node's UUID is found with the machting key 'key' """ for (node_uuid, pub_keys) in self._public_keys[file_node_name].items(): if key in pub_keys and key_node_uuid == node_uuid: return True return False def NodeHasAuthorizedKey(self, file_node_name, key): """Checks whether a node has a particular key in its authorized_keys file. @type file_node_name: string @param file_node_name: name of the node whose authorized_key file is inspected @type key: string @param key: key which is expected to be found in the node's authorized_key file @rtype: boolean @return: True if the key is found in the node's authorized_key file """ return key in self._authorized_keys[file_node_name] def AssertNodeSetOnlyHasAuthorizedKey(self, node_set, query_node_key): """Check if nodes in the given set only have a particular authorized key. @type node_set: list of strings @param node_set: list of nodes who are supposed to have the key @type query_node_key: string @param query_node_key: key which is looked for """ assert isinstance(node_set, list) for node_name in self._all_node_data: if node_name in node_set: if not self.NodeHasAuthorizedKey(node_name, query_node_key): raise Exception("Node '%s' does not have authorized key '%s'." % (node_name, query_node_key)) else: if self.NodeHasAuthorizedKey(node_name, query_node_key): raise Exception("Node '%s' has authorized key '%s' although it" " should not." % (node_name, query_node_key)) def AssertAllNodesHaveAuthorizedKey(self, key): """Check if all nodes have a particular key in their auth. keys file. @type key: string @param key: key exptected to be present in all node's authorized_keys file @raise Exception: if a node does not have the authorized key. """ self.AssertNodeSetOnlyHasAuthorizedKey(list(self._all_node_data), key) def AssertNoNodeHasAuthorizedKey(self, key): """Check if none of the nodes has a particular key in their auth. keys file. @type key: string @param key: key exptected to be present in all node's authorized_keys file @raise Exception: if a node *does* have the authorized key. """ self.AssertNodeSetOnlyHasAuthorizedKey([], key) def AssertNodeSetOnlyHasPublicKey(self, node_set, query_node_uuid, query_node_key): """Check if nodes in the given set only have a particular public key. @type node_set: list of strings @param node_set: list of nodes who are supposed to have the key @type query_node_uuid: string @param query_node_uuid: uuid of the node whose key is looked for @type query_node_key: string @param query_node_key: key which is looked for """ for node_name in self._all_node_data: if node_name in node_set: if not self.NodeHasPublicKey(node_name, query_node_uuid, query_node_key): raise Exception("Node '%s' does not have public key '%s' of node" " '%s'." % (node_name, query_node_key, query_node_uuid)) else: if self.NodeHasPublicKey(node_name, query_node_uuid, query_node_key): raise Exception("Node '%s' has public key '%s' of node" " '%s' although it should not." % (node_name, query_node_key, query_node_uuid)) def AssertNoNodeHasPublicKey(self, uuid, key): """Check if none of the nodes have the given public key in their file. @type uuid: string @param uuid: UUID of the node whose key is looked for @raise Exception: if a node *does* have the public key. """ self.AssertNodeSetOnlyHasPublicKey([], uuid, key) def AssertPotentialMasterCandidatesOnlyHavePublicKey(self, query_node_name): """Checks if the node's key is on all potential master candidates only. This ensures that the node's key is in all public key files of all potential master candidates, and it also checks whether the key is *not* in all other nodes's key files. @param query_node_name: name of the node whose key is expected to be in the public key file of all potential master candidates @type query_node_name: string @raise Exception: when a potential master candidate does not have the public key or a normal node *does* have a public key. """ query_node_uuid, query_node_key, _, _, _ = \ self._all_node_data[query_node_name] potential_master_candidates = self.GetAllPotentialMasterCandidateNodeNames() self.AssertNodeSetOnlyHasPublicKey( potential_master_candidates, query_node_uuid, query_node_key) def _AssertTypePublicKeys(self): """Asserts that the public key dictionary has the right types. This is helpful as an invariant that shall not be violated during the tests due to type errors. """ assert isinstance(self._public_keys, dict) for node_file, pub_keys in self._public_keys.items(): assert isinstance(node_file, str) assert isinstance(pub_keys, dict) for node_key, keys in pub_keys.items(): assert isinstance(node_key, str) assert isinstance(keys, list) for key in keys: assert isinstance(key, str) def _AssertTypeAuthorizedKeys(self): """Asserts that the authorized keys dictionary has the right types. This is useful to check as an invariant that is not supposed to be violated during the tests. """ assert isinstance(self._authorized_keys, dict) for node_file, auth_keys in self._authorized_keys.items(): assert isinstance(node_file, str) assert isinstance(auth_keys, set) for key in auth_keys: assert isinstance(key, str) # Disabling a pylint warning about unused parameters. Those need # to be here to properly mock the real methods. # pylint: disable=W0613 def RunCommand(self, cluster_name, node, base_cmd, port, data, debug=False, verbose=False, use_cluster_key=False, ask_key=False, strict_host_check=False, ensure_version=False): """This emulates ssh.RunSshCmdWithStdin calling ssh_update. While in real SSH operations, ssh.RunSshCmdWithStdin is called with the command ssh_update to manipulate a remote node's SSH key files (authorized_keys and ganeti_pub_key) file, this method emulates the operation by manipulating only its internal dictionaries of SSH keys. No actual key files of any node is touched. """ if node in self._max_retries: if node not in self._retries: self._retries[node] = 0 self._retries[node] += 1 if self._retries[node] < self._max_retries[node]: raise errors.OpExecError("(Fake) SSH connection to node '%s' failed." % node) assert base_cmd == pathutils.SSH_UPDATE if constants.SSHS_SSH_AUTHORIZED_KEYS in data: instructions_auth = data[constants.SSHS_SSH_AUTHORIZED_KEYS] self._HandleAuthorizedKeys(instructions_auth, node) if constants.SSHS_SSH_PUBLIC_KEYS in data: instructions_pub = data[constants.SSHS_SSH_PUBLIC_KEYS] self._HandlePublicKeys(instructions_pub, node) # pylint: enable=W0613 def _EnsureAuthKeyFile(self, file_node_name): if file_node_name not in self._authorized_keys: self._authorized_keys[file_node_name] = set() self._AssertTypePublicKeys() self._AssertTypeAuthorizedKeys() def _AddAuthorizedKeys(self, file_node_name, ssh_keys): """Mocks adding the given keys to the authorized_keys file.""" assert isinstance(ssh_keys, list) self._EnsureAuthKeyFile(file_node_name) for key in ssh_keys: self._authorized_keys[file_node_name].add(key) self._AssertTypePublicKeys() self._AssertTypeAuthorizedKeys() def _RemoveAuthorizedKeys(self, file_node_name, keys): """Mocks removing the keys from authorized_keys on the given node. @param keys: list of ssh keys @type keys: list of strings """ self._EnsureAuthKeyFile(file_node_name) self._authorized_keys[file_node_name] = \ set([k for k in self._authorized_keys[file_node_name] if k not in keys]) self._AssertTypeAuthorizedKeys() def _HandleAuthorizedKeys(self, instructions, node): (action, authorized_keys) = instructions ssh_key_sets = list(authorized_keys.values()) if action == constants.SSHS_ADD: for ssh_keys in ssh_key_sets: self._AddAuthorizedKeys(node, ssh_keys) elif action == constants.SSHS_REMOVE: for ssh_keys in ssh_key_sets: self._RemoveAuthorizedKeys(node, ssh_keys) else: raise Exception("Unsupported action: %s" % action) self._AssertTypeAuthorizedKeys() def _EnsurePublicKeyFile(self, file_node_name): if file_node_name not in self._public_keys: self._public_keys[file_node_name] = {} self._AssertTypePublicKeys() def _ClearPublicKeys(self, file_node_name): self._public_keys[file_node_name] = {} self._AssertTypePublicKeys() def _OverridePublicKeys(self, ssh_keys, file_node_name): assert isinstance(ssh_keys, dict) self._ClearPublicKeys(file_node_name) for key_node_uuid, node_keys in ssh_keys.items(): assert isinstance(node_keys, list) if key_node_uuid in self._public_keys[file_node_name]: raise Exception("Duplicate node in ssh_update data.") self._public_keys[file_node_name][key_node_uuid] = node_keys self._AssertTypePublicKeys() def _ReplaceOrAddPublicKeys(self, public_keys, file_node_name): assert isinstance(public_keys, dict) self._EnsurePublicKeyFile(file_node_name) for key_node_uuid, keys in public_keys.items(): assert isinstance(keys, list) self._public_keys[file_node_name][key_node_uuid] = keys self._AssertTypePublicKeys() def _RemovePublicKeys(self, public_keys, file_node_name): assert isinstance(public_keys, dict) self._EnsurePublicKeyFile(file_node_name) for key_node_uuid in public_keys: if key_node_uuid in self._public_keys[file_node_name]: self._public_keys[file_node_name][key_node_uuid] = [] self._AssertTypePublicKeys() def _HandlePublicKeys(self, instructions, node): (action, public_keys) = instructions if action == constants.SSHS_OVERRIDE: self._OverridePublicKeys(public_keys, node) elif action == constants.SSHS_ADD: self._ReplaceOrAddPublicKeys(public_keys, node) elif action == constants.SSHS_REPLACE_OR_ADD: self._ReplaceOrAddPublicKeys(public_keys, node) elif action == constants.SSHS_REMOVE: self._RemovePublicKeys(public_keys, node) elif action == constants.SSHS_CLEAR: self._ClearPublicKeys(node) else: raise Exception("Unsupported action: %s." % action) self._AssertTypePublicKeys() # pylint: disable=W0613 def AddAuthorizedKeys(self, file_obj, keys): """Emulates ssh.AddAuthorizedKeys on the master node. Instead of actually mainpulating the authorized_keys file, this method keeps the state of the file in a dictionary in memory. @see: C{ssh.AddAuthorizedKeys} """ assert isinstance(keys, list) assert self._master_node_name self._AddAuthorizedKeys(self._master_node_name, keys) self._AssertTypeAuthorizedKeys() def RemoveAuthorizedKeys(self, file_name, keys): """Emulates ssh.RemoveAuthorizeKeys on the master node. Instead of actually mainpulating the authorized_keys file, this method keeps the state of the file in a dictionary in memory. @see: C{ssh.RemoveAuthorizedKeys} """ assert isinstance(keys, list) assert self._master_node_name self._RemoveAuthorizedKeys(self._master_node_name, keys) self._AssertTypeAuthorizedKeys() def AddPublicKey(self, new_uuid, new_key, **kwargs): """Emulates ssh.AddPublicKey on the master node. Instead of actually mainpulating the authorized_keys file, this method keeps the state of the file in a dictionary in memory. @see: C{ssh.AddPublicKey} """ assert self._master_node_name assert isinstance(new_key, str) key_dict = {new_uuid: [new_key]} self._ReplaceOrAddPublicKeys(key_dict, self._master_node_name) self._AssertTypePublicKeys() def RemovePublicKey(self, target_uuid, **kwargs): """Emulates ssh.RemovePublicKey on the master node. Instead of actually mainpulating the authorized_keys file, this method keeps the state of the file in a dictionary in memory. @see: {ssh.RemovePublicKey} """ assert self._master_node_name key_dict = {target_uuid: []} self._RemovePublicKeys(key_dict, self._master_node_name) self._AssertTypePublicKeys() def QueryPubKeyFile(self, target_uuids, **kwargs): """Emulates ssh.QueryPubKeyFile on the master node. Instead of actually mainpulating the authorized_keys file, this method keeps the state of the file in a dictionary in memory. @see: C{ssh.QueryPubKey} """ assert self._master_node_name all_keys = target_uuids is None if all_keys: return self._public_keys[self._master_node_name] if isinstance(target_uuids, str): target_uuids = [target_uuids] result_dict = {} for key_node_uuid, keys in \ self._public_keys[self._master_node_name].items(): if key_node_uuid in target_uuids: result_dict[key_node_uuid] = keys self._AssertTypePublicKeys() return result_dict def ReplaceNameByUuid(self, node_uuid, node_name, **kwargs): """Emulates ssh.ReplaceNameByUuid on the master node. Instead of actually mainpulating the authorized_keys file, this method keeps the state of the file in a dictionary in memory. @see: C{ssh.ReplacenameByUuid} """ assert isinstance(node_uuid, str) assert isinstance(node_name, str) assert self._master_node_name if node_name in self._public_keys[self._master_node_name]: self._public_keys[self._master_node_name][node_uuid] = \ self._public_keys[self._master_node_name][node_name][:] del self._public_keys[self._master_node_name][node_name] self._AssertTypePublicKeys() # pylint: enable=W0613 ganeti-3.1.0~rc2/test/py/unit/000075500000000000000000000000001476477700300161725ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/unit/hypervisor/000075500000000000000000000000001476477700300204045ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/unit/hypervisor/hv_kvm/000075500000000000000000000000001476477700300216765ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/unit/hypervisor/hv_kvm/test_kvm_runtime.py000064400000000000000000000041531476477700300256520ustar00rootroot00000000000000import os import pytest from ganeti.hypervisor.hv_kvm.kvm_runtime import KVMRuntime from ganeti import objects from ganeti import serializer kvm_cmd = ['/usr/bin/kvm', 'dummy'] up_hvp = { 'acpi': True, 'boot_order': 'disk' } class TestKVMRuntime: @pytest.fixture def kvm_disks(self): # Get the list for disks from the json because of complexity with open("./test/py/unit/test_data/serialized_disks.json") as file: data = serializer.LoadJson(file.read()) disks = [(objects.Disk.FromDict(sdisk), link, uri) for sdisk, link, uri in data] yield disks @pytest.fixture def kvm_nics(self): # Get the list for nics from the json because of complexity with open("./test/py/unit/test_data/serialized_nics.json") as file: data = serializer.LoadJson(file.read()) nics = [objects.NIC.FromDict(nic) for nic in data] yield nics def test_properties(self, kvm_disks, kvm_nics): kvm_runtime = KVMRuntime([kvm_cmd, kvm_nics, up_hvp, kvm_disks]) assert kvm_runtime.kvm_cmd == kvm_cmd assert kvm_runtime.kvm_nics == kvm_nics assert kvm_runtime.up_hvp == up_hvp assert kvm_runtime.kvm_disks == kvm_disks def test_serialize(self, kvm_disks, kvm_nics): kvm_runtime = KVMRuntime([kvm_cmd, kvm_nics, up_hvp, kvm_disks]) serialized_runtime = kvm_runtime.serialize() # do not update the runtime fpr equality check deserialized_runtime = KVMRuntime.from_serialized(serialized_runtime, False) assert deserialized_runtime.kvm_cmd == kvm_runtime.kvm_cmd assert deserialized_runtime.up_hvp == kvm_runtime.up_hvp # check only the uuid for disks and nics # because the equal operator is not implemented for index in range(len(kvm_nics)): assert (deserialized_runtime.kvm_nics[index].uuid == kvm_runtime.kvm_nics[index].uuid) for index in range(len(kvm_disks)): assert (deserialized_runtime.kvm_disks[index][0].uuid == kvm_runtime.kvm_disks[index][0].uuid) # assert deserialized_runtime.kvm_nics == kvm_runtime.kvm_nics # assert deserialized_runtime.kvm_disks == kvm_runtime.kvm_disks ganeti-3.1.0~rc2/test/py/unit/hypervisor/hv_kvm/test_monitor.py000064400000000000000000000161501476477700300250010ustar00rootroot00000000000000# # # Copyright (C) 2024 the Ganeti project # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import threading import socket import tempfile import time from typing import Dict, List import pytest from ganeti.hypervisor.hv_kvm.monitor import QmpConnection, QmpMessage from ganeti import serializer QMP_VERSION_MICRO = 50 QMP_VERSION_MINOR = 13 QMP_VERSION_MAJOR = 0 QMP_BANNER_DATA = { "QMP": { "version": { "package": "", "qemu": { "micro": QMP_VERSION_MICRO, "minor": QMP_VERSION_MINOR, "major": QMP_VERSION_MAJOR, }, "capabilities": [], }, } } EMPTY_RESPONSE = { "return": [], } FAKE_QMP_COMMANDS = {} def simulate_qmp(command: str): """Register a function that will be executed by the given qmp command. @param command: The command on which the function listens """ def decorator(func): FAKE_QMP_COMMANDS[command] = func return func return decorator def encode_data(data: dict) -> bytes: return serializer.DumpJson(data) + QmpConnection._MESSAGE_END_TOKEN def get_qmp_commands() -> List[str]: return list(FAKE_QMP_COMMANDS.keys()) def get_supported_commands() -> Dict: commands = {'return': []} for cmd in get_qmp_commands(): command_item = { 'name': cmd.replace('_', '-') } commands['return'].append(command_item) return commands @simulate_qmp('test-command') def simulate_test_command(sock: socket.socket, arguments: Dict): sock.send(encode_data({"return": arguments})) @simulate_qmp('test-fire-event') def simulate_test_fire_event(sock: socket.socket, arguments: Dict): sock.send(encode_data({"return": arguments})) event_data = { "event": "TEST_EVENT", "timestamp": { "seconds": 1401385907, "microseconds": 422329 }, "data": {} } time.sleep(0.2) sock.send(encode_data(event_data)) class FakeQmpSocket(threading.Thread): def __init__(self, socket_path): threading.Thread.__init__(self) self._is_running = True self._is_simulate = True self._conn = None self.socket_path = socket_path self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.socket.bind(self.socket_path) self.socket.listen(1) def run(self): conn, _ = self.socket.accept() self._conn = conn # send the banner first conn.send(encode_data(QMP_BANNER_DATA)) # Expect qmp_capabilities and return an empty response conn.recv(4096) conn.send(encode_data(EMPTY_RESPONSE)) # Expect query-commands and return the list of supported commands conn.recv(4096) conn.send(encode_data(get_supported_commands())) while self._is_running: while self._is_simulate: data = conn.recv(4096) if data != b'': msg = QmpMessage.build_from_json_string(data.decode('utf-8')) cmd = msg['execute'] # check if the function exists with simulate_qmp decorator if cmd in FAKE_QMP_COMMANDS.keys(): func = FAKE_QMP_COMMANDS[cmd] func(conn, msg.data.get('arguments', {})) conn.close() def send(self, data: bytes): self._is_simulate = False self._conn.send(data) self._is_simulate = True def stop(self): self._is_running = False self.socket.close() class TestQmpConnection: @pytest.fixture def fake_socket_path(self) -> str: return tempfile.NamedTemporaryFile().name @pytest.fixture def fake_qmp_socket(self, fake_socket_path): fake_qmp_socket = FakeQmpSocket(fake_socket_path) fake_qmp_socket.daemon = True fake_qmp_socket.start() yield fake_qmp_socket fake_qmp_socket.stop() @pytest.fixture def fake_qmp(self, fake_qmp_socket, fake_socket_path): qmp = QmpConnection(fake_socket_path) qmp.timeout = 1 yield qmp if qmp.is_connected(): qmp.close() def test_connect(self, fake_qmp: QmpConnection): fake_qmp.connect() # check version is successfully parsed assert fake_qmp.version == ( QMP_VERSION_MAJOR, QMP_VERSION_MINOR, QMP_VERSION_MICRO ) # check supported commands assert (fake_qmp.supported_commands == frozenset( item["name"] for item in get_supported_commands()['return'])) def test_recv_qmp(self, fake_qmp: QmpConnection, fake_qmp_socket): fake_qmp.connect() # get one qmp message with multiple socket send pieces send_pieces = ['{"ret', 'ur', 'n": {}}\r\n'] for piece in send_pieces: fake_qmp_socket.send(piece.encode('utf-8')) qmp = fake_qmp.recv_qmp() assert qmp == QmpMessage.build_from_json_string("".join(send_pieces)) # send two messages in one send and parse two two_msgs = ['{"return": [{"name": "quit"}, {"name": "eject"}]}\r\n', '{"return": {"running": true, "singlestep": false}}\r\n'] # combine the two strings into one fake_qmp_socket.send("".join(two_msgs).encode('utf-8')) qmp_msg0 = fake_qmp.recv_qmp() qmp_msg1 = fake_qmp.recv_qmp() assert (qmp_msg0 == QmpMessage.build_from_json_string(two_msgs[0]) and qmp_msg1 == QmpMessage.build_from_json_string(two_msgs[1])) def test_execute_qmp(self, fake_qmp: QmpConnection): arguments = { 'test1': 123, 'test2': "test" } fake_qmp.connect() # run test command and check the returned arguments msg = fake_qmp.execute_qmp("test-command", arguments) assert msg == arguments # run command that does not exist with pytest.raises(Exception) as exc_info: fake_qmp.execute_qmp("non_existing_command") assert exc_info.type.__name__ == "QmpCommandNotSupported" def test_wait_for_qmp_event(self, fake_qmp: QmpConnection): fake_qmp.connect() # test None if timeout exceeds none_event = fake_qmp.wait_for_qmp_event(['NONE_EXISTING_EVENT'], 0.1) assert none_event is None fake_qmp.execute_qmp("test-fire-event") test_event = fake_qmp.wait_for_qmp_event(['TEST_EVENT'], 0.3) assert test_event.event_type == "TEST_EVENT" ganeti-3.1.0~rc2/test/py/unit/test_data/000075500000000000000000000000001476477700300201425ustar00rootroot00000000000000ganeti-3.1.0~rc2/test/py/unit/test_data/serialized_disks.json000064400000000000000000000047711476477700300243760ustar00rootroot00000000000000[ [ { "dev_type": "drbd", "logical_id": [ "3e9d956e-2db0-4d7a-80fb-543d25bb1840", "6f34bb1a-baba-4a69-b5b2-1804694adcd0", 11004, 0, 1, null ], "children": [ { "dev_type": "plain", "logical_id": [ "ganeti", "25f537bb-03bd-406e-b64a-9a9712bfcfa5.disk0_data" ], "children": [], "nodes": [ "3e9d956e-2db0-4d7a-80fb-543d25bb1840", "6f34bb1a-baba-4a69-b5b2-1804694adcd0" ], "iv_name": "", "size": 10240, "mode": "rw", "params": { "stripes": 1 }, "serial_no": 1, "uuid": "ee2c4570-165b-4abb-9f1b-62f19c4b405e", "ctime": 1721112669.1474214, "mtime": 1721112669.1474178 }, { "dev_type": "plain", "logical_id": [ "ganeti", "25f537bb-03bd-406e-b64a-9a9712bfcfa5.disk0_meta" ], "children": [], "nodes": [ "3e9d956e-2db0-4d7a-80fb-543d25bb1840", "6f34bb1a-baba-4a69-b5b2-1804694adcd0" ], "iv_name": "", "size": 128, "mode": "rw", "params": { "stripes": 1 }, "serial_no": 1, "uuid": "a2ceea83-6e4a-4976-9073-8551250cbce6", "ctime": 1721112669.147439, "mtime": 1721112669.1474364 } ], "nodes": [ "3e9d956e-2db0-4d7a-80fb-543d25bb1840", "6f34bb1a-baba-4a69-b5b2-1804694adcd0" ], "iv_name": "disk/0", "size": 10240, "mode": "rw", "params": { "c-delay-target": 1, "c-fill-target": 0, "c-max-rate": 100000, "c-min-rate": 100000, "c-plan-ahead": 0, "default-metavg": "ganeti", "disable-meta-flush": true, "disabled-barriers": "bf", "disk-custom": "--c-plan-ahead 0", "dynamic-resync": false, "net-custom": "--max-buffers 32000 --max-epoch-size 8000 --verify-alg crc32c", "protocol": "C", "resync-rate": 100000 }, "hvinfo": { "driver": "virtio-blk-pci", "id": "disk-8db0f07a-f486-481e", "bus": "pci.0", "addr": "0xc" }, "serial_no": 1, "uuid": "8db0f07a-f486-481e-b254-4c90123f61f9", "ctime": 1721112669.1474493, "mtime": 1721112669.1474466 }, "/var/run/ganeti/instance-disks/test2:0", null ] ]ganeti-3.1.0~rc2/test/py/unit/test_data/serialized_nics.json000064400000000000000000000023441476477700300242070ustar00rootroot00000000000000[ { "mac": "aa:00:00:f6:4d:b4", "ip": "10.10.1.18", "network": "e0400df0-d58c-4bed-8849-5b6da160b8bb", "nicparams": { "link": "gnt-br0", "mode": "bridged", "vlan": "" }, "netinfo": { "name": "ganeti_test_42", "serial_no": 2, "network": "10.10.1.0/24", "gateway": "10.10.1.1", "reservations": "0000000000000000101000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "ext_reservations": "1111111111111111010010011111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "ctime": 1720789965.8390286, "mtime": 1720789974.890768, "uuid": "e0400df0-d58c-4bed-8849-5b6da160b8bb", "tags": [] }, "hvinfo": { "driver": "virtio-net-pci", "id": "nic-4564e5d5-8df4-40ec", "bus": "pci.0", "addr": "0xd" }, "uuid": "4564e5d5-8df4-40ec-8dee-b5b0d6067495" } ]ganeti-3.1.0~rc2/tools/000075500000000000000000000000001476477700300147445ustar00rootroot00000000000000ganeti-3.1.0~rc2/tools/cfgshell000075500000000000000000000231341476477700300164640ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tool to do manual changes to the config file. """ # functions in this module need to have a given name structure, so: # pylint: disable=C0103 import optparse import cmd try: import readline _wd = readline.get_completer_delims() _wd = _wd.replace("-", "") readline.set_completer_delims(_wd) del _wd except ImportError: pass from ganeti import errors from ganeti import config from ganeti import objects class ConfigShell(cmd.Cmd): """Command tool for editing the config file. Note that although we don't do saves after remove, the current ConfigWriter code does that; so we can't prevent someone from actually breaking the config with this tool. It's the users' responsibility to know what they're doing. """ # all do_/complete_* functions follow the same API # pylint: disable=W0613 prompt = "(/) " def __init__(self, cfg_file=None): """Constructor for the ConfigShell object. The optional cfg_file argument will be used to load a config file at startup. """ cmd.Cmd.__init__(self) self.cfg = None self.parents = [] self.path = [] if cfg_file: self.do_load(cfg_file) self.postcmd(False, "") def emptyline(self): """Empty line handling. Note that the default will re-run the last command. We don't want that, and just ignore the empty line. """ return False @staticmethod def _get_entries(obj): """Computes the list of subdirs and files in the given object. This, depending on the passed object entry, look at each logical child of the object and decides if it's a container or a simple object. Based on this, it computes the list of subdir and files. """ dirs = [] entries = [] if isinstance(obj, objects.ConfigObject): for name in obj.GetAllSlots(): child = getattr(obj, name, None) if isinstance(child, (list, dict, tuple, objects.ConfigObject)): dirs.append(name) else: entries.append(name) elif isinstance(obj, (list, tuple)): for idx, child in enumerate(obj): if isinstance(child, (list, dict, tuple, objects.ConfigObject)): dirs.append(str(idx)) else: entries.append(str(idx)) elif isinstance(obj, dict): dirs = obj.keys() return dirs, entries def precmd(self, line): """Precmd hook to prevent commands in invalid states. This will prevent everything except load and quit when no configuration is loaded. """ if line.startswith("load") or line == "EOF" or line == "quit": return line if not self.parents or self.cfg is None: print("No config data loaded") return "" return line def postcmd(self, stop, line): """Postcmd hook to update the prompt. We show the current location in the prompt and this function is used to update it; this is only needed after cd and load, but we update it anyway. """ if self.cfg is None: self.prompt = "(#no config) " else: self.prompt = "(/%s) " % ("/".join(self.path),) return stop def do_load(self, line): """Load function. Syntax: load [/path/to/config/file] This will load a new configuration, discarding any existing data (if any). If no argument has been passed, it will use the default config file location. """ if line: arg = line else: arg = None try: self.cfg = config.ConfigWriter(cfg_file=arg, offline=True) self.parents = [self.cfg._config_data] # pylint: disable=W0212 self.path = [] except errors.ConfigurationError as err: print("Error: %s" % str(err)) return False def do_ls(self, line): """List the current entry. This will show directories with a slash appended and files normally. """ dirs, entries = self._get_entries(self.parents[-1]) for i in dirs: print(i + "/") for i in entries: print(i) return False def complete_cd(self, text, line, begidx, endidx): """Completion function for the cd command. """ pointer = self.parents[-1] dirs, _ = self._get_entries(pointer) matches = [str(name) for name in dirs if name.startswith(text)] return matches def do_cd(self, line): """Changes the current path. Valid arguments: either .., /, "" (no argument) or a child of the current object. """ if line == "..": if self.path: self.path.pop() self.parents.pop() return False else: print("Already at top level") return False elif len(line) == 0 or line == "/": self.parents = self.parents[0:1] self.path = [] return False pointer = self.parents[-1] dirs, _ = self._get_entries(pointer) if line not in dirs: print("No such child") return False if isinstance(pointer, (dict, list, tuple)): if isinstance(pointer, (list, tuple)): line = int(line) new_obj = pointer[line] else: new_obj = getattr(pointer, line) self.parents.append(new_obj) self.path.append(str(line)) return False def do_pwd(self, line): """Shows the current path. This duplicates the prompt functionality, but it's reasonable to have. """ print("/" + "/".join(self.path)) return False def complete_cat(self, text, line, begidx, endidx): """Completion for the cat command. """ pointer = self.parents[-1] _, entries = self._get_entries(pointer) matches = [name for name in entries if name.startswith(text)] return matches def do_cat(self, line): """Shows the contents of the given file. This will display the contents of the given file, which must be a child of the current path (as shows by `ls`). """ pointer = self.parents[-1] _, entries = self._get_entries(pointer) if line not in entries: print("No such entry") return False if isinstance(pointer, (dict, list, tuple)): if isinstance(pointer, (list, tuple)): line = int(line) val = pointer[line] else: val = getattr(pointer, line) print(val) return False def do_verify(self, line): """Verify the configuration. This verifies the contents of the configuration file (and not the in-memory data, as every modify operation automatically saves the file). """ vdata = self.cfg.VerifyConfig() if vdata: print("Validation failed. Errors:") for text in vdata: print(text) return False def do_save(self, line): """Saves the configuration data. Note that is redundant (all modify operations automatically save the data), but it is good to use it as in the future that could change. """ if self.cfg.VerifyConfig(): print("Config data does not validate, refusing to save.") return False self.cfg._WriteConfig() # pylint: disable=W0212 def do_rm(self, line): """Removes an instance or a node. This function works only on instances or nodes. You must be in either `/nodes` or `/instances` and give a valid argument. """ pointer = self.parents[-1] data = self.cfg._config_data # pylint: disable=W0212 if pointer not in (data.instances, data.nodes): print("Can only delete instances and nodes") return False if pointer == data.instances: if line in data.instances: self.cfg.RemoveInstance(line) else: print("Invalid instance name") else: if line in data.nodes: self.cfg.RemoveNode(line) else: print("Invalid node name") @staticmethod def do_EOF(line): """Exit the application. """ print() return True @staticmethod def do_quit(line): """Exit the application. """ print() return True class Error(Exception): """Generic exception""" pass def ParseOptions(): """Parses the command line options. In case of command line errors, it will show the usage and exit the program. @return: a tuple (options, args), as returned by OptionParser.parse_args """ parser = optparse.OptionParser() options, args = parser.parse_args() return options, args def main(): """Application entry point. """ _, args = ParseOptions() if args: cfg_file = args[0] else: cfg_file = None shell = ConfigShell(cfg_file=cfg_file) shell.cmdloop() if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/cfgupgrade000075500000000000000000000036111476477700300170020ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tool to upgrade the configuration file. This code handles only the types supported by the json python build-in module. As an example, 'set' is a 'list'. """ from ganeti.tools.cfgupgrade import CfgUpgrade, Error, ParseOptions if __name__ == "__main__": opts, args = ParseOptions() try: CfgUpgrade(opts, args).Run() except Error as e: if opts.debug: # If debugging, we want to see the original stack trace. raise else: # Else silence it for the sake of convenience. raise SystemExit(e) ganeti-3.1.0~rc2/tools/cfgupgrade12000075500000000000000000000316211476477700300171470ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2007, 2008, 2009 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # pylint: disable=C0103,E1103 # C0103: invalid name NoDefault # E1103: Instance of 'foor' has no 'bar' member (but some types could # not be inferred) """Tool to upgrade the configuration file. This code handles only the types supported by the json python build-in module. As an example, 'set' is a 'list'. @note: this has lots of duplicate content with C{cfgupgrade}. Ideally, it should be merged. """ import os import os.path import sys import optparse import logging import errno from ganeti import constants from ganeti import serializer from ganeti import utils from ganeti import cli from ganeti import pathutils from ganeti.utils import version options = None args = None # Unique object to identify calls without default value NoDefault = object() # Dictionary with instance old keys, and new hypervisor keys INST_HV_CHG = { "hvm_pae": constants.HV_PAE, "vnc_bind_address": constants.HV_VNC_BIND_ADDRESS, "initrd_path": constants.HV_INITRD_PATH, "hvm_nic_type": constants.HV_NIC_TYPE, "kernel_path": constants.HV_KERNEL_PATH, "hvm_acpi": constants.HV_ACPI, "hvm_cdrom_image_path": constants.HV_CDROM_IMAGE_PATH, "hvm_boot_order": constants.HV_BOOT_ORDER, "hvm_disk_type": constants.HV_DISK_TYPE, } # Instance beparams changes INST_BE_CHG = { "vcpus": constants.BE_VCPUS, "memory": constants.BE_MEMORY, "auto_balance": constants.BE_AUTO_BALANCE, } # Field names F_SERIAL = "serial_no" class Error(Exception): """Generic exception""" pass def SsconfName(key): """Returns the file name of an (old) ssconf key. """ return "%s/ssconf_%s" % (options.data_dir, key) def ReadFile(file_name, default=NoDefault): """Reads a file. """ logging.debug("Reading %s", file_name) try: fh = open(file_name, "r") except IOError as err: if default is not NoDefault and err.errno == errno.ENOENT: return default raise try: return fh.read() finally: fh.close() def WriteFile(file_name, data): """Writes a configuration file. """ logging.debug("Writing %s", file_name) utils.WriteFile(file_name=file_name, data=data, mode=0o600, dry_run=options.dry_run, backup=True) def GenerateSecret(all_secrets): """Generate an unique DRBD secret. This is a copy from ConfigWriter. """ retries = 64 while retries > 0: secret = utils.GenerateSecret() if secret not in all_secrets: break retries -= 1 else: raise Error("Can't generate unique DRBD secret") return secret def SetupLogging(): """Configures the logging module. """ formatter = logging.Formatter("%(asctime)s: %(message)s") stderr_handler = logging.StreamHandler() stderr_handler.setFormatter(formatter) if options.debug: stderr_handler.setLevel(logging.NOTSET) elif options.verbose: stderr_handler.setLevel(logging.INFO) else: stderr_handler.setLevel(logging.CRITICAL) root_logger = logging.getLogger("") root_logger.setLevel(logging.NOTSET) root_logger.addHandler(stderr_handler) def Cluster12To20(cluster): """Upgrades the cluster object from 1.2 to 2.0. """ logging.info("Upgrading the cluster object") # Upgrade the configuration version if "config_version" in cluster: del cluster["config_version"] # Add old ssconf keys back to config logging.info(" - importing ssconf keys") for key in ("master_node", "master_ip", "master_netdev", "cluster_name"): if key not in cluster: cluster[key] = ReadFile(SsconfName(key)).strip() if "default_hypervisor" not in cluster: old_hyp = ReadFile(SsconfName("hypervisor")).strip() if old_hyp == "xen-3.0": hyp = "xen-pvm" elif old_hyp == "xen-hvm-3.1": hyp = "xen-hvm" elif old_hyp == "fake": hyp = "fake" else: raise Error("Unknown old hypervisor name '%s'" % old_hyp) logging.info("Setting the default and enabled hypervisor") cluster["default_hypervisor"] = hyp cluster["enabled_hypervisors"] = [hyp] # hv/be params if "hvparams" not in cluster: logging.info(" - adding hvparams") cluster["hvparams"] = constants.HVC_DEFAULTS if "beparams" not in cluster: logging.info(" - adding beparams") cluster["beparams"] = {constants.PP_DEFAULT: constants.BEC_DEFAULTS} # file storage if "file_storage_dir" not in cluster: cluster["file_storage_dir"] = pathutils.DEFAULT_FILE_STORAGE_DIR # candidate pool size if "candidate_pool_size" not in cluster: cluster["candidate_pool_size"] = constants.MASTER_POOL_SIZE_DEFAULT def Node12To20(node): """Upgrades a node from 1.2 to 2.0. """ logging.info("Upgrading node %s", node['name']) if F_SERIAL not in node: node[F_SERIAL] = 1 if "master_candidate" not in node: node["master_candidate"] = True for key in "offline", "drained": if key not in node: node[key] = False def Instance12To20(drbd_minors, secrets, hypervisor, instance): """Upgrades an instance from 1.2 to 2.0. """ if F_SERIAL not in instance: instance[F_SERIAL] = 1 if "hypervisor" not in instance: instance["hypervisor"] = hypervisor # hvparams changes if "hvparams" not in instance: instance["hvparams"] = hvp = {} for old, new in INST_HV_CHG.items(): if old in instance: if (instance[old] is not None and instance[old] != constants.VALUE_DEFAULT and # no longer valid in 2.0 new in constants.HVC_DEFAULTS[hypervisor]): hvp[new] = instance[old] del instance[old] # beparams changes if "beparams" not in instance: instance["beparams"] = bep = {} for old, new in INST_BE_CHG.items(): if old in instance: if instance[old] is not None: bep[new] = instance[old] del instance[old] # disk changes for disk in instance["disks"]: Disk12To20(drbd_minors, secrets, disk) # other instance changes if "status" in instance: instance["admin_up"] = instance["status"] == "up" del instance["status"] def Disk12To20(drbd_minors, secrets, disk): """Upgrades a disk from 1.2 to 2.0. """ if "mode" not in disk: disk["mode"] = constants.DISK_RDWR if disk["dev_type"] == constants.DT_DRBD8: old_lid = disk["logical_id"] for node in old_lid[:2]: if node not in drbd_minors: raise Error("Can't find node '%s' while upgrading disk" % node) drbd_minors[node] += 1 minor = drbd_minors[node] old_lid.append(minor) old_lid.append(GenerateSecret(secrets)) del disk["physical_id"] if disk["children"]: for child in disk["children"]: Disk12To20(drbd_minors, secrets, child) def _ParseOptions(): parser = optparse.OptionParser(usage="%prog [--debug|--verbose] [--force]") parser.add_option("--dry-run", dest="dry_run", action="store_true", help="Try to do the conversion, but don't write" " output file") parser.add_option(cli.FORCE_OPT) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option("--path", help="Convert configuration in this" " directory instead of '%s'" % pathutils.DATA_DIR, default=pathutils.DATA_DIR, dest="data_dir") return parser.parse_args() def _ComposePaths(oobj): # We need to keep filenames locally because they might be renamed between # versions. oobj.data_dir = os.path.abspath(oobj.data_dir) oobj.CONFIG_DATA_PATH = oobj.data_dir + "/config.data" oobj.SERVER_PEM_PATH = oobj.data_dir + "/server.pem" oobj.KNOWN_HOSTS_PATH = oobj.data_dir + "/known_hosts" oobj.RAPI_CERT_FILE = oobj.data_dir + "/rapi.pem" def _WriteConfiguration(config_data, known_hosts): try: logging.info("Writing configuration file") WriteFile(options.CONFIG_DATA_PATH, serializer.DumpJson(config_data)) if known_hosts is not None: logging.info("Writing SSH known_hosts file (%s)", known_hosts.strip()) WriteFile(options.KNOWN_HOSTS_PATH, known_hosts) if not options.dry_run: if not os.path.exists(options.RAPI_CERT_FILE): logging.debug("Writing RAPI certificate to %s", options.RAPI_CERT_FILE) utils.GenerateSelfSignedSslCert(options.RAPI_CERT_FILE, 1) except: logging.critical("Writing configuration failed. It is probably in an" " inconsistent state and needs manual intervention.") raise logging.info("Configuration file updated.") def main(): """Main program. """ # pylint: disable=W0603 global options, args program = os.path.basename(sys.argv[0]) (options, args) = _ParseOptions() _ComposePaths(options) SetupLogging() # Option checking if args: raise Error("No arguments expected") if not options.force: usertext = ("%s MUST be run on the master node. Is this the master" " node and are ALL instances down?" % program) if not cli.AskUser(usertext): sys.exit(1) # Check whether it's a Ganeti configuration directory if not (os.path.isfile(options.CONFIG_DATA_PATH) and os.path.isfile(options.SERVER_PEM_PATH) or os.path.isfile(options.KNOWN_HOSTS_PATH)): raise Error(("%s does not seem to be a known Ganeti configuration" " directory") % options.data_dir) config_version = ReadFile(SsconfName("config_version"), "1.2").strip() logging.info("Found configuration version %s", config_version) config_data = serializer.LoadJson(ReadFile(options.CONFIG_DATA_PATH)) # Ganeti 1.2? if config_version == "1.2": logging.info("Found a Ganeti 1.2 configuration") cluster = config_data["cluster"] old_config_version = cluster.get("config_version", None) logging.info("Found old configuration version %s", old_config_version) if old_config_version not in (3, ): raise Error("Unsupported configuration version: %s" % old_config_version) if "version" not in config_data: config_data["version"] = version.BuildVersion(2, 0, 0) if F_SERIAL not in config_data: config_data[F_SERIAL] = 1 # Make sure no instance uses remote_raid1 anymore remote_raid1_instances = [] for instance in config_data["instances"].values(): if instance["disk_template"] == "remote_raid1": remote_raid1_instances.append(instance["name"]) if remote_raid1_instances: for name in remote_raid1_instances: logging.error("Instance %s still using remote_raid1 disk template", name) raise Error("Unable to convert configuration as long as there are" " instances using remote_raid1 disk template") # Build content of new known_hosts file cluster_name = ReadFile(SsconfName("cluster_name")).rstrip() cluster_key = cluster["rsahostkeypub"] known_hosts = "%s ssh-rsa %s\n" % (cluster_name, cluster_key) Cluster12To20(cluster) # Add node attributes logging.info("Upgrading nodes") # stable-sort the names to have repeatable runs for node_name in utils.NiceSort(config_data["nodes"].keys()): Node12To20(config_data["nodes"][node_name]) # Instance changes logging.info("Upgrading instances") drbd_minors = dict.fromkeys(config_data["nodes"], 0) secrets = set() # stable-sort the names to have repeatable runs for instance_name in utils.NiceSort(config_data["instances"].keys()): Instance12To20(drbd_minors, secrets, cluster["default_hypervisor"], config_data["instances"][instance_name]) else: logging.info("Found a Ganeti 2.0 configuration") if "config_version" in config_data["cluster"]: raise Error("Inconsistent configuration: found config_data in" " configuration file") known_hosts = None _WriteConfiguration(config_data, known_hosts) if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/check-cert-expired000075500000000000000000000050571476477700300203470ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tool to detect expired X509 certificates. """ # pylint: disable=C0103 # C0103: Invalid name check-cert-expired import os.path import sys import OpenSSL from ganeti import constants from ganeti import cli from ganeti import utils def main(): """Main routine. """ program = os.path.basename(sys.argv[0]) if len(sys.argv) != 2: cli.ToStderr("Usage: %s ", program) sys.exit(constants.EXIT_FAILURE) filename = sys.argv[1] # Read certificate try: cert_pem = utils.ReadFile(filename) except EnvironmentError as err: cli.ToStderr("Unable to read %s: %s", filename, err) sys.exit(constants.EXIT_FAILURE) # Check validity try: cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_pem) (errcode, msg) = utils.VerifyX509Certificate(cert, None, None) if msg: cli.ToStderr("%s: %s", filename, msg) if errcode == utils.CERT_ERROR: sys.exit(constants.EXIT_SUCCESS) except (KeyboardInterrupt, SystemExit): raise except Exception as err: # pylint: disable=W0703 cli.ToStderr("Unable to check %s: %s", filename, err) sys.exit(constants.EXIT_FAILURE) if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/cluster-merge000075500000000000000000000751411476477700300174600ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tool to merge two or more clusters together. The clusters have to run the same version of Ganeti! """ # pylint: disable=C0103 # C0103: Invalid name cluster-merge import logging import os import optparse import shutil import sys import tempfile from ganeti import cli from ganeti import config from ganeti import constants from ganeti import errors from ganeti import ssh from ganeti import utils from ganeti import pathutils from ganeti import compat _GROUPS_MERGE = "merge" _GROUPS_RENAME = "rename" _CLUSTERMERGE_ECID = "clustermerge-ecid" _RESTART_ALL = "all" _RESTART_UP = "up" _RESTART_NONE = "none" _RESTART_CHOICES = (_RESTART_ALL, _RESTART_UP, _RESTART_NONE) _PARAMS_STRICT = "strict" _PARAMS_WARN = "warn" _PARAMS_CHOICES = (_PARAMS_STRICT, _PARAMS_WARN) PAUSE_PERIOD_OPT = cli.cli_option("-p", "--watcher-pause-period", default=1800, action="store", type="int", dest="pause_period", help=("Amount of time in seconds watcher" " should be suspended from running")) GROUPS_OPT = cli.cli_option("--groups", default=None, metavar="STRATEGY", choices=(_GROUPS_MERGE, _GROUPS_RENAME), dest="groups", help=("How to handle groups that have the" " same name (One of: %s/%s)" % (_GROUPS_MERGE, _GROUPS_RENAME))) PARAMS_OPT = cli.cli_option("--parameter-conflicts", default=_PARAMS_STRICT, metavar="STRATEGY", choices=_PARAMS_CHOICES, dest="params", help=("How to handle params that have" " different values (One of: %s/%s)" % _PARAMS_CHOICES)) RESTART_OPT = cli.cli_option("--restart", default=_RESTART_ALL, metavar="STRATEGY", choices=_RESTART_CHOICES, dest="restart", help=("How to handle restarting instances" " same name (One of: %s/%s/%s)" % _RESTART_CHOICES)) SKIP_STOP_INSTANCES_OPT = \ cli.cli_option("--skip-stop-instances", default=True, action="store_false", dest="stop_instances", help=("Don't stop the instances on the clusters, just check " "that none is running")) def Flatten(unflattened_list): """Flattens a list. @param unflattened_list: A list of unflattened list objects. @return: A flattened list """ flattened_list = [] for item in unflattened_list: if isinstance(item, list): flattened_list.extend(Flatten(item)) else: flattened_list.append(item) return flattened_list class MergerData(object): """Container class to hold data used for merger. """ def __init__(self, cluster, key_path, nodes, instances, master_node, config_path=None): """Initialize the container. @param cluster: The name of the cluster @param key_path: Path to the ssh private key used for authentication @param nodes: List of online nodes in the merging cluster @param instances: List of instances running on merging cluster @param master_node: Name of the master node @param config_path: Path to the merging cluster config """ self.cluster = cluster self.key_path = key_path self.nodes = nodes self.instances = instances self.master_node = master_node self.config_path = config_path class Merger(object): """Handling the merge. """ RUNNING_STATUSES = compat.UniqueFrozenset([ constants.INSTST_RUNNING, constants.INSTST_ERRORUP, ]) def __init__(self, clusters, pause_period, groups, restart, params, stop_instances): """Initialize object with sane defaults and infos required. @param clusters: The list of clusters to merge in @param pause_period: The time watcher shall be disabled for @param groups: How to handle group conflicts @param restart: How to handle instance restart @param stop_instances: Indicates whether the instances must be stopped (True) or if the Merger must only check if no instances are running on the mergee clusters (False) """ self.merger_data = [] self.clusters = clusters self.pause_period = pause_period self.work_dir = tempfile.mkdtemp(suffix="cluster-merger") (self.cluster_name, ) = cli.GetClient().QueryConfigValues(["cluster_name"]) self.ssh_runner = ssh.SshRunner(self.cluster_name) self.groups = groups self.restart = restart self.params = params self.stop_instances = stop_instances if self.restart == _RESTART_UP: raise NotImplementedError def Setup(self): """Sets up our end so we can do the merger. This method is setting us up as a preparation for the merger. It makes the initial contact and gathers information needed. @raise errors.RemoteError: for errors in communication/grabbing """ (remote_path, _, _) = ssh.GetUserFiles("root") if self.cluster_name in self.clusters: raise errors.CommandError("Cannot merge cluster %s with itself" % self.cluster_name) # Fetch remotes private key for cluster in self.clusters: result = self._RunCmd(cluster, "cat %s" % remote_path, batch=False, ask_key=False) if result.failed: raise errors.RemoteError("There was an error while grabbing ssh private" " key from %s. Fail reason: %s; output: %s" % (cluster, result.fail_reason, result.output)) key_path = utils.PathJoin(self.work_dir, cluster) utils.WriteFile(key_path, mode=0o600, data=result.stdout) result = self._RunCmd(cluster, "gnt-node list -o name,offline" " --no-headers --separator=,", private_key=key_path) if result.failed: raise errors.RemoteError("Unable to retrieve list of nodes from %s." " Fail reason: %s; output: %s" % (cluster, result.fail_reason, result.output)) nodes_statuses = [line.split(",") for line in result.stdout.splitlines()] nodes = [node_status[0] for node_status in nodes_statuses if node_status[1] == "N"] result = self._RunCmd(cluster, "gnt-instance list -o name --no-headers", private_key=key_path) if result.failed: raise errors.RemoteError("Unable to retrieve list of instances from" " %s. Fail reason: %s; output: %s" % (cluster, result.fail_reason, result.output)) instances = result.stdout.splitlines() path = utils.PathJoin(pathutils.DATA_DIR, "ssconf_%s" % constants.SS_MASTER_NODE) result = self._RunCmd(cluster, "cat %s" % path, private_key=key_path) if result.failed: raise errors.RemoteError("Unable to retrieve the master node name from" " %s. Fail reason: %s; output: %s" % (cluster, result.fail_reason, result.output)) master_node = result.stdout.strip() self.merger_data.append(MergerData(cluster, key_path, nodes, instances, master_node)) def _PrepareAuthorizedKeys(self): """Prepare the authorized_keys on every merging node. This method add our public key to remotes authorized_key for further communication. """ (_, pub_key_file, auth_keys) = ssh.GetUserFiles("root") pub_key = utils.ReadFile(pub_key_file) for data in self.merger_data: for node in data.nodes: result = self._RunCmd(node, ("cat >> %s << '!EOF.'\n%s!EOF.\n" % (auth_keys, pub_key)), private_key=data.key_path, max_attempts=3) if result.failed: raise errors.RemoteError("Unable to add our public key to %s in %s." " Fail reason: %s; output: %s" % (node, data.cluster, result.fail_reason, result.output)) def _RunCmd(self, hostname, command, user="root", use_cluster_key=False, strict_host_check=False, private_key=None, batch=True, ask_key=False, max_attempts=1): """Wrapping SshRunner.Run with default parameters. For explanation of parameters see L{ganeti.ssh.SshRunner.Run}. """ for _ in range(max_attempts): result = self.ssh_runner.Run(hostname=hostname, command=command, user=user, use_cluster_key=use_cluster_key, strict_host_check=strict_host_check, private_key=private_key, batch=batch, ask_key=ask_key) if not result.failed: break return result def _CheckRunningInstances(self): """Checks if on the clusters to be merged there are running instances @rtype: boolean @return: True if there are running instances, False otherwise """ for cluster in self.clusters: result = self._RunCmd(cluster, "gnt-instance list -o status") if self.RUNNING_STATUSES.intersection(result.output.splitlines()): return True return False def _StopMergingInstances(self): """Stop instances on merging clusters. """ for cluster in self.clusters: result = self._RunCmd(cluster, "gnt-instance shutdown --all" " --force-multiple") if result.failed: raise errors.RemoteError("Unable to stop instances on %s." " Fail reason: %s; output: %s" % (cluster, result.fail_reason, result.output)) def _DisableWatcher(self): """Disable watch on all merging clusters, including ourself. """ for cluster in ["localhost"] + self.clusters: result = self._RunCmd(cluster, "gnt-cluster watcher pause %d" % self.pause_period) if result.failed: raise errors.RemoteError("Unable to pause watcher on %s." " Fail reason: %s; output: %s" % (cluster, result.fail_reason, result.output)) def _RemoveMasterIps(self): """Removes the master IPs from the master nodes of each cluster. """ for data in self.merger_data: result = self._RunCmd(data.master_node, "gnt-cluster deactivate-master-ip --yes") if result.failed: raise errors.RemoteError("Unable to remove master IP on %s." " Fail reason: %s; output: %s" % (data.master_node, result.fail_reason, result.output)) def _StopDaemons(self): """Stop all daemons on merging nodes. """ cmd = "%s stop-all" % pathutils.DAEMON_UTIL for data in self.merger_data: for node in data.nodes: result = self._RunCmd(node, cmd, max_attempts=3) if result.failed: raise errors.RemoteError("Unable to stop daemons on %s." " Fail reason: %s; output: %s." % (node, result.fail_reason, result.output)) def _FetchRemoteConfig(self): """Fetches and stores remote cluster config from the master. This step is needed before we can merge the config. """ for data in self.merger_data: result = self._RunCmd(data.cluster, "cat %s" % pathutils.CLUSTER_CONF_FILE) if result.failed: raise errors.RemoteError("Unable to retrieve remote config on %s." " Fail reason: %s; output %s" % (data.cluster, result.fail_reason, result.output)) data.config_path = utils.PathJoin(self.work_dir, "%s_config.data" % data.cluster) utils.WriteFile(data.config_path, data=result.stdout) # R0201: Method could be a function def _KillMasterDaemon(self): # pylint: disable=R0201 """Kills the local master daemon. @raise errors.CommandError: If unable to kill """ result = utils.RunCmd([pathutils.DAEMON_UTIL, "stop-master"]) if result.failed: raise errors.CommandError("Unable to stop master daemons." " Fail reason: %s; output: %s" % (result.fail_reason, result.output)) def _MergeConfig(self): """Merges all foreign config into our own config. """ my_config = config.ConfigWriter(offline=True) fake_ec_id = 0 # Needs to be uniq over the whole config merge for data in self.merger_data: other_config = config.ConfigWriter(data.config_path, accept_foreign=True) self._MergeClusterConfigs(my_config, other_config) self._MergeNodeGroups(my_config, other_config) for node in other_config.GetNodeList(): node_info = other_config.GetNodeInfo(node) # Offline the node, it will be reonlined later at node readd node_info.master_candidate = False node_info.drained = False node_info.offline = True my_config.AddNode(node_info, _CLUSTERMERGE_ECID + str(fake_ec_id)) fake_ec_id += 1 for instance in other_config.GetInstanceList(): instance_info = other_config.GetInstanceInfo(instance) # Update the DRBD port assignments # This is a little bit hackish for dsk in instance_info.disks: if dsk.dev_type in constants.DTS_DRBD: port = my_config.AllocatePort() logical_id = list(dsk.logical_id) logical_id[2] = port dsk.logical_id = tuple(logical_id) my_config.AddInstance(instance_info, _CLUSTERMERGE_ECID + str(fake_ec_id)) fake_ec_id += 1 def _MergeClusterConfigs(self, my_config, other_config): """Checks that all relevant cluster parameters are compatible """ my_cluster = my_config.GetClusterInfo() other_cluster = other_config.GetClusterInfo() err_count = 0 # # Generic checks # check_params = [ "beparams", "default_iallocator", "drbd_usermode_helper", "hidden_os", "maintain_node_health", "master_netdev", "ndparams", "nicparams", "primary_ip_family", "tags", "uid_pool", ] check_params_strict = [ "volume_group_name", ] if my_cluster.IsFileStorageEnabled() or \ other_cluster.IsFileStorageEnabled(): check_params_strict.append("file_storage_dir") if my_cluster.IsSharedFileStorageEnabled() or \ other_cluster.IsSharedFileStorageEnabled(): check_params_strict.append("shared_file_storage_dir") check_params.extend(check_params_strict) params_strict = (self.params == _PARAMS_STRICT) for param_name in check_params: my_param = getattr(my_cluster, param_name) other_param = getattr(other_cluster, param_name) if my_param != other_param: logging.error("The value (%s) of the cluster parameter %s on %s" " differs to this cluster's value (%s)", other_param, param_name, other_cluster.cluster_name, my_param) if params_strict or param_name in check_params_strict: err_count += 1 # # Custom checks # # Check default hypervisor my_defhyp = my_cluster.enabled_hypervisors[0] other_defhyp = other_cluster.enabled_hypervisors[0] if my_defhyp != other_defhyp: logging.warning("The default hypervisor (%s) differs on %s, new" " instances will be created with this cluster's" " default hypervisor (%s)", other_defhyp, other_cluster.cluster_name, my_defhyp) if (set(my_cluster.enabled_hypervisors) != set(other_cluster.enabled_hypervisors)): logging.error("The set of enabled hypervisors (%s) on %s differs to" " this cluster's set (%s)", other_cluster.enabled_hypervisors, other_cluster.cluster_name, my_cluster.enabled_hypervisors) err_count += 1 # Check hypervisor params for hypervisors we care about for hyp in my_cluster.enabled_hypervisors: for param in my_cluster.hvparams[hyp]: my_value = my_cluster.hvparams[hyp][param] other_value = other_cluster.hvparams[hyp][param] if my_value != other_value: logging.error("The value (%s) of the %s parameter of the %s" " hypervisor on %s differs to this cluster's parameter" " (%s)", other_value, param, hyp, other_cluster.cluster_name, my_value) if params_strict: err_count += 1 # Check os hypervisor params for hypervisors we care about for os_name in set(my_cluster.os_hvp.keys() + other_cluster.os_hvp.keys()): for hyp in my_cluster.enabled_hypervisors: my_os_hvp = self._GetOsHypervisor(my_cluster, os_name, hyp) other_os_hvp = self._GetOsHypervisor(other_cluster, os_name, hyp) if my_os_hvp != other_os_hvp: logging.error("The OS parameters (%s) for the %s OS for the %s" " hypervisor on %s differs to this cluster's parameters" " (%s)", other_os_hvp, os_name, hyp, other_cluster.cluster_name, my_os_hvp) if params_strict: err_count += 1 # # Warnings # if my_cluster.modify_etc_hosts != other_cluster.modify_etc_hosts: logging.warning("The modify_etc_hosts value (%s) differs on %s," " this cluster's value (%s) will take precedence", other_cluster.modify_etc_hosts, other_cluster.cluster_name, my_cluster.modify_etc_hosts) if my_cluster.modify_ssh_setup != other_cluster.modify_ssh_setup: logging.warning("The modify_ssh_setup value (%s) differs on %s," " this cluster's value (%s) will take precedence", other_cluster.modify_ssh_setup, other_cluster.cluster_name, my_cluster.modify_ssh_setup) # # Actual merging # my_cluster.reserved_lvs = list(set(my_cluster.reserved_lvs + other_cluster.reserved_lvs)) if my_cluster.prealloc_wipe_disks != other_cluster.prealloc_wipe_disks: logging.warning("The prealloc_wipe_disks value (%s) on %s differs to this" " cluster's value (%s). The least permissive value (%s)" " will be used", other_cluster.prealloc_wipe_disks, other_cluster.cluster_name, my_cluster.prealloc_wipe_disks, True) my_cluster.prealloc_wipe_disks = True for os_, osparams in other_cluster.osparams.items(): if os_ not in my_cluster.osparams: my_cluster.osparams[os_] = osparams elif my_cluster.osparams[os_] != osparams: logging.error("The OS parameters (%s) for the %s OS on %s differs to" " this cluster's parameters (%s)", osparams, os_, other_cluster.cluster_name, my_cluster.osparams[os_]) if params_strict: err_count += 1 if err_count: raise errors.ConfigurationError("Cluster config for %s has incompatible" " values, please fix and re-run" % other_cluster.cluster_name) # R0201: Method could be a function def _GetOsHypervisor(self, cluster, os_name, hyp): # pylint: disable=R0201 if os_name in cluster.os_hvp: return cluster.os_hvp[os_name].get(hyp, None) else: return None # R0201: Method could be a function def _MergeNodeGroups(self, my_config, other_config): """Adds foreign node groups ConfigWriter.AddNodeGroup takes care of making sure there are no conflicts. """ # pylint: disable=R0201 logging.info("Node group conflict strategy: %s", self.groups) my_grps = my_config.GetAllNodeGroupsInfo().values() other_grps = other_config.GetAllNodeGroupsInfo().values() # Check for node group naming conflicts: conflicts = [] for other_grp in other_grps: for my_grp in my_grps: if other_grp.name == my_grp.name: conflicts.append(other_grp) if conflicts: conflict_names = utils.CommaJoin([g.name for g in conflicts]) logging.info("Node groups in both local and remote cluster: %s", conflict_names) # User hasn't specified how to handle conflicts if not self.groups: raise errors.CommandError("The following node group(s) are in both" " clusters, and no merge strategy has been" " supplied (see the --groups option): %s" % conflict_names) # User wants to rename conflicts elif self.groups == _GROUPS_RENAME: for grp in conflicts: new_name = "%s-%s" % (grp.name, other_config.GetClusterName()) logging.info("Renaming remote node group from %s to %s" " to resolve conflict", grp.name, new_name) grp.name = new_name # User wants to merge conflicting groups elif self.groups == _GROUPS_MERGE: for other_grp in conflicts: logging.info("Merging local and remote '%s' groups", other_grp.name) for node_name in other_grp.members[:]: node = other_config.GetNodeInfo(node_name) # Access to a protected member of a client class # pylint: disable=W0212 other_config._UnlockedRemoveNodeFromGroup(node) # Access to a protected member of a client class # pylint: disable=W0212 my_grp_uuid = my_config._UnlockedLookupNodeGroup(other_grp.name) # Access to a protected member of a client class # pylint: disable=W0212 my_config._UnlockedAddNodeToGroup(node, my_grp_uuid) node.group = my_grp_uuid # Remove from list of groups to add other_grps.remove(other_grp) for grp in other_grps: #TODO: handle node group conflicts my_config.AddNodeGroup(grp, _CLUSTERMERGE_ECID) # R0201: Method could be a function def _StartMasterDaemon(self, no_vote=False): # pylint: disable=R0201 """Starts the local master daemon. @param no_vote: Should the masterd started without voting? default: False @raise errors.CommandError: If unable to start daemon. """ env = {} if no_vote: env["EXTRA_MASTERD_ARGS"] = "--no-voting --yes-do-it" result = utils.RunCmd([pathutils.DAEMON_UTIL, "start-master"], env=env) if result.failed: raise errors.CommandError("Couldn't start ganeti master." " Fail reason: %s; output: %s" % (result.fail_reason, result.output)) def _ReaddMergedNodesAndRedist(self): """Readds all merging nodes and make sure their config is up-to-date. @raise errors.CommandError: If anything fails. """ for data in self.merger_data: for node in data.nodes: logging.info("Readding node %s", node) result = utils.RunCmd(["gnt-node", "add", "--readd", "--no-ssh-key-check", node]) if result.failed: logging.error("%s failed to be readded. Reason: %s, output: %s", node, result.fail_reason, result.output) result = utils.RunCmd(["gnt-cluster", "redist-conf"]) if result.failed: raise errors.CommandError("Redistribution failed. Fail reason: %s;" " output: %s" % (result.fail_reason, result.output)) # R0201: Method could be a function def _StartupAllInstances(self): # pylint: disable=R0201 """Starts up all instances (locally). @raise errors.CommandError: If unable to start clusters """ result = utils.RunCmd(["gnt-instance", "startup", "--all", "--force-multiple"]) if result.failed: raise errors.CommandError("Unable to start all instances." " Fail reason: %s; output: %s" % (result.fail_reason, result.output)) # R0201: Method could be a function # TODO: make this overridable, for some verify errors def _VerifyCluster(self): # pylint: disable=R0201 """Runs gnt-cluster verify to verify the health. @raise errors.ProgrammError: If cluster fails on verification """ result = utils.RunCmd(["gnt-cluster", "verify"]) if result.failed: raise errors.CommandError("Verification of cluster failed." " Fail reason: %s; output: %s" % (result.fail_reason, result.output)) def Merge(self): """Does the actual merge. It runs all the steps in the right order and updates the user about steps taken. Also it keeps track of rollback_steps to undo everything. """ rbsteps = [] try: logging.info("Pre cluster verification") self._VerifyCluster() logging.info("Prepare authorized_keys") rbsteps.append("Remove our key from authorized_keys on nodes:" " %(nodes)s") self._PrepareAuthorizedKeys() rbsteps.append("Start all instances again on the merging" " clusters: %(clusters)s") if self.stop_instances: logging.info("Stopping merging instances (takes a while)") self._StopMergingInstances() logging.info("Checking that no instances are running on the mergees") instances_running = self._CheckRunningInstances() if instances_running: raise errors.CommandError("Some instances are still running on the" " mergees") logging.info("Disable watcher") self._DisableWatcher() logging.info("Merging config") self._FetchRemoteConfig() logging.info("Removing master IPs on mergee master nodes") self._RemoveMasterIps() logging.info("Stop daemons on merging nodes") self._StopDaemons() logging.info("Stopping master daemon") self._KillMasterDaemon() rbsteps.append("Restore %s from another master candidate" " and restart master daemon" % pathutils.CLUSTER_CONF_FILE) self._MergeConfig() self._StartMasterDaemon(no_vote=True) # Point of no return, delete rbsteps del rbsteps[:] logging.warning("We are at the point of no return. Merge can not easily" " be undone after this point.") logging.info("Readd nodes") self._ReaddMergedNodesAndRedist() logging.info("Merge done, restart master daemon normally") self._KillMasterDaemon() self._StartMasterDaemon() if self.restart == _RESTART_ALL: logging.info("Starting instances again") self._StartupAllInstances() else: logging.info("Not starting instances again") logging.info("Post cluster verification") self._VerifyCluster() except errors.GenericError as e: logging.exception(e) if rbsteps: nodes = Flatten([data.nodes for data in self.merger_data]) info = { "clusters": self.clusters, "nodes": nodes, } logging.critical("In order to rollback do the following:") for step in rbsteps: logging.critical(" * %s", step % info) else: logging.critical("Nothing to rollback.") # TODO: Keep track of steps done for a flawless resume? def Cleanup(self): """Clean up our environment. This cleans up remote private keys and configs and after that deletes the temporary directory. """ shutil.rmtree(self.work_dir) def main(): """Main routine. """ program = os.path.basename(sys.argv[0]) parser = optparse.OptionParser(usage="%%prog [options...] ", prog=program) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option(PAUSE_PERIOD_OPT) parser.add_option(GROUPS_OPT) parser.add_option(RESTART_OPT) parser.add_option(PARAMS_OPT) parser.add_option(SKIP_STOP_INSTANCES_OPT) (options, args) = parser.parse_args() utils.SetupToolLogging(options.debug, options.verbose) if not args: parser.error("No clusters specified") cluster_merger = Merger(utils.UniqueSequence(args), options.pause_period, options.groups, options.restart, options.params, options.stop_instances) try: try: cluster_merger.Setup() cluster_merger.Merge() except errors.GenericError as e: logging.exception(e) return constants.EXIT_FAILURE finally: cluster_merger.Cleanup() return constants.EXIT_SUCCESS if __name__ == "__main__": sys.exit(main()) ganeti-3.1.0~rc2/tools/confd-client000075500000000000000000000223761476477700300172510ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # pylint: disable=C0103 """confd client program This is can be used to test and debug confd daemon functionality. """ from __future__ import print_function import sys import optparse import time from ganeti import constants from ganeti import cli from ganeti import utils from ganeti import pathutils from ganeti.confd import client as confd_client USAGE = ("\tconfd-client [--addr=host] [--hmac=key]") LOG_HEADERS = { 0: "- ", 1: "* ", 2: "", } OPTIONS = [ cli.cli_option("--hmac", dest="hmac", default=None, help="Specify HMAC key instead of reading" " it from the filesystem", metavar=""), cli.cli_option("-a", "--address", dest="mc", default="localhost", help="Server IP to query (default: 127.0.0.1)", metavar="
"), cli.cli_option("-r", "--requests", dest="requests", default=100, help="Number of requests for the timing tests", type="int", metavar=""), ] def Log(msg, *args, **kwargs): """Simple function that prints out its argument. """ if args: msg = msg % args indent = kwargs.get("indent", 0) sys.stdout.write("%*s%s%s\n" % (2 * indent, "", LOG_HEADERS.get(indent, " "), msg)) sys.stdout.flush() def LogAtMost(msgs, count, **kwargs): """Log at most count of given messages. """ for m in msgs[:count]: Log(m, **kwargs) if len(msgs) > count: Log("...", **kwargs) def Err(msg, exit_code=1): """Simple error logging that prints to stderr. """ sys.stderr.write(msg + "\n") sys.stderr.flush() sys.exit(exit_code) def Usage(): """Shows program usage information and exits the program.""" print("Usage:", file=sys.stderr) print(USAGE, file=sys.stderr) sys.exit(2) class TestClient(object): """Confd test client.""" def __init__(self): """Constructor.""" self.opts = None self.cluster_master = None self.instance_ips = None self.is_timing = False self.ParseOptions() def ParseOptions(self): """Parses the command line options. In case of command line errors, it will show the usage and exit the program. """ parser = optparse.OptionParser(usage="\n%s" % USAGE, version=("%%prog (ganeti) %s" % constants.RELEASE_VERSION), option_list=OPTIONS) options, args = parser.parse_args() if args: Usage() if options.hmac is None: options.hmac = utils.ReadFile(pathutils.CONFD_HMAC_KEY) self.hmac_key = options.hmac self.mc_list = [options.mc] self.opts = options @staticmethod def _ProcessNotOk(reply, reqtype, is_timing): Log("Query %s gave non-ok status %s: %s" % (reply.orig_request, reply.server_reply.status, reply.server_reply)) if is_timing: Err("Aborting timing tests") if reqtype == constants.CONFD_REQ_CLUSTER_MASTER: Err("Cannot continue after master query failure") if reqtype == constants.CONFD_REQ_INSTANCES_IPS_LIST: Err("Cannot continue after instance IP list query failure") @staticmethod def _ProcessIpList(answer): Log("Instance primary IP query: OK") if not answer: Log("no IPs received", indent=1) else: LogAtMost(answer, 5, indent=1) @staticmethod def _ProcessMapping(answer): Log("Instance IP to node IP query: OK") if not answer: Log("no mapping received", indent=1) else: LogAtMost(answer, 5, indent=1) def ConfdCallback(self, reply): """Callback for confd queries""" if reply.type == confd_client.UPCALL_REPLY: answer = reply.server_reply.answer reqtype = reply.orig_request.type if reply.server_reply.status != constants.CONFD_REPL_STATUS_OK: self._ProcessNotOk(reply, reqtype, self.is_timing) return if self.is_timing: return if reqtype == constants.CONFD_REQ_PING: Log("Ping: OK") elif reqtype == constants.CONFD_REQ_CLUSTER_MASTER: Log("Master: OK (%s)", answer) if self.cluster_master is None: # only assign the first time, in the plain query self.cluster_master = answer elif reqtype == constants.CONFD_REQ_NODE_ROLE_BYNAME: if answer == constants.CONFD_NODE_ROLE_MASTER: Log("Node role for master: OK",) else: Err("Node role for master: wrong: %s" % answer) elif reqtype == constants.CONFD_REQ_NODE_PIP_LIST: Log("Node primary ip query: OK") LogAtMost(answer, 5, indent=1) elif reqtype == constants.CONFD_REQ_MC_PIP_LIST: Log("Master candidates primary IP query: OK") LogAtMost(answer, 5, indent=1) elif reqtype == constants.CONFD_REQ_INSTANCES_IPS_LIST: self.instance_ips = self._ProcessIpList(answer) elif reqtype == constants.CONFD_REQ_NODE_PIP_BY_INSTANCE_IP: self._ProcessMapping(answer) else: Log("Unhandled reply %s, please fix the client", reqtype) print(answer) def DoConfdRequestReply(self, req): self.confd_counting_callback.RegisterQuery(req.rsalt) self.confd_client.SendRequest(req, async_=False) while not self.confd_counting_callback.AllAnswered(): if not self.confd_client.ReceiveReply(): Err("Did not receive all expected confd replies") break def TestConfd(self): """Run confd queries for the cluster. """ Log("Checking confd results") filter_callback = confd_client.ConfdFilterCallback(self.ConfdCallback) counting_callback = confd_client.ConfdCountingCallback(filter_callback) self.confd_counting_callback = counting_callback self.confd_client = confd_client.ConfdClient(self.hmac_key, self.mc_list, counting_callback) tests = [ {"type": constants.CONFD_REQ_PING}, {"type": constants.CONFD_REQ_CLUSTER_MASTER}, {"type": constants.CONFD_REQ_CLUSTER_MASTER, "query": {constants.CONFD_REQQ_FIELDS: [str(constants.CONFD_REQFIELD_NAME), str(constants.CONFD_REQFIELD_IP), str(constants.CONFD_REQFIELD_MNODE_PIP), ]}}, {"type": constants.CONFD_REQ_NODE_ROLE_BYNAME}, {"type": constants.CONFD_REQ_NODE_PIP_LIST}, {"type": constants.CONFD_REQ_MC_PIP_LIST}, {"type": constants.CONFD_REQ_INSTANCES_IPS_LIST, "query": None}, {"type": constants.CONFD_REQ_NODE_PIP_BY_INSTANCE_IP}, ] for kwargs in tests: if kwargs["type"] == constants.CONFD_REQ_NODE_ROLE_BYNAME: assert self.cluster_master is not None kwargs["query"] = self.cluster_master elif kwargs["type"] == constants.CONFD_REQ_NODE_PIP_BY_INSTANCE_IP: kwargs["query"] = {constants.CONFD_REQQ_IPLIST: self.instance_ips} req = confd_client.ConfdClientRequest(**kwargs) self.DoConfdRequestReply(req) def TestTiming(self): """Run timing tests. """ # timing tests if self.opts.requests <= 0: return Log("Timing tests") self.is_timing = True self.TimingOp("ping", {"type": constants.CONFD_REQ_PING}) self.TimingOp("instance ips", {"type": constants.CONFD_REQ_INSTANCES_IPS_LIST}) def TimingOp(self, name, kwargs): """Run a single timing test. """ start = time.time() for _ in range(self.opts.requests): req = confd_client.ConfdClientRequest(**kwargs) self.DoConfdRequestReply(req) stop = time.time() per_req = 1000 * (stop - start) / self.opts.requests Log("%.3fms per %s request", per_req, name, indent=1) def Run(self): """Run all the tests. """ self.TestConfd() self.TestTiming() def main(): """Main function. """ return TestClient().Run() if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/fmtjson000075500000000000000000000034601476477700300163550ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tool to format JSON data. """ import sys import json def main(): """Main routine. """ if len(sys.argv) > 1: sys.stderr.write("Read JSON data from standard input and write a" " formatted version on standard output. There are" " no options or arguments.\n") sys.exit(1) data = json.load(sys.stdin) json.dump(data, sys.stdout, indent=2, sort_keys=True) sys.stdout.write("\n") if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/ganeti-listrunner000075500000000000000000000471341476477700300203550ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2010, 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Run an executable on a list of hosts. Script to serially run an executable on a list of hosts via ssh with password auth as root. If the provided log dir does not yet exist, it will try to create it. Implementation: - the main process spawns up to batch_size children, which: - connects to the remote host via ssh as root - uploads the executable with a random name to /tmp via sftp - chmod 500s it - via ssh: chdirs into the upload directory and runs the script - deletes it - writes status messages and all output to one logfile per host - the main process gathers then the status of the children and reports the success/failure ratio - entire script can be aborted with Ctrl-C Security considerations: - the root password for the remote hosts is stored in memory for the runtime of the script - the executable to be run on the remote host is handled the following way: - try to create a random directory with permissions 700 on the remote host, abort furter processing on this host if this failes - upload the executable with to a random filename in that directory - set executable permissions to 500 - run the executable - delete the execuable and the directory on the remote host """ # pylint: disable=C0103 # C0103: Invalid name ganeti-listrunner from __future__ import print_function import errno import optparse import getpass import logging import os import random import select import socket import sys import time import traceback try: import paramiko except ImportError: print("The \"paramiko\" module could not be imported. Install it from your" " distribution's repository. The package is usually named" " \"python-paramiko\".", file=sys.stderr) sys.exit(1) REMOTE_PATH_BASE = "/tmp/listrunner" USAGE = ("%prog -l logdir {-c command | -x /path/to/file} [-b batch_size]" " {-f hostfile|-h hosts} [-u username]" " [-p password_file | -A]") def LogDirUseable(logdir): """Ensure log file directory is available and usable.""" testfile = "%s/test-%s-%s.deleteme" % (logdir, random.random(), random.random()) try: os.mkdir(logdir) except OSError as err: if err.errno != errno.EEXIST: raise try: logtest = open(testfile, "a+") logtest.writelines("log file writeability test\n") logtest.close() os.unlink(testfile) return True except (OSError, IOError): return False def GetTimeStamp(timestamp=None): """Return ISO8601 timestamp. Returns ISO8601 timestamp, optionally expects a time.localtime() tuple in timestamp, but will use the current time if this argument is not supplied. """ if timestamp is None: timestamp = time.localtime() isotime = time.strftime("%Y-%m-%dT%H:%M:%S", timestamp) return isotime def PingByTcp(target, port, timeout=10, live_port_needed=False, source=None): """Simple ping implementation using TCP connect(2). Try to do a TCP connect(2) from an optional source IP to the specified target IP and the specified target port. If the optional parameter live_port_needed is set to true, requires the remote end to accept the connection. The timeout is specified in seconds and defaults to 10 seconds. If the source optional argument is not passed, the source address selection is left to the kernel, otherwise we try to connect using the passed address (failures to bind other than EADDRNOTAVAIL will be ignored). """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) success = False if source is not None: try: sock.bind((source, 0)) except socket.error as err: if err.errno == errno.EADDRNOTAVAIL: success = False sock.settimeout(timeout) try: sock.connect((target, port)) sock.close() success = True except socket.timeout: success = False except socket.error as err: success = (not live_port_needed) and (err.errno == errno.ECONNREFUSED) return success def GetHosts(hostsfile): """Return list of hosts from hostfile. Reads the hostslist file and returns a list of hosts. Expects the hostslist file to contain one hostname per line. """ try: datafile = open(hostsfile, "r") except IOError as msg: print("Failed to open hosts file %s: %s" % (hostsfile, msg)) sys.exit(2) hosts = datafile.readlines() datafile.close() return hosts def WriteLog(message, logfile): """Writes message, terminated by newline, to logfile.""" try: logfile = open(logfile, "a+") except IOError as msg: print("failed to open log file %s: %s" % (logfile, msg)) print("log message was: %s" % message) sys.exit(1) # no being able to log is critical try: timestamp = GetTimeStamp() logfile.writelines("%s %s\n" % (timestamp, message)) logfile.close() except IOError as msg: print("failed to write to logfile %s: %s" % (logfile, msg)) print("log message was: %s" % message) sys.exit(1) # no being able to log is critical def GetAgentKeys(): """Tries to get a list of ssh keys from an agent.""" try: agent = paramiko.Agent() return list(agent.get_keys()) except paramiko.SSHException: return [] def SetupSshConnection(host, username, password, use_agent, logfile): """Setup the ssh connection used for all later steps. This function sets up the ssh connection that will be used both for upload and remote command execution. On success, it will return paramiko.Transport object with an already logged in session. On failure, False will be returned. """ # check if target is willing to talk to us at all if not PingByTcp(host, 22, live_port_needed=True): WriteLog("ERROR: FAILURE_NOT_REACHABLE", logfile) print(" - ERROR: host not reachable on 22/tcp") return False if use_agent: keys = GetAgentKeys() else: keys = [] all_kwargs = [{"pkey": k} for k in keys] all_desc = ["key %d" % d for d in range(len(keys))] if password is not None: all_kwargs.append({"password": password}) all_desc.append("password") # deal with logging out of paramiko.transport handler = None for desc, kwargs in zip(all_desc, all_kwargs): try: transport = paramiko.Transport((host, 22)) # only try to setup the logging handler once if not handler: handler = logging.StreamHandler() handler.setLevel(logging.ERROR) log = logging.getLogger(transport.get_log_channel()) log.addHandler(handler) transport.connect(username=username, **kwargs) WriteLog("ssh connection established using %s" % desc, logfile) # strange ... when establishing the session and the immediately # setting up the channels for sftp & shell from that, it sometimes # fails, but waiting 1 second after session setup makes it always work # time.sleep(1) # FIXME apparently needfull to give sshd some time return transport except (socket.gaierror, socket.error, paramiko.SSHException): continue methods = ", ".join(all_desc) WriteLog("ERROR: FAILURE_CONNECTION_SETUP (tried %s) " % methods, logfile) WriteLog("aborted", logfile) print(" - ERROR: connection setup failed (tried %s)" % methods) return False def UploadFiles(connection, executable, filelist, logfile): """Uploads the specified files via sftp. Uploads the specified files to a random, freshly created directory with a temporary name under /tmp. All uploaded files are chmod 0o400 after upload with the exception of executable, with is chmod 500. Upon success, returns the absolute path to the remote upload directory, but will return False upon failure. """ remote_dir = "%s.%s-%s" % (REMOTE_PATH_BASE, random.random(), random.random()) try: sftp = paramiko.SFTPClient.from_transport(connection) sftp.mkdir(remote_dir, mode=0o700) for item in filelist: remote_file = "%s/%s" % (remote_dir, os.path.basename(item)) WriteLog("uploading %s to remote %s" % (item, remote_file), logfile) sftp.put(item, remote_file) if item == executable: sftp.chmod(remote_file, 0o500) else: sftp.chmod(remote_file, 0o400) sftp.close() except IOError as err: WriteLog("ERROR: FAILURE_UPLOAD: %s" % err, logfile) return False return remote_dir def CleanupRemoteDir(connection, upload_dir, filelist, logfile): """Cleanes out and removes the remote work directory.""" try: sftp = paramiko.SFTPClient.from_transport(connection) for item in filelist: fullpath = "%s/%s" % (upload_dir, os.path.basename(item)) WriteLog("removing remote %s" % fullpath, logfile) sftp.remove(fullpath) sftp.rmdir(upload_dir) sftp.close() except IOError as err: WriteLog("ERROR: FAILURE_CLEANUP: %s" % err, logfile) return False return True def RunRemoteCommand(connection, command, logfile): """Execute the command via ssh on the remote host.""" session = connection.open_session() session.setblocking(0) # the following dance is needed because paramiko changed APIs: # from returning True/False for success to always returning None # and throwing an exception in case of problems. # And I want to support both the old and the new API. result = True # being optimistic here, I know message = None try: if session.exec_command("%s 2>&1" % command) is False: result = False except paramiko.SSHException as message: result = False if not result: WriteLog("ERROR: FAILURE_COMMAND_EXECUTION: %s" % message, logfile) return False ### Read when data is available output = "" while select.select([session], [], []): try: data = session.recv(1024) except socket.timeout as err: data = None WriteLog("FAILED: socket.timeout %s" % err, logfile) except socket.error as err: data = None WriteLog("FAILED: socket.error %s" % err, logfile) if not data: break output += data select.select([], [], [], .1) WriteLog("SUCCESS: command output follows", logfile) for line in output.splitlines(): WriteLog("output = %s" % line, logfile) WriteLog("command execution completed", logfile) session.close() return True def HostWorker(logdir, username, password, use_agent, hostname, executable, exec_args, command, filelist): """Per-host worker. This function does not return - it's the main code of the childs, which exit at the end of this function. The exit code 0 or 1 will be interpreted by the parent. @param logdir: the directory where the logfiles must be created @param username: SSH username @param password: SSH password @param use_agent: whether we should instead use an agent @param hostname: the hostname to connect to @param executable: the executable to upload, if not None @param exec_args: Additional arguments for executable @param command: the command to run @param filelist: auxiliary files to upload """ # in the child/worker process logfile = "%s/%s.log" % (logdir, hostname) print("%s - starting" % hostname) result = 0 # optimism, I know try: connection = SetupSshConnection(hostname, username, password, use_agent, logfile) if connection is not False: if executable is not None: print(" %s: uploading files" % hostname) upload_dir = UploadFiles(connection, executable, filelist, logfile) command = ("cd %s && ./%s" % (upload_dir, os.path.basename(executable))) if exec_args: command += " %s" % exec_args print(" %s: executing remote command" % hostname) cmd_result = RunRemoteCommand(connection, command, logfile) if cmd_result is True: print(" %s: remote command execution successful" % hostname) else: print(" %s: remote command execution failed," " check log for details" % hostname) result = 1 if executable is not None: print(" %s: cleaning up remote work dir" % hostname) cln_result = CleanupRemoteDir(connection, upload_dir, filelist, logfile) if cln_result is False: print(" %s: remote work dir cleanup failed, check" " log for details" % hostname) result = 1 connection.close() else: print(" %s: connection setup failed, skipping" % hostname) result = 1 except KeyboardInterrupt: print(" %s: received KeyboardInterrupt, aborting" % hostname) WriteLog("ERROR: ABORT_KEYBOARD_INTERRUPT", logfile) result = 1 except Exception as err: # pylint: disable=W0703 result = 1 trace = traceback.format_exc() msg = "ERROR: UNHANDLED_EXECPTION_ERROR: %s\nTrace: %s" % (err, trace) WriteLog(msg, logfile) print(" %s: %s" % (hostname, msg)) # and exit with exit code 0 or 1, so the parent can compute statistics sys.exit(result) def LaunchWorker(child_pids, logdir, username, password, use_agent, hostname, executable, exec_args, command, filelist): """Launch the per-host worker. Arguments are the same as for HostWorker, except for child_pids, which is a dictionary holding the pid-to-hostname mapping. """ hostname = hostname.rstrip("\n") pid = os.fork() if pid > 0: # controller just record the pids child_pids[pid] = hostname else: HostWorker(logdir, username, password, use_agent, hostname, executable, exec_args, command, filelist) def ParseOptions(): """Parses the command line options. In case of command line errors, it will show the usage and exit the program. @return: the options in a tuple """ # resolve because original used -h for hostfile, which conflicts # with -h for help parser = optparse.OptionParser(usage="\n%s" % USAGE, conflict_handler="resolve") parser.add_option("-l", dest="logdir", default=None, help="directory to write logfiles to") parser.add_option("-x", dest="executable", default=None, help="executable to run on remote host(s)",) parser.add_option("-f", dest="hostfile", default=None, help="hostlist file (one host per line)") parser.add_option("-h", dest="hostlist", default=None, metavar="HOSTS", help="comma-separated list of hosts or single hostname",) parser.add_option("-a", dest="auxfiles", action="append", default=[], help="optional auxiliary file to upload" " (can be given multiple times)", metavar="FILE") parser.add_option("-c", dest="command", default=None, help="shell command to run on remote host(s)") parser.add_option("-b", dest="batch_size", default=15, type="int", help="batch-size, how many hosts to process" " in parallel [15]") parser.add_option("-u", dest="username", default="root", help="username used to connect [root]") parser.add_option("-p", dest="password", default=None, help="password used to authenticate (when not" " using an agent)") parser.add_option("-A", dest="use_agent", default=False, action="store_true", help="instead of password, use keys from an SSH agent") parser.add_option("--args", dest="exec_args", default=None, help="Arguments to be passed to executable (-x)") opts, args = parser.parse_args() if opts.executable and opts.command: parser.error("Options -x and -c conflict with each other") if not (opts.executable or opts.command): parser.error("One of -x and -c must be given") if opts.command and opts.exec_args: parser.error("Can't specify arguments when using custom command") if not opts.logdir: parser.error("Option -l is required") if opts.hostfile and opts.hostlist: parser.error("Options -f and -h conflict with each other") if not (opts.hostfile or opts.hostlist): parser.error("One of -f or -h must be given") if args: parser.error("This program doesn't take any arguments, passed in: %s" % ", ".join(args)) return (opts.logdir, opts.executable, opts.exec_args, opts.hostfile, opts.hostlist, opts.command, opts.use_agent, opts.auxfiles, opts.username, opts.password, opts.batch_size) def _GetPassword(use_agent, password, username): if use_agent: pass elif password: try: fh = open(password) pwvalue = fh.readline().strip() fh.close() except IOError as e: print("error: can not read in from password file %s: %s" % (password, e)) sys.exit(1) password = pwvalue else: password = getpass.getpass("%s's password for all nodes: " % username) return password def _GetHosts(hostfile, hostlist): if hostfile: hosts = GetHosts(hostfile) else: if "," in hostlist: hostlist = hostlist.rstrip(",") # commandline robustness hosts = hostlist.split(",") else: hosts = [hostlist] return hosts def main(): """main.""" (logdir, executable, exec_args, hostfile, hostlist, command, use_agent, auxfiles, username, password, batch_size) = ParseOptions() ### Unbuffered sys.stdout sys.stdout = os.fdopen(1, "w", 0) if LogDirUseable(logdir) is False: print("ERROR: cannot create logfiles in dir %s, aborting" % logdir) sys.exit(1) password = _GetPassword(use_agent, password, username) hosts = _GetHosts(hostfile, hostlist) successes = failures = 0 filelist = auxfiles[:] filelist.append(executable) # initial batch batch = hosts[:batch_size] hosts = hosts[batch_size:] child_pids = {} for hostname in batch: LaunchWorker(child_pids, logdir, username, password, use_agent, hostname, executable, exec_args, command, filelist) while child_pids: pid, status = os.wait() hostname = child_pids.pop(pid, "") print(" %s: done (in parent)" % hostname) if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0: successes += 1 else: failures += 1 if hosts: LaunchWorker(child_pids, logdir, username, password, use_agent, hosts.pop(0), executable, exec_args, command, filelist) print() print("All done, %s successful and %s failed hosts" % (successes, failures)) sys.exit(0) if __name__ == "__main__": try: main() except KeyboardInterrupt: print("Received KeyboardInterrupt, aborting") sys.exit(1) ganeti-3.1.0~rc2/tools/ifup-os.in000064400000000000000000000226071476477700300166650ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # This script is a hook called to configure new TAP network interfaces # used for instance communication, and it should be called whenever a # new instance is started. # # This script configures the new interface but it also performs # maintenance on the network interfaces that have been configured # before, by checking whether those TAP interfaces still exist, etc. # # This script also controls the DHCP server that leases IP address for # instances, i.e., the NICs inside the instances, not the TAP # interfaces. The DHCP server is started and restarted # as necessary and always with up-to-date configuration files. # # This script expects the following environment variables # # INTERFACE: network interface name to be configured # MODE: networking mode for 'INTERFACE' (must be 'routed') # MAC: MAC address for 'INTERFACE' # IP: IP address for 'INTERFACE' source @PKGLIBDIR@/net-common readonly NETMASK=255.255.255.255 readonly DNSMASQ_CONF=/var/run/ganeti/dnsmasq.conf readonly DNSMASQ_HOSTS=/var/run/ganeti/dnsmasq.hosts readonly DNSMASQ_PID=/var/run/ganeti/dnsmasq.pid # join intercalates a sequence of arguments using the given separator function join { local IFS="$1" shift echo "$*" } # restart_dnsmasq restarts the DHCP server dnsmasq with the (possibly # up-to-date) configuration file. # # If all instances have been terminated, which means there are no more # TAP network interfaces to monitor or IP addresses to lease, the DHCP # server is terminated through 'SIGTERM'. # # If there are still instances running, it will be restarted and the # configuration file will be passed it. function restart_dnsmasq { local RUNNING= local PID if [ -f "$DNSMASQ_PID" ] then PID=$(cat $DNSMASQ_PID) if [ -n "$PID" ] && ps -p "$PID" then RUNNING=yes fi fi if [ "$RUNNING" = yes ] then kill -TERM $PID # wait for the process to die while kill -0 $PID 2>/dev/null do sleep 1 done rm -f $DNSMASQ_PID fi if [ -n "$ALIVE_INTERFACES" -a -n "$ALIVE_LEASES" ] then dnsmasq -C $DNSMASQ_CONF fi return 0 } # Check that environment variable 'INTERFACE' exists. # # This environment variable holds the TAP network interface that # should be configured by this script. Ganeti always passes it, # but... :) if [ -z "$INTERFACE" ] then echo ifup-os: Failed to configure communication mechanism \ interface because the \'INTERFACE\' environment variable was \ not specified to the script exit 1 fi # Check that environment variable 'MODE' exists. # # See comment about environment variable 'INTERFACE'. if [ -z "$MODE" ] then echo ifup-os: Failed to configure communication mechanism \ interface because the \'MODE\' environment variable was \ not specified to the script exit 1 fi # Check whether the interface being configured has instance # communication enabled, otherwise exit this script. if ! is_instance_communication_tap; then exit 0; fi # Check that environment variable 'MAC' exists. # # See comment about environment variable 'INTERFACE'. if [ -z "$MAC" ] then echo ifup-os: Failed to configure communication mechanism \ interface because the \'MAC\' environment variable was \ not specified to the script exit 1 fi # Check that environment variable 'IP' exists. # # See comment about environment variable 'INTERFACE'. if [ -z "$IP" ] then echo ifup-os: Failed to configure communication mechanism \ interface because the \'IP\' environment variable was \ not specified to the script exit 1 fi # Configure the TAP interface # # Ganeti defers the configuration of instance network interfaces to # hooks, therefore, we must configure the interface's network address, # netmask, and IP address. # # The TAP network interface, which is used by the instance # communication, is part of the network 169.254.0.0/16 and has the IP # 169.254.169.254. Because all network interfaces used in the # instance communication have the same IP, the routing table must also # be configured, and that is done at a later step. # # Note the interface must be marked as up before configuring the # routing table and before starting/restarting the DHCP server. # # Note also that we don't have to check whether the interface is # already configured because reconfiguring the interface with the same # parameters does not produce an error. ip link set $INTERFACE up ip addr add 169.254.169.254/$NETMASK dev $INTERFACE # There is a known bug where UDP packets comming from a host to a XEN # guest are missing checksums. There are several ways how to tackle the # issue, for example fixing the checksums using iptables (requires a # newer version): # # iptables -A POSTROUTING -t mangle -p udp --dport bootpc -j CHECKSUM \ # --checksum-fill # # The easiest one currently seems to be to just turn checksumming off # for this direction: ethtool -K $INTERFACE tx off || true # Configure the routing table # # Given that all TAP network interfaces in the instance communication # have the same IP address, the routing table must be configured in # order to properly route traffic from the host to the guests. # # Note that we must first check if a duplicate routing rule has # already been added to the routing table, as this operation will fail # if we try to add a routing rule that already exists. ACTIVE_IP=$(ip route | grep "dev $INTERFACE" | awk '{ print $1 }') if [ -z "$ACTIVE_IP" -o "$ACTIVE_IP" != "$IP" ] then ip route add $IP/32 dev $INTERFACE fi # Ensure the DHCP server configuration files exist touch $DNSMASQ_CONF chmod 0644 $DNSMASQ_CONF touch $DNSMASQ_HOSTS chmod 0644 $DNSMASQ_HOSTS # Determine dnsmasq operational mode. # # The DHCP server dnsmasq can run in different modes. In this version # of the script, only the mode 'bind-dynamic' is supported. Please # refer to the dnsmasq FAQ for a detailed of each mode. # # Note that dnsmasq might already be running, therefore, we don't need # to determine which modes are supported by this DHCP server. # Instead, we just read the current mode from the configuration file. DNSMASQ_MODE=$(head -n 1 $DNSMASQ_CONF) if [ -z "$DNSMASQ_MODE" ] then BIND_DYNAMIC=$(dnsmasq --help | grep -e --bind-dynamic) if [ -z "$BIND_DYNAMIC" ] then echo ifup-os: dnsmasq mode \"bind-dynamic\" is not supported exit 1 fi DNSMASQ_MODE=bind-dynamic fi # Determine the interfaces that should go in the configuration file. # # The TAP network interfaces used by the instance communication are # named after the following pattern # # gnt.com.%d # # where '%d' is a unique number within the host. Fortunately, dnsmasq # supports binding to specific network interfaces via a pattern. ALIVE_INTERFACES=${GANETI_TAP}.* # Determine which of the leases are not duplicated and should go in # the new configuration file for the DHCP server. # # Given that instances come and go, it is possible that we offer more # leases that necessary and, worse, that we have duplicate leases, # that is, the same IP address for the same/different MAC addresses. # Duplicate leases must be eliminated before being written to the # configuration file. CONF_LEASES=$(cat $DNSMASQ_HOSTS) CONF_LEASES=$(join $'\n' $CONF_LEASES | sort -u) ALIVE_LEASES=( $MAC,$IP ) for i in $CONF_LEASES do LEASE_MAC=$(echo $i | cut -d "," -f 1) LEASE_IP=$(echo $i | cut -d "," -f 2) if [ "$LEASE_MAC" != "$MAC" -a "$LEASE_IP" != "$IP" ] then ALIVE_LEASES=( ${ALIVE_LEASES[@]} $i ) fi done ALIVE_LEASES=$(echo ${ALIVE_LEASES[@]} | sort -u) # Update dnsmasq configuration. # # Write the parameters we have collected before into the new dnsmasq # configuration file. Also, write the new leases into the new dnsmasq # hosts file. Finally, restart dnsmasq with the new configuration # files. cat > $DNSMASQ_CONF <> $DNSMASQ_CONF; done echo -n > $DNSMASQ_HOSTS for i in $ALIVE_LEASES; do echo $i >> $DNSMASQ_HOSTS; done restart_dnsmasq ganeti-3.1.0~rc2/tools/kvm-console-wrapper000075500000000000000000000040631476477700300206100ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. SOCAT="$1" INSTANCE="$2" MONITOR="$3" PARAMS="$4" CONSOLE="$5" unpause() { echo "info status" | "$SOCAT" STDIO "UNIX-CONNECT:$MONITOR" 2>/dev/null | grep -q '^VM status: paused' || return # As there is no way to be sure when the main socat has actually connected to # the instance console, sleep for a few seconds before unpausing the # instance. This is a tradeoff between missing some console output if the # node is overloaded and making the user wait everytime when the node isn't # so busy. sleep 3 # Send \r\n after notice as terminal is in raw mode printf "Instance $INSTANCE is paused, unpausing\r\n" echo "c" | "$SOCAT" STDIO "UNIX-CONNECT:$MONITOR" &>/dev/null } unpause & exec "$SOCAT" "$PARAMS" "$CONSOLE" ganeti-3.1.0~rc2/tools/kvm-ifup.in000064400000000000000000000034131476477700300170330ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2011, 2012, 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. source @PKGLIBDIR@/net-common check # Execute the script for setting up the communication with the # instance OS if is_instance_communication_tap && [ -x "$CONF_DIR/kvm-ifup-os" ]; then . $CONF_DIR/kvm-ifup-os fi # Execute the user-supplied network script, if applicable if [ -x "$CONF_DIR/kvm-vif-bridge" ]; then exec $CONF_DIR/kvm-vif-bridge fi if ! is_instance_communication_tap; then setup_bridge setup_ovs setup_route fi ganeti-3.1.0~rc2/tools/lvmstrap000075500000000000000000000726701476477700300165560ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2006, 2007, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Program which configures LVM on the Ganeti nodes. This program wipes disks and creates a volume group on top of them. It can also show disk information to help you decide which disks you want to wipe. The error handling is done by raising our own exceptions from most of the functions; these exceptions then handled globally in the main() function. The exceptions that each function can raise are not documented individually, since almost every error path ends in a raise. Another two exceptions that are handled globally are IOError and OSError. The idea behind this is, since we run as root, we should usually not get these errors, but if we do it's most probably a system error, so they should be handled and the user instructed to report them. """ from __future__ import print_function import os import sys import optparse import time import errno import re from ganeti.utils import RunCmd, ReadFile from ganeti import constants from ganeti import cli from ganeti import compat USAGE = ("\tlvmstrap diskinfo\n" "\tlvmstrap [--vg-name=NAME] [--allow-removable]" " { --alldisks | --disks DISKLIST } [--use-sfdisk]" " create") verbose_flag = False #: Supported disk types (as prefixes) SUPPORTED_TYPES = [ "hd", "sd", "md", "ubd", ] #: Excluded filesystem types EXCLUDED_FS = compat.UniqueFrozenset([ "nfs", "nfs4", "autofs", "tmpfs", "proc", "sysfs", "usbfs", "devpts", ]) #: A regular expression that matches partitions (must be kept in sync # with L{SUPPORTED_TYPES} PART_RE = re.compile("^((?:h|s|m|ub)d[a-z]{1,2})[0-9]+$") #: Minimum partition size to be considered (1 GB) PART_MINSIZE = 1024 * 1024 * 1024 MBR_MAX_SIZE = 2 * (10 ** 12) class Error(Exception): """Generic exception""" pass class ProgrammingError(Error): """Exception denoting invalid assumptions in programming. This should catch sysfs tree changes, or otherwise incorrect assumptions about the contents of the /sys/block/... directories. """ pass class SysconfigError(Error): """Exception denoting invalid system configuration. If the system configuration is somehow wrong (e.g. /dev files missing, or having mismatched major/minor numbers relative to /sys/block devices), this exception will be raised. This should usually mean that the installation of the Xen node failed in some steps. """ pass class PrereqError(Error): """Exception denoting invalid prerequisites. If the node does not meet the requirements for cluster membership, this exception will be raised. Things like wrong kernel version, or no free disks, etc. belong here. This should usually mean that the build steps for the Xen node were not followed correctly. """ pass class OperationalError(Error): """Exception denoting actual errors. Errors during the bootstrapping are signaled using this exception. """ pass class ParameterError(Error): """Exception denoting invalid input from user. Wrong disks given as parameters will be signaled using this exception. """ pass def Usage(): """Shows program usage information and exits the program. """ print("Usage:", file=sys.stderr) print(USAGE, file=sys.stderr) sys.exit(2) def ParseOptions(): """Parses the command line options. In case of command line errors, it will show the usage and exit the program. @rtype: tuple @return: a tuple of (options, args), as returned by OptionParser.parse_args """ global verbose_flag # pylint: disable=W0603 parser = optparse.OptionParser(usage="\n%s" % USAGE, version="%%prog (ganeti) %s" % constants.RELEASE_VERSION) parser.add_option("--alldisks", dest="alldisks", help="erase ALL disks", action="store_true", default=False) parser.add_option("-d", "--disks", dest="disks", help="Choose disks (e.g. hda,hdg)", metavar="DISKLIST") parser.add_option(cli.VERBOSE_OPT) parser.add_option("-r", "--allow-removable", action="store_true", dest="removable_ok", default=False, help="allow and use removable devices too") parser.add_option("-g", "--vg-name", type="string", dest="vgname", default="xenvg", metavar="NAME", help="the volume group to be created [default: xenvg]") parser.add_option("--use-sfdisk", dest="use_sfdisk", action="store_true", default=False, help="use sfdisk instead of parted") options, args = parser.parse_args() if len(args) != 1: Usage() verbose_flag = options.verbose return options, args def IsPartitioned(disk): """Returns whether a given disk should be used partitioned or as-is. Currently only md devices are used as is. """ return not (disk.startswith("md") or PART_RE.match(disk)) def DeviceName(disk): """Returns the appropriate device name for a disk. For non-partitioned devices, it returns the name as is, otherwise it returns the first partition. """ if IsPartitioned(disk): device = "/dev/%s1" % disk else: device = "/dev/%s" % disk return device def SysfsName(disk): """Returns the sysfs name for a disk or partition. """ match = PART_RE.match(disk) if match: # this is a partition, which resides in /sys/block under a different name disk = "%s/%s" % (match.group(1), disk) return "/sys/block/%s" % disk def ExecCommand(command): """Executes a command. This is just a wrapper around commands.getstatusoutput, with the difference that if the command line argument -v has been given, it will print the command line and the command output on stdout. @param command: the command line to be executed @rtype: tuple @return: a tuple of (status, output) where status is the exit status and output the stdout and stderr of the command together """ if verbose_flag: print(command) result = RunCmd(command) if verbose_flag: print(result.output) return result def CheckPrereq(): """Check the prerequisites of this program. It check that it runs on Linux 2.6, and that /sys is mounted and the fact that /sys/block is a directory. """ if os.getuid() != 0: raise PrereqError("This tool runs as root only. Really.") osname, _, release, _, _ = os.uname() if osname != "Linux": raise PrereqError("This tool only runs on Linux" " (detected OS: %s)." % osname) if not (release.startswith("2.6.") or release.startswith("3.")): raise PrereqError("Wrong major kernel version (detected %s, needs" " 2.6.* or 3.*)" % release) if not os.path.ismount("/sys"): raise PrereqError("Can't find a filesystem mounted at /sys." " Please mount /sys.") if not os.path.isdir("/sys/block"): raise SysconfigError("Can't find /sys/block directory. Has the" " layout of /sys changed?") if not os.path.ismount("/proc"): raise PrereqError("Can't find a filesystem mounted at /proc." " Please mount /proc.") if not os.path.exists("/proc/mounts"): raise SysconfigError("Can't find /proc/mounts") def CheckVGExists(vgname): """Checks to see if a volume group exists. @param vgname: the volume group name @return: a four-tuple (exists, lv_count, vg_size, vg_free), where: - exists: True if the volume exists, otherwise False; if False, all other members of the tuple are None - lv_count: The number of logical volumes in the volume group - vg_size: The total size of the volume group (in gibibytes) - vg_free: The available space in the volume group """ result = ExecCommand("vgs --nohead -o lv_count,vg_size,vg_free" " --nosuffix --units g" " --ignorelockingfailure %s" % vgname) if not result.failed: try: lv_count, vg_size, vg_free = result.stdout.strip().split() except ValueError: # This means the output of vgdisplay can't be parsed raise PrereqError("cannot parse output of vgs (%s)" % result.stdout) else: lv_count = vg_size = vg_free = None return not result.failed, lv_count, vg_size, vg_free def CheckSysDev(name, devnum): """Checks consistency between /sys and /dev trees. In /sys/block//dev and /sys/block///dev are the kernel-known device numbers. The /dev/ block/char devices are created by userspace and thus could differ from the kernel view. This function checks the consistency between the device number read from /sys and the actual device number in /dev. Note that since the system could be using udev which removes and recreates the device nodes on partition table rescan, we need to do some retries here. Since we only do a stat, we can afford to do many short retries. @param name: the device name, e.g. 'sda' @param devnum: the device number, e.g. 0x803 (2051 in decimal) for sda3 @raises SysconfigError: in case of failure of the check """ path = "/dev/%s" % name for _ in range(40): if os.path.exists(path): break time.sleep(0.250) else: raise SysconfigError("the device file %s does not exist, but the block" " device exists in the /sys/block tree" % path) rdev = os.stat(path).st_rdev if devnum != rdev: raise SysconfigError("For device %s, the major:minor in /dev is %04x" " while the major:minor in sysfs is %s" % (path, rdev, devnum)) def ReadDev(syspath): """Reads the device number from a sysfs path. The device number is given in sysfs under a block device directory in a file named 'dev' which contains major:minor (in ASCII). This function reads that file and converts the major:minor pair to a dev number. @type syspath: string @param syspath: the path to a block device dir in sysfs, e.g. C{/sys/block/sda} @return: the device number """ if not os.path.exists("%s/dev" % syspath): raise ProgrammingError("Invalid path passed to ReadDev: %s" % syspath) f = open("%s/dev" % syspath) data = f.read().strip() f.close() major, minor = data.split(":", 1) major = int(major) minor = int(minor) dev = os.makedev(major, minor) return dev def ReadSize(syspath): """Reads the size from a sysfs path. The size is given in sysfs under a block device directory in a file named 'size' which contains the number of sectors (in ASCII). This function reads that file and converts the number in sectors to the size in bytes. @type syspath: string @param syspath: the path to a block device dir in sysfs, e.g. C{/sys/block/sda} @rtype: int @return: the device size in bytes """ if not os.path.exists("%s/size" % syspath): raise ProgrammingError("Invalid path passed to ReadSize: %s" % syspath) f = open("%s/size" % syspath) data = f.read().strip() f.close() size = 512 * int(data) return size def ReadPV(name): """Reads physical volume information. This function tries to see if a block device is a physical volume. @type name: string @param name: the device name (e.g. sda) @return: the name of the volume group to which this PV belongs, or "" if this PV is not in use, or None if this is not a PV """ result = ExecCommand("pvdisplay -c /dev/%s" % name) if result.failed: return None vgname = result.stdout.strip().split(":")[1] return vgname def GetDiskList(opts): """Computes the block device list for this system. This function examines the /sys/block tree and using information therein, computes the status of the block device. @return: a list like [(name, size, dev, partitions, inuse), ...], where: - name is the block device name (e.g. sda) - size the size in bytes - dev is the device number (e.g. 8704 for hdg) - partitions is [(name, size, dev), ...] mirroring the disk list data inuse is a boolean showing the in-use status of the disk, computed as the possibility of re-reading the partition table (the meaning of the operation varies with the kernel version, but is usually accurate; a mounted disk/partition or swap-area or PV with active LVs on it is busy) """ dlist = [] for name in os.listdir("/sys/block"): if not compat.any([name.startswith(pfx) for pfx in SUPPORTED_TYPES]): continue disksysfsname = "/sys/block/%s" % name size = ReadSize(disksysfsname) f = open("/sys/block/%s/removable" % name) removable = int(f.read().strip()) f.close() if removable and not opts.removable_ok: continue dev = ReadDev(disksysfsname) CheckSysDev(name, dev) inuse = InUse(name) # Enumerate partitions of the block device partitions = [] for partname in os.listdir(disksysfsname): if not partname.startswith(name): continue partsysfsname = "%s/%s" % (disksysfsname, partname) partdev = ReadDev(partsysfsname) partsize = ReadSize(partsysfsname) if partsize >= PART_MINSIZE: CheckSysDev(partname, partdev) partinuse = InUse(partname) partitions.append((partname, partsize, partdev, partinuse)) partitions.sort() dlist.append((name, size, dev, partitions, inuse)) dlist.sort() return dlist def GetMountInfo(): """Reads /proc/mounts and computes the mountpoint-devnum mapping. This function reads /proc/mounts, finds the mounted filesystems (excepting a hard-coded blacklist of network and virtual filesystems) and does a stat on these mountpoints. The st_dev number of the results is memorised for later matching against the /sys/block devices. @rtype: dict @return: a {mountpoint: device number} dictionary """ mountlines = ReadFile("/proc/mounts").splitlines() mounts = {} for line in mountlines: _, mountpoint, fstype, _ = line.split(None, 3) # fs type blacklist if fstype in EXCLUDED_FS: continue try: dev = os.stat(mountpoint).st_dev except OSError as err: # this should be a fairly rare error, since we are blacklisting # network filesystems; with this in mind, we'll ignore it, # since the rereadpt check catches in-use filesystems, # and this is used for disk information only print("Can't stat mountpoint '%s': %s" % (mountpoint, err), file=sys.stderr) print("Ignoring.", file=sys.stderr) continue mounts[dev] = mountpoint return mounts def GetSwapInfo(): """Reads /proc/swaps and returns the list of swap backing stores. """ swaplines = ReadFile("/proc/swaps").splitlines()[1:] return [line.split(None, 1)[0] for line in swaplines] def DevInfo(name, dev, mountinfo): """Computes miscellaneous information about a block device. @type name: string @param name: the device name, e.g. sda @return: a tuple (mpath, whatvg, fileinfo), where: - mpath is the mount path where this device is mounted or None - whatvg is the result of the ReadPV function - fileinfo is the output of file -bs on the device """ if dev in mountinfo: mpath = mountinfo[dev] else: mpath = None whatvg = ReadPV(name) result = ExecCommand("file -bs /dev/%s" % name) if result.failed: fileinfo = "" % result.stderr fileinfo = result.stdout[:45] return mpath, whatvg, fileinfo def ShowDiskInfo(opts): """Shows a nicely formatted block device list for this system. This function shows the user a table with the information gathered by the other functions defined, in order to help the user make a choice about which disks should be allocated to our volume group. """ def _inuse(inuse): if inuse: return "yes" else: return "no" mounts = GetMountInfo() dlist = GetDiskList(opts) print("------- Disk information -------") headers = { "name": "Name", "size": "Size[M]", "used": "Used", "mount": "Mount", "lvm": "LVM?", "info": "Info", } fields = ["name", "size", "used", "mount", "lvm", "info"] flatlist = [] # Flatten the [(disk, [partition,...]), ...] list for name, size, dev, parts, inuse in dlist: flatlist.append((name, size, dev, _inuse(inuse))) for partname, partsize, partdev, partinuse in parts: flatlist.append((partname, partsize, partdev, _inuse(partinuse))) strlist = [] for name, size, dev, in_use in flatlist: mp, vgname, fileinfo = DevInfo(name, dev, mounts) if mp is None: mp = "-" if vgname is None: lvminfo = "-" elif vgname == "": lvminfo = "yes,free" else: lvminfo = "in %s" % vgname if len(name) > 3: # Indent partitions name = " %s" % name strlist.append([name, "%.2f" % (float(size) / 1024 / 1024), in_use, mp, lvminfo, fileinfo]) data = cli.GenerateTable(headers, fields, None, strlist, numfields=["size"]) for line in data: print(line) def CheckSysfsHolders(name): """Check to see if a device is 'hold' at sysfs level. This is usually the case for Physical Volumes under LVM. @rtype: boolean @return: true if the device is available according to sysfs """ try: contents = os.listdir("%s/holders/" % SysfsName(name)) except OSError as err: if err.errno == errno.ENOENT: contents = [] else: raise return not bool(contents) def CheckReread(name): """Check to see if a block device is in use. Uses blockdev to reread the partition table of a block device (or fuser if the device is not partitionable), and thus compute the in-use status. See the discussion in GetDiskList about the meaning of 'in use'. @rtype: boolean @return: the in-use status of the device """ use_blockdev = IsPartitioned(name) if use_blockdev: cmd = "blockdev --rereadpt /dev/%s" % name else: cmd = "fuser -vam /dev/%s" % name for _ in range(3): result = ExecCommand(cmd) if not use_blockdev and result.failed: break elif use_blockdev and not result.failed: break time.sleep(2) if use_blockdev: return not result.failed else: return result.failed def CheckMounted(name): """Check to see if a block device is a mountpoint. In recent distros/kernels, this is reported directly via fuser, but on older ones not, so we do an additional check here (manually). """ minfo = GetMountInfo() dev = ReadDev(SysfsName(name)) return dev not in minfo def CheckSwap(name): """Check to see if a block device is being used as swap. """ name = "/dev/%s" % name return name not in GetSwapInfo() def InUse(name): """Returns if a disk is in use or not. """ return not (CheckSysfsHolders(name) and CheckReread(name) and CheckMounted(name) and CheckSwap(name)) def WipeDisk(name): """Wipes a block device. This function wipes a block device, by clearing and re-reading the partition table. If not successful, it writes back the old partition data, and leaves the cleanup to the user. @param name: the device name (e.g. sda) """ if InUse(name): raise OperationalError("CRITICAL: disk %s you selected seems to be in" " use. ABORTING!" % name) fd = os.open("/dev/%s" % name, os.O_RDWR | os.O_SYNC) olddata = os.read(fd, 512) if len(olddata) != 512: raise OperationalError("CRITICAL: Can't read partition table information" " from /dev/%s (needed 512 bytes, got %d" % (name, len(olddata))) newdata = b"\x00" * 512 os.lseek(fd, 0, 0) bytes_written = os.write(fd, newdata) os.close(fd) if bytes_written != 512: raise OperationalError("CRITICAL: Can't write partition table information" " to /dev/%s (tried to write 512 bytes, written" " %d. I don't know how to cleanup. Sorry." % (name, bytes_written)) if InUse(name): # try to restore the data fd = os.open("/dev/%s" % name, os.O_RDWR | os.O_SYNC) os.write(fd, olddata) os.close(fd) raise OperationalError("CRITICAL: disk %s which I have just wiped cannot" " reread partition table. Most likely, it is" " in use. You have to clean after this yourself." " I tried to restore the old partition table," " but I cannot guarantee nothing has broken." % name) def PartitionDisk(name, use_sfdisk): """Partitions a disk. This function creates a single partition spanning the entire disk, by means of fdisk. @param name: the device name, e.g. sda """ # Check that parted exists result = ExecCommand("parted --help") if result.failed: use_sfdisk = True print("Unable to execute \"parted --help\"," " falling back to sfdisk.", file=sys.stderr) # Check disk size - over 2TB means we need to use GPT size = ReadSize("/sys/block/%s" % name) if size > MBR_MAX_SIZE: label_type = "gpt" if use_sfdisk: raise OperationalError("Critical: Disk larger than 2TB detected, but" " parted is either not installed or --use-sfdisk" " has been specified") else: label_type = "msdos" if use_sfdisk: result = ExecCommand( "echo ,,8e, | sfdisk /dev/%s" % name) if result.failed: raise OperationalError("CRITICAL: disk %s which I have just partitioned" " cannot reread its partition table, or there" " is some other sfdisk error. Likely, it is in" " use. You have to clean this yourself. Error" " message from sfdisk: %s" % (name, result.output)) else: result = ExecCommand("parted -s /dev/%s mklabel %s" % (name, label_type)) if result.failed: raise OperationalError("Critical: failed to create %s label on %s" % (label_type, name)) result = ExecCommand("parted -s /dev/%s mkpart pri ext2 1 100%%" % name) if result.failed: raise OperationalError("Critical: failed to create partition on %s" % name) result = ExecCommand("parted -s /dev/%s set 1 lvm on" % name) if result.failed: raise OperationalError("Critical: failed to set partition on %s to LVM" % name) def CreatePVOnDisk(name): """Creates a physical volume on a block device. This function creates a physical volume on a block device, overriding all warnings. So it can wipe existing PVs and PVs which are in a VG. @param name: the device name, e.g. sda """ device = DeviceName(name) result = ExecCommand("pvcreate -yff %s" % device) if result.failed: raise OperationalError("I cannot create a physical volume on" " %s. Error message: %s." " Please clean up yourself." % (device, result.output)) def CreateVG(vgname, disks): """Creates the volume group. This function creates a volume group named `vgname` on the disks given as parameters. The physical extent size is set to 64MB. @param disks: a list of disk names, e.g. ['sda','sdb'] """ pnames = [DeviceName(d) for d in disks] result = ExecCommand("vgcreate -s 64MB '%s' %s" % (vgname, " ".join(pnames))) if result.failed: raise OperationalError("I cannot create the volume group %s from" " disks %s. Error message: %s. Please clean up" " yourself." % (vgname, " ".join(disks), result.output)) def _ComputeSysdFreeUsed(sysdisks): sysd_free = [] sysd_used = [] for name, _, _, parts, used in sysdisks: if used: sysd_used.append(name) for partname, _, _, partused in parts: if partused: sysd_used.append(partname) else: sysd_free.append(partname) else: sysd_free.append(name) if not sysd_free: raise PrereqError("no free disks found! (%d in-use disks)" % len(sysd_used)) return (sysd_free, sysd_used) def ValidateDiskList(options): """Validates or computes the disk list for create. This function either computes the available disk list (if the user gave --alldisks option), or validates the user-given disk list (by using the --disks option) such that all given disks are present and not in use. @param options: the options returned from OptParser.parse_options @return: a list of disk names, e.g. ['sda', 'sdb'] """ sysdisks = GetDiskList(options) if not sysdisks: raise PrereqError("no disks found (I looked for" " non-removable block devices).") (sysd_free, sysd_used) = _ComputeSysdFreeUsed(sysdisks) if options.alldisks: disklist = sysd_free elif options.disks: disklist = options.disks.split(",") for name in disklist: if name in sysd_used: raise ParameterError("disk %s is in use, cannot wipe!" % name) if name not in sysd_free: raise ParameterError("cannot find disk %s!" % name) else: raise ParameterError("Please use either --alldisks or --disks!") return disklist def BootStrap(): """Actual main routine. """ CheckPrereq() options, args = ParseOptions() vgname = options.vgname command = args.pop(0) if command == "diskinfo": ShowDiskInfo(options) return if command != "create": Usage() exists, lv_count, vg_size, vg_free = CheckVGExists(vgname) if exists: raise PrereqError("It seems volume group '%s' already exists:\n" " LV count: %s, size: %s, free: %s." % (vgname, lv_count, vg_size, vg_free)) disklist = ValidateDiskList(options) for disk in disklist: WipeDisk(disk) if IsPartitioned(disk): PartitionDisk(disk, options.use_sfdisk) for disk in disklist: CreatePVOnDisk(disk) CreateVG(vgname, disklist) status, lv_count, size, _ = CheckVGExists(vgname) if status: print("Done! %s: size %s GiB, disks: %s" % (vgname, size, ",".join(disklist))) else: raise OperationalError("Although everything seemed ok, the volume" " group did not get created.") def main(): """Application entry point. This is just a wrapper over BootStrap, to handle our own exceptions. """ try: BootStrap() except PrereqError as err: print("The prerequisites for running this tool are not met.", file=sys.stderr) print("Please make sure you followed all the steps in" " the build document.", file=sys.stderr) print("Description: %s" % str(err), file=sys.stderr) sys.exit(1) except SysconfigError as err: print("This system's configuration seems wrong, at" " least is not what I expect.", file=sys.stderr) print("Please check that the installation didn't fail" " at some step.", file=sys.stderr) print("Description: %s" % str(err), file=sys.stderr) sys.exit(1) except ParameterError as err: print("Some parameters you gave to the program or the" " invocation is wrong. ", file=sys.stderr) print("Description: %s" % str(err), file=sys.stderr) Usage() except OperationalError as err: print("A serious error has happened while modifying" " the system's configuration.", file=sys.stderr) print("Please review the error message below and make" " sure you clean up yourself.", file=sys.stderr) print("It is most likely that the system configuration" " has been partially altered.", file=sys.stderr) print(str(err), file=sys.stderr) sys.exit(1) except ProgrammingError as err: print("Internal application error. Please report this" " to the Ganeti developer list.", file=sys.stderr) print("Error description: %s" % str(err), file=sys.stderr) sys.exit(1) except Error as err: print("Unhandled application error: %s" % err, file=sys.stderr) sys.exit(1) except (IOError, OSError) as err: print("I/O error detected, please report.", file=sys.stderr) print("Description: %s" % str(err), file=sys.stderr) sys.exit(1) if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/master-ip-setup000075500000000000000000000070771476477700300177440ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e -u USAGE_MSG="Usage: $0 {start|stop}" PATH=$PATH:/sbin:/usr/sbin:/usr/local/sbin FPING_PACKET_COUNT=5 FPING_PACKET_INTERVAL_MS=200 # Start the master IP start() { case $CLUSTER_IP_VERSION in 4) ARP_COMMAND="arping -q -U -c 3 -I $MASTER_NETDEV -s $MASTER_IP $MASTER_IP" ;; 6) ARP_COMMAND="ndisc6 -q -r 3 $MASTER_IP $MASTER_NETDEV" ;; *) echo "Invalid cluster IP version specified: $CLUSTER_IP_VERSION" >&2 exit 1 ;; esac # Check if the master IP address is already configured on this machine if fping -c $FPING_PACKET_COUNT -p $FPING_PACKET_INTERVAL_MS \ -S 127.0.0.1 $MASTER_IP >/dev/null 2>&1; then echo "Master IP address already configured on this machine. Doing nothing." exit 0 fi # Check if the master IP address is already configured on another machine if fping -c $FPING_PACKET_COUNT -p $FPING_PACKET_INTERVAL_MS \ $MASTER_IP >/dev/null 2>&1; then echo "Error: master IP address configured on another machine." >&2 exit 1 fi if ! ip addr add $MASTER_IP/$MASTER_NETMASK \ dev $MASTER_NETDEV label $MASTER_NETDEV:0 preferred_lft 0; then echo "Error during the activation of the master IP address" >&2 exit 1 fi # Send gratuituous ARP to update neighbours' ARP cache $ARP_COMMAND || : } # Stop the master IP stop() { # Check if the master IP address is still configured on this machine if ! ip addr show dev $MASTER_NETDEV | \ grep -F " $MASTER_IP/$MASTER_NETMASK" >/dev/null 2>&1; then # Check if the master IP address is configured on a wrong device if fping -c $FPING_PACKET_COUNT -p $FPING_PACKET_INTERVAL_MS \ -S 127.0.0.1 $MASTER_IP >/dev/null 2>&1; then echo "Error: master IP address configured on wrong device," \ "can't shut it down." >&2 exit 1 else echo "Master IP address not configured on this machine. Doing nothing." exit 0 fi fi if ! ip addr del $MASTER_IP/$MASTER_NETMASK dev $MASTER_NETDEV; then echo "Error during the deactivation of the master IP address" >&2 exit 1 fi } if (( $# < 1 )); then echo $USAGE_MSG >&2 exit 1 fi case "$1" in start) start ;; stop) stop ;; *) echo $USAGE_MSG >&2 exit 1 ;; esac exit 0 ganeti-3.1.0~rc2/tools/move-instance000075500000000000000000001172471476477700300174560ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010, 2011, 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tool to move instances from one cluster to another. """ # pylint: disable=C0103 # C0103: Invalid name move-instance import os import sys import time import logging import optparse import random import threading from ganeti import cli from ganeti import constants from ganeti import utils from ganeti import workerpool from ganeti import objects from ganeti import compat from ganeti import rapi from ganeti import errors from ganeti.rapi.client import UsesRapiClient import ganeti.rapi.client_utils # pylint: disable=W0611 SRC_RAPI_PORT_OPT = \ cli.cli_option("--src-rapi-port", action="store", type="int", dest="src_rapi_port", default=constants.DEFAULT_RAPI_PORT, help=("Source cluster RAPI port (defaults to %s)" % constants.DEFAULT_RAPI_PORT)) SRC_CA_FILE_OPT = \ cli.cli_option("--src-ca-file", action="store", type="string", dest="src_ca_file", help=("File containing source cluster Certificate" " Authority (CA) in PEM format")) SRC_USERNAME_OPT = \ cli.cli_option("--src-username", action="store", type="string", dest="src_username", default=None, help="Source cluster username") SRC_PASSWORD_FILE_OPT = \ cli.cli_option("--src-password-file", action="store", type="string", dest="src_password_file", help="File containing source cluster password") DEST_RAPI_PORT_OPT = \ cli.cli_option("--dest-rapi-port", action="store", type="int", dest="dest_rapi_port", default=constants.DEFAULT_RAPI_PORT, help=("Destination cluster RAPI port (defaults to source" " cluster RAPI port)")) DEST_CA_FILE_OPT = \ cli.cli_option("--dest-ca-file", action="store", type="string", dest="dest_ca_file", help=("File containing destination cluster Certificate" " Authority (CA) in PEM format (defaults to source" " cluster CA)")) DEST_USERNAME_OPT = \ cli.cli_option("--dest-username", action="store", type="string", dest="dest_username", default=None, help=("Destination cluster username (defaults to" " source cluster username)")) DEST_PASSWORD_FILE_OPT = \ cli.cli_option("--dest-password-file", action="store", type="string", dest="dest_password_file", help=("File containing destination cluster password" " (defaults to source cluster password)")) DEST_INSTANCE_NAME_OPT = \ cli.cli_option("--dest-instance-name", action="store", type="string", dest="dest_instance_name", help=("Instance name on destination cluster (only" " when moving exactly one instance)")) DEST_PRIMARY_NODE_OPT = \ cli.cli_option("--dest-primary-node", action="store", type="string", dest="dest_primary_node", help=("Primary node on destination cluster (only" " when moving exactly one instance)")) DEST_SECONDARY_NODE_OPT = \ cli.cli_option("--dest-secondary-node", action="store", type="string", dest="dest_secondary_node", help=("Secondary node on destination cluster (only" " when moving exactly one instance)")) DEST_DISK_TEMPLATE_OPT = \ cli.cli_option("--dest-disk-template", action="store", type="string", dest="dest_disk_template", default=None, help="Disk template to use on destination cluster") COMPRESS_OPT = \ cli.cli_option("--compress", action="store", type="string", dest="compress", default="none", help="Compression mode to use during the move (this mode has" " to be supported by both clusters)") PARALLEL_OPT = \ cli.cli_option("-p", "--parallel", action="store", type="int", default=1, dest="parallel", metavar="", help="Number of instances to be moved simultaneously") OPPORTUNISTIC_TRIES_OPT = \ cli.cli_option("--opportunistic-tries", action="store", type="int", dest="opportunistic_tries", metavar="", help="Number of opportunistic instance creation attempts" " before a normal creation is performed. An opportunistic" " attempt will use the iallocator with all the nodes" " currently unlocked, failing if not enough nodes are" " available. Even though it will succeed (or fail) more" " quickly, it can result in suboptimal instance" " placement") OPPORTUNISTIC_DELAY_OPT = \ cli.cli_option("--opportunistic-delay", action="store", type="int", dest="opportunistic_delay", metavar="", help="The delay between successive opportunistic instance" " creation attempts, in seconds") KEEP_SOURCE_INSTANCE = \ cli.cli_option("--keep-source-instance", action="store_true", dest="keep_source_instance", default=False, help="Keep source instance after successful transfer") class Error(Exception): """Generic error. """ class Abort(Error): """Special exception for aborting import/export. """ class RapiClientFactory(object): """Factory class for creating RAPI clients. @ivar src_cluster_name: Source cluster name @ivar dest_cluster_name: Destination cluster name @ivar GetSourceClient: Callable returning new client for source cluster @ivar GetDestClient: Callable returning new client for destination cluster """ def __init__(self, options, src_cluster_name, dest_cluster_name): """Initializes this class. @param options: Program options @type src_cluster_name: string @param src_cluster_name: Source cluster name @type dest_cluster_name: string @param dest_cluster_name: Destination cluster name """ self.src_cluster_name = src_cluster_name self.dest_cluster_name = dest_cluster_name # TODO: Implement timeouts for RAPI connections # TODO: Support for using system default paths for verifying SSL certificate logging.debug("Using '%s' as source CA", options.src_ca_file) src_curl_config = rapi.client.GenericCurlConfig(cafile=options.src_ca_file) if options.dest_ca_file: logging.debug("Using '%s' as destination CA", options.dest_ca_file) dest_curl_config = \ rapi.client.GenericCurlConfig(cafile=options.dest_ca_file) else: logging.debug("Using source CA for destination") dest_curl_config = src_curl_config logging.debug("Source RAPI server is %s:%s", src_cluster_name, options.src_rapi_port) logging.debug("Source username is '%s'", options.src_username) if options.src_username is None: src_username = "" else: src_username = options.src_username if options.src_password_file: logging.debug("Reading '%s' for source password", options.src_password_file) src_password = utils.ReadOneLineFile(options.src_password_file, strict=True) else: logging.debug("Source has no password") src_password = None self.GetSourceClient = lambda: \ rapi.client.GanetiRapiClient(src_cluster_name, port=options.src_rapi_port, curl_config_fn=src_curl_config, username=src_username, password=src_password) if options.dest_rapi_port: dest_rapi_port = options.dest_rapi_port else: dest_rapi_port = options.src_rapi_port if options.dest_username is None: dest_username = src_username else: dest_username = options.dest_username logging.debug("Destination RAPI server is %s:%s", dest_cluster_name, dest_rapi_port) logging.debug("Destination username is '%s'", dest_username) if options.dest_password_file: logging.debug("Reading '%s' for destination password", options.dest_password_file) dest_password = utils.ReadOneLineFile(options.dest_password_file, strict=True) else: logging.debug("Using source password for destination") dest_password = src_password self.GetDestClient = lambda: \ rapi.client.GanetiRapiClient(dest_cluster_name, port=dest_rapi_port, curl_config_fn=dest_curl_config, username=dest_username, password=dest_password) class MoveJobPollReportCb(cli.JobPollReportCbBase): def __init__(self, abort_check_fn, remote_import_fn): """Initializes this class. @type abort_check_fn: callable @param abort_check_fn: Function to check whether move is aborted @type remote_import_fn: callable or None @param remote_import_fn: Callback for reporting received remote import information """ cli.JobPollReportCbBase.__init__(self) self._abort_check_fn = abort_check_fn self._remote_import_fn = remote_import_fn def ReportLogMessage(self, job_id, serial, timestamp, log_type, log_msg): """Handles a log message. """ if log_type == constants.ELOG_REMOTE_IMPORT: logging.debug("Received remote import information") if not self._remote_import_fn: raise RuntimeError("Received unexpected remote import information") assert "x509_ca" in log_msg assert "disks" in log_msg self._remote_import_fn(log_msg) return logging.info("[%s] %s", time.ctime(utils.MergeTime(timestamp)), cli.FormatLogMessage(log_type, log_msg)) def ReportNotChanged(self, job_id, status): """Called if a job hasn't changed in a while. """ try: # Check whether we were told to abort by the other thread self._abort_check_fn() except Abort: logging.warning("Aborting despite job %s still running", job_id) raise class InstanceMove(object): """Status class for instance moves. """ def __init__(self, src_instance_name, dest_instance_name, dest_pnode, dest_snode, compress, dest_iallocator, dest_disk_template, hvparams, beparams, osparams, nics, opportunistic_tries, opportunistic_delay, keep_source_instance): """Initializes this class. @type src_instance_name: string @param src_instance_name: Instance name on source cluster @type dest_instance_name: string @param dest_instance_name: Instance name on destination cluster @type dest_pnode: string or None @param dest_pnode: Name of primary node on destination cluster @type dest_snode: string or None @param dest_snode: Name of secondary node on destination cluster @type compress; string @param compress: Compression mode to use (has to be supported on both clusters) @type dest_iallocator: string or None @param dest_iallocator: Name of iallocator to use @type dest_disk_template: string or None @param dest_disk_template: Disk template to use instead of the original one @type hvparams: dict or None @param hvparams: Hypervisor parameters to override @type beparams: dict or None @param beparams: Backend parameters to override @type osparams: dict or None @param osparams: OS parameters to override @type nics: dict or None @param nics: NICs to override @type opportunistic_tries: int or None @param opportunistic_tries: Number of opportunistic creation attempts to perform @type opportunistic_delay: int or None @param opportunistic_delay: Delay between successive creation attempts, in seconds @type keep_source_instance: bool @param keep_source_instance: Remove instance after successful export """ self.src_instance_name = src_instance_name self.dest_instance_name = dest_instance_name self.dest_pnode = dest_pnode self.dest_snode = dest_snode self.compress = compress self.dest_iallocator = dest_iallocator self.dest_disk_template = dest_disk_template self.hvparams = hvparams self.beparams = beparams self.osparams = osparams self.nics = nics self.keep_source_instance = keep_source_instance if opportunistic_tries is not None: self.opportunistic_tries = opportunistic_tries else: self.opportunistic_tries = 0 if opportunistic_delay is not None: self.opportunistic_delay = opportunistic_delay else: self.opportunistic_delay = constants.DEFAULT_OPPORTUNISTIC_RETRY_INTERVAL self.error_message = None class MoveRuntime(object): """Class to keep track of instance move. """ def __init__(self, move): """Initializes this class. @type move: L{InstanceMove} """ self.move = move # Thread synchronization self.lock = threading.Lock() self.source_to_dest = threading.Condition(self.lock) self.dest_to_source = threading.Condition(self.lock) # Source information self.src_error_message = None self.src_expinfo = None self.src_instinfo = None # Destination information self.dest_error_message = None self.dest_impinfo = None def HandleErrors(self, prefix, fn, *args): """Wrapper to catch errors and abort threads. @type prefix: string @param prefix: Variable name prefix ("src" or "dest") @type fn: callable @param fn: Function """ assert prefix in ("dest", "src") try: # Call inner function fn(*args) errmsg = None except Abort: errmsg = "Aborted" except Exception as err: # pylint: disable=W0703 logging.exception("Caught unhandled exception") errmsg = str(err) setattr(self, "%s_error_message" % prefix, errmsg) self.lock.acquire() try: self.source_to_dest.notify_all() self.dest_to_source.notify_all() finally: self.lock.release() def CheckAbort(self): """Check whether thread should be aborted. @raise Abort: When thread should be aborted """ if not (self.src_error_message is None and self.dest_error_message is None): logging.info("Aborting") raise Abort() def Wait(self, cond, check_fn): """Waits for a condition to become true. @type cond: threading.Condition @param cond: Threading condition @type check_fn: callable @param check_fn: Function to check whether condition is true """ cond.acquire() try: while check_fn(self): self.CheckAbort() cond.wait() finally: cond.release() def PollJob(self, cl, job_id, remote_import_fn=None): """Wrapper for polling a job. @type cl: L{rapi.client.GanetiRapiClient} @param cl: RAPI client @type job_id: string @param job_id: Job ID @type remote_import_fn: callable or None @param remote_import_fn: Callback for reporting received remote import information @return: opreturn of the move job @raise errors.JobLost: If job can't be found @raise errors.OpExecError: If job didn't succeed @see: L{ganeti.rapi.client_utils.PollJob} """ return rapi.client_utils.PollJob(cl, job_id, MoveJobPollReportCb(self.CheckAbort, remote_import_fn)) class MoveDestExecutor(object): def __init__(self, dest_client, mrt): """Destination side of an instance move. @type dest_client: L{rapi.client.GanetiRapiClient} @param dest_client: RAPI client @type mrt: L{MoveRuntime} @param mrt: Instance move runtime information """ logging.debug("Waiting for instance information to become available") mrt.Wait(mrt.source_to_dest, lambda mrt: mrt.src_instinfo is None or mrt.src_expinfo is None) logging.info("Creating instance %s in remote-import mode", mrt.move.dest_instance_name) # Depending on whether opportunistic tries are enabled, we may have to # make multiple creation attempts creation_attempts = [True] * mrt.move.opportunistic_tries # But the last one is never opportunistic, and will block until completion # or failure creation_attempts.append(False) # Initiate the RNG for the variations random.seed() for is_attempt_opportunistic in creation_attempts: job_id = self._CreateInstance(dest_client, mrt.move.dest_instance_name, mrt.move.dest_pnode, mrt.move.dest_snode, mrt.move.compress, mrt.move.dest_iallocator, mrt.move.dest_disk_template, mrt.src_instinfo, mrt.src_expinfo, mrt.move.hvparams, mrt.move.beparams, mrt.move.osparams, mrt.move.nics, is_attempt_opportunistic ) try: # The completion of this block signifies that the import has been # completed successfullly mrt.PollJob(dest_client, job_id, remote_import_fn=compat.partial(self._SetImportInfo, mrt)) logging.info("Import successful") return except errors.OpPrereqError as err: # Any exception in the non-opportunistic creation is to be passed on, # as well as exceptions apart from resources temporarily unavailable if not is_attempt_opportunistic or \ err.args[1] != rapi.client.ECODE_TEMP_NORES: raise delay_to_use = MoveDestExecutor._VaryDelay(mrt) logging.info("Opportunistic attempt unsuccessful, waiting %.2f seconds" " before another creation attempt is made", delay_to_use) time.sleep(delay_to_use) @staticmethod def _VaryDelay(mrt): """ Varies the opportunistic delay by a small amount. """ MAX_VARIATION = 0.15 variation_factor = (1.0 + random.uniform(-MAX_VARIATION, MAX_VARIATION)) return mrt.move.opportunistic_delay * variation_factor @staticmethod def _SetImportInfo(mrt, impinfo): """Sets the remote import information and notifies source thread. @type mrt: L{MoveRuntime} @param mrt: Instance move runtime information @param impinfo: Remote import information """ mrt.dest_to_source.acquire() try: mrt.dest_impinfo = impinfo mrt.dest_to_source.notify_all() finally: mrt.dest_to_source.release() @staticmethod def _GetDisks(instance): disks = [] for idisk in instance["disks"]: odisk = { constants.IDISK_SIZE: idisk["size"], constants.IDISK_MODE: idisk["mode"], constants.IDISK_NAME: str(idisk.get("name")), } spindles = idisk.get("spindles") if spindles is not None: odisk[constants.IDISK_SPINDLES] = spindles disks.append(odisk) return disks @staticmethod def _GetNics(instance, override_nics): try: nics = [{ constants.INIC_IP: ip, constants.INIC_MAC: mac, constants.INIC_MODE: mode, constants.INIC_LINK: link, constants.INIC_VLAN: vlan, constants.INIC_NETWORK: network, constants.INIC_NAME: nic_name } for nic_name, _, ip, mac, mode, link, vlan, network, _ in instance["nics"]] except ValueError: raise Error("Received NIC information does not match expected format; " "Do the versions of this tool and the source cluster match?") if len(override_nics) > len(nics): raise Error("Can not create new NICs") if override_nics: assert len(override_nics) <= len(nics) for idx, (nic, override) in enumerate(zip(nics, override_nics)): nics[idx] = objects.FillDict(nic, override) return nics @staticmethod def _CreateInstance(cl, name, pnode, snode, compress, iallocator, dest_disk_template, instance, expinfo, override_hvparams, override_beparams, override_osparams, override_nics, is_attempt_opportunistic): """Starts the instance creation in remote import mode. @type cl: L{rapi.client.GanetiRapiClient} @param cl: RAPI client @type name: string @param name: Instance name @type pnode: string or None @param pnode: Name of primary node on destination cluster @type snode: string or None @param snode: Name of secondary node on destination cluster @type compress: string @param compress: Compression mode to use @type iallocator: string or None @param iallocator: Name of iallocator to use @type dest_disk_template: string or None @param dest_disk_template: Disk template to use instead of the original one @type instance: dict @param instance: Instance details from source cluster @type expinfo: dict @param expinfo: Prepared export information from source cluster @type override_hvparams: dict or None @param override_hvparams: Hypervisor parameters to override @type override_beparams: dict or None @param override_beparams: Backend parameters to override @type override_osparams: dict or None @param override_osparams: OS parameters to override @type override_nics: dict or None @param override_nics: NICs to override @type is_attempt_opportunistic: bool @param is_attempt_opportunistic: Whether to use opportunistic locking or not @return: Job ID """ if dest_disk_template: disk_template = dest_disk_template else: disk_template = instance["disk_template"] disks = MoveDestExecutor._GetDisks(instance) nics = MoveDestExecutor._GetNics(instance, override_nics) os_type = instance.get("os", None) # TODO: Should this be the actual up/down status? (run_state) start = (instance["config_state"] == "up") assert len(disks) == len(instance["disks"]) assert len(nics) == len(instance["nics"]) inst_beparams = instance.get("be_instance", {}) inst_hvparams = instance.get("hv_instance", {}) inst_osparams = instance.get("os_instance", {}) return cl.CreateInstance(constants.INSTANCE_REMOTE_IMPORT, name, disk_template, disks, nics, os=os_type, pnode=pnode, snode=snode, start=start, ip_check=False, iallocator=iallocator, hypervisor=instance["hypervisor"], source_handshake=expinfo["handshake"], source_x509_ca=expinfo["x509_ca"], compress=compress, source_instance_name=instance["name"], beparams=objects.FillDict(inst_beparams, override_beparams), hvparams=objects.FillDict(inst_hvparams, override_hvparams), osparams=objects.FillDict(inst_osparams, override_osparams), opportunistic_locking=is_attempt_opportunistic, tags=instance["tags"], wait_for_sync=False, ) class MoveSourceExecutor(object): def __init__(self, src_client, mrt): """Source side of an instance move. @type src_client: L{rapi.client.GanetiRapiClient} @param src_client: RAPI client @type mrt: L{MoveRuntime} @param mrt: Instance move runtime information """ logging.info("Checking whether instance exists") self._CheckInstance(src_client, mrt.move.src_instance_name) logging.info("Retrieving instance information from source cluster") instinfo = self._GetInstanceInfo(src_client, mrt.PollJob, mrt.move.src_instance_name) instinfo["tags"] = self._GetInstanceTags(src_client, mrt.move.src_instance_name) logging.info("Preparing export on source cluster") expinfo = self._PrepareExport(src_client, mrt.PollJob, mrt.move.src_instance_name) assert "handshake" in expinfo assert "x509_key_name" in expinfo assert "x509_ca" in expinfo # Hand information to destination thread mrt.source_to_dest.acquire() try: mrt.src_instinfo = instinfo mrt.src_expinfo = expinfo mrt.source_to_dest.notify_all() finally: mrt.source_to_dest.release() logging.info("Waiting for destination information to become available") mrt.Wait(mrt.dest_to_source, lambda mrt: mrt.dest_impinfo is None) logging.info("Starting remote export on source cluster") self._ExportInstance(src_client, mrt.PollJob, mrt.move.src_instance_name, expinfo["x509_key_name"], mrt.move.compress, mrt.dest_impinfo, mrt.move.keep_source_instance) logging.info("Export successful") @staticmethod def _CheckInstance(cl, name): """Checks whether the instance exists on the source cluster. @type cl: L{rapi.client.GanetiRapiClient} @param cl: RAPI client @type name: string @param name: Instance name """ try: cl.GetInstance(name) except rapi.client.GanetiApiError as err: if err.code == rapi.client.HTTP_NOT_FOUND: raise Error("Instance %s not found (%s)" % (name, str(err))) raise @staticmethod def _GetInstanceInfo(cl, poll_job_fn, name): """Retrieves detailed instance information from source cluster. @type cl: L{rapi.client.GanetiRapiClient} @param cl: RAPI client @type poll_job_fn: callable @param poll_job_fn: Function to poll for job result @type name: string @param name: Instance name """ job_id = cl.GetInstanceInfo(name, static=True) result = poll_job_fn(cl, job_id) assert len(result[0].keys()) == 1 return result[0][list(result[0].keys())[0]] @staticmethod def _GetInstanceTags(cl, name): """Retrieves instance tags (as they are not returned by _GetInstanceInfo) @type cl: L{rapi.client.GanetiRapiClient} @param cl: RAPI client @type name: string @param name: Instance name @return: list of strings """ return cl.GetInstance(name)["tags"] @staticmethod def _PrepareExport(cl, poll_job_fn, name): """Prepares export on source cluster. @type cl: L{rapi.client.GanetiRapiClient} @param cl: RAPI client @type poll_job_fn: callable @param poll_job_fn: Function to poll for job result @type name: string @param name: Instance name """ job_id = cl.PrepareExport(name, constants.EXPORT_MODE_REMOTE) return poll_job_fn(cl, job_id)[0] @staticmethod def _ExportInstance(cl, poll_job_fn, name, x509_key_name, compress, impinfo, keep_source_instance): """Exports instance from source cluster. @type cl: L{rapi.client.GanetiRapiClient} @param cl: RAPI client @type poll_job_fn: callable @param poll_job_fn: Function to poll for job result @type name: string @param name: Instance name @param x509_key_name: Source X509 key @type compress: string @param compress: Compression mode to use @param impinfo: Import information from destination cluster @type keep_source_instance: bool @param keep_source_instance: Remove instance after successful export """ job_id = cl.ExportInstance(name, constants.EXPORT_MODE_REMOTE, impinfo["disks"], shutdown=True, remove_instance=not keep_source_instance, x509_key_name=x509_key_name, destination_x509_ca=impinfo["x509_ca"], compress=compress) (fin_resu, dresults) = poll_job_fn(cl, job_id)[0] if not (fin_resu and compat.all(dresults)): raise Error("Export failed for disks %s" % utils.CommaJoin(str(idx) for idx, result in enumerate(dresults) if not result)) class MoveSourceWorker(workerpool.BaseWorker): def RunTask(self, rapi_factory, move): # pylint: disable=W0221 """Executes an instance move. @type rapi_factory: L{RapiClientFactory} @param rapi_factory: RAPI client factory @type move: L{InstanceMove} @param move: Instance move information """ try: logging.info("Preparing to move %s from cluster %s to %s as %s", move.src_instance_name, rapi_factory.src_cluster_name, rapi_factory.dest_cluster_name, move.dest_instance_name) mrt = MoveRuntime(move) logging.debug("Starting destination thread") dest_thread = threading.Thread(name="DestFor%s" % self.name, target=mrt.HandleErrors, args=("dest", MoveDestExecutor, rapi_factory.GetDestClient(), mrt, )) dest_thread.start() try: mrt.HandleErrors("src", MoveSourceExecutor, rapi_factory.GetSourceClient(), mrt) finally: dest_thread.join() if mrt.src_error_message or mrt.dest_error_message: move.error_message = ("Source error: %s, destination error: %s" % (mrt.src_error_message, mrt.dest_error_message)) else: move.error_message = None except Exception as err: # pylint: disable=W0703 logging.exception("Caught unhandled exception") move.error_message = str(err) def CheckRapiSetup(rapi_factory): """Checks the RAPI setup by retrieving the version. @type rapi_factory: L{RapiClientFactory} @param rapi_factory: RAPI client factory """ src_client = rapi_factory.GetSourceClient() logging.info("Connecting to source RAPI server") logging.info("Source cluster RAPI version: %s", src_client.GetVersion()) dest_client = rapi_factory.GetDestClient() logging.info("Connecting to destination RAPI server") logging.info("Destination cluster RAPI version: %s", dest_client.GetVersion()) def ParseOptions(): """Parses options passed to program. """ program = os.path.basename(sys.argv[0]) parser = optparse.OptionParser(usage=("%prog [--debug|--verbose]" " " " "), prog=program) parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option(cli.IALLOCATOR_OPT) parser.add_option(cli.BACKEND_OPT) parser.add_option(cli.HVOPTS_OPT) parser.add_option(cli.OSPARAMS_OPT) parser.add_option(cli.NET_OPT) parser.add_option(SRC_RAPI_PORT_OPT) parser.add_option(SRC_CA_FILE_OPT) parser.add_option(SRC_USERNAME_OPT) parser.add_option(SRC_PASSWORD_FILE_OPT) parser.add_option(DEST_RAPI_PORT_OPT) parser.add_option(DEST_CA_FILE_OPT) parser.add_option(DEST_USERNAME_OPT) parser.add_option(DEST_PASSWORD_FILE_OPT) parser.add_option(DEST_INSTANCE_NAME_OPT) parser.add_option(DEST_PRIMARY_NODE_OPT) parser.add_option(DEST_SECONDARY_NODE_OPT) parser.add_option(DEST_DISK_TEMPLATE_OPT) parser.add_option(COMPRESS_OPT) parser.add_option(PARALLEL_OPT) parser.add_option(OPPORTUNISTIC_TRIES_OPT) parser.add_option(OPPORTUNISTIC_DELAY_OPT) parser.add_option(KEEP_SOURCE_INSTANCE) (options, args) = parser.parse_args() return (parser, options, args) def _CheckAllocatorOptions(parser, options): if (bool(options.iallocator) and bool(options.dest_primary_node or options.dest_secondary_node)): parser.error("Destination node and iallocator options exclude each other") if (not options.iallocator and options.opportunistic_tries is not None and options.opportunistic_tries > 0): parser.error("Opportunistic instance creation can only be used with an" " iallocator") def _CheckOpportunisticLockingOptions(parser, options): tries_specified = options.opportunistic_tries is not None delay_specified = options.opportunistic_delay is not None if tries_specified: if options.opportunistic_tries < 0: parser.error("Number of opportunistic creation attempts must be >= 0") if delay_specified: if options.opportunistic_delay <= 0: parser.error("The delay between two successive creation attempts must" " be greater than zero") elif delay_specified: parser.error("Opportunistic delay can only be specified when opportunistic" " tries are used") else: # The default values will be provided later pass def _CheckInstanceOptions(parser, options, instance_names): if len(instance_names) == 1: # Moving one instance only if options.hvparams: utils.ForceDictType(options.hvparams, constants.HVS_PARAMETER_TYPES) if options.beparams: utils.ForceDictType(options.beparams, constants.BES_PARAMETER_TYPES) if options.nics: options.nics = cli.ParseNicOption(options.nics) # allow user to override instance parameters by hardcoding the # empty string as "ignore this setting" shortcut for nic in options.nics: for k, v in nic.items(): if v == "": nic[k] = None else: # Moving more than one instance if compat.any(options.dest_instance_name, options.dest_primary_node, options.dest_secondary_node, options.hvparams, options.beparams, options.osparams, options.nics): parser.error("The options --dest-instance-name, --dest-primary-node," " --dest-secondary-node, --hypervisor-parameters," " --backend-parameters, --os-parameters and --net can" " only be used when moving exactly one instance") def CheckOptions(parser, options, args): """Checks options and arguments for validity. """ if len(args) < 3: parser.error("Not enough arguments") src_cluster_name = args.pop(0) dest_cluster_name = args.pop(0) instance_names = args assert len(instance_names) > 0 # TODO: Remove once using system default paths for SSL certificate # verification is implemented if not options.src_ca_file: parser.error("Missing source cluster CA file") if options.parallel < 1: parser.error("Number of simultaneous moves must be >= 1") _CheckAllocatorOptions(parser, options) _CheckOpportunisticLockingOptions(parser, options) _CheckInstanceOptions(parser, options, instance_names) return (src_cluster_name, dest_cluster_name, instance_names) def DestClusterHasDefaultIAllocator(rapi_factory): """Determines if a given cluster has a default iallocator. """ result = rapi_factory.GetDestClient().GetInfo() ia_name = "default_iallocator" return ia_name in result and result[ia_name] def ExitWithError(message): """Exits after an error and shows a message. """ sys.stderr.write("move-instance: error: " + message + "\n") sys.exit(constants.EXIT_FAILURE) def _PrepareListOfInstanceMoves(options, instance_names): moves = [] for src_instance_name in instance_names: if options.dest_instance_name: assert len(instance_names) == 1 # Rename instance dest_instance_name = options.dest_instance_name else: dest_instance_name = src_instance_name moves.append(InstanceMove(src_instance_name, dest_instance_name, options.dest_primary_node, options.dest_secondary_node, options.compress, options.iallocator, options.dest_disk_template, options.hvparams, options.beparams, options.osparams, options.nics, options.opportunistic_tries, options.opportunistic_delay, options.keep_source_instance)) assert len(moves) == len(instance_names) return moves @UsesRapiClient def main(): """Main routine. """ (parser, options, args) = ParseOptions() utils.SetupToolLogging(options.debug, options.verbose, threadname=True) (src_cluster_name, dest_cluster_name, instance_names) = \ CheckOptions(parser, options, args) logging.info("Source cluster: %s", src_cluster_name) logging.info("Destination cluster: %s", dest_cluster_name) logging.info("Instances to be moved: %s", utils.CommaJoin(instance_names)) rapi_factory = RapiClientFactory(options, src_cluster_name, dest_cluster_name) CheckRapiSetup(rapi_factory) has_iallocator = options.iallocator or \ DestClusterHasDefaultIAllocator(rapi_factory) if len(instance_names) > 1 and not has_iallocator: ExitWithError("When moving multiple nodes, an iallocator must be used. " "None was provided and the target cluster does not have " "a default iallocator.") if (len(instance_names) == 1 and not (has_iallocator or options.dest_primary_node or options.dest_secondary_node)): ExitWithError("Target cluster does not have a default iallocator, " "please specify either destination nodes or an iallocator.") moves = _PrepareListOfInstanceMoves(options, instance_names) # Start workerpool wp = workerpool.WorkerPool("Move", options.parallel, MoveSourceWorker) try: # Add instance moves to workerpool for move in moves: wp.AddTask((rapi_factory, move)) # Wait for all moves to finish wp.Quiesce() finally: wp.TerminateWorkers() # There should be no threads running at this point, hence not using locks # anymore logging.info("Instance move results:") for move in moves: if move.dest_instance_name == move.src_instance_name: name = move.src_instance_name else: name = "%s as %s" % (move.src_instance_name, move.dest_instance_name) if move.error_message: msg = "Failed (%s)" % move.error_message else: msg = "Success" logging.info("%s: %s", name, msg) if compat.any(move.error_message for move in moves): sys.exit(constants.EXIT_FAILURE) sys.exit(constants.EXIT_SUCCESS) if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/net-common.in000064400000000000000000000150601476477700300173520ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @SHELL_ENV_INIT@ readonly GANETI_TAP="gnt.com" function check { if [ -z "$INTERFACE" ]; then echo "No network interface specified" exit 1 fi if [ -z "$MODE" ]; then echo "MODE not specified" exit 1 fi } function is_instance_communication_tap { COMMUNICATION=$(echo "$INTERFACE" | cut -d "." -f 1-2) if [ "$MODE" = "routed" -a "$COMMUNICATION" = "$GANETI_TAP" ] then return 0 else return 1 fi } function fix_mac { # Fix the autogenerated MAC to have the first octet set to "fe" # to discourage the bridge from using the TAP dev's MAC FIXED_MAC=$(ip link show $INTERFACE | \ awk '{if ($1 == "link/ether") printf("fe%s",substr($2,3,15))}') # in case of a vif (xen_netback device) this action is not allowed ip link set $INTERFACE address $FIXED_MAC || true } function log_info() { echo "${@}" >&2 logger -p local1.info -t ${0}[$$] -- "${@}" } function bridge_vlan_add { local VID="${1}" shift # when the VID is in use by traditional VLAN interface, then the # tap on the VLAN aware bridge will not receive traffic if [ -r /proc/net/vlan/config ]; then local vlan_interface="$(awk -F '[| ]*' -v vlan="^${VID}$" 'match($2, vlan) { print $1 }' /proc/net/vlan/config)" local lower_devs="$(awk -F '[| ]*' -v vlan="^${VID}$" 'match($2, vlan) { print $3 }' /proc/net/vlan/config)" if [ -n "${vlan_interface}" ]; then for i in ${lower_devs}; do # allow bridge stacking and vlan overlap for veth devices if [ "$(ip -o link show dev ${i} type veth | wc -l)" -ne 1 ]; then echo "VLAN ${VID} is in use by lower interface ${i}" exit 1 fi done fi fi # add the VLAN to the tap # $@ is either empty or contains "pvid untagged" bridge vlan add dev ${INTERFACE} vid ${VID} "${@}" master # add the VLAN to the bridge uplinks local uplink for uplink in ${BRIDGE_UPLINKS}; do log_info "configuring VLAN ${VID} on interface ${uplink} (reason: instance ${INSTANCE})" bridge vlan add dev ${uplink} vid ${VID} master done } function setup_bridge_vlan_aware { # enforce the admin to configure vlan filtering explicit if [ $( must be untagged (access or hybrid) if [ "${VLAN:0:1}" != ":" ]; then bridge_vlan_add ${VLANS[$i]#.} pvid untagged else bridge_vlan_add ${VLANS[$i]} fi # all other VLANs are tagged else bridge_vlan_add ${VLANS[$i]} fi done } function setup_bridge { if [ "$MODE" = "bridged" ]; then fix_mac ip link set $INTERFACE up ip link set $INTERFACE mtu $( /proc/sys/net/ipv4/conf/$INTERFACE/proxy_arp echo 1 > /proc/sys/net/ipv4/conf/$INTERFACE/forwarding fi if [ -d "/proc/sys/net/ipv6/conf/$INTERFACE" ]; then echo 1 > /proc/sys/net/ipv6/conf/$INTERFACE/proxy_ndp echo 1 > /proc/sys/net/ipv6/conf/$INTERFACE/forwarding fi fi } ganeti-3.1.0~rc2/tools/ovfconverter000075500000000000000000000154351476477700300174240ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tool to translate between ovf and ganeti backup format. """ import logging import optparse import os from ganeti import cli from ganeti import constants from ganeti import errors from ganeti import ovf from ganeti import utils IMPORT_MODE = "import" EXPORT_MODE = "export" def CheckOptions(parser, options_dict, required, forbidden, excluding, mode): """Performes check on the command line options. Checks whether the required arguments are present and if none of the arguments not supported for the current mode are given. @type options_dict: list @param options_dict: dictionary containing all the options from the command line @type required: list @param required: list of pairs (option, argument) where 'option' is required in mode 'mode' @type forbidden: list @param forbidden: list of pairs (option, argument) which are not allowed in mode 'mode' @type excluding: list @param excluding: list of pairs (argument1, argument2); each pair contains mutually exclusive arguments @type mode: string @param mode: current mode of the converter """ for (option, argument) in required: if not options_dict[option]: parser.error("Argument %s is required for %s" % (argument, mode)) for (option, argument) in forbidden: if options_dict[option]: parser.error("Argument %s is not allowed in %s mode" % (argument, mode)) for (arg1, arg2) in excluding: if options_dict[arg1] and options_dict[arg2]: parser.error("Arguments %s and %s exclude each other" % (arg1, arg2)) def ParseOptions(): """Parses the command line options and arguments. In case of mismatching parameters, it will show the correct usage and exit. @rtype: tuple @return: (mode, sourcefile to read from, additional options) """ usage = ("%%prog {%s|%s} [options...]" % (IMPORT_MODE, EXPORT_MODE)) parser = optparse.OptionParser(usage=usage) #global options parser.add_option(cli.DEBUG_OPT) parser.add_option(cli.VERBOSE_OPT) parser.add_option("-n", "--name", dest="name", action="store", help="Name of the instance") parser.add_option("--output-dir", dest="output_dir", help="Path to the output directory") #import options import_group = optparse.OptionGroup(parser, "Import options") import_group.add_option(cli.BACKEND_OPT) import_group.add_option(cli.DISK_OPT) import_group.add_option(cli.DISK_TEMPLATE_OPT) import_group.add_option(cli.HYPERVISOR_OPT) import_group.add_option(cli.NET_OPT) import_group.add_option(cli.NONICS_OPT) import_group.add_option(cli.OS_OPT) import_group.add_option(cli.OSPARAMS_OPT) import_group.add_option(cli.TAG_ADD_OPT) parser.add_option_group(import_group) #export options export_group = optparse.OptionGroup(parser, "Export options") export_group.add_option("--compress", dest="compression", action="store_true", default=False, help="The exported disk will be compressed to tar.gz") export_group.add_option("--external", dest="ext_usage", action="store_true", default=False, help="The package will be used externally (ommits the" " Ganeti-specific parts of configuration)") export_group.add_option("-f", "--format", dest="disk_format", action="store", choices=("raw", "cow", "vmdk"), help="Disk format for export (one of raw/cow/vmdk)") export_group.add_option("--ova", dest="ova_package", action="store_true", default=False, help="Export everything into OVA package") parser.add_option_group(export_group) options, args = parser.parse_args() if len(args) != 2: parser.error("Wrong number of arguments") mode = args.pop(0) input_path = os.path.abspath(args.pop(0)) if mode == IMPORT_MODE: required = [] forbidden = [ ("compression", "--compress"), ("disk_format", "--format"), ("ext_usage", "--external"), ("ova_package", "--ova"), ] excluding = [("nics", "no_nics")] elif mode == EXPORT_MODE: required = [("disk_format", "--format")] forbidden = [ ("beparams", "--backend-parameters"), ("disk_template", "--disk-template"), ("disks", "--disk"), ("hypervisor", "--hypervisor-parameters"), ("nics", "--net"), ("no_nics", "--no-nics"), ("os", "--os-type"), ("osparams", "--os-parameters"), ("tags", "--tags"), ] excluding = [] else: parser.error("First argument should be either '%s' or '%s'" % (IMPORT_MODE, EXPORT_MODE)) options_dict = vars(options) CheckOptions(parser, options_dict, required, forbidden, excluding, mode) return (mode, input_path, options) def main(): """Main routine. """ (mode, input_path, options) = ParseOptions() utils.SetupToolLogging(options.debug, options.verbose) logging.info("Chosen %s mode, reading the %s file", mode, input_path) assert mode in (IMPORT_MODE, EXPORT_MODE) converter = None try: if mode == IMPORT_MODE: converter = ovf.OVFImporter(input_path, options) elif mode == EXPORT_MODE: converter = ovf.OVFExporter(input_path, options) converter.Parse() converter.Save() except errors.OpPrereqError as err: if converter: converter.Cleanup() logging.exception(err) return constants.EXIT_FAILURE if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/post-upgrade000064400000000000000000000047341476477700300173110ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # pylint: disable=C0103 """Hook to be run after upgrading to this version. """ import sys from ganeti import utils from ganeti import cli def main(): """Main program. """ if len(sys.argv) != 2: cli.ToStderr("Expecting precisely one argument, the version upgrading from") return 1 versionstring = sys.argv[1] version = utils.version.ParseVersion(versionstring) error_code = 0 if utils.version.IsBefore(version, 2, 12, 5): result = utils.RunCmd(["gnt-cluster", "renew-crypto", "--new-node-certificates", "-f", "-d"]) if result.failed: cli.ToStderr("Failed to create node certificates: %s; Output %s" % (result.fail_reason, result.output)) error_code = 1 if utils.version.IsBefore(version, 2, 13, 0): result = utils.RunCmd(["gnt-cluster", "renew-crypto", "--new-ssh-keys", "--no-ssh-key-check", "-f", "-d"]) if result.failed: cli.ToStderr("Failed to create SSH keys: %s; Output %s" % (result.fail_reason, result.output)) error_code = 1 return error_code if __name__ == "__main__": exit(main()) ganeti-3.1.0~rc2/tools/query-config000075500000000000000000000106601476477700300173050ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2014 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Tool to query the cluster configuration over RConfD """ # tool name shouldn't follow module naming. # pylint: disable=C0103 import optparse import sys from ganeti import constants from ganeti import cli from ganeti import utils from ganeti import pathutils from ganeti.confd import client as confd_client USAGE = ("\tquery-config [--addr=host] [--hmac=key] QUERY [QUERY...]") OPTIONS = [ cli.cli_option("--hmac", dest="hmac", default=None, help="Specify HMAC key instead of reading" " it from the filesystem", metavar=""), cli.cli_option("-a", "--address", dest="mc", default="127.0.0.1", help="Server IP to query (default: 127.0.0.1)", metavar="
") ] def Err(msg, exit_code=1): """Simple error logging that prints to stderr. """ sys.stderr.write(msg + "\n") sys.stderr.flush() sys.exit(exit_code) def Usage(): """Shows program usage information and exits the program. """ print("Usage:", file=sys.stderr) print(USAGE, file=sys.stderr) sys.exit(2) class QueryClient(object): """Confd client for querying the configuration JSON. """ def __init__(self): """Constructor. """ self.opts = None self.cluster_master = None self.instance_ips = None self.is_timing = False self.ParseOptions() def ParseOptions(self): """Parses the command line options. In case of command line errors, it will show the usage and exit the program. @return: a tuple (options, args), as returned by OptionParser.parse_args """ parser = optparse.OptionParser(usage="\n%s" % USAGE, version=("%%prog (ganeti) %s" % constants.RELEASE_VERSION), option_list=OPTIONS) options, args = parser.parse_args() if args == []: Usage() self.paths = args if options.hmac is None: options.hmac = utils.ReadFile(pathutils.CONFD_HMAC_KEY) self.hmac_key = options.hmac self.mc_list = [options.mc] self.opts = options def Run(self): self.store_callback = confd_client.StoreResultCallback() self.confd_client = confd_client.ConfdClient( self.hmac_key, self.mc_list, self.store_callback) responses = [] for path in self.paths: req = confd_client.ConfdClientRequest( type=constants.CONFD_REQ_CONFIG_QUERY, query=path) _, response = self.DoConfdRequestReply(req) responses.append(str(response.server_reply.answer)) table = zip(self.paths, responses) longest_path = max(len(p) for p in self.paths) for p, a in table: print("%s\t%s" % (p.ljust(longest_path), a)) def DoConfdRequestReply(self, req): """Send request to Confd and await all responses. """ self.confd_client.SendRequest(req, async_=False) if not self.confd_client.ReceiveReply(): Err("Did not receive all expected confd replies") return self.store_callback.GetResponse(req.rsalt) def main(): """Application entry point. """ QueryClient().Run() if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/sanitize-config000075500000000000000000000213651476477700300177720ustar00rootroot00000000000000#!/usr/bin/python3 # # Copyright (C) 2010 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # pylint: disable=C0103 """Tool to sanitize/randomize the configuration file. """ import sys import os import os.path import optparse from ganeti import constants from ganeti import serializer from ganeti import utils from ganeti import pathutils from ganeti import cli from ganeti.cli import cli_option OPTS = [ cli.VERBOSE_OPT, cli_option("--path", help="Convert this configuration file" " instead of '%s'" % pathutils.CLUSTER_CONF_FILE, default=pathutils.CLUSTER_CONF_FILE, dest="CONFIG_DATA_PATH"), cli_option("--sanitize-names", default="yes", type="bool", help="Randomize the cluster, node and instance names [yes]"), cli_option("--sanitize-ips", default="yes", type="bool", help="Randomize the cluster, node and instance IPs [yes]"), cli_option("--sanitize-lvs", default="no", type="bool", help="Randomize the LV names (for old clusters) [no]"), cli_option("--sanitize-os-names", default="yes", type="bool", help="Randomize the OS names [yes]"), cli_option("--no-randomization", default=False, action="store_true", help="Disable all name randomization (only randomize secrets)"), cli_option("--base-domain", default="example.com", help="The base domain used for new names [example.com]"), ] def Error(txt, *args): """Writes a message to standard error and exits. """ cli.ToStderr(txt, *args) sys.exit(1) def GenerateNameMap(opts, names, base): """For a given set of names, generate a list of sane new names. """ names = utils.NiceSort(names) name_map = {} for idx, old_name in enumerate(names): new_name = "%s%d.%s" % (base, idx + 1, opts.base_domain) if new_name in names: Error("Name conflict for %s: %s already exists", base, new_name) name_map[old_name] = new_name return name_map def SanitizeSecrets(opts, cfg): # pylint: disable=W0613 """Cleanup configuration secrets. """ cfg["cluster"]["rsahostkeypub"] = "" cfg["cluster"]["dsahostkeypub"] = "" for instance in cfg["instances"].values(): for disk in instance["disks"]: RandomizeDiskSecrets(disk) def SanitizeCluster(opts, cfg): """Sanitize the cluster names. """ cfg["cluster"]["cluster_name"] = "cluster." + opts.base_domain def SanitizeNodes(opts, cfg): """Sanitize node names. """ old_names = cfg["nodes"].keys() old_map = GenerateNameMap(opts, old_names, "node") # rename nodes RenameDictKeys(cfg["nodes"], old_map, True) # update master node cfg["cluster"]["master_node"] = old_map[cfg["cluster"]["master_node"]] # update instance configuration for instance in cfg["instances"].values(): instance["primary_node"] = old_map[instance["primary_node"]] for disk in instance["disks"]: RenameDiskNodes(disk, old_map) def SanitizeInstances(opts, cfg): """Sanitize instance names. """ old_names = cfg["instances"].keys() old_map = GenerateNameMap(opts, old_names, "instance") RenameDictKeys(cfg["instances"], old_map, True) def SanitizeIps(opts, cfg): # pylint: disable=W0613 """Sanitize the IP names. @note: we're interested in obscuring the old IPs, not in generating actually valid new IPs, so we chose to simply put IPv4 addresses, irrelevant of whether IPv6 or IPv4 addresses existed before. """ def _Get(old): if old in ip_map: return ip_map[old] idx = len(ip_map) + 1 rest, d_octet = divmod(idx, 256) rest, c_octet = divmod(rest, 256) rest, b_octet = divmod(rest, 256) if rest > 0: Error("Too many IPs!") new_ip = "%d.%d.%d.%d" % (10, b_octet, c_octet, d_octet) ip_map[old] = new_ip return new_ip ip_map = {} cfg["cluster"]["master_ip"] = _Get(cfg["cluster"]["master_ip"]) for node in cfg["nodes"].values(): node["primary_ip"] = _Get(node["primary_ip"]) node["secondary_ip"] = _Get(node["secondary_ip"]) for instance in cfg["instances"].values(): for nic in instance["nics"]: if "ip" in nic and nic["ip"]: nic["ip"] = _Get(nic["ip"]) def SanitizeOsNames(opts, cfg): # pylint: disable=W0613 """Sanitize the OS names. """ def _Get(old): if old in os_map: return os_map[old] os_map[old] = "ganeti-os%d" % (len(os_map) + 1) return os_map[old] os_map = {} for instance in cfg["instances"].values(): instance["os"] = _Get(instance["os"]) if "os_hvp" in cfg["cluster"]: for os_name in cfg["cluster"]["os_hvp"]: # force population of the entire os map _Get(os_name) RenameDictKeys(cfg["cluster"]["os_hvp"], os_map, False) def SanitizeDisks(opts, cfg): # pylint: disable=W0613 """Cleanup disks disks. """ def _Get(old): if old in lv_map: return old lv_map[old] = utils.NewUUID() return lv_map[old] def helper(disk): if "children" in disk and disk["children"]: for child in disk["children"]: helper(child) if disk["dev_type"] == constants.DT_PLAIN and opts.sanitize_lvs: disk["logical_id"][1] = _Get(disk["logical_id"][1]) lv_map = {} for instance in cfg["instances"].values(): for disk in instance["disks"]: helper(disk) def RandomizeDiskSecrets(disk): """Randomize a disks' secrets (if any). """ if "children" in disk and disk["children"]: for child in disk["children"]: RandomizeDiskSecrets(child) # only disk type to contain secrets is the drbd one if disk["dev_type"] == constants.DT_DRBD8: disk["logical_id"][5] = utils.GenerateSecret() def RenameDiskNodes(disk, node_map): """Rename nodes in the disk config. """ if "children" in disk and disk["children"]: for child in disk["children"]: RenameDiskNodes(child, node_map) # only disk type to contain nodes is the drbd one if disk["dev_type"] == constants.DT_DRBD8: lid = disk["logical_id"] lid[0] = node_map[lid[0]] lid[1] = node_map[lid[1]] def RenameDictKeys(a_dict, name_map, update_name): """Rename the dictionary keys based on a name map. """ for old_name in a_dict.keys(): new_name = name_map[old_name] a_dict[new_name] = a_dict[old_name] del a_dict[old_name] if update_name: a_dict[new_name]["name"] = new_name def main(): """Main program. """ # Option parsing parser = optparse.OptionParser(usage="%prog [--verbose] output_file") for o in OPTS: parser.add_option(o) (opts, args) = parser.parse_args() if opts.no_randomization: opts.sanitize_names = opts.sanitize_ips = opts.sanitize_os_names = \ opts.sanitize_lvs = False # Option checking if len(args) != 1: Error("Usage: sanitize-config [options] { | -}") # Check whether it's a Ganeti configuration directory if not os.path.isfile(opts.CONFIG_DATA_PATH): Error("Cannot find Ganeti configuration file %s", opts.CONFIG_DATA_PATH) config_data = serializer.LoadJson(utils.ReadFile(opts.CONFIG_DATA_PATH)) # Randomize LVM names SanitizeDisks(opts, config_data) SanitizeSecrets(opts, config_data) if opts.sanitize_names: SanitizeCluster(opts, config_data) SanitizeNodes(opts, config_data) SanitizeInstances(opts, config_data) if opts.sanitize_ips: SanitizeIps(opts, config_data) if opts.sanitize_os_names: SanitizeOsNames(opts, config_data) data = serializer.DumpJson(config_data) if args[0] == "-": sys.stdout.write(data) else: utils.WriteFile(file_name=args[0], data=data, mode=0o600, backup=True) if __name__ == "__main__": main() ganeti-3.1.0~rc2/tools/vcluster-setup.in000064400000000000000000000206221476477700300203030ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2012 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -e -u -o pipefail shopt -s extglob readonly self=$(readlink -f $0) readonly ensure_dirs=@PKGLIBDIR@/ensure-dirs readonly action_shortcuts=( start stop restart status watcher ) readonly default_nodecount=5 readonly default_instcount=10 readonly default_netprefix=192.0.2 readonly default_netdev=eth0 readonly default_initscript=@SYSCONFDIR@/init.d/ganeti readonly cluster_name=cluster readonly etc_hosts_filename=/etc/hosts # IP address space: # Cluster: .1 # Nodes: .10-.99 # Instances: .100-.254 readonly first_node_ipaddr_octet=10 readonly first_inst_ipaddr_octet=100 readonly max_node_count=$((first_inst_ipaddr_octet - first_node_ipaddr_octet)) readonly max_instance_count=$((255 - first_inst_ipaddr_octet)) usage() { echo "Usage: $0 [-E] [-N] [-c ] [-i ] [-p ]"\ '[-n ] [-I ] ' echo echo 'Options:' echo " -c Number of virtual nodes (defaults to $default_nodecount)" echo " -i Number of instances (defaults to $default_instcount)" echo " -p IPv4 network prefix (defaults to $default_netprefix)" echo ' -n Network device for virtual IP addresses (defaults to'\ "$default_netdev)" echo " -I Path to init script (defaults to $default_initscript)" echo " -E Do not modify $etc_hosts_filename" echo ' -N Do not configure networking' } # Variables for options nodecount=$default_nodecount instcount=$default_instcount netprefix=$default_netprefix netdev=$default_netdev initscript=$default_initscript etchosts=1 networking=1 # Parse options while getopts :hENc:p:n:i:I: opt; do case "$opt" in h) usage exit 0 ;; c) nodecount="$OPTARG" if [[ "$nodecount" != +([0-9]) ]]; then echo "Invalid node count number: $nodecount" >&2 exit 1 elif (( nodecount > max_node_count )); then echo "Node count must be $max_node_count or lower" >&2 exit 1 fi ;; i) instcount="$OPTARG" if [[ "$instcount" != +([0-9]) ]]; then echo "Invalid instance count number: $instcount" >&2 exit 1 elif (( instcount > max_instance_count )); then echo "Instance count must be $max_instance_count or lower" >&2 exit 1 fi ;; p) netprefix="$OPTARG" if [[ "$netprefix" != +([0-9]).+([0-9]).+([0-9]) ]]; then echo "Invalid network prefix: $netprefix" >&2 exit 1 fi ;; n) netdev="$OPTARG" if ! ip link show $netdev >/dev/null; then echo "Invalid network device: $netdev" >&2 exit 1 fi ;; I) initscript="$OPTARG" if [[ ! -x $initscript ]]; then echo "Init script '$initscript' is not executable" >&2 exit 1 fi ;; E) etchosts= ;; N) networking= ;; \?) echo "Invalid option: -$OPTARG" >&2 usage >&2 exit 1 ;; :) echo "Option -$OPTARG requires an argument" >&2 usage >&2 exit 1 ;; esac done shift $((OPTIND - 1)) if [[ "$#" != 1 ]]; then usage exit 1 fi readonly rootdir=$1; shift if [[ ! -d "$rootdir" ]]; then echo "Directory '$rootdir' does not exist!" >&2 exit 1 fi if (( $nodecount < 1 )); then echo "Must create at least one node, currently requested $nodecount" >&2 exit 1 fi node_hostname() { local -r number="$1" echo "node$((number + 1))" } instance_hostname() { local -r number="$1" echo "instance$((number + 1))" } node_ipaddr() { local -r number="$1" echo "$netprefix.$((first_node_ipaddr_octet + number))" } instance_ipaddr() { local -r number="$1" echo "$netprefix.$((first_inst_ipaddr_octet + number))" } setup_node() { local -r number="$1" local -r nodedir=$rootdir/$(node_hostname $number) echo "Setting up node '$(node_hostname $number)' ..." >&2 if [[ ! -d $nodedir ]]; then mkdir $nodedir fi mkdir -p \ $nodedir@SYSCONFDIR@/{default,ganeti} \ $nodedir@LOCALSTATEDIR@/lock\ $nodedir@LOCALSTATEDIR@/{lib,log,run}/ganeti GANETI_HOSTNAME=$(node_hostname $number) \ GANETI_ROOTDIR=$nodedir \ $ensure_dirs local -r daemon_args="-b $(node_ipaddr $number)" cat > $nodedir/etc/default/ganeti < $nodedir/cmd <&2 ( set -e -u local -r tmpfile=$(mktemp $etc_hosts_filename.vcluster.XXXXX) trap "rm -f $tmpfile" EXIT { egrep -v "^$netprefix.[[:digit:]]+[[:space:]]" $etc_hosts_filename echo "$netprefix.1 $cluster_name" for ((i=0; i < nodecount; ++i)); do echo "$(node_ipaddr $i) $(node_hostname $i)" done for ((i=0; i < instcount; ++i)); do echo "$(instance_ipaddr $i) $(instance_hostname $i)" done } > $tmpfile && \ chmod 0644 $tmpfile && \ mv $tmpfile $etc_hosts_filename && \ trap - EXIT ) } setup_network_interfaces() { echo 'Configuring network ...' >&2 for ((i=0; i < nodecount; ++i)); do local ipaddr="$(node_ipaddr $i)/32" ip addr del "$ipaddr" dev "$netdev" || : ip addr add "$ipaddr" dev "$netdev" done route add -net $netprefix.0 netmask 255.255.255.0 dev $netdev || : } setup_scripts() { echo 'Configuring helper scripts ...' >&2 for action in "${action_shortcuts[@]}"; do { echo '#!/bin/bash' for ((i=0; i < nodecount; ++i)); do local name=$(node_hostname $i) if [[ $action = watcher ]]; then echo "echo 'Running watcher for virtual node \"$name\" ..." echo "$name/cmd ganeti-watcher \"\$@\"" else echo "echo 'Action \"$action\" for virtual node \"$name\" ...'" echo "$name/cmd $initscript $action \"\$@\"" fi done } > $rootdir/$action-all chmod +x $rootdir/$action-all done } show_info() { cat </dev/null ; then echo "$2" fi } xenstore_write() { xenstore-write "$XENBUS_PATH/$1" "$2" } log() { local level="$1" ; shift logger -p daemon."${level}" -t "$0" -- "$@" } : "${XENBUS_PATH:?}" "${vif:?}" vifname=$(xenstore_read "vifname" "") if [ -n "$vifname" ] ; then if [ "$1" = "online" ] && ! ip link show "$vifname" >/dev/null >&2 ; then log debug "Renaming interface ${vif} to ${vifname}" ip link set "$vif" name "$vifname" fi vif="$vifname" fi case "$1" in online) # Tell Xen we're connected xenstore_write "hotplug-status" "connected" ;; offline) ifconfig "$vif" down || true ;; esac ganeti-3.1.0~rc2/tools/vif-ganeti.in000075500000000000000000000034231476477700300173320ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2011, 2012, 2013 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. if [ -x "@XEN_CONFIG_DIR@/scripts/vif-custom" ]; then exec @XEN_CONFIG_DIR@/scripts/vif-custom $* fi source @PKGLIBDIR@/net-common dir=$(dirname "$0") . "$dir"/vif-common.sh # taken from older vif-common.sh dev=$vif dev_=${dev#vif} domid=${dev_%.*} devid=${dev_#*.} domname=$(xm domname $domid) NIC_DIR=$RUN_DIR/xen-hypervisor/nic INTERFACE=$dev INSTANCE=$domname source $NIC_DIR/$domname/$devid setup_bridge setup_ovs setup_route success ganeti-3.1.0~rc2/tools/xen-console-wrapper000075500000000000000000000040111476477700300205760ustar00rootroot00000000000000#!/bin/bash # # Copyright (C) 2011 Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. XEN_CMD="$1" INSTANCE="$2" unpause() { ispaused=$(${XEN_CMD} list $INSTANCE 2>/dev/null | tail -n1 | awk '{print $5}' | sed -n 's/..\(.\).../\1/p') [[ "$ispaused" == "p" ]] || return # As there is no way to be sure when xen console has actually connected to the # instance, sleep for a few seconds before unpausing the instance. This is a # tradeoff between missing some console output if the node is overloaded and # making the user wait everytime when the node isn't so busy. sleep 3 # Send \r\n after notice as terminal is in raw mode printf "Instance $INSTANCE is paused, unpausing\r\n" $(${XEN_CMD} unpause "$INSTANCE") } unpause & exec $XEN_CMD console "$INSTANCE"